Fixes #4295: reworked message extraction for PO files

framework/console/controllers/MessageController.php

@@ -12,6 +12,7 @@ use yii\console\Controller;
 use yii\console\Exception;
 use yii\helpers\FileHelper;
 use yii\helpers\VarDumper;
+use yii\i18n\GettextPoFile;
 
 /**
  * Extracts messages to be translated from source files.
@@ -88,14 +89,16 @@ class MessageController extends Controller
             'format' => 'php',
         ], require($configFile));
 
-        if (!isset($config['sourcePath'], $config['messagePath'], $config['languages'])) {
-            throw new Exception('The configuration file must specify "sourcePath", "messagePath" and "languages".');
+        if (!isset($config['sourcePath'], $config['languages'])) {
+            throw new Exception('The configuration file must specify "sourcePath" and "languages".');
         }
         if (!is_dir($config['sourcePath'])) {
             throw new Exception("The source path {$config['sourcePath']} is not a valid directory.");
         }
         if (in_array($config['format'], ['php', 'po'])) {
-            if (!is_dir($config['messagePath'])) {
+            if (!isset($config['messagePath'])) {
+                throw new Exception('The configuration file must specify "messagePath".');
+            } elseif (!is_dir($config['messagePath'])) {
                 throw new Exception("The message path {$config['messagePath']} is not a valid directory.");
             }
         }
@@ -118,14 +121,11 @@ class MessageController extends Controller
                 if (!is_dir($dir)) {
                     @mkdir($dir);
                 }
-                foreach ($messages as $category => $msgs) {
-                    $file = str_replace("\\", '/', "$dir/$category." . $config['format']);
-                    $path = dirname($file);
-                    if (!is_dir($path)) {
-                        mkdir($path, 0755, true);
-                    }
-                    $msgs = array_values(array_unique($msgs));
-                    $this->generateMessageFile($msgs, $file, $config['overwrite'], $config['removeUnused'], $config['sort'], $config['format']);
+                if ($config['format'] === 'po') {
+                    $catalog = isset($config['catalog']) ? $config['catalog'] : 'messages';
+                    $this->saveMessagesToPO($messages, $dir, $config['overwrite'], $config['removeUnused'], $config['sort'], $catalog);
+                } else {
+                    $this->saveMessagesToPHP($messages, $dir, $config['overwrite'], $config['removeUnused'], $config['sort']);
                 }
             }
         } elseif ($config['format'] === 'db') {
@@ -200,11 +200,11 @@ class MessageController extends Controller
                 $savedFlag = true;
 
                 $db->createCommand()
-                ->insert($sourceMessageTable, ['category' => $category, 'message' => $m])->execute();
+                   ->insert($sourceMessageTable, ['category' => $category, 'message' => $m])->execute();
                 $lastId = $db->getLastInsertID();
                 foreach ($languages as $language) {
                     $db->createCommand()
-                    ->insert($messageTable, ['id' => $lastId, 'language' => $language])->execute();
+                       ->insert($messageTable, ['id' => $lastId, 'language' => $language])->execute();
                 }
             }
         }
@@ -217,19 +217,19 @@ class MessageController extends Controller
         } else {
             if ($removeUnused) {
                 $db->createCommand()
-                ->delete($sourceMessageTable, ['in', 'id', $obsolete])->execute();
+                   ->delete($sourceMessageTable, ['in', 'id', $obsolete])->execute();
                 echo "deleted.\n";
             } else {
                 $last_id = $db->getLastInsertID();
                 $db->createCommand()
-                    ->update(
-                        $sourceMessageTable,
-                        ['message' => new \yii\db\Expression("CONCAT('@@',message,'@@')")],
-                        ['in', 'id', $obsolete]
-                    )->execute();
+                   ->update(
+                       $sourceMessageTable,
+                       ['message' => new \yii\db\Expression("CONCAT('@@',message,'@@')")],
+                       ['in', 'id', $obsolete]
+                   )->execute();
                 foreach ($languages as $language) {
                     $db->createCommand()
-                    ->insert($messageTable, ['id' => $last_id, 'language' => $language])->execute();
+                       ->insert($messageTable, ['id' => $last_id, 'language' => $language])->execute();
                 }
                 echo "updated.\n";
             }
@@ -273,27 +273,41 @@ class MessageController extends Controller
     }
 
     /**
-     * Writes messages into file
+     * Writes messages into PHP files
+     *
+     * @param array $messages
+     * @param string $dirName name of the directory to write to
+     * @param boolean $overwrite if existing file should be overwritten without backup
+     * @param boolean $removeUnused if obsolete translations should be removed
+     * @param boolean $sort if translations should be sorted
+     */
+    protected function saveMessagesToPHP($messages, $dirName, $overwrite, $removeUnused, $sort)
+    {
+        foreach ($messages as $category => $msgs) {
+            $file = str_replace("\\", '/', "$dirName/$category.php");
+            $path = dirname($file);
+            if (!is_dir($path)) {
+                mkdir($path, 0755, true);
+            }
+            $msgs = array_values(array_unique($msgs));
+            echo "Saving messages to $file...\n";
+            $this->saveMessagesCategoryToPHP($msgs, $file, $overwrite, $removeUnused, $sort);
+        }
+    }
+
+    /**
+     * Writes category messages into PHP file
      *
      * @param array $messages
      * @param string $fileName name of the file to write to
      * @param boolean $overwrite if existing file should be overwritten without backup
      * @param boolean $removeUnused if obsolete translations should be removed
      * @param boolean $sort if translations should be sorted
-     * @param string $format output format
      */
-    protected function generateMessageFile($messages, $fileName, $overwrite, $removeUnused, $sort, $format)
+    protected function saveMessagesCategoryToPHP($messages, $fileName, $overwrite, $removeUnused, $sort)
     {
-        echo "Saving messages to $fileName...";
         if (is_file($fileName)) {
-            if ($format === 'po') {
-                $translated = file_get_contents($fileName);
-                preg_match_all('/(?<=msgid ").*(?="\n(#*)msgstr)/', $translated, $keys);
-                preg_match_all('/(?<=msgstr ").*(?="\n\n)/', $translated, $values);
-                $translated = array_combine($keys[0], $values[0]);
-            } else {
-                $translated = require($fileName);
-            }
+            $translated = require($fileName);
             sort($messages);
             ksort($translated);
             if (array_keys($translated) == $messages) {
@@ -304,9 +318,6 @@ class MessageController extends Controller
             $merged = [];
             $untranslated = [];
             foreach ($messages as $message) {
-                if ($format === 'po') {
-                    $message = preg_replace('/\"/', '\"', $message);
-                }
                 if (array_key_exists($message, $translated) && strlen($translated[$message]) > 0) {
                     $merged[$message] = $translated[$message];
                 } else {
@@ -336,47 +347,19 @@ class MessageController extends Controller
             if (false === $overwrite) {
                 $fileName .= '.merged';
             }
-            if ($format === 'po') {
-                $output = '';
-                foreach ($merged as $k => $v) {
-                    $k = preg_replace('/(\")|(\\\")/', "\\\"", $k);
-                    $v = preg_replace('/(\")|(\\\")/', "\\\"", $v);
-                    if (substr($v, 0, 2) === '@@' && substr($v, -2) === '@@') {
-                        $output .= "#msgid \"$k\"\n";
-                        $output .= "#msgstr \"$v\"\n";
-                    } else {
-                        $output .= "msgid \"$k\"\n";
-                        $output .= "msgstr \"$v\"\n";
-                    }
-                    $output .= "\n";
-                }
-                $merged = $output;
-            }
-            echo "translation merged.\n";
+            echo "Translation merged.\n";
         } else {
-            if ($format === 'po') {
-                $merged = '';
-                sort($messages);
-                foreach ($messages as $message) {
-                    $message = preg_replace('/(\")|(\\\")/', '\\\"', $message);
-                    $merged .= "msgid \"$message\"\n";
-                    $merged .= "msgstr \"\"\n";
-                    $merged .= "\n";
-                }
-            } else {
-                $merged = [];
-                foreach ($messages as $message) {
-                    $merged[$message] = '';
-                }
-                ksort($merged);
+            $merged = [];
+            foreach ($messages as $message) {
+                $merged[$message] = '';
             }
-            echo "saved.\n";
+            ksort($merged);
         }
-        if ($format === 'po') {
-            $content = $merged;
-        } else {
-            $array = VarDumper::export($merged);
-            $content = <<<EOD
+        echo "Saved.\n";
+
+
+        $array = VarDumper::export($merged);
+        $content = <<<EOD
 <?php
 /**
  * Message translations.
@@ -398,7 +381,95 @@ class MessageController extends Controller
 return $array;
 
 EOD;
-        }
+
         file_put_contents($fileName, $content);
     }
+
+    /**
+     * Writes messages into PO file
+     *
+     * @param array $messages
+     * @param string $dirName name of the directory to write to
+     * @param boolean $overwrite if existing file should be overwritten without backup
+     * @param boolean $removeUnused if obsolete translations should be removed
+     * @param boolean $sort if translations should be sorted
+     * @param string $catalog message catalog
+     */
+    protected function saveMessagesToPO($messages, $dirName, $overwrite, $removeUnused, $sort, $catalog)
+    {
+        $file = str_replace("\\", '/', "$dirName/$catalog.po");
+        $path = dirname($file);
+        if (!is_dir($path)) {
+            mkdir($path, 0755, true);
+        }
+        echo "Saving messages to $file...\n";
+
+        $poFile = new GettextPoFile();
+
+
+        $merged = [];
+        $notTranslatedYet = [];
+        $todos = [];
+
+        foreach ($messages as $category => $msgs) {
+            $msgs = array_values(array_unique($msgs));
+
+            if (is_file($file)) {
+                $existingMessages = $poFile->load($file, $category);
+
+                sort($msgs);
+                ksort($existingMessages);
+                if (array_keys($existingMessages) == $msgs) {
+                    echo "Nothing new... skipped.\n";
+                    return self::EXIT_CODE_NORMAL;
+                }
+
+                // merge existing message translations with new message translations
+                foreach ($msgs as $message) {
+                    if (array_key_exists($message, $existingMessages) && strlen($existingMessages[$message]) > 0) {
+                        $merged[$category . chr(4) . $message] = $existingMessages[$message];
+                    } else {
+                        $notTranslatedYet[] = $message;
+                    }
+                }
+                ksort($merged);
+                sort($notTranslatedYet);
+
+                // collect not yet translated messages
+                foreach ($notTranslatedYet as $message) {
+                    $todos[$category . chr(4) . $message] = '';
+                }
+
+                // add obsolete unused messages
+                foreach ($existingMessages as $message => $translation) {
+                    if (!isset($merged[$category . chr(4) . $message]) && !isset($todos[$category . chr(4) . $message]) && !$removeUnused) {
+                        if (substr($translation, 0, 2) === '@@' && substr($translation, -2) === '@@') {
+                            $todos[$category . chr(4) . $message] = $translation;
+                        } else {
+                            $todos[$category . chr(4) . $message] = '@@' . $translation . '@@';
+                        }
+                    }
+                }
+
+                $merged = array_merge($todos, $merged);
+                if ($sort) {
+                    ksort($merged);
+                }
+
+                if ($overwrite === false) {
+                    $file .= '.merged';
+                }
+
+                echo "Translation merged.\n";
+            } else {
+                sort($msgs);
+                foreach ($msgs as $message) {
+                    $merged[$category . chr(4) . $message] = '';
+                }
+                ksort($merged);
+            }
+        }
+        $poFile->save($file, $merged);
+        echo "Saved.\n";
+    }
 }

framework/views/messageConfig.php

@@ -3,8 +3,6 @@
 return [
     // string, required, root directory of all source files
     'sourcePath' => __DIR__,
-    // string, required, root directory containing message translations.
-    'messagePath' => __DIR__ . DIRECTORY_SEPARATOR . 'messages',
     // array, required, list of language codes that the extracted messages
     // should be translated to. For example, ['zh-CN', 'de'].
     'languages' => ['de'],
@@ -44,9 +42,30 @@ return [
         '.hgkeep',
         '/messages',
     ],
-    // Generated file format. Can be either "php", "po" or "db".
+
+    // 'php' output format is for saving messages to php files.
     'format' => 'php',
-    // When format is "db", you may specify the following two options
-    //'db' => 'db',
-    //'sourceMessageTable' => '{{%source_message}}',
+    // Root directory containing message translations.
+    'messagePath' => __DIR__ . DIRECTORY_SEPARATOR . 'messages',
+
+
+    /*
+    // 'db' output format is for saving messages to database.
+    'format' => 'db',
+    // Connection component to use. Optional.
+    'db' => 'db',
+    // Custom source message table. Optional.
+    // 'sourceMessageTable' => '{{%source_message}}',
+    // Custom name for translation message table. Optional.
+    // 'messageTable' => '{{%message}}',
+    */
+
+    /*
+    // 'po' output format is for saving messages to gettext po files.
+    'format' => 'po',
+    // Root directory containing message translations.
+    'messagePath' => __DIR__ . DIRECTORY_SEPARATOR . 'messages',
+    // Name of the file that will be used for translations.
+    'catalog' => 'messages',
+    */
 ];

tests/unit/framework/console/controllers/MessageControllerTest.php

@@ -255,13 +255,13 @@ class MessageControllerTest extends TestCase
             'messagePath' => $this->messagePath,
             'overwrite' => true,
         ]);
-        $this->runMessageControllerAction('extract', [$this->configFileName]);
+        $commandOutput = $this->runMessageControllerAction('extract', [$this->configFileName]);
 
         $messages = require($this->messagePath . DIRECTORY_SEPARATOR . $messageFileName);
-        $this->assertTrue(array_key_exists($newMessage, $messages), 'Unable to add new message!');
-        $this->assertTrue(array_key_exists($existingMessage, $messages), 'Unable to keep existing message!');
-        $this->assertEquals('', $messages[$newMessage], 'Wrong new message content!');
-        $this->assertEquals($existingMessageContent, $messages[$existingMessage], 'Unable to keep existing message content!');
+        $this->assertTrue(array_key_exists($newMessage, $messages), 'Unable to add new message: "' . $newMessage . '". Command output was:' . "\n" . $commandOutput);
+        $this->assertTrue(array_key_exists($existingMessage, $messages), 'Unable to keep existing message: "' . $existingMessage . '". Command output was:' . "\n" . $commandOutput);
+        $this->assertEquals('', $messages[$newMessage], 'Wrong new message content!. Command output was:\n' . $commandOutput);
+        $this->assertEquals($existingMessageContent, $messages[$existingMessage], 'Unable to keep existing message content!. Command output was:' . "\n" . $commandOutput);
     }
 
     /**