i18n.js

// Copyright 2017 The Kubernetes Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/**
 * @fileoverview Gulp tasks for the extraction of translatable messages.
 */
import childProcess from 'child_process';
import fileExists from 'file-exists';
import gulp from 'gulp';
import cheerio from 'gulp-cheerio';
import freplace from 'gulp-findreplace';
import gulpUtil from 'gulp-util';
import xslt from 'gulp-xslt';
import jsesc from 'jsesc';
import path from 'path';
import q from 'q';
import regexpClone from 'regexp-clone';

import conf from './conf';

/**
 * Extracts the translatable text messages for the given language key from the pre-compiled
 * files under conf.paths.{serve|messagesForExtraction}.
 * @param  {string} langKey - the locale key
 * @return {!Promise} A promise object.
 */
function extractForLanguage(langKey) {
  let deferred = q.defer();

  let translationBundle = path.join(conf.paths.base, `i18n/messages-${langKey}.xtb`);
  let codeSource = path.join(conf.paths.serve, '**.js');
  let messagesSource = path.join(conf.paths.messagesForExtraction, '**.js');
  let command = `java -jar ${conf.paths.xtbgenerator} --lang ${langKey}` +
      ` --xtb_output_file ${translationBundle} --js ${codeSource} --js ${messagesSource}`;
  if (fileExists.sync(translationBundle)) {
    command = `${command} --translations_file ${translationBundle}`;
  }

  childProcess.exec(command, function(err, stdout, stderr) {
    if (err) {
      gulpUtil.log(stdout);
      gulpUtil.log(stderr);
      deferred.reject(new Error(err));
    }
    deferred.resolve();
  });

  return deferred.promise;
}

gulp.task('generate-xtbs', [
  'extract-translations',
  'remove-unused-translations',
  'remove-duplicated-translations',
  'sort-translations',
  'set-prod-node-env',
]);

let prevMsgs = {};

gulp.task('buildExistingI18nCache', function() {
  return gulp.src('i18n/messages-en.xtb').pipe(cheerio((doc) => {
    doc('translation').each((i, translation) => {
      let key = translation.attribs.key;
      let index = key.lastIndexOf('_');
      if (index !== -1) {
        let lastpart = key.substring(index + 1);
        if (/^[0-9]+$/.test(lastpart)) {
          let indexSuffix = Number(lastpart);
          let filePrefix = key.substring(0, index);
          if (!prevMsgs[filePrefix]) {
            prevMsgs[filePrefix] = [];
          }
          prevMsgs[filePrefix].push({
            index: indexSuffix,
            key: key,
            text: doc(translation).text(),
            desc: translation.attribs.desc,
            used: false,
          });
        }
      }
    });
  }));
});

/**
 * Extracts all translation messages into XTB bundles.
 *
 * Cleans up the data from the previous run to prevent cross-pollination between branches.
 */
gulp.task(
    'extract-translations', ['scripts', 'angular-templates', 'clean-messages-for-extraction'],
    function() {
      let promises = conf.translations.map((translation) => extractForLanguage(translation.key));
      return q.all(promises);
    });

/**
 * Task to sort translations.
 */
gulp.task('sort-translations', ['remove-duplicated-translations'], function() {
  return gulp.src('i18n/messages-*.xtb')
      .pipe(xslt('build/sort-translations.xslt'))
      .pipe(gulp.dest('i18n'));
});

/**
 * Task to remove duplicated translations (should remove old translations from XTB when entries are
 * updated in source code). It has to be ran before 'sort-translations' as original translation
 * order is required.
 */
gulp.task('remove-duplicated-translations', ['extract-translations'], function() {
  return gulp.src('i18n/messages-*.xtb')
      .pipe(xslt('build/remove-duplicated-translations.xslt'))
      .pipe(gulp.dest('i18n'));
});

/**
 * Task to used to find translations used in JavaScript files. Do not run manually. It should be
 * invoked as a part of 'remove-unused-translations'.
 */
gulp.task('find-translations-used-in-js', function() {
  let jsSource = path.join(conf.paths.frontendSrc, '**/*.js');
  return gulp.src(jsSource).pipe(freplace(/MSG_\w*/g, function(match) {
    // Mark every message found in JavaScript files as used, it will allow deletion of unused
    // messages afterwards.
    translationsManager.addUsed(match);
  }));
});

/**
 * Task to remove unused translations. Do not run manually. It should be invoked as a part of
 * 'generate-xtbs'.
 */
gulp.task(
    'remove-unused-translations', ['angular-templates', 'find-translations-used-in-js'],
    function() {
      // Get translations used in JavaScript and HTML files. These will not be removed.
      let used = translationsManager.getUsed();

      return gulp.src('i18n/messages-*.xtb')
          .pipe(cheerio((doc) => {
            let unused = new Set();

            // Find translations to remove.
            doc('translation').each((i, translation) => {
              let key = translation.attribs.key;
              if (!used.has(key)) {
                unused.add(key);
              }
            });

            // Remove unused translations.
            unused.forEach((r) => {
              doc(`translation[key=${r}]`).remove();
            });
          }))
          .pipe(gulp.dest('i18n/'));
    });

/**
 * Translations manager is a closure function allowing to manage translations.
 * Allows removing unused translations after marking certain as used.
 *
 * @type {{markAsUsed, removeUnused}}
 */
export let translationsManager = (function() {

  /**
   * Set of translations marked as used.
   *
   * @type {Set}
   */
  let used = new Set();

  /**
   * Function used to mark translations as used.
   *
   * @param {string} key
   */
  function addUsed(key) {
    used.add(key);
  }

  /**
   * Function used to get used translations.
   *
   * @return {Set}
   */
  function getUsed() {
    return used;
  }

  return {
    addUsed: addUsed,
    getUsed: getUsed,
  };
})();

// Regex to match [[Foo | Bar]] or [[Foo]] i18n placeholders.
// Technical details:
// * First capturing group is lazy math for any string not-containing |. This is to make
//   both [[ message | description ]] and [[ message ]] work.
// * Second is non-capturing and optional. It has a capturing group inside. This is to
//   extract description that is optional.
const I18N_REGEX = /\[\[([^|]*?)(?:\|(.*?))?\]\]/g;

export function processI18nMessages(file, minifiedHtml) {
  let content = jsesc(minifiedHtml);
  let pureHtmlContent = `${content}`;
  let filePath = path.relative(file.base, file.path);
  let messageVarPrefix = filePath.toUpperCase().split('/').join('_').replace('.HTML', '');
  let used = new Set();

  /**
   * Finds all i18n messages inside a template and returns its text, description and original
   * string.
   * @param {string} htmlContent
   * @return {!Array<{text: string, desc: string, original: string}>}
   */
  function findI18nMessages(htmlContent) {
    let matches = htmlContent.match(I18N_REGEX);
    if (matches) {
      return matches.map((match) => {
        let exec = regexpClone(I18N_REGEX).exec(match);
        // Default to no description when it is not provided.
        let desc = (exec[2] || '(no description provided)').trim();
        // replace {{$variableName}} with {{ $variableName}} to avoid {$ getting recognised as
        // google.getMsg format
        let text = exec[1].replace('{$', '{ $');
        let varName = undefined;
        if (prevMsgs[`MSG_${messageVarPrefix}`]) {
          for (let msg of prevMsgs[`MSG_${messageVarPrefix}`]) {
            if (msg.text === text && msg.desc === desc && !msg.used) {
              varName = msg.key;
              msg.used = true;
              used.add(msg.index);
              break;
            }
          }
        }
        return {text: text, desc: desc, original: match, varName: varName};
      });
    }
    return [];
  }

  let i18nMessages = findI18nMessages(content);

  /**
   * @param {number} index
   * @return {string}
   */
  function createMessageVarName() {
    for (let i = 0;; i++) {
      if (!used.has(i)) {
        used.add(i);
        return `MSG_${messageVarPrefix}_${i}`;
      }
    }
  }

  i18nMessages.forEach((message) => {
    message.varName = message.varName || createMessageVarName();
    // Replace i18n messages with english messages for testing and MSG_ vars invocations
    // for compiler passses.
    content = content.replace(message.original, `' + ${message.varName} + '`);
    pureHtmlContent = pureHtmlContent.replace(message.original, message.text);

    // Mark every message found in this HTML file as used, it will allow deletion of unused messages
    // afterwards.
    translationsManager.addUsed(message.varName);
  });

  let messageVariables = i18nMessages.map((message) => {
    return `/** @desc ${message.desc} */\n` +
        `var ${message.varName} = goog.getMsg('${message.text}');\n`;
  });

  file.messages = messageVariables.join('\n');
  // Eval pure HTML content, because it has been jsescaped previously. This is safe to eval since
  // it was escaped by jsecs previously.
  file.pureHtmlContent = eval(`'${pureHtmlContent}'`);
  file.moduleContent = `` +
      `import module from '/index_module';\n\n${file.messages}\n` +
      `module.run(['$templateCache', ($templateCache) => {\n` +
      `    $templateCache.put('${filePath}', '${content}');\n` +
      `}]);\n`;

  return minifiedHtml;
}