Merge pull request #23281 from michelkaporin/i18n

Added ability for JSON/XLIFF conversion and Push/Pull from Transifex

Merge pull request #23281 from michelkaporin/i18n
Added ability for JSON/XLIFF conversion and Push/Pull from Transifex
96bd2ed7 · Michel Kaporin · GitHub · 4c5030a3 · ac301ba0 · 96bd2ed7
9 changed file
--- a/build/gulpfile.vscode.js
+++ b/build/gulpfile.vscode.js
@@ -29,6 +29,7 @@ const packageJson = require('../package.json');
 const product = require('../product.json');
 const shrinkwrap = require('../npm-shrinkwrap.json');
 const crypto = require('crypto');
+const i18n = require('./lib/i18n');

 const dependencies = Object.keys(shrinkwrap.dependencies)
 	.concat(Array.isArray(product.extraNodeModules) ? product.extraNodeModules : []); // additional dependencies from our product configuration
@@ -340,6 +341,54 @@ gulp.task('vscode-linux-ia32-min', ['minify-vscode', 'clean-vscode-linux-ia32'],
 gulp.task('vscode-linux-x64-min', ['minify-vscode', 'clean-vscode-linux-x64'], packageTask('linux', 'x64', { minified: true }));
 gulp.task('vscode-linux-arm-min', ['minify-vscode', 'clean-vscode-linux-arm'], packageTask('linux', 'arm', { minified: true }));

+// Transifex Localizations
+const vscodeLanguages = [
+	'chs',
+	'cht',
+	'jpn',
+	'kor',
+	'deu',
+	'fra',
+	'esn',
+	'rus',
+	'ita'
+];
+
+const setupDefaultLanguages = [
+	'chs',
+	'cht',
+	'kor'
+];
+
+const apiHostname = process.env.TRANSIFEX_API_URL;
+const apiName = process.env.TRANSIFEX_API_NAME;
+const apiToken = process.env.TRANSIFEX_API_TOKEN;
+
+gulp.task('vscode-translations-push', function() {
+	const pathToMetadata = './out-vscode/nls.metadata.json';
+	const pathToExtensions = './extensions/**/*.nls.json';
+	const pathToSetup = 'build/win32/**/{Default.isl,messages.en.isl}';
+
+	return es.merge(
+		gulp.src(pathToMetadata).pipe(i18n.prepareXlfFiles()),
+		gulp.src(pathToSetup).pipe(i18n.prepareXlfFiles()),
+		gulp.src(pathToExtensions).pipe(i18n.prepareXlfFiles('vscode-extensions'))
+	).pipe(i18n.pushXlfFiles(apiHostname, apiName, apiToken));
+});
+
+gulp.task('vscode-translations-pull', function() {
+	return es.merge(
+		i18n.pullXlfFiles('vscode-editor', apiHostname, apiName, apiToken, vscodeLanguages),
+		i18n.pullXlfFiles('vscode-workbench', apiHostname, apiName, apiToken, vscodeLanguages),
+		i18n.pullXlfFiles('vscode-extensions', apiHostname, apiName, apiToken, vscodeLanguages),
+		i18n.pullXlfFiles('vscode-setup', apiHostname, apiName, apiToken, setupDefaultLanguages)
+	).pipe(vfs.dest('../vscode-localization'));
+});
+
+gulp.task('vscode-translations-import', function() {
+	return gulp.src('../vscode-localization/**/*.xlf').pipe(i18n.prepareJsonFiles()).pipe(vfs.dest('./i18n'));
+});
+
 // Sourcemaps

 gulp.task('upload-vscode-sourcemaps', ['minify-vscode'], () => {

--- a/build/lib/i18n.js
+++ b/build/lib/i18n.js
--- a/build/lib/i18n.ts
+++ b/build/lib/i18n.ts
--- a/build/lib/test/i18n.test.js
+++ b/build/lib/test/i18n.test.js
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+var assert = require("assert");
+var i18n = require("../i18n");
+suite('XLF Parser Tests', function () {
+    var sampleXlf = '<?xml version="1.0" encoding="utf-8"?><xliff version="1.2" xmlns="urn:oasis:names:tc:xliff:document:1.2"><file original="vs/base/common/keybinding" source-language="en" datatype="plaintext"><body><trans-unit id="key1"><source xml:lang="en">Key #1</source></trans-unit><trans-unit id="key2"><source xml:lang="en">Key #2 &amp;</source></trans-unit></body></file></xliff>';
+    var sampleTranslatedXlf = '<?xml version="1.0" encoding="utf-8"?><xliff version="1.2" xmlns="urn:oasis:names:tc:xliff:document:1.2"><file original="vs/base/common/keybinding" source-language="en" target-language="ru" datatype="plaintext"><body><trans-unit id="key1"><source xml:lang="en">Key #1</source><target>Кнопка #1</target></trans-unit><trans-unit id="key2"><source xml:lang="en">Key #2 &amp;</source><target>Кнопка #2 &amp;</target></trans-unit></body></file></xliff>';
+    var originalFilePath = 'vs/base/common/keybinding';
+    var keys = ['key1', 'key2'];
+    var messages = ['Key #1', 'Key #2 &'];
+    var translatedMessages = { key1: 'Кнопка #1', key2: 'Кнопка #2 &' };
+    test('Keys & messages to XLF conversion', function () {
+        var xlf = new i18n.XLF('vscode-workbench');
+        xlf.addFile(originalFilePath, keys, messages);
+        var xlfString = xlf.toString();
+        assert.strictEqual(xlfString.replace(/\s{2,}/g, ''), sampleXlf);
+    });
+    test('XLF to keys & messages conversion', function () {
+        i18n.XLF.parse(sampleTranslatedXlf).then(function (resolvedFiles) {
+            assert.deepEqual(resolvedFiles[0].messages, translatedMessages);
+            assert.strictEqual(resolvedFiles[0].originalFilePath, originalFilePath);
+        });
+    });
+    test('JSON file source path to Transifex resource match', function () {
+        var editorProject = 'vscode-editor', workbenchProject = 'vscode-workbench';
+        var platform = { name: 'vs/platform', project: editorProject }, editorContrib = { name: 'vs/editor/contrib', project: editorProject }, editor = { name: 'vs/editor', project: editorProject }, base = { name: 'vs/base', project: editorProject }, code = { name: 'vs/code', project: workbenchProject }, workbenchParts = { name: 'vs/workbench/parts/html', project: workbenchProject }, workbenchServices = { name: 'vs/workbench/services/files', project: workbenchProject }, workbench = { name: 'vs/workbench', project: workbenchProject };
+        assert.deepEqual(i18n.getResource('vs/platform/actions/browser/menusExtensionPoint'), platform);
+        assert.deepEqual(i18n.getResource('vs/editor/contrib/clipboard/browser/clipboard'), editorContrib);
+        assert.deepEqual(i18n.getResource('vs/editor/common/modes/modesRegistry'), editor);
+        assert.deepEqual(i18n.getResource('vs/base/common/errorMessage'), base);
+        assert.deepEqual(i18n.getResource('vs/code/electron-main/window'), code);
+        assert.deepEqual(i18n.getResource('vs/workbench/parts/html/browser/webview'), workbenchParts);
+        assert.deepEqual(i18n.getResource('vs/workbench/services/files/node/fileService'), workbenchServices);
+        assert.deepEqual(i18n.getResource('vs/workbench/browser/parts/panel/panelActions'), workbench);
+    });
+});
--- a/build/lib/test/i18n.test.ts
+++ b/build/lib/test/i18n.test.ts
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import assert = require('assert');
+import i18n = require('../i18n');
+
+suite('XLF Parser Tests', () => {
+	const sampleXlf = '<?xml version="1.0" encoding="utf-8"?><xliff version="1.2" xmlns="urn:oasis:names:tc:xliff:document:1.2"><file original="vs/base/common/keybinding" source-language="en" datatype="plaintext"><body><trans-unit id="key1"><source xml:lang="en">Key #1</source></trans-unit><trans-unit id="key2"><source xml:lang="en">Key #2 &amp;</source></trans-unit></body></file></xliff>';
+	const sampleTranslatedXlf = '<?xml version="1.0" encoding="utf-8"?><xliff version="1.2" xmlns="urn:oasis:names:tc:xliff:document:1.2"><file original="vs/base/common/keybinding" source-language="en" target-language="ru" datatype="plaintext"><body><trans-unit id="key1"><source xml:lang="en">Key #1</source><target>Кнопка #1</target></trans-unit><trans-unit id="key2"><source xml:lang="en">Key #2 &amp;</source><target>Кнопка #2 &amp;</target></trans-unit></body></file></xliff>';
+	const originalFilePath = 'vs/base/common/keybinding';
+	const keys = ['key1', 'key2'];
+	const messages = ['Key #1', 'Key #2 &'];
+	const translatedMessages = { key1: 'Кнопка #1', key2: 'Кнопка #2 &' };
+
+	test('Keys & messages to XLF conversion', () => {
+		let xlf = new i18n.XLF('vscode-workbench');
+		xlf.addFile(originalFilePath, keys, messages);
+		const xlfString = xlf.toString();
+
+		assert.strictEqual(xlfString.replace(/\s{2,}/g, ''), sampleXlf);
+	});
+
+	test('XLF to keys & messages conversion', () => {
+		i18n.XLF.parse(sampleTranslatedXlf).then(function(resolvedFiles) {
+			assert.deepEqual(resolvedFiles[0].messages, translatedMessages);
+			assert.strictEqual(resolvedFiles[0].originalFilePath, originalFilePath);
+		});
+	});
+
+	test('JSON file source path to Transifex resource match', () => {
+		const editorProject: string = 'vscode-editor',
+			workbenchProject: string = 'vscode-workbench';
+
+		const platform: i18n.Resource = { name: 'vs/platform', project: editorProject },
+			editorContrib = { name: 'vs/editor/contrib', project: editorProject },
+			editor = { name: 'vs/editor', project: editorProject },
+			base = { name: 'vs/base', project: editorProject },
+			code = { name: 'vs/code', project: workbenchProject },
+			workbenchParts = { name: 'vs/workbench/parts/html', project: workbenchProject },
+			workbenchServices = { name: 'vs/workbench/services/files', project: workbenchProject },
+			workbench = { name: 'vs/workbench', project: workbenchProject};
+
+		assert.deepEqual(i18n.getResource('vs/platform/actions/browser/menusExtensionPoint'), platform);
+		assert.deepEqual(i18n.getResource('vs/editor/contrib/clipboard/browser/clipboard'), editorContrib);
+		assert.deepEqual(i18n.getResource('vs/editor/common/modes/modesRegistry'), editor);
+		assert.deepEqual(i18n.getResource('vs/base/common/errorMessage'), base);
+		assert.deepEqual(i18n.getResource('vs/code/electron-main/window'), code);
+		assert.deepEqual(i18n.getResource('vs/workbench/parts/html/browser/webview'), workbenchParts);
+		assert.deepEqual(i18n.getResource('vs/workbench/services/files/node/fileService'), workbenchServices);
+		assert.deepEqual(i18n.getResource('vs/workbench/browser/parts/panel/panelActions'), workbench);
+	});
+});
\ No newline at end of file
--- a/build/lib/typings/event-stream.d.ts
+++ b/build/lib/typings/event-stream.d.ts
 declare module "event-stream" {
 	import { Stream } from 'stream';
 	import { ThroughStream } from 'through';
+	import { MapStream } from 'map-stream';

 	function merge(streams: Stream[]): ThroughStream;
 	function merge(...streams: Stream[]): ThroughStream;
@@ -16,4 +17,5 @@ declare module "event-stream" {
 	function mapSync<I,O>(cb: (data:I) => O): ThroughStream;
 	function map<I,O>(cb: (data:I, cb:(err?:Error, data?: O)=>void) => O): ThroughStream;

+	function readable(asyncFunction: Function): MapStream;
 }
\ No newline at end of file
--- a/build/npm/postinstall.js
+++ b/build/npm/postinstall.js
@@ -35,4 +35,6 @@ const extensions = [
 	'gulp'
 ];

-extensions.forEach(extension => npmInstall(`extensions/${extension}`));
\ No newline at end of file
+extensions.forEach(extension => npmInstall(`extensions/${extension}`));
+
+npmInstall(`build`); // node modules required for build
\ No newline at end of file
--- a/build/package.json
+++ b/build/package.json
+{
+  "name": "code-oss-dev-build",
+  "version": "1.0.0",
+  "devDependencies": {
+	"@types/xml2js": "^0.0.33",
+	"xml2js": "^0.4.17"
+  }
+}
\ No newline at end of file
--- a/extensions/typescript/package.json
+++ b/extensions/typescript/package.json
@@ -425,4 +425,4 @@
      }
    ]
  }
-}
\ No newline at end of file
+}