Separated Default.isl and messages.isl in two separate projects. Introduced...

Separated Default.isl and messages.isl in two separate projects. Introduced language parameter for pulling XLFs.

build/gulpfile.vscode.js

@@ -340,6 +340,25 @@ gulp.task('vscode-linux-ia32-min', ['minify-vscode', 'clean-vscode-linux-ia32'],
 gulp.task('vscode-linux-x64-min', ['minify-vscode', 'clean-vscode-linux-x64'], packageTask('linux', 'x64', { minified: true }));
 gulp.task('vscode-linux-arm-min', ['minify-vscode', 'clean-vscode-linux-arm'], packageTask('linux', 'arm', { minified: true }));
 
+// Transifex Localizations
+const vscodeLanguages = [
+	'chs',
+	'cht',
+	'jpn',
+	'kor',
+	'deu',
+	'fra',
+	'esn',
+	'rus',
+	'ita'
+];
+
+const setupDefaultLanguages = [
+	'chs',
+	'cht',
+	'kor'
+];
+
 const apiHostname = process.env.TRANSIFEX_API_URL;
 const apiName = process.env.TRANSIFEX_API_NAME;
 const apiToken = process.env.TRANSIFEX_API_TOKEN;
@@ -358,8 +377,10 @@ gulp.task('vscode-translations-push', function() {
 
 gulp.task('vscode-translations-pull', function() {
 	return es.merge(
-		i18n.pullXlfFiles('vscode-editor-workbench', apiHostname, apiName, apiToken),
-		i18n.pullXlfFiles('vscode-extensions', apiHostname, apiName, apiToken)
+		i18n.pullXlfFiles('vscode-editor', apiHostname, apiName, apiToken, vscodeLanguages),
+		i18n.pullXlfFiles('vscode-workbench', apiHostname, apiName, apiToken, vscodeLanguages),
+		i18n.pullXlfFiles('vscode-extensions', apiHostname, apiName, apiToken, vscodeLanguages),
+		i18n.pullXlfFiles('vscode-setup', apiHostname, apiName, apiToken, setupDefaultLanguages)
 	).pipe(vfs.dest('../vscode-localization'));
 });
 

build/lib/i18n.js

@@ -501,17 +501,19 @@ function prepareXlfFiles(projectName, extensionName) {
     });
 }
 exports.prepareXlfFiles = prepareXlfFiles;
-var editorProject = 'vscode-editor', workbenchProject = 'vscode-workbench';
+var editorProject = 'vscode-editor', workbenchProject = 'vscode-workbench', setupProject = 'vscode-setup';
 /**
- * Ensure to update this when new resources are pushed to Transifex.
+ * Ensure to update those arrays when new resources are pushed to Transifex.
  * Used because Transifex does not have API method to pull all project resources.
  */
-var editorWorkbenchResources = [
+var editorResources = [
     { name: 'vs/platform', project: editorProject },
     { name: 'vs/editor/contrib', project: editorProject },
     { name: 'vs/editor', project: editorProject },
     { name: 'vs/base', project: editorProject },
-    { name: 'vs/code', project: workbenchProject },
+    { name: 'vs/code', project: workbenchProject }
+];
+var workbenchResources = [
     { name: 'vs/workbench', project: workbenchProject },
     { name: 'vs/workbench/parts/cli', project: workbenchProject },
     { name: 'vs/workbench/parts/codeEditor', project: workbenchProject },
@@ -548,7 +550,7 @@ var editorWorkbenchResources = [
     { name: 'vs/workbench/services/mode', project: workbenchProject },
     { name: 'vs/workbench/services/textfile', project: workbenchProject },
     { name: 'vs/workbench/services/themes', project: workbenchProject },
-    { name: 'setup', project: workbenchProject }
+    { name: 'setup_messages', project: workbenchProject }
 ];
 function getResource(sourceFile) {
     var resource;
@@ -638,11 +640,17 @@ function importModuleOrPackageJson(file, json, projectName, stream, extensionNam
         stream.emit('data', xlfFile);
     }
 }
-var islXlf, islProcessed = 0;
 function importIsl(file, stream) {
-    var islFiles = ['Default.isl', 'messages.en.isl'];
-    var projectName = workbenchProject;
-    var xlf = islXlf ? islXlf : islXlf = new XLF(projectName), keys = [], messages = [];
+    var projectName, resourceFile;
+    if (path.basename(file.path) === 'Default.isl') {
+        projectName = setupProject;
+        resourceFile = 'setup_default.xlf';
+    }
+    else {
+        projectName = workbenchProject;
+        resourceFile = 'setup_messages.xlf';
+    }
+    var xlf = new XLF(projectName), keys = [], messages = [];
     var model = new TextModel(file.contents.toString());
     var inMessageSection = false;
     model.lines.forEach(function (line) {
@@ -677,11 +685,9 @@ function importIsl(file, stream) {
     var originalPath = file.path.substring(file.cwd.length + 1, file.path.split('.')[0].length).replace(/\\/g, '/');
     xlf.addFile(originalPath, keys, messages);
     // Emit only upon all ISL files combined into single XLF instance
-    if (++islProcessed === islFiles.length) {
-        var newFilePath = path.join(projectName, 'setup.xlf');
-        var xlfFile = new File({ path: newFilePath, contents: new Buffer(xlf.toString(), 'utf-8') });
-        stream.emit('data', xlfFile);
-    }
+    var newFilePath = path.join(projectName, resourceFile);
+    var xlfFile = new File({ path: newFilePath, contents: new Buffer(xlf.toString(), 'utf-8') });
+    stream.emit('data', xlfFile);
 }
 function pushXlfFiles(apiHostname, username, password) {
     var tryGetPromises = [];
@@ -814,14 +820,16 @@ function updateResource(project, slug, xlfFile, apiHostname, credentials) {
     });
 }
 function obtainProjectResources(projectName) {
-    var resources;
-    if (projectName === 'vscode-editor-workbench') {
-        resources = editorWorkbenchResources;
+    var resources = [];
+    if (projectName === 'vscode-editor') {
+        resources = editorResources;
+    }
+    else if (projectName === 'vscode-workbench') {
+        resources = workbenchResources;
     }
     else if (projectName === 'vscode-extensions') {
         var extensionsToLocalize = glob.sync('./extensions/**/*.nls.json').map(function (extension) { return extension.split('/')[2]; });
         var resourcesToPull_1 = [];
-        resources = [];
         extensionsToLocalize.forEach(function (extension) {
             if (resourcesToPull_1.indexOf(extension) === -1) {
                 resourcesToPull_1.push(extension);
@@ -829,9 +837,12 @@ function obtainProjectResources(projectName) {
             }
         });
     }
+    else if (projectName === 'vscode-setup') {
+        resources.push({ name: 'setup_default', project: setupProject });
+    }
     return resources;
 }
-function pullXlfFiles(projectName, apiHostname, username, password, resources) {
+function pullXlfFiles(projectName, apiHostname, username, password, languages, resources) {
     if (!resources) {
         resources = obtainProjectResources(projectName);
     }
@@ -839,7 +850,7 @@ function pullXlfFiles(projectName, apiHostname, username, password, resources) {
         throw new Error('Transifex projects and resources must be defined to be able to pull translations from Transifex.');
     }
     var credentials = username + ":" + password;
-    var expectedTranslationsCount = vscodeLanguages.length * resources.length;
+    var expectedTranslationsCount = languages.length * resources.length;
     var translationsRetrieved = 0, called = false;
     return event_stream_1.readable(function (count, callback) {
         // Mark end of stream when all resources were retrieved
@@ -849,34 +860,13 @@ function pullXlfFiles(projectName, apiHostname, username, password, resources) {
         if (!called) {
             called = true;
             var stream_1 = this;
-            vscodeLanguages.map(function (language) {
+            // Retrieve XLF files from main projects
+            languages.map(function (language) {
                 resources.map(function (resource) {
-                    var slug = resource.name.replace(/\//g, '_');
-                    var project = resource.project;
-                    var iso639 = iso639_3_to_2[language];
-                    var options = {
-                        hostname: apiHostname,
-                        path: "/api/2/project/" + project + "/resource/" + slug + "/translation/" + iso639 + "?file&mode=onlyreviewed",
-                        auth: credentials,
-                        method: 'GET'
-                    };
-                    var request = http.request(options, function (res) {
-                        var xlfBuffer = '';
-                        res.on('data', function (data) { return xlfBuffer += data; });
-                        res.on('end', function () {
-                            if (res.statusCode === 200) {
-                                stream_1.emit('data', new File({ contents: new Buffer(xlfBuffer), path: project + "/" + language + "/" + slug + ".xlf" }));
-                            }
-                            else {
-                                throw new Error(slug + " in " + project + " returned no data. Response code: " + res.statusCode + ".");
-                            }
-                            translationsRetrieved++;
-                        });
-                    });
-                    request.on('error', function (err) {
-                        throw new Error("Failed to query resource " + slug + " with the following error: " + err);
-                    });
-                    request.end();
+                    retrieveResource(language, resource, apiHostname, credentials).then(function (file) {
+                        stream_1.emit('data', file);
+                        translationsRetrieved++;
+                    }).catch(function (error) { throw new Error(error); });
                 });
             });
         }
@@ -884,6 +874,33 @@ function pullXlfFiles(projectName, apiHostname, username, password, resources) {
     });
 }
 exports.pullXlfFiles = pullXlfFiles;
+function retrieveResource(language, resource, apiHostname, credentials) {
+    return new Promise(function (resolve, reject) {
+        var slug = resource.name.replace(/\//g, '_');
+        var project = resource.project;
+        var iso639 = iso639_3_to_2[language];
+        var options = {
+            hostname: apiHostname,
+            path: "/api/2/project/" + project + "/resource/" + slug + "/translation/" + iso639 + "?file&mode=onlyreviewed",
+            auth: credentials,
+            method: 'GET'
+        };
+        var request = http.request(options, function (res) {
+            var xlfBuffer = '';
+            res.on('data', function (data) { return xlfBuffer += data; });
+            res.on('end', function () {
+                if (res.statusCode === 200) {
+                    resolve(new File({ contents: new Buffer(xlfBuffer), path: project + "/" + language + "/" + slug + ".xlf" }));
+                }
+                reject(slug + " in " + project + " returned no data. Response code: " + res.statusCode + ".");
+            });
+        });
+        request.on('error', function (err) {
+            reject("Failed to query resource " + slug + " with the following error: " + err);
+        });
+        request.end();
+    });
+}
 function prepareJsonFiles() {
     return event_stream_1.through(function (xlf) {
         var stream = this;

build/lib/i18n.ts

@@ -570,18 +570,21 @@ export function prepareXlfFiles(projectName?: string, extensionName?: string): T
 }
 
 const editorProject: string = 'vscode-editor',
-	workbenchProject: string = 'vscode-workbench';
+	workbenchProject: string = 'vscode-workbench',
+	setupProject: string = 'vscode-setup';
 
 /**
- * Ensure to update this when new resources are pushed to Transifex.
+ * Ensure to update those arrays when new resources are pushed to Transifex.
  * Used because Transifex does not have API method to pull all project resources.
  */
-const editorWorkbenchResources: Resource[] = [
+const editorResources: Resource[] = [
 	{ name: 'vs/platform', project: editorProject },
 	{ name: 'vs/editor/contrib', project: editorProject },
 	{ name: 'vs/editor', project: editorProject },
 	{ name: 'vs/base', project: editorProject },
-	{ name: 'vs/code', project: workbenchProject },
+	{ name: 'vs/code', project: workbenchProject }
+];
+const workbenchResources: Resource[] = [
 	{ name: 'vs/workbench', project: workbenchProject },
 	{ name: 'vs/workbench/parts/cli', project: workbenchProject },
 	{ name: 'vs/workbench/parts/codeEditor', project: workbenchProject },
@@ -618,7 +621,7 @@ const editorWorkbenchResources: Resource[] = [
 	{ name: 'vs/workbench/services/mode', project: workbenchProject },
 	{ name: 'vs/workbench/services/textfile', project: workbenchProject },
 	{ name: 'vs/workbench/services/themes', project: workbenchProject },
-	{ name: 'setup', project: workbenchProject }
+	{ name: 'setup_messages', project: workbenchProject }
 ];
 
 export function getResource(sourceFile: string): Resource {
@@ -715,14 +718,18 @@ function importModuleOrPackageJson(file: File, json: ModuleJsonFormat | PackageJ
 	}
 }
 
-var islXlf: XLF,
-	islProcessed: number = 0;
-
 function importIsl(file: File, stream: ThroughStream) {
-	const islFiles = ['Default.isl', 'messages.en.isl'];
-	const projectName = workbenchProject;
+	let projectName: string,
+		resourceFile: string;
+	if (path.basename(file.path) === 'Default.isl') {
+		projectName = setupProject;
+		resourceFile = 'setup_default.xlf';
+	} else {
+		projectName = workbenchProject;
+		resourceFile = 'setup_messages.xlf';
+	}
 
-	let xlf = islXlf ? islXlf : islXlf = new XLF(projectName),
+	let xlf = new XLF(projectName),
 		keys: string[] = [],
 		messages: string[] = [];
 
@@ -761,11 +768,9 @@ function importIsl(file: File, stream: ThroughStream) {
 	xlf.addFile(originalPath, keys, messages);
 
 	// Emit only upon all ISL files combined into single XLF instance
-	if (++islProcessed === islFiles.length) {
-		const newFilePath = path.join(projectName, 'setup.xlf');
-		const xlfFile = new File({ path: newFilePath, contents: new Buffer(xlf.toString(), 'utf-8')});
-		stream.emit('data', xlfFile);
-	}
+	const newFilePath = path.join(projectName, resourceFile);
+	const xlfFile = new File({ path: newFilePath, contents: new Buffer(xlf.toString(), 'utf-8')});
+	stream.emit('data', xlfFile);
 }
 
 export function pushXlfFiles(apiHostname: string, username: string, password: string): ThroughStream {
@@ -906,14 +911,15 @@ function updateResource(project: string, slug: string, xlfFile: File, apiHostnam
 }
 
 function obtainProjectResources(projectName: string): Resource[] {
-	let resources: Resource[];
+	let resources: Resource[] = [];
 
-	if (projectName === 'vscode-editor-workbench') {
-		resources = editorWorkbenchResources;
+	if (projectName === 'vscode-editor') {
+		resources = editorResources;
+	} else if (projectName === 'vscode-workbench') {
+		resources = workbenchResources;
 	} else if (projectName === 'vscode-extensions') {
 		let extensionsToLocalize: string[] = glob.sync('./extensions/**/*.nls.json').map(extension => extension.split('/')[2]);
 		let resourcesToPull: string[] = [];
-		resources = [];
 
 		extensionsToLocalize.forEach(extension => {
 			if (resourcesToPull.indexOf(extension) === -1) { // remove duplicate elements returned by glob
@@ -921,12 +927,14 @@ function obtainProjectResources(projectName: string): Resource[] {
 				resources.push({ name: extension, project: projectName });
 			}
 		});
+	} else if (projectName === 'vscode-setup') {
+		resources.push({ name: 'setup_default', project: setupProject });
 	}
 
 	return resources;
 }
 
-export function pullXlfFiles(projectName: string, apiHostname: string, username: string, password: string, resources?: Resource[]): NodeJS.ReadableStream {
+export function pullXlfFiles(projectName: string, apiHostname: string, username: string, password: string, languages: string[], resources?: Resource[]): NodeJS.ReadableStream {
 	if (!resources) {
 		resources = obtainProjectResources(projectName);
 	}
@@ -935,7 +943,7 @@ export function pullXlfFiles(projectName: string, apiHostname: string, username:
 	}
 
 	const credentials = `${username}:${password}`;
-	let expectedTranslationsCount = vscodeLanguages.length * resources.length;
+	let expectedTranslationsCount = languages.length * resources.length;
 	let translationsRetrieved = 0, called = false;
 
 	return readable(function(count, callback) {
@@ -948,34 +956,13 @@ export function pullXlfFiles(projectName: string, apiHostname: string, username:
 			called = true;
 			const stream = this;
 
-			vscodeLanguages.map(function(language) {
+			// Retrieve XLF files from main projects
+			languages.map(function(language) {
 				resources.map(function(resource) {
-					const slug = resource.name.replace(/\//g, '_');
-					const project = resource.project;
-					const iso639 = iso639_3_to_2[language];
-					const options = {
-						hostname: apiHostname,
-						path: `/api/2/project/${project}/resource/${slug}/translation/${iso639}?file&mode=onlyreviewed`,
-						auth: credentials,
-						method: 'GET'
-					};
-
-					let request = http.request(options, (res) => {
-							let xlfBuffer: string = '';
-							res.on('data', (data) => xlfBuffer += data);
-							res.on('end', () => {
-								if (res.statusCode === 200) {
-									stream.emit('data', new File({ contents: new Buffer(xlfBuffer), path: `${project}/${language}/${slug}.xlf` }));
-								} else {
-									throw new Error(`${slug} in ${project} returned no data. Response code: ${res.statusCode}.`);
-								}
-								translationsRetrieved++;
-							});
-					});
-					request.on('error', (err) => {
-						throw new Error(`Failed to query resource ${slug} with the following error: ${err}`);
-					});
-					request.end();
+					retrieveResource(language, resource, apiHostname, credentials).then((file: File) => {
+						stream.emit('data', file);
+						translationsRetrieved++;
+					}).catch(error => { throw new Error(error); });
 				});
 			});
 		}
@@ -984,6 +971,35 @@ export function pullXlfFiles(projectName: string, apiHostname: string, username:
 	});
 }
 
+function retrieveResource(language: string, resource: Resource, apiHostname, credentials): Promise<File> {
+	return new Promise<File>((resolve, reject) => {
+		const slug = resource.name.replace(/\//g, '_');
+		const project = resource.project;
+		const iso639 = iso639_3_to_2[language];
+		const options = {
+			hostname: apiHostname,
+			path: `/api/2/project/${project}/resource/${slug}/translation/${iso639}?file&mode=onlyreviewed`,
+			auth: credentials,
+			method: 'GET'
+		};
+
+		let request = http.request(options, (res) => {
+				let xlfBuffer: string = '';
+				res.on('data', (data) => xlfBuffer += data);
+				res.on('end', () => {
+					if (res.statusCode === 200) {
+						resolve(new File({ contents: new Buffer(xlfBuffer), path: `${project}/${language}/${slug}.xlf` }));
+					}
+					reject(`${slug} in ${project} returned no data. Response code: ${res.statusCode}.`);
+				});
+		});
+		request.on('error', (err) => {
+			reject(`Failed to query resource ${slug} with the following error: ${err}`);
+		});
+		request.end();
+	});
+}
+
 export function prepareJsonFiles(): ThroughStream {
 	return through(function(xlf: File) {
 		let stream = this;