From 394e3d2cbe4e62d63c8c9c787b46527bbf72d4d8 Mon Sep 17 00:00:00 2001 From: Michel Kaporin Date: Wed, 5 Apr 2017 18:12:24 +0200 Subject: [PATCH] Separated Default.isl and messages.isl in two separate projects. Introduced language parameter for pulling XLFs. --- build/gulpfile.vscode.js | 25 ++++++++- build/lib/i18n.js | 111 ++++++++++++++++++++++---------------- build/lib/i18n.ts | 114 ++++++++++++++++++++++----------------- 3 files changed, 152 insertions(+), 98 deletions(-) diff --git a/build/gulpfile.vscode.js b/build/gulpfile.vscode.js index 7fdca5a65cd..da70c16af7d 100644 --- a/build/gulpfile.vscode.js +++ b/build/gulpfile.vscode.js @@ -340,6 +340,25 @@ gulp.task('vscode-linux-ia32-min', ['minify-vscode', 'clean-vscode-linux-ia32'], gulp.task('vscode-linux-x64-min', ['minify-vscode', 'clean-vscode-linux-x64'], packageTask('linux', 'x64', { minified: true })); gulp.task('vscode-linux-arm-min', ['minify-vscode', 'clean-vscode-linux-arm'], packageTask('linux', 'arm', { minified: true })); +// Transifex Localizations +const vscodeLanguages = [ + 'chs', + 'cht', + 'jpn', + 'kor', + 'deu', + 'fra', + 'esn', + 'rus', + 'ita' +]; + +const setupDefaultLanguages = [ + 'chs', + 'cht', + 'kor' +]; + const apiHostname = process.env.TRANSIFEX_API_URL; const apiName = process.env.TRANSIFEX_API_NAME; const apiToken = process.env.TRANSIFEX_API_TOKEN; @@ -358,8 +377,10 @@ gulp.task('vscode-translations-push', function() { gulp.task('vscode-translations-pull', function() { return es.merge( - i18n.pullXlfFiles('vscode-editor-workbench', apiHostname, apiName, apiToken), - i18n.pullXlfFiles('vscode-extensions', apiHostname, apiName, apiToken) + i18n.pullXlfFiles('vscode-editor', apiHostname, apiName, apiToken, vscodeLanguages), + i18n.pullXlfFiles('vscode-workbench', apiHostname, apiName, apiToken, vscodeLanguages), + i18n.pullXlfFiles('vscode-extensions', apiHostname, apiName, apiToken, vscodeLanguages), + i18n.pullXlfFiles('vscode-setup', apiHostname, apiName, apiToken, setupDefaultLanguages) ).pipe(vfs.dest('../vscode-localization')); }); diff --git a/build/lib/i18n.js b/build/lib/i18n.js index f7798ac0a12..0f50749fbcb 100644 --- a/build/lib/i18n.js +++ b/build/lib/i18n.js @@ -501,17 +501,19 @@ function prepareXlfFiles(projectName, extensionName) { }); } exports.prepareXlfFiles = prepareXlfFiles; -var editorProject = 'vscode-editor', workbenchProject = 'vscode-workbench'; +var editorProject = 'vscode-editor', workbenchProject = 'vscode-workbench', setupProject = 'vscode-setup'; /** - * Ensure to update this when new resources are pushed to Transifex. + * Ensure to update those arrays when new resources are pushed to Transifex. * Used because Transifex does not have API method to pull all project resources. */ -var editorWorkbenchResources = [ +var editorResources = [ { name: 'vs/platform', project: editorProject }, { name: 'vs/editor/contrib', project: editorProject }, { name: 'vs/editor', project: editorProject }, { name: 'vs/base', project: editorProject }, - { name: 'vs/code', project: workbenchProject }, + { name: 'vs/code', project: workbenchProject } +]; +var workbenchResources = [ { name: 'vs/workbench', project: workbenchProject }, { name: 'vs/workbench/parts/cli', project: workbenchProject }, { name: 'vs/workbench/parts/codeEditor', project: workbenchProject }, @@ -548,7 +550,7 @@ var editorWorkbenchResources = [ { name: 'vs/workbench/services/mode', project: workbenchProject }, { name: 'vs/workbench/services/textfile', project: workbenchProject }, { name: 'vs/workbench/services/themes', project: workbenchProject }, - { name: 'setup', project: workbenchProject } + { name: 'setup_messages', project: workbenchProject } ]; function getResource(sourceFile) { var resource; @@ -638,11 +640,17 @@ function importModuleOrPackageJson(file, json, projectName, stream, extensionNam stream.emit('data', xlfFile); } } -var islXlf, islProcessed = 0; function importIsl(file, stream) { - var islFiles = ['Default.isl', 'messages.en.isl']; - var projectName = workbenchProject; - var xlf = islXlf ? islXlf : islXlf = new XLF(projectName), keys = [], messages = []; + var projectName, resourceFile; + if (path.basename(file.path) === 'Default.isl') { + projectName = setupProject; + resourceFile = 'setup_default.xlf'; + } + else { + projectName = workbenchProject; + resourceFile = 'setup_messages.xlf'; + } + var xlf = new XLF(projectName), keys = [], messages = []; var model = new TextModel(file.contents.toString()); var inMessageSection = false; model.lines.forEach(function (line) { @@ -677,11 +685,9 @@ function importIsl(file, stream) { var originalPath = file.path.substring(file.cwd.length + 1, file.path.split('.')[0].length).replace(/\\/g, '/'); xlf.addFile(originalPath, keys, messages); // Emit only upon all ISL files combined into single XLF instance - if (++islProcessed === islFiles.length) { - var newFilePath = path.join(projectName, 'setup.xlf'); - var xlfFile = new File({ path: newFilePath, contents: new Buffer(xlf.toString(), 'utf-8') }); - stream.emit('data', xlfFile); - } + var newFilePath = path.join(projectName, resourceFile); + var xlfFile = new File({ path: newFilePath, contents: new Buffer(xlf.toString(), 'utf-8') }); + stream.emit('data', xlfFile); } function pushXlfFiles(apiHostname, username, password) { var tryGetPromises = []; @@ -814,14 +820,16 @@ function updateResource(project, slug, xlfFile, apiHostname, credentials) { }); } function obtainProjectResources(projectName) { - var resources; - if (projectName === 'vscode-editor-workbench') { - resources = editorWorkbenchResources; + var resources = []; + if (projectName === 'vscode-editor') { + resources = editorResources; + } + else if (projectName === 'vscode-workbench') { + resources = workbenchResources; } else if (projectName === 'vscode-extensions') { var extensionsToLocalize = glob.sync('./extensions/**/*.nls.json').map(function (extension) { return extension.split('/')[2]; }); var resourcesToPull_1 = []; - resources = []; extensionsToLocalize.forEach(function (extension) { if (resourcesToPull_1.indexOf(extension) === -1) { resourcesToPull_1.push(extension); @@ -829,9 +837,12 @@ function obtainProjectResources(projectName) { } }); } + else if (projectName === 'vscode-setup') { + resources.push({ name: 'setup_default', project: setupProject }); + } return resources; } -function pullXlfFiles(projectName, apiHostname, username, password, resources) { +function pullXlfFiles(projectName, apiHostname, username, password, languages, resources) { if (!resources) { resources = obtainProjectResources(projectName); } @@ -839,7 +850,7 @@ function pullXlfFiles(projectName, apiHostname, username, password, resources) { throw new Error('Transifex projects and resources must be defined to be able to pull translations from Transifex.'); } var credentials = username + ":" + password; - var expectedTranslationsCount = vscodeLanguages.length * resources.length; + var expectedTranslationsCount = languages.length * resources.length; var translationsRetrieved = 0, called = false; return event_stream_1.readable(function (count, callback) { // Mark end of stream when all resources were retrieved @@ -849,34 +860,13 @@ function pullXlfFiles(projectName, apiHostname, username, password, resources) { if (!called) { called = true; var stream_1 = this; - vscodeLanguages.map(function (language) { + // Retrieve XLF files from main projects + languages.map(function (language) { resources.map(function (resource) { - var slug = resource.name.replace(/\//g, '_'); - var project = resource.project; - var iso639 = iso639_3_to_2[language]; - var options = { - hostname: apiHostname, - path: "/api/2/project/" + project + "/resource/" + slug + "/translation/" + iso639 + "?file&mode=onlyreviewed", - auth: credentials, - method: 'GET' - }; - var request = http.request(options, function (res) { - var xlfBuffer = ''; - res.on('data', function (data) { return xlfBuffer += data; }); - res.on('end', function () { - if (res.statusCode === 200) { - stream_1.emit('data', new File({ contents: new Buffer(xlfBuffer), path: project + "/" + language + "/" + slug + ".xlf" })); - } - else { - throw new Error(slug + " in " + project + " returned no data. Response code: " + res.statusCode + "."); - } - translationsRetrieved++; - }); - }); - request.on('error', function (err) { - throw new Error("Failed to query resource " + slug + " with the following error: " + err); - }); - request.end(); + retrieveResource(language, resource, apiHostname, credentials).then(function (file) { + stream_1.emit('data', file); + translationsRetrieved++; + }).catch(function (error) { throw new Error(error); }); }); }); } @@ -884,6 +874,33 @@ function pullXlfFiles(projectName, apiHostname, username, password, resources) { }); } exports.pullXlfFiles = pullXlfFiles; +function retrieveResource(language, resource, apiHostname, credentials) { + return new Promise(function (resolve, reject) { + var slug = resource.name.replace(/\//g, '_'); + var project = resource.project; + var iso639 = iso639_3_to_2[language]; + var options = { + hostname: apiHostname, + path: "/api/2/project/" + project + "/resource/" + slug + "/translation/" + iso639 + "?file&mode=onlyreviewed", + auth: credentials, + method: 'GET' + }; + var request = http.request(options, function (res) { + var xlfBuffer = ''; + res.on('data', function (data) { return xlfBuffer += data; }); + res.on('end', function () { + if (res.statusCode === 200) { + resolve(new File({ contents: new Buffer(xlfBuffer), path: project + "/" + language + "/" + slug + ".xlf" })); + } + reject(slug + " in " + project + " returned no data. Response code: " + res.statusCode + "."); + }); + }); + request.on('error', function (err) { + reject("Failed to query resource " + slug + " with the following error: " + err); + }); + request.end(); + }); +} function prepareJsonFiles() { return event_stream_1.through(function (xlf) { var stream = this; diff --git a/build/lib/i18n.ts b/build/lib/i18n.ts index 37424500101..aeb03f601f2 100644 --- a/build/lib/i18n.ts +++ b/build/lib/i18n.ts @@ -570,18 +570,21 @@ export function prepareXlfFiles(projectName?: string, extensionName?: string): T } const editorProject: string = 'vscode-editor', - workbenchProject: string = 'vscode-workbench'; + workbenchProject: string = 'vscode-workbench', + setupProject: string = 'vscode-setup'; /** - * Ensure to update this when new resources are pushed to Transifex. + * Ensure to update those arrays when new resources are pushed to Transifex. * Used because Transifex does not have API method to pull all project resources. */ -const editorWorkbenchResources: Resource[] = [ +const editorResources: Resource[] = [ { name: 'vs/platform', project: editorProject }, { name: 'vs/editor/contrib', project: editorProject }, { name: 'vs/editor', project: editorProject }, { name: 'vs/base', project: editorProject }, - { name: 'vs/code', project: workbenchProject }, + { name: 'vs/code', project: workbenchProject } +]; +const workbenchResources: Resource[] = [ { name: 'vs/workbench', project: workbenchProject }, { name: 'vs/workbench/parts/cli', project: workbenchProject }, { name: 'vs/workbench/parts/codeEditor', project: workbenchProject }, @@ -618,7 +621,7 @@ const editorWorkbenchResources: Resource[] = [ { name: 'vs/workbench/services/mode', project: workbenchProject }, { name: 'vs/workbench/services/textfile', project: workbenchProject }, { name: 'vs/workbench/services/themes', project: workbenchProject }, - { name: 'setup', project: workbenchProject } + { name: 'setup_messages', project: workbenchProject } ]; export function getResource(sourceFile: string): Resource { @@ -715,14 +718,18 @@ function importModuleOrPackageJson(file: File, json: ModuleJsonFormat | PackageJ } } -var islXlf: XLF, - islProcessed: number = 0; - function importIsl(file: File, stream: ThroughStream) { - const islFiles = ['Default.isl', 'messages.en.isl']; - const projectName = workbenchProject; + let projectName: string, + resourceFile: string; + if (path.basename(file.path) === 'Default.isl') { + projectName = setupProject; + resourceFile = 'setup_default.xlf'; + } else { + projectName = workbenchProject; + resourceFile = 'setup_messages.xlf'; + } - let xlf = islXlf ? islXlf : islXlf = new XLF(projectName), + let xlf = new XLF(projectName), keys: string[] = [], messages: string[] = []; @@ -761,11 +768,9 @@ function importIsl(file: File, stream: ThroughStream) { xlf.addFile(originalPath, keys, messages); // Emit only upon all ISL files combined into single XLF instance - if (++islProcessed === islFiles.length) { - const newFilePath = path.join(projectName, 'setup.xlf'); - const xlfFile = new File({ path: newFilePath, contents: new Buffer(xlf.toString(), 'utf-8')}); - stream.emit('data', xlfFile); - } + const newFilePath = path.join(projectName, resourceFile); + const xlfFile = new File({ path: newFilePath, contents: new Buffer(xlf.toString(), 'utf-8')}); + stream.emit('data', xlfFile); } export function pushXlfFiles(apiHostname: string, username: string, password: string): ThroughStream { @@ -906,14 +911,15 @@ function updateResource(project: string, slug: string, xlfFile: File, apiHostnam } function obtainProjectResources(projectName: string): Resource[] { - let resources: Resource[]; + let resources: Resource[] = []; - if (projectName === 'vscode-editor-workbench') { - resources = editorWorkbenchResources; + if (projectName === 'vscode-editor') { + resources = editorResources; + } else if (projectName === 'vscode-workbench') { + resources = workbenchResources; } else if (projectName === 'vscode-extensions') { let extensionsToLocalize: string[] = glob.sync('./extensions/**/*.nls.json').map(extension => extension.split('/')[2]); let resourcesToPull: string[] = []; - resources = []; extensionsToLocalize.forEach(extension => { if (resourcesToPull.indexOf(extension) === -1) { // remove duplicate elements returned by glob @@ -921,12 +927,14 @@ function obtainProjectResources(projectName: string): Resource[] { resources.push({ name: extension, project: projectName }); } }); + } else if (projectName === 'vscode-setup') { + resources.push({ name: 'setup_default', project: setupProject }); } return resources; } -export function pullXlfFiles(projectName: string, apiHostname: string, username: string, password: string, resources?: Resource[]): NodeJS.ReadableStream { +export function pullXlfFiles(projectName: string, apiHostname: string, username: string, password: string, languages: string[], resources?: Resource[]): NodeJS.ReadableStream { if (!resources) { resources = obtainProjectResources(projectName); } @@ -935,7 +943,7 @@ export function pullXlfFiles(projectName: string, apiHostname: string, username: } const credentials = `${username}:${password}`; - let expectedTranslationsCount = vscodeLanguages.length * resources.length; + let expectedTranslationsCount = languages.length * resources.length; let translationsRetrieved = 0, called = false; return readable(function(count, callback) { @@ -948,34 +956,13 @@ export function pullXlfFiles(projectName: string, apiHostname: string, username: called = true; const stream = this; - vscodeLanguages.map(function(language) { + // Retrieve XLF files from main projects + languages.map(function(language) { resources.map(function(resource) { - const slug = resource.name.replace(/\//g, '_'); - const project = resource.project; - const iso639 = iso639_3_to_2[language]; - const options = { - hostname: apiHostname, - path: `/api/2/project/${project}/resource/${slug}/translation/${iso639}?file&mode=onlyreviewed`, - auth: credentials, - method: 'GET' - }; - - let request = http.request(options, (res) => { - let xlfBuffer: string = ''; - res.on('data', (data) => xlfBuffer += data); - res.on('end', () => { - if (res.statusCode === 200) { - stream.emit('data', new File({ contents: new Buffer(xlfBuffer), path: `${project}/${language}/${slug}.xlf` })); - } else { - throw new Error(`${slug} in ${project} returned no data. Response code: ${res.statusCode}.`); - } - translationsRetrieved++; - }); - }); - request.on('error', (err) => { - throw new Error(`Failed to query resource ${slug} with the following error: ${err}`); - }); - request.end(); + retrieveResource(language, resource, apiHostname, credentials).then((file: File) => { + stream.emit('data', file); + translationsRetrieved++; + }).catch(error => { throw new Error(error); }); }); }); } @@ -984,6 +971,35 @@ export function pullXlfFiles(projectName: string, apiHostname: string, username: }); } +function retrieveResource(language: string, resource: Resource, apiHostname, credentials): Promise { + return new Promise((resolve, reject) => { + const slug = resource.name.replace(/\//g, '_'); + const project = resource.project; + const iso639 = iso639_3_to_2[language]; + const options = { + hostname: apiHostname, + path: `/api/2/project/${project}/resource/${slug}/translation/${iso639}?file&mode=onlyreviewed`, + auth: credentials, + method: 'GET' + }; + + let request = http.request(options, (res) => { + let xlfBuffer: string = ''; + res.on('data', (data) => xlfBuffer += data); + res.on('end', () => { + if (res.statusCode === 200) { + resolve(new File({ contents: new Buffer(xlfBuffer), path: `${project}/${language}/${slug}.xlf` })); + } + reject(`${slug} in ${project} returned no data. Response code: ${res.statusCode}.`); + }); + }); + request.on('error', (err) => { + reject(`Failed to query resource ${slug} with the following error: ${err}`); + }); + request.end(); + }); +} + export function prepareJsonFiles(): ThroughStream { return through(function(xlf: File) { let stream = this; -- GitLab