"use strict"; /*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ Object.defineProperty(exports, "__esModule", { value: true }); const path = require("path"); const fs = require("fs"); const event_stream_1 = require("event-stream"); const File = require("vinyl"); const Is = require("is"); const xml2js = require("xml2js"); const glob = require("glob"); const https = require("https"); const gulp = require("gulp"); const fancyLog = require("fancy-log"); const ansiColors = require("ansi-colors"); const iconv = require("iconv-lite"); const NUMBER_OF_CONCURRENT_DOWNLOADS = 4; function log(message, ...rest) { fancyLog(ansiColors.green('[i18n]'), message, ...rest); } exports.defaultLanguages = [ { id: 'zh-tw', folderName: 'cht', translationId: 'zh-hant' }, { id: 'zh-cn', folderName: 'chs', translationId: 'zh-hans' }, { id: 'ja', folderName: 'jpn' }, { id: 'ko', folderName: 'kor' }, { id: 'de', folderName: 'deu' }, { id: 'fr', folderName: 'fra' }, { id: 'es', folderName: 'esn' }, { id: 'ru', folderName: 'rus' }, { id: 'it', folderName: 'ita' } ]; // languages requested by the community to non-stable builds exports.extraLanguages = [ { id: 'pt-br', folderName: 'ptb' }, { id: 'hu', folderName: 'hun' }, { id: 'tr', folderName: 'trk' } ]; // non built-in extensions also that are transifex and need to be part of the language packs exports.externalExtensionsWithTranslations = { 'vscode-chrome-debug': 'msjsdiag.debugger-for-chrome', 'vscode-node-debug': 'ms-vscode.node-debug', 'vscode-node-debug2': 'ms-vscode.node-debug2' }; var LocalizeInfo; (function (LocalizeInfo) { function is(value) { let candidate = value; return Is.defined(candidate) && Is.string(candidate.key) && (Is.undef(candidate.comment) || (Is.array(candidate.comment) && candidate.comment.every(element => Is.string(element)))); } LocalizeInfo.is = is; })(LocalizeInfo || (LocalizeInfo = {})); var BundledFormat; (function (BundledFormat) { function is(value) { if (Is.undef(value)) { return false; } let candidate = value; let length = Object.keys(value).length; return length === 3 && Is.defined(candidate.keys) && Is.defined(candidate.messages) && Is.defined(candidate.bundles); } BundledFormat.is = is; })(BundledFormat || (BundledFormat = {})); var PackageJsonFormat; (function (PackageJsonFormat) { function is(value) { if (Is.undef(value) || !Is.object(value)) { return false; } return Object.keys(value).every(key => { let element = value[key]; return Is.string(element) || (Is.object(element) && Is.defined(element.message) && Is.defined(element.comment)); }); } PackageJsonFormat.is = is; })(PackageJsonFormat || (PackageJsonFormat = {})); class Line { constructor(indent = 0) { this.buffer = []; if (indent > 0) { this.buffer.push(new Array(indent + 1).join(' ')); } } append(value) { this.buffer.push(value); return this; } toString() { return this.buffer.join(''); } } exports.Line = Line; class TextModel { constructor(contents) { this._lines = contents.split(/\r\n|\r|\n/); } get lines() { return this._lines; } } class XLF { constructor(project) { this.project = project; this.buffer = []; this.files = Object.create(null); this.numberOfMessages = 0; } toString() { this.appendHeader(); for (let file in this.files) { this.appendNewLine(``, 2); for (let item of this.files[file]) { this.addStringItem(item); } this.appendNewLine('', 2); } this.appendFooter(); return this.buffer.join('\r\n'); } addFile(original, keys, messages) { if (keys.length === 0) { console.log('No keys in ' + original); return; } if (keys.length !== messages.length) { throw new Error(`Unmatching keys(${keys.length}) and messages(${messages.length}).`); } this.numberOfMessages += keys.length; this.files[original] = []; let existingKeys = new Set(); for (let i = 0; i < keys.length; i++) { let key = keys[i]; let realKey; let comment; if (Is.string(key)) { realKey = key; comment = undefined; } else if (LocalizeInfo.is(key)) { realKey = key.key; if (key.comment && key.comment.length > 0) { comment = key.comment.map(comment => encodeEntities(comment)).join('\r\n'); } } if (!realKey || existingKeys.has(realKey)) { continue; } existingKeys.add(realKey); let message = encodeEntities(messages[i]); this.files[original].push({ id: realKey, message: message, comment: comment }); } } addStringItem(item) { if (!item.id || !item.message) { throw new Error(`No item ID or value specified: ${JSON.stringify(item)}`); } this.appendNewLine(``, 4); this.appendNewLine(`${item.message}`, 6); if (item.comment) { this.appendNewLine(`${item.comment}`, 6); } this.appendNewLine('', 4); } appendHeader() { this.appendNewLine('', 0); this.appendNewLine('', 0); } appendFooter() { this.appendNewLine('', 0); } appendNewLine(content, indent) { let line = new Line(indent); line.append(content); this.buffer.push(line.toString()); } } exports.XLF = XLF; XLF.parsePseudo = function (xlfString) { return new Promise((resolve) => { let parser = new xml2js.Parser(); let files = []; parser.parseString(xlfString, function (_err, result) { const fileNodes = result['xliff']['file']; fileNodes.forEach(file => { const originalFilePath = file.$.original; const messages = {}; const transUnits = file.body[0]['trans-unit']; if (transUnits) { transUnits.forEach((unit) => { const key = unit.$.id; const val = pseudify(unit.source[0]['_'].toString()); if (key && val) { messages[key] = decodeEntities(val); } }); files.push({ messages: messages, originalFilePath: originalFilePath, language: 'ps' }); } }); resolve(files); }); }); }; XLF.parse = function (xlfString) { return new Promise((resolve, reject) => { let parser = new xml2js.Parser(); let files = []; parser.parseString(xlfString, function (err, result) { if (err) { reject(new Error(`XLF parsing error: Failed to parse XLIFF string. ${err}`)); } const fileNodes = result['xliff']['file']; if (!fileNodes) { reject(new Error(`XLF parsing error: XLIFF file does not contain "xliff" or "file" node(s) required for parsing.`)); } fileNodes.forEach((file) => { const originalFilePath = file.$.original; if (!originalFilePath) { reject(new Error(`XLF parsing error: XLIFF file node does not contain original attribute to determine the original location of the resource file.`)); } let language = file.$['target-language']; if (!language) { reject(new Error(`XLF parsing error: XLIFF file node does not contain target-language attribute to determine translated language.`)); } const messages = {}; const transUnits = file.body[0]['trans-unit']; if (transUnits) { transUnits.forEach((unit) => { const key = unit.$.id; if (!unit.target) { return; // No translation available } let val = unit.target[0]; if (typeof val !== 'string') { val = val._; } if (key && val) { messages[key] = decodeEntities(val); } else { reject(new Error(`XLF parsing error: XLIFF file ${originalFilePath} does not contain full localization data. ID or target translation for one of the trans-unit nodes is not present.`)); } }); files.push({ messages: messages, originalFilePath: originalFilePath, language: language.toLowerCase() }); } }); resolve(files); }); }); }; class Limiter { constructor(maxDegreeOfParalellism) { this.maxDegreeOfParalellism = maxDegreeOfParalellism; this.outstandingPromises = []; this.runningPromises = 0; } queue(factory) { return new Promise((c, e) => { this.outstandingPromises.push({ factory, c, e }); this.consume(); }); } consume() { while (this.outstandingPromises.length && this.runningPromises < this.maxDegreeOfParalellism) { const iLimitedTask = this.outstandingPromises.shift(); this.runningPromises++; const promise = iLimitedTask.factory(); promise.then(iLimitedTask.c).catch(iLimitedTask.e); promise.then(() => this.consumed()).catch(() => this.consumed()); } } consumed() { this.runningPromises--; this.consume(); } } exports.Limiter = Limiter; function sortLanguages(languages) { return languages.sort((a, b) => { return a.id < b.id ? -1 : (a.id > b.id ? 1 : 0); }); } function stripComments(content) { /** * First capturing group matches double quoted string * Second matches single quotes string * Third matches block comments * Fourth matches line comments */ const regexp = /("(?:[^\\\"]*(?:\\.)?)*")|('(?:[^\\\']*(?:\\.)?)*')|(\/\*(?:\r?\n|.)*?\*\/)|(\/{2,}.*?(?:(?:\r?\n)|$))/g; let result = content.replace(regexp, (match, _m1, _m2, m3, m4) => { // Only one of m1, m2, m3, m4 matches if (m3) { // A block comment. Replace with nothing return ''; } else if (m4) { // A line comment. If it ends in \r?\n then keep it. let length = m4.length; if (length > 2 && m4[length - 1] === '\n') { return m4[length - 2] === '\r' ? '\r\n' : '\n'; } else { return ''; } } else { // We match a string return match; } }); return result; } function escapeCharacters(value) { const result = []; for (let i = 0; i < value.length; i++) { const ch = value.charAt(i); switch (ch) { case '\'': result.push('\\\''); break; case '"': result.push('\\"'); break; case '\\': result.push('\\\\'); break; case '\n': result.push('\\n'); break; case '\r': result.push('\\r'); break; case '\t': result.push('\\t'); break; case '\b': result.push('\\b'); break; case '\f': result.push('\\f'); break; default: result.push(ch); } } return result.join(''); } function processCoreBundleFormat(fileHeader, languages, json, emitter) { let keysSection = json.keys; let messageSection = json.messages; let bundleSection = json.bundles; let statistics = Object.create(null); let defaultMessages = Object.create(null); let modules = Object.keys(keysSection); modules.forEach((module) => { let keys = keysSection[module]; let messages = messageSection[module]; if (!messages || keys.length !== messages.length) { emitter.emit('error', `Message for module ${module} corrupted. Mismatch in number of keys and messages.`); return; } let messageMap = Object.create(null); defaultMessages[module] = messageMap; keys.map((key, i) => { if (typeof key === 'string') { messageMap[key] = messages[i]; } else { messageMap[key.key] = messages[i]; } }); }); let languageDirectory = path.join(__dirname, '..', '..', '..', 'vscode-loc', 'i18n'); if (!fs.existsSync(languageDirectory)) { log(`No VS Code localization repository found. Looking at ${languageDirectory}`); log(`To bundle translations please check out the vscode-loc repository as a sibling of the vscode repository.`); } let sortedLanguages = sortLanguages(languages); sortedLanguages.forEach((language) => { if (process.env['VSCODE_BUILD_VERBOSE']) { log(`Generating nls bundles for: ${language.id}`); } statistics[language.id] = 0; let localizedModules = Object.create(null); let languageFolderName = language.translationId || language.id; let i18nFile = path.join(languageDirectory, `vscode-language-pack-${languageFolderName}`, 'translations', 'main.i18n.json'); let allMessages; if (fs.existsSync(i18nFile)) { let content = stripComments(fs.readFileSync(i18nFile, 'utf8')); allMessages = JSON.parse(content); } modules.forEach((module) => { let order = keysSection[module]; let moduleMessage; if (allMessages) { moduleMessage = allMessages.contents[module]; } if (!moduleMessage) { if (process.env['VSCODE_BUILD_VERBOSE']) { log(`No localized messages found for module ${module}. Using default messages.`); } moduleMessage = defaultMessages[module]; statistics[language.id] = statistics[language.id] + Object.keys(moduleMessage).length; } let localizedMessages = []; order.forEach((keyInfo) => { let key = null; if (typeof keyInfo === 'string') { key = keyInfo; } else { key = keyInfo.key; } let message = moduleMessage[key]; if (!message) { if (process.env['VSCODE_BUILD_VERBOSE']) { log(`No localized message found for key ${key} in module ${module}. Using default message.`); } message = defaultMessages[module][key]; statistics[language.id] = statistics[language.id] + 1; } localizedMessages.push(message); }); localizedModules[module] = localizedMessages; }); Object.keys(bundleSection).forEach((bundle) => { let modules = bundleSection[bundle]; let contents = [ fileHeader, `define("${bundle}.nls.${language.id}", {` ]; modules.forEach((module, index) => { contents.push(`\t"${module}": [`); let messages = localizedModules[module]; if (!messages) { emitter.emit('error', `Didn't find messages for module ${module}.`); return; } messages.forEach((message, index) => { contents.push(`\t\t"${escapeCharacters(message)}${index < messages.length ? '",' : '"'}`); }); contents.push(index < modules.length - 1 ? '\t],' : '\t]'); }); contents.push('});'); emitter.queue(new File({ path: bundle + '.nls.' + language.id + '.js', contents: Buffer.from(contents.join('\n'), 'utf-8') })); }); }); Object.keys(statistics).forEach(key => { let value = statistics[key]; log(`${key} has ${value} untranslated strings.`); }); sortedLanguages.forEach(language => { let stats = statistics[language.id]; if (Is.undef(stats)) { log(`\tNo translations found for language ${language.id}. Using default language instead.`); } }); } function processNlsFiles(opts) { return event_stream_1.through(function (file) { let fileName = path.basename(file.path); if (fileName === 'nls.metadata.json') { let json = null; if (file.isBuffer()) { json = JSON.parse(file.contents.toString('utf8')); } else { this.emit('error', `Failed to read component file: ${file.relative}`); return; } if (BundledFormat.is(json)) { processCoreBundleFormat(opts.fileHeader, opts.languages, json, this); } } this.queue(file); }); } exports.processNlsFiles = processNlsFiles; const editorProject = 'vscode-editor', workbenchProject = 'vscode-workbench', extensionsProject = 'vscode-extensions', setupProject = 'vscode-setup'; function getResource(sourceFile) { let resource; if (/^vs\/platform/.test(sourceFile)) { return { name: 'vs/platform', project: editorProject }; } else if (/^vs\/editor\/contrib/.test(sourceFile)) { return { name: 'vs/editor/contrib', project: editorProject }; } else if (/^vs\/editor/.test(sourceFile)) { return { name: 'vs/editor', project: editorProject }; } else if (/^vs\/base/.test(sourceFile)) { return { name: 'vs/base', project: editorProject }; } else if (/^vs\/code/.test(sourceFile)) { return { name: 'vs/code', project: workbenchProject }; } else if (/^vs\/workbench\/contrib/.test(sourceFile)) { resource = sourceFile.split('/', 4).join('/'); return { name: resource, project: workbenchProject }; } else if (/^vs\/workbench\/services/.test(sourceFile)) { resource = sourceFile.split('/', 4).join('/'); return { name: resource, project: workbenchProject }; } else if (/^vs\/workbench/.test(sourceFile)) { return { name: 'vs/workbench', project: workbenchProject }; } throw new Error(`Could not identify the XLF bundle for ${sourceFile}`); } exports.getResource = getResource; function createXlfFilesForCoreBundle() { return event_stream_1.through(function (file) { const basename = path.basename(file.path); if (basename === 'nls.metadata.json') { if (file.isBuffer()) { const xlfs = Object.create(null); const json = JSON.parse(file.contents.toString('utf8')); for (let coreModule in json.keys) { const projectResource = getResource(coreModule); const resource = projectResource.name; const project = projectResource.project; const keys = json.keys[coreModule]; const messages = json.messages[coreModule]; if (keys.length !== messages.length) { this.emit('error', `There is a mismatch between keys and messages in ${file.relative} for module ${coreModule}`); return; } else { let xlf = xlfs[resource]; if (!xlf) { xlf = new XLF(project); xlfs[resource] = xlf; } xlf.addFile(`src/${coreModule}`, keys, messages); } } for (let resource in xlfs) { const xlf = xlfs[resource]; const filePath = `${xlf.project}/${resource.replace(/\//g, '_')}.xlf`; const xlfFile = new File({ path: filePath, contents: Buffer.from(xlf.toString(), 'utf8') }); this.queue(xlfFile); } } else { this.emit('error', new Error(`File ${file.relative} is not using a buffer content`)); return; } } else { this.emit('error', new Error(`File ${file.relative} is not a core meta data file.`)); return; } }); } exports.createXlfFilesForCoreBundle = createXlfFilesForCoreBundle; function createXlfFilesForExtensions() { let counter = 0; let folderStreamEnded = false; let folderStreamEndEmitted = false; return event_stream_1.through(function (extensionFolder) { const folderStream = this; const stat = fs.statSync(extensionFolder.path); if (!stat.isDirectory()) { return; } let extensionName = path.basename(extensionFolder.path); if (extensionName === 'node_modules') { return; } counter++; let _xlf; function getXlf() { if (!_xlf) { _xlf = new XLF(extensionsProject); } return _xlf; } gulp.src([`./extensions/${extensionName}/package.nls.json`, `./extensions/${extensionName}/**/nls.metadata.json`], { allowEmpty: true }).pipe(event_stream_1.through(function (file) { if (file.isBuffer()) { const buffer = file.contents; const basename = path.basename(file.path); if (basename === 'package.nls.json') { const json = JSON.parse(buffer.toString('utf8')); const keys = Object.keys(json); const messages = keys.map((key) => { const value = json[key]; if (Is.string(value)) { return value; } else if (value) { return value.message; } else { return `Unknown message for key: ${key}`; } }); getXlf().addFile(`extensions/${extensionName}/package`, keys, messages); } else if (basename === 'nls.metadata.json') { const json = JSON.parse(buffer.toString('utf8')); const relPath = path.relative(`./extensions/${extensionName}`, path.dirname(file.path)); for (let file in json) { const fileContent = json[file]; getXlf().addFile(`extensions/${extensionName}/${relPath}/${file}`, fileContent.keys, fileContent.messages); } } else { this.emit('error', new Error(`${file.path} is not a valid extension nls file`)); return; } } }, function () { if (_xlf) { let xlfFile = new File({ path: path.join(extensionsProject, extensionName + '.xlf'), contents: Buffer.from(_xlf.toString(), 'utf8') }); folderStream.queue(xlfFile); } this.queue(null); counter--; if (counter === 0 && folderStreamEnded && !folderStreamEndEmitted) { folderStreamEndEmitted = true; folderStream.queue(null); } })); }, function () { folderStreamEnded = true; if (counter === 0) { folderStreamEndEmitted = true; this.queue(null); } }); } exports.createXlfFilesForExtensions = createXlfFilesForExtensions; function createXlfFilesForIsl() { return event_stream_1.through(function (file) { let projectName, resourceFile; if (path.basename(file.path) === 'Default.isl') { projectName = setupProject; resourceFile = 'setup_default.xlf'; } else { projectName = workbenchProject; resourceFile = 'setup_messages.xlf'; } let xlf = new XLF(projectName), keys = [], messages = []; let model = new TextModel(file.contents.toString()); let inMessageSection = false; model.lines.forEach(line => { if (line.length === 0) { return; } let firstChar = line.charAt(0); switch (firstChar) { case ';': // Comment line; return; case '[': inMessageSection = '[Messages]' === line || '[CustomMessages]' === line; return; } if (!inMessageSection) { return; } let sections = line.split('='); if (sections.length !== 2) { throw new Error(`Badly formatted message found: ${line}`); } else { let key = sections[0]; let value = sections[1]; if (key.length > 0 && value.length > 0) { keys.push(key); messages.push(value); } } }); const originalPath = file.path.substring(file.cwd.length + 1, file.path.split('.')[0].length).replace(/\\/g, '/'); xlf.addFile(originalPath, keys, messages); // Emit only upon all ISL files combined into single XLF instance const newFilePath = path.join(projectName, resourceFile); const xlfFile = new File({ path: newFilePath, contents: Buffer.from(xlf.toString(), 'utf-8') }); this.queue(xlfFile); }); } exports.createXlfFilesForIsl = createXlfFilesForIsl; function pushXlfFiles(apiHostname, username, password) { let tryGetPromises = []; let updateCreatePromises = []; return event_stream_1.through(function (file) { const project = path.dirname(file.relative); const fileName = path.basename(file.path); const slug = fileName.substr(0, fileName.length - '.xlf'.length); const credentials = `${username}:${password}`; // Check if resource already exists, if not, then create it. let promise = tryGetResource(project, slug, apiHostname, credentials); tryGetPromises.push(promise); promise.then(exists => { if (exists) { promise = updateResource(project, slug, file, apiHostname, credentials); } else { promise = createResource(project, slug, file, apiHostname, credentials); } updateCreatePromises.push(promise); }); }, function () { // End the pipe only after all the communication with Transifex API happened Promise.all(tryGetPromises).then(() => { Promise.all(updateCreatePromises).then(() => { this.queue(null); }).catch((reason) => { throw new Error(reason); }); }).catch((reason) => { throw new Error(reason); }); }); } exports.pushXlfFiles = pushXlfFiles; function getAllResources(project, apiHostname, username, password) { return new Promise((resolve, reject) => { const credentials = `${username}:${password}`; const options = { hostname: apiHostname, path: `/api/2/project/${project}/resources`, auth: credentials, method: 'GET' }; const request = https.request(options, (res) => { let buffer = []; res.on('data', (chunk) => buffer.push(chunk)); res.on('end', () => { if (res.statusCode === 200) { let json = JSON.parse(Buffer.concat(buffer).toString()); if (Array.isArray(json)) { resolve(json.map(o => o.slug)); return; } reject(`Unexpected data format. Response code: ${res.statusCode}.`); } else { reject(`No resources in ${project} returned no data. Response code: ${res.statusCode}.`); } }); }); request.on('error', (err) => { reject(`Failed to query resources in ${project} with the following error: ${err}. ${options.path}`); }); request.end(); }); } function findObsoleteResources(apiHostname, username, password) { let resourcesByProject = Object.create(null); resourcesByProject[extensionsProject] = [].concat(exports.externalExtensionsWithTranslations); // clone return event_stream_1.through(function (file) { const project = path.dirname(file.relative); const fileName = path.basename(file.path); const slug = fileName.substr(0, fileName.length - '.xlf'.length); let slugs = resourcesByProject[project]; if (!slugs) { resourcesByProject[project] = slugs = []; } slugs.push(slug); this.push(file); }, function () { const json = JSON.parse(fs.readFileSync('./build/lib/i18n.resources.json', 'utf8')); let i18Resources = [...json.editor, ...json.workbench].map((r) => r.project + '/' + r.name.replace(/\//g, '_')); let extractedResources = []; for (let project of [workbenchProject, editorProject]) { for (let resource of resourcesByProject[project]) { if (resource !== 'setup_messages') { extractedResources.push(project + '/' + resource); } } } if (i18Resources.length !== extractedResources.length) { console.log(`[i18n] Obsolete resources in file 'build/lib/i18n.resources.json': JSON.stringify(${i18Resources.filter(p => extractedResources.indexOf(p) === -1)})`); console.log(`[i18n] Missing resources in file 'build/lib/i18n.resources.json': JSON.stringify(${extractedResources.filter(p => i18Resources.indexOf(p) === -1)})`); } let promises = []; for (let project in resourcesByProject) { promises.push(getAllResources(project, apiHostname, username, password).then(resources => { let expectedResources = resourcesByProject[project]; let unusedResources = resources.filter(resource => resource && expectedResources.indexOf(resource) === -1); if (unusedResources.length) { console.log(`[transifex] Obsolete resources in project '${project}': ${unusedResources.join(', ')}`); } })); } return Promise.all(promises).then(_ => { this.push(null); }).catch((reason) => { throw new Error(reason); }); }); } exports.findObsoleteResources = findObsoleteResources; function tryGetResource(project, slug, apiHostname, credentials) { return new Promise((resolve, reject) => { const options = { hostname: apiHostname, path: `/api/2/project/${project}/resource/${slug}/?details`, auth: credentials, method: 'GET' }; const request = https.request(options, (response) => { if (response.statusCode === 404) { resolve(false); } else if (response.statusCode === 200) { resolve(true); } else { reject(`Failed to query resource ${project}/${slug}. Response: ${response.statusCode} ${response.statusMessage}`); } }); request.on('error', (err) => { reject(`Failed to get ${project}/${slug} on Transifex: ${err}`); }); request.end(); }); } function createResource(project, slug, xlfFile, apiHostname, credentials) { return new Promise((_resolve, reject) => { const data = JSON.stringify({ 'content': xlfFile.contents.toString(), 'name': slug, 'slug': slug, 'i18n_type': 'XLIFF' }); const options = { hostname: apiHostname, path: `/api/2/project/${project}/resources`, headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(data) }, auth: credentials, method: 'POST' }; let request = https.request(options, (res) => { if (res.statusCode === 201) { log(`Resource ${project}/${slug} successfully created on Transifex.`); } else { reject(`Something went wrong in the request creating ${slug} in ${project}. ${res.statusCode}`); } }); request.on('error', (err) => { reject(`Failed to create ${project}/${slug} on Transifex: ${err}`); }); request.write(data); request.end(); }); } /** * The following link provides information about how Transifex handles updates of a resource file: * https://dev.befoolish.co/tx-docs/public/projects/updating-content#what-happens-when-you-update-files */ function updateResource(project, slug, xlfFile, apiHostname, credentials) { return new Promise((resolve, reject) => { const data = JSON.stringify({ content: xlfFile.contents.toString() }); const options = { hostname: apiHostname, path: `/api/2/project/${project}/resource/${slug}/content`, headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(data) }, auth: credentials, method: 'PUT' }; let request = https.request(options, (res) => { if (res.statusCode === 200) { res.setEncoding('utf8'); let responseBuffer = ''; res.on('data', function (chunk) { responseBuffer += chunk; }); res.on('end', () => { const response = JSON.parse(responseBuffer); log(`Resource ${project}/${slug} successfully updated on Transifex. Strings added: ${response.strings_added}, updated: ${response.strings_added}, deleted: ${response.strings_added}`); resolve(); }); } else { reject(`Something went wrong in the request updating ${slug} in ${project}. ${res.statusCode}`); } }); request.on('error', (err) => { reject(`Failed to update ${project}/${slug} on Transifex: ${err}`); }); request.write(data); request.end(); }); } // cache resources let _coreAndExtensionResources; function pullCoreAndExtensionsXlfFiles(apiHostname, username, password, language, externalExtensions) { if (!_coreAndExtensionResources) { _coreAndExtensionResources = []; // editor and workbench const json = JSON.parse(fs.readFileSync('./build/lib/i18n.resources.json', 'utf8')); _coreAndExtensionResources.push(...json.editor); _coreAndExtensionResources.push(...json.workbench); // extensions let extensionsToLocalize = Object.create(null); glob.sync('./extensions/**/*.nls.json').forEach(extension => extensionsToLocalize[extension.split('/')[2]] = true); glob.sync('./extensions/*/node_modules/vscode-nls').forEach(extension => extensionsToLocalize[extension.split('/')[2]] = true); Object.keys(extensionsToLocalize).forEach(extension => { _coreAndExtensionResources.push({ name: extension, project: extensionsProject }); }); if (externalExtensions) { for (let resourceName in externalExtensions) { _coreAndExtensionResources.push({ name: resourceName, project: extensionsProject }); } } } return pullXlfFiles(apiHostname, username, password, language, _coreAndExtensionResources); } exports.pullCoreAndExtensionsXlfFiles = pullCoreAndExtensionsXlfFiles; function pullSetupXlfFiles(apiHostname, username, password, language, includeDefault) { let setupResources = [{ name: 'setup_messages', project: workbenchProject }]; if (includeDefault) { setupResources.push({ name: 'setup_default', project: setupProject }); } return pullXlfFiles(apiHostname, username, password, language, setupResources); } exports.pullSetupXlfFiles = pullSetupXlfFiles; function pullXlfFiles(apiHostname, username, password, language, resources) { const credentials = `${username}:${password}`; let expectedTranslationsCount = resources.length; let translationsRetrieved = 0, called = false; return event_stream_1.readable(function (_count, callback) { // Mark end of stream when all resources were retrieved if (translationsRetrieved === expectedTranslationsCount) { return this.emit('end'); } if (!called) { called = true; const stream = this; resources.map(function (resource) { retrieveResource(language, resource, apiHostname, credentials).then((file) => { if (file) { stream.emit('data', file); } translationsRetrieved++; }).catch(error => { throw new Error(error); }); }); } callback(); }); } const limiter = new Limiter(NUMBER_OF_CONCURRENT_DOWNLOADS); function retrieveResource(language, resource, apiHostname, credentials) { return limiter.queue(() => new Promise((resolve, reject) => { const slug = resource.name.replace(/\//g, '_'); const project = resource.project; let transifexLanguageId = language.id === 'ps' ? 'en' : language.translationId || language.id; const options = { hostname: apiHostname, path: `/api/2/project/${project}/resource/${slug}/translation/${transifexLanguageId}?file&mode=onlyreviewed`, auth: credentials, port: 443, method: 'GET' }; console.log('[transifex] Fetching ' + options.path); let request = https.request(options, (res) => { let xlfBuffer = []; res.on('data', (chunk) => xlfBuffer.push(chunk)); res.on('end', () => { if (res.statusCode === 200) { resolve(new File({ contents: Buffer.concat(xlfBuffer), path: `${project}/${slug}.xlf` })); } else if (res.statusCode === 404) { console.log(`[transifex] ${slug} in ${project} returned no data.`); resolve(null); } else { reject(`${slug} in ${project} returned no data. Response code: ${res.statusCode}.`); } }); }); request.on('error', (err) => { reject(`Failed to query resource ${slug} with the following error: ${err}. ${options.path}`); }); request.end(); })); } function prepareI18nFiles() { let parsePromises = []; return event_stream_1.through(function (xlf) { let stream = this; let parsePromise = XLF.parse(xlf.contents.toString()); parsePromises.push(parsePromise); parsePromise.then(resolvedFiles => { resolvedFiles.forEach(file => { let translatedFile = createI18nFile(file.originalFilePath, file.messages); stream.queue(translatedFile); }); }); }, function () { Promise.all(parsePromises) .then(() => { this.queue(null); }) .catch(reason => { throw new Error(reason); }); }); } exports.prepareI18nFiles = prepareI18nFiles; function createI18nFile(originalFilePath, messages) { let result = Object.create(null); result[''] = [ '--------------------------------------------------------------------------------------------', 'Copyright (c) Microsoft Corporation. All rights reserved.', 'Licensed under the MIT License. See License.txt in the project root for license information.', '--------------------------------------------------------------------------------------------', 'Do not edit this file. It is machine generated.' ]; for (let key of Object.keys(messages)) { result[key] = messages[key]; } let content = JSON.stringify(result, null, '\t'); if (process.platform === 'win32') { content = content.replace(/\n/g, '\r\n'); } return new File({ path: path.join(originalFilePath + '.i18n.json'), contents: Buffer.from(content, 'utf8') }); } const i18nPackVersion = "1.0.0"; function pullI18nPackFiles(apiHostname, username, password, language, resultingTranslationPaths) { return pullCoreAndExtensionsXlfFiles(apiHostname, username, password, language, exports.externalExtensionsWithTranslations) .pipe(prepareI18nPackFiles(exports.externalExtensionsWithTranslations, resultingTranslationPaths, language.id === 'ps')); } exports.pullI18nPackFiles = pullI18nPackFiles; function prepareI18nPackFiles(externalExtensions, resultingTranslationPaths, pseudo = false) { let parsePromises = []; let mainPack = { version: i18nPackVersion, contents: {} }; let extensionsPacks = {}; let errors = []; return event_stream_1.through(function (xlf) { let project = path.basename(path.dirname(xlf.relative)); let resource = path.basename(xlf.relative, '.xlf'); let contents = xlf.contents.toString(); let parsePromise = pseudo ? XLF.parsePseudo(contents) : XLF.parse(contents); parsePromises.push(parsePromise); parsePromise.then(resolvedFiles => { resolvedFiles.forEach(file => { const path = file.originalFilePath; const firstSlash = path.indexOf('/'); if (project === extensionsProject) { let extPack = extensionsPacks[resource]; if (!extPack) { extPack = extensionsPacks[resource] = { version: i18nPackVersion, contents: {} }; } const externalId = externalExtensions[resource]; if (!externalId) { // internal extension: remove 'extensions/extensionId/' segnent const secondSlash = path.indexOf('/', firstSlash + 1); extPack.contents[path.substr(secondSlash + 1)] = file.messages; } else { extPack.contents[path] = file.messages; } } else { mainPack.contents[path.substr(firstSlash + 1)] = file.messages; } }); }).catch(reason => { errors.push(reason); }); }, function () { Promise.all(parsePromises) .then(() => { if (errors.length > 0) { throw errors; } const translatedMainFile = createI18nFile('./main', mainPack); resultingTranslationPaths.push({ id: 'vscode', resourceName: 'main.i18n.json' }); this.queue(translatedMainFile); for (let extension in extensionsPacks) { const translatedExtFile = createI18nFile(`./extensions/${extension}`, extensionsPacks[extension]); this.queue(translatedExtFile); const externalExtensionId = externalExtensions[extension]; if (externalExtensionId) { resultingTranslationPaths.push({ id: externalExtensionId, resourceName: `extensions/${extension}.i18n.json` }); } else { resultingTranslationPaths.push({ id: `vscode.${extension}`, resourceName: `extensions/${extension}.i18n.json` }); } } this.queue(null); }) .catch((reason) => { this.emit('error', reason); }); }); } exports.prepareI18nPackFiles = prepareI18nPackFiles; function prepareIslFiles(language, innoSetupConfig) { let parsePromises = []; return event_stream_1.through(function (xlf) { let stream = this; let parsePromise = XLF.parse(xlf.contents.toString()); parsePromises.push(parsePromise); parsePromise.then(resolvedFiles => { resolvedFiles.forEach(file => { if (path.basename(file.originalFilePath) === 'Default' && !innoSetupConfig.defaultInfo) { return; } let translatedFile = createIslFile(file.originalFilePath, file.messages, language, innoSetupConfig); stream.queue(translatedFile); }); }).catch(reason => { this.emit('error', reason); }); }, function () { Promise.all(parsePromises) .then(() => { this.queue(null); }) .catch(reason => { this.emit('error', reason); }); }); } exports.prepareIslFiles = prepareIslFiles; function createIslFile(originalFilePath, messages, language, innoSetup) { let content = []; let originalContent; if (path.basename(originalFilePath) === 'Default') { originalContent = new TextModel(fs.readFileSync(originalFilePath + '.isl', 'utf8')); } else { originalContent = new TextModel(fs.readFileSync(originalFilePath + '.en.isl', 'utf8')); } originalContent.lines.forEach(line => { if (line.length > 0) { let firstChar = line.charAt(0); if (firstChar === '[' || firstChar === ';') { if (line === '; *** Inno Setup version 5.5.3+ English messages ***') { content.push(`; *** Inno Setup version 5.5.3+ ${innoSetup.defaultInfo.name} messages ***`); } else { content.push(line); } } else { let sections = line.split('='); let key = sections[0]; let translated = line; if (key) { if (key === 'LanguageName') { translated = `${key}=${innoSetup.defaultInfo.name}`; } else if (key === 'LanguageID') { translated = `${key}=${innoSetup.defaultInfo.id}`; } else if (key === 'LanguageCodePage') { translated = `${key}=${innoSetup.codePage.substr(2)}`; } else { let translatedMessage = messages[key]; if (translatedMessage) { translated = `${key}=${translatedMessage}`; } } } content.push(translated); } } }); const basename = path.basename(originalFilePath); const filePath = `${basename}.${language.id}.isl`; return new File({ path: filePath, contents: iconv.encode(Buffer.from(content.join('\r\n'), 'utf8').toString(), innoSetup.codePage) }); } function encodeEntities(value) { let result = []; for (let i = 0; i < value.length; i++) { let ch = value[i]; switch (ch) { case '<': result.push('<'); break; case '>': result.push('>'); break; case '&': result.push('&'); break; default: result.push(ch); } } return result.join(''); } function decodeEntities(value) { return value.replace(/</g, '<').replace(/>/g, '>').replace(/&/g, '&'); } function pseudify(message) { return '\uFF3B' + message.replace(/[aouei]/g, '$&$&') + '\uFF3D'; }