diff --git a/dev-packages/cli/src/theia.ts b/dev-packages/cli/src/theia.ts index b41ab5032622c..d783700ce81e5 100644 --- a/dev-packages/cli/src/theia.ts +++ b/dev-packages/cli/src/theia.ts @@ -25,7 +25,7 @@ import * as ffmpeg from '@theia/ffmpeg'; import checkHoisted from './check-hoisting'; import downloadPlugins from './download-plugins'; import runTest from './run-test'; -import { extract } from '@theia/localization-manager'; +import { LocalizationManager, extract } from '@theia/localization-manager'; process.on('unhandledRejection', (reason, promise) => { throw reason; @@ -110,6 +110,7 @@ async function theiaCli(): Promise { // affecting the global `yargs` instance used by the CLI. const { appTarget } = defineCommonOptions(yargsFactory()).help(false).parse(); const manager = new ApplicationPackageManager({ projectPath, appTarget }); + const localizationManager = new LocalizationManager(); const { target } = manager.pck; defineCommonOptions(yargs) .command<{ @@ -228,6 +229,46 @@ async function theiaCli(): Promise { await downloadPlugins({ packed }); }, }) + .command<{ + freeApi?: boolean, + deeplKey: string, + file: string, + languages: string[], + sourceLanguage?: string + }>({ + command: 'nls-localize [languages...]', + describe: 'Localize json files using the DeepL API', + builder: { + 'file': { + alias: 'f', + describe: 'The source file which should be translated', + demandOption: true + }, + 'deepl-key': { + alias: 'k', + describe: 'DeepL key used for API access. See https://www.deepl.com/docs-api for more information', + demandOption: true + }, + 'free-api': { + describe: 'Indicates whether the specified DeepL API key belongs to the free API', + boolean: true, + default: false, + }, + 'source-language': { + alias: 's', + describe: 'The source language of the translation file' + } + }, + handler: async ({ freeApi, deeplKey, file, sourceLanguage, languages = [] }) => { + await localizationManager.localize({ + sourceFile: file, + freeApi: freeApi ?? true, + authKey: deeplKey, + targetLanguages: languages, + sourceLanguage + }); + } + }) .command<{ root: string, output: string, diff --git a/dev-packages/localization-manager/README.md b/dev-packages/localization-manager/README.md index b6860583565ff..b4cffc00468c2 100644 --- a/dev-packages/localization-manager/README.md +++ b/dev-packages/localization-manager/README.md @@ -12,8 +12,45 @@ ## Description -The `@theia/localization-manager` package is used easily create localizations of Theia and Theia extensions for different languages. -Its main use case is to extract localization keys and default values from `nls.localize` calls within the codebase. +The `@theia/localization-manager` package is used easily create localizations of Theia and Theia extensions for different languages. It has two main use cases. + +First, it allows to extract localization keys and default values from `nls.localize` calls within the codebase using the `nls-extract` Theia-CLI command. Take this code for example: + +```ts +const hi = nls.localize('greetings/hi', 'Hello'); +const bye = nls.localize('greetings/bye', 'Bye'); +``` + +It will be converted into this JSON file (`nls.json`): + +```json +{ + "greetings": { + "hi": "Hello", + "bye": "Bye" + } +} +``` + +Afterwards, any manual or automatic translation approach can be used to translate this file into other languages. These JSON files are supposed to be picked up by `LocalizationContribution`s. + +Additionally, Theia provides a simple way to translate the generated JSON files out of the box using the [DeepL API](https://www.deepl.com/docs-api). For this, a [DeepL free or pro account](https://www.deepl.com/pro) is needed. Using the `nls-localize` command of the Theia-CLI, a target file can be translated into different languages. For example, when calling the command using the previous JSON file with the `fr` (french) language, the following `nls.fr.json` file will be created in the same directory as the translation source: + +```json +{ + "greetings": { + "hi": "Bonjour", + "bye": "Au revoir" + } +} +``` + + +Only JSON entries without corresponding translations are translated using DeepL. This ensures that manual changes to the translated files aren't overwritten and only new translation entries are actually sent to DeepL. + +Use `theia nls-localize --help` for more information on how to use the command and supply DeepL API keys. + +For more information, see the [internationalization documentation](https://theia-ide.org/docs/i18n/). ## Additional Information diff --git a/dev-packages/localization-manager/package.json b/dev-packages/localization-manager/package.json index 241977029546f..a019c28cd5343 100644 --- a/dev-packages/localization-manager/package.json +++ b/dev-packages/localization-manager/package.json @@ -29,7 +29,10 @@ "watch": "theiaext watch" }, "dependencies": { + "@types/bent": "^7.0.1", "@types/fs-extra": "^4.0.2", + "bent": "^7.1.0", + "chalk": "4.0.0", "deepmerge": "^4.2.2", "fs-extra": "^4.0.2", "glob": "^7.2.0", diff --git a/dev-packages/localization-manager/src/common.ts b/dev-packages/localization-manager/src/common.ts new file mode 100644 index 0000000000000..a23d798122396 --- /dev/null +++ b/dev-packages/localization-manager/src/common.ts @@ -0,0 +1,19 @@ +/******************************************************************************** + * Copyright (C) 2021 TypeFox and others. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * This Source Code may also be made available under the following Secondary + * Licenses when the conditions for such availability set forth in the Eclipse + * Public License v. 2.0 are satisfied: GNU General Public License, version 2 + * with the GNU Classpath Exception which is available at + * https://www.gnu.org/software/classpath/license.html. + * + * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 + ********************************************************************************/ + +export interface Localization { + [key: string]: string | Localization +} diff --git a/dev-packages/localization-manager/src/deepl-api.ts b/dev-packages/localization-manager/src/deepl-api.ts new file mode 100644 index 0000000000000..ac03647e41331 --- /dev/null +++ b/dev-packages/localization-manager/src/deepl-api.ts @@ -0,0 +1,122 @@ +/******************************************************************************** + * Copyright (C) 2021 TypeFox and others. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * This Source Code may also be made available under the following Secondary + * Licenses when the conditions for such availability set forth in the Eclipse + * Public License v. 2.0 are satisfied: GNU General Public License, version 2 + * with the GNU Classpath Exception which is available at + * https://www.gnu.org/software/classpath/license.html. + * + * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 + ********************************************************************************/ + +import * as bent from 'bent'; + +const post = bent('POST', 'json', 200); +// 50 is the maximum amount of translations per request +const deeplLimit = 50; + +export async function deepl( + parameters: DeeplParameters +): Promise { + const sub_domain = parameters.free_api ? 'api-free' : 'api'; + const textChunks: string[][] = []; + const textArray = [...parameters.text]; + while (textArray.length > 0) { + textChunks.push(textArray.splice(0, deeplLimit)); + } + const responses: DeeplResponse[] = await Promise.all(textChunks.map(chunk => { + const parameterCopy: DeeplParameters = { ...parameters, text: chunk }; + return post(`https://${sub_domain}.deepl.com/v2/translate`, Buffer.from(toFormData(parameterCopy)), { + 'Content-Type': 'application/x-www-form-urlencoded', + 'User-Agent': 'Theia-Localization-Manager' + }); + })); + const mergedResponse: DeeplResponse = { translations: [] }; + for (const response of responses) { + mergedResponse.translations.push(...response.translations); + } + return mergedResponse; +} + +function toFormData(parameters: DeeplParameters): string { + const str: string[] = []; + for (const [key, value] of Object.entries(parameters)) { + if (typeof value === 'string') { + str.push(encodeURIComponent(key) + '=' + encodeURIComponent(value.toString())); + } else if (Array.isArray(value)) { + for (const item of value) { + str.push(encodeURIComponent(key) + '=' + encodeURIComponent(item.toString())); + } + } + } + return str.join('&'); +} + +export type DeeplLanguage = + | 'BG' + | 'CS' + | 'DA' + | 'DE' + | 'EL' + | 'EN-GB' + | 'EN-US' + | 'EN' + | 'ES' + | 'ET' + | 'FI' + | 'FR' + | 'HU' + | 'IT' + | 'JA' + | 'LT' + | 'LV' + | 'NL' + | 'PL' + | 'PT-PT' + | 'PT-BR' + | 'PT' + | 'RO' + | 'RU' + | 'SK' + | 'SL' + | 'SV' + | 'ZH'; + +export const supportedLanguages = [ + 'BG', 'CD', 'DA', 'DE', 'EL', 'EN-GB', 'EN-US', 'EN', 'ES', 'ET', 'FI', 'FR', 'HU', 'IT', + 'JA', 'LT', 'LV', 'NL', 'PL', 'PT-PT', 'PT-BR', 'PT', 'RO', 'RU', 'SK', 'SL', 'SV', 'ZH' +]; + +export function isSupportedLanguage(language: string): language is DeeplLanguage { + return supportedLanguages.includes(language.toUpperCase()); +} + +export interface DeeplParameters { + free_api: Boolean + auth_key: string + text: string[] + source_lang?: DeeplLanguage + target_lang: DeeplLanguage + split_sentences?: '0' | '1' | 'nonewlines' + preserve_formatting?: '0' | '1' + formality?: 'default' | 'more' | 'less' + tag_handling?: string[] + non_splitting_tags?: string[] + outline_detection?: string + splitting_tags?: string[] + ignore_tags?: string[] +} + +export interface DeeplResponse { + translations: DeeplTranslation[] +} + +export interface DeeplTranslation { + detected_source_language: string + text: string +} diff --git a/dev-packages/localization-manager/src/index.ts b/dev-packages/localization-manager/src/index.ts index 7de6b549fe8d5..cec1aa9a8b67b 100644 --- a/dev-packages/localization-manager/src/index.ts +++ b/dev-packages/localization-manager/src/index.ts @@ -14,4 +14,6 @@ * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 ********************************************************************************/ +export * from './common'; export * from './localization-extractor'; +export * from './localization-manager'; diff --git a/dev-packages/localization-manager/src/localization-extractor.ts b/dev-packages/localization-manager/src/localization-extractor.ts index e366199a276f3..fc956bf1f5226 100644 --- a/dev-packages/localization-manager/src/localization-extractor.ts +++ b/dev-packages/localization-manager/src/localization-extractor.ts @@ -21,13 +21,10 @@ import * as path from 'path'; import { glob } from 'glob'; import { promisify } from 'util'; import deepmerge = require('deepmerge'); +import { Localization } from './common'; const globPromise = promisify(glob); -export interface Localization { - [key: string]: string | Localization -} - export interface ExtractionOptions { root: string output: string diff --git a/dev-packages/localization-manager/src/localization-manager.spec.ts b/dev-packages/localization-manager/src/localization-manager.spec.ts new file mode 100644 index 0000000000000..f75ed1c901990 --- /dev/null +++ b/dev-packages/localization-manager/src/localization-manager.spec.ts @@ -0,0 +1,80 @@ +/******************************************************************************** + * Copyright (C) 2021 TypeFox and others. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * This Source Code may also be made available under the following Secondary + * Licenses when the conditions for such availability set forth in the Eclipse + * Public License v. 2.0 are satisfied: GNU General Public License, version 2 + * with the GNU Classpath Exception which is available at + * https://www.gnu.org/software/classpath/license.html. + * + * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 + ********************************************************************************/ + +import * as assert from 'assert'; +import { DeeplParameters, DeeplResponse } from './deepl-api'; +import { LocalizationManager, LocalizationOptions } from './localization-manager'; + +describe('localization-manager#translateLanguage', () => { + + async function mockLocalization(parameters: DeeplParameters): Promise { + return { + translations: parameters.text.map(value => ({ + detected_source_language: '', + text: `[${value}]` + })) + }; + } + + const manager = new LocalizationManager(mockLocalization); + const defaultOptions: LocalizationOptions = { + authKey: '', + freeApi: false, + sourceFile: '', + targetLanguages: ['EN'] + }; + + it('should translate a single value', async () => { + const input = { + key: 'value' + }; + const target = {}; + await manager.translateLanguage(input, target, 'EN', defaultOptions); + assert.deepStrictEqual(target, { + key: '[value]' + }); + }); + + it('should translate nested values', async () => { + const input = { + a: { + b: 'b' + }, + c: 'c' + }; + const target = {}; + await manager.translateLanguage(input, target, 'EN', defaultOptions); + assert.deepStrictEqual(target, { + a: { + b: '[b]' + }, + c: '[c]' + }); + }); + + it('should not override existing targets', async () => { + const input = { + a: 'a' + }; + const target = { + a: 'b' + }; + await manager.translateLanguage(input, target, 'EN', defaultOptions); + assert.deepStrictEqual(target, { + a: 'b' + }); + }); +}); diff --git a/dev-packages/localization-manager/src/localization-manager.ts b/dev-packages/localization-manager/src/localization-manager.ts new file mode 100644 index 0000000000000..135fa8f1a87f5 --- /dev/null +++ b/dev-packages/localization-manager/src/localization-manager.ts @@ -0,0 +1,147 @@ +/******************************************************************************** + * Copyright (C) 2021 TypeFox and others. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * This Source Code may also be made available under the following Secondary + * Licenses when the conditions for such availability set forth in the Eclipse + * Public License v. 2.0 are satisfied: GNU General Public License, version 2 + * with the GNU Classpath Exception which is available at + * https://www.gnu.org/software/classpath/license.html. + * + * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 + ********************************************************************************/ + +import * as chalk from 'chalk'; +import * as fs from 'fs-extra'; +import * as path from 'path'; +import { Localization } from './common'; +import { deepl, DeeplLanguage, DeeplParameters, isSupportedLanguage, supportedLanguages } from './deepl-api'; + +export interface LocalizationOptions { + freeApi: Boolean + authKey: string + sourceFile: string + sourceLanguage?: string + targetLanguages: string[] +} + +export type LocalizationFunction = (parameters: DeeplParameters) => Promise; + +export class LocalizationManager { + + constructor(private localizationFn = deepl) { } + + async localize(options: LocalizationOptions): Promise { + let source: Localization = {}; + try { + source = await fs.readJson(options.sourceFile); + } catch { + console.log(chalk.red(`Could not read file "${options.sourceFile}"`)); + process.exit(1); + } + const languages: string[] = []; + for (const targetLanguage of options.targetLanguages) { + if (!isSupportedLanguage(targetLanguage)) { + console.log(chalk.yellow(`Language "${targetLanguage}" is not supported for automatic localization`)); + } else { + languages.push(targetLanguage); + } + } + if (languages.length !== options.targetLanguages.length) { + console.log('Supported languages: ' + supportedLanguages.join(', ')); + } + const existingTranslations: Map = new Map(); + for (const targetLanguage of languages) { + try { + const targetPath = this.translationFileName(options.sourceFile, targetLanguage); + existingTranslations.set(targetLanguage, await fs.readJson(targetPath)); + } catch { + existingTranslations.set(targetLanguage, {}); + } + } + await Promise.all(languages.map(language => this.translateLanguage(source, existingTranslations.get(language)!, language, options))); + + for (const targetLanguage of languages) { + const targetPath = this.translationFileName(options.sourceFile, targetLanguage); + try { + await fs.writeFile(targetPath, JSON.stringify(existingTranslations.get(targetLanguage)!, undefined, 4)); + } catch { + console.error(chalk.red(`Error writing translated file to '${targetPath}'`)); + } + } + } + + protected translationFileName(original: string, language: string): string { + const directory = path.dirname(original); + const fileName = path.basename(original, '.json'); + return path.join(directory, `${fileName}.${language.toLowerCase()}.json`); + } + + async translateLanguage(source: Localization, target: Localization, targetLanguage: string, options: LocalizationOptions): Promise { + const map = this.buildLocalizationMap(source, target); + if (map.text.length > 0) { + try { + const translationResponse = await this.localizationFn({ + auth_key: options.authKey, + free_api: options.freeApi, + target_lang: targetLanguage.toUpperCase() as DeeplLanguage, + source_lang: options.sourceLanguage?.toUpperCase() as DeeplLanguage, + text: map.text + }); + translationResponse.translations.forEach(({ text }, i) => { + map.localize(i, text); + }); + console.log(chalk.green(`Successfully translated ${map.text.length} value${map.text.length > 1 ? 's' : ''} for language "${targetLanguage}"`)); + } catch (e) { + console.log(chalk.red(`Could not translate into language "${targetLanguage}"`), e); + } + } else { + console.log(`No translation necessary for language "${targetLanguage}"`); + } + } + + protected buildLocalizationMap(source: Localization, target: Localization): LocalizationMap { + const functionMap = new Map void>(); + const text: string[] = []; + const process = (s: Localization, t: Localization) => { + // Delete all extra keys in the target translation first + for (const key of Object.keys(t)) { + if (!(key in s)) { + delete t[key]; + } + } + for (const [key, value] of Object.entries(s)) { + if (!(key in t)) { + if (typeof value === 'string') { + functionMap.set(text.length, translation => t[key] = translation); + text.push(value); + } else { + const newLocalization: Localization = {}; + t[key] = newLocalization; + process(value, newLocalization); + } + } else if (typeof value === 'object') { + if (typeof t[key] === 'string') { + t[key] = {}; + } + process(value, t[key] as Localization); + } + } + }; + + process(source, target); + + return { + text, + localize: (index, value) => functionMap.get(index)!(value) + }; + } +} + +export interface LocalizationMap { + text: string[] + localize: (index: number, value: string) => void +}