diff --git a/bun.lockb b/bun.lockb index 358e28a..78367a6 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/examples/email/languine.config.ts b/examples/email/languine.config.ts index 0d6c1b5..c127af1 100644 --- a/examples/email/languine.config.ts +++ b/examples/email/languine.config.ts @@ -15,4 +15,5 @@ export default defineConfig({ provider: "openai", model: "gpt-4-turbo", }, + extract: ["./emails/**/*.tsx"], }); diff --git a/examples/email/locales/en.json b/examples/email/locales/en.json index 8be44dc..4a2d9f3 100644 --- a/examples/email/locales/en.json +++ b/examples/email/locales/en.json @@ -1,15 +1,12 @@ { - "previewText": "Join %{invitedByUsername} on %{company}", - "company": "%{company}", - "logoAlt": "Vercel Logo", - "joinTeamHeading": "Join %{teamName} on %{company}", - "greeting": "Hi %{username},", - "invitationText": - "%{invitedByUsername} (%{email}) has invited you to join the %{teamName} team on %{company}.", - "invitedToAlt": "Invited to", - "joinTeamButton": "Join the team", - "copyUrlText": "Or copy and paste this URL into your browser:", - "footerText": - "This invitation was intended for %{username} (%{ip} from %{location}). If you were not expecting this invitation, you can ignore this email. If you are concerned about your account's safety, please reply to this email to get in touch with us." - } - \ No newline at end of file + "previewText": "Join %{invitedByUsername} on %{company}", + "company": "%{company}", + "logoAlt": "Vercel Logo", + "joinTeamHeading": "Join %{teamName} on %{company}", + "greeting": "Hi %{username},", + "invitationText": "%{invitedByUsername} (%{email}) has invited you to join the %{teamName} team on %{company}.", + "invitedToAlt": "Invited to", + "joinTeamButton": "Join the team", + "copyUrlText": "Or copy and paste this URL into your browser:", + "footerText": "This invitation was intended for %{username} (%{ip} from %{location}). If you were not expecting this invitation, you can ignore this email. If you are concerned about your account's safety, please reply to this email to get in touch with us." +} \ No newline at end of file diff --git a/examples/email/locales/es.json b/examples/email/locales/es.json index 40bdab2..5c61614 100644 --- a/examples/email/locales/es.json +++ b/examples/email/locales/es.json @@ -8,5 +8,5 @@ "invitedToAlt": "Invitado a", "joinTeamButton": "Únete al equipo", "copyUrlText": "O copia y pega esta URL en tu navegador:", - "footerText": "Esta invitación fue destinada para %{username} (%{ip} desde %{location}). Si no esperabas esta invitación, puedes ignorar este correo electrónico. Si te preocupa la seguridad de tu cuenta, por favor responde a este correo electrónico para ponerte en contacto con nosotros." + "footerText": "Esta invitación fue destinada para %{username} (%{ip} desde %{location}). Si no esperabas esta invitación, puedes ignorar este correo. Si te preocupa la seguridad de tu cuenta, por favor responde a este correo para ponerte en contacto con nosotros." } \ No newline at end of file diff --git a/examples/email/locales/pt.json b/examples/email/locales/pt.json index 2bed970..c9b343f 100644 --- a/examples/email/locales/pt.json +++ b/examples/email/locales/pt.json @@ -7,6 +7,6 @@ "invitationText": "%{invitedByUsername} (%{email}) convidou você para se juntar ao time %{teamName} na %{company}.", "invitedToAlt": "Convidado para", "joinTeamButton": "Entrar no time", - "copyUrlText": "Ou copie e cole este URL no seu navegador:", + "copyUrlText": "Ou copie e cole esta URL no seu navegador:", "footerText": "Este convite foi destinado para %{username} (%{ip} de %{location}). Se você não estava esperando este convite, pode ignorar este e-mail. Se estiver preocupado com a segurança da sua conta, por favor responda a este e-mail para entrar em contato conosco." } \ No newline at end of file diff --git a/packages/cli/package.json b/packages/cli/package.json index 93d70ae..3915b97 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "languine", - "version": "0.7.0", + "version": "0.7.1", "type": "module", "bin": "dist/index.js", "main": "dist/config.js", @@ -13,24 +13,27 @@ "typecheck": "tsc --noEmit", "build": "tsup --clean", "dev": "tsup --watch --clean", - "start": "node dist/index.js" + "start": "node dist/index.js", + "test": "vitest" }, "files": ["dist", "README.md"], "dependencies": { - "jiti": "^2.4.2", - "sucrase": "^3.35.0", "@ai-sdk/openai": "^1.0.11", + "@babel/parser": "^7.26.3", "@clack/prompts": "^0.9.0", "ai": "^4.0.22", "chalk": "^5.4.1", "dedent": "^1.5.3", "diff": "^7.0.0", "dotenv": "^16.4.7", + "glob": "^11.0.0", + "jiti": "^2.4.2", "ollama": "^0.5.11", "ollama-ai-provider": "^1.1.0", "plist": "^3.1.0", "preferred-pm": "^4.0.0", "simple-git": "^3.27.0", + "sucrase": "^3.35.0", "xml2js": "^0.6.2", "yaml": "^2.6.1", "zod": "^3.24.1" diff --git a/packages/cli/src/commands/extract.ts b/packages/cli/src/commands/extract.ts new file mode 100644 index 0000000..26fc513 --- /dev/null +++ b/packages/cli/src/commands/extract.ts @@ -0,0 +1,130 @@ +import fs from "node:fs"; +import { confirm, intro, outro, text } from "@clack/prompts"; +import chalk from "chalk"; +import { glob } from "glob"; +import { parseJS } from "../parsers/js.js"; +import { getConfig, updateConfig } from "../utils.js"; + +export async function extract(update = false) { + intro("Extracting translation keys..."); + + const config = await getConfig(); + + if (!config.extract?.length) { + const shouldContinue = await confirm({ + message: "Would you like to add extract patterns to your config?", + }); + + if (!shouldContinue) { + process.exit(1); + } + + const pattern = await text({ + message: "Where would you like to extract translations from?", + defaultValue: "./src/**/*.{ts,tsx}", + placeholder: "./src/**/*.{ts,tsx}", + }); + + if (typeof pattern === "symbol") { + process.exit(0); + } + + // Add extract pattern and save config + await updateConfig({ + ...config, + extract: [pattern], + }); + + outro("Updated config with extract patterns"); + return []; + } + + const foundKeys = new Set(); + + for (const pattern of config.extract) { + const files = glob.sync(pattern); + + for (const file of files) { + const code = fs.readFileSync(file, "utf8"); + const keys = parseJS(code); + + for (const key of keys) { + foundKeys.add(key); + } + } + } + + const keys = Array.from(foundKeys); + + if (config.files.json.include.length === 0) { + outro(chalk.red("No translation files found in config")); + return []; + } + + // Get source locale file path from config + const sourceLocale = config.locale.source; + const sourceFile = + typeof config.files.json.include[0] === "string" + ? config.files.json.include[0].replace("[locale]", sourceLocale) + : config.files.json.include[0].from.replace("[locale]", sourceLocale); + + // Read existing translations if any + let translations: Record = {}; + if (fs.existsSync(sourceFile)) { + const content = fs.readFileSync(sourceFile, "utf8"); + const ext = sourceFile.split(".").pop()?.toLowerCase(); + + if (ext === "json") { + translations = JSON.parse(content); + } else if (ext === "ts" || ext === "js") { + // For TS/JS files, use jiti to safely load the module + const jiti = require("jiti")(process.cwd()); + const mod = jiti(sourceFile); + translations = mod.default || mod; + } else if (ext === "md" || ext === "mdx") { + // For MD/MDX files, parse frontmatter + const match = content.match(/^---\n([\s\S]*?)\n---/); + if (match) { + translations = JSON.parse(match[1]); + } + } + } + + // Add new keys with empty translations + const newKeys = keys.filter((key) => !translations[key]); + + if (update) { + for (const key of keys) { + if (!translations[key]) { + translations[key] = ""; + } + } + + // Write back to source file based on extension + const ext = sourceFile.split(".").pop()?.toLowerCase(); + let output = ""; + + if (ext === "json") { + output = JSON.stringify(translations, null, 2); + } else if (ext === "ts") { + output = `export default ${JSON.stringify(translations, null, 2)}`; + } else if (ext === "js") { + output = `module.exports = ${JSON.stringify(translations, null, 2)}`; + } else if (ext === "md" || ext === "mdx") { + output = `---\n${JSON.stringify(translations, null, 2)}\n---\n`; + } + + fs.writeFileSync(sourceFile, output); + } + if (newKeys.length > 0) { + outro( + chalk.green( + `Found ${newKeys.length} new translation keys from source files${update ? ` and saved them to ${sourceFile}` : ""}`, + ), + ); + } else { + outro(chalk.yellow("No new translation keys found")); + } + + return keys; +} diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index fcd0691..f50d614 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -6,29 +6,32 @@ import chalk from "chalk"; import dedent from "dedent"; import { clean } from "./commands/clean.js"; import { diff } from "./commands/diff.js"; +import { extract } from "./commands/extract.js"; import { init } from "./commands/init.js"; import { instructions } from "./commands/instructions.js"; import { translate } from "./commands/translate.js"; -console.log( - ` +if (!process.argv[2]) { + console.log( + ` ██╗ █████╗ ███╗ ██╗ ██████╗ ██╗ ██╗██╗███╗ ██╗███████╗ ██║ ██╔══██╗████╗ ██║██╔════╝ ██║ ██║██║████╗ ██║██╔════╝ ██║ ███████║██╔██╗ ██║██║ ███╗██║ ██║██║██║██╗ ██║█████╗ ██║ ██╔══██║██║╚██╗██║██║ ██║██║ ██║██║██║╚██╗██║██╔══╝ ███████╗██║ ██║██║ ╚████║╚██████╔╝╚██████╔╝██║██║ ╚████║███████╗ ╚══════╝╚═╝ ╚═╝╚═╝ ╚═══╝ ╚═════╝ ╚═════╝ ╚═╝╚═╝ ╚═══╝╚══════╝ - `, -); + `, + ); -console.log( - chalk.gray(dedent` - Translate your application with Languine CLI powered by AI. - Website: ${chalk.bold("https://languine.ai")} - `), -); + console.log( + chalk.gray(dedent` + Translate your application with Languine CLI powered by AI. + Website: ${chalk.bold("https://languine.ai")} + `), + ); -console.log(); + console.log(); +} // Parse command line arguments const command = @@ -40,6 +43,7 @@ const command = { value: "translate", label: "Translate to target languages" }, { value: "add", label: "Add a new language" }, { value: "instructions", label: "Add custom translation instructions" }, + { value: "extract", label: "Extract translation keys from source files" }, { value: "diff", label: "Check for changes in source locale file" }, { value: "clean", label: "Clean unused translations" }, { value: "available", label: "Available commands" }, @@ -55,13 +59,16 @@ const preset = process.argv.includes("--preset") ? process.argv[process.argv.indexOf("--preset") + 1] : undefined; const force = process.argv.includes("--force") || process.argv.includes("-f"); - if (command === "init") { await init(preset); } else if (command === "translate") { await translate(targetLocale, force); } else if (command === "instructions") { await instructions(); +} else if (command === "extract") { + const update = + process.argv.includes("--update") || process.argv.includes("-u"); + await extract(update); } else if (command === "diff") { await diff(); } else if (command === "clean") { @@ -74,6 +81,8 @@ if (command === "init") { ${chalk.cyan("translate")} ${chalk.gray("")} Translate to a specific locale ${chalk.cyan("translate")} ${chalk.gray("--force")} Force translate all keys ${chalk.cyan("instructions")} Add custom translation instructions + ${chalk.cyan("extract")} Extract translation keys from source files + ${chalk.cyan("extract")} ${chalk.gray("-u, --update")} Update source locale file with new keys ${chalk.cyan("diff")} Check for changes in source locale file ${chalk.cyan("clean")} Clean unused translations ${chalk.cyan("available")} Show available commands diff --git a/packages/cli/src/parsers/js.ts b/packages/cli/src/parsers/js.ts new file mode 100644 index 0000000..0778e27 --- /dev/null +++ b/packages/cli/src/parsers/js.ts @@ -0,0 +1,115 @@ +import * as babelParser from "@babel/parser"; +import type { Node, ObjectExpression, Parser, StringLiteral } from "./types.js"; + +class FunctionCallParser implements Parser { + constructor(private functions: string[]) {} + + parse(node: Node, keys: string[]): void { + if ( + node.type === "CallExpression" && + node.callee?.type === "Identifier" && + this.functions.includes(node.callee.name || "") + ) { + const firstArg = node.arguments?.[0] as StringLiteral | undefined; + if (firstArg?.type === "StringLiteral") { + keys.push(firstArg.value); + } + } + } +} + +class IntlFormatParser implements Parser { + parse(node: Node, keys: string[]): void { + if ( + node.type === "CallExpression" && + node.callee?.type === "MemberExpression" && + node.callee.object?.name === "intl" && + node.callee.property?.name === "formatMessage" + ) { + const firstArg = node.arguments?.[0] as ObjectExpression | undefined; + if (firstArg?.type === "ObjectExpression") { + const idProperty = firstArg.properties?.find( + (prop) => prop.key?.name === "id", + ); + if (idProperty?.value?.type === "StringLiteral") { + keys.push(idProperty.value.value); + } + } + } + } +} + +class I18nParser implements Parser { + parse(node: Node, keys: string[]): void { + if ( + node.type === "CallExpression" && + node.callee?.type === "MemberExpression" && + node.callee.object?.name === "i18n" && + node.callee.property?.name === "t" + ) { + const firstArg = node.arguments?.[0] as StringLiteral | undefined; + if (firstArg?.type === "StringLiteral") { + keys.push(firstArg.value); + } + } + } +} + +class JSXComponentParser implements Parser { + constructor(private components: string[]) {} + + parse(node: Node, keys: string[]): void { + if ( + node.type === "JSXOpeningElement" && + this.components.includes(node.name?.name || "") + ) { + const idAttr = node.attributes?.find((attr) => attr.name?.name === "id"); + if (idAttr?.value?.type === "StringLiteral") { + keys.push(idAttr.value.value); + } + } + } +} + +export const parseJS = ( + code: string, + functions: string[] = ["t"], + components: string[] = ["FormattedMessage", "Trans"], +): string[] => { + const ast = babelParser.parse(code, { + sourceType: "module", + plugins: ["jsx", "typescript"], + }); + + const keys: string[] = []; + const parsers: Parser[] = [ + new FunctionCallParser(functions), + new IntlFormatParser(), + new I18nParser(), + new JSXComponentParser(components), + ]; + const traverseNode = (node: Node) => { + if (!node) return; + + // Apply all parsers + for (const parser of parsers) { + parser.parse(node, keys); + } + + // Recursively traverse child nodes + for (const key in node) { + const value = node[key as keyof Node]; + if (Array.isArray(value)) { + for (const child of value) { + traverseNode(child as Node); + } + } else if (typeof value === "object" && value !== null) { + traverseNode(value as Node); + } + } + }; + + traverseNode(ast as Node); + + return keys; +}; diff --git a/packages/cli/src/parsers/types.ts b/packages/cli/src/parsers/types.ts new file mode 100644 index 0000000..85dbbe3 --- /dev/null +++ b/packages/cli/src/parsers/types.ts @@ -0,0 +1,51 @@ +export interface StringLiteral { + type: "StringLiteral"; + value: string; +} + +export interface ObjectExpression { + type: "ObjectExpression"; + properties: Property[]; +} + +export interface Property { + key?: { + name: string; + }; + value?: StringLiteral; +} + +export interface JSXAttribute { + name?: { + name: string; + }; + value?: StringLiteral; +} + +export type Node = { + type: string; + callee?: { + type: string; + name?: string; + object?: { + name: string; + }; + property?: { + name: string; + }; + }; + arguments?: (StringLiteral | ObjectExpression)[]; + name?: { + name: string; + }; + attributes?: JSXAttribute[]; + value?: string; + key?: { + name: string; + }; + properties?: Property[]; +}; + +export interface Parser { + parse(node: Node, keys: string[]): void; +} diff --git a/packages/cli/src/types.ts b/packages/cli/src/types.ts index caf87b0..f2b5dc8 100644 --- a/packages/cli/src/types.ts +++ b/packages/cli/src/types.ts @@ -2,26 +2,47 @@ import type { LanguageModelV1 } from "ai"; export type Provider = "openai" | "ollama"; +/** + * Configuration interface for Languine + */ export interface Config { + /** Version of the Languine configuration */ version: string; + /** Locale configuration */ locale: { + /** Source language code (e.g. 'en') */ source: string; + /** Target language codes to translate to */ targets: string[]; }; + /** File configuration by format type */ files: { + /** Configuration for each file format */ [format: string]: { + /** Glob patterns or path mappings to include */ include: Include[]; }; }; + /** Glob patterns to extract translation keys from source files */ + extract?: string[]; + /** Language model configuration */ llm: { + /** LLM provider ('openai' or 'ollama') */ provider: Provider; + /** Model name to use */ model: string; + /** Temperature for model responses (0-1) */ temperature?: number; }; + /** Custom translation instructions */ instructions?: string; + /** Hook functions */ hooks?: { + /** Hook called after translation is complete */ afterTranslate?: (args: { + /** Translated content */ content: string; + /** Path to the translated file */ filePath: string; }) => Promise; }; diff --git a/packages/cli/src/utils.ts b/packages/cli/src/utils.ts index 7d4e0d1..ecd61e3 100644 --- a/packages/cli/src/utils.ts +++ b/packages/cli/src/utils.ts @@ -73,6 +73,7 @@ export function generateConfig({ provider, model, configType, + extractPatterns, }: { version: string; sourceLanguage: string; @@ -82,6 +83,7 @@ export function generateConfig({ provider: string; model: string; configType: string; + extractPatterns?: string[]; }) { const formatKey = fileFormat.includes("-") ? `"${fileFormat}"` : fileFormat; @@ -100,6 +102,7 @@ export function generateConfig({ provider: "${provider}", model: "${model}", }, + ${extractPatterns ? `extract: [${extractPatterns.map((p) => `"${p}"`).join(", ")}]` : ""} }`; if (configType === "mjs") { @@ -169,6 +172,24 @@ export async function getConfig(): Promise { } } +export async function updateConfig(config: Config): Promise { + const { path: filePath, format } = await configFile(); + const configContent = generateConfig({ + version: config.version, + sourceLanguage: config.locale.source, + targetLanguages: config.locale.targets, + fileFormat: Object.keys(config.files)[0], + filesPatterns: config.files[Object.keys(config.files)[0]] + .include as string[], + provider: config.llm.provider, + model: config.llm.model, + configType: format, + extractPatterns: config.extract as string[], + }); + + await fs.writeFile(filePath, configContent); +} + export async function execAsync(command: string) { return await new Promise((resolve, reject) => { exec(command, (error) => { diff --git a/packages/cli/test/extract.test.ts b/packages/cli/test/extract.test.ts new file mode 100644 index 0000000..fccb255 --- /dev/null +++ b/packages/cli/test/extract.test.ts @@ -0,0 +1,88 @@ +import { expect, test } from "vitest"; +import { parseJS } from "../src/parsers/js.js"; + +test("extracts translation keys from t() function calls", () => { + const code = ` + t("hello.world"); + t("another.key"); + notT("ignored"); + `; + + const keys = parseJS(code); + expect(keys).toEqual(["hello.world", "another.key"]); +}); + +test("extracts translation keys from intl.formatMessage()", () => { + const code = ` + intl.formatMessage({ id: "welcome" }); + intl.formatMessage({ id: "goodbye" }); + intl.something({ id: "ignored" }); + `; + + const keys = parseJS(code); + expect(keys).toEqual(["welcome", "goodbye"]); +}); + +test("extracts translation keys from i18n.t() calls", () => { + const code = ` + i18n.t("dashboard.title"); + i18n.t("user.profile"); + other.t("ignored"); + `; + + const keys = parseJS(code); + expect(keys).toEqual(["dashboard.title", "user.profile"]); +}); + +test("extracts translation keys from JSX components", () => { + const code = ` + <> + + + + + `; + + const keys = parseJS(code); + expect(keys).toEqual(["header.title", "footer.copyright"]); +}); + +test("extracts translation keys from mixed usage", () => { + const code = ` + function Component() { + return ( +
+ {t("greeting")} + + {intl.formatMessage({ id: "start" })} + {i18n.t("continue")} +
+ ); + } + `; + + const keys = parseJS(code); + expect(keys).toEqual(["greeting", "welcome", "start", "continue"]); +}); + +test("supports custom function names", () => { + const code = ` + translate("custom.key"); + __("another.key"); + `; + + const keys = parseJS(code, ["translate", "__"]); + expect(keys).toEqual(["custom.key", "another.key"]); +}); + +test("supports custom component names", () => { + const code = ` + <> + + + + `; + + const keys = parseJS(code, ["t"], ["Message", "Translate"]); + expect(keys).toEqual(["custom.message", "another.message"]); +}); diff --git a/packages/react-email/package.json b/packages/react-email/package.json index 7d53747..6966990 100644 --- a/packages/react-email/package.json +++ b/packages/react-email/package.json @@ -1,6 +1,6 @@ { "name": "@languine/react-email", - "version": "0.1.0", + "version": "0.2.0", "files": ["dist", "README.md"], "main": "dist/index.mjs", "types": "dist/index.d.ts",