Skip to content

Commit

Permalink
Extract keys from source (#30)
Browse files Browse the repository at this point in the history
* Extract keys

* wip

* Only show logo on root command

* Bump version
  • Loading branch information
pontusab authored Dec 29, 2024
1 parent efbe5e5 commit 15a59e2
Show file tree
Hide file tree
Showing 14 changed files with 469 additions and 33 deletions.
Binary file modified bun.lockb
Binary file not shown.
1 change: 1 addition & 0 deletions examples/email/languine.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,4 +15,5 @@ export default defineConfig({
provider: "openai",
model: "gpt-4-turbo",
},
extract: ["./emails/**/*.tsx"],
});
25 changes: 11 additions & 14 deletions examples/email/locales/en.json
Original file line number Diff line number Diff line change
@@ -1,15 +1,12 @@
{
"previewText": "Join %{invitedByUsername} on %{company}",
"company": "%{company}",
"logoAlt": "Vercel Logo",
"joinTeamHeading": "Join %{teamName} on %{company}",
"greeting": "Hi %{username},",
"invitationText":
"%{invitedByUsername} (%{email}) has invited you to join the %{teamName} team on %{company}.",
"invitedToAlt": "Invited to",
"joinTeamButton": "Join the team",
"copyUrlText": "Or copy and paste this URL into your browser:",
"footerText":
"This invitation was intended for %{username} (%{ip} from %{location}). If you were not expecting this invitation, you can ignore this email. If you are concerned about your account's safety, please reply to this email to get in touch with us."
}

"previewText": "Join %{invitedByUsername} on %{company}",
"company": "%{company}",
"logoAlt": "Vercel Logo",
"joinTeamHeading": "Join %{teamName} on %{company}",
"greeting": "Hi %{username},",
"invitationText": "%{invitedByUsername} (%{email}) has invited you to join the %{teamName} team on %{company}.",
"invitedToAlt": "Invited to",
"joinTeamButton": "Join the team",
"copyUrlText": "Or copy and paste this URL into your browser:",
"footerText": "This invitation was intended for %{username} (%{ip} from %{location}). If you were not expecting this invitation, you can ignore this email. If you are concerned about your account's safety, please reply to this email to get in touch with us."
}
2 changes: 1 addition & 1 deletion examples/email/locales/es.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,5 @@
"invitedToAlt": "Invitado a",
"joinTeamButton": "Únete al equipo",
"copyUrlText": "O copia y pega esta URL en tu navegador:",
"footerText": "Esta invitación fue destinada para %{username} (%{ip} desde %{location}). Si no esperabas esta invitación, puedes ignorar este correo electrónico. Si te preocupa la seguridad de tu cuenta, por favor responde a este correo electrónico para ponerte en contacto con nosotros."
"footerText": "Esta invitación fue destinada para %{username} (%{ip} desde %{location}). Si no esperabas esta invitación, puedes ignorar este correo. Si te preocupa la seguridad de tu cuenta, por favor responde a este correo para ponerte en contacto con nosotros."
}
2 changes: 1 addition & 1 deletion examples/email/locales/pt.json
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,6 @@
"invitationText": "%{invitedByUsername} (%{email}) convidou você para se juntar ao time %{teamName} na %{company}.",
"invitedToAlt": "Convidado para",
"joinTeamButton": "Entrar no time",
"copyUrlText": "Ou copie e cole este URL no seu navegador:",
"copyUrlText": "Ou copie e cole esta URL no seu navegador:",
"footerText": "Este convite foi destinado para %{username} (%{ip} de %{location}). Se você não estava esperando este convite, pode ignorar este e-mail. Se estiver preocupado com a segurança da sua conta, por favor responda a este e-mail para entrar em contato conosco."
}
11 changes: 7 additions & 4 deletions packages/cli/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "languine",
"version": "0.7.0",
"version": "0.7.1",
"type": "module",
"bin": "dist/index.js",
"main": "dist/config.js",
Expand All @@ -13,24 +13,27 @@
"typecheck": "tsc --noEmit",
"build": "tsup --clean",
"dev": "tsup --watch --clean",
"start": "node dist/index.js"
"start": "node dist/index.js",
"test": "vitest"
},
"files": ["dist", "README.md"],
"dependencies": {
"jiti": "^2.4.2",
"sucrase": "^3.35.0",
"@ai-sdk/openai": "^1.0.11",
"@babel/parser": "^7.26.3",
"@clack/prompts": "^0.9.0",
"ai": "^4.0.22",
"chalk": "^5.4.1",
"dedent": "^1.5.3",
"diff": "^7.0.0",
"dotenv": "^16.4.7",
"glob": "^11.0.0",
"jiti": "^2.4.2",
"ollama": "^0.5.11",
"ollama-ai-provider": "^1.1.0",
"plist": "^3.1.0",
"preferred-pm": "^4.0.0",
"simple-git": "^3.27.0",
"sucrase": "^3.35.0",
"xml2js": "^0.6.2",
"yaml": "^2.6.1",
"zod": "^3.24.1"
Expand Down
130 changes: 130 additions & 0 deletions packages/cli/src/commands/extract.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
import fs from "node:fs";
import { confirm, intro, outro, text } from "@clack/prompts";
import chalk from "chalk";
import { glob } from "glob";
import { parseJS } from "../parsers/js.js";
import { getConfig, updateConfig } from "../utils.js";

export async function extract(update = false) {
intro("Extracting translation keys...");

const config = await getConfig();

if (!config.extract?.length) {
const shouldContinue = await confirm({
message: "Would you like to add extract patterns to your config?",
});

if (!shouldContinue) {
process.exit(1);
}

const pattern = await text({
message: "Where would you like to extract translations from?",
defaultValue: "./src/**/*.{ts,tsx}",
placeholder: "./src/**/*.{ts,tsx}",
});

if (typeof pattern === "symbol") {
process.exit(0);
}

// Add extract pattern and save config
await updateConfig({
...config,
extract: [pattern],
});

outro("Updated config with extract patterns");
return [];
}

const foundKeys = new Set<string>();

for (const pattern of config.extract) {
const files = glob.sync(pattern);

for (const file of files) {
const code = fs.readFileSync(file, "utf8");
const keys = parseJS(code);

for (const key of keys) {
foundKeys.add(key);
}
}
}

const keys = Array.from(foundKeys);

if (config.files.json.include.length === 0) {
outro(chalk.red("No translation files found in config"));
return [];
}

// Get source locale file path from config
const sourceLocale = config.locale.source;
const sourceFile =
typeof config.files.json.include[0] === "string"
? config.files.json.include[0].replace("[locale]", sourceLocale)
: config.files.json.include[0].from.replace("[locale]", sourceLocale);

// Read existing translations if any
let translations: Record<string, string> = {};
if (fs.existsSync(sourceFile)) {
const content = fs.readFileSync(sourceFile, "utf8");
const ext = sourceFile.split(".").pop()?.toLowerCase();

if (ext === "json") {
translations = JSON.parse(content);
} else if (ext === "ts" || ext === "js") {
// For TS/JS files, use jiti to safely load the module
const jiti = require("jiti")(process.cwd());
const mod = jiti(sourceFile);
translations = mod.default || mod;
} else if (ext === "md" || ext === "mdx") {
// For MD/MDX files, parse frontmatter
const match = content.match(/^---\n([\s\S]*?)\n---/);
if (match) {
translations = JSON.parse(match[1]);
}
}
}

// Add new keys with empty translations
const newKeys = keys.filter((key) => !translations[key]);

if (update) {
for (const key of keys) {
if (!translations[key]) {
translations[key] = "";
}
}

// Write back to source file based on extension
const ext = sourceFile.split(".").pop()?.toLowerCase();
let output = "";

if (ext === "json") {
output = JSON.stringify(translations, null, 2);
} else if (ext === "ts") {
output = `export default ${JSON.stringify(translations, null, 2)}`;
} else if (ext === "js") {
output = `module.exports = ${JSON.stringify(translations, null, 2)}`;
} else if (ext === "md" || ext === "mdx") {
output = `---\n${JSON.stringify(translations, null, 2)}\n---\n`;
}

fs.writeFileSync(sourceFile, output);
}
if (newKeys.length > 0) {
outro(
chalk.green(
`Found ${newKeys.length} new translation keys from source files${update ? ` and saved them to ${sourceFile}` : ""}`,
),
);
} else {
outro(chalk.yellow("No new translation keys found"));
}

return keys;
}
33 changes: 21 additions & 12 deletions packages/cli/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,29 +6,32 @@ import chalk from "chalk";
import dedent from "dedent";
import { clean } from "./commands/clean.js";
import { diff } from "./commands/diff.js";
import { extract } from "./commands/extract.js";
import { init } from "./commands/init.js";
import { instructions } from "./commands/instructions.js";
import { translate } from "./commands/translate.js";

console.log(
`
if (!process.argv[2]) {
console.log(
`
██╗ █████╗ ███╗ ██╗ ██████╗ ██╗ ██╗██╗███╗ ██╗███████╗
██║ ██╔══██╗████╗ ██║██╔════╝ ██║ ██║██║████╗ ██║██╔════╝
██║ ███████║██╔██╗ ██║██║ ███╗██║ ██║██║██║██╗ ██║█████╗
██║ ██╔══██║██║╚██╗██║██║ ██║██║ ██║██║██║╚██╗██║██╔══╝
███████╗██║ ██║██║ ╚████║╚██████╔╝╚██████╔╝██║██║ ╚████║███████╗
╚══════╝╚═╝ ╚═╝╚═╝ ╚═══╝ ╚═════╝ ╚═════╝ ╚═╝╚═╝ ╚═══╝╚══════╝
`,
);
`,
);

console.log(
chalk.gray(dedent`
Translate your application with Languine CLI powered by AI.
Website: ${chalk.bold("https://languine.ai")}
`),
);
console.log(
chalk.gray(dedent`
Translate your application with Languine CLI powered by AI.
Website: ${chalk.bold("https://languine.ai")}
`),
);

console.log();
console.log();
}

// Parse command line arguments
const command =
Expand All @@ -40,6 +43,7 @@ const command =
{ value: "translate", label: "Translate to target languages" },
{ value: "add", label: "Add a new language" },
{ value: "instructions", label: "Add custom translation instructions" },
{ value: "extract", label: "Extract translation keys from source files" },
{ value: "diff", label: "Check for changes in source locale file" },
{ value: "clean", label: "Clean unused translations" },
{ value: "available", label: "Available commands" },
Expand All @@ -55,13 +59,16 @@ const preset = process.argv.includes("--preset")
? process.argv[process.argv.indexOf("--preset") + 1]
: undefined;
const force = process.argv.includes("--force") || process.argv.includes("-f");

if (command === "init") {
await init(preset);
} else if (command === "translate") {
await translate(targetLocale, force);
} else if (command === "instructions") {
await instructions();
} else if (command === "extract") {
const update =
process.argv.includes("--update") || process.argv.includes("-u");
await extract(update);
} else if (command === "diff") {
await diff();
} else if (command === "clean") {
Expand All @@ -74,6 +81,8 @@ if (command === "init") {
${chalk.cyan("translate")} ${chalk.gray("<locale>")} Translate to a specific locale
${chalk.cyan("translate")} ${chalk.gray("--force")} Force translate all keys
${chalk.cyan("instructions")} Add custom translation instructions
${chalk.cyan("extract")} Extract translation keys from source files
${chalk.cyan("extract")} ${chalk.gray("-u, --update")} Update source locale file with new keys
${chalk.cyan("diff")} Check for changes in source locale file
${chalk.cyan("clean")} Clean unused translations
${chalk.cyan("available")} Show available commands
Expand Down
Loading

0 comments on commit 15a59e2

Please sign in to comment.