Skip to content

Commit

Permalink
Extract keys
Browse files Browse the repository at this point in the history
  • Loading branch information
pontusab committed Dec 28, 2024
1 parent 77b3928 commit a90921a
Show file tree
Hide file tree
Showing 12 changed files with 437 additions and 20 deletions.
Binary file modified bun.lockb
Binary file not shown.
3 changes: 2 additions & 1 deletion examples/email/languine.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,4 +15,5 @@ export default defineConfig({
provider: "openai",
model: "gpt-4-turbo",
},
});
extract: ["./emails/**/*.tsx"],
});
25 changes: 11 additions & 14 deletions examples/email/locales/en.json
Original file line number Diff line number Diff line change
@@ -1,15 +1,12 @@
{
"previewText": "Join %{invitedByUsername} on %{company}",
"company": "%{company}",
"logoAlt": "Vercel Logo",
"joinTeamHeading": "Join %{teamName} on %{company}",
"greeting": "Hi %{username},",
"invitationText":
"%{invitedByUsername} (%{email}) has invited you to join the %{teamName} team on %{company}.",
"invitedToAlt": "Invited to",
"joinTeamButton": "Join the team",
"copyUrlText": "Or copy and paste this URL into your browser:",
"footerText":
"This invitation was intended for %{username} (%{ip} from %{location}). If you were not expecting this invitation, you can ignore this email. If you are concerned about your account's safety, please reply to this email to get in touch with us."
}

"previewText": "Join %{invitedByUsername} on %{company}",
"company": "%{company}",
"logoAlt": "Vercel Logo",
"joinTeamHeading": "Join %{teamName} on %{company}",
"greeting": "Hi %{username},",
"invitationText": "%{invitedByUsername} (%{email}) has invited you to join the %{teamName} team on %{company}.",
"invitedToAlt": "Invited to",
"joinTeamButton": "Join the team",
"copyUrlText": "Or copy and paste this URL into your browser:",
"footerText": "This invitation was intended for %{username} (%{ip} from %{location}). If you were not expecting this invitation, you can ignore this email. If you are concerned about your account's safety, please reply to this email to get in touch with us."
}
9 changes: 6 additions & 3 deletions packages/cli/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,24 +13,27 @@
"typecheck": "tsc --noEmit",
"build": "tsup --clean",
"dev": "tsup --watch --clean",
"start": "node dist/index.js"
"start": "node dist/index.js",
"test": "vitest"
},
"files": ["dist", "README.md"],
"dependencies": {
"jiti": "^2.4.2",
"sucrase": "^3.35.0",
"@ai-sdk/openai": "^1.0.11",
"@babel/parser": "^7.26.3",
"@clack/prompts": "^0.9.0",
"ai": "^4.0.22",
"chalk": "^5.4.1",
"dedent": "^1.5.3",
"diff": "^7.0.0",
"dotenv": "^16.4.7",
"glob": "^11.0.0",
"jiti": "^2.4.2",
"ollama": "^0.5.11",
"ollama-ai-provider": "^1.1.0",
"plist": "^3.1.0",
"preferred-pm": "^4.0.0",
"simple-git": "^3.27.0",
"sucrase": "^3.35.0",
"xml2js": "^0.6.2",
"yaml": "^2.6.1",
"zod": "^3.24.1"
Expand Down
113 changes: 113 additions & 0 deletions packages/cli/src/commands/extract.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
import fs from "node:fs";
import { confirm, intro, outro, text } from "@clack/prompts";
import { glob } from "glob";
import { parseJS } from "../parsers/js.js";
import { getConfig, updateConfig } from "../utils.js";

export async function extract(update = false) {
intro("Extracting translation keys...");

const config = await getConfig();

if (!config.extract?.length) {
const shouldContinue = await confirm({
message: "Would you like to add extract patterns to your config?",
});

if (!shouldContinue) {
process.exit(1);
}

const pattern = await text({
message: "Where would you like to extract translations from?",
defaultValue: "./src/**/*.{ts,tsx}",
placeholder: "./src/**/*.{ts,tsx}",
});

if (typeof pattern === "symbol") {
process.exit(0);
}

// Add extract pattern and save config
await updateConfig({
...config,
extract: [pattern],
});

outro("Updated config with extract patterns");
return [];
}

const extractedKeys = new Set<string>();

for (const pattern of config.extract) {
const files = glob.sync(pattern);

for (const file of files) {
const code = fs.readFileSync(file, "utf8");
const keys = parseJS(code);

for (const key of keys) {
extractedKeys.add(key);
}
}
}

const keys = Array.from(extractedKeys);

if (update) {
// Get source locale file path from config
const sourceLocale = config.locale.source;
const sourceFile = config.files.json.include[0].replace(
"[locale]",
sourceLocale,
);

// Read existing translations if any
let translations: Record<string, string> = {};
if (fs.existsSync(sourceFile)) {
const content = fs.readFileSync(sourceFile, "utf8");
const ext = sourceFile.split(".").pop()?.toLowerCase();

if (ext === "json") {
translations = JSON.parse(content);
} else if (ext === "ts" || ext === "js") {
// For TS/JS files, evaluate the content
const mod = eval(content);
translations = mod.default || mod;
} else if (ext === "md" || ext === "mdx") {
// For MD/MDX files, parse frontmatter
const match = content.match(/^---\n([\s\S]*?)\n---/);
if (match) {
translations = JSON.parse(match[1]);
}
}
}

// Add new keys with empty translations
for (const key of keys) {
if (!translations[key]) {
translations[key] = "";
}
}

// Write back to source file based on extension
const ext = sourceFile.split(".").pop()?.toLowerCase();
let output = "";

if (ext === "json") {
output = JSON.stringify(translations, null, 2);
} else if (ext === "ts") {
output = `export default ${JSON.stringify(translations, null, 2)}`;
} else if (ext === "js") {
output = `module.exports = ${JSON.stringify(translations, null, 2)}`;
} else if (ext === "md" || ext === "mdx") {
output = `---\n${JSON.stringify(translations, null, 2)}\n---\n`;
}

fs.writeFileSync(sourceFile, output);
outro("Updated source locale file with new keys");
}

return keys;
}
9 changes: 8 additions & 1 deletion packages/cli/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import chalk from "chalk";
import dedent from "dedent";
import { clean } from "./commands/clean.js";
import { diff } from "./commands/diff.js";
import { extract } from "./commands/extract.js";
import { init } from "./commands/init.js";
import { instructions } from "./commands/instructions.js";
import { translate } from "./commands/translate.js";
Expand Down Expand Up @@ -40,6 +41,7 @@ const command =
{ value: "translate", label: "Translate to target languages" },
{ value: "add", label: "Add a new language" },
{ value: "instructions", label: "Add custom translation instructions" },
{ value: "extract", label: "Extract translation keys from source files" },
{ value: "diff", label: "Check for changes in source locale file" },
{ value: "clean", label: "Clean unused translations" },
{ value: "available", label: "Available commands" },
Expand All @@ -55,13 +57,16 @@ const preset = process.argv.includes("--preset")
? process.argv[process.argv.indexOf("--preset") + 1]
: undefined;
const force = process.argv.includes("--force") || process.argv.includes("-f");

if (command === "init") {
await init(preset);
} else if (command === "translate") {
await translate(targetLocale, force);
} else if (command === "instructions") {
await instructions();
} else if (command === "extract") {
const update =
process.argv.includes("--update") || process.argv.includes("-u");
await extract(update);
} else if (command === "diff") {
await diff();
} else if (command === "clean") {
Expand All @@ -74,6 +79,8 @@ if (command === "init") {
${chalk.cyan("translate")} ${chalk.gray("<locale>")} Translate to a specific locale
${chalk.cyan("translate")} ${chalk.gray("--force")} Force translate all keys
${chalk.cyan("instructions")} Add custom translation instructions
${chalk.cyan("extract")} Extract translation keys from source files
${chalk.cyan("extract")} ${chalk.gray("-u, --update")} Update source locale file with new keys
${chalk.cyan("diff")} Check for changes in source locale file
${chalk.cyan("clean")} Clean unused translations
${chalk.cyan("available")} Show available commands
Expand Down
115 changes: 115 additions & 0 deletions packages/cli/src/parsers/js.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
import * as babelParser from "@babel/parser";
import type { Node, ObjectExpression, Parser, StringLiteral } from "./types.js";

class FunctionCallParser implements Parser {
constructor(private functions: string[]) {}

parse(node: Node, keys: string[]): void {
if (
node.type === "CallExpression" &&
node.callee?.type === "Identifier" &&
this.functions.includes(node.callee.name || "")
) {
const firstArg = node.arguments?.[0] as StringLiteral | undefined;
if (firstArg?.type === "StringLiteral") {
keys.push(firstArg.value);
}
}
}
}

class IntlFormatParser implements Parser {
parse(node: Node, keys: string[]): void {
if (
node.type === "CallExpression" &&
node.callee?.type === "MemberExpression" &&
node.callee.object?.name === "intl" &&
node.callee.property?.name === "formatMessage"
) {
const firstArg = node.arguments?.[0] as ObjectExpression | undefined;
if (firstArg?.type === "ObjectExpression") {
const idProperty = firstArg.properties?.find(
(prop) => prop.key?.name === "id",
);
if (idProperty?.value?.type === "StringLiteral") {
keys.push(idProperty.value.value);
}
}
}
}
}

class I18nParser implements Parser {
parse(node: Node, keys: string[]): void {
if (
node.type === "CallExpression" &&
node.callee?.type === "MemberExpression" &&
node.callee.object?.name === "i18n" &&
node.callee.property?.name === "t"
) {
const firstArg = node.arguments?.[0] as StringLiteral | undefined;
if (firstArg?.type === "StringLiteral") {
keys.push(firstArg.value);
}
}
}
}

class JSXComponentParser implements Parser {
constructor(private components: string[]) {}

parse(node: Node, keys: string[]): void {
if (
node.type === "JSXOpeningElement" &&
this.components.includes(node.name?.name || "")
) {
const idAttr = node.attributes?.find((attr) => attr.name?.name === "id");
if (idAttr?.value?.type === "StringLiteral") {
keys.push(idAttr.value.value);
}
}
}
}

export const parseJS = (
code: string,
functions: string[] = ["t"],
components: string[] = ["FormattedMessage", "Trans"],
): string[] => {
const ast = babelParser.parse(code, {
sourceType: "module",
plugins: ["jsx", "typescript"],
});

const keys: string[] = [];
const parsers: Parser[] = [
new FunctionCallParser(functions),
new IntlFormatParser(),
new I18nParser(),
new JSXComponentParser(components),
];
const traverseNode = (node: Node) => {
if (!node) return;

// Apply all parsers
for (const parser of parsers) {
parser.parse(node, keys);
}

// Recursively traverse child nodes
for (const key in node) {
const value = node[key as keyof Node];
if (Array.isArray(value)) {
for (const child of value) {
traverseNode(child as Node);
}
} else if (typeof value === "object" && value !== null) {
traverseNode(value as Node);
}
}
};

traverseNode(ast as Node);

return keys;
};
51 changes: 51 additions & 0 deletions packages/cli/src/parsers/types.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
export interface StringLiteral {
type: "StringLiteral";
value: string;
}

export interface ObjectExpression {
type: "ObjectExpression";
properties: Property[];
}

export interface Property {
key?: {
name: string;
};
value?: StringLiteral;
}

export interface JSXAttribute {
name?: {
name: string;
};
value?: StringLiteral;
}

export type Node = {
type: string;
callee?: {
type: string;
name?: string;
object?: {
name: string;
};
property?: {
name: string;
};
};
arguments?: (StringLiteral | ObjectExpression)[];
name?: {
name: string;
};
attributes?: JSXAttribute[];
value?: string;
key?: {
name: string;
};
properties?: Property[];
};

export interface Parser {
parse(node: Node, keys: string[]): void;
}
Loading

0 comments on commit a90921a

Please sign in to comment.