Skip to content

Commit

Permalink
Fix ollama
Browse files Browse the repository at this point in the history
  • Loading branch information
pontusab committed Dec 25, 2024
1 parent 34841e1 commit 93ee9c3
Show file tree
Hide file tree
Showing 22 changed files with 109 additions and 87 deletions.
7 changes: 4 additions & 3 deletions examples/next-international/languine.config.mjs
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
export default {
version: "1.0.0",
version: "0.5.5",
locale: {
source: "en",
targets: ["fr"],
Expand All @@ -9,7 +9,8 @@ export default {
include: ["locales/[locale].ts"],
},
},
openai: {
model: "gpt-4-turbo",
llm: {
provider: "ollama",
model: "mistral:latest",
},
};
3 changes: 2 additions & 1 deletion examples/next-international/locales/en.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
export default {
hello: "Hello",
welcome: "Hello {name}!",
"about.you": "Hello {name}! You have {age} yo",
"about.you": "Hello {name}! You have {age} years",
"scope.test": "A scope",
"scope.more.test": "A scope",
"scope.more.param": "A scope with {param}",
Expand All @@ -11,4 +11,5 @@ export default {
"missing.translation.in.fr": "This should work",
"cows#one": "A cow",
"cows#other": "{count} cows",
"languine.hello": "Hello Languine",
} as const;
13 changes: 7 additions & 6 deletions examples/next-international/locales/fr.ts
Original file line number Diff line number Diff line change
@@ -1,14 +1,15 @@
export default {
hello: "Bonjour",
welcome: "Bonjour {name} !",
"about.you": "Bonjour {name} ! Tu as {age} ans",
"scope.test": "Un domaine",
"scope.more.test": "Un domaine",
"scope.more.param": "Un domaine avec {param}",
"scope.more.and.more.test": "Un domaine",
welcome: "Bonjour {name}!",
"about.you": "Bonjour {name}! Vous avez {age} ans",
"scope.test": "Un scope",
"scope.more.test": "Un scope",
"scope.more.param": "Un scope avec {param}",
"scope.more.and.more.test": "Un scope",
"scope.more.stars#one": "1 étoile sur GitHub",
"scope.more.stars#other": "{count} étoiles sur GitHub",
"missing.translation.in.fr": "Cela devrait fonctionner",
"cows#one": "Une vache",
"cows#other": "{count} vaches",
"languine.hello": "Hello Languine",
} as const;
13 changes: 7 additions & 6 deletions packages/cli/src/commands/init.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { execSync } from "node:child_process";
import fs from "node:fs/promises";
import path from "node:path";
import { intro, outro, select, text } from "@clack/prompts";
import { providers } from "../model-providers.js";
import { providers } from "../providers.js";
import type { Provider } from "../types.js";
import { configPath } from "../utils.js";

Expand Down Expand Up @@ -83,17 +83,18 @@ export async function init() {
version: "${require("../../package.json").version}",
locale: {
source: "${sourceLanguage}",
targets: ${JSON.stringify(targetLanguages.split(",").map((l) => l.trim()))}
targets: ${JSON.stringify(targetLanguages.split(",").map((l) => l.trim()))},
},
files: {
${fileFormat}: {
include: ["${filesDirectory}/[locale].${fileFormat}"]
}
include: ["${filesDirectory}/[locale].${fileFormat}"],
},
},
llm: {
provider: "${provider}",
model: "${model}"
}
model: "${model}",
temperature: 0,
},
}`;

try {
Expand Down
12 changes: 6 additions & 6 deletions packages/cli/src/commands/translate.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,11 +43,6 @@ export async function translate(targetLocale?: string, force = false) {

const git = simpleGit();

// Initialize OpenAI
const openai = createOpenAI({
apiKey: await getApiKey("OpenAI", "OPENAI_API_KEY"),
});

const model = getModel(config);

const s = spinner();
Expand Down Expand Up @@ -135,7 +130,12 @@ export async function translate(targetLocale?: string, force = false) {
summary,
};
} catch (error) {
return { locale, sourcePath, success: false, error };
return {
locale,
sourcePath,
success: false,
error,
};
}
}),
),
Expand Down
2 changes: 2 additions & 0 deletions packages/cli/src/envs.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
import dotenv from "dotenv";
dotenv.config();
4 changes: 1 addition & 3 deletions packages/cli/src/index.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
#!/usr/bin/env node

import dotenv from "dotenv";
dotenv.config();

import "./envs.js";
import { select } from "@clack/prompts";
import chalk from "chalk";
import dedent from "dedent";
Expand Down
1 change: 0 additions & 1 deletion packages/cli/src/prompt.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ Translation Requirements:
- Keep all technical identifiers unchanged
- Keep consistent capitalization, spacing, and line breaks
- Respect existing whitespace and newline patterns
- Never add space before a ! or ?
`;

export function createBasePrompt(text: string, options: PromptOptions) {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import { outro } from "@clack/prompts";
import chalk from "chalk";
import ollama from "ollama";
import type { Provider } from "./types.js";

Expand Down Expand Up @@ -28,11 +30,21 @@ export const providers: Record<Provider, ProviderConfig> = {
value: "ollama",
label: "Ollama",
getModels: async () => {
const { models } = await ollama.list();
return models.map((model) => ({
value: model.name,
label: model.name,
}));
try {
const { models } = await ollama.list();

return models.map((model) => ({
value: model.name,
label: model.name,
}));
} catch {
outro(
chalk.red(
"Failed to get models from Ollama, is it installed and running?",
),
);
process.exit(1);
}
},
},
};
33 changes: 23 additions & 10 deletions packages/cli/src/translators/js.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,13 @@ function getStrings(code: string) {
const strings: StringMatch[] = [];

while (match) {
strings.push({
index: match.index,
content: match[0],
});
// Skip if the string is a key (contains dots or #)
if (!match[0].includes(".") && !match[0].includes("#")) {
strings.push({
index: match.index,
content: match[0],
});
}

match = quotesRegex.exec(code);
}
Expand All @@ -52,11 +55,19 @@ function replaceStrings(

replaces.forEach((replace, i) => {
const original = strings[i];
if (!original) return; // Skip if no matching original string

const offset = out.length - code.length;
const quote = original.content[0]; // Get the quote character used

// Keep original quotes but ensure replacement content is complete
const wrappedReplace = replace.startsWith(quote)
? replace
: `${quote}${replace}${quote}`;

out =
out.slice(0, original.index + offset) +
replace +
wrappedReplace +
out.slice(original.index + original.content.length + offset);
});

Expand Down Expand Up @@ -96,12 +107,13 @@ export const javascript: Translator = {
const { object } = await generateObject({
model: options.model,
prompt: getPrompt(toTranslate, options),
temperature: options.config.llm?.temperature ?? 0,
schema: z.object({
translations: z.array(z.string()),
items: z.array(z.string()),
}),
});

translated = object.translations;
translated = object.items;
}

const output = replaceStrings(
Expand Down Expand Up @@ -129,13 +141,14 @@ export const javascript: Translator = {
const { object } = await generateObject({
model: options.model,
prompt: getPrompt(strings, options),
temperature: options.config.llm?.temperature ?? 0,
schema: z.object({
translations: z.array(z.string()),
items: z.array(z.string()),
}),
});

return {
content: replaceStrings(options.content, strings, object.translations),
content: replaceStrings(options.content, strings, object.items),
};
},
};
Expand All @@ -145,9 +158,9 @@ function getPrompt(strings: StringMatch[], options: PromptOptions) {
${baseRequirements}
- Preserve all object/property keys, syntax characters, and punctuation marks exactly
- Only translate text content within quotation marks
- Don't remove the quotes around the keys and values
A list of javascript codeblocks, return the translated javascript string in a JSON array of string:`;

const codeblocks = strings
.map((v) => {
return `\`\`\`${options.format}\n${v.content}\n\`\`\``;
Expand Down
3 changes: 3 additions & 0 deletions packages/cli/src/translators/json.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ export const json: Translator = {
if (changes.addedKeys.length > 0) {
translated = await generateObject({
model: options.model,
temperature: options.config.llm?.temperature ?? 0,
prompt: getPrompt(
JSON.stringify(
getContentToTranslate(sourceObj, changes.addedKeys),
Expand Down Expand Up @@ -43,6 +44,7 @@ export const json: Translator = {
const { object } = await generateObject({
model: options.model,
prompt: getPrompt(options.content, options),
temperature: options.config.llm?.temperature ?? 0,
output: "no-schema",
});

Expand Down Expand Up @@ -181,5 +183,6 @@ function getPrompt(json: string, options: PromptOptions) {
Source content (JSON), Return only the translated content with identical structure:
`;

return createBasePrompt(`${text}\n${json}`, options);
}
1 change: 1 addition & 0 deletions packages/cli/src/translators/md.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ export const markdown: Translator = {

const { object: arr } = await generateObject({
model: options.model,
temperature: options.config.llm?.temperature ?? 0,
schema: z.array(z.string()),
prompt: getPrompt(
`
Expand Down
1 change: 1 addition & 0 deletions packages/cli/src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ export interface Config {
llm: {
provider: Provider;
model: string;
temperature: number;
};
instructions?: string;
hooks?: {
Expand Down
36 changes: 20 additions & 16 deletions packages/cli/test/js.test.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
import { expect, test } from "vitest";
import { Config } from "../src/types.js";
import { readFile } from "node:fs/promises";
import { fileURLToPath } from "node:url";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { MockLanguageModelV1 } from "ai/test";
import { expect, test } from "vitest";
import { javascript } from "../src/translators/js.js";
import type { Config } from "../src/types.js";
import { getPromptText } from "./test-utils.js";

const dir = path.dirname(fileURLToPath(import.meta.url));
Expand All @@ -29,13 +29,15 @@ test("JavaScript adapter: new", async () => {
rawCall: { rawPrompt: null, rawSettings: {} },
finishReason: "stop",
usage: { promptTokens: 10, completionTokens: 20 },
text: JSON.stringify([
'"标题"',
'"介绍"',
"'在开始之前,请确保您具备以下条件:\\n一个 GitHub 账户'",
"`您可以在自己的云基础设施上自托管 Midday,以便更好地控制您的数据。\\n 本指南将引导您完成设置 Midday 的整个过程。`",
"`当前时间是 ${Date.now()}`",
]),
text: JSON.stringify({
items: [
"标题",
"介绍",
"在开始之前,请确保您具备以下条件:\n一个 GitHub 账户",
"您可以在自己的云基础设施上自托管 Midday,以便更好地控制您的数据。\n 本指南将引导您完成设置 Midday 的整个过程。",
`当前时间是 ${Date.now()}`,
],
}),
};
},
}),
Expand Down Expand Up @@ -70,12 +72,14 @@ test("JavaScript adapter: diff", async () => {
rawCall: { rawPrompt: null, rawSettings: {} },
finishReason: "stop",
usage: { promptTokens: 10, completionTokens: 20 },
text: JSON.stringify([
'"title"',
'"Updated"',
"`Updated\nUpdated`",
"`Updated ${Date.now()}`",
]),
text: JSON.stringify({
items: [
"title",
"Updated",
"Updated\nUpdated",
`Updated ${Date.now()}`,
],
}),
};
},
}),
Expand Down
6 changes: 3 additions & 3 deletions packages/cli/test/json.test.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
import { expect, test } from "vitest";
import { Config } from "../src/types.js";
import { readFile } from "node:fs/promises";
import { fileURLToPath } from "node:url";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { MockLanguageModelV1 } from "ai/test";
import { expect, test } from "vitest";
import { json } from "../src/translators/json.js";
import type { Config } from "../src/types.js";
import { getPromptText } from "./test-utils.js";

const dir = path.dirname(fileURLToPath(import.meta.url));
Expand Down
12 changes: 6 additions & 6 deletions packages/cli/test/md.test.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
import { getPromptText } from "./test-utils.js";
import { expect, test } from "vitest";
import { markdown } from "../src/translators/md.js";
import { Config } from "../src/types.js";
import { readFile } from "node:fs/promises";
import { fileURLToPath } from "node:url";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { MockLanguageModelV1 } from "ai/test";
import { expect, test } from "vitest";
import { markdown } from "../src/translators/md.js";
import type { Config } from "../src/types.js";
import { getPromptText } from "./test-utils.js";

const dir = path.dirname(fileURLToPath(import.meta.url));

Expand All @@ -24,7 +24,7 @@ test("markdown adapter: new", async () => {
rawCall: { rawPrompt: null, rawSettings: {} },
finishReason: "stop",
usage: { promptTokens: 10, completionTokens: 20 },
text: `你好,世界,这是一个用于测试翻译的测试文档。`,
text: "你好,世界,这是一个用于测试翻译的测试文档。",
}),
}),
});
Expand Down
2 changes: 1 addition & 1 deletion packages/cli/test/snapshots/js-diff.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,5 @@ export default {
content: `Updated
Updated`
},
dynamic: `Updated ${Date.now()}`
dynamic: `Translated ${Date.now()}`
}
Loading

0 comments on commit 93ee9c3

Please sign in to comment.