Use a single thread for the reranking service

felladrin · Jan 25, 2025 · 34d2404 · 34d2404
1 parent e717da0
commit 34d2404
Showing 1 changed file with 4 additions and 7 deletions.
diff --git a/server/rerankerService.ts b/server/rerankerService.ts
@@ -40,9 +40,7 @@ export async function startRerankerService() {
   await ensureModelExists(modelPath);
   printMessage("Starting service...");
 
-  const contextSize = 8192;
-  const threads = Math.min(Math.max(1, os.cpus().length - 2), 16);
-  const batchSize = Math.ceil(contextSize / threads);
+  const contextSize = 2048;
 
   const serverProcess = spawn(
     "llama-server",
@@ -51,19 +49,18 @@ export async function startRerankerService() {
       modelPath,
       "--ctx-size",
       contextSize.toString(),
-      "--parallel",
-      threads.toString(),
       "--batch-size",
-      batchSize.toString(),
+      contextSize.toString(),
       "--ubatch-size",
-      batchSize.toString(),
+      contextSize.toString(),
       "--flash-attn",
       "--host",
       SERVER_HOST,
       "--port",
       SERVER_PORT.toString(),
       "--log-verbosity",
       VERBOSE_MODE ? "1" : "0",
+      "--no-warmup",
       "--reranking",
       "--pooling",
       "rank",