Skip to content

Commit

Permalink
chore: support chunks for embeddings api
Browse files Browse the repository at this point in the history
  • Loading branch information
meysamhadeli committed Nov 26, 2024
1 parent 21718ba commit ed9b970
Showing 1 changed file with 9 additions and 10 deletions.
19 changes: 9 additions & 10 deletions cmd/code.go
Original file line number Diff line number Diff line change
Expand Up @@ -114,28 +114,28 @@ func handleCodeCommand(rootDependencies *RootDependencies) {
// Split file into chunks of up to 8000 tokens
fileChunks, err := utils.SplitIntoChunks(dataFile.Code, maxTokens)
if err != nil {
spinnerLoadContext.Stop()
spinnerEmbeddingContext.Stop()
fmt.Print("\r")
fmt.Println(lipgloss.Red.Render(fmt.Sprintf("Failed to split file '%s' into chunks: %v", dataFile.RelativePath, err)))
break
return
}

for _, chunk := range fileChunks {
tokenCount, err := utils.TokenCount(chunk)
if err != nil {
spinnerLoadContext.Stop()
spinnerEmbeddingContext.Stop()
fmt.Print("\r")
fmt.Println(lipgloss.Red.Render(fmt.Sprintf("Failed to calculate token count: %v", err)))
break
return
}

if currentTokenCount+tokenCount > maxTokens {
// Process the current chunks before adding more
if err := processEmbeddingsChunks(); err != nil {
spinnerLoadContext.Stop()
spinnerEmbeddingContext.Stop()
fmt.Print("\r")
fmt.Println(lipgloss.Red.Render(fmt.Sprintf("Failed to process chunk: %v", err)))
break
return
}
}

Expand All @@ -150,9 +150,10 @@ func handleCodeCommand(rootDependencies *RootDependencies) {

// Process any remaining chunks
if err := processEmbeddingsChunks(); err != nil {
spinnerLoadContext.Stop()
spinnerEmbeddingContext.Stop()
fmt.Print("\r")
fmt.Println(lipgloss.Red.Render(fmt.Sprintf("Failed to process remaining chunks: %v", err)))
return
}

spinnerEmbeddingContext.Stop()
Expand Down Expand Up @@ -317,9 +318,7 @@ startLoop: // Label for the start loop
RelativePath: change.RelativePath,
})

err = processEmbeddingsChunks()

if err != nil {
if err := processEmbeddingsChunks(); err != nil {
spinnerUpdateContext.Stop()
fmt.Print("\r")
fmt.Println(lipgloss.Red.Render(fmt.Sprintf("%v", err)))
Expand Down

0 comments on commit ed9b970

Please sign in to comment.