Skip to content

Commit

Permalink
Bug - cmd with quotes doesnt work (#17)
Browse files Browse the repository at this point in the history
The LLMs sometimes suggested commands which required shell quoting,
which got escaped into commands that didn't act as the llm expected.

Example, LLM suggested:
find ./ -name "testfile", which will be executed as 
find ./ -name \"testfile\" since the quotes becomes escaped.

* Setup tests to confirm bug
* Fixed issue by removing all single and double quotes before executing command
  • Loading branch information
baalimago authored Jul 19, 2024
1 parent c561591 commit 5b7534b
Show file tree
Hide file tree
Showing 2 changed files with 103 additions and 10 deletions.
35 changes: 25 additions & 10 deletions internal/text/querier_cmd_mode.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,11 @@ import (
"github.com/baalimago/go_away_boilerplate/pkg/ancli"
)

var (
errFormat = "code: %v, stderr: '%v', stdout: '%v'\n"
okFormat = "stdout on new line:\n%v\n"
)

func (q *Querier[C]) handleCmdMode() error {
// Tokens stream end without endline
fmt.Println()
Expand All @@ -23,27 +28,39 @@ func (q *Querier[C]) handleCmdMode() error {
case "q":
return nil
case "e":
return q.executeAiCmd()
out, err := q.executeLlmCmd()
if err == nil {
ancli.PrintOK(fmt.Sprintf("%v\n", out))
return nil
} else {
return fmt.Errorf("failed to execute cmd: %v", err)
}
default:
ancli.PrintWarn(fmt.Sprintf("unrecognized command: %v, please try again\n", input))
}
}
}

func (q *Querier[C]) executeAiCmd() error {
func (q *Querier[C]) executeLlmCmd() (string, error) {
fullMsg, err := utils.ReplaceTildeWithHome(q.fullMsg)
if err != nil {
return fmt.Errorf("parseGlob, ReplaceTildeWithHome: %w", err)
return "", fmt.Errorf("parseGlob, ReplaceTildeWithHome: %w", err)
}
// Quotes are, in 99% of the time, expanded by the shell in
// different ways and then passed into the shell. So when LLM
// suggests a command, executeAiCmd needs to act the same (meaning)
// remove/expand the quotes
fullMsg = strings.ReplaceAll(fullMsg, "\"", "")
fullMsg = strings.ReplaceAll(fullMsg, "'", "")
split := strings.Split(fullMsg, " ")
if len(split) < 1 {
return errors.New("Querier.executeAiCmd: too few tokens in q.fullMsg")
return "", errors.New("Querier.executeAiCmd: too few tokens in q.fullMsg")
}
cmd := split[0]
args := split[1:]

if len(cmd) == 0 {
return errors.New("Querier.executeAiCmd: command is empty")
return "", errors.New("Querier.executeAiCmd: command is empty")
}

command := exec.Command(cmd, args...)
Expand All @@ -58,13 +75,11 @@ func (q *Querier[C]) executeAiCmd() error {
if err != nil {
cast := &exec.ExitError{}
if errors.As(err, &cast) {
ancli.PrintErr(fmt.Sprintf("code: %v, stderr: '%v', stdout: '%v'\n", cast.ExitCode(), errStr, outStr))
return nil
return "", fmt.Errorf(errFormat, cast.ExitCode(), errStr, outStr)
} else {
return fmt.Errorf("Querier.executeAiCmd - run error: %w", err)
return "", fmt.Errorf("Querier.executeAiCmd - run error: %w", err)
}
}

ancli.PrintOK(fmt.Sprintf("stdout on new line:\n%v\n", outStr))
return nil
return fmt.Sprintf(okFormat, outStr), nil
}
78 changes: 78 additions & 0 deletions internal/text/querier_cmd_mode_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
package text

import (
"context"
"fmt"
"os"
"path/filepath"
"testing"

"github.com/baalimago/clai/internal/models"
"github.com/baalimago/go_away_boilerplate/pkg/testboil"
)

type mockCompleter struct{}

func (m mockCompleter) Setup() error {
return nil
}

func (m mockCompleter) StreamCompletions(ctx context.Context, c models.Chat) (chan models.CompletionEvent, error) {
return nil, nil
}

func Test_executeAiCmd(t *testing.T) {
testCases := []struct {
description string
setup func(t *testing.T)
given string
want string
wantErr error
}{
{
description: "it should run shell cmd",
given: "printf 'test'",
want: fmt.Sprintf(okFormat, "test"),
wantErr: nil,
},
{
description: "it should work with quotes",
setup: func(t *testing.T) {
t.Helper()
os.Chdir(filepath.Dir(testboil.CreateTestFile(t, "testfile").Name()))
},
given: "find ./ -name \"testfile\"",
want: fmt.Sprintf(okFormat, "./testfile\n"),
wantErr: nil,
},
{
description: "it should work without quotes",
setup: func(t *testing.T) {
t.Helper()
os.Chdir(filepath.Dir(testboil.CreateTestFile(t, "testfile").Name()))
},
given: "find ./ -name testfile",
want: fmt.Sprintf(okFormat, "./testfile\n"),
wantErr: nil,
},
}

for _, tc := range testCases {
t.Run(tc.description, func(t *testing.T) {
q := Querier[mockCompleter]{}
if tc.setup != nil {
tc.setup(t)
}
q.fullMsg = tc.given
gotFormated, gotErr := q.executeLlmCmd()

if gotFormated != tc.want {
t.Fatalf("expected: %v, got: %v", tc.want, gotFormated)
}

if gotErr != tc.wantErr {
t.Fatalf("expected error: %v, got: %v", tc.wantErr, gotErr)
}
})
}
}

0 comments on commit 5b7534b

Please sign in to comment.