Update containerd-shim-runhcs-v1 tests (#1783) (#1892)

Update shim tests to match current shim behavior.
Run shim tests in GitHub CI.

(cherry picked from commit 55f8c42)

Signed-off-by: Hamza El-Saawy <[email protected]>

test/containerd-shim-runhcs-v1/delete_test.go

@@ -3,8 +3,6 @@
 package main
 
 import (
-	"io/ioutil"
-	"os"
 	"testing"
 	"time"
 
@@ -17,7 +15,7 @@ func verifyDeleteCommandSuccess(t *testing.T, stdout, stderr string, runerr erro
 		t.Fatalf("expected `delete` command success got err: %v", runerr)
 	}
 	if stdout == "" {
-		t.Fatalf("expected `delete` command stdout to be non-empty, stderr: %v", stderr)
+		t.Fatalf("expected `delete` command stdout to be non-empty, stdout: %v", stdout)
 	}
 	var resp task.DeleteResponse
 	if err := proto.Unmarshal([]byte(stdout), &resp); err != nil {
@@ -29,9 +27,6 @@ func verifyDeleteCommandSuccess(t *testing.T, stdout, stderr string, runerr erro
 	if begin.After(resp.ExitedAt) || end.Before(resp.ExitedAt) {
 		t.Fatalf("DeleteResponse.ExitedAt should be between, %v and %v, got: %v", begin, end, resp.ExitedAt)
 	}
-	if stderr != "" {
-		t.Fatalf("expected `delete` command stderr to be empty got: %s", stderr)
-	}
 }
 
 func Test_Delete_No_Bundle_Arg(t *testing.T) {
@@ -71,13 +66,10 @@ func Test_Delete_No_Bundle_Path(t *testing.T) {
 }
 
 func Test_Delete_HcsSystem_NotFound(t *testing.T) {
-	dir, err := ioutil.TempDir("", "")
-	if err != nil {
-		t.Fatal("failed to create tmpdir")
-	}
-	defer func() {
-		os.RemoveAll(dir)
-	}()
+	// `delete` no longer removes bundle, but still create a directory regardless
+	//
+	// https://github.com/microsoft/hcsshim/commit/450cdb150a74aa594d7fe63bb0b3a2a37f5dd782
+	dir := t.TempDir()
 
 	before := time.Now()
 	stdout, stderr, err := runGlobalCommand(
@@ -95,7 +87,4 @@ func Test_Delete_HcsSystem_NotFound(t *testing.T) {
 		t,
 		stdout, stderr, err,
 		before, after)
-	if _, err := os.Stat(dir); err == nil || !os.IsNotExist(err) {
-		t.Fatalf("expected the bundle dir to be cleaned up. Got err: %v", err)
-	}
 }

test/containerd-shim-runhcs-v1/global_command_test.go

@@ -29,17 +29,19 @@ func runGlobalCommand(t *testing.T, args []string) (string, string, error) {
 	return outb.String(), errb.String(), err
 }
 
-func verifyGlobalCommandSuccess(t *testing.T, expectedStdout, stdout, expectedStderr, stderr string, runerr error) {
-	if runerr != nil {
-		t.Fatalf("expected no error got stdout: '%s', stderr: '%s', err: '%v'", stdout, stderr, runerr)
+func verifyGlobalCommandSuccess(t *testing.T, expectedStdout, stdout, expectedStderr, stderr string, runErr error) {
+	t.Helper()
+	if runErr != nil {
+		t.Fatalf("expected no error got stdout: '%s', stderr: '%s', err: '%v'", stdout, stderr, runErr)
 	}
 
 	verifyGlobalCommandOut(t, expectedStdout, stdout, expectedStderr, stderr)
 }
 
-func verifyGlobalCommandFailure(t *testing.T, expectedStdout, stdout, expectedStderr, stderr string, runerr error) {
-	if runerr == nil || runerr.Error() != "exit status 1" {
-		t.Fatalf("expected error: 'exit status 1', got: '%v'", runerr)
+func verifyGlobalCommandFailure(t *testing.T, expectedStdout, stdout, expectedStderr, stderr string, runErr error) {
+	t.Helper()
+	if runErr == nil || runErr.Error() != "exit status 1" {
+		t.Fatalf("expected error: 'exit status 1', got: '%v'", runErr)
 	}
 
 	verifyGlobalCommandOut(t, expectedStdout, stdout, expectedStderr, stderr)
@@ -49,14 +51,14 @@ func verifyGlobalCommandOut(t *testing.T, expectedStdout, stdout, expectedStderr
 	// stdout verify
 	if expectedStdout == "" && expectedStdout != stdout {
 		t.Fatalf("expected stdout empty got: %s", stdout)
-	} else if !strings.HasPrefix(stdout, expectedStdout) {
+	} else if !strings.Contains(stdout, expectedStdout) {
 		t.Fatalf("expected stdout to begin with: %s, got: %s", expectedStdout, stdout)
 	}
 
 	// stderr verify
 	if expectedStderr == "" && expectedStderr != stderr {
 		t.Fatalf("expected stderr empty got: %s", stderr)
-	} else if !strings.HasPrefix(stderr, expectedStderr) {
+	} else if !strings.Contains(stderr, expectedStderr) {
 		t.Fatalf("expected stderr to begin with: %s, got: %s", expectedStderr, stderr)
 	}
 }

test/containerd-shim-runhcs-v1/start_test.go

@@ -1,3 +1,4 @@
+//go:build functional
 // +build functional
 
 package main
@@ -13,6 +14,7 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/Microsoft/go-winio"
 	"github.com/containerd/containerd/runtime/v2/task"
@@ -51,7 +53,18 @@ func createStartCommandWithID(t *testing.T, id string) (*exec.Cmd, *bytes.Buffer
 }
 
 func cleanupTestBundle(t *testing.T, dir string) {
-	err := os.RemoveAll(dir)
+	t.Helper()
+	var err error
+	for i := 0; i < 2; i++ {
+		// sporadic access-denies errors if trying to delete bundle (namely panic.log) before OS realizes
+		// shim exited and releases file handle
+		if err = os.RemoveAll(dir); err == nil {
+			// does not os.RemoveAll does not if path doesn't exist
+			return
+		}
+		time.Sleep(time.Millisecond)
+	}
+
 	if err != nil {
 		t.Errorf("failed removing test bundle with: %v", err)
 	}
@@ -98,7 +111,7 @@ func verifyStartCommandSuccess(t *testing.T, expectedNamespace, expectedID strin
 
 	cl.Close()
 	c.Close()
-	if err != nil && !strings.HasPrefix(err.Error(), "ttrpc: client shutting down: ttrpc: closed") {
+	if err != nil && !strings.HasPrefix(err.Error(), "ttrpc: closed") {
 		t.Fatalf("failed to shutdown shim with: %v", err)
 	}
 }