diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ca9f3c685d..24ed236cae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -250,8 +250,7 @@ jobs: run: ${{ env.GOTESTCMD }} -gcflags=all=-d=checkptr -tags admin ./... - name: Run non-functional tests - run: ${{ env.GOTESTCMD }} -mod=mod -gcflags=all=-d=checkptr -tags admin ./... - + run: ${{ env.GOTESTCMD }} -mod=mod -gcflags=all=-d=checkptr ./internal/... ./pkg/... working-directory: test - name: Run containerd-shim-runhcs-v1 tests diff --git a/test/cri-containerd/container_gmsa_test.go b/test/cri-containerd/container_gmsa_test.go index 14be0d585b..5d34c3426c 100644 --- a/test/cri-containerd/container_gmsa_test.go +++ b/test/cri-containerd/container_gmsa_test.go @@ -16,6 +16,7 @@ import ( func Test_RunContainer_GMSA_WCOW(t *testing.T) { requireFeatures(t, featureGMSA) + requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor) credSpec := gmsaSetup(t) pullRequiredImages(t, []string{imageWindowsNanoserver}) @@ -98,6 +99,7 @@ func Test_RunContainer_GMSA_WCOW(t *testing.T) { } func Test_RunContainer_GMSA_Disabled(t *testing.T) { requireFeatures(t, featureGMSA) + requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor) credSpec := "totally real and definitely not a fake or arbitrary gMSA credential spec that is 1000%% properly formatted as JSON" pullRequiredImages(t, []string{imageWindowsNanoserver}) diff --git a/test/cri-containerd/container_network_test.go b/test/cri-containerd/container_network_test.go index 6e4888ea85..269f7beeab 100644 --- a/test/cri-containerd/container_network_test.go +++ b/test/cri-containerd/container_network_test.go @@ -84,6 +84,8 @@ func Test_Container_Network_LCOW(t *testing.T) { } func Test_Container_Network_Hostname(t *testing.T) { + requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor, featureLCOW) + type config struct { name string requiredFeatures []string diff --git a/test/cri-containerd/container_update_test.go b/test/cri-containerd/container_update_test.go index 53939a0021..af3831ebc6 100644 --- a/test/cri-containerd/container_update_test.go +++ b/test/cri-containerd/container_update_test.go @@ -34,7 +34,9 @@ func calculateJobCPURate(hostProcs uint32, processorCount uint32) uint32 { } func Test_Container_UpdateResources_CPUShare(t *testing.T) { + requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor, featureLCOW) require.Build(t, osversion.V20H2) + type config struct { name string requiredFeatures []string @@ -142,6 +144,9 @@ func Test_Container_UpdateResources_CPUShare(t *testing.T) { } func Test_Container_UpdateResources_CPUShare_NotRunning(t *testing.T) { + requireFeatures(t, featureCRIUpdateContainer) + requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor, featureLCOW) + type config struct { name string requiredFeatures []string @@ -153,7 +158,7 @@ func Test_Container_UpdateResources_CPUShare_NotRunning(t *testing.T) { tests := []config{ { name: "WCOW_Process", - requiredFeatures: []string{featureWCOWProcess, featureCRIUpdateContainer}, + requiredFeatures: []string{featureWCOWProcess}, runtimeHandler: wcowProcessRuntimeHandler, sandboxImage: imageWindowsNanoserver, containerImage: imageWindowsNanoserver, @@ -161,7 +166,7 @@ func Test_Container_UpdateResources_CPUShare_NotRunning(t *testing.T) { }, { name: "WCOW_Hypervisor", - requiredFeatures: []string{featureWCOWHypervisor, featureCRIUpdateContainer}, + requiredFeatures: []string{featureWCOWHypervisor}, runtimeHandler: wcowHypervisorRuntimeHandler, sandboxImage: imageWindowsNanoserver, containerImage: imageWindowsNanoserver, @@ -169,7 +174,7 @@ func Test_Container_UpdateResources_CPUShare_NotRunning(t *testing.T) { }, { name: "LCOW", - requiredFeatures: []string{featureLCOW, featureCRIUpdateContainer}, + requiredFeatures: []string{featureLCOW}, runtimeHandler: lcowRuntimeHandler, sandboxImage: imageLcowK8sPause, containerImage: imageLcowAlpine, @@ -249,6 +254,9 @@ func Test_Container_UpdateResources_CPUShare_NotRunning(t *testing.T) { } func Test_Container_UpdateResources_Memory(t *testing.T) { + requireFeatures(t, featureCRIUpdateContainer) + requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor, featureLCOW) + type config struct { name string requiredFeatures []string @@ -260,7 +268,7 @@ func Test_Container_UpdateResources_Memory(t *testing.T) { tests := []config{ { name: "WCOW_Process", - requiredFeatures: []string{featureWCOWProcess, featureCRIUpdateContainer}, + requiredFeatures: []string{featureWCOWProcess}, runtimeHandler: wcowProcessRuntimeHandler, sandboxImage: imageWindowsNanoserver, containerImage: imageWindowsNanoserver, @@ -268,7 +276,7 @@ func Test_Container_UpdateResources_Memory(t *testing.T) { }, { name: "WCOW_Hypervisor", - requiredFeatures: []string{featureWCOWHypervisor, featureCRIUpdateContainer}, + requiredFeatures: []string{featureWCOWHypervisor}, runtimeHandler: wcowHypervisorRuntimeHandler, sandboxImage: imageWindowsNanoserver, containerImage: imageWindowsNanoserver, @@ -276,7 +284,7 @@ func Test_Container_UpdateResources_Memory(t *testing.T) { }, { name: "LCOW", - requiredFeatures: []string{featureLCOW, featureCRIUpdateContainer}, + requiredFeatures: []string{featureLCOW}, runtimeHandler: lcowRuntimeHandler, sandboxImage: imageLcowK8sPause, containerImage: imageLcowAlpine, diff --git a/test/cri-containerd/containerdrestart_test.go b/test/cri-containerd/containerdrestart_test.go index b39c10b686..c9ba734983 100644 --- a/test/cri-containerd/containerdrestart_test.go +++ b/test/cri-containerd/containerdrestart_test.go @@ -77,6 +77,7 @@ func Test_ContainerdRestart_LCOW(t *testing.T) { // test restarting containers and pods func Test_Container_CRI_Restart(t *testing.T) { requireFeatures(t, featureCRIPlugin) + requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor, featureLCOW) client := newTestRuntimeClient(t) pluginClient := newTestPluginClient(t) @@ -225,6 +226,7 @@ func Test_Container_CRI_Restart_State(t *testing.T) { wcowTestFile := `C:\Users\ContainerUser\t.txt` requireFeatures(t, featureCRIPlugin) + requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor, featureLCOW) client := newTestRuntimeClient(t) ctx, cancel := context.WithCancel(context.Background()) diff --git a/test/cri-containerd/extended_task_test.go b/test/cri-containerd/extended_task_test.go index 87c8772fcb..b718153e5b 100644 --- a/test/cri-containerd/extended_task_test.go +++ b/test/cri-containerd/extended_task_test.go @@ -24,6 +24,8 @@ func getPodProcessorInfo(ctx context.Context, podID string) (*extendedtask.Compu } func Test_ExtendedTask_ProcessorInfo(t *testing.T) { + requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor, featureLCOW) + type config struct { name string requiredFeatures []string diff --git a/test/cri-containerd/jobcontainer_test.go b/test/cri-containerd/jobcontainer_test.go index ec5e0fd64f..421f195223 100644 --- a/test/cri-containerd/jobcontainer_test.go +++ b/test/cri-containerd/jobcontainer_test.go @@ -419,9 +419,10 @@ func Test_RunContainer_HostVolumes_JobContainer_WCOW(t *testing.T) { } func Test_RunContainer_JobContainer_VolumeMount(t *testing.T) { - client := newTestRuntimeClient(t) + requireFeatures(t, featureWCOWProcess, featureHostProcess) require.ExactBuild(t, osversion.RS5) + client := newTestRuntimeClient(t) dir := t.TempDir() tmpfn := filepath.Join(dir, "tmpfile") @@ -451,49 +452,43 @@ func Test_RunContainer_JobContainer_VolumeMount(t *testing.T) { } type config struct { - name string - containerName string - requiredFeatures []string - sandboxImage string - containerImage string - exec []string - mounts []*runtime.Mount + name string + containerName string + sandboxImage string + containerImage string + exec []string + mounts []*runtime.Mount } tests := []config{ { - name: "JobContainer_VolumeMount_DriveLetter", - containerName: t.Name() + "-Container-DriveLetter", - requiredFeatures: []string{featureWCOWProcess, featureHostProcess}, - sandboxImage: imageWindowsNanoserver, - containerImage: imageWindowsNanoserver, - mounts: mountDriveLetter, - exec: []string{"cmd", "/c", "dir", "%CONTAINER_SANDBOX_MOUNT_POINT%\\path\\in\\container\\tmpfile"}, + name: "JobContainer_VolumeMount_DriveLetter", + containerName: t.Name() + "-Container-DriveLetter", + sandboxImage: imageWindowsNanoserver, + containerImage: imageWindowsNanoserver, + mounts: mountDriveLetter, + exec: []string{"cmd", "/c", "dir", "%CONTAINER_SANDBOX_MOUNT_POINT%\\path\\in\\container\\tmpfile"}, }, { - name: "JobContainer_VolumeMount_NoDriveLetter", - containerName: t.Name() + "-Container-NoDriveLetter", - requiredFeatures: []string{featureWCOWProcess, featureHostProcess}, - sandboxImage: imageWindowsNanoserver, - containerImage: imageWindowsNanoserver, - mounts: mountNoDriveLetter, - exec: []string{"cmd", "/c", "dir", "%CONTAINER_SANDBOX_MOUNT_POINT%\\path\\in\\container\\tmpfile"}, + name: "JobContainer_VolumeMount_NoDriveLetter", + containerName: t.Name() + "-Container-NoDriveLetter", + sandboxImage: imageWindowsNanoserver, + containerImage: imageWindowsNanoserver, + mounts: mountNoDriveLetter, + exec: []string{"cmd", "/c", "dir", "%CONTAINER_SANDBOX_MOUNT_POINT%\\path\\in\\container\\tmpfile"}, }, { - name: "JobContainer_VolumeMount_SingleFile", - containerName: t.Name() + "-Container-SingleFile", - requiredFeatures: []string{featureWCOWProcess, featureHostProcess}, - sandboxImage: imageWindowsNanoserver, - containerImage: imageWindowsNanoserver, - mounts: mountSingleFile, - exec: []string{"cmd", "/c", "type", "%CONTAINER_SANDBOX_MOUNT_POINT%\\path\\in\\container\\testfile"}, + name: "JobContainer_VolumeMount_SingleFile", + containerName: t.Name() + "-Container-SingleFile", + sandboxImage: imageWindowsNanoserver, + containerImage: imageWindowsNanoserver, + mounts: mountSingleFile, + exec: []string{"cmd", "/c", "type", "%CONTAINER_SANDBOX_MOUNT_POINT%\\path\\in\\container\\testfile"}, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - requireFeatures(t, test.requiredFeatures...) - requiredImages := []string{test.sandboxImage, test.containerImage} pullRequiredImages(t, requiredImages) @@ -525,25 +520,25 @@ func Test_RunContainer_JobContainer_VolumeMount(t *testing.T) { } func Test_RunContainer_JobContainer_Environment(t *testing.T) { + requireFeatures(t, featureWCOWProcess, featureHostProcess) + client := newTestRuntimeClient(t) type config struct { - name string - containerName string - requiredFeatures []string - sandboxImage string - containerImage string - env []*runtime.KeyValue - exec []string + name string + containerName string + sandboxImage string + containerImage string + env []*runtime.KeyValue + exec []string } tests := []config{ { - name: "JobContainer_Env_NoMountPoint", - containerName: t.Name() + "-Container-WithNoMountPoint", - requiredFeatures: []string{featureWCOWProcess, featureHostProcess}, - sandboxImage: imageWindowsNanoserver, - containerImage: imageWindowsNanoserver, + name: "JobContainer_Env_NoMountPoint", + containerName: t.Name() + "-Container-WithNoMountPoint", + sandboxImage: imageWindowsNanoserver, + containerImage: imageWindowsNanoserver, env: []*runtime.KeyValue{ { Key: "PATH", Value: "C:\\Windows\\system32;C:\\Windows", @@ -552,11 +547,10 @@ func Test_RunContainer_JobContainer_Environment(t *testing.T) { exec: []string{"cmd", "/c", "IF", "%PATH%", "==", "C:\\Windows\\system32;C:\\Windows", "( exit 0 )", "ELSE", "(exit -1)"}, }, { - name: "JobContainer_VolumeMount_WithMountPoint", - containerName: t.Name() + "-Container-WithMountPoint", - requiredFeatures: []string{featureWCOWProcess, featureHostProcess}, - sandboxImage: imageWindowsNanoserver, - containerImage: imageWindowsNanoserver, + name: "JobContainer_VolumeMount_WithMountPoint", + containerName: t.Name() + "-Container-WithMountPoint", + sandboxImage: imageWindowsNanoserver, + containerImage: imageWindowsNanoserver, env: []*runtime.KeyValue{ { Key: "PATH", Value: "%CONTAINER_SANDBOX_MOUNT_POINT%\\apps\\vim\\;C:\\Windows\\system32;C:\\Windows", @@ -568,8 +562,6 @@ func Test_RunContainer_JobContainer_Environment(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - requireFeatures(t, test.requiredFeatures...) - requiredImages := []string{test.sandboxImage, test.containerImage} pullRequiredImages(t, requiredImages) @@ -602,57 +594,52 @@ func Test_RunContainer_JobContainer_Environment(t *testing.T) { } func Test_RunContainer_WorkingDirectory_JobContainer_WCOW(t *testing.T) { - client := newTestRuntimeClient(t) + requireFeatures(t, featureWCOWProcess, featureHostProcess) require.ExactBuild(t, osversion.RS5) + client := newTestRuntimeClient(t) + type config struct { - name string - containerName string //nolint:unused // may be used in future tests - workDir string - requiredFeatures []string - sandboxImage string - containerImage string - cmd []string + name string + containerName string //nolint:unused // may be used in future tests + workDir string + sandboxImage string + containerImage string + cmd []string } tests := []config{ { - name: "JobContainer_WorkDir_DriveLetter", - workDir: "C:\\go\\", - requiredFeatures: []string{featureWCOWProcess, featureHostProcess}, - sandboxImage: imageWindowsNanoserver, - containerImage: imageJobContainerWorkDir, - cmd: []string{"src\\workdir\\workdir.exe"}, + name: "JobContainer_WorkDir_DriveLetter", + workDir: "C:\\go\\", + sandboxImage: imageWindowsNanoserver, + containerImage: imageJobContainerWorkDir, + cmd: []string{"src\\workdir\\workdir.exe"}, }, { - name: "JobContainer_WorkDir_NoDriveLetter", - workDir: "/go", - requiredFeatures: []string{featureWCOWProcess, featureHostProcess}, - sandboxImage: imageWindowsNanoserver, - containerImage: imageJobContainerWorkDir, - cmd: []string{"src/workdir/workdir.exe"}, + name: "JobContainer_WorkDir_NoDriveLetter", + workDir: "/go", + sandboxImage: imageWindowsNanoserver, + containerImage: imageJobContainerWorkDir, + cmd: []string{"src/workdir/workdir.exe"}, }, { - name: "JobContainer_WorkDir_Default", // Just use the workdir from the image, which is C:\\go\\src\\workdir - requiredFeatures: []string{featureWCOWProcess, featureHostProcess}, - sandboxImage: imageWindowsNanoserver, - containerImage: imageJobContainerWorkDir, - cmd: []string{"workdir.exe"}, + name: "JobContainer_WorkDir_Default", // Just use the workdir from the image, which is C:\\go\\src\\workdir + sandboxImage: imageWindowsNanoserver, + containerImage: imageJobContainerWorkDir, + cmd: []string{"workdir.exe"}, }, { - name: "JobContainer_WorkDir_EnvVar", // Test that putting the envvar in the workdir functions. - workDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT\\go\\src\\workdir\\", - requiredFeatures: []string{featureWCOWProcess, featureHostProcess}, - sandboxImage: imageWindowsNanoserver, - containerImage: imageJobContainerWorkDir, - cmd: []string{"workdir.exe"}, + name: "JobContainer_WorkDir_EnvVar", // Test that putting the envvar in the workdir functions. + workDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT\\go\\src\\workdir\\", + sandboxImage: imageWindowsNanoserver, + containerImage: imageJobContainerWorkDir, + cmd: []string{"workdir.exe"}, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - requireFeatures(t, test.requiredFeatures...) - requiredImages := []string{test.sandboxImage, test.containerImage} pullRequiredImages(t, requiredImages) diff --git a/test/cri-containerd/logging_binary_test.go b/test/cri-containerd/logging_binary_test.go index 2bae198fbd..450147bb62 100644 --- a/test/cri-containerd/logging_binary_test.go +++ b/test/cri-containerd/logging_binary_test.go @@ -12,6 +12,7 @@ import ( "testing" "time" + "github.com/Microsoft/hcsshim/test/pkg/require" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) @@ -21,12 +22,13 @@ import ( // which this test will use to construct logPath for CreateContainerRequest and as // the location of stdout artifacts created by the binary func Test_Run_Container_With_Binary_Logger(t *testing.T) { + requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor, featureLCOW) + binaryPath := require.Binary(t, "sample-logging-driver.exe") + client := newTestRuntimeClient(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - binaryPath := requireBinary(t, "sample-logging-driver.exe") - logPath := "binary:///" + binaryPath type config struct { diff --git a/test/cri-containerd/main_test.go b/test/cri-containerd/main_test.go index de43d4063b..e89dd37aa9 100644 --- a/test/cri-containerd/main_test.go +++ b/test/cri-containerd/main_test.go @@ -9,7 +9,6 @@ import ( "flag" "fmt" "os" - "path/filepath" "testing" "time" @@ -145,23 +144,13 @@ func requireFeatures(tb testing.TB, features ...string) { require.Features(tb, flagFeatures, features...) } -// requireBinary checks if `binary` exists in the same directory as the test -// binary. -// Returns full binary path if it exists, otherwise, skips the test. -func requireBinary(tb testing.TB, binary string) string { +// requireAnyFeatures checks in flagFeatures if at least one of the required features +// was enabled, and skips the test if all are missing. +// +// See: [requireFeatures] +func requireAnyFeature(tb testing.TB, features ...string) { tb.Helper() - executable, err := os.Executable() - if err != nil { - tb.Skipf("error locating executable: %s", err) - return "" - } - baseDir := filepath.Dir(executable) - binaryPath := filepath.Join(baseDir, binary) - if _, err := os.Stat(binaryPath); os.IsNotExist(err) { - tb.Skipf("binary not found: %s", binaryPath) - return "" - } - return binaryPath + require.AnyFeature(tb, flagFeatures, features...) } func getWindowsNanoserverImage(build uint16) string { diff --git a/test/cri-containerd/pod_update_test.go b/test/cri-containerd/pod_update_test.go index 6cb541ebc1..ff5bb8ff21 100644 --- a/test/cri-containerd/pod_update_test.go +++ b/test/cri-containerd/pod_update_test.go @@ -16,6 +16,8 @@ import ( ) func Test_Pod_UpdateResources_Memory(t *testing.T) { + requireAnyFeature(t, featureLCOW, featureWCOWHypervisor) + type config struct { name string requiredFeatures []string @@ -87,6 +89,8 @@ func Test_Pod_UpdateResources_Memory(t *testing.T) { } func Test_Pod_UpdateResources_Memory_PA(t *testing.T) { + requireAnyFeature(t, featureLCOW, featureWCOWHypervisor) + type config struct { name string requiredFeatures []string @@ -159,6 +163,8 @@ func Test_Pod_UpdateResources_Memory_PA(t *testing.T) { } func Test_Pod_UpdateResources_CPUShares(t *testing.T) { + requireAnyFeature(t, featureLCOW, featureWCOWHypervisor) + type config struct { name string requiredFeatures []string @@ -222,6 +228,8 @@ func Test_Pod_UpdateResources_CPUShares(t *testing.T) { func Test_Pod_UpdateResources_CPUGroup(t *testing.T) { t.Skip("Skipping for now") + requireAnyFeature(t, featureLCOW, featureWCOWHypervisor) + ctx := context.Background() processorTopology, err := processorinfo.HostProcessorInfo(ctx) diff --git a/test/cri-containerd/policy_test.go b/test/cri-containerd/policy_test.go index 0bad446faa..b6e906dd79 100644 --- a/test/cri-containerd/policy_test.go +++ b/test/cri-containerd/policy_test.go @@ -22,6 +22,7 @@ import ( "github.com/Microsoft/hcsshim/pkg/annotations" "github.com/Microsoft/hcsshim/pkg/securitypolicy" "github.com/Microsoft/hcsshim/test/pkg/definitions/tools/securitypolicy/helpers" + "github.com/Microsoft/hcsshim/test/pkg/require" ) var validPolicyAlpineCommand = []string{"ash", "-c", "echo 'Hello'"} @@ -1025,13 +1026,12 @@ func Test_RunPodSandboxNotAllowed_WithPolicy_EncryptedScratchPolicy(t *testing.T func Test_RunContainer_WithPolicy_And_Binary_Logger_Without_Stdio(t *testing.T) { requireFeatures(t, featureLCOWIntegrity) + binaryPath := require.Binary(t, "sample-logging-driver.exe") client := newTestRuntimeClient(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - binaryPath := requireBinary(t, "sample-logging-driver.exe") - logPath := "binary:///" + binaryPath pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) diff --git a/test/cri-containerd/removepodsandbox_test.go b/test/cri-containerd/removepodsandbox_test.go index f17823fdd5..10304b3098 100644 --- a/test/cri-containerd/removepodsandbox_test.go +++ b/test/cri-containerd/removepodsandbox_test.go @@ -85,6 +85,8 @@ func runContainerInSandboxTest(t *testing.T, tc *TestConfig) { } func Test_RunPodSandbox_Without_Sandbox_Stop(t *testing.T) { + requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor, featureLCOW) + tests := []TestConfig{ { name: "WCOW_Process", @@ -123,6 +125,8 @@ func Test_RunPodSandbox_Without_Sandbox_Stop(t *testing.T) { } func Test_RunContainer_Without_Sandbox_Stop(t *testing.T) { + requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor, featureLCOW) + tests := []TestConfig{ { name: "WCOW_Process", diff --git a/test/cri-containerd/runpodsandbox_test.go b/test/cri-containerd/runpodsandbox_test.go index 2d18980ba0..7a269a7681 100644 --- a/test/cri-containerd/runpodsandbox_test.go +++ b/test/cri-containerd/runpodsandbox_test.go @@ -984,7 +984,9 @@ func Test_RunPodSandbox_Mount_SandboxDir_NoShare_WCOW(t *testing.T) { } func Test_RunPodSandbox_CPUGroup(t *testing.T) { + requireAnyFeature(t, featureLCOW, featureWCOWHypervisor) require.Build(t, osversion.V21H1) + ctx := context.Background() presentID := "FA22A12C-36B3-486D-A3E9-BC526C2B450B" diff --git a/test/cri-containerd/stats_test.go b/test/cri-containerd/stats_test.go index 05c73dd269..15e2040982 100644 --- a/test/cri-containerd/stats_test.go +++ b/test/cri-containerd/stats_test.go @@ -186,6 +186,8 @@ func Test_SandboxStats_List_PodID_LCOW(t *testing.T) { } func Test_ContainerStats_ContainerID(t *testing.T) { + requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor, featureLCOW) + type config struct { name string requiredFeatures []string @@ -260,6 +262,8 @@ func Test_ContainerStats_ContainerID(t *testing.T) { } func Test_ContainerStats_List_ContainerID(t *testing.T) { + requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor, featureLCOW) + type config struct { name string requiredFeatures []string @@ -334,6 +338,8 @@ func Test_ContainerStats_List_ContainerID(t *testing.T) { } func Test_SandboxStats_WorkingSet_PhysicallyBacked(t *testing.T) { + requireAnyFeature(t, featureLCOW, featureWCOWHypervisor) + type config struct { name string requiredFeatures []string diff --git a/test/cri-containerd/stopcontainer_test.go b/test/cri-containerd/stopcontainer_test.go index ec5756659d..d5343cead2 100644 --- a/test/cri-containerd/stopcontainer_test.go +++ b/test/cri-containerd/stopcontainer_test.go @@ -183,6 +183,7 @@ func Test_StopContainer_ReusePod_LCOW(t *testing.T) { func Test_GracefulTermination(t *testing.T) { // The test image is based on 2022 servercore and nanoserver images. Ensure // that we are running on the correct OS version + requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor) require.Build(t, osversion.V21H2Server) for name, tc := range map[string]struct { diff --git a/test/go.mod b/test/go.mod index 4b71cc5f04..2060fa127d 100644 --- a/test/go.mod +++ b/test/go.mod @@ -20,6 +20,7 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.8.4 go.opencensus.io v0.24.0 + golang.org/x/exp v0.0.0-20220827204233-334a2380cb91 golang.org/x/sync v0.3.0 golang.org/x/sys v0.11.0 google.golang.org/grpc v1.57.0 diff --git a/test/go.sum b/test/go.sum index 28575b7735..195647d046 100644 --- a/test/go.sum +++ b/test/go.sum @@ -1909,6 +1909,7 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20220827204233-334a2380cb91 h1:tnebWN09GYg9OLPss1KXj8txwZc6X6uMr6VFdcGNbHw= golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= diff --git a/test/pkg/flag/flag.go b/test/pkg/flag/flag.go index b76c6a1be8..c5f81cae20 100644 --- a/test/pkg/flag/flag.go +++ b/test/pkg/flag/flag.go @@ -1,6 +1,3 @@ -// This package augments the default "flags" package with functionality similar -// to that in "github.com/urfave/cli", since the two packages do not mix easily -// and the "testing" package uses a default flagset that we cannot easily update. package flag import ( @@ -8,24 +5,101 @@ import ( "strings" "github.com/sirupsen/logrus" + "golang.org/x/exp/maps" + "golang.org/x/exp/slices" ) -const FeatureFlagName = "feature" +const ( + FeatureFlagName = "feature" + ExcludeFeatureFlagName = "exclude" +) + +// NewFeatureFlag defines two flags, [FeatureFlagName] and [ExcludeFeatureFlagName], to +// allow setting and excluding certain features. +func NewFeatureFlag(features []string) *IncludeExcludeStringSet { + fs := NewStringSet(FeatureFlagName, + "`features` to test; can be set multiple times, with a comma-separated list, or both. "+ + "Leave empty to enable all features. "+ + "(supported features: "+strings.Join(features, ", ")+")", false) + + return NewIncludeExcludeStringSet(fs, ExcludeFeatureFlagName, + "`features` to exclude from tests (see "+FeatureFlagName+" for more details)", + features) +} + +// IncludeExcludeStringSet allows unsetting strings seen in a [StringSet]. +type IncludeExcludeStringSet struct { + // flags explicitly included + inc *StringSet + // flags explicitly excluded + exc *StringSet + // def value, if no values set + // we don't error if an unknown value is provided + def []string +} + +// NewIncludeExcludeStringSet returns a new NewIncludeExcludeStringSet. +func NewIncludeExcludeStringSet(include *StringSet, name, usage string, all []string) *IncludeExcludeStringSet { + es := &IncludeExcludeStringSet{ + inc: include, + exc: &StringSet{ + s: make(map[string]struct{}), + cs: include.cs, + }, + def: slices.Clone(all), + } + flag.Var(es, name, usage) + return es +} + +var _ flag.Value = &IncludeExcludeStringSet{} + +func (es *IncludeExcludeStringSet) Set(s string) error { return es.exc.Set(s) } + +func (es *IncludeExcludeStringSet) String() string { + if es == nil { // may be called by flag package on nil receiver + return "" + } + ss := es.strings() + if len(ss) == 0 { + return "" + } + return "[" + strings.Join(ss, ", ") + "]" +} + +func (es *IncludeExcludeStringSet) Strings() []string { return es.strings() } +func (es *IncludeExcludeStringSet) Len() int { return len(es.strings()) } -func NewFeatureFlag(all []string) *StringSet { - return NewStringSet(FeatureFlagName, - "set of `features` to test; can be set multiple times, with a comma-separated list, or both "+ - "(supported features: "+strings.Join(all, ", ")+")", - false, - ) +func (es *IncludeExcludeStringSet) strings() []string { + ss := es.def + set := make([]string, 0, len(ss)) + if es.inc != nil && es.inc.Len() > 0 { + // include values were explicitly set + ss = es.inc.Strings() + } + for _, s := range ss { + if !es.exc.IsSet(s) { + set = append(set, s) + } + } + return set +} + +func (es *IncludeExcludeStringSet) IsSet(s string) bool { + if es.inc == nil || es.inc.Len() == 0 || es.inc.IsSet(s) { + // either no values were included, or value was explicitly provided + return !es.exc.IsSet(s) + } + return false } // StringSet is a type to be used with the standard library's flag.Var // function as a custom flag value, similar to "github.com/urfave/cli".StringSet, // but it only tracks unique instances. +// // It takes either a comma-separated list of strings, or repeated invocations. type StringSet struct { - s map[string]struct{} + s stringSet // cs indicates if the set is case sensitive or not cs bool } @@ -43,36 +117,28 @@ func NewStringSet(name, usage string, caseSensitive bool) *StringSet { } // Strings returns a string slice of the flags provided to the flag -func (ss *StringSet) Strings() []string { - a := make([]string, 0, len(ss.s)) - for k := range ss.s { - a = append(a, k) - } - - return a -} +func (ss *StringSet) Strings() []string { return maps.Keys(ss.s) } func (ss *StringSet) String() string { + if ss == nil || ss.Len() == 0 { // may be called by flag package on nil receiver + return "" + } return "[" + strings.Join(ss.Strings(), ", ") + "]" } func (ss *StringSet) Len() int { return len(ss.s) } -func (ss *StringSet) IsSet(s string) bool { - _, ok := ss.s[ss.standardize(s)] - return ok -} +func (ss *StringSet) IsSet(s string) bool { return ss.s.isSet(ss.standardize(s)) } -// Set is called by `flag` each time the flag is seen when parsing the -// command line. +// Set is called by `flag` each time the flag is seen when parsing the command line. func (ss *StringSet) Set(s string) error { for _, f := range strings.Split(s, ",") { - ss.s[ss.standardize(f)] = struct{}{} + ss.s.set(ss.standardize(f)) } return nil } -// Standardize formats the feature flag s to be consistent (ie, trim and to lowercase) +// standardize formats the feature flag s to be consistent (ie, trim and to lowercase) func (ss *StringSet) standardize(s string) string { s = strings.TrimSpace(s) if !ss.cs { @@ -81,7 +147,16 @@ func (ss *StringSet) standardize(s string) string { return s } -// LogrusLevel is a flag that accepts logrus logging levels, as strings. +// stringSet is a set of strings. +type stringSet map[string]struct{} + +func (ss stringSet) set(s string) { ss[s] = struct{}{} } +func (ss stringSet) isSet(s string) bool { + _, ok := ss[s] + return ok +} + +// LogrusLevel is a flag that accepts logrus logging levels as strings. type LogrusLevel struct { Level logrus.Level } diff --git a/test/pkg/flag/flag_test.go b/test/pkg/flag/flag_test.go new file mode 100644 index 0000000000..04b4f5ef58 --- /dev/null +++ b/test/pkg/flag/flag_test.go @@ -0,0 +1,126 @@ +package flag + +import ( + "fmt" + "testing" + + "golang.org/x/exp/slices" +) + +// tests for testing fixtures ... + +// calling New(IncludeExclude)?StringSet will add it to the default flag set, +// which may cause problems since [testing] already defines flags + +func Test_ExcludeStringSetFlag(t *testing.T) { + all := []string{"one", "two", "three", "four", "zero"} + es := &IncludeExcludeStringSet{ + inc: &StringSet{ + s: make(map[string]struct{}), + cs: false, + }, + exc: &StringSet{ + s: make(map[string]struct{}), + cs: false, + }, + def: slices.Clone(all), + } + + orderlessEq(t, all, es.Strings()) + for _, s := range all { + assert(t, es.IsSet(s), s+" is expected to be set, but is not") + } + + for i, tc := range []struct { + set []string + unset []string + exp []string + }{ + { + unset: []string{"one", "five"}, + exp: []string{"two", "three", "four", "zero"}, + }, + { + unset: []string{"one, three"}, + exp: []string{"two", "four", "zero"}, + }, + { + set: []string{"two,one"}, + exp: []string{"two"}, + }, + { + set: []string{"three,one", " zero "}, + exp: []string{"two", "zero"}, + }, + } { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + for _, s := range tc.set { + must(t, es.inc.Set(s)) + } + for _, s := range tc.unset { + must(t, es.exc.Set(s)) + } + + orderlessEq(t, tc.exp, es.Strings()) + + for _, s := range all { + if slices.Contains(tc.exp, s) { + assert(t, es.IsSet(s), s+" is expected to be set, but is not") + } else { + assert(t, !es.IsSet(s), s+" is not expected to be set, but is") + } + } + }) + } +} + +func Test_StringSetFlag(t *testing.T) { + ss := &StringSet{ + s: make(map[string]struct{}), + cs: false, + } + + must(t, ss.Set("hi,bye,HI")) + must(t, ss.Set("Bye")) + must(t, ss.Set("not a word")) + exp := []string{"hi", "bye", "not a word"} + + orderlessEq(t, exp, ss.Strings()) + for _, s := range exp { + assert(t, ss.IsSet(s), s+"is expected to be set, but is not") + } + for _, s := range []string{"HI", "bYe", "BYE", " not A wOrD"} { + assert(t, ss.IsSet(s), s+"is expected to be set, but is not") + } + for _, s := range []string{"hello", "goodbye", "also not a word"} { + assert(t, !ss.IsSet(s), s+"is not expected to be set, but is") + } +} + +func orderlessEq(tb testing.TB, exp, got []string) { + tb.Helper() + if len(exp) != len(got) { + tb.Fatalf("expected length %d (%s), got %d (%s)", len(exp), exp, len(got), got) + } + ss := stringSet{} + for _, s := range exp { + ss.set(s) + } + for _, s := range got { + assert(tb, ss.isSet(s), "unexpected value: "+s) + } +} + +func assert(tb testing.TB, b bool, msg any) { + tb.Helper() + if !b { + tb.Fatal("assertion failed", msg) + } +} + +func must(tb testing.TB, err error) { + tb.Helper() + if err != nil { + tb.Fatal(err) + } +} diff --git a/test/pkg/require/requires.go b/test/pkg/require/requires.go index b042d9f9ce..877ddb70e6 100644 --- a/test/pkg/require/requires.go +++ b/test/pkg/require/requires.go @@ -9,9 +9,12 @@ import ( ) // Features checks the wanted features are present in given, -// and skips the test if any are missing. If the given set is empty, -// the function returns (by default all features are enabled). -func Features(tb testing.TB, given *flag.StringSet, want ...string) { +// and skips the test if any are missing or explicitly excluded. +// If the given set is empty, the function returns +// (by default, all features are enabled). +// +// See [flag.NewFeatureFlag] and [flag.IncludeExcludeStringSet] for more details. +func Features(tb testing.TB, given *flag.IncludeExcludeStringSet, want ...string) { tb.Helper() if given.Len() == 0 { return @@ -23,6 +26,22 @@ func Features(tb testing.TB, given *flag.StringSet, want ...string) { } } +// AnyFeature checks if at least one of the features are enabled. +// +// See [Features] for more information. +func AnyFeature(tb testing.TB, given *flag.IncludeExcludeStringSet, want ...string) { + tb.Helper() + if given.Len() == 0 { + return + } + for _, f := range want { + if given.IsSet(f) { + return + } + } + tb.Skipf("skipping test due to missing features: %s", want) +} + // Binary checks if `binary` exists in the same directory as the test // binary. // Returns full binary path if it exists, otherwise, skips the test.