Add the new datamodule tests

.github/workflows/code_scan.yml

@@ -7,6 +7,26 @@ on:
     - cron: "0 18 * * 1-5"
 
 jobs:
+  Bandit:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: CHECKOUT REPOSITORY
+        uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+      - name: Install dependencies
+        run: python -m pip install tox
+      - name: Bandit Scanning
+        run: tox -e bandit-scan
+      - name: UPLOAD BANDIT REPORT
+        uses: actions/upload-artifact@v3
+        with:
+          name: bandit-report
+          path: .tox/bandit-report.txt
+        # Use always() to always run this step to publish scan results when there are test failures
+        if: ${{ always() }}
   Snyk-scan:
     runs-on: [self-hosted, linux, x64]
     permissions:
@@ -26,9 +46,10 @@ jobs:
         uses: github/codeql-action/upload-sarif@v2
         with:
           sarif_file: .tox/snyk.sarif
-      - name: UPLOAD REPORT
+      - name: UPLOAD SNYK REPORT
         uses: actions/upload-artifact@v3
         with:
+          name: snyk-report
           path: .tox/snyk.html
         # Use always() to always run this step to publish scan results when there are test failures
         if: ${{ always() }}

requirements/dev.txt

@@ -1,3 +1,5 @@
+flaky
+nbmake
 pre-commit
 pytest
 pytest-cov

src/anomalib/data/avenue.py

@@ -172,7 +172,7 @@ class Avenue(AnomalibVideoDataModule):
         gt_dir (Path | str): Path to the ground truth files
         clip_length_in_frames (int, optional): Number of video frames in each clip.
         frames_between_clips (int, optional): Number of frames between each consecutive video clip.
-        task TaskType): Task type, 'classification', 'detection' or 'segmentation'
+        task (TaskType): Task type, 'classification', 'detection' or 'segmentation'
         image_size (int | tuple[int, int] | None, optional): Size of the input image.
             Defaults to None.
         center_crop (int | tuple[int, int] | None, optional): When provided, the images will be center-cropped

tests/unit/__init__.py

@@ -0,0 +1,4 @@
+"""Tests - Unit."""
+
+# Copyright (C) 2022 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0

tests/unit/data/__init__.py

@@ -0,0 +1,4 @@
+"""Unit Tests - Data."""
+
+# Copyright (C) 2023 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0

tests/unit/data/base/__init__.py

@@ -0,0 +1,17 @@
+"""Unit Tests - Base Datamodules."""
+
+# Copyright (C) 2023 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+from .test_datamodule import _TestAnomalibDataModule
+from .test_depth import _TestAnomalibDepthDatamodule
+from .test_image import _TestAnomalibImageDatamodule
+from .test_video import _TestAnomalibVideoDatamodule
+
+
+__all__ = [
+    "_TestAnomalibDataModule",
+    "_TestAnomalibDepthDatamodule",
+    "_TestAnomalibImageDatamodule",
+    "_TestAnomalibVideoDatamodule",
+]

tests/unit/data/base/test_datamodule.py

@@ -0,0 +1,24 @@
+"""Datamodule Unit Tests."""
+
+# Copyright (C) 2023 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+
+from anomalib.data import AnomalibDataModule
+import pytest
+from torch.utils.data import DataLoader
+
+
+class _TestAnomalibDataModule:
+    """Base test class for ``AnomalibDataModule``.
+
+    This is a base class for testing the AnomalibDataModule. Since
+    ``AnomalibDataModule`` has methods that are yet to be implemented, this base
+    test class is not meant to be used directly.
+    """
+
+    @pytest.mark.parametrize("subset", ["train", "val", "test"])
+    def test_datamodule_has_dataloader_attributes(self, datamodule: AnomalibDataModule, subset: str) -> None:
+        """Test that the datamodule has the correct dataloader attributes."""
+        dataloader = f"{subset}_dataloader"
+        assert hasattr(datamodule, dataloader) and isinstance(getattr(datamodule, dataloader)(), DataLoader)

tests/unit/data/base/test_depth.py

@@ -0,0 +1,45 @@
+"""Unit Tests - Base Depth Datamodules."""
+
+# Copyright (C) 2023 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+from anomalib.data import AnomalibDataModule
+import pytest
+from .test_datamodule import _TestAnomalibDataModule
+
+
+class _TestAnomalibDepthDatamodule(_TestAnomalibDataModule):
+    @pytest.mark.parametrize("subset", ["train", "val", "test"])
+    def test_get_item_returns_correct_keys_and_shapes(self, datamodule: AnomalibDataModule, subset: str) -> None:
+        """Test that the datamodule __getitem__ returns the correct keys and shapes."""
+
+        # Get the dataloader.
+        dataloader = getattr(datamodule, f"{subset}_dataloader")()
+
+        # Get the first batch.
+        batch = next(iter(dataloader))
+
+        # Check that the batch has the correct keys.
+        expected_keys = {"image_path", "depth_path", "label", "image", "depth_image"}
+
+        if dataloader.dataset.task in ("detection", "segmentation"):
+            expected_keys |= {"mask_path", "mask"}
+
+            if dataloader.dataset.task == "detection":
+                expected_keys |= {"boxes"}
+
+        assert batch.keys() == expected_keys
+
+        # Check that the batch has the correct shape.
+        assert len(batch["image_path"]) == 4
+        assert len(batch["depth_path"]) == 4
+        assert batch["image"].shape == (4, 3, 256, 256)
+        assert batch["depth_image"].shape == (4, 3, 256, 256)
+        assert batch["label"].shape == (4,)
+
+        # TODO: Detection task should return bounding boxes.
+        # if dataloader.dataset.task == "detection":
+        #     assert batch["boxes"].shape == (4, 4)
+
+        if dataloader.dataset.task in ("detection", "segmentation"):
+            assert batch["mask"].shape == (4, 256, 256)

tests/unit/data/base/test_image.py

@@ -0,0 +1,31 @@
+"""Unit Tests - Base Image Datamodules."""
+
+# Copyright (C) 2023 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+import pytest
+from anomalib.data import AnomalibDataModule
+from .test_datamodule import _TestAnomalibDataModule
+
+
+class _TestAnomalibImageDatamodule(_TestAnomalibDataModule):
+    @pytest.mark.parametrize("subset", ["train", "val", "test"])
+    def test_get_item_returns_correct_keys_and_shapes(self, datamodule: AnomalibDataModule, subset: str) -> None:
+        """Test that the datamodule __getitem__ returns image, mask, label and boxes."""
+
+        # Get the dataloader.
+        dataloader = getattr(datamodule, f"{subset}_dataloader")()
+
+        # Get the first batch.
+        batch = next(iter(dataloader))
+
+        # Check that the batch has the correct shape.
+        assert batch["image"].shape == (4, 3, 256, 256)
+        assert batch["label"].shape == (4,)
+
+        # TODO: Detection task should return bounding boxes.
+        # if dataloader.dataset.task == "detection":
+        #     assert batch["boxes"].shape == (4, 4)
+
+        if dataloader.dataset.task in ("detection", "segmentation"):
+            assert batch["mask"].shape == (4, 256, 256)

tests/unit/data/base/test_video.py

@@ -0,0 +1,47 @@
+"""Unit Tests - Base Video Datamodules."""
+
+# Copyright (C) 2023 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+import pytest
+from anomalib.data import AnomalibDataModule
+from .test_datamodule import _TestAnomalibDataModule
+
+
+class _TestAnomalibVideoDatamodule(_TestAnomalibDataModule):
+    @pytest.mark.parametrize("subset", ["train", "val", "test"])
+    def test_get_item_returns_correct_keys_and_shapes(self, datamodule: AnomalibDataModule, subset: str) -> None:
+        """Test that the datamodule __getitem__ returns image, mask, label and boxes."""
+
+        # Get the dataloader.
+        dataloader = getattr(datamodule, f"{subset}_dataloader")()
+
+        # Get the first batch.
+        batch = next(iter(dataloader))
+
+        # Check that the batch has the correct keys.
+        expected_train_keys = {"image", "video_path", "frames", "last_frame", "original_image"}
+        expected_eval_keys = expected_train_keys | {"label", "mask"}
+        expected_eval_detection_keys = expected_eval_keys | {"boxes"}
+
+        if subset == "train":
+            expected_keys = expected_train_keys
+        else:
+            if dataloader.dataset.task == "detection":
+                expected_keys = expected_eval_detection_keys
+            else:
+                expected_keys = expected_eval_keys
+
+        assert batch.keys() == expected_keys
+
+        # Check that the batch has the correct shape.
+        assert batch["image"].shape == (4, 3, 256, 256)
+        assert len(batch["video_path"]) == 4
+        assert len(batch["frames"]) == 4
+        assert len(batch["last_frame"]) == 4
+        # We don't know the shape of the original image, so we only check that it is a list of 4 images.
+        assert batch["original_image"].shape[0] == 4
+
+        if subset in ("val", "test"):
+            assert len(batch["label"]) == 4
+            assert batch["mask"].shape == (4, 256, 256)

tests/unit/data/conftest.py

@@ -0,0 +1,13 @@
+"""Unit Tests - Datamodules Configurations."""
+
+# Copyright (C) 2023 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+from anomalib.data import TaskType
+import pytest
+
+
+@pytest.fixture(params=[TaskType.CLASSIFICATION, TaskType.DETECTION, TaskType.SEGMENTATION])
+def task_type(request) -> str:
+    """Create and return a task type."""
+    return request.param

tests/unit/data/test_avenue.py

@@ -0,0 +1,37 @@
+"""Unit Tests - Avenue Datamodule."""
+
+# Copyright (C) 2023 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+from anomalib.data import Avenue, TaskType
+import pytest
+from .base import _TestAnomalibVideoDatamodule
+
+from tests.helpers.dataset import get_dataset_path
+
+
+class TestAvenue(_TestAnomalibVideoDatamodule):
+    """Avenue Datamodule Unit Tests."""
+
+    @pytest.fixture
+    def datamodule(self, task_type: TaskType) -> Avenue:
+        # Create and prepare the dataset
+        root = get_dataset_path("avenue")
+        gt_dir = os.path.join(root, "ground_truth_demo")
+
+        # TODO: Fix the Avenue dataset path via get_dataset_path
+        _datamodule = Avenue(
+            root=root,
+            gt_dir=gt_dir,
+            image_size=256,
+            task=task_type,
+            num_workers=0,
+            train_batch_size=4,
+            eval_batch_size=4,
+        )
+
+        _datamodule.prepare_data()
+        _datamodule.setup()
+
+        return _datamodule

tests/unit/data/test_btech.py

@@ -0,0 +1,31 @@
+"""Unit Tests - BTech Datamodule."""
+
+# Copyright (C) 2023 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+from anomalib.data import BTech, TaskType
+import pytest
+from .base import _TestAnomalibImageDatamodule
+
+from tests.helpers.dataset import get_dataset_path
+
+
+class TestBTech(_TestAnomalibImageDatamodule):
+    """MVTec Datamodule Unit Tests."""
+
+    @pytest.fixture
+    def datamodule(self, task_type: TaskType) -> BTech:
+        # Create and prepare the dataset
+        _datamodule = BTech(
+            root=get_dataset_path("BTech"),
+            category="01",
+            task=task_type,
+            image_size=256,
+            train_batch_size=4,
+            eval_batch_size=4,
+        )
+
+        _datamodule.prepare_data()
+        _datamodule.setup()
+
+        return _datamodule

tests/unit/data/test_folder.py

@@ -0,0 +1,49 @@
+"""Unit Tests - Folder Datamodule."""
+
+# Copyright (C) 2023 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+from anomalib.data import Folder, TaskType
+from .base import _TestAnomalibImageDatamodule
+import pytest
+
+from tests.helpers.dataset import get_dataset_path
+
+
+class TestFolder(_TestAnomalibImageDatamodule):
+    """Folder Datamodule Unit Tests.
+
+    All of the folder datamodule tests are placed in ``TestFolder`` class.
+    """
+
+    @pytest.fixture(params=[None, "good_test"])
+    def normal_test_dir(self, request) -> str:
+        """Create and return a normal test directory."""
+        return request.param
+
+    @pytest.fixture
+    def datamodule(self, task_type: TaskType, normal_test_dir: str) -> Folder:
+        """Create and return a folder datamodule.
+
+        This datamodule uses the MVTec bottle dataset for testing.
+        """
+        # Make sure to use a mask directory for segmentation. Folder datamodule
+        # expects a relative directory to the root.
+        mask_dir = None if task_type == TaskType.CLASSIFICATION else "ground_truth/broken_large"
+
+        # Create and prepare the dataset
+        _datamodule = Folder(
+            root=get_dataset_path("bottle"),
+            normal_dir="good",
+            abnormal_dir="broken_large",
+            normal_test_dir=normal_test_dir,
+            mask_dir=mask_dir,
+            image_size=(256, 256),
+            train_batch_size=4,
+            eval_batch_size=4,
+            num_workers=0,
+            task=task_type,
+        )
+        _datamodule.setup()
+
+        return _datamodule