diff --git a/requirements/dev.txt b/requirements/dev.txt index 98b81bbd28..4816a99c16 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,6 +1,7 @@ pre-commit pytest pytest-cov +pytest-order pytest-sugar pytest-xdist coverage[toml] diff --git a/tests/pre_merge/tools/__init__.py b/tests/pre_merge/tools/__init__.py new file mode 100644 index 0000000000..a35a271287 --- /dev/null +++ b/tests/pre_merge/tools/__init__.py @@ -0,0 +1,8 @@ +"""Test entry point scripts. + + +Note: This might be removed when migration to CLI is complete. +""" + +# Copyright (C) 2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 diff --git a/tests/pre_merge/tools/conftest.py b/tests/pre_merge/tools/conftest.py new file mode 100644 index 0000000000..14bf1184e0 --- /dev/null +++ b/tests/pre_merge/tools/conftest.py @@ -0,0 +1,56 @@ +"""Fixtures for the tools tests.""" + +# Copyright (C) 2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +from tempfile import TemporaryDirectory +from typing import Generator + +import albumentations as A +import cv2 +import numpy as np +import pytest +from albumentations.pytorch import ToTensorV2 + +from anomalib.config import get_configurable_parameters + + +@pytest.fixture(scope="package") +def project_path() -> Generator[str, None, None]: + with TemporaryDirectory() as project_dir: + yield project_dir + + +@pytest.fixture(scope="package") +def get_config(project_path): + def get_config( + model_name: str | None = None, + config_path: str | None = None, + weight_file: str | None = None, + ): + """Gets config for testing.""" + config = get_configurable_parameters(model_name, config_path, weight_file) + config.dataset.image_size = (100, 100) + config.model.input_size = (100, 100) + config.project.path = project_path + config.trainer.max_epochs = 1 + config.trainer.check_val_every_n_epoch = 1 + config.trainer.limit_train_batches = 1 + config.trainer.limit_predict_batches = 1 + return config + + yield get_config + + +@pytest.fixture(scope="package") +def get_dummy_inference_image(project_path) -> Generator[str, None, None]: + image = np.zeros((100, 100, 3), dtype=np.uint8) + cv2.imwrite(project_path + "/dummy_image.png", image) + yield project_path + "/dummy_image.png" + + +@pytest.fixture(scope="package") +def transforms_config() -> dict: + """Note: this is computed using trainer.datamodule.test_data.transform.to_dict()""" + return A.Compose([A.ToFloat(max_value=255), ToTensorV2()]).to_dict() diff --git a/tests/pre_merge/tools/test_gradio_entrypoint.py b/tests/pre_merge/tools/test_gradio_entrypoint.py new file mode 100644 index 0000000000..d733bbb00f --- /dev/null +++ b/tests/pre_merge/tools/test_gradio_entrypoint.py @@ -0,0 +1,93 @@ +"""Test Gradio inference entrypoint script.""" + +# Copyright (C) 2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + + +from __future__ import annotations + +import sys +from importlib.util import find_spec + +import pytest + +from anomalib.data import TaskType +from anomalib.deploy import ExportMode, OpenVINOInferencer, TorchInferencer, export +from anomalib.models import get_model + +sys.path.append("tools/inference") + + +@pytest.mark.order(5) +class TestGradioInferenceEntrypoint: + """This tests whether the entrypoints run without errors without quantitative measure of the outputs. + + Note: This does not launch the gradio server. It only checks if the right inferencer is called. + """ + + @pytest.fixture + def get_functions(self): + """Get functions from Gradio_inference.py""" + if find_spec("gradio_inference") is not None: + from tools.inference.gradio_inference import get_inferencer, get_parser + else: + raise Exception("Unable to import gradio_inference.py for testing") + return get_parser, get_inferencer + + def test_torch_inference(self, get_functions, project_path, get_config, transforms_config): + """Test gradio_inference.py""" + parser, inferencer = get_functions + model = get_model(get_config("padim")) + + # export torch model + export( + task=TaskType.SEGMENTATION, + transform=transforms_config, + input_size=(100, 100), + model=model, + export_mode=ExportMode.TORCH, + export_root=project_path, + ) + + arguments = parser().parse_args( + [ + "--weights", + project_path + "/weights/torch/model.pt", + ] + ) + assert isinstance(inferencer(arguments.weights, arguments.metadata), TorchInferencer) + + def test_openvino_inference(self, get_functions, project_path, get_config, transforms_config): + """Test gradio_inference.py""" + parser, inferencer = get_functions + model = get_model(get_config("padim")) + + # export OpenVINO model + export( + task=TaskType.SEGMENTATION, + transform=transforms_config, + input_size=(100, 100), + model=model, + export_mode=ExportMode.OPENVINO, + export_root=project_path, + ) + + arguments = parser().parse_args( + [ + "--weights", + project_path + "/weights/openvino/model.bin", + "--metadata", + project_path + "/weights/openvino/metadata.json", + ] + ) + assert isinstance(inferencer(arguments.weights, arguments.metadata), OpenVINOInferencer) + + # test error is raised when metadata is not provided to openvino model + with pytest.raises(ValueError): + arguments = parser().parse_args( + [ + "--weights", + project_path + "/weights/openvino/model.bin", + ] + ) + inferencer(arguments.weights, arguments.metadata) diff --git a/tests/pre_merge/tools/test_lightning_entrypoint.py b/tests/pre_merge/tools/test_lightning_entrypoint.py new file mode 100644 index 0000000000..cf7e8d1129 --- /dev/null +++ b/tests/pre_merge/tools/test_lightning_entrypoint.py @@ -0,0 +1,47 @@ +"""Test lightning inference entrypoint script.""" + +# Copyright (C) 2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + + +from __future__ import annotations + +import sys +from importlib.util import find_spec + +import pytest + +sys.path.append("tools/inference") +from unittest.mock import patch + + +@pytest.mark.order(3) +class TestLightningInferenceEntrypoint: + """This tests whether the entrypoints run without errors without quantitative measure of the outputs.""" + + @pytest.fixture + def get_functions(self): + """Get functions from lightning_inference.py""" + if find_spec("lightning_inference") is not None: + from tools.inference.lightning_inference import get_parser, infer + else: + raise Exception("Unable to import lightning_inference.py for testing") + return get_parser, infer + + def test_lightning_inference(self, get_functions, get_config, project_path, get_dummy_inference_image): + """Test lightning_inferenc.py""" + get_parser, infer = get_functions + with patch("tools.inference.lightning_inference.get_configurable_parameters", side_effect=get_config): + arguments = get_parser().parse_args( + [ + "--config", + "src/anomalib/models/padim/config.yaml", + "--weights", + project_path + "/weights/lightning/model.ckpt", + "--input", + get_dummy_inference_image, + "--output", + project_path + "/output", + ] + ) + infer(arguments) diff --git a/tests/pre_merge/tools/test_openvino_entrypoint.py b/tests/pre_merge/tools/test_openvino_entrypoint.py new file mode 100644 index 0000000000..ebf3f7d39e --- /dev/null +++ b/tests/pre_merge/tools/test_openvino_entrypoint.py @@ -0,0 +1,66 @@ +"""Test OpenVINO inference entrypoint script.""" + +# Copyright (C) 2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + + +from __future__ import annotations + +import sys +from importlib.util import find_spec + +import pytest + +from anomalib.data import TaskType +from anomalib.deploy import ExportMode, export +from anomalib.models import get_model + +sys.path.append("tools/inference") + + +@pytest.mark.order(4) +class TestOpenVINOInferenceEntrypoint: + """This tests whether the entrypoints run without errors without quantitative measure of the outputs.""" + + @pytest.fixture + def get_functions(self): + """Get functions from openvino_inference.py""" + if find_spec("openvino_inference") is not None: + from tools.inference.openvino_inference import get_parser, infer + else: + raise Exception("Unable to import openvino_inference.py for testing") + return get_parser, infer + + def test_openvino_inference( + self, get_functions, get_config, project_path, get_dummy_inference_image, transforms_config + ): + """Test openvino_inference.py""" + get_parser, infer = get_functions + + model = get_model(get_config("padim")) + + # export OpenVINO model + export( + task=TaskType.SEGMENTATION, + transform=transforms_config, + input_size=(100, 100), + model=model, + export_mode=ExportMode.OPENVINO, + export_root=project_path, + ) + + arguments = get_parser().parse_args( + [ + "--config", + "src/anomalib/models/padim/config.yaml", + "--weights", + project_path + "/weights/openvino/model.bin", + "--metadata", + project_path + "/weights/openvino/metadata.json", + "--input", + get_dummy_inference_image, + "--output", + project_path + "/output", + ] + ) + infer(arguments) diff --git a/tests/pre_merge/tools/test_test_entrypoint.py b/tests/pre_merge/tools/test_test_entrypoint.py new file mode 100644 index 0000000000..a79db2eb29 --- /dev/null +++ b/tests/pre_merge/tools/test_test_entrypoint.py @@ -0,0 +1,54 @@ +"""Test test.py entrypoint script.""" + +# Copyright (C) 2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import sys +from importlib.util import find_spec + +import pytest + +sys.path.append("tools") +from unittest.mock import patch + + +@pytest.mark.order(2) +class TestTestEntrypoint: + """This tests whether the entrypoints run without errors without quantitative measure of the outputs.""" + + @pytest.fixture + def get_functions(self): + """Get functions from test.py""" + if find_spec("test") is not None: + from tools.test import get_parser, test + else: + raise Exception("Unable to import test.py for testing") + return get_parser, test + + def test_test(self, get_functions, get_config, project_path): + """Test test.py""" + get_parser, test = get_functions + with patch("tools.test.get_configurable_parameters", side_effect=get_config): + # Test when model key is passed + arguments = get_parser().parse_args( + ["--model", "padim", "--weight_file", project_path + "/weights/lightning/model.ckpt"] + ) + test(arguments) + + # Test when weight file is incorrect + arguments = get_parser().parse_args(["--model", "padim"]) + with pytest.raises(FileNotFoundError): + test(arguments) + + # Test when config key is passed + arguments = get_parser().parse_args( + [ + "--config", + "src/anomalib/models/padim/config.yaml", + "--weight_file", + project_path + "/weights/lightning/model.ckpt", + ] + ) + test(arguments) diff --git a/tests/pre_merge/tools/test_torch_entrypoint.py b/tests/pre_merge/tools/test_torch_entrypoint.py new file mode 100644 index 0000000000..9917f5909e --- /dev/null +++ b/tests/pre_merge/tools/test_torch_entrypoint.py @@ -0,0 +1,58 @@ +"""Test torch inference entrypoint script.""" + +# Copyright (C) 2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + + +from __future__ import annotations + +import sys +from importlib.util import find_spec + +import pytest + +from anomalib.data import TaskType +from anomalib.deploy import ExportMode, export +from anomalib.models import get_model + +sys.path.append("tools/inference") + + +@pytest.mark.order(4) +class TestTorchInferenceEntrypoint: + """This tests whether the entrypoints run without errors without quantitative measure of the outputs.""" + + @pytest.fixture + def get_functions(self): + """Get functions from torch_inference.py""" + if find_spec("torch_inference") is not None: + from tools.inference.torch_inference import get_parser, infer + else: + raise Exception("Unable to import torch_inference.py for testing") + return get_parser, infer + + def test_torch_inference( + self, get_functions, get_config, project_path, get_dummy_inference_image, transforms_config + ): + """Test torch_inference.py""" + get_parser, infer = get_functions + model = get_model(get_config("padim")) + export( + task=TaskType.SEGMENTATION, + transform=transforms_config, + input_size=(100, 100), + model=model, + export_mode=ExportMode.TORCH, + export_root=project_path, + ) + arguments = get_parser().parse_args( + [ + "--weights", + project_path + "/weights/torch/model.pt", + "--input", + get_dummy_inference_image, + "--output", + project_path + "/output", + ] + ) + infer(arguments) diff --git a/tests/pre_merge/tools/test_train_entrypoint.py b/tests/pre_merge/tools/test_train_entrypoint.py new file mode 100644 index 0000000000..50aa951b8c --- /dev/null +++ b/tests/pre_merge/tools/test_train_entrypoint.py @@ -0,0 +1,44 @@ +"""Test train entrypoint""" + + +# Copyright (C) 2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +import sys +from importlib.util import find_spec +from unittest.mock import patch + +import pytest + +sys.path.append("tools") # This is the path to the tools folder as it is not part of the anomalib package + + +@pytest.mark.order(1) +class TestTrainEntrypoint: + """This tests whether the entrypoints run without errors without quantitative measure of the outputs.""" + + @pytest.fixture + def get_functions(self): + """Get functions from train.py""" + if find_spec("train") is not None: + from tools.train import get_parser, train + else: + raise Exception("Unable to import train.py for testing") + return get_parser, train + + def test_train(self, get_functions, get_config): + """Test train.py.""" + with patch("tools.train.get_configurable_parameters", side_effect=get_config): + get_parser, train = get_functions + # Test when model key is passed + args = get_parser().parse_args(["--model", "padim"]) + train(args) + + # Don't run fit and test the second time + with patch("tools.train.Trainer"): + # Test when config key is passed + args = get_parser().parse_args(["--config", "src/anomalib/models/padim/config.yaml"]) + + train(args) + # Test when log_level key is passed + args = get_parser().parse_args(["--log-level", "ERROR"]) + train(args) diff --git a/tools/inference/gradio_inference.py b/tools/inference/gradio_inference.py index 048b0bbc5f..7d4e5f5ca7 100644 --- a/tools/inference/gradio_inference.py +++ b/tools/inference/gradio_inference.py @@ -8,7 +8,7 @@ from __future__ import annotations -from argparse import ArgumentParser, Namespace +from argparse import ArgumentParser from importlib import import_module from pathlib import Path @@ -20,8 +20,8 @@ from anomalib.deploy import Inferencer -def get_args() -> Namespace: - r"""Get command line arguments. +def get_parser() -> ArgumentParser: + """Get command line arguments. Example: @@ -30,14 +30,14 @@ def get_args() -> Namespace: ... --weights ./results/padim/mvtec/bottle/weights/torch/model.pt Returns: - Namespace: List of arguments. + ArgumentParser: Argument parser for gradio inference. """ parser = ArgumentParser() parser.add_argument("--weights", type=Path, required=True, help="Path to model weights") parser.add_argument("--metadata", type=Path, required=False, help="Path to a JSON file containing the metadata.") parser.add_argument("--share", type=bool, required=False, default=False, help="Share Gradio `share_url`") - return parser.parse_args() + return parser def get_inferencer(weight_path: Path, metadata_path: Path | None = None) -> Inferencer: @@ -96,7 +96,7 @@ def infer(image: np.ndarray, inferencer: Inferencer) -> tuple[np.ndarray, np.nda if __name__ == "__main__": - args = get_args() + args = get_parser().parse_args() gradio_inferencer = get_inferencer(args.weights, args.metadata) interface = gr.Interface( diff --git a/tools/inference/lightning_inference.py b/tools/inference/lightning_inference.py index c00c0428d4..0474200154 100644 --- a/tools/inference/lightning_inference.py +++ b/tools/inference/lightning_inference.py @@ -16,11 +16,11 @@ from anomalib.utils.callbacks import get_callbacks -def get_args() -> Namespace: - """Get command line arguments. +def get_parser() -> ArgumentParser: + """Get parser. Returns: - Namespace: List of arguments. + ArgumentParser: The parser object. """ parser = ArgumentParser() parser.add_argument("--config", type=Path, required=True, help="Path to a config file") @@ -42,13 +42,11 @@ def get_args() -> Namespace: help="Show the visualized predictions on the screen.", ) - args = parser.parse_args() - return args + return parser -def infer(): +def infer(args: Namespace): """Run inference.""" - args = get_args() config = get_configurable_parameters(config_path=args.config) config.trainer.resume_from_checkpoint = str(args.weights) config.visualization.show_images = args.show @@ -76,7 +74,9 @@ def infer(): ) # create the dataset - dataset = InferenceDataset(args.input, image_size=tuple(config.dataset.image_size), transform=transform) + dataset = InferenceDataset( + args.input, image_size=tuple(config.dataset.image_size), transform=transform # type: ignore + ) dataloader = DataLoader(dataset) # generate predictions @@ -84,4 +84,5 @@ def infer(): if __name__ == "__main__": - infer() + args = get_parser().parse_args() + infer(args) diff --git a/tools/inference/openvino_inference.py b/tools/inference/openvino_inference.py index ac0ce0c6f7..df1848fc3c 100644 --- a/tools/inference/openvino_inference.py +++ b/tools/inference/openvino_inference.py @@ -11,20 +11,16 @@ from argparse import ArgumentParser, Namespace from pathlib import Path -from anomalib.data.utils import ( - generate_output_image_filename, - get_image_filenames, - read_image, -) +from anomalib.data.utils import generate_output_image_filename, get_image_filenames, read_image from anomalib.deploy import OpenVINOInferencer from anomalib.post_processing import Visualizer -def get_args() -> Namespace: - """Get command line arguments. +def get_parser() -> ArgumentParser: + """Get parser. Returns: - Namespace: List of arguments. + ArgumentParser: The parser object. """ parser = ArgumentParser() parser.add_argument("--config", type=Path, required=True, help="Path to a config file") @@ -63,21 +59,17 @@ def get_args() -> Namespace: help="Show the visualized predictions on the screen.", ) - args = parser.parse_args() + return parser - return args - -def infer() -> None: +def infer(args: Namespace) -> None: """Infer predictions. Show/save the output if path is to an image. If the path is a directory, go over each image in the directory. - """ - # Get the command line arguments, and config from the config.yaml file. - # This config file is also used for training and contains all the relevant - # information regarding the data, model, train and inference details. - args = get_args() + Args: + args (Namespace): The arguments from the command line. + """ # Get the inferencer. inferencer = OpenVINOInferencer(path=args.weights, metadata_path=args.metadata, device=args.device) visualizer = Visualizer(mode=args.visualization_mode, task=args.task) @@ -103,4 +95,5 @@ def infer() -> None: if __name__ == "__main__": - infer() + args = get_parser().parse_args() + infer(args) diff --git a/tools/inference/torch_inference.py b/tools/inference/torch_inference.py index 3848408728..bc4f661b79 100644 --- a/tools/inference/torch_inference.py +++ b/tools/inference/torch_inference.py @@ -13,20 +13,16 @@ import torch -from anomalib.data.utils import ( - generate_output_image_filename, - get_image_filenames, - read_image, -) +from anomalib.data.utils import generate_output_image_filename, get_image_filenames, read_image from anomalib.deploy import TorchInferencer from anomalib.post_processing import Visualizer -def get_args() -> Namespace: - """Get command line arguments. +def get_parser() -> ArgumentParser: + """Get parser. Returns: - Namespace: List of arguments. + ArgumentParser: The parser object. """ parser = ArgumentParser() parser.add_argument("--weights", type=Path, required=True, help="Path to model weights") @@ -63,18 +59,17 @@ def get_args() -> Namespace: help="Show the visualized predictions on the screen.", ) - args = parser.parse_args() + return parser - return args - -def infer() -> None: +def infer(args: Namespace) -> None: """Infer predictions. Show/save the output if path is to an image. If the path is a directory, go over each image in the directory. + + Args: + args (Namespace): The arguments from the command line. """ - # Get the command line arguments. - args = get_args() torch.set_grad_enabled(False) @@ -103,4 +98,5 @@ def infer() -> None: if __name__ == "__main__": - infer() + args = get_parser().parse_args() + infer(args=args) diff --git a/tools/test.py b/tools/test.py index b2772aaf5d..6278d00017 100644 --- a/tools/test.py +++ b/tools/test.py @@ -13,27 +13,26 @@ from anomalib.utils.callbacks import get_callbacks -def get_args() -> Namespace: - """Get CLI arguments. +def get_parser() -> ArgumentParser: + """Get parser. Returns: - Namespace: CLI arguments. + ArgumentParser: The parser object. """ parser = ArgumentParser() parser.add_argument("--model", type=str, default="stfpm", help="Name of the algorithm to train/test") parser.add_argument("--config", type=str, required=False, help="Path to a model config file") parser.add_argument("--weight_file", type=str, default="weights/model.ckpt") - args = parser.parse_args() - return args + return parser -def test(): - """Test an anomaly classification and segmentation model that is initially trained via `tools/train.py`. +def test(args: Namespace): + """Test an anomaly model. - The script is able to write the results into both filesystem and a logger such as Tensorboard. + Args: + args (Namespace): The arguments from the command line. """ - args = get_args() config = get_configurable_parameters( model_name=args.model, config_path=args.config, @@ -53,4 +52,5 @@ def test(): if __name__ == "__main__": - test() + args = get_parser().parse_args() + test(args) diff --git a/tools/train.py b/tools/train.py index f5ffb8ef9b..3b0a1c88a0 100644 --- a/tools/train.py +++ b/tools/train.py @@ -24,24 +24,27 @@ logger = logging.getLogger("anomalib") -def get_args() -> Namespace: - """Get command line arguments. +def get_parser() -> ArgumentParser: + """Get parser. Returns: - Namespace: List of arguments. + ArgumentParser: The parser object. """ parser = ArgumentParser() parser.add_argument("--model", type=str, default="padim", help="Name of the algorithm to train/test") parser.add_argument("--config", type=str, required=False, help="Path to a model config file") parser.add_argument("--log-level", type=str, default="INFO", help="") - args = parser.parse_args() - return args + return parser -def train(): - """Train an anomaly classification or segmentation model based on a provided configuration file.""" - args = get_args() +def train(args: Namespace): + """Train an anomaly model. + + Args: + args (Namespace): The arguments from the command line. + """ + configure_logger(level=args.log_level) if args.log_level == "ERROR": @@ -72,4 +75,5 @@ def train(): if __name__ == "__main__": - train() + args = get_parser().parse_args() + train(args) diff --git a/tox.ini b/tox.ini index 34a0e381cc..0fdfe9e9c9 100644 --- a/tox.ini +++ b/tox.ini @@ -24,6 +24,7 @@ deps = coverage[toml] pytest pytest-cov + pytest-order flaky nbmake -r{toxinidir}/requirements/base.txt