diff --git a/README.md b/README.md index 16ee29e39b..56f9683f33 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ [![python](https://img.shields.io/badge/python-3.7%2B-green)]() [![pytorch](https://img.shields.io/badge/pytorch-1.8.1%2B-orange)]() [![openvino](https://img.shields.io/badge/openvino-2021.4.2-purple)]() +[![comet](https://custom-icon-badges.herokuapp.com/badge/comet__ml-3.31.7-orange?logo=logo_comet_ml)](https://www.comet.com/site/products/ml-experiment-tracking/?utm_source=anomalib&utm_medium=referral) [![Codacy Badge](https://app.codacy.com/project/badge/Grade/684927c1c76c4c5e94bb53480812fbbb)](https://www.codacy.com/gh/openvinotoolkit/anomalib/dashboard?utm_source=github.com&utm_medium=referral&utm_content=openvinotoolkit/anomalib&utm_campaign=Badge_Grade) [![black](https://img.shields.io/badge/code%20style-black-000000.svg)]() [![Nightly-Regression Test](https://github.com/openvinotoolkit/anomalib/actions/workflows/nightly.yml/badge.svg)](https://github.com/openvinotoolkit/anomalib/actions/workflows/nightly.yml) @@ -244,19 +245,25 @@ python tools/benchmarking/benchmark.py \ Refer to the [Benchmarking Documentation](https://openvinotoolkit.github.io/anomalib/guides/benchmarking.html) for more details. -# Logging Images +# Experiment Management -You can save images locally or to a logger such TensorBoard or Weights and Biases by setting the following configuration. +Anomablib is integrated with various libraries for experiment tracking such as Comet, tensorboard, and wandb through [pytorch lighting loggers](https://pytorch-lightning.readthedocs.io/en/stable/extensions/logging.html). + +Below is an example of how to enable logging for hyper-parameters, metrics, model graphs, and predictions on images in the test data-set ```yaml -logging: - logger: [tensorboard, wandb] - log_graph: false +visualization: + log_images: True # log images to the available loggers (if any) + mode: full # options: ["full", "simple"] + + logging: + logger: [comet, tensorboard, wandb] + log_graph: True ``` -For more information on logging images, refer to the [Logging Documentation](https://openvinotoolkit.github.io/anomalib/guides/logging.html) +For more information, refer to the [Logging Documentation](https://openvinotoolkit.github.io/anomalib/guides/logging.html) ---- +Note: Set your API Key for [Comet.ml](https://www.comet.com/signup?utm_source=anomalib&utm_medium=referral) via `comet_ml.init()` in interactive python or simply run `export COMET_API_KEY=` # Datasets diff --git a/anomalib/models/cflow/config.yaml b/anomalib/models/cflow/config.yaml index 257ed11915..be166c2417 100644 --- a/anomalib/models/cflow/config.yaml +++ b/anomalib/models/cflow/config.yaml @@ -59,7 +59,7 @@ project: path: ./results logging: - logger: [] # options: [tensorboard, wandb, csv] or combinations. + logger: [] # options: [comet, tensorboard, wandb, csv] or combinations. log_graph: false # Logs the model graph to respective logger. optimization: diff --git a/anomalib/models/dfkde/config.yaml b/anomalib/models/dfkde/config.yaml index 68e5d80bd7..070f0c2456 100644 --- a/anomalib/models/dfkde/config.yaml +++ b/anomalib/models/dfkde/config.yaml @@ -47,7 +47,7 @@ project: path: ./results logging: - logger: [] # options: [tensorboard, wandb, csv] or combinations. + logger: [] # options: [comet, tensorboard, wandb, csv] or combinations. log_graph: false # Logs the model graph to respective logger. optimization: diff --git a/anomalib/models/dfm/config.yaml b/anomalib/models/dfm/config.yaml index 0d3dbfc2a0..47db50fb4e 100755 --- a/anomalib/models/dfm/config.yaml +++ b/anomalib/models/dfm/config.yaml @@ -47,7 +47,7 @@ project: path: ./results logging: - logger: [] # options: [tensorboard, wandb, csv] or combinations. + logger: [] # options: [comet, tensorboard, wandb, csv] or combinations. log_graph: false # Logs the model graph to respective logger. optimization: diff --git a/anomalib/models/draem/config.yaml b/anomalib/models/draem/config.yaml index 4dffb12bc9..9f3326daa1 100644 --- a/anomalib/models/draem/config.yaml +++ b/anomalib/models/draem/config.yaml @@ -56,7 +56,7 @@ project: path: ./results logging: - logger: [] # options: [tensorboard, wandb, csv] or combinations. + logger: [] # options: [comet, tensorboard, wandb, csv] or combinations. log_graph: false # Logs the model graph to respective logger. optimization: diff --git a/anomalib/models/fastflow/config.yaml b/anomalib/models/fastflow/config.yaml index 6d64da88b6..b02f430fd9 100644 --- a/anomalib/models/fastflow/config.yaml +++ b/anomalib/models/fastflow/config.yaml @@ -59,7 +59,7 @@ project: path: ./results logging: - logger: [] # options: [tensorboard, wandb, csv] or combinations. + logger: [] # options: [comet, tensorboard, wandb, csv] or combinations. log_graph: false # Logs the model graph to respective logger. optimization: diff --git a/anomalib/models/ganomaly/config.yaml b/anomalib/models/ganomaly/config.yaml index 1fd3d8019c..2e5dfb6bba 100644 --- a/anomalib/models/ganomaly/config.yaml +++ b/anomalib/models/ganomaly/config.yaml @@ -59,7 +59,7 @@ project: path: ./results logging: - logger: [] # options: [tensorboard, wandb, csv] or combinations. + logger: [] # options: [comet, tensorboard, wandb, csv] or combinations. log_graph: false # Logs the model graph to respective logger. optimization: diff --git a/anomalib/models/padim/config.yaml b/anomalib/models/padim/config.yaml index 83e2762237..92e66618dc 100644 --- a/anomalib/models/padim/config.yaml +++ b/anomalib/models/padim/config.yaml @@ -54,7 +54,7 @@ project: path: ./results logging: - logger: [] # options: [tensorboard, wandb, csv] or combinations. + logger: [] # options: [comet, tensorboard, wandb, csv] or combinations. log_graph: false # Logs the model graph to respective logger. optimization: diff --git a/anomalib/models/patchcore/config.yaml b/anomalib/models/patchcore/config.yaml index f7d37e24f7..31567ad530 100644 --- a/anomalib/models/patchcore/config.yaml +++ b/anomalib/models/patchcore/config.yaml @@ -55,7 +55,7 @@ project: path: ./results logging: - logger: [] # options: [tensorboard, wandb, csv] or combinations. + logger: [] # options: [comet, tensorboard, wandb, csv] or combinations. log_graph: false # Logs the model graph to respective logger. optimization: diff --git a/anomalib/models/reverse_distillation/config.yaml b/anomalib/models/reverse_distillation/config.yaml index f7b741c3da..d30f3baedf 100644 --- a/anomalib/models/reverse_distillation/config.yaml +++ b/anomalib/models/reverse_distillation/config.yaml @@ -63,7 +63,7 @@ project: path: ./results logging: - logger: [] # options: [tensorboard, wandb, csv] or combinations. + logger: [] # options: [comet, tensorboard, wandb, csv] or combinations. log_graph: false # Logs the model graph to respective logger. optimization: diff --git a/anomalib/models/stfpm/config.yaml b/anomalib/models/stfpm/config.yaml index 48ed3acb75..fe3637bf27 100644 --- a/anomalib/models/stfpm/config.yaml +++ b/anomalib/models/stfpm/config.yaml @@ -61,7 +61,7 @@ project: path: ./results logging: - logger: [] # options: [tensorboard, wandb, csv] or combinations. + logger: [] # options: [comet, tensorboard, wandb, csv] or combinations. log_graph: false # Logs the model graph to respective logger. optimization: diff --git a/anomalib/utils/callbacks/graph.py b/anomalib/utils/callbacks/graph.py index 321862a309..c38475756b 100644 --- a/anomalib/utils/callbacks/graph.py +++ b/anomalib/utils/callbacks/graph.py @@ -7,7 +7,7 @@ from pytorch_lightning import Callback, LightningModule, Trainer from pytorch_lightning.utilities.cli import CALLBACK_REGISTRY -from anomalib.utils.loggers import AnomalibTensorBoardLogger, AnomalibWandbLogger +from anomalib.utils.loggers import AnomalibTensorBoardLogger, AnomalibWandbLogger, AnomalibCometLogger @CALLBACK_REGISTRY @@ -38,7 +38,7 @@ def on_train_end(self, trainer: Trainer, pl_module: LightningModule) -> None: """ for logger in trainer.loggers: - if isinstance(logger, AnomalibTensorBoardLogger): + if isinstance(logger, (AnomalibCometLogger, AnomalibTensorBoardLogger)): logger.log_graph(pl_module, input_array=torch.ones((1, 3, 256, 256))) elif isinstance(logger, AnomalibWandbLogger): logger.unwatch(pl_module) # type: ignore diff --git a/anomalib/utils/loggers/__init__.py b/anomalib/utils/loggers/__init__.py index e98a8c578f..fd563ef156 100644 --- a/anomalib/utils/loggers/__init__.py +++ b/anomalib/utils/loggers/__init__.py @@ -14,14 +14,18 @@ from .tensorboard import AnomalibTensorBoardLogger from .wandb import AnomalibWandbLogger +from .comet import AnomalibCometLogger __all__ = [ + "AnomalibCometLogger", "AnomalibTensorBoardLogger", "AnomalibWandbLogger", "configure_logger", "get_experiment_logger", ] -AVAILABLE_LOGGERS = ["tensorboard", "wandb", "csv"] + + +AVAILABLE_LOGGERS = ["tensorboard", "wandb", "csv", "comet"] logger = logging.getLogger(__name__) @@ -112,6 +116,21 @@ def get_experiment_logger( save_dir=wandb_logdir, ) ) + elif experiment_logger=="comet": + comet_logdir = os.path.join(config.project.path, "logs") + os.makedirs(comet_logdir, exist_ok=True) + run_name = ( + config.model.name + if "category" not in config.dataset.keys() + else f"{config.dataset.category} {config.model.name}" + ) + logger_list.append( + AnomalibCometLogger( + project_name=config.dataset.name, + experiment_name=run_name, + save_dir=comet_logdir + ) + ) elif experiment_logger == "csv": logger_list.append(CSVLogger(save_dir=os.path.join(config.project.path, "logs"))) else: diff --git a/anomalib/utils/loggers/comet.py b/anomalib/utils/loggers/comet.py new file mode 100644 index 0000000000..9d16e6b2f3 --- /dev/null +++ b/anomalib/utils/loggers/comet.py @@ -0,0 +1,114 @@ +"""comet logger with add image interface.""" + +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +from typing import Any, Optional, Union + +import numpy as np +from matplotlib.figure import Figure +from pytorch_lightning.loggers.comet import CometLogger +from pytorch_lightning.utilities import rank_zero_only + + +from .base import ImageLoggerBase + + +class AnomalibCometLogger(ImageLoggerBase, CometLogger): + """Logger for comet. + + Adds interface for `add_image` in the logger rather than calling the experiment object. + Note: + Same as the CometLogger provided by PyTorch Lightning and the doc string is reproduced below. + + Track your parameters, metrics, source code and more using + `Comet `_. + + Install it with pip: + + .. code-block:: bash + + pip install comet-ml + + Comet requires either an API Key (online mode) or a local directory path (offline mode). + + Args: + api_key: Required in online mode. API key, found on Comet.ml. If not given, this + will be loaded from the environment variable COMET_API_KEY or ~/.comet.config + if either exists. + save_dir: Required in offline mode. The path for the directory to save local + comet logs. If given, this also sets the directory for saving checkpoints. + project_name: Optional. Send your experiment to a specific project. + Otherwise will be sent to Uncategorized Experiments. + If the project name does not already exist, Comet.ml will create a new project. + rest_api_key: Optional. Rest API key found in Comet.ml settings. + This is used to determine version number + experiment_name: Optional. String representing the name for this particular experiment on Comet.ml. + experiment_key: Optional. If set, restores from existing experiment. + offline: If api_key and save_dir are both given, this determines whether + the experiment will be in online or offline mode. This is useful if you use + save_dir to control the checkpoints directory and have a ~/.comet.config + file but still want to run offline experiments. + prefix: A string to put at the beginning of metric keys. + kwargs: Additional arguments like `workspace`, `log_code`, etc. used by + :class:`CometExperiment` can be passed as keyword arguments in this logger. + + Raises: + ModuleNotFoundError: + If required Comet package is not installed on the device. + MisconfigurationException: + If neither ``api_key`` nor ``save_dir`` are passed as arguments. + Example: + >>> from anomalib.utils.loggers import AnomalibCometLogger + >>> from pytorch_lightning import Trainer + >>> comet_logger = AnomalibCometLogger() + >>> trainer = Trainer(logger=comet_logger) + + See Also: + - `Comet Documentation `__ + """ + + def __init__( + self, + api_key: Optional[str] = None, + save_dir: Optional[str] = None, + project_name: Optional[str] = None, + rest_api_key: Optional[str] = None, + experiment_name: Optional[str] = None, + experiment_key: Optional[str] = None, + offline: bool = False, + prefix: str = "", + **kwargs + ) -> None: + super().__init__( + api_key=api_key, + save_dir=save_dir, + project_name=project_name, + rest_api_key=rest_api_key, + experiment_name=experiment_name, + experiment_key=experiment_key, + offline=offline, + prefix=prefix, + **kwargs + ) + self.experiment.log_other("Created from", "Anomalib") + + @rank_zero_only + def add_image(self, image: Union[np.ndarray, Figure], name: Optional[str] = None, **kwargs: Any): + """Interface to add image to comet logger. + + Args: + image (Union[np.ndarray, Figure]): Image to log + name (Optional[str]): The tag of the image + kwargs: Accepts only `global_step` (int). The step at which to log the image. + """ + if "global_step" not in kwargs: + raise ValueError("`global_step` is required for comet logger") + + global_step = kwargs["global_step"] + # Need to call different functions of `Experiment` for Figure vs np.ndarray + + if isinstance(image, Figure): + self.experiment.log_figure(figure_name=name, figure=image, step=global_step) + else: + self.experiment.log_image(name=name, image_data=image, step=global_step) diff --git a/docs/blog/001-train-custom-dataset/README.md b/docs/blog/001-train-custom-dataset/README.md index 9559f205de..1a1f7a695a 100644 --- a/docs/blog/001-train-custom-dataset/README.md +++ b/docs/blog/001-train-custom-dataset/README.md @@ -151,11 +151,11 @@ Here is an example of the generated results for a toy dataset containing Hazelnu ## Logging and Experiment Management -While it is delightful to know how good your model performed on your preferred metric, it is even more exciting to see the predicted outputs. Anomalib provides a couple of ways to log and track experiments. These can be used individually or in a combination. As of the current release, you can save images to a local folder, or upload to weights and biases, or TensorBoard. +While it is delightful to know how good your model performed on your preferred metric, it is even more exciting to see the predicted outputs. Anomalib provides a couple of ways to log and track experiments. These can be used individually or in a combination. As of the current release, you can save images to a local folder, or upload to comet, weights and biases, or TensorBoard. -To select where you would like to save the images, change the `log_images_to` parameter in the `project` section in the config file. +To select where you would like to save the images, change the `log_images` parameter in the `Visualization` section in the config file to true. -For example, setting the following `log_images_to: ["local"]` will result in saving the images in the results folder as shown in the tree structure below: +For example, setting the following `log_images: True` will result in saving the images in the results folder as shown in the tree structure below: ```bash results @@ -176,14 +176,21 @@ results ### Logging to Tensorboard and/or W&B -To use TensorBoard and/or W&B logger, ensure that the logger parameter is set to `tensorboard`, `wandb` or `[tensorboard, wandb]` in the `logging` section of the config file. +To use TensorBoard and/or W&B logger and/or Comet logger, ensure that the logger parameter is set to `comet`, `tensorboard`, `wandb` or `[tensorboard, wandb]` in the `logging` section of the config file. -An example configuration for saving to TensorBoard is shown in the figure below. Similarly after setting logger to `wandb` you will see the images on your wandb project dashboard. +An example configuration for saving to TensorBoard is shown in the figure below. Similarly after setting logger to `wandb` or 'comet' you will see the images on your wandb and/or comet project dashboard. ```yaml -logging: - log_images_to: [tensorboard] - logger: tensorboard # options: [tensorboard, wandb, csv] or combinations. + visualization: + show_images: False # show images on the screen + save_images: False # save images to the file system + log_images: True # log images to the available loggers (if any) + image_save_path: null # path to which images will be saved + mode: full # options: ["full", "simple"] + + logging: + logger: [comet, tensorboard, wandb] #Choose any combination of these 3 + log_graph: false ```
diff --git a/docs/source/guides/logging.rst b/docs/source/guides/logging.rst index 3c38f78ea4..38eba09179 100644 --- a/docs/source/guides/logging.rst +++ b/docs/source/guides/logging.rst @@ -9,7 +9,7 @@ Anomalib offers various mechanisms for logging metrics and predicted masks. Enabling Logging ***************** -These can be enabled using the ``logger`` parameter in ``logging`` section of each model configuration file. The available options are ``tensorboard``, ``wandb`` and ``csv``. +These can be enabled using the ``logger`` parameter in ``logging`` section of each model configuration file. The available options are ``comet``, ``tensorboard``, ``wandb`` and ``csv``. For example, to log to TensorBoard: .. code-block:: yaml @@ -23,26 +23,38 @@ You can also pass a list of loggers to enable multiple loggers. For example: .. code-block:: yaml logging: - logger: [tensorboard, wandb] + logger: [comet, tensorboard, wandb] log_graph: false Logging Images ************** -Anomalib allows you to save predictions to the file system by setting ``log_images_to: local``. As of the current version, Anomalib also supports TensorBoard and Weights and Biases loggers for logging images. These loggers extend upon the base loggers by providing a common interface for logging images. You can access the required logger from ``trainer.loggers``. Then you can use ``logger.add_image`` method to log images. For a complete overview of this method refer to our `API documentation `_. +Anomalib allows you to save predictions to the file system by setting ``log_images: True`` in the visualization section . As of the current version, Anomalib also supports Comet, TensorBoard and Weights and Biases loggers for logging images. These loggers extend upon the base loggers by providing a common interface for logging images. You can access the required logger from ``trainer.loggers``. Then you can use ``logger.add_image`` method to log images. For a complete overview of this method refer to our `API documentation `_. .. code-block:: yaml + + visualization: + show_images: False # show images on the screen + save_images: False # save images to the file system + log_images: True # log images to the available loggers (if any) + image_save_path: null # path to which images will be saved + mode: full # options: ["full", "simple"] + logging: - log_images_to: ["local", "tensorboard", "wandb"] - logger: [tensorboard, wandb] + logger: [comet, tensorboard, wandb] log_graph: false .. note:: - Logging images to TensorBoard and wandb won't work if you don't have ``logger: [tensorboard, wandb]`` set as well. This ensures that the respective logger is passed to the trainer object. + Logging images to Comet,TensorBoard and wandb won't work if you don't have ``logger: [comet, tensorboard, wandb]`` set as well. This ensures that the respective logger is passed to the trainer object. + +.. figure:: ../images/logging/comet_media.jpg + :alt: comet dashboard showing logged images + + Comet Images in TensorBoard Dashboard .. figure:: ../images/logging/tensorboard_media.jpg :alt: tensorboard dashboard showing logged images @@ -64,14 +76,19 @@ To log other artifacts to the logger, you can directly access the logger object When accessing the base ``logger/logger.experiment`` object, refer to the documentation of the respective logger for the list of available methods. -Anomalib makes it easier to log your model graph to TensorBoard or Weights and Biases. Just set ``log_graph`` to True under ``logging`` parameter of the model configuration file. +Anomalib makes it easier to log your model graph to Comet, TensorBoard or Weights and Biases. Just set ``log_graph`` to True under ``logging`` parameter of the model configuration file. .. code-block:: yaml logging: - logger: [tensorboard] + logger: [comet, tensorboard] log_graph: true +.. figure:: ../images/logging/comet_graph.jpg + :alt: comet dashboard showing model graph + + Model Graph in Comet Dashboard + .. figure:: ../images/logging/tensorboard_graph.jpg :alt: tensorboard dashboard showing model graph diff --git a/docs/source/images/logging/comet_graph.png b/docs/source/images/logging/comet_graph.png new file mode 100644 index 0000000000..d55e18f956 Binary files /dev/null and b/docs/source/images/logging/comet_graph.png differ diff --git a/docs/source/images/logging/comet_media.png b/docs/source/images/logging/comet_media.png new file mode 100644 index 0000000000..869fb27e64 Binary files /dev/null and b/docs/source/images/logging/comet_media.png differ diff --git a/requirements/base.txt b/requirements/base.txt index e7de2fbbbc..a2d3e494d9 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,4 +1,5 @@ albumentations>=1.1.0 +comet-ml>=3.31.7 einops>=0.3.2 gradio>=2.9.4 imgaug==0.4.0 diff --git a/tests/pre_merge/utils/loggers/test_get_logger.py b/tests/pre_merge/utils/loggers/test_get_logger.py index f3c239aec6..3468832a73 100644 --- a/tests/pre_merge/utils/loggers/test_get_logger.py +++ b/tests/pre_merge/utils/loggers/test_get_logger.py @@ -15,6 +15,7 @@ from anomalib.utils.loggers import ( AnomalibTensorBoardLogger, AnomalibWandbLogger, + AnomalibCometLogger, UnknownLogger, get_experiment_logger, ) @@ -50,17 +51,23 @@ def test_get_experiment_logger(): logger = get_experiment_logger(config=config) assert isinstance(logger[0], AnomalibWandbLogger) + # get comet logger + config.project.logger = "comet" + logger = get_experiment_logger(config=config) + assert isinstance(logger[0], AnomalibCometLogger) + # get csv logger. config.project.logger = "csv" logger = get_experiment_logger(config=config) assert isinstance(logger[0], CSVLogger) # get multiple loggers - config.project.logger = ["tensorboard", "wandb", "csv"] + config.project.logger = ["tensorboard", "wandb", "csv", "comet"] logger = get_experiment_logger(config=config) assert isinstance(logger[0], AnomalibTensorBoardLogger) assert isinstance(logger[1], AnomalibWandbLogger) assert isinstance(logger[2], CSVLogger) + assert isinstance(logger[3], AnomalibCometLogger) # raise unknown with pytest.raises(UnknownLogger):