diff --git a/tools/benchmarking/benchmark.py b/tools/benchmarking/benchmark.py index e5f87a413f..244be2cab9 100644 --- a/tools/benchmarking/benchmark.py +++ b/tools/benchmarking/benchmark.py @@ -25,9 +25,10 @@ import warnings from argparse import ArgumentParser from concurrent.futures import ProcessPoolExecutor, as_completed +from datetime import datetime from pathlib import Path from tempfile import TemporaryDirectory -from typing import Dict, List, Union, cast +from typing import Dict, List, Optional, Union, cast import torch from omegaconf import DictConfig, ListConfig, OmegaConf @@ -147,11 +148,11 @@ def get_single_model_metrics(model_config: Union[DictConfig, ListConfig], openvi return data -def compute_on_cpu(sweep_config: Union[DictConfig, ListConfig]): +def compute_on_cpu(sweep_config: Union[DictConfig, ListConfig], folder: Optional[str] = None): """Compute all run configurations over a sigle CPU.""" for run_config in get_run_config(sweep_config.grid_search): model_metrics = sweep(run_config, 0, sweep_config.seed, False) - write_metrics(model_metrics, sweep_config.writer) + write_metrics(model_metrics, sweep_config.writer, folder) def compute_on_gpu( @@ -159,6 +160,7 @@ def compute_on_gpu( device: int, seed: int, writers: List[str], + folder: Optional[str] = None, compute_openvino: bool = False, ): """Go over each run config and collect the result. @@ -168,19 +170,20 @@ def compute_on_gpu( device (int): The GPU id used for running the sweep. seed (int): Fix a seed. writers (List[str]): Destinations to write to. + folder (optional, str): Sub-directory to which runs are written to. Defaults to None. If none writes to root. compute_openvino (bool, optional): Compute OpenVINO throughput. Defaults to False. """ for run_config in run_configs: if isinstance(run_config, (DictConfig, ListConfig)): model_metrics = sweep(run_config, device, seed, compute_openvino) - write_metrics(model_metrics, writers) + write_metrics(model_metrics, writers, folder) else: raise ValueError( f"Expecting `run_config` of type DictConfig or ListConfig. Got {type(run_config)} instead." ) -def distribute_over_gpus(sweep_config: Union[DictConfig, ListConfig]): +def distribute_over_gpus(sweep_config: Union[DictConfig, ListConfig], folder: Optional[str] = None): """Distribute metric collection over all available GPUs. This is done by splitting the list of configurations.""" with ProcessPoolExecutor( max_workers=torch.cuda.device_count(), mp_context=multiprocessing.get_context("spawn") @@ -197,6 +200,7 @@ def distribute_over_gpus(sweep_config: Union[DictConfig, ListConfig]): device_id + 1, sweep_config.seed, sweep_config.writer, + folder, sweep_config.compute_openvino, ) ) @@ -214,24 +218,28 @@ def distribute(config: Union[DictConfig, ListConfig]): config: (Union[DictConfig, ListConfig]): Sweep configuration. """ + runs_folder = datetime.strftime(datetime.now(), "%Y_%m_%d-%H_%M_%S") devices = config.hardware if not torch.cuda.is_available() and "gpu" in devices: pl_logger.warning("Config requested GPU benchmarking but torch could not detect any cuda enabled devices") elif {"cpu", "gpu"}.issubset(devices): # Create process for gpu and cpu with ProcessPoolExecutor(max_workers=2, mp_context=multiprocessing.get_context("spawn")) as executor: - jobs = [executor.submit(compute_on_cpu, config), executor.submit(distribute_over_gpus, config)] + jobs = [ + executor.submit(compute_on_cpu, config, runs_folder), + executor.submit(distribute_over_gpus, config, runs_folder), + ] for job in as_completed(jobs): try: job.result() except Exception as exception: raise Exception(f"Error occurred while computing benchmark on device {job}") from exception elif "cpu" in devices: - compute_on_cpu(config) + compute_on_cpu(config, folder=runs_folder) elif "gpu" in devices: - distribute_over_gpus(config) + distribute_over_gpus(config, folder=runs_folder) if "wandb" in config.writer: - upload_to_wandb(team="anomalib") + upload_to_wandb(team="anomalib", folder=runs_folder) def sweep( diff --git a/tools/benchmarking/utils/metrics.py b/tools/benchmarking/utils/metrics.py index 51236ac75d..2cece6d149 100644 --- a/tools/benchmarking/utils/metrics.py +++ b/tools/benchmarking/utils/metrics.py @@ -17,7 +17,7 @@ import string from glob import glob from pathlib import Path -from typing import Dict, List, Union +from typing import Dict, List, Optional, Union import pandas as pd from torch.utils.tensorboard.writer import SummaryWriter @@ -25,12 +25,17 @@ import wandb -def write_metrics(model_metrics: Dict[str, Union[str, float]], writers: List[str]): +def write_metrics( + model_metrics: Dict[str, Union[str, float]], + writers: List[str], + folder: Optional[str] = None, +): """Writes metrics to destination provided in the sweep config. Args: model_metrics (Dict): Dictionary to be written writers (List[str]): List of destinations. + folder (optional, str): Sub-directory to which runs are written to. Defaults to None. If none writes to root. """ # Write to file as each run is computed if model_metrics == {} or model_metrics is None: @@ -38,7 +43,8 @@ def write_metrics(model_metrics: Dict[str, Union[str, float]], writers: List[str # Write to CSV metrics_df = pd.DataFrame(model_metrics, index=[0]) - result_path = Path(f"runs/{model_metrics['model_name']}_{model_metrics['device']}.csv") + result_folder = Path("runs") if folder is None else Path(f"runs/{folder}") + result_path = result_folder / f"{model_metrics['model_name']}_{model_metrics['device']}.csv" Path.mkdir(result_path.parent, parents=True, exist_ok=True) if not result_path.is_file(): metrics_df.to_csv(result_path) @@ -93,7 +99,10 @@ def get_unique_key(str_len: int) -> str: return "".join([random.choice(string.ascii_lowercase) for _ in range(str_len)]) -def upload_to_wandb(team: str = "anomalib"): +def upload_to_wandb( + team: str = "anomalib", + folder: Optional[str] = None, +): """Upload the data in csv files to wandb. Creates a project named benchmarking_[two random characters]. This is so that the project names are unique. @@ -102,10 +111,12 @@ def upload_to_wandb(team: str = "anomalib"): Args: team (str, optional): Name of the team on wandb. This can also be the id of your personal account. Defaults to "anomalib". + folder (optional, str): Sub-directory from which runs are picked up. Defaults to None. If none picks from runs. """ project = f"benchmarking_{get_unique_key(2)}" tag_list = ["dataset.category", "model_name", "dataset.image_size", "model.backbone", "device"] - for csv_file in glob("runs/*.csv"): + search_path = "runs/*.csv" if folder is None else f"runs/{folder}/*.csv" + for csv_file in glob(search_path): table = pd.read_csv(csv_file) for index, row in table.iterrows(): row = dict(row[1:]) # remove index column