Merge pull request #682 from gammasim/separate_ctapipe_imports

Improve startup time of applications

docs/source/io_operations.rst

@@ -0,0 +1,22 @@
+.. _IO:
+
+I/O
+---
+
+This module include I/O related functionality.
+
+.. _iomodule:
+
+io_handler
+**********
+
+.. automodule:: io_operations.io_handler
+   :members:
+
+.. _iohdf5:
+
+hdf5_handler
+************
+
+.. automodule:: io_operations.hdf5_handler
+   :members:

docs/source/library.rst

@@ -15,7 +15,7 @@ simtools is composed of the following modules and module groups:
   corsika_simtel
   data_model
   db_handler
-  io_handler
+  io_operations
   job_execution
   layout_array
   mc_model

simtools/applications/compare_cumulative_psf.py

@@ -86,8 +86,8 @@
 import yaml
 
 import simtools.utils.general as gen
-from simtools import io_handler
 from simtools.configuration import configurator
+from simtools.io_operations import io_handler
 from simtools.model.telescope_model import TelescopeModel
 from simtools.ray_tracing import RayTracing
 from simtools.visualization import visualize

simtools/applications/generate_corsika_histograms.py

@@ -170,10 +170,10 @@
 import numpy as np
 
 import simtools.utils.general as gen
-from simtools import io_handler
 from simtools.configuration import configurator
 from simtools.corsika import corsika_histograms_visualize
 from simtools.corsika.corsika_histograms import CorsikaHistograms
+from simtools.io_operations import io_handler
 
 logger = logging.getLogger()
 

simtools/applications/generate_simtel_array_histograms.py

@@ -55,8 +55,8 @@
 from matplotlib.backends.backend_pdf import PdfPages
 
 import simtools.utils.general as gen
-from simtools import io_handler
 from simtools.configuration import configurator
+from simtools.io_operations import io_handler
 from simtools.simtel.simtel_histograms import SimtelHistograms
 
 

simtools/applications/make_regular_arrays.py

@@ -40,8 +40,9 @@
 import astropy.units as u
 
 import simtools.utils.general as gen
-from simtools import db_handler, io_handler
+from simtools import db_handler
 from simtools.configuration import configurator
+from simtools.io_operations import io_handler
 from simtools.layout.layout_array import LayoutArray
 
 

simtools/applications/plot_layout_array.py

@@ -44,8 +44,8 @@
 from astropy import units as u
 
 import simtools.utils.general as gen
-from simtools import io_handler
 from simtools.configuration import configurator
+from simtools.io_operations import io_handler
 from simtools.layout.layout_array import LayoutArray
 from simtools.visualization.visualize import plot_array
 

simtools/applications/sim_showers_for_trigger_rates.py

@@ -68,8 +68,8 @@
 
 import astropy.units as u
 
-from simtools import io_handler
 from simtools.configuration import configurator
+from simtools.io_operations import io_handler
 from simtools.simulator import Simulator
 from simtools.utils import general as gen
 

simtools/applications/tune_psf.py

@@ -94,8 +94,8 @@
 from matplotlib.backends.backend_pdf import PdfPages
 
 import simtools.utils.general as gen
-from simtools import io_handler
 from simtools.configuration import configurator
+from simtools.io_operations import io_handler
 from simtools.model.model_utils import split_simtel_parameter
 from simtools.model.telescope_model import TelescopeModel
 from simtools.ray_tracing import RayTracing

simtools/applications/validate_camera_efficiency.py

@@ -57,9 +57,9 @@
 from pathlib import Path
 
 import simtools.utils.general as gen
-from simtools import io_handler
 from simtools.camera_efficiency import CameraEfficiency
 from simtools.configuration import configurator
+from simtools.io_operations import io_handler
 from simtools.model.telescope_model import TelescopeModel
 
 

simtools/applications/validate_camera_fov.py

@@ -59,8 +59,8 @@
 from pathlib import Path
 
 import simtools.utils.general as gen
-from simtools import io_handler
 from simtools.configuration import configurator
+from simtools.io_operations import io_handler
 from simtools.model.telescope_model import TelescopeModel
 
 

simtools/applications/validate_optics.py

@@ -77,8 +77,8 @@
 from matplotlib.backends.backend_pdf import PdfPages
 
 import simtools.utils.general as gen
-from simtools import io_handler
 from simtools.configuration import configurator
+from simtools.io_operations import io_handler
 from simtools.model.telescope_model import TelescopeModel
 from simtools.ray_tracing import RayTracing
 

simtools/camera_efficiency.py

@@ -8,7 +8,7 @@
 from astropy.table import Table
 
 import simtools.utils.general as gen
-from simtools import io_handler
+from simtools.io_operations import io_handler
 from simtools.model.telescope_model import TelescopeModel
 from simtools.simtel.simtel_runner_camera_efficiency import SimtelRunnerCameraEfficiency
 from simtools.utils import names

simtools/configuration/configurator.py

@@ -8,7 +8,7 @@
 from dotenv import load_dotenv
 
 import simtools.configuration.commandline_parser as argparser
-from simtools import io_handler
+from simtools.io_operations import io_handler
 
 __all__ = [
     "Configurator",

simtools/corsika/corsika_config.py

@@ -7,7 +7,7 @@
 from astropy.io.misc import yaml
 
 import simtools.utils.general as gen
-from simtools import io_handler
+from simtools.io_operations import io_handler
 from simtools.layout.layout_array import LayoutArray
 from simtools.utils import names
 from simtools.utils.general import collect_data_from_yaml_or_dict

simtools/corsika/corsika_histograms.py

@@ -13,11 +13,12 @@
 from ctapipe.io import write_table
 from eventio import IACTFile
 
-from simtools import io_handler, version
+from simtools import version
+from simtools.io_operations import io_handler
+from simtools.io_operations.hdf5_handler import fill_hdf5_table
 from simtools.utils.general import (
     collect_data_from_yaml_or_dict,
     convert_2D_to_radial_distr,
-    fill_hdf5_table,
     rotate,
     save_dict_to_file,
 )

simtools/corsika/corsika_runner.py

@@ -3,11 +3,11 @@
 from copy import copy
 from pathlib import Path
 
-from simtools import io_handler
 from simtools.corsika.corsika_config import (
     CorsikaConfig,
     MissingRequiredInputInCorsikaConfigData,
 )
+from simtools.io_operations import io_handler
 from simtools.utils import names
 from simtools.utils.general import collect_data_from_yaml_or_dict
 

simtools/data_model/metadata_collector.py

@@ -4,8 +4,8 @@
 
 import simtools.utils.general as gen
 import simtools.version
-from simtools import io_handler
 from simtools.data_model import metadata_model, validate_schema
+from simtools.io_operations import io_handler
 from simtools.utils import names
 
 __all__ = ["MetadataCollector"]

simtools/data_model/model_data_writer.py

@@ -5,7 +5,7 @@
 import yaml
 
 import simtools.utils.general as gen
-from simtools import io_handler
+from simtools.io_operations import io_handler
 
 __all__ = ["ModelDataWriter"]
 

simtools/db_handler.py

@@ -12,7 +12,7 @@
 from pymongo.errors import BulkWriteError
 
 import simtools.utils.general as gen
-from simtools import io_handler
+from simtools.io_operations import io_handler
 from simtools.model.model_utils import get_telescope_class
 from simtools.utils import names
 

simtools/io_operations/hdf5_handler.py

@@ -0,0 +1,112 @@
+import logging
+from pathlib import PosixPath
+
+import astropy.units as u
+import tables
+from astropy.table import Table
+from ctapipe.io import read_table
+
+from simtools.utils.names import sanitize_name
+
+__all__ = [
+    "fill_hdf5_table",
+    "read_hdf5",
+]
+
+_logger = logging.getLogger(__name__)
+
+
+def fill_hdf5_table(hist, x_bin_edges, y_bin_edges, x_label, y_label, meta_data):
+    """
+    Create and fill an hdf5 table with the histogram information.
+    It works for both 1D and 2D distributions.
+
+    Parameters
+    ----------
+    hist: numpy.ndarray
+        The counts of the histograms.
+    x_bin_edges: numpy.array
+        The x bin edges of the histograms.
+    y_bin_edges: numpy.array
+        The y bin edges of the histograms.
+        Use None for 1D histograms.
+    x_label: str
+        X bin edges label.
+    y_label: str
+        Y bin edges label.
+        Use None for 1D histograms.
+    meta_data: dict
+        Dictionary with the histogram metadata.
+    """
+
+    # Complement metadata
+    if x_label is not None:
+        meta_data["x bin edges"] = sanitize_name(x_label)
+    meta_data["x bin edges unit"] = (
+        x_bin_edges.unit if isinstance(x_bin_edges, u.Quantity) else u.dimensionless_unscaled
+    )
+
+    if y_bin_edges is not None:
+        if y_label is not None:
+            meta_data["y bin edges"] = sanitize_name(y_label)
+            names = [
+                f"{meta_data['y bin edges'].split('__')[0]}_{i}"
+                for i in range(len(y_bin_edges[:-1]))
+            ]
+        else:
+            names = [
+                f"{meta_data['Title'].split('__')[0]}_{i}" for i in range(len(y_bin_edges[:-1]))
+            ]
+        meta_data["y bin edges unit"] = (
+            y_bin_edges.unit if isinstance(y_bin_edges, u.Quantity) else u.dimensionless_unscaled
+        )
+
+        table = Table(
+            [hist[i, :] for i in range(len(y_bin_edges[:-1]))],
+            names=names,
+            meta=meta_data,
+        )
+
+    else:
+        if x_label is not None:
+            meta_data["x bin edges"] = sanitize_name(x_label)
+            names = meta_data["x bin edges"]
+        else:
+            names = meta_data["Title"]
+        table = Table(
+            [
+                x_bin_edges[:-1],
+                hist,
+            ],
+            names=(names, sanitize_name("Values")),
+            meta=meta_data,
+        )
+    return table
+
+
+def read_hdf5(hdf5_file_name):
+    """
+    Read a hdf5 output file.
+
+    Parameters
+    ----------
+    hdf5_file_name: str or Path
+        Name or Path of the hdf5 file to read from.
+
+    Returns
+    -------
+    list
+        The list with the astropy.Table instances for the various 1D and 2D histograms saved
+        in the hdf5 file.
+    """
+    if isinstance(hdf5_file_name, PosixPath):
+        hdf5_file_name = hdf5_file_name.absolute().as_posix()
+
+    tables_list = []
+
+    with tables.open_file(hdf5_file_name, mode="r") as file:
+        for node in file.walk_nodes("/", "Table"):
+            table_path = node._v_pathname
+            table = read_table(hdf5_file_name, table_path)
+            tables_list.append(table)
+    return tables_list

simtools/layout/layout_array.py

@@ -5,7 +5,8 @@
 import numpy as np
 from astropy.table import QTable
 
-from simtools import db_handler, io_handler
+from simtools import db_handler
+from simtools.io_operations import io_handler
 from simtools.layout.geo_coordinates import GeoCoordinates
 from simtools.layout.telescope_position import TelescopePosition
 from simtools.utils import names

simtools/model/array_model.py

@@ -1,7 +1,8 @@
 import logging
 from copy import copy
 
-from simtools import db_handler, io_handler
+from simtools import db_handler
+from simtools.io_operations import io_handler
 from simtools.layout.layout_array import LayoutArray
 from simtools.model.telescope_model import TelescopeModel
 from simtools.simtel.simtel_config_writer import SimtelConfigWriter

simtools/model/telescope_model.py

@@ -9,7 +9,8 @@
 from astropy.table import Table
 
 import simtools.utils.general as gen
-from simtools import db_handler, io_handler
+from simtools import db_handler
+from simtools.io_operations import io_handler
 from simtools.model.camera import Camera
 from simtools.model.mirrors import Mirrors
 from simtools.simtel.simtel_config_writer import SimtelConfigWriter

simtools/ray_tracing.py

@@ -14,7 +14,7 @@
 from astropy.table import QTable
 
 import simtools.utils.general as gen
-from simtools import io_handler
+from simtools.io_operations import io_handler
 from simtools.model.model_utils import compute_telescope_transmission
 from simtools.model.telescope_model import TelescopeModel
 from simtools.psf_analysis import PSFImage

simtools/simtel/simtel_histograms.py

@@ -9,7 +9,7 @@
 from matplotlib.backends.backend_pdf import PdfPages
 
 from simtools import version
-from simtools.utils.general import fill_hdf5_table
+from simtools.io_operations.hdf5_handler import fill_hdf5_table
 from simtools.utils.names import sanitize_name
 
 __all__ = ["BadHistogramFormat", "SimtelHistograms"]