MOBT-809: Backport weather symbol release changes into master

improver/categorical/modal_code.py

@@ -23,7 +23,7 @@
 from improver.utilities.cube_manipulation import MergeCubes
 
 from ..metadata.forecast_times import forecast_period_coord
-from .utilities import day_night_map
+from .utilities import day_night_map, dry_map
 
 
 class BaseModalCategory(BasePlugin):
@@ -358,6 +358,12 @@ class ModalFromGroupings(BaseModalCategory):
     Where there are different categories available for night and day, the
     modal code returned is always a day code, regardless of the times
     covered by the input files.
+
+    If a location is to return a dry code after consideration of the various
+    weightings, the wet codes for that location are converted into the best
+    matching dry cloud code and these are included in determining the resulting
+    dry code. The wet bias has no impact on the weight of these converted wet
+    codes, but the day weighting still applies.
     """
 
     # Day length set to aid testing.
@@ -420,6 +426,8 @@ def __init__(
                 constructing the categories.
         """
         super().__init__(decision_tree)
+        self.dry_map = dry_map(self.decision_tree)
+
         self.broad_categories = broad_categories
         self.wet_categories = wet_categories
         self.intensity_categories = intensity_categories
@@ -744,6 +752,38 @@ def _set_blended_times(cube: Cube, result: Cube) -> None:
             )
             result.replace_coord(new_coord)
 
+    def _get_dry_equivalents(
+        self, cube: Cube, dry_indices: np.ndarray, time_axis: int
+    ) -> Cube:
+        """
+        Returns a cube with only dry codes in which all wet codes have
+        been replaced by their nearest dry cloud equivalent. For example a
+        shower code is replaced with a partly cloudy code, a light rain code
+        is replaced with a cloud code, and a heavy rain code is replaced with
+        an overcast cloud code.
+
+        Args:
+            cube: Weather code cube.
+            dry_indices: An array of bools which are true for locations where
+                         the summary weather code will be dry.
+            time_axis: The time coordinate dimension.
+
+        Returns:
+            cube: Wet codes converted to their dry equivalent for those points
+                  that will receive a dry summary weather code.
+        """
+        dry_cube = cube.copy()
+        for value, target in self.dry_map.items():
+            dry_cube.data = np.where(cube.data == value, target, dry_cube.data)
+
+        # Note that np.rollaxis returns a new view of the input data. As such
+        # changes to `original` here are also changes to cube.data.
+        original = np.rollaxis(cube.data, time_axis)
+        dried = np.rollaxis(dry_cube.data, time_axis)
+        original[..., dry_indices] = dried[..., dry_indices]
+
+        return cube
+
     def process(self, cubes: CubeList) -> Cube:
         """Calculate the modal categorical code by grouping weather codes.
 
@@ -767,14 +807,19 @@ def process(self, cubes: CubeList) -> Cube:
         if len(cube.coord("time").points) == 1:
             result = cube
         else:
-            original_cube = self._emphasise_day_period(cube.copy())
-            cube = self._consolidate_intensity_categories(cube)
             cube = self._emphasise_day_period(cube)
 
             result = cube[0].copy()
             (time_axis,) = cube.coord_dims("time")
 
             wet_indices = self._find_wet_indices(cube, time_axis)
+
+            # For dry locations convert the wet codes to their equivalent dry
+            # codes for use in determining the summary symbol.
+            cube = self._get_dry_equivalents(cube, ~wet_indices, time_axis)
+
+            original_cube = cube.copy()
+            cube = self._consolidate_intensity_categories(cube)
             result = self._find_most_significant_dry_code(cube, result, ~wet_indices)
 
             result = self._get_most_likely_following_grouping(

improver/categorical/utilities.py

@@ -24,7 +24,7 @@
 ]
 
 LEAF_REQUIRED_KEY_WORDS = ["leaf"]
-LEAF_OPTIONAL_KEY_WORDS = ["if_night", "is_unreachable", "group"]
+LEAF_OPTIONAL_KEY_WORDS = ["if_night", "is_unreachable", "group", "dry_equivalent"]
 
 OPTIONAL_KEY_WORDS = [
     "if_masked",
@@ -576,3 +576,22 @@ def day_night_map(decision_tree: Dict[str, Dict[str, Union[str, List]]]) -> Dict
         for k, v in decision_tree.items()
         if "if_night" in v.keys()
     }
+
+
+def dry_map(decision_tree: Dict[str, Dict[str, Union[str, List]]]) -> Dict:
+    """Returns a dict showing which dry values are linked to which wet values.
+    This is used to produce cloud contributions from wet codes when determining
+    a dry summary symbol.
+
+    Args:
+        decision_tree:
+            Decision tree definition, provided as a dictionary.
+
+    Returns:
+        dict showing which dry categories (values) are linked to which wet categories (keys)
+    """
+    return {
+        v["leaf"]: v["dry_equivalent"]
+        for v in decision_tree.values()
+        if "dry_equivalent" in v.keys()
+    }

improver_tests/acceptance/SHA256SUMS

@@ -937,7 +937,7 @@ c44a00d49912f13fb17256fa0e01d9425977b5c94f2d41c602c657d39deb74cc  ./weather-symb
 90a6495f65e053e943dd4dcc9731da274f11bc19a3817a70c057218ed8eaa4de  ./weather-symbol-modes/blend_mismatch_inputs/20201209T1600Z-weather_symbols-PT01H.nc
 a981f6238828b63d7a9f7c597dd234dcc86dd35e2e3c4ba6c3b6feb491ec872e  ./weather-symbol-modes/blend_mismatch_inputs/20201209T1700Z-weather_symbols-PT01H.nc
 de4a49bca4cd930328942cf6bb9a1bc1bdd2589120a57a2335ee4ef2449dbd5d  ./weather-symbol-modes/blend_mismatch_inputs/20201209T1800Z-weather_symbols-PT01H.nc
-9ee56eeaf451b360120a1e52b95435bc5fa1089f6447d92244afcb0ca84aa781  ./weather-symbol-modes/blend_mismatch_inputs/kgo.nc
+eaf72bd6c75204e4cfbdeb975cce0d2995544c72f135ddb142d7ad418f99a308  ./weather-symbol-modes/blend_mismatch_inputs/kgo.nc
 bdaffce463e5d9e30ea6cfae58d4decfa266600cbb51480161d4170f46bb6ba3  ./weather-symbol-modes/broad_categories.json
 deb7f4effb821b2808b647e02ac955c91adae4baa33765b16378cff40e3ec5e8  ./weather-symbol-modes/gridded_input/20201209T0700Z-weather_symbols-PT01H.nc
 a61a70b0ce9e70577ba177462b9f1bfbda2457cc3975f0e9a562e1311e86e671  ./weather-symbol-modes/gridded_input/20201209T0800Z-weather_symbols-PT01H.nc
@@ -951,7 +951,7 @@ bcd90ab1d28fd736d4a3d9e481374348438d7716b543f9c7d435b003ba10c344  ./weather-symb
 973c60900aa526818e7119ed016997170055017ee1bbda279b9e640750f96f61  ./weather-symbol-modes/gridded_input/20201209T1600Z-weather_symbols-PT01H.nc
 571bff58be29197e5f946745ed565889ec81499521c38d8f7286488079afb46d  ./weather-symbol-modes/gridded_input/20201209T1700Z-weather_symbols-PT01H.nc
 2af4455b0ba7c4124e49eb1ff004e770b6239a9e2e1513f60ba4db3f0beb02cf  ./weather-symbol-modes/gridded_input/20201209T1800Z-weather_symbols-PT01H.nc
-c00571c03d331d661066607cbf51cb53e9a3e9c573c9c782361e41e9a643275f  ./weather-symbol-modes/gridded_input/kgo.nc
+da324b0903f7ad8a9f06782ee819a55a7a1570b3a3e830b1b65e0293cf717d37  ./weather-symbol-modes/gridded_input/kgo.nc
 96a8462af571f06dbd8b91a7a90aaef403eefd2b73929a5c6d8a3fbb01159aca  ./weather-symbol-modes/gridded_ties/20201209T0700Z-weather_symbols-PT01H.nc
 9f64c7a8aa7cf0e87799f96ebffe1e449e1f5174fb583d44f2479e085672dc84  ./weather-symbol-modes/gridded_ties/20201209T0800Z-weather_symbols-PT01H.nc
 c698b9599219fe89374a2565e55a374d9236904c9dd99a2ae61b5416506e98d3  ./weather-symbol-modes/gridded_ties/20201209T0900Z-weather_symbols-PT01H.nc
@@ -965,7 +965,7 @@ c39ea98f6fe64788c4ea7ea242111a5c8bbeacfaf52b2ead0cf0aed0007d46ab  ./weather-symb
 6a5b04644ab11d077809f615bac2829656127a0eeee3843940f8d33673bd70c8  ./weather-symbol-modes/gridded_ties/20201209T1700Z-weather_symbols-PT01H.nc
 39d0fa291798366a00ecae79a65de0b0692d5b4db17ac98a97d48e54b75e5dd4  ./weather-symbol-modes/gridded_ties/20201209T1800Z-weather_symbols-PT01H.nc
 fc922ede9e118dea3e7e3ba354664151f09407b32450688ed5c9870de5307c14  ./weather-symbol-modes/gridded_ties/kgo.nc
-a534bf31735a0b4a00755447b35d9531da01eb5feb1a09d1e2cc68cdd93e5d09  ./weather-symbol-modes/intensity_categories.json
+fc023594fb4ff913345e553a5bade1d51b30942476ff04745c62f0b8d826cf2e  ./weather-symbol-modes/intensity_categories.json
 89ba47a99c53d23b5490254366211a7cc0a5c8633c9faee97c091ee48a366b87  ./weather-symbol-modes/single_input/20201210T0000Z-weather_symbols-PT01H.nc
 d64efaa75b03aa4ba1fb16caa31891492e9fc5f967a584de42a4a59dc2f54237  ./weather-symbol-modes/single_input/kgo.nc
 f4e13dec400ec945ba5bd03a286780b183665c22d707c038c0990ef3491e888e  ./weather-symbol-modes/spot_input/20201209T0700Z-weather_symbols-PT01H.nc
@@ -980,7 +980,7 @@ c44a00d49912f13fb17256fa0e01d9425977b5c94f2d41c602c657d39deb74cc  ./weather-symb
 90a6495f65e053e943dd4dcc9731da274f11bc19a3817a70c057218ed8eaa4de  ./weather-symbol-modes/spot_input/20201209T1600Z-weather_symbols-PT01H.nc
 a981f6238828b63d7a9f7c597dd234dcc86dd35e2e3c4ba6c3b6feb491ec872e  ./weather-symbol-modes/spot_input/20201209T1700Z-weather_symbols-PT01H.nc
 de4a49bca4cd930328942cf6bb9a1bc1bdd2589120a57a2335ee4ef2449dbd5d  ./weather-symbol-modes/spot_input/20201209T1800Z-weather_symbols-PT01H.nc
-16d57e10b3c6cb8bddc9048c1ab09e11503cecdbe569aef746e95c8cbc23ded1  ./weather-symbol-modes/spot_input/kgo.nc
+3f43cd5fb973e4edb1a4f6ce832d0c2752fac186e1819ae1477fc466b0857e1b  ./weather-symbol-modes/spot_input/kgo.nc
 36f26203008ac401e361f549e39c5c1a0334d31eef2e064528d2c11ba029d1d2  ./weather-symbol-modes/spot_ties/20201209T0700Z-weather_symbols-PT01H.nc
 8543d8168e23975f537767a55a8f6fbd7d15f187556748ab62e4edc3f70a84d3  ./weather-symbol-modes/spot_ties/20201209T0800Z-weather_symbols-PT01H.nc
 cb1a6c410f37132f0faa541a68411e00c38bf77c719f98adcff664f6699d4bf5  ./weather-symbol-modes/spot_ties/20201209T0900Z-weather_symbols-PT01H.nc
@@ -995,7 +995,7 @@ ea67ae7a7f5363ae3692b29c93fb9989449d96e49dc613a1297d98ddb12c578a  ./weather-symb
 134fb1750cf47e868ee67801b1ea5b1f17120e6f58a787a69e54638d7d88ff82  ./weather-symbol-modes/spot_ties/20201209T1800Z-weather_symbols-PT01H.nc
 391a9bd50824318fb8f147db3b14bcd56f5b0f0e0d5479b3579e08d70ef6ba77  ./weather-symbol-modes/spot_ties/kgo.nc
 2d66678569a1af9b3357cd1c9d6b75dfedd912a99c3ed77135b2a2169b12687e  ./weather-symbol-modes/wet_categories.json
-2ec32b8654824a633fcc9d7405d4d3a8b487514bddb9e8ec6aae66c9a96c2565  ./weather-symbol-modes/wx_decision_tree.json
+6a362aa64925b9674c2806a9cafafb3d4b2d36bf8afc2f775c9b0615721e8183  ./weather-symbol-modes/wx_decision_tree.json
 bd51a8596838e355c78509b8b3b1215a3f1e3b798d5b300d4aff8032a26507b7  ./weighted_blending/accum_cycle_blend/kgo.nc
 c465dc56d83e9e866c1140033c65f07a02db154792b505d4a704d2773c3744e4  ./weighted_blending/accum_cycle_blend/ukv_prob_accum_PT3H.nc
 08e196cdc9f733a56f16fcca3b2444aac847e5c14538ddff4dd0da94efabe475  ./weighted_blending/accum_cycle_blend/ukv_prob_accum_PT4H.nc

improver_tests/acceptance/test_weather_symbol_modes.py

@@ -44,7 +44,7 @@ def test_expected(tmp_path, test_path):
     intensity_categories = (
         acc.kgo_root() / "weather-symbol-modes" / "intensity_categories.json"
     )
-    wxtree = acc.kgo_root() / "categorical-modes" / "wx_decision_tree.json"
+    wxtree = acc.kgo_root() / "weather-symbol-modes" / "wx_decision_tree.json"
     output_path = tmp_path / "output.nc"
     args = [
         *input_paths,
@@ -74,7 +74,7 @@ def test_no_input(tmp_path):
     intensity_categories = (
         acc.kgo_root() / "weather-symbol-modes" / "intensity_categories.json"
     )
-    wxtree = acc.kgo_root() / "categorical-modes" / "wx_decision_tree.json"
+    wxtree = acc.kgo_root() / "weather-symbol-modes" / "wx_decision_tree.json"
     output_path = tmp_path / "output.nc"
     args = [
         "--decision-tree",

improver_tests/categorical/decision_tree/__init__.py

@@ -488,56 +488,63 @@ def wxcode_decision_tree(accumulation: bool = False) -> Dict[str, Dict[str, Any]
         "Fog": {"leaf": 6, "group": "visibility"},
         "Cloudy": {"leaf": 7},
         "Overcast": {"leaf": 8},
-        "Light_Shower_Night": {"leaf": 9},
+        "Light_Shower_Night": {"leaf": 9, "dry_equivalent": 2},
         "Light_Shower_Day": {
             "leaf": 10,
             "if_night": "Light_Shower_Night",
             "group": "rain",
+            "dry_equivalent": 3,
         },
-        "Drizzle": {"leaf": 11, "group": "rain"},
-        "Light_Rain": {"leaf": 12, "group": "rain"},
-        "Heavy_Shower_Night": {"leaf": 13},
+        "Drizzle": {"leaf": 11, "group": "rain", "dry_equivalent": 8},
+        "Light_Rain": {"leaf": 12, "group": "rain", "dry_equivalent": 8},
+        "Heavy_Shower_Night": {"leaf": 13, "dry_equivalent": 2},
         "Heavy_Shower_Day": {
             "leaf": 14,
             "if_night": "Heavy_Shower_Night",
             "group": "rain",
+            "dry_equivalent": 3,
         },
-        "Heavy_Rain": {"leaf": 15, "group": "rain"},
-        "Sleet_Shower_Night": {"leaf": 16},
+        "Heavy_Rain": {"leaf": 15, "group": "rain", "dry_equivalent": 8},
+        "Sleet_Shower_Night": {"leaf": 16, "dry_equivalent": 2},
         "Sleet_Shower_Day": {
             "leaf": 17,
             "if_night": "Sleet_Shower_Night",
             "group": "sleet",
+            "dry_equivalent": 3,
         },
-        "Sleet": {"leaf": 18, "group": "sleet"},
-        "Hail_Shower_Night": {"leaf": 19},
+        "Sleet": {"leaf": 18, "group": "sleet", "dry_equivalent": 8},
+        "Hail_Shower_Night": {"leaf": 19, "dry_equivalent": 2},
         "Hail_Shower_Day": {
             "leaf": 20,
             "if_night": "Hail_Shower_Night",
             "group": "convection",
+            "dry_equivalent": 3,
         },
-        "Hail": {"leaf": 21, "group": "convection"},
-        "Light_Snow_Shower_Night": {"leaf": 22},
+        "Hail": {"leaf": 21, "group": "convection", "dry_equivalent": 8},
+        "Light_Snow_Shower_Night": {"leaf": 22, "dry_equivalent": 2},
         "Light_Snow_Shower_Day": {
             "leaf": 23,
             "if_night": "Light_Snow_Shower_Night",
             "group": "snow",
+            "dry_equivalent": 3,
         },
-        "Light_Snow": {"leaf": 24, "group": "snow"},
-        "Heavy_Snow_Shower_Night": {"leaf": 25},
+        "Light_Snow": {"leaf": 24, "group": "snow", "dry_equivalent": 8},
+        "Heavy_Snow_Shower_Night": {"leaf": 25, "dry_equivalent": 2},
         "Heavy_Snow_Shower_Day": {
             "leaf": 26,
             "if_night": "Heavy_Snow_Shower_Night",
             "group": "snow",
+            "dry_equivalent": 3,
         },
-        "Heavy_Snow": {"leaf": 27, "group": "snow"},
-        "Thunder_Shower_Night": {"leaf": 28},
+        "Heavy_Snow": {"leaf": 27, "group": "snow", "dry_equivalent": 8},
+        "Thunder_Shower_Night": {"leaf": 28, "dry_equivalent": 2},
         "Thunder_Shower_Day": {
             "leaf": 29,
             "if_night": "Thunder_Shower_Night",
             "group": "convection",
+            "dry_equivalent": 3,
         },
-        "Thunder": {"leaf": 30, "group": "convection"},
+        "Thunder": {"leaf": 30, "group": "convection", "dry_equivalent": 8},
     }
 
     if accumulation: