Skip to content

Commit

Permalink
feat(pid.py and image_records.py): Changed compression handling and a…
Browse files Browse the repository at this point in the history 
…dded icer_byte_quota and icer_minloss.
  • Loading branch information
@rbeyer
rbeyer committed Feb 21, 2024
1 parent 9371d80 commit 1509546
Show file tree
Hide file tree
Showing 8 changed files with 327 additions and 88 deletions.
6 changes: 6 additions & 0 deletions CHANGELOG.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,12 @@ Changed
- create_mmgis_pano.py - create() now takes a thumbsize int or tuple of ints that
will control the creation and size of an output thumbnail JPG file, with naming
convention set by Yamcs/OpenMCT.
- image_records.py - Added icer_byte_quota and icer_minloss parameters to ImageRecord,
as well as a variety of improved handling related to this change and in pid.py.
- pid.py - Changed handling of the compression value to match the kinds of data we'll
get from telemetry, so that the letters are assigned to cover an interval of possible
compression ratios, and to reflect the new default byte quota value for driving
images.

0.7.0 (2023-02-05)
------------------
Expand Down
48 changes: 38 additions & 10 deletions src/vipersci/pds/pid.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#!/usr/bin/env python
"""This module contains classes for VIPER Product IDs."""

# Copyright 2022-2023, United States Government as represented by the
# Copyright 2022-2024, United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All rights reserved.
#
Expand Down Expand Up @@ -73,7 +73,7 @@
a=1, # 1:1 Lossless compression
b=5, # 5:1 compression
c=16, # 16:1 compression
d=64, # 64:1 compression
d=24, # 24:1 compression
s="SLoG", # SLoG compression
z=None, # Uncompressed
)
Expand Down Expand Up @@ -319,15 +319,8 @@ def __init__(self, *args):

instrument = self.instrument_name(instrument)

if compression in vis_compression:
pass
elif compression in vis_compression.values():
compression = get_key(compression, vis_compression)
else:
raise ValueError(f"{args[3]} is not one of {vis_compression.keys()}")

super().__init__(date, time, instrument)
self.compression = compression
self.compression = self.compression_letter(compression)

def __str__(self):
return "-".join((super().__str__(), self.compression))
Expand Down Expand Up @@ -380,6 +373,41 @@ def compression_class(self):
else:
return "Lossy"

@staticmethod
def compression_letter(compression):
"""Returns the letter code from the pid.vis_compression dictionary that matches
the value provided via *compression*.
"""
if compression in vis_compression:
return compression
elif compression in vis_compression.values():
return get_key(compression, vis_compression)
elif isinstance(compression, (int, float)):
compression_ratios = []
for v in vis_compression.values():
if isinstance(v, (int, float)):
compression_ratios.append(v)

if len(compression_ratios) == 0:
raise ValueError(
"There are no numeric values in vis_compression "
f"({vis_compression})."
)

for r in sorted(compression_ratios, reverse=True):
if compression >= r:
return get_key(r, vis_compression)
else:
raise ValueError(
f"The numeric value of {compression} is not greater than one "
f"of {compression_ratios}"
)
else:
raise ValueError(
f"Could not determine one of {vis_compression.keys()} from "
f"compression ({compression})."
)

@staticmethod
def best_compression(identifiers: Iterable):
"""
Expand Down
136 changes: 73 additions & 63 deletions src/vipersci/vis/db/image_records.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

"""Defines the VIS image_records table using the SQLAlchemy ORM."""

# Copyright 2022-2023, United States Government as represented by the
# Copyright 2022-2024, United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All rights reserved.
#
Expand Down Expand Up @@ -44,7 +44,7 @@
from sqlalchemy.orm import mapped_column, relationship, synonym, validates

from vipersci.pds import Purpose
from vipersci.pds.pid import VISID, vis_instruments, vis_compression
from vipersci.pds.pid import VISID, vis_instruments
from vipersci.pds.xml import find_text, ns
from vipersci.pds.datetime import fromisozformat, isozformat
from vipersci.vis.header import pga_gain as header_pga_gain
Expand All @@ -67,15 +67,6 @@ class ImageType(enum.Flag):
def _missing_(cls, value):
return None

def compression_ratio(self):
# This mapping reflects the current settings in RFSW
if self == ImageType.LOSSLESS_ICER_IMAGE or self == ImageType.SLOG_ICER_IMAGE:
return 1
if self == ImageType.LOSSY_ICER_IMAGE:
return 16
else:
return None


class ProcessingStage(enum.Flag):
# PROCESS_RESERVED = 1
Expand Down Expand Up @@ -155,6 +146,14 @@ class ImageRecord(Base):
doc="The absolute path (POSIX style) that contains the Array_2D_Image "
"that this metadata refers to.",
)
icer_byte_quota = mapped_column(
Integer,
doc="The byteQuota value during onboard ICER compression. In the returned "
"Yamcs info, the value is in kilobytes, but this value is in bytes.",
)
icer_minloss = mapped_column(
Integer, doc="The minLoss value during onboard ICER compression."
)
image_id = mapped_column(
Integer,
nullable=False,
Expand Down Expand Up @@ -366,6 +365,15 @@ def __init__(self, **kwargs):
else:
pid = False

# Adjust the byteQuota value
if "byteQuota" in kwargs and "icer_byte_quota" not in kwargs:
kwargs["icer_byte_quota"] = int(kwargs["byteQuota"]) * 1000
del kwargs["byteQuota"]

if "minLoss" in kwargs and "icer_minloss" not in kwargs:
kwargs["icer_minloss"] = int(kwargs["minLoss"])
del kwargs["minLoss"]

rpargs = dict()
otherargs = dict()
for k, v in kwargs.items():
Expand Down Expand Up @@ -438,6 +446,7 @@ def __init__(self, **kwargs):

# Ensure product_id consistency
if pid:
# Check datetimes
if "lobt" in kwargs:
if pid.datetime() != lobt_dt:
raise ValueError(
Expand All @@ -451,6 +460,7 @@ def __init__(self, **kwargs):
f"provided start_time ({kwargs['start_time']}) disagree."
)

# Check instrument
if (
self.instrument_name is not None
and vis_instruments[pid.instrument] != self.instrument_name
Expand All @@ -461,84 +471,77 @@ def __init__(self, **kwargs):
f"({self.instrument_name}) disagree."
)

# Check compression letter
if self.output_image_mask is None:
if self.processing_info is not None:
ps = ProcessingStage(self.processing_info)
if ProcessingStage.SLOG in ps and pid.compression != "s":
if self.yamcs_name is not None:
if "slog" in self.yamcs_name and pid.compression != "s":
raise ValueError(
f"The product_id compression code ({pid.compression}) is "
"not s, but processing_info indicates it should be "
f"({self.processing_info}). "
)
elif ProcessingStage.SLOG not in ps and pid.compression == "s":
raise ValueError(
"The product_id compression code is s, but "
"processing_info indicates it shouldn't be "
f"({self.processing_info}). "
"not s, but yamcs_name indicates it should be "
f"({self.yamcs_name}). "
)
else:
t = ImageType(self.output_image_mask)
if ImageType.SLOG_ICER_IMAGE == t and pid.compression == "s":
pass
elif t.compression_ratio() != vis_compression[pid.compression]:
elif (
VISID.compression_letter(compression_ratio(self.icer_byte_quota))
!= pid.compression
):
raise ValueError(
f"The product_id compression code ({pid.compression}) and "
f"the compression ratio ({t.compression_ratio()}) based on "
f"the output_image_mask ({self.output_image_mask}) disagree."
"the compression ratio "
f"({compression_ratio(self.icer_byte_quota)}) based on "
f"the icer_byte_quota ({self.icer_byte_quota}) disagree."
)
elif (
self.start_time is not None
and self.instrument_name is not None
and self.output_image_mask is not None
):
try:
t = ImageType(self.output_image_mask)
c = None
if (
self.processing_info == ProcessingStage.SLOG
or t == ImageType.SLOG_ICER_IMAGE
):
c = "s"
else:
c = t.compression_ratio()
pid = VISID(
self.start_time.date(),
self.start_time.time(),
self.instrument_name,
c,
elif self.start_time is not None and self.instrument_name is not None:
c = None
if self.output_image_mask is not None:
try:
if ImageType(self.output_image_mask) == ImageType.SLOG_ICER_IMAGE:
c = "s"
except ValueError:

Check warning on line 503 in src/vipersci/vis/db/image_records.py

View check run for this annotation

Codecov / codecov/patch

src/vipersci/vis/db/image_records.py#L503 

Added line #L503 was not covered by tests
# output_image_mask has bad value
pass

Check warning on line 505 in src/vipersci/vis/db/image_records.py

View check run for this annotation

Codecov / codecov/patch

src/vipersci/vis/db/image_records.py#L505 

Added line #L505 was not covered by tests

if c is None and self.yamcs_name is not None and "slog" in self.yamcs_name:
c = "s"

if c is None and self.icer_byte_quota is not None:
c = compression_ratio(self.icer_byte_quota)

if c is None:
raise ValueError(
"Could not determine the compression information "
f"from output_image_mask ({self.output_image_mask}), "
f"processing_info ({self.processing_info}), or "
f"icer_byte_quota ({self.icer_byte_quota})."
)
except ValueError as err:
# output_image_mask has bad value, last try
if self.yamcs_name is None:
raise err
else:
if "slog" in self.yamcs_name:
pid = VISID(
self.start_time.date(),
self.start_time.time(),
self.instrument_name,
"s",
)
else:
raise ValueError(
"Could not determine the compression information "
f"from output_image_mask ({self.output_image_mask})."
)

pid = VISID(
self.start_time.date(),
self.start_time.time(),
self.instrument_name,
c,
)
else:
got = dict()
for k in (
"product_id",
"start_time",
"instrument_name",
"output_image_mask",
"processing_info",
"icer_byte_quota",
):
v = getattr(self, k)
if v is not None:
got[k] = v

raise ValueError(
"Either product_id must be given, or each of start_time, "
f"instrument_name, and output_image_mask. Got: {got}"
f"instrument_name, and output_image_mask plus some other things."
f"Got: {got}"
)

self._pid = str(pid)
Expand Down Expand Up @@ -737,3 +740,10 @@ def update(self, other):
setattr(self, k, v)
else:
self.labelmeta[k] = v


def compression_ratio(byte_quota):
"""Returns the result of dividing the number of bytes in a grayscale image
(2048 * 2048 * 2 == 8,388,608) by the byte_quota of the returned image.
"""
return (2048 * 2048 * 2) / byte_quota
Loading

0 comments on commit 1509546

Please sign in to comment.