Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix reading of CA-CN structures #1065

Open
wants to merge 3 commits into
base: development
Choose a base branch
from
Open

Fix reading of CA-CN structures #1065

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5bc31a4
Update docs & support to pass args to pytest
VorobiovM Aug 8, 2024
ff2d104
fix last channel in CA-CN structure having incorrect dtype
VorobiovM Aug 9, 2024
9289616
Suppress F401
VorobiovM Aug 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions CONTRIBUTING.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@ When submitting PR's please take into account:

# Testing

You can use tox to run tests locally. Example for the unit tests with Python version 3.7:
You can use tox to run tests locally. Example for the unit tests with Python version 3.9:

```console
tox -e py27
tox -e py39
```

Otherwise, you can just push and the tests will be run by GitHub Actions.
Expand Down
187 changes: 91 additions & 96 deletions src/asammdf/blocks/mdf_v4.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1044,6 +1044,13 @@ def _read_channels(
first_dep = ca_block = ChannelArrayBlock(address=component_addr, stream=stream, mapped=mapped)
dependencies[index] = [first_dep]

ca_cnt = len(dependencies[index])
byte_offset_factors = []
bit_pos_inval_factors = []
dimensions = []
total_elem = 1

# recurse into CA structure
while ca_block.composition_addr:
stream.seek(ca_block.composition_addr)
blk_id = stream.read(4)
Expand Down Expand Up @@ -1071,105 +1078,9 @@ def _read_channels(
mapped=mapped,
)

ca_cnt = len(dependencies[index])
if ret_composition:
dependencies[index].extend(ret_composition)

byte_offset_factors = []
bit_pos_inval_factors = []
dimensions = []
total_elem = 1

for ca_blck in dependencies[index][:ca_cnt]:
# only consider CN templates
if ca_blck.ca_type != v4c.CA_STORAGE_TYPE_CN_TEMPLATE:
logger.warning("Only CN template arrays are supported")
continue

# 1D array with dimensions
for i in range(ca_blck.dims):
dim_size = ca_blck[f"dim_size_{i}"]
dimensions.append(dim_size)
total_elem *= dim_size

# 1D arrays for byte offset and invalidation bit pos calculations
byte_offset_factors.extend(ca_blck.get_byte_offset_factors())
bit_pos_inval_factors.extend(ca_blck.get_bit_pos_inval_factors())

multipliers = [1] * len(dimensions)
for i in range(len(dimensions) - 2, -1, -1):
multipliers[i] = multipliers[i + 1] * dimensions[i + 1]

def _get_nd_coords(index, factors: list[int]) -> list[int]:
"""Convert 1D index to CA's nD coordinates"""
coords = [0] * len(factors)
for i, factor in enumerate(factors):
coords[i] = index // factor
index %= factor
return coords

def _get_name_with_indices(ch_name: str, ch_parent_name: str, indices: list[int]) -> str:
coords = "[" + "][".join(str(coord) for coord in indices) + "]"
m = re.match(ch_parent_name, ch_name)
n = re.search(r"\[\d+\]", ch_name)
if m:
name = ch_name[: m.end()] + coords + ch_name[m.end() :]
elif n:
name = ch_name[: n.start()] + coords + ch_name[n.start() :]
else:
name = ch_name + coords
return name

ch_len = len(channels)
for elem_id in range(1, total_elem):
for cn_id in range(index, ch_len):
nd_coords = _get_nd_coords(elem_id, multipliers)

# copy composition block
new_block = deepcopy(channels[cn_id])

# update byte offset & position of invalidation bit
byte_offset = bit_offset = 0
for coord, byte_factor, bit_factor in zip(
nd_coords, byte_offset_factors, bit_pos_inval_factors
):
byte_offset += coord * byte_factor
bit_offset += coord * bit_factor
new_block.byte_offset += byte_offset
new_block.pos_invalidation_bit += bit_offset

# update channel name
new_block.name = _get_name_with_indices(new_block.name, channel.name, nd_coords)

# append to channel list
channels.append(new_block)

# update channel dependencies
if dependencies[cn_id] is not None:
deps = []
for dep in dependencies[cn_id]:
if not isinstance(dep, ChannelArrayBlock):
dep_entry = (dep[0], dep[1] + (ch_len - index) * elem_id)
deps.append(dep_entry)
dependencies.append(deps)
else:
dependencies.append(None)

# update channels db
entry = (dg_cntr, ch_cntr)
self.channels_db.add(new_block.name, entry)
ch_cntr += 1

# modify channels' names found recursively in-place
orig_name = channel.name
for cn_id in range(index, ch_len):
nd_coords = _get_nd_coords(0, multipliers)
name = _get_name_with_indices(channels[cn_id].name, orig_name, nd_coords)
entry = self.channels_db.pop(channels[cn_id].name)
channels[cn_id].name = name
# original channel entry will only contain single source tuple
self.channels_db.add(name, entry[0])

break

else:
Expand All @@ -1178,6 +1089,90 @@ def _get_name_with_indices(ch_name: str, ch_parent_name: str, indices: list[int]
)
break

# create channels from ca block
for ca_blck in dependencies[index][:ca_cnt]:
# only consider CN templates
if ca_blck.ca_type != v4c.CA_STORAGE_TYPE_CN_TEMPLATE:
logger.warning("Only CN template arrays are supported")
continue

# 1D array with dimensions
for i in range(ca_blck.dims):
dim_size = ca_blck[f"dim_size_{i}"]
dimensions.append(dim_size)
total_elem *= dim_size

# 1D arrays for byte offset and invalidation bit pos calculations
byte_offset_factors.extend(ca_blck.get_byte_offset_factors())
bit_pos_inval_factors.extend(ca_blck.get_bit_pos_inval_factors())

multipliers = [1] * len(dimensions)
for i in range(len(dimensions) - 2, -1, -1):
multipliers[i] = multipliers[i + 1] * dimensions[i + 1]

def _get_nd_coords(index, factors: list[int]) -> list[int]:
"""Convert 1D index to CA's nD coordinates"""
coords = [0] * len(factors)
for i, factor in enumerate(factors):
coords[i] = index // factor
index %= factor
return coords

def _get_name_with_indices(ch_name: str, ch_parent_name: str, indices: list[int]) -> str:
coords = "[" + "][".join(str(coord) for coord in indices) + "]"
m = re.match(ch_parent_name, ch_name)
n = re.search(r"\[\d+\]", ch_name)
if m:
name = ch_name[: m.end()] + coords + ch_name[m.end() :]
elif n:
name = ch_name[: n.start()] + coords + ch_name[n.start() :]
else:
name = ch_name + coords
return name

ch_len = len(channels)
for elem_id in range(total_elem):
for cn_id in range(index, ch_len):
nd_coords = _get_nd_coords(elem_id, multipliers)

# copy composition block
new_block = deepcopy(channels[cn_id])

# update byte offset & position of invalidation bit
byte_offset = bit_offset = 0
for coord, byte_factor, bit_factor in zip(
nd_coords, byte_offset_factors, bit_pos_inval_factors
):
byte_offset += coord * byte_factor
bit_offset += coord * bit_factor
new_block.byte_offset += byte_offset
new_block.pos_invalidation_bit += bit_offset

# update channel name
new_block.name = _get_name_with_indices(new_block.name, channel.name, nd_coords)

# append to channel list
channels.append(new_block)

# update channel dependencies
if dependencies[cn_id] is not None:
deps = []
for dep in dependencies[cn_id]:
if not isinstance(dep, ChannelArrayBlock):
dep_entry = (dep[0], dep[1] + (ch_len - index) * elem_id)
deps.append(dep_entry)
if deps:
dependencies.append(deps)
else:
dependencies.append(None)
else:
dependencies.append(None)

# update channels db
entry = (dg_cntr, ch_cntr)
self.channels_db.add(new_block.name, entry)
ch_cntr += 1

else:
dependencies.append(None)

Expand Down
6 changes: 3 additions & 3 deletions src/asammdf/gui/widgets/batch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,21 +104,21 @@ def __init__(
formats.append("MAT")
except ImportError:
try:
from scipy.io import savemat
from scipy.io import savemat # noqa: F401

formats.append("MAT")
except ImportError:
pass

try:
from h5py import File as HDF5
from h5py import File as HDF5 # noqa: F401

formats.append("HDF5")
except ImportError:
pass

try:
from fastparquet import write as write_parquet
from fastparquet import write as write_parquet # noqa: F401

formats.append("Parquet")
except ImportError:
Expand Down
6 changes: 3 additions & 3 deletions src/asammdf/gui/widgets/file.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -368,21 +368,21 @@ def __init__(
formats.append("MAT")
except ImportError:
try:
from scipy.io import savemat
from scipy.io import savemat # noqa: F401

formats.append("MAT")
except ImportError:
pass

try:
from h5py import File as HDF5
from h5py import File as HDF5 # noqa: F401

formats.append("HDF5")
except ImportError:
pass

try:
from fastparquet import write as write_parquet
from fastparquet import write as write_parquet # noqa: F401

formats.append("Parquet")
except ImportError:
Expand Down
4 changes: 2 additions & 2 deletions tox.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,12 @@ deps =
pytest-timeout
pyautogui; sys_platform != "win32"
pywin32; sys_platform == "win32"
passenv = DISPLAY,XAUTHORITY
passenv = DISPLAY,XAUTHORITY,HTTP_PROXY,HTTPS_PROXY
setenv =
DISPLAY = :0
QT_DEBUG_PLUGINS = 1
commands =
coverage run --rcfile=pyproject.toml --module pytest
coverage run --rcfile=pyproject.toml --module pytest {posargs}
coverage report --rcfile=pyproject.toml

[testenv:style]
Expand Down
Loading