add dynamic AUTOSAR PDU-container extraction to CAN bus logging

.github/workflows/build.yml

@@ -30,11 +30,22 @@ jobs:
           libxcb-icccm4 libxcb-image0 libxcb-cursor0 libxcb-keysyms1 libxcb-xkb1 libxcb-shape0 \
           libxkbcommon-x11-0 libxcb-render-util0
 
+      - name: Install Ninja (Windows)
+        if: runner.os == 'Windows'
+        run: choco install ninja -y
+
+      - name: Install Visual Studio Build Tools (Windows)
+        if: runner.os == 'Windows'
+        run: choco install visualstudio2022-workload-vctools -y
+
       - name: Install Python packages
         run: |
           pip install tox
 
       - name: Build the executable
+        shell: bash
+        env:
+          CMAKE_GENERATOR: "Ninja"
         run: |
           tox -e pyinstaller -- --distpath dist/${{ matrix.os }} --noconfirm --clean
 

.github/workflows/wheels.yml

@@ -24,16 +24,27 @@ jobs:
       - name: Install cibuildwheel
         run: python -m pip install cibuildwheel
 
+      - name: Install Ninja (Windows)
+        if: runner.os == 'Windows'
+        run: choco install ninja -y
+
+      - name: Install Visual Studio Build Tools (Windows)
+        if: runner.os == 'Windows'
+        run: choco install visualstudio2022-workload-vctools -y
+
+      - name: Install library dev packages (Linux)
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt update
+          sudo apt-get install libegl1 libopengl0 libxml2 libxml2-dev libxslt1-dev libxslt1.1 libxslt-dev python3-dev
+
       - name: Build wheels
         shell: bash
         env:
           CIBW_TEST_COMMAND: "pytest {project}/test --ignore={project}/test/asammdf/gui"
-        run: |
-          if [ "$RUNNER_OS" = "Linux" ]; then
-            sudo apt update
-            sudo apt-get install libegl1 libopengl0 libxml2 libxml2-dev libxslt1-dev libxslt1.1
-          fi
-          python -m cibuildwheel --output-dir wheelhouse
+          CIBW_CMAKE_GENERATOR: "Ninja"          # force Ninja generator
+          CMAKE_GENERATOR: "Ninja"
+        run: python -m cibuildwheel --output-dir wheelhouse
 
       - uses: actions/upload-artifact@v4
         with:

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.12.9
+    rev: v0.15.16
     hooks:
       - id: ruff-check
         args: [--fix]

CMakeLists.txt

@@ -24,8 +24,8 @@ python_add_library(cutils MODULE WITH_SOABI USE_SABI 3.10 src/asammdf/blocks/cut
 if (WITH_SYSTEM_DEFLATE)
   find_package(libdeflate REQUIRED)
   find_package(lz4 REQUIRED)
-  find_package(libzstd REQUIRED)
-  target_link_libraries(cutils PRIVATE Python::NumPy libdeflate::libdeflate_shared lz4_shared libzstd_shared)
+  find_package(zstd REQUIRED)
+  target_link_libraries(cutils PRIVATE Python::NumPy libdeflate::libdeflate_shared LZ4::lz4_shared zstd::libzstd_shared)
 else()
   # Add submodule libdeflate
   set(CMAKE_POSITION_INDEPENDENT_CODE ON)

README.md

@@ -173,23 +173,29 @@ asammdf uses the following libraries
 - lxml : for canmatrix arxml support
 - lz4 : to speed up the disk IO performance
 - python-dateutil : measurement start time handling
+- chardet : to detect non-standard Unicode encodings
+- typing-extensions
+- zstd (if Python < 3.14)
 
 Optional dependencies needed for exports
 
 - h5py : for HDF5 export
 - hdf5storage : for Matlab v7.3 .mat export
 - pyarrow : for parquet export
 - scipy: for Matlab v4 and v5 .mat export
+- python-snappy
+- polars
 
 Other optional dependencies
 
 - PySide6 : for GUI tool
 - pyqtgraph : for GUI tool and Signal plotting
 - matplotlib : as fallback for Signal plotting
-- chardet : to detect non-standard Unicode encodings
 - pyqtlet2 : for the GPS window
-- isal : for faster zlib compression/decompression
+- deflate : for faster zlib compression/decompression
 - fsspec : access files stored in the cloud
+- cryptography
+- keyring
 
 ## Benchmarks
 

doc/buslogging.rst

@@ -41,6 +41,39 @@ There multiple ways to address this channel in this situation:
 #. CAN bus ID, ASAM conformant message ID and short signal name, delimited by dot
 
     .. code:: python
-
-        mdf.get('CAN1.CAN_DataFrame_123.EngineStatus')   
-
+
+        mdf.get('CAN1.CAN_DataFrame_123.EngineStatus')
+
+
+Container I-PDUs
+================
+
+AUTOSAR **container I-PDUs** pack several smaller PDUs into a single CAN(-FD)
+frame. When the database describes a frame as a PDU container, each contained
+PDU is decoded and gets **its own channel group**, so its signals are addressed
+exactly like any other bus logging signal:
+
+.. code:: python
+
+    mdf.get('EngineStatus')                       # short signal name
+    mdf.get('CAN1.VehicleStatus.EngineStatus')    # container frame name
+
+Both container layouts are supported: *dynamic* containers, where every
+contained PDU is prefixed by a header (``Header_ID`` + ``Header_DLC``) and its
+offset therefore changes from frame to frame, and *static* containers, where the
+contained PDUs sit at fixed offsets.
+
+A sender may transmit a contained PDU shorter than the length declared in the
+database — the header DLC is the authority. Signals that reach past the
+transmitted length would otherwise be decoded out of the neighbouring PDU or out
+of the container padding, so those samples are marked invalid:
+
+.. code:: python
+
+    sig = mdf.get('SomeSignal')
+    sig.invalidation_bits    # None, or True where the bytes were not transmitted
+
+Two limitations are worth noting: a multiplexed *contained* PDU is not modelled
+by **canmatrix**, so it cannot be decoded, and container I-PDUs are only handled
+for CAN — not for LIN.
+

doc/intro.rst

@@ -87,13 +87,18 @@ Dependencies
 * lxml : for canmatrix arxml support
 * lz4 : to speed up the disk IO performance
 * python-dateutil : measurement start time handling
+* chardet : to detect non-standard Unicode encodings
+* typing-extensions
+* zstd (if Python < 3.14)
 
 Optional dependencies needed for exports
 
 * h5py : for HDF5 export
 * hdf5storage : for Matlab v7.3 .mat export
 * pyarrow : for parquet export
 * scipy: for Matlab v4 and v5 .mat export
+* python-snappy
+* polars
 
 Other optional dependencies
 
@@ -102,8 +107,10 @@ Other optional dependencies
 * matplotlib : as fallback for Signal plotting
 * chardet : to detect non-standard Unicode encodings 
 * pyqtlet2 : for the GPS window
-* isal : for faster zlib compression/decompression
+* deflate : for faster zlib compression/decompression
 * fsspec : access files stored in the cloud
+* cryptography
+* keyring
 
 
 Installation

doc/pdu_container_extraction.md

@@ -0,0 +1,147 @@
+# AUTOSAR PDU-container extraction (CAN bus logging)
+
+Developer note for the container-PDU support in
+`src/asammdf/blocks/bus_logging_utils.py` (`extract_pdus`) wired into
+`MDF._extract_can_logging` (`mdf_v4.py`).
+
+The user-facing documentation is the "Container I-PDUs" section of
+`doc/buslogging.rst`. This note is not part of the built docs (sphinx is
+configured for `.rst` only) — it records *why* the decoder looks the way it
+does, for whoever touches it next.
+
+## What
+
+AUTOSAR **container I-PDUs** pack several contained PDUs into one CAN(-FD)
+frame. `extract_mux` only understood `is_multiplexed` frames, so container
+frames (`message.is_pdu_container`) were routed through it and mis-decoded.
+`extract_pdus` decodes them and gives **each contained PDU its own channel
+group**, reusing the existing channel-group machinery unchanged.
+
+Two container layouts are supported:
+
+- **Dynamic** — each contained PDU is prefixed by a per-PDU header
+  (`Header_ID` + `Header_DLC`). A PDU's byte offset depends on the lengths of
+  the PDUs before it, so the frame is walked header-by-header.
+- **Static** — no per-PDU header; contained PDUs sit at fixed byte offsets.
+
+## How
+
+`_extract_can_logging` routes `message.is_pdu_container` frames to
+`extract_pdus`; everything else keeps using `extract_mux`. `extract_pdus`
+branches on whether the frame exposes the `Header_ID`/`Header_DLC` synthetic
+signals that canmatrix injects.
+
+### Dynamic containers
+
+1. Header geometry is derived from the header signals (short header = 24 + 8
+   bits, long header = 32 + 32); it is **not** hardcoded.
+2. Headers are walked across all frames one "slot" per iteration. Frames
+   diverge in offset after the first PDU, so a per-frame byte offset is
+   advanced independently and each frame's current header window is gathered
+   vectorized. Unknown/padding ids advance by their DLC, which guarantees
+   termination.
+   `Header_ID`/`Header_DLC` are read **unsigned** even when the database marks
+   them signed — canmatrix's ARXML parser does mark them signed, which makes a
+   0xFF padding byte decode as a DLC of −1 and walks the offset *backwards*
+   over a padded tail, rescanning the frame misaligned and inventing contained
+   PDUs out of padding.
+3. Per contained PDU, its payload slices are gathered into a 2-D array and its
+   signals extracted. A contained PDU's signal `start_bit` values are
+   **PDU-payload-relative**, so once the byte-aligned PDU payload slice is
+   isolated the regular `extract_signal` machinery applies unchanged.
+4. A sender may transmit a contained PDU **shorter than its declared size** (the
+   header DLC is the authority). The gather is fixed-width, so signals reaching
+   past the transmitted length are decoded from the next PDU or from container
+   padding — those samples are flagged through `invalidation_bits` rather than
+   reported as measured data. Real OEM data hits this on ~2.4 % of
+   contained-PDU occurrences.
+
+### Static containers
+
+canmatrix cannot decode static containers at runtime — `Frame.unpack` raises
+`DecodingConatainerPdu` on them. But its ARXML parser has already rebased each
+contained PDU's signal `start_bit` values to be **frame-relative** (via the
+PDU's `OFFSET`). So every contained PDU decodes straight from the full frame
+payload; each still becomes its own channel group. Static PDUs carry no header
+id (`pdu.id is None`), so the channel-group identity keys on the PDU name.
+
+### Shared emission
+
+Both paths call `_emit_pdu_signals(...)`, which extracts a single PDU's signals
+into a dedicated channel-group entry (the PDU identity is carried in the entry's
+`muxer` slot via `_contained_pdu_muxer`). The only difference is the payload it
+is handed: the sliced PDU payload (dynamic) or the full frame (static).
+
+### CAN-FD note
+
+Container frames ride on CAN-FD in practice (they need > 8 bytes; canmatrix
+marks every container frame `is_fd = True`). Extraction itself is purely
+`CAN_DataFrame.DataBytes`-width driven — `_extract_can_logging` does not read
+`EDL`/`DataLength`/`DLC` — so the FD flag has no functional effect on decoding;
+the wide `DataBytes` array is all that matters.
+
+## Not handled
+
+- **PDU-internal multiplexing** — a multiplexed contained PDU is not modelled by
+  canmatrix (it only reads `I-SIGNAL-TO-I-PDU-MAPPING`, no `DYNAMIC/STATIC-PART`
+  under a contained PDU), so it is out of reach for this metadata-only approach.
+- **LIN container frames** — container I-PDUs are a CAN-FD/FlexRay/Ethernet
+  mechanism; bus logging support here is scoped to CAN.
+
+## Testing
+
+Fully offline in `test/test_CAN_pdu_extraction.py`; containers are built
+in-memory with canmatrix.
+
+- `test_extract_pdus_matches_canmatrix_unpack` — dynamic containers vs the
+  `canmatrix.Frame.unpack` oracle, across big/little-endian headers and
+  0x00/0xFF padding, with unique multi-PDU frames including bit-packed,
+  non-byte-aligned **signed** signals.
+- `test_extract_pdus_static_container` — static containers vs an equivalent
+  **flat** canmatrix frame carrying the same frame-relative signals (needed
+  because `Frame.unpack` refuses static containers); covers byte-aligned LE/BE
+  and a non-byte-aligned signed field, and asserts one channel group per PDU.
+- `test_extract_bus_logging_container_e2e` — full `MDF.extract_bus_logging`
+  pipeline on a 32-byte container.
+- `test_extract_bus_logging_canfd_container_e2e` — full pipeline on genuine
+  CAN-FD frames: 64-byte payload with the `EDL` flag and `DataLength` members
+  set.
+- `test_extract_pdus_wide_signed_signal` — a contained-PDU signal wider than 64
+  bits and declared **signed** (real OEM containers carry 216/288/400-bit
+  signed blobs) comes back as raw bytes instead of raising `OverflowError` in
+  `as_non_byte_sized_signed_int`.
+- `test_extract_pdus_short_header_dlc_invalidates` — header DLC shorter than the
+  declared PDU size: transmitted signals decode normally, signals past the
+  transmitted length are flagged invalid.
+
+Run them with:
+
+```bash
+python -m unittest test.test_CAN_pdu_extraction -v
+```
+
+The existing `test/test_CAN_bus_logging.py` (real OBD/J1939 data, downloaded)
+continues to pass, confirming no regression to the `extract_mux` path.
+
+## Validation on real measurements
+
+Validated against two real OEM CAN-FD bus logs (5.1 M and 2.3 M CAN frames) and
+three production ARXML databases covering three CAN channels, with
+`canmatrix.Frame.unpack` as the oracle:
+
+- **1 237 247** decoded signal values across **15** real container messages,
+  **0** mismatches.
+- Full `extract_bus_logging`: master yields only `Header_ID`/`Header_DLC` for
+  container messages; this branch yields 789 additional real signals on the
+  first measurement (58 container-derived channel groups) and 203
+  container-derived groups on the second.
+- All 541 non-container signals are bit-identical to master, and 676 008
+  non-container values were separately confirmed against the oracle.
+- Decoded values are physically plausible: HV DC-link 400 V mean / 794 V peak on
+  an 800 V platform, inverter and coolant temperatures 25–33 °C, 14.5 V rail.
+- The same drive was also recorded **signal-based** (decoded on the fly by the
+  logger toolchain). Cross-checking our container decode against that recording
+  — an oracle sharing no code with asammdf or canmatrix — 298 of 303 comparable
+  signals agree on every one of 114 251 samples; the remaining five are
+  free-running sequence counters / CRCs whose sample instants differ between the
+  two recordings by more than the comparison window.

pyproject.toml

@@ -21,6 +21,7 @@ classifiers = [
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
     "Programming Language :: Python :: 3.14",
+    "Programming Language :: Python :: 3.15",
 ]
 keywords = [
     "read",
@@ -36,20 +37,18 @@ keywords = [
 dependencies = [
     "chardet>=7.0.0",
     "canmatrix[arxml,dbc]>=1.2",
-    "isal; platform_machine == 'x86_64' or platform_machine == 'AMD64'",
-    "deflate; platform_machine == 'x86_64' or platform_machine == 'AMD64'",
     "lxml>=4.9.3",
     "lz4",
     "numexpr>=2.10.0",
     "numpy>=2.0.0",
     "pandas>=2.2.2",
     "python-dateutil",
     "typing-extensions>=4.10",
-    "zstd",
+    "zstd; python_version < '3.14'",
 ]
 
 [project.optional-dependencies]
-decode = ["chardet>=7.0.0"]
+compression = ["deflate"]
 export = [
     "pyarrow>=17.0.0",
     "h5py>=3.11",
@@ -99,7 +98,7 @@ test-command = "pytest {project}/test"
 build-frontend = "build"
 archs = ["auto64"]                        # only build for 64bit architectures
 # skip pypy and irrelevant architectures
-skip = ["pp*", "*_ppc64le", "*-musllinux*", "*_s390x", "cp314*"]
+skip = ["pp*", "*_ppc64le", "*-musllinux*", "*_s390x", "cp315*", "cp314t*"]
 
 [tool.mypy]
 files = "."
@@ -115,7 +114,6 @@ module = [
     "fsspec.*",
     "h5py.*",
     "hdf5storage.*",
-    "isal.*",
     "lz4.*",
     "mdfreader.*",
     "mfile.*",

run_ruff.bat

@@ -1,3 +1,3 @@
-pip install --upgrade ruff && ^
+pip install ruff~=0.15.0 && ^
 ruff check --fix && ^
 ruff format