Adding experimental "chunked compression"

.github/workflows/cmake.yml

@@ -42,11 +42,11 @@ jobs:
 
           vcpkg install azure-storage-blobs-cpp:x64-windows-static
           vcpkg install azure-identity-cpp:x64-windows-static
-          vcpkg install rapidjson 'curl[ssl]' --triplet x64-windows-static
+          vcpkg install rapidjson 'curl[ssl]' lz4 --triplet x64-windows-static
           # for the ARM64 cross-compilation build, we also need to install the ARM64 versions of the dependencies
           vcpkg install azure-storage-blobs-cpp:arm64-windows-static
           vcpkg install azure-identity-cpp:arm64-windows-static
-          vcpkg install rapidjson 'curl[ssl]' --triplet arm64-windows-static
+          vcpkg install rapidjson 'curl[ssl]' lz4 --triplet arm64-windows-static
 
       - name: Install dependencies (Linux)
         if: ${{ (matrix.os == 'ubuntu-latest') }}
@@ -59,7 +59,7 @@ jobs:
           sudo apt-get install libfreetype6-dev -y
           sudo apt-get install rapidjson-dev -y
           sudo apt-get install libssl-dev -y
-          vcpkg install azure-storage-blobs-cpp azure-identity-cpp
+          vcpkg install azure-storage-blobs-cpp azure-identity-cpp lz4
 
       - name: Install Azurite (for Azure SDK based stream tests)
         shell: bash
@@ -74,7 +74,17 @@ jobs:
           # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
           # Note that we need to point CMake to the vcpkg-toolchain-file, and also specify the target triplet, c.f. https://learn.microsoft.com/en-us/vcpkg/concepts/triplets
           # since we aim for a static build.
-          cmake -B "${{github.workspace}}/build" -A x64 -DCMAKE_BUILD_TYPE=${{matrix.build}} -DLIBCZI_BUILD_CZICMD=ON -DLIBCZI_BUILD_CURL_BASED_STREAM=ON -DLIBCZI_BUILD_AZURESDK_BASED_STREAM=ON -DLIBCZI_BUILD_PREFER_EXTERNALPACKAGE_LIBCURL=ON -DLIBCZI_BUILD_LIBCZIAPI=ON -DCMAKE_TOOLCHAIN_FILE="$env:VCPKG_INSTALLATION_ROOT/scripts/buildsystems/vcpkg.cmake" -DVCPKG_TARGET_TRIPLET=x64-windows-static
+          cmake -B "${{github.workspace}}/build" -A x64 `
+              -DCMAKE_BUILD_TYPE=${{matrix.build}} `
+              -DLIBCZI_BUILD_CZICMD=ON `
+              -DLIBCZI_BUILD_CURL_BASED_STREAM=ON `
+              -DLIBCZI_BUILD_AZURESDK_BASED_STREAM=ON `
+              -DLIBCZI_BUILD_PREFER_EXTERNALPACKAGE_LIBCURL=ON `
+              -DLIBCZI_BUILD_LIBCZIAPI=ON `
+              -DLIBCZI_BUILD_ENABLE_EXPERIMENTAL_FUNCTIONALITY=ON `
+              -DLIBCZI_BUILD_PREFER_EXTERNALPACKAGE_LZ4=ON `
+              -DCMAKE_TOOLCHAIN_FILE="$env:VCPKG_INSTALLATION_ROOT/scripts/buildsystems/vcpkg.cmake" `
+              -DVCPKG_TARGET_TRIPLET=x64-windows-static
 
       - name: Configure CMake (Windows ARM64)
         if: ${{ (matrix.os == 'windows-latest') }}
@@ -92,6 +102,8 @@ jobs:
               -DLIBCZI_BUILD_PREFER_EXTERNALPACKAGE_LIBCURL=ON `
               -DLIBCZI_BUILD_LIBCZIAPI=ON `
               -DLIBCZI_BUILD_UNITTESTS=OFF `
+              -DLIBCZI_BUILD_ENABLE_EXPERIMENTAL_FUNCTIONALITY=ON `
+              -DLIBCZI_BUILD_PREFER_EXTERNALPACKAGE_LZ4=ON `
               -DCMAKE_TOOLCHAIN_FILE="$env:VCPKG_INSTALLATION_ROOT/scripts/buildsystems/vcpkg.cmake" `
               -DVCPKG_TARGET_TRIPLET=arm64-windows-static `
               -DCRASH_ON_UNALIGNED_ACCESS=OFF `
@@ -102,7 +114,16 @@ jobs:
         if: ${{ (matrix.os == 'ubuntu-latest') }}
         shell: bash
         run: |
-          cmake -B "${{github.workspace}}/build" -DCMAKE_BUILD_TYPE=${{matrix.build}} -DLIBCZI_BUILD_CZICMD=ON -DLIBCZI_BUILD_CURL_BASED_STREAM=ON -DLIBCZI_BUILD_PREFER_EXTERNALPACKAGE_LIBCURL=OFF -DLIBCZI_BUILD_AZURESDK_BASED_STREAM=ON -DLIBCZI_BUILD_LIBCZIAPI=ON -DCMAKE_TOOLCHAIN_FILE="${VCPKG_INSTALLATION_ROOT}/scripts/buildsystems/vcpkg.cmake"
+          cmake -B "${{github.workspace}}/build" \
+              -DCMAKE_BUILD_TYPE=${{matrix.build}} \
+              -DLIBCZI_BUILD_CZICMD=ON \
+              -DLIBCZI_BUILD_CURL_BASED_STREAM=ON \
+              -DLIBCZI_BUILD_PREFER_EXTERNALPACKAGE_LIBCURL=OFF \
+              -DLIBCZI_BUILD_AZURESDK_BASED_STREAM=ON \
+              -DLIBCZI_BUILD_LIBCZIAPI=ON \
+              -DLIBCZI_BUILD_ENABLE_EXPERIMENTAL_FUNCTIONALITY=ON \
+              -DLIBCZI_BUILD_PREFER_EXTERNALPACKAGE_LZ4=ON \
+              -DCMAKE_TOOLCHAIN_FILE="${VCPKG_INSTALLATION_ROOT}/scripts/buildsystems/vcpkg.cmake"
 
       - name: Build (Linux)
         if: ${{ (matrix.os == 'ubuntu-latest') }}

CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.15)
 cmake_policy(SET CMP0091 NEW) # enable new "MSVC runtime library selection" (https://cmake.org/cmake/help/latest/variable/CMAKE_MSVC_RUNTIME_LIBRARY.html)
 
 project(libCZI 
-      VERSION 0.68.0
+      VERSION 0.69.0
       HOMEPAGE_URL "https://github.com/ZEISS/libczi"
       DESCRIPTION "libCZI is an Open Source Cross-Platform C++ library to read and write CZI")
 
@@ -136,6 +136,72 @@ option(LIBCZI_BUILD_PREFER_EXTERNALPACKAGE_ZSTD "Prefer a ZSTD-package present o
 #  copy of RapidJSON during the CMake-run. This option is only relevant if building CZICmd.
 option(LIBCZI_BUILD_PREFER_EXTERNALPACKAGE_RAPIDJSON "Prefer an RapidJSON-package present on the system" ON)
 
+# Experimental functionality policy.
+#
+# This cache variable intentionally is not declared with option(), because it has
+# three states:
+#
+#   AUTO - Do not force experimental features globally. Each experimental feature
+#          keeps its own default. This is the default and preserves the normal,
+#          conservative build behavior.
+#
+#   ON   - Enable all experimental features by default.
+#
+#   OFF  - Disable all experimental features by default.
+#
+# Individual experimental feature options remain available and may still be set
+# explicitly by the user. If both this policy variable and an individual feature
+# option are specified, the individual feature option wins. This allows package
+# managers and advanced users to enable the global policy while still opting out
+# of one specific experimental feature if needed.
+#
+# Example:
+#   -DLIBCZI_BUILD_ENABLE_EXPERIMENTAL_FUNCTIONALITY=ON
+#
+# Example with explicit override:
+#   -DLIBCZI_BUILD_ENABLE_EXPERIMENTAL_FUNCTIONALITY=ON
+#   -DLIBCZI_BUILD_EXPERIMENTAL_CHUNKED_COMPRESSION=OFF
+set(LIBCZI_BUILD_ENABLE_EXPERIMENTAL_FUNCTIONALITY "AUTO" CACHE STRING
+    "Experimental functionality policy: AUTO, ON, or OFF.")
+set_property(CACHE LIBCZI_BUILD_ENABLE_EXPERIMENTAL_FUNCTIONALITY PROPERTY STRINGS AUTO ON OFF)
+string(TOUPPER "${LIBCZI_BUILD_ENABLE_EXPERIMENTAL_FUNCTIONALITY}" _libczi_experimental_functionality_policy)
+
+if (NOT _libczi_experimental_functionality_policy STREQUAL "AUTO" AND
+    NOT _libczi_experimental_functionality_policy STREQUAL "ON" AND
+    NOT _libczi_experimental_functionality_policy STREQUAL "OFF")
+    message(FATAL_ERROR
+        "LIBCZI_BUILD_ENABLE_EXPERIMENTAL_FUNCTIONALITY must be one of: AUTO, ON, OFF.")
+endif()
+
+# Default for experimental chunked compression.
+#
+# The default is derived from LIBCZI_BUILD_ENABLE_EXPERIMENTAL_FUNCTIONALITY:
+#
+#   AUTO -> OFF
+#   ON   -> ON
+#   OFF  -> OFF
+#
+# The value is used only as the default for option(). If the user already
+# supplied LIBCZI_BUILD_EXPERIMENTAL_CHUNKED_COMPRESSION in the CMake cache or
+# via -D, that explicit value is preserved by option().
+set(_libczi_default_experimental_chunked_compression OFF)
+
+if (_libczi_experimental_functionality_policy STREQUAL "ON")
+    set(_libczi_default_experimental_chunked_compression ON)
+elseif (_libczi_experimental_functionality_policy STREQUAL "OFF")
+    set(_libczi_default_experimental_chunked_compression OFF)
+endif()
+
+option(LIBCZI_BUILD_EXPERIMENTAL_CHUNKED_COMPRESSION "Enable experimental chunked compression support in the build." ${_libczi_default_experimental_chunked_compression})
+
+# Controls how the mandatory LZ4 dependency is resolved when experimental
+# chunked compression is enabled. When ON, CMake requires an externally
+# provided lz4 package via find_package(lz4 CONFIG REQUIRED). When OFF,
+# CMake fetches and builds a private copy of LZ4 using FetchContent.
+# This option has no effect when LIBCZI_BUILD_EXPERIMENTAL_CHUNKED_COMPRESSION is OFF.
+option(LIBCZI_BUILD_PREFER_EXTERNALPACKAGE_LZ4 "Prefer an LZ4-package present on the system" OFF)
+
+
 
 if (LIBCZI_BUILD_UNITTESTS)
  include (CTest)

Src/libCZI/BitmapOperations.cpp

@@ -8,7 +8,7 @@
 #include "utilities.h"
 #include "libCZI.h"
 #include "Site.h"
-#include "inc_libCZI_Config.h"
+#include "libCZI_Config_Internal.h"
 
 using namespace libCZI;
 using namespace libCZI::detail;

Src/libCZI/BitmapOperationsBitonal.cpp

@@ -4,16 +4,16 @@
 
 #include "BitmapOperationsBitonal.h"
 
+#include "libCZI_Config_Internal.h"
+#include "BitmapOperations.h"
+#include "libCZI_Utilities.h"
+#include "utilities.h"
+
 #include <cstdint>
 #include <algorithm>
 #include <sstream>
 #include <stdexcept>
 
-#include "inc_libCZI_Config.h"
-#include "BitmapOperations.h"
-#include <libCZI_Utilities.h>
-#include "utilities.h"
-
 #if LIBCZI_HAS_BUILTIN_BSWAP32
     #include <byteswap.h>
 #endif

Src/libCZI/CMakeLists.txt

@@ -75,7 +75,6 @@ set(LIBCZISRCFILES
             decoder_zstd.h
             FileHeaderSegmentData.h
             ImportExport.h
-            inc_libCZI_Config.h
             IndexSet.h
             libCZI.h
             libCZI_Compositor.h
@@ -131,6 +130,11 @@ set(LIBCZISRCFILES
             SubblockAttachmentAccessor.cpp
             BitmapOperationsBitonal.h
             BitmapOperationsBitonal.cpp
+            chunkedCompress.cpp
+            decoder_chunkedcompression.h
+            decoder_chunkedcompression.cpp
+            compresscommon.h
+            compresscommon.cpp
 )
 
 # prepare the configuration-file "libCZI_Config.h"
@@ -305,6 +309,37 @@ else()
   add_subdirectory(${zstd_SOURCE_DIR}/build/cmake ${zstd_BINARY_DIR})
 endif()
 
+if (LIBCZI_BUILD_EXPERIMENTAL_CHUNKED_COMPRESSION)
+    if (LIBCZI_BUILD_PREFER_EXTERNALPACKAGE_LZ4)
+      find_package(lz4 CONFIG REQUIRED)
+      if (TARGET LZ4::lz4_static)
+        set(LIBCZI_LZ4_LINK_TARGET "LZ4::lz4_static")
+      elseif (TARGET LZ4::lz4_shared)
+        set(LIBCZI_LZ4_LINK_TARGET "LZ4::lz4_shared")
+      else()
+        message(FATAL_ERROR "No suitable lz4 target found")
+      endif()
+    else()
+      FetchContent_Declare(
+        lz4
+        GIT_REPOSITORY https://github.com/lz4/lz4.git
+        GIT_TAG "v1.10.0"
+        SOURCE_SUBDIR "build/cmake"
+        PATCH_COMMAND # raises cmake_minimum_required to VERSION 3.10 (silences CMake 4.x deprecation warning)
+          "${CMAKE_COMMAND}"
+          -DFILE=<SOURCE_DIR>/build/cmake/CMakeLists.txt
+          -P "${CMAKE_CURRENT_SOURCE_DIR}/../../cmake/patch_cmake_minimum_version.cmake"
+      )
+
+      message(STATUS "Fetching lz4...")
+      set(LZ4_BUILD_CLI           OFF CACHE BOOL "" FORCE)
+      set(LZ4_BUILD_LEGACY_LZ4C   OFF CACHE BOOL "" FORCE)
+
+      FetchContent_MakeAvailable(lz4)
+      set(LIBCZI_LZ4_LINK_TARGET "lz4_static")
+    endif()
+endif()
+
 if (LIBCZI_HAVE_WIN32_API)
   set(libCZI_WINDOWSAPIAVAILABLE 1)
 else()
@@ -317,16 +352,31 @@ else()
   set(libCZI_WINDOWSAPI_UWP_AVAILABLE 0)
 endif()
 
+# Export the experimental chunked-compression availability into libCZI_Config.h.
+# Public headers use this macro to expose or hide experimental API declarations
+# consistently with the way the library was built.
+if (LIBCZI_BUILD_EXPERIMENTAL_CHUNKED_COMPRESSION)
+  set(libCZI_Experimental_Chunked_Compression_Available 1)
+else()
+  set(libCZI_Experimental_Chunked_Compression_Available 0)
+endif()
+
 configure_file (
   "${CMAKE_CURRENT_SOURCE_DIR}/libCZI_Config.h.in"
   "${CMAKE_CURRENT_BINARY_DIR}/libCZI_Config.h"
   ESCAPE_QUOTES @ONLY)
 
+configure_file (
+  "${CMAKE_CURRENT_SOURCE_DIR}/libCZI_Config_Internal.h.in"
+  "${CMAKE_CURRENT_BINARY_DIR}/libCZI_Config_Internal.h"
+  ESCAPE_QUOTES @ONLY)
+
 
 #  Define headers for this library. PUBLIC headers are used for compiling the library, and will be added to consumers' build paths.
 set(libCZIPublicHeaders "ImportExport.h" "libCZI.h" "libCZI_Compositor.h" "libCZI_DimCoordinate.h" "libCZI_exceptions.h"
                "libCZI_Helpers.h" "libCZI_Metadata.h" "libCZI_Metadata2.h" "libCZI_Pixels.h" "libCZI_ReadWrite.h"
-               "libCZI_Site.h" "libCZI_Utilities.h" "libCZI_Write.h" "libCZI_compress.h" "libCZI_StreamsLib.h" "libCZI_SubBlock.h")
+               "libCZI_Site.h" "libCZI_Utilities.h" "libCZI_Write.h" "libCZI_compress.h" "libCZI_StreamsLib.h" "libCZI_SubBlock.h"
+               "${CMAKE_CURRENT_BINARY_DIR}/libCZI_Config.h")
 
 #
 #define the shared libCZI - library
@@ -338,8 +388,6 @@ if (LIBCZI_BUILD_DYNLIB)
   SET_TARGET_PROPERTIES (libCZI PROPERTIES DEFINE_SYMBOL  "LIBCZI_EXPORTS" )
   set_target_properties(libCZI PROPERTIES VERSION ${PROJECT_VERSION} SOVERSION 1)
   # add the binary tree to the search path for include files so that we will find libCZI_Config.h
-  target_include_directories(libCZI PRIVATE  "${CMAKE_CURRENT_BINARY_DIR}")
-  target_include_directories(libCZI PRIVATE  ${EIGEN3_INCLUDE_DIR})
   target_link_libraries(libCZI PRIVATE  ${ADDITIONAL_LIBS_REQUIRED_FOR_ATOMIC})
   set_target_properties(libCZI PROPERTIES DEBUG_POSTFIX "d")
   if (LIBCZI_BUILD_PREFER_EXTERNALPACKAGE_ZSTD)
@@ -348,6 +396,12 @@ if (LIBCZI_BUILD_DYNLIB)
    target_link_libraries(libCZI PRIVATE libzstd_static)
    target_include_directories(libCZI PRIVATE ${zstd_SOURCE_DIR}/lib)
   endif()
+  if (LIBCZI_BUILD_EXPERIMENTAL_CHUNKED_COMPRESSION)
+    target_link_libraries(libCZI PRIVATE ${LIBCZI_LZ4_LINK_TARGET})
+    if (NOT LIBCZI_BUILD_PREFER_EXTERNALPACKAGE_LZ4)
+      target_include_directories(libCZI PRIVATE ${lz4_SOURCE_DIR}/lib)
+    endif()
+  endif()
   if (LIBCZI_BUILD_CURL_BASED_STREAM)
     target_link_libraries(libCZI PRIVATE CURL::libcurl)
   endif()
@@ -365,10 +419,13 @@ if (LIBCZI_BUILD_DYNLIB)
   if(LIBCZI_HAVE_WIN32_API)
     target_link_libraries(libCZI PRIVATE ole32 Windowscodecs)
   endif()
-  target_include_directories(libCZI
-      PUBLIC
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-        $<INSTALL_INTERFACE:include>)
+    target_include_directories(libCZI
+        PUBLIC
+            $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+            $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>      # add the binary tree to the search path for include files so that we will find libCZI_Config.h
+            $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+        PRIVATE
+            ${EIGEN3_INCLUDE_DIR})
 endif(LIBCZI_BUILD_DYNLIB)
 
 #
@@ -380,15 +437,21 @@ set_target_properties(libCZIStatic PROPERTIES CXX_STANDARD 14)
 target_compile_definitions(libCZIStatic PRIVATE _LIBCZISTATICLIB)
 set_target_properties(libCZIStatic PROPERTIES VERSION ${PROJECT_VERSION})
 # add the binary tree to the search path for include files so that we will find libCZI_Config.h
-target_include_directories(libCZIStatic PRIVATE  "${CMAKE_CURRENT_BINARY_DIR}")
-target_include_directories(libCZIStatic PRIVATE  ${EIGEN3_INCLUDE_DIR})
 target_link_libraries(libCZIStatic PRIVATE  ${ADDITIONAL_LIBS_REQUIRED_FOR_ATOMIC})
 set_target_properties(libCZIStatic PROPERTIES DEBUG_POSTFIX "d")
 if (LIBCZI_BUILD_PREFER_EXTERNALPACKAGE_ZSTD)
-   target_link_libraries(libCZIStatic PUBLIC ${LIBCZI_ZSTD_LINK_TARGET})
+  target_link_libraries(libCZIStatic PUBLIC ${LIBCZI_ZSTD_LINK_TARGET})
 else()
-   target_link_libraries(libCZIStatic PRIVATE libzstd_static)
-   target_include_directories(libCZIStatic PRIVATE ${zstd_SOURCE_DIR}/lib)
+  target_link_libraries(libCZIStatic PRIVATE libzstd_static)
+  target_include_directories(libCZIStatic PRIVATE ${zstd_SOURCE_DIR}/lib)
+endif()
+if (LIBCZI_BUILD_EXPERIMENTAL_CHUNKED_COMPRESSION)
+  if (LIBCZI_BUILD_PREFER_EXTERNALPACKAGE_LZ4)
+    target_link_libraries(libCZIStatic PUBLIC ${LIBCZI_LZ4_LINK_TARGET})
+  else()
+    target_link_libraries(libCZIStatic PUBLIC ${LIBCZI_LZ4_LINK_TARGET})
+    target_include_directories(libCZIStatic PRIVATE ${lz4_SOURCE_DIR}/lib)
+  endif()
 endif()
 
 if (LIBCZI_BUILD_CURL_BASED_STREAM)
@@ -414,9 +477,12 @@ target_compile_definitions(libCZIStatic PUBLIC _LIBCZISTATICLIB)    # make sure
 
 # Static library include directories
 target_include_directories(libCZIStatic
-  PUBLIC
-    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-    $<INSTALL_INTERFACE:include>)
+    PUBLIC
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+        $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+    PRIVATE
+        ${EIGEN3_INCLUDE_DIR})
 
 if(LIBCZI_ENABLE_INSTALL)
 
@@ -444,7 +510,7 @@ if(LIBCZI_ENABLE_INSTALL)
                 CONFIGURATIONS Release RelWithDebInfo MinSizeRel)
     endif()
 
-    # Manual install of public headers to include/libCZI (simulate structure without moving files)
+    # Manual install of public headers and generated public configuration header
     install(FILES ${libCZIPublicHeaders}
         DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/libCZI)