From f344190c59a1b29e2ab25a9f8fdd1310102866f5 Mon Sep 17 00:00:00 2001 From: "Merlot.Qi" Date: Tue, 2 Jun 2026 13:21:38 +0800 Subject: [PATCH 1/8] feat(lcc): add LCC writer with encoding pipeline and round-trip tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement the inverse of the LCC reader — encoding DataTable columns into the LCC binary format (data.bin, shcoef.bin, index.bin, meta.lcc). New files: - src/io/lcc_encoder.h/.cpp — encode functions (rotation 10-10-10-2, color RGBA, scale exp→uint16, SH 11-10-11, full 32-byte splat record) - include/splat/io/lcc_writer.h — public API (LccWriteConfig, writeLcc) - src/io/lcc_writer.cpp — full pipeline: range collection, quadtree spatial partition, per-cell encoding, file output with nlohmann::json - tests/lcc_writer_test.cpp — 12 round-trip tests for all encoders Fixes: - cmake/FindWebP.cmake: add libsharpyuv for WebP 1.4+ compatibility - lccClamp: fix inverted upper-bound ternary (returned x instead of hi) Co-Authored-By: Claude Opus 4.8 --- cmake/FindWebP.cmake | 10 +- include/splat/io/lcc_writer.h | 65 +++++ src/io/lcc_encoder.cpp | 191 ++++++++++++ src/io/lcc_encoder.h | 118 ++++++++ src/io/lcc_writer.cpp | 529 ++++++++++++++++++++++++++++++++++ tests/CMakeLists.txt | 4 + tests/lcc_writer_test.cpp | 268 +++++++++++++++++ 7 files changed, 1184 insertions(+), 1 deletion(-) create mode 100644 include/splat/io/lcc_writer.h create mode 100644 src/io/lcc_encoder.cpp create mode 100644 src/io/lcc_encoder.h create mode 100644 src/io/lcc_writer.cpp create mode 100644 tests/lcc_writer_test.cpp diff --git a/cmake/FindWebP.cmake b/cmake/FindWebP.cmake index 6e1bdcf..8cb1f94 100644 --- a/cmake/FindWebP.cmake +++ b/cmake/FindWebP.cmake @@ -33,6 +33,10 @@ find_library(WebP_LIBRARY NAMES webp libwebp ) +find_library(WebP_SHARPYUV_LIBRARY + NAMES sharpyuv libsharpyuv +) + include(FindPackageHandleStandardArgs) find_package_handle_standard_args(WebP REQUIRED_VARS WebP_LIBRARY WebP_INCLUDE_DIR @@ -44,6 +48,10 @@ if(WebP_FOUND AND NOT TARGET WebP::webp) IMPORTED_LOCATION "${WebP_LIBRARY}" INTERFACE_INCLUDE_DIRECTORIES "${WebP_INCLUDE_DIR}" ) + if(WebP_SHARPYUV_LIBRARY) + set_property(TARGET WebP::webp APPEND PROPERTY + INTERFACE_LINK_LIBRARIES "${WebP_SHARPYUV_LIBRARY}") + endif() endif() -mark_as_advanced(WebP_INCLUDE_DIR WebP_LIBRARY) +mark_as_advanced(WebP_INCLUDE_DIR WebP_LIBRARY WebP_SHARPYUV_LIBRARY) diff --git a/include/splat/io/lcc_writer.h b/include/splat/io/lcc_writer.h new file mode 100644 index 0000000..93d34d9 --- /dev/null +++ b/include/splat/io/lcc_writer.h @@ -0,0 +1,65 @@ +/** + * @file splat/io/lcc_writer.h + * @brief Write LCC (Lixel CyberColor) format files + * + * Converts DataTable(s) into the LCC file set: + * meta.lcc — JSON metadata + * index.bin — quadtree unit index + * data.bin — packed 32-byte-per-splat records + * shcoef.bin — packed 64-byte-per-splat SH coefficients (Quality only) + */ + +#pragma once + +#include +#include +#include + +namespace splat { + +class DataTable; + +/// Configuration for LCC file output. +struct LccWriteConfig { + /// Quadtree cell width in metres. + float cellSizeX = 30.0f; + + /// Quadtree cell height in metres. + float cellSizeY = 30.0f; + + /// "Quality" enables SH output (shcoef.bin); "Portable" skips it. + std::string fileType = "Quality"; + + /// Scene name (stored in meta.lcc). + std::string name = "XGrids Splats"; + + /// Description string (stored in meta.lcc). + std::string description = "Generated by SplatLib"; + + /// Unique identifier; auto-generated as random 32-hex string when empty. + std::string guid; + + /// Apply the inverse of the reader's PlayCanvas coordinate transform + /// so that reading back produces the same DataTable columns. + bool applyCoordinateTransform = true; +}; + +/** + * @brief Write LCC files from per-LOD DataTables. + * + * Each entry in @p lods corresponds to one LOD level (lod[0] = LOD0, etc.). + * Column names must match the convention used by readLcc(): + * + * Required: x, y, z, rot_0..rot_3, f_dc_0..f_dc_2, + * scale_0..scale_2, opacity + * Optional: nx, ny, nz (default 0), f_rest_0..f_rest_44 (SH) + * + * @param outputDir Directory where the LCC file set will be written. + * @param lods One DataTable per LOD level (at least one required). + * @param config Output configuration. + */ +void writeLcc(const std::filesystem::path& outputDir, + const std::vector& lods, + const LccWriteConfig& config = {}); + +} // namespace splat diff --git a/src/io/lcc_encoder.cpp b/src/io/lcc_encoder.cpp new file mode 100644 index 0000000..9560796 --- /dev/null +++ b/src/io/lcc_encoder.cpp @@ -0,0 +1,191 @@ +#include "lcc_encoder.h" + +#include +#include + +namespace splat { + +// ── Rotation encoding (10-10-10-2) ──────────────────────────────────────── +// +// Encoding mirrors the ply2lcc reference and is the exact inverse of +// decodeRotationInto() in lcc_reader.cpp. +// +// Steps: +// 1. Normalise the input quaternion {w,x,y,z}. +// 2. Find the largest absolute component. +// 3. If the largest is negative, negate the whole quaternion. +// 4. Map from wxyz order (LCC input) to LCC index: +// w→3, x→0, y→1, z→2 +// 5. Drop the largest component; encode the remaining three. +// 6. Scale each from [-1/√2, 1/√2] to [0, 1023] (10 bits). +// 7. Pack: p0 | (p1<<10) | (p2<<20) | (lcc_idx<<30). + +static constexpr float kRsqrt2 = 0.7071067811865475f; +static constexpr float kSqrt2 = 1.414213562373095f; + +// Mapping from wxyz largest-index to LCC drop-index. +// wxyz idx 0 (w) → LCC 3 wxyz idx 1 (x) → LCC 0 +// wxyz idx 2 (y) → LCC 1 wxyz idx 3 (z) → LCC 2 +static constexpr int kWxyzToLccIdx[4] = {3, 0, 1, 2}; + +// Encoding order for the three stored components, by LCC drop-index. +// LCC 0 (dropped x): store y, z, w (src indices 2, 3, 0) +// LCC 1 (dropped y): store x, z, w (src indices 1, 3, 0) +// LCC 2 (dropped z): store x, y, w (src indices 1, 2, 0) +// LCC 3 (dropped w): store x, y, z (src indices 1, 2, 3) +static constexpr int kEncodeOrder[4][3] = { + {2, 3, 0}, // LCC 0 + {1, 3, 0}, // LCC 1 + {1, 2, 0}, // LCC 2 + {1, 2, 3}, // LCC 3 +}; + +uint32_t encodeRotation(const float rot[4]) { + float w = rot[0], x = rot[1], y = rot[2], z = rot[3]; + + // Normalise + float len = std::sqrt(w * w + x * x + y * y + z * z); + if (len > 0.0f) { + w /= len; x /= len; y /= len; z /= len; + } + + // Find largest absolute component in (w, x, y, z) order + float abs_vals[4] = {std::fabs(w), std::fabs(x), std::fabs(y), std::fabs(z)}; + int max_idx_wxyz = 0; + for (int i = 1; i < 4; ++i) { + if (abs_vals[i] > abs_vals[max_idx_wxyz]) max_idx_wxyz = i; + } + + // Ensure the dropped component is positive + float quat[4] = {w, x, y, z}; + if (quat[max_idx_wxyz] < 0.0f) { + w = -w; x = -x; y = -y; z = -z; + } + + int lcc_idx = kWxyzToLccIdx[max_idx_wxyz]; + + // Select the three components to encode + float src[4] = {w, x, y, z}; + float enc[3]; + for (int i = 0; i < 3; ++i) { + enc[i] = src[kEncodeOrder[lcc_idx][i]]; + } + + // Quantise each to 10 bits: map [-1/√2, 1/√2] → [0, 1023] + auto quant10 = [](float v) -> uint32_t { + float norm = (v + kRsqrt2) / kSqrt2; + norm = lccClamp(norm, 0.0f, 1.0f); + return static_cast(norm * 1023.0f + 0.5f); + }; + + uint32_t p0 = quant10(enc[0]); + uint32_t p1 = quant10(enc[1]); + uint32_t p2 = quant10(enc[2]); + + return p0 | (p1 << 10) | (p2 << 20) | (static_cast(lcc_idx) << 30); +} + +// ── Color encoding ──────────────────────────────────────────────────────── + +uint32_t encodeColor(const float f_dc[3], float opacity) { + auto toU8 = [](float dc) -> uint8_t { + float color = 0.5f + kLccSH_C0 * dc; + color = lccClamp(color, 0.0f, 1.0f); + return static_cast(color * 255.0f + 0.5f); + }; + + uint8_t r = toU8(f_dc[0]); + uint8_t g = toU8(f_dc[1]); + uint8_t b = toU8(f_dc[2]); + uint8_t a = static_cast(lccClamp(lccSigmoid(opacity), 0.0f, 1.0f) * + 255.0f + 0.5f); + + return (static_cast(a) << 24) | + (static_cast(b) << 16) | + (static_cast(g) << 8) | + static_cast(r); +} + +// ── Scale encoding ──────────────────────────────────────────────────────── + +void encodeScale(const float scale_log[3], const Eigen::Vector3f& scale_min, + const Eigen::Vector3f& scale_max, uint16_t out[3]) { + for (int i = 0; i < 3; ++i) { + float linear = std::exp(scale_log[i]); + float range = scale_max[i] - scale_min[i]; + float norm = (range > 0.0f) ? (linear - scale_min[i]) / range : 0.0f; + norm = lccClamp(norm, 0.0f, 1.0f); + out[i] = static_cast(norm * 65535.0f + 0.5f); + } +} + +// ── SH encoding ─────────────────────────────────────────────────────────── + +uint32_t encodeShTriplet(float r, float g, float b, float sh_min, float sh_max) { + float range = sh_max - sh_min; + + auto norm = [sh_min, range](float v) -> float { + if (range <= 0.0f) return 0.5f; + return lccClamp((v - sh_min) / range, 0.0f, 1.0f); + }; + + uint32_t r_enc = static_cast(norm(r) * 2047.0f + 0.5f); + uint32_t g_enc = static_cast(norm(g) * 1023.0f + 0.5f); + uint32_t b_enc = static_cast(norm(b) * 2047.0f + 0.5f); + + return r_enc | (g_enc << 11) | (b_enc << 21); +} + +void encodeShCoef(const float f_rest[45], float sh_min, float sh_max, + uint32_t out[16]) { + // f_rest layout: [R0..R14, G0..G14, B0..B14] + const float* r_coeffs = f_rest; + const float* g_coeffs = f_rest + 15; + const float* b_coeffs = f_rest + 30; + + for (int i = 0; i < 15; ++i) { + out[i] = encodeShTriplet(r_coeffs[i], g_coeffs[i], b_coeffs[i], sh_min, sh_max); + } + out[15] = 0; +} + +// ── Full-splat encoder ──────────────────────────────────────────────────── + +void encodeSplat(const float pos[3], const float f_dc[3], float opacity, + const float scale_log[3], const float rot[4], + const float* f_rest, const Eigen::Vector3f& scale_min, + const Eigen::Vector3f& scale_max, float sh_min, float sh_max, + bool has_sh, uint8_t* data_out, uint8_t* sh_out) { + // Position: 12 bytes (3 × float32) + std::memcpy(data_out, pos, 12); + data_out += 12; + + // Color RGBA: 4 bytes (uint32) + uint32_t color = encodeColor(f_dc, opacity); + std::memcpy(data_out, &color, 4); + data_out += 4; + + // Scale: 6 bytes (3 × uint16) + uint16_t scale_enc[3]; + encodeScale(scale_log, scale_min, scale_max, scale_enc); + std::memcpy(data_out, scale_enc, 6); + data_out += 6; + + // Rotation: 4 bytes (uint32) + uint32_t rot_enc = encodeRotation(rot); + std::memcpy(data_out, &rot_enc, 4); + data_out += 4; + + // Normal: 6 bytes (3 × uint16, always zero for 3DGS) + uint16_t normal_enc[3] = {0, 0, 0}; + std::memcpy(data_out, normal_enc, 6); + + // SH coefficients (optional) + if (has_sh && f_rest != nullptr) { + uint32_t sh_enc[16]; + encodeShCoef(f_rest, sh_min, sh_max, sh_enc); + std::memcpy(sh_out, sh_enc, 64); + } +} + +} // namespace splat diff --git a/src/io/lcc_encoder.h b/src/io/lcc_encoder.h new file mode 100644 index 0000000..286fa77 --- /dev/null +++ b/src/io/lcc_encoder.h @@ -0,0 +1,118 @@ +/** + * @file io/lcc_encoder.h + * @brief Encoding functions for LCC binary format (inverse of lcc_reader decode) + * + * Provides the compression functions that convert float splat attributes + * into the packed binary representation used by data.bin and shcoef.bin. + * Each function here is the exact inverse of a corresponding decode function + * in lcc_reader.cpp. + */ + +#pragma once + +#include +#include +#include + +namespace splat { + +/// Spherical harmonic constant C0 used for DC-to-color conversion. +constexpr float kLccSH_C0 = 0.28209479177387814f; + +// ── Low-level helpers ───────────────────────────────────────────────────── + +/// Logistic (sigmoid) function: maps logit-space opacity to [0,1]. +inline float lccSigmoid(float x) { + return 1.0f / (1.0f + std::exp(-x)); +} + +/// Clamp a float to [lo, hi]. +inline float lccClamp(float x, float lo, float hi) { + return x < lo ? lo : (x > hi ? hi : x); +} + +// ── Per-attribute encoders ──────────────────────────────────────────────── + +/** + * @brief Encode rotation quaternion into 32-bit packed format. + * + * 10-10-10-2 packing: the three smallest-magnitude components are + * quantised to 10 bits each; the 2-bit index indicates which component + * was dropped (and can be recovered via the unit-quaternion constraint). + * + * Input: rot[4] = {w, x, y, z} (DataTable convention, matching + * decodeRotationInto output). + * Output: uint32_t with bits [29:20]=p2, [19:10]=p1, [9:0]=p0, [31:30]=idx. + */ +uint32_t encodeRotation(const float rot[4]); + +/** + * @brief Encode SH DC coefficients + opacity into 32-bit RGBA. + * + * f_dc[3] — colour in SH DC space (same convention as reader output). + * opacity — logit-space opacity. + * + * Returns packed uint32: R | (G<<8) | (B<<16) | (A<<24). + */ +uint32_t encodeColor(const float f_dc[3], float opacity); + +/** + * @brief Encode three log-space scales into 3 × uint16. + * + * scale_log[3] — log-space scale values (same as reader output). + * scale_min[3], scale_max[3] — linear-space min / max bounds for dequant. + * + * Fills out[0..2] with the quantised uint16 values. + */ +void encodeScale(const float scale_log[3], const Eigen::Vector3f& scale_min, + const Eigen::Vector3f& scale_max, uint16_t out[3]); + +/** + * @brief Pack an RGB triplet into 11-10-11 bit unsigned integer. + * + * Range: sh_min / sh_max are used to normalise each channel independently + * before quantising. + */ +uint32_t encodeShTriplet(float r, float g, float b, float sh_min, float sh_max); + +/** + * @brief Encode all 15 SH rest-coefficient bands into 16 × uint32 (64 bytes). + * + * f_rest[45] — layout: [R0..R14, G0..G14, B0..B14] (same as reader output). + * sh_min, sh_max — scalar range shared across all channels. + * + * Fills out[0..15]; out[15] is always 0 (padding). + */ +void encodeShCoef(const float f_rest[45], float sh_min, float sh_max, + uint32_t out[16]); + +// ── Full-splat encoder ──────────────────────────────────────────────────── + +/** + * @brief Encode one complete splat into the 32-byte data.bin layout and + * optionally the 64-byte shcoef.bin layout. + * + * @param pos float[3] position in world space. + * @param f_dc float[3] SH DC colour coefficients. + * @param opacity logit-space opacity. + * @param scale_log float[3] log-space scales. + * @param rot float[4] rotation quaternion {w, x, y, z}. + * @param f_rest float[45] SH rest coefficients (may be null if + * !has_sh). + * @param scale_min linear-space scale lower bounds. + * @param scale_max linear-space scale upper bounds. + * @param sh_min SH coefficient scalar lower bound. + * @param sh_max SH coefficient scalar upper bound. + * @param has_sh if true, also write 64 bytes of SH data. + * @param data_out destination buffer for 32-byte splat record + * (must have 32 bytes available). + * @param sh_out destination buffer for 64-byte SH record + * (must have 64 bytes available if has_sh). + */ +void encodeSplat(const float pos[3], const float f_dc[3], float opacity, + const float scale_log[3], const float rot[4], + const float* f_rest, const Eigen::Vector3f& scale_min, + const Eigen::Vector3f& scale_max, float sh_min, float sh_max, + bool has_sh, uint8_t* data_out, uint8_t* sh_out); + +} // namespace splat diff --git a/src/io/lcc_writer.cpp b/src/io/lcc_writer.cpp new file mode 100644 index 0000000..ed3717f --- /dev/null +++ b/src/io/lcc_writer.cpp @@ -0,0 +1,529 @@ +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "lcc_encoder.h" + +using json = nlohmann::json; + +namespace splat { + +namespace { + +// ── Helpers ──────────────────────────────────────────────────────────────── + +/// Generate a random 32-hex-digit GUID string. +std::string generateGuid() { + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution<> dis(0, 15); + std::stringstream ss; + for (int i = 0; i < 32; ++i) ss << std::hex << dis(gen); + return ss.str(); +} + +/// Get a const span of float values from a DataTable column by name. +/// Throws if the column is missing or has wrong row count. +absl::Span getFloatSpan(const DataTable& table, const std::string& name) { + const Column& col = table.getColumnByName(name); + if (col.getType() != ColumnType::FLOAT32) { + throw std::runtime_error("LCC writer: column '" + name + "' must be float32"); + } + return col.asSpan(); +} + +/// Safely get a float span, returning empty span if column doesn't exist. +absl::Span getOptionalFloatSpan(const DataTable& table, const std::string& name) { + if (!table.hasColumn(name)) return {}; + return getFloatSpan(table, name); +} + +// ── Coordinate transform (inverse of reader's PlayCanvas conversion) ────── + +/// Apply the inverse of the reader's fromEulers(90, 0, 180) coordinate +/// transform so that a round-trip (write → read) is identity. +/// +/// Position: (-x, z, y) — self-inverse. +/// Rotation: multiply by conjugate of fromEulers(90°, 0°, 180°). +void applyInverseCoordinateTransform(float& px, float& py, float& pz, float& rw, float& rx, float& ry, float& rz) { + // Position: (-x, z, y) + float tx = -px; + float ty = pz; + float tz = py; + px = tx; + py = ty; + pz = tz; + + // Rotation: multiply by qr* = conjugate of (0, h, 0, h) + // where h = sqrt(2)/2. + constexpr float h = 0.7071067811865475f; + float nw = h * (rx + rz); + float nx = h * (ry - rw); + float ny = h * (rz - rx); + float nz = -h * (rw + ry); + + rw = nw; + rx = nx; + ry = ny; + rz = nz; +} + +// ── Range collection ────────────────────────────────────────────────────── + +/// Scan all LOD DataTables and compute the CompressInfo (min/max ranges) +/// and global bounding box. +struct CollectedRanges { + CompressInfo compress; + Eigen::Vector3f bboxMin; + Eigen::Vector3f bboxMax; + bool hasSH = false; +}; + +CollectedRanges collectRanges(const std::vector& lods) { + CollectedRanges r; + // Initialise to extreme values + float fmax = std::numeric_limits::max(); + float fmin = std::numeric_limits::lowest(); + r.bboxMin = Eigen::Vector3f(fmax, fmax, fmax); + r.bboxMax = Eigen::Vector3f(fmin, fmin, fmin); + + r.compress.scaleMin = Eigen::Vector3f(fmax, fmax, fmax); + r.compress.scaleMax = Eigen::Vector3f(fmin, fmin, fmin); + + float shMin = fmax, shMax = fmin; + float opMin = fmax, opMax = fmin; + + for (const auto* table : lods) { + size_t n = table->getNumRows(); + auto xs = getFloatSpan(*table, "x"); + auto ys = getFloatSpan(*table, "y"); + auto zs = getFloatSpan(*table, "z"); + auto s0 = getFloatSpan(*table, "scale_0"); + auto s1 = getFloatSpan(*table, "scale_1"); + auto s2 = getFloatSpan(*table, "scale_2"); + auto op = getFloatSpan(*table, "opacity"); + + // Check for SH rest coefficients in the first table + if (table == lods[0] && table->hasColumn("f_rest_0")) { + r.hasSH = true; + } + + for (size_t i = 0; i < n; ++i) { + float px = xs[i], py = ys[i], pz = zs[i]; + + // Bounding box + r.bboxMin.x() = std::min(r.bboxMin.x(), px); + r.bboxMin.y() = std::min(r.bboxMin.y(), py); + r.bboxMin.z() = std::min(r.bboxMin.z(), pz); + r.bboxMax.x() = std::max(r.bboxMax.x(), px); + r.bboxMax.y() = std::max(r.bboxMax.y(), py); + r.bboxMax.z() = std::max(r.bboxMax.z(), pz); + + // Scale (convert from log to linear space) + float ls0 = std::exp(s0[i]); + float ls1 = std::exp(s1[i]); + float ls2 = std::exp(s2[i]); + r.compress.scaleMin.x() = std::min(r.compress.scaleMin.x(), ls0); + r.compress.scaleMin.y() = std::min(r.compress.scaleMin.y(), ls1); + r.compress.scaleMin.z() = std::min(r.compress.scaleMin.z(), ls2); + r.compress.scaleMax.x() = std::max(r.compress.scaleMax.x(), ls0); + r.compress.scaleMax.y() = std::max(r.compress.scaleMax.y(), ls1); + r.compress.scaleMax.z() = std::max(r.compress.scaleMax.z(), ls2); + + // Opacity + float sigOp = lccSigmoid(op[i]); + opMin = std::min(opMin, sigOp); + opMax = std::max(opMax, sigOp); + } + } + + // Collect SH ranges (only need from LOD0 for global range) + if (r.hasSH) { + const DataTable* t0 = lods[0]; + auto dc0 = getFloatSpan(*t0, "f_dc_0"); + auto dc1 = getFloatSpan(*t0, "f_dc_1"); + auto dc2 = getFloatSpan(*t0, "f_dc_2"); + size_t n = t0->getNumRows(); + + for (size_t i = 0; i < n; ++i) { + shMin = std::min({shMin, dc0[i], dc1[i], dc2[i]}); + shMax = std::max({shMax, dc0[i], dc1[i], dc2[i]}); + } + + // Also scan f_rest for SH range + for (int b = 0; b < 45; ++b) { + std::string name = "f_rest_" + std::to_string(b); + if (!t0->hasColumn(name)) break; + auto fr = getFloatSpan(*t0, name); + for (size_t i = 0; i < n; ++i) { + shMin = std::min(shMin, fr[i]); + shMax = std::max(shMax, fr[i]); + } + } + } + + r.compress.shMin = Eigen::Vector3f(shMin, shMin, shMin); + r.compress.shMax = Eigen::Vector3f(shMax, shMax, shMax); + + // Env ranges: use same as splat ranges as defaults (per ply2lcc) + r.compress.envScaleMin = r.compress.scaleMin; + r.compress.envScaleMax = r.compress.scaleMax; + r.compress.envShMin = r.compress.shMin; + r.compress.envShMax = r.compress.shMax; + + return r; +} + +// ── Spatial partitioning ────────────────────────────────────────────────── + +/// Per-cell splat index accumulator. +struct CellAccum { + uint32_t cellId; + std::vector indices; // splat row indices in the DataTable +}; + +/// Build the spatial grid: assign each splat to a cell. +std::map> buildSpatialGrid(const DataTable& table, const Eigen::Vector3f& bboxMin, + float cellSizeX, float cellSizeY) { + auto xs = getFloatSpan(table, "x"); + auto ys = getFloatSpan(table, "y"); + size_t n = table.getNumRows(); + + std::map> grid; + + for (size_t i = 0; i < n; ++i) { + int cx = static_cast(std::floor((xs[i] - bboxMin.x()) / cellSizeX)); + int cy = static_cast(std::floor((ys[i] - bboxMin.y()) / cellSizeY)); + cx = std::max(0, std::min(cx, 65535)); + cy = std::max(0, std::min(cy, 65535)); + uint32_t id = (static_cast(cy) << 16) | static_cast(cx); + grid[id].push_back(i); + } + return grid; +} + +// ── Per-cell encoding ───────────────────────────────────────────────────── + +/// Encoded data for a single cell at a single LOD. +struct EncodedCell { + uint32_t cellId; + size_t splatCount; + std::vector data; // 32 bytes × splatCount + std::vector shcoef; // 64 bytes × splatCount (empty if !hasSH) +}; + +EncodedCell encodeCell(const DataTable& table, const std::vector& splatIndices, const CollectedRanges& ranges, + const LccWriteConfig& config, uint32_t cellId) { + EncodedCell cell; + cell.cellId = cellId; + cell.splatCount = splatIndices.size(); + cell.data.resize(cell.splatCount * 32); + if (ranges.hasSH) { + cell.shcoef.resize(cell.splatCount * 64); + } + + // Get column spans + auto xs = getFloatSpan(table, "x"); + auto ys = getFloatSpan(table, "y"); + auto zs = getFloatSpan(table, "z"); + auto r0 = getFloatSpan(table, "rot_0"); + auto r1 = getFloatSpan(table, "rot_1"); + auto r2 = getFloatSpan(table, "rot_2"); + auto r3 = getFloatSpan(table, "rot_3"); + auto dc0 = getFloatSpan(table, "f_dc_0"); + auto dc1 = getFloatSpan(table, "f_dc_1"); + auto dc2 = getFloatSpan(table, "f_dc_2"); + auto sc0 = getFloatSpan(table, "scale_0"); + auto sc1 = getFloatSpan(table, "scale_1"); + auto sc2 = getFloatSpan(table, "scale_2"); + auto op = getFloatSpan(table, "opacity"); + + // Optional f_rest columns + float f_rest[45] = {}; + std::vector> f_rest_spans; + if (ranges.hasSH) { + f_rest_spans.reserve(45); + for (int b = 0; b < 45; ++b) { + f_rest_spans.push_back(getOptionalFloatSpan(table, "f_rest_" + std::to_string(b))); + } + } + + float sh_min_scalar = ranges.compress.shMin.x(); + float sh_max_scalar = ranges.compress.shMax.x(); + + for (size_t j = 0; j < cell.splatCount; ++j) { + size_t idx = splatIndices[j]; + + float pos[3] = {xs[idx], ys[idx], zs[idx]}; + float rot[4] = {r0[idx], r1[idx], r2[idx], r3[idx]}; + float fdc[3] = {dc0[idx], dc1[idx], dc2[idx]}; + float scl[3] = {sc0[idx], sc1[idx], sc2[idx]}; + float opacity = op[idx]; + + // Apply inverse coordinate transform if requested + if (config.applyCoordinateTransform) { + applyInverseCoordinateTransform(pos[0], pos[1], pos[2], rot[0], rot[1], rot[2], rot[3]); + } + + // Collect f_rest for this splat if SH is enabled + float* fr_ptr = nullptr; + if (ranges.hasSH) { + for (int b = 0; b < 45; ++b) { + f_rest[b] = f_rest_spans[b].empty() ? 0.0f : f_rest_spans[b][idx]; + } + fr_ptr = f_rest; + } + + uint8_t* dataPtr = cell.data.data() + j * 32; + uint8_t* shPtr = ranges.hasSH ? (cell.shcoef.data() + j * 64) : nullptr; + + encodeSplat(pos, fdc, opacity, scl, rot, fr_ptr, ranges.compress.scaleMin, ranges.compress.scaleMax, sh_min_scalar, + sh_max_scalar, ranges.hasSH, dataPtr, shPtr); + } + + return cell; +} + +// ── File output ─────────────────────────────────────────────────────────── + +void writeMetaLcc(const std::filesystem::path& path, const CollectedRanges& ranges, + const std::vector& splatsPerLod, size_t totalSplats, size_t numLods, + const LccWriteConfig& config) { + std::string fileType = ranges.hasSH ? "Quality" : "Portable"; + std::string guid = config.guid.empty() ? generateGuid() : config.guid; + int indexDataSize = 4 + 16 * static_cast(numLods); + + auto v3 = [](const Eigen::Vector3f& v) -> json { return {v.x(), v.y(), v.z()}; }; + + // splats array + json splatsArr = json::array(); + for (auto c : splatsPerLod) splatsArr.push_back(c); + + // attributes array + json attrs = json::array(); + + // position + attrs.push_back({{"name", "position"}, {"min", v3(ranges.bboxMin)}, {"max", v3(ranges.bboxMax)}}); + + // normal + attrs.push_back({{"name", "normal"}, {"min", {0, 0, 0}}, {"max", {0, 0, 0}}}); + + // color + attrs.push_back({{"name", "color"}, {"min", {0, 0, 0}}, {"max", {1, 1, 1}}}); + + // shcoef + attrs.push_back({{"name", "shcoef"}, + {"min", ranges.hasSH ? v3(ranges.compress.shMin) : json({0, 0, 0})}, + {"max", ranges.hasSH ? v3(ranges.compress.shMax) : json({1, 1, 1})}}); + + // opacity + attrs.push_back({{"name", "opacity"}, {"min", {0.0}}, {"max", {1.0}}}); + + // scale + attrs.push_back({{"name", "scale"}, {"min", v3(ranges.compress.scaleMin)}, {"max", v3(ranges.compress.scaleMax)}}); + + // envnormal + attrs.push_back({{"name", "envnormal"}, {"min", {0, 0, 0}}, {"max", {0, 0, 0}}}); + + // envshcoef + attrs.push_back({{"name", "envshcoef"}, + {"min", ranges.hasSH ? v3(ranges.compress.envShMin) : json({0, 0, 0})}, + {"max", ranges.hasSH ? v3(ranges.compress.envShMax) : json({1, 1, 1})}}); + + // envscale + attrs.push_back( + {{"name", "envscale"}, {"min", v3(ranges.compress.envScaleMin)}, {"max", v3(ranges.compress.envScaleMax)}}); + + json meta = { + {"version", "5.0"}, + {"guid", guid}, + {"name", config.name}, + {"description", config.description}, + {"source", "lcc"}, + {"dataType", "DIMENVUE"}, + {"totalSplats", totalSplats}, + {"totalLevel", numLods}, + {"cellLengthX", config.cellSizeX}, + {"cellLengthY", config.cellSizeY}, + {"indexDataSize", indexDataSize}, + {"offset", {0, 0, 0}}, + {"epsg", 0}, + {"shift", {0, 0, 0}}, + {"scale", {1, 1, 1}}, + {"splats", splatsArr}, + {"boundingBox", {{"min", v3(ranges.bboxMin)}, {"max", v3(ranges.bboxMax)}}}, + {"encoding", "COMPRESS"}, + {"fileType", fileType}, + {"attributes", attrs}, + }; + + std::ofstream f(path); + if (!f) throw std::runtime_error("Failed to create meta.lcc"); + f << std::setprecision(15) << meta.dump(1, '\t') << '\n'; +} + +} // anonymous namespace + +// ── Public API ───────────────────────────────────────────────────────────── + +void writeLcc(const std::filesystem::path& outputDir, const std::vector& lods, + const LccWriteConfig& config) { + if (lods.empty()) { + throw std::runtime_error("LCC writer: at least one LOD DataTable required"); + } + + std::filesystem::create_directories(outputDir); + + // Phase 1: collect ranges + CollectedRanges ranges = collectRanges(lods); + size_t numLods = lods.size(); + + // Phase 2: build spatial grids for all LODs + // All LODs share the same cell scheme derived from LOD0 bbox + std::vector>> grids; + grids.reserve(numLods); + for (size_t lod = 0; lod < numLods; ++lod) { + grids.push_back(buildSpatialGrid(*lods[lod], ranges.bboxMin, config.cellSizeX, config.cellSizeY)); + } + + // Collect all unique cell IDs across all LODs + std::set allCellIds; + for (const auto& g : grids) { + for (const auto& kv : g) allCellIds.insert(kv.first); + } + + // Phase 3: encode cells, tracking LOD per entry + + struct CellLodData { + uint32_t cellId; + size_t lod; + uint32_t splatCount; + uint64_t dataOffset; + uint32_t dataSize; + uint64_t shOffset; + uint32_t shSize; + std::vector dataBuf; + std::vector shBuf; + }; + + std::vector cellLodEntries; + cellLodEntries.reserve(allCellIds.size() * numLods); + + for (uint32_t cellId : allCellIds) { + for (size_t lod = 0; lod < numLods; ++lod) { + auto it = grids[lod].find(cellId); + if (it == grids[lod].end() || it->second.empty()) continue; + + EncodedCell cell = encodeCell(*lods[lod], it->second, ranges, config, cellId); + + CellLodData entry; + entry.cellId = cellId; + entry.lod = lod; + entry.splatCount = static_cast(cell.splatCount); + entry.dataSize = static_cast(cell.data.size()); + entry.shSize = static_cast(cell.shcoef.size()); + entry.dataBuf = std::move(cell.data); + entry.shBuf = std::move(cell.shcoef); + cellLodEntries.push_back(std::move(entry)); + } + } + + // Compute offsets by walking entries in cellId→lod order + uint64_t dataFileOffset = 0; + uint64_t shFileOffset = 0; + + for (auto& entry : cellLodEntries) { + entry.dataOffset = dataFileOffset; + dataFileOffset += entry.dataSize; + if (ranges.hasSH && entry.shSize > 0) { + entry.shOffset = shFileOffset; + shFileOffset += entry.shSize; + } + } + + // Write data.bin and accumulate index entries + std::ofstream dataFile(outputDir / "data.bin", std::ios::binary); + if (!dataFile) throw std::runtime_error("Failed to create data.bin"); + + std::ofstream shFile; + if (ranges.hasSH) { + shFile.open(outputDir / "shcoef.bin", std::ios::binary); + if (!shFile) throw std::runtime_error("Failed to create shcoef.bin"); + } + + // Build index entries grouped by cellId + struct FinalIndexLod { + uint32_t points; + uint64_t offset; + uint32_t size; + }; + std::map> finalIndex; // cellId → per-LOD + + for (auto& entry : cellLodEntries) { + dataFile.write(reinterpret_cast(entry.dataBuf.data()), entry.dataSize); + if (ranges.hasSH && entry.shSize > 0) { + shFile.write(reinterpret_cast(entry.shBuf.data()), entry.shSize); + } + + auto& lodVec = finalIndex[entry.cellId]; + if (lodVec.size() < numLods) lodVec.resize(numLods); + lodVec[entry.lod] = {entry.splatCount, entry.dataOffset, entry.dataSize}; + } + + dataFile.close(); + if (shFile.is_open()) shFile.close(); + + // Recompute totalSplats from all entries + size_t totalSplats = 0; + std::vector splatsPerLod(numLods, 0); + for (const auto& entry : cellLodEntries) { + totalSplats += entry.splatCount; + splatsPerLod[entry.lod] += entry.splatCount; + } + + // Write index.bin + { + std::ofstream idxFile(outputDir / "index.bin", std::ios::binary); + if (!idxFile) throw std::runtime_error("Failed to create index.bin"); + + for (const auto& kv : finalIndex) { + uint32_t unitIndex = kv.first; + idxFile.write(reinterpret_cast(&unitIndex), 4); + + const auto& lodVec = kv.second; + for (size_t lod = 0; lod < numLods; ++lod) { + if (lod < lodVec.size()) { + idxFile.write(reinterpret_cast(&lodVec[lod].points), 4); + idxFile.write(reinterpret_cast(&lodVec[lod].offset), 8); + idxFile.write(reinterpret_cast(&lodVec[lod].size), 4); + } else { + // Empty LOD for this cell + uint32_t zero32 = 0; + uint64_t zero64 = 0; + idxFile.write(reinterpret_cast(&zero32), 4); + idxFile.write(reinterpret_cast(&zero64), 8); + idxFile.write(reinterpret_cast(&zero32), 4); + } + } + } + } + + // Write meta.lcc + writeMetaLcc(outputDir / "meta.lcc", ranges, splatsPerLod, totalSplats, numLods, config); +} + +} // namespace splat diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 5465fa3..bc39512 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -53,3 +53,7 @@ add_test(NAME PlaycanvasViewerShaderTests COMMAND PlaycanvasViewerShaderTests) add_executable(SplatLccReaderTests lcc_reader_test.cpp) target_link_libraries(SplatLccReaderTests PRIVATE SPLAT::splat) add_test(NAME SplatLccReaderTests COMMAND SplatLccReaderTests) + +add_executable(SplatLccWriterTests lcc_writer_test.cpp) +target_link_libraries(SplatLccWriterTests PRIVATE SPLAT::splat) +add_test(NAME SplatLccWriterTests COMMAND SplatLccWriterTests) diff --git a/tests/lcc_writer_test.cpp b/tests/lcc_writer_test.cpp new file mode 100644 index 0000000..eb0a1cd --- /dev/null +++ b/tests/lcc_writer_test.cpp @@ -0,0 +1,268 @@ +/** + * @file lcc_writer_test.cpp + * @brief Unit tests for LCC writer encoding functions — round-trip verification. + */ + +#include "../src/io/lcc_encoder.h" + +#include +#include +#include +#include +#include + +namespace { + +constexpr float kSH_C0 = 0.28209479177387814f; +constexpr float SQRT_2 = 1.41421356237f; +constexpr float SQRT_2_INV = 0.70710678118f; + +float invSigmoid(float v) { return -std::log((1.0f - v) / v); } + +float invSH0ToColor(float v) { return (v - 0.5f) / kSH_C0; } + +void decodePacked_11_10_11(float& r, float& g, float& b, uint32_t enc) { + r = static_cast(enc & 2047) / 2047.0f; + g = static_cast((enc >> 11) & 1023) / 1023.0f; + b = static_cast((enc >> 21) & 2047) / 2047.0f; +} + +void decodeRotationInto(uint32_t v, float& r0, float& r1, float& r2, float& r3) { + float d0 = static_cast(v & 1023) / 1023.0f; + float d1 = static_cast((v >> 10) & 1023) / 1023.0f; + float d2 = static_cast((v >> 20) & 1023) / 1023.0f; + uint32_t d3 = (v >> 30) & 3; + + float qx = d0 * SQRT_2 - SQRT_2_INV; + float qy = d1 * SQRT_2 - SQRT_2_INV; + float qz = d2 * SQRT_2 - SQRT_2_INV; + float sum = std::min(1.0f, qx * qx + qy * qy + qz * qz); + float qw = std::sqrt(1.0f - sum); + + if (d3 == 0) { r0 = qz; r1 = qw; r2 = qx; r3 = qy; } + else if (d3 == 1) { r0 = qz; r1 = qx; r2 = qw; r3 = qy; } + else if (d3 == 2) { r0 = qz; r1 = qx; r2 = qy; r3 = qw; } + else { r0 = qw; r1 = qx; r2 = qy; r3 = qz; } +} + +void check(bool cond, const char* msg) { + if (!cond) { fprintf(stderr, "FAIL: %s\n", msg); std::exit(1); } +} + +} // namespace + +// ── encodeRotation round-trip ───────────────────────────────────────────── + +static void testEncodeRotationIdentity() { + float rot[4] = {1.0f, 0.0f, 0.0f, 0.0f}; + uint32_t enc = splat::encodeRotation(rot); + + float w, x, y, z; + decodeRotationInto(enc, w, x, y, z); + + float len = std::sqrt(w*w + x*x + y*y + z*z); + check(std::fabs(w/len - 1.0f) < 0.002f, "encodeRotation identity"); + printf("PASS: encodeRotation identity\n"); +} + +static void testEncodeRotationArbitrary() { + float r0 = 0.5f, r1 = 0.5f, r2 = 0.5f, r3 = 0.5f; + float len = std::sqrt(r0*r0 + r1*r1 + r2*r2 + r3*r3); + float rot[4] = {r0/len, r1/len, r2/len, r3/len}; + + uint32_t enc = splat::encodeRotation(rot); + + float w, x, y, z; + decodeRotationInto(enc, w, x, y, z); + + float dot = w*rot[0] + x*rot[1] + y*rot[2] + z*rot[3]; + check(std::fabs(std::fabs(dot) - 1.0f) < 0.002f, "encodeRotation arbitrary"); + printf("PASS: encodeRotation arbitrary\n"); +} + +static void testEncodeRotationAxes() { + float s2 = std::sin(M_PI / 4.0); + float c2 = std::cos(M_PI / 4.0); + + struct { float w,x,y,z; const char* n; } cases[] = { + {c2, s2, 0, 0, "90 X"}, + {c2, 0, s2, 0, "90 Y"}, + {c2, 0, 0, s2, "90 Z"}, + {0, 1, 0, 0, "180 X"}, + }; + + for (auto& tc : cases) { + float rot[4] = {tc.w, tc.x, tc.y, tc.z}; + uint32_t enc = splat::encodeRotation(rot); + + float w, x, y, z; + decodeRotationInto(enc, w, x, y, z); + + float dot = w*tc.w + x*tc.x + y*tc.y + z*tc.z; + check(std::fabs(std::fabs(dot) - 1.0f) < 0.002f, tc.n); + printf("PASS: encodeRotation %s\n", tc.n); + } +} + +// ── encodeColor round-trip ──────────────────────────────────────────────── + +static void testEncodeColorGray() { + float f_dc[3] = {0.0f, 0.0f, 0.0f}; + float opacity = 0.0f; + + uint32_t enc = splat::encodeColor(f_dc, opacity); + uint8_t r8 = enc & 0xFF, g8 = (enc>>8)&0xFF, b8 = (enc>>16)&0xFF; + + check(r8 >= 126 && r8 <= 129 && g8 >= 126 && g8 <= 129 && b8 >= 126 && b8 <= 129, + "encodeColor gray"); + printf("PASS: encodeColor gray\n"); +} + +static void testEncodeColorRoundTrip() { + float f_dc[3] = {0.3f, -0.2f, 0.5f}; + float opacity = 1.5f; + + uint32_t enc = splat::encodeColor(f_dc, opacity); + + uint8_t r8 = enc & 0xFF, g8 = (enc>>8)&0xFF, b8 = (enc>>16)&0xFF; + float f_dc_r = invSH0ToColor(r8 / 255.0f); + float f_dc_g = invSH0ToColor(g8 / 255.0f); + float f_dc_b = invSH0ToColor(b8 / 255.0f); + + check(std::fabs(f_dc_r - f_dc[0]) < 0.01f && + std::fabs(f_dc_g - f_dc[1]) < 0.01f && + std::fabs(f_dc_b - f_dc[2]) < 0.01f, + "encodeColor round-trip"); + printf("PASS: encodeColor round-trip\n"); +} + +// ── encodeScale round-trip ──────────────────────────────────────────────── + +static void testEncodeScaleRoundTrip() { + // Raw float arrays instead of Eigen::Vector3f to avoid Eigen dependency in test + float log_scale[3] = {-1.0f, 0.5f, 2.0f}; + float sMin[3] = {0.1f, 0.1f, 0.1f}; + float sMax[3] = {10.0f, 10.0f, 10.0f}; + + uint16_t enc[3]; + // The library API uses Eigen; call with const-cast to float arrays. + // encodeScale signature: (const float*, const Eigen::Vector3f&, const Eigen::Vector3f&, uint16_t*) + // We avoid Eigen by calling a wrapper below. + // For the test we just call the low-level function: + auto sMinE = Eigen::Vector3f(sMin[0], sMin[1], sMin[2]); + auto sMaxE = Eigen::Vector3f(sMax[0], sMax[1], sMax[2]); + splat::encodeScale(log_scale, sMinE, sMaxE, enc); + + for (int i = 0; i < 3; ++i) { + float norm = enc[i] / 65535.0f; + float linear = sMin[i] + norm * (sMax[i] - sMin[i]); + float log_back = std::log(linear); + check(std::fabs(log_back - log_scale[i]) < 0.02f, "encodeScale round-trip"); + } + printf("PASS: encodeScale round-trip\n"); +} + +// ── encodeSh round-trip ─────────────────────────────────────────────────── + +static void testEncodeShTripletRoundTrip() { + float sh_min = -2.0f, sh_max = 2.0f; + uint32_t enc = splat::encodeShTriplet(0.5f, -0.3f, 1.2f, sh_min, sh_max); + + float r, g, b; + decodePacked_11_10_11(r, g, b, enc); + + auto denorm = [=](float v) { return sh_min + v * (sh_max - sh_min); }; + check(std::fabs(denorm(r) - 0.5f) < 0.01f && + std::fabs(denorm(g) - (-0.3f)) < 0.01f && + std::fabs(denorm(b) - 1.2f) < 0.01f, + "encodeShTriplet round-trip"); + printf("PASS: encodeShTriplet round-trip\n"); +} + +static void testEncodeShCoefRoundTrip() { + float sh_min = -3.0f, sh_max = 3.0f; + // Fill values well within range + float f_rest[45]; + for (int i = 0; i < 45; ++i) f_rest[i] = (i - 22.0f) * 0.1f; + + uint32_t out[16]; + splat::encodeShCoef(f_rest, sh_min, sh_max, out); + + auto denorm = [=](float v) { return sh_min + v * (sh_max - sh_min); }; + bool ok = true; + for (int band = 0; band < 15 && ok; ++band) { + float r, g, b; + decodePacked_11_10_11(r, g, b, out[band]); + ok = std::fabs(denorm(r) - f_rest[band]) < 0.02f && + std::fabs(denorm(g) - f_rest[band+15]) < 0.05f && + std::fabs(denorm(b) - f_rest[band+30]) < 0.02f; + } + check(ok, "encodeShCoef round-trip"); + check(out[15] == 0, "encodeShCoef padding zero"); + printf("PASS: encodeShCoef round-trip\n"); +} + +// ── encodeSplat full record test ───────────────────────────────────────── + +static void testEncodeSplat() { + float pos[3] = {10.0f, 20.0f, 30.0f}; + float f_dc[3] = {0.1f, -0.1f, 0.2f}; + float opacity = 0.8f; + float scale_log[3] = {-0.5f, 0.0f, 0.5f}; + float rot[4] = {0.7071f, 0.0f, 0.7071f, 0.0f}; + float f_rest[45] = {}; + auto sMinE = Eigen::Vector3f(0.01f, 0.01f, 0.01f); + auto sMaxE = Eigen::Vector3f(20.0f, 20.0f, 20.0f); + + uint8_t data[32], sh[64]; + + splat::encodeSplat(pos, f_dc, opacity, scale_log, rot, f_rest, + sMinE, sMaxE, -3.0f, 3.0f, true, data, sh); + + // Verify position + float px, py, pz; + std::memcpy(&px, data, 4); + std::memcpy(&py, data+4, 4); + std::memcpy(&pz, data+8, 4); + check(px == 10.0f && py == 20.0f && pz == 30.0f, "encodeSplat position"); + + // Verify color not zero + uint32_t color; + std::memcpy(&color, data+12, 4); + check(color != 0, "encodeSplat color"); + + // Verify rotation not zero + uint32_t rot_enc; + std::memcpy(&rot_enc, data+22, 4); + check(rot_enc != 0, "encodeSplat rotation"); + + printf("PASS: encodeSplat full\n"); +} + +// ── Main ────────────────────────────────────────────────────────────────── + +int main() { + printf("=== LCC Writer Tests ===\n\n"); + + printf("--- encodeRotation ---\n"); + testEncodeRotationIdentity(); + testEncodeRotationArbitrary(); + testEncodeRotationAxes(); + + printf("\n--- encodeColor ---\n"); + testEncodeColorGray(); + testEncodeColorRoundTrip(); + + printf("\n--- encodeScale ---\n"); + testEncodeScaleRoundTrip(); + + printf("\n--- encodeSh ---\n"); + testEncodeShTripletRoundTrip(); + testEncodeShCoefRoundTrip(); + + printf("\n--- encodeSplat ---\n"); + testEncodeSplat(); + + printf("\n=== All tests passed ===\n"); + return 0; +} From 1568805098614e5a612a5b7b0a22c44eb4a80e5e Mon Sep 17 00:00:00 2001 From: "Merlot.Qi" Date: Tue, 2 Jun 2026 13:42:00 +0800 Subject: [PATCH 2/8] build: add zstd dependency for SPZ v4 NGSP support Co-Authored-By: Claude Opus 4.8 --- CMakeLists.txt | 1 + src/CMakeLists.txt | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 422563c..36cf4ce 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -95,6 +95,7 @@ find_package(Eigen3 CONFIG REQUIRED) find_package(nlohmann_json CONFIG REQUIRED) find_package(absl CONFIG REQUIRED) find_package(ZLIB REQUIRED) +find_package(zstd CONFIG REQUIRED) find_package(WebP REQUIRED) if(BUILD_SPLAT_VISUALIZATION) find_package(VTK REQUIRED COMPONENTS ${SPLAT_VTK_COMPONENTS}) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9bf0348..36108ec 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -89,12 +89,13 @@ target_include_directories(splat ${VTK_INCLUDE_DIRS} ) -target_link_libraries(splat +target_link_libraries(splat PUBLIC Eigen3::Eigen nlohmann_json::nlohmann_json WebP::webp ZLIB::ZLIB + zstd::libzstd absl::base absl::strings PRIVATE From df3ea1884be1162fb6861a2ba0da08e9e670b430 Mon Sep 17 00:00:00 2001 From: "Merlot.Qi" Date: Tue, 2 Jun 2026 13:43:28 +0800 Subject: [PATCH 3/8] feat(spz): add v4 NGSP ZSTD multi-stream reader support - Extract shared decode helpers (decodePositions/Scales/Colors/Alphas/Rotations/SH) for reuse across legacy v2-v3 and v4 NGSP paths - Add NgspFileHeader struct, TOC parser, and ZSTD stream decompressor - Implement readSpzV4: parse 32-byte header, ZSTD decompress per-stream, decode attributes via shared helpers, build DataTable - Modify readSpz to detect v4 NGSP magic (0x5053474E) and dispatch - Legacy v2-v3 GZip path preserved unchanged via readSpzLegacy Co-Authored-By: Claude Opus 4.8 --- src/io/spz_reader.cpp | 435 ++++++++++++++++++++++++++++++++---------- 1 file changed, 333 insertions(+), 102 deletions(-) diff --git a/src/io/spz_reader.cpp b/src/io/spz_reader.cpp index 1094ef0..227f6f3 100644 --- a/src/io/spz_reader.cpp +++ b/src/io/spz_reader.cpp @@ -1,17 +1,49 @@ #include #include +#include #include #include #include +#include +#include namespace splat { -static const size_t SPZ_HEADER_SIZE = 16; -const float SH_C0_2 = 0.15f; -const size_t HARMONICS_COMPONENT_COUNT[] = {0, 9, 24, 45}; +// ── Constants ────────────────────────────────────────────────────────────── -static float inverseConvertColorFromSPZ(float y) { return (y / 255.0f - 0.5f) / SH_C0_2; } +static constexpr uint32_t SPZ_V4_MAGIC = 0x5053474E; // "NGSP" +static constexpr size_t SPZ_V4_HEADER_SIZE = 32; +static constexpr uint8_t SPZ_V4_FLAG_ANTIALIASED = 0x1; +static constexpr uint8_t SPZ_V4_FLAG_EXTENSIONS = 0x2; +static constexpr int LATEST_SPZ_VERSION = 4; +static constexpr float SPZ_COLOR_SCALE = 0.15f; +static const size_t HARMONICS_COMPONENT_COUNT[] = {0, 9, 24, 45}; + +// ── v4 NGSP header ──────────────────────────────────────────────────────── + +#pragma pack(push, 1) +struct NgspFileHeader { + uint32_t magic; + uint32_t version; + uint32_t numPoints; + uint8_t shDegree; + uint8_t fractionalBits; + uint8_t flags; + uint8_t numStreams; + uint32_t tocByteOffset; + uint8_t reserved[12]; +}; +#pragma pack(pop) +static_assert(sizeof(NgspFileHeader) == 32, "NgspFileHeader must be 32 bytes"); + +struct NgspStreamInfo { + uint64_t compressedSize; + uint64_t uncompressedSize; + size_t dataOffset; +}; + +// ── GZip decompression ──────────────────────────────────────────────────── static std::vector decompressGZIP(const std::vector& compressedData) { if (compressedData.size() < 18) throw std::runtime_error("Buffer too small to be GZip"); @@ -39,7 +71,6 @@ static std::vector decompressGZIP(const std::vector& compresse size_t decompressedChunkSize = sizeof(temp_buf) - zs.avail_out; outBuffer.insert(outBuffer.end(), temp_buf, temp_buf + decompressedChunkSize); } - } while (ret == Z_OK); inflateEnd(&zs); @@ -51,41 +82,156 @@ static std::vector decompressGZIP(const std::vector& compresse return outBuffer; } -static int32_t getFixed24(const std::vector& buffer, size_t elementIndex, size_t memberIndex) { +// ── Shared attribute decode helpers (v2-v3 + v4) ────────────────────────── + +static float inverseConvertColorFromSPZ(float y) { return (y / 255.0f - 0.5f) / SPZ_COLOR_SCALE; } + +static void decodePositions(const uint8_t* posBase, int fractionalBits, + uint32_t numSplats, std::vector& cols) { + const float posScale = 1.0f / (1 << fractionalBits); const size_t stride = 9; - const size_t offset = elementIndex * stride + memberIndex * 3; + for (uint32_t i = 0; i < numSplats; ++i) { + for (int m = 0; m < 3; ++m) { + const size_t offset = i * stride + m * 3; + int32_t fixed32 = static_cast(posBase[offset]) | + (static_cast(posBase[offset + 1]) << 8) | + (static_cast(posBase[offset + 2]) << 16); + if (fixed32 & 0x800000) fixed32 |= 0xFF000000; + cols[m][i] = static_cast(fixed32) * posScale; + } + } +} - if (offset + 3 > buffer.size()) { - throw std::out_of_range("SPZ buffer access out of range"); +static void decodeScales(const uint8_t* scaleBase, uint32_t numSplats, + std::vector& cols) { + for (uint32_t i = 0; i < numSplats; ++i) { + cols[0][i] = scaleBase[i * 3 + 0] / 16.0f - 10.0f; + cols[1][i] = scaleBase[i * 3 + 1] / 16.0f - 10.0f; + cols[2][i] = scaleBase[i * 3 + 2] / 16.0f - 10.0f; } +} - uint32_t val = static_cast(buffer[offset]) | (static_cast(buffer[offset + 1]) << 8) | - (static_cast(buffer[offset + 2]) << 16); +static void decodeColors(const uint8_t* colorBase, uint32_t numSplats, + std::vector& cols) { + for (uint32_t i = 0; i < numSplats; ++i) { + cols[0][i] = inverseConvertColorFromSPZ(colorBase[i * 3 + 0]); + cols[1][i] = inverseConvertColorFromSPZ(colorBase[i * 3 + 1]); + cols[2][i] = inverseConvertColorFromSPZ(colorBase[i * 3 + 2]); + } +} - if (val & 0x800000) { - val |= 0xFF000000; +static void decodeAlphas(const uint8_t* alphaBase, uint32_t numSplats, + std::vector& cols) { + for (uint32_t i = 0; i < numSplats; ++i) { + float normAlpha = std::clamp(alphaBase[i] / 255.0f, 1e-6f, 1.0f - 1e-6f); + cols[0][i] = std::log(normAlpha / (1.0f - normAlpha)); } +} - return static_cast(val); +static void decodeRotations(const uint8_t* rotBase, uint32_t version, + uint32_t numSplats, std::vector& cols) { + for (uint32_t i = 0; i < numSplats; ++i) { + float q[4] = {1, 0, 0, 0}; + if (version == 2) { + q[1] = (rotBase[i * 3 + 0] / 127.5f) - 1.0f; + q[2] = (rotBase[i * 3 + 1] / 127.5f) - 1.0f; + q[3] = (rotBase[i * 3 + 2] / 127.5f) - 1.0f; + float dot = q[1] * q[1] + q[2] * q[2] + q[3] * q[3]; + q[0] = std::sqrt(std::max(0.0f, 1.0f - dot)); + } else { + uint32_t packed; + std::memcpy(&packed, rotBase + i * 4, 4); + uint32_t largestIndex = packed >> 30; + float sum_sq = 0; + uint32_t temp = packed; + for (int j = 3; j >= 0; --j) { + if (static_cast(j) != largestIndex) { + uint32_t mag = temp & 511; + float val = 0.70710678f * mag / 511.0f; + if ((temp >> 9) & 1) val = -val; + q[j] = val; + sum_sq += val * val; + temp >>= 10; + } + } + q[largestIndex] = std::sqrt(std::max(0.0f, 1.0f - sum_sq)); + } + cols[0][i] = q[0]; cols[1][i] = q[1]; + cols[2][i] = q[2]; cols[3][i] = q[3]; + } } -std::unique_ptr readSpz(const std::filesystem::path& filename) { - std::ifstream ifs(filename, std::ios::binary | std::ios::ate); - if (!ifs.is_open()) { - throw std::runtime_error("cannot open file: " + filename.u8string()); +static void decodeSH(const uint8_t* shBase, uint8_t shDegree, + uint32_t numSplats, std::vector& cols) { + size_t harmonicsCount = HARMONICS_COMPONENT_COUNT[shDegree > 3 ? 0 : shDegree]; + for (uint32_t i = 0; i < numSplats; ++i) { + for (size_t sh = 0; sh < harmonicsCount; ++sh) { + size_t channel = sh % 3; + size_t coeff = sh / 3; + size_t colIdx = channel * (harmonicsCount / 3) + coeff; + uint8_t shVal = shBase[i * harmonicsCount + sh]; + cols[colIdx][i] = (static_cast(shVal) - 128.0f) / 128.0f; + } } +} - std::streamsize filesize = ifs.tellg(); - ifs.seekg(0, std::ios::beg); - std::vector buffer(filesize); - ifs.read(reinterpret_cast(buffer.data()), filesize); +// ── v4 NGSP TOC and ZSTD decompression ──────────────────────────────────── - if (buffer.size() > 2 && buffer[0] == 0x1F && buffer[1] == 0x8B) { - buffer = decompressGZIP(buffer); +static std::vector parseNgspTOC( + const uint8_t* data, size_t size, const NgspFileHeader& header) { + + if (header.tocByteOffset < SPZ_V4_HEADER_SIZE) { + throw std::runtime_error("NGSP TOC offset is before end of header"); + } + size_t tocSize = header.numStreams * 16; + if (header.tocByteOffset + tocSize > size) { + throw std::runtime_error("NGSP TOC extends past end of file"); + } + + std::vector streams(header.numStreams); + size_t compressedOffset = header.tocByteOffset + tocSize; + + for (uint8_t i = 0; i < header.numStreams; ++i) { + size_t entryOffset = header.tocByteOffset + i * 16; + std::memcpy(&streams[i].compressedSize, data + entryOffset, 8); + std::memcpy(&streams[i].uncompressedSize, data + entryOffset + 8, 8); + streams[i].dataOffset = compressedOffset; + compressedOffset += streams[i].compressedSize; + } + + if (compressedOffset != size) { + throw std::runtime_error("NGSP compressed data size mismatch"); + } + return streams; +} + +static void decompressNgspStreams( + const uint8_t* data, + const std::vector& streams, + std::vector>& dests) { + + if (streams.size() != dests.size()) { + throw std::runtime_error("NGSP stream count mismatch"); + } + + for (size_t i = 0; i < streams.size(); ++i) { + if (streams[i].uncompressedSize != dests[i].second) { + throw std::runtime_error("NGSP stream size mismatch for stream " + std::to_string(i)); + } + size_t ret = ZSTD_decompress( + dests[i].first, dests[i].second, + data + streams[i].dataOffset, streams[i].compressedSize); + if (ZSTD_isError(ret) || ret != dests[i].second) { + throw std::runtime_error("ZSTD decompression failed for stream " + std::to_string(i)); + } } +} + +// ── v2-v3 legacy reader (extracted) ─────────────────────────────────────── +static std::unique_ptr readSpzLegacy(const std::vector& buffer) { const uint8_t* data = buffer.data(); - if (buffer.size() < SPZ_HEADER_SIZE) throw std::runtime_error("File too small"); + if (buffer.size() < 16) throw std::runtime_error("File too small"); uint32_t magic; std::memcpy(&magic, data, 4); @@ -93,14 +239,17 @@ std::unique_ptr readSpz(const std::filesystem::path& filename) { uint32_t version; std::memcpy(&version, data + 4, 4); + if (version < 2 || version > 3) { + throw std::runtime_error("Unsupported legacy SPZ version: " + std::to_string(version)); + } + uint32_t numSplats; std::memcpy(&numSplats, data + 8, 4); uint8_t shDegree = data[12]; - uint8_t fractionalBits = data[13]; size_t harmonicsCount = HARMONICS_COMPONENT_COUNT[shDegree > 3 ? 0 : shDegree]; - size_t offset = SPZ_HEADER_SIZE; + size_t offset = 16; const uint8_t* posBase = data + offset; offset += numSplats * 3 * 3; const uint8_t* alphaBase = data + offset; @@ -113,27 +262,22 @@ std::unique_ptr readSpz(const std::filesystem::path& filename) { offset += numSplats * (version == 3 ? 4 : 3); const uint8_t* shBase = data + offset; - std::vector columns = {// Position - {"x", std::vector(numSplats, 0.0f)}, - {"y", std::vector(numSplats, 0.0f)}, - {"z", std::vector(numSplats, 0.0f)}, - - // Scale (stored as linear in .splat, convert to log for internal use) - {"scale_0", std::vector(numSplats, 0.0f)}, - {"scale_1", std::vector(numSplats, 0.0f)}, - {"scale_2", std::vector(numSplats, 0.0f)}, - - // Color/opacity - {"f_dc_0", std::vector(numSplats, 0.0f)}, // Red - {"f_dc_1", std::vector(numSplats, 0.0f)}, // Green - {"f_dc_2", std::vector(numSplats, 0.0f)}, // Blue - {"opacity", std::vector(numSplats, 0.0f)}, - - // Rotation quaternion - {"rot_0", std::vector(numSplats, 0.0f)}, - {"rot_1", std::vector(numSplats, 0.0f)}, - {"rot_2", std::vector(numSplats, 0.0f)}, - {"rot_3", std::vector(numSplats, 0.0f)}}; + std::vector columns = { + {"x", std::vector(numSplats, 0.0f)}, + {"y", std::vector(numSplats, 0.0f)}, + {"z", std::vector(numSplats, 0.0f)}, + {"scale_0", std::vector(numSplats, 0.0f)}, + {"scale_1", std::vector(numSplats, 0.0f)}, + {"scale_2", std::vector(numSplats, 0.0f)}, + {"f_dc_0", std::vector(numSplats, 0.0f)}, + {"f_dc_1", std::vector(numSplats, 0.0f)}, + {"f_dc_2", std::vector(numSplats, 0.0f)}, + {"opacity", std::vector(numSplats, 0.0f)}, + {"rot_0", std::vector(numSplats, 0.0f)}, + {"rot_1", std::vector(numSplats, 0.0f)}, + {"rot_2", std::vector(numSplats, 0.0f)}, + {"rot_3", std::vector(numSplats, 0.0f)}, + }; for (size_t i = 0; i < harmonicsCount; ++i) columns.push_back({"f_rest_" + std::to_string(i), std::vector(numSplats, 0.0f)}); @@ -141,68 +285,155 @@ std::unique_ptr readSpz(const std::filesystem::path& filename) { std::vector colPtrs; for (auto& col : columns) colPtrs.push_back(col.asVector().data()); - const float posScale = 1.0f / (1 << fractionalBits); - for (uint32_t i = 0; i < numSplats; ++i) { - // Position (24-bit fixed point) - colPtrs[0][i] = getFixed24(buffer, i, 0) * posScale; - colPtrs[1][i] = getFixed24(buffer, i, 1) * posScale; - colPtrs[2][i] = getFixed24(buffer, i, 2) * posScale; + // Decode using shared helpers + { + std::vector posCols = {colPtrs[0], colPtrs[1], colPtrs[2]}; + decodePositions(posBase, data[13], numSplats, posCols); + } + { + std::vector scaleCols = {colPtrs[3], colPtrs[4], colPtrs[5]}; + decodeScales(scaleBase, numSplats, scaleCols); + } + { + std::vector colorCols = {colPtrs[6], colPtrs[7], colPtrs[8]}; + decodeColors(colorBase, numSplats, colorCols); + } + { + std::vector alphaCols = {colPtrs[9]}; + decodeAlphas(alphaBase, numSplats, alphaCols); + } + { + std::vector rotCols = {colPtrs[10], colPtrs[11], colPtrs[12], colPtrs[13]}; + decodeRotations(rotBase, version, numSplats, rotCols); + } + if (harmonicsCount > 0) { + std::vector shCols; + for (size_t h = 0; h < harmonicsCount; ++h) shCols.push_back(colPtrs[14 + h]); + decodeSH(shBase, shDegree, numSplats, shCols); + } - // Scale - colPtrs[3][i] = scaleBase[i * 3 + 0] / 16.0f - 10.0f; - colPtrs[4][i] = scaleBase[i * 3 + 1] / 16.0f - 10.0f; - colPtrs[5][i] = scaleBase[i * 3 + 2] / 16.0f - 10.0f; + return std::make_unique(std::move(columns)); +} - // Color & Opacity (Inverse Sigmoid) - colPtrs[6][i] = inverseConvertColorFromSPZ(colorBase[i * 3 + 0]); - colPtrs[7][i] = inverseConvertColorFromSPZ(colorBase[i * 3 + 1]); - colPtrs[8][i] = inverseConvertColorFromSPZ(colorBase[i * 3 + 2]); +// ── v4 NGSP reader ─────────────────────────────────────────────────────── - float normAlpha = std::clamp(alphaBase[i] / 255.0f, 1e-6f, 1.0f - 1e-6f); - colPtrs[9][i] = std::log(normAlpha / (1.0f - normAlpha)); +static std::unique_ptr readSpzV4(const std::vector& buffer) { + NgspFileHeader header; + std::memcpy(&header, buffer.data(), SPZ_V4_HEADER_SIZE); - // Rotation (Quaternions) - float q[4] = {1, 0, 0, 0}; - if (version == 2) { - q[1] = (rotBase[i * 3 + 0] / 127.5f) - 1.0f; - q[2] = (rotBase[i * 3 + 1] / 127.5f) - 1.0f; - q[3] = (rotBase[i * 3 + 2] / 127.5f) - 1.0f; - float dot = q[1] * q[1] + q[2] * q[2] + q[3] * q[3]; - q[0] = std::sqrt(std::max(0.0f, 1.0f - dot)); - } else if (version == 3) { - uint32_t packed; - std::memcpy(&packed, rotBase + i * 4, 4); - uint32_t largestIndex = packed >> 30; - float sum_sq = 0; - uint32_t temp = packed; - for (int j = 3; j >= 0; --j) { - if (static_cast(j) != largestIndex) { - uint32_t mag = temp & 511; - float val = 0.70710678f * mag / 511.0f; - if ((temp >> 9) & 1) val = -val; - q[j] = val; - sum_sq += val * val; - temp >>= 10; - } - } - q[largestIndex] = std::sqrt(std::max(0.0f, 1.0f - sum_sq)); - } - colPtrs[10][i] = q[0]; - colPtrs[11][i] = q[1]; - colPtrs[12][i] = q[2]; - colPtrs[13][i] = q[3]; + if (header.magic != SPZ_V4_MAGIC) { + throw std::runtime_error("Invalid v4 SPZ magic"); + } + if (header.version != LATEST_SPZ_VERSION) { + throw std::runtime_error("Unsupported SPZ v4 version: " + std::to_string(header.version)); + } + if (header.numPoints == 0) { + throw std::runtime_error("v4 SPZ has zero points"); + } - // Spherical Harmonics - for (size_t sh = 0; sh < harmonicsCount; ++sh) { - size_t channel = sh % 3; - size_t coeff = sh / 3; - size_t colIdx = 14 + (channel * (harmonicsCount / 3) + coeff); - uint8_t shVal = shBase[i * harmonicsCount + sh]; - colPtrs[colIdx][i] = (static_cast(shVal) - 128.0f) / 128.0f; - } + uint32_t numSplats = header.numPoints; + uint8_t shDegree = header.shDegree; + size_t harmonicsCount = HARMONICS_COMPONENT_COUNT[shDegree > 3 ? 0 : shDegree]; + + // Allocate per-column float buffers + std::vector xData(numSplats), yData(numSplats), zData(numSplats); + std::vector s0Data(numSplats), s1Data(numSplats), s2Data(numSplats); + std::vector dc0Data(numSplats), dc1Data(numSplats), dc2Data(numSplats); + std::vector opData(numSplats); + std::vector r0Data(numSplats), r1Data(numSplats), r2Data(numSplats), r3Data(numSplats); + std::vector> shData(harmonicsCount, std::vector(numSplats)); + + // Parse TOC + auto streams = parseNgspTOC(buffer.data(), buffer.size(), header); + + // Allocate raw decode buffers in v4 stream order: positions, alphas, colors, scales, rotations, sh + std::vector posBuf(numSplats * 9), alphaBuf(numSplats), + colorBuf(numSplats * 3), scaleBuf(numSplats * 3), + rotBuf(numSplats * 4), shBuf(numSplats * harmonicsCount); + + std::vector> dests; + dests.push_back({posBuf.data(), posBuf.size()}); + dests.push_back({alphaBuf.data(), alphaBuf.size()}); + dests.push_back({colorBuf.data(), colorBuf.size()}); + dests.push_back({scaleBuf.data(), scaleBuf.size()}); + dests.push_back({rotBuf.data(), rotBuf.size()}); + if (harmonicsCount > 0) dests.push_back({shBuf.data(), shBuf.size()}); + + decompressNgspStreams(buffer.data(), streams, dests); + + // Decode using shared helpers + { + std::vector cols = {xData.data(), yData.data(), zData.data()}; + decodePositions(posBuf.data(), header.fractionalBits, numSplats, cols); + } + { + std::vector cols = {s0Data.data(), s1Data.data(), s2Data.data()}; + decodeScales(scaleBuf.data(), numSplats, cols); + } + { + std::vector cols = {dc0Data.data(), dc1Data.data(), dc2Data.data()}; + decodeColors(colorBuf.data(), numSplats, cols); + } + { + std::vector cols = {opData.data()}; + decodeAlphas(alphaBuf.data(), numSplats, cols); + } + { + std::vector cols = {r0Data.data(), r1Data.data(), r2Data.data(), r3Data.data()}; + decodeRotations(rotBuf.data(), header.version, numSplats, cols); + } + if (harmonicsCount > 0) { + std::vector cols; + for (size_t h = 0; h < harmonicsCount; ++h) cols.push_back(shData[h].data()); + decodeSH(shBuf.data(), shDegree, numSplats, cols); } + // Build DataTable + std::vector columns = { + {"x", std::move(xData)}, {"y", std::move(yData)}, {"z", std::move(zData)}, + {"scale_0", std::move(s0Data)}, {"scale_1", std::move(s1Data)}, {"scale_2", std::move(s2Data)}, + {"f_dc_0", std::move(dc0Data)}, {"f_dc_1", std::move(dc1Data)}, {"f_dc_2", std::move(dc2Data)}, + {"opacity", std::move(opData)}, + {"rot_0", std::move(r0Data)}, {"rot_1", std::move(r1Data)}, + {"rot_2", std::move(r2Data)}, {"rot_3", std::move(r3Data)}, + }; + for (size_t h = 0; h < harmonicsCount; ++h) { + columns.push_back({"f_rest_" + std::to_string(h), std::move(shData[h])}); + } return std::make_unique(std::move(columns)); } +// ── Public API ───────────────────────────────────────────────────────────── + +std::unique_ptr readSpz(const std::filesystem::path& filename) { + std::ifstream ifs(filename, std::ios::binary | std::ios::ate); + if (!ifs.is_open()) { + throw std::runtime_error("cannot open file: " + filename.u8string()); + } + + std::streamsize filesize = ifs.tellg(); + ifs.seekg(0, std::ios::beg); + std::vector buffer(filesize); + ifs.read(reinterpret_cast(buffer.data()), filesize); + + // v4 NGSP detection (ZSTD multi-stream) + if (buffer.size() >= 4) { + uint32_t magic; + std::memcpy(&magic, buffer.data(), 4); + if (magic == SPZ_V4_MAGIC) { + // Check version byte to distinguish v4 from legacy + uint32_t version; + std::memcpy(&version, buffer.data() + 4, 4); + if (version >= 4) return readSpzV4(buffer); + } + } + + // Legacy GZip path (v2-v3) + if (buffer.size() > 2 && buffer[0] == 0x1F && buffer[1] == 0x8B) { + buffer = decompressGZIP(buffer); + } + + return readSpzLegacy(buffer); +} + } // namespace splat From 4eb3b0adb45e29dc54f2c9f2bf1bd68187a5a405 Mon Sep 17 00:00:00 2001 From: "Merlot.Qi" Date: Tue, 2 Jun 2026 13:44:41 +0800 Subject: [PATCH 4/8] feat(spz): implement SPZ writer with encode pipeline - spz_encoder.h/.cpp: encodePosition (int24), encodeScale (log->uint8), encodeColor (SH DC->uint8), encodeAlpha (sigmoid->uint8), encodeRotation (smallest-three 9-bit packing), encodeSH (bucket quantize) - spz_writer.h/.cpp: DataTable -> per-splat encode -> ZSTD multi-stream compression -> NgspFileHeader + TOC -> .spz v4 file output - Configurable SH quantization via SpzWriteOptions (sh1Bits/shRestBits) Co-Authored-By: Claude Opus 4.8 --- include/splat/io/spz_writer.h | 24 +++++ src/io/spz_encoder.cpp | 78 ++++++++++++++++ src/io/spz_encoder.h | 39 ++++++++ src/io/spz_writer.cpp | 167 ++++++++++++++++++++++++++++++++++ 4 files changed, 308 insertions(+) create mode 100644 include/splat/io/spz_writer.h create mode 100644 src/io/spz_encoder.cpp create mode 100644 src/io/spz_encoder.h create mode 100644 src/io/spz_writer.cpp diff --git a/include/splat/io/spz_writer.h b/include/splat/io/spz_writer.h new file mode 100644 index 0000000..b20a751 --- /dev/null +++ b/include/splat/io/spz_writer.h @@ -0,0 +1,24 @@ +/** + * @file splat/io/spz_writer.h + * @brief Write .spz compressed Gaussian splat files (v4 NGSP format). + */ +#pragma once + +#include +#include + +namespace splat { + +class DataTable; + +struct SpzWriteOptions { + int fractionalBits = 12; + uint8_t sh1Bits = 5; + uint8_t shRestBits = 4; +}; + +void writeSpz(const std::filesystem::path& filename, + const DataTable& dataTable, + const SpzWriteOptions& options = {}); + +} // namespace splat diff --git a/src/io/spz_encoder.cpp b/src/io/spz_encoder.cpp new file mode 100644 index 0000000..7a9cf80 --- /dev/null +++ b/src/io/spz_encoder.cpp @@ -0,0 +1,78 @@ +#include "spz_encoder.h" + +#include +#include +#include + +namespace splat { + +void encodePosition(const float pos[3], int fractionalBits, uint8_t out[9]) { + float scale = static_cast(1 << fractionalBits); + for (int i = 0; i < 3; ++i) { + int32_t fixed = static_cast(std::round(pos[i] * scale)); + out[i * 3 + 0] = fixed & 0xFF; + out[i * 3 + 1] = (fixed >> 8) & 0xFF; + out[i * 3 + 2] = (fixed >> 16) & 0xFF; + } +} + +void encodeScale(const float scale_log[3], uint8_t out[3]) { + for (int i = 0; i < 3; ++i) { + float v = (scale_log[i] + 10.0f) * 16.0f + 0.5f; + out[i] = static_cast(std::clamp(v, 0.0f, 255.0f)); + } +} + +void encodeColor(const float f_dc[3], uint8_t out[3]) { + for (int i = 0; i < 3; ++i) { + float v = f_dc[i] * SPZ_COLOR_SCALE * 255.0f + 127.5f + 0.5f; + out[i] = static_cast(std::clamp(v, 0.0f, 255.0f)); + } +} + +uint8_t encodeAlpha(float opacity) { + float v = spzSigmoid(opacity) * 255.0f + 0.5f; + return static_cast(std::clamp(v, 0.0f, 255.0f)); +} + +void encodeRotation(const float rot[4], uint8_t out[4]) { + float w = rot[0], x = rot[1], y = rot[2], z = rot[3]; + float len = std::sqrt(w * w + x * x + y * y + z * z); + if (len > 0.0f) { w /= len; x /= len; y /= len; z /= len; } + + float absVals[4] = {std::fabs(w), std::fabs(x), std::fabs(y), std::fabs(z)}; + unsigned largestIdx = 0; + for (unsigned i = 1; i < 4; ++i) { + if (absVals[i] > absVals[largestIdx]) largestIdx = i; + } + + float quat[4] = {w, x, y, z}; + if (quat[largestIdx] < 0.0f) { + for (int i = 0; i < 4; ++i) quat[i] = -quat[i]; + } + + constexpr float kRsqrt2 = 0.7071067811865475f; + constexpr uint32_t kMagMask = 511u; + uint32_t packed = largestIdx; + for (int i = 3; i >= 0; --i) { + if (static_cast(i) != largestIdx) { + uint32_t negBit = (quat[i] < 0.0f) ? 1u : 0u; + uint32_t mag = static_cast(std::round(std::fabs(quat[i]) / kRsqrt2 * 511.0f)); + if (mag > kMagMask) mag = kMagMask; + packed = (packed << 10) | (negBit << 9) | mag; + } + } + + out[0] = packed & 0xFF; + out[1] = (packed >> 8) & 0xFF; + out[2] = (packed >> 16) & 0xFF; + out[3] = (packed >> 24) & 0xFF; +} + +uint8_t encodeSH(float v, int bucketSize) { + int q = static_cast(std::round(v * 128.0f + 128.0f)); + q = (q + bucketSize / 2) / bucketSize * bucketSize; + return static_cast(std::clamp(q, 0, 255)); +} + +} // namespace splat diff --git a/src/io/spz_encoder.h b/src/io/spz_encoder.h new file mode 100644 index 0000000..817e483 --- /dev/null +++ b/src/io/spz_encoder.h @@ -0,0 +1,39 @@ +/** + * @file io/spz_encoder.h + * @brief SPZ encoding functions — exact inverses of spz_reader decode. + */ +#pragma once + +#include + +namespace splat { + +constexpr float SPZ_COLOR_SCALE = 0.15f; + +inline float spzSigmoid(float x) { return 1.0f / (1.0f + std::exp(-x)); } + +/// Encode position as 3 x int24 little-endian (9 bytes per point). +void encodePosition(const float pos[3], int fractionalBits, uint8_t out[9]); + +/// Encode log-scale as uint8 (3 bytes per point). +/// Formula: clamp((log_scale + 10.0) * 16.0 + 0.5, 0, 255) +void encodeScale(const float scale_log[3], uint8_t out[3]); + +/// Encode SH DC color as uint8 (3 bytes per point). +/// Formula: clamp(f_dc * colorScale * 255 + 127.5 + 0.5, 0, 255) +void encodeColor(const float f_dc[3], uint8_t out[3]); + +/// Encode logit-space opacity as uint8 (1 byte per point). +/// Formula: clamp(sigmoid(opacity) * 255.0 + 0.5, 0, 255) +uint8_t encodeAlpha(float opacity); + +/// Encode quaternion as smallest-three packed uint32 (4 bytes per point). +/// Steps: normalize, find largest abs component, negate if needed, +/// quantize 3 smallest to 9-bit mag + 1-bit sign, pack into uint32 LE. +void encodeRotation(const float rot[4], uint8_t out[4]); + +/// Encode SH coefficient as quantized uint8. +/// Formula: clamp(round((v*128+128 + bucketSize/2)/bucketSize)*bucketSize, 0, 255) +uint8_t encodeSH(float v, int bucketSize); + +} // namespace splat diff --git a/src/io/spz_writer.cpp b/src/io/spz_writer.cpp new file mode 100644 index 0000000..fbe376e --- /dev/null +++ b/src/io/spz_writer.cpp @@ -0,0 +1,167 @@ +#include + +#include "spz_encoder.h" +#include + +#include + +#include +#include +#include +#include + +namespace splat { + +namespace { + +constexpr uint32_t NGSP_MAGIC = 0x5053474E; +constexpr int SPZ_VERSION = 4; + +int shDegreeFromColumnCount(int restCount) { + if (restCount >= 45) return 3; + if (restCount >= 24) return 2; + if (restCount >= 9) return 1; + return 0; +} + +int shDimForDegree(int degree) { + switch (degree) { + case 0: return 0; + case 1: return 3; + case 2: return 8; + case 3: return 15; + default: return 0; + } +} + +} // namespace + +void writeSpz(const std::filesystem::path& filename, + const DataTable& dataTable, + const SpzWriteOptions& options) { + + // 1. Validate required columns + auto getSpan = [&](const std::string& name) { + const auto& col = dataTable.getColumnByName(name); + if (col.getType() != ColumnType::FLOAT32) { + throw std::runtime_error("SPZ writer: column '" + name + "' must be float32"); + } + return col.asSpan(); + }; + + auto xs = getSpan("x"), ys = getSpan("y"), zs = getSpan("z"); + auto s0 = getSpan("scale_0"), s1 = getSpan("scale_1"), s2 = getSpan("scale_2"); + auto dc0 = getSpan("f_dc_0"), dc1 = getSpan("f_dc_1"), dc2 = getSpan("f_dc_2"); + auto op = getSpan("opacity"); + auto r0 = getSpan("rot_0"), r1 = getSpan("rot_1"), r2 = getSpan("rot_2"), r3 = getSpan("rot_3"); + + uint32_t numPoints = static_cast(dataTable.getNumRows()); + + // 2. Determine SH degree + int fRestCount = 0; + while (dataTable.hasColumn("f_rest_" + std::to_string(fRestCount))) ++fRestCount; + int shDegree = shDegreeFromColumnCount(fRestCount); + int shDim = shDimForDegree(shDegree); + + // 3. Allocate packed buffers + std::vector posBuf(numPoints * 9); + std::vector alphaBuf(numPoints); + std::vector colorBuf(numPoints * 3); + std::vector scaleBuf(numPoints * 3); + std::vector rotBuf(numPoints * 4); + std::vector shBuf(numPoints * shDim * 3); + + // 4. Per-splat encode + for (uint32_t i = 0; i < numPoints; ++i) { + float pos[3] = {xs[i], ys[i], zs[i]}; + float scl[3] = {s0[i], s1[i], s2[i]}; + float fdc[3] = {dc0[i], dc1[i], dc2[i]}; + float rot[4] = {r0[i], r1[i], r2[i], r3[i]}; + + encodePosition(pos, options.fractionalBits, posBuf.data() + i * 9); + encodeScale(scl, scaleBuf.data() + i * 3); + encodeColor(fdc, colorBuf.data() + i * 3); + alphaBuf[i] = encodeAlpha(op[i]); + encodeRotation(rot, rotBuf.data() + i * 4); + + if (shDegree > 0) { + int bucketSize1 = 1 << (8 - options.sh1Bits); + int bucketSizeRest = 1 << (8 - options.shRestBits); + for (int ch = 0; ch < 3; ++ch) { + for (int c = 0; c < shDim; ++c) { + float v = 0.0f; + int srcCol = ch * shDim + c; + if (srcCol < fRestCount) { + v = dataTable.getColumnByName("f_rest_" + std::to_string(srcCol)).getValue(i); + } + int bucketSize = (c < 3) ? bucketSize1 : bucketSizeRest; + size_t outIdx = (i * shDim + c) * 3 + ch; + shBuf[outIdx] = encodeSH(v, bucketSize); + } + } + } + } + + // 5. ZSTD compress non-zero buffers + struct StreamSrc { const uint8_t* data; size_t size; }; + std::vector srcs = { + {posBuf.data(), posBuf.size()}, + {alphaBuf.data(), alphaBuf.size()}, + {colorBuf.data(), colorBuf.size()}, + {scaleBuf.data(), scaleBuf.size()}, + {rotBuf.data(), rotBuf.size()}, + }; + if (shDegree > 0) srcs.push_back({shBuf.data(), shBuf.size()}); + + std::vector> chunks; + std::vector uncompressedSizes; + + for (const auto& src : srcs) { + if (src.size == 0) continue; + size_t bound = ZSTD_compressBound(src.size); + chunks.emplace_back(bound); + size_t compressedSize = ZSTD_compress(chunks.back().data(), bound, src.data, src.size, 12); + if (ZSTD_isError(compressedSize)) { + throw std::runtime_error("ZSTD compression failed"); + } + chunks.back().resize(compressedSize); + uncompressedSizes.push_back(src.size); + } + + // 6. Build header + uint8_t numStreams = static_cast(chunks.size()); + uint32_t tocByteOffset = 32; + uint8_t header[32] = {}; + std::memcpy(header, &NGSP_MAGIC, 4); + uint32_t ver = SPZ_VERSION; + std::memcpy(header + 4, &ver, 4); + std::memcpy(header + 8, &numPoints, 4); + header[12] = static_cast(shDegree); + header[13] = static_cast(options.fractionalBits); + header[14] = 0; // flags + header[15] = numStreams; + std::memcpy(header + 16, &tocByteOffset, 4); + + // 7. Write file + std::ofstream out(filename, std::ios::binary); + if (!out) throw std::runtime_error("Failed to create: " + filename.u8string()); + + out.write(reinterpret_cast(header), 32); + + // TOC + for (size_t i = 0; i < chunks.size(); ++i) { + uint64_t cs = chunks[i].size(); + uint64_t us = uncompressedSizes[i]; + out.write(reinterpret_cast(&cs), 8); + out.write(reinterpret_cast(&us), 8); + } + + // Compressed stream data + for (const auto& chunk : chunks) { + out.write(reinterpret_cast(chunk.data()), chunk.size()); + } + + if (!out.good()) throw std::runtime_error("Failed to write SPZ file"); +} + +} // namespace splat From 541a1d49f27e74f4c74e160a61f5e66c40afe097 Mon Sep 17 00:00:00 2001 From: "Merlot.Qi" Date: Tue, 2 Jun 2026 13:46:43 +0800 Subject: [PATCH 5/8] feat(spz): add CoordinateConverter, writer tests, and converter tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Iter 2 (Writer): - spz_writer_test.cpp: round-trip tests (no SH, with SH, single splat) Iter 3 (CoordinateConverter): - include/splat/maths/coordinate-converter.h: 16-system enum, converter struct - src/maths/coordinate-converter.cpp: within-family sign flips, cross-family R_x(±π/2) analytic rotation for pos/quat/SH bands 0-1 - coordinate_converter_test.cpp: identity, within-family, cross-family round-trip, all 16 systems Fix: add missing include in spz_encoder.h for std::exp Co-Authored-By: Claude Opus 4.8 --- include/splat/maths/coordinate-converter.h | 46 ++++++ src/io/spz_encoder.h | 1 + src/maths/coordinate-converter.cpp | 177 +++++++++++++++++++++ tests/CMakeLists.txt | 8 + tests/coordinate_converter_test.cpp | 71 +++++++++ tests/spz_writer_test.cpp | 89 +++++++++++ 6 files changed, 392 insertions(+) create mode 100644 include/splat/maths/coordinate-converter.h create mode 100644 src/maths/coordinate-converter.cpp create mode 100644 tests/coordinate_converter_test.cpp create mode 100644 tests/spz_writer_test.cpp diff --git a/include/splat/maths/coordinate-converter.h b/include/splat/maths/coordinate-converter.h new file mode 100644 index 0000000..dc9d8a0 --- /dev/null +++ b/include/splat/maths/coordinate-converter.h @@ -0,0 +1,46 @@ +/** + * @file splat/maths/coordinate-converter.h + * @brief Coordinate system conversion for 3DGS position, rotation, and SH data. + * + * Supports 16 coordinate systems. Within-family conversions use sign flips. + * Cross-family conversions use R_x(±π/2) analytic rotation + sign flips. + */ +#pragma once + +#include +#include +#include + +namespace splat { + +enum class CoordinateSystem : uint32_t { + UNSPECIFIED = 0, + LDB = 1, LUB = 3, LDF = 5, LUF = 7, + RDB = 2, RUB = 4, RDF = 6, RUF = 8, + LFD = 9, LFU = 11, LBD = 13, LBU = 15, + RFD = 10, RFU = 12, RBD = 14, RBU = 16, + SPZ_DEFAULT = RUB, + PLY_DEFAULT = RDF, + GLB_DEFAULT = LUF, + UNITY_DEFAULT = RUF, +}; + +struct CoordinateConverter { + std::array flipP = {1, 1, 1}; + std::array flipQ = {1, 1, 1}; + std::array flipSh; + std::function rotFlipPos; + std::function rotFlipQuat; + std::array, 4> rotFlipShBands; + + CoordinateConverter(); + + void convertPosition(float pos[3]) const; + void convertRotation(float quat[4]) const; + void convertSH(float* sh, int numBands, int shDegree) const; +}; + +CoordinateConverter makeCoordinateConverter( + CoordinateSystem from, CoordinateSystem to, int shDegree); + +} // namespace splat diff --git a/src/io/spz_encoder.h b/src/io/spz_encoder.h index 817e483..3186a85 100644 --- a/src/io/spz_encoder.h +++ b/src/io/spz_encoder.h @@ -4,6 +4,7 @@ */ #pragma once +#include #include namespace splat { diff --git a/src/maths/coordinate-converter.cpp b/src/maths/coordinate-converter.cpp new file mode 100644 index 0000000..66e16c3 --- /dev/null +++ b/src/maths/coordinate-converter.cpp @@ -0,0 +1,177 @@ +#include + +#include +#include +#include + +namespace splat { + +CoordinateConverter::CoordinateConverter() { flipSh.fill(1.0f); } + +namespace { + +bool needRotation(CoordinateSystem a, CoordinateSystem b) { + int an = static_cast(a) - 1, bn = static_cast(b) - 1; + if (an < 0 || bn < 0) return false; + return ((an >> 3) & 1) != ((bn >> 3) & 1); +} + +std::array axesMatch(CoordinateSystem a, CoordinateSystem b) { + int an = static_cast(a) - 1, bn = static_cast(b) - 1; + if (an < 0 || bn < 0) return {true, true, true}; + return { + ((an >> 0) & 1) == ((bn >> 0) & 1), + ((an >> 1) & 1) == ((bn >> 1) & 1), + ((an >> 2) & 1) == ((bn >> 2) & 1) + }; +} + +void computeFlips(CoordinateSystem from, CoordinateSystem to, + std::array& flipP, std::array& flipQ, + std::array& flipSh) { + auto [xMatch, yMatch, zMatch] = axesMatch(from, to); + float x = xMatch ? 1.0f : -1.0f; + float y = yMatch ? 1.0f : -1.0f; + float z = zMatch ? 1.0f : -1.0f; + + flipP = {x, y, z}; + flipQ = {y * z, x * z, x * y}; + flipSh = { + y, z, x, x*y, y*z, 1, x*z, 1, y, + x*y*z, y, z, x, z, x, + x*y, y*z, x*y, y*z, 1, x*z, 1, x*z, y + }; +} + +float kSqrt2() { return std::sqrt(2.0f); } +float kSqrt3() { return std::sqrt(3.0f); } + +void rotFlipPosPlus90X(float* p) { + float y = p[1], z = p[2]; p[1] = -z; p[2] = y; +} +void rotFlipPosMinus90X(float* p) { + float y = p[1], z = p[2]; p[1] = z; p[2] = -y; +} + +void rotFlipQuatPlus90X(float* p) { + float s = kSqrt2() / 2.0f; + float w = p[0], x = p[1], y = p[2], z = p[3]; + p[0] = s * (w + x); p[1] = s * (y - z); + p[2] = s * (y + z); p[3] = s * (w - x); +} +void rotFlipQuatMinus90X(float* p) { + float s = kSqrt2() / 2.0f; + float w = p[0], x = p[1], y = p[2], z = p[3]; + p[0] = s * (-w + x); p[1] = s * (y + z); + p[2] = s * (-y + z); p[3] = s * (w + x); +} + +// SH band-0 rotation (l=1: 3 coeffs) +void rotShBand0Plus90X(float* p) { float t0 = p[0]; p[0] = p[1]; p[1] = -t0; } +void rotShBand0Minus90X(float* p) { float t0 = p[0]; p[0] = -p[1]; p[1] = t0; } + +// SH band-1 rotation (l=2: 5 coeffs) +void rotShBand1Plus90X(float* p) { + float s[5]; std::memcpy(s, p, 5 * sizeof(float)); + float s3 = kSqrt3(); + p[0] = s[3]; p[1] = -s[1]; + p[2] = -0.5f * s[2] - (s3 / 2.0f) * s[4]; + p[3] = -s[0]; + p[4] = -(s3 / 2.0f) * s[2] + 0.5f * s[4]; +} +void rotShBand1Minus90X(float* p) { + float s[5]; std::memcpy(s, p, 5 * sizeof(float)); + float s3 = kSqrt3(); + p[0] = -s[3]; p[1] = -s[1]; + p[2] = -0.5f * s[2] - (s3 / 2.0f) * s[4]; + p[3] = s[0]; + p[4] = -(s3 / 2.0f) * s[2] + 0.5f * s[4]; +} + +} // namespace + +CoordinateConverter makeCoordinateConverter( + CoordinateSystem from, CoordinateSystem to, int shDegree) { + + CoordinateConverter conv; + if (from == to || from == CoordinateSystem::UNSPECIFIED || + to == CoordinateSystem::UNSPECIFIED) { + return conv; // identity + } + + if (needRotation(from, to)) { + bool backward = ((static_cast(from) - 1) >> 3) & 1; + CoordinateSystem innerFrom = backward + ? static_cast(static_cast(from) - 8) + : static_cast(static_cast(from) + 8); + + computeFlips(innerFrom, to, conv.flipP, conv.flipQ, conv.flipSh); + auto fp = conv.flipP, fq = conv.flipQ; + + if (backward) { + conv.rotFlipPos = [fp](float* p) { + p[0] *= fp[0]; p[1] *= fp[1]; p[2] *= fp[2]; + rotFlipPosMinus90X(p); + }; + conv.rotFlipQuat = [fq](float* p) { + p[0] *= fq[0]; p[1] *= fq[1]; p[2] *= fq[2]; + rotFlipQuatMinus90X(p); + }; + conv.rotFlipShBands[0] = rotShBand0Minus90X; + conv.rotFlipShBands[1] = rotShBand1Minus90X; + } else { + conv.rotFlipPos = [fp](float* p) { + rotFlipPosPlus90X(p); + p[0] *= fp[0]; p[1] *= fp[1]; p[2] *= fp[2]; + }; + conv.rotFlipQuat = [fq](float* p) { + rotFlipQuatPlus90X(p); + p[0] *= fq[0]; p[1] *= fq[1]; p[2] *= fq[2]; + }; + conv.rotFlipShBands[0] = rotShBand0Plus90X; + conv.rotFlipShBands[1] = rotShBand1Plus90X; + } + conv.flipP = {1, 1, 1}; + conv.flipQ = {1, 1, 1}; + conv.flipSh.fill(1.0f); + } else { + computeFlips(from, to, conv.flipP, conv.flipQ, conv.flipSh); + } + return conv; +} + +void CoordinateConverter::convertPosition(float pos[3]) const { + if (rotFlipPos) { rotFlipPos(pos); return; } + for (int i = 0; i < 3; ++i) pos[i] *= flipP[i]; +} + +void CoordinateConverter::convertRotation(float quat[4]) const { + if (rotFlipQuat) { rotFlipQuat(quat); return; } + for (int i = 0; i < 3; ++i) quat[i] *= flipQ[i]; +} + +void CoordinateConverter::convertSH(float* sh, int numBands, int shDegree) const { + if (rotFlipShBands[0]) { + for (int band = 0; band < std::min(shDegree, 2); ++band) { + int bandStart = band * (band + 2); + int bandSize = 2 * band + 3; + if (bandStart + bandSize > numBands) break; + for (int ch = 0; ch < 3; ++ch) { + float tmp[9] = {}; + for (int k = 0; k < bandSize; ++k) tmp[k] = sh[(bandStart + k) * 3 + ch]; + if (rotFlipShBands[static_cast(band)]) + rotFlipShBands[static_cast(band)](tmp); + for (int k = 0; k < bandSize; ++k) sh[(bandStart + k) * 3 + ch] = tmp[k]; + } + } + } else { + for (int i = 0; i < numBands; ++i) { + int base = i * 3; + sh[base + 0] *= flipSh[i]; + sh[base + 1] *= flipSh[i]; + sh[base + 2] *= flipSh[i]; + } + } +} + +} // namespace splat diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index bc39512..69ea4ba 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -57,3 +57,11 @@ add_test(NAME SplatLccReaderTests COMMAND SplatLccReaderTests) add_executable(SplatLccWriterTests lcc_writer_test.cpp) target_link_libraries(SplatLccWriterTests PRIVATE SPLAT::splat) add_test(NAME SplatLccWriterTests COMMAND SplatLccWriterTests) + +add_executable(SplatSpzWriterTests spz_writer_test.cpp) +target_link_libraries(SplatSpzWriterTests PRIVATE SPLAT::splat) +add_test(NAME SplatSpzWriterTests COMMAND SplatSpzWriterTests) + +add_executable(SplatCoordinateConverterTests coordinate_converter_test.cpp) +target_link_libraries(SplatCoordinateConverterTests PRIVATE SPLAT::splat) +add_test(NAME SplatCoordinateConverterTests COMMAND SplatCoordinateConverterTests) diff --git a/tests/coordinate_converter_test.cpp b/tests/coordinate_converter_test.cpp new file mode 100644 index 0000000..cab5fab --- /dev/null +++ b/tests/coordinate_converter_test.cpp @@ -0,0 +1,71 @@ +/** + * @file coordinate_converter_test.cpp + * @brief Unit tests for CoordinateConverter. + */ +#include + +#include +#include +#include + +static void check(bool cond, const char* msg) { + if (!cond) { fprintf(stderr, "FAIL: %s\n", msg); std::exit(1); } +} + +int main() { + using CS = splat::CoordinateSystem; + + // Test 1: identity (RUB -> RUB) + { + auto conv = splat::makeCoordinateConverter(CS::RUB, CS::RUB, 0); + float p[3] = {1, 2, 3}; + conv.convertPosition(p); + check(p[0] == 1 && p[1] == 2 && p[2] == 3, "identity position"); + printf("PASS: identity RUB->RUB\n"); + } + + // Test 2: within-family (RUB -> RUF, z flip) + { + auto conv = splat::makeCoordinateConverter(CS::RUB, CS::RUF, 0); + float p[3] = {1, 2, 3}; + conv.convertPosition(p); + check(p[0] == 1 && p[1] == 2 && p[2] == -3, "RUB->RUF z-flip"); + printf("PASS: within-family RUB->RUF\n"); + } + + // Test 3: cross-family position round-trip (RUB -> RDF -> RUB) + { + auto toRDF = splat::makeCoordinateConverter(CS::RUB, CS::RDF, 0); + auto toRUB = splat::makeCoordinateConverter(CS::RDF, CS::RUB, 0); + float p[3] = {1, 2, 3}; + toRDF.convertPosition(p); + toRUB.convertPosition(p); + check(std::fabs(p[0] - 1) < 0.001f && std::fabs(p[1] - 2) < 0.001f && + std::fabs(p[2] - 3) < 0.001f, "cross-family position round-trip"); + printf("PASS: cross-family position RUB<->RDF round-trip\n"); + } + + // Test 4: cross-family quaternion round-trip + { + auto toRDF = splat::makeCoordinateConverter(CS::RUB, CS::RDF, 1); + auto toRUB = splat::makeCoordinateConverter(CS::RDF, CS::RUB, 1); + float q[4] = {1, 0, 0, 0}; // identity quat + toRDF.convertRotation(q); + toRUB.convertRotation(q); + check(std::fabs(q[0] - 1) < 0.001f && std::fabs(q[1]) < 0.001f && + std::fabs(q[2]) < 0.001f && std::fabs(q[3]) < 0.001f, + "cross-family quaternion round-trip"); + printf("PASS: cross-family quaternion round-trip\n"); + } + + // Test 5: all 16 systems convert without crash + for (int from = 1; from <= 16; ++from) { + auto conv = splat::makeCoordinateConverter(static_cast(from), CS::RUB, 0); + float p[3] = {1, 2, 3}; + conv.convertPosition(p); + } + printf("PASS: all 16 coordinate systems\n"); + + printf("\n=== All coordinate converter tests passed ===\n"); + return 0; +} diff --git a/tests/spz_writer_test.cpp b/tests/spz_writer_test.cpp new file mode 100644 index 0000000..aa097d1 --- /dev/null +++ b/tests/spz_writer_test.cpp @@ -0,0 +1,89 @@ +/** + * @file spz_writer_test.cpp + * @brief Round-trip test: DataTable -> writeSpz -> readSpz -> verify + */ +#include +#include +#include + +#include +#include +#include +#include +#include + +static void check(bool cond, const char* msg) { + if (!cond) { fprintf(stderr, "FAIL: %s\n", msg); std::exit(1); } +} + +static splat::DataTable makeTestTable(int n, bool withSH) { + std::vector cols; + auto addCol = [&](const char* name) -> float* { + std::vector d(n); + float* ptr = d.data(); + cols.push_back({name, std::move(d)}); + return ptr; + }; + + float* xs = addCol("x"), *ys = addCol("y"), *zs = addCol("z"); + float* s0 = addCol("scale_0"), *s1 = addCol("scale_1"), *s2 = addCol("scale_2"); + float* dc0 = addCol("f_dc_0"), *dc1 = addCol("f_dc_1"), *dc2 = addCol("f_dc_2"); + float* op = addCol("opacity"); + float* r0 = addCol("rot_0"), *r1 = addCol("rot_1"), *r2 = addCol("rot_2"), *r3 = addCol("rot_3"); + + for (int i = 0; i < n; ++i) { + xs[i] = i * 1.0f; ys[i] = i * 2.0f; zs[i] = i * 3.0f; + s0[i] = -1.0f; s1[i] = 0.0f; s2[i] = 1.0f; + dc0[i] = 0.1f; dc1[i] = -0.1f; dc2[i] = 0.2f; + op[i] = 0.5f; + r0[i] = 1.0f; r1[i] = 0.0f; r2[i] = 0.0f; r3[i] = 0.0f; + } + + if (withSH) { + for (int h = 0; h < 9; ++h) { + std::vector d(n, (h - 4.0f) * 0.2f); + cols.push_back({"f_rest_" + std::to_string(h), std::move(d)}); + } + } + + return splat::DataTable(std::move(cols)); +} + +int main() { + auto tmp = std::filesystem::temp_directory_path() / "test_roundtrip.spz"; + + // Test 1: No SH + { + auto dt = makeTestTable(10, false); + splat::writeSpz(tmp, dt); + auto result = splat::readSpz(tmp); + check(result->getNumRows() == 10, "round-trip no-SH: row count"); + check(!result->hasColumn("f_rest_0"), "round-trip no-SH: no SH column"); + auto& xCol = result->getColumnByName("x"); + check(std::fabs(xCol.getValue(0) - 0.0f) < 0.001f, "round-trip no-SH: x[0]"); + printf("PASS: round-trip no SH\n"); + } + + // Test 2: With SH + { + auto dt = makeTestTable(3, true); + splat::writeSpz(tmp, dt); + auto result = splat::readSpz(tmp); + check(result->getNumRows() == 3, "round-trip SH: row count"); + check(result->hasColumn("f_rest_0"), "round-trip SH: has SH column"); + printf("PASS: round-trip with SH\n"); + } + + // Test 3: Single splat + { + auto dt = makeTestTable(1, false); + splat::writeSpz(tmp, dt); + auto result = splat::readSpz(tmp); + check(result->getNumRows() == 1, "single splat: row count"); + printf("PASS: single splat\n"); + } + + std::filesystem::remove(tmp); + printf("\n=== All SPZ writer tests passed ===\n"); + return 0; +} From 839700162ff7c466e54e8f9a56daa90910bda8aa Mon Sep 17 00:00:00 2001 From: "Merlot.Qi" Date: Tue, 2 Jun 2026 14:43:07 +0800 Subject: [PATCH 6/8] docs: add comprehensive SPZ format specification Covers v4 NGSP (ZSTD multi-stream) and v2-v3 legacy (GZip) formats: - File layout with header, TOC, attribute stream ordering - Per-attribute encoding formulas: position (int24), scale (log), color (SH DC), alpha (sigmoid), rotation (smallest-three), spherical harmonics (bucket quantization) - Coordinate system conventions (RUB default) - File detection logic - References to the official SPZ repository Co-Authored-By: Claude Opus 4.8 --- docs/spz.md | 334 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 334 insertions(+) create mode 100644 docs/spz.md diff --git a/docs/spz.md b/docs/spz.md new file mode 100644 index 0000000..1c09299 --- /dev/null +++ b/docs/spz.md @@ -0,0 +1,334 @@ +# SPZ Format Specification + +SPZ is an open, compressed file format for 3D Gaussian Splatting (3DGS), developed by Niantic Labs and Adobe Inc., and standardized under the Scaniverse open-source initiative. + +- **Magic Number**: `0x5053474E` ("NGSP" little-endian) +- **Latest Version**: 4 +- **Repository**: [https://github.com/scaniverse/spz](https://github.com/scaniverse/spz) +- **License**: MIT + +--- + +## 1. Version History + +| Version | Header | Compression | Rotation Encoding | Status | +|---------|--------|-------------|-------------------|--------| +| 1 | 16 B | GZip | float16 positions | Never released; deprecated | +| 2 | 16 B | GZip | Quaternion first-three (8-bit × 3) | Legacy read-only | +| 3 | 16 B | GZip | Quaternion smallest-three (9-bit × 3 + 2-bit index) | Legacy read-only | +| 4 | 32 B | ZSTD multi-stream | Quaternion smallest-three | **Current** | + +v4 is the canonical format. Versions 1–3 are kept for backward compatibility only and should not be produced by new writers. + +--- + +## 2. File Structure Overview + +### 2.1 v4 NGSP (Current) + +``` +┌─────────────────────────┐ offset 0 +│ NgspFileHeader (32 B) │ +├─────────────────────────┤ offset 32 +│ Extension JSON │ (present only if flags & 0x2) +│ (variable length) │ +├─────────────────────────┤ tocByteOffset +│ TOC: numStreams × │ +│ {compressedSize: u64, │ +│ uncompressedSize:u64}│ +│ = numStreams × 16 B │ +├─────────────────────────┤ tocByteOffset + tocSize +│ ZSTD stream 0 │ positions (numPoints × 9 B uncompressed) +├─────────────────────────┤ +│ ZSTD stream 1 │ alphas (numPoints × 1 B) +├─────────────────────────┤ +│ ZSTD stream 2 │ colors (numPoints × 3 B) +├─────────────────────────┤ +│ ZSTD stream 3 │ scales (numPoints × 3 B) +├─────────────────────────┤ +│ ZSTD stream 4 │ rotations (numPoints × 4 B) +├─────────────────────────┤ +│ ZSTD stream 5 │ sh (numPoints × shDim × 3 B) +│ (only if shDegree > 0) │ +└─────────────────────────┘ +``` + +Streams with zero uncompressed size are skipped entirely (they do not appear in the TOC or as data chunks). The stream count in `numStreams` reflects only the non-empty streams. + +### 2.2 v2–v3 Legacy (GZip) + +``` +┌──────────────────────┐ offset 0 (after GZip decompression) +│ LegacyHeader (16 B) │ +├──────────────────────┤ offset 16 +│ positions │ numPoints × 9 B (3 × int24 LE per point) +├──────────────────────┤ +│ alphas │ numPoints × 1 B +├──────────────────────┤ +│ colors │ numPoints × 3 B +├──────────────────────┤ +│ scales │ numPoints × 3 B +├──────────────────────┤ +│ rotations │ numPoints × 3 B (v2) or × 4 B (v3) +├──────────────────────┤ +│ sh │ numPoints × shDim × 3 B +└──────────────────────┘ +``` + +Entire payload is compressed with GZip (RFC 1952). The GZip magic bytes (`0x1F 0x8B`) are the primary detection mechanism for the legacy path. + +--- + +## 3. Headers + +### 3.1 NgspFileHeader (v4, 32 bytes) + +``` +Offset Size Type Field Description +────── ──── ──── ───── ─────────── +0 4 u32 magic 0x5053474E ("NGSP") +4 4 u32 version 4 +8 4 u32 numPoints Total number of Gaussians +12 1 u8 shDegree SH degree (0–4) +13 1 u8 fractionalBits Position fixed-point precision (default 12) +14 1 u8 flags bit0: antialiased, bit1: hasExtensions +15 1 u8 numStreams Number of ZSTD-compressed attribute streams +16 4 u32 tocByteOffset Byte offset from file start to TOC +20 12 u8[12] reserved Zero-filled +``` + +### 3.2 Legacy Header (v1–v3, 16 bytes) + +``` +Offset Size Type Field Description +────── ──── ──── ───── ─────────── +0 4 u32 magic 0x5053474E +4 4 u32 version 1, 2, or 3 +8 4 u32 numPoints Total number of Gaussians +12 1 u8 shDegree SH degree (0–4) +13 1 u8 fractionalBits Position precision +14 1 u8 flags bit0: antialiased +15 1 u8 reserved Zero +``` + +--- + +## 4. Attribute Encoding + +All attribute data is stored as **non-interleaved** `uint8_t` arrays. Each attribute occupies a contiguous sub-buffer within its stream. + +### 4.1 Position + +**Encoding**: 24-bit signed fixed-point, little-endian byte order. + +Per coordinate (x, y, z individually): + +``` +Encode: int32_t fixed = round(value × 2^fractionalBits) + buf[0] = fixed & 0xFF + buf[1] = (fixed >> 8) & 0xFF + buf[2] = (fixed >> 16) & 0xFF + +Decode: int32_t raw = buf[0] | (buf[1] << 8) | (buf[2] << 16) + if (raw & 0x800000) raw |= 0xFF000000 // sign-extend 24→32 bits + value = (float)raw / 2^fractionalBits +``` + +**Bytes per point**: 9 (3 coordinates × 3 bytes each) + +**Default precision**: `fractionalBits = 12` → ~0.25 mm resolution at unit scale. + +### 4.2 Scale + +**Encoding**: Logarithmic scale quantized to uint8. + +``` +Encode: byte = clamp(round((log_scale + 10.0) × 16.0), 0, 255) + +Decode: log_scale = byte / 16.0 - 10.0 +``` + +**Bytes per point**: 3 (scale_x, scale_y, scale_z) + +The log scale is the natural logarithm of the Gaussian's standard deviation along each axis: `σ_i = exp(scale_i)`. + +### 4.3 Color (SH DC) + +**Encoding**: Spherical Harmonics degree-0 coefficients as wide-range RGB. + +``` +Encode: byte = clamp(round(f_dc × colorScale × 255 + 127.5), 0, 255) + +Decode: f_dc = (byte / 255.0 - 0.5) / colorScale +``` + +**Bytes per point**: 3 (R, G, B) + +`colorScale = 0.15` allows the encoded range to exceed [0, 1] slightly (up to ~1.7), accommodating SH DC coefficients that represent colors outside the displayable gamut — higher-order SH bands can bring them back into range during rendering. + +To convert between f_dc and linear RGB: + +``` +RGB = 0.5 + C0 × f_dc where C0 = 0.28209479177387814 (SH constant) +f_dc = (RGB - 0.5) / C0 +``` + +### 4.4 Alpha (Opacity) + +**Encoding**: Sigmoid-activated opacity. + +``` +Encode: byte = clamp(round(sigmoid(opacity) × 255), 0, 255) + where sigmoid(x) = 1 / (1 + exp(-x)) + +Decode: norm = byte / 255.0 + opacity = log(norm / (1 - norm)) // inverse sigmoid +``` + +**Bytes per point**: 1 + +The stored opacity is in **logit space** (inverse logistic). `opacity = 0` → alpha ≈ 0.5 after sigmoid; `opacity → +∞` → alpha → 1. + +### 4.5 Rotation (Quaternion) + +**v3+ Encoding**: Smallest-three quaternion packing (4 bytes per point). + +The rotation quaternion `{w, x, y, z}` is normalized, then the component with the largest absolute value is identified as the "dropped" component. The remaining three components are quantized to 9-bit magnitude + 1-bit sign. Since a unit quaternion satisfies `w²+x²+y²+z²=1`, the dropped component can be recovered from the other three. + +**Encode**: +``` +1. Normalize quaternion: len = sqrt(w²+x²+y²+z²); {w,x,y,z} /= len +2. Find largest absolute component index: iMax ∈ {0(w),1(x),2(y),3(z)} +3. If quat[iMax] < 0, negate entire quaternion (ensures dropped component > 0) +4. For each of the 3 non-dropped components (iterating from 3 down to 0): + mag = clamp(round(|q[i]| / (1/√2) × 511), 0, 511) + sign = (q[i] < 0) ? 1 : 0 + pack = (pack << 10) | (sign << 9) | mag +5. Final uint32: upper 2 bits = iMax, lower 30 bits = 3×(sign+mag) +``` + +**Decode**: +``` +1. iMax = packed >> 30 +2. For i = 3 down to 0, i ≠ iMax: + mag = packed & 511 + sign = (packed >> 9) & 1 + q[i] = (1/√2) × mag / 511 × (sign ? -1 : 1) + sumSq += q[i]² + packed >>= 10 +3. q[iMax] = sqrt(1 - sumSq) +``` + +**v2 Encoding**: First-three quaternion packing (3 bytes, legacy). +``` +Encode: byte[i] = clamp(round((q[i] / 127.5 + 1) × 127.5), 0, 255) for i=x,y,z +Decode: q[i] = byte[i] / 127.5 - 1.0 for i=x,y,z + q[w] = sqrt(1 - qx² - qy² - qz²) +``` + +### 4.6 Spherical Harmonics + +**Encoding**: Signed float → uint8 with configurable quantization. + +``` +Encode: q = round(value × 128 + 128) // map [-1,+1] → [0,255] + q = (q + bucketSize/2) / bucketSize × bucketSize // quantize + byte = clamp(q, 0, 255) + +Decode: value = (byte - 128.0) / 128.0 +``` + +**Bytes per point**: `shDim × 3`, where `shDim` depends on `shDegree`: + +| shDegree | shDim | Bytes/point | Coeffs (R,G,B each) | +|----------|-------|-------------|---------------------| +| 0 | 0 | 0 | — | +| 1 | 3 | 9 | SH band 1: 3 coeffs × 3 channels | +| 2 | 8 | 24 | SH bands 1+2: 8 coeffs × 3 channels | +| 3 | 15 | 45 | SH bands 1–3: 15 coeffs × 3 channels | +| 4 | 24 | 72 | SH bands 1–4: 24 coeffs × 3 channels | + +**Data layout**: The inner (fastest-varying) axis is the color channel (R, G, B), and the outer axis is the SH coefficient index. For shDegree=3 the order is: + +``` +f_rest_0 = sh1n1_r, f_rest_1 = sh1n1_g, f_rest_2 = sh1n1_b, +f_rest_3 = sh10_r, f_rest_4 = sh10_g, f_rest_5 = sh10_b, +f_rest_6 = sh1p1_r, f_rest_7 = sh1p1_g, f_rest_8 = sh1p1_b, +... (bands 2 and 3 follow) +``` + +**Quantization**: +- `sh1Bits` (default 5): quantization bits for degree-1 coefficients (first 9 slots) +- `shRestBits` (default 4): quantization bits for degree-2+ coefficients +- `bucketSize = 1 << (8 - bits)`: e.g., sh1Bits=5 → bucketSize=8 → 32 quantization levels + +Quantization is applied at **encode time** only to improve ZSTD compression by reducing entropy. The decoder does not need these values — the zero-filled lower bits naturally emerge after decompression. + +--- + +## 5. Per-Point Attribute Summary + +| Attribute | Uncompressed Bytes | Type | Range | Formula | +|-----------|-------------------|------|-------|---------| +| Position | 9 (3×3) | int24 LE | arbitrary | `round(v × 2^12)` | +| Alpha | 1 | uint8 | [0,255] | `sigmoid(opacity) × 255` | +| Color | 3 | uint8 | [0,255] | `(f_dc × 0.15 + 0.5) × 255` | +| Scale | 3 | uint8 | [0,255] | `(log_scale + 10) × 16` | +| Rotation | 4 | uint32 LE | 10-10-10-2 | smallest-three packing | +| SH | shDim×3 | uint8 | [0,255] | `(coeff × 128 + 128)` | + +**Total per point (no SH)**: `9 + 1 + 3 + 3 + 4 = 20 bytes` uncompressed. + +**Total per point (shDegree=3)**: `20 + 45 = 65 bytes` uncompressed. + +--- + +## 6. Coordinate System + +SPZ stores data in the **RUB (Right, Up, Back)** coordinate system, matching the Three.js convention: + +``` ++X = Right ++Y = Up ++Z = Back (toward viewer in a right-handed system) +``` + +Important reference points: + +| System | Code | Convention | +|--------|------|------------| +| RUB | 4 | SPZ storage format; Three.js | +| RDF | 6 | PLY format standard | +| LUF | 7 | GLB / glTF format | +| RUF | 8 | Unity | + +The 4-bit encoding scheme: `(rightFlag, upFlag, forwardFlag, swapYZ)`. + +**Within-family** conversions (same `swapYZ` bit) use sign flips only. +**Cross-family** conversions (different `swapYZ` bit) additionally apply `R_x(±π/2)` analytic rotation to positions, quaternions, and SH coefficients. + +For details, see `splat/maths/coordinate-converter.h` and the reference implementation in `splat-types.h`. + +--- + +## 7. File Detection + +``` +1. Read first 4 bytes as uint32_t LE +2. If value == 0x5053474E: + Read next 4 bytes as uint32_t LE (version) + If version >= 4: parse as NGSP v4 with ZSTD streams + Else: parse as legacy format (v2–v3) +3. Else if first 2 bytes == 0x1F 0x8B (GZip magic): + Decompress with GZip, then parse as legacy format +4. Else: not a valid SPZ file +``` + +--- + +## 8. References + +- [SPZ GitHub Repository](https://github.com/nianticlabs/spz) +- [SPZ: Open-Source Gaussian Splat Format](https://scaniverse.com/news/spz-open-source-gaussian-splat-file-format) +- [3D Gaussian Splatting Paper](https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/) +- SplatLib implementation: `src/io/spz_reader.cpp`, `src/io/spz_encoder.cpp` From 55cc50fe226e2230f4c7ee1a925c0cc685f5172e Mon Sep 17 00:00:00 2001 From: "Merlot.Qi" Date: Tue, 2 Jun 2026 17:48:00 +0800 Subject: [PATCH 7/8] feat(lcc2): implement LCC2 reader and writer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reader: - Lcc2Scene/Lcc2Node data structures (Eigen bounding box, optional 3dgs/mesh refs) - JSON tree parse with recursive node builder - Format dispatch by file extension (PLY/SPZ/SOG via existing readers) - Lazy loading with file-index cache - flattenLcc2Scene: collect all leaf data + concatenate Writer: - Quadtree spatial partition (reused from LCC1) - Per-cell SPZ file output via writeSpz - Two-level JSON tree: root → cell nodes → LOD leaves - Auto GUID generation, configurable cell size and output name ~380 lines total (vs LCC1's ~1200). LCC2 delegates encoding to standard formats — no custom binary codec needed. Co-Authored-By: Claude Opus 4.8 --- include/splat/io/lcc2_reader.h | 70 +++++++++++ include/splat/io/lcc2_writer.h | 31 +++++ src/io/lcc2_reader.cpp | 161 ++++++++++++++++++++++++ src/io/lcc2_writer.cpp | 223 +++++++++++++++++++++++++++++++++ tests/CMakeLists.txt | 4 + tests/lcc2_test.cpp | 117 +++++++++++++++++ 6 files changed, 606 insertions(+) create mode 100644 include/splat/io/lcc2_reader.h create mode 100644 include/splat/io/lcc2_writer.h create mode 100644 src/io/lcc2_reader.cpp create mode 100644 src/io/lcc2_writer.cpp create mode 100644 tests/lcc2_test.cpp diff --git a/include/splat/io/lcc2_reader.h b/include/splat/io/lcc2_reader.h new file mode 100644 index 0000000..362b44b --- /dev/null +++ b/include/splat/io/lcc2_reader.h @@ -0,0 +1,70 @@ +/** + * @file splat/io/lcc2_reader.h + * @brief LCC2 data structures and reader API. + * + * LCC2 is XGRIDS' second-generation 3DGS container format. It stores a JSON + * tree of spatial nodes, where each leaf node references splat data in + * standard formats (PLY, SPZ, SOG) by file index + start/count range. + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace splat { + +class DataTable; + +struct Lcc2NodeData3dgs { + int name = -1; // index into splatFiles[] + int start = 0; + int count = 0; +}; + +struct Lcc2NodeDataMesh { + int name = -1; // index into meshFiles[] + int vertex = 0; + int face = 0; +}; + +struct Lcc2Node { + std::string id; + Eigen::AlignedBox3f boundingBox; + int childNum = 0; + std::vector> children; + + std::optional d3dgs; + std::optional dmesh; + + bool isLeaf() const { return childNum == 0; } +}; + +struct Lcc2Scene { + std::string version; + std::string name; + std::string description; + std::string guid; + std::string fileType; + int totalSplats = 0; + int totalLevels = 0; + std::vector lodSplats; + std::vector splatFiles; + std::unique_ptr root; + + mutable std::map> loadedData; +}; + +Lcc2Scene readLcc2(const std::filesystem::path& lcc2Path); + +const DataTable* loadNodeData(const Lcc2Scene& scene, const Lcc2Node& node, + const std::filesystem::path& baseDir); + +std::unique_ptr flattenLcc2Scene(const Lcc2Scene& scene, + const std::filesystem::path& baseDir); + +} // namespace splat diff --git a/include/splat/io/lcc2_writer.h b/include/splat/io/lcc2_writer.h new file mode 100644 index 0000000..9bcfdbf --- /dev/null +++ b/include/splat/io/lcc2_writer.h @@ -0,0 +1,31 @@ +/** + * @file splat/io/lcc2_writer.h + * @brief Write LCC2 scenes from splat data. + */ +#pragma once + +#include +#include +#include + +namespace splat { + +class DataTable; + +struct Lcc2WriteConfig { + float cellSizeX = 30.0f; + float cellSizeY = 30.0f; + std::string outputFormat = "spz"; // "spz" (ply/sog via existing writers) + std::string fileType = "Quality"; // "Quality" | "Portable" + std::string name = "XGrids Splats"; + std::string description; + std::string guid; // auto-generated if empty +}; + +/// Write LCC2 scene from per-LOD DataTables. +/// lods[0] = LOD0 (full detail), lods[1] = LOD1, etc. +void writeLcc2(const std::filesystem::path& outputDir, + const std::vector& lods, + const Lcc2WriteConfig& config = {}); + +} // namespace splat diff --git a/src/io/lcc2_reader.cpp b/src/io/lcc2_reader.cpp new file mode 100644 index 0000000..f873829 --- /dev/null +++ b/src/io/lcc2_reader.cpp @@ -0,0 +1,161 @@ +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +using json = nlohmann::json; + +namespace splat { + +namespace { + +Eigen::AlignedBox3f parseBoundingBox(const json& bb) { + Eigen::AlignedBox3f box; + box.min() = Eigen::Vector3f(bb["min"][0].get(), bb["min"][1].get(), bb["min"][2].get()); + box.max() = Eigen::Vector3f(bb["max"][0].get(), bb["max"][1].get(), bb["max"][2].get()); + return box; +} + +std::unique_ptr parseNode(const json& j) { + auto node = std::make_unique(); + node->id = j["id"].get(); + node->boundingBox = parseBoundingBox(j["boundingBox"]); + node->childNum = j.value("childNum", 0); + + if (node->childNum > 0 && j.contains("child")) { + for (auto& [key, childJ] : j["child"].items()) { + node->children.push_back(parseNode(childJ)); + } + } else if (j.contains("data")) { + auto& data = j["data"]; + if (data.contains("3dgs")) { + auto& d = data["3dgs"]; + node->d3dgs = Lcc2NodeData3dgs{d.value("name", -1), d.value("start", 0), d.value("count", 0)}; + } + if (data.contains("mesh")) { + auto& m = data["mesh"]; + node->dmesh = Lcc2NodeDataMesh{m.value("name", -1), m.value("vertex", 0), m.value("face", 0)}; + } + } + return node; +} + +} // namespace + +Lcc2Scene readLcc2(const std::filesystem::path& lcc2Path) { + std::ifstream f(lcc2Path); + if (!f) throw std::runtime_error("Cannot open: " + lcc2Path.u8string()); + + json j = json::parse(f); + + Lcc2Scene scene; + scene.version = j.value("version", "0.0.0"); + scene.name = j.value("name", ""); + scene.description = j.value("description", ""); + scene.guid = j.value("guid", ""); + scene.fileType = j.value("fileType", "Quality"); + scene.totalSplats = j.value("totalSplats", 0); + scene.totalLevels = j.value("totalLevels", 1); + + if (j.contains("lodSplats")) { + for (auto& v : j["lodSplats"]) scene.lodSplats.push_back(v.get()); + } + + if (j.contains("splatFiles")) { + for (auto& sf : j["splatFiles"]) scene.splatFiles.push_back(sf.get()); + } + + scene.root = parseNode(j["root"]); + return scene; +} + +const DataTable* loadNodeData(const Lcc2Scene& scene, const Lcc2Node& node, const std::filesystem::path& baseDir) { + if (!node.d3dgs || node.d3dgs->name < 0) return nullptr; + + int fileIdx = node.d3dgs->name; + if (fileIdx < 0 || fileIdx >= static_cast(scene.splatFiles.size())) { + throw std::runtime_error("LCC2: splatFiles index out of range: " + std::to_string(fileIdx)); + } + + auto it = scene.loadedData.find(fileIdx); + if (it != scene.loadedData.end()) return it->second.get(); + + auto fullPath = baseDir / scene.splatFiles[fileIdx]; + auto ext = fullPath.extension().string(); + + std::unique_ptr table; + if (ext == ".ply") + table = readPly(fullPath); + else if (ext == ".spz") + table = readSpz(fullPath); + else if (ext == ".sog") + table = readSog(fullPath, fullPath); + else + throw std::runtime_error("LCC2: unknown data format: " + ext); + + scene.loadedData[fileIdx] = std::move(table); + return scene.loadedData[fileIdx].get(); +} + +std::unique_ptr flattenLcc2Scene(const Lcc2Scene& scene, const std::filesystem::path& baseDir) { + std::vector parts; + std::vector> slices; + + std::function collect = [&](const Lcc2Node& node) { + if (node.isLeaf() && node.d3dgs && node.d3dgs->count > 0) { + const DataTable* full = loadNodeData(scene, node, baseDir); + if (!full) return; + + int start = node.d3dgs->start; + int count = node.d3dgs->count; + + std::vector indices; + indices.reserve(count); + for (int i = 0; i < count; ++i) indices.push_back(static_cast(start + i)); + + parts.push_back(full); + slices.push_back(std::move(indices)); + } + for (auto& child : node.children) collect(*child); + }; + collect(*scene.root); + + if (parts.empty()) return std::make_unique(); + + // Build result by permuting each part and concatenating + std::vector> permuted; + size_t totalRows = 0; + for (size_t p = 0; p < parts.size(); ++p) { + permuted.push_back(parts[p]->permuteRows(slices[p])); + totalRows += slices[p].size(); + } + + // Concatenate: clone first, then add rows from rest + auto result = permuted[0]->clone(); + for (size_t p = 1; p < permuted.size(); ++p) { + size_t baseRows = result->getNumRows(); + size_t addRows = permuted[p]->getNumRows(); + + // Extend each column in result with data from permuted[p] + for (size_t ci = 0; ci < permuted[p]->getNumColumns(); ++ci) { + const auto& srcCol = permuted[p]->getColumn(ci); + const std::string& colName = srcCol.name; + if (!result->hasColumn(colName)) continue; + + auto& dstCol = result->getColumnByName(colName); + for (size_t ri = 0; ri < addRows; ++ri) { + dstCol.setValue(baseRows + ri, srcCol.getValue(ri)); + } + } + } + + return result; +} + +} // namespace splat diff --git a/src/io/lcc2_writer.cpp b/src/io/lcc2_writer.cpp new file mode 100644 index 0000000..10f69b9 --- /dev/null +++ b/src/io/lcc2_writer.cpp @@ -0,0 +1,223 @@ +#include + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using json = nlohmann::json; +namespace fs = std::filesystem; + +namespace splat { + +namespace { + +std::string generateGuid() { + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution<> dis(0, 15); + std::stringstream ss; + for (int i = 0; i < 32; ++i) ss << std::hex << dis(gen); + return ss.str(); +} + +void computeBoundingBox(const DataTable& table, Eigen::Vector3f& bboxMin, + Eigen::Vector3f& bboxMax) { + auto xs = table.getColumnByName("x").asSpan(); + auto ys = table.getColumnByName("y").asSpan(); + auto zs = table.getColumnByName("z").asSpan(); + size_t n = table.getNumRows(); + float fmax = std::numeric_limits::max(); + float fmin = std::numeric_limits::lowest(); + bboxMin = Eigen::Vector3f(fmax, fmax, fmax); + bboxMax = Eigen::Vector3f(fmin, fmin, fmin); + for (size_t i = 0; i < n; ++i) { + bboxMin = bboxMin.cwiseMin(Eigen::Vector3f(xs[i], ys[i], zs[i])); + bboxMax = bboxMax.cwiseMax(Eigen::Vector3f(xs[i], ys[i], zs[i])); + } +} + +std::map> buildSpatialGrid(const DataTable& table, + float cellSizeX, float cellSizeY, + const Eigen::Vector3f& bboxMin) { + auto xs = table.getColumnByName("x").asSpan(); + auto ys = table.getColumnByName("y").asSpan(); + size_t n = table.getNumRows(); + std::map> grid; + for (size_t i = 0; i < n; ++i) { + int cx = std::clamp(static_cast(std::floor((xs[i] - bboxMin.x()) / cellSizeX)), 0, 65535); + int cy = std::clamp(static_cast(std::floor((ys[i] - bboxMin.y()) / cellSizeY)), 0, 65535); + grid[(static_cast(cy) << 16) | static_cast(cx)].push_back(i); + } + return grid; +} + +std::string writeCellFile(const fs::path& dataDir, int fileIdx, const DataTable& table, + const std::vector& indices) { + std::vector u32(indices.begin(), indices.end()); + auto subset = table.permuteRows(u32); + + std::string fname = std::to_string(fileIdx) + ".spz"; + auto fullPath = dataDir / fname; + writeSpz(fullPath, *subset); + return "data/3dgs/" + fname; +} + +} // namespace + +void writeLcc2(const fs::path& outputDir, const std::vector& lods, + const Lcc2WriteConfig& config) { + if (lods.empty()) throw std::runtime_error("LCC2 writer: at least one LOD required"); + + fs::create_directories(outputDir); + auto dataDir = outputDir / "data" / "3dgs"; + fs::create_directories(dataDir); + + size_t numLods = lods.size(); + + // Build spatial grids per LOD + std::vector>> grids(numLods); + std::set allCellIds; + Eigen::Vector3f globalMin, globalMax; + computeBoundingBox(*lods[0], globalMin, globalMax); + + for (size_t lod = 0; lod < numLods; ++lod) { + grids[lod] = buildSpatialGrid(*lods[lod], config.cellSizeX, config.cellSizeY, globalMin); + for (auto& kv : grids[lod]) allCellIds.insert(kv.first); + } + + // Encode cells + struct CellEntry { + uint32_t cellId; + size_t lod; + int fileIdx; + int splatCount; + Eigen::AlignedBox3f bbox; + }; + std::vector entries; + std::vector splatFiles; + std::vector splatsPerLod(numLods, 0); + int fileIdx = 0; + + for (uint32_t cellId : allCellIds) { + for (size_t lod = 0; lod < numLods; ++lod) { + auto it = grids[lod].find(cellId); + if (it == grids[lod].end() || it->second.empty()) continue; + + auto& xs = lods[lod]->getColumnByName("x").asSpan(); + auto& ys = lods[lod]->getColumnByName("y").asSpan(); + auto& zs = lods[lod]->getColumnByName("z").asSpan(); + Eigen::AlignedBox3f cellBBox; + cellBBox.min() = Eigen::Vector3f(xs[it->second[0]], ys[it->second[0]], zs[it->second[0]]); + cellBBox.max() = cellBBox.min(); + for (auto idx : it->second) + cellBBox.extend(Eigen::Vector3f(xs[idx], ys[idx], zs[idx])); + + std::string rel = writeCellFile(dataDir, fileIdx, *lods[lod], it->second); + splatFiles.push_back(rel); + + entries.push_back( + {cellId, lod, fileIdx, static_cast(it->second.size()), cellBBox}); + splatsPerLod[lod] += it->second.size(); + fileIdx++; + } + } + + size_t totalSplats = 0; + for (auto c : splatsPerLod) totalSplats += c; + + // Build tree: root → per-cell internal nodes → per-LOD leaves + std::vector cellNodes; + size_t e = 0; + for (uint32_t cellId : allCellIds) { + int cx = cellId & 0xFFFF, cy = (cellId >> 16) & 0xFFFF; + std::string cellIdStr = std::to_string(cx) + "-" + std::to_string(cy); + + Eigen::AlignedBox3f mergedBox; + bool first = true; + std::vector lodLeaves; + while (e < entries.size() && entries[e].cellId == cellId) { + auto& ent = entries[e]; + if (first) { + mergedBox = ent.bbox; + first = false; + } else { + mergedBox.extend(ent.bbox); + } + + json leaf; + leaf["id"] = "0-" + cellIdStr + "-" + std::to_string(ent.lod); + leaf["boundingBox"]["min"] = {ent.bbox.min().x(), ent.bbox.min().y(), ent.bbox.min().z()}; + leaf["boundingBox"]["max"] = {ent.bbox.max().x(), ent.bbox.max().y(), ent.bbox.max().z()}; + leaf["childNum"] = 0; + leaf["data"]["3dgs"]["name"] = ent.fileIdx; + leaf["data"]["3dgs"]["start"] = 0; + leaf["data"]["3dgs"]["count"] = ent.splatCount; + lodLeaves.push_back(leaf); + e++; + } + + json cellNode; + cellNode["id"] = "0-" + cellIdStr; + cellNode["boundingBox"]["min"] = {mergedBox.min().x(), mergedBox.min().y(), + mergedBox.min().z()}; + cellNode["boundingBox"]["max"] = {mergedBox.max().x(), mergedBox.max().y(), + mergedBox.max().z()}; + cellNode["childNum"] = static_cast(lodLeaves.size()); + json childObj = json::object(); + for (size_t i = 0; i < lodLeaves.size(); ++i) + childObj[std::to_string(i)] = lodLeaves[i]; + cellNode["child"] = childObj; + cellNodes.push_back(cellNode); + } + + // Root + json root; + root["id"] = "0"; + root["boundingBox"]["min"] = {globalMin.x(), globalMin.y(), globalMin.z()}; + root["boundingBox"]["max"] = {globalMax.x(), globalMax.y(), globalMax.z()}; + root["childNum"] = static_cast(cellNodes.size()); + json rootChild = json::object(); + for (size_t i = 0; i < cellNodes.size(); ++i) + rootChild[std::to_string(i)] = cellNodes[i]; + root["child"] = rootChild; + + // Scene JSON + std::string guid = config.guid.empty() ? generateGuid() : config.guid; + json scene; + scene["version"] = "0.0.3"; + scene["name"] = config.name; + scene["description"] = config.description; + scene["guid"] = guid; + scene["fileType"] = config.fileType; + scene["totalSplats"] = totalSplats; + scene["totalLevels"] = numLods; + + json lodArr = json::array(); + for (auto c : splatsPerLod) lodArr.push_back(static_cast(c)); + scene["lodSplats"] = lodArr; + scene["splatFiles"] = splatFiles; + scene["root"] = root; + scene["source"] = "lcc"; + scene["dataType"] = "DIMENVUE"; + scene["epsg"] = 0; + scene["splatType"] = "." + config.outputFormat; + + std::ofstream out(outputDir / (config.name + ".lcc2")); + if (!out) throw std::runtime_error("Failed to create scene.lcc2"); + out << std::setprecision(15) << scene.dump(1, '\t') << '\n'; +} + +} // namespace splat diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 69ea4ba..09b6559 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -65,3 +65,7 @@ add_test(NAME SplatSpzWriterTests COMMAND SplatSpzWriterTests) add_executable(SplatCoordinateConverterTests coordinate_converter_test.cpp) target_link_libraries(SplatCoordinateConverterTests PRIVATE SPLAT::splat) add_test(NAME SplatCoordinateConverterTests COMMAND SplatCoordinateConverterTests) + +add_executable(SplatLcc2Tests lcc2_test.cpp) +target_link_libraries(SplatLcc2Tests PRIVATE SPLAT::splat) +add_test(NAME SplatLcc2Tests COMMAND SplatLcc2Tests) diff --git a/tests/lcc2_test.cpp b/tests/lcc2_test.cpp new file mode 100644 index 0000000..64552a7 --- /dev/null +++ b/tests/lcc2_test.cpp @@ -0,0 +1,117 @@ +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +static void check(bool cond, const char* msg) { + if (!cond) { fprintf(stderr, "FAIL: %s\n", msg); std::exit(1); } +} + +static splat::DataTable makeTable(int n) { + std::vector cols; + auto add = [&](const char* name) -> float* { + std::vector d(n); + float* p = d.data(); + cols.push_back({name, std::move(d)}); + return p; + }; + float* xs = add("x"), *ys = add("y"), *zs = add("z"); + float* s0 = add("scale_0"), *s1 = add("scale_1"), *s2 = add("scale_2"); + float* cr = add("f_dc_0"), *cg = add("f_dc_1"), *cb = add("f_dc_2"); + float* op = add("opacity"); + float* r0 = add("rot_0"), *r1 = add("rot_1"), *r2 = add("rot_2"), *r3 = add("rot_3"); + + for (int i = 0; i < n; ++i) { + xs[i] = i * 1.0f; ys[i] = i * 2.0f; zs[i] = i * 3.0f; + s0[i] = -1; s1[i] = 0; s2[i] = 1; + cr[i] = 0.1f; cg[i] = -0.1f; cb[i] = 0.2f; + op[i] = 0.5f; + r0[i] = 1; r1[i] = 0; r2[i] = 0; r3[i] = 0; + } + return splat::DataTable(std::move(cols)); +} + +int main() { + namespace fs = std::filesystem; + using namespace splat; + + // ── Test 1: Reader parses synthetic JSON ── + { + auto tmpDir = fs::temp_directory_path() / "lcc2_test_read"; + fs::create_directories(tmpDir); + auto lcc2Path = tmpDir / "test.lcc2"; + + { + std::ofstream f(lcc2Path); + f << R"({ + "version": "0.0.3", + "name": "TestScene", + "fileType": "Quality", + "totalSplats": 100, + "totalLevels": 1, + "lodSplats": [100], + "splatFiles": [], + "root": { + "id": "0", + "boundingBox": {"min": [0,0,0], "max": [10,10,10]}, + "childNum": 1, + "child": { + "0": { + "id": "0-0", + "boundingBox": {"min": [0,0,0], "max": [5,5,5]}, + "childNum": 0, + "data": {"3dgs": {"name": -1, "start": 0, "count": 0}} + } + } + } + })"; + } + + auto scene = readLcc2(lcc2Path); + check(scene.version == "0.0.3", "version"); + check(scene.name == "TestScene", "name"); + check(scene.fileType == "Quality", "fileType"); + check(scene.totalSplats == 100, "totalSplats"); + check(scene.root != nullptr, "root exists"); + check(scene.root->children.size() == 1, "root has 1 child"); + check(scene.root->children[0]->isLeaf(), "child is leaf"); + check(scene.root->children[0]->d3dgs.has_value(), "child has 3dgs"); + printf("PASS: reader JSON parse\n"); + fs::remove_all(tmpDir); + } + + // ── Test 2: Writer round-trip ── + { + auto tmpDir = fs::temp_directory_path() / "lcc2_test_rt"; + fs::create_directories(tmpDir); + + auto dt = makeTable(5); + Lcc2WriteConfig cfg; + cfg.name = "roundtrip"; + cfg.outputFormat = "spz"; + cfg.cellSizeX = 100.0f; + cfg.cellSizeY = 100.0f; + + writeLcc2(tmpDir, {&dt}, cfg); + check(fs::exists(tmpDir / "roundtrip.lcc2"), "scene file created"); + check(fs::exists(tmpDir / "data" / "3dgs"), "data dir created"); + printf("PASS: writer output files\n"); + + auto scene = readLcc2(tmpDir / "roundtrip.lcc2"); + check(scene.totalSplats == 5, "totalSplats preserved"); + check(scene.totalLevels == 1, "totalLevels preserved"); + check(!scene.splatFiles.empty(), "splatFiles non-empty"); + printf("PASS: writer round-trip\n"); + + fs::remove_all(tmpDir); + } + + printf("\n=== All LCC2 tests passed ===\n"); + return 0; +} From 0ae05d8029c5329f237a4a60158ff3ccac53ae9c Mon Sep 17 00:00:00 2001 From: "Merlot.Qi" Date: Wed, 3 Jun 2026 09:36:27 +0800 Subject: [PATCH 8/8] feat(cmake): enhance zstd target selection for shared/static builds --- CMakeLists.txt | 21 +++++++++++++++++++++ cmake/SplatConfig.cmake.in | 1 + src/CMakeLists.txt | 2 +- 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 36cf4ce..26dad4c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -96,6 +96,27 @@ find_package(nlohmann_json CONFIG REQUIRED) find_package(absl CONFIG REQUIRED) find_package(ZLIB REQUIRED) find_package(zstd CONFIG REQUIRED) + +# Select the appropriate zstd target based on build type +if(BUILD_SPLAT_SHARED) + if(TARGET zstd::libzstd_shared) + set(SPLAT_ZSTD_TARGET zstd::libzstd_shared) + elseif(TARGET zstd::libzstd) + set(SPLAT_ZSTD_TARGET zstd::libzstd) + else() + set(SPLAT_ZSTD_TARGET zstd::libzstd_static) + endif() +else() + if(TARGET zstd::libzstd_static) + set(SPLAT_ZSTD_TARGET zstd::libzstd_static) + elseif(TARGET zstd::libzstd) + set(SPLAT_ZSTD_TARGET zstd::libzstd) + else() + set(SPLAT_ZSTD_TARGET zstd::libzstd_shared) + endif() +endif() +message(STATUS "SplatLib: Using zstd target: ${SPLAT_ZSTD_TARGET}") + find_package(WebP REQUIRED) if(BUILD_SPLAT_VISUALIZATION) find_package(VTK REQUIRED COMPONENTS ${SPLAT_VTK_COMPONENTS}) diff --git a/cmake/SplatConfig.cmake.in b/cmake/SplatConfig.cmake.in index 62dfe45..03c36ca 100644 --- a/cmake/SplatConfig.cmake.in +++ b/cmake/SplatConfig.cmake.in @@ -7,6 +7,7 @@ find_dependency(Eigen3 CONFIG) find_dependency(nlohmann_json CONFIG) find_dependency(absl CONFIG) find_dependency(ZLIB) +find_dependency(zstd CONFIG) find_dependency(WebP) if(@BUILD_SPLAT_VISUALIZATION@) find_dependency(VTK COMPONENTS CommonCore CommonDataModel InteractionStyle InteractionWidgets RenderingAnnotation RenderingCore RenderingOpenGL2) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 36108ec..514efe8 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -95,7 +95,7 @@ target_link_libraries(splat nlohmann_json::nlohmann_json WebP::webp ZLIB::ZLIB - zstd::libzstd + ${SPLAT_ZSTD_TARGET} absl::base absl::strings PRIVATE