diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9844da0..d6b820b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,7 +4,6 @@ on: push: tags: - "v*" - - "!v*-runtime-green" permissions: contents: read diff --git a/.gitignore b/.gitignore index 650c80a..8706554 100644 --- a/.gitignore +++ b/.gitignore @@ -51,3 +51,9 @@ desktop.ini tools/siegfried/windows-x86_64/sf.exe tools/siegfried/windows-x86_64/default.sig tools/siegfried/windows-x86_64/tool-manifest.json +/target-alpha6-cli-crash/ +/target-alpha6-cli-delta/ + +/target-alpha6-cli-soak/ + +/target-alpha6-cli-watch-v3/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 229f572..471528c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,65 +2,14 @@ All notable changes to MailVault Collection Profiler are documented here. -The project follows Semantic Versioning once the public API reaches a stable release. Alpha builds -may change local profiler schemas and command contracts through explicit migrations. - -## 0.1.0-alpha.4 — 2026-07-21 - -- Added exact format identification with pinned Siegfried `1.11.6` and PRONOM `v124`. -- Added versioned PUID assertions, all-match retention, bounded batches, resume checkpoints and format UI/CLI. -- Added verified sidecar acquisition, Tauri resource bundling and third-party notices. -- Added migration `0006_exact_format_identification.sql`. -- Updated English/Persian documentation and sanitized release screenshots. -- Fixed Windows JSON probing and real-batch execution by resolving the verified signature through - `-home` and passing only `default.sig`, avoiding Siegfried 1.11.6's unescaped header path. -- Added Windows regression coverage for the sidecar command contract. - - -### Added - -- Existing workspace inspection, compatibility reporting and explicit open modes. -- Workspace schema 5 with metadata, append-only review events and review projections. -- Pre-migration SQLite backup and retained migration-failure marker. -- Run catalog and completed-run reopening after a full process restart. -- Single review writer with read-only fallback when another process holds the workspace lock. -- Finding review statuses, required-note policy and append-only SHA-256 event chains. -- Review history integrity validation and write-disable fallback after detected tampering. -- Findings categories for requires-attention, informational evidence, reviewed and all findings. -- Sanitized JSON summary and CSV finding exports using non-reversible short tokens. -- CLI commands for workspace inspection, run listing, finding review and sanitized export. -- Desktop start journey for profiling a new archive or opening an existing workspace. -- Integration tests for reopen persistence, lock contention, review history and sanitized exports. - -### Changed - -- Public application version advanced to `0.1.0-alpha.4`; WiX/MSI version maps to `0.1.0.4`. -- Informational filename/content relationships are separated from warnings requiring attention. -- Quality, CI and release commands use locked Cargo resolution and validate alpha.3 release notes. - -### Fixed during release hardening - -- Centralized signed-to-unsigned SQLite count conversion for explorer, review and workspace queries. -- Split review-event persistence into transactional preparation, insert and projection helpers. -- Replaced manual guarded division with checked review-completion arithmetic. -- Replaced the oversized CLI dispatcher with command-specific handlers. -- Split workspace inspection/opening into focused compatibility, access and integrity helpers. -- Replaced the flat eight-argument Tauri findings command with a typed request object. -- Kept the desktop findings request contract synchronized with the nested Tauri payload. -- Made the documentation screenshot gate recurse through versioned screenshot directories and require the public alpha.3 images explicitly. - -### Validation - -- Passed the complete Windows quality gate, including strict Clippy, 36 Rust tests, TypeScript, Vite, native desktop compilation and npm audit with 0 vulnerabilities. -- Completed the full private exact-format run with 13,684/13,684 objects, 13,636 identified, 46 unknown, 0 ambiguous, 0 tool errors, 51 extension mismatches and 64 distinct PUIDs. -- Verified schema 5-to-6 workspace migration, complete byte coverage and unchanged canonical MailVault SHA-256. - -### Security and privacy - -- Review data stays outside MailVault and original runtime evidence. -- Review notes are excluded from application logs and sanitized exports. -- Existing profiler databases are never created implicitly during reopen. -- Workspace/source overlap is checked after canonicalization. +## 0.1.0-alpha.6 — 2026-07-28 + +- Unified initial, incremental and rebuild profiling into one crash-resumable pipeline. +- Added full consistent metadata reconciliation on every source revision while reusing successful SHA-256 file-stat and exact-format results. +- Added automatic Siegfried/PRONOM identification before a physical run can succeed. +- Added retryable unavailable/tool-error states, stable collection identity aliases and bounded snapshot retention. +- Added Windows gates for initial/no-change, real delta, force-kill resume, 100-cycle soak and independent rebuild equivalence. +- Workspace schema advanced to 7; public application version is `0.1.0-alpha.6` and MSI version is `0.1.0.6`. ## 0.1.0-alpha.2 — 2026-07-19 diff --git a/CITATION.cff b/CITATION.cff index 03c4316..e4aa7be 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -6,7 +6,7 @@ authors: - name: FireXCore repository-code: https://github.com/FireXCore/mailvault-collection-profiler url: https://github.com/FireXCore/mailvault-collection-profiler -version: 0.1.0-alpha.4 +version: 0.1.0-alpha.6 date-released: 2026-07-19 license: Apache-2.0 abstract: >- diff --git a/Cargo.lock b/Cargo.lock index c27b1bb..c390e2b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1926,7 +1926,7 @@ checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "mailvault-profiler-cli" -version = "0.1.0-alpha.4" +version = "0.1.0-alpha.6" dependencies = [ "clap", "profiler-adapter-mailvault", @@ -1941,7 +1941,7 @@ dependencies = [ [[package]] name = "mailvault-profiler-desktop" -version = "0.1.0-alpha.4" +version = "0.1.0-alpha.6" dependencies = [ "profiler-adapter-mailvault", "profiler-core", @@ -2573,7 +2573,7 @@ dependencies = [ [[package]] name = "profiler-adapter-mailvault" -version = "0.1.0-alpha.4" +version = "0.1.0-alpha.6" dependencies = [ "fs2", "hex", @@ -2589,7 +2589,7 @@ dependencies = [ [[package]] name = "profiler-core" -version = "0.1.0-alpha.4" +version = "0.1.0-alpha.6" dependencies = [ "hex", "serde", @@ -2602,7 +2602,7 @@ dependencies = [ [[package]] name = "profiler-engine" -version = "0.1.0-alpha.4" +version = "0.1.0-alpha.6" dependencies = [ "fs2", "hex", @@ -2622,7 +2622,7 @@ dependencies = [ [[package]] name = "profiler-format-siegfried" -version = "0.1.0-alpha.4" +version = "0.1.0-alpha.6" dependencies = [ "hex", "profiler-core", @@ -2636,7 +2636,7 @@ dependencies = [ [[package]] name = "profiler-storage-sqlite" -version = "0.1.0-alpha.4" +version = "0.1.0-alpha.6" dependencies = [ "profiler-core", "rusqlite", diff --git a/Cargo.toml b/Cargo.toml index 51a47a4..26e2d97 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ members = [ resolver = "3" [workspace.package] -version = "0.1.0-alpha.4" +version = "0.1.0-alpha.6" edition = "2024" rust-version = "1.97.1" authors = ["FireXCore"] diff --git a/README.md b/README.md index cfa541e..b5f164e 100644 --- a/README.md +++ b/README.md @@ -16,9 +16,9 @@ -> **Development pre-release:** `0.1.0-alpha.4` adds exact format identification as a versioned, -> resumable profiler stage. Windows release builds bundle a pinned Siegfried `1.11.6` sidecar and -> PRONOM `v124` signature database. The source archive remains read-only. Installers are unsigned. +> **Final feature pre-release:** `0.1.0-alpha.6` combines continuous incremental profiling, +> same-run crash recovery and automatic exact-format identification. Windows builds bundle pinned +> Siegfried `1.11.6` and PRONOM `v124`. MailVault remains read-only and installers are unsigned. ## What the product does @@ -28,11 +28,11 @@ index over that evidence: ```text MailVault archive (read-only) → consistent SQLite snapshot - → physical content inventory - → exact SHA-256 identity and occurrence history - → bounded file-stat verification - → exact format identification with PUID evidence - → searchable desktop and CLI views + → full metadata reconciliation from an immutable snapshot + → cumulative SHA-256 identity and occurrence history + → selective file-stat verification and retry + → automatic exact-format identification with PUID evidence + → durable incremental generations and searchable desktop/CLI views ``` The profiler does not download mail, mutate MailVault, execute attachments, perform OCR, expand @@ -55,10 +55,10 @@ sample-only data: | Physical inventory findings | 1,484 | | Physical inventory warnings / errors | 2 / 0 | -Alpha 3 validated the physical inventory on Windows. Alpha 4 preserves that baseline, adds schema -migration `0006`, and completed a private full-scale exact-format run: 13,684/13,684 objects, -13,636 identified, 46 unknown, 0 ambiguous and 0 tool errors. The canonical MailVault database -SHA-256 remained unchanged. +Alpha 4 completed the Windows quality gate and exact-format run on all 13,684 real objects. Alpha 6 +adds schema migration `0007`, continuous generations, update-aware metadata reconciliation, selective +content reuse, same-run recovery and bounded snapshot retention. Publication still requires the Alpha 6 +real-runtime acceptance matrix to pass on Windows. ## Implemented capabilities @@ -73,7 +73,7 @@ SHA-256 remained unchanged. - Cursor-paginated inventory, content-object detail and append-only finding review. - Sanitized summary and finding export without local paths, filenames, addresses or review notes. -### Exact format identification — Alpha 4 +### Exact format identification - One identification job per unique content object, never per duplicate occurrence. - Pinned Siegfried `1.11.6` executable and PRONOM `v124` signature database. @@ -115,23 +115,47 @@ See [Exact format identification](docs/FORMAT_IDENTIFICATION.md), ```text E:\MailVault-E -E:\MailVault-Profiler-Alpha4 -E:\MailVault-Profiler-Evidence-Alpha4 +E:\MailVault-Profiler-Alpha6 +E:\MailVault-Profiler-Evidence-Alpha6 ``` Full instructions: [Windows installation](docs/INSTALLATION_WINDOWS.md). ## CLI workflow +Run the complete initial or incremental pipeline: + +```powershell +.\target\release\mailvault-profiler.exe profile ` + --archive "E:\MailVault-E" ` + --workspace "E:\MailVault-Profiler-Alpha6" ` + --siegfried ".\tools\siegfried\windows-x86_64\sf.exe" ` + --signature ".\tools\siegfried\windows-x86_64\default.sig" ` + --retain-successful-snapshots 2 +``` + +Keep the same workspace current after future MailVault syncs: + +```powershell +.\target\release\mailvault-profiler.exe profile ` + --archive "E:\MailVault-E" ` + --workspace "E:\MailVault-Profiler-Alpha6" ` + --watch ` + --poll-seconds 300 +``` + +The standalone `formats` commands remain diagnostic tools. The normal `profile` command now runs exact +format identification automatically before the generation can succeed. + Create or reopen the physical inventory first: ```powershell .\target\release\mailvault-profiler.exe workspace inspect ` - --workspace "E:\MailVault-Profiler-Alpha4" ` + --workspace "E:\MailVault-Profiler-Alpha6" ` --json .\target\release\mailvault-profiler.exe runs list ` - --workspace "E:\MailVault-Profiler-Alpha4" ` + --workspace "E:\MailVault-Profiler-Alpha6" ` --json ``` @@ -148,7 +172,7 @@ Run exact identification against a completed physical baseline: ```powershell .\target\release\mailvault-profiler.exe formats identify ` - --workspace "E:\MailVault-Profiler-Alpha4" ` + --workspace "E:\MailVault-Profiler-Alpha6" ` --run "" ` --siegfried ".\tools\siegfried\windows-x86_64\sf.exe" ` --signature ".\tools\siegfried\windows-x86_64\default.sig" ` @@ -165,12 +189,12 @@ Inspect aggregate and object-level results: ```powershell .\target\release\mailvault-profiler.exe formats summary ` - --workspace "E:\MailVault-Profiler-Alpha4" ` + --workspace "E:\MailVault-Profiler-Alpha6" ` --run "" ` --json .\target\release\mailvault-profiler.exe formats list ` - --workspace "E:\MailVault-Profiler-Alpha4" ` + --workspace "E:\MailVault-Profiler-Alpha6" ` --run "" ` --state ambiguous ` --json @@ -190,7 +214,7 @@ Requirements: ```powershell npm ci .\scripts\install-siegfried.ps1 -.\scripts\quality.ps1 +.\scripts\quality-alpha6.ps1 npm run tauri:desktop:bundle ``` @@ -199,22 +223,24 @@ The installer build embeds the verified sidecar resources. Generated `sf.exe`, ` ## Validation status -The following checks were completed in the supplied build environment: +The trusted Alpha 4 predecessor passed 36 Rust tests, strict Clippy, TypeScript, Vite, native Tauri and +the full private exact-format run. Alpha 6 static package gates validate version alignment, +documentation/privacy, JSON/TOML and SQLite schema migration. -- TypeScript type-check and Vite production build: passed. -- Tree-sitter parse of all Rust source files: passed. -- Alpha 3 real profiler database migration from schema 5 to 6: passed. -- SQLite `quick_check` and `foreign_key_check` after migration: passed. -- Real baseline counts preserved: 13,684 objects, 22,068 occurrences, 1,484 findings. -- Source Alpha 3 profiler database SHA-256 unchanged after migration test. -- Synthetic exact-format projection against the migrated schema: passed. +Before publishing Alpha 6, run: -A Rust toolchain was not available in the current isolated build environment, so local semantic -`cargo check`, Clippy, unit tests, Tauri compilation and a real Siegfried run were not claimed. -GitHub Windows CI and the release workflow are configured to install the pinned sidecar and run the -full Rust/desktop gates before producing installers. +```powershell +.\scripts\quality-alpha6.ps1 -BuildInstallers +.\scripts\validate-alpha6-real.ps1 ... +.\scripts\validate-alpha6-delta.ps1 ... +.\scripts\validate-alpha6-crash.ps1 ... +.\scripts\validate-alpha6-soak.ps1 -Cycles 100 ... +.\scripts\validate-alpha6-rebuild-equivalence.ps1 ... +``` -See [Alpha 4 validation evidence](docs/VALIDATION_0.1.0-alpha.4.md). +See [Alpha 6 validation contract](docs/VALIDATION_0.1.0-alpha.6.md), +[continuous profiling](docs/CONTINUOUS_PROFILING.md) and +[real acceptance runbook](docs/ALPHA6_REAL_ACCEPTANCE_FA.md). ## Documentation @@ -231,7 +257,7 @@ See [Alpha 4 validation evidence](docs/VALIDATION_0.1.0-alpha.4.md). - [Privacy](docs/PRIVACY.md) - [Real archive baseline](docs/REAL_ARCHIVE_BASELINE.md) - [Release process](docs/RELEASE_PROCESS.md) -- [Alpha 4 release notes](docs/releases/v0.1.0-alpha.4.md) +- [Alpha 6 release notes](docs/releases/v0.1.0-alpha.6.md) ## Scope intentionally deferred diff --git a/README_FA.md b/README_FA.md index e617342..7c6e750 100644 --- a/README_FA.md +++ b/README_FA.md @@ -4,162 +4,89 @@ # MailVault Collection Profiler -**موجودی فیزیکی، کنترل کیفیت و تشخیص دقیق فرمت فایل برای آرشیوهای MailVault؛ کاملاً محلی و فقط‌خواندنی.** +**پروفایل فنی محلی، فقط‌خواندنی، پیوسته و قابل Resume برای آرشیوهای MailVault.** -[English](README.md) · [دریافت](https://github.com/FireXCore/mailvault-collection-profiler/releases) · [شروع سریع](docs/GETTING_STARTED.md) · [تشخیص دقیق فرمت](docs/FORMAT_IDENTIFICATION.md) · [امنیت](SECURITY.md) +[English](README.md) · [دریافت](https://github.com/FireXCore/mailvault-collection-profiler/releases) · [شروع سریع](docs/GETTING_STARTED.md) · [امنیت](SECURITY.md) -> **نسخه توسعه‌ای:** `0.1.0-alpha.4` مرحله تشخیص دقیق فرمت را به‌صورت Versioned، Resume-safe و -> قابل ممیزی اضافه می‌کند. Build ویندوز، Siegfried `1.11.6` و Signature Database استاندارد -> PRONOM `v124` را به‌صورت Pin‌شده همراه برنامه بسته‌بندی می‌کند. آرشیو MailVault تغییر نمی‌کند. +> `0.1.0-alpha.6` آخرین نسخه Feature است. این نسخه Profile اولیه، ادامه Incremental، Resume همان Run +> پس از قطع، Retry فایل‌های ناموفق و تشخیص خودکار فرمت با Siegfried `1.11.6` و PRONOM `v124` را +> در یک Pipeline واحد ادغام می‌کند. بعد از این Release فقط Maintenance، Security و Compatibility مجاز است. -## این برنامه دقیقاً چه کاری می‌کند؟ - -MailVault مدرک اصلی ایمیل را نگه می‌دارد. Profiler یک Index فنی و قابل بازسازی روی همان مدرک می‌سازد: +## Pipeline نهایی ```text MailVault فقط‌خواندنی - ← Snapshot سازگار SQLite - ← Physical Inventory - ← هویت دقیق SHA-256 و تاریخچه occurrence - ← کنترل فیزیکی فایل‌ها - ← تشخیص دقیق فرمت و PUID - ← Explorer دسکتاپ و CLI + → Snapshot سازگار با SQLite Online Backup + → Reconciliation کامل metadata + → Projection تجمعی بر پایه SHA-256 + → File-stat انتخابی و Retry خطاها + → Exact-format خودکار و PUID + → Generation موفق + Retention محدود Snapshot ``` -Profiler ایمیل دانلود نمی‌کند، MailVault را تغییر نمی‌دهد، فایل را اجرا نمی‌کند، OCR انجام نمی‌دهد، -ZIP/RAR را باز نمی‌کند، سند را Invoice یا Quotation تشخیص نمی‌دهد و چیزی در RMS نمی‌نویسد. +اجرای بعدی روی همان Workspace، Metadata Snapshot جدید را کامل مقایسه می‌کند اما File-stat و Format +موفق و بدون تغییر را دوباره اجرا نمی‌کند. Objectهای Missing، Mismatch، Unavailable یا Tool Error در +Run بعد دوباره امتحان می‌شوند. قطع Process باعث ساخت Profile جدید یا شروع از صفر نمی‌شود؛ همان Run و +Checkpointهای Commit‌شده ادامه پیدا می‌کنند. -## مبنای واقعی طراحی +## مبنای واقعی | معیار | مقدار ثبت‌شده | |---|---:| -| حجم آرشیو | حدود ۲۰ تا ۳۰ گیگابایت | | پیام | 17,296 | | MIME part | 54,450 | | Content object | 13,684 | -| Content occurrence | 22,068 | | Message relationship | 12,115 | -| حجم Blobهای منحصربه‌فرد | 6,467,253,277 بایت | -| Findings مرحله فیزیکی | 1,484 | -| Warning / Error | 2 / 0 | - -Alpha 3 روی آرشیو واقعی Windows اعتبارسنجی شد. Alpha 4 همان Baseline را حفظ می‌کند، Migration شماره -`0006` را اضافه می‌کند و اجرای خصوصی کامل تشخیص فرمت را نیز با پوشش 13,684 از 13,684 Object، -13,636 مورد شناسایی‌شده، 46 مورد Unknown، صفر Ambiguous و صفر Tool error با موفقیت پشت سر گذاشته -است. SHA-256 دیتابیس اصلی MailVault قبل و بعد از اجرا بدون تغییر باقی ماند. - -## قابلیت‌های Alpha 4 - -- اجرای Identification فقط روی فایل‌های منحصربه‌فرد، نه هر occurrence تکراری؛ -- Siegfried `1.11.6` و PRONOM `v124` Pin‌شده؛ -- ثبت SHA-256 ابزار و Signature، نسخه‌ها، زمان ساخت و Identifierها؛ -- نگهداری تمام Matchها و انتخاب Primary Assertion بدون حذف Ambiguity؛ -- ذخیره PUID، نام و نسخه فرمت، MIME، Evidence basis، Warning و Format class؛ -- وضعیت‌های `identified`، `unknown`، `ambiguous`، `empty`، `skipped_unavailable` و `tool_error`؛ -- ثبت Extension mismatch فقط وقتی Alias امن با Extension واقعاً بررسی شده باشد؛ -- Batch محدود، Timeout، محدودیت خروجی Process و جداسازی خودکار Batch خراب؛ -- Checkpoint/Resume براساس Configuration fingerprint؛ -- Lock اختصاصی Workspace برای جلوگیری از دو Writer هم‌زمان؛ -- عدم Container expansion، OCR، Preview و Classification تجاری. - -

داشبورد تشخیص دقیق فرمت

- -

جزئیات Format Assertion

+| Blob bytes | 6,467,253,277 | -

مرز معماری Alpha 4

- -## اجرای CLI - -ابتدا Workspace و Run فیزیکی را پیدا کن: - -```powershell -.\target\release\mailvault-profiler.exe runs list ` - --workspace "E:\MailVault-Profiler-Alpha4" ` - --json -``` +Alpha 4 روی Windows با ۳۶ تست Rust و ۱۳٬۶۸۴/۱۳٬۶۸۴ Object سبز شد: ۱۳٬۶۳۶ Identified، ۴۶ +Unknown، صفر Ambiguous و صفر Tool Error. Alpha 6 باید Matrix واقعی Initial، No-change، Delta، +Force-kill، Soak و Rebuild equivalence را قبل از انتشار GitHub پاس کند. -Toolchain را Probe کن: +## اجرای کامل CLI ```powershell -.\target\release\mailvault-profiler.exe formats probe ` +.\target\release\mailvault-profiler.exe profile ` + --archive "E:\MailVault-E" ` + --workspace "E:\MailVault-Profiler-Alpha6" ` --siegfried ".\tools\siegfried\windows-x86_64\sf.exe" ` --signature ".\tools\siegfried\windows-x86_64\default.sig" ` - --json + --retain-successful-snapshots 2 ``` -Identification را اجرا کن: +برای ادامه دائمی بعد از Syncهای جدید: ```powershell -.\target\release\mailvault-profiler.exe formats identify ` - --workspace "E:\MailVault-Profiler-Alpha4" ` - --run "" ` - --siegfried ".\tools\siegfried\windows-x86_64\sf.exe" ` - --signature ".\tools\siegfried\windows-x86_64\default.sig" ` - --batch-size 2048 ` - --workers 0 ` - --timeout-seconds 900 ` - --resume true ` - --allow-migration ` - 1> format-result.json ` - 2> format-progress.jsonl +.\target\release\mailvault-profiler.exe profile ` + --archive "E:\MailVault-E" ` + --workspace "E:\MailVault-Profiler-Alpha6" ` + --watch ` + --poll-seconds 300 ``` -راهنمای کامل: [تشخیص دقیق فرمت](docs/FORMAT_IDENTIFICATION.md) و -[Runbook اجرایی](docs/FORMAT_IDENTIFICATION_RUNBOOK.md). - -## Build از Source +## Gate نهایی ```powershell -npm ci -.\scripts\install-siegfried.ps1 -.\scripts\quality.ps1 -npm run tauri:desktop:bundle +.\scripts\quality-alpha6.ps1 -BuildInstallers +.\scripts\validate-alpha6-real.ps1 ... +.\scripts\validate-alpha6-delta.ps1 ... +.\scripts\validate-alpha6-crash.ps1 ... +.\scripts\validate-alpha6-soak.ps1 -Cycles 100 ... +.\scripts\validate-alpha6-rebuild-equivalence.ps1 ... ``` -فایل‌های تولیدشده `sf.exe`، `default.sig` و `tool-manifest.json` داخل Git Commit نمی‌شوند؛ Release -Workflow آن‌ها را از Release رسمی دریافت، Hash را بررسی و به‌عنوان Resource داخل Installer قرار می‌دهد. - -## وضعیت Validation - -در محیط فعلی این موارد Pass شده‌اند: - -- TypeScript type-check و Vite production build؛ -- Parse نحوی تمام فایل‌های Rust با Tree-sitter؛ -- Migration دیتابیس واقعی Alpha 3 از Schema 5 به 6؛ -- `quick_check` و `foreign_key_check`؛ -- حفظ 13,684 Object، 22,068 Occurrence و 1,484 Finding؛ -- ثابت‌ماندن SHA-256 دیتابیس Source؛ -- Projection آزمایشی Format Assertion روی Schema مهاجرت‌داده‌شده. - -در Container فعلی Rust toolchain وجود نداشت؛ بنابراین ادعای دروغ درباره `cargo check`، Clippy، تست‌های -Rust، Build Native Tauri یا اجرای واقعی Siegfried نشده است. CI ویندوز برای اجرای همین Gateهای کامل -تنظیم شده است. - -گزارش: [Validation Alpha 4](docs/VALIDATION_0.1.0-alpha.4.md). - -## مستندات اصلی - -- [فهرست کامل](docs/INDEX.md) -- [شروع سریع](docs/GETTING_STARTED.md) -- [نصب Windows](docs/INSTALLATION_WINDOWS.md) -- [راهنمای GUI](docs/GUI_GUIDE.md) -- [مرجع CLI](docs/CLI_REFERENCE.md) -- [تشخیص دقیق فرمت](docs/FORMAT_IDENTIFICATION.md) -- [Runbook تشخیص فرمت](docs/FORMAT_IDENTIFICATION_RUNBOOK.md) -- [معماری](docs/ARCHITECTURE.md) -- [مدل امنیتی](docs/SECURITY_MODEL.md) -- [حریم خصوصی](docs/PRIVACY.md) -- [Baseline واقعی](docs/REAL_ARCHIVE_BASELINE.md) -- [Release Notes Alpha 4](docs/releases/v0.1.0-alpha.4.md) +راهنماها: [Continuous profiling](docs/CONTINUOUS_PROFILING.md)، +[قرارداد Validation](docs/VALIDATION_0.1.0-alpha.6.md)، +[پذیرش واقعی فارسی](docs/ALPHA6_REAL_ACCEPTANCE_FA.md) و +[Release Notes](docs/releases/v0.1.0-alpha.6.md). -## خارج از Scope فعلی +## خارج از Scope -Resume اجرای Physical Profile قطع‌شده، Full Fixity Hash، Container expansion، JHOVE، استخراج متن، -OCR، Embedding، LLM، تشخیص Invoice/Quotation، اتصال خودکار به RFQ و هر نوع نوشتن در RMS. +OCR، استخراج متن، Embedding، Vector DB، RAG، LLM، Classification تجاری، RMS integration، اجرای +Attachment و Container expansion داخل این Profiler نیستند. ## مجوز -Apache License 2.0. فایل‌های [LICENSE](LICENSE)، [NOTICE](NOTICE) و -[THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md) را ببین. +Apache License 2.0. diff --git a/SOURCE_PROVENANCE.json b/SOURCE_PROVENANCE.json new file mode 100644 index 0000000..0b50a4b --- /dev/null +++ b/SOURCE_PROVENANCE.json @@ -0,0 +1,18 @@ +{ + "schema": "mailvault.profiler.source-provenance.v1", + "version": "0.1.0-alpha.6", + "workspace_schema": 7, + "canonical_predecessor_archive": "mailvault-collection-profiler(9).zip", + "canonical_predecessor_sha256": "afe3f97ac891b602a0d578818167a3112f66dea9ac6c55fceba86aa4d8964296", + "predecessor_release": "0.1.0-alpha.4", + "feature_scope": [ + "continuous_full-snapshot metadata reconciliation", + "selective content-addressed file-stat reuse and retry", + "automatic exact-format identification", + "same-run crash resume", + "bounded snapshot retention", + "real Windows acceptance gates" + ], + "runtime_status": "awaiting Windows real-archive acceptance", + "publication_rule": "do not tag or publish until every required acceptance gate is green" +} diff --git a/apps/cli/src/main.rs b/apps/cli/src/main.rs index c457d75..3a8bfea 100644 --- a/apps/cli/src/main.rs +++ b/apps/cli/src/main.rs @@ -2,6 +2,8 @@ use std::{ path::{Path, PathBuf}, process::ExitCode, str::FromStr, + thread, + time::Duration, }; use clap::{Parser, Subcommand}; @@ -13,7 +15,11 @@ use profiler_core::{ WorkspaceOpenMode, }; use profiler_engine::{ - ExactFormatEngine, ProfileEngine, ProfileOptions, ProfileRequest, + ExactFormatEngine, + continuous::{ + ContinuousProfileEngine, ContinuousProfileOptions, ContinuousProfileRequest, + profile_needed, source_revision_fingerprint, + }, workspace::{ WorkspaceSession, add_review_note, clear_review_status, export_sanitized_run, finding_detail, findings_page, list_runs, set_review_status, @@ -56,6 +62,26 @@ enum Command { file_stat_workers: u32, #[arg(long, default_value_t = 512)] file_stat_batch_size: u32, + #[arg(long)] + siegfried: Option, + #[arg(long)] + signature: Option, + #[arg(long, default_value_t = 2_048)] + format_batch_size: u32, + #[arg(long, default_value_t = 0)] + format_workers: u32, + #[arg(long, default_value_t = 900)] + format_timeout_seconds: u64, + #[arg(long)] + skip_format: bool, + #[arg(long)] + rebuild: bool, + #[arg(long)] + watch: bool, + #[arg(long, default_value_t = 300)] + poll_seconds: u64, + #[arg(long, default_value_t = 2)] + retain_successful_snapshots: u32, }, Workspace { #[command(subcommand)] @@ -231,31 +257,6 @@ enum FormatsCommand { }, } -#[derive(Debug)] -struct IdentifyFormatsArgs { - workspace: PathBuf, - run: String, - siegfried: Option, - signature: Option, - batch_size: u32, - workers: u32, - timeout_seconds: u64, - resume: bool, - allow_migration: bool, -} - -#[derive(Debug)] -struct ListFormatsArgs { - workspace: PathBuf, - run: String, - state: Option, - puid: Option, - mismatch_only: bool, - search: Option, - limit: u32, - json: bool, -} - #[derive(Debug, Subcommand)] enum ExportCommand { SanitizedSummary { @@ -318,13 +319,33 @@ fn run(arguments: Arguments) -> ProfilerResult<()> { batch_size, file_stat_workers, file_stat_batch_size, - } => run_profile( + siegfried, + signature, + format_batch_size, + format_workers, + format_timeout_seconds, + skip_format, + rebuild, + watch, + poll_seconds, + retain_successful_snapshots, + } => run_profile(&ProfileCommandOptions { archive, workspace, batch_size, file_stat_workers, file_stat_batch_size, - ), + siegfried, + signature, + format_batch_size, + format_workers, + format_timeout_seconds, + skip_format, + rebuild, + watch, + poll_seconds, + retain_successful_snapshots, + }), Command::Workspace { command } => run_workspace_command(command), Command::Runs { command } => run_runs_command(command), Command::Findings { command } => run_findings_command(command), @@ -377,25 +398,82 @@ fn run_snapshot( print_json(&MailVaultAdapter.create_snapshot(&request, &JsonLineProgress)?) } -fn run_profile( +#[derive(Debug)] +struct ProfileCommandOptions { archive: PathBuf, workspace: PathBuf, batch_size: u32, file_stat_workers: u32, file_stat_batch_size: u32, -) -> ProfilerResult<()> { - let mut options = ProfileOptions::default(); - options.inventory.batch_size = batch_size; - options.file_stat.workers = file_stat_workers; - options.file_stat.batch_size = file_stat_batch_size; - print_json(&ProfileEngine.profile( - &ProfileRequest { - archive_root: archive, - workspace_root: workspace, - options, - }, - &JsonLineProgress, - )?) + siegfried: Option, + signature: Option, + format_batch_size: u32, + format_workers: u32, + format_timeout_seconds: u64, + skip_format: bool, + rebuild: bool, + watch: bool, + poll_seconds: u64, + retain_successful_snapshots: u32, +} + +fn run_profile(command: &ProfileCommandOptions) -> ProfilerResult<()> { + if command.poll_seconds < 5 { + return Err(profiler_core::ProfilerError::InvalidArgument( + "poll-seconds must be at least 5".into(), + )); + } + + let mut options = continuous_profile_options(command); + loop { + let preflight = MailVaultAdapter.preflight(&command.archive)?; + let revision = source_revision_fingerprint(&preflight)?; + let should_profile = !command.watch || profile_needed(&command.workspace, &revision)?; + if should_profile { + let execution = ContinuousProfileEngine.profile( + &ContinuousProfileRequest { + archive_root: command.archive.clone(), + workspace_root: command.workspace.clone(), + siegfried_path: command.siegfried.clone(), + signature_path: command.signature.clone(), + options: options.clone(), + }, + &JsonLineProgress, + ); + match execution { + Ok(result) => { + print_json(&result)?; + options.rebuild = false; + } + Err(error) if command.watch && error.report().retryable => { + eprintln!( + "{}", + serde_json::to_string(&error.report()) + .unwrap_or_else(|_| error.to_string()) + ); + } + Err(error) => return Err(error), + } + } + if !command.watch { + return Ok(()); + } + thread::sleep(Duration::from_secs(command.poll_seconds)); + } +} + +fn continuous_profile_options(command: &ProfileCommandOptions) -> ContinuousProfileOptions { + let mut options = ContinuousProfileOptions::default(); + options.profile.inventory.batch_size = command.batch_size; + options.profile.file_stat.workers = command.file_stat_workers; + options.profile.file_stat.batch_size = command.file_stat_batch_size; + options.format.batch_size = command.format_batch_size; + options.format.workers = command.format_workers; + options.format.timeout_seconds = command.format_timeout_seconds; + options.format_required = !command.skip_format; + options.rebuild = command.rebuild; + options.retain_successful_snapshots = command.retain_successful_snapshots; + options } fn run_workspace_command(command: WorkspaceCommand) -> ProfilerResult<()> { @@ -617,7 +695,7 @@ fn run_formats_command(command: FormatsCommand) -> ProfilerResult<()> { signature, workers, json, - } => run_format_probe(siegfried, signature, workers, json), + } => run_formats_probe(siegfried, signature, workers, json), FormatsCommand::Identify { workspace, run, @@ -628,22 +706,24 @@ fn run_formats_command(command: FormatsCommand) -> ProfilerResult<()> { timeout_seconds, resume, allow_migration, - } => run_format_identification(IdentifyFormatsArgs { - workspace, + } => run_formats_identify( + &workspace, run, siegfried, signature, - batch_size, - workers, - timeout_seconds, - resume, + FormatOptions { + batch_size, + workers, + timeout_seconds, + resume, + }, allow_migration, - }), + ), FormatsCommand::Summary { workspace, run, json, - } => run_format_summary(&workspace, &run, json), + } => run_formats_summary(&workspace, &run, json), FormatsCommand::List { workspace, run, @@ -653,20 +733,22 @@ fn run_formats_command(command: FormatsCommand) -> ProfilerResult<()> { search, limit, json, - } => run_format_list(ListFormatsArgs { - workspace, - run, - state, - puid, - mismatch_only, - search, - limit, - json, - }), + } => run_formats_list( + &workspace, + FormatListCommandOptions { + run, + state, + puid, + mismatch_only, + search, + limit, + json, + }, + ), } } -fn run_format_probe( +fn run_formats_probe( siegfried: Option, signature: Option, workers: u32, @@ -684,34 +766,34 @@ fn run_format_probe( } } -fn run_format_identification(args: IdentifyFormatsArgs) -> ProfilerResult<()> { - let session = open_review_workspace(&args.workspace, args.allow_migration)?; +fn run_formats_identify( + workspace: &Path, + run: String, + siegfried: Option, + signature: Option, + options: FormatOptions, + allow_migration: bool, +) -> ProfilerResult<()> { + let session = open_review_workspace(workspace, allow_migration)?; let context = session.context(); let archive_root = single_source_root(&context.source_roots)?; - print_json(&ExactFormatEngine.identify( &FormatIdentificationRequest { - baseline_run_id: args.run, + baseline_run_id: run, workspace_root: context.root_path, profiler_database: context.profiler_database, archive_root, - siegfried_path: args.siegfried, - signature_path: args.signature, - options: FormatOptions { - batch_size: args.batch_size, - workers: args.workers, - timeout_seconds: args.timeout_seconds, - resume: args.resume, - }, + siegfried_path: siegfried, + signature_path: signature, + options, }, &JsonLineProgress, )?) } -fn run_format_summary(workspace: &Path, run: &str, json: bool) -> ProfilerResult<()> { +fn run_formats_summary(workspace: &Path, run: &str, json: bool) -> ProfilerResult<()> { let session = open_read_only_workspace(workspace)?; let summary = ExactFormatEngine.summary(&session.context().profiler_database, run)?; - if json { print_json(&summary) } else { @@ -733,32 +815,46 @@ fn run_format_summary(workspace: &Path, run: &str, json: bool) -> ProfilerResult } } -fn run_format_list(args: ListFormatsArgs) -> ProfilerResult<()> { - let session = open_read_only_workspace(&args.workspace)?; +#[derive(Debug)] +struct FormatListCommandOptions { + run: String, + state: Option, + puid: Option, + mismatch_only: bool, + search: Option, + limit: u32, + json: bool, +} + +fn run_formats_list(workspace: &Path, command: FormatListCommandOptions) -> ProfilerResult<()> { + let session = open_read_only_workspace(workspace)?; let page = ExactFormatEngine.page( &session.context().profiler_database, &FormatPageRequest { - baseline_run_id: args.run, + baseline_run_id: command.run, filters: FormatFilters { - search: args.search, - state: args.state.as_deref().map(parse_format_state).transpose()?, - puid: args.puid, - mismatch_only: args.mismatch_only, + search: command.search, + state: command + .state + .as_deref() + .map(parse_format_state) + .transpose()?, + puid: command.puid, + mismatch_only: command.mismatch_only, }, after_sha256: None, - limit: args.limit, + limit: command.limit, }, )?; - - if args.json { + if command.json { print_json(&page) } else { for item in page.items { println!( "{} {} {} {}", item.state, - item.primary_identifier.as_deref().unwrap_or("—"), - item.primary_format_name.as_deref().unwrap_or("—"), + item.primary_identifier.as_deref().unwrap_or("—"), + item.primary_format_name.as_deref().unwrap_or("—"), item.sha256 ); } diff --git a/apps/desktop/package.json b/apps/desktop/package.json index d2c8b76..5d9b403 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -1,7 +1,7 @@ { "name": "@firexcore/mailvault-profiler-desktop", "private": true, - "version": "0.1.0-alpha.4", + "version": "0.1.0-alpha.6", "type": "module", "scripts": { "dev": "vite --port 1420 --strictPort", @@ -12,17 +12,17 @@ "dependencies": { "@tauri-apps/api": "^2.11.1", "@tauri-apps/plugin-dialog": "^2.7.2", - "react": "^19.2.7", - "react-dom": "^19.2.7" + "react": "^19.2.8", + "react-dom": "^19.2.8" }, "devDependencies": { "@tauri-apps/cli": "^2.11.4", - "@types/react": "^19.2.0", - "@types/react-dom": "^19.2.0", - "@vitejs/plugin-react": "^6.0.3", + "@types/react": "^19.2.18", + "@types/react-dom": "^19.2.4", + "@vitejs/plugin-react": "^6.0.5", "typescript": "^7.0.2", - "vite": "^8.1.5", - "@types/node": "^26.1.1" + "vite": "^8.2.0", + "@types/node": "^26.1.2" }, "description": "Windows desktop interface for MailVault Collection Profiler.", "license": "Apache-2.0", diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 03940ea..38183f0 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -11,7 +11,8 @@ use profiler_core::{ SnapshotRequest, SnapshotResult, WorkspaceInspection, WorkspaceOpenMode, }; use profiler_engine::{ - ExactFormatEngine, ProfileEngine, ProfileOptions, ProfileRequest, ProfileResult, + ExactFormatEngine, ProfileResult, + continuous::{ContinuousProfileEngine, ContinuousProfileOptions, ContinuousProfileRequest}, workspace::{ WorkspaceContext, WorkspaceSession, add_review_note, clear_review_status, export_sanitized_run, finding_detail, findings_page as query_findings_page, @@ -275,15 +276,20 @@ async fn profile_collection( root: String, workspace: String, on_event: Channel, + app: AppHandle, state: State<'_, DesktopState>, ) -> Result { let workspace_path = PathBuf::from(&workspace); + let (siegfried_path, signature_path) = + bundled_siegfried_paths(&app).map_err(|error| error.report())?; let result = tauri::async_runtime::spawn_blocking(move || { - ProfileEngine.profile( - &ProfileRequest { + ContinuousProfileEngine.profile( + &ContinuousProfileRequest { archive_root: PathBuf::from(root), workspace_root: PathBuf::from(workspace), - options: ProfileOptions::default(), + siegfried_path, + signature_path, + options: ContinuousProfileOptions::default(), }, &ChannelProgressSink { channel: on_event }, ) diff --git a/apps/desktop/src-tauri/tauri.conf.json b/apps/desktop/src-tauri/tauri.conf.json index 210d89d..e4f04ce 100644 --- a/apps/desktop/src-tauri/tauri.conf.json +++ b/apps/desktop/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "MailVault Collection Profiler", - "version": "0.1.0-alpha.4", + "version": "0.1.0-alpha.6", "identifier": "com.firexcore.mailvault-profiler", "build": { "beforeDevCommand": "npm run dev", @@ -40,7 +40,7 @@ ], "windows": { "wix": { - "version": "0.1.0.4" + "version": "0.1.0.6" } }, "resources": { diff --git a/apps/desktop/src/types.ts b/apps/desktop/src/types.ts index 03cfacf..827f660 100644 --- a/apps/desktop/src/types.ts +++ b/apps/desktop/src/types.ts @@ -39,6 +39,7 @@ export interface PreflightReport { databasePath: string; databaseBytes: number; archiveIdentity: string | null; + archiveIdentityAliases: string[]; schemaVersion: number | null; journalMode: string | null; lockState: LockState; @@ -153,6 +154,12 @@ export interface SnapshotResult { manifest: SnapshotManifest; } +export interface FormatIdentificationResult { + formatRunId: string; + baselineRunId: string; + configurationFingerprint: string; + summary: FormatSummary; +} export interface ProfileResult { runId: string; collectionId: string; @@ -162,6 +169,13 @@ export interface ProfileResult { snapshot: SnapshotResult; inventory: InventoryResult; fileStat: FileStatResult; + runKind: string; + generation: number; + resumed: boolean; + resumeCount: number; + sourceRevisionFingerprint: string; + format: FormatIdentificationResult | null; + snapshotsDeleted: number; } export interface InventoryFilters { diff --git a/crates/profiler-adapter-mailvault/src/preflight.rs b/crates/profiler-adapter-mailvault/src/preflight.rs index 33fefaa..4aa947a 100644 --- a/crates/profiler-adapter-mailvault/src/preflight.rs +++ b/crates/profiler-adapter-mailvault/src/preflight.rs @@ -26,12 +26,15 @@ pub(crate) fn run_preflight(archive_root: &Path) -> ProfilerResult ProfilerResult ProfilerResult ProfilerResu } pub(crate) fn read_archive_identity_value(connection: &Connection) -> ProfilerResult { + if let Some(archive_uuid) = connection + .query_row( + "SELECT value FROM schema_meta WHERE key='archive_uuid'", + [], + |row| row.get::<_, String>(0), + ) + .optional() + .map_err(|source| sqlite_error("reading MailVault archive UUID", source))? + .filter(|value| !value.trim().is_empty()) + { + return Ok(identity_digest( + "mailvault-profiler-archive-uuid-v1", + &[archive_uuid], + )); + } + let anchor = connection + .query_row( + "SELECT archive_id FROM accounts ORDER BY id LIMIT 1", + [], + |row| row.get::<_, String>(0), + ) + .optional() + .map_err(|source| sqlite_error("reading MailVault archive identity anchor", source))? + .ok_or_else(|| { + ProfilerError::IncompatibleSource( + "MailVault archive contains no account identities".into(), + ) + })?; + Ok(identity_digest( + "mailvault-profiler-collection-v2", + &[anchor], + )) +} + +pub(crate) fn read_archive_identity_aliases( + connection: &Connection, +) -> ProfilerResult> { let mut statement = connection .prepare("SELECT archive_id FROM accounts ORDER BY archive_id") - .map_err(|source| sqlite_error("preparing MailVault archive identity", source))?; + .map_err(|source| sqlite_error("preparing MailVault identity aliases", source))?; let archive_ids = statement .query_map([], |row| row.get::<_, String>(0)) - .map_err(|source| sqlite_error("querying MailVault archive identity", source))? + .map_err(|source| sqlite_error("querying MailVault identity aliases", source))? .collect::, _>>() - .map_err(|source| sqlite_error("collecting MailVault archive identity", source))?; + .map_err(|source| sqlite_error("collecting MailVault identity aliases", source))?; if archive_ids.is_empty() { - return Err(ProfilerError::IncompatibleSource( - "MailVault archive contains no account identities".into(), - )); + return Ok(Vec::new()); } + let mut aliases = archive_ids + .iter() + .cloned() + .map(|value| identity_digest("mailvault-profiler-account-v1", &[value])) + .collect::>(); + aliases.push(identity_digest( + "mailvault-profiler-collection-v1", + &archive_ids, + )); + aliases.sort(); + aliases.dedup(); + Ok(aliases) +} +fn identity_digest(domain: &str, values: &[String]) -> String { let mut digest = Sha256::new(); - digest.update(b"mailvault-profiler-collection-v1\0"); - for archive_id in archive_ids { - let bytes = archive_id.as_bytes(); + digest.update(domain.as_bytes()); + digest.update(b"\0"); + for value in values { + let bytes = value.as_bytes(); digest.update((bytes.len() as u64).to_be_bytes()); digest.update(bytes); } - Ok(hex::encode(digest.finalize())) + hex::encode(digest.finalize()) } fn read_archive_identity( connection: &Connection, checks: &mut Vec, -) -> Option { - match read_archive_identity_value(connection) { - Ok(identity) => { +) -> (Option, Vec) { + match ( + read_archive_identity_value(connection), + read_archive_identity_aliases(connection), + ) { + (Ok(identity), Ok(aliases)) => { checks.push(passed( "archive_identity", "Stable archive identity", CheckLevel::Required, format!("{}…", &identity[..12]), )); - Some(identity) + (Some(identity), aliases) } - Err(error) => { + (Err(error), _) | (_, Err(error)) => { checks.push(failed( "archive_identity", "Stable archive identity", CheckLevel::Required, error.to_string(), )); - None + (None, Vec::new()) } } } @@ -430,20 +490,24 @@ fn check_required_path( }); } -fn finalize_report( - layout: &MailVaultLayout, +struct FinalizeReportInput { archive_identity: Option, + archive_identity_aliases: Vec, schema_version: Option, journal_mode: Option, lock_state: LockState, metrics: ArchiveMetrics, checks: Vec, -) -> PreflightReport { - let errors_count = checks +} + +fn finalize_report(layout: &MailVaultLayout, input: FinalizeReportInput) -> PreflightReport { + let errors_count = input + .checks .iter() .filter(|check| check.status == CheckStatus::Failed) .count() as u64; - let warnings_count = checks + let warnings_count = input + .checks .iter() .filter(|check| check.status == CheckStatus::Warning) .count() as u64; @@ -453,12 +517,13 @@ fn finalize_report( archive_root: layout.root.to_string_lossy().into_owned(), database_path: layout.database.to_string_lossy().into_owned(), database_bytes: 0, - archive_identity, - schema_version, - journal_mode, - lock_state, - metrics, - checks, + archive_identity: input.archive_identity, + archive_identity_aliases: input.archive_identity_aliases, + schema_version: input.schema_version, + journal_mode: input.journal_mode, + lock_state: input.lock_state, + metrics: input.metrics, + checks: input.checks, warnings_count, errors_count, inspected_at: OffsetDateTime::now_utc(), diff --git a/crates/profiler-adapter-mailvault/src/snapshot.rs b/crates/profiler-adapter-mailvault/src/snapshot.rs index cb60d75..020112d 100644 --- a/crates/profiler-adapter-mailvault/src/snapshot.rs +++ b/crates/profiler-adapter-mailvault/src/snapshot.rs @@ -294,7 +294,7 @@ fn recover_or_reuse_snapshot( snapshot_database: &Path, manifest_path: &Path, archive_identity: &str, - expected_metrics: &profiler_core::ArchiveMetrics, + _expected_metrics: &profiler_core::ArchiveMetrics, sink: &dyn ProgressSink, ) -> ProfilerResult { let started = Instant::now(); @@ -308,9 +308,6 @@ fn recover_or_reuse_snapshot( ))); } let snapshot_metrics = read_metrics(&snapshot)?; - if &snapshot_metrics != expected_metrics { - return Err(ProfilerError::SourceChanged); - } let schema_version = read_schema_version_value(&snapshot)?; let snapshot_archive_identity = read_archive_identity_value(&snapshot)?; if snapshot_archive_identity != archive_identity { @@ -352,7 +349,7 @@ fn recover_or_reuse_snapshot( snapshot_sha256, snapshot_bytes, schema_version, - source_metrics: expected_metrics.clone(), + source_metrics: snapshot_metrics.clone(), snapshot_metrics, created_at: OffsetDateTime::now_utc(), }; diff --git a/crates/profiler-core/src/file_stat.rs b/crates/profiler-core/src/file_stat.rs index 1528d29..672a7c5 100644 --- a/crates/profiler-core/src/file_stat.rs +++ b/crates/profiler-core/src/file_stat.rs @@ -193,10 +193,15 @@ pub trait FileStatStore { fn load_file_stat_checkpoint(&self, run_id: &str) -> ProfilerResult>; - fn count_file_stat_objects(&self, collection_id: &str) -> ProfilerResult<(u64, u64)>; + fn count_file_stat_objects( + &self, + run_id: &str, + collection_id: &str, + ) -> ProfilerResult<(u64, u64)>; fn load_file_stat_batch( &self, + run_id: &str, collection_id: &str, after_sha256: Option<&str>, limit: u32, diff --git a/crates/profiler-core/src/run.rs b/crates/profiler-core/src/run.rs index 9815a7f..11afffc 100644 --- a/crates/profiler-core/src/run.rs +++ b/crates/profiler-core/src/run.rs @@ -63,9 +63,13 @@ pub fn validate_transition(from: RunState, to: RunState) -> ProfilerResult<()> { RunState::Pausing | RunState::Cancelling | RunState::Succeeded | RunState::Failed ), RunState::Pausing => matches!(to, RunState::Paused | RunState::Failed), - RunState::Paused => matches!(to, RunState::Running | RunState::Cancelling), + RunState::Paused => matches!( + to, + RunState::Preflighting | RunState::Running | RunState::Cancelling + ), RunState::Cancelling => matches!(to, RunState::Cancelled | RunState::Failed), - RunState::Cancelled | RunState::Succeeded | RunState::Failed => false, + RunState::Failed => matches!(to, RunState::Preflighting), + RunState::Cancelled | RunState::Succeeded => false, }; if valid { diff --git a/crates/profiler-core/src/source.rs b/crates/profiler-core/src/source.rs index 6c5ee5c..cd1b81e 100644 --- a/crates/profiler-core/src/source.rs +++ b/crates/profiler-core/src/source.rs @@ -59,6 +59,8 @@ pub struct PreflightReport { pub database_path: String, pub database_bytes: u64, pub archive_identity: Option, + #[serde(default)] + pub archive_identity_aliases: Vec, pub schema_version: Option, pub journal_mode: Option, pub lock_state: LockState, diff --git a/crates/profiler-engine/Cargo.toml b/crates/profiler-engine/Cargo.toml index 5bddc23..f78a5ae 100644 --- a/crates/profiler-engine/Cargo.toml +++ b/crates/profiler-engine/Cargo.toml @@ -16,6 +16,7 @@ profiler-core = { path = "../profiler-core" } profiler-format-siegfried = { path = "../profiler-format-siegfried" } profiler-storage-sqlite = { path = "../profiler-storage-sqlite" } rayon.workspace = true +rusqlite.workspace = true serde.workspace = true serde_json.workspace = true sha2.workspace = true @@ -25,5 +26,4 @@ uuid.workspace = true workspace = true [dev-dependencies] -rusqlite.workspace = true tempfile.workspace = true diff --git a/crates/profiler-engine/src/continuous.rs b/crates/profiler-engine/src/continuous.rs new file mode 100644 index 0000000..2ce5794 --- /dev/null +++ b/crates/profiler-engine/src/continuous.rs @@ -0,0 +1,722 @@ +use std::{ + collections::BTreeSet, + fs::{self, File, OpenOptions}, + io::Write, + path::{Path, PathBuf}, +}; + +use fs2::FileExt; +use profiler_adapter_mailvault::MailVaultAdapter; +use profiler_core::{ + CollectionAdapter, FormatIdentificationRequest, FormatOptions, PreflightReport, ProfilerError, + ProfilerResult, ProgressSink, RunState, +}; +use profiler_storage_sqlite::ProfilerStore; +use rusqlite::OptionalExtension; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use uuid::Uuid; + +use crate::{ExactFormatEngine, ProfileEngine, ProfileOptions, ProfileRequest, ProfileResult}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ContinuousProfileOptions { + pub profile: ProfileOptions, + pub format: FormatOptions, + pub format_required: bool, + pub rebuild: bool, + pub retain_successful_snapshots: u32, +} + +impl Default for ContinuousProfileOptions { + fn default() -> Self { + Self { + profile: ProfileOptions::default(), + format: FormatOptions::default(), + format_required: true, + rebuild: false, + retain_successful_snapshots: 2, + } + } +} + +#[derive(Debug, Clone)] +pub struct ContinuousProfileRequest { + pub archive_root: PathBuf, + pub workspace_root: PathBuf, + pub siegfried_path: Option, + pub signature_path: Option, + pub options: ContinuousProfileOptions, +} + +#[derive(Debug, Default, Clone, Copy)] +pub struct ContinuousProfileEngine; + +impl ContinuousProfileEngine { + pub fn profile( + &self, + request: &ContinuousProfileRequest, + progress: &dyn ProgressSink, + ) -> ProfilerResult { + validate_request(request)?; + fs::create_dir_all(&request.workspace_root).map_err(|source| ProfilerError::Io { + operation: "creating continuous profiler workspace", + path: request.workspace_root.clone(), + source, + })?; + let _lock = ProfileLock::acquire(&request.workspace_root)?; + + let adapter = MailVaultAdapter; + let preflight = adapter.preflight(&request.archive_root)?; + if !preflight.compatible { + return Err(ProfilerError::IncompatibleSource(format!( + "preflight failed with {} required errors", + preflight.errors_count + ))); + } + let archive_identity = preflight.archive_identity.as_deref().ok_or_else(|| { + ProfilerError::IncompatibleSource( + "preflight did not produce an archive identity".into(), + ) + })?; + + let database = request + .workspace_root + .join("profiler") + .join("profiler.sqlite3"); + let mut store = ProfilerStore::open(&database)?; + let collection_id = store.register_continuous_collection( + adapter.kind(), + archive_identity, + &preflight.archive_identity_aliases, + &preflight.archive_root, + )?; + let configuration_fingerprint = semantic_configuration_fingerprint(request)?; + let source_revision = source_revision_fingerprint(&preflight)?; + let registration = begin_or_resume_run( + &store, + &collection_id, + &configuration_fingerprint, + &source_revision, + request.options.rebuild, + )?; + let run_id = registration.run_id.clone(); + + let physical = ProfileEngine::profile_registered_run( + &ProfileRequest { + archive_root: request.archive_root.clone(), + workspace_root: request.workspace_root.clone(), + options: request.options.profile.clone(), + }, + progress, + adapter, + &preflight, + &mut store, + &run_id, + &collection_id, + &database, + &configuration_fingerprint, + false, + ); + let mut result = match physical { + Ok(result) => result, + Err(error) => { + mark_failure(&store, &run_id, &error)?; + return Err(error); + } + }; + + if request.options.format_required { + let format = ExactFormatEngine.identify( + &FormatIdentificationRequest { + baseline_run_id: run_id.clone(), + workspace_root: request.workspace_root.clone(), + profiler_database: database.clone(), + archive_root: request.archive_root.clone(), + siegfried_path: request.siegfried_path.clone(), + signature_path: request.signature_path.clone(), + options: request.options.format, + }, + progress, + ); + match format { + Ok(format) => result.format = Some(format), + Err(error) => { + mark_failure(&store, &run_id, &error)?; + return Err(error); + } + } + } + + store.transition_run(&run_id, RunState::Succeeded)?; + result.run_kind = registration.run_kind; + result.generation = registration.generation; + result.resumed = registration.resumed; + result.resume_count = registration.resume_count; + result.source_revision_fingerprint = source_revision; + result.snapshots_deleted = prune_snapshots( + &store, + &request.workspace_root, + &collection_id, + request.options.retain_successful_snapshots.max(1), + )?; + Ok(result) + } +} + +#[derive(Debug)] +struct RunRegistration { + run_id: String, + run_kind: String, + generation: u64, + resumed: bool, + resume_count: u64, +} + +pub fn profile_needed(workspace: &Path, source_revision: &str) -> ProfilerResult { + let database = workspace.join("profiler").join("profiler.sqlite3"); + if !database.is_file() { + return Ok(true); + } + let store = ProfilerStore::open_existing(&database)?; + let connection = store.connection(); + let resumable: bool = connection + .query_row( + "SELECT EXISTS(SELECT 1 FROM profiler_runs + WHERE state IN ('pending','preflighting','snapshotting','ready','running', + 'pausing','paused','cancelling') + OR (state='failed' AND failure_retryable=1))", + [], + |row| row.get(0), + ) + .map_err(|source| sqlite_error("checking resumable profile work", &source))?; + if resumable { + return Ok(true); + } + let latest: Option = connection + .query_row( + "SELECT source_revision_fingerprint FROM profiler_runs + WHERE state='succeeded' AND source_revision_fingerprint IS NOT NULL + ORDER BY generation DESC, created_at DESC LIMIT 1", + [], + |row| row.get(0), + ) + .optional() + .map_err(|source| sqlite_error("loading latest successful source revision", &source))?; + Ok(latest.as_deref() != Some(source_revision)) +} + +fn begin_or_resume_run( + store: &ProfilerStore, + collection_id: &str, + configuration_fingerprint: &str, + source_revision_fingerprint: &str, + rebuild: bool, +) -> ProfilerResult { + let connection = store.connection(); + let candidate = connection + .query_row( + "SELECT id, run_kind, generation, resume_count FROM profiler_runs + WHERE collection_id=?1 AND configuration_fingerprint=?2 + AND (state IN ('pending','preflighting','snapshotting','ready','running', + 'pausing','paused','cancelling') + OR (state='failed' AND failure_retryable=1)) + ORDER BY generation DESC, created_at DESC LIMIT 1", + rusqlite::params![collection_id, configuration_fingerprint], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, i64>(3)?, + )) + }, + ) + .optional() + .map_err(|source| sqlite_error("finding resumable profile run", &source))?; + + if let Some((run_id, run_kind, generation, resume_count)) = candidate { + if rebuild { + return Err(ProfilerError::InvalidArgument( + "a resumable run exists; resume or remove it before a rebuild".into(), + )); + } + let now = now_text(); + connection + .execute( + "UPDATE profiler_runs SET state='pending', resume_count=resume_count+1, + last_resumed_at=?1, last_heartbeat_at=?1, failure_code=NULL, + failure_message=NULL, failure_retryable=0, finished_at=NULL, + source_revision_fingerprint=?2, updated_at=?1 WHERE id=?3", + rusqlite::params![now, source_revision_fingerprint, run_id], + ) + .map_err(|source| sqlite_error("resuming profile run", &source))?; + return Ok(RunRegistration { + run_id, + run_kind, + generation: to_u64(generation)?, + resumed: true, + resume_count: to_u64(resume_count + 1)?, + }); + } + + let previous = connection + .query_row( + "SELECT id, generation FROM profiler_runs + WHERE collection_id=?1 AND state='succeeded' + ORDER BY generation DESC, created_at DESC LIMIT 1", + [collection_id], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)), + ) + .optional() + .map_err(|source| sqlite_error("loading previous profile run", &source))?; + let (parent_run_id, generation, run_kind) = match previous { + Some((id, generation)) => ( + Some(id), + generation + 1, + if rebuild { "rebuild" } else { "incremental" }, + ), + None => (None, 1, if rebuild { "rebuild" } else { "initial" }), + }; + let run_id = Uuid::now_v7().to_string(); + let now = now_text(); + connection + .execute( + "INSERT INTO profiler_runs( + id, collection_id, state, pipeline_version, configuration_fingerprint, + created_at, updated_at, run_kind, parent_run_id, generation, + last_heartbeat_at, source_revision_fingerprint + ) VALUES(?1,?2,'pending',?3,?4,?5,?5,?6,?7,?8,?5,?9)", + rusqlite::params![ + run_id, + collection_id, + env!("CARGO_PKG_VERSION"), + configuration_fingerprint, + now, + run_kind, + parent_run_id, + generation, + source_revision_fingerprint, + ], + ) + .map_err(|source| sqlite_error("creating continuous profile run", &source))?; + Ok(RunRegistration { + run_id, + run_kind: run_kind.into(), + generation: to_u64(generation)?, + resumed: false, + resume_count: 0, + }) +} + +fn mark_failure(store: &ProfilerStore, run_id: &str, error: &ProfilerError) -> ProfilerResult<()> { + let report = error.report(); + let now = now_text(); + store + .connection() + .execute( + "UPDATE profiler_runs SET state='failed', failure_code=?1, failure_message=?2, + failure_retryable=?3, finished_at=?4, updated_at=?4 + WHERE id=?5 AND state NOT IN ('cancelled','succeeded')", + rusqlite::params![ + format!("{:?}", report.code).to_ascii_lowercase(), + report.message, + i64::from(report.retryable), + now, + run_id, + ], + ) + .map_err(|source| sqlite_error("recording continuous profile failure", &source))?; + Ok(()) +} + +pub fn source_revision_fingerprint(preflight: &PreflightReport) -> ProfilerResult { + let mut digest = Sha256::new(); + digest.update(b"mailvault-source-revision-v3\0"); + + // SQLite's shared-memory file is coordination state, not archive content. + // Read-only connections may update its metadata/read marks, so including + // `-shm` makes an unchanged WAL archive appear revised on every poll. + // Database and WAL metadata plus stable archive metrics form the watch + // signal; the consistent snapshot remains the authoritative content proof. + for path in [ + PathBuf::from(&preflight.database_path), + PathBuf::from(format!("{}-wal", preflight.database_path)), + ] { + digest.update(path.to_string_lossy().as_bytes()); + match fs::metadata(&path) { + Ok(metadata) => { + digest.update(metadata.len().to_be_bytes()); + if let Ok(modified) = metadata.modified() + && let Ok(delta) = modified.duration_since(std::time::UNIX_EPOCH) + { + digest.update(delta.as_nanos().to_be_bytes()); + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + digest.update(b"missing"); + } + Err(source) => { + return Err(ProfilerError::Io { + operation: "reading MailVault revision metadata", + path, + source, + }); + } + } + } + let metrics = serde_json::to_vec(&preflight.metrics).map_err(|error| { + ProfilerError::Internal(format!("serializing source revision metrics: {error}")) + })?; + digest.update(metrics); + Ok(hex::encode(digest.finalize())) +} + +fn semantic_configuration_fingerprint( + request: &ContinuousProfileRequest, +) -> ProfilerResult { + let payload = serde_json::to_vec(&serde_json::json!({ + "contract": "mailvault-continuous-profile-v6", + "formatRequired": request.options.format_required, + "containerExpansion": false, + "contentIdentity": "sha256", + "metadataReconciliation": "full-consistent-snapshot", + })) + .map_err(|error| ProfilerError::Internal(format!("serializing profile contract: {error}")))?; + let mut digest = Sha256::new(); + digest.update(b"mailvault-continuous-profile-v6\0"); + digest.update(payload); + Ok(hex::encode(digest.finalize())) +} + +fn prune_snapshots( + store: &ProfilerStore, + workspace: &Path, + collection_id: &str, + keep: u32, +) -> ProfilerResult { + let connection = store.connection(); + let root = workspace.join("snapshots"); + let candidates = snapshot_retention_candidates(connection, collection_id, keep)?; + let mut deleted = delete_snapshot_candidates(connection, &root, collection_id, candidates)?; + let referenced = referenced_snapshot_directories(connection, collection_id)?; + deleted = deleted.saturating_add(delete_orphan_snapshot_directories(&root, &referenced)?); + Ok(deleted) +} + +fn snapshot_retention_candidates( + connection: &rusqlite::Connection, + collection_id: &str, + keep: u32, +) -> ProfilerResult> { + let mut statement = connection + .prepare( + "SELECT snapshot.id, snapshot.snapshot_database_path + FROM source_snapshots AS snapshot + WHERE snapshot.collection_id=?1 + AND snapshot.retention_state<>'pinned' + AND snapshot.id NOT IN ( + SELECT source_snapshot_id FROM profiler_runs + WHERE collection_id=?1 AND source_snapshot_id IS NOT NULL + AND state NOT IN ('succeeded','cancelled') + ) + AND snapshot.id NOT IN ( + SELECT source_snapshot_id FROM profiler_runs + WHERE collection_id=?1 AND state='succeeded' + AND source_snapshot_id IS NOT NULL + ORDER BY generation DESC LIMIT ?2 + )", + ) + .map_err(|source| sqlite_error("preparing snapshot retention", &source))?; + let rows = statement + .query_map(rusqlite::params![collection_id, i64::from(keep)], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + }) + .map_err(|source| sqlite_error("querying snapshot retention", &source))? + .collect::, _>>() + .map_err(|source| sqlite_error("collecting snapshot retention", &source))?; + Ok(rows) +} + +fn delete_snapshot_candidates( + connection: &rusqlite::Connection, + root: &Path, + collection_id: &str, + candidates: Vec<(String, String)>, +) -> ProfilerResult { + let mut deleted = 0_u64; + for (snapshot_id, database_path) in candidates { + delete_snapshot_directory(root, &database_path)?; + connection + .execute("DELETE FROM source_snapshots WHERE id=?1", [&snapshot_id]) + .map_err(|source| sqlite_error("deleting expired source snapshot", &source))?; + connection + .execute( + "INSERT INTO snapshot_retention_events( + id, snapshot_id, collection_id, action, reason, detail_json, occurred_at + ) VALUES(?1,?2,?3,'deleted','retention_limit','{}',?4)", + rusqlite::params![ + Uuid::now_v7().to_string(), + snapshot_id, + collection_id, + now_text(), + ], + ) + .map_err(|source| sqlite_error("recording snapshot retention", &source))?; + deleted = deleted.saturating_add(1); + } + Ok(deleted) +} + +fn delete_snapshot_directory(root: &Path, database_path: &str) -> ProfilerResult<()> { + let Some(directory) = PathBuf::from(database_path).parent().map(Path::to_path_buf) else { + return Ok(()); + }; + if !directory.starts_with(root) { + return Err(ProfilerError::InvalidPath { + message: "snapshot retention candidate escapes the workspace".into(), + path: directory, + }); + } + if directory.exists() { + fs::remove_dir_all(&directory).map_err(|source| ProfilerError::Io { + operation: "deleting expired profiler snapshot", + path: directory, + source, + })?; + } + Ok(()) +} + +fn referenced_snapshot_directories( + connection: &rusqlite::Connection, + collection_id: &str, +) -> ProfilerResult> { + let mut statement = connection + .prepare("SELECT snapshot_database_path FROM source_snapshots WHERE collection_id=?1") + .map_err(|source| sqlite_error("preparing referenced snapshot query", &source))?; + let paths = statement + .query_map([collection_id], |row| row.get::<_, String>(0)) + .map_err(|source| sqlite_error("loading referenced snapshots", &source))? + .collect::, _>>() + .map_err(|source| sqlite_error("collecting referenced snapshots", &source))?; + Ok(paths + .into_iter() + .filter_map(|path| PathBuf::from(path).parent().map(Path::to_path_buf)) + .collect()) +} + +fn delete_orphan_snapshot_directories( + root: &Path, + referenced: &BTreeSet, +) -> ProfilerResult { + if !root.is_dir() { + return Ok(0); + } + let mut deleted = 0_u64; + for entry in fs::read_dir(root).map_err(|source| ProfilerError::Io { + operation: "scanning snapshot directory", + path: root.to_path_buf(), + source, + })? { + let entry = entry.map_err(|source| ProfilerError::Io { + operation: "reading snapshot directory entry", + path: root.to_path_buf(), + source, + })?; + let path = entry.path(); + if entry.file_type().is_ok_and(|kind| kind.is_dir()) && !referenced.contains(&path) { + fs::remove_dir_all(&path).map_err(|source| ProfilerError::Io { + operation: "deleting orphan profiler snapshot", + path: path.clone(), + source, + })?; + deleted = deleted.saturating_add(1); + } + } + Ok(deleted) +} + +fn validate_request(request: &ContinuousProfileRequest) -> ProfilerResult<()> { + if !request.archive_root.is_dir() { + return Err(ProfilerError::InvalidPath { + message: "MailVault archive root is not a directory".into(), + path: request.archive_root.clone(), + }); + } + if request.workspace_root.starts_with(&request.archive_root) + || request.archive_root.starts_with(&request.workspace_root) + { + return Err(ProfilerError::InvalidArgument( + "profiler workspace and MailVault archive must not overlap".into(), + )); + } + if request.options.retain_successful_snapshots == 0 { + return Err(ProfilerError::InvalidArgument( + "at least one successful snapshot must be retained".into(), + )); + } + Ok(()) +} + +struct ProfileLock { + file: File, +} + +impl ProfileLock { + fn acquire(workspace: &Path) -> ProfilerResult { + let path = workspace.join(".mailvault-profiler.profile.lock"); + let mut file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(&path) + .map_err(|source| ProfilerError::Io { + operation: "opening continuous profile lock", + path: path.clone(), + source, + })?; + file.try_lock_exclusive().map_err(|_| { + ProfilerError::SourceBusy("another profile writer owns this workspace".into()) + })?; + file.set_len(0).map_err(|source| ProfilerError::Io { + operation: "resetting continuous profile lock", + path: path.clone(), + source, + })?; + writeln!( + file, + "pid={} version={}", + std::process::id(), + env!("CARGO_PKG_VERSION") + ) + .map_err(|source| ProfilerError::Io { + operation: "writing continuous profile lock", + path, + source, + })?; + Ok(Self { file }) + } +} + +impl Drop for ProfileLock { + fn drop(&mut self) { + let _ = FileExt::unlock(&self.file); + } +} + +fn sqlite_error(operation: &'static str, source: &rusqlite::Error) -> ProfilerError { + ProfilerError::Sqlite { + operation, + message: source.to_string(), + } +} + +fn now_text() -> String { + time::OffsetDateTime::now_utc() + .format(&time::format_description::well_known::Rfc3339) + .unwrap_or_else(|_| "1970-01-01T00:00:00Z".into()) +} + +fn to_u64(value: i64) -> ProfilerResult { + u64::try_from(value) + .map_err(|_| ProfilerError::Internal("negative continuous-profile counter".into())) +} + +#[cfg(test)] +mod tests { + use super::*; + use profiler_core::{ArchiveMetrics, LockState}; + use time::OffsetDateTime; + + fn preflight_report(database: &Path) -> PreflightReport { + PreflightReport { + adapter: "mailvault".into(), + compatible: true, + archive_root: database + .parent() + .expect("database parent") + .to_string_lossy() + .into_owned(), + database_path: database.to_string_lossy().into_owned(), + database_bytes: fs::metadata(database).expect("database metadata").len(), + archive_identity: Some("archive-id".into()), + archive_identity_aliases: Vec::new(), + schema_version: Some(3), + journal_mode: Some("wal".into()), + lock_state: LockState::Absent, + metrics: ArchiveMetrics { + accounts: 1, + messages: 2, + message_occurrences: 2, + mime_parts: 3, + attachment_occurrences: 1, + blobs: 1, + blob_bytes: 4, + message_relations: 0, + participants: 2, + }, + checks: Vec::new(), + warnings_count: 0, + errors_count: 0, + inspected_at: OffsetDateTime::UNIX_EPOCH, + } + } + + #[test] + fn source_revision_ignores_sqlite_shared_memory_churn() { + let directory = tempfile::tempdir().expect("temporary directory"); + let database = directory.path().join("mailvault.sqlite3"); + let wal = PathBuf::from(format!("{}-wal", database.to_string_lossy())); + let shm = PathBuf::from(format!("{}-shm", database.to_string_lossy())); + + fs::write(&database, b"database").expect("write database"); + fs::write(&wal, b"wal").expect("write wal"); + fs::write(&shm, b"shared-memory-a").expect("write shm"); + + let report = preflight_report(&database); + let before = source_revision_fingerprint(&report).expect("fingerprint before"); + + fs::write(&shm, b"shared-memory-b-with-different-size").expect("rewrite shm"); + let after = source_revision_fingerprint(&report).expect("fingerprint after"); + + assert_eq!(before, after); + } + + #[test] + fn source_revision_changes_when_wal_changes() { + let directory = tempfile::tempdir().expect("temporary directory"); + let database = directory.path().join("mailvault.sqlite3"); + let wal = PathBuf::from(format!("{}-wal", database.to_string_lossy())); + + fs::write(&database, b"database").expect("write database"); + fs::write(&wal, b"wal-a").expect("write wal"); + + let report = preflight_report(&database); + let before = source_revision_fingerprint(&report).expect("fingerprint before"); + + fs::write(&wal, b"wal-b-with-different-size").expect("rewrite wal"); + let after = source_revision_fingerprint(&report).expect("fingerprint after"); + + assert_ne!(before, after); + } + + #[test] + fn source_revision_changes_when_logical_metrics_change() { + let directory = tempfile::tempdir().expect("temporary directory"); + let database = directory.path().join("mailvault.sqlite3"); + + fs::write(&database, b"database").expect("write database"); + + let before_report = preflight_report(&database); + let mut after_report = before_report.clone(); + after_report.metrics.messages += 1; + + let before = source_revision_fingerprint(&before_report).expect("fingerprint before"); + let after = source_revision_fingerprint(&after_report).expect("fingerprint after"); + + assert_ne!(before, after); + } +} diff --git a/crates/profiler-engine/src/file_stat.rs b/crates/profiler-engine/src/file_stat.rs index eec2593..749315c 100644 --- a/crates/profiler-engine/src/file_stat.rs +++ b/crates/profiler-engine/src/file_stat.rs @@ -24,7 +24,8 @@ pub(crate) fn run_file_stat( ProfilerError::Internal(format!("creating file-stat worker pool: {error}")) })?; - let (total_objects, total_bytes) = store.count_file_stat_objects(&request.collection_id)?; + let (total_objects, total_bytes) = + store.count_file_stat_objects(&request.run_id, &request.collection_id)?; let mut checkpoint = store .load_file_stat_checkpoint(&request.run_id)? .unwrap_or_else(FileStatCheckpoint::empty); @@ -48,6 +49,7 @@ pub(crate) fn run_file_stat( loop { let batch = store.load_file_stat_batch( + &request.run_id, &request.collection_id, checkpoint.last_sha256.as_deref(), request.options.batch_size, diff --git a/crates/profiler-engine/src/format.rs b/crates/profiler-engine/src/format.rs index dfba509..ee0bb41 100644 --- a/crates/profiler-engine/src/format.rs +++ b/crates/profiler-engine/src/format.rs @@ -434,9 +434,9 @@ fn resolve_object_path(archive_root: &Path, locator: &str) -> ProfilerResult ProfilerResult { let payload = serde_json::to_vec(&serde_json::json!({ "contract": "mailvault-profiler-exact-format-v1", @@ -446,9 +446,7 @@ fn configuration_fingerprint( "signatureVersion": tool.signature_version, "signatureSha256": tool.signature_sha256, "extensionEvidence": "symlink-alias-or-unchecked", - "batchSize": request.options.batch_size, - "workers": workers, - "timeoutSeconds": request.options.timeout_seconds, + "workContract": "content-addressed-selective-v2", "containerExpansion": false, })) .map_err(|error| { diff --git a/crates/profiler-engine/src/lib.rs b/crates/profiler-engine/src/lib.rs index ae543ef..08d9aea 100644 --- a/crates/profiler-engine/src/lib.rs +++ b/crates/profiler-engine/src/lib.rs @@ -1,3 +1,4 @@ +pub mod continuous; mod file_stat; pub mod format; pub mod workspace; @@ -44,6 +45,13 @@ pub struct ProfileResult { pub snapshot: SnapshotResult, pub inventory: InventoryResult, pub file_stat: FileStatResult, + pub run_kind: String, + pub generation: u64, + pub resumed: bool, + pub resume_count: u64, + pub source_revision_fingerprint: String, + pub format: Option, + pub snapshots_deleted: u64, } #[derive(Debug, Default, Clone, Copy)] @@ -104,6 +112,7 @@ impl ProfileEngine { &collection_id, &profiler_database, &configuration_fingerprint, + true, ); if let Err(error) = &result { let _ = store.fail_run(&run_id, error); @@ -112,7 +121,7 @@ impl ProfileEngine { } #[allow(clippy::too_many_arguments)] - fn profile_registered_run( + pub(crate) fn profile_registered_run( request: &ProfileRequest, progress: &dyn ProgressSink, adapter: MailVaultAdapter, @@ -122,6 +131,7 @@ impl ProfileEngine { collection_id: &str, profiler_database: &std::path::Path, configuration_fingerprint: &str, + complete_run: bool, ) -> ProfilerResult { store.transition_run(run_id, RunState::Preflighting)?; progress.send(ProgressEvent { @@ -199,7 +209,9 @@ impl ProfileEngine { &resolver, progress, )?; - store.transition_run(run_id, RunState::Succeeded)?; + if complete_run { + store.transition_run(run_id, RunState::Succeeded)?; + } Ok(ProfileResult { run_id: run_id.to_owned(), @@ -210,6 +222,13 @@ impl ProfileEngine { snapshot, inventory, file_stat, + run_kind: "initial".into(), + generation: 1, + resumed: false, + resume_count: 0, + source_revision_fingerprint: String::new(), + format: None, + snapshots_deleted: 0, }) } } diff --git a/crates/profiler-engine/tests/profile_pipeline.rs b/crates/profiler-engine/tests/profile_pipeline.rs index 112751f..df341ec 100644 --- a/crates/profiler-engine/tests/profile_pipeline.rs +++ b/crates/profiler-engine/tests/profile_pipeline.rs @@ -11,6 +11,7 @@ use profiler_core::{ }; use profiler_engine::{ ProfileEngine, ProfileOptions, ProfileRequest, ProfileResult as EngineProfileResult, + continuous::{ContinuousProfileEngine, ContinuousProfileOptions, ContinuousProfileRequest}, workspace::{ WorkspaceSession, add_review_note, export_sanitized_run, findings_page, list_runs, open_run, set_review_status, @@ -248,6 +249,158 @@ fn sanitized_exports_contain_no_source_paths_filenames_or_notes() { assert!(!csv.contains("private review note")); } +#[test] +fn continuous_profile_reconciles_metadata_updates_and_bounds_snapshots() { + let fixture = Fixture::create(); + let first = run_continuous_profile(&fixture, &RecordingProgress::default()); + assert_eq!(first.run_kind, "initial"); + assert_eq!(first.generation, 1); + + let source = Connection::open(fixture.archive.join("database/mailvault.sqlite3")) + .expect("open source for metadata update"); + source + .execute( + "UPDATE message_occurrences SET labels_json='[\"INBOX\",\"STARRED\"]', + updated_at='2026-07-28T12:00:00Z' WHERE id=1", + [], + ) + .expect("update occurrence labels"); + drop(source); + + let second = run_continuous_profile(&fixture, &RecordingProgress::default()); + assert_eq!(second.run_kind, "incremental"); + assert_eq!(second.generation, 2); + let profiler = Connection::open(&second.profiler_database).expect("open profiler database"); + let labels: String = profiler + .query_row( + "SELECT labels_json FROM source_message_occurrences + WHERE snapshot_id=?1 AND source_occurrence_id=1", + [&second.source_snapshot_id], + |row| row.get(0), + ) + .expect("load reconciled occurrence"); + assert_eq!(labels, "[\"INBOX\",\"STARRED\"]"); + let snapshots: i64 = profiler + .query_row("SELECT COUNT(*) FROM source_snapshots", [], |row| { + row.get(0) + }) + .expect("count retained snapshots"); + assert_eq!(snapshots, 1); +} + +#[test] +fn continuous_profile_reuses_successful_stat_and_retries_failures() { + let fixture = Fixture::create(); + let first = run_continuous_profile(&fixture, &RecordingProgress::default()); + let second = run_continuous_profile(&fixture, &RecordingProgress::default()); + let profiler = Connection::open(&second.profiler_database).expect("open profiler database"); + + let first_success: i64 = profiler + .query_row( + "SELECT COUNT(*) FROM file_stat_observations + WHERE run_id=?1 AND sha256=?2", + params![first.run_id, HASH_AVAILABLE], + |row| row.get(0), + ) + .expect("count initial success"); + let second_success: i64 = profiler + .query_row( + "SELECT COUNT(*) FROM file_stat_observations + WHERE run_id=?1 AND sha256=?2", + params![second.run_id, HASH_AVAILABLE], + |row| row.get(0), + ) + .expect("count repeated success"); + let second_missing: i64 = profiler + .query_row( + "SELECT COUNT(*) FROM file_stat_observations + WHERE run_id=?1 AND sha256=?2", + params![second.run_id, HASH_MISSING], + |row| row.get(0), + ) + .expect("count missing retry"); + assert_eq!(first_success, 1); + assert_eq!(second_success, 0); + assert_eq!(second_missing, 1); +} + +#[derive(Debug)] +struct InterruptingProgress { + interrupted: Mutex, +} + +impl InterruptingProgress { + fn new() -> Self { + Self { + interrupted: Mutex::new(false), + } + } +} + +impl ProgressSink for InterruptingProgress { + fn send(&self, event: ProgressEvent) -> ProfilerResult<()> { + let mut interrupted = self.interrupted.lock().expect("interrupt mutex poisoned"); + if !*interrupted + && event.stage == RunStage::MetadataInventory + && event.stage_state == StageState::Running + { + *interrupted = true; + return Err(profiler_core::ProfilerError::ProgressDelivery( + "controlled interruption".into(), + )); + } + Ok(()) + } +} + +#[test] +fn continuous_profile_resumes_the_same_run_after_committed_interruption() { + let fixture = Fixture::create(); + let interrupted = run_continuous_profile_result(&fixture, &InterruptingProgress::new()); + assert!(interrupted.is_err()); + let database = fixture.workspace.join("profiler/profiler.sqlite3"); + let profiler = Connection::open(&database).expect("open interrupted profiler database"); + let interrupted_run: String = profiler + .query_row( + "SELECT id FROM profiler_runs ORDER BY created_at DESC LIMIT 1", + [], + |row| row.get(0), + ) + .expect("load interrupted run"); + drop(profiler); + + let resumed = run_continuous_profile(&fixture, &RecordingProgress::default()); + assert_eq!(resumed.run_id, interrupted_run); + assert!(resumed.resumed); + assert_eq!(resumed.resume_count, 1); +} + +fn run_continuous_profile(fixture: &Fixture, progress: &dyn ProgressSink) -> EngineProfileResult { + run_continuous_profile_result(fixture, progress).expect("continuous profile succeeds") +} + +fn run_continuous_profile_result( + fixture: &Fixture, + progress: &dyn ProgressSink, +) -> ProfilerResult { + let mut options = ContinuousProfileOptions::default(); + options.profile.inventory.batch_size = 1; + options.profile.file_stat.batch_size = 1; + options.profile.file_stat.workers = 1; + options.format_required = false; + options.retain_successful_snapshots = 1; + ContinuousProfileEngine.profile( + &ContinuousProfileRequest { + archive_root: fixture.archive.clone(), + workspace_root: fixture.workspace.clone(), + siegfried_path: None, + signature_path: None, + options, + }, + progress, + ) +} + fn run_profile(fixture: &Fixture, progress: &RecordingProgress) -> EngineProfileResult { let mut options = ProfileOptions::default(); options.inventory.batch_size = 2; diff --git a/crates/profiler-storage-sqlite/migrations/0007_continuous_profile.sql b/crates/profiler-storage-sqlite/migrations/0007_continuous_profile.sql new file mode 100644 index 0000000..d281389 --- /dev/null +++ b/crates/profiler-storage-sqlite/migrations/0007_continuous_profile.sql @@ -0,0 +1,44 @@ +-- MailVault Collection Profiler Alpha 6 continuous lifecycle. +ALTER TABLE collections ADD COLUMN archive_identity_aliases_json TEXT NOT NULL DEFAULT '[]' + CHECK(json_valid(archive_identity_aliases_json)); + +ALTER TABLE source_snapshots ADD COLUMN retention_state TEXT NOT NULL DEFAULT 'retained' + CHECK(retention_state IN ('retained', 'pinned')); +ALTER TABLE source_snapshots ADD COLUMN retention_reason TEXT; + +ALTER TABLE profiler_runs ADD COLUMN run_kind TEXT NOT NULL DEFAULT 'initial' + CHECK(run_kind IN ('initial', 'incremental', 'rebuild')); +ALTER TABLE profiler_runs ADD COLUMN parent_run_id TEXT REFERENCES profiler_runs(id) ON DELETE SET NULL; +ALTER TABLE profiler_runs ADD COLUMN generation INTEGER NOT NULL DEFAULT 1 CHECK(generation > 0); +ALTER TABLE profiler_runs ADD COLUMN resume_count INTEGER NOT NULL DEFAULT 0 CHECK(resume_count >= 0); +ALTER TABLE profiler_runs ADD COLUMN last_resumed_at TEXT; +ALTER TABLE profiler_runs ADD COLUMN last_heartbeat_at TEXT; +ALTER TABLE profiler_runs ADD COLUMN failure_retryable INTEGER NOT NULL DEFAULT 0 CHECK(failure_retryable IN (0,1)); +ALTER TABLE profiler_runs ADD COLUMN source_revision_fingerprint TEXT; + +ALTER TABLE content_objects ADD COLUMN active INTEGER NOT NULL DEFAULT 1 CHECK(active IN (0,1)); +ALTER TABLE content_objects ADD COLUMN last_seen_snapshot_id TEXT REFERENCES source_snapshots(id) ON DELETE SET NULL; +ALTER TABLE content_objects ADD COLUMN stat_input_fingerprint TEXT; +ALTER TABLE content_objects ADD COLUMN last_stat_success_fingerprint TEXT; +ALTER TABLE content_objects ADD COLUMN stat_retry_count INTEGER NOT NULL DEFAULT 0 CHECK(stat_retry_count >= 0); +ALTER TABLE content_objects ADD COLUMN format_input_fingerprint TEXT; +ALTER TABLE content_objects ADD COLUMN last_format_success_fingerprint TEXT; +ALTER TABLE content_objects ADD COLUMN format_retry_count INTEGER NOT NULL DEFAULT 0 CHECK(format_retry_count >= 0); + +CREATE TABLE snapshot_retention_events ( + id TEXT PRIMARY KEY, + snapshot_id TEXT NOT NULL, + collection_id TEXT NOT NULL REFERENCES collections(id) ON DELETE CASCADE, + action TEXT NOT NULL CHECK(action IN ('deleted','delete_failed')), + reason TEXT NOT NULL, + detail_json TEXT NOT NULL CHECK(json_valid(detail_json)), + occurred_at TEXT NOT NULL +) STRICT; + +CREATE UNIQUE INDEX idx_runs_one_live_per_collection + ON profiler_runs(collection_id) + WHERE state IN ('pending','preflighting','snapshotting','ready','running','pausing','paused','cancelling'); +CREATE INDEX idx_runs_collection_generation ON profiler_runs(collection_id, generation DESC); +CREATE INDEX idx_content_objects_collection_active_sha ON content_objects(collection_id, active, sha256); +CREATE INDEX idx_snapshots_collection_retention_created + ON source_snapshots(collection_id, retention_state, created_at DESC); diff --git a/crates/profiler-storage-sqlite/src/format.rs b/crates/profiler-storage-sqlite/src/format.rs index 3570dda..3fb84b1 100644 --- a/crates/profiler-storage-sqlite/src/format.rs +++ b/crates/profiler-storage-sqlite/src/format.rs @@ -61,7 +61,10 @@ impl ProfilerStore { ORDER BY variant.occurrence_count DESC, variant.normalized_filename LIMIT 1) FROM content_objects AS content - WHERE content.collection_id=:collection_id + WHERE content.collection_id=:collection_id AND content.active=1 + AND (content.last_format_success_fingerprint IS NULL + OR content.last_format_success_fingerprint<>content.format_input_fingerprint + OR content.format_state IN ('uninspected','skipped_unavailable','tool_error')) AND (:after_sha256 IS NULL OR content.sha256 > :after_sha256) ORDER BY content.sha256 LIMIT :limit", @@ -159,6 +162,22 @@ impl ProfilerStore { ]), )); } + transaction.execute( + "UPDATE content_objects SET + last_format_success_fingerprint=format_input_fingerprint, + format_retry_count=0 + WHERE id IN (SELECT content_object_id FROM format_observations + WHERE format_run_id=?1 AND state IN ('identified','unknown','ambiguous','empty'))", + [format_run_id], + ).map_err(|source| sqlite_error("publishing exact-format cache", source))?; + transaction + .execute( + "UPDATE content_objects SET format_retry_count=format_retry_count+1 + WHERE id IN (SELECT content_object_id FROM format_observations + WHERE format_run_id=?1 AND state IN ('skipped_unavailable','tool_error'))", + [format_run_id], + ) + .map_err(|source| sqlite_error("recording exact-format retries", source))?; transaction .execute( "UPDATE format_identification_runs @@ -249,7 +268,6 @@ impl ProfilerStore { .query_row( format_count_sql(), named_params! { - ":baseline_run_id": request.baseline_run_id.as_str(), ":collection_id": collection_id.as_str(), ":search_like": search_like, ":state": state, @@ -267,7 +285,6 @@ impl ProfilerStore { let mut items = statement .query_map( named_params! { - ":baseline_run_id": request.baseline_run_id.as_str(), ":collection_id": collection_id.as_str(), ":search_like": search_like, ":state": state, @@ -377,6 +394,9 @@ impl ResumableFormatRun { struct PersistedFormatSummary { format_run_id: String, state: String, + total_objects: i64, + eligible_objects: i64, + total_bytes: i64, completed_objects: i64, completed_bytes: i64, identified: i64, @@ -405,33 +425,39 @@ impl PersistedFormatSummary { Ok(Self { format_run_id: row.get(0)?, state: row.get(1)?, - completed_objects: row.get(2)?, - completed_bytes: row.get(3)?, - identified: row.get(4)?, - unknown: row.get(5)?, - ambiguous: row.get(6)?, - empty: row.get(7)?, - skipped_unavailable: row.get(8)?, - tool_errors: row.get(9)?, - extension_mismatches: row.get(10)?, - started_at: row.get(11)?, - finished_at: row.get(12)?, - tool_name: row.get(13)?, - tool_version: row.get(14)?, - executable_path: row.get(15)?, - executable_sha256: row.get(16)?, - signature_path: row.get(17)?, - signature_sha256: row.get(18)?, - signature_version: row.get(19)?, - signature_created: row.get(20)?, - identifiers_json: row.get(21)?, - probed_at: row.get(22)?, + total_objects: row.get(2)?, + eligible_objects: row.get(3)?, + total_bytes: row.get(4)?, + completed_objects: row.get(5)?, + completed_bytes: row.get(6)?, + identified: row.get(7)?, + unknown: row.get(8)?, + ambiguous: row.get(9)?, + empty: row.get(10)?, + skipped_unavailable: row.get(11)?, + tool_errors: row.get(12)?, + extension_mismatches: row.get(13)?, + started_at: row.get(14)?, + finished_at: row.get(15)?, + tool_name: row.get(16)?, + tool_version: row.get(17)?, + executable_path: row.get(18)?, + executable_sha256: row.get(19)?, + signature_path: row.get(20)?, + signature_sha256: row.get(21)?, + signature_version: row.get(22)?, + signature_created: row.get(23)?, + identifiers_json: row.get(24)?, + probed_at: row.get(25)?, }) } fn apply_to(self, summary: &mut FormatSummary) -> ProfilerResult<()> { summary.latest_format_run_id = Some(self.format_run_id); summary.latest_run_state = Some(self.state); + summary.total_objects = to_u64(self.total_objects, "format total objects")?; + summary.eligible_objects = to_u64(self.eligible_objects, "format eligible objects")?; + summary.total_bytes = to_u64(self.total_bytes, "format total bytes")?; summary.completed_objects = to_u64(self.completed_objects, "format completed objects")?; summary.completed_bytes = to_u64(self.completed_bytes, "format completed bytes")?; summary.identified = to_u64(self.identified, "identified formats")?; @@ -481,10 +507,10 @@ fn load_format_baseline( .ok_or_else(|| { ProfilerError::contract(ErrorCode::RunNotFound, "baseline run was not found", false) })?; - if state != "succeeded" { + if state != "succeeded" && state != "running" { return Err(ProfilerError::contract( ErrorCode::RunNotBrowsable, - "exact format identification requires a completed physical profile", + "exact format identification requires an active or completed physical profile", false, )); } @@ -583,7 +609,11 @@ fn load_format_workload( COALESCE(SUM(CASE WHEN availability_state='available' AND expected_size_bytes>0 THEN expected_size_bytes ELSE 0 END), 0) - FROM content_objects WHERE collection_id=?1", + FROM content_objects WHERE collection_id=?1 AND active=1 + AND format_input_fingerprint IS NOT NULL + AND (last_format_success_fingerprint IS NULL + OR last_format_success_fingerprint<>format_input_fingerprint + OR format_state IN ('uninspected','skipped_unavailable','tool_error'))", [collection_id], |row| { Ok(FormatWorkload { @@ -823,10 +853,10 @@ fn load_latest_format_run( ) -> ProfilerResult> { connection .query_row( - "SELECT run.id, run.state, run.completed_objects, run.completed_bytes, - run.identified, run.unknown, run.ambiguous, run.empty_objects, - run.skipped_unavailable, run.tool_errors, run.extension_mismatches, - run.started_at, run.finished_at, + "SELECT run.id, run.state, run.total_objects, run.eligible_objects, run.total_bytes, + run.completed_objects, run.completed_bytes, run.identified, run.unknown, + run.ambiguous, run.empty_objects, run.skipped_unavailable, run.tool_errors, + run.extension_mismatches, run.started_at, run.finished_at, tool.tool_name, tool.tool_version, tool.executable_path, tool.executable_sha256, tool.signature_path, tool.signature_sha256, tool.signature_version, tool.signature_created, tool.identifiers_json, @@ -1077,58 +1107,40 @@ fn format_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { } fn format_count_sql() -> &'static str { - "WITH latest_run AS ( - SELECT id FROM format_identification_runs - WHERE baseline_run_id=:baseline_run_id - ORDER BY started_at DESC LIMIT 1 - ) - SELECT COUNT(*) FROM content_objects AS content - LEFT JOIN format_observations AS observation - ON observation.content_object_id=content.id - AND observation.format_run_id=(SELECT id FROM latest_run) - WHERE content.collection_id=:collection_id - AND (:state IS NULL OR COALESCE(observation.state, 'uninspected')=:state) - AND (:puid IS NULL OR observation.primary_identifier=:puid) - AND (:mismatch_only=0 OR COALESCE(observation.extension_mismatch, 0)=1) + "SELECT COUNT(*) FROM content_objects AS content + WHERE content.collection_id=:collection_id AND content.active=1 + AND (:state IS NULL OR content.format_state=:state) + AND (:puid IS NULL OR content.primary_puid=:puid) + AND (:mismatch_only=0 OR content.extension_mismatch=1) AND (:search_like IS NULL OR lower(content.sha256) LIKE :search_like OR lower(content.source_detected_mime_type) LIKE :search_like - OR lower(COALESCE(observation.primary_identifier, '')) LIKE :search_like - OR lower(COALESCE(observation.primary_format_name, '')) LIKE :search_like + OR lower(COALESCE(content.primary_puid, '')) LIKE :search_like + OR lower(COALESCE(content.primary_format_name, '')) LIKE :search_like OR EXISTS(SELECT 1 FROM filename_variants AS variant WHERE variant.content_object_id=content.id AND lower(variant.display_filename) LIKE :search_like))" } fn format_page_sql() -> &'static str { - "WITH latest_run AS ( - SELECT id FROM format_identification_runs - WHERE baseline_run_id=:baseline_run_id - ORDER BY started_at DESC LIMIT 1 - ) - SELECT content.id, content.sha256, + "SELECT content.id, content.sha256, COALESCE((SELECT variant.display_filename FROM filename_variants AS variant WHERE variant.content_object_id=content.id ORDER BY variant.occurrence_count DESC, variant.normalized_filename LIMIT 1), '[unnamed]'), content.expected_size_bytes, content.source_detected_mime_type, - COALESCE(observation.state, 'uninspected'), observation.primary_identifier, - observation.primary_format_name, observation.primary_format_version, - observation.primary_mime_type, COALESCE(observation.match_count, 0), - COALESCE(observation.extension_checked, 0), - COALESCE(observation.extension_mismatch, 0) + content.format_state, content.primary_puid, content.primary_format_name, + content.primary_format_version, content.primary_format_mime_type, + content.format_match_count, content.extension_checked, content.extension_mismatch FROM content_objects AS content - LEFT JOIN format_observations AS observation - ON observation.content_object_id=content.id - AND observation.format_run_id=(SELECT id FROM latest_run) - WHERE content.collection_id=:collection_id + WHERE content.collection_id=:collection_id AND content.active=1 AND (:after_sha256 IS NULL OR content.sha256>:after_sha256) - AND (:state IS NULL OR COALESCE(observation.state, 'uninspected')=:state) - AND (:puid IS NULL OR observation.primary_identifier=:puid) - AND (:mismatch_only=0 OR COALESCE(observation.extension_mismatch, 0)=1) + AND (:state IS NULL OR content.format_state=:state) + AND (:puid IS NULL OR content.primary_puid=:puid) + AND (:mismatch_only=0 OR content.extension_mismatch=1) AND (:search_like IS NULL OR lower(content.sha256) LIKE :search_like OR lower(content.source_detected_mime_type) LIKE :search_like - OR lower(COALESCE(observation.primary_identifier, '')) LIKE :search_like - OR lower(COALESCE(observation.primary_format_name, '')) LIKE :search_like + OR lower(COALESCE(content.primary_puid, '')) LIKE :search_like + OR lower(COALESCE(content.primary_format_name, '')) LIKE :search_like OR EXISTS(SELECT 1 FROM filename_variants AS variant WHERE variant.content_object_id=content.id AND lower(variant.display_filename) LIKE :search_like)) diff --git a/crates/profiler-storage-sqlite/src/lib.rs b/crates/profiler-storage-sqlite/src/lib.rs index 4b2eb31..e6fe1b0 100644 --- a/crates/profiler-storage-sqlite/src/lib.rs +++ b/crates/profiler-storage-sqlite/src/lib.rs @@ -10,6 +10,7 @@ pub fn migration_failure_marker_path(database_path: &Path) -> PathBuf { } use std::{ + collections::BTreeSet, fs::{self, OpenOptions}, io::Write, path::{Path, PathBuf}, @@ -33,7 +34,7 @@ use unicode_normalization::UnicodeNormalization; use uuid::Uuid; const APPLICATION_ID: i64 = 0x4D56_5046; // MVPF -pub const CURRENT_USER_VERSION: i64 = 6; +pub const CURRENT_USER_VERSION: i64 = 7; #[derive(Debug, Clone, Copy)] struct Migration { @@ -73,6 +74,11 @@ const MIGRATIONS: &[Migration] = &[ user_version: 6, sql: include_str!("../migrations/0006_exact_format_identification.sql"), }, + Migration { + id: "0007_continuous_profile", + user_version: 7, + sql: include_str!("../migrations/0007_continuous_profile.sql"), + }, ]; #[derive(Debug)] @@ -251,6 +257,174 @@ impl ProfilerStore { Ok(id) } + pub fn register_continuous_collection( + &self, + adapter_kind: &str, + archive_identity: &str, + aliases: &[String], + archive_root_display: &str, + ) -> ProfilerResult { + if let Some((id, payload)) = self.find_exact_collection(adapter_kind, archive_identity)? { + let mut merged = parse_identity_aliases(&payload)?; + merged.insert(archive_identity.to_owned()); + merged.extend(aliases.iter().cloned()); + self.update_continuous_collection( + &id, + archive_identity, + &merged, + archive_root_display, + false, + )?; + return Ok(id); + } + + let requested = aliases + .iter() + .cloned() + .chain(std::iter::once(archive_identity.to_owned())) + .collect::>(); + let mut matches = self.find_collection_alias_matches(adapter_kind, &requested)?; + if matches.len() > 1 { + return Err(ProfilerError::contract( + profiler_core::ErrorCode::WorkspaceSourceOverlap, + "archive identity aliases match more than one profiler collection", + false, + )); + } + if let Some((id, mut merged)) = matches.pop() { + merged.extend(requested); + self.update_continuous_collection( + &id, + archive_identity, + &merged, + archive_root_display, + true, + )?; + return Ok(id); + } + + self.insert_continuous_collection( + adapter_kind, + archive_identity, + &requested, + archive_root_display, + ) + } + + fn find_exact_collection( + &self, + adapter_kind: &str, + archive_identity: &str, + ) -> ProfilerResult> { + self.connection + .query_row( + "SELECT id, archive_identity_aliases_json FROM collections + WHERE adapter_kind=?1 AND archive_identity=?2", + params![adapter_kind, archive_identity], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + ) + .optional() + .map_err(|source| sqlite_error("finding exact collection identity", source)) + } + + fn find_collection_alias_matches( + &self, + adapter_kind: &str, + requested: &BTreeSet, + ) -> ProfilerResult)>> { + let mut statement = self + .connection + .prepare( + "SELECT id, archive_identity, archive_identity_aliases_json + FROM collections WHERE adapter_kind=?1", + ) + .map_err(|source| sqlite_error("preparing collection alias lookup", source))?; + let candidates = statement + .query_map([adapter_kind], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + )) + }) + .map_err(|source| sqlite_error("querying collection aliases", source))? + .collect::, _>>() + .map_err(|source| sqlite_error("collecting collection aliases", source))?; + + candidates + .into_iter() + .filter_map(|(id, identity, payload)| { + let mut persisted = match parse_identity_aliases(&payload) { + Ok(value) => value, + Err(error) => return Some(Err(error)), + }; + persisted.insert(identity); + (!persisted.is_disjoint(requested)).then_some(Ok((id, persisted))) + }) + .collect() + } + + fn update_continuous_collection( + &self, + id: &str, + archive_identity: &str, + aliases: &BTreeSet, + archive_root_display: &str, + migrate_identity: bool, + ) -> ProfilerResult<()> { + let aliases_json = serialize_identity_aliases(aliases)?; + if migrate_identity { + self.connection + .execute( + "UPDATE collections SET archive_identity=?1, archive_identity_aliases_json=?2, + archive_root_display=?3, updated_at=?4 WHERE id=?5", + params![ + archive_identity, + aliases_json, + archive_root_display, + now_text(), + id + ], + ) + .map_err(|source| sqlite_error("migrating collection identity", source))?; + } else { + self.connection + .execute( + "UPDATE collections SET archive_identity_aliases_json=?1, + archive_root_display=?2, updated_at=?3 WHERE id=?4", + params![aliases_json, archive_root_display, now_text(), id], + ) + .map_err(|source| sqlite_error("updating continuous collection", source))?; + } + Ok(()) + } + + fn insert_continuous_collection( + &self, + adapter_kind: &str, + archive_identity: &str, + aliases: &BTreeSet, + archive_root_display: &str, + ) -> ProfilerResult { + let id = Uuid::now_v7().to_string(); + self.connection + .execute( + "INSERT INTO collections(id, adapter_kind, archive_identity, archive_root_display, + created_at, updated_at, archive_identity_aliases_json) + VALUES(?1,?2,?3,?4,?5,?5,?6)", + params![ + id, + adapter_kind, + archive_identity, + archive_root_display, + now_text(), + serialize_identity_aliases(aliases)? + ], + ) + .map_err(|source| sqlite_error("registering continuous collection", source))?; + Ok(id) + } + pub fn register_source_snapshot( &self, collection_id: &str, @@ -472,15 +646,26 @@ impl InventorySink for ProfilerStore { transaction .execute( - "UPDATE content_objects AS content \ - SET occurrence_count=(\ - SELECT COUNT(*) FROM content_occurrences AS occurrence \ - WHERE occurrence.content_object_id=content.id\ - ), updated_at=?1 \ - WHERE collection_id=?2", - params![now_text(), request.collection_id], + "UPDATE content_objects AS content SET + active=CASE WHEN EXISTS(SELECT 1 FROM source_blobs b + WHERE b.snapshot_id=?1 AND b.sha256=content.sha256) + OR EXISTS(SELECT 1 FROM source_parts p + WHERE p.snapshot_id=?1 AND p.sha256=content.sha256) THEN 1 ELSE 0 END, + last_seen_snapshot_id=CASE WHEN EXISTS(SELECT 1 FROM source_blobs b + WHERE b.snapshot_id=?1 AND b.sha256=content.sha256) + OR EXISTS(SELECT 1 FROM source_parts p + WHERE p.snapshot_id=?1 AND p.sha256=content.sha256) + THEN ?1 ELSE last_seen_snapshot_id END, + occurrence_count=(SELECT COUNT(*) FROM content_occurrences o + WHERE o.content_object_id=content.id AND o.snapshot_id=?1), + updated_at=?2 WHERE collection_id=?3", + params![ + request.source_snapshot_id, + now_text(), + request.collection_id + ], ) - .map_err(|source| sqlite_error("updating content occurrence counts", source))?; + .map_err(|source| sqlite_error("updating current content projection", source))?; transaction .execute( @@ -499,10 +684,10 @@ impl InventorySink for ProfilerStore { SELECT content_object_id, filename_normalized, MIN(filename_original), COUNT(*), \ MIN(message_date), MAX(message_date) \ FROM content_occurrences \ - WHERE filename_normalized IS NOT NULL \ - AND content_object_id IN (SELECT id FROM content_objects WHERE collection_id=?1) \ + WHERE snapshot_id=?1 AND filename_normalized IS NOT NULL \ + AND content_object_id IN (SELECT id FROM content_objects WHERE collection_id=?2 AND active=1) \ GROUP BY content_object_id, filename_normalized", - [request.collection_id.as_str()], + params![request.source_snapshot_id, request.collection_id], ) .map_err(|source| sqlite_error("building filename variants", source))?; @@ -537,44 +722,52 @@ impl FileStatStore for ProfilerStore { .transpose() } - fn count_file_stat_objects(&self, collection_id: &str) -> ProfilerResult<(u64, u64)> { - let (objects, bytes): (i64, i64) = self - .connection - .query_row( - "SELECT COUNT(*), COALESCE(SUM(expected_size_bytes), 0) \ - FROM content_objects WHERE collection_id=?1", - [collection_id], - |row| Ok((row.get(0)?, row.get(1)?)), - ) - .map_err(|source| sqlite_error("counting file-stat objects", source))?; + fn count_file_stat_objects( + &self, + run_id: &str, + collection_id: &str, + ) -> ProfilerResult<(u64, u64)> { + let (objects, bytes):(i64,i64)=self.connection.query_row( + "SELECT COUNT(*), COALESCE(SUM(expected_size_bytes),0) FROM content_objects c + WHERE c.collection_id=?1 AND c.active=1 AND ( + EXISTS(SELECT 1 FROM file_stat_observations o WHERE o.run_id=?2 AND o.content_object_id=c.id) + OR c.last_stat_success_fingerprint IS NULL + OR c.last_stat_success_fingerprint<>c.stat_input_fingerprint + OR c.availability_state<>'available' OR c.size_state<>'match')", + params![collection_id,run_id], |row| Ok((row.get(0)?,row.get(1)?)), + ).map_err(|source| sqlite_error("counting selective file-stat workload", source))?; Ok(( - u64::try_from(objects).map_err(|_| { - ProfilerError::Internal("negative file-stat object count returned".into()) - })?, - u64::try_from(bytes).map_err(|_| { - ProfilerError::Internal("negative file-stat byte count returned".into()) - })?, + u64::try_from(objects) + .map_err(|_| ProfilerError::Internal("negative file-stat count".into()))?, + u64::try_from(bytes) + .map_err(|_| ProfilerError::Internal("negative file-stat bytes".into()))?, )) } fn load_file_stat_batch( &self, + run_id: &str, collection_id: &str, after_sha256: Option<&str>, limit: u32, ) -> ProfilerResult> { - let mut statement = self - .connection - .prepare_cached( - "SELECT id, sha256, expected_size_bytes, canonical_path_display \ - FROM content_objects \ - WHERE collection_id=?1 AND sha256>?2 \ - ORDER BY sha256 LIMIT ?3", - ) - .map_err(|source| sqlite_error("preparing file-stat work batch", source))?; + let mut statement=self.connection.prepare_cached( + "SELECT c.id,c.sha256,c.expected_size_bytes,c.canonical_path_display + FROM content_objects c WHERE c.collection_id=?1 AND c.active=1 AND c.sha256>?3 + AND NOT EXISTS(SELECT 1 FROM file_stat_observations o WHERE o.run_id=?2 AND o.content_object_id=c.id) + AND (c.last_stat_success_fingerprint IS NULL + OR c.last_stat_success_fingerprint<>c.stat_input_fingerprint + OR c.availability_state<>'available' OR c.size_state<>'match') + ORDER BY c.sha256 LIMIT ?4" + ).map_err(|source| sqlite_error("preparing selective file-stat batch",source))?; statement .query_map( - params![collection_id, after_sha256.unwrap_or(""), i64::from(limit)], + params![ + collection_id, + run_id, + after_sha256.unwrap_or(""), + i64::from(limit) + ], |row| { let size: i64 = row.get(2)?; Ok(FileStatWorkItem { @@ -586,9 +779,9 @@ impl FileStatStore for ProfilerStore { }) }, ) - .map_err(|source| sqlite_error("querying file-stat work batch", source))? + .map_err(|source| sqlite_error("querying selective file-stat batch", source))? .collect::, _>>() - .map_err(|source| sqlite_error("collecting file-stat work batch", source)) + .map_err(|source| sqlite_error("collecting selective file-stat batch", source)) } fn commit_file_stat_batch( @@ -611,7 +804,7 @@ impl FileStatStore for ProfilerStore { fn finalize_file_stat(&mut self, request: &FileStatRequest) -> ProfilerResult { let summary = file_stat_summary(&self.connection, request)?; let (expected_objects, expected_bytes) = - self.count_file_stat_objects(&request.collection_id)?; + self.count_file_stat_objects(&request.run_id, &request.collection_id)?; if summary.total_objects != expected_objects || summary.expected_bytes != expected_bytes { return Err(ProfilerError::Internal(format!( "file-stat reconciliation failed: observations={}/{}, bytes={}/{}", @@ -757,15 +950,19 @@ fn ingest_blobs( "INSERT INTO content_objects(\ id, collection_id, sha256, expected_size_bytes, source_detected_mime_type, \ canonical_path_display, availability_state, integrity_state, security_state, \ - first_seen_at, last_seen_at, occurrence_count, created_at, updated_at\ - ) VALUES(?1, ?2, ?3, ?4, ?5, ?6, 'uninspected', ?7, 'unknown', ?8, ?8, 0, ?9, ?9) \ + first_seen_at, last_seen_at, occurrence_count, created_at, updated_at, active, + last_seen_snapshot_id, stat_input_fingerprint, format_input_fingerprint\ + ) VALUES(?1, ?2, ?3, ?4, ?5, ?6, 'uninspected', ?7, 'unknown', ?8, ?8, 0, ?9, ?9, 1, ?10, ?11, ?12) \ ON CONFLICT(collection_id, sha256) DO UPDATE SET \ expected_size_bytes=excluded.expected_size_bytes, \ source_detected_mime_type=excluded.source_detected_mime_type, \ canonical_path_display=excluded.canonical_path_display, \ - integrity_state=excluded.integrity_state, \ + integrity_state=excluded.integrity_state, active=1, + last_seen_snapshot_id=excluded.last_seen_snapshot_id, + stat_input_fingerprint=excluded.stat_input_fingerprint, + format_input_fingerprint=excluded.format_input_fingerprint, \ first_seen_at=COALESCE(content_objects.first_seen_at, excluded.first_seen_at), \ - updated_at=excluded.updated_at", + last_seen_at=excluded.last_seen_at, updated_at=excluded.updated_at", ) .map_err(|source| sqlite_error("preparing blob ingestion", source))?; for row in rows { @@ -808,6 +1005,15 @@ fn ingest_blobs( }, row.first_seen_at, now, + request.source_snapshot_id, + format!( + "stat-v1:{}:{}:{}", + row.sha256, row.size_bytes, row.storage_path + ), + format!( + "format-v2:{}:{}:{}", + row.sha256, row.size_bytes, row.detected_mime_type + ), ]) .map_err(|source| sqlite_error("ingesting content object", source))?; } @@ -845,9 +1051,19 @@ fn ingest_parts( "INSERT INTO content_objects(\ id, collection_id, sha256, expected_size_bytes, source_detected_mime_type, \ canonical_path_display, availability_state, integrity_state, security_state, \ - first_seen_at, last_seen_at, occurrence_count, created_at, updated_at\ - ) VALUES(?1, ?2, ?3, ?4, ?5, ?6, 'uninspected', 'not_verified', 'unknown', ?7, ?7, 0, ?8, ?8) \ - ON CONFLICT(collection_id, sha256) DO NOTHING", + first_seen_at, last_seen_at, occurrence_count, created_at, updated_at, active, + last_seen_snapshot_id, stat_input_fingerprint, format_input_fingerprint\ + ) VALUES(?1, ?2, ?3, ?4, ?5, ?6, 'uninspected', 'not_verified', 'unknown', ?7, ?7, 0, ?8, ?8, 1, ?9, ?10, ?11) \ + ON CONFLICT(collection_id, sha256) DO UPDATE SET active=1, + last_seen_snapshot_id=excluded.last_seen_snapshot_id, + expected_size_bytes=excluded.expected_size_bytes, + source_detected_mime_type=excluded.source_detected_mime_type, + canonical_path_display=CASE WHEN excluded.canonical_path_display<>'' + THEN excluded.canonical_path_display ELSE content_objects.canonical_path_display END, + stat_input_fingerprint=CASE WHEN excluded.canonical_path_display<>'' + THEN excluded.stat_input_fingerprint ELSE content_objects.stat_input_fingerprint END, + format_input_fingerprint=excluded.format_input_fingerprint, + updated_at=excluded.updated_at", ) .map_err(|source| sqlite_error("preparing fallback content ingestion", source))?; let mut occurrence_statement = transaction @@ -921,6 +1137,21 @@ fn ingest_parts( row.blob_path.as_deref().unwrap_or(""), row.message_date, now, + request.source_snapshot_id, + format!( + "stat-v1:{}:{}:{}", + sha256, + row.size_bytes, + row.blob_path.as_deref().unwrap_or("") + ), + format!( + "format-v2:{}:{}:{}", + sha256, + row.size_bytes, + row.detected_mime_type + .as_deref() + .unwrap_or(&row.declared_mime_type) + ), ]) .map_err(|source| sqlite_error("ingesting fallback content object", source))?; @@ -1050,7 +1281,11 @@ fn persist_file_stat_observations( WHEN integrity_state='source_verified' THEN 'source_verified' \ ELSE 'size_verified' \ END, \ - modified_unix_ns=?4, last_stat_run_id=?5, last_stat_at=?6, updated_at=?6 \ + modified_unix_ns=?4, last_stat_run_id=?5, last_stat_at=?6, + last_stat_success_fingerprint=CASE WHEN ?2='available' AND ?3='match' + THEN stat_input_fingerprint ELSE last_stat_success_fingerprint END, + stat_retry_count=CASE WHEN ?2='available' AND ?3='match' THEN 0 ELSE stat_retry_count+1 END, + updated_at=?6 \ WHERE id=?7 AND collection_id=?8", ) .map_err(|source| sqlite_error("preparing content-object file-stat update", source))?; @@ -1532,7 +1767,7 @@ fn inventory_summary( )?, content_objects: count_where( connection, - "SELECT COUNT(*) FROM content_objects WHERE collection_id=?1", + "SELECT COUNT(*) FROM content_objects WHERE collection_id=?1 AND active=1", &request.collection_id, )?, content_occurrences: count_where( @@ -1547,7 +1782,7 @@ fn inventory_summary( )?, zero_byte_content_objects: count_where( connection, - "SELECT COUNT(*) FROM content_objects WHERE collection_id=?1 AND expected_size_bytes=0", + "SELECT COUNT(*) FROM content_objects WHERE collection_id=?1 AND active=1 AND expected_size_bytes=0", &request.collection_id, )?, same_hash_different_names: count_where( @@ -2033,6 +2268,19 @@ fn sqlite_error(operation: &'static str, source: rusqlite::Error) -> ProfilerErr } } +fn parse_identity_aliases(payload: &str) -> ProfilerResult> { + serde_json::from_str::>(payload) + .map(|values| values.into_iter().collect()) + .map_err(|error| { + ProfilerError::Internal(format!("invalid collection aliases JSON: {error}")) + }) +} + +fn serialize_identity_aliases(aliases: &BTreeSet) -> ProfilerResult { + serde_json::to_string(aliases) + .map_err(|error| ProfilerError::Internal(format!("serializing identity aliases: {error}"))) +} + #[cfg(test)] mod tests { use super::*; @@ -2050,7 +2298,7 @@ mod tests { row.get(0) }) .unwrap(); - assert_eq!(count, 6); + assert_eq!(count, 7); let user_version: i64 = second .connection() .pragma_query_value(None, "user_version", |row| row.get(0)) diff --git a/docs/ALPHA6_PACKAGE_VALIDATION.md b/docs/ALPHA6_PACKAGE_VALIDATION.md new file mode 100644 index 0000000..f7e4fdb --- /dev/null +++ b/docs/ALPHA6_PACKAGE_VALIDATION.md @@ -0,0 +1,18 @@ +# Alpha 6 package validation + +The Alpha 6 source package was assembled from the canonical Alpha 4 source archive and includes +workspace schema 7. Before delivery, the following package-level checks passed: + +- application, npm and Tauri versions align at `0.1.0-alpha.6`; +- MSI maps to `0.1.0.6`; +- tag/version gate accepts `v0.1.0-alpha.6`; +- documentation/privacy and local-link gates pass; +- JSON and TOML parse successfully; +- SQLite migrations 1 through 7 apply from an empty database; +- migration 6 to 7 preserves an existing succeeded run; +- `PRAGMA quick_check` is `ok` and `foreign_key_check` is empty; +- `git diff --check` reports no whitespace errors. + +Rust compilation, strict Clippy, the full test suite, Tauri installers and the real MailVault acceptance +matrix must run on the owner's Windows environment. This document deliberately does not claim those +runtime gates before their evidence exists. diff --git a/docs/ALPHA6_REAL_ACCEPTANCE_FA.md b/docs/ALPHA6_REAL_ACCEPTANCE_FA.md new file mode 100644 index 0000000..d2dca53 --- /dev/null +++ b/docs/ALPHA6_REAL_ACCEPTANCE_FA.md @@ -0,0 +1,28 @@ +# پذیرش واقعی MailVault Collection Profiler 0.1.0-alpha.6 + +این نسخه زمانی قابل انتشار است که Pipeline واحد زیر روی Copy واقعی Archive سبز شود: + +`preflight → online backup snapshot → full metadata reconciliation → selective file stat → selective exact format → final reconciliation → succeeded` + +## ترتیب اجرا + +1. `scripts/quality-alpha6.ps1 -BuildInstallers` +2. `scripts/validate-alpha6-real.ps1` +3. `scripts/validate-alpha6-delta.ps1` بعد از یک IMAP sync واقعی +4. `scripts/validate-alpha6-crash.ps1` برای هر چهار Stage +5. `scripts/validate-alpha6-soak.ps1 -Cycles 100` +6. `scripts/validate-alpha6-rebuild-equivalence.ps1` + +## معیار Closure + +- `PRAGMA quick_check=ok` و Foreign-key violation صفر؛ +- MailVault SHA-256 قبل و بعد یکسان؛ +- اجرای بدون تغییر، Object موفق قدیمی را دوباره Stat یا Identify نکند؛ +- Object ناموفق در Run بعد Retry شود؛ +- Attachment تکراری فقط Occurrence جدید بسازد و Content Object قبلی را reuse کند؛ +- Force-kill همان Run را با `resume_count>0` ادامه دهد؛ +- حداکثر Snapshotهای موفق مطابق Retention باقی بماند؛ +- نتیجه تجمعی Incremental با Rebuild مستقل byte-for-byte در Projection منطقی برابر باشد؛ +- MSI و NSIS، Source ZIP و SHA256SUMS از Commit و Tag واحد ساخته شوند. + +تا قبل از سبز شدن این Matrix، Release در GitHub نباید منتشر شود. diff --git a/docs/CONTINUOUS_PROFILING.md b/docs/CONTINUOUS_PROFILING.md new file mode 100644 index 0000000..a615365 --- /dev/null +++ b/docs/CONTINUOUS_PROFILING.md @@ -0,0 +1,17 @@ +# Continuous profiling — Alpha 6 final + +Alpha 6 is the final feature release of MailVault Collection Profiler. A workspace is bound to one +stable MailVault archive identity. Every run takes a consistent SQLite Online Backup snapshot, +reconciles all metadata from that immutable snapshot, and processes only content whose physical or +exact-format input fingerprint is new, changed, or retryable. + +The canonical pipeline is: + +`preflight → snapshot → inventory/reconciliation → selective file-stat → selective exact-format → success` + +A failed process resumes the same run and snapshot. Successful SHA-256 content is not re-stat-ed or +re-identified. Missing, mismatched, unavailable and tool-error objects remain retryable. Snapshot +retention keeps incomplete runs and the configured number of recent successful snapshots. + +`--rebuild` creates a new complete generation after any resumable run is resolved. `--watch` polls the +MailVault source revision and starts work only after the archive changes. diff --git a/docs/GITHUB_PUBLISHING_GUIDE_FA.md b/docs/GITHUB_PUBLISHING_GUIDE_FA.md index e8add49..d09a80e 100644 --- a/docs/GITHUB_PUBLISHING_GUIDE_FA.md +++ b/docs/GITHUB_PUBLISHING_GUIDE_FA.md @@ -90,7 +90,7 @@ git init git branch -M main git add . git status --short -git commit -m "Release MailVault Collection Profiler 0.1.0-alpha.4" +git commit -m "Release MailVault Collection Profiler 0.1.0-alpha.6" git remote add origin https://github.com/FireXCore/mailvault-collection-profiler.git git push -u origin main ``` @@ -262,7 +262,7 @@ Release فقط با Tag نسخه اجرا می‌شود. اگر CI قرمز است، Tag و Release نساز. انتهای Build موفق Frontend به‌تنهایی به معنی Gate سبز نیست. -## ۱۰. آماده‌سازی Release `0.1.0-alpha.4` +## ۱۰. آماده‌سازی Release `0.1.0-alpha.6` نسخه باید در این فایل‌ها هماهنگ باشد: @@ -278,13 +278,13 @@ apps/desktop/src-tauri/tauri.conf.json ```cmd npm run check:release-config -node scripts\check-tag-version.cjs v0.1.0-alpha.4 +node scripts\check-tag-version.cjs v0.1.0-alpha.6 ``` Release Notes عمومی در این فایل آماده است: ```text -docs/releases/v0.1.0-alpha.4.md +docs/releases/v0.1.0-alpha.6.md ``` ## ۱۱. ساخت Tag و اجرای Release Workflow @@ -293,8 +293,8 @@ docs/releases/v0.1.0-alpha.4.md git checkout main git pull --ff-only git status --short -git tag -a v0.1.0-alpha.4 -m "MailVault Collection Profiler 0.1.0-alpha.4" -git push origin v0.1.0-alpha.4 +git tag -a v0.1.0-alpha.6 -m "MailVault Collection Profiler 0.1.0-alpha.6" +git push origin v0.1.0-alpha.6 ``` Workflow زیر اجرا می‌شود: @@ -325,7 +325,7 @@ Releases → Draft release قبل از Publish کنترل کن: -- Tag دقیقاً `v0.1.0-alpha.4` باشد +- Tag دقیقاً `v0.1.0-alpha.6` باشد - گزینه `This is a pre-release` روشن باشد - Release Notes کامل باشد - NSIS `-setup.exe` وجود داشته باشد @@ -392,14 +392,14 @@ Publish release اگر قبل از Publish مشکل وجود دارد، Draft را حذف کن، Tag را فقط در صورت منتشرنشدن حذف و اصلاح کن: ```cmd -git push --delete origin v0.1.0-alpha.4 -git tag -d v0.1.0-alpha.4 +git push --delete origin v0.1.0-alpha.6 +git tag -d v0.1.0-alpha.6 ``` اگر Release منتشر شده است، Tag را بازنویسی نکن. یک نسخه جدید مانند زیر بساز: ```text -0.1.0-alpha.4 +0.1.0-alpha.6 ``` Release منتشرشده باید Immutable تلقی شود. diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index d434e81..ab9458f 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -1,40 +1,42 @@ -# Implementation status — 0.1.0-alpha.4 +# Implementation status — 0.1.0-alpha.6 ## Implemented -- read-only MailVault v3 preflight and SQLite snapshot; -- physical inventory, content/occurrence separation and file-stat checks; -- findings, append-only review, sanitized export and explorer UI; -- exact format core/storage/engine/sidecar/CLI/desktop modules; -- pinned Siegfried/PRONOM acquisition, verification and Tauri resource bundling; -- schema migration 6 and format filtering/progress/resume contract; -- publication-ready English/Persian documentation and sanitized screenshots. - -## Validated - -- complete Windows release configuration and documentation/privacy gate; -- Rust formatting and strict workspace Clippy with warnings denied; -- complete Rust test suite: 36 passed, 0 failed; -- frontend TypeScript check and Vite production build; -- native Tauri desktop compilation; -- npm audit: 0 vulnerabilities; -- Windows PowerShell 5.1 sidecar installation and independent verification; -- pinned Siegfried `1.11.6` and PRONOM `v124` runtime identity; -- Windows-safe home-relative signature argument contract; -- real Alpha 3 workspace migration from schema 5 to 6; -- full private 13,684-object exact-format run; -- 13,684/13,684 object coverage and 6,466,878,455/6,466,878,455 byte coverage; -- 13,636 identified, 46 unknown, 0 ambiguous and 0 tool errors; -- 51 extension mismatches and 64 distinct PUIDs recorded for review; -- canonical MailVault SHA-256 unchanged before and after identification; -- sanitized aggregate acceptance evidence and evidence checksum manifest. - -## Runtime status - -`0.1.0-alpha.4` is runtime green for the supplied private MailVault collection. Public Windows -installers remain unsigned and the release is intended for controlled technical evaluation. - -## Deferred - -Physical-profile pause/resume, full fixity, container expansion, JHOVE, extraction, OCR, semantic -search, procurement classification, RMS integration and automatic application updates. +- read-only MailVault schema-v3 preflight and SQLite Online Backup snapshot; +- stable collection binding with archive identity aliases and no path-based fallback; +- full metadata reconciliation from every immutable source snapshot; +- cumulative SHA-256 content projection with current/inactive state; +- selective file-stat reuse for successful unchanged objects; +- automatic retry for missing, mismatched, unavailable and I/O-error objects; +- automatic exact-format identification in the physical profile pipeline; +- Siegfried 1.11.6 and PRONOM v124 tool/signature evidence; +- same-run resume after interrupted snapshot, inventory, file-stat or format stages; +- initial, incremental and rebuild generations in one workspace; +- bounded successful-snapshot retention and orphan cleanup; +- continuous CLI watch with source-revision polling and retryable-error backoff; +- schema migration 6 to 7; +- Windows quality and real-runtime acceptance scripts. + +## Trusted predecessor evidence + +Alpha 4 completed the Windows quality gate with 36 Rust tests and processed 13,684/13,684 real +content objects with 13,636 identified, 46 unknown, zero ambiguous and zero tool errors. Alpha 6 +preserves that exact-format implementation and adds the continuous lifecycle around it. + +## Required before GitHub publication + +- `cargo fmt`, strict workspace Clippy and all Rust tests; +- TypeScript, Vite and native Tauri checks; +- MSI and NSIS build; +- real initial and immediate no-change profile; +- real IMAP delta and duplicate-SHA reuse; +- force-kill resume in all four stages; +- 100-cycle bounded-growth soak; +- independent rebuild equivalence; +- unchanged canonical MailVault SHA-256; +- sanitized public evidence and SHA256SUMS. + +## Permanently outside this profiler + +OCR, text extraction, embeddings, vector databases, RAG, LLM classification, procurement semantics, +RMS integration, attachment execution and container expansion. diff --git a/docs/INDEX.md b/docs/INDEX.md index 66c100e..d8f619f 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -34,3 +34,10 @@ - [Alpha 4 release notes](releases/v0.1.0-alpha.4.md) - [GitHub publishing guide — Persian](GITHUB_PUBLISHING_GUIDE_FA.md) - [Repository release handoff — Persian](REPOSITORY_RELEASE_HANDOFF_FA.md) + +- [Continuous profiling](CONTINUOUS_PROFILING.md) +- [Alpha 6 validation](VALIDATION_0.1.0-alpha.6.md) +- [Alpha 6 release notes](releases/v0.1.0-alpha.6.md) + +- [Alpha 6 real acceptance (FA)](ALPHA6_REAL_ACCEPTANCE_FA.md) +- [Alpha 6 package validation](ALPHA6_PACKAGE_VALIDATION.md) diff --git a/docs/INSTALLATION_WINDOWS.md b/docs/INSTALLATION_WINDOWS.md index 6c8ae50..1095173 100644 --- a/docs/INSTALLATION_WINDOWS.md +++ b/docs/INSTALLATION_WINDOWS.md @@ -2,7 +2,7 @@ ## Supported target -`0.1.0-alpha.4` is validated for Windows x64 using the MSVC Rust target. The project may compile on +`0.1.0-alpha.6` is validated for Windows x64 using the MSVC Rust target. The project may compile on other desktop platforms after installing normal Tauri system libraries, but the canonical native release gate is Windows. @@ -108,7 +108,7 @@ target\release\bundle\nsis target\release\bundle\msi ``` -The public SemVer remains `0.1.0-alpha.4`. WiX/MSI uses the numeric package version `0.1.0.4` +The public SemVer remains `0.1.0-alpha.6`. WiX/MSI uses the numeric package version `0.1.0.6` because Windows Installer does not accept an arbitrary prerelease label in ProductVersion. ## Run in development diff --git a/docs/RELEASE_PROCESS.md b/docs/RELEASE_PROCESS.md index a67b625..aab530a 100644 --- a/docs/RELEASE_PROCESS.md +++ b/docs/RELEASE_PROCESS.md @@ -36,7 +36,7 @@ apps/desktop/src-tauri/tauri.conf.json bundle.windows.wix.version ``` The public version may contain a prerelease suffix. MSI uses a separate numeric four-part version. -For `0.1.0-alpha.4`, the MSI version is `0.1.0.4`. +For `0.1.0-alpha.6`, the MSI version is `0.1.0.6`. ## 2. Write release notes and changelog @@ -67,7 +67,7 @@ Open an x64 Visual Studio Developer Command Prompt: call "C:\Program Files\Microsoft Visual Studio\18\Community\Common7\Tools\VsDevCmd.bat" -arch=amd64 -host_arch=amd64 cd /d D:\mailvault-collection-profiler npm ci -node scripts\check-tag-version.cjs v0.1.0-alpha.4 +node scripts\check-tag-version.cjs v0.1.0-alpha.6 powershell -NoProfile -ExecutionPolicy Bypass -File .\scripts\quality.ps1 npm run tauri:desktop:bundle powershell -NoProfile -ExecutionPolicy Bypass -File .\scripts\generate-release-checksums.ps1 @@ -101,10 +101,10 @@ Repeat the install/uninstall check with the MSI package. ```cmd git add . -git commit -m "Release MailVault Collection Profiler 0.1.0-alpha.4" +git commit -m "Release MailVault Collection Profiler 0.1.0-alpha.6" git push origin main -git tag -a v0.1.0-alpha.4 -m "MailVault Collection Profiler 0.1.0-alpha.4" -git push origin v0.1.0-alpha.4 +git tag -a v0.1.0-alpha.6 -m "MailVault Collection Profiler 0.1.0-alpha.6" +git push origin v0.1.0-alpha.6 ``` The tag starts `.github/workflows/release.yml`. diff --git a/docs/REPOSITORY_RELEASE_HANDOFF_FA.md b/docs/REPOSITORY_RELEASE_HANDOFF_FA.md index 6bb078f..e3f4ce4 100644 --- a/docs/REPOSITORY_RELEASE_HANDOFF_FA.md +++ b/docs/REPOSITORY_RELEASE_HANDOFF_FA.md @@ -1,4 +1,4 @@ -# تحویل انتشار GitHub — `0.1.0-alpha.4` +# تحویل انتشار GitHub — `0.1.0-alpha.6` این سند ترتیب امن انتقال Candidate آلفا ۴ به مخزن عمومی را مشخص می‌کند. آلفا ۴ قابلیت **Exact Format Identification** را به Profiler اضافه می‌کند، اما تا قبل از سبز شدن Windows CI و @@ -54,7 +54,7 @@ Live Siegfried installation/probe Real 13,684-object format run ``` -مدرک کامل: `docs/VALIDATION_0.1.0-alpha.4.md`. +مدرک کامل: `docs/VALIDATION_0.1.0-alpha.6.md`. ## مرحله ۱: ساخت Branch @@ -64,7 +64,7 @@ Real 13,684-object format run ```powershell git checkout main git pull --ff-only -git checkout -b feat/alpha4-exact-format-identification +git checkout -b feat/alpha6-exact-format-identification ``` فایل‌های بسته را روی Repository جایگزین کن. پوشه‌های `node_modules`، `target`، `dist`، دیتابیس‌های @@ -87,7 +87,7 @@ npm run tauri:desktop:bundle ```powershell npm run check:release-config -node .\scripts\check-tag-version.cjs v0.1.0-alpha.4 +node .\scripts\check-tag-version.cjs v0.1.0-alpha.6 ``` هیچ Tag یا Release تا قبل از سبز شدن این Gate ساخته نشود. @@ -98,7 +98,7 @@ node .\scripts\check-tag-version.cjs v0.1.0-alpha.4 git add . git status --short git commit -m "feat: add exact PRONOM format identification" -git push -u origin feat/alpha4-exact-format-identification +git push -u origin feat/alpha6-exact-format-identification ``` Pull Request باید تمام Required Checkهای Branch Protection را پاس کند. Diff اصلی شامل ۱۸ فایل @@ -132,8 +132,8 @@ Raw Profiler DB، Snapshot، Path، Filename، Subject، Domain و Progress JSON ```powershell git checkout main git pull --ff-only -git tag -a v0.1.0-alpha.4 -m "MailVault Collection Profiler 0.1.0-alpha.4" -git push origin v0.1.0-alpha.4 +git tag -a v0.1.0-alpha.6 -m "MailVault Collection Profiler 0.1.0-alpha.6" +git push origin v0.1.0-alpha.6 ``` Workflow `release.yml` یک Draft Release می‌سازد. Draft باید شامل NSIS، MSI و @@ -157,7 +157,7 @@ docs/assets/social-preview.png Release Notes: ```text -docs/releases/v0.1.0-alpha.4.md +docs/releases/v0.1.0-alpha.6.md ``` ## مرز ادعا diff --git a/docs/VALIDATION_0.1.0-alpha.4.md b/docs/VALIDATION_0.1.0-alpha.4.md index 8d840cd..c891126 100644 --- a/docs/VALIDATION_0.1.0-alpha.4.md +++ b/docs/VALIDATION_0.1.0-alpha.4.md @@ -2,11 +2,26 @@ ## Scope -This document records the completed release evidence for exact format identification in -MailVault Collection Profiler `0.1.0-alpha.4`. Private databases, paths, filenames and raw runtime -evidence are not published. +This document separates completed evidence from work that still requires Windows CI or a private +real-archive execution. It must not be read as a fabricated claim that exact format identification +has already been run across the production collection. -## Pinned format-identification contract +## Source examined + +- user-supplied `mailvault-collection-profiler` Alpha 3 source; +- private Alpha 3 real-profile workspace; +- Alpha 4 implementation candidate in this package; +- target application version `0.1.0-alpha.4`; +- profiler schema migration `0006_exact_format_identification.sql`. + +## Research decisions + +Alpha 4 uses a pinned Siegfried/PRONOM sidecar rather than an in-process magic-byte table or DROID +runtime because it provides machine-readable batch identification and persistent PRONOM IDs while +remaining practical to bundle in a Windows desktop application. The design records tool/signature +identity and retains all matches. + +Pinned contract: ```text Siegfried 1.11.6 @@ -14,94 +29,115 @@ PRONOM v124 container expansion disabled ``` -The executable and signature SHA-256 values are recorded in the private acceptance evidence and in -the profiler run metadata. - -## Completed Windows quality gate +## Completed static and frontend gates | Gate | Result | |---|---| -| Release/version configuration | passed | -| Documentation/privacy/local-link gate | passed | -| Rust formatting | passed | -| Strict Clippy with `-D warnings` | passed | -| Rust tests | passed, 36 | | TypeScript strict type-check | passed | | Vite production build | passed | -| Native Tauri desktop compilation | passed | -| Rust syntax validation | passed, 34 source files | -| npm audit | passed, 0 vulnerabilities | -| Pinned Siegfried installation | passed | -| Independent Siegfried verification | passed | -| Siegfried version | `1.11.6` | -| PRONOM signature | `v124` | -| Windows-safe home-relative signature contract | passed | +| Rust source Tree-sitter syntax parse | passed, 34 Rust files | +| Release configuration parser | passed after MSI mapping correction | +| Documentation/privacy/link gate | passed in packaged candidate | +| Screenshot dimensions and deterministic names | passed | +| npm production audit | passed, 0 vulnerabilities | +| npm complete audit | passed, 0 vulnerabilities | -## Workspace migration +Tree-sitter proves syntactic parsability only; it is not a substitute for Rust type checking. -A private copy of the real Alpha 3 profiler workspace was migrated from schema 5 to schema 6. +## Real Alpha 3 database migration test + +A private copy of the real Alpha 3 profiler database was migrated from schema 5 to schema 6. | Check | Result | |---|---| -| compatibility after migration | compatible | -| schema version | 6 | -| migration required | false | -| active workspace lock after completion | false | -| last migrated by | `0.1.0-alpha.4` | +| source database SHA-256 before/after test | unchanged | +| migrated database `PRAGMA quick_check(1)` | `ok` | +| `PRAGMA foreign_key_check` | no violations | | content objects preserved | 13,684 | -| source/snapshot clone hashes | matched | - -## Full real-archive exact-format run - -| Metric | Result | -|---|---:| -| Baseline run | `019f83f1-1687-7032-be61-5a9a1085ad51` | -| Format run | `019f84e2-037f-7271-b5d7-e814314dd5ba` | -| State | `succeeded` | -| Total objects | 13,684 | -| Eligible objects | 13,682 | -| Completed objects | 13,684 | -| Total bytes | 6,466,878,455 | -| Completed bytes | 6,466,878,455 | -| Identified | 13,636 | -| Unknown | 46 | -| Ambiguous | 0 | -| Empty | 1 | -| Skipped unavailable | 1 | -| Tool errors | 0 | -| Extension mismatches | 51 | -| Distinct PUIDs | 64 | - -State accounting is complete: +| content occurrences preserved | 22,068 | +| findings preserved | 1,484 | +| synthetic format projection | passed | +| baseline-scoped format projection isolation | passed | + +Source database SHA-256 used in the private migration test: ```text -13,636 identified -+ 46 unknown -+ 0 ambiguous -+ 1 empty -+ 1 skipped unavailable -+ 0 tool errors -= 13,684 total objects +d73b040cc7a136a80d0fcab6d8194fa66c2496958bfe4e6d69f06b849d1766b8 ``` -The canonical MailVault database SHA-256 matched before and after the run. +This hash is evidence for the provided private profiler database, not a public MailVault content +hash. -## Evidence boundary +## Implemented safety tests in code -The retained private evidence bundle includes: +The source includes unit/integration coverage for: -- workspace inspection after migration; -- exact-format summary; -- unknown, extension-mismatch and tool-error review queues; -- source SHA-256 before/after evidence; -- acceptance JSON; -- SHA-256 manifest for the evidence files. +- PRONOM version parsing; +- stronger signature evidence outranking extension-only evidence; +- exclusion of extension-only alternatives from false ambiguity; +- migration idempotency and expected migration count; +- format projection, baseline isolation and cursor query behavior; +- checkpoint/run completion constraints; +- physical-profile source immutability inherited from Alpha 3. -Raw archives, databases, filenames, local paths and profiler progress logs are excluded from the -public repository and release. +These Rust tests require semantic compilation in CI before release. -## Release status +## Windows runtime evidence supplied by the maintainer -The implementation, complete Windows quality gate, schema migration and full private real-archive -exact-format run are **runtime green**. Public installers remain unsigned and are distributed as an -Alpha prerelease for controlled technical evaluation. +The following gates were executed on Windows PowerShell 5.1 with Rust 1.97.1 tooling: + +| Gate | Result | +|---|---| +| pinned Siegfried installation | passed | +| independent Siegfried verification | passed | +| Siegfried version | `1.11.6` | +| PRONOM signature | `v124` | +| Windows-safe relative signature JSON contract | passed | +| `cargo test -p profiler-format-siegfried --locked` | passed, 5 tests | +| `cargo fmt --all -- --check` | passed | + +The first strict workspace Clippy run reached semantic compilation and reported structural/style +findings in the new exact-format modules. The implementation was then refactored without lint +suppression: options are borrowed, run-start parameters are grouped, large functions are decomposed, +large read buffers are heap-backed, and SQLite row/projection work is split into focused helpers. +This refactor still requires a fresh Windows `cargo clippy` execution before it can be recorded as +passed. + +## Not yet completed + +These results are deliberately **not claimed** until the updated source is rerun: + +- strict workspace Clippy after the structural refactor; +- complete Rust workspace test suite; +- native Tauri compile or installer build; +- real 13,684-object Siegfried identification run; +- real throughput, ETA accuracy, format distribution or peak memory. + +## Release CI requirements + +A publishable tag must pass the configured Windows workflow: + +1. install Rust `1.97.1` and Node `24`; +2. install and verify pinned Siegfried resources; +3. run formatting, Clippy and all Rust tests with `--locked`; +4. run TypeScript checks and production build; +5. run documentation/privacy gate; +6. compile Tauri desktop targets; +7. build NSIS and MSI installers; +8. generate `SHA256SUMS.txt` and artifact attestations where supported. + +## Runtime gate after CI + +Before calling Alpha 4 **runtime green**, execute the private runbook against the real workspace and +record: + +- exact tool and signature hashes; +- elapsed time and peak memory; +- identified, unknown, ambiguous, mismatch and tool-error counts; +- PUID/format distribution; +- checkpoint/resume evidence; +- source non-mutation evidence; +- sanitized aggregate export. + +Until that evidence exists, this package is an implementation/release candidate, not a completed +real-archive Alpha 4 benchmark. diff --git a/docs/VALIDATION_0.1.0-alpha.6.md b/docs/VALIDATION_0.1.0-alpha.6.md new file mode 100644 index 0000000..1b93100 --- /dev/null +++ b/docs/VALIDATION_0.1.0-alpha.6.md @@ -0,0 +1,14 @@ +# Alpha 6 validation contract + +Alpha 4 is the trusted static baseline. Alpha 6 must additionally prove on a copied real workspace: + +1. initial profile and exact-format completion; +2. immediate no-change run with no successful object reprocessing; +3. one real new-message delta; +4. duplicate attachment SHA reuse; +5. metadata-only update detection; +6. repair and retry of an unavailable object; +7. force-kill and same-run resume during snapshot, inventory, file-stat and format; +8. bounded snapshot/database growth under repeated cycles; +9. source MailVault SHA-256 unchanged; +10. cumulative result equivalent to a clean rebuild. diff --git a/docs/releases/v0.1.0-alpha.4.md b/docs/releases/v0.1.0-alpha.4.md index ebf9ef8..0445bba 100644 --- a/docs/releases/v0.1.0-alpha.4.md +++ b/docs/releases/v0.1.0-alpha.4.md @@ -1,52 +1,22 @@ # MailVault Collection Profiler `v0.1.0-alpha.4` -Alpha 4 adds exact technical file-format identification to the existing read-only physical -profiler. This is an unsigned Windows prerelease intended for controlled technical evaluation. - -It does not create a second archive, mutate MailVault, expand containers, execute attachments, -perform OCR or add procurement classification. +Alpha 4 adds exact technical file-format identification to the existing read-only physical profiler. +It does not create a second archive and does not add OCR or procurement classification. ## Highlights -- pinned Siegfried `1.11.6` and PRONOM `v124` Windows sidecar; +- pinned Siegfried `1.11.6` + PRONOM `v124` Windows sidecar; - verified release-asset acquisition and local tool/signature manifest; - PUID, format/version, MIME, evidence basis and all alternative matches; - deterministic primary assertion with explicit ambiguity preservation; -- safe extension-evidence aliases and an honest **Not checked** state; +- safe extension-evidence aliases and honest **Not checked** state; - bounded batch runner with timeout, output limits and adaptive failure isolation; - durable format checkpoints and same-fingerprint resume; - exclusive format-stage workspace lock; - desktop **Exact formats** dashboard and CLI commands; -- profiler workspace schema migration 6; +- profiler schema migration 6; - sanitized aggregate format summary schema 2; -- updated English/Persian documentation and sanitized release screenshots. - -## Real-archive validation - -The release candidate completed the full Windows quality gate and a private, full-scale MailVault -run. Only aggregate evidence is published. - -| Metric | Result | -|---|---:| -| Baseline run | `019f83f1-1687-7032-be61-5a9a1085ad51` | -| Exact-format run | `019f84e2-037f-7271-b5d7-e814314dd5ba` | -| Total objects | 13,684 | -| Eligible physical objects | 13,682 | -| Completed objects | 13,684 | -| Identified | 13,636 | -| Unknown | 46 | -| Ambiguous | 0 | -| Empty | 1 | -| Skipped unavailable | 1 | -| Tool errors | 0 | -| Extension mismatches | 51 | -| Distinct PUIDs | 64 | -| Completed bytes | 6,466,878,455 | -| Canonical MailVault source | SHA-256 unchanged | - -The complete Windows gate passed with Rust formatting, strict Clippy (`-D warnings`), 36 Rust tests, -TypeScript checking, Vite production build, native desktop compilation, pinned sidecar verification -and an npm audit reporting zero vulnerabilities. +- updated English/Persian docs and publication-ready screenshots. ## CLI @@ -63,28 +33,22 @@ See [CLI reference](../CLI_REFERENCE.md) and ## Security and privacy - MailVault remains read-only. -- Sidecar resources are versioned and SHA-256 recorded. +- Sidecar resources are versioned and hashed. - Containers are not expanded. - No attachment is executed or rendered. - Raw profiler databases, paths, filenames and progress evidence remain private. -- Public validation includes aggregate counts only. ## Upgrade Back up an Alpha 3 workspace before using `--allow-migration`. Migration `0006` preserves existing objects, occurrences, findings and review history while adding exact-format tables and indexes. -## Verify downloads - -After downloading the NSIS or MSI artifact and `SHA256SUMS.txt`: - -```powershell -Get-FileHash .\MailVault-Collection-Profiler_*.exe -Algorithm SHA256 -Get-FileHash .\MailVault-Collection-Profiler_*.msi -Algorithm SHA256 -Get-Content .\SHA256SUMS.txt -``` +## Validation status -Installers are unsigned and may trigger Windows SmartScreen. +Static/frontend gates and migration of a private real Alpha 3 workspace passed. Full Rust semantic +compilation, Windows installer creation and the first private real-archive Siegfried run are required +before publishing runtime-green performance claims. See +[Alpha 4 validation evidence](../VALIDATION_0.1.0-alpha.4.md). ## Known limitations @@ -93,5 +57,5 @@ Installers are unsigned and may trigger Windows SmartScreen. - no JHOVE validation; - no extraction, OCR or preview; - no document classification or RMS integration; -- no automatic application updater; -- unsigned Windows installers. +- unsigned Windows installers; +- no public real-archive format distribution is bundled in this source candidate. diff --git a/docs/releases/v0.1.0-alpha.6.md b/docs/releases/v0.1.0-alpha.6.md new file mode 100644 index 0000000..3a2d231 --- /dev/null +++ b/docs/releases/v0.1.0-alpha.6.md @@ -0,0 +1,8 @@ +# MailVault Collection Profiler v0.1.0-alpha.6 + +Final feature-complete prerelease combining incremental profiling, continuous watch, crash-safe +resume, content-addressed file/format reuse, update-aware full-snapshot metadata reconciliation, +stable archive identity migration and bounded snapshot retention. + +Publication requires the Windows quality gate and the real-archive initial/no-change/delta/crash/soak +acceptance scripts to pass. After Alpha 6 only maintenance, security and compatibility fixes are in scope. diff --git a/package-lock.json b/package-lock.json index 1d07296..46e5014 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "mailvault-profiler-workspace", - "version": "0.1.0-alpha.4", + "version": "0.1.0-alpha.6", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "mailvault-profiler-workspace", - "version": "0.1.0-alpha.4", + "version": "0.1.0-alpha.6", "license": "Apache-2.0", "workspaces": [ "apps/desktop" @@ -22,40 +22,40 @@ }, "apps/desktop": { "name": "@firexcore/mailvault-profiler-desktop", - "version": "0.1.0-alpha.4", + "version": "0.1.0-alpha.6", "license": "Apache-2.0", "dependencies": { "@tauri-apps/api": "^2.11.1", "@tauri-apps/plugin-dialog": "^2.7.2", - "react": "^19.2.7", - "react-dom": "^19.2.7" + "react": "^19.2.8", + "react-dom": "^19.2.8" }, "devDependencies": { "@tauri-apps/cli": "^2.11.4", - "@types/node": "^26.1.1", - "@types/react": "^19.2.0", - "@types/react-dom": "^19.2.0", - "@vitejs/plugin-react": "^6.0.3", + "@types/node": "^26.1.2", + "@types/react": "^19.2.18", + "@types/react-dom": "^19.2.4", + "@vitejs/plugin-react": "^6.0.5", "typescript": "^7.0.2", - "vite": "^8.1.5" + "vite": "^8.2.0" } }, "node_modules/@emnapi/core": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.1.tgz", - "integrity": "sha512-RSvbQmHzdKzNsLYa/wHrbc3KN4sYLKAdPZxqiM2HATqv/SBk2/ENSHpvXGaLOMcsAyz0poEGqkmmKYG3OWiJEQ==", + "version": "2.0.0-alpha.3", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-2.0.0-alpha.3.tgz", + "integrity": "sha512-AZypUeJ/yByuxyS7BlSNRDOMLMlROYtjYdIAuBmJssVz1UJDSeYxLrdizhXCFYhedC5bqd/ASy8EuNXbVVXp9g==", "dev": true, "license": "MIT", "optional": true, "dependencies": { - "@emnapi/wasi-threads": "1.2.2", + "@emnapi/wasi-threads": "2.0.1", "tslib": "^2.4.0" } }, "node_modules/@emnapi/runtime": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.1.tgz", - "integrity": "sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw==", + "version": "2.0.0-alpha.3", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-2.0.0-alpha.3.tgz", + "integrity": "sha512-hFPAhMUjJD9BSyCANEISPOogeXC9Zo9ZQl7L6vKnaVsMkCtzznaW/naYypeyl0Gv5rYfWYsZbpixTMpjDJzQeA==", "dev": true, "license": "MIT", "optional": true, @@ -64,9 +64,9 @@ } }, "node_modules/@emnapi/wasi-threads": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.2.tgz", - "integrity": "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-2.0.1.tgz", + "integrity": "sha512-9DsSk+o5NBX0CCJT8s0EROGSGxjR/tKu6aBTaVyq+SjAEQH4XcdcRxPBRzsBLizTTJ49MJjF+jgu3qnO9GLQcQ==", "dev": true, "license": "MIT", "optional": true, @@ -79,28 +79,31 @@ "link": true }, "node_modules/@napi-rs/wasm-runtime": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.6.tgz", - "integrity": "sha512-ZLv/JdUfkvOy9eCnnBaGfiO+XimbjebAeO+MRQqD/B+FR1tnRN0tpKSJHRbE8sFfS6aqsXZ67TQjfwfsxULVbg==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.2.2.tgz", + "integrity": "sha512-JfB4kuJQjaoHuCTseIINHtHWeJnvgEcxjwA5t/Y00ZgaOO1Crz3fjT/p8kT28zA/Caz7oiUMn3d6H2yOVCVwuw==", "dev": true, "license": "MIT", "optional": true, "dependencies": { "@tybys/wasm-util": "^0.10.3" }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=23.5.0" + }, "funding": { "type": "github", "url": "https://github.com/sponsors/Brooooooklyn" }, "peerDependencies": { - "@emnapi/core": "^1.7.1", - "@emnapi/runtime": "^1.7.1" + "@emnapi/core": "^1.7.1 || ^2.0.0-alpha.3", + "@emnapi/runtime": "^1.7.1 || ^2.0.0-alpha.3" } }, "node_modules/@oxc-project/types": { - "version": "0.139.0", - "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.139.0.tgz", - "integrity": "sha512-r9gHphtCs+1M7J0pw6Sn/hh/Wpa/iQrOOkrNAlVLF/gHq+/CJmHIWKKUUhdWjcD6CIa8idarspCsASiXCXvFUw==", + "version": "0.142.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.142.0.tgz", + "integrity": "sha512-7W+2q5AKQVU36fkaryontrHn3YDt1RyUYXatw9i5H8ocYe2sPKSFB6eS8WNPeRKiN1qAWWZUPm7gwFzJGrccqQ==", "dev": true, "license": "MIT", "funding": { @@ -108,9 +111,9 @@ } }, "node_modules/@rolldown/binding-android-arm64": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.1.5.tgz", - "integrity": "sha512-lZg8fqIv2v7FF237bwMgzGZEJvGL79/s5knJ/i6FmsGF4XXlzccZ4jb+TrFIxtSSxFtIpdsgrPZeMk1I9AFcyQ==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.1.tgz", + "integrity": "sha512-02hOeOSryYxVrOIphmLAsqnCJWxwlzFk+pEt/N/i6OgT3lShHO7xGCU5cpgchRDHboAEbSjzgGh+O/u1GswQmA==", "cpu": [ "arm64" ], @@ -125,9 +128,9 @@ } }, "node_modules/@rolldown/binding-darwin-arm64": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.1.5.tgz", - "integrity": "sha512-51Bnx9pNiMRKSUNtBfySkNJ9vMU9Hh3I1ozDd6gyPPYzaXCfnptUcEZxXGYFn+ul2dtcMUiqGR1Yai2K10uoTw==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.2.1.tgz", + "integrity": "sha512-fMsTOnN0OjFm3CyppWPitKnc8UlliVARUULW6cfU6AIqjdtgmSFWSk9vecHzZduv/yMWIHDlRhM1e8Iff9uAfA==", "cpu": [ "arm64" ], @@ -142,9 +145,9 @@ } }, "node_modules/@rolldown/binding-darwin-x64": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.1.5.tgz", - "integrity": "sha512-Tm+gbfC0aHu1tBA/JvKQh32S0K6YgCHkiAF4/W6xX0K0RmNuc94VeK419dJoE65R5aRxmo+noZQSWrAMF6yb6g==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.2.1.tgz", + "integrity": "sha512-1wjKdz/XLGKHaTNHjQveQ/B23TKx4ItAqm1JbyVuvNPc4Ze0Fb48s49TAd/2zcplPl8okE/UbTgmlVfwT7eFeQ==", "cpu": [ "x64" ], @@ -159,9 +162,9 @@ } }, "node_modules/@rolldown/binding-freebsd-x64": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.1.5.tgz", - "integrity": "sha512-JMzDKCCXq93YccG5gz3hvOs1oXRKAf0XYpfOS88e+wZrC8Iugj6j68867vrYZkvpDDpKn/KoKORThmchMpF6TA==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.2.1.tgz", + "integrity": "sha512-Fa0jHR07E7YBN4vOEsbVf2briYNsuOowfLJaXULZM0ldMlaCaj2LJgLMbMe4iacRyZmvR8efFhgR9wKuGclQUg==", "cpu": [ "x64" ], @@ -176,9 +179,9 @@ } }, "node_modules/@rolldown/binding-linux-arm-gnueabihf": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.1.5.tgz", - "integrity": "sha512-uML21j2K5TfPGutKxub+M+nLjZIrWjXQ5Grx4lCe/nimTj9B4L63zHpjXLl4y0L3mcm2htEQIb06oCG/szerNw==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.2.1.tgz", + "integrity": "sha512-pzkgu1SSHGgRRyRZ4fbmSgmajbVt+epaLP99NDjFft69v/ypfTi6swBMiVdh2EkQ0OSnHE1lZDM7DRGkyAzUpA==", "cpu": [ "arm" ], @@ -193,9 +196,9 @@ } }, "node_modules/@rolldown/binding-linux-arm64-gnu": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.1.5.tgz", - "integrity": "sha512-navSiuTMogvnQoZoM/v+l3ZWo50/NTwSHSzheABx/RCnmUPaKwq9qSo4Br2OYRs21+Fz8uFqITZM3H4opOB0/Q==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.2.1.tgz", + "integrity": "sha512-QI5SEDY8cbiYWHx0VO4vIc3UlS6a32vXHjU8Qy/17adEmZIPuByJg13UEvo9c/UCiUkdcVWY83C+b+JrwnNyUg==", "cpu": [ "arm64" ], @@ -213,9 +216,9 @@ } }, "node_modules/@rolldown/binding-linux-arm64-musl": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.1.5.tgz", - "integrity": "sha512-lAryqH7IteztmCXQXk0etKj4wBQ7Gx5S6LjKhsgp9zb8I5bsuvU/2llH1hDQcjsFeqIsovMVN339/8pUDDBXxA==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.2.1.tgz", + "integrity": "sha512-Sm41FyCeXqmYcERoYOCbGIL5hNfd8w9LQ7Y61Bev48HkcjaJqV/iiVOaiDxjVTRMS+QKrZmD8cfPt4uMVnvM+A==", "cpu": [ "arm64" ], @@ -233,9 +236,9 @@ } }, "node_modules/@rolldown/binding-linux-ppc64-gnu": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.1.5.tgz", - "integrity": "sha512-fsK/sNBnxzBlL4O1JNrZakVQxPspqpED5dLtNsZS9oOKmtSpdNIzxH2kkol5HYTWJN47sE20ztMJPxfZ89qGOg==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.2.1.tgz", + "integrity": "sha512-2x+WhXTGl9yJYPbltW/BSEPTVz9OIWQyER4N+gJEDWkkn904eRcBzELqh/Hf7K0w/ubGbKNMv0ZC+94QK/IFEg==", "cpu": [ "ppc64" ], @@ -253,9 +256,9 @@ } }, "node_modules/@rolldown/binding-linux-s390x-gnu": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.1.5.tgz", - "integrity": "sha512-gLYb4BIadlfTOYT5gO503n8zQjXflgzpD0FcyKh0Mzx3rqCZKnHoJWV9xe1KXUJ5lx2JfcSHr/mhzS0PC/McAA==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.2.1.tgz", + "integrity": "sha512-eEjmQpuRQayHPWWnywaWHkFT3ToPbP3RYy42VVd/B9aBGDA+Ol25EIWHxKQST3IiWJjikCWUF7KtbfqwZrzVwQ==", "cpu": [ "s390x" ], @@ -273,9 +276,9 @@ } }, "node_modules/@rolldown/binding-linux-x64-gnu": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.1.5.tgz", - "integrity": "sha512-FjcpEKUyJygHgs1o50VYNvkt5+7Le/VEdYt0AkRpkL33MnyQfwr8l5mXwMmfmTbyMPr5vJLC+8/Gd9gXnwU1QQ==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.2.1.tgz", + "integrity": "sha512-/Orga1fZYkLc/56jBICcHrKchl8Z2UKdDSr3LG9ToWO1lQ6a4Livk9Xz+9WN91zsz5QR3XQz2NNoSDEvP6qadw==", "cpu": [ "x64" ], @@ -293,9 +296,9 @@ } }, "node_modules/@rolldown/binding-linux-x64-musl": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.1.5.tgz", - "integrity": "sha512-Me+PfPI2TMeOQk0gYWfLQZtTktrmzbr8cDboqX83XKc7UrgAi55gF+2dUkWdxd19n55Essp2yeca+O9N5rBxHg==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.2.1.tgz", + "integrity": "sha512-xxBJRL+0q0Kce7orznGWLuylHDY65vuARXZRpX+hPdv+DqK2c3NlCsVA98tlWzWNEE7yPqA/1NQ5nnCrj49Y5A==", "cpu": [ "x64" ], @@ -313,9 +316,9 @@ } }, "node_modules/@rolldown/binding-openharmony-arm64": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.1.5.tgz", - "integrity": "sha512-yc5WrLzXks6zCQfn9Oxr8pORKyl/pF+QjHmW/Qx3qu0oyrrNC+y2JLTU1E2rcWYAmzlnqngWXHQjy51VzW70Vw==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.2.1.tgz", + "integrity": "sha512-M6AdXIXw3s+/8XpKMzdGDEXGS1S7kwUsy+rcTIUIOx5Ge4nXKCtAFHFV9YKkXvGcC5WMoTjAteLzlsQROVI0Yw==", "cpu": [ "arm64" ], @@ -330,28 +333,25 @@ } }, "node_modules/@rolldown/binding-wasm32-wasi": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.1.5.tgz", - "integrity": "sha512-VbQGPX2b4r48TAMIM2cjgluIM1HYutm4pcTEJsle7iEP7sB1dFqtPLBVbdLAZCxy1txCcPxf4QFf4v8uvltPqA==", - "cpu": [ - "wasm32" - ], + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.2.1.tgz", + "integrity": "sha512-/TX0SoRGojHzSAHpfVBbavRVSazg5U3h3Y3VXfcc0cdugq6kxdqw8LPGFiPr+/7gE/60zRcsOY2Vi9b9eT0jww==", "dev": true, "license": "MIT", "optional": true, "dependencies": { - "@emnapi/core": "1.11.1", - "@emnapi/runtime": "1.11.1", - "@napi-rs/wasm-runtime": "^1.1.6" + "@emnapi/core": "2.0.0-alpha.3", + "@emnapi/runtime": "2.0.0-alpha.3", + "@napi-rs/wasm-runtime": "^1.2.0" }, "engines": { - "node": "^20.19.0 || >=22.12.0" + "node": "^20.19.0 || ^22.13.0 || >=23.5.0" } }, "node_modules/@rolldown/binding-win32-arm64-msvc": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.1.5.tgz", - "integrity": "sha512-gHv82k63z4qpV5+Q1y/12KrK0ltWBukVDI8nZcbT7Tt/ZlOIVwppazneq0F93oDxTo3IgAMEDIoQh3E2n6mVsw==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.2.1.tgz", + "integrity": "sha512-EvRrivJieyHG+AO9lleZWgq+g0+S7oV2C51yuqlcyU/R9net+sI4Pj0F+lUoP2bEr6TWX3SqFaaS0SzfLxSzkw==", "cpu": [ "arm64" ], @@ -366,9 +366,9 @@ } }, "node_modules/@rolldown/binding-win32-x64-msvc": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.1.5.tgz", - "integrity": "sha512-tTZuDBPw85tEN5PQi1pnEBzDy0Z49HtScLAbD5t6hyeU92A95pRWaSMw1GZZi/RwgSgUIl0xrSlXIT/9QzvYSA==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.2.1.tgz", + "integrity": "sha512-Z4eCmn5QJ/5+azF9knpLWKfVd9aidn0mAe9TpJgvBLId9Ax3t0+JVxBmT25Bv7NBbVW1TZyKjQjQReouMeH5UQ==", "cpu": [ "x64" ], @@ -652,9 +652,9 @@ } }, "node_modules/@types/node": { - "version": "26.1.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-26.1.1.tgz", - "integrity": "sha512-nxAkRSVkN1Y0JC1W8ky/fTfkGsMmcrRsbx+3XoZE+rMOX71kLYTV7fLXpqud1GpbpP5TuffXFqfX7fH2GgZREw==", + "version": "26.1.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.1.2.tgz", + "integrity": "sha512-Vu4a5UFA9rIIFJ7rB/Vaafh9lrCQszopTCx6KjFboXTGQbPNasehVR5TEiithSDGyd1DEiUByggTZsg8jukeIg==", "dev": true, "license": "MIT", "dependencies": { @@ -662,9 +662,9 @@ } }, "node_modules/@types/react": { - "version": "19.2.17", - "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.17.tgz", - "integrity": "sha512-MXfmqaVPEVgkBT/aY0aGCkRWWtByiYQXo3xdQ8r5RzuFrPiRn8Gar2tQdXSUQ2GKV3bkXckek89V8wQBY2Q/Aw==", + "version": "19.2.18", + "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.18.tgz", + "integrity": "sha512-AnzbBERsrLKtk2XSfTbYRLjQPdy116Sty4q+T+Bp3IC4l6jNBvreVPAHmpq9qhXQM7CXZPjLVmGMw9sy+hxQ3w==", "dev": true, "license": "MIT", "dependencies": { @@ -672,9 +672,9 @@ } }, "node_modules/@types/react-dom": { - "version": "19.2.3", - "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.2.3.tgz", - "integrity": "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==", + "version": "19.2.4", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.2.4.tgz", + "integrity": "sha512-Bsc+QHgp+P/F02XDzNCY9jnZNCUuLki36KT7VKrTXXLdHf+vHMNZnW1rVu5DNW/rCK+fya3DATySbLM4yhtKUw==", "dev": true, "license": "MIT", "peerDependencies": { @@ -1022,9 +1022,9 @@ } }, "node_modules/@vitejs/plugin-react": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-6.0.3.tgz", - "integrity": "sha512-vmFvco5/QuC2f9Oj+wTk0+9XeDFkHxSamwZKYc7MxYwKICfvUvlMhqKI0VuICPltGqh1neqBKDvO4kes1ya8vg==", + "version": "6.0.5", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-6.0.5.tgz", + "integrity": "sha512-BOVzne/NL162sMdResB25mUv+vWMF5NoAjNf09TeGlE7ZpszZWSD3winycicLJw72yeVsoCn/2kOhEuCvEShMA==", "dev": true, "license": "MIT", "dependencies": { @@ -1417,9 +1417,9 @@ } }, "node_modules/postcss": { - "version": "8.5.20", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.20.tgz", - "integrity": "sha512-lW616l85ucIQL+FocMmL7pQFPqBmwejrCMg+iPxyImlrANNJG9NHq/RkyCZopDhd8C3LA03PHRJDjkbGu8vvug==", + "version": "8.5.25", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.25.tgz", + "integrity": "sha512-DTPx3RWSSnWyzLxQnlH0rJP+EW5ekl16ZU4/psbIhA0e53kJfdgaN5vKM+xP7yJtXVu+nfdVFmlgFDEKAe4Pyw==", "dev": true, "funding": [ { @@ -1446,34 +1446,34 @@ } }, "node_modules/react": { - "version": "19.2.7", - "resolved": "https://registry.npmjs.org/react/-/react-19.2.7.tgz", - "integrity": "sha512-HNe9WslTbXmFK8o8cmwgAeJFSBvt1bPdHCVKtaaV+WlAN36mpT4hcRpwbf3fY56ar2oIXzsBpOAiIRHAdY0OlQ==", + "version": "19.2.8", + "resolved": "https://registry.npmjs.org/react/-/react-19.2.8.tgz", + "integrity": "sha512-PWaYA1L/q9u2u7xYQi+Y3L3Yfnie7XyLeaJICV1MGD6LprsBxcAqGjYyr0eY3p+QdsA+x/Irkt4Qif8D63+Sbw==", "license": "MIT", "engines": { "node": ">=0.10.0" } }, "node_modules/react-dom": { - "version": "19.2.7", - "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.7.tgz", - "integrity": "sha512-t0BRVXvbiE/o20Hfw669rLbMCDWtYZLvmJigy2f0MxsXF+71pxhR3xOkspmsO8h3ZlNzyibAmtCa3l4lYKk6gQ==", + "version": "19.2.8", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.8.tgz", + "integrity": "sha512-rVprimfGBG3DR+Tq0IQG2DT5PxKth1WIGDmj5yPmlzr4YBe7uyE+Du4oVqTDXZSHGGGXRtTJEGSSePyQCMBglQ==", "license": "MIT", "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { - "react": "^19.2.7" + "react": "^19.2.8" } }, "node_modules/rolldown": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.1.5.tgz", - "integrity": "sha512-t9z29cJjXf/vxQ8dyhCSpt6H6aSwHTk8cT5I3iy6SMXuFpk5mB6PL6XfC8PCwrPTx93udwKUm9HRteAlTGBLiA==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.2.1.tgz", + "integrity": "sha512-4FKJhg8d3OiyQOA6Q1Q0hoFFpW9/OoX+VsHzpECsdsIZoOArrAK90gl59YK/Z+gnDel45bgJZK03ozH/9bCqEw==", "dev": true, "license": "MIT", "dependencies": { - "@oxc-project/types": "=0.139.0", + "@oxc-project/types": "=0.142.0", "@rolldown/pluginutils": "^1.0.0" }, "bin": { @@ -1483,21 +1483,21 @@ "node": "^20.19.0 || >=22.12.0" }, "optionalDependencies": { - "@rolldown/binding-android-arm64": "1.1.5", - "@rolldown/binding-darwin-arm64": "1.1.5", - "@rolldown/binding-darwin-x64": "1.1.5", - "@rolldown/binding-freebsd-x64": "1.1.5", - "@rolldown/binding-linux-arm-gnueabihf": "1.1.5", - "@rolldown/binding-linux-arm64-gnu": "1.1.5", - "@rolldown/binding-linux-arm64-musl": "1.1.5", - "@rolldown/binding-linux-ppc64-gnu": "1.1.5", - "@rolldown/binding-linux-s390x-gnu": "1.1.5", - "@rolldown/binding-linux-x64-gnu": "1.1.5", - "@rolldown/binding-linux-x64-musl": "1.1.5", - "@rolldown/binding-openharmony-arm64": "1.1.5", - "@rolldown/binding-wasm32-wasi": "1.1.5", - "@rolldown/binding-win32-arm64-msvc": "1.1.5", - "@rolldown/binding-win32-x64-msvc": "1.1.5" + "@rolldown/binding-android-arm64": "1.2.1", + "@rolldown/binding-darwin-arm64": "1.2.1", + "@rolldown/binding-darwin-x64": "1.2.1", + "@rolldown/binding-freebsd-x64": "1.2.1", + "@rolldown/binding-linux-arm-gnueabihf": "1.2.1", + "@rolldown/binding-linux-arm64-gnu": "1.2.1", + "@rolldown/binding-linux-arm64-musl": "1.2.1", + "@rolldown/binding-linux-ppc64-gnu": "1.2.1", + "@rolldown/binding-linux-s390x-gnu": "1.2.1", + "@rolldown/binding-linux-x64-gnu": "1.2.1", + "@rolldown/binding-linux-x64-musl": "1.2.1", + "@rolldown/binding-openharmony-arm64": "1.2.1", + "@rolldown/binding-wasm32-wasi": "1.2.1", + "@rolldown/binding-win32-arm64-msvc": "1.2.1", + "@rolldown/binding-win32-x64-msvc": "1.2.1" } }, "node_modules/scheduler": { @@ -1584,16 +1584,16 @@ "license": "MIT" }, "node_modules/vite": { - "version": "8.1.5", - "resolved": "https://registry.npmjs.org/vite/-/vite-8.1.5.tgz", - "integrity": "sha512-7ULLwsCdYx/nRyrpiEwvqb5TFHrMVZyBt+rg/OAXT7rgj/z+DtTDyKFeLAdDkubDVDKD8jOsndmy7m55XcfUsw==", + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.2.0.tgz", + "integrity": "sha512-pn+CFpM0lwDeKwmOq1ZaBK/9sjorZcgqxki6MbY/jPEVd9vichIlmlD4HmQ5wdP5EgqQCFRaACBxMC7uEGc6lQ==", "dev": true, "license": "MIT", "dependencies": { - "lightningcss": "^1.32.0", + "lightningcss": "^1.33.0", "picomatch": "^4.0.5", - "postcss": "^8.5.17", - "rolldown": "~1.1.5", + "postcss": "^8.5.23", + "rolldown": "~1.2.0", "tinyglobby": "^0.2.17" }, "bin": { @@ -1610,7 +1610,7 @@ }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", - "@vitejs/devtools": "^0.3.0", + "@vitejs/devtools": "^0.4.0", "esbuild": "^0.27.0 || ^0.28.0", "jiti": ">=1.21.0", "less": "^4.0.0", diff --git a/package.json b/package.json index a85ad1c..05dcaf3 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "mailvault-profiler-workspace", "private": true, - "version": "0.1.0-alpha.4", + "version": "0.1.0-alpha.6", "workspaces": [ "apps/desktop" ], diff --git a/scripts/acceptance-alpha6.ps1 b/scripts/acceptance-alpha6.ps1 new file mode 100644 index 0000000..2b1c8ae --- /dev/null +++ b/scripts/acceptance-alpha6.ps1 @@ -0,0 +1,26 @@ +param( + [Parameter(Mandatory=$true)][string]$ArchiveRoot, + [Parameter(Mandatory=$true)][string]$WorkspaceRoot, + [string]$Siegfried, + [string]$Signature, + [switch]$AllowKnownMissing, + [switch]$BuildInstallers +) +$ErrorActionPreference='Stop' +$EvidenceRoot=Join-Path $WorkspaceRoot 'acceptance-evidence' +New-Item -ItemType Directory -Force -Path $EvidenceRoot|Out-Null +& (Join-Path $PSScriptRoot 'quality-alpha6.ps1') -BuildInstallers:$BuildInstallers 2>&1 | + Tee-Object -FilePath (Join-Path $EvidenceRoot '01-quality.log') +& (Join-Path $PSScriptRoot 'validate-alpha6-real.ps1') -ArchiveRoot $ArchiveRoot ` + -WorkspaceRoot $WorkspaceRoot -Siegfried $Siegfried -Signature $Signature -SkipQuality -AllowKnownMissing:$AllowKnownMissing 2>&1 | + Tee-Object -FilePath (Join-Path $EvidenceRoot '02-initial-no-change.log') +$Manifest=[ordered]@{ + schema='mailvault.profiler.alpha6.acceptance.v1' + generatedAt=(Get-Date).ToUniversalTime().ToString('o') + archiveRootToken=(Get-FileHash (Join-Path $ArchiveRoot 'database\mailvault.sqlite3') -Algorithm SHA256).Hash + workspace=$WorkspaceRoot + completed=@('quality','initial','no_change','source_immutability') + pending=@('real_delta','force_kill_each_stage','soak','rebuild_equivalence') +} +$Manifest|ConvertTo-Json -Depth 6|Set-Content (Join-Path $EvidenceRoot 'acceptance-manifest.json') -Encoding UTF8 +$Manifest|ConvertTo-Json -Depth 6 diff --git a/scripts/apply-alpha6-final.ps1 b/scripts/apply-alpha6-final.ps1 new file mode 100644 index 0000000..89f1e5c --- /dev/null +++ b/scripts/apply-alpha6-final.ps1 @@ -0,0 +1,27 @@ +param( + [Parameter(Mandatory=$true)][string]$RepositoryRoot, + [Parameter(Mandatory=$true)][string]$SourceZip, + [string]$Branch='feat/alpha6-final-continuous-profile' +) +$ErrorActionPreference='Stop' +$RepositoryRoot=(Resolve-Path $RepositoryRoot).Path +if((git -C $RepositoryRoot status --short)){throw 'Repository must be clean before applying Alpha 6.'} +$Temp=Join-Path $env:TEMP ('mailvault-alpha6-'+[guid]::NewGuid().ToString('N')) +New-Item -ItemType Directory -Path $Temp|Out-Null +try{ + Expand-Archive -LiteralPath $SourceZip -DestinationPath $Temp -Force + $Source=Get-ChildItem $Temp -Directory|Select-Object -First 1 + if(!$Source){throw 'Source ZIP has no root directory.'} + git -C $RepositoryRoot switch -c $Branch + robocopy $Source.FullName $RepositoryRoot /MIR /XD .git target node_modules tools /XF '*.sqlite3' '*.log' + if($LASTEXITCODE -ge 8){throw "robocopy failed: $LASTEXITCODE"} + Push-Location $RepositoryRoot + try{ + cargo fmt --all + cargo fmt --all -- --check + git diff --check + node scripts/check-tag-version.cjs v0.1.0-alpha.6 + node scripts/check-docs.cjs + }finally{Pop-Location} + Write-Host 'Alpha 6 source applied. Run scripts\quality-alpha6.ps1 next.' -ForegroundColor Green +}finally{Remove-Item $Temp -Recurse -Force -ErrorAction SilentlyContinue} diff --git a/scripts/check-docs.cjs b/scripts/check-docs.cjs index e442490..5e5d84e 100644 --- a/scripts/check-docs.cjs +++ b/scripts/check-docs.cjs @@ -26,8 +26,13 @@ const requiredFiles = [ "docs/FINDINGS_REVIEW.md", "docs/VALIDATION_0.1.0-alpha.3.md", "docs/VALIDATION_0.1.0-alpha.4.md", + "docs/VALIDATION_0.1.0-alpha.6.md", + "docs/ALPHA6_PACKAGE_VALIDATION.md", + "docs/CONTINUOUS_PROFILING.md", + "docs/ALPHA6_REAL_ACCEPTANCE_FA.md", "docs/releases/v0.1.0-alpha.3.md", "docs/releases/v0.1.0-alpha.4.md", + "docs/releases/v0.1.0-alpha.6.md", "docs/FORMAT_IDENTIFICATION.md", "docs/FORMAT_IDENTIFICATION_RUNBOOK.md", "THIRD_PARTY_NOTICES.md", diff --git a/scripts/quality-alpha6.ps1 b/scripts/quality-alpha6.ps1 new file mode 100644 index 0000000..9547a60 --- /dev/null +++ b/scripts/quality-alpha6.ps1 @@ -0,0 +1,15 @@ +param([switch]$BuildInstallers) +Set-StrictMode -Version Latest +$ErrorActionPreference='Stop' +$Root=(Resolve-Path (Join-Path $PSScriptRoot '..')).Path +Push-Location $Root +try { + & (Join-Path $PSScriptRoot 'quality.ps1') + node scripts/check-tag-version.cjs v0.1.0-alpha.6 + if($LASTEXITCODE -ne 0){throw 'Alpha 6 tag/version gate failed.'} + if($BuildInstallers){ + $Npm=if($env:OS -eq 'Windows_NT'){'npm.cmd'}else{'npm'} + & $Npm run tauri:desktop:bundle + if($LASTEXITCODE -ne 0){throw 'Tauri installer build failed.'} + } +} finally { Pop-Location } diff --git a/scripts/run-real-archive-profile.ps1 b/scripts/run-real-archive-profile.ps1 index 9402f4d..ad9f898 100644 --- a/scripts/run-real-archive-profile.ps1 +++ b/scripts/run-real-archive-profile.ps1 @@ -420,7 +420,7 @@ finally { $manifest = [ordered]@{ schemaVersion = 1 - profilerVersion = "0.1.0-alpha.4" + profilerVersion = "0.1.0-alpha.6" startedAt = $startedAt.ToString("o") finishedAt = $finishedAt.ToString("o") elapsedMilliseconds = [long] $elapsedMilliseconds diff --git a/scripts/validate-alpha6-crash.ps1 b/scripts/validate-alpha6-crash.ps1 new file mode 100644 index 0000000..5809fe2 --- /dev/null +++ b/scripts/validate-alpha6-crash.ps1 @@ -0,0 +1,181 @@ +param( + [Parameter(Mandatory = $true)] + [string]$ArchiveRoot, + + [Parameter(Mandatory = $true)] + [string]$WorkspaceRoot, + + [ValidateSet('source_snapshot', 'metadata_inventory', 'file_stat', 'format_identification')] + [string]$Stage = 'metadata_inventory', + + [string]$Siegfried, + [string]$Signature +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +$Repo = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path +$Cargo = (Get-Command cargo.exe -ErrorAction Stop).Source +$CliTargetDir = Join-Path $Repo 'target-alpha6-cli-crash' +$CliExecutable = Join-Path $CliTargetDir 'release\mailvault-profiler.exe' +$PreviousCargoTargetDir = $env:CARGO_TARGET_DIR + +$Stdout = Join-Path $env:TEMP "mailvault-alpha6-crash-$Stage.out.jsonl" +$Stderr = Join-Path $env:TEMP "mailvault-alpha6-crash-$Stage.err.jsonl" + +function Restore-CargoTargetDirectory { + if ($null -eq $PreviousCargoTargetDir) { + Remove-Item Env:CARGO_TARGET_DIR -ErrorAction SilentlyContinue + } else { + $env:CARGO_TARGET_DIR = $PreviousCargoTargetDir + } +} + +function Read-LogTail { + param( + [Parameter(Mandatory = $true)] + [string]$Path, + + [int]$Lines = 80 + ) + + if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { + return '' + } + + return (Get-Content -LiteralPath $Path -Tail $Lines -ErrorAction SilentlyContinue) -join "`r`n" +} + +Push-Location $Repo +try { + # The desktop and CLI targets both produce mailvault-profiler.exe in the default + # Cargo target directory. Build the CLI in an isolated target directory and run + # that exact executable so a Tauri bundle cannot overwrite the crash harness target. + $env:CARGO_TARGET_DIR = $CliTargetDir + + & $Cargo build --locked --release -p mailvault-profiler-cli --bin mailvault-profiler + if ($LASTEXITCODE -ne 0) { + throw "Building the isolated Alpha 6 CLI failed with exit code $LASTEXITCODE." + } + + if (-not (Test-Path -LiteralPath $CliExecutable -PathType Leaf)) { + throw "Isolated CLI executable was not produced: $CliExecutable" + } + + if (Test-Path -LiteralPath $WorkspaceRoot) { + Remove-Item -LiteralPath $WorkspaceRoot -Recurse -Force + } + + Remove-Item $Stdout, $Stderr -Force -ErrorAction SilentlyContinue + + $CliArgs = @( + 'profile', + '--archive', $ArchiveRoot, + '--workspace', $WorkspaceRoot, + '--batch-size', '25', + '--file-stat-batch-size', '25', + '--format-batch-size', '25' + ) + + if ($Siegfried) { + $CliArgs += @('--siegfried', $Siegfried) + } + + if ($Signature) { + $CliArgs += @('--signature', $Signature) + } + + $Process = Start-Process ` + -FilePath $CliExecutable ` + -ArgumentList $CliArgs ` + -RedirectStandardError $Stderr ` + -RedirectStandardOutput $Stdout ` + -PassThru + + $Deadline = (Get-Date).AddMinutes(15) + $Killed = $false + $StagePattern = '"stage":"' + $Stage + '"' + + while ((Get-Date) -lt $Deadline) { + Start-Sleep -Milliseconds 100 + $Process.Refresh() + + $StageObserved = + (Test-Path -LiteralPath $Stderr -PathType Leaf) -and + ((Get-Content -LiteralPath $Stderr -Raw -ErrorAction SilentlyContinue) -match $StagePattern) + + if ($StageObserved) { + Stop-Process -Id $Process.Id -Force + [void]$Process.WaitForExit(10000) + $Killed = $true + break + } + + if ($Process.HasExited) { + break + } + } + + if (-not $Killed) { + if (-not $Process.HasExited) { + Stop-Process -Id $Process.Id -Force + [void]$Process.WaitForExit(10000) + } + + $StdoutTail = Read-LogTail -Path $Stdout + $StderrTail = Read-LogTail -Path $Stderr + + throw @" +Stage $Stage was not observed before process exit/timeout. +CLI executable: $CliExecutable +STDOUT tail: +$StdoutTail +STDERR tail: +$StderrTail +"@ + } + + # Resume by executing the exact same isolated CLI binary, not `cargo run`. + & $CliExecutable @CliArgs + if ($LASTEXITCODE -ne 0) { + throw "Resume after force-kill failed with exit code $LASTEXITCODE." + } +} finally { + Restore-CargoTargetDirectory + Pop-Location +} + +$Db = Join-Path $WorkspaceRoot 'profiler\profiler.sqlite3' +$Python = @' +import json +import sqlite3 +import sys + +connection = sqlite3.connect(sys.argv[1]) +assert connection.execute('pragma quick_check').fetchone()[0] == 'ok' +assert not connection.execute('pragma foreign_key_check').fetchall() +run = connection.execute( + 'select id,state,resume_count from profiler_runs order by generation desc limit 1' +).fetchone() +assert run[1] == 'succeeded' and run[2] > 0 +assert connection.execute('select count(*) from profiler_runs').fetchone()[0] == 1 +print(json.dumps({'runId': run[0], 'state': run[1], 'resumeCount': run[2]})) +'@ + +$Audit = Join-Path $env:TEMP 'mailvault-alpha6-crash.py' +[System.IO.File]::WriteAllText( + $Audit, + $Python.TrimEnd("`r", "`n") + "`n", + [System.Text.UTF8Encoding]::new($false) +) + +& python $Audit $Db +if ($LASTEXITCODE -ne 0) { + throw 'Force-kill resume assertions failed.' +} + +[pscustomobject]@{ + Status = 'FORCE_KILL_RESUME_GREEN' + Stage = $Stage +} | ConvertTo-Json diff --git a/scripts/validate-alpha6-delta.ps1 b/scripts/validate-alpha6-delta.ps1 new file mode 100644 index 0000000..b3bbba5 --- /dev/null +++ b/scripts/validate-alpha6-delta.ps1 @@ -0,0 +1,398 @@ +param( + [Parameter(Mandatory = $true)] + [string]$ArchiveRoot, + + [Parameter(Mandatory = $true)] + [string]$WorkspaceRoot, + + [string]$Siegfried, + [string]$Signature, + + [ValidateRange(1, 20)] + [int]$RetainSuccessfulSnapshots = 2 +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +$Repo = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path +$Cargo = (Get-Command cargo.exe -ErrorAction Stop).Source +$CliTargetDir = Join-Path $Repo 'target-alpha6-cli-delta' +$CliExecutable = Join-Path $CliTargetDir 'release\mailvault-profiler.exe' +$PreviousCargoTargetDir = $env:CARGO_TARGET_DIR + +$ProfilerDb = Join-Path $WorkspaceRoot 'profiler\profiler.sqlite3' +$SourceDb = Join-Path $ArchiveRoot 'database\mailvault.sqlite3' +$Evidence = Join-Path $WorkspaceRoot 'acceptance-evidence' + +foreach ($Required in @($ProfilerDb, $SourceDb)) { + if (-not (Test-Path -LiteralPath $Required -PathType Leaf)) { + throw "Required file is missing: $Required" + } +} + +New-Item -ItemType Directory -Force -Path $Evidence | Out-Null + +$Nonce = [Guid]::NewGuid().ToString('N') +$Marker = Join-Path $Evidence "alpha6-duplicate-sha-marker-$Nonce.bin" +$MarkerText = @" +MailVault Alpha 6 duplicate SHA acceptance marker +contract=v1 +nonce=$Nonce +"@ +$MarkerBytes = [System.Text.Encoding]::UTF8.GetBytes( + $MarkerText.TrimEnd("`r", "`n") + "`n" +) +[System.IO.File]::WriteAllBytes($Marker, $MarkerBytes) + +$Sha = ( + Get-FileHash -LiteralPath $Marker -Algorithm SHA256 +).Hash.ToLowerInvariant() + +$Audit = Join-Path $env:TEMP "mailvault-alpha6-delta-$Nonce.py" +$Python = @' +import json +import sqlite3 +import sys + +database = sys.argv[1] +sha256 = sys.argv[2] + +connection = sqlite3.connect(database) +assert connection.execute("PRAGMA quick_check").fetchone()[0] == "ok" +assert not connection.execute("PRAGMA foreign_key_check").fetchall() + +run = connection.execute( + """ + SELECT id, generation, source_snapshot_id + FROM profiler_runs + WHERE state = 'succeeded' + ORDER BY generation DESC + LIMIT 1 + """ +).fetchone() +assert run is not None, "no succeeded profiler run exists" + +snapshot = connection.execute( + "SELECT source_metrics_json FROM source_snapshots WHERE id = ?", + (run[2],), +).fetchone() +assert snapshot is not None, "latest run has no source snapshot" + +metrics = json.loads(snapshot[0]) + +content_object = connection.execute( + """ + SELECT + COUNT(*), + COALESCE(MAX(occurrence_count), 0), + COALESCE(MAX(format_state), 'absent'), + COALESCE(MAX(availability_state), 'absent'), + COALESCE(MAX(size_state), 'absent') + FROM content_objects + WHERE active = 1 + AND sha256 = ? + """, + (sha256,), +).fetchone() + +print(json.dumps({ + "run": run[0], + "generation": run[1], + "sourceSnapshotId": run[2], + "metrics": metrics, + "objectRows": content_object[0], + "occurrences": content_object[1], + "formatState": content_object[2], + "availability": content_object[3], + "sizeState": content_object[4], +})) +'@ + +[System.IO.File]::WriteAllText( + $Audit, + $Python.TrimEnd("`r", "`n") + "`n", + [System.Text.UTF8Encoding]::new($false) +) + +function Restore-CargoTargetDirectory { + if ($null -eq $PreviousCargoTargetDir) { + Remove-Item Env:CARGO_TARGET_DIR -ErrorAction SilentlyContinue + } else { + $env:CARGO_TARGET_DIR = $PreviousCargoTargetDir + } +} + +function Read-State { + $Output = & python $Audit $ProfilerDb $Sha + if ($LASTEXITCODE -ne 0) { + throw "Delta state audit failed with exit code $LASTEXITCODE." + } + + return ($Output | ConvertFrom-Json) +} + +function Get-OptionalFileState { + param( + [Parameter(Mandatory = $true)] + [string]$Path + ) + + if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { + return $null + } + + $Item = Get-Item -LiteralPath $Path + return [ordered]@{ + Path = $Item.FullName + Bytes = $Item.Length + LastWriteTimeUtc = $Item.LastWriteTimeUtc.ToString('o') + SHA256 = (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash + } +} + +function Get-SourceState { + return [ordered]@{ + Database = Get-OptionalFileState -Path $SourceDb + Wal = Get-OptionalFileState -Path ($SourceDb + '-wal') + } +} + +function Assert-SourceStateUnchanged { + param( + [Parameter(Mandatory = $true)] + [object]$Before, + + [Parameter(Mandatory = $true)] + [object]$After, + + [Parameter(Mandatory = $true)] + [string]$Label + ) + + $BeforeJson = $Before | ConvertTo-Json -Depth 6 -Compress + $AfterJson = $After | ConvertTo-Json -Depth 6 -Compress + + if ($BeforeJson -ne $AfterJson) { + throw "The profiler changed MailVault database/WAL state during '$Label'." + } +} + +function Invoke-Profile { + param( + [Parameter(Mandatory = $true)] + [string]$Label + ) + + $CliArgs = @( + 'profile', + '--archive', $ArchiveRoot, + '--workspace', $WorkspaceRoot, + '--retain-successful-snapshots', $RetainSuccessfulSnapshots.ToString() + ) + + if ($Siegfried) { + $CliArgs += @('--siegfried', $Siegfried) + } + + if ($Signature) { + $CliArgs += @('--signature', $Signature) + } + + $StdoutLog = Join-Path $Evidence "alpha6-delta-$Nonce-$Label.stdout.log" + $StderrLog = Join-Path $Evidence "alpha6-delta-$Nonce-$Label.stderr.log" + + Remove-Item ` + -LiteralPath $StdoutLog, $StderrLog ` + -Force ` + -ErrorAction SilentlyContinue + + $Process = Start-Process ` + -FilePath $CliExecutable ` + -ArgumentList $CliArgs ` + -RedirectStandardOutput $StdoutLog ` + -RedirectStandardError $StderrLog ` + -PassThru ` + -Wait + + if ($Process.ExitCode -ne 0) { + $StderrTail = '' + if (Test-Path -LiteralPath $StderrLog -PathType Leaf) { + $StderrTail = ( + Get-Content -LiteralPath $StderrLog -Tail 100 -ErrorAction SilentlyContinue + ) -join "`r`n" + } + + throw @" +Delta profile '$Label' failed with exit code $($Process.ExitCode). +STDOUT: $StdoutLog +STDERR: $StderrLog +STDERR tail: +$StderrTail +"@ + } + + return [pscustomobject]@{ + StdoutLog = $StdoutLog + StderrLog = $StderrLog + } +} + +Push-Location $Repo +try { + # The desktop and CLI packages both produce mailvault-profiler.exe in the + # default Cargo target directory. Build and run the CLI in an isolated target. + $env:CARGO_TARGET_DIR = $CliTargetDir + + & $Cargo build --locked --release -p mailvault-profiler-cli --bin mailvault-profiler + if ($LASTEXITCODE -ne 0) { + throw "Building the isolated Alpha 6 delta CLI failed with exit code $LASTEXITCODE." + } + + if (-not (Test-Path -LiteralPath $CliExecutable -PathType Leaf)) { + throw "Isolated CLI executable was not produced: $CliExecutable" + } + + $Before = Read-State + if ([int64]$Before.objectRows -ne 0) { + throw "The generated marker SHA already exists in the active profile projection." + } + + $BaselineSourceState = Get-SourceState + + Write-Host @" + +STEP 1 OF 2 +Attach this exact file to ONE new email sent to a mailbox archived by MailVault: + +$Marker + +SHA-256: $Sha + +Then run the normal MailVault IMAP sync. Do not copy the marker into the +MailVault object store manually. After the sync completes, return here. +"@ -ForegroundColor Cyan + [void](Read-Host 'Press Enter after the first MailVault sync has completed') + + $FirstSourceStateBeforeProfile = Get-SourceState + if ( + $FirstSourceStateBeforeProfile.Database.SHA256 -eq + $BaselineSourceState.Database.SHA256 + ) { + throw "The MailVault database hash did not change after the first requested sync." + } + + $FirstExecution = Invoke-Profile -Label 'first' + $FirstSourceStateAfterProfile = Get-SourceState + + Assert-SourceStateUnchanged ` + -Before $FirstSourceStateBeforeProfile ` + -After $FirstSourceStateAfterProfile ` + -Label 'first delta profile' + + $First = Read-State + + if ([int64]$First.generation -le [int64]$Before.generation) { + throw 'Generation did not advance after the first real delta.' + } + if ([int64]$First.objectRows -ne 1) { + throw 'The marker did not produce exactly one active content object.' + } + if ([int64]$First.occurrences -lt 1) { + throw 'The marker occurrence was not projected.' + } + if ($First.availability -ne 'available' -or $First.sizeState -ne 'match') { + throw 'The marker did not pass file-stat.' + } + if ($First.formatState -in @('uninspected', 'tool_error', 'absent')) { + throw "The marker did not complete exact-format profiling: $($First.formatState)" + } + if ( + [int64]$First.metrics.attachmentOccurrences -le + [int64]$Before.metrics.attachmentOccurrences + ) { + throw 'Attachment occurrence metrics did not advance after the first delta.' + } + + Write-Host @" + +STEP 2 OF 2 +Send the SAME marker file in a SECOND new email: + +$Marker + +Run the normal MailVault IMAP sync again. After the sync completes, return here. +"@ -ForegroundColor Cyan + [void](Read-Host 'Press Enter after the second MailVault sync has completed') + + $SecondSourceStateBeforeProfile = Get-SourceState + if ( + $SecondSourceStateBeforeProfile.Database.SHA256 -eq + $FirstSourceStateAfterProfile.Database.SHA256 + ) { + throw "The MailVault database hash did not change after the second requested sync." + } + + $SecondExecution = Invoke-Profile -Label 'second' + $SecondSourceStateAfterProfile = Get-SourceState + + Assert-SourceStateUnchanged ` + -Before $SecondSourceStateBeforeProfile ` + -After $SecondSourceStateAfterProfile ` + -Label 'second delta profile' + + $Second = Read-State + + if ([int64]$Second.generation -le [int64]$First.generation) { + throw 'Generation did not advance after the duplicate-SHA delta.' + } + if ([int64]$Second.objectRows -ne 1) { + throw 'Duplicate attachment created more than one active content object.' + } + if ([int64]$Second.occurrences -le [int64]$First.occurrences) { + throw 'Duplicate attachment did not add an occurrence.' + } + if ( + [int64]$Second.metrics.attachmentOccurrences -le + [int64]$First.metrics.attachmentOccurrences + ) { + throw 'Attachment occurrence metrics did not advance after the duplicate-SHA delta.' + } + if ($Second.availability -ne 'available' -or $Second.sizeState -ne 'match') { + throw 'The duplicate marker no longer passes file-stat.' + } + if ($Second.formatState -in @('uninspected', 'tool_error', 'absent')) { + throw "The duplicate marker has an invalid format state: $($Second.formatState)" + } + + $Summary = [ordered]@{ + Status = 'REAL_DELTA_AND_SHA_REUSE_GREEN' + CandidateHead = (git rev-parse HEAD).Trim() + MarkerPath = $Marker + SHA256 = $Sha + BaselineSourceState = $BaselineSourceState + FirstSourceState = $FirstSourceStateAfterProfile + SecondSourceState = $SecondSourceStateAfterProfile + FirstProfilerReadOnly = $true + SecondProfilerReadOnly = $true + Before = $Before + First = $First + Second = $Second + FirstExecution = $FirstExecution + SecondExecution = $SecondExecution + RecordedAtUtc = [DateTime]::UtcNow.ToString('o') + } + + $SummaryPath = Join-Path $Evidence "alpha6-real-delta-$Nonce.json" + [System.IO.File]::WriteAllText( + $SummaryPath, + ($Summary | ConvertTo-Json -Depth 12) + "`n", + [System.Text.UTF8Encoding]::new($false) + ) + + $Summary | ConvertTo-Json -Depth 12 + Get-FileHash -LiteralPath $SummaryPath -Algorithm SHA256 +} finally { + Restore-CargoTargetDirectory + Pop-Location +} diff --git a/scripts/validate-alpha6-real.ps1 b/scripts/validate-alpha6-real.ps1 new file mode 100644 index 0000000..63b0819 --- /dev/null +++ b/scripts/validate-alpha6-real.ps1 @@ -0,0 +1,57 @@ +param( + [Parameter(Mandatory=$true)][string]$ArchiveRoot, + [Parameter(Mandatory=$true)][string]$WorkspaceRoot, + [string]$Siegfried, + [string]$Signature, + [switch]$SkipQuality, + [switch]$AllowKnownMissing +) +Set-StrictMode -Version Latest +$ErrorActionPreference='Stop' +$Repo=(Resolve-Path (Join-Path $PSScriptRoot '..')).Path +$SourceDb=Join-Path $ArchiveRoot 'database\mailvault.sqlite3' +if(!(Test-Path -LiteralPath $SourceDb -PathType Leaf)){throw "MailVault database not found: $SourceDb"} +if(Test-Path -LiteralPath $WorkspaceRoot){throw 'Use a new empty validation workspace.'} +$Before=(Get-FileHash -LiteralPath $SourceDb -Algorithm SHA256).Hash +Push-Location $Repo +try { + if(!$SkipQuality){& (Join-Path $PSScriptRoot 'quality-alpha6.ps1')} + $Args=@('run','--locked','--release','-p','mailvault-profiler-cli','--bin','mailvault-profiler','--', + 'profile','--archive',$ArchiveRoot,'--workspace',$WorkspaceRoot, + '--retain-successful-snapshots','2') + if($Siegfried){$Args+=@('--siegfried',$Siegfried)} + if($Signature){$Args+=@('--signature',$Signature)} + & cargo @Args + if($LASTEXITCODE -ne 0){throw 'Initial Alpha 6 profile failed.'} + & cargo @Args + if($LASTEXITCODE -ne 0){throw 'Immediate no-change Alpha 6 profile failed.'} +} finally { Pop-Location } +$ProfilerDb=Join-Path $WorkspaceRoot 'profiler\profiler.sqlite3' +$Python=@' +import json,sqlite3,sys +c=sqlite3.connect(sys.argv[1]) +assert c.execute('pragma quick_check').fetchone()[0]=='ok' +assert not c.execute('pragma foreign_key_check').fetchall() +runs=c.execute("select id,run_kind,generation,state,resume_count,source_snapshot_id from profiler_runs order by generation").fetchall() +assert len(runs)>=2 and all(row[3]=='succeeded' for row in runs[-2:]) +first,latest=runs[-2],runs[-1] +assert latest[2]==first[2]+1 +success_reprocessed=c.execute("select count(*) from file_stat_observations where run_id=? and availability_state='available' and size_state='match'",(latest[0],)).fetchone()[0] +format_row=c.execute("select total_objects,identified,unknown,ambiguous,empty_objects,skipped_unavailable,tool_errors,state from format_identification_runs where baseline_run_id=? order by started_at desc limit 1",(latest[0],)).fetchone() +assert format_row is not None and format_row[7]=='succeeded' +assert success_reprocessed==0, f'unchanged successful file-stat objects were reprocessed: {success_reprocessed}' +assert sum(format_row[1:5])==0, f'unchanged successful format objects were reidentified: {format_row}' +missing=c.execute("select count(*) from content_objects where active=1 and availability_state<>'available'").fetchone()[0] +snapshots=c.execute('select count(*) from source_snapshots').fetchone()[0] +assert snapshots<=2 +print(json.dumps({'runs':runs[-2:],'successfulFileStatReprocessed':success_reprocessed,'latestFormatWork':format_row,'activeUnavailable':missing,'retainedSnapshots':snapshots},indent=2)) +if missing and sys.argv[2]!='1': raise SystemExit('active unavailable objects remain; resolve them or rerun with -AllowKnownMissing after documenting the exception') +'@ +$Audit=Join-Path $env:TEMP 'mailvault-alpha6-real.py' +Set-Content -LiteralPath $Audit -Value $Python -Encoding UTF8 +$AllowKnownMissingFlag = if ($AllowKnownMissing.IsPresent) { '1' } else { '0' } +python $Audit $ProfilerDb $AllowKnownMissingFlag +if($LASTEXITCODE -ne 0){throw 'Initial/no-change database assertions failed.'} +$After=(Get-FileHash -LiteralPath $SourceDb -Algorithm SHA256).Hash +if($Before -ne $After){throw 'Profiler modified canonical MailVault database.'} +[pscustomobject]@{Status='INITIAL_NO_CHANGE_GREEN';MailVaultSHA256=$After;Workspace=$WorkspaceRoot}|ConvertTo-Json diff --git a/scripts/validate-alpha6-rebuild-equivalence.ps1 b/scripts/validate-alpha6-rebuild-equivalence.ps1 new file mode 100644 index 0000000..5c78425 --- /dev/null +++ b/scripts/validate-alpha6-rebuild-equivalence.ps1 @@ -0,0 +1,51 @@ +param( + [Parameter(Mandatory=$true)][string]$ArchiveRoot, + [Parameter(Mandatory=$true)][string]$ContinuousWorkspace, + [Parameter(Mandatory=$true)][string]$RebuildWorkspace, + [string]$Siegfried, + [string]$Signature +) +$ErrorActionPreference='Stop' +if(Test-Path $RebuildWorkspace){throw 'Rebuild workspace must be new and empty.'} +$Args=@('run','--locked','--release','-p','mailvault-profiler-cli','--bin','mailvault-profiler','--', + 'profile','--archive',$ArchiveRoot,'--workspace',$RebuildWorkspace,'--rebuild') +if($Siegfried){$Args+=@('--siegfried',$Siegfried)} +if($Signature){$Args+=@('--signature',$Signature)} +& cargo @Args +if($LASTEXITCODE -ne 0){throw 'Independent rebuild failed.'} +$ContinuousDb=Join-Path $ContinuousWorkspace 'profiler\profiler.sqlite3' +$RebuildDb=Join-Path $RebuildWorkspace 'profiler\profiler.sqlite3' +$Python=@' +import hashlib,json,sqlite3,sys + +def digest(db): + c=sqlite3.connect(db) + run=c.execute("select id,source_snapshot_id from profiler_runs where state='succeeded' order by generation desc limit 1").fetchone() + if not run: raise SystemExit(f'no successful run in {db}') + run_id,snapshot=run + queries={ + 'messages':("select source_message_id,archive_id,account_id,provider_thread_namespace,provider_thread_value,rfc_message_id,subject_raw,subject_normalized,header_date,raw_path,raw_sha256,raw_size_bytes,parse_defects_json from source_messages where snapshot_id=? order by source_message_id",(snapshot,)), + 'occurrences':("select source_occurrence_id,source_message_id,generation_id,uid,flags_json,labels_json,internal_date,rfc822_size,modseq,selected_for_raw,fetch_status,last_error from source_message_occurrences where snapshot_id=? order by source_occurrence_id",(snapshot,)), + 'participants':("select source_participant_id,source_message_id,role,ordinal,name,address,domain from source_participants where snapshot_id=? order by source_participant_id",(snapshot,)), + 'parts':("select source_part_id,source_message_id,part_path,parent_part_path,role,declared_mime_type,detected_mime_type,content_disposition,content_id,filename_original,filename_safe,charset,transfer_encoding,size_bytes,sha256,blob_path,headers_json,defects_json from source_parts where snapshot_id=? order by source_part_id",(snapshot,)), + 'relations':("select source_relation_id,source_message_id,target_message_id,relation_type,evidence_type,confidence from source_message_relations where snapshot_id=? order by source_relation_id",(snapshot,)), + 'content':("select sha256,expected_size_bytes,source_detected_mime_type,canonical_path_display,active,availability_state,size_state,format_state,primary_puid,primary_format_name,primary_format_version,primary_format_mime_type,format_match_count,extension_checked,extension_mismatch from content_objects where active=1 order by sha256",()), + } + out={} + for name,(sql,args) in queries.items(): + h=hashlib.sha256(); count=0 + for row in c.execute(sql,args): + h.update(json.dumps(row,ensure_ascii=False,separators=(',',':'),default=str).encode());h.update(b'\n');count+=1 + out[name]={'count':count,'sha256':h.hexdigest()} + assert c.execute('pragma quick_check').fetchone()[0]=='ok' + assert not c.execute('pragma foreign_key_check').fetchall() + return out +left=digest(sys.argv[1]);right=digest(sys.argv[2]) +print(json.dumps({'continuous':left,'rebuild':right,'equal':left==right},indent=2)) +if left!=right: raise SystemExit('cumulative incremental projection differs from independent rebuild') +'@ +$Audit=Join-Path $env:TEMP 'mailvault-alpha6-equivalence.py' +Set-Content $Audit $Python -Encoding UTF8 +python $Audit $ContinuousDb $RebuildDb +if($LASTEXITCODE -ne 0){throw 'Rebuild equivalence failed.'} +[pscustomobject]@{Status='REBUILD_EQUIVALENCE_GREEN';Continuous=$ContinuousWorkspace;Rebuild=$RebuildWorkspace}|ConvertTo-Json diff --git a/scripts/validate-alpha6-soak.ps1 b/scripts/validate-alpha6-soak.ps1 new file mode 100644 index 0000000..a1c1f07 --- /dev/null +++ b/scripts/validate-alpha6-soak.ps1 @@ -0,0 +1,355 @@ +param( + [Parameter(Mandatory = $true)] + [string]$ArchiveRoot, + + [Parameter(Mandatory = $true)] + [string]$WorkspaceRoot, + + [ValidateRange(1, 1000)] + [int]$Cycles = 100, + + [string]$Siegfried, + [string]$Signature, + + [ValidateRange(1, 20)] + [int]$RetainSuccessfulSnapshots = 2 +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +$Repo = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path +$Cargo = (Get-Command cargo.exe -ErrorAction Stop).Source +$CliTargetDir = Join-Path $Repo 'target-alpha6-cli-soak' +$CliExecutable = Join-Path $CliTargetDir 'release\mailvault-profiler.exe' +$PreviousCargoTargetDir = $env:CARGO_TARGET_DIR + +$SourceDb = Join-Path $ArchiveRoot 'database\mailvault.sqlite3' + +if (-not (Test-Path -LiteralPath $SourceDb -PathType Leaf)) { + throw "MailVault database was not found: $SourceDb" +} + +if (Test-Path -LiteralPath $WorkspaceRoot) { + throw "Use a new soak workspace. Existing path: $WorkspaceRoot" +} + +$EvidenceRoot = Join-Path $WorkspaceRoot 'acceptance-evidence' +New-Item -ItemType Directory -Force -Path $EvidenceRoot | Out-Null + +$RunNonce = [Guid]::NewGuid().ToString('N') + +function Restore-CargoTargetDirectory { + if ($null -eq $PreviousCargoTargetDir) { + Remove-Item Env:CARGO_TARGET_DIR -ErrorAction SilentlyContinue + } else { + $env:CARGO_TARGET_DIR = $PreviousCargoTargetDir + } +} + +function Get-OptionalFileState { + param( + [Parameter(Mandatory = $true)] + [string]$Path + ) + + if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { + return $null + } + + $Item = Get-Item -LiteralPath $Path + return [ordered]@{ + Path = $Item.FullName + Bytes = $Item.Length + LastWriteTimeUtc = $Item.LastWriteTimeUtc.ToString('o') + SHA256 = (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash + } +} + +function Get-SourceState { + return [ordered]@{ + Database = Get-OptionalFileState -Path $SourceDb + Wal = Get-OptionalFileState -Path ($SourceDb + '-wal') + } +} + +function Assert-SourceStateUnchanged { + param( + [Parameter(Mandatory = $true)] + [object]$Before, + + [Parameter(Mandatory = $true)] + [object]$After + ) + + $BeforeJson = $Before | ConvertTo-Json -Depth 6 -Compress + $AfterJson = $After | ConvertTo-Json -Depth 6 -Compress + + if ($BeforeJson -ne $AfterJson) { + throw 'MailVault database/WAL state changed during the soak acceptance.' + } +} + +function Invoke-SoakCycle { + param( + [Parameter(Mandatory = $true)] + [int]$Cycle + ) + + $CycleText = $Cycle.ToString('D3') + $StdoutLog = Join-Path $EvidenceRoot "alpha6-soak-$RunNonce-cycle-$CycleText.stdout.log" + $StderrLog = Join-Path $EvidenceRoot "alpha6-soak-$RunNonce-cycle-$CycleText.stderr.log" + + $Arguments = @( + 'profile', + '--archive', $ArchiveRoot, + '--workspace', $WorkspaceRoot, + '--retain-successful-snapshots', $RetainSuccessfulSnapshots.ToString() + ) + + if ($Siegfried) { + $Arguments += @('--siegfried', $Siegfried) + } + + if ($Signature) { + $Arguments += @('--signature', $Signature) + } + + Write-Host "Alpha 6 soak cycle $Cycle / $Cycles" -ForegroundColor Cyan + + $Process = Start-Process ` + -FilePath $CliExecutable ` + -ArgumentList $Arguments ` + -RedirectStandardOutput $StdoutLog ` + -RedirectStandardError $StderrLog ` + -PassThru ` + -Wait + + if ($Process.ExitCode -ne 0) { + $StderrTail = '' + if (Test-Path -LiteralPath $StderrLog -PathType Leaf) { + $StderrTail = ( + Get-Content -LiteralPath $StderrLog -Tail 100 -ErrorAction SilentlyContinue + ) -join "`r`n" + } + + throw @" +Soak cycle $Cycle failed with exit code $($Process.ExitCode). +STDOUT: $StdoutLog +STDERR: $StderrLog +STDERR tail: +$StderrTail +"@ + } +} + +Push-Location $Repo +try { + # The desktop and CLI packages both produce mailvault-profiler.exe in the + # default Cargo target directory. Build and run the CLI in an isolated target. + $env:CARGO_TARGET_DIR = $CliTargetDir + + & $Cargo build --locked --release -p mailvault-profiler-cli --bin mailvault-profiler + if ($LASTEXITCODE -ne 0) { + throw "Building the isolated Alpha 6 soak CLI failed with exit code $LASTEXITCODE." + } + + if (-not (Test-Path -LiteralPath $CliExecutable -PathType Leaf)) { + throw "Isolated CLI executable was not produced: $CliExecutable" + } + + $SourceStateBefore = Get-SourceState + + for ($Cycle = 1; $Cycle -le $Cycles; $Cycle++) { + Invoke-SoakCycle -Cycle $Cycle + } + + $SourceStateAfter = Get-SourceState + Assert-SourceStateUnchanged -Before $SourceStateBefore -After $SourceStateAfter +} finally { + Restore-CargoTargetDirectory + Pop-Location +} + +$ProfilerDb = Join-Path $WorkspaceRoot 'profiler\profiler.sqlite3' +if (-not (Test-Path -LiteralPath $ProfilerDb -PathType Leaf)) { + throw "Profiler database was not produced: $ProfilerDb" +} + +$Audit = Join-Path $env:TEMP "mailvault-alpha6-soak-$RunNonce.py" +$Python = @' +import json +import os +import sqlite3 +import sys + +database = sys.argv[1] +expected_cycles = int(sys.argv[2]) +snapshot_limit = int(sys.argv[3]) + +connection = sqlite3.connect(database) +assert connection.execute("PRAGMA quick_check").fetchone()[0] == "ok" +assert not connection.execute("PRAGMA foreign_key_check").fetchall() + +runs = connection.execute( + """ + SELECT id, run_kind, generation, state, resume_count + FROM profiler_runs + ORDER BY generation + """ +).fetchall() + +assert len(runs) == expected_cycles, ( + f"expected {expected_cycles} runs, found {len(runs)}" +) +assert all(row[3] == "succeeded" for row in runs), "not all runs succeeded" +assert runs[0][1] == "initial", f"first run was {runs[0][1]}" +assert all(row[1] == "incremental" for row in runs[1:]), ( + "non-initial soak runs must be incremental" +) +assert [row[2] for row in runs] == list(range(1, expected_cycles + 1)), ( + "run generations are not contiguous" +) + +snapshots = connection.execute( + "SELECT COUNT(*) FROM source_snapshots" +).fetchone()[0] +assert snapshots <= snapshot_limit, ( + f"snapshot retention exceeded: {snapshots} > {snapshot_limit}" +) + +live_runs = connection.execute( + """ + SELECT COUNT(*) + FROM profiler_runs + WHERE state NOT IN ('succeeded', 'failed', 'cancelled') + """ +).fetchone()[0] +assert live_runs == 0, f"live runs remain: {live_runs}" + +orphan_file_stats = connection.execute( + """ + SELECT COUNT(*) + FROM file_stat_observations observation + LEFT JOIN profiler_runs run ON run.id = observation.run_id + WHERE run.id IS NULL + """ +).fetchone()[0] +assert orphan_file_stats == 0, ( + f"orphan file-stat observations remain: {orphan_file_stats}" +) + +successful_file_stat_reprocessed = connection.execute( + """ + SELECT COUNT(*) + FROM file_stat_observations observation + JOIN profiler_runs run ON run.id = observation.run_id + WHERE run.generation > 1 + AND observation.availability_state = 'available' + AND observation.size_state = 'match' + """ +).fetchone()[0] +assert successful_file_stat_reprocessed == 0, ( + "unchanged successful file-stat objects were reprocessed: " + f"{successful_file_stat_reprocessed}" +) + +format_rows = connection.execute( + """ + SELECT + run.generation, + format_run.state, + format_run.identified, + format_run.unknown, + format_run.ambiguous, + format_run.empty_objects, + format_run.tool_errors + FROM format_identification_runs format_run + JOIN profiler_runs run ON run.id = format_run.baseline_run_id + WHERE run.generation > 1 + ORDER BY run.generation + """ +).fetchall() + +assert len(format_rows) == max(expected_cycles - 1, 0), ( + f"expected {expected_cycles - 1} incremental format runs, " + f"found {len(format_rows)}" +) +assert all(row[1] == "succeeded" for row in format_rows), ( + "an incremental format run did not succeed" +) + +reidentified = sum(sum(row[2:6]) for row in format_rows) +tool_errors = sum(row[6] for row in format_rows) + +assert reidentified == 0, ( + f"unchanged successful format objects were reidentified: {reidentified}" +) +assert tool_errors == 0, ( + f"incremental format tool errors were recorded: {tool_errors}" +) + +result = { + "runs": len(runs), + "firstGeneration": runs[0][2], + "lastGeneration": runs[-1][2], + "snapshots": snapshots, + "snapshotLimit": snapshot_limit, + "liveRuns": live_runs, + "orphanFileStatObservations": orphan_file_stats, + "successfulFileStatReprocessed": successful_file_stat_reprocessed, + "incrementalFormatRuns": len(format_rows), + "successfulFormatsReidentified": reidentified, + "formatToolErrors": tool_errors, + "databaseBytes": os.path.getsize(database), +} + +print(json.dumps(result, indent=2)) +'@ + +[System.IO.File]::WriteAllText( + $Audit, + $Python.TrimEnd("`r", "`n") + "`n", + [System.Text.UTF8Encoding]::new($false) +) + +$AuditOutput = @( + & python $Audit $ProfilerDb $Cycles $RetainSuccessfulSnapshots +) + +if ($LASTEXITCODE -ne 0) { + throw 'Soak integrity assertions failed.' +} + +$AuditText = $AuditOutput -join "`r`n" +$AuditText | Write-Host +$AuditResult = $AuditText | ConvertFrom-Json + +$CandidateHead = '' +Push-Location $Repo +try { + $CandidateHead = (git rev-parse HEAD).Trim() +} finally { + Pop-Location +} + +$Summary = [ordered]@{ + Status = 'SOAK_GREEN' + CandidateHead = $CandidateHead + Cycles = $Cycles + Workspace = $WorkspaceRoot + SourceStateBefore = $SourceStateBefore + SourceStateAfter = $SourceStateAfter + Audit = $AuditResult + RecordedAtUtc = [DateTime]::UtcNow.ToString('o') +} + +$SummaryPath = Join-Path $EvidenceRoot "alpha6-soak-$RunNonce-summary.json" +[System.IO.File]::WriteAllText( + $SummaryPath, + ($Summary | ConvertTo-Json -Depth 12) + "`n", + [System.Text.UTF8Encoding]::new($false) +) + +$Summary | ConvertTo-Json -Depth 12 +Get-FileHash -LiteralPath $SummaryPath -Algorithm SHA256