diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..8431941 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Runtime-data checksums must see identical bytes on every build host. +* text=auto eol=lf diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index eed1d9f..0b8bbf0 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -17,7 +17,7 @@ Closes # - [ ] `cargo test --workspace` - [ ] `cargo test -p firmware-analysis-toolkit --test test_kernel_profile_consistency` - [ ] `python3 scripts/build-runtime-data-bundle.py --check` -- [ ] `uv run mkdocs build --strict` (only if `docs/` changed) +- [ ] Documentation links and rendered Markdown checked (if documentation changed) ## Notes diff --git a/.github/release-build-setup.yml b/.github/release-build-setup.yml new file mode 100644 index 0000000..2b0bf1c --- /dev/null +++ b/.github/release-build-setup.yml @@ -0,0 +1,14 @@ +- name: Use bundled compression libraries + shell: bash + run: | + echo 'LZMA_API_STATIC=1' >> "$GITHUB_ENV" + echo 'BZIP2_NO_PKG_CONFIG=1' >> "$GITHUB_ENV" +- name: Install license-report tool + uses: taiki-e/install-action@v2 + with: + tool: cargo-about@0.8.2 +- name: Stage runtime data and dependency licenses + shell: bash + run: | + python3 scripts/prepare-release-resources.py + bash scripts/generate-third-party-licenses.sh target/release-resources/third-party-licenses.json diff --git a/.github/workflows/release-smoke.yml b/.github/workflows/release-smoke.yml new file mode 100644 index 0000000..ca00446 --- /dev/null +++ b/.github/workflows/release-smoke.yml @@ -0,0 +1,57 @@ +name: Test release archives +on: + workflow_call: + inputs: + plan: + required: true + type: string +permissions: + contents: read +jobs: + smoke: + strategy: + fail-fast: false + matrix: + include: + - runner: macos-14 + target: aarch64-apple-darwin + extension: tar.gz + - runner: macos-15-intel + target: x86_64-apple-darwin + extension: tar.gz + - runner: ubuntu-22.04 + target: x86_64-unknown-linux-gnu + extension: tar.gz + - runner: ubuntu-24.04-arm + target: aarch64-unknown-linux-gnu + extension: tar.gz + - runner: windows-2022 + target: x86_64-pc-windows-msvc + extension: zip + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - uses: actions/download-artifact@v8 + with: + pattern: artifacts-* + merge-multiple: true + path: artifacts + - name: Run packaged executable and verify bundled data + shell: bash + env: + VERSION: ${{ fromJson(inputs.plan).releases[0].app_version }} + ARCHIVE: artifacts/firmware-analysis-toolkit-${{ matrix.target }}.${{ matrix.extension }} + run: python3 scripts/smoke-release.py "$ARCHIVE" --version "$VERSION" + - name: Verify Linux archive on Fedora + if: runner.os == 'Linux' + shell: bash + env: + VERSION: ${{ fromJson(inputs.plan).releases[0].app_version }} + ARCHIVE: artifacts/firmware-analysis-toolkit-${{ matrix.target }}.${{ matrix.extension }} + run: | + docker run --rm -v "$PWD:/source:ro" -w /source \ + -e VERSION -e ARCHIVE fedora:42 \ + sh -c 'dnf -y install python3 && python3 scripts/smoke-release.py "$ARCHIVE" --version "$VERSION"' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..b43381c --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,326 @@ +# This file was autogenerated by dist: https://axodotdev.github.io/cargo-dist +# +# Copyright 2022-2024, axodotdev +# SPDX-License-Identifier: MIT or Apache-2.0 +# +# CI that: +# +# * checks for a Git Tag that looks like a release +# * builds artifacts with dist (archives, installers, hashes) +# * uploads those artifacts to temporary workflow zip +# * on success, uploads the artifacts to a GitHub Release +# +# Note that a GitHub Release with this tag is assumed to exist as a draft +# with the appropriate title/body, and will be undrafted for you. + +name: Release +permissions: + "contents": "write" + +# This task will run whenever you workflow_dispatch with a tag that looks like a version +# like "1.0.0", "v0.1.0-prerelease.1", "my-app/0.1.0", "releases/v1.0.0", etc. +# Various formats will be parsed into a VERSION and an optional PACKAGE_NAME, where +# PACKAGE_NAME must be the name of a Cargo package in your workspace, and VERSION +# must be a Cargo-style SemVer Version (must have at least major.minor.patch). +# +# If PACKAGE_NAME is specified, then the announcement will be for that +# package (erroring out if it doesn't have the given version or isn't dist-able). +# +# If PACKAGE_NAME isn't specified, then the announcement will be for all +# (dist-able) packages in the workspace with that version (this mode is +# intended for workspaces with only one dist-able package, or with all dist-able +# packages versioned/released in lockstep). +# +# If you push multiple tags at once, separate instances of this workflow will +# spin up, creating an independent announcement for each one. However, GitHub +# will hard limit this to 3 tags per commit, as it will assume more tags is a +# mistake. +# +# If there's a prerelease-style suffix to the version, then the release(s) +# will be marked as a prerelease. +on: + pull_request: + workflow_dispatch: + inputs: + tag: + description: Release Tag + required: true + default: dry-run + type: string + +jobs: + # Run 'dist plan' (or host) to determine what tasks we need to do + plan: + runs-on: "ubuntu-22.04" + outputs: + val: ${{ steps.plan.outputs.manifest }} + tag: ${{ (inputs.tag != 'dry-run' && inputs.tag) || '' }} + tag-flag: ${{ inputs.tag && inputs.tag != 'dry-run' && format('--tag={0}', inputs.tag) || '' }} + publishing: ${{ inputs.tag && inputs.tag != 'dry-run' }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + submodules: recursive + - name: Install dist + # we specify bash to get pipefail; it guards against the `curl` command + # failing. otherwise `sh` won't catch that `curl` returned non-0 + shell: bash + run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.33.0/cargo-dist-installer.sh | sh" + - name: Cache dist + uses: actions/upload-artifact@v7 + with: + name: cargo-dist-cache + path: ~/.cargo/bin/dist + # sure would be cool if github gave us proper conditionals... + # so here's a doubly-nested ternary-via-truthiness to try to provide the best possible + # functionality based on whether this is a pull_request, and whether it's from a fork. + # (PRs run on the *source* but secrets are usually on the *target* -- that's *good* + # but also really annoying to build CI around when it needs secrets to work right.) + - id: plan + run: | + dist ${{ (inputs.tag && inputs.tag != 'dry-run' && format('host --steps=create --tag={0}', inputs.tag)) || 'plan' }} --output-format=json > plan-dist-manifest.json + echo "dist ran successfully" + cat plan-dist-manifest.json + echo "manifest=$(jq -c "." plan-dist-manifest.json)" >> "$GITHUB_OUTPUT" + - name: "Upload dist-manifest.json" + uses: actions/upload-artifact@v7 + with: + name: artifacts-plan-dist-manifest + path: plan-dist-manifest.json + + # Build and packages all the platform-specific things + build-local-artifacts: + name: build-local-artifacts (${{ join(matrix.targets, ', ') }}) + # Let the initial task tell us to not run (currently very blunt) + needs: + - plan + if: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix.include != null && (needs.plan.outputs.publishing == 'true' || fromJson(needs.plan.outputs.val).ci.github.pr_run_mode == 'upload') || inputs.tag == 'dry-run' }} + strategy: + fail-fast: false + # Target platforms/runners are computed by dist in create-release. + # Each member of the matrix has the following arguments: + # + # - runner: the github runner + # - dist-args: cli flags to pass to dist + # - install-dist: expression to run to install dist on the runner + # + # Typically there will be: + # - 1 "global" task that builds universal installers + # - N "local" tasks that build each platform's binaries and platform-specific installers + matrix: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix }} + runs-on: ${{ matrix.runner }} + container: ${{ matrix.container && matrix.container.image || null }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BUILD_MANIFEST_NAME: target/distrib/${{ join(matrix.targets, '-') }}-dist-manifest.json + steps: + - name: enable windows longpaths + run: | + git config --global core.longpaths true + - uses: actions/checkout@v6 + with: + persist-credentials: false + submodules: recursive + - name: Install Rust non-interactively if not already installed + if: ${{ matrix.container }} + run: | + if ! command -v cargo > /dev/null 2>&1; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + echo "$HOME/.cargo/bin" >> $GITHUB_PATH + fi + - name: "Use bundled compression libraries" + run: | + echo 'LZMA_API_STATIC=1' >> "$GITHUB_ENV" + echo 'BZIP2_NO_PKG_CONFIG=1' >> "$GITHUB_ENV" + shell: "bash" + - name: "Install license-report tool" + uses: "taiki-e/install-action@v2" + with: + "tool": "cargo-about@0.8.2" + - name: "Stage runtime data and dependency licenses" + run: | + python3 scripts/prepare-release-resources.py + bash scripts/generate-third-party-licenses.sh target/release-resources/third-party-licenses.json + shell: "bash" + - uses: swatinem/rust-cache@v2 + with: + key: ${{ join(matrix.targets, '-') }} + cache-provider: ${{ matrix.cache_provider }} + - name: Install dist + run: ${{ matrix.install_dist.run }} + # Get the dist-manifest + - name: Fetch local artifacts + uses: actions/download-artifact@v8 + with: + pattern: artifacts-* + path: target/distrib/ + merge-multiple: true + - name: Install dependencies + run: | + ${{ matrix.packages_install }} + - name: Build artifacts + run: | + # Actually do builds and make zips and whatnot + dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json + echo "dist ran successfully" + - id: cargo-dist + name: Post-build + # We force bash here just because github makes it really hard to get values up + # to "real" actions without writing to env-vars, and writing to env-vars has + # inconsistent syntax between shell and powershell. + shell: bash + run: | + # Parse out what we just built and upload it to scratch storage + echo "paths<> "$GITHUB_OUTPUT" + dist print-upload-files-from-manifest --manifest dist-manifest.json >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + + cp dist-manifest.json "$BUILD_MANIFEST_NAME" + - name: "Upload artifacts" + uses: actions/upload-artifact@v7 + with: + name: artifacts-build-local-${{ join(matrix.targets, '_') }} + path: | + ${{ steps.cargo-dist.outputs.paths }} + ${{ env.BUILD_MANIFEST_NAME }} + + # Build and package all the platform-agnostic(ish) things + build-global-artifacts: + needs: + - plan + - build-local-artifacts + runs-on: "ubuntu-22.04" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BUILD_MANIFEST_NAME: target/distrib/global-dist-manifest.json + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + submodules: recursive + - name: Install cached dist + uses: actions/download-artifact@v8 + with: + name: cargo-dist-cache + path: ~/.cargo/bin/ + - run: chmod +x ~/.cargo/bin/dist + # Get all the local artifacts for the global tasks to use (for e.g. checksums) + - name: Fetch local artifacts + uses: actions/download-artifact@v8 + with: + pattern: artifacts-* + path: target/distrib/ + merge-multiple: true + - id: cargo-dist + shell: bash + run: | + dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json "--artifacts=global" > dist-manifest.json + echo "dist ran successfully" + + # Parse out what we just built and upload it to scratch storage + echo "paths<> "$GITHUB_OUTPUT" + jq --raw-output ".upload_files[]" dist-manifest.json >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + + cp dist-manifest.json "$BUILD_MANIFEST_NAME" + - name: "Upload artifacts" + uses: actions/upload-artifact@v7 + with: + name: artifacts-build-global + path: | + ${{ steps.cargo-dist.outputs.paths }} + ${{ env.BUILD_MANIFEST_NAME }} + + custom-release-smoke: + needs: + - plan + - build-local-artifacts + uses: ./.github/workflows/release-smoke.yml + with: + plan: ${{ needs.plan.outputs.val }} + secrets: inherit + # Determines if we should publish/announce + host: + needs: + - plan + - build-local-artifacts + - build-global-artifacts + - custom-release-smoke + # Only run if we're "publishing", and only if plan, local and global didn't fail (skipped is fine) + if: ${{ always() && needs.plan.result == 'success' && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.custom-release-smoke.result == 'skipped' || needs.custom-release-smoke.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + runs-on: "ubuntu-22.04" + outputs: + val: ${{ steps.host.outputs.manifest }} + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + submodules: recursive + - name: Install cached dist + uses: actions/download-artifact@v8 + with: + name: cargo-dist-cache + path: ~/.cargo/bin/ + - run: chmod +x ~/.cargo/bin/dist + # Fetch artifacts from scratch-storage + - name: Fetch artifacts + uses: actions/download-artifact@v8 + with: + pattern: artifacts-* + path: target/distrib/ + merge-multiple: true + - id: host + shell: bash + run: | + dist host ${{ needs.plan.outputs.tag-flag }} --steps=upload --steps=release --output-format=json > dist-manifest.json + echo "artifacts uploaded and released successfully" + cat dist-manifest.json + echo "manifest=$(jq -c "." dist-manifest.json)" >> "$GITHUB_OUTPUT" + - name: "Upload dist-manifest.json" + uses: actions/upload-artifact@v7 + with: + # Overwrite the previous copy + name: artifacts-dist-manifest + path: dist-manifest.json + # Create a GitHub Release while uploading all files to it + - name: "Download GitHub Artifacts" + uses: actions/download-artifact@v8 + with: + pattern: artifacts-* + path: artifacts + merge-multiple: true + - name: Cleanup + run: | + # Remove the granular manifests + rm -f artifacts/*-dist-manifest.json + - name: Create GitHub Release + env: + PRERELEASE_FLAG: "${{ fromJson(steps.host.outputs.manifest).announcement_is_prerelease && '--prerelease' || '' }}" + RELEASE_COMMIT: "${{ github.sha }}" + run: | + # If we're editing a release in place, we need to upload things ahead of time + gh release upload "${{ needs.plan.outputs.tag }}" artifacts/* + + gh release edit "${{ needs.plan.outputs.tag }}" --target "$RELEASE_COMMIT" $PRERELEASE_FLAG --draft=false + + announce: + needs: + - plan + - host + # use "always() && ..." to allow us to wait for all publish jobs while + # still allowing individual publish jobs to skip themselves (for prereleases). + # "host" however must run to completion, no skipping allowed! + if: ${{ always() && needs.host.result == 'success' }} + runs-on: "ubuntu-22.04" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + submodules: recursive diff --git a/AGENTS.md b/AGENTS.md index e8a696e..f2a4bd4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,6 +9,14 @@ Keep dependency upgrades separate from edition changes. `rustfmt.toml` retains the existing formatting style so edition changes do not reformat unrelated code. Use `tests/support/subprocess.rs` for tests that need a custom environment or working directory; configure the child command instead of mutating the test process. +## Releases + +Follow [RELEASING.md](RELEASING.md) for versioning, validation, packaging, and publication. +Use one `vMAJOR.MINOR.PATCH` product tag and one shared workspace version. +Keep runtime-data and citation versions synchronized through `cargo-release` replacements. +Release notes describe user-facing changes; the first regular release introduces the product and its migration path. +Use `cargo-dist` for native GitHub platform builds and archive smoke tests; publishing is an explicit maintainer workflow dispatch. + ## MCU analysis and terminal reports **Raw MCU identification** (`fat_analyze::mcu`): Structural vector candidates, family evidence, and code mapping are independent. diff --git a/CHANGELOG.md b/CHANGELOG.md index 76ccf5e..c0734af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,17 @@ ## [Unreleased] +## [2.0.0] + +First regular release of the Rust-based FAT CLI. +See the [release notes](release-notes/v2.0.0.md) for capabilities, installation, and migration from FAT 1.x. + ### Added +- A shared Rust CLI for firmware identification, extraction, binary and filesystem analysis, and emulation workflows. +- Versioned runtime-data bundles with installation and integrity verification. +- Portable archives with bundled runtime data for macOS Apple Silicon and Intel, Linux x86-64 and ARM64, and Windows x86-64. +- A maintainer release workflow using `cargo-release`, `git-cliff`, `cargo-dist`, and GitHub CLI, with synchronized versions and release links in the README. - `identify` and `inspect mcu` show compact overviews by default. `--details` expands related findings into grouped reference tables; JSON stays unchanged. - `inspect mcu` reports bounded Thumb code and function candidates, referenced strings, recovered initialization descriptors, and decoded MMIO accesses. diff --git a/CITATION.cff b/CITATION.cff index 2428a3d..7aee3c6 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -1,5 +1,6 @@ cff-version: 1.2.0 type: software +version: "2.0.0" title: "FAT — Firmware Analysis Toolkit" message: >- If you use FAT in your research or publications, please cite diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2df8e23..5d395ac 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,12 +41,8 @@ cargo test -p firmware-analysis-toolkit --test test_kernel_profile_consistency python3 scripts/build-runtime-data-bundle.py --check ``` -If you touched anything under `docs/`, build the documentation too. It's strict, so a broken link fails the build: - -```bash -uv sync --extra docs -uv run mkdocs build --strict -``` +Documentation is published as Markdown in this repository. +When changing it, check relative links and preview tables and code blocks in GitHub's rendered view. The clippy policy lives in the root `Cargo.toml` under `[workspace.lints]`. A handful of design-level lints are allowed there; everything else has to pass. @@ -67,6 +63,9 @@ Two asks. Don't upload vendor firmware images, extracted filesystems, or anythin Keep them focused and reference the issue they close (`Closes #NN`). Match the style of the code around you, and add or update tests when behavior changes. If it changes user-facing behavior, add a line under `Unreleased` in `CHANGELOG.md`. +Use Conventional Commit PR titles: `feat:` for compatible additions, `fix:` for fixes, and `!` for breaking changes, such as `feat(cli)!: rename an option`. +These feed release-note drafts; maintainers review the version and user-facing wording before publication. +See [RELEASING.md](RELEASING.md) for the release workflow. If it changes the workspace layout, a key abstraction, or the CLI surface, update `AGENTS.md` — it's the shared instruction file for coding agents, and `CLAUDE.md` is just a one-line import of it. If you're planning something large, open an issue first so we can talk it through before you spend the time on it. diff --git a/Cargo.lock b/Cargo.lock index 69835a7..eb1ee66 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -427,7 +427,7 @@ checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "fat-toolkit-analyze" -version = "2.0.0-alpha.1" +version = "2.0.0" dependencies = [ "aho-corasick", "anyhow", @@ -450,7 +450,7 @@ dependencies = [ [[package]] name = "fat-toolkit-backend" -version = "2.0.0-alpha.1" +version = "2.0.0" dependencies = [ "fat-toolkit-core", "fat-toolkit-family", @@ -462,7 +462,7 @@ dependencies = [ [[package]] name = "fat-toolkit-bootloader" -version = "2.0.0-alpha.1" +version = "2.0.0" dependencies = [ "fat-toolkit-core", "fat-toolkit-extract", @@ -474,7 +474,7 @@ dependencies = [ [[package]] name = "fat-toolkit-core" -version = "2.0.0-alpha.1" +version = "2.0.0" dependencies = [ "rusqlite", "semver", @@ -488,7 +488,7 @@ dependencies = [ [[package]] name = "fat-toolkit-emulate" -version = "2.0.0-alpha.1" +version = "2.0.0" dependencies = [ "fat-toolkit-backend", "fat-toolkit-core", @@ -501,7 +501,7 @@ dependencies = [ [[package]] name = "fat-toolkit-extract" -version = "2.0.0-alpha.1" +version = "2.0.0" dependencies = [ "crc32fast", "flate2", @@ -515,11 +515,11 @@ dependencies = [ [[package]] name = "fat-toolkit-family" -version = "2.0.0-alpha.1" +version = "2.0.0" [[package]] name = "fat-toolkit-package" -version = "2.0.0-alpha.1" +version = "2.0.0" dependencies = [ "md-5", "num-bigint", @@ -528,7 +528,7 @@ dependencies = [ [[package]] name = "fat-toolkit-plugin-api" -version = "2.0.0-alpha.1" +version = "2.0.0" dependencies = [ "fat-toolkit-core", "serde", @@ -536,7 +536,7 @@ dependencies = [ [[package]] name = "fat-toolkit-plugin-host" -version = "2.0.0-alpha.1" +version = "2.0.0" dependencies = [ "fat-toolkit-analyze", "fat-toolkit-core", @@ -546,7 +546,7 @@ dependencies = [ [[package]] name = "fat-toolkit-query" -version = "2.0.0-alpha.1" +version = "2.0.0" dependencies = [ "clang", "fat-toolkit-core", @@ -564,7 +564,7 @@ dependencies = [ [[package]] name = "fat-toolkit-taint" -version = "2.0.0-alpha.1" +version = "2.0.0" dependencies = [ "dirs", "fat-toolkit-core", @@ -603,7 +603,7 @@ checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "firmware-analysis-toolkit" -version = "2.0.0-alpha.1" +version = "2.0.0" dependencies = [ "clap", "crc32fast", diff --git a/Cargo.toml b/Cargo.toml index 03cd15e..9446e6b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ members = [ resolver = "3" [workspace.package] -version = "2.0.0-alpha.1" +version = "2.0.0" edition = "2024" rust-version = "1.90" license = "FSL-1.1-ALv2" @@ -36,3 +36,7 @@ result_large_err = "allow" type_complexity = "allow" # Nested guards remain valid style; Rust 2024 need not rewrite them as let chains. collapsible_if = "allow" + +[profile.dist] +inherits = "release" +lto = "thin" diff --git a/INSTALL.md b/INSTALL.md index 107c4e2..0435281 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -2,6 +2,49 @@ The minimal quick start (clone, build, install) lives in the [README](README.md). This page is the full installation reference: installer options, external tools, and optional components. +## Release downloads + +Choose a bundle for your operating system and CPU from [FAT Releases](https://github.com/attify/firmware-analysis-toolkit/releases/latest). +Binary bundles contain `fat` (`fat.exe` on Windows), the matching `share/fat` runtime data, and dependency license notices. +Choose the target matching your machine: + +| System | CPU | Target | +| --- | --- | --- | +| macOS | Apple Silicon | `aarch64-apple-darwin` | +| macOS | Intel | `x86_64-apple-darwin` | +| Linux | x86-64 | `x86_64-unknown-linux-gnu` | +| Linux | ARM64 | `aarch64-unknown-linux-gnu` | +| Windows | x86-64 | `x86_64-pc-windows-msvc` | + +Linux x86-64 requires glibc 2.35 or newer; Linux ARM64 requires glibc 2.39 or newer. +The archives are tested on Ubuntu 22.04 (x86-64), Ubuntu 24.04 (ARM64), and Fedora 42 (both CPUs). + +Verify the archive against its adjacent `.sha256` file, extract it, and keep the executable and `share` directory together: + +```bash +# Substitute the target from the release's asset list. +shasum -a 256 -c firmware-analysis-toolkit-TARGET.tar.gz.sha256 +tar -xzf firmware-analysis-toolkit-TARGET.tar.gz +cd firmware-analysis-toolkit-TARGET +./fat --version +./fat data verify +./fat doctor +export PATH="$PWD:$PATH" +``` + +On Windows, use `Get-FileHash` to compare the ZIP's SHA-256 with its `.sha256` file, extract with `Expand-Archive`, and run `.\fat.exe data verify` from the extracted directory. +The Windows binary supports native analysis commands; workflows using Linux filesystem semantics or external Unix tools should run under WSL with the Linux build. + +For a Cargo-installed executable, download the same version's `fat-data-VERSION.zip` and run `fat data install --archive ./fat-data-VERSION.zip`. +The source installer below installs both the executable and its runtime data. + +## Upgrading from FAT 1.x + +FAT 2 is a Rust CLI with a new installation and project workflow. +Install it in a separate directory, run `fat doctor`, and create a project with `fat new ./firmware.bin`. +Use the `fat` commands in the [quick start](README.md#quick-start); the FAT 1.x `setup.sh`, `fat.py`, and `fat.config` instructions apply to the older release. +Keep existing FAT 1.x workspaces until you have recreated the projects you need in FAT 2. + ## Installer options The source installer requires Rust 1.90+, Python 3, and a C compiler. The default extraction profile also needs Binwalk's native build dependencies; use `--install-system-deps` to install the supported host packages. diff --git a/README.md b/README.md index 0e10857..f37bb53 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ # FAT — Firmware Analysis Toolkit +[![FAT release](https://img.shields.io/github/v/release/attify/firmware-analysis-toolkit?filter=v*&sort=semver&label=FAT)](https://github.com/attify/firmware-analysis-toolkit/releases/latest) +[Download the latest release](https://github.com/attify/firmware-analysis-toolkit/releases/latest) · [Changelog](CHANGELOG.md) · [Installation](INSTALL.md) + **Firmware Analysis Toolkit** is a security research toolkit for investigating firmware security for IoT, Physical AI, Mobile Devices & Robotics. It is primarily built for the ["Offensive IoT Exploitation"](https://www.attify.com/training) training conducted by [Attify](https://attify.com). It has been built based on actual hands-on field experience of what really matters when it comes to real firmware security investigations that matter. @@ -21,13 +24,15 @@ Some of the features include classifying unknown firmware, extracts filesystems, ## Quick start -Clone the repository, build with Rust 1.90 or newer, and install: +Download a binary bundle from [Releases](https://github.com/attify/firmware-analysis-toolkit/releases/latest), or build the latest regular release with Rust 1.90 or newer: ```bash -git clone https://github.com/attify/firmware-analysis-toolkit.git +git clone --branch v2.0.0 --depth 1 https://github.com/attify/firmware-analysis-toolkit.git cd firmware-analysis-toolkit cargo build --release --locked cargo install --path crates/fat_cli --locked +python3 scripts/build-runtime-data-bundle.py --output ./dist/fat-data.zip +fat data install --archive ./dist/fat-data.zip fat doctor ``` @@ -149,6 +154,7 @@ fat verify --help | [docs/architecture.md](docs/architecture.md) | Workspace crates and extension points | | [DEVELOPMENT.md](DEVELOPMENT.md) | Build and test workflow | | [CONTRIBUTING.md](CONTRIBUTING.md) | Setup, PR checklist, and contribution guide | +| [RELEASING.md](RELEASING.md) | Version naming and the maintainer release workflow | ## Citation diff --git a/RELEASING.md b/RELEASING.md new file mode 100644 index 0000000..bade926 --- /dev/null +++ b/RELEASING.md @@ -0,0 +1,169 @@ +# Releasing FAT + +FAT releases are created on request from tested commits on `master`. +Use `cargo-release` for shared version updates, `git-cliff` for release-note drafts, `cargo-dist` for platform archives and release automation, and GitHub CLI for maintainer actions. +The GitHub release is the download page; the README badge and latest-release link follow it automatically. + +## Version and timing + +| Kind | Example | Use | +| --- | --- | --- | +| Major | `2.4.3` → `3.0.0` | Breaking changes to documented commands, JSON, configuration, or plugin interfaces | +| Minor | `2.4.3` → `2.5.0` | Compatible features | +| Patch | `2.4.3` → `2.4.4` | Compatible fixes | +| Preview | `2.5.0-alpha.1`, `2.5.0-rc.1` | Testing before a regular release | + +Use one product tag, `vMAJOR.MINOR.PATCH`, and the release title `FAT MAJOR.MINOR.PATCH`. +All workspace crates and the accompanying runtime data use that version. +Runtime-data bundles declare compatibility with the exact executable version they accompany. +Review accumulated changes roughly every two weeks; release when there is a useful tested update, and ship urgent fixes as needed. +Regular releases are marked Latest; previews are prereleases. + +"Prepare the next FAT release" means a reviewed version update and draft release. +"Publish the next FAT minor release from master" includes publishing after the checks below pass. + +## Tools + +Install the tested maintainer tools, or use their official prebuilt binaries: + +```bash +cargo install cargo-release --version 1.1.6 --locked +cargo install git-cliff --version 2.14.2 --locked +cargo install cargo-about --version 0.8.2 --locked +cargo install cargo-dist --version 0.33.0 --locked +gh auth status +``` + +GitHub CLI (`gh`) is installed separately through its [installation guide](https://cli.github.com/manual/installation). +The workspace remains `publish = false`; this workflow distributes FAT through GitHub. + +## Prepare a version + +Start in a clean, dedicated worktree based on current `origin/master`. +For the first regular release use `release_version=2.0.0`; subsequently choose the next SemVer version. + +```bash +release_version=2.0.0 +release_tag="v$release_version" +cargo release version "$release_version" --workspace +cargo release version "$release_version" --workspace --execute --no-confirm +cargo release replace --workspace --execute --no-confirm +python3 scripts/build-runtime-data-bundle.py --check +``` + +The first command previews the changes. +The execute steps update Cargo manifests, dependency requirements, the lockfile, runtime-data metadata, citation metadata, runtime-data asset names in `dist-workspace.toml`, and the README's pinned source-install command. +README source-install instructions stay on the regular release when preparing a preview. +Review the diff before committing it. + +## Release notes + +The first regular release uses the curated introduction in [release-notes/v2.0.0.md](release-notes/v2.0.0.md). +It describes the product, highlights, installation, and migration from FAT 1.x. +It is the baseline for subsequent release notes. + +```bash +release_notes=release-notes/v2.0.0.md +``` + +For later releases, generate a draft from the previous published FAT tag: + +```bash +previous_tag=v2.0.0 +mkdir -p release-notes +release_notes="release-notes/$release_tag.md" +git cliff "$previous_tag..HEAD" --tag "$release_tag" --output "$release_notes" +``` + +`cliff.toml` groups features, fixes, performance improvements, and other changes. +It recognizes FAT SemVer tags and excludes historical QEMU tags. +Nonconventional commits remain visible for review; routine documentation, test, CI, style, and release-maintenance commits are omitted unless breaking. +Review the generated notes, preserve useful entries already under `Unreleased`, and add the finalized release section to `CHANGELOG.md` while retaining its history. +Keep the notes focused on user-visible changes, including migration instructions for breaking changes. +Include an absolute link to the tagged installation guide so the notes can also serve as the binary bundle's README. +Commit the version, notes, and changelog through a release PR. + +## Validate the release PR + +Run the checks from [CONTRIBUTING.md](CONTRIBUTING.md): + +```bash +cargo fmt --all --check +cargo clippy --workspace --all-targets --all-features --locked -- -D warnings +cargo test --workspace --locked -- --test-threads=1 +python3 -m unittest discover -s tests/release -p 'test_*.py' +python3 scripts/build-runtime-data-bundle.py --check +dist generate --check +dist plan +``` + +The Release workflow builds portable archives on native GitHub runners for macOS Apple Silicon and Intel, Linux x86-64 and ARM64, and Windows x86-64. +Linux x86-64 uses Ubuntu 22.04 (glibc 2.35); ARM64 uses Ubuntu 24.04 (glibc 2.39). +Both Linux archives are also tested on Fedora 42. +Each archive contains the executable, matching `share/fat` runtime data, release notes, and dependency license notices. +Compression libraries are built from their bundled sources to avoid dependencies on runner-specific Homebrew or Linux packages. + +Pull requests build and test the archives without publishing them. +The smoke checks extract each archive into a fresh directory, verify its checksum, run the executable, verify the bundled data, and exercise identification and MCU inspection with a synthetic input. +They also inspect native shared-library dependencies on macOS and Linux. +The workflow stops before publication if any build or smoke check fails. + +`dist-workspace.toml` is the source of truth for the generated `.github/workflows/release.yml`. +Change the configuration or reusable setup/smoke workflow, then run `dist generate`; do not hand-edit the generated workflow. +Release builds use the compiler pinned in `rust-toolchain.toml`; the minimum supported source-build compiler remains Rust 1.90. + +For a local archive build: + +```bash +python3 scripts/prepare-release-resources.py +bash scripts/generate-third-party-licenses.sh target/release-resources/third-party-licenses.json +LZMA_API_STATIC=1 BZIP2_NO_PKG_CONFIG=1 dist build --artifacts=local --target TARGET +python3 scripts/smoke-release.py target/distrib/firmware-analysis-toolkit-TARGET.tar.gz --version "$release_version" +``` + +Replace `TARGET` with the host's Rust target triple and use `.zip` on Windows. +The archive contains `fat` or `fat.exe` alongside `share`, so keep them together after extraction. +Archives use the package name and target triple; the enclosing GitHub release tag identifies their version. +The separately installable runtime-data archive retains the name `fat-data-VERSION.zip`. + +## Merge, tag, and publish + +Merge the release PR after its validation and platform checks pass. +Use a clean checkout of the exact merged commit on `origin/master`, and verify the workspace version matches the intended tag. +Create the product tag on that commit: + +```bash +release_tag="v$release_version" +cargo release tag -p firmware-analysis-toolkit +cargo release tag -p firmware-analysis-toolkit --execute --no-confirm +git push origin "refs/tags/$release_tag" +``` + +Create a draft with the reviewed release notes: + +```bash +gh release create "$release_tag" --verify-tag --draft \ + --title "FAT $release_version" --notes-file "$release_notes" +``` + +If a draft already exists, update its notes and target commit instead of creating another. +Remove obsolete assets from that draft before the first cargo-dist run. +Never replace assets or move tags belonging to a published version. + +Publication is an explicit workflow dispatch; pushing a tag alone does not publish: + +```bash +gh workflow run release.yml --ref "$release_tag" -f tag="$release_tag" +``` + +The workflow rebuilds and tests the tagged code, uploads the archives and checksums to the draft, and publishes it only after every platform passes. +A dry run uses `-f tag=dry-run` and leaves releases untouched. +For a regular release, confirm it is marked Latest: + +```bash +gh release edit "$release_tag" --prerelease=false --latest +``` + +For a preview, use `--prerelease=true --latest=false` instead. +Verify the published tag, release notes, asset list, and checksums by downloading the release assets. +The GitHub sidebar and README badge follow the latest regular release. diff --git a/cliff.toml b/cliff.toml new file mode 100644 index 0000000..720bee8 --- /dev/null +++ b/cliff.toml @@ -0,0 +1,29 @@ +[changelog] +header = "# Changelog\n\n" +body = """ +{% if version %}## [{{ version | trim_start_matches(pat="v") }}]{% else %}## [Unreleased]{% endif %} +{% for group, commits in commits | group_by(attribute="group") %} +### {{ group }} +{% for commit in commits %} +- {% if commit.breaking %}**Breaking:** {% endif %}{{ commit.message | split(pat="\n") | first | trim }} +{% endfor %} +{% endfor %} +""" +trim = true + +[git] +conventional_commits = true +filter_unconventional = false +protect_breaking_commits = true +commit_parsers = [ + { message = "^Merge ", skip = true }, + { message = "^feat", group = "Added" }, + { message = "^fix", group = "Fixed" }, + { message = "^perf", group = "Performance" }, + { message = "^refactor", group = "Changed" }, + { message = "^(docs|test|ci|style|chore)(\\(|:)", skip = true }, + { message = ".*", group = "Changed" }, +] +# Exclude historical QEMU download tags and use FAT SemVer tags only. +tag_pattern = "^v[0-9]+\\.[0-9]+\\.[0-9]+(-[0-9A-Za-z.-]+)?$" +sort_commits = "oldest" diff --git a/crates/fat_analyze/Cargo.toml b/crates/fat_analyze/Cargo.toml index ca9273c..ddec2a9 100644 --- a/crates/fat_analyze/Cargo.toml +++ b/crates/fat_analyze/Cargo.toml @@ -17,9 +17,9 @@ name = "fat_analyze" path = "src/lib.rs" [dependencies] -fat-core = { version = "=2.0.0-alpha.1", path = "../fat_core", package = "fat-toolkit-core" } -fat-family = { version = "=2.0.0-alpha.1", path = "../fat_family", package = "fat-toolkit-family" } -fat-plugin-api = { version = "=2.0.0-alpha.1", path = "../fat_plugin_api", package = "fat-toolkit-plugin-api" } +fat-core = { version = "=2.0.0", path = "../fat_core", package = "fat-toolkit-core" } +fat-family = { version = "=2.0.0", path = "../fat_family", package = "fat-toolkit-family" } +fat-plugin-api = { version = "=2.0.0", path = "../fat_plugin_api", package = "fat-toolkit-plugin-api" } anyhow = "1" aho-corasick = "1" regex = "1" diff --git a/crates/fat_backend/Cargo.toml b/crates/fat_backend/Cargo.toml index d4e28dd..a5e1787 100644 --- a/crates/fat_backend/Cargo.toml +++ b/crates/fat_backend/Cargo.toml @@ -17,8 +17,8 @@ name = "fat_backend" path = "src/lib.rs" [dependencies] -fat-core = { version = "=2.0.0-alpha.1", path = "../fat_core", package = "fat-toolkit-core" } -fat-family = { version = "=2.0.0-alpha.1", path = "../fat_family", package = "fat-toolkit-family" } +fat-core = { version = "=2.0.0", path = "../fat_core", package = "fat-toolkit-core" } +fat-family = { version = "=2.0.0", path = "../fat_family", package = "fat-toolkit-family" } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/crates/fat_bootloader/Cargo.toml b/crates/fat_bootloader/Cargo.toml index 128b716..266da0f 100644 --- a/crates/fat_bootloader/Cargo.toml +++ b/crates/fat_bootloader/Cargo.toml @@ -17,8 +17,8 @@ name = "fat_bootloader" path = "src/lib.rs" [dependencies] -fat-core = { version = "=2.0.0-alpha.1", path = "../fat_core", package = "fat-toolkit-core" } -fat-extract = { version = "=2.0.0-alpha.1", path = "../fat_extract", package = "fat-toolkit-extract" } +fat-core = { version = "=2.0.0", path = "../fat_core", package = "fat-toolkit-core" } +fat-extract = { version = "=2.0.0", path = "../fat_extract", package = "fat-toolkit-extract" } serde = { version = "1", features = ["derive"] } serde_json = "1" walkdir = "2" diff --git a/crates/fat_cli/Cargo.toml b/crates/fat_cli/Cargo.toml index 96215e3..a3de3ce 100644 --- a/crates/fat_cli/Cargo.toml +++ b/crates/fat_cli/Cargo.toml @@ -12,20 +12,34 @@ categories.workspace = true description = "FAT — command-line firmware analysis toolkit" publish = false +[package.metadata.dist] +dist = true + +# Only the CLI owns the product tag and non-Cargo version fields. +[package.metadata.release] +tag = true +pre-release-replacements = [ + { file = "../../dist-workspace.toml", search = 'fat-data-[^"/]+\.zip', replace = 'fat-data-{{version}}.zip', exactly = 2, prerelease = true }, + { file = "../../share/fat/manifest.json", search = '"data_version": "[^"]+"', replace = '"data_version": "{{version}}"', exactly = 1, prerelease = true }, + { file = "../../share/fat/manifest.json", search = '"compatible_fat": "[^"]+"', replace = '"compatible_fat": "={{version}}"', exactly = 1, prerelease = true }, + { file = "../../CITATION.cff", search = '(?m)^version: "[^"]+"$', replace = 'version: "{{version}}"', exactly = 1, prerelease = true }, + { file = "../../README.md", search = 'git clone --branch v[^ ]+ --depth 1', replace = 'git clone --branch v{{version}} --depth 1', exactly = 1 }, +] + [dependencies] clap = { version = "4", features = ["derive", "color"] } -fat_analyze = { version = "=2.0.0-alpha.1", path = "../fat_analyze", package = "fat-toolkit-analyze" } -fat_backend = { version = "=2.0.0-alpha.1", path = "../fat_backend", package = "fat-toolkit-backend" } -fat_bootloader = { version = "=2.0.0-alpha.1", path = "../fat_bootloader", package = "fat-toolkit-bootloader" } -fat_family = { version = "=2.0.0-alpha.1", path = "../fat_family", package = "fat-toolkit-family" } -fat_extract = { version = "=2.0.0-alpha.1", path = "../fat_extract", package = "fat-toolkit-extract" } -fat_package = { version = "=2.0.0-alpha.1", path = "../fat_package", package = "fat-toolkit-package" } -fat_core = { version = "=2.0.0-alpha.1", path = "../fat_core", package = "fat-toolkit-core" } -fat_emulate = { version = "=2.0.0-alpha.1", path = "../fat_emulate", package = "fat-toolkit-emulate" } -fat_taint = { version = "=2.0.0-alpha.1", path = "../fat_taint", package = "fat-toolkit-taint" } -fat_plugin_api = { version = "=2.0.0-alpha.1", path = "../fat_plugin_api", package = "fat-toolkit-plugin-api" } -fat_plugin_host = { version = "=2.0.0-alpha.1", path = "../fat_plugin_host", package = "fat-toolkit-plugin-host" } -fat_query = { version = "=2.0.0-alpha.1", path = "../fat_query", package = "fat-toolkit-query" } +fat_analyze = { version = "=2.0.0", path = "../fat_analyze", package = "fat-toolkit-analyze" } +fat_backend = { version = "=2.0.0", path = "../fat_backend", package = "fat-toolkit-backend" } +fat_bootloader = { version = "=2.0.0", path = "../fat_bootloader", package = "fat-toolkit-bootloader" } +fat_family = { version = "=2.0.0", path = "../fat_family", package = "fat-toolkit-family" } +fat_extract = { version = "=2.0.0", path = "../fat_extract", package = "fat-toolkit-extract" } +fat_package = { version = "=2.0.0", path = "../fat_package", package = "fat-toolkit-package" } +fat_core = { version = "=2.0.0", path = "../fat_core", package = "fat-toolkit-core" } +fat_emulate = { version = "=2.0.0", path = "../fat_emulate", package = "fat-toolkit-emulate" } +fat_taint = { version = "=2.0.0", path = "../fat_taint", package = "fat-toolkit-taint" } +fat_plugin_api = { version = "=2.0.0", path = "../fat_plugin_api", package = "fat-toolkit-plugin-api" } +fat_plugin_host = { version = "=2.0.0", path = "../fat_plugin_host", package = "fat-toolkit-plugin-host" } +fat_query = { version = "=2.0.0", path = "../fat_query", package = "fat-toolkit-query" } r2pipe = "0.8.0" regex = "1" serde = { version = "1", features = ["derive"] } @@ -48,7 +62,7 @@ serde_yaml = "0.9" # Integration tests under tests/integration/ are registered as fat_cli test # targets, so the fixture loaders they use are enabled here rather than in # fat_query's default features. -fat_query = { version = "=2.0.0-alpha.1", path = "../fat_query", package = "fat-toolkit-query", features = ["test-fixtures"] } +fat_query = { version = "=2.0.0", path = "../fat_query", package = "fat-toolkit-query", features = ["test-fixtures"] } flate2 = "1" tempfile = "3" zip = "2" @@ -492,4 +506,3 @@ path = "../../tests/integration/test_taint_cross_state_profile_cli.rs" [lints] workspace = true - diff --git a/crates/fat_cli/src/data_cmd.rs b/crates/fat_cli/src/data_cmd.rs index 4dcba08..b861a10 100644 --- a/crates/fat_cli/src/data_cmd.rs +++ b/crates/fat_cli/src/data_cmd.rs @@ -120,7 +120,9 @@ fn status(explicit_root: Option, json: bool) -> DynResult<()> { let data_root = resolver .roots() .iter() - .find(|root| root.origin == resolved.origin) + .find(|root| { + root.origin == resolved.origin && resolved.root.starts_with(&root.path) + }) .map(|root| root.path.display().to_string()) .unwrap_or_else(|| resolved.root.display().to_string()); DataStatus { @@ -318,7 +320,7 @@ fn verify(explicit_root: Option, json: bool) -> DynResult<()> { let data_root = resolver .roots() .iter() - .find(|root| root.origin == resolved.origin) + .find(|root| root.origin == resolved.origin && resolved.root.starts_with(&root.path)) .map(|root| root.path.display().to_string()) .unwrap_or_else(|| resolved.root.display().to_string()); let result = InstallResult { diff --git a/crates/fat_core/src/data_dir.rs b/crates/fat_core/src/data_dir.rs index 3bb6174..b9fe07a 100644 --- a/crates/fat_core/src/data_dir.rs +++ b/crates/fat_core/src/data_dir.rs @@ -94,11 +94,16 @@ impl DataResolver { .and_then(Path::parent) .and_then(Path::parent) .map(|prefix| prefix.join("share").join("fat")); + let portable_root = executable + .as_deref() + .and_then(Path::parent) + .map(|directory| directory.join("share").join("fat")); let development_root = development_root.filter(|root| is_development_checkout(root)); let candidates = [ explicit_root.map(|path| (path, DataRootOrigin::Explicit)), environment_root.map(|path| (path, DataRootOrigin::Environment)), + portable_root.map(|path| (path, DataRootOrigin::ExecutableRelative)), executable_root.map(|path| (path, DataRootOrigin::ExecutableRelative)), user_root.map(|path| (path, DataRootOrigin::User)), development_root.map(|path| (path, DataRootOrigin::Development)), diff --git a/crates/fat_core/tests/data_dir.rs b/crates/fat_core/tests/data_dir.rs index 385f9a1..b9dcb5f 100644 --- a/crates/fat_core/tests/data_dir.rs +++ b/crates/fat_core/tests/data_dir.rs @@ -90,6 +90,29 @@ fn resolver_falls_back_through_environment_installed_user_and_development_roots( } } +#[test] +fn portable_bundle_uses_adjacent_data_before_parent_prefix_or_user_data() { + let temp = tempdir().unwrap(); + let prefix = temp.path().join("portable bundle"); + let adjacent = resource(&prefix.join("share/fat"), "profiles/rehosting"); + resource(&temp.path().join("share/fat"), "profiles/rehosting"); + let user = temp.path().join("user"); + resource(&user, "profiles/rehosting"); + + for name in ["fat", "fat.exe"] { + let resolver = DataResolver::from_paths( + None, + None, + Some(prefix.join(name)), + Some(user.clone()), + None, + ); + let resolved = resolver.resolve_required("profiles/rehosting").unwrap(); + assert_eq!(resolved.origin, DataRootOrigin::ExecutableRelative); + assert_eq!(resolved.path, adjacent); + } +} + #[test] fn unmarked_development_directory_is_not_a_candidate() { let temp = tempdir().unwrap(); @@ -142,7 +165,13 @@ fn managed_root_resolves_resources_from_the_active_version() { std::fs::create_dir_all(&resource).unwrap(); std::fs::write( active_root.join("manifest.json"), - r#"{"schema_version":1,"data_version":"0.1.4","compatible_fat":">=2.0.0-alpha.1,<2.1.0","files":[]}"#, + serde_json::json!({ + "schema_version": 1, + "data_version": "0.1.4", + "compatible_fat": format!("={}", env!("CARGO_PKG_VERSION")), + "files": [], + }) + .to_string(), ) .unwrap(); std::fs::write( @@ -168,7 +197,13 @@ fn managed_active_version_outranks_stale_direct_files() { let active = resource(&active_root, "profiles/rehosting"); std::fs::write( active_root.join("manifest.json"), - r#"{"schema_version":1,"data_version":"0.1.4","compatible_fat":">=2.0.0-alpha.1,<2.1.0","files":[]}"#, + serde_json::json!({ + "schema_version": 1, + "data_version": "0.1.4", + "compatible_fat": format!("={}", env!("CARGO_PKG_VERSION")), + "files": [], + }) + .to_string(), ) .unwrap(); std::fs::write( diff --git a/crates/fat_emulate/Cargo.toml b/crates/fat_emulate/Cargo.toml index ca8537c..5e52232 100644 --- a/crates/fat_emulate/Cargo.toml +++ b/crates/fat_emulate/Cargo.toml @@ -17,9 +17,9 @@ name = "fat_emulate" path = "src/lib.rs" [dependencies] -fat-backend = { version = "=2.0.0-alpha.1", path = "../fat_backend", package = "fat-toolkit-backend" } -fat-family = { version = "=2.0.0-alpha.1", path = "../fat_family", package = "fat-toolkit-family" } -fat-core = { version = "=2.0.0-alpha.1", path = "../fat_core", package = "fat-toolkit-core" } +fat-backend = { version = "=2.0.0", path = "../fat_backend", package = "fat-toolkit-backend" } +fat-family = { version = "=2.0.0", path = "../fat_family", package = "fat-toolkit-family" } +fat-core = { version = "=2.0.0", path = "../fat_core", package = "fat-toolkit-core" } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/crates/fat_plugin_api/Cargo.toml b/crates/fat_plugin_api/Cargo.toml index 6d3f929..ccfcc77 100644 --- a/crates/fat_plugin_api/Cargo.toml +++ b/crates/fat_plugin_api/Cargo.toml @@ -17,7 +17,7 @@ name = "fat_plugin_api" path = "src/lib.rs" [dependencies] -fat_core = { version = "=2.0.0-alpha.1", path = "../fat_core", package = "fat-toolkit-core" } +fat_core = { version = "=2.0.0", path = "../fat_core", package = "fat-toolkit-core" } serde = { version = "1", features = ["derive"] } [lints] diff --git a/crates/fat_plugin_host/Cargo.toml b/crates/fat_plugin_host/Cargo.toml index c5054aa..0cb7a30 100644 --- a/crates/fat_plugin_host/Cargo.toml +++ b/crates/fat_plugin_host/Cargo.toml @@ -17,9 +17,9 @@ name = "fat_plugin_host" path = "src/lib.rs" [dependencies] -fat_analyze = { version = "=2.0.0-alpha.1", path = "../fat_analyze", package = "fat-toolkit-analyze" } -fat_core = { version = "=2.0.0-alpha.1", path = "../fat_core", package = "fat-toolkit-core" } -fat_plugin_api = { version = "=2.0.0-alpha.1", path = "../fat_plugin_api", package = "fat-toolkit-plugin-api" } +fat_analyze = { version = "=2.0.0", path = "../fat_analyze", package = "fat-toolkit-analyze" } +fat_core = { version = "=2.0.0", path = "../fat_core", package = "fat-toolkit-core" } +fat_plugin_api = { version = "=2.0.0", path = "../fat_plugin_api", package = "fat-toolkit-plugin-api" } tempfile = "3" [[test]] diff --git a/crates/fat_query/Cargo.toml b/crates/fat_query/Cargo.toml index f103ae3..20b5f4e 100644 --- a/crates/fat_query/Cargo.toml +++ b/crates/fat_query/Cargo.toml @@ -23,7 +23,7 @@ test-fixtures = [] [dependencies] clang = { version = "2", features = ["runtime"] } -fat_core = { version = "=2.0.0-alpha.1", path = "../fat_core", package = "fat-toolkit-core" } +fat_core = { version = "=2.0.0", path = "../fat_core", package = "fat-toolkit-core" } rusqlite = { version = "0.32", features = ["bundled"] } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/crates/fat_taint/Cargo.toml b/crates/fat_taint/Cargo.toml index 2765f27..5b7b01b 100644 --- a/crates/fat_taint/Cargo.toml +++ b/crates/fat_taint/Cargo.toml @@ -17,8 +17,8 @@ name = "fat_taint" path = "src/lib.rs" [dependencies] -fat-core = { version = "=2.0.0-alpha.1", path = "../fat_core", package = "fat-toolkit-core" } -fat-query = { version = "=2.0.0-alpha.1", path = "../fat_query", package = "fat-toolkit-query" } +fat-core = { version = "=2.0.0", path = "../fat_core", package = "fat-toolkit-core" } +fat-query = { version = "=2.0.0", path = "../fat_query", package = "fat-toolkit-query" } serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yaml = "0.9" diff --git a/dist-workspace.toml b/dist-workspace.toml new file mode 100644 index 0000000..99bcfad --- /dev/null +++ b/dist-workspace.toml @@ -0,0 +1,43 @@ +[workspace] +members = ["cargo:."] + +[dist] +cargo-dist-version = "0.33.0" +packages = ["firmware-analysis-toolkit"] +ci = ["github"] +dispatch-releases = true +pr-run-mode = "upload" +create-release = false +installers = [] +targets = [ + "aarch64-apple-darwin", + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "aarch64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", +] +unix-archive = ".tar.gz" +windows-archive = ".zip" +auto-includes = false +include = [ + "target/release-resources/share", + "target/release-resources/README.md", + "target/release-resources/third-party-licenses.json", + "LICENSE", + "LICENSING.md", + "THIRD_PARTY_NOTICES.md", + "LICENSES", +] +github-build-setup = "../release-build-setup.yml" +global-artifacts-jobs = ["./release-smoke"] + +[dist.github-custom-runners] +aarch64-apple-darwin = "macos-14" +x86_64-apple-darwin = "macos-15-intel" +x86_64-unknown-linux-gnu = "ubuntu-22.04" +aarch64-unknown-linux-gnu = "ubuntu-24.04-arm" +x86_64-pc-windows-msvc = "windows-2022" + +[[dist.extra-artifacts]] +artifacts = ["dist/fat-data-2.0.0.zip", "dist/fat-data-2.0.0.zip.sha256"] +build = ["python3", "scripts/prepare-release-resources.py"] diff --git a/docs/runtime.md b/docs/runtime.md index 76abf64..7c0e587 100644 --- a/docs/runtime.md +++ b/docs/runtime.md @@ -13,18 +13,22 @@ fat data list `fat data status` reports which origin supplied the active tree: an explicit path, `FAT_DATA_DIR` (`environment`), executable-relative bundled `share/fat`, the managed user root, or a marked development checkout. `FAT_DATA_DIR` may point directly at an unversioned reviewed data tree. -The managed user-data root -instead uses `active.json`; FAT verifies that referenced version and its manifest before it is considered ready. An empty managed root with no active record is reported as `missing`. A present malformed active record, or one that points to missing, incompatible, or unverifiable data, is reported as structured `broken` status (including origin/path fields in JSON), never as ready. +The managed user-data root instead uses `active.json`; FAT verifies that referenced version and its manifest before it is considered ready. +An empty managed root with no active record is reported as `missing`. +A present malformed active record, or one that points to missing, incompatible, or unverifiable data, is reported as structured `broken` status (including origin/path fields in JSON), never as ready. Release maintainers build the downloadable ZIP from the reviewed source files: ```bash python3 scripts/build-runtime-data-bundle.py --check +release_version="$(awk -F'\"' '/^version = / { print $2; exit }' Cargo.toml)" python3 scripts/build-runtime-data-bundle.py \ - --output dist/fat-data-2.0.0-alpha.1.zip + --output "dist/fat-data-$release_version.zip" ``` -The bundle contains two example bootloader profiles, 21 kernel metadata and configuration files, three Android runtime helpers, and a README for each group. Its manifest pins every file by SHA-256 and declares compatibility with FAT 2.0.0 alpha releases. Linux kernels, U-Boot images, the kernel builder image, APKs, and Android instrumentation runners are separate prerequisites. +The bundle contains example bootloader profiles, kernel metadata and configuration files, Android runtime helpers, setup READMEs, and license notices. +Its manifest pins every file by SHA-256 and declares compatibility with the accompanying FAT version. +Linux kernels, U-Boot images, the kernel builder image, APKs, and Android instrumentation runners are separate prerequisites. The packaged READMEs explain each group's purpose and setup: @@ -34,16 +38,17 @@ The packaged READMEs explain each group's purpose and setup: The kernel builder record's `localhost/...` reference names a local image, not a download. The exact pinned image must be present before the offline kernel build can run. Container rebuilds require a digest comparison; a clean source checkout by itself does not supply a verified builder. -Official data assets use the exact URL pattern: +Download the data ZIP for your executable version from [FAT Releases](https://github.com/attify/firmware-analysis-toolkit/releases). +Asset URLs use this pattern, with `VERSION` replaced by the release version: ```text -https://github.com/attify/firmware-analysis-toolkit/releases/download/v2.0.0-alpha.1/fat-data-2.0.0-alpha.1.zip -https://github.com/attify/firmware-analysis-toolkit/releases/download/v2.0.0-alpha.1/fat-data-2.0.0-alpha.1.zip.sha256 +https://github.com/attify/firmware-analysis-toolkit/releases/download/vVERSION/fat-data-VERSION.zip +https://github.com/attify/firmware-analysis-toolkit/releases/download/vVERSION/fat-data-VERSION.zip.sha256 ``` Verify the adjacent `.sha256` file before `fat data install`. Installation preflights the ZIP and rejects oversized archives or central directories, path traversal, symlinks, duplicate or undeclared payloads, more than 1,024 entries, manifests over 1 MiB, and expanded output over 32 MiB. Streamed bytes are bounded independently of declared sizes, and no partially verified version is activated. -Resolution order is: command-level path, `FAT_DATA_DIR`, executable-relative `../share/fat`, platform user data, then a marked development checkout. Within a managed root, a verified active version takes precedence over stale direct -files. +Resolution order is: command-level path, `FAT_DATA_DIR`, executable-relative `../share/fat`, platform user data, then a marked development checkout. +Within a managed root, a verified active version takes precedence over stale direct files. Normal commands do not access the network. SHA-256 verification detects file changes but does not by itself authenticate a publisher; release signing is a separate launch gate. diff --git a/release-notes/v2.0.0.md b/release-notes/v2.0.0.md new file mode 100644 index 0000000..f55783e --- /dev/null +++ b/release-notes/v2.0.0.md @@ -0,0 +1,53 @@ +# FAT 2.0.0 + +FAT 2.0 is the first regular release of the Rust-based Firmware Analysis Toolkit. +It brings firmware identification, extraction, binary and filesystem analysis, and emulation workflows into the `fat` command-line interface. + +## Highlights + +- **Identify unknown firmware:** recognize containers, compressed data, raw MCU images, and repeated regions before choosing an analysis workflow. +- **Inspect Cortex-M firmware:** recover vector tables, address mappings, startup paths, referenced strings, and decoded peripheral accesses, with profile-backed device-family information. +- **Explore binaries and filesystems:** inspect ELF files, trace data flow, map startup behavior, and search extracted firmware. +- **Work through a firmware project:** create a project, extract its files, analyze it, and prepare QEMU-based emulation with explicit backend prerequisites. +- **Read focused reports:** compact summaries, semantic colors, bordered tables, `--details` for supporting tables, and JSON for structured output. +- **Install versioned runtime data:** ship the matching profiles and helper resources with the executable and verify their contents with `fat data verify`. + +## Installation + +Download the archive matching your operating system and CPU: + +| System | CPUs | Format | +| --- | --- | --- | +| macOS | Apple Silicon, Intel | `.tar.gz` | +| Linux, including Fedora | x86-64, ARM64 | `.tar.gz` | +| Windows | x86-64 | `.zip` | + +Verify it against its adjacent `.sha256` file and extract it. +Keep the executable and `share` directory together, then run: + +```bash +./fat --version +./fat data verify +./fat doctor +./fat identify ./firmware.bin +``` + +On Windows, use `.\fat.exe` instead of `./fat`. +Linux x86-64 requires glibc 2.35 or newer; Linux ARM64 requires glibc 2.39 or newer. +Linux bundles are tested on Ubuntu and Fedora 42. + +The release also provides a runtime-data ZIP for Cargo installations and tagged source archives. +See the [installation guide](https://github.com/attify/firmware-analysis-toolkit/blob/v2.0.0/INSTALL.md) for source installation and external analysis tools. + +## Upgrading from FAT 1.x + +FAT 2 uses a new Rust CLI and project workflow. +Install it separately from FAT 1.x and follow the current installation guide; the older `setup.sh`, `fat.py`, and `fat.config` instructions apply to FAT 1.x. +Keep existing FAT 1.x workspaces and recreate the projects you need with `fat new ./firmware.bin`. + +## License + +FAT 2.0 is source available under [FSL-1.1-ALv2](https://github.com/attify/firmware-analysis-toolkit/blob/v2.0.0/LICENSE). +Third-party components retain their own licenses, included with the binary bundles. + +See the [changelog](https://github.com/attify/firmware-analysis-toolkit/blob/v2.0.0/CHANGELOG.md) for detailed changes. diff --git a/release.toml b/release.toml new file mode 100644 index 0000000..64f6fae --- /dev/null +++ b/release.toml @@ -0,0 +1,10 @@ +# One FAT version across the workspace; distribution happens through GitHub. +shared-version = true +dependent-version = "upgrade" +consolidate-commits = true +publish = false +push = false +tag = false +tag-name = "v{{version}}" +tag-prefix = "" +pre-release-commit-message = "chore(release): prepare FAT {{version}}" diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 73cb934..0d8ed42 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "stable" +channel = "1.94.0" components = ["rustfmt", "clippy"] diff --git a/scripts/prepare-release-resources.py b/scripts/prepare-release-resources.py new file mode 100644 index 0000000..6659e53 --- /dev/null +++ b/scripts/prepare-release-resources.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +"""Stage verified runtime data and the release README for cargo-dist archives.""" + +from pathlib import Path +import runpy +import shutil +import zipfile + + +REPO = Path(__file__).resolve().parent.parent +builder = runpy.run_path(str(REPO / "scripts/build-runtime-data-bundle.py")) +manifest, entries = builder["load_and_verify"]() +version = manifest["data_version"] +archive = REPO / "dist" / f"fat-data-{version}.zip" +builder["build"](archive, manifest, entries) +Path(f"{archive}.sha256").write_text(builder["sha256"](archive) + "\n", encoding="ascii") +stage = REPO / "target/release-resources" +data = stage / "share/fat" +if data.exists(): + shutil.rmtree(data) +with zipfile.ZipFile(archive) as source: + source.extractall(data) +shutil.copyfile(REPO / f"release-notes/v{version}.md", stage / "README.md") +print(f"Staged FAT {version}: {len(entries)} runtime files and release notes") diff --git a/scripts/smoke-release.py b/scripts/smoke-release.py new file mode 100644 index 0000000..1bbf5dc --- /dev/null +++ b/scripts/smoke-release.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Smoke-test an extracted release archive outside its source checkout.""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import struct +import subprocess +import tarfile +import tempfile +import zipfile + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("archive", type=Path) + parser.add_argument("--version", required=True) + args = parser.parse_args() + archive = args.archive.resolve() + checksum = Path(f"{archive}.sha256").read_text().split()[0] + assert hashlib.sha256(archive.read_bytes()).hexdigest() == checksum, "archive checksum mismatch" + + with tempfile.TemporaryDirectory(prefix="fat release smoke ") as scratch: + root = Path(scratch) + if archive.suffix == ".zip": + with zipfile.ZipFile(archive) as source: + source.extractall(root) + else: + with tarfile.open(archive) as source: + source.extractall(root, filter="data") + binaries = list(root.rglob("fat.exe" if os.name == "nt" else "fat")) + binary, = [path for path in binaries if path.is_file()] + data_root = binary.parent / "share/fat" + manifest = json.loads((data_root / "manifest.json").read_text()) + assert manifest["data_version"] == args.version + licenses = json.loads((binary.parent / "third-party-licenses.json").read_text()) + assert licenses["packages"] and licenses["licenses"], "dependency licenses missing" + environment = os.environ.copy() + environment.pop("FAT_DATA_DIR", None) + + def run(*arguments: str) -> str: + result = subprocess.run( + [str(binary), *arguments], cwd=root, env=environment, + text=True, capture_output=True, timeout=60, check=True, + ) + return result.stdout + + assert run("--version").strip() == f"fat {args.version}" + verified = json.loads(run("data", "verify", "--json")) + assert verified["data_version"] == args.version + assert verified["verified_files"] == len(manifest["files"]) + assert data_root.resolve() == Path(verified["data_root"]).resolve(), verified + status = json.loads(run("data", "status", "--json")) + assert status["status"] == "ready", status + assert data_root.resolve() == Path(status["data_root"]).resolve(), status + blob = bytearray(512) + for index, value in enumerate([0x20001000, 0x08000101] + [0x08000121] * 14): + struct.pack_into("=2.0.0-alpha.1,<2.1.0", + "data_version": "2.0.0", + "compatible_fat": "=2.0.0", "files": [ { "path": "LICENSE", diff --git a/tests/integration/test_cli_bootstrap.rs b/tests/integration/test_cli_bootstrap.rs index 014f0bb..c19936c 100644 --- a/tests/integration/test_cli_bootstrap.rs +++ b/tests/integration/test_cli_bootstrap.rs @@ -66,8 +66,7 @@ fn fat_cli_shows_version() { assert!(output.status.success()); let stdout = String::from_utf8_lossy(&output.stdout); - assert!(stdout.contains("fat")); - assert!(stdout.contains("2.0.0-alpha.1")); + assert_eq!(stdout.trim(), concat!("fat ", env!("CARGO_PKG_VERSION"))); } #[test] diff --git a/tests/integration/test_data_cli.rs b/tests/integration/test_data_cli.rs index 4ced666..c9618f5 100644 --- a/tests/integration/test_data_cli.rs +++ b/tests/integration/test_data_cli.rs @@ -17,7 +17,7 @@ fn write_data_tree(root: &Path, version: &str, profile: &[u8]) { let manifest = json!({ "schema_version": 1, "data_version": version, - "compatible_fat": ">=2.0.0-alpha.1,<2.1.0", + "compatible_fat": format!("={}", env!("CARGO_PKG_VERSION")), "files": [{ "path": "profiles/rehosting/acme/router.yaml", "sha256": digest(profile), @@ -147,6 +147,7 @@ fn data_status_resolves_and_verifies_executable_relative_share_data() { assert_eq!(status["status"], "ready"); assert_eq!(status["data_version"], "2.0.0-alpha.7"); assert_eq!(status["origin"], "executable-relative"); + assert_eq!(status["data_root"], data_root.display().to_string()); assert_eq!(status["active_path"], data_root.display().to_string()); let human = Command::new(&installed_fat) @@ -444,7 +445,7 @@ fn data_install_accepts_a_safe_zip_archive() { let manifest = json!({ "schema_version": 1, "data_version": "2.0.0-alpha.3", - "compatible_fat": ">=2.0.0-alpha.1,<2.1.0", + "compatible_fat": format!("={}", env!("CARGO_PKG_VERSION")), "files": [{ "path": "profiles/rehosting/acme/router.yaml", "sha256": digest(profile), @@ -686,7 +687,7 @@ fn write_zip_with_manifest( let manifest = json!({ "schema_version": 1, "data_version": version, - "compatible_fat": ">=2.0.0-alpha.1,<2.1.0", + "compatible_fat": format!("={}", env!("CARGO_PKG_VERSION")), "files": files, }); let archive_file = fs::File::create(archive).unwrap(); diff --git a/tests/release/test_install_script.py b/tests/release/test_install_script.py index d39dca8..7b74ee7 100755 --- a/tests/release/test_install_script.py +++ b/tests/release/test_install_script.py @@ -5,6 +5,7 @@ import os from pathlib import Path +import re import subprocess import sys import tempfile @@ -14,6 +15,11 @@ REPO = Path(__file__).resolve().parents[2] INSTALLER = REPO / "scripts" / "install.sh" +FAT_VERSION = re.search( + r'^version\s*=\s*"([^"]+)"', + (REPO / "Cargo.toml").read_text(encoding="utf-8"), + re.MULTILINE, +).group(1) def write_executable(path: Path, text: str) -> None: @@ -72,7 +78,7 @@ def setUp(self) -> None: output.write("fat " + " ".join(args) + "\\n") if args == ["--version"]: - print("fat 2.0.0-alpha.1") + print("fat {FAT_VERSION}") raise SystemExit(0) if len(args) >= 2 and args[:2] == ["data", "install"]: data_dir = Path(args[args.index("--data-dir") + 1])