diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 14bb931d..9b94e24a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,9 +180,27 @@ jobs: with: targets: wasm32-unknown-unknown + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.12" + cache: pip + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: "22" + - name: Test crate and build wasm32 run: | cargo test --manifest-path crates/fuckmark-scan/Cargo.toml cargo build --release --target wasm32-unknown-unknown --manifest-path crates/fuckmark-scan/Cargo.toml test -f crates/fuckmark-scan/target/wasm32-unknown-unknown/release/fuckmark_scan.wasm + - name: Build host library and language bindings + run: | + ./crates/fuckmark-scan/build-native.sh + python -m pip install --upgrade pip + python -m pip install -e ".[dev]" + python -m pytest -q tests/test_language_bindings.py tests/test_scan_wasm.py + diff --git a/CHANGELOG.md b/CHANGELOG.md index 29cc3eff..1ab90e96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ - Zero-install hidden-Unicode scan page: `docs/scan.html` (packaged as `fuckmark/webui/scan.html`, engine copy `scan.js`). `fuckmark web` serves it at `/scan.html`. After deploy: `mark.q1z.org/scan.html`. Scanning runs in the browser; a local Python `/api/scan` confirm is optional when the CLI server is up. - Chromium browser extension under `editors/browser`: popup scanner, on-page reveal (badges plus CSS highlights), context-menu scan, per-tab toolbar badge, and optional paste-safe stripping that keeps real emoji ZWJ sequences (both neighbors emoji) and stops page paste handlers from seeing the raw clipboard. Zero remote code; `scan.js` matches the VS Code engine copy. Load unpacked from `chrome://extensions`. - WASM build of `fuckmark-hidden-scan-v1` (`crates/fuckmark-scan`, committed `fuckmark_scan.wasm`). The scan page and the Chromium popup load it when `fetch` works so classification stays on-device; `scan.js` remains the `file://` fallback. Lone UTF-16 surrogates route through the JS engine (UTF-8 cannot carry them). Empty category arrays stay empty selections. The extension CSP allows `wasm-unsafe-eval` so Chromium can instantiate the module. Page reveal and paste-safe keep the synchronous JS port. Rebuild with `crates/fuckmark-scan/build-wasm.sh`. +- Language bindings for the same Rust scan ABI: C header (`crates/fuckmark-scan/include/fuckmark_scan.h`), optional Python ctypes loader (`fuckmark.native_scan`), and a Node package under `bindings/node` (WASM + JS fallback). Host library build: `crates/fuckmark-scan/build-native.sh`. Lone UTF-16 surrogates cannot cross the UTF-8 ABI, so Python `scan_text` / `clean_text` delegate those strings to the Python reference scanner, and the WASM wrapper normalizes the JS fallback result to the same `index` / `truncated` / `highest_severity` / `source_length` schema. The C example is linked with `-rpath,$ORIGIN` and written next to `libfuckmark_scan.so`. Overview: `bindings/README.md`. - `fuckmark normalize` (`fuckmark-normalize-v1`): NFC, optional identifier lookalike fold (UTS #39-inspired subset, not a full confusable map), then strip of the security category set, plus a JSON receipt (`input_sha256` / `output_sha256` / `steps` / `report_hash`). `--clean` remains the blunt strip with no receipt. HTTP: `POST /api/normalize`. Python: `normalize_text`, `skeleton_fold`. Reference: `docs/normalize.md`. - VS Code / Cursor: `fuckmark.cleanOnSave` (default off) strips hidden Unicode on save. - Frozen confirmation files, hashes, mix mechanisms, and the published wheel are unchanged. Package version stays 0.4.1. diff --git a/README.md b/README.md index 30f4d261..ead6c83a 100644 --- a/README.md +++ b/README.md @@ -250,6 +250,16 @@ Eligible Latin, Greek, Cyrillic, Han, Kana, Hangul syllable, and emoji clusters Frozen evidence, hashes, and protocols: [`docs/research.md`](docs/research.md). Product contract: [`docs/product-contract.md`](docs/product-contract.md). CLI: [`docs/cli.md`](docs/cli.md). Limits matrix: [`docs/limits.md`](docs/limits.md). Demo: [`docs/demo.html`](docs/demo.html). +## Language bindings (Rust scan ABI) + +The hidden-scan classifier also ships as a small Rust crate with a C ABI, an optional Python native loader (`fuckmark.native_scan`), and a Node package (`bindings/node`). Overview: [`bindings/README.md`](bindings/README.md). + +```text +./crates/fuckmark-scan/build-native.sh +python -c "from fuckmark.native_scan import scan_text; print(scan_text('a\u202eb')['total'])" +cd bindings/node && node sync-assets.js && node test.js +``` + ## Development ```text diff --git a/bindings/README.md b/bindings/README.md new file mode 100644 index 00000000..e0d52832 --- /dev/null +++ b/bindings/README.md @@ -0,0 +1,57 @@ +# Language bindings + +The classifier core lives in `crates/fuckmark-scan` (`fuckmark-hidden-scan-v1`). +Python remains the reference implementation in `fuckmark.product.scan`. These +bindings call the same table through the shared C / WASM ABI. + +| Binding | Path | Engine | +| --- | --- | --- | +| C / FFI header | `crates/fuckmark-scan/include/fuckmark_scan.h` | host `cdylib` | +| Python (optional native) | `fuckmark/native_scan.py` | host `cdylib` via ctypes | +| Node / JS | `bindings/node` | WASM + JS fallback | +| Browser / extension | `editors/wasm`, `docs/scan.html` | WASM + JS fallback | + +## Build the host library + +```text +./crates/fuckmark-scan/build-native.sh +``` + +That prints the path to `libfuckmark_scan.so` (Linux), `.dylib` (macOS), or +`fuckmark_scan.dll` (Windows). Override discovery with `FUCKMARK_SCAN_LIB`. + +Categories ABI: pass `*` for every category, an empty string for an empty +selection, or a comma-separated list such as `bidi_control,tag`. + +Packed return values begin with a little-endian `u32` length followed by UTF-8 +JSON. + +## Python + +```text +./crates/fuckmark-scan/build-native.sh +python -c "from fuckmark.native_scan import scan_text; print(scan_text('a\u202eb'))" +``` + +If the library is missing, `available()` is false and callers should use +`fuckmark.product.scan`. + +## Node + +```text +cd bindings/node +node sync-assets.js +node test.js +``` + +## C example + +```text +./crates/fuckmark-scan/build-native.sh +cc -I crates/fuckmark-scan/include \ + crates/fuckmark-scan/examples/scan_cli.c \ + -L crates/fuckmark-scan/target/release -lfuckmark_scan \ + -Wl,-rpath,\$ORIGIN \ + -o crates/fuckmark-scan/target/release/fm-scan +./crates/fuckmark-scan/target/release/fm-scan $'a\u202eb' +``` diff --git a/bindings/node/README.md b/bindings/node/README.md new file mode 100644 index 00000000..9a0bfe07 --- /dev/null +++ b/bindings/node/README.md @@ -0,0 +1,21 @@ +# @fuckmark/scan (Node) + +Node binding for `fuckmark-hidden-scan-v1`. Loads the committed WASM module and +keeps the JS port as a fallback for lone UTF-16 surrogates / `file`-style +environments. + +```text +cd bindings/node +node sync-assets.js +node test.js +``` + +```js +const scan = require("./index.js"); + +const result = await scan.scanText(source, null, "javascript"); +const cleaned = await scan.cleanText(source, scan.fallback.DEFAULT_SECURITY_CATEGORIES); +``` + +Assets are copied from `editors/wasm` and `editors/vscode/scan.js`. Re-run +`npm run sync` after rebuilding the WASM module. diff --git a/bindings/node/fuckmark_scan.wasm b/bindings/node/fuckmark_scan.wasm new file mode 100755 index 00000000..5c459b0f Binary files /dev/null and b/bindings/node/fuckmark_scan.wasm differ diff --git a/bindings/node/index.js b/bindings/node/index.js new file mode 100644 index 00000000..448739f4 --- /dev/null +++ b/bindings/node/index.js @@ -0,0 +1,56 @@ +"use strict"; + +const fs = require("fs"); +const path = require("path"); +const loader = require("./scan_wasm.js"); +const fallback = require("./scan.js"); + +let ready = null; + +function wasmPath() { + return path.join(__dirname, "fuckmark_scan.wasm"); +} + +async function createScanEngine() { + globalThis.FuckMarkScan = fallback; + const bytes = fs.readFileSync(wasmPath()); + return loader.loadFuckMarkScanWasm(bytes); +} + +function getEngine() { + if (!ready) { + ready = createScanEngine(); + } + return ready; +} + +async function scanText(text, categories, language) { + const api = await getEngine(); + return api.scanText(text, categories, language); +} + +async function cleanText(text, categories) { + const api = await getEngine(); + return api.cleanText(text, categories); +} + +async function autofixTrojanSource(text) { + const api = await getEngine(); + return api.autofixTrojanSource(text); +} + +async function classify(codepoint) { + const api = await getEngine(); + return api.classify(codepoint); +} + +module.exports = { + SCAN_ALGORITHM_VERSION: "fuckmark-hidden-scan-v1", + createScanEngine, + getEngine, + scanText, + cleanText, + autofixTrojanSource, + classify, + fallback, +}; diff --git a/bindings/node/package.json b/bindings/node/package.json new file mode 100644 index 00000000..b1fea195 --- /dev/null +++ b/bindings/node/package.json @@ -0,0 +1,34 @@ +{ + "name": "@fuckmark/scan", + "version": "0.4.1", + "description": "fuckmark-hidden-scan-v1 Node binding (WASM engine with JS fallback)", + "license": "MIT", + "type": "commonjs", + "main": "index.js", + "files": [ + "index.js", + "scan_wasm.js", + "fuckmark_scan.wasm", + "scan.js", + "README.md" + ], + "engines": { + "node": ">=18" + }, + "scripts": { + "test": "node test.js", + "sync": "node sync-assets.js" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/byte271/FuckMark.git", + "directory": "bindings/node" + }, + "keywords": [ + "unicode", + "trojan-source", + "hidden-text", + "wasm", + "security" + ] +} diff --git a/bindings/node/scan.js b/bindings/node/scan.js new file mode 100644 index 00000000..f28c001f --- /dev/null +++ b/bindings/node/scan.js @@ -0,0 +1,569 @@ +// Faithful JavaScript port of fuckmark-hidden-scan-v1 (fuckmark/product/scan.py). +// Kept in lockstep with the Python engine by tests/test_vscode_scanner_parity.py. + +"use strict"; + +const SCAN_ALGORITHM_VERSION = "fuckmark-hidden-scan-v1"; + +const CATEGORY_BIDI_CONTROL = "bidi_control"; +const CATEGORY_ZERO_WIDTH = "zero_width"; +const CATEGORY_VARIATION_SELECTOR = "variation_selector"; +const CATEGORY_TAG = "tag"; +const CATEGORY_ENCLOSING_MARK = "enclosing_mark"; +const CATEGORY_LINE_SEPARATOR = "line_separator"; +const CATEGORY_DEPRECATED = "deprecated"; +const CATEGORY_FORMAT = "format"; +const CATEGORY_CONTROL = "control"; +const CATEGORY_PRIVATE_USE = "private_use"; +const CATEGORY_NONCHARACTER = "noncharacter"; +const CATEGORY_SURROGATE = "surrogate"; + +const SCAN_CATEGORIES = [ + CATEGORY_BIDI_CONTROL, + CATEGORY_ZERO_WIDTH, + CATEGORY_VARIATION_SELECTOR, + CATEGORY_TAG, + CATEGORY_ENCLOSING_MARK, + CATEGORY_LINE_SEPARATOR, + CATEGORY_DEPRECATED, + CATEGORY_FORMAT, + CATEGORY_CONTROL, + CATEGORY_PRIVATE_USE, + CATEGORY_NONCHARACTER, + CATEGORY_SURROGATE, +]; + +const CATEGORY_DESCRIPTIONS = { + [CATEGORY_BIDI_CONTROL]: "bidirectional override/isolate (Trojan Source reordering)", + [CATEGORY_ZERO_WIDTH]: "zero-width or invisible spacing character", + [CATEGORY_VARIATION_SELECTOR]: "variation selector (glyph/steganography carrier)", + [CATEGORY_TAG]: "Unicode tag character (hidden text / prompt-injection smuggling)", + [CATEGORY_ENCLOSING_MARK]: "enclosing combining mark (alters surrounding glyph)", + [CATEGORY_LINE_SEPARATOR]: "line/paragraph separator (breaks parsers, invisible)", + [CATEGORY_DEPRECATED]: "deprecated format control or interlinear annotation", + [CATEGORY_FORMAT]: "general Unicode format control (Cf)", + [CATEGORY_CONTROL]: "C0/C1 control character", + [CATEGORY_PRIVATE_USE]: "private-use codepoint (renderer-defined)", + [CATEGORY_NONCHARACTER]: "Unicode noncharacter or unassigned codepoint", + [CATEGORY_SURROGATE]: "lone surrogate codepoint", +}; + +const DEFAULT_SECURITY_CATEGORIES = [ + CATEGORY_BIDI_CONTROL, + CATEGORY_ZERO_WIDTH, + CATEGORY_TAG, + CATEGORY_CONTROL, + CATEGORY_NONCHARACTER, + CATEGORY_SURROGATE, +]; + +const TROJAN_SOURCE_CATEGORIES = [CATEGORY_BIDI_CONTROL]; + +const ALLOWED_WHITESPACE = new Set([0x09, 0x0a, 0x0d, 0x20]); + +function inRange(cp, lo, hi) { + return cp >= lo && cp <= hi; +} + +const BIDI_CONTROL = new Set([0x061c, 0x200e, 0x200f]); +for (let cp = 0x202a; cp <= 0x202e; cp += 1) BIDI_CONTROL.add(cp); +for (let cp = 0x2066; cp <= 0x2069; cp += 1) BIDI_CONTROL.add(cp); + +const DEPRECATED = new Set([0xfff9, 0xfffa, 0xfffb]); +for (let cp = 0x206a; cp <= 0x206f; cp += 1) DEPRECATED.add(cp); + +const ZERO_WIDTH = new Set([ + 0x00ad, 0x034f, 0x115f, 0x1160, 0x17b4, 0x17b5, 0x180e, 0x200b, 0x200c, + 0x200d, 0x2060, 0x2061, 0x2062, 0x2063, 0x2064, 0x3164, 0xfeff, 0xffa0, +]); + +function isVariationSelector(cp) { + return inRange(cp, 0xfe00, 0xfe0f) || inRange(cp, 0xe0100, 0xe01ef); +} + +function isTag(cp) { + return inRange(cp, 0xe0000, 0xe007f); +} + +function isNoncharacter(cp) { + if (inRange(cp, 0xfdd0, 0xfdef)) return true; + const low = cp & 0xffff; + return low === 0xfffe || low === 0xffff; +} + +function isEnclosingMark(cp) { + // Unicode general category Me. Enumerated to avoid a runtime Unicode DB. + return ( + inRange(cp, 0x0488, 0x0489) || + cp === 0x1abe || + inRange(cp, 0x20dd, 0x20e0) || + inRange(cp, 0x20e2, 0x20e4) || + inRange(cp, 0xa670, 0xa672) + ); +} + +function isControl(cp) { + return inRange(cp, 0x00, 0x1f) || inRange(cp, 0x7f, 0x9f); +} + +function isLineSeparator(cp) { + return cp === 0x2028 || cp === 0x2029; +} + +// General category Cf ranges (Unicode 15-era), excluding codepoints already +// classified above. Mirrors what unicodedata.category returns as "Cf". +const FORMAT_RANGES = [ + [0x0600, 0x0605], + [0x06dd, 0x06dd], + [0x070f, 0x070f], + [0x0890, 0x0891], + [0x08e2, 0x08e2], + [0x110bd, 0x110bd], + [0x110cd, 0x110cd], + [0x13430, 0x1343f], + [0x1bca0, 0x1bca3], + [0x1d173, 0x1d17a], +]; + +function isFormat(cp) { + for (const [lo, hi] of FORMAT_RANGES) { + if (inRange(cp, lo, hi)) return true; + } + return false; +} + +function isPrivateUse(cp) { + return ( + inRange(cp, 0xe000, 0xf8ff) || + inRange(cp, 0xf0000, 0xffffd) || + inRange(cp, 0x100000, 0x10fffd) + ); +} + +function classify(cp) { + if (ALLOWED_WHITESPACE.has(cp)) return null; + if (BIDI_CONTROL.has(cp)) return CATEGORY_BIDI_CONTROL; + if (DEPRECATED.has(cp)) return CATEGORY_DEPRECATED; + if (ZERO_WIDTH.has(cp)) return CATEGORY_ZERO_WIDTH; + if (isVariationSelector(cp)) return CATEGORY_VARIATION_SELECTOR; + if (isTag(cp)) return CATEGORY_TAG; + if (isNoncharacter(cp)) return CATEGORY_NONCHARACTER; + if (cp >= 0xd800 && cp <= 0xdfff) return CATEGORY_SURROGATE; + if (isEnclosingMark(cp)) return CATEGORY_ENCLOSING_MARK; + if (isLineSeparator(cp)) return CATEGORY_LINE_SEPARATOR; + if (isControl(cp)) return CATEGORY_CONTROL; + if (isFormat(cp)) return CATEGORY_FORMAT; + if (isPrivateUse(cp)) return CATEGORY_PRIVATE_USE; + return null; +} + +function codepointToken(cp) { + return "U+" + cp.toString(16).toUpperCase().padStart(4, "0"); +} + +function isEmojiish(cp) { + if (cp < 0) return false; + if (cp >= 0x1f1e6 && cp <= 0x1f1ff) return true; + if (cp >= 0x1f000 && cp <= 0x1faff) return true; + if (cp >= 0x2600 && cp <= 0x27bf) return true; + return cp === 0x200d || [0x00a9, 0x00ae, 0x203c, 0x2049, 0x2122, 0x2139, 0x3030, 0x303d, 0x3297, 0x3299].includes(cp); +} + +function isIdentChar(ch) { + return ch === "_" || /\p{L}|\p{N}/u.test(ch); +} + +function neighborBefore(text, offset) { + if (offset <= 0) return { cp: -1, ch: "" }; + if (offset >= 2) { + const cp = text.codePointAt(offset - 2); + if (cp > 0xffff) return { cp, ch: String.fromCodePoint(cp) }; + } + const cp = text.codePointAt(offset - 1); + return { cp, ch: String.fromCodePoint(cp) }; +} + +function neighborAfter(text, offset, length) { + const nextIndex = offset + length; + if (nextIndex >= text.length) return { cp: -1, ch: "" }; + const cp = text.codePointAt(nextIndex); + return { cp, ch: String.fromCodePoint(cp) }; +} + +const LANGUAGE_ALIASES = { + auto: "auto", + javascript: "javascript", + js: "javascript", + ts: "javascript", + typescript: "javascript", + jsx: "javascript", + tsx: "javascript", + c: "c", + h: "c", + cc: "c", + cpp: "c", + cxx: "c", + java: "c", + go: "c", + rs: "c", + rust: "c", + cs: "c", + css: "c", + jsonc: "c", + python: "python", + py: "python", + pyi: "python", + hash: "python", + sh: "python", + bash: "python", + zsh: "python", + shell: "python", + shellscript: "python", + yaml: "python", + yml: "python", + rb: "python", + ruby: "python", + toml: "python", + html: "html", + htm: "html", + xml: "html", + sql: "sql", +}; + +function normalizeLanguage(language) { + if (!language) return "auto"; + const key = String(language).trim().toLowerCase(); + return LANGUAGE_ALIASES[key] || "auto"; +} + +function slashComments(language) { + return language === "auto" || language === "javascript" || language === "c" || language === "sql"; +} + +function blockComments(language) { + return slashComments(language); +} + +function hashComments(language) { + return language === "python"; +} + +function sqlLineComments(language) { + return language === "sql"; +} + +function htmlComments(language) { + return language === "html"; +} + +function sourceRoles(text, language) { + const lang = normalizeLanguage(language); + const roles = new Array(text.length).fill("code"); + let index = 0; + let inString = ""; + let escape = false; + while (index < text.length) { + const ch = text[index]; + if (inString) { + roles[index] = "string"; + if (escape) { + escape = false; + index += 1; + continue; + } + if (ch === "\\" && index + 1 < text.length) { + escape = true; + index += 1; + continue; + } + if (ch === inString) inString = ""; + index += 1; + continue; + } + const pair = text.slice(index, index + 2); + if (slashComments(lang) && pair === "//" && !(index > 0 && text[index - 1] === ":")) { + let cursor = index; + while (cursor < text.length && text[cursor] !== "\n" && text[cursor] !== "\r") { + roles[cursor] = "comment"; + cursor += 1; + } + index = cursor; + continue; + } + if (sqlLineComments(lang) && pair === "--") { + let cursor = index; + while (cursor < text.length && text[cursor] !== "\n" && text[cursor] !== "\r") { + roles[cursor] = "comment"; + cursor += 1; + } + index = cursor; + continue; + } + if (blockComments(lang) && pair === "/*") { + const close = text.indexOf("*/", index + 2); + const stop = close >= 0 ? close + 2 : text.length; + for (let cursor = index; cursor < stop; cursor += 1) roles[cursor] = "comment"; + index = stop; + continue; + } + if (hashComments(lang) && ch === "#") { + let cursor = index; + while (cursor < text.length && text[cursor] !== "\n" && text[cursor] !== "\r") { + roles[cursor] = "comment"; + cursor += 1; + } + index = cursor; + continue; + } + if (htmlComments(lang) && text.slice(index, index + 4) === "", index + 4); + const stop = close >= 0 ? close + 3 : text.length; + for (let cursor = index; cursor < stop; cursor += 1) roles[cursor] = "comment"; + index = stop; + continue; + } + if (ch === '"' || ch === "'" || ch === "`") { + inString = ch; + roles[index] = "string"; + index += 1; + continue; + } + index += 1; + } + return roles; +} + +function classifyContext(text, offset, role, category) { + const length = text.codePointAt(offset) > 0xffff ? 2 : 1; + const prev = neighborBefore(text, offset); + const next = neighborAfter(text, offset, length); + if ((role === "comment" || role === "string") && category === CATEGORY_BIDI_CONTROL) { + return role; + } + if ( + isEmojiish(prev.cp) || + isEmojiish(next.cp) || + (prev.cp >= 0xfe00 && prev.cp <= 0xfe0f) || + (next.cp >= 0xfe00 && next.cp <= 0xfe0f) + ) { + return "emoji"; + } + if (role === "comment") return "comment"; + if (role === "string") return "string"; + if (isIdentChar(prev.ch) || isIdentChar(next.ch)) return "identifier"; + return "prose"; +} + +function scoreSeverity(category, context) { + if (category === CATEGORY_TAG) return "critical"; + if (category === CATEGORY_BIDI_CONTROL) { + if (context === "identifier" || context === "comment" || context === "string") return "critical"; + return "high"; + } + if (category === CATEGORY_ZERO_WIDTH) { + if (context === "emoji") return "info"; + if (context === "identifier") return "high"; + return "medium"; + } + if (category === CATEGORY_VARIATION_SELECTOR) return context === "emoji" ? "info" : "medium"; + if (category === CATEGORY_CONTROL || category === CATEGORY_NONCHARACTER || category === CATEGORY_SURROGATE) { + return "high"; + } + return "medium"; +} + +function explainFinding(category, context, severity) { + if (category === CATEGORY_BIDI_CONTROL && context === "identifier") { + return { + why: "Bidirectional override sits inside an identifier, so the glyphs can read differently than the bytes (Trojan Source).", + remedy: "Strip the bidi control and keep the identifier left-to-right.", + }; + } + if (category === CATEGORY_BIDI_CONTROL && context === "comment") { + return { + why: "Bidirectional override sits inside a comment, so commented-out code can appear to run (Trojan Source commenting-out).", + remedy: "Strip the bidi control from the comment.", + }; + } + if (category === CATEGORY_BIDI_CONTROL && context === "string") { + return { + why: "Bidirectional override sits inside a string, so the literal can appear to close early (Trojan Source stretched-string).", + remedy: "Strip the bidi control from the string.", + }; + } + if (category === CATEGORY_BIDI_CONTROL) { + return { + why: "Bidirectional override can reorder nearby glyphs (Trojan Source class, CVE-2021-42574).", + remedy: "Strip U+202A-U+202E / U+2066-U+2069 and rewrite the text left-to-right.", + }; + } + if (category === CATEGORY_TAG) { + return { + why: "Unicode tag characters encode a second ASCII string that models read and humans do not.", + remedy: "Strip U+E0020-U+E007F; inspect tag_payload for the smuggled text.", + }; + } + if (category === CATEGORY_ZERO_WIDTH && context === "emoji") { + return { + why: "Zero-width joiner or invisible mark inside an emoji cluster; usually a legitimate emoji sequence.", + remedy: "Leave emoji ZWJ sequences unless you are sanitizing for a security boundary.", + }; + } + if (category === CATEGORY_ZERO_WIDTH && context === "identifier") { + return { + why: "Zero-width character splits an identifier, breaking search and some compilers while looking unchanged.", + remedy: "Strip the zero-width character from the identifier.", + }; + } + if (category === CATEGORY_ZERO_WIDTH) { + return { + why: "Invisible spacing or joining character that changes the byte stream without changing the glyphs.", + remedy: "Strip the zero-width character.", + }; + } + if (category === CATEGORY_VARIATION_SELECTOR && context === "emoji") { + return { + why: "Variation selector tunes an emoji glyph; usually benign.", + remedy: "Keep emoji variation selectors unless you are stripping all hidden marks.", + }; + } + if (severity === "high") { + return { + why: "Hidden or non-text codepoint that should not appear in ordinary source or prompts.", + remedy: "Strip the character.", + }; + } + return { + why: CATEGORY_DESCRIPTIONS[category] || "Hidden or format codepoint that is invisible or renderer-defined.", + remedy: "Strip the character if this text crosses a trust boundary.", + }; +} + +function scanText(text, categories, language) { + const selected = categories ? new Set(categories) : null; + const roles = sourceRoles(text, language); + const counts = {}; + const findings = []; + let total = 0; + let offset = 0; + while (offset < text.length) { + const cp = text.codePointAt(offset); + const length = cp > 0xffff ? 2 : 1; + const category = classify(cp); + if (category !== null && (selected === null || selected.has(category))) { + total += 1; + counts[category] = (counts[category] || 0) + 1; + const context = classifyContext(text, offset, roles[offset] || "code", category); + const severity = scoreSeverity(category, context); + const explained = explainFinding(category, context, severity); + findings.push({ + offset, + length, + codepoint: cp, + category, + context, + severity, + why: explained.why, + remedy: explained.remedy, + }); + } + offset += length; + } + return { total, counts, findings }; +} + +function cleanText(text, categories) { + const selected = categories ? new Set(categories) : null; + let out = ""; + let removed = 0; + let offset = 0; + while (offset < text.length) { + const cp = text.codePointAt(offset); + const length = cp > 0xffff ? 2 : 1; + const category = classify(cp); + if (category !== null && (selected === null || selected.has(category))) { + removed += 1; + } else { + out += text.slice(offset, offset + length); + } + offset += length; + } + return { cleaned: out, removed }; +} + +function autofixTrojanSource(text) { + return cleanText(text, TROJAN_SOURCE_CATEGORIES); +} + +function languageFromDocumentId(languageId) { + return normalizeLanguage(languageId); +} + +const LANGUAGE_BY_SUFFIX = { + ".js": "javascript", + ".mjs": "javascript", + ".cjs": "javascript", + ".ts": "javascript", + ".jsx": "javascript", + ".tsx": "javascript", + ".c": "c", + ".h": "c", + ".hh": "c", + ".cc": "c", + ".cpp": "c", + ".cxx": "c", + ".java": "c", + ".go": "c", + ".rs": "c", + ".cs": "c", + ".css": "c", + ".py": "python", + ".pyi": "python", + ".sh": "python", + ".bash": "python", + ".zsh": "python", + ".yaml": "python", + ".yml": "python", + ".rb": "python", + ".toml": "python", + ".html": "html", + ".htm": "html", + ".xml": "html", + ".sql": "sql", +}; + +function languageFromPath(path) { + const name = String(path || ""); + const slash = Math.max(name.lastIndexOf("/"), name.lastIndexOf("\\")); + const base = slash >= 0 ? name.slice(slash + 1) : name; + const dot = base.lastIndexOf("."); + if (dot <= 0) return "auto"; + const suffix = base.slice(dot).toLowerCase(); + return LANGUAGE_BY_SUFFIX[suffix] || "auto"; +} + +const FuckMarkScan = { + SCAN_ALGORITHM_VERSION, + SCAN_CATEGORIES, + CATEGORY_DESCRIPTIONS, + DEFAULT_SECURITY_CATEGORIES, + TROJAN_SOURCE_CATEGORIES, + classify, + classifyContext, + scoreSeverity, + normalizeLanguage, + languageFromDocumentId, + languageFromPath, + sourceRoles, + codepointToken, + scanText, + cleanText, + autofixTrojanSource, +}; + +if (typeof module !== "undefined" && module.exports) { + module.exports = FuckMarkScan; +} +if (typeof globalThis !== "undefined") { + globalThis.FuckMarkScan = FuckMarkScan; +} diff --git a/bindings/node/scan_wasm.js b/bindings/node/scan_wasm.js new file mode 100644 index 00000000..4e8efdda --- /dev/null +++ b/bindings/node/scan_wasm.js @@ -0,0 +1,270 @@ +"use strict"; + +const WASM_CATEGORIES = [ + "bidi_control", + "zero_width", + "variation_selector", + "tag", + "enclosing_mark", + "line_separator", + "deprecated", + "format", + "control", + "private_use", + "noncharacter", + "surrogate", +]; + +function utf16Offsets(text) { + const map = []; + let offset = 0; + while (offset < text.length) { + map.push(offset); + const cp = text.codePointAt(offset); + offset += cp > 0xffff ? 2 : 1; + } + return map; +} + +function codepointToken(cp) { + return "U+" + Number(cp).toString(16).toUpperCase().padStart(4, "0"); +} + +function hasLoneSurrogate(text) { + const source = String(text || ""); + for (let index = 0; index < source.length; index += 1) { + const unit = source.charCodeAt(index); + if (unit >= 0xd800 && unit <= 0xdbff) { + if (index + 1 >= source.length) return true; + const next = source.charCodeAt(index + 1); + if (next < 0xdc00 || next > 0xdfff) return true; + index += 1; + continue; + } + if (unit >= 0xdc00 && unit <= 0xdfff) return true; + } + return false; +} + +function encodeCategories(categories) { + if (categories == null) return "*"; + if (Array.isArray(categories)) return categories.join(","); + return String(categories); +} + +const SEVERITY_RANK = { info: 0, medium: 1, high: 2, critical: 3 }; + +function codePointCount(text) { + let count = 0; + for (let offset = 0; offset < text.length; ) { + const cp = text.codePointAt(offset); + offset += cp > 0xffff ? 2 : 1; + count += 1; + } + return count; +} + +function utf16OffsetToIndex(text, offset) { + let index = 0; + let pos = 0; + while (pos < offset && pos < text.length) { + const cp = text.codePointAt(pos); + pos += cp > 0xffff ? 2 : 1; + index += 1; + } + return index; +} + +function peakSeverity(findings) { + let peak = ""; + for (const finding of findings) { + if ((SEVERITY_RANK[finding.severity] || -1) > (SEVERITY_RANK[peak] || -1)) { + peak = finding.severity; + } + } + return peak; +} + +function normalizeScanResult(source, result) { + const findings = (result.findings || []).map((finding) => { + const offset = finding.offset != null ? finding.offset : 0; + const length = finding.length != null ? finding.length : finding.codepoint > 0xffff ? 2 : 1; + const index = finding.index != null ? finding.index : utf16OffsetToIndex(source, offset); + return { + offset, + length, + index, + codepoint: finding.codepoint, + category: finding.category, + context: finding.context, + severity: finding.severity, + why: finding.why, + remedy: finding.remedy, + }; + }); + return { + total: result.total || 0, + counts: result.counts || {}, + findings, + truncated: Boolean(result.truncated), + highest_severity: result.highest_severity || peakSeverity(findings), + source_length: result.source_length != null ? result.source_length : codePointCount(source), + }; +} + +function writeUtf8(memory, alloc, bytes) { + const ptr = alloc(bytes.length); + new Uint8Array(memory.buffer, ptr, bytes.length).set(bytes); + return ptr; +} + +function readPackedJson(memory, dealloc, ptr) { + const view = new DataView(memory.buffer, ptr, 4); + const length = view.getUint32(0, true); + const bytes = new Uint8Array(memory.buffer, ptr + 4, length); + const text = new TextDecoder("utf-8").decode(bytes); + dealloc(ptr, 4 + length); + return JSON.parse(text); +} + +function wrapInstance(instance, fallback) { + const { memory, fm_alloc, fm_dealloc, fm_classify, fm_scan, fm_clean } = instance.exports; + const encoder = new TextEncoder(); + + function callScan(text, language, categories, maxFindings) { + const textBytes = encoder.encode(String(text || "")); + const langBytes = encoder.encode(String(language || "auto")); + const catBytes = encoder.encode(encodeCategories(categories)); + const textPtr = writeUtf8(memory, fm_alloc, textBytes); + const langPtr = writeUtf8(memory, fm_alloc, langBytes); + const catPtr = writeUtf8(memory, fm_alloc, catBytes); + const resultPtr = fm_scan( + textPtr, + textBytes.length, + langPtr, + langBytes.length, + catPtr, + catBytes.length, + maxFindings == null ? -1 : maxFindings + ); + fm_dealloc(textPtr, textBytes.length); + fm_dealloc(langPtr, langBytes.length); + fm_dealloc(catPtr, catBytes.length); + return readPackedJson(memory, fm_dealloc, resultPtr); + } + + function callClean(text, categories) { + const textBytes = encoder.encode(String(text || "")); + const catBytes = encoder.encode(encodeCategories(categories)); + const textPtr = writeUtf8(memory, fm_alloc, textBytes); + const catPtr = writeUtf8(memory, fm_alloc, catBytes); + const resultPtr = fm_clean(textPtr, textBytes.length, catPtr, catBytes.length); + fm_dealloc(textPtr, textBytes.length); + fm_dealloc(catPtr, catBytes.length); + return readPackedJson(memory, fm_dealloc, resultPtr); + } + + function scanText(text, categories, language) { + const source = String(text || ""); + if (fallback && hasLoneSurrogate(source)) { + return normalizeScanResult(source, fallback.scanText(source, categories, language)); + } + const payload = callScan(source, language || "auto", categories, -1); + const offsets = utf16Offsets(source); + const findings = (payload.findings || []).map((finding) => { + const offset = offsets[finding.index] || 0; + const length = finding.codepoint > 0xffff ? 2 : 1; + return { + offset, + length, + index: finding.index, + codepoint: finding.codepoint, + category: finding.category, + context: finding.context, + severity: finding.severity, + why: finding.why, + remedy: finding.remedy, + }; + }); + return { + total: payload.total || 0, + counts: payload.counts || {}, + findings, + truncated: Boolean(payload.truncated), + highest_severity: payload.highest_severity || "", + source_length: payload.source_length || 0, + }; + } + + function cleanText(text, categories) { + const source = String(text || ""); + if (fallback && hasLoneSurrogate(source)) { + return fallback.cleanText(source, categories); + } + return callClean(source, categories); + } + + const api = Object.assign({}, fallback || {}, { + engine: "wasm", + SCAN_ALGORITHM_VERSION: "fuckmark-hidden-scan-v1", + classify(cp) { + const index = fm_classify(cp >>> 0); + return index < 0 ? null : WASM_CATEGORIES[index]; + }, + scanText, + cleanText, + autofixTrojanSource(text) { + return cleanText(text, ["bidi_control"]); + }, + codepointToken: (fallback && fallback.codepointToken) || codepointToken, + hasLoneSurrogate, + encodeCategories, + }); + return api; +} + +async function instantiateWasm(source) { + let buffer; + if (source instanceof ArrayBuffer) { + buffer = source; + } else if (ArrayBuffer.isView(source)) { + buffer = source.buffer.slice(source.byteOffset, source.byteOffset + source.byteLength); + } else if (typeof source === "string") { + const response = await fetch(source); + if (!response.ok) { + throw new Error("fuckmark_scan.wasm missing"); + } + buffer = await response.arrayBuffer(); + } else { + throw new Error("wasm source required"); + } + const result = await WebAssembly.instantiate(buffer, {}); + return result.instance; +} + +async function loadFuckMarkScanWasm(source) { + const fallback = typeof globalThis !== "undefined" ? globalThis.FuckMarkScan : undefined; + const instance = await instantiateWasm(source || "fuckmark_scan.wasm"); + const api = wrapInstance(instance, fallback); + if (typeof globalThis !== "undefined") { + globalThis.FuckMarkScan = api; + globalThis.FuckMarkScanWasm = api; + } + if (typeof module !== "undefined" && module.exports) { + module.exports.loadFuckMarkScanWasm = loadFuckMarkScanWasm; + module.exports.wrapInstance = wrapInstance; + } + return api; +} + +if (typeof module !== "undefined" && module.exports) { + module.exports.loadFuckMarkScanWasm = loadFuckMarkScanWasm; + module.exports.wrapInstance = wrapInstance; + module.exports.WASM_CATEGORIES = WASM_CATEGORIES; + module.exports.hasLoneSurrogate = hasLoneSurrogate; + module.exports.encodeCategories = encodeCategories; + module.exports.normalizeScanResult = normalizeScanResult; +} +if (typeof globalThis !== "undefined") { + globalThis.loadFuckMarkScanWasm = loadFuckMarkScanWasm; +} diff --git a/bindings/node/sync-assets.js b/bindings/node/sync-assets.js new file mode 100644 index 00000000..acb3321e --- /dev/null +++ b/bindings/node/sync-assets.js @@ -0,0 +1,18 @@ +"use strict"; + +const fs = require("fs"); +const path = require("path"); + +const root = path.resolve(__dirname, "../.."); +const copies = [ + ["editors/wasm/scan_wasm.js", "scan_wasm.js"], + ["editors/wasm/fuckmark_scan.wasm", "fuckmark_scan.wasm"], + ["editors/vscode/scan.js", "scan.js"], +]; + +for (const [srcRel, destName] of copies) { + const src = path.join(root, srcRel); + const dest = path.join(__dirname, destName); + fs.copyFileSync(src, dest); + process.stdout.write(destName + "\n"); +} diff --git a/bindings/node/test.js b/bindings/node/test.js new file mode 100644 index 00000000..a58b4445 --- /dev/null +++ b/bindings/node/test.js @@ -0,0 +1,33 @@ +"use strict"; + +const assert = require("assert"); +const scan = require("./index.js"); + +(async () => { + const rlo = String.fromCodePoint(0x202e); + const result = await scan.scanText("a" + rlo + "b", null, "auto"); + assert.strictEqual(result.total, 1); + assert.strictEqual(result.findings[0].category, "bidi_control"); + assert.strictEqual(result.findings[0].severity, "critical"); + const cleaned = await scan.cleanText("a" + rlo + "b", ["bidi_control"]); + assert.strictEqual(cleaned.removed, 1); + assert.strictEqual(cleaned.cleaned, "ab"); + const empty = await scan.scanText("a" + rlo + "b", [], "auto"); + assert.strictEqual(empty.total, 0); + const lone = await scan.scanText("\uD800", null, "auto"); + assert.strictEqual(lone.total, 1); + assert.strictEqual(lone.findings[0].category, "surrogate"); + assert.strictEqual(lone.findings[0].index, 0); + assert.strictEqual(lone.findings[0].offset, 0); + assert.strictEqual(lone.truncated, false); + assert.strictEqual(lone.highest_severity, "high"); + assert.strictEqual(lone.source_length, 1); + const mixed = await scan.scanText("a\uD800b", null, "auto"); + assert.strictEqual(mixed.total, 1); + assert.strictEqual(mixed.findings[0].index, 1); + assert.strictEqual(mixed.source_length, 3); + process.stdout.write("ok\n"); +})().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/crates/fuckmark-scan/README.md b/crates/fuckmark-scan/README.md index 1dbb4c44..60e72f5b 100644 --- a/crates/fuckmark-scan/README.md +++ b/crates/fuckmark-scan/README.md @@ -1,18 +1,20 @@ -# fuckmark-scan (Rust / WASM) +# fuckmark-scan (Rust / WASM / native) Native and `wasm32` build of `fuckmark-hidden-scan-v1`. Same category table as `editors/vscode/scan.js` and `fuckmark.product.scan`. No network, no extra -crate dependencies. +crate dependencies. C ABI header: `include/fuckmark_scan.h`. ```text rustup target add wasm32-unknown-unknown ./crates/fuckmark-scan/build-wasm.sh +./crates/fuckmark-scan/build-native.sh ``` -That writes `fuckmark_scan.wasm` next to the scan page, the packaged web UI, -and the Chromium popup. The page and popup load it when `fetch` works and keep -`scan.js` as the `file://` fallback. +`build-wasm.sh` writes `fuckmark_scan.wasm` next to the scan page, packaged web +UI, and Chromium popup. `build-native.sh` builds the host `cdylib` for the +Python ctypes binding and C callers. -`cargo test` in this directory covers classification, Trojan Source roles, and -clean. Conformance vectors are also replayed from Python via Node on the -committed module. +Category filter encoding over the ABI: `*` means all categories, an empty +string means an empty selection, otherwise a comma-separated list. + +Language bindings overview: [`bindings/README.md`](../../bindings/README.md). diff --git a/crates/fuckmark-scan/build-native.sh b/crates/fuckmark-scan/build-native.sh new file mode 100755 index 00000000..9773cbc6 --- /dev/null +++ b/crates/fuckmark-scan/build-native.sh @@ -0,0 +1,21 @@ +#!/bin/sh +set -eu +root="$(CDPATH= cd -- "$(dirname "$0")/../.." && pwd)" +crate="$root/crates/fuckmark-scan" +cd "$crate" +cargo test --offline --quiet 2>/dev/null || cargo test --quiet +cargo build --release --quiet +lib="" +for candidate in \ + "$crate/target/release/libfuckmark_scan.so" \ + "$crate/target/release/libfuckmark_scan.dylib" \ + "$crate/target/release/fuckmark_scan.dll" +do + if test -f "$candidate"; then + lib="$candidate" + break + fi +done +test -n "$lib" +printf '%s\n' "$lib" +ls -l "$lib" diff --git a/crates/fuckmark-scan/examples/scan_cli.c b/crates/fuckmark-scan/examples/scan_cli.c new file mode 100644 index 00000000..116a8ed6 --- /dev/null +++ b/crates/fuckmark-scan/examples/scan_cli.c @@ -0,0 +1,35 @@ +#include +#include +#include +#include "fuckmark_scan.h" + +static void print_packed_json(uint8_t *ptr) { + uint32_t length = (uint32_t)ptr[0] + | ((uint32_t)ptr[1] << 8) + | ((uint32_t)ptr[2] << 16) + | ((uint32_t)ptr[3] << 24); + fwrite(ptr + 4, 1, length, stdout); + fputc('\n', stdout); + fm_dealloc(ptr, 4 + length); +} + +int main(int argc, char **argv) { + const char *text = argc > 1 ? argv[1] : "a\xE2\x80\xAE" "b"; + const char *language = argc > 2 ? argv[2] : "auto"; + const char *categories = "*"; + uint8_t *result = fm_scan( + (const uint8_t *)text, + (uint32_t)strlen(text), + (const uint8_t *)language, + (uint32_t)strlen(language), + (const uint8_t *)categories, + (uint32_t)strlen(categories), + -1 + ); + if (result == NULL) { + fprintf(stderr, "fm_scan failed\n"); + return 1; + } + print_packed_json(result); + return 0; +} diff --git a/crates/fuckmark-scan/include/fuckmark_scan.h b/crates/fuckmark-scan/include/fuckmark_scan.h new file mode 100644 index 00000000..7f85ed56 --- /dev/null +++ b/crates/fuckmark-scan/include/fuckmark_scan.h @@ -0,0 +1,35 @@ +#ifndef FUCKMARK_SCAN_H +#define FUCKMARK_SCAN_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +uint8_t *fm_alloc(uint32_t size); +void fm_dealloc(uint8_t *ptr, uint32_t size); +int32_t fm_classify(uint32_t codepoint); + +uint8_t *fm_scan( + const uint8_t *text_ptr, + uint32_t text_len, + const uint8_t *lang_ptr, + uint32_t lang_len, + const uint8_t *cats_ptr, + uint32_t cats_len, + int32_t max_findings +); + +uint8_t *fm_clean( + const uint8_t *text_ptr, + uint32_t text_len, + const uint8_t *cats_ptr, + uint32_t cats_len +); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/docs/scan_wasm.js b/docs/scan_wasm.js index 5f01e7cd..4e8efdda 100644 --- a/docs/scan_wasm.js +++ b/docs/scan_wasm.js @@ -52,6 +52,66 @@ function encodeCategories(categories) { return String(categories); } +const SEVERITY_RANK = { info: 0, medium: 1, high: 2, critical: 3 }; + +function codePointCount(text) { + let count = 0; + for (let offset = 0; offset < text.length; ) { + const cp = text.codePointAt(offset); + offset += cp > 0xffff ? 2 : 1; + count += 1; + } + return count; +} + +function utf16OffsetToIndex(text, offset) { + let index = 0; + let pos = 0; + while (pos < offset && pos < text.length) { + const cp = text.codePointAt(pos); + pos += cp > 0xffff ? 2 : 1; + index += 1; + } + return index; +} + +function peakSeverity(findings) { + let peak = ""; + for (const finding of findings) { + if ((SEVERITY_RANK[finding.severity] || -1) > (SEVERITY_RANK[peak] || -1)) { + peak = finding.severity; + } + } + return peak; +} + +function normalizeScanResult(source, result) { + const findings = (result.findings || []).map((finding) => { + const offset = finding.offset != null ? finding.offset : 0; + const length = finding.length != null ? finding.length : finding.codepoint > 0xffff ? 2 : 1; + const index = finding.index != null ? finding.index : utf16OffsetToIndex(source, offset); + return { + offset, + length, + index, + codepoint: finding.codepoint, + category: finding.category, + context: finding.context, + severity: finding.severity, + why: finding.why, + remedy: finding.remedy, + }; + }); + return { + total: result.total || 0, + counts: result.counts || {}, + findings, + truncated: Boolean(result.truncated), + highest_severity: result.highest_severity || peakSeverity(findings), + source_length: result.source_length != null ? result.source_length : codePointCount(source), + }; +} + function writeUtf8(memory, alloc, bytes) { const ptr = alloc(bytes.length); new Uint8Array(memory.buffer, ptr, bytes.length).set(bytes); @@ -107,7 +167,7 @@ function wrapInstance(instance, fallback) { function scanText(text, categories, language) { const source = String(text || ""); if (fallback && hasLoneSurrogate(source)) { - return fallback.scanText(source, categories, language); + return normalizeScanResult(source, fallback.scanText(source, categories, language)); } const payload = callScan(source, language || "auto", categories, -1); const offsets = utf16Offsets(source); @@ -203,6 +263,7 @@ if (typeof module !== "undefined" && module.exports) { module.exports.WASM_CATEGORIES = WASM_CATEGORIES; module.exports.hasLoneSurrogate = hasLoneSurrogate; module.exports.encodeCategories = encodeCategories; + module.exports.normalizeScanResult = normalizeScanResult; } if (typeof globalThis !== "undefined") { globalThis.loadFuckMarkScanWasm = loadFuckMarkScanWasm; diff --git a/editors/browser/scan_wasm.js b/editors/browser/scan_wasm.js index 5f01e7cd..4e8efdda 100644 --- a/editors/browser/scan_wasm.js +++ b/editors/browser/scan_wasm.js @@ -52,6 +52,66 @@ function encodeCategories(categories) { return String(categories); } +const SEVERITY_RANK = { info: 0, medium: 1, high: 2, critical: 3 }; + +function codePointCount(text) { + let count = 0; + for (let offset = 0; offset < text.length; ) { + const cp = text.codePointAt(offset); + offset += cp > 0xffff ? 2 : 1; + count += 1; + } + return count; +} + +function utf16OffsetToIndex(text, offset) { + let index = 0; + let pos = 0; + while (pos < offset && pos < text.length) { + const cp = text.codePointAt(pos); + pos += cp > 0xffff ? 2 : 1; + index += 1; + } + return index; +} + +function peakSeverity(findings) { + let peak = ""; + for (const finding of findings) { + if ((SEVERITY_RANK[finding.severity] || -1) > (SEVERITY_RANK[peak] || -1)) { + peak = finding.severity; + } + } + return peak; +} + +function normalizeScanResult(source, result) { + const findings = (result.findings || []).map((finding) => { + const offset = finding.offset != null ? finding.offset : 0; + const length = finding.length != null ? finding.length : finding.codepoint > 0xffff ? 2 : 1; + const index = finding.index != null ? finding.index : utf16OffsetToIndex(source, offset); + return { + offset, + length, + index, + codepoint: finding.codepoint, + category: finding.category, + context: finding.context, + severity: finding.severity, + why: finding.why, + remedy: finding.remedy, + }; + }); + return { + total: result.total || 0, + counts: result.counts || {}, + findings, + truncated: Boolean(result.truncated), + highest_severity: result.highest_severity || peakSeverity(findings), + source_length: result.source_length != null ? result.source_length : codePointCount(source), + }; +} + function writeUtf8(memory, alloc, bytes) { const ptr = alloc(bytes.length); new Uint8Array(memory.buffer, ptr, bytes.length).set(bytes); @@ -107,7 +167,7 @@ function wrapInstance(instance, fallback) { function scanText(text, categories, language) { const source = String(text || ""); if (fallback && hasLoneSurrogate(source)) { - return fallback.scanText(source, categories, language); + return normalizeScanResult(source, fallback.scanText(source, categories, language)); } const payload = callScan(source, language || "auto", categories, -1); const offsets = utf16Offsets(source); @@ -203,6 +263,7 @@ if (typeof module !== "undefined" && module.exports) { module.exports.WASM_CATEGORIES = WASM_CATEGORIES; module.exports.hasLoneSurrogate = hasLoneSurrogate; module.exports.encodeCategories = encodeCategories; + module.exports.normalizeScanResult = normalizeScanResult; } if (typeof globalThis !== "undefined") { globalThis.loadFuckMarkScanWasm = loadFuckMarkScanWasm; diff --git a/editors/wasm/scan_wasm.js b/editors/wasm/scan_wasm.js index 5f01e7cd..4e8efdda 100644 --- a/editors/wasm/scan_wasm.js +++ b/editors/wasm/scan_wasm.js @@ -52,6 +52,66 @@ function encodeCategories(categories) { return String(categories); } +const SEVERITY_RANK = { info: 0, medium: 1, high: 2, critical: 3 }; + +function codePointCount(text) { + let count = 0; + for (let offset = 0; offset < text.length; ) { + const cp = text.codePointAt(offset); + offset += cp > 0xffff ? 2 : 1; + count += 1; + } + return count; +} + +function utf16OffsetToIndex(text, offset) { + let index = 0; + let pos = 0; + while (pos < offset && pos < text.length) { + const cp = text.codePointAt(pos); + pos += cp > 0xffff ? 2 : 1; + index += 1; + } + return index; +} + +function peakSeverity(findings) { + let peak = ""; + for (const finding of findings) { + if ((SEVERITY_RANK[finding.severity] || -1) > (SEVERITY_RANK[peak] || -1)) { + peak = finding.severity; + } + } + return peak; +} + +function normalizeScanResult(source, result) { + const findings = (result.findings || []).map((finding) => { + const offset = finding.offset != null ? finding.offset : 0; + const length = finding.length != null ? finding.length : finding.codepoint > 0xffff ? 2 : 1; + const index = finding.index != null ? finding.index : utf16OffsetToIndex(source, offset); + return { + offset, + length, + index, + codepoint: finding.codepoint, + category: finding.category, + context: finding.context, + severity: finding.severity, + why: finding.why, + remedy: finding.remedy, + }; + }); + return { + total: result.total || 0, + counts: result.counts || {}, + findings, + truncated: Boolean(result.truncated), + highest_severity: result.highest_severity || peakSeverity(findings), + source_length: result.source_length != null ? result.source_length : codePointCount(source), + }; +} + function writeUtf8(memory, alloc, bytes) { const ptr = alloc(bytes.length); new Uint8Array(memory.buffer, ptr, bytes.length).set(bytes); @@ -107,7 +167,7 @@ function wrapInstance(instance, fallback) { function scanText(text, categories, language) { const source = String(text || ""); if (fallback && hasLoneSurrogate(source)) { - return fallback.scanText(source, categories, language); + return normalizeScanResult(source, fallback.scanText(source, categories, language)); } const payload = callScan(source, language || "auto", categories, -1); const offsets = utf16Offsets(source); @@ -203,6 +263,7 @@ if (typeof module !== "undefined" && module.exports) { module.exports.WASM_CATEGORIES = WASM_CATEGORIES; module.exports.hasLoneSurrogate = hasLoneSurrogate; module.exports.encodeCategories = encodeCategories; + module.exports.normalizeScanResult = normalizeScanResult; } if (typeof globalThis !== "undefined") { globalThis.loadFuckMarkScanWasm = loadFuckMarkScanWasm; diff --git a/fuckmark/native_scan.py b/fuckmark/native_scan.py new file mode 100644 index 00000000..bc62a722 --- /dev/null +++ b/fuckmark/native_scan.py @@ -0,0 +1,246 @@ +from __future__ import annotations + +import json +import os +import sys +from ctypes import ( + CDLL, + POINTER, + c_char, + c_char_p, + c_int32, + c_uint32, + create_string_buffer, + string_at, +) +from pathlib import Path + + +_ROOT = Path(__file__).resolve().parents[1] +_LIB: CDLL | None = None + + +def _library_candidates() -> list[Path]: + env = os.environ.get("FUCKMARK_SCAN_LIB", "").strip() + names = ( + "libfuckmark_scan.so", + "libfuckmark_scan.dylib", + "fuckmark_scan.dll", + "fuckmark_scan.so", + ) + paths: list[Path] = [] + if env: + paths.append(Path(env)) + release = _ROOT / "crates" / "fuckmark-scan" / "target" / "release" + debug = _ROOT / "crates" / "fuckmark-scan" / "target" / "debug" + for folder in (release, debug): + for name in names: + paths.append(folder / name) + return paths + + +def library_path() -> Path | None: + for path in _library_candidates(): + if path.is_file(): + return path + return None + + +def load_library() -> CDLL | None: + global _LIB + if _LIB is not None: + return _LIB + path = library_path() + if path is None: + return None + lib = CDLL(str(path)) + lib.fm_alloc.argtypes = [c_uint32] + lib.fm_alloc.restype = POINTER(c_char) + lib.fm_dealloc.argtypes = [POINTER(c_char), c_uint32] + lib.fm_dealloc.restype = None + lib.fm_classify.argtypes = [c_uint32] + lib.fm_classify.restype = c_int32 + lib.fm_scan.argtypes = [ + c_char_p, + c_uint32, + c_char_p, + c_uint32, + c_char_p, + c_uint32, + c_int32, + ] + lib.fm_scan.restype = POINTER(c_char) + lib.fm_clean.argtypes = [c_char_p, c_uint32, c_char_p, c_uint32] + lib.fm_clean.restype = POINTER(c_char) + _LIB = lib + return _LIB + + +def available() -> bool: + return load_library() is not None + + +def _read_packed_json(lib: CDLL, ptr) -> dict: + if not ptr: + raise RuntimeError("native scan returned a null pointer") + header = string_at(ptr, 4) + length = int.from_bytes(header, "little", signed=False) + payload = string_at(ptr, 4 + length)[4:] + lib.fm_dealloc(ptr, c_uint32(4 + length)) + return json.loads(payload.decode("utf-8")) + + +def _contains_surrogate(text: str) -> bool: + return any(0xD800 <= ord(character) <= 0xDFFF for character in text) + + +def _scan_via_python( + text: str, + *, + language: str, + categories: list[str] | None, + max_findings: int, +) -> dict: + from .product.scan import SCAN_ALGORITHM_VERSION, scan_hidden_characters + + cap = max(len(text), 1) if max_findings < 0 else max_findings + full = scan_hidden_characters(text, max_findings=cap, language=language or "auto") + if categories is None: + selected = None + else: + from .product.scan import normalize_scan_categories + + selected = normalize_scan_categories(categories) + findings: list[dict[str, object]] = [] + counts: dict[str, int] = {} + peak = "" + rank = {"info": 0, "medium": 1, "high": 2, "critical": 3} + for finding in full.findings: + if selected is not None and finding.category not in selected: + continue + findings.append( + { + "index": finding.index, + "codepoint": finding.codepoint, + "category": finding.category, + "context": finding.context, + "severity": finding.severity, + "why": finding.why, + "remedy": finding.remedy, + } + ) + counts[finding.category] = counts.get(finding.category, 0) + 1 + if rank.get(finding.severity, -1) > rank.get(peak, -1): + peak = finding.severity + return { + "algorithm_version": SCAN_ALGORITHM_VERSION, + "source_length": full.source_length, + "total": len(findings), + "truncated": full.truncated, + "highest_severity": peak, + "counts": counts, + "findings": findings, + } + + +def _clean_via_python(text: str, *, categories: list[str] | None) -> tuple[str, int]: + from .product.scan import clean_hidden_characters + + return clean_hidden_characters(text, categories=categories) + + +def classify(codepoint: int) -> str | None: + lib = load_library() + if lib is None: + raise RuntimeError("native fuckmark-scan library is not available") + if not isinstance(codepoint, int): + raise TypeError("codepoint must be an int") + index = int(lib.fm_classify(c_uint32(codepoint & 0xFFFFFFFF))) + if index < 0: + return None + from .product.scan import SCAN_CATEGORIES + + return SCAN_CATEGORIES[index] + + +def scan_text( + text: str, + *, + language: str = "auto", + categories: list[str] | None = None, + max_findings: int = -1, +) -> dict: + if not isinstance(text, str): + raise TypeError("text must be a string") + if _contains_surrogate(text): + return _scan_via_python( + text, + language=language, + categories=categories, + max_findings=max_findings, + ) + lib = load_library() + if lib is None: + raise RuntimeError("native fuckmark-scan library is not available") + text_bytes = text.encode("utf-8") + lang_bytes = str(language or "auto").encode("utf-8") + if categories is None: + cat_bytes = b"*" + else: + cat_bytes = ",".join(categories).encode("utf-8") + text_buf = create_string_buffer(text_bytes) + lang_buf = create_string_buffer(lang_bytes) + cat_buf = create_string_buffer(cat_bytes) if cat_bytes else create_string_buffer(1) + ptr = lib.fm_scan( + text_buf, + c_uint32(len(text_bytes)), + lang_buf, + c_uint32(len(lang_bytes)), + cat_buf, + c_uint32(len(cat_bytes)), + c_int32(max_findings), + ) + return _read_packed_json(lib, ptr) + + +def clean_text(text: str, *, categories: list[str] | None = None) -> tuple[str, int]: + if not isinstance(text, str): + raise TypeError("text must be a string") + if _contains_surrogate(text): + return _clean_via_python(text, categories=categories) + lib = load_library() + if lib is None: + raise RuntimeError("native fuckmark-scan library is not available") + text_bytes = text.encode("utf-8") + if categories is None: + cat_bytes = b"*" + else: + cat_bytes = ",".join(categories).encode("utf-8") + text_buf = create_string_buffer(text_bytes) + cat_buf = create_string_buffer(cat_bytes) if cat_bytes else create_string_buffer(1) + ptr = lib.fm_clean( + text_buf, + c_uint32(len(text_bytes)), + cat_buf, + c_uint32(len(cat_bytes)), + ) + payload = _read_packed_json(lib, ptr) + return str(payload.get("cleaned", "")), int(payload.get("removed", 0)) + + +def main(argv: list[str] | None = None) -> int: + args = list(sys.argv[1:] if argv is None else argv) + if not available(): + sys.stderr.write( + "fuckmark native-scan: library not found; run crates/fuckmark-scan/build-native.sh\n" + ) + return 1 + text = args[0] if args else "a\u202eb" + language = args[1] if len(args) > 1 else "auto" + payload = scan_text(text, language=language) + sys.stdout.write(json.dumps(payload, ensure_ascii=False) + "\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/fuckmark/webui/scan_wasm.js b/fuckmark/webui/scan_wasm.js index 5f01e7cd..4e8efdda 100644 --- a/fuckmark/webui/scan_wasm.js +++ b/fuckmark/webui/scan_wasm.js @@ -52,6 +52,66 @@ function encodeCategories(categories) { return String(categories); } +const SEVERITY_RANK = { info: 0, medium: 1, high: 2, critical: 3 }; + +function codePointCount(text) { + let count = 0; + for (let offset = 0; offset < text.length; ) { + const cp = text.codePointAt(offset); + offset += cp > 0xffff ? 2 : 1; + count += 1; + } + return count; +} + +function utf16OffsetToIndex(text, offset) { + let index = 0; + let pos = 0; + while (pos < offset && pos < text.length) { + const cp = text.codePointAt(pos); + pos += cp > 0xffff ? 2 : 1; + index += 1; + } + return index; +} + +function peakSeverity(findings) { + let peak = ""; + for (const finding of findings) { + if ((SEVERITY_RANK[finding.severity] || -1) > (SEVERITY_RANK[peak] || -1)) { + peak = finding.severity; + } + } + return peak; +} + +function normalizeScanResult(source, result) { + const findings = (result.findings || []).map((finding) => { + const offset = finding.offset != null ? finding.offset : 0; + const length = finding.length != null ? finding.length : finding.codepoint > 0xffff ? 2 : 1; + const index = finding.index != null ? finding.index : utf16OffsetToIndex(source, offset); + return { + offset, + length, + index, + codepoint: finding.codepoint, + category: finding.category, + context: finding.context, + severity: finding.severity, + why: finding.why, + remedy: finding.remedy, + }; + }); + return { + total: result.total || 0, + counts: result.counts || {}, + findings, + truncated: Boolean(result.truncated), + highest_severity: result.highest_severity || peakSeverity(findings), + source_length: result.source_length != null ? result.source_length : codePointCount(source), + }; +} + function writeUtf8(memory, alloc, bytes) { const ptr = alloc(bytes.length); new Uint8Array(memory.buffer, ptr, bytes.length).set(bytes); @@ -107,7 +167,7 @@ function wrapInstance(instance, fallback) { function scanText(text, categories, language) { const source = String(text || ""); if (fallback && hasLoneSurrogate(source)) { - return fallback.scanText(source, categories, language); + return normalizeScanResult(source, fallback.scanText(source, categories, language)); } const payload = callScan(source, language || "auto", categories, -1); const offsets = utf16Offsets(source); @@ -203,6 +263,7 @@ if (typeof module !== "undefined" && module.exports) { module.exports.WASM_CATEGORIES = WASM_CATEGORIES; module.exports.hasLoneSurrogate = hasLoneSurrogate; module.exports.encodeCategories = encodeCategories; + module.exports.normalizeScanResult = normalizeScanResult; } if (typeof globalThis !== "undefined") { globalThis.loadFuckMarkScanWasm = loadFuckMarkScanWasm; diff --git a/tests/test_language_bindings.py b/tests/test_language_bindings.py new file mode 100644 index 00000000..6e24a211 --- /dev/null +++ b/tests/test_language_bindings.py @@ -0,0 +1,106 @@ +import json +import shutil +import subprocess +from pathlib import Path + +import pytest + +from fuckmark.native_scan import available, clean_text, library_path, scan_text +from fuckmark.product.scan import scan_hidden_characters + + +ROOT = Path(__file__).resolve().parents[1] +VECTORS = ROOT / "specs" / "fuckmark-hidden-scan-v1.vectors.json" +CRATE = ROOT / "crates" / "fuckmark-scan" +NODE_BIND = ROOT / "bindings" / "node" + + +def test_c_header_and_binding_docs_exist() -> None: + assert (CRATE / "include" / "fuckmark_scan.h").is_file() + header = (CRATE / "include" / "fuckmark_scan.h").read_text(encoding="utf-8") + assert "fm_scan" in header + assert "fm_clean" in header + assert "fm_classify" in header + assert (ROOT / "bindings" / "README.md").is_file() + assert "FUCKMARK_SCAN_LIB" in (ROOT / "bindings" / "README.md").read_text(encoding="utf-8") + example = (ROOT / "bindings" / "README.md").read_text(encoding="utf-8") + assert "-Wl,-rpath,\\$ORIGIN" in example + assert "/tmp/fm-scan" not in example + assert "crates/fuckmark-scan/target/release/fm-scan" in example + + +def test_native_library_matches_python_vectors() -> None: + if not available(): + cargo = shutil.which("cargo") + if cargo is None: + pytest.skip("cargo is not available") + subprocess.run( + ["sh", str(CRATE / "build-native.sh")], + check=True, + timeout=180, + ) + assert available() + assert library_path() is not None + payload = json.loads(VECTORS.read_text(encoding="utf-8")) + for vector in payload["vectors"]: + text = "".join(chr(code) for code in vector["codepoints"]) + native = scan_text(text, language=vector.get("language") or "auto") + python = scan_hidden_characters(text, language=vector.get("language")) + assert native["total"] == python.total, vector["id"] + assert len(native["findings"]) == len(python.findings), vector["id"] + for left, right in zip(native["findings"], python.findings, strict=True): + assert left["index"] == right.index, vector["id"] + assert left["codepoint"] == right.codepoint, vector["id"] + assert left["category"] == right.category, vector["id"] + assert left["context"] == right.context, vector["id"] + assert left["severity"] == right.severity, vector["id"] + + +def test_native_empty_categories_and_clean() -> None: + if not available(): + pytest.skip("native library is not available") + text = "a\u202eb\u200bc" + empty = scan_text(text, categories=[]) + assert empty["total"] == 0 + cleaned, removed = clean_text(text, categories=["bidi_control"]) + assert removed == 1 + assert cleaned == "ab\u200bc" + + +def test_node_binding_assets_and_tests() -> None: + node = shutil.which("node") + if node is None: + pytest.skip("node is not available") + subprocess.run([node, str(NODE_BIND / "sync-assets.js")], check=True, timeout=30) + for name in ("scan_wasm.js", "fuckmark_scan.wasm", "scan.js"): + assert (NODE_BIND / name).is_file() + completed = subprocess.run( + [node, str(NODE_BIND / "test.js")], + check=True, + capture_output=True, + text=True, + timeout=30, + ) + assert completed.stdout.strip() == "ok" + + +def test_native_scan_reports_lone_surrogates() -> None: + lone = "\ud800" + payload = scan_text(lone) + assert payload["total"] == 1 + assert payload["source_length"] == 1 + assert payload["truncated"] is False + assert payload["highest_severity"] + assert payload["findings"][0]["index"] == 0 + assert payload["findings"][0]["category"] == "surrogate" + assert payload["findings"][0]["codepoint"] == 0xD800 + mixed = scan_text("a" + lone + "b") + assert mixed["total"] == 1 + assert mixed["findings"][0]["index"] == 1 + assert mixed["source_length"] == 3 + cleaned, removed = clean_text(lone) + assert removed == 1 + assert cleaned == "" + kept, unchanged = clean_text(lone, categories=[]) + assert unchanged == 0 + assert kept == lone diff --git a/tests/test_scan_wasm.py b/tests/test_scan_wasm.py index a7db98b4..01e5d113 100644 --- a/tests/test_scan_wasm.py +++ b/tests/test_scan_wasm.py @@ -179,6 +179,13 @@ def test_wasm_routes_lone_surrogates_through_js_fallback() -> None: const scanned = api.scanText(lone, null, "auto"); if (scanned.total !== 1) throw new Error("scan total " + scanned.total); if (scanned.findings[0].category !== "surrogate") throw new Error("category " + scanned.findings[0].category); + if (scanned.findings[0].index !== 0) throw new Error("missing index " + scanned.findings[0].index); + if (scanned.truncated !== false) throw new Error("truncated " + scanned.truncated); + if (!scanned.highest_severity) throw new Error("missing highest_severity"); + if (scanned.source_length !== 1) throw new Error("source_length " + scanned.source_length); + const mixed = api.scanText("a\uD800b", null, "auto"); + if (mixed.findings[0].index !== 1) throw new Error("mixed index " + mixed.findings[0].index); + if (mixed.source_length !== 3) throw new Error("mixed source_length " + mixed.source_length); const cleaned = api.cleanText(lone, null); if (cleaned.removed !== 1) throw new Error("removed " + cleaned.removed); if (cleaned.cleaned !== "") throw new Error("cleaned leftover");