From 6050f182f4da3be02e8c272f38df14114985adeb Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 2 Sep 2026 02:44:50 +0000 Subject: [PATCH 1/2] Add C, Python, and Node bindings for the Rust scan ABI Expose fuckmark-hidden-scan-v1 through a C header and host cdylib, an optional Python ctypes loader, and a Node package that wraps the committed WASM engine with the JS fallback. Co-authored-by: Byte271 --- .github/workflows/ci.yml | 18 + CHANGELOG.md | 1 + README.md | 10 + bindings/README.md | 57 ++ bindings/node/README.md | 21 + bindings/node/fuckmark_scan.wasm | Bin 0 -> 45182 bytes bindings/node/index.js | 56 ++ bindings/node/package.json | 34 ++ bindings/node/scan.js | 569 +++++++++++++++++++ bindings/node/scan_wasm.js | 209 +++++++ bindings/node/sync-assets.js | 18 + bindings/node/test.js | 24 + crates/fuckmark-scan/README.md | 18 +- crates/fuckmark-scan/build-native.sh | 21 + crates/fuckmark-scan/examples/scan_cli.c | 35 ++ crates/fuckmark-scan/include/fuckmark_scan.h | 35 ++ fuckmark/native_scan.py | 178 ++++++ tests/test_language_bindings.py | 80 +++ 18 files changed, 1376 insertions(+), 8 deletions(-) create mode 100644 bindings/README.md create mode 100644 bindings/node/README.md create mode 100755 bindings/node/fuckmark_scan.wasm create mode 100644 bindings/node/index.js create mode 100644 bindings/node/package.json create mode 100644 bindings/node/scan.js create mode 100644 bindings/node/scan_wasm.js create mode 100644 bindings/node/sync-assets.js create mode 100644 bindings/node/test.js create mode 100755 crates/fuckmark-scan/build-native.sh create mode 100644 crates/fuckmark-scan/examples/scan_cli.c create mode 100644 crates/fuckmark-scan/include/fuckmark_scan.h create mode 100644 fuckmark/native_scan.py create mode 100644 tests/test_language_bindings.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 14bb931d..9b94e24a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,9 +180,27 @@ jobs: with: targets: wasm32-unknown-unknown + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.12" + cache: pip + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: "22" + - name: Test crate and build wasm32 run: | cargo test --manifest-path crates/fuckmark-scan/Cargo.toml cargo build --release --target wasm32-unknown-unknown --manifest-path crates/fuckmark-scan/Cargo.toml test -f crates/fuckmark-scan/target/wasm32-unknown-unknown/release/fuckmark_scan.wasm + - name: Build host library and language bindings + run: | + ./crates/fuckmark-scan/build-native.sh + python -m pip install --upgrade pip + python -m pip install -e ".[dev]" + python -m pytest -q tests/test_language_bindings.py tests/test_scan_wasm.py + diff --git a/CHANGELOG.md b/CHANGELOG.md index 29cc3eff..588a65d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ - Zero-install hidden-Unicode scan page: `docs/scan.html` (packaged as `fuckmark/webui/scan.html`, engine copy `scan.js`). `fuckmark web` serves it at `/scan.html`. After deploy: `mark.q1z.org/scan.html`. Scanning runs in the browser; a local Python `/api/scan` confirm is optional when the CLI server is up. - Chromium browser extension under `editors/browser`: popup scanner, on-page reveal (badges plus CSS highlights), context-menu scan, per-tab toolbar badge, and optional paste-safe stripping that keeps real emoji ZWJ sequences (both neighbors emoji) and stops page paste handlers from seeing the raw clipboard. Zero remote code; `scan.js` matches the VS Code engine copy. Load unpacked from `chrome://extensions`. - WASM build of `fuckmark-hidden-scan-v1` (`crates/fuckmark-scan`, committed `fuckmark_scan.wasm`). The scan page and the Chromium popup load it when `fetch` works so classification stays on-device; `scan.js` remains the `file://` fallback. Lone UTF-16 surrogates route through the JS engine (UTF-8 cannot carry them). Empty category arrays stay empty selections. The extension CSP allows `wasm-unsafe-eval` so Chromium can instantiate the module. Page reveal and paste-safe keep the synchronous JS port. Rebuild with `crates/fuckmark-scan/build-wasm.sh`. +- Language bindings for the same Rust scan ABI: C header (`crates/fuckmark-scan/include/fuckmark_scan.h`), optional Python ctypes loader (`fuckmark.native_scan`), and a Node package under `bindings/node` (WASM + JS fallback). Host library build: `crates/fuckmark-scan/build-native.sh`. Overview: `bindings/README.md`. - `fuckmark normalize` (`fuckmark-normalize-v1`): NFC, optional identifier lookalike fold (UTS #39-inspired subset, not a full confusable map), then strip of the security category set, plus a JSON receipt (`input_sha256` / `output_sha256` / `steps` / `report_hash`). `--clean` remains the blunt strip with no receipt. HTTP: `POST /api/normalize`. Python: `normalize_text`, `skeleton_fold`. Reference: `docs/normalize.md`. - VS Code / Cursor: `fuckmark.cleanOnSave` (default off) strips hidden Unicode on save. - Frozen confirmation files, hashes, mix mechanisms, and the published wheel are unchanged. Package version stays 0.4.1. diff --git a/README.md b/README.md index 30f4d261..ead6c83a 100644 --- a/README.md +++ b/README.md @@ -250,6 +250,16 @@ Eligible Latin, Greek, Cyrillic, Han, Kana, Hangul syllable, and emoji clusters Frozen evidence, hashes, and protocols: [`docs/research.md`](docs/research.md). Product contract: [`docs/product-contract.md`](docs/product-contract.md). CLI: [`docs/cli.md`](docs/cli.md). Limits matrix: [`docs/limits.md`](docs/limits.md). Demo: [`docs/demo.html`](docs/demo.html). +## Language bindings (Rust scan ABI) + +The hidden-scan classifier also ships as a small Rust crate with a C ABI, an optional Python native loader (`fuckmark.native_scan`), and a Node package (`bindings/node`). Overview: [`bindings/README.md`](bindings/README.md). + +```text +./crates/fuckmark-scan/build-native.sh +python -c "from fuckmark.native_scan import scan_text; print(scan_text('a\u202eb')['total'])" +cd bindings/node && node sync-assets.js && node test.js +``` + ## Development ```text diff --git a/bindings/README.md b/bindings/README.md new file mode 100644 index 00000000..f2746c08 --- /dev/null +++ b/bindings/README.md @@ -0,0 +1,57 @@ +# Language bindings + +The classifier core lives in `crates/fuckmark-scan` (`fuckmark-hidden-scan-v1`). +Python remains the reference implementation in `fuckmark.product.scan`. These +bindings call the same table through the shared C / WASM ABI. + +| Binding | Path | Engine | +| --- | --- | --- | +| C / FFI header | `crates/fuckmark-scan/include/fuckmark_scan.h` | host `cdylib` | +| Python (optional native) | `fuckmark/native_scan.py` | host `cdylib` via ctypes | +| Node / JS | `bindings/node` | WASM + JS fallback | +| Browser / extension | `editors/wasm`, `docs/scan.html` | WASM + JS fallback | + +## Build the host library + +```text +./crates/fuckmark-scan/build-native.sh +``` + +That prints the path to `libfuckmark_scan.so` (Linux), `.dylib` (macOS), or +`fuckmark_scan.dll` (Windows). Override discovery with `FUCKMARK_SCAN_LIB`. + +Categories ABI: pass `*` for every category, an empty string for an empty +selection, or a comma-separated list such as `bidi_control,tag`. + +Packed return values begin with a little-endian `u32` length followed by UTF-8 +JSON. + +## Python + +```text +./crates/fuckmark-scan/build-native.sh +python -c "from fuckmark.native_scan import scan_text; print(scan_text('a\u202eb'))" +``` + +If the library is missing, `available()` is false and callers should use +`fuckmark.product.scan`. + +## Node + +```text +cd bindings/node +node sync-assets.js +node test.js +``` + +## C example + +```text +./crates/fuckmark-scan/build-native.sh +cc -I crates/fuckmark-scan/include \ + crates/fuckmark-scan/examples/scan_cli.c \ + -L crates/fuckmark-scan/target/release -lfuckmark_scan \ + -Wl,-rpath,\$ORIGIN/../../crates/fuckmark-scan/target/release \ + -o /tmp/fm-scan +/tmp/fm-scan $'a\u202eb' +``` diff --git a/bindings/node/README.md b/bindings/node/README.md new file mode 100644 index 00000000..9a0bfe07 --- /dev/null +++ b/bindings/node/README.md @@ -0,0 +1,21 @@ +# @fuckmark/scan (Node) + +Node binding for `fuckmark-hidden-scan-v1`. Loads the committed WASM module and +keeps the JS port as a fallback for lone UTF-16 surrogates / `file`-style +environments. + +```text +cd bindings/node +node sync-assets.js +node test.js +``` + +```js +const scan = require("./index.js"); + +const result = await scan.scanText(source, null, "javascript"); +const cleaned = await scan.cleanText(source, scan.fallback.DEFAULT_SECURITY_CATEGORIES); +``` + +Assets are copied from `editors/wasm` and `editors/vscode/scan.js`. Re-run +`npm run sync` after rebuilding the WASM module. diff --git a/bindings/node/fuckmark_scan.wasm b/bindings/node/fuckmark_scan.wasm new file mode 100755 index 0000000000000000000000000000000000000000..5c459b0fc27996ab930f71e1a9b7eb17ea246282 GIT binary patch literal 45182 zcmdVDdw?BBedpWN-RCjqoRN;?u_bF{sn4+%BrMsZ*NkKcs0|hfJHTBMZ^AXP#?r`| zhi0Uikt~CQ#*P64CYS_*F@{KBz)4JS5{K{*L~P7scOh|do#5RiK?Y|-E-X>9aN`Ao z_w)T#_j!y)#%BMx_sTxg-Szm@uYRxk)vv0$gW;J2E(ij5w7xq$b}ZN(9Xoc+9n(F3 zo&V!ggWU!4R3`4YP^xIjq?i4Tij=b3_8kR-ATP^<2j-4NA9WvgQ4qw1uo#qrLR5(3 zAS{=|m@7}kf(ygI6$)TkEEI~RxLAmLVv1A1Ij1sVPce!r#cEh6REm{i7zI(S92Ux< z>QF8SV^`unj_iSqC=LSU(q<3>D(sDe5=}Tj6kH()9Sw(JxF}rgN(C21v=kTILAQE! zA#%Zz;)T&Mm(InUAJyUWD`BL}9YZwiC*z5~|{PfSeh30_&@cJGLP zcvY2$Jrlz-Go$-%3jU80PcwUlCxd^h@_QymxP5iyy6g51&kkQVGPyT$)$6X?KQesq zy6cB$MxwB1y$e!TUG*N<*I0ets&~4H*IvK-+SmN@O-F7#`YVTLX0MxbZ*gyR$8K?V zx!-b?&)yq;*!?y59}9Qrza&^2)|#$2NP|YGhjPxMWN`*9H8=mkwb|9+Kk3hH)a5vaS|tPJc+LA2Vyk|7gs3@ zko4G$Z`@Z56GyeKTH!cNHJ$1uQ3^Kw)xFLgyLgU&L7I#=!!>rBglU<^__HnyhN2*$ zm-$^ZW%8>*GzA^&i%WJua<{)!%Ntada%}%^l7abDB{n%Ig4| z&ZvXFGof|orN-4_Eu?k2Qyy!U)PAheMcBqo+$^g#kQd)e0EtB6u>M9hjaYzrkWdCv zsf?J{!e*tH6QiZ?pD?KZhV^bB00~6nmAx=qjn&_F3m}milbCT25(Yv`$!GOO7v^1? zFAji(BETL`=|=r_BTJa}NgR-~OccU(QF$orgIRqDN&PKWN|KaBnYh4!bz$F-7}1uH z4l#tJaYoWue0B(Gm_+s8HBgYN+=%Fv=ti%Mk~QCt$Ozja4SHeZt<`VyoqlbIJ$^$b z?KG21JG5!_t^Q$;@jAE|N$x>i(81t}VTY67g~&X+;!0|7Yl$4hIKIk z9-H-uBvo%ybrfN4?x6q?p&hUESCEcTggi9|{wcDsR@FgpJdMVhaO#M38hPB+4CSd_lWcX8U6^W!LQ$8y)iz8Ej~q_PeP-m%2&2K*{LsL za8Q4bL4la%hNmg!v8^}|OTA5k{_1bKF172Tl@tMl%VrOx2pO$3T09hB9bmrMT$lr@ zM!Z%}$|8~{QT-#rMv@KzykUemJRs_7ATW#2p-C_4!pY@oZLKubl$iu;TY4%BSpRk3 zaxBhas!^vkmoX#>-YsbP95$m|#+(R~7*4pIL_W9eP? zaQ`hN)!!vpLG>KA<(<~G=RmF$i)R=_o(F;Cb^|55|EWG zZ8qz|Q*LXL(1uOa9X+QfPTgR9jgms9rRe^0)RRkJqXfp>AU!a&%&$ko6)AC}LKSXS zHp_{|T;Pr;rLks@8ZR+`g^X5}`g_FX=(EcPX!*UdgREgcf&t48)p6(C|y*HdgylEfC0^rAFw zof>QOh}=C%IU5OGB5RMWY#Y22GV6v^klrZ!yPUxkK?>8~o||L1#~L5%MYPNnOy}IL zUc?o8qI^f>7V?Ct7?9MuG}+lJ6}jN0y^$n-!_G!ni$EcrJ00jAst4&)?oj&GPn?;*nMY zjvEQJc7n5nrAI{JGpZ7#al;AMPB^V)3*s4k7DBP=`G_K<1&lm{D9gdS2vjktz- zD`}RmqhYN29HU?o83mEXxc&iI2yfZ37x(IEMfF#@2Bx-}$dNLG>>U9(H7)=!_I?|V(Mzk? zHw|XO_8szw$)dsB{D_QA0r`o$@t;0H8lfJEkwDmJXej)QHv#v?zsTw~{H3YbWV zSP^B8xH)=szy2X(hWP}Ve5e%VGFLom8@&6Ri}H@NTNHbsA+`y6dYB?8B=vqd3bovP z0dD$KgfXL0kn)?dt!#D8Pi+>~$R-07a`qvw(V&L`kWLo7ImmRf<%@pX&2#88q8qLl ziZKL+$wM@L=wpfd*c+O~E0Wtk3U0-ZLMEs&ca%Jab$aHf5=9hJ9R{pBwCbh&%$#9+KKF3{Z~{lT{o?rHD}uxCzX_K6e(L?+VTm7B$!#t1+f zw5eIcMi{Y;bX0?ee}9q-r2iOk-T`ecdcqi~5SZVP3`>bSVkx1iUxb>kDxfwoOmo zdJ0#Ij@3UcDnUBLU*2{4_-*9KuS>sTWkB1Pp-9bpCML)(z57p(jV-DW;VwA)M|WKM zHzcNoUAXivee>MSV`+rHoi0_y^ic!2GzX|gANH*wuj%LfE%grKnLmk(EBA5kCodht z>E6{F!z3P$kvF%EKBb@W<#DT{TA`YLyjA^y+h{fc1>sAbpF9;E(jNeN7=r(RCh6%% zG8jD`#$DOtBCGeI&oNz#)6!M=U&NM}t<>Ub`d@sRYFeQVYUa{A|L9k(n{w=)I6}Y$ zW5Vs{w)$D0OW$xPz3-G<&-D1O8FAI!cV=ystS!)0om(JYL$>IFV_6Ki&}3lnu~M6Z z UM7$!~W+cGFcpv38I?M$eSw^&WBd?QS&*38@~NHe#|H#Rr-k}=Um?O8G01McQ_ z_|$$c575vC13!4m)>MDpwcRj=xR(8wCCURHp9ZxMV&)|QkzhUB>d!9*9y|0Y&WIk=%6*Sxvu&Li=P|O?DcxSV!`aM2T{-3#i z%V8q9eRu<=WD(w05-Q4HP?Q20*gzz}rP3lzeOK&C3$sZ!p;W)E!5;8b&f7@iql}qa@DZY?yXG+>HY6~=Al+jCmU^DXV~R3;6( z(oT1$p&Y*cH!Z!p{&y`c>Z<=w-D&B+=oz#LEm)iOjANPOnWz5`tJg}mvfsC~Zv6hA zZunO2e=d;rq@}g!VdduDbasDlT_Ei?OY5$C`vT?OYH8j5f4ij>TK)SCJ=6bU)_;3W zi+O$`-rQw%y8C&zrFFx8zom8Kbz*_KA6lT?M;CziyO!3%H4!c!x3p>(exJ6q?sh+C zY2EmJ-qO1J@n;Lv{Ze;Y`k(adMLQ7=_S}|bOZy9})7{=*b*H6&ZO^6&R>wbQb( zO=#(X8_d1*I}3E!M45SvKcYE-g67hh474N!CH{R8x$kkwq*LjJ!;D zdeQ6y|U?{tYZChW^kD#^)cXascY^dBrvU5 z*!o+2e!51JVe*q^Sidi?vy$nMd_o`grQZx)s;1vn{e*u@un1qRQ5AXe|B_1L6p!=G zr1x7-J}H>yib2P+L6rmGp_8ORtVsY%5t)LoT1!uMRn?kD+$d{0hu=N`Nu&Co3hb45 zSA}$7Ed7d=e=tiERQbRe9*iK5biqq6#{&X$j|9$@TF=^c>FsY3M{~5(%FcGyUENu| zK5Gp$P4Ba;PYHw6;!=Kawt!RtB2(m(9YqvhOPD$lBu1Qiayl3$^1r{%l2!?)1s*V= zBFetAKMTP#BL&s9;hMqlW&Un$6A$4f+;iEXi?Y`fbGteb4Ne7P^*0MIqeuOZv)hN; zD?Y{plv@#gV&V+U9v+ikc#vXbjb-eD9^GWsxap(BHM;TAUIQ ztx*&GL^bkIJl^X+z>UIp9d(HYD=oxk5L$bD#tSqDRyUtB# ziqQ2Ja0wE5o3Fo$Kk-zgi6On| zL&lA$&5fGpM)_HBgFy4sb3?O*oEw?0WBfrePMfk*ar#_;ku%PamQm!v4vLtbdQ~Q7 z*(htGQ`L;p>n3Q+g~xtZe_mr9h7#}$6f9n$*N)q{<42ikT%g64AbtBO^jbG_4`P*0 z*_80hG7D)X3HT&Z#?x>aL8Qc6|`d* z*qRS8kKcl5Lj9nALU-B~l)h>#t?bhLq55#tHDf5RF-gYWSZo!uqlgNdPc)q&f}%iZ zLGv-1U_jM`_V%}ey#>xZg^T_YtAuQD#4y2@Bi80@45%{aC@D`?9}u$oS}$QNA-`E{ zl;98f<@AhMOS3mkg<8w$6YUh_sY*&2%BuaASxSE>T}i*Mzw%R}w3M9p z3H9WnnQ^187%Njp?Q;5vwN8-DI8#XPX_HjKWc(;}61pSx-p;Zp(ET0H=4kloeOk&K zVjFCIh$5|ZrC9_b?a9;92Q#b?iJHbdG5b^-y5`4U_C*wouv=f}tfI}hpIK&%AaGMs z=})U-XX&rB0naOK3r%sxl6?B6aA%y!THZ6CW|G&Mo}i@G6k%lMoh7jiRU#(kX=iP)_6t4T#8Y6TUr|+Dyz_FNo=cgK_6Lex2dl&sdTnNAkHO(Bkky*v@%G4 zPH=^>ckcwLbIg6Z|J*L0si>gciEvx;7UvJeWb+s?|5ud1Xr?YZj ze0x^T)+P+~4rLPAjgcLQGh|tJmXD=hZ_BT7_j2FNhu_xf%;&Ra9#=_hhE`C6w9l9O zgF9N~{wgc?4PRAKKsnVGC%*a4R>7PTZ}UkOND9*bkQMxK8{#%ER$uTb1d~eb3F4R- z!U&M*Z#G4yvneB0mSu(@S1DyN?qykCk2`9y>otwA@&ajQjU`))l=N4hD!59nVRNn{ zSO+$VYlp2Am0F|Cr2{BA_em2=%Z4hNjMDXE=>zY91b)B~n#`&qEepONiLQ{&YLng; z%4!vh9QS;;SH@y6-0jOPYpz#+zg!9tX*m!bZi((!T66%1kuaXBHLLQci5$O@ zdhD(tvpWdN+Hi|Jec@LB17#>_n`X8Y z)$ZqJP4$_12pYKxGmJH{^Dn^dWOk+n43f{?Q4s?FhNU*@tJ8CCY>l3S>UTPPy)#4% zTG4841^lF>8m44nT8%gC5=#&a=0GdGRCxi){{oxsg>MsRn1AEgqCzodpB;@ldDjg7 zHi^~zOFrw)UV7=Y8iChAi%8FL)s>{_IsKGjknJyF9mA)7rYqIhNZHSJrD}*4gg@;{ zMeZ&2^IfUXC$hCcR91Rsb3G(X6BoOn_34*Y1*P{X+b9+>KGPp31mqV z>wS<2MPo0Vx|0U1K>K|dBtW9uT~cm!eolIQOuv^aL{wfIzO0F4uMMwm)+|^Q>){2B zUR`!H7U{C3u~?UZMqQWnUUYXhmSF6bCe_AywB5(=d?q>shw$pS%c?MVrfMH-J7!;C!rNl64;p&e$l*yr0nFIlFi zWkADJS7ii~73rBbLDXbLf5N_YU4+a-nWUl&JPN0JbEPhQ&5P<^$THtWCzZQ0xky>% z<_cYE&HnlqbJ8Doja5p#$2C^#dY5Y?y1vUb8p(>}0)GAVFInjelJouD%bTVC#sx`# za(=>%1hbSZmjKXB)MRZqmoXX3TmN|8(rvD>bbfD_5&#^4)eLbmHjrO?yj`+t5QE?n zT*ZV-A}8M^4H;yY5OaZA$qG#XTyl}TGMB8B1JBqhNg?hmNhvG1Xd7}-(FJRD!w~o-2?fcMUVmsOCOd&&bJEuR@k254s-^-Eu`>qkJFIfTd4#v4L%kf_`-*o=Xira#4g@yYCuN=3f;cMr)Y!Bbg@9 z&TE~jSme)z3?PWO#b$B$)^o>YEl8}znBZTUCqRgh4Ji_HJNrZVY^$A32}Ln_QGjf1 zf@$WPQAR=6HNsDE9F=N~qBsVRRNGTXIk;_;AGd5gNU_bi{vQPblH($sw&5`!_$}!% zOX|ypqcX$zSo#A?TUP%Mt(p?`-TJe)vtYI&WN_}d3$CS0IZFzGB89q4L|)6~&xRNe8>DIu zTvoI;VT<2pzu0YWo_JA)RebrcR(?^t{2FWpGEKvY4TFVDKF!**Y$dx+z4e(}g;BTX ze>6CmTjj;e(z_Mg{ATuer?jn%e5Fn8=G+EQP7gS z9~0=-xtOwo)KWU{;22pn`-2TKbFUhbg8*Sg_?O@$eK6}SwK`>e?!A)rS}WUnj%m<9 z7e6aOxx*l}1jW-(A_t^rCvxvu7$+h}43y6brwY(GlB zBGl_|zlB^pcXHWwk82QUO^DGlV!0v_h_73@_ZfXd$dfe@SGTpfLenlwmm5vffp0g_ zKC6bnv?>PGmvvRla0WvyL%>el6S=+EG`7*|NqY=(vn&TP2UV@8ozDuh>NmP&q!m2Y z0$-3i4s`P;=m!z%^w)n=-41uqsFY4_xo+0#&-%i_z2Epv*dFT8|GAzUfT9yEeIi!z>%JVw=ePcMF*TY0-o~N&TzVQ*N^cbBd@! z)_1{g-R2ZokP_@V&*eU-o`?8KGwT1{h)iH`admN^6g=GTDQ$6eSyo2Yss2w^4x_NR zUH448F2oVECwnqQM(4txO`nN!NF)<_6pGg|w{H*^;8y_)tH zThpFbx<@Tw$AaoVgk8I2TQ-lzhz;s&BBq9tH6s~GPj^UhvCLx15kxzGC|lWeWO0!t zbR1diCD?P;X~R}2JE3?Mklr++4j#0C=cgmt^x&Ja6N&`1Sg6m)B}%caeLO!_rs$98 z_Fs)(%CxP+B9Qhh$Qa}Y6cbpJ*g16C^Wd~Wo;unejD>$pFeLR6lSW&p%fJv6*U;4( z5IB-|+I4NrbuGacI0W$+AiG6WHhxy?@Cp`yT@u@QyfUqo?at2UP$?HIV^L@VY)4R0 zDIcGp2}y726#chU!WBqw${4~8o`Xp*UDT-PBS_kFtjBr|T@Tqo885Z~mk74R_LXfB zY!Th|5QvO);+#YWqx!wKK*|+HaXXO+#XjQe&CQ{kStj;q?^b=F(wbW}36D-uiC(8|xI9d+I zfiYcrb!(Iz=zwGy;wX=HWr#M*^CJq#t(T>=FRd%5kIe+O+rqJi&JvI<@kxT|f_Fav zU-k{+Wd?|tb{()2GT~HC;*cbgC3*6k`gn6pLtcuYyBsj0Y0($1Yy$y3HN2AeVEXM( zVsr?4iE6ePCQ9omR!_rNu(KbSVm(VM{!l^oC`?m8+;)=M=^G5%jfV#Pb4G6^!Ymmh z@bNuCCd;e|THU#=^49nO}j`v@CgmP2ssX3IlU1Wp-?tJUMN z&Y&O|#7)Hm6^)zdsy`4J9@4~|G{Uqh<5VJv#yLSp;+;ZBY7QCWl(QKe#gLCi+qWwV zG%V*0Aj`_Utje_R3_HIv%Lx0pSqL?52&e`KghC)xEAJ90zlN8$xs!d9^g2Cyi0HA^ z7givsLC%^`kAz1cp=)8@K3YgLprYl$QF?!O65DseP~?fg%C3ZyccsR84K z=th2YWVR{#K?!gI9m`C@%qGj6nr${G8S0AD^wnP^x?;GJt>>QicNjg@LBip7-zMk0 z9%?8u@~b96bCu2^+JlGaIpiVw`woaX^^lOi2+>lhS(_f|YC?{jntkdit-^B0u{Yie za6>a3*LwqCo0}a^UkG^+y(?HyMl;mEmE^QE$PN-gIj<2e;k8Gx37vXQ_hn2JtK~`L z;X-r zjbRCzI3Wx%+5rg@DFC|=)DCE@T=Hmxn^#R?BKX6yrL*q0RYPN2W@I#K;xQLl{( zIAV{%3jD@0OcJt;Kt)~$y#7Hg(?8i7{64aS5TQ&$Er7SB8d7!ZA1HZ_+6I6)lW{fX zl_N6jB+J;PXZkaQ`Q+NXVW&uQ9OcV3RnX} z{9`ej-ReT>snzyJFB~%frX=XsS!$AplL^A=Y{gR-X!f{}ZN@*}8|ld! z5Pm#0LZz5W^U{!YbT<9A(D(ZzP)8Ro-{^WV?{0q zEvysnJXauo&}qs{@`Q^wWvC1JGG&1`W$?CT%A$sCmqF~x4ULd$U*kU)t>>Twx|Cy>rfb5sD@H(_#sV1ktFI9vk#TtX5AyC+umT2X03%B)Xmd zf&LG&7l#DtSD$*lc6$aO&N_aOR<*a|=%EH2$SjJsKl5fBJ44ED$IP}x*tXDYTi|UQ z-1oMPlZr58-0WhZZPvsug5YY2pU&^<7Na5?#QG;TsNii7<}44IH+Awts>D7Fxj$US zxi`-ewUUX@*Bo;|Svuh@#{in=!*a)+Bn3!<~(PO?7PU(}=72V`d*q@?V&8 zdb7>+O`cfF@(^B6x#hNCOmVJ3PWCpkA)7mGsWn?>7No|8L+G~F8w+&Da~C~_v^m$% zqGA&4?`D7^E|AmMlkh&DNSz?DF74M%pWQ$|j`3wsd8pB!qWl@oX+szzZ{R(`lDV4Z zvyfcYoVu%ns^ZVb5Z-8%BZBU1Iw(QK+ye#UJ}td!%n21vmJMT*lW=BO@)s7P&0a`W={`NlTqxmXcGm1kwue=;t~X0^v@LR> zO|I|=51b6R-^$P?Sj)(SkI9JiUVHI8Ik>a^<*gB0M>n8En+|2aWY(qwdrNXUnAUdF z0lkF|<(v-X79C0)3V;p>Sy>yo3@@G70X_L$#zO136~h_|GmL|A2vOm|OO)4H({*T= zlL@4aOjl{UlT^}5(L7$PmGW(1W)8Np>4v?aFFU=-&YjAGkh6m!7c1|y`K zw9siV5H?VnXD~{g2E*p2h7zQiKQ|S$w!sMDSNp%V{6Q4rke*)NeIdn92Z%%7Z*!C= z{rDUYb8|Nzs{cguCvauNkMj|-#^y(!dHaPoS$A12^9g#)=duK!caOPHVg^4s2$tY6 z6IR}-(w4@|feH{7x~Rd>OH8vorfDQ8<^^isrmCy*y#^c+lMV*rHOdaBES-CNlFUgY zoqa0bmy5E8UM@&K~VL7(UYd_;B`U*TL}X-H(rEk9HjlAMbuVojuwW{&YWn zJA1S%jO>0qlResXFnqH6@rmrwuJHFW8Gm(!zu9A!JC{A)?sMOjJz6q0F?+nz=iZY& zW`!*kO5Wvj@9i#ZspKB_x%X$eSz${h_dcKdV3wN|wp4N-@VO_m+^n#rlKV-Y`%spf z6}D7zPx;)hWVu;kOC|TqKKGFH$i7AdSez|=#oa75q zi4&3~=Aj0l%!utLXI(VH56WnC7Ey?Jc%pFn?&*mFMM~*z5nizn1|s~(`WRD?a!Q|} z;5u(nwHKERJ=@G)tLf#W!dr`6H&gnFq?Qp?tIxRmzo(|55 zTv5m&7Yri#3nr$lDUl6KqLx^NNGN`pF-LUdbXDe@mAzCeQJj_AHm<-{YqYd5s-Cil zP8{x#Kg{r~V6KJG5LzilF^jUwEft8Q2uwRyG&^FcW{;XYSlyM2OB7e_qLo_$vTQkC}6|#6PmP|0qbpiy9Sq11Zg# zxoEZqX?2nPusMbou3Q$e?%lGOQ(LPV`No_isQxGnhwqcQDE`L>*VzXatFK|z277N%u|QDa@mfWl15>1D?jbC-Q#QpOH!z>eEc zU@_vvNoHwOC@bPUZZG1LY4Yr1Mah)v&~z&LXn_f9&ZdW!su4Y8S2+^^hFM<-NLnT8 zDnv^C?}fMU!l==2f$|Vl95ZHfR^SqM_s!Eg0^i7Qeaz^g7-*{%p@N*#0U3(i9MiT8 zvCL*>YKWBjmJvuvy#%9Lg$(uY2%n((kn7qZ=GKrrN51AX!0CL8o4_s#11F(iLgyMG z476#Uc6F;;-e6z9Dwa+cwp=ITVwVEn^hB|k|XqVn5a3<7Oum_))$t$Y7o{TpO zz{jpg-o*X-v5(z06eVPaX##`zNCyN=?^WIUomwXc(Xh4gr@A@_nR%u?(pm?7|530M z0rwJH)$3?8QMH5qFb)2)moC*-f{0~CcApV<7Z`gj0R2y&JH`v;1mhn(wDhsNX+FI) zz4LuS;~gH2^thf#eYi9A_e*?jLoNFM60HMi@hV^uK;x~oTu{rUtYvGkQ3e4EBN6ab z>sj^CcMW;-UE_@dD3KO45N5UuKJ|KDTZtA&na*XVf1mRHqgwVX+ zZ_<-_79>bju&X*lWE;nVmO{zj(@LdyVXDrip>yeyVJqx`#t4lr!)Nvm$u4w51cbA2 zL_YG1Uiml$j+|P@4fZr@!PDA2Vf;f$DYq!N-raztc>HV!j=G}a?ZR0GSo zqN9flMH#bQ{eBZlgMwb__30mSq*YkJc8L`phN%&!rsk2i8J;VuHZ(xy;eBdM{%oPYO_Yc12{Ytlb{w~zM z0$1G@c7G7Y<-^9hQ0Bm8ralRW7&NuzxBh?2fCbe@!jR8b<)4x{#TnbsR|O-2a)D;F zb4L~Lw4j})5-ZJ*gl;Cm`9UQu^siNp92M3txub1%H~Ms9o*FpwB%KE)bFByGH}$E$ z|7OMtl9~Jg%cL?_m34f)p)eG&WyhNFV_0EGt0?+zllJ-K*r-*UU5(IO)Db^$>C)cX zgcJiW2zPK^EB0HQ>|2^8@Pn2He$-YCp2}ZORsB2H zN!88)@LWW-^~RaT|r?YWJs?`TI34lqMg^_GzVhPXs;)O-i=Maw|m-y;0F|c>=nBr^-F}mCS6E?|qvAW#JYI%U>?BH>w z4|Q|6+P6?*jYJba$fy$5W723f`#{2*!}eZMh*ik-3$b(cf0i2RZJ~_+#3Jzg89B7f zW7}u}M`xEv2Lt<~)2VC2N6mcjS%ItlWWnr#B?6QDujd|CLAXJ)4hRXO;^o0bTsV@% zK|@U(*tE-;3rTssRUHHre_0!z7O-v@EgW==h4;jO^`J*eMk=#c-rCcC{YOH-+WJz& zkM+1Buy-blT!QvUW)N>z9y|9s5Duylc4|8U2b=3hb}-Vaj@(5qq-m)E<8o{#dj!pc zml;p%e8+~zz;7FV*Zb)NsQ6FvBmfI{h)3DPg*%zW*0y2$%*7q5)B4QCOS>dKSYzz) zh_Kry#RSy2oKhV_?TXm;KuMavuEg9$J^lab4Xls5nNSfv zGpGN*-n}MZ-xT(`?$4ksCiqu`q1#PdN7AA?(Cr=kP)gZm~An9n3mjXhoy- z0qb(+mkH9C6$WZK-7P7LcpGaw>7Y--Qd!_u{PV9>ipB{q(13k%&hJwp_5ehMUTTeQ zF<=NOelp&|g!qWMDfD&5-Q|7JL)iHcwH`1uM3$gnQ+sn*2(|dWK-tE;$mnJODJhWb z8ETW)x`NIqtjlHGW297&#^5K5;xKR_W~pF^>W!BQ@f=aJ0Mnhl{Ry}>RO`jnA|0)W zl)h&gg>i!B_Hxvxr##pocCdxoojjqoy;&79MnajfL&gZFUaw4z z)Lq3q7HpO5W4yjY1)qlN_Nv${p35AjWvDdV$+mc_A)goPB(r5ci#aLeF2@2Pu5I%( z*mkDdjs?QRtLK>Vs(!Y>!59it>;ED`1=S@cne-Kh^0&>-p6qx-UZ(-o`xf*rtOQJu zELle|m5l=S8ZJB~K2dg*r9XcFzRcCS(e3Ows>5L2#aSUS8Po;ygH6Nh|5LG%BgG6 zIr|_6GNl*hv8CMOtkbT>8abL>&;*H{-Bk~XO9qK;W(G;GgsbR`Y7embgp63|&vGqD zb&Q^;Ize8>&fK6Fi|u`YMmdp|A-)Alyjy^dati5|;==SA3Pi013Zi!3@ZEVv*D6m~4hdzTGhFAZXr?s!EGg4Zuv#sGZKXi-Vu zp~2X@7-2@M!05!t(b-WEki!5lUIf*f%G$YC_xcw@&7Fp_^ryu(%tBVp!=}})YI)I9 z)bq9AvAk6H^Re5Kp4$``4rInPz}w57{6eDaiG5-YiNv+YL`tKxO`mWb#CZ)j5+ejp zXF0h+lFj+`E-a;iTz!XY+kJ(a3v00Ja}rz{*1Mf4g{Pxgliqs;1ikr#3vsIS%iaei zRweIu4Jq$eWm_FotV&MkHc~&(ky^IY0buDhRQ4JzY!=RZ<_3?o@#!2rJUWHscIWUdk`5MnLuSD->Rskl&;WS#*cVROy^G6sobQ;);+Z-GU(pzlW z7BGEFzbjJK-zF3gV&Y!ulS)Zhj#68cPXRh!f3HdaiWpn|cC)1YEo~<>nL5`x#;odJ z6p0x~I}<01B=Ipnpz=yA7m-g6<1&Ql+o{?pPE{N^+Xi7V5KGZC zUE?v6V((|lqEeEDZm49-ZfMLQZ?~XF=4U!G^(kgj@H5Q>DV@^{;`>DR%&7xYtzX{37sLSdd9)-F)kW?0SYeBPS-5|fS8+jiw#MY`d zR}+E;(FJ?msEU{Lb-d*d2=%5j|2)9(7q!CX!%Y~mBMr)es_r-l012`qf!25V=hnqO zjVE1QsmMK9xGP4b3f<`>3Sa$Rh)Z5LqaAdg)oHopZ11$M7tZ_BsSQ2S1uxvRln9KS^+UgMU@$@8m}Y9?)#US+Ha;-# z#mn#!ydIFhS!>Q&9c%`s!v7hbrU<7TV6{V*j z5P#TUs!c>Tr{>d(a-~Ar#!eXFfP1EN2v1w-N!)NE36k1Gb!XDj5mdN%WIDxnD&vZ2 zCUB07_*@ymf*GL&HyNoJzGMTuNuej&vx(M2AxKv~l0J3IlO>(8w(A&sG~x-CLJ}qi zIH|JplrE-E$`Qf0m2ZmM?orpPMl#e*C{S~ekTT~$Rw}{~ovaXVjLrasZ9pQl6w#x$ zQv1Qq!b}!QV}RG@wr$3t$B%%}It%H&ECqz4JO$D-X;whp9l@y8=)SZFExasVQd9o2F;4a<_p;b7tDaL+dtc z)m79!cL=3=LYwWRk@}Aoy;d{BP_tZ6X#>mV3ms7vwQ!N-Pcp$G77dM9xt8~@GlJ}F zleQ4zmi+LbPq{a>RmhKg?8y_YiGvyl6XS_SA=;77A<#gAVs?C}Fw-m%dSaHKbOqWq z$sx5k&LKL!yJ?dLf6PQ8w*V1432kps7}( z2JF8xVK0MaD3Gi8Oaqj-imY*@-~Ax|i+O7chDLMOOtkS)(gYNu1w*6X3JDWtp$>u* z(Hi_oe4|Ph@#1{WpL1FW2Kx`6kPbUIGmJk59oYneOjw~wwm?yKLWdil(q~K`4<@m` zZZ`diysfdzg5de$Ak!#<17)%&#(ly0MKF$vF|bbj`LA7emF!@HdYb*^ko%hP2emgmf#50e?R+leH(U3mJ`OSi}i>u9=3upmg!#}P@t zQ*Fgew@6LMiIs!Cn}(aVTGdmnjuX-e4zWiRCSznU#A8?k0KWc712L+i^FE2RoBG z8FxPU58oGOjH^uNjJxfR&c!&tgHKfKDswjD7R+vQ=2>UYZ}Ke0t!pvP!|q_*3D%G@ zL%r_l88_GJ_)lTn361$D6yiBI>4% zoGw^r2R|7VWx;K5WaLL^vR4RbkU{D&-PJ#Kz{)66_omQ)g&x%p6oJAh*gF+8c3~%@ z2m@*BUsx8HNMI2wgA>RU9}9v1Dxa1atZ+-6LRp<7RY-LU1~7zZbSFOGtDD1X%Ap?7 z1TYxZ7H%u3em=}SX;v+s3EGL%YXksSgg}~STU>;bDGPO?q)y=-nVWL3^0T-MV}0=jz&%f7bF1vp5jg6dbdg+ahUu^@=~McK8$ zHUH%le5FKqf?W~^vLYt>B*73$?#+{GCe&togAJV zIeKtp&+N$FiJLB)K0GtK=d$es*KgiBJh*A|_C1^TZo7W(@b!DPZXOsO93I%Vad6+z zzJ1%a4-X6v@4IYb^!n-H>60$r~=6o(Y2Mf_iWjxV*j+1dF)d zI6Ap^>c(Vd^mQZ20oqBfA4w*sCO5opWP0jKq4Kk~zGrHB#5X=Mx@Y9F(bOimPb%Las}>KA$V(3YK#fZ;v~FeN9%b z%2jf>ge%<*0>s^he-JDy)C1Ye(aC*Nz1)j$ zh{x>cp5ckhd5(F0+2~#XCzwXBVu^&{;MC~kY%;rlcs3cGNk%7+jLwW+KQW?G(<766 zL1B8s-jRK%|K3YqjSW0V*^#8Be97oO5~w*la&$J?1NJi`Gs$o=D`8HqpE^9b7dv`s zyJ;ClU$(pPnf+6TC-!1DXOrQB2SrPt(U zAj!=@GRr6cK8y*XWOBpAO$YZsKRG;e7|VVWhI(@JhRI8>8W}m56=+$K7Kj-Jl8KR- zndGLa!^!aUNFud4co37603+G&iE7sA(p@d`%^V!wqau)T3<~SEO*5l5kCQimo5+dc z8r+fL1Idk}=>1`6seIK_0mHkq)3R?H%)YhZUiz8^5)($t(ODxwTddY6*H4cOj|=(? zq~5b13hjl+2jDE^9i14No=I-p&o!BtnzC|-WpF{bQwkPBBH0In4Ci)porsAVBh#Ww zTh>H89%rc0X4RevBxq#1&8*>MV&n!Sn!#Zt(G{UExbreL5s_k$z_Rd1_$kVaJ%(p zdgMmv1X)1L6nJ7}-|U9jsSVQ@pmSn3BiYMYn^9{$l4f8K%#msOqAp}gKRcCZz=xN^ z(Aq!C%RWo90#=fq87 zJ1%P|ind!s^E!FW;QFy@Jz&@kn%6e3z4qG4Yp1WBz4kEAe@41|`FtE*#X~R8uWt@d z;NxK14qS)i&LG6i%l94LGcMiV;Em{p8A!b0$j0?eZ|<+d`@3OwfAjM7&Dp8h;R(Be z?&i!qPTt?-Qz1+RNsqy*9$X=^HF*3{;*<@;$_x;`Uz1l&G z-{Kh;99m3`XXvj7#4z4o^YZ55*?rrZzj_dVKt6%|1C8184df@tr_AR^8rua|&o;HV zjBdw7Ox4%&p}g+J#fE|(++MklY6 zRoRdI3bysugLS~a4*m4DLz}m4-o9mE%f>C6wrt+AWy{trgIk8SY}>MZ>%i8HTQ_apymiag zty>4T4sG4Gb^GAJ;KspCgPRAp3~n7992^?lHn@FgU})perlHM4TZXm{4Gs+rZ5!IY zjrhQ}P1`na+p=xzw!v*f+qP}nz8w^|)BJWoZ>QRJlBVS$!H3~#SZIXGN$PVnu+L~N zFDiTMmIT2U>z;m(a&1BnUl!*(gHc=J`IhHr&aelRL0P*FKLa~2|5>x={fXvxIM1f? z-9^^}Px8*E$!tzE3qQm+_y6)gQ25{c*RgYJf!k=_zgYh|T5QScVebK6mR+ygJ3Kp# z;NG^>%lj*+vzA}}{2K0MJN`WaziVj_9=Kq~mJ3z~+dr{p?#f{O?pG8Bj=v(+J5`4hoO_o3jC#)m%gH^cV^-*CTk=Qkhwy@wxv?DtRq!|=z0?=>D6_~*tS65;vu z>5nu&bNQcK@tF_)`K_OM>^tGfU-{k@Cxa($J-OQ*cFz!)h>u@iwY&=f>Y#*vCmgqVO-z|m?f}UQmjU<2h3EP@{#Vt$Ssb~ zkExf#RgS?07w&sq9rPpClR%OlL?I$miwmUif6=Y&1Z_h#>odH<+LrLVMR4}Tzg)Vcu(Pt-)#9FItHMHceY|3DW&EPBZ5rBU&0pz$)h~1KP!It3fi_pL3d$_IS>dSgcK#63VN~3Ub??usOS6l4r^-94F^(1Qkz|)Ew!NMf=YRL(LK-gT#0D8 z^`d&Xtg@_UbqdtV1(g*$w-v)3khLIbyXbirMHgLMb}Rb_$vdxi>Ey1V&App@*EUze ztz|!ao@Bco4Y}x|iYs66yrq|(-@mM9=Ou;8_8!-J;j+z3;_}5P&<+>(h*9q1rF`}R zb$g-78u(R8+m|8tnSQqPNLo@Tt)vQAJ_n7^6|7_;)?Wc;abL9R3i0BTgJJ2yV)I2Z zAI~2Q-Ant!Mdz13wf%-2t5GbZD0Hi^FevnE%i*P=1SRa>99EXRtl*aJ`ch+ckJ}*m z+PDe}v1pg*wVqbcN&bY;Ds=ybAGRP^*0aQ|i=rv_%3$Brl81uf#le!|h4RG+5tddj zdF7H-O9$%l>hlZd1snda9dlR3+e=G!fMYFOga^5<5@1w-6)biaq4A4iyvp%HZ`4z& z#4h+n_fi@Zsxf;99VJsVugrBNts@&z$pzts?n3Nm_pcnRE*Af~RYpvLxz3?=L+6C72uxuC)~sr+bb( zH>v)3{qbsP;B@sjHYT;?Bd2Qv_w8PE=J!r7sz0^6UUDbukA;ubf2n+8$q#Fd^L8(K z?1ESIomlp&3mPjIZ%FYT7c=S(|BFYd{=Bgj& z?h}=3s9&rEE-Dr)U)&rPf)~8v0s`8Y)C+sqiyptVva0MBCzaJeV81RHAGmxMw8n)o8j|C8Ua zP;09{P4&d9UU9nv1?AoGbgz52n=18" + }, + "scripts": { + "test": "node test.js", + "sync": "node sync-assets.js" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/byte271/FuckMark.git", + "directory": "bindings/node" + }, + "keywords": [ + "unicode", + "trojan-source", + "hidden-text", + "wasm", + "security" + ] +} diff --git a/bindings/node/scan.js b/bindings/node/scan.js new file mode 100644 index 00000000..f28c001f --- /dev/null +++ b/bindings/node/scan.js @@ -0,0 +1,569 @@ +// Faithful JavaScript port of fuckmark-hidden-scan-v1 (fuckmark/product/scan.py). +// Kept in lockstep with the Python engine by tests/test_vscode_scanner_parity.py. + +"use strict"; + +const SCAN_ALGORITHM_VERSION = "fuckmark-hidden-scan-v1"; + +const CATEGORY_BIDI_CONTROL = "bidi_control"; +const CATEGORY_ZERO_WIDTH = "zero_width"; +const CATEGORY_VARIATION_SELECTOR = "variation_selector"; +const CATEGORY_TAG = "tag"; +const CATEGORY_ENCLOSING_MARK = "enclosing_mark"; +const CATEGORY_LINE_SEPARATOR = "line_separator"; +const CATEGORY_DEPRECATED = "deprecated"; +const CATEGORY_FORMAT = "format"; +const CATEGORY_CONTROL = "control"; +const CATEGORY_PRIVATE_USE = "private_use"; +const CATEGORY_NONCHARACTER = "noncharacter"; +const CATEGORY_SURROGATE = "surrogate"; + +const SCAN_CATEGORIES = [ + CATEGORY_BIDI_CONTROL, + CATEGORY_ZERO_WIDTH, + CATEGORY_VARIATION_SELECTOR, + CATEGORY_TAG, + CATEGORY_ENCLOSING_MARK, + CATEGORY_LINE_SEPARATOR, + CATEGORY_DEPRECATED, + CATEGORY_FORMAT, + CATEGORY_CONTROL, + CATEGORY_PRIVATE_USE, + CATEGORY_NONCHARACTER, + CATEGORY_SURROGATE, +]; + +const CATEGORY_DESCRIPTIONS = { + [CATEGORY_BIDI_CONTROL]: "bidirectional override/isolate (Trojan Source reordering)", + [CATEGORY_ZERO_WIDTH]: "zero-width or invisible spacing character", + [CATEGORY_VARIATION_SELECTOR]: "variation selector (glyph/steganography carrier)", + [CATEGORY_TAG]: "Unicode tag character (hidden text / prompt-injection smuggling)", + [CATEGORY_ENCLOSING_MARK]: "enclosing combining mark (alters surrounding glyph)", + [CATEGORY_LINE_SEPARATOR]: "line/paragraph separator (breaks parsers, invisible)", + [CATEGORY_DEPRECATED]: "deprecated format control or interlinear annotation", + [CATEGORY_FORMAT]: "general Unicode format control (Cf)", + [CATEGORY_CONTROL]: "C0/C1 control character", + [CATEGORY_PRIVATE_USE]: "private-use codepoint (renderer-defined)", + [CATEGORY_NONCHARACTER]: "Unicode noncharacter or unassigned codepoint", + [CATEGORY_SURROGATE]: "lone surrogate codepoint", +}; + +const DEFAULT_SECURITY_CATEGORIES = [ + CATEGORY_BIDI_CONTROL, + CATEGORY_ZERO_WIDTH, + CATEGORY_TAG, + CATEGORY_CONTROL, + CATEGORY_NONCHARACTER, + CATEGORY_SURROGATE, +]; + +const TROJAN_SOURCE_CATEGORIES = [CATEGORY_BIDI_CONTROL]; + +const ALLOWED_WHITESPACE = new Set([0x09, 0x0a, 0x0d, 0x20]); + +function inRange(cp, lo, hi) { + return cp >= lo && cp <= hi; +} + +const BIDI_CONTROL = new Set([0x061c, 0x200e, 0x200f]); +for (let cp = 0x202a; cp <= 0x202e; cp += 1) BIDI_CONTROL.add(cp); +for (let cp = 0x2066; cp <= 0x2069; cp += 1) BIDI_CONTROL.add(cp); + +const DEPRECATED = new Set([0xfff9, 0xfffa, 0xfffb]); +for (let cp = 0x206a; cp <= 0x206f; cp += 1) DEPRECATED.add(cp); + +const ZERO_WIDTH = new Set([ + 0x00ad, 0x034f, 0x115f, 0x1160, 0x17b4, 0x17b5, 0x180e, 0x200b, 0x200c, + 0x200d, 0x2060, 0x2061, 0x2062, 0x2063, 0x2064, 0x3164, 0xfeff, 0xffa0, +]); + +function isVariationSelector(cp) { + return inRange(cp, 0xfe00, 0xfe0f) || inRange(cp, 0xe0100, 0xe01ef); +} + +function isTag(cp) { + return inRange(cp, 0xe0000, 0xe007f); +} + +function isNoncharacter(cp) { + if (inRange(cp, 0xfdd0, 0xfdef)) return true; + const low = cp & 0xffff; + return low === 0xfffe || low === 0xffff; +} + +function isEnclosingMark(cp) { + // Unicode general category Me. Enumerated to avoid a runtime Unicode DB. + return ( + inRange(cp, 0x0488, 0x0489) || + cp === 0x1abe || + inRange(cp, 0x20dd, 0x20e0) || + inRange(cp, 0x20e2, 0x20e4) || + inRange(cp, 0xa670, 0xa672) + ); +} + +function isControl(cp) { + return inRange(cp, 0x00, 0x1f) || inRange(cp, 0x7f, 0x9f); +} + +function isLineSeparator(cp) { + return cp === 0x2028 || cp === 0x2029; +} + +// General category Cf ranges (Unicode 15-era), excluding codepoints already +// classified above. Mirrors what unicodedata.category returns as "Cf". +const FORMAT_RANGES = [ + [0x0600, 0x0605], + [0x06dd, 0x06dd], + [0x070f, 0x070f], + [0x0890, 0x0891], + [0x08e2, 0x08e2], + [0x110bd, 0x110bd], + [0x110cd, 0x110cd], + [0x13430, 0x1343f], + [0x1bca0, 0x1bca3], + [0x1d173, 0x1d17a], +]; + +function isFormat(cp) { + for (const [lo, hi] of FORMAT_RANGES) { + if (inRange(cp, lo, hi)) return true; + } + return false; +} + +function isPrivateUse(cp) { + return ( + inRange(cp, 0xe000, 0xf8ff) || + inRange(cp, 0xf0000, 0xffffd) || + inRange(cp, 0x100000, 0x10fffd) + ); +} + +function classify(cp) { + if (ALLOWED_WHITESPACE.has(cp)) return null; + if (BIDI_CONTROL.has(cp)) return CATEGORY_BIDI_CONTROL; + if (DEPRECATED.has(cp)) return CATEGORY_DEPRECATED; + if (ZERO_WIDTH.has(cp)) return CATEGORY_ZERO_WIDTH; + if (isVariationSelector(cp)) return CATEGORY_VARIATION_SELECTOR; + if (isTag(cp)) return CATEGORY_TAG; + if (isNoncharacter(cp)) return CATEGORY_NONCHARACTER; + if (cp >= 0xd800 && cp <= 0xdfff) return CATEGORY_SURROGATE; + if (isEnclosingMark(cp)) return CATEGORY_ENCLOSING_MARK; + if (isLineSeparator(cp)) return CATEGORY_LINE_SEPARATOR; + if (isControl(cp)) return CATEGORY_CONTROL; + if (isFormat(cp)) return CATEGORY_FORMAT; + if (isPrivateUse(cp)) return CATEGORY_PRIVATE_USE; + return null; +} + +function codepointToken(cp) { + return "U+" + cp.toString(16).toUpperCase().padStart(4, "0"); +} + +function isEmojiish(cp) { + if (cp < 0) return false; + if (cp >= 0x1f1e6 && cp <= 0x1f1ff) return true; + if (cp >= 0x1f000 && cp <= 0x1faff) return true; + if (cp >= 0x2600 && cp <= 0x27bf) return true; + return cp === 0x200d || [0x00a9, 0x00ae, 0x203c, 0x2049, 0x2122, 0x2139, 0x3030, 0x303d, 0x3297, 0x3299].includes(cp); +} + +function isIdentChar(ch) { + return ch === "_" || /\p{L}|\p{N}/u.test(ch); +} + +function neighborBefore(text, offset) { + if (offset <= 0) return { cp: -1, ch: "" }; + if (offset >= 2) { + const cp = text.codePointAt(offset - 2); + if (cp > 0xffff) return { cp, ch: String.fromCodePoint(cp) }; + } + const cp = text.codePointAt(offset - 1); + return { cp, ch: String.fromCodePoint(cp) }; +} + +function neighborAfter(text, offset, length) { + const nextIndex = offset + length; + if (nextIndex >= text.length) return { cp: -1, ch: "" }; + const cp = text.codePointAt(nextIndex); + return { cp, ch: String.fromCodePoint(cp) }; +} + +const LANGUAGE_ALIASES = { + auto: "auto", + javascript: "javascript", + js: "javascript", + ts: "javascript", + typescript: "javascript", + jsx: "javascript", + tsx: "javascript", + c: "c", + h: "c", + cc: "c", + cpp: "c", + cxx: "c", + java: "c", + go: "c", + rs: "c", + rust: "c", + cs: "c", + css: "c", + jsonc: "c", + python: "python", + py: "python", + pyi: "python", + hash: "python", + sh: "python", + bash: "python", + zsh: "python", + shell: "python", + shellscript: "python", + yaml: "python", + yml: "python", + rb: "python", + ruby: "python", + toml: "python", + html: "html", + htm: "html", + xml: "html", + sql: "sql", +}; + +function normalizeLanguage(language) { + if (!language) return "auto"; + const key = String(language).trim().toLowerCase(); + return LANGUAGE_ALIASES[key] || "auto"; +} + +function slashComments(language) { + return language === "auto" || language === "javascript" || language === "c" || language === "sql"; +} + +function blockComments(language) { + return slashComments(language); +} + +function hashComments(language) { + return language === "python"; +} + +function sqlLineComments(language) { + return language === "sql"; +} + +function htmlComments(language) { + return language === "html"; +} + +function sourceRoles(text, language) { + const lang = normalizeLanguage(language); + const roles = new Array(text.length).fill("code"); + let index = 0; + let inString = ""; + let escape = false; + while (index < text.length) { + const ch = text[index]; + if (inString) { + roles[index] = "string"; + if (escape) { + escape = false; + index += 1; + continue; + } + if (ch === "\\" && index + 1 < text.length) { + escape = true; + index += 1; + continue; + } + if (ch === inString) inString = ""; + index += 1; + continue; + } + const pair = text.slice(index, index + 2); + if (slashComments(lang) && pair === "//" && !(index > 0 && text[index - 1] === ":")) { + let cursor = index; + while (cursor < text.length && text[cursor] !== "\n" && text[cursor] !== "\r") { + roles[cursor] = "comment"; + cursor += 1; + } + index = cursor; + continue; + } + if (sqlLineComments(lang) && pair === "--") { + let cursor = index; + while (cursor < text.length && text[cursor] !== "\n" && text[cursor] !== "\r") { + roles[cursor] = "comment"; + cursor += 1; + } + index = cursor; + continue; + } + if (blockComments(lang) && pair === "/*") { + const close = text.indexOf("*/", index + 2); + const stop = close >= 0 ? close + 2 : text.length; + for (let cursor = index; cursor < stop; cursor += 1) roles[cursor] = "comment"; + index = stop; + continue; + } + if (hashComments(lang) && ch === "#") { + let cursor = index; + while (cursor < text.length && text[cursor] !== "\n" && text[cursor] !== "\r") { + roles[cursor] = "comment"; + cursor += 1; + } + index = cursor; + continue; + } + if (htmlComments(lang) && text.slice(index, index + 4) === "", index + 4); + const stop = close >= 0 ? close + 3 : text.length; + for (let cursor = index; cursor < stop; cursor += 1) roles[cursor] = "comment"; + index = stop; + continue; + } + if (ch === '"' || ch === "'" || ch === "`") { + inString = ch; + roles[index] = "string"; + index += 1; + continue; + } + index += 1; + } + return roles; +} + +function classifyContext(text, offset, role, category) { + const length = text.codePointAt(offset) > 0xffff ? 2 : 1; + const prev = neighborBefore(text, offset); + const next = neighborAfter(text, offset, length); + if ((role === "comment" || role === "string") && category === CATEGORY_BIDI_CONTROL) { + return role; + } + if ( + isEmojiish(prev.cp) || + isEmojiish(next.cp) || + (prev.cp >= 0xfe00 && prev.cp <= 0xfe0f) || + (next.cp >= 0xfe00 && next.cp <= 0xfe0f) + ) { + return "emoji"; + } + if (role === "comment") return "comment"; + if (role === "string") return "string"; + if (isIdentChar(prev.ch) || isIdentChar(next.ch)) return "identifier"; + return "prose"; +} + +function scoreSeverity(category, context) { + if (category === CATEGORY_TAG) return "critical"; + if (category === CATEGORY_BIDI_CONTROL) { + if (context === "identifier" || context === "comment" || context === "string") return "critical"; + return "high"; + } + if (category === CATEGORY_ZERO_WIDTH) { + if (context === "emoji") return "info"; + if (context === "identifier") return "high"; + return "medium"; + } + if (category === CATEGORY_VARIATION_SELECTOR) return context === "emoji" ? "info" : "medium"; + if (category === CATEGORY_CONTROL || category === CATEGORY_NONCHARACTER || category === CATEGORY_SURROGATE) { + return "high"; + } + return "medium"; +} + +function explainFinding(category, context, severity) { + if (category === CATEGORY_BIDI_CONTROL && context === "identifier") { + return { + why: "Bidirectional override sits inside an identifier, so the glyphs can read differently than the bytes (Trojan Source).", + remedy: "Strip the bidi control and keep the identifier left-to-right.", + }; + } + if (category === CATEGORY_BIDI_CONTROL && context === "comment") { + return { + why: "Bidirectional override sits inside a comment, so commented-out code can appear to run (Trojan Source commenting-out).", + remedy: "Strip the bidi control from the comment.", + }; + } + if (category === CATEGORY_BIDI_CONTROL && context === "string") { + return { + why: "Bidirectional override sits inside a string, so the literal can appear to close early (Trojan Source stretched-string).", + remedy: "Strip the bidi control from the string.", + }; + } + if (category === CATEGORY_BIDI_CONTROL) { + return { + why: "Bidirectional override can reorder nearby glyphs (Trojan Source class, CVE-2021-42574).", + remedy: "Strip U+202A-U+202E / U+2066-U+2069 and rewrite the text left-to-right.", + }; + } + if (category === CATEGORY_TAG) { + return { + why: "Unicode tag characters encode a second ASCII string that models read and humans do not.", + remedy: "Strip U+E0020-U+E007F; inspect tag_payload for the smuggled text.", + }; + } + if (category === CATEGORY_ZERO_WIDTH && context === "emoji") { + return { + why: "Zero-width joiner or invisible mark inside an emoji cluster; usually a legitimate emoji sequence.", + remedy: "Leave emoji ZWJ sequences unless you are sanitizing for a security boundary.", + }; + } + if (category === CATEGORY_ZERO_WIDTH && context === "identifier") { + return { + why: "Zero-width character splits an identifier, breaking search and some compilers while looking unchanged.", + remedy: "Strip the zero-width character from the identifier.", + }; + } + if (category === CATEGORY_ZERO_WIDTH) { + return { + why: "Invisible spacing or joining character that changes the byte stream without changing the glyphs.", + remedy: "Strip the zero-width character.", + }; + } + if (category === CATEGORY_VARIATION_SELECTOR && context === "emoji") { + return { + why: "Variation selector tunes an emoji glyph; usually benign.", + remedy: "Keep emoji variation selectors unless you are stripping all hidden marks.", + }; + } + if (severity === "high") { + return { + why: "Hidden or non-text codepoint that should not appear in ordinary source or prompts.", + remedy: "Strip the character.", + }; + } + return { + why: CATEGORY_DESCRIPTIONS[category] || "Hidden or format codepoint that is invisible or renderer-defined.", + remedy: "Strip the character if this text crosses a trust boundary.", + }; +} + +function scanText(text, categories, language) { + const selected = categories ? new Set(categories) : null; + const roles = sourceRoles(text, language); + const counts = {}; + const findings = []; + let total = 0; + let offset = 0; + while (offset < text.length) { + const cp = text.codePointAt(offset); + const length = cp > 0xffff ? 2 : 1; + const category = classify(cp); + if (category !== null && (selected === null || selected.has(category))) { + total += 1; + counts[category] = (counts[category] || 0) + 1; + const context = classifyContext(text, offset, roles[offset] || "code", category); + const severity = scoreSeverity(category, context); + const explained = explainFinding(category, context, severity); + findings.push({ + offset, + length, + codepoint: cp, + category, + context, + severity, + why: explained.why, + remedy: explained.remedy, + }); + } + offset += length; + } + return { total, counts, findings }; +} + +function cleanText(text, categories) { + const selected = categories ? new Set(categories) : null; + let out = ""; + let removed = 0; + let offset = 0; + while (offset < text.length) { + const cp = text.codePointAt(offset); + const length = cp > 0xffff ? 2 : 1; + const category = classify(cp); + if (category !== null && (selected === null || selected.has(category))) { + removed += 1; + } else { + out += text.slice(offset, offset + length); + } + offset += length; + } + return { cleaned: out, removed }; +} + +function autofixTrojanSource(text) { + return cleanText(text, TROJAN_SOURCE_CATEGORIES); +} + +function languageFromDocumentId(languageId) { + return normalizeLanguage(languageId); +} + +const LANGUAGE_BY_SUFFIX = { + ".js": "javascript", + ".mjs": "javascript", + ".cjs": "javascript", + ".ts": "javascript", + ".jsx": "javascript", + ".tsx": "javascript", + ".c": "c", + ".h": "c", + ".hh": "c", + ".cc": "c", + ".cpp": "c", + ".cxx": "c", + ".java": "c", + ".go": "c", + ".rs": "c", + ".cs": "c", + ".css": "c", + ".py": "python", + ".pyi": "python", + ".sh": "python", + ".bash": "python", + ".zsh": "python", + ".yaml": "python", + ".yml": "python", + ".rb": "python", + ".toml": "python", + ".html": "html", + ".htm": "html", + ".xml": "html", + ".sql": "sql", +}; + +function languageFromPath(path) { + const name = String(path || ""); + const slash = Math.max(name.lastIndexOf("/"), name.lastIndexOf("\\")); + const base = slash >= 0 ? name.slice(slash + 1) : name; + const dot = base.lastIndexOf("."); + if (dot <= 0) return "auto"; + const suffix = base.slice(dot).toLowerCase(); + return LANGUAGE_BY_SUFFIX[suffix] || "auto"; +} + +const FuckMarkScan = { + SCAN_ALGORITHM_VERSION, + SCAN_CATEGORIES, + CATEGORY_DESCRIPTIONS, + DEFAULT_SECURITY_CATEGORIES, + TROJAN_SOURCE_CATEGORIES, + classify, + classifyContext, + scoreSeverity, + normalizeLanguage, + languageFromDocumentId, + languageFromPath, + sourceRoles, + codepointToken, + scanText, + cleanText, + autofixTrojanSource, +}; + +if (typeof module !== "undefined" && module.exports) { + module.exports = FuckMarkScan; +} +if (typeof globalThis !== "undefined") { + globalThis.FuckMarkScan = FuckMarkScan; +} diff --git a/bindings/node/scan_wasm.js b/bindings/node/scan_wasm.js new file mode 100644 index 00000000..5f01e7cd --- /dev/null +++ b/bindings/node/scan_wasm.js @@ -0,0 +1,209 @@ +"use strict"; + +const WASM_CATEGORIES = [ + "bidi_control", + "zero_width", + "variation_selector", + "tag", + "enclosing_mark", + "line_separator", + "deprecated", + "format", + "control", + "private_use", + "noncharacter", + "surrogate", +]; + +function utf16Offsets(text) { + const map = []; + let offset = 0; + while (offset < text.length) { + map.push(offset); + const cp = text.codePointAt(offset); + offset += cp > 0xffff ? 2 : 1; + } + return map; +} + +function codepointToken(cp) { + return "U+" + Number(cp).toString(16).toUpperCase().padStart(4, "0"); +} + +function hasLoneSurrogate(text) { + const source = String(text || ""); + for (let index = 0; index < source.length; index += 1) { + const unit = source.charCodeAt(index); + if (unit >= 0xd800 && unit <= 0xdbff) { + if (index + 1 >= source.length) return true; + const next = source.charCodeAt(index + 1); + if (next < 0xdc00 || next > 0xdfff) return true; + index += 1; + continue; + } + if (unit >= 0xdc00 && unit <= 0xdfff) return true; + } + return false; +} + +function encodeCategories(categories) { + if (categories == null) return "*"; + if (Array.isArray(categories)) return categories.join(","); + return String(categories); +} + +function writeUtf8(memory, alloc, bytes) { + const ptr = alloc(bytes.length); + new Uint8Array(memory.buffer, ptr, bytes.length).set(bytes); + return ptr; +} + +function readPackedJson(memory, dealloc, ptr) { + const view = new DataView(memory.buffer, ptr, 4); + const length = view.getUint32(0, true); + const bytes = new Uint8Array(memory.buffer, ptr + 4, length); + const text = new TextDecoder("utf-8").decode(bytes); + dealloc(ptr, 4 + length); + return JSON.parse(text); +} + +function wrapInstance(instance, fallback) { + const { memory, fm_alloc, fm_dealloc, fm_classify, fm_scan, fm_clean } = instance.exports; + const encoder = new TextEncoder(); + + function callScan(text, language, categories, maxFindings) { + const textBytes = encoder.encode(String(text || "")); + const langBytes = encoder.encode(String(language || "auto")); + const catBytes = encoder.encode(encodeCategories(categories)); + const textPtr = writeUtf8(memory, fm_alloc, textBytes); + const langPtr = writeUtf8(memory, fm_alloc, langBytes); + const catPtr = writeUtf8(memory, fm_alloc, catBytes); + const resultPtr = fm_scan( + textPtr, + textBytes.length, + langPtr, + langBytes.length, + catPtr, + catBytes.length, + maxFindings == null ? -1 : maxFindings + ); + fm_dealloc(textPtr, textBytes.length); + fm_dealloc(langPtr, langBytes.length); + fm_dealloc(catPtr, catBytes.length); + return readPackedJson(memory, fm_dealloc, resultPtr); + } + + function callClean(text, categories) { + const textBytes = encoder.encode(String(text || "")); + const catBytes = encoder.encode(encodeCategories(categories)); + const textPtr = writeUtf8(memory, fm_alloc, textBytes); + const catPtr = writeUtf8(memory, fm_alloc, catBytes); + const resultPtr = fm_clean(textPtr, textBytes.length, catPtr, catBytes.length); + fm_dealloc(textPtr, textBytes.length); + fm_dealloc(catPtr, catBytes.length); + return readPackedJson(memory, fm_dealloc, resultPtr); + } + + function scanText(text, categories, language) { + const source = String(text || ""); + if (fallback && hasLoneSurrogate(source)) { + return fallback.scanText(source, categories, language); + } + const payload = callScan(source, language || "auto", categories, -1); + const offsets = utf16Offsets(source); + const findings = (payload.findings || []).map((finding) => { + const offset = offsets[finding.index] || 0; + const length = finding.codepoint > 0xffff ? 2 : 1; + return { + offset, + length, + index: finding.index, + codepoint: finding.codepoint, + category: finding.category, + context: finding.context, + severity: finding.severity, + why: finding.why, + remedy: finding.remedy, + }; + }); + return { + total: payload.total || 0, + counts: payload.counts || {}, + findings, + truncated: Boolean(payload.truncated), + highest_severity: payload.highest_severity || "", + source_length: payload.source_length || 0, + }; + } + + function cleanText(text, categories) { + const source = String(text || ""); + if (fallback && hasLoneSurrogate(source)) { + return fallback.cleanText(source, categories); + } + return callClean(source, categories); + } + + const api = Object.assign({}, fallback || {}, { + engine: "wasm", + SCAN_ALGORITHM_VERSION: "fuckmark-hidden-scan-v1", + classify(cp) { + const index = fm_classify(cp >>> 0); + return index < 0 ? null : WASM_CATEGORIES[index]; + }, + scanText, + cleanText, + autofixTrojanSource(text) { + return cleanText(text, ["bidi_control"]); + }, + codepointToken: (fallback && fallback.codepointToken) || codepointToken, + hasLoneSurrogate, + encodeCategories, + }); + return api; +} + +async function instantiateWasm(source) { + let buffer; + if (source instanceof ArrayBuffer) { + buffer = source; + } else if (ArrayBuffer.isView(source)) { + buffer = source.buffer.slice(source.byteOffset, source.byteOffset + source.byteLength); + } else if (typeof source === "string") { + const response = await fetch(source); + if (!response.ok) { + throw new Error("fuckmark_scan.wasm missing"); + } + buffer = await response.arrayBuffer(); + } else { + throw new Error("wasm source required"); + } + const result = await WebAssembly.instantiate(buffer, {}); + return result.instance; +} + +async function loadFuckMarkScanWasm(source) { + const fallback = typeof globalThis !== "undefined" ? globalThis.FuckMarkScan : undefined; + const instance = await instantiateWasm(source || "fuckmark_scan.wasm"); + const api = wrapInstance(instance, fallback); + if (typeof globalThis !== "undefined") { + globalThis.FuckMarkScan = api; + globalThis.FuckMarkScanWasm = api; + } + if (typeof module !== "undefined" && module.exports) { + module.exports.loadFuckMarkScanWasm = loadFuckMarkScanWasm; + module.exports.wrapInstance = wrapInstance; + } + return api; +} + +if (typeof module !== "undefined" && module.exports) { + module.exports.loadFuckMarkScanWasm = loadFuckMarkScanWasm; + module.exports.wrapInstance = wrapInstance; + module.exports.WASM_CATEGORIES = WASM_CATEGORIES; + module.exports.hasLoneSurrogate = hasLoneSurrogate; + module.exports.encodeCategories = encodeCategories; +} +if (typeof globalThis !== "undefined") { + globalThis.loadFuckMarkScanWasm = loadFuckMarkScanWasm; +} diff --git a/bindings/node/sync-assets.js b/bindings/node/sync-assets.js new file mode 100644 index 00000000..acb3321e --- /dev/null +++ b/bindings/node/sync-assets.js @@ -0,0 +1,18 @@ +"use strict"; + +const fs = require("fs"); +const path = require("path"); + +const root = path.resolve(__dirname, "../.."); +const copies = [ + ["editors/wasm/scan_wasm.js", "scan_wasm.js"], + ["editors/wasm/fuckmark_scan.wasm", "fuckmark_scan.wasm"], + ["editors/vscode/scan.js", "scan.js"], +]; + +for (const [srcRel, destName] of copies) { + const src = path.join(root, srcRel); + const dest = path.join(__dirname, destName); + fs.copyFileSync(src, dest); + process.stdout.write(destName + "\n"); +} diff --git a/bindings/node/test.js b/bindings/node/test.js new file mode 100644 index 00000000..7337c737 --- /dev/null +++ b/bindings/node/test.js @@ -0,0 +1,24 @@ +"use strict"; + +const assert = require("assert"); +const scan = require("./index.js"); + +(async () => { + const rlo = String.fromCodePoint(0x202e); + const result = await scan.scanText("a" + rlo + "b", null, "auto"); + assert.strictEqual(result.total, 1); + assert.strictEqual(result.findings[0].category, "bidi_control"); + assert.strictEqual(result.findings[0].severity, "critical"); + const cleaned = await scan.cleanText("a" + rlo + "b", ["bidi_control"]); + assert.strictEqual(cleaned.removed, 1); + assert.strictEqual(cleaned.cleaned, "ab"); + const empty = await scan.scanText("a" + rlo + "b", [], "auto"); + assert.strictEqual(empty.total, 0); + const lone = await scan.scanText("\uD800", null, "auto"); + assert.strictEqual(lone.total, 1); + assert.strictEqual(lone.findings[0].category, "surrogate"); + process.stdout.write("ok\n"); +})().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/crates/fuckmark-scan/README.md b/crates/fuckmark-scan/README.md index 1dbb4c44..60e72f5b 100644 --- a/crates/fuckmark-scan/README.md +++ b/crates/fuckmark-scan/README.md @@ -1,18 +1,20 @@ -# fuckmark-scan (Rust / WASM) +# fuckmark-scan (Rust / WASM / native) Native and `wasm32` build of `fuckmark-hidden-scan-v1`. Same category table as `editors/vscode/scan.js` and `fuckmark.product.scan`. No network, no extra -crate dependencies. +crate dependencies. C ABI header: `include/fuckmark_scan.h`. ```text rustup target add wasm32-unknown-unknown ./crates/fuckmark-scan/build-wasm.sh +./crates/fuckmark-scan/build-native.sh ``` -That writes `fuckmark_scan.wasm` next to the scan page, the packaged web UI, -and the Chromium popup. The page and popup load it when `fetch` works and keep -`scan.js` as the `file://` fallback. +`build-wasm.sh` writes `fuckmark_scan.wasm` next to the scan page, packaged web +UI, and Chromium popup. `build-native.sh` builds the host `cdylib` for the +Python ctypes binding and C callers. -`cargo test` in this directory covers classification, Trojan Source roles, and -clean. Conformance vectors are also replayed from Python via Node on the -committed module. +Category filter encoding over the ABI: `*` means all categories, an empty +string means an empty selection, otherwise a comma-separated list. + +Language bindings overview: [`bindings/README.md`](../../bindings/README.md). diff --git a/crates/fuckmark-scan/build-native.sh b/crates/fuckmark-scan/build-native.sh new file mode 100755 index 00000000..9773cbc6 --- /dev/null +++ b/crates/fuckmark-scan/build-native.sh @@ -0,0 +1,21 @@ +#!/bin/sh +set -eu +root="$(CDPATH= cd -- "$(dirname "$0")/../.." && pwd)" +crate="$root/crates/fuckmark-scan" +cd "$crate" +cargo test --offline --quiet 2>/dev/null || cargo test --quiet +cargo build --release --quiet +lib="" +for candidate in \ + "$crate/target/release/libfuckmark_scan.so" \ + "$crate/target/release/libfuckmark_scan.dylib" \ + "$crate/target/release/fuckmark_scan.dll" +do + if test -f "$candidate"; then + lib="$candidate" + break + fi +done +test -n "$lib" +printf '%s\n' "$lib" +ls -l "$lib" diff --git a/crates/fuckmark-scan/examples/scan_cli.c b/crates/fuckmark-scan/examples/scan_cli.c new file mode 100644 index 00000000..116a8ed6 --- /dev/null +++ b/crates/fuckmark-scan/examples/scan_cli.c @@ -0,0 +1,35 @@ +#include +#include +#include +#include "fuckmark_scan.h" + +static void print_packed_json(uint8_t *ptr) { + uint32_t length = (uint32_t)ptr[0] + | ((uint32_t)ptr[1] << 8) + | ((uint32_t)ptr[2] << 16) + | ((uint32_t)ptr[3] << 24); + fwrite(ptr + 4, 1, length, stdout); + fputc('\n', stdout); + fm_dealloc(ptr, 4 + length); +} + +int main(int argc, char **argv) { + const char *text = argc > 1 ? argv[1] : "a\xE2\x80\xAE" "b"; + const char *language = argc > 2 ? argv[2] : "auto"; + const char *categories = "*"; + uint8_t *result = fm_scan( + (const uint8_t *)text, + (uint32_t)strlen(text), + (const uint8_t *)language, + (uint32_t)strlen(language), + (const uint8_t *)categories, + (uint32_t)strlen(categories), + -1 + ); + if (result == NULL) { + fprintf(stderr, "fm_scan failed\n"); + return 1; + } + print_packed_json(result); + return 0; +} diff --git a/crates/fuckmark-scan/include/fuckmark_scan.h b/crates/fuckmark-scan/include/fuckmark_scan.h new file mode 100644 index 00000000..7f85ed56 --- /dev/null +++ b/crates/fuckmark-scan/include/fuckmark_scan.h @@ -0,0 +1,35 @@ +#ifndef FUCKMARK_SCAN_H +#define FUCKMARK_SCAN_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +uint8_t *fm_alloc(uint32_t size); +void fm_dealloc(uint8_t *ptr, uint32_t size); +int32_t fm_classify(uint32_t codepoint); + +uint8_t *fm_scan( + const uint8_t *text_ptr, + uint32_t text_len, + const uint8_t *lang_ptr, + uint32_t lang_len, + const uint8_t *cats_ptr, + uint32_t cats_len, + int32_t max_findings +); + +uint8_t *fm_clean( + const uint8_t *text_ptr, + uint32_t text_len, + const uint8_t *cats_ptr, + uint32_t cats_len +); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/fuckmark/native_scan.py b/fuckmark/native_scan.py new file mode 100644 index 00000000..ed08c453 --- /dev/null +++ b/fuckmark/native_scan.py @@ -0,0 +1,178 @@ +from __future__ import annotations + +import json +import os +import sys +from ctypes import ( + CDLL, + POINTER, + c_char, + c_char_p, + c_int32, + c_uint32, + create_string_buffer, + string_at, +) +from pathlib import Path + + +_ROOT = Path(__file__).resolve().parents[1] +_LIB: CDLL | None = None + + +def _library_candidates() -> list[Path]: + env = os.environ.get("FUCKMARK_SCAN_LIB", "").strip() + names = ( + "libfuckmark_scan.so", + "libfuckmark_scan.dylib", + "fuckmark_scan.dll", + "fuckmark_scan.so", + ) + paths: list[Path] = [] + if env: + paths.append(Path(env)) + release = _ROOT / "crates" / "fuckmark-scan" / "target" / "release" + debug = _ROOT / "crates" / "fuckmark-scan" / "target" / "debug" + for folder in (release, debug): + for name in names: + paths.append(folder / name) + return paths + + +def library_path() -> Path | None: + for path in _library_candidates(): + if path.is_file(): + return path + return None + + +def load_library() -> CDLL | None: + global _LIB + if _LIB is not None: + return _LIB + path = library_path() + if path is None: + return None + lib = CDLL(str(path)) + lib.fm_alloc.argtypes = [c_uint32] + lib.fm_alloc.restype = POINTER(c_char) + lib.fm_dealloc.argtypes = [POINTER(c_char), c_uint32] + lib.fm_dealloc.restype = None + lib.fm_classify.argtypes = [c_uint32] + lib.fm_classify.restype = c_int32 + lib.fm_scan.argtypes = [ + c_char_p, + c_uint32, + c_char_p, + c_uint32, + c_char_p, + c_uint32, + c_int32, + ] + lib.fm_scan.restype = POINTER(c_char) + lib.fm_clean.argtypes = [c_char_p, c_uint32, c_char_p, c_uint32] + lib.fm_clean.restype = POINTER(c_char) + _LIB = lib + return _LIB + + +def available() -> bool: + return load_library() is not None + + +def _read_packed_json(lib: CDLL, ptr) -> dict: + if not ptr: + raise RuntimeError("native scan returned a null pointer") + header = string_at(ptr, 4) + length = int.from_bytes(header, "little", signed=False) + payload = string_at(ptr, 4 + length)[4:] + lib.fm_dealloc(ptr, c_uint32(4 + length)) + return json.loads(payload.decode("utf-8")) + + +def classify(codepoint: int) -> str | None: + lib = load_library() + if lib is None: + raise RuntimeError("native fuckmark-scan library is not available") + if not isinstance(codepoint, int): + raise TypeError("codepoint must be an int") + index = int(lib.fm_classify(c_uint32(codepoint & 0xFFFFFFFF))) + if index < 0: + return None + from .product.scan import SCAN_CATEGORIES + + return SCAN_CATEGORIES[index] + + +def scan_text( + text: str, + *, + language: str = "auto", + categories: list[str] | None = None, + max_findings: int = -1, +) -> dict: + if not isinstance(text, str): + raise TypeError("text must be a string") + lib = load_library() + if lib is None: + raise RuntimeError("native fuckmark-scan library is not available") + text_bytes = text.encode("utf-8") + lang_bytes = str(language or "auto").encode("utf-8") + if categories is None: + cat_bytes = b"*" + else: + cat_bytes = ",".join(categories).encode("utf-8") + text_buf = create_string_buffer(text_bytes) + lang_buf = create_string_buffer(lang_bytes) + cat_buf = create_string_buffer(cat_bytes) if cat_bytes else create_string_buffer(1) + ptr = lib.fm_scan( + text_buf, + c_uint32(len(text_bytes)), + lang_buf, + c_uint32(len(lang_bytes)), + cat_buf, + c_uint32(len(cat_bytes)), + c_int32(max_findings), + ) + return _read_packed_json(lib, ptr) + + +def clean_text(text: str, *, categories: list[str] | None = None) -> tuple[str, int]: + if not isinstance(text, str): + raise TypeError("text must be a string") + lib = load_library() + if lib is None: + raise RuntimeError("native fuckmark-scan library is not available") + text_bytes = text.encode("utf-8") + if categories is None: + cat_bytes = b"*" + else: + cat_bytes = ",".join(categories).encode("utf-8") + text_buf = create_string_buffer(text_bytes) + cat_buf = create_string_buffer(cat_bytes) if cat_bytes else create_string_buffer(1) + ptr = lib.fm_clean( + text_buf, + c_uint32(len(text_bytes)), + cat_buf, + c_uint32(len(cat_bytes)), + ) + payload = _read_packed_json(lib, ptr) + return str(payload.get("cleaned", "")), int(payload.get("removed", 0)) + + +def main(argv: list[str] | None = None) -> int: + args = list(sys.argv[1:] if argv is None else argv) + if not available(): + sys.stderr.write( + "fuckmark native-scan: library not found; run crates/fuckmark-scan/build-native.sh\n" + ) + return 1 + text = args[0] if args else "a\u202eb" + language = args[1] if len(args) > 1 else "auto" + payload = scan_text(text, language=language) + sys.stdout.write(json.dumps(payload, ensure_ascii=False) + "\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_language_bindings.py b/tests/test_language_bindings.py new file mode 100644 index 00000000..20de93f5 --- /dev/null +++ b/tests/test_language_bindings.py @@ -0,0 +1,80 @@ +import json +import shutil +import subprocess +from pathlib import Path + +import pytest + +from fuckmark.native_scan import available, clean_text, library_path, scan_text +from fuckmark.product.scan import scan_hidden_characters + + +ROOT = Path(__file__).resolve().parents[1] +VECTORS = ROOT / "specs" / "fuckmark-hidden-scan-v1.vectors.json" +CRATE = ROOT / "crates" / "fuckmark-scan" +NODE_BIND = ROOT / "bindings" / "node" + + +def test_c_header_and_binding_docs_exist() -> None: + assert (CRATE / "include" / "fuckmark_scan.h").is_file() + header = (CRATE / "include" / "fuckmark_scan.h").read_text(encoding="utf-8") + assert "fm_scan" in header + assert "fm_clean" in header + assert "fm_classify" in header + assert (ROOT / "bindings" / "README.md").is_file() + assert "FUCKMARK_SCAN_LIB" in (ROOT / "bindings" / "README.md").read_text(encoding="utf-8") + + +def test_native_library_matches_python_vectors() -> None: + if not available(): + cargo = shutil.which("cargo") + if cargo is None: + pytest.skip("cargo is not available") + subprocess.run( + ["sh", str(CRATE / "build-native.sh")], + check=True, + timeout=180, + ) + assert available() + assert library_path() is not None + payload = json.loads(VECTORS.read_text(encoding="utf-8")) + for vector in payload["vectors"]: + text = "".join(chr(code) for code in vector["codepoints"]) + native = scan_text(text, language=vector.get("language") or "auto") + python = scan_hidden_characters(text, language=vector.get("language")) + assert native["total"] == python.total, vector["id"] + assert len(native["findings"]) == len(python.findings), vector["id"] + for left, right in zip(native["findings"], python.findings, strict=True): + assert left["index"] == right.index, vector["id"] + assert left["codepoint"] == right.codepoint, vector["id"] + assert left["category"] == right.category, vector["id"] + assert left["context"] == right.context, vector["id"] + assert left["severity"] == right.severity, vector["id"] + + +def test_native_empty_categories_and_clean() -> None: + if not available(): + pytest.skip("native library is not available") + text = "a\u202eb\u200bc" + empty = scan_text(text, categories=[]) + assert empty["total"] == 0 + cleaned, removed = clean_text(text, categories=["bidi_control"]) + assert removed == 1 + assert cleaned == "ab\u200bc" + + +def test_node_binding_assets_and_tests() -> None: + node = shutil.which("node") + if node is None: + pytest.skip("node is not available") + subprocess.run([node, str(NODE_BIND / "sync-assets.js")], check=True, timeout=30) + for name in ("scan_wasm.js", "fuckmark_scan.wasm", "scan.js"): + assert (NODE_BIND / name).is_file() + completed = subprocess.run( + [node, str(NODE_BIND / "test.js")], + check=True, + capture_output=True, + text=True, + timeout=30, + ) + assert completed.stdout.strip() == "ok" From b76f013bc717943f41fbd33e5e42896e5fe121e5 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 2 Sep 2026 03:04:25 +0000 Subject: [PATCH 2/2] Harden language bindings for surrogates and the C example rpath Route lone UTF-16 surrogates through the Python scanner so the native binding can report the surrogate category. Normalize the WASM JS fallback result to the same schema, and link the C example next to the built library with \$ORIGIN. Co-authored-by: Byte271 --- CHANGELOG.md | 2 +- bindings/README.md | 6 +-- bindings/node/scan_wasm.js | 63 +++++++++++++++++++++++++++++- bindings/node/test.js | 9 +++++ docs/scan_wasm.js | 63 +++++++++++++++++++++++++++++- editors/browser/scan_wasm.js | 63 +++++++++++++++++++++++++++++- editors/wasm/scan_wasm.js | 63 +++++++++++++++++++++++++++++- fuckmark/native_scan.py | 68 +++++++++++++++++++++++++++++++++ fuckmark/webui/scan_wasm.js | 63 +++++++++++++++++++++++++++++- tests/test_language_bindings.py | 26 +++++++++++++ tests/test_scan_wasm.py | 7 ++++ 11 files changed, 424 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 588a65d3..1ab90e96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ - Zero-install hidden-Unicode scan page: `docs/scan.html` (packaged as `fuckmark/webui/scan.html`, engine copy `scan.js`). `fuckmark web` serves it at `/scan.html`. After deploy: `mark.q1z.org/scan.html`. Scanning runs in the browser; a local Python `/api/scan` confirm is optional when the CLI server is up. - Chromium browser extension under `editors/browser`: popup scanner, on-page reveal (badges plus CSS highlights), context-menu scan, per-tab toolbar badge, and optional paste-safe stripping that keeps real emoji ZWJ sequences (both neighbors emoji) and stops page paste handlers from seeing the raw clipboard. Zero remote code; `scan.js` matches the VS Code engine copy. Load unpacked from `chrome://extensions`. - WASM build of `fuckmark-hidden-scan-v1` (`crates/fuckmark-scan`, committed `fuckmark_scan.wasm`). The scan page and the Chromium popup load it when `fetch` works so classification stays on-device; `scan.js` remains the `file://` fallback. Lone UTF-16 surrogates route through the JS engine (UTF-8 cannot carry them). Empty category arrays stay empty selections. The extension CSP allows `wasm-unsafe-eval` so Chromium can instantiate the module. Page reveal and paste-safe keep the synchronous JS port. Rebuild with `crates/fuckmark-scan/build-wasm.sh`. -- Language bindings for the same Rust scan ABI: C header (`crates/fuckmark-scan/include/fuckmark_scan.h`), optional Python ctypes loader (`fuckmark.native_scan`), and a Node package under `bindings/node` (WASM + JS fallback). Host library build: `crates/fuckmark-scan/build-native.sh`. Overview: `bindings/README.md`. +- Language bindings for the same Rust scan ABI: C header (`crates/fuckmark-scan/include/fuckmark_scan.h`), optional Python ctypes loader (`fuckmark.native_scan`), and a Node package under `bindings/node` (WASM + JS fallback). Host library build: `crates/fuckmark-scan/build-native.sh`. Lone UTF-16 surrogates cannot cross the UTF-8 ABI, so Python `scan_text` / `clean_text` delegate those strings to the Python reference scanner, and the WASM wrapper normalizes the JS fallback result to the same `index` / `truncated` / `highest_severity` / `source_length` schema. The C example is linked with `-rpath,$ORIGIN` and written next to `libfuckmark_scan.so`. Overview: `bindings/README.md`. - `fuckmark normalize` (`fuckmark-normalize-v1`): NFC, optional identifier lookalike fold (UTS #39-inspired subset, not a full confusable map), then strip of the security category set, plus a JSON receipt (`input_sha256` / `output_sha256` / `steps` / `report_hash`). `--clean` remains the blunt strip with no receipt. HTTP: `POST /api/normalize`. Python: `normalize_text`, `skeleton_fold`. Reference: `docs/normalize.md`. - VS Code / Cursor: `fuckmark.cleanOnSave` (default off) strips hidden Unicode on save. - Frozen confirmation files, hashes, mix mechanisms, and the published wheel are unchanged. Package version stays 0.4.1. diff --git a/bindings/README.md b/bindings/README.md index f2746c08..e0d52832 100644 --- a/bindings/README.md +++ b/bindings/README.md @@ -51,7 +51,7 @@ node test.js cc -I crates/fuckmark-scan/include \ crates/fuckmark-scan/examples/scan_cli.c \ -L crates/fuckmark-scan/target/release -lfuckmark_scan \ - -Wl,-rpath,\$ORIGIN/../../crates/fuckmark-scan/target/release \ - -o /tmp/fm-scan -/tmp/fm-scan $'a\u202eb' + -Wl,-rpath,\$ORIGIN \ + -o crates/fuckmark-scan/target/release/fm-scan +./crates/fuckmark-scan/target/release/fm-scan $'a\u202eb' ``` diff --git a/bindings/node/scan_wasm.js b/bindings/node/scan_wasm.js index 5f01e7cd..4e8efdda 100644 --- a/bindings/node/scan_wasm.js +++ b/bindings/node/scan_wasm.js @@ -52,6 +52,66 @@ function encodeCategories(categories) { return String(categories); } +const SEVERITY_RANK = { info: 0, medium: 1, high: 2, critical: 3 }; + +function codePointCount(text) { + let count = 0; + for (let offset = 0; offset < text.length; ) { + const cp = text.codePointAt(offset); + offset += cp > 0xffff ? 2 : 1; + count += 1; + } + return count; +} + +function utf16OffsetToIndex(text, offset) { + let index = 0; + let pos = 0; + while (pos < offset && pos < text.length) { + const cp = text.codePointAt(pos); + pos += cp > 0xffff ? 2 : 1; + index += 1; + } + return index; +} + +function peakSeverity(findings) { + let peak = ""; + for (const finding of findings) { + if ((SEVERITY_RANK[finding.severity] || -1) > (SEVERITY_RANK[peak] || -1)) { + peak = finding.severity; + } + } + return peak; +} + +function normalizeScanResult(source, result) { + const findings = (result.findings || []).map((finding) => { + const offset = finding.offset != null ? finding.offset : 0; + const length = finding.length != null ? finding.length : finding.codepoint > 0xffff ? 2 : 1; + const index = finding.index != null ? finding.index : utf16OffsetToIndex(source, offset); + return { + offset, + length, + index, + codepoint: finding.codepoint, + category: finding.category, + context: finding.context, + severity: finding.severity, + why: finding.why, + remedy: finding.remedy, + }; + }); + return { + total: result.total || 0, + counts: result.counts || {}, + findings, + truncated: Boolean(result.truncated), + highest_severity: result.highest_severity || peakSeverity(findings), + source_length: result.source_length != null ? result.source_length : codePointCount(source), + }; +} + function writeUtf8(memory, alloc, bytes) { const ptr = alloc(bytes.length); new Uint8Array(memory.buffer, ptr, bytes.length).set(bytes); @@ -107,7 +167,7 @@ function wrapInstance(instance, fallback) { function scanText(text, categories, language) { const source = String(text || ""); if (fallback && hasLoneSurrogate(source)) { - return fallback.scanText(source, categories, language); + return normalizeScanResult(source, fallback.scanText(source, categories, language)); } const payload = callScan(source, language || "auto", categories, -1); const offsets = utf16Offsets(source); @@ -203,6 +263,7 @@ if (typeof module !== "undefined" && module.exports) { module.exports.WASM_CATEGORIES = WASM_CATEGORIES; module.exports.hasLoneSurrogate = hasLoneSurrogate; module.exports.encodeCategories = encodeCategories; + module.exports.normalizeScanResult = normalizeScanResult; } if (typeof globalThis !== "undefined") { globalThis.loadFuckMarkScanWasm = loadFuckMarkScanWasm; diff --git a/bindings/node/test.js b/bindings/node/test.js index 7337c737..a58b4445 100644 --- a/bindings/node/test.js +++ b/bindings/node/test.js @@ -17,6 +17,15 @@ const scan = require("./index.js"); const lone = await scan.scanText("\uD800", null, "auto"); assert.strictEqual(lone.total, 1); assert.strictEqual(lone.findings[0].category, "surrogate"); + assert.strictEqual(lone.findings[0].index, 0); + assert.strictEqual(lone.findings[0].offset, 0); + assert.strictEqual(lone.truncated, false); + assert.strictEqual(lone.highest_severity, "high"); + assert.strictEqual(lone.source_length, 1); + const mixed = await scan.scanText("a\uD800b", null, "auto"); + assert.strictEqual(mixed.total, 1); + assert.strictEqual(mixed.findings[0].index, 1); + assert.strictEqual(mixed.source_length, 3); process.stdout.write("ok\n"); })().catch((error) => { console.error(error); diff --git a/docs/scan_wasm.js b/docs/scan_wasm.js index 5f01e7cd..4e8efdda 100644 --- a/docs/scan_wasm.js +++ b/docs/scan_wasm.js @@ -52,6 +52,66 @@ function encodeCategories(categories) { return String(categories); } +const SEVERITY_RANK = { info: 0, medium: 1, high: 2, critical: 3 }; + +function codePointCount(text) { + let count = 0; + for (let offset = 0; offset < text.length; ) { + const cp = text.codePointAt(offset); + offset += cp > 0xffff ? 2 : 1; + count += 1; + } + return count; +} + +function utf16OffsetToIndex(text, offset) { + let index = 0; + let pos = 0; + while (pos < offset && pos < text.length) { + const cp = text.codePointAt(pos); + pos += cp > 0xffff ? 2 : 1; + index += 1; + } + return index; +} + +function peakSeverity(findings) { + let peak = ""; + for (const finding of findings) { + if ((SEVERITY_RANK[finding.severity] || -1) > (SEVERITY_RANK[peak] || -1)) { + peak = finding.severity; + } + } + return peak; +} + +function normalizeScanResult(source, result) { + const findings = (result.findings || []).map((finding) => { + const offset = finding.offset != null ? finding.offset : 0; + const length = finding.length != null ? finding.length : finding.codepoint > 0xffff ? 2 : 1; + const index = finding.index != null ? finding.index : utf16OffsetToIndex(source, offset); + return { + offset, + length, + index, + codepoint: finding.codepoint, + category: finding.category, + context: finding.context, + severity: finding.severity, + why: finding.why, + remedy: finding.remedy, + }; + }); + return { + total: result.total || 0, + counts: result.counts || {}, + findings, + truncated: Boolean(result.truncated), + highest_severity: result.highest_severity || peakSeverity(findings), + source_length: result.source_length != null ? result.source_length : codePointCount(source), + }; +} + function writeUtf8(memory, alloc, bytes) { const ptr = alloc(bytes.length); new Uint8Array(memory.buffer, ptr, bytes.length).set(bytes); @@ -107,7 +167,7 @@ function wrapInstance(instance, fallback) { function scanText(text, categories, language) { const source = String(text || ""); if (fallback && hasLoneSurrogate(source)) { - return fallback.scanText(source, categories, language); + return normalizeScanResult(source, fallback.scanText(source, categories, language)); } const payload = callScan(source, language || "auto", categories, -1); const offsets = utf16Offsets(source); @@ -203,6 +263,7 @@ if (typeof module !== "undefined" && module.exports) { module.exports.WASM_CATEGORIES = WASM_CATEGORIES; module.exports.hasLoneSurrogate = hasLoneSurrogate; module.exports.encodeCategories = encodeCategories; + module.exports.normalizeScanResult = normalizeScanResult; } if (typeof globalThis !== "undefined") { globalThis.loadFuckMarkScanWasm = loadFuckMarkScanWasm; diff --git a/editors/browser/scan_wasm.js b/editors/browser/scan_wasm.js index 5f01e7cd..4e8efdda 100644 --- a/editors/browser/scan_wasm.js +++ b/editors/browser/scan_wasm.js @@ -52,6 +52,66 @@ function encodeCategories(categories) { return String(categories); } +const SEVERITY_RANK = { info: 0, medium: 1, high: 2, critical: 3 }; + +function codePointCount(text) { + let count = 0; + for (let offset = 0; offset < text.length; ) { + const cp = text.codePointAt(offset); + offset += cp > 0xffff ? 2 : 1; + count += 1; + } + return count; +} + +function utf16OffsetToIndex(text, offset) { + let index = 0; + let pos = 0; + while (pos < offset && pos < text.length) { + const cp = text.codePointAt(pos); + pos += cp > 0xffff ? 2 : 1; + index += 1; + } + return index; +} + +function peakSeverity(findings) { + let peak = ""; + for (const finding of findings) { + if ((SEVERITY_RANK[finding.severity] || -1) > (SEVERITY_RANK[peak] || -1)) { + peak = finding.severity; + } + } + return peak; +} + +function normalizeScanResult(source, result) { + const findings = (result.findings || []).map((finding) => { + const offset = finding.offset != null ? finding.offset : 0; + const length = finding.length != null ? finding.length : finding.codepoint > 0xffff ? 2 : 1; + const index = finding.index != null ? finding.index : utf16OffsetToIndex(source, offset); + return { + offset, + length, + index, + codepoint: finding.codepoint, + category: finding.category, + context: finding.context, + severity: finding.severity, + why: finding.why, + remedy: finding.remedy, + }; + }); + return { + total: result.total || 0, + counts: result.counts || {}, + findings, + truncated: Boolean(result.truncated), + highest_severity: result.highest_severity || peakSeverity(findings), + source_length: result.source_length != null ? result.source_length : codePointCount(source), + }; +} + function writeUtf8(memory, alloc, bytes) { const ptr = alloc(bytes.length); new Uint8Array(memory.buffer, ptr, bytes.length).set(bytes); @@ -107,7 +167,7 @@ function wrapInstance(instance, fallback) { function scanText(text, categories, language) { const source = String(text || ""); if (fallback && hasLoneSurrogate(source)) { - return fallback.scanText(source, categories, language); + return normalizeScanResult(source, fallback.scanText(source, categories, language)); } const payload = callScan(source, language || "auto", categories, -1); const offsets = utf16Offsets(source); @@ -203,6 +263,7 @@ if (typeof module !== "undefined" && module.exports) { module.exports.WASM_CATEGORIES = WASM_CATEGORIES; module.exports.hasLoneSurrogate = hasLoneSurrogate; module.exports.encodeCategories = encodeCategories; + module.exports.normalizeScanResult = normalizeScanResult; } if (typeof globalThis !== "undefined") { globalThis.loadFuckMarkScanWasm = loadFuckMarkScanWasm; diff --git a/editors/wasm/scan_wasm.js b/editors/wasm/scan_wasm.js index 5f01e7cd..4e8efdda 100644 --- a/editors/wasm/scan_wasm.js +++ b/editors/wasm/scan_wasm.js @@ -52,6 +52,66 @@ function encodeCategories(categories) { return String(categories); } +const SEVERITY_RANK = { info: 0, medium: 1, high: 2, critical: 3 }; + +function codePointCount(text) { + let count = 0; + for (let offset = 0; offset < text.length; ) { + const cp = text.codePointAt(offset); + offset += cp > 0xffff ? 2 : 1; + count += 1; + } + return count; +} + +function utf16OffsetToIndex(text, offset) { + let index = 0; + let pos = 0; + while (pos < offset && pos < text.length) { + const cp = text.codePointAt(pos); + pos += cp > 0xffff ? 2 : 1; + index += 1; + } + return index; +} + +function peakSeverity(findings) { + let peak = ""; + for (const finding of findings) { + if ((SEVERITY_RANK[finding.severity] || -1) > (SEVERITY_RANK[peak] || -1)) { + peak = finding.severity; + } + } + return peak; +} + +function normalizeScanResult(source, result) { + const findings = (result.findings || []).map((finding) => { + const offset = finding.offset != null ? finding.offset : 0; + const length = finding.length != null ? finding.length : finding.codepoint > 0xffff ? 2 : 1; + const index = finding.index != null ? finding.index : utf16OffsetToIndex(source, offset); + return { + offset, + length, + index, + codepoint: finding.codepoint, + category: finding.category, + context: finding.context, + severity: finding.severity, + why: finding.why, + remedy: finding.remedy, + }; + }); + return { + total: result.total || 0, + counts: result.counts || {}, + findings, + truncated: Boolean(result.truncated), + highest_severity: result.highest_severity || peakSeverity(findings), + source_length: result.source_length != null ? result.source_length : codePointCount(source), + }; +} + function writeUtf8(memory, alloc, bytes) { const ptr = alloc(bytes.length); new Uint8Array(memory.buffer, ptr, bytes.length).set(bytes); @@ -107,7 +167,7 @@ function wrapInstance(instance, fallback) { function scanText(text, categories, language) { const source = String(text || ""); if (fallback && hasLoneSurrogate(source)) { - return fallback.scanText(source, categories, language); + return normalizeScanResult(source, fallback.scanText(source, categories, language)); } const payload = callScan(source, language || "auto", categories, -1); const offsets = utf16Offsets(source); @@ -203,6 +263,7 @@ if (typeof module !== "undefined" && module.exports) { module.exports.WASM_CATEGORIES = WASM_CATEGORIES; module.exports.hasLoneSurrogate = hasLoneSurrogate; module.exports.encodeCategories = encodeCategories; + module.exports.normalizeScanResult = normalizeScanResult; } if (typeof globalThis !== "undefined") { globalThis.loadFuckMarkScanWasm = loadFuckMarkScanWasm; diff --git a/fuckmark/native_scan.py b/fuckmark/native_scan.py index ed08c453..bc62a722 100644 --- a/fuckmark/native_scan.py +++ b/fuckmark/native_scan.py @@ -90,6 +90,65 @@ def _read_packed_json(lib: CDLL, ptr) -> dict: return json.loads(payload.decode("utf-8")) +def _contains_surrogate(text: str) -> bool: + return any(0xD800 <= ord(character) <= 0xDFFF for character in text) + + +def _scan_via_python( + text: str, + *, + language: str, + categories: list[str] | None, + max_findings: int, +) -> dict: + from .product.scan import SCAN_ALGORITHM_VERSION, scan_hidden_characters + + cap = max(len(text), 1) if max_findings < 0 else max_findings + full = scan_hidden_characters(text, max_findings=cap, language=language or "auto") + if categories is None: + selected = None + else: + from .product.scan import normalize_scan_categories + + selected = normalize_scan_categories(categories) + findings: list[dict[str, object]] = [] + counts: dict[str, int] = {} + peak = "" + rank = {"info": 0, "medium": 1, "high": 2, "critical": 3} + for finding in full.findings: + if selected is not None and finding.category not in selected: + continue + findings.append( + { + "index": finding.index, + "codepoint": finding.codepoint, + "category": finding.category, + "context": finding.context, + "severity": finding.severity, + "why": finding.why, + "remedy": finding.remedy, + } + ) + counts[finding.category] = counts.get(finding.category, 0) + 1 + if rank.get(finding.severity, -1) > rank.get(peak, -1): + peak = finding.severity + return { + "algorithm_version": SCAN_ALGORITHM_VERSION, + "source_length": full.source_length, + "total": len(findings), + "truncated": full.truncated, + "highest_severity": peak, + "counts": counts, + "findings": findings, + } + + +def _clean_via_python(text: str, *, categories: list[str] | None) -> tuple[str, int]: + from .product.scan import clean_hidden_characters + + return clean_hidden_characters(text, categories=categories) + + def classify(codepoint: int) -> str | None: lib = load_library() if lib is None: @@ -113,6 +172,13 @@ def scan_text( ) -> dict: if not isinstance(text, str): raise TypeError("text must be a string") + if _contains_surrogate(text): + return _scan_via_python( + text, + language=language, + categories=categories, + max_findings=max_findings, + ) lib = load_library() if lib is None: raise RuntimeError("native fuckmark-scan library is not available") @@ -140,6 +206,8 @@ def scan_text( def clean_text(text: str, *, categories: list[str] | None = None) -> tuple[str, int]: if not isinstance(text, str): raise TypeError("text must be a string") + if _contains_surrogate(text): + return _clean_via_python(text, categories=categories) lib = load_library() if lib is None: raise RuntimeError("native fuckmark-scan library is not available") diff --git a/fuckmark/webui/scan_wasm.js b/fuckmark/webui/scan_wasm.js index 5f01e7cd..4e8efdda 100644 --- a/fuckmark/webui/scan_wasm.js +++ b/fuckmark/webui/scan_wasm.js @@ -52,6 +52,66 @@ function encodeCategories(categories) { return String(categories); } +const SEVERITY_RANK = { info: 0, medium: 1, high: 2, critical: 3 }; + +function codePointCount(text) { + let count = 0; + for (let offset = 0; offset < text.length; ) { + const cp = text.codePointAt(offset); + offset += cp > 0xffff ? 2 : 1; + count += 1; + } + return count; +} + +function utf16OffsetToIndex(text, offset) { + let index = 0; + let pos = 0; + while (pos < offset && pos < text.length) { + const cp = text.codePointAt(pos); + pos += cp > 0xffff ? 2 : 1; + index += 1; + } + return index; +} + +function peakSeverity(findings) { + let peak = ""; + for (const finding of findings) { + if ((SEVERITY_RANK[finding.severity] || -1) > (SEVERITY_RANK[peak] || -1)) { + peak = finding.severity; + } + } + return peak; +} + +function normalizeScanResult(source, result) { + const findings = (result.findings || []).map((finding) => { + const offset = finding.offset != null ? finding.offset : 0; + const length = finding.length != null ? finding.length : finding.codepoint > 0xffff ? 2 : 1; + const index = finding.index != null ? finding.index : utf16OffsetToIndex(source, offset); + return { + offset, + length, + index, + codepoint: finding.codepoint, + category: finding.category, + context: finding.context, + severity: finding.severity, + why: finding.why, + remedy: finding.remedy, + }; + }); + return { + total: result.total || 0, + counts: result.counts || {}, + findings, + truncated: Boolean(result.truncated), + highest_severity: result.highest_severity || peakSeverity(findings), + source_length: result.source_length != null ? result.source_length : codePointCount(source), + }; +} + function writeUtf8(memory, alloc, bytes) { const ptr = alloc(bytes.length); new Uint8Array(memory.buffer, ptr, bytes.length).set(bytes); @@ -107,7 +167,7 @@ function wrapInstance(instance, fallback) { function scanText(text, categories, language) { const source = String(text || ""); if (fallback && hasLoneSurrogate(source)) { - return fallback.scanText(source, categories, language); + return normalizeScanResult(source, fallback.scanText(source, categories, language)); } const payload = callScan(source, language || "auto", categories, -1); const offsets = utf16Offsets(source); @@ -203,6 +263,7 @@ if (typeof module !== "undefined" && module.exports) { module.exports.WASM_CATEGORIES = WASM_CATEGORIES; module.exports.hasLoneSurrogate = hasLoneSurrogate; module.exports.encodeCategories = encodeCategories; + module.exports.normalizeScanResult = normalizeScanResult; } if (typeof globalThis !== "undefined") { globalThis.loadFuckMarkScanWasm = loadFuckMarkScanWasm; diff --git a/tests/test_language_bindings.py b/tests/test_language_bindings.py index 20de93f5..6e24a211 100644 --- a/tests/test_language_bindings.py +++ b/tests/test_language_bindings.py @@ -23,6 +23,10 @@ def test_c_header_and_binding_docs_exist() -> None: assert "fm_classify" in header assert (ROOT / "bindings" / "README.md").is_file() assert "FUCKMARK_SCAN_LIB" in (ROOT / "bindings" / "README.md").read_text(encoding="utf-8") + example = (ROOT / "bindings" / "README.md").read_text(encoding="utf-8") + assert "-Wl,-rpath,\\$ORIGIN" in example + assert "/tmp/fm-scan" not in example + assert "crates/fuckmark-scan/target/release/fm-scan" in example def test_native_library_matches_python_vectors() -> None: @@ -78,3 +82,25 @@ def test_node_binding_assets_and_tests() -> None: timeout=30, ) assert completed.stdout.strip() == "ok" + + +def test_native_scan_reports_lone_surrogates() -> None: + lone = "\ud800" + payload = scan_text(lone) + assert payload["total"] == 1 + assert payload["source_length"] == 1 + assert payload["truncated"] is False + assert payload["highest_severity"] + assert payload["findings"][0]["index"] == 0 + assert payload["findings"][0]["category"] == "surrogate" + assert payload["findings"][0]["codepoint"] == 0xD800 + mixed = scan_text("a" + lone + "b") + assert mixed["total"] == 1 + assert mixed["findings"][0]["index"] == 1 + assert mixed["source_length"] == 3 + cleaned, removed = clean_text(lone) + assert removed == 1 + assert cleaned == "" + kept, unchanged = clean_text(lone, categories=[]) + assert unchanged == 0 + assert kept == lone diff --git a/tests/test_scan_wasm.py b/tests/test_scan_wasm.py index a7db98b4..01e5d113 100644 --- a/tests/test_scan_wasm.py +++ b/tests/test_scan_wasm.py @@ -179,6 +179,13 @@ def test_wasm_routes_lone_surrogates_through_js_fallback() -> None: const scanned = api.scanText(lone, null, "auto"); if (scanned.total !== 1) throw new Error("scan total " + scanned.total); if (scanned.findings[0].category !== "surrogate") throw new Error("category " + scanned.findings[0].category); + if (scanned.findings[0].index !== 0) throw new Error("missing index " + scanned.findings[0].index); + if (scanned.truncated !== false) throw new Error("truncated " + scanned.truncated); + if (!scanned.highest_severity) throw new Error("missing highest_severity"); + if (scanned.source_length !== 1) throw new Error("source_length " + scanned.source_length); + const mixed = api.scanText("a\uD800b", null, "auto"); + if (mixed.findings[0].index !== 1) throw new Error("mixed index " + mixed.findings[0].index); + if (mixed.source_length !== 3) throw new Error("mixed source_length " + mixed.source_length); const cleaned = api.cleanText(lone, null); if (cleaned.removed !== 1) throw new Error("removed " + cleaned.removed); if (cleaned.cleaned !== "") throw new Error("cleaned leftover");