From bc58d9bb80db25e1199bbf9465c2f92674bf35c4 Mon Sep 17 00:00:00 2001 From: Marco Filho Date: Wed, 1 Jul 2026 03:46:57 -0300 Subject: [PATCH] chore(epic-3): CI, tests to ~84% coverage, ADRs, API docs and 0.2.0 changelog - Add createScraper unit tests via an injected fake daemon (scraper.js coverage 11% -> 94%); full suite is now 22 tests at ~84% line coverage. Add a test:cov script (native node:test coverage). - GitHub Actions CI (.github/workflows/ci.yml) running build + unit tests on Node 22 and 24 (22.12+ for the MCP SDK's require(esm)). - Architecture Decision Records (docs/adr/0001-0003) and docs/API.md. - CHANGELOG 0.2.0 entry and version bump to 0.2.0; README CI/node/license badges. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/ci.yml | 27 ++++++ CHANGELOG.md | 41 ++++++++ README.md | 4 + docs/API.md | 58 ++++++++++++ docs/adr/0001-persistent-daemon.md | 24 +++++ docs/adr/0002-ndjson-ipc.md | 24 +++++ docs/adr/0003-mcp-primary-agent-interface.md | 25 +++++ package.json | 4 +- tests/scraper.test.js | 99 ++++++++++++++++++++ 9 files changed, 305 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ci.yml create mode 100644 docs/API.md create mode 100644 docs/adr/0001-persistent-daemon.md create mode 100644 docs/adr/0002-ndjson-ipc.md create mode 100644 docs/adr/0003-mcp-primary-agent-interface.md create mode 100644 tests/scraper.test.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..31e2ad4 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,27 @@ +name: ci + +on: + push: + branches: [main] + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # The MCP server dynamically loads the ESM MCP SDK, which needs the + # require(esm) support that is stable from Node 22.12+. + node: [22, 24] + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} + cache: npm + + - run: npm ci + - run: npm run build + - run: npm run test:unit diff --git a/CHANGELOG.md b/CHANGELOG.md index acdf2a8..6fcb909 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,46 @@ # Changelog +## [0.2.0] - 2026-07-01 + +Agent-ready release: cloudscraper.js can now be driven by AI agents, keeps solved +sessions hot, and is safe to install in CI/Docker. + +### ✨ New Features + +- **Reusable hot sessions** via `createScraper()` backed by a long-lived Python + daemon (`daemon.py`) β€” repeated requests skip re-solving the challenge. +- **AI-agent interfaces** (closes #1): an MCP server (`cloudscraper-mcp`) exposing + `fetch_protected_url`, `get_cookies`, `solve_challenge`; a LangChain + `DynamicStructuredTool` (`createCloudScraperTool`); and function-calling JSON + Schemas (`functionSchemas`). +- **HTML β†’ Markdown** output (`format: "markdown"`, `htmlToMarkdown`). +- Full HTTP methods on the new SDK (GET/POST/PUT/DELETE/PATCH/HEAD) plus + `cookies()` / `tokens()`, with per-session `proxy`, `retries` (backoff), + `rateLimitPerHost` and `timeoutMs`. + +### πŸ”’ Security + +- **Removed the privileged `postinstall`** (no more `sudo` / Homebrew / + `curl | bash` on `npm install`). Python setup is now opt-in + (`npm run install-deps`), making the package safe for CI/Docker/serverless. + +### πŸ”§ Technical + +- Robust **NDJSON IPC** replacing the fragile positional stdout parsing. +- `HttpMethod` is now a proper string-literal union; fixed the malformed + `repository.url`. +- Upgraded TypeScript to 5.x (required by zod v4); enabled `skipLibCheck`. + +### πŸ§ͺ Quality + +- Offline unit tests (`node --test`) for IPC, daemon client, scraper, tools, + schemas and MCP wiring β€” 22 tests, ~84% line coverage (`npm run test:unit`). +- GitHub Actions CI (Node 22/24) and Architecture Decision Records (`docs/adr/`). + +### πŸ“š Docs + +- README architecture + agent usage; `docs/API.md`; agent examples in `examples/`. + ## [0.1.1] - 2025-06-20 ### ✨ New Features diff --git a/README.md b/README.md index 9400f9e..cdab086 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ # CloudScraper.js +[![ci](https://github.com/maarkN/cloudscraper.js/actions/workflows/ci.yml/badge.svg)](https://github.com/maarkN/cloudscraper.js/actions/workflows/ci.yml) +![node](https://img.shields.io/badge/node-%E2%89%A520-brightgreen) +![license](https://img.shields.io/badge/license-ISC-blue) + A Node.js wrapper for Python-based CloudFlare bypass functionality. This library is a JavaScript port of the popular `cloudscraper` Python library, designed to help developers bypass CloudFlare protection mechanisms in their Node.js applications. ## 🎯 Purpose diff --git a/docs/API.md b/docs/API.md new file mode 100644 index 0000000..d5b65f3 --- /dev/null +++ b/docs/API.md @@ -0,0 +1,58 @@ +# API Reference (v0.2) + +Two ways to use the library: + +- **Legacy** β€” `new CloudScraper()` (one Python process per request). Still supported. +- **v0.2** β€” `createScraper()` (hot, reusable sessions via a daemon) + agent interfaces. + +## `createScraper(options?): Promise` + +```ts +import { createScraper } from "cloudscraper.js"; +const scraper = await createScraper({ format: "markdown", retries: 3 }); +``` + +`CreateScraperOptions`: + +| option | type | default | notes | +|---|---|---|---| +| `usePython3` | boolean | `true` | run the daemon with `python3` vs `python` | +| `proxy` | string | – | `http://user:pass@host:port` | +| `retries` | number | `2` | retries on 429/5xx/network, exponential backoff | +| `rateLimitPerHost` | number | – | max requests/second per host | +| `timeoutMs` | number | `30000` | default per-request timeout | +| `headers` | object | – | default headers merged into every request | +| `format` | `"html" \| "markdown"` | `"html"` | default output format | + +### `Scraper` + +```ts +scraper.get(url, opts?) // also: post, put, delete, patch, head +scraper.cookies(url) // -> Record +scraper.tokens(url) // -> { tokens?, userAgent? } +scraper.close() // release the hot session +``` + +`ScraperResponse`: `{ status, ok, headers, cookies, text(), json(), error }`. +`text()` returns markdown when `format: "markdown"`. + +## AI agents + +```ts +import { + createMcpServer, startStdioMcpServer, // MCP server + createCloudScraperTool, // LangChain DynamicStructuredTool (peer: @langchain/core) + fetchProtectedUrl, getCookies, solveChallenge, // framework-agnostic handlers + functionSchemas, // OpenAI/Anthropic function-calling schemas + htmlToMarkdown, // HTML -> Markdown +} from "cloudscraper.js"; +``` + +- **MCP**: run the `cloudscraper-mcp` binary (see `examples/mcp-client-config.json`). Tools: + `fetch_protected_url`, `get_cookies`, `solve_challenge`. Requires Node β‰₯ 22.12. +- **LangChain**: `const tool = await createCloudScraperTool(await createScraper())`. +- **Function calling**: advertise `functionSchemas` to the model; run `fetchProtectedUrl(scraper, input)` when it calls the tool. + +## Requirements + +Node β‰₯ 20 (β‰₯ 22.12 for the MCP server) Β· Python 3 with `pip install cloudscraper`. diff --git a/docs/adr/0001-persistent-daemon.md b/docs/adr/0001-persistent-daemon.md new file mode 100644 index 0000000..952f989 --- /dev/null +++ b/docs/adr/0001-persistent-daemon.md @@ -0,0 +1,24 @@ +# ADR 0001 β€” Persistent daemon instead of spawn-per-request + +Status: accepted Β· Date: 2026-07-01 + +## Context +The original design spawned a fresh Python process (running `cloudscraper`) for +every request, calling `create_scraper()` each time. This re-solved the +Cloudflare challenge on every call and discarded the solved cookies, so every +request paid ~1–3s of cold-start + solve. It also spawned one process per +concurrent request. + +## Decision +Introduce a single long-lived **daemon** (`daemon.py`) that keeps solved +`cloudscraper` sessions hot in a dict keyed by `sessionId`, served by a thread +pool. The Node SDK manages one daemon per process and multiplexes all requests +over it. + +## Consequences +- Repeated requests on a session skip the challenge β†’ p50 target ~150ms. +- One process serves N sessions instead of one process per request. +- Adds lifecycle management (ready handshake, crash auto-restart) in the client. +- The Python `cloudscraper` engine stays encapsulated behind the daemon, which + keeps the door open to swapping it for a native backend (`cloudscraper-go`) + without changing the SDK or agent interfaces. diff --git a/docs/adr/0002-ndjson-ipc.md b/docs/adr/0002-ndjson-ipc.md new file mode 100644 index 0000000..94ea008 --- /dev/null +++ b/docs/adr/0002-ndjson-ipc.md @@ -0,0 +1,24 @@ +# ADR 0002 β€” NDJSON IPC instead of positional stdout parsing + +Status: accepted Β· Date: 2026-07-01 + +## Context +The old bridge parsed the Python stdout positionally (line 0 = body, line 1 = +status, line 2 = headers), assuming stdout arrives "in lines". A pipe delivers +arbitrary byte chunks, so a large response split across chunks broke that logic β€” +it only worked by luck because the body was single-line base64. + +## Decision +Use **NDJSON** (one JSON object per line) for all Node↔Python IPC. Requests and +responses are correlated by an `id`. The Node side uses a buffered +`NdjsonDecoder` that accumulates chunks and only emits a message on a complete +`\n`-terminated line. Response bodies are carried as base64 in a `bodyB64` field. + +Alternative considered: length-prefixed framing β€” rejected for being harder to +inspect/debug than line-delimited JSON. + +## Consequences +- Robust to arbitrary chunk boundaries and multiple messages per chunk + (covered by tests with a 200 KB body split into 3 KB chunks). +- Enables request multiplexing (concurrent in-flight requests by `id`). +- Minor serialization overhead vs. raw framing β€” acceptable. diff --git a/docs/adr/0003-mcp-primary-agent-interface.md b/docs/adr/0003-mcp-primary-agent-interface.md new file mode 100644 index 0000000..9de4195 --- /dev/null +++ b/docs/adr/0003-mcp-primary-agent-interface.md @@ -0,0 +1,25 @@ +# ADR 0003 β€” MCP as the primary agent interface + +Status: accepted Β· Date: 2026-07-01 + +## Context +Issue #1 asked for "agent" support. There are several ways to expose a tool to +LLM agents: raw function-calling schemas, framework-specific tools (LangChain), +or the Model Context Protocol (MCP). MCP has become the interoperable standard +(Claude, IDEs, generic MCP clients) in 2025–2026. + +## Decision +Ship an **MCP server** (`cloudscraper-mcp`) as the primary interface, exposing +`fetch_protected_url`, `get_cookies` and `solve_challenge`. Provide a LangChain +`DynamicStructuredTool` and provider-agnostic function-calling JSON Schemas as +secondary adapters. All three delegate to the same framework-agnostic handlers +in `src/mcp/tools.ts`. + +The MCP SDK, zod and `@langchain/core` are loaded via **dynamic import** so the +base package stays light and free of ESM/CJS friction; `@langchain/core` is an +optional peer dependency. + +## Consequences +- Maximum interoperability + positions the library on the current agent frontier. +- Cost: the MCP server dynamically loads the ESM SDK, so it requires Node β‰₯ 22.12. +- Shared handlers keep the three interfaces consistent and independently testable. diff --git a/package.json b/package.json index 86a7ff7..7685f50 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "cloudscraper.js", "description": "Python based CloudFlare bypass. Utilizes [VeNoMouS's CloudScraper](https://github.com/VeNoMouS/cloudscraper) project to bypass CloudFlare. All credit goes to him.", - "version": "0.1.1", + "version": "0.2.0", "main": "./built/cloudscraper-js.js", "bin": { "cloudscraper-mcp": "./built/mcp/bin.js" @@ -10,6 +10,8 @@ "test": "node test.js", "pretest:unit": "npm run build", "test:unit": "node --test tests/*.test.js", + "pretest:cov": "npm run build", + "test:cov": "node --test --experimental-test-coverage tests/*.test.js", "build": "rm -rf built && npm run build:ts", "build:ts": "tsc --build", "clean": "rm -rf built && tsc --build --clean", diff --git a/tests/scraper.test.js b/tests/scraper.test.js new file mode 100644 index 0000000..506920b --- /dev/null +++ b/tests/scraper.test.js @@ -0,0 +1,99 @@ +"use strict"; +const test = require("node:test"); +const assert = require("node:assert"); +const { createScraper } = require("../built/scraper.js"); + +// A fake DaemonClient β€” lets us test createScraper end-to-end without Python. +function daemonReturning(fn) { + return { + calls: [], + async send(req) { + this.calls.push(req); + return fn(req, this.calls.length); + }, + async close() {}, + }; +} +const okHtml = (html) => ({ + ok: true, + status: 200, + headers: {}, + cookies: {}, + bodyB64: Buffer.from(html, "utf8").toString("base64"), +}); + +test("get returns HTML by default and hits the daemon with method GET", async () => { + const daemon = daemonReturning(() => okHtml("

Hi

")); + const s = await createScraper({ daemon }); + const res = await s.get("https://x.test/a"); + assert.equal(res.ok, true); + assert.equal(res.status, 200); + assert.equal(res.text(), "

Hi

"); + assert.equal(daemon.calls[0].op, "request"); + assert.equal(daemon.calls[0].method, "GET"); + assert.equal(daemon.calls[0].url, "https://x.test/a"); +}); + +test("format markdown converts HTML (default and per-request override)", async () => { + const d1 = daemonReturning(() => okHtml("

Hi

")); + const s1 = await createScraper({ daemon: d1, format: "markdown" }); + assert.match((await s1.get("https://x.test")).text(), /# Hi/); + + const d2 = daemonReturning(() => okHtml("

Hi

")); + const s2 = await createScraper({ daemon: d2, format: "html" }); + assert.match((await s2.get("https://x.test", { format: "markdown" })).text(), /# Hi/); +}); + +test("post/put/delete/patch/head send the right method and serialize object bodies", async () => { + const daemon = daemonReturning(() => okHtml("ok")); + const s = await createScraper({ daemon }); + await s.post("https://x.test", { body: { a: 1 } }); + await s.put("https://x.test"); + await s.delete("https://x.test"); + await s.patch("https://x.test"); + await s.head("https://x.test"); + assert.deepEqual( + daemon.calls.map((c) => c.method), + ["POST", "PUT", "DELETE", "PATCH", "HEAD"], + ); + assert.equal(daemon.calls[0].body, JSON.stringify({ a: 1 })); +}); + +test("non-retryable error (403) returns an error response without retrying", async () => { + const daemon = daemonReturning(() => ({ + ok: false, + status: 403, + error: { code: "HTTP", message: "denied" }, + })); + const s = await createScraper({ daemon, retries: 2 }); + const res = await s.get("https://x.test"); + assert.equal(res.ok, false); + assert.equal(res.status, 403); + assert.match(res.error.message, /denied/); + assert.throws(() => res.json()); + assert.equal(daemon.calls.length, 1); +}); + +test("retries a transient 5xx then succeeds", async () => { + const daemon = daemonReturning((_req, n) => + n === 1 ? { ok: false, status: 503, error: { code: "HTTP", message: "busy" } } : okHtml("recovered"), + ); + const s = await createScraper({ daemon, retries: 2 }); + const res = await s.get("https://x.test"); + assert.equal(res.ok, true); + assert.equal(res.text(), "recovered"); + assert.equal(daemon.calls.length, 2); +}); + +test("cookies / tokens / close talk to the daemon", async () => { + const daemon = daemonReturning((req) => { + if (req.op === "cookies") return { ok: true, cookies: { cf: "1" } }; + if (req.op === "tokens") return { ok: true, tokens: { cf: "1" }, userAgent: "UA/9" }; + return { ok: true }; + }); + const s = await createScraper({ daemon }); + assert.deepEqual(await s.cookies("https://x.test"), { cf: "1" }); + assert.equal((await s.tokens("https://x.test")).userAgent, "UA/9"); + await s.close(); + assert.ok(daemon.calls.some((c) => c.op === "close_session")); +});