diff --git a/src/index.ts b/src/index.ts index 1ed703e..409454e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,11 +1,11 @@ import { URL } from "url"; import { Parser } from "htmlparser2"; -import nodeFetch from "node-fetch"; import UnexpectedError from "./unexpectedError"; import { schema, keys } from "./schema"; import { Metadata, Opts } from "./types"; import { decode as he_decode } from "he"; import { decode as iconv_decode } from "iconv-lite"; +import { safeFetch } from "./ssrfGuard"; type ParserContext = { isHtml?: boolean; @@ -48,11 +48,12 @@ function unfurl(url: string, opts?: Opts): Promise { async function getPage(url: string, opts: Opts) { const res = await (opts.fetch ? opts.fetch(url) - : nodeFetch(new URL(url), { + : safeFetch(url, { headers: opts.headers, size: opts.size, follow: opts.follow, timeout: opts.timeout, + allowPrivateIPs: opts.allowPrivateIPs, })); const buf = Buffer.from(await res.arrayBuffer()); @@ -122,15 +123,18 @@ async function getPage(url: string, opts: Opts) { return buf.toString(); } -function getRemoteMetadata(url: string, { fetch = nodeFetch }: Opts) { +function getRemoteMetadata(url: string, { fetch, allowPrivateIPs }: Opts) { return async function ({ oembed, metadata }) { if (!oembed) { return metadata; } const target = new URL(he_decode(oembed.href), url); + const doFetch = fetch + ? (u: string) => fetch(u) + : (u: string) => safeFetch(u, { allowPrivateIPs }); - let res = await fetch(target.href); + let res = await doFetch(target.href); let contentType = res.headers.get("Content-Type"); const status = res.status; @@ -138,7 +142,7 @@ function getRemoteMetadata(url: string, { fetch = nodeFetch }: Opts) { // try again using HTTPS target.protocol = "https:"; - res = await fetch(target.href); + res = await doFetch(target.href); contentType = res.headers.get("Content-Type"); } diff --git a/src/ssrfGuard.ts b/src/ssrfGuard.ts new file mode 100644 index 0000000..d59d000 --- /dev/null +++ b/src/ssrfGuard.ts @@ -0,0 +1,190 @@ +/** + * SSRF guard — resolves a URL's hostname and rejects requests targeting + * private, loopback, link-local, or otherwise unsafe IP ranges. + * + * Used to prevent attackers from coercing unfurl into making HTTP requests + * to internal infrastructure (cloud metadata endpoints, internal APIs, + * private network services, etc). + */ + +import { URL } from "url"; +import nodeFetch, { RequestInit, Response } from "node-fetch"; +import { promises as dns } from "dns"; +import { isIP } from "net"; + +export class SSRFError extends Error { + constructor(message: string) { + super(message); + this.name = "SSRFError"; + } +} + +const ALLOWED_PROTOCOLS = new Set(["http:", "https:"]); + +/** + * Returns true if the given IP address is in a range that should not be + * reachable from a public-input URL fetcher. Covers IPv4 and IPv6. + */ +export function isPrivateOrReservedIP(ip: string): boolean { + const family = isIP(ip); + if (family === 0) return false; + ip = ip.toLowerCase(); + + if (family === 4) { + const [a, b] = ip.split(".").map((p) => parseInt(p, 10)); + + // 0.0.0.0/8 — current network + if (a === 0) return true; + // 10.0.0.0/8 — private + if (a === 10) return true; + // 127.0.0.0/8 — loopback + if (a === 127) return true; + // 169.254.0.0/16 — link-local (includes cloud metadata 169.254.169.254) + if (a === 169 && b === 254) return true; + // 172.16.0.0/12 — private + if (a === 172 && b >= 16 && b <= 31) return true; + // 192.0.0.0/24, 192.0.2.0/24 — reserved/documentation + if (a === 192 && b === 0) return true; + // 192.168.0.0/16 — private + if (a === 192 && b === 168) return true; + // 198.18.0.0/15 — benchmarking + if (a === 198 && (b === 18 || b === 19)) return true; + // 198.51.100.0/24 — documentation + if (a === 198 && b === 51) return true; + // 203.0.113.0/24 — documentation + if (a === 203 && b === 0) return true; + // 224.0.0.0/4 — multicast + if (a >= 224 && a <= 239) return true; + // 240.0.0.0/4 — reserved (includes 255.255.255.255 broadcast) + if (a >= 240) return true; + + return false; + } + + // IPv6 + // Loopback ::1 + if (ip === "::1") return true; + // Unspecified :: + if (ip === "::") return true; + // IPv4-mapped IPv6 (::ffff:x.x.x.x) — recurse on the embedded v4 + const v4MappedMatch = ip.match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/); + if (v4MappedMatch) return isPrivateOrReservedIP(v4MappedMatch[1]); + // Unique local fc00::/7 + if (ip.startsWith("fc") || ip.startsWith("fd")) return true; + // Link-local fe80::/10 + if ( + ip.startsWith("fe8") || + ip.startsWith("fe9") || + ip.startsWith("fea") || + ip.startsWith("feb") + ) + return true; + // Multicast ff00::/8 + if (ip.startsWith("ff")) return true; + + return false; +} + +/** + * Validates that a URL is safe to fetch from a public-input link-preview + * context: must be http(s) and must not resolve to a private/reserved IP. + * + * Throws SSRFError if the URL fails any check. Resolves silently if safe. + */ +export async function assertSafeURL( + rawUrl: string, + allowPrivateIPs = false +): Promise { + if (allowPrivateIPs) return; + let parsed: URL; + try { + parsed = new URL(rawUrl); + } catch { + throw new SSRFError(`Invalid URL: ${rawUrl}`); + } + + if (!ALLOWED_PROTOCOLS.has(parsed.protocol)) { + throw new SSRFError(`Disallowed protocol: ${parsed.protocol}`); + } + + const hostname = parsed.hostname; + if (!hostname) { + throw new SSRFError("URL has no hostname"); + } + + // If the hostname is already a literal IP, validate it directly. + if (isIP(hostname) !== 0) { + if (isPrivateOrReservedIP(hostname)) { + throw new SSRFError(`Disallowed destination IP: ${hostname}`); + } + return; + } + + // Otherwise resolve all addresses and reject if ANY resolve to a + // forbidden range. (Defense against DNS responses with mixed records.) + let addresses: { address: string; family: number }[]; + try { + addresses = await dns.lookup(hostname, { all: true }); + } catch (err) { + throw new SSRFError( + `Failed to resolve ${hostname}: ${(err as Error).message}` + ); + } + + if (addresses.length === 0) { + throw new SSRFError(`No addresses resolved for ${hostname}`); + } + + for (const { address } of addresses) { + if (isPrivateOrReservedIP(address)) { + throw new SSRFError( + `Hostname ${hostname} resolves to disallowed IP ${address}` + ); + } + } +} + +/** + * Wraps node-fetch with manual redirect handling so that each redirect + * target is re-validated against the SSRF guard. node-fetch's automatic + * redirect-following bypasses any one-shot pre-fetch validation, so we + * have to walk the chain ourselves. + */ +export async function safeFetch( + initialUrl: string, + init: RequestInit & { follow?: number; allowPrivateIPs?: boolean } = {} +): Promise { + const maxRedirects = typeof init.follow === "number" ? init.follow : 20; + const allowPrivateIPs = init.allowPrivateIPs === true; + + // Strip our custom keys before passing to node-fetch. + delete init.follow; + delete init.allowPrivateIPs; + + let currentUrl = initialUrl; + + for (let hop = 0; hop <= maxRedirects; hop++) { + await assertSafeURL(currentUrl, allowPrivateIPs); + + const res = await nodeFetch(currentUrl, { + ...init, + redirect: "manual", + }); + + // Not a redirect — return as-is. + if (res.status < 300 || res.status >= 400) { + return res; + } + + const location = res.headers.get("location"); + if (!location) { + // Redirect status with no Location header — return what we got. + return res; + } + + // Resolve relative redirects against the URL that produced them. + currentUrl = new URL(location, currentUrl).href; + } + + throw new SSRFError(`Too many redirects (>${maxRedirects})`); +} diff --git a/src/types.ts b/src/types.ts index af5db5a..3ff3079 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,5 +1,6 @@ import { HeadersInit } from "node-fetch"; export type Opts = { + allowPrivateIPs?: boolean; /** support retreiving oembed metadata */ oembed?: boolean; /** req/res timeout in ms, it resets on redirect. 0 to disable (OS limit applies) */ diff --git a/test/basic/test.ts b/test/basic/test.ts index 3940e89..fec7892 100644 --- a/test/basic/test.ts +++ b/test/basic/test.ts @@ -8,7 +8,10 @@ test("should handle content which is escaped badly", async () => { "Content-Type": "text/html", }); - const result = await unfurl("http://localhost/html/double-escaped-edge-case"); + const result = await unfurl( + "http://localhost/html/double-escaped-edge-case", + { allowPrivateIPs: true } + ); expect(result.description).toEqual('"'); }); @@ -20,7 +23,9 @@ test("should detect title, description, keywords and canonical URL", async () => "Content-Type": "text/html", }); - const result = await unfurl("http://localhost/html/basic"); + const result = await unfurl("http://localhost/html/basic", { + allowPrivateIPs: true, + }); const expected = { favicon: "http://localhost/favicon.ico", @@ -42,7 +47,9 @@ test("should detect title, description, keywords and canonical URL even when the "Content-Type": "text/html", }); - const result = await unfurl("http://localhost/html/basic-body"); + const result = await unfurl("http://localhost/html/basic-body", { + allowPrivateIPs: true, + }); const expected = { favicon: "http://localhost/favicon.ico", @@ -62,7 +69,9 @@ test("should detect last dupe of title, description and keywords", async () => { "Content-Type": "text/html", }); - const result = await unfurl("http://localhost/html/basic-duplicates"); + const result = await unfurl("http://localhost/html/basic-duplicates", { + allowPrivateIPs: true, + }); const expected = { favicon: "http://localhost/favicon.ico", @@ -81,7 +90,9 @@ test("should detect last dupe of title, description and keywords", async () => { "Content-Type": "text/html", }); - const result = await unfurl("http://localhost/html/keyword-edge-cases"); + const result = await unfurl("http://localhost/html/keyword-edge-cases", { + allowPrivateIPs: true, + }); const expected = { favicon: "http://localhost/favicon.ico", diff --git a/test/encoding/test.ts b/test/encoding/test.ts index 0b35a93..47fae34 100644 --- a/test/encoding/test.ts +++ b/test/encoding/test.ts @@ -9,7 +9,9 @@ test("should detect GB2312 charset (HTML 4) and convert to UTF-8", async () => { "Content-Type": "text/html", }); - const result = await unfurl("http://localhost/html4/gb2312"); + const result = await unfurl("http://localhost/html4/gb2312", { + allowPrivateIPs: true, + }); const expected = { description: @@ -28,7 +30,9 @@ test("should detect GB2312 charset (HTML 5) and convert to UTF-8", async () => { "Content-Type": "text/html", }); - const result = await unfurl("http://localhost/html5/gb2312"); + const result = await unfurl("http://localhost/html5/gb2312", { + allowPrivateIPs: true, + }); const expected = { description: @@ -47,7 +51,9 @@ test("should detect EUC-JP charset (HTML 5) and convert to UTF-8", async () => { "Content-Type": "text/html", }); - const result = await unfurl("http://localhost/html5/euc-jp"); + const result = await unfurl("http://localhost/html5/euc-jp", { + allowPrivateIPs: true, + }); const expected = { description: diff --git a/test/general/content-type.test.ts b/test/general/content-type.test.ts index fa35ef8..760cfd0 100644 --- a/test/general/content-type.test.ts +++ b/test/general/content-type.test.ts @@ -9,7 +9,7 @@ test("should throw bad content type error", async () => { "Content-Type": "image/png", }); - await unfurl("http://localhost/image"); + await unfurl("http://localhost/image", { allowPrivateIPs: true }); } catch (err) { expect(err.name).toEqual(UnexpectedError.EXPECTED_HTML.name); expect(err.message).toEqual(UnexpectedError.EXPECTED_HTML.message); diff --git a/test/general/options.test.ts b/test/general/options.test.ts index 5de4567..077e89d 100644 --- a/test/general/options.test.ts +++ b/test/general/options.test.ts @@ -23,6 +23,7 @@ test("should respect oembed", async () => { const result = await unfurl("http://localhost/html/oembed", { oembed: false, + allowPrivateIPs: true, }); expect(result.oEmbed).toEqual(undefined); diff --git a/test/general/status-code.test.ts b/test/general/status-code.test.ts index ba28d6e..2a57591 100644 --- a/test/general/status-code.test.ts +++ b/test/general/status-code.test.ts @@ -5,9 +5,9 @@ import UnexpectedError from "../../src/unexpectedError"; test("should throw if status code not 200", () => { nock("http://localhost").get("/html/return-404").reply(404); - return expect(unfurl("http://localhost/html/return-404")).rejects.toThrow( - new UnexpectedError(UnexpectedError.BAD_HTTP_STATUS) - ); + return expect( + unfurl("http://localhost/html/return-404", { allowPrivateIPs: true }) + ).rejects.toThrow(new UnexpectedError(UnexpectedError.BAD_HTTP_STATUS)); }); test("should not throw if status code is 200", async () => { @@ -16,6 +16,6 @@ test("should not throw if status code is 200", async () => { }); return expect( - unfurl("http://localhost/html/return-200") + unfurl("http://localhost/html/return-200", { allowPrivateIPs: true }) ).resolves.toBeTruthy(); }); diff --git a/test/general/url.test.ts b/test/general/url.test.ts index c43ccf3..fd4ba1e 100644 --- a/test/general/url.test.ts +++ b/test/general/url.test.ts @@ -5,7 +5,9 @@ test("should not throw when provided non-ascii url", async () => { let err; try { - await unfurl("http://localhost/日本語urlってどうよ"); + await unfurl("http://localhost/日本語urlってどうよ", { + allowPrivateIPs: true, + }); } catch (e) { err = e; } finally { diff --git a/test/oembed/test.ts b/test/oembed/test.ts index 144f4a5..9b4f837 100644 --- a/test/oembed/test.ts +++ b/test/oembed/test.ts @@ -12,7 +12,9 @@ test("should noop and not throw for wrong content type", async () => { "Content-Type": "text/html", }); - const result = await unfurl("http://localhost/html/oembed-broken"); + const result = await unfurl("http://localhost/html/oembed-broken", { + allowPrivateIPs: true, + }); expect(result.oEmbed).toEqual(undefined); }); @@ -30,7 +32,9 @@ test("width/height should be numbers", async () => { "Content-Type": "application/json", }); - const result = await unfurl("http://localhost/html/oembed"); + const result = await unfurl("http://localhost/html/oembed", { + allowPrivateIPs: true, + }); expect(result.oEmbed?.type).toEqual("video"); const oEmbed = @@ -56,7 +60,9 @@ test("should decode entities in OEmbed URL", async () => { "Content-Type": "application/json", }); - const result = await unfurl("http://localhost/html/oembed"); + const result = await unfurl("http://localhost/html/oembed", { + allowPrivateIPs: true, + }); expect(result.oEmbed?.type).toEqual("video"); const oEmbed = @@ -82,7 +88,9 @@ test("should prefer fetching JSON oEmbed", async () => { "Content-Type": "application/json", }); - const result = await unfurl("http://localhost/html/oembed-multi"); + const result = await unfurl("http://localhost/html/oembed-multi", { + allowPrivateIPs: true, + }); const expected = { version: "1.0", @@ -124,7 +132,9 @@ test("should upgrade to HTTPS if needed", async () => { "Content-Type": "application/json", }); - const result = await unfurl("http://localhost/html/oembed-http"); + const result = await unfurl("http://localhost/html/oembed-http", { + allowPrivateIPs: true, + }); expect(result.oEmbed?.version).toEqual("1.0"); }); @@ -142,7 +152,9 @@ test("should build oEmbed from JSON", async () => { "Content-Type": "application/json", }); - const result = await unfurl("http://localhost/html/oembed"); + const result = await unfurl("http://localhost/html/oembed", { + allowPrivateIPs: true, + }); const expected = { version: "1.0", @@ -178,7 +190,9 @@ test("should build oEmbed from XML", async () => { "Content-Type": "text/xml", }); - const result = await unfurl("http://localhost/html/oembed-xml"); + const result = await unfurl("http://localhost/html/oembed-xml", { + allowPrivateIPs: true, + }); const expected = { html: '', @@ -216,7 +230,9 @@ test("should build oEmbed from XML with CDATA", async () => { "Content-Type": "text/xml", }); - const result = await unfurl("http://localhost/html/oembed-xml-cdata"); + const result = await unfurl("http://localhost/html/oembed-xml-cdata", { + allowPrivateIPs: true, + }); const expected = { height: 450, diff --git a/test/open_graph/test.ts b/test/open_graph/test.ts index 686603e..c08139c 100644 --- a/test/open_graph/test.ts +++ b/test/open_graph/test.ts @@ -8,7 +8,9 @@ test("should build videos[]", async () => { "Content-Type": "text/html", }); - const result = await unfurl("http://localhost/open_graph/videos"); + const result = await unfurl("http://localhost/open_graph/videos", { + allowPrivateIPs: true, + }); const expected = { videos: [ { @@ -41,7 +43,9 @@ test("should build images[]", async () => { "Content-Type": "text/html", }); - const result = await unfurl("http://localhost/open_graph/images"); + const result = await unfurl("http://localhost/open_graph/images", { + allowPrivateIPs: true, + }); const expected = { images: [ { @@ -79,7 +83,9 @@ test("should build audio[]", async () => { "Content-Type": "text/html", }); - const result = await unfurl("http://localhost/open_graph/audio"); + const result = await unfurl("http://localhost/open_graph/audio", { + allowPrivateIPs: true, + }); const expected = { audio: [ { @@ -109,7 +115,9 @@ test("should quality relative urls", async () => { "Content-Type": "text/html", }); - const result = await unfurl("http://localhost/open_graph/relative_url"); + const result = await unfurl("http://localhost/open_graph/relative_url", { + allowPrivateIPs: true, + }); const expected = { images: [ { @@ -134,7 +142,9 @@ test("should build article[]", async () => { "Content-Type": "text/html", }); - const result = await unfurl("http://localhost/open_graph/article"); + const result = await unfurl("http://localhost/open_graph/article", { + allowPrivateIPs: true, + }); const expected = { type: "article", articles: [ diff --git a/test/ssrf/test.ts b/test/ssrf/test.ts new file mode 100644 index 0000000..c0b25f8 --- /dev/null +++ b/test/ssrf/test.ts @@ -0,0 +1,127 @@ +import { + assertSafeURL, + isPrivateOrReservedIP, + SSRFError, + safeFetch, +} from "../../src/ssrfGuard"; +import { unfurl } from "../../src/index"; +import nock from "nock"; + +describe("isPrivateOrReservedIP", () => { + test.each([ + ["127.0.0.1", true], + ["10.0.0.1", true], + ["10.255.255.255", true], + ["172.16.0.1", true], + ["172.31.255.255", true], + ["192.168.1.1", true], + ["169.254.169.254", true], // AWS metadata + ["0.0.0.0", true], + ["255.255.255.255", true], + ["224.0.0.1", true], // multicast + ["::1", true], // IPv6 loopback + ["fe80::1", true], // IPv6 link-local + ["fc00::1", true], // IPv6 unique local + ["::ffff:127.0.0.1", true], // IPv4-mapped IPv6 + ])("rejects private/reserved IP %s", (ip, expected) => { + expect(isPrivateOrReservedIP(ip)).toBe(expected); + }); + + test.each([ + ["8.8.8.8", false], + ["1.1.1.1", false], + ["172.15.0.1", false], // just outside private range + ["172.32.0.1", false], // just outside private range + ["2606:4700:4700::1111", false], // Cloudflare DNS + ])("allows public IP %s", (ip, expected) => { + expect(isPrivateOrReservedIP(ip)).toBe(expected); + }); +}); + +describe("assertSafeURL", () => { + test("rejects loopback literal", async () => { + await expect(assertSafeURL("http://127.0.0.1/foo")).rejects.toThrow( + SSRFError + ); + }); + + test("rejects AWS metadata IP", async () => { + await expect( + assertSafeURL("http://169.254.169.254/latest/meta-data/") + ).rejects.toThrow(SSRFError); + }); + + test("rejects private IP", async () => { + await expect(assertSafeURL("http://192.168.1.1/admin")).rejects.toThrow( + SSRFError + ); + }); + + test("rejects file:// protocol", async () => { + await expect(assertSafeURL("file:///etc/passwd")).rejects.toThrow( + SSRFError + ); + }); + + test("rejects gopher:// protocol", async () => { + await expect(assertSafeURL("gopher://localhost/")).rejects.toThrow( + SSRFError + ); + }); + + test("rejects malformed URL", async () => { + await expect(assertSafeURL("not a url")).rejects.toThrow(SSRFError); + }); + + test("allows public hostname", async () => { + await expect(assertSafeURL("https://example.com")).resolves.toBeUndefined(); + }); + + test("rejects localhost hostname (resolves to loopback)", async () => { + await expect(assertSafeURL("http://localhost/")).rejects.toThrow(SSRFError); + }); +}); + +describe("unfurl SSRF integration", () => { + test("unfurl rejects direct loopback", async () => { + await expect(unfurl("http://127.0.0.1:8080")).rejects.toThrow(SSRFError); + }); + + test("unfurl rejects AWS metadata endpoint", async () => { + await expect( + unfurl( + "http://169.254.169.254/latest/meta-data/iam/security-credentials/" + ) + ).rejects.toThrow(SSRFError); + }); +}); + +describe("safeFetch redirect handling", () => { + afterEach(() => { + nock.cleanAll(); + }); + + test("follows redirects when validation is bypassed", async () => { + nock("http://localhost") + .get("/hop1") + .reply(302, "", { Location: "http://localhost/hop2" }) + .get("/hop2") + .reply(200, "ok"); + + const res = await safeFetch("http://localhost/hop1", { + allowPrivateIPs: true, + }); + expect(res.status).toBe(200); + }); + + test("rejects too many redirects", async () => { + nock("http://localhost") + .get("/loop") + .times(25) + .reply(302, "", { Location: "http://localhost/loop" }); + + await expect( + safeFetch("http://localhost/loop", { allowPrivateIPs: true, follow: 3 }) + ).rejects.toThrow(SSRFError); + }); +}); diff --git a/test/twitter_card/test.ts b/test/twitter_card/test.ts index a48f018..0dfdad0 100644 --- a/test/twitter_card/test.ts +++ b/test/twitter_card/test.ts @@ -9,7 +9,9 @@ test("should build players[]", async () => { "Content-Type": "text/html", }); - const result = await unfurl("http://localhost/twitter_card/players"); + const result = await unfurl("http://localhost/twitter_card/players", { + allowPrivateIPs: true, + }); const expected = { players: [ { @@ -35,7 +37,9 @@ test("should build images[]", async () => { "Content-Type": "text/html", }); - const result = await unfurl("http://localhost/twitter_card/images"); + const result = await unfurl("http://localhost/twitter_card/images", { + allowPrivateIPs: true, + }); const expected = { images: [ { @@ -59,7 +63,9 @@ test("should build apps[]", async () => { "Content-Type": "text/html", }); - const result = await unfurl("http://localhost/twitter_card/apps"); + const result = await unfurl("http://localhost/twitter_card/apps", { + allowPrivateIPs: true, + }); const expected = { apps: { googleplay: { @@ -86,7 +92,9 @@ test("should quality relative urls", async () => { "Content-Type": "text/html", }); - const result = await unfurl("http://localhost/twitter_card/relative_url"); + const result = await unfurl("http://localhost/twitter_card/relative_url", { + allowPrivateIPs: true, + }); const expected = { images: [ { @@ -118,7 +126,9 @@ test("should build card", async () => { "Content-Type": "text/html", }); - const result = await unfurl("http://localhost/twitter_card/multi"); + const result = await unfurl("http://localhost/twitter_card/multi", { + allowPrivateIPs: true, + }); const expected = { apps: {