diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ffd9969..51239e6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,21 +9,6 @@ on: permissions: contents: read -env: - COMPAT_PKGS: >- - github.com/chainreactors/fingers - github.com/chainreactors/fingers/alias - github.com/chainreactors/fingers/common - github.com/chainreactors/fingers/ehole - github.com/chainreactors/fingers/favicon - github.com/chainreactors/fingers/fingerprinthub - github.com/chainreactors/fingers/fingers - github.com/chainreactors/fingers/goby - github.com/chainreactors/fingers/nmap - github.com/chainreactors/fingers/resources - github.com/chainreactors/fingers/wappalyzer - github.com/chainreactors/fingers/xray - jobs: test: name: test @@ -36,52 +21,70 @@ jobs: - name: Setup Go uses: actions/setup-go@v5 with: - go-version: '1.22.x' + go-version: 'stable' cache: true + # go.mod declares go 1.17: vet's stdversion check flags any newer + # standard library API, and tidy must not raise the go line. + - name: Tidy + run: | + go mod tidy -go=1.17 -compat=1.17 + git diff --exit-code go.mod go.sum - name: Vet - run: go vet -tags "re2_cgo re2_static" ./... + run: go vet ./... - name: Build - run: go build -tags "re2_cgo re2_static" ./... + run: go build ./... - name: Test - run: go test -tags "re2_cgo re2_static" -short -timeout 300s -race -count=1 ./... + run: go test -short -timeout 300s -race -count=1 ./... - go117-build: - name: Go 1.17 compatibility build + re2: + name: re2 module runs-on: ubuntu-latest + defaults: + run: + working-directory: re2 steps: - name: Checkout uses: actions/checkout@v4 - - name: Setup Go for vendoring + - name: Setup Go uses: actions/setup-go@v5 with: - go-version: '1.22.x' + go-version: '1.24.x' cache: true - - name: Prepare vendor tree - run: go mod vendor + - name: Vet + run: go vet -tags "re2_cgo re2_static" ./... + + - name: Test + run: go test -tags "re2_cgo re2_static" -count=1 ./... + + go117: + name: Go 1.17 module build and test + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 - name: Setup Go 1.17 uses: actions/setup-go@v5 with: go-version: '1.17.x' - - name: Build with Go 1.17 + - name: Build env: - GO111MODULE: off - CGO_ENABLED: '0' - run: | - export GOPATH="${RUNNER_TEMP}/gopath" - src="${GOPATH}/src/github.com/chainreactors/fingers" - mkdir -p "$(dirname "${src}")" - cp -a . "${src}" - cd "${src}" - go build -tags="noembed goregexp" ${COMPAT_PKGS} + GOTOOLCHAIN: local + run: go build ./... + + - name: Test + env: + GOTOOLCHAIN: local + run: go test -short -timeout 300s -count=1 ./... go111-build: name: Go 1.11 compatibility build diff --git a/.gitignore b/.gitignore index a7c8889..c132f3b 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,5 @@ resources/nmap-service-probes.txt /CLAUDE.md /MIGRATION.md vendor/ +/.judge-data/ +/scripts/__pycache__/ diff --git a/README.md b/README.md index 4538fb0..97580b6 100644 --- a/README.md +++ b/README.md @@ -1,177 +1,210 @@ - -## Introduce - -多指纹库聚合识别引擎. 当前支持`fingers(主指纹库)` `wappalyzer`, `fingerprinthub`, `ehole`, `goby` 指纹 - -不用再挑选指纹识别的工具, AllInOne一站式实现 - -使用了fingers的工具: - -* ⭐ [spray](https://github.com/chainreactors/spray) **最佳实践**, 集合了目录爆破, 指纹识别, 信息收集等等功能的超强性能的http fuzz工具 -* [gogo](https://github.com/chainreactors/gogo), 使用了fingers原生指纹库, 红队向的自动化扫描引擎 -* [zombie](https://github.com/chainreactors/zombie), 在爆破前使用fingers进行指纹验证, 提高爆破效率 - -(任何使用了fingers的工具欢迎在issue中告诉我, 我会将你的工具添加到这里) - -## Features - -* 支持多指纹库聚合识别 - * ✅ fingers 原生指纹库 - * ✅ [wappalyzer](https://github.com/projectdiscovery/wappalyzergo) - * ✅ [fingerprinthub](https://github.com/0x727/FingerprintHub) - * ✅ [ehole](https://github.com/EdgeSecurityTeam/EHole) - * ✅ goby -* 支持多指纹源favicon识别 -* 超强性能, 单个站点识别 <100ms. 重写了各指纹库的引擎, 并极大优化了性能 -* 聚合输出, 多指纹库的结果将会自动整合 -* 支持CPE的URI, FSB, WFN格式输出 - -### morefingers - -https://github.com/chainreactors/morefingers - -fingers的拓展引擎, 有更全更大的指纹库. - -从对闭源工具的逆向得到的指纹库, 为了避免可能存在的纠纷, 不提供开源版本. - -## QuickStart - -`go get github.com/chainreactors/fingers@master` - -### Example - -document: https://chainreactors.github.io/wiki/libs/fingers/ - -调用内置所有进行指纹引擎识别, 示例: - -```golang -func TestEngine(t *testing.T) { - engine, err := NewEngine() - if err != nil { - panic(err) - } - resp, err := http.Get("http://127.0.0.1:8080/") - if err != nil { - return - } - content := httputils.ReadRaw(resp) - frames, err := engine.DetectContent(content) - if err != nil { - return - } - fmt.Println(frames.String()) -} -``` - -调用SDK识别Favicon指纹, 示例: - -```golang -func TestFavicon(t *testing.T) { - engine, err := NewEngine() - if err != nil { - panic(err) - } - resp, err := http.Get("http://baidu.com/favicon.ico") - if err != nil { - return - } - content := httputils.ReadRaw(resp) - body, _, _ := httputils.SplitHttpRaw(content) - frame := engine.DetectFavicon(body) - fmt.Println(frame.String()) -} -``` - -更多用法请见: https://chainreactors.github.io/wiki/libs/fingers/sdk/ - -## fingers 引擎 - -fingers指纹引擎是目前特性最丰富, 性能最强的指纹规则库. - -* 支持多种方式规则配置 -* 支持多种方式的版本号匹配 -* 404/favicon/waf/cdn/供应链指纹识别 -* 主动指纹识别 -* 超强性能, 采用了缓存,正则预编译,默认端口,优先级等等算法提高引擎性能 -* 重点指纹,指纹来源与tag标记 - - -### 内置指纹库 - -指纹库位于: https://github.com/chainreactors/templates/tree/master/fingers - -文档: https://chainreactors.github.io/wiki/libs/fingers/rule/ - -tcp指纹与http指纹为同一格式, 但通过不同的文件进行管理 - -### 完整的配置 - -fingers设计的核心思路是命中一个指纹仅需要一条规则, 因此配置的多条规则中, 只需要任意一条命中即标记为命中, 需要在编写指纹的时候注意找到最能匹配目标框架的那条规则. - -一个完整的配置: - -```yaml -- name: frame # 指纹名字, 匹配到的时候输出的值 - default_port: # 指纹的默认端口, 加速匹配. tcp指纹如果匹配到第一个就会结束指纹匹配, http则会继续匹配, 所以默认端口对http没有特殊优化 - - '1111' - protocol: http # tcp/http, 默认为http - rule: - - version: v1.1.1 # 可不填, 默认为空, 表示无具体版本 - regexps: # 匹配的方式 - vuln: # 匹配到vuln的正则, 如果匹配到, 会输出framework为name的同时, 还会添加vuln为vuln的漏洞信息 - - version:(.*) # vuln只支持正则, 同时支持版本号匹配, 使用括号的正则分组. 只支持第一组 - regexp: # 匹配指纹正则 - - "finger.*test" - # 除了正则, 还支持其他类型的匹配, 包括以下方式 - header: # 仅http协议可用, 匹配header中包含的数据 - - string - body: # 包含匹配, 非正则表达式 - - string - md5: # 匹配body的md5hash - - [md5] - mmh3: # 匹配body的mmh3hash - - [mmh3] - - # 只有上面规则中的至少一条命中才会执行version - version: - - version:(.*) # 某些情况下难以同时编写指纹的正则与关于版本的正则, 可以特地为version写一条正则 - - favicon: # favicon的hash值, 仅http生效 - md5: - - f7e3d97f404e71d302b3239eef48d5f2 - mmh3: - - '516963061' - level: 1 # 0代表不需要主动发包, 1代表需要额外主动发起请求. 如果当前level为0则不会发送数据, 但是依旧会进行被动的指纹匹配. - send_data: "info\n" # 匹配指纹需要主动发送的数据 - vuln: frame_unauthorized # 如果regexps中的vuln命中, 则会输出漏洞名称. 某些漏洞也可以通过匹配关键字识别, 因此一些简单的poc使用指纹的方式实现, 复杂的poc请使用-e下的nuclei yaml配置 - -``` - -为了压缩体积, 没有特别指定的参数可以留空会使用默认值。 - -在两个配置文件中包含大量案例可供参考。 - -但实际上大部分字段都不需要配置, 仅作为特殊情况下的能力储备。 - -每个指纹都可以有多个rule, 每个rule中都有一个regexps, 每个regexps有多条不同种类的字符串/正则/hash - - -## TODO - -- [x] 指纹名重定向, 统一多指纹库的同一指纹不同名问题 -- [x] 指纹黑名单, 用于过滤指纹库中的垃圾指纹 -- [x] 更丰富的CPE相关特性支持 -- [ ] 更优雅的与nuclei或其他漏洞库联动 -- 支持更多引擎 - - [ ] [nuclei technologies](https://github.com/projectdiscovery/nuclei-templates/tree/main/http/technologies) 实现 - - [ ] fingerprinthub v4 - - [ ] tidefinger - - [ ] kscan - - [ ] nmap - -## Thanks - -* [wappalyzer](https://github.com/projectdiscovery/wappalyzergo) -* [fingerprinthub](https://github.com/0x727/FingerprintHub) -* [ehole](https://github.com/EdgeSecurityTeam/EHole) -* goby @XiaoliChan @9bie + +## Introduce + +多指纹库聚合识别引擎. 当前支持`fingers(主指纹库)` `wappalyzer`, `fingerprinthub`, `ehole`, `goby` 指纹 + +不用再挑选指纹识别的工具, AllInOne一站式实现 + +使用了fingers的工具: + +* ⭐ [spray](https://github.com/chainreactors/spray) **最佳实践**, 集合了目录爆破, 指纹识别, 信息收集等等功能的超强性能的http fuzz工具 +* [gogo](https://github.com/chainreactors/gogo), 使用了fingers原生指纹库, 红队向的自动化扫描引擎 +* [zombie](https://github.com/chainreactors/zombie), 在爆破前使用fingers进行指纹验证, 提高爆破效率 + +(任何使用了fingers的工具欢迎在issue中告诉我, 我会将你的工具添加到这里) + +## Features + +* 支持多指纹库聚合识别 + * ✅ fingers 原生指纹库 + * ✅ [wappalyzer](https://github.com/projectdiscovery/wappalyzergo) + * ✅ [fingerprinthub](https://github.com/0x727/FingerprintHub) + * ✅ [ehole](https://github.com/EdgeSecurityTeam/EHole) + * ✅ goby +* 支持多指纹源favicon识别 +* 超强性能, 单个站点识别 <100ms. 重写了各指纹库的引擎, 并极大优化了性能 +* 聚合输出, 多指纹库的结果将会自动整合 +* 支持CPE的URI, FSB, WFN格式输出 + +### morefingers + +https://github.com/chainreactors/morefingers + +fingers的拓展引擎, 有更全更大的指纹库. + +从对闭源工具的逆向得到的指纹库, 为了避免可能存在的纠纷, 不提供开源版本. + +## QuickStart + +`go get github.com/chainreactors/fingers@master` + +支持 Go 1.17 及以上。规则正则默认使用标准库；在 Go 1.24 及以上可以换成更快的 RE2，只需额外引入一行： + +```golang +import _ "github.com/chainreactors/fingers/re2" // go get github.com/chainreactors/fingers/re2 +``` + +### Example + +document: https://chainreactors.github.io/wiki/libs/fingers/ + +调用内置所有进行指纹引擎识别, 示例: + +```golang +func TestEngine(t *testing.T) { + engine, err := NewEngine() + if err != nil { + panic(err) + } + resp, err := http.Get("http://127.0.0.1:8080/") + if err != nil { + return + } + content := httputils.ReadRaw(resp) + frames, err := engine.DetectContent(content) + if err != nil { + return + } + fmt.Println(frames.String()) +} +``` + +调用SDK识别Favicon指纹, 示例: + +```golang +func TestFavicon(t *testing.T) { + engine, err := NewEngine() + if err != nil { + panic(err) + } + resp, err := http.Get("http://baidu.com/favicon.ico") + if err != nil { + return + } + content := httputils.ReadRaw(resp) + body, _, _ := httputils.SplitHttpRaw(content) + frame := engine.DetectFavicon(body) + fmt.Println(frame.String()) +} +``` + +更多用法请见: https://chainreactors.github.io/wiki/libs/fingers/sdk/ + +## 用 Jev 提升识别效果 + +规则引擎召回能力强，但会有误报、多个引擎的同名重复，版本号也难以拿准。配置一个 [TypeSafe Jev](https://docs.typesafe.ai/api.md) key，引擎就会审查每一次 Web 匹配的规则结果，只返回证据支持的结果：去掉误报和重复，补全版本。调用方式不变。 + +```golang +engine, _ := fingers.NewEngine() +engine.EnableJudge("") // 空字符串读取 TYPESAFE_API_KEY +frames, _ := engine.DetectContent(raw) // WebMatch、Match、DetectResponse 同样生效 +``` + +命令行：`engine -u https://example.com --jev-key `。 + +- **只审查，不识别**：Jev 只判断规则给出的结论是否成立，从不说出产品名，因此不会凭空多出结果。 +- **失败时退回规则结果**：请求失败或 key 无效时，返回的就是纯规则结果。 +- **每条结果都带依据**：保留下来的结果带有 `Framework.Judge`（选项、Outcome、置信度、证据），随现有输出一起序列化。 +- **代价**：每个有命中的页面约 1～2 个请求，相同页面命中缓存。 + +漏报不在扫描时处理，因为 Jev 不说出产品名。维护者用 `judge/maintain` 离线发现规则漏掉的产品，交给人命名后用 `judge/gen` 生成规则，更新到指纹库后所有用户受益。 + +需要自定义 Provider、阈值或查看被剔除条目及原因时，直接使用 `judge` 包，或给 `engine.Judge` 赋值，见 [judge/README.md](judge/README.md)。判定机制只有一个 Claim：代码提出断言并附上证据，Provider 选出证据支持的选项，选项的 Outcome（holds / refuted / insufficient）驱动固定动作。Claim 合约、缓存和 Jev 客户端在 [`github.com/chainreactors/utils/jev`](https://github.com/chainreactors/utils/tree/master/jev)。 + +**数据外发**：判定时会把响应的精简视图发给 Provider，包括响应头（去掉 Date、Set-Cookie 值等无关头）、Cookie 名、标题、generator/description、脚本和样式路径、内联脚本开头、HTML 注释、表单字段名，以及正文前 1500 字。扫描授权范围内的目标前，请确认允许把这些内容发送给第三方服务。 + +当前实现于 2026-09-27 在 69 份已标注的真实响应上复测（jev-1.13.0）：误报 24 → 0，真实命中误删 0，版本正确 36、错误 0；不加载新增规则时 Discover 找出全部 3 个漏报产品，加载后无误报警。见 [复测报告](docs/jev-claim-retest-20260927.md)。 + +设计讨论见 [#34](https://github.com/chainreactors/fingers/issues/34)。 + +## fingers 引擎 + +fingers指纹引擎是目前特性最丰富, 性能最强的指纹规则库. + +* 支持多种方式规则配置 +* 支持多种方式的版本号匹配 +* 404/favicon/waf/cdn/供应链指纹识别 +* 主动指纹识别 +* 超强性能, 采用了缓存,正则预编译,默认端口,优先级等等算法提高引擎性能 +* 重点指纹,指纹来源与tag标记 + + +### 内置指纹库 + +指纹库位于: https://github.com/chainreactors/templates/tree/master/fingers + +文档: https://chainreactors.github.io/wiki/libs/fingers/rule/ + +tcp指纹与http指纹为同一格式, 但通过不同的文件进行管理 + +### 完整的配置 + +fingers设计的核心思路是命中一个指纹仅需要一条规则, 因此配置的多条规则中, 只需要任意一条命中即标记为命中, 需要在编写指纹的时候注意找到最能匹配目标框架的那条规则. + +一个完整的配置: + +```yaml +- name: frame # 指纹名字, 匹配到的时候输出的值 + default_port: # 指纹的默认端口, 加速匹配. tcp指纹如果匹配到第一个就会结束指纹匹配, http则会继续匹配, 所以默认端口对http没有特殊优化 + - '1111' + protocol: http # tcp/http, 默认为http + rule: + - version: v1.1.1 # 可不填, 默认为空, 表示无具体版本 + regexps: # 匹配的方式 + vuln: # 匹配到vuln的正则, 如果匹配到, 会输出framework为name的同时, 还会添加vuln为vuln的漏洞信息 + - version:(.*) # vuln只支持正则, 同时支持版本号匹配, 使用括号的正则分组. 只支持第一组 + regexp: # 匹配指纹正则 + - "finger.*test" + # 除了正则, 还支持其他类型的匹配, 包括以下方式 + header: # 仅http协议可用, 匹配header中包含的数据 + - string + body: # 包含匹配, 非正则表达式 + - string + md5: # 匹配body的md5hash + - [md5] + mmh3: # 匹配body的mmh3hash + - [mmh3] + + # 只有上面规则中的至少一条命中才会执行version + version: + - version:(.*) # 某些情况下难以同时编写指纹的正则与关于版本的正则, 可以特地为version写一条正则 + + favicon: # favicon的hash值, 仅http生效 + md5: + - f7e3d97f404e71d302b3239eef48d5f2 + mmh3: + - '516963061' + level: 1 # 0代表不需要主动发包, 1代表需要额外主动发起请求. 如果当前level为0则不会发送数据, 但是依旧会进行被动的指纹匹配. + send_data: "info\n" # 匹配指纹需要主动发送的数据 + vuln: frame_unauthorized # 如果regexps中的vuln命中, 则会输出漏洞名称. 某些漏洞也可以通过匹配关键字识别, 因此一些简单的poc使用指纹的方式实现, 复杂的poc请使用-e下的nuclei yaml配置 + +``` + +为了压缩体积, 没有特别指定的参数可以留空会使用默认值。 + +在两个配置文件中包含大量案例可供参考。 + +但实际上大部分字段都不需要配置, 仅作为特殊情况下的能力储备。 + +每个指纹都可以有多个rule, 每个rule中都有一个regexps, 每个regexps有多条不同种类的字符串/正则/hash + + +## TODO + +- [x] 指纹名重定向, 统一多指纹库的同一指纹不同名问题 +- [x] 指纹黑名单, 用于过滤指纹库中的垃圾指纹 +- [x] 更丰富的CPE相关特性支持 +- [ ] 更优雅的与nuclei或其他漏洞库联动 +- 支持更多引擎 + - [ ] [nuclei technologies](https://github.com/projectdiscovery/nuclei-templates/tree/main/http/technologies) 实现 + - [ ] fingerprinthub v4 + - [ ] tidefinger + - [ ] kscan + - [ ] nmap + +## Thanks + +* [wappalyzer](https://github.com/projectdiscovery/wappalyzergo) +* [fingerprinthub](https://github.com/0x727/FingerprintHub) +* [ehole](https://github.com/EdgeSecurityTeam/EHole) +* goby @XiaoliChan @9bie diff --git a/alias_attributes_test.go b/alias_attributes_test.go new file mode 100644 index 0000000..1a42b88 --- /dev/null +++ b/alias_attributes_test.go @@ -0,0 +1,26 @@ +package fingers + +import "testing" + +// A version written on one page's frameworks must not reach the next page: +// alias attributes are shared by every page. +func TestMergeFrameworksCopiesAliasAttributes(t *testing.T) { + e, err := NewEngine(FingersEngine, WappalyzerEngine, GobyEngine, EHoleEngine, FingerPrintEngine) + if err != nil { + t.Fatal(err) + } + raw := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\n" + + "") + for i := 0; i < 30; i++ { // which pointer survives depends on map order + a, _ := e.DetectContent(raw) + for _, f := range a { + f.Attributes.Version = "9.9.9" + } + b, _ := e.DetectContent(raw) + for _, f := range b { + if f.Version == "9.9.9" { + t.Fatalf("version leaked into the next page: %s", f.Name) + } + } + } +} diff --git a/cmd/engine/example.go b/cmd/engine/example.go index 472311e..c75e780 100644 --- a/cmd/engine/example.go +++ b/cmd/engine/example.go @@ -1,216 +1,225 @@ -package main - -import ( - "crypto/tls" - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "os" - "path/filepath" - "strings" - - "github.com/chainreactors/fingers" - "github.com/chainreactors/fingers/common" - "github.com/chainreactors/fingers/resources" - "github.com/chainreactors/utils/encode" - "github.com/chainreactors/utils/httputils" - "github.com/jessevdk/go-flags" - "gopkg.in/yaml.v3" -) - -var opts struct { - // 指定要使用的引擎，多个引擎用逗号分隔 - Engines string `short:"e" long:"engines" description:"Specify engines to use (comma separated)" default:"fingers,fingerprinthub,wappalyzer,ehole,goby"` - - // 是否忽略SSL证书验证 - InsecureSSL bool `short:"k" long:"insecure" description:"Skip SSL certificate verification"` - - // 目标URL - URL string `short:"u" long:"url" description:"Target URL to fingerprint" required:"true"` - - // 是否显示详细信息 - Verbose bool `short:"v" long:"verbose" description:"Show verbose debug information"` - - // 是否只检测favicon - FaviconOnly bool `short:"f" long:"favicon" description:"Only detect favicon"` - - // 资源文件覆盖 - GobyFile string `long:"goby" description:"Override goby.json.gz with custom file"` - FingerprintHubWebFile string `long:"fingerprinthub-web" description:"Override fingerprinthub_web.json.gz with custom file"` - FingerprintHubServiceFile string `long:"fingerprinthub-service" description:"Override fingerprinthub_service.json.gz with custom file"` - EholeFile string `long:"ehole" description:"Override ehole.json.gz with custom file"` - FingersFile string `long:"fingers" description:"Override fingers_http.json.gz with custom file"` - WappalyzerFile string `long:"wappalyzer" description:"Override wappalyzer.json.gz with custom file"` - AliasesFile string `long:"aliases" description:"Override aliases.yaml with custom file"` -} - -// 处理资源文件覆盖 -func processResourceFile(filePath string) ([]byte, error) { - if filePath == "" { - return nil, nil - } - - data, err := ioutil.ReadFile(filePath) - if err != nil { - return nil, fmt.Errorf("failed to read file %s: %v", filePath, err) - } - - ext := strings.ToLower(filepath.Ext(filePath)) - - // 如果是JSON文件，转换为YAML - if ext == ".json" { - var jsonData interface{} - if err := json.Unmarshal(data, &jsonData); err != nil { - return nil, fmt.Errorf("failed to parse JSON file %s: %v", filePath, err) - } - - data, err = yaml.Marshal(jsonData) - if err != nil { - return nil, fmt.Errorf("failed to convert JSON to YAML for file %s: %v", filePath, err) - } - } - - // 如果文件不是.gz结尾，进行gzip压缩 - if !strings.HasSuffix(filePath, ".gz") { - compressedData, err := encode.GzipCompress(data) - if err != nil { - return nil, fmt.Errorf("failed to compress file %s: %v", filePath, err) - } - data = compressedData - } - - return data, nil -} - -func main() { - // 解析命令行参数 - _, err := flags.Parse(&opts) - if err != nil { - os.Exit(1) - } - - // 处理资源文件覆盖 - if opts.GobyFile != "" { - if data, err := processResourceFile(opts.GobyFile); err != nil { - fmt.Println(err) - os.Exit(1) - } else { - resources.GobyData = data - } - } - - if opts.FingerprintHubWebFile != "" { - if data, err := processResourceFile(opts.FingerprintHubWebFile); err != nil { - fmt.Println(err) - os.Exit(1) - } else { - resources.FingerprinthubWebData = data - } - } - - if opts.FingerprintHubServiceFile != "" { - if data, err := processResourceFile(opts.FingerprintHubServiceFile); err != nil { - fmt.Println(err) - os.Exit(1) - } else { - resources.FingerprinthubServiceData = data - } - } - - if opts.EholeFile != "" { - if data, err := processResourceFile(opts.EholeFile); err != nil { - fmt.Println(err) - os.Exit(1) - } else { - resources.EholeData = data - } - } - - if opts.FingersFile != "" { - if data, err := processResourceFile(opts.FingersFile); err != nil { - fmt.Println(err) - os.Exit(1) - } else { - resources.FingersHTTPData = data - } - } - - if opts.WappalyzerFile != "" { - if data, err := processResourceFile(opts.WappalyzerFile); err != nil { - fmt.Println(err) - os.Exit(1) - } else { - resources.WappalyzerData = data - } - } - - if opts.AliasesFile != "" { - if data, err := processResourceFile(opts.AliasesFile); err != nil { - fmt.Println(err) - os.Exit(1) - } else { - resources.AliasesData = data - } - } - - // 创建HTTP客户端 - client := &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: opts.InsecureSSL, - }, - }, - } - - // 创建引擎实例 - var engineNames []string - if opts.Engines != "" { - engineNames = strings.Split(opts.Engines, ",") - } - - engine, err := fingers.NewEngine(engineNames...) - if err != nil { - fmt.Printf("Failed to create engine: %v\n", err) - os.Exit(1) - } - - if opts.Verbose { - fmt.Printf("Loaded engines: %s\n", engine.String()) - } - - // 发送HTTP请求 - resp, err := client.Get(opts.URL) - if err != nil { - fmt.Printf("Failed to request URL %s: %v\n", opts.URL, err) - os.Exit(1) - } - defer resp.Body.Close() - - // 检测指纹 - var frames common.Frameworks - if opts.FaviconOnly { - content := httputils.ReadBody(resp) - frame := engine.DetectFavicon(content) - if frame != nil { - frames.Add(frame) - } - } else { - frames = engine.Match(resp) - } - - // 输出结果 - if opts.Verbose { - fmt.Printf("\nDetected frameworks for %s:\n", opts.URL) - for _, frame := range frames { - fmt.Printf("Name: %s\n", frame.Name) - fmt.Printf("Vendor: %s\n", frame.Vendor) - fmt.Printf("Product: %s\n", frame.Product) - fmt.Printf("Version: %s\n", frame.Version) - fmt.Printf("CPE: %s\n", frame.CPE()) - fmt.Printf("---\n") - } - } else { - fmt.Println(frames.String()) - } -} +package main + +import ( + "crypto/tls" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/chainreactors/fingers" + "github.com/chainreactors/fingers/common" + "github.com/chainreactors/fingers/resources" + "github.com/chainreactors/utils/encode" + "github.com/chainreactors/utils/httputils" + "github.com/jessevdk/go-flags" + "gopkg.in/yaml.v3" +) + +var opts struct { + // 指定要使用的引擎，多个引擎用逗号分隔 + Engines string `short:"e" long:"engines" description:"Specify engines to use (comma separated)" default:"fingers,fingerprinthub,wappalyzer,ehole,goby"` + + // 是否忽略SSL证书验证 + InsecureSSL bool `short:"k" long:"insecure" description:"Skip SSL certificate verification"` + + // 目标URL + URL string `short:"u" long:"url" description:"Target URL to fingerprint" required:"true"` + + // 是否显示详细信息 + Verbose bool `short:"v" long:"verbose" description:"Show verbose debug information"` + + // 是否只检测favicon + FaviconOnly bool `short:"f" long:"favicon" description:"Only detect favicon"` + + // 配置 Jev key 后, 引擎自动去除误报与重复并补全版本 + JevKey string `long:"jev-key" description:"Jev API key: review rule results to drop false positives and fill versions"` + + // 资源文件覆盖 + GobyFile string `long:"goby" description:"Override goby.json.gz with custom file"` + FingerprintHubWebFile string `long:"fingerprinthub-web" description:"Override fingerprinthub_web.json.gz with custom file"` + FingerprintHubServiceFile string `long:"fingerprinthub-service" description:"Override fingerprinthub_service.json.gz with custom file"` + EholeFile string `long:"ehole" description:"Override ehole.json.gz with custom file"` + FingersFile string `long:"fingers" description:"Override fingers_http.json.gz with custom file"` + WappalyzerFile string `long:"wappalyzer" description:"Override wappalyzer.json.gz with custom file"` + AliasesFile string `long:"aliases" description:"Override aliases.yaml with custom file"` +} + +// 处理资源文件覆盖 +func processResourceFile(filePath string) ([]byte, error) { + if filePath == "" { + return nil, nil + } + + data, err := ioutil.ReadFile(filePath) + if err != nil { + return nil, fmt.Errorf("failed to read file %s: %v", filePath, err) + } + + ext := strings.ToLower(filepath.Ext(filePath)) + + // 如果是JSON文件，转换为YAML + if ext == ".json" { + var jsonData interface{} + if err := json.Unmarshal(data, &jsonData); err != nil { + return nil, fmt.Errorf("failed to parse JSON file %s: %v", filePath, err) + } + + data, err = yaml.Marshal(jsonData) + if err != nil { + return nil, fmt.Errorf("failed to convert JSON to YAML for file %s: %v", filePath, err) + } + } + + // 如果文件不是.gz结尾，进行gzip压缩 + if !strings.HasSuffix(filePath, ".gz") { + compressedData, err := encode.GzipCompress(data) + if err != nil { + return nil, fmt.Errorf("failed to compress file %s: %v", filePath, err) + } + data = compressedData + } + + return data, nil +} + +func main() { + // 解析命令行参数 + _, err := flags.Parse(&opts) + if err != nil { + os.Exit(1) + } + + // 处理资源文件覆盖 + if opts.GobyFile != "" { + if data, err := processResourceFile(opts.GobyFile); err != nil { + fmt.Println(err) + os.Exit(1) + } else { + resources.GobyData = data + } + } + + if opts.FingerprintHubWebFile != "" { + if data, err := processResourceFile(opts.FingerprintHubWebFile); err != nil { + fmt.Println(err) + os.Exit(1) + } else { + resources.FingerprinthubWebData = data + } + } + + if opts.FingerprintHubServiceFile != "" { + if data, err := processResourceFile(opts.FingerprintHubServiceFile); err != nil { + fmt.Println(err) + os.Exit(1) + } else { + resources.FingerprinthubServiceData = data + } + } + + if opts.EholeFile != "" { + if data, err := processResourceFile(opts.EholeFile); err != nil { + fmt.Println(err) + os.Exit(1) + } else { + resources.EholeData = data + } + } + + if opts.FingersFile != "" { + if data, err := processResourceFile(opts.FingersFile); err != nil { + fmt.Println(err) + os.Exit(1) + } else { + resources.FingersHTTPData = data + } + } + + if opts.WappalyzerFile != "" { + if data, err := processResourceFile(opts.WappalyzerFile); err != nil { + fmt.Println(err) + os.Exit(1) + } else { + resources.WappalyzerData = data + } + } + + if opts.AliasesFile != "" { + if data, err := processResourceFile(opts.AliasesFile); err != nil { + fmt.Println(err) + os.Exit(1) + } else { + resources.AliasesData = data + } + } + + // 创建HTTP客户端 + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: opts.InsecureSSL, + }, + }, + } + + // 创建引擎实例 + var engineNames []string + if opts.Engines != "" { + engineNames = strings.Split(opts.Engines, ",") + } + + engine, err := fingers.NewEngine(engineNames...) + if err != nil { + fmt.Printf("Failed to create engine: %v\n", err) + os.Exit(1) + } + if opts.JevKey != "" { + if err := engine.EnableJudge(opts.JevKey); err != nil { + fmt.Printf("Failed to enable Jev: %v\n", err) + os.Exit(1) + } + } + + if opts.Verbose { + fmt.Printf("Loaded engines: %s\n", engine.String()) + } + + // 发送HTTP请求 + resp, err := client.Get(opts.URL) + if err != nil { + fmt.Printf("Failed to request URL %s: %v\n", opts.URL, err) + os.Exit(1) + } + defer resp.Body.Close() + + // 检测指纹 + var frames common.Frameworks + if opts.FaviconOnly { + content := httputils.ReadBody(resp) + frame := engine.DetectFavicon(content) + if frame != nil { + frames.Add(frame) + } + } else { + frames = engine.Match(resp) + } + + // 输出结果 + if opts.Verbose { + fmt.Printf("\nDetected frameworks for %s:\n", opts.URL) + for _, frame := range frames { + fmt.Printf("Name: %s\n", frame.Name) + fmt.Printf("Vendor: %s\n", frame.Vendor) + fmt.Printf("Product: %s\n", frame.Product) + fmt.Printf("Version: %s\n", frame.Version) + fmt.Printf("CPE: %s\n", frame.CPE()) + fmt.Printf("---\n") + } + } else { + fmt.Println(frames.String()) + } +} diff --git a/cmd/judgeeval/README.md b/cmd/judgeeval/README.md new file mode 100644 index 0000000..2d155eb --- /dev/null +++ b/cmd/judgeeval/README.md @@ -0,0 +1,97 @@ +# judgeeval：纯规则 vs 规则 + judge + +## 可回放真实数据评估 + +`-manifest` 模式覆盖保存结果清洗、逐产品补全版本、漏报聚类和原生指纹生成，结果写入独立目录。 + +```powershell +# urls.txt 每行一个 URL；新采集用新目录。IP 证书不匹配时显式加 --insecure。 +python scripts/capture_judge.py --urls urls.txt --out .judge-data/corpus --workers 6 +go build -o bin/judgeeval.exe ./cmd/judgeeval +# TYPESAFE_API_KEY 仅通过进程环境提供。 +./bin/judgeeval.exe -manifest .judge-data/corpus/manifest.json -cache .judge-data/cache -out .judge-data/run-01 -rps 8 +# 同一快照及原结果用于验证历史清洗机制。 +./bin/judgeeval.exe -manifest .judge-data/corpus/manifest.json -history .judge-data/run-01/baseline.jsonl -cache .judge-data/cache -out .judge-data/run-02 +# 用运行目录内归档的响应严格离线复现；无须 API key，缓存未命中直接失败。 +./bin/judgeeval.exe -offline -manifest .judge-data/run-02/manifest.json -history .judge-data/run-02/baseline.jsonl -cache .judge-data/cache -out .judge-data/run-offline +``` + +采集器保存完整响应、来源、时间、SHA256 和失败记录，不自动跳转。第三方原文保存在 git 忽略的 `.judge-data/`。输出目录不能复用覆盖。 + +### 标注 + +为 manifest 的每份快照填写 `labels`：`product`、`present`、原文片段 `evidence` 和结论依据 `basis`。`version` 缺省表示未标注；空字符串表示未声明版本；非空字符串为准确版本。`aliases` 显式列出同义名，计分不使用子串匹配。加载时验证响应哈希及证据片段，片段是否足以支持结论仍需评审。 + +格式示例（仅示意，不作为真实数据）： + +```json +{"product":"Apache","aliases":["apache http server"],"present":true,"version":"2.4.38","evidence":["Server: Apache/2.4.38"],"basis":"响应头声明产品和版本"} +``` + +真值范围是响应证据支持的产品与版本，不等价于主机安装审计。标注来自响应头、generator meta、资源路径和人工评审；不以 Jev 回答作真值，不向清洗和版本判定传入标签。 + +### 生成与独立验证 + +`generation` 每项填写 `name`（输出文件名）、`product`、`positive`/`negative`（训练快照 ID）。其他有该产品标注的快照用于测试；排除训练主机、相同响应和重复测试响应。使用普通 `Positive` 自动推断版本，测试标签只打分。 + +主动规则另填 `probe`（请求字符串），各样本 `probes` 将请求映射到同主机已采集响应 ID。回放不发送网络请求。导出 YAML 后重新加载、编译并验证。结论为 `passed_holdout`、`failed_holdout`、`insufficient_holdout`；至少需要独立正反例且版本无错漏才能在本批次通过。单主机新产品仅能产生待验证草案。生成文件供审阅，不自动写入指纹库。 + +### 输出 + +- `manifest.json` / `samples/*.http` / `build.json`：本次实际输入及 Go 构建信息，响应可独立加载；离线模型答案仍需同时保留 `-cache` 目录。 +- `baseline.jsonl`：原生 Frameworks，来源是当前规则或 `-history`。历史记录按 ID、SHA256 严格对齐，缺失即报错。 +- `cleaned.jsonl`：成功清洗并补版本的原生 Frameworks。 +- `rows.jsonl`：`id/url/sha256/baseline/judged/error/rule_ms/judge_ms`，两种输入模式相同；派生状态不重复存储。 +- `metrics.json` / `report.md`：标注范围内的 TP/FP/FN/TN、误删、自然漏报恢复、版本正确/错误/缺失/无依据填值。失败和未标注输出单列。 +- `generation.json` / `fingerprints/*.yaml`：候选指纹、逐样本原生 Framework 结果及排除的样本。`training` / `holdout` 按样本 ID 索引，null 表示已评估且未命中，缺失 key 表示未评估。产品、标签、训练计划统一读取 manifest；状态与分数由报告推导，不再写入另一套逐样本 DTO。 + +`Inspect` 一次完成存在性和版本判断；报告比较 Baseline 与 Judged.Accepted() 两个阶段。已有非空版本保留，错误由报告揭示。缓存按模型、endpoint、完整 state 和 Claim 精确复用，使用 `claim-report-v3` 命名空间隔离旧缓存，Ruling 保存为 `option/confidence`。metrics.json 的 schema 为 3，统计使用 `claims/options`。 + +新增版本按产品及显式别名合并计数，原基线任一别名已有版本即不算新增。漏报簇的候选名称单独统计，不能算作已补回漏报。生成器训练样本的自动版本保存在 `training` 内的 Framework；训练标签只在导出后打分。训练结果错误或缺失得到 `failed_training`，测试有错得到 `failed_holdout`，未完成全部测试或缺少独立正反例得到 `insufficient_holdout`。报告与维护收录共用这一评估逻辑。 + +新采集快照的历史回放验证的是清洗机制，不代表已验证生产历史分布。样本量、同主机依赖和标签覆盖范围必须随指标报告。 + +### 维护原生指纹库 + +`-maintain` 在同一次 manifest 回放中核验新产品、生成原生 YAML、验证候选，再通过实际引擎加载新增指纹。`-library` 明确指定本地已有扩展库，先加载再评估；不要求新增 SDK 请求/响应结构。 + +```powershell +# 审核新增产品并输出本次通过验证的增量库。 +./bin/judgeeval.exe -maintain -manifest .judge-data/corpus/manifest.json -cache .judge-data/cache -out .judge-data/maintain-01 +# 下一轮维护使用已有库作为基线，防止重复收录。 +./bin/judgeeval.exe -maintain -library .judge-data/maintain-01/library.yaml -manifest .judge-data/corpus/manifest.json -cache .judge-data/cache -out .judge-data/maintain-02 +# 完整离线复现；如果原运行用了扩展库，也必须传入归档的 input-library.yaml。 +./bin/judgeeval.exe -offline -maintain -library .judge-data/maintain-02/input-library.yaml -manifest .judge-data/maintain-02/manifest.json -history .judge-data/maintain-02/baseline.jsonl -cache .judge-data/cache -out .judge-data/maintain-offline +``` + +新增被动指纹的收录条件： + +1. 在五个内置 HTTP 引擎目录、别名表和实际加载的 fingers 运行时目录中均无同名产品。按规范名及显式别名审计，记录各目录 SHA256；并非覆盖所有外部指纹源。 +2. 普通 `Positive` 为指定名称自动确认版本，通过训练集、独立正反例、版本验证；不把标签版本传给生成器。 +3. 至少两个独立正例主机组、五个独立反例主机组。相同主机名或同属一个 `group` 的样本合并计数，且排除训练主机和重复内容。 +4. 调用 `engine.Fingers().LoadFromYAML(...)` 和 `engine.Compile()` 后重放全部样本。新增产品的标注识别/版本必须全部正确，已有命中和非空版本必须保留。 + +输出 `catalog-before.json`、`maintenance.json`；通过时再将 `library.candidate.yaml` 提升为 `library.yaml`。已有产品为 `already_catalogued`；没有足够独立样本的候选保留为 `insufficient_holdout` 或 `insufficient_independent_hosts`，不会混入输出库。主动规则暂不进入此被动库维护流程。`-library` 的原文另存 `input-library.yaml`，便于准确复现。 + +此收录检查验证**原生规则加载**。加载后再由 Jev 清洗是否误删、错误召回或错误补版本，要用上述第二轮当前规则基线单独验证；见 `rows.jsonl` / `metrics.json`，不能把去重成功当成整个清洗链路准确。扩展库需要调用方显式加载，命令不重写内置压缩资源或发布外部仓库。 + + +## 目录模式 + +在一批原始 HTTP 响应上，用和 SDK 调用方完全相同的路径（`engine.DetectContent` → `j.Inspect` → `all.Accepted()`）跑一遍，逐项对比纯规则结果和经过判定之后的结果。它也用来校准新的 Provider。 + +```bash +go build -buildvcs=false -o judgeeval ./cmd/judgeeval + +# 标注集（仓库自带 40 页合成样本，含真值） +TYPESAFE_API_KEY=... ./judgeeval -provider jev -samples testdata/samples -labels testdata/labels.json + +# 任意真实页面目录（*.http，完整的原始响应） +TYPESAFE_API_KEY=... ./judgeeval -provider jev -samples cc/samples -cache judgecache -out judgereport -rps 15 +``` + +- 两种模式共用一个磁盘 Provider 包装器，保存完整批次的有效 Ruling、统计调用和缓存命中。缓存写入使用临时文件再重命名；失败与非法回答不落盘。 +- `rows.jsonl` 保存唯一的 Baseline/Judged 记录；`metrics.json` / `report.md` 从同一结果统计，未标注预测单列，不使用正文提及或 generator 代理真值。 +- 目录模式的 `-labels` 是样本文件名到 productLabel 数组的映射（格式同 manifest 的 `labels`），统一使用规范名和显式 aliases 打分。 +- `ledger.json` 保存规则审计快照；manifest 模式的 `discovery.json` 直接保存 maintain.Cluster，以样本 ID 关联，不在每行复制漏报候选。 +- 新 Provider 在 `main.go` 的 `newProvider` 注册，并实现 Claim/Ruling 合约。 diff --git a/cmd/judgeeval/main.go b/cmd/judgeeval/main.go new file mode 100644 index 0000000..bb87233 --- /dev/null +++ b/cmd/judgeeval/main.go @@ -0,0 +1,242 @@ +// judgeeval evaluates saved HTTP responses through the same Inspect pipeline as SDK callers. +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "math" + "net/http" + "os" + "path/filepath" + "sort" + "sync" + "sync/atomic" + "time" + + "github.com/chainreactors/fingers" + "github.com/chainreactors/fingers/judge" + "github.com/chainreactors/fingers/judge/maintain" + "github.com/chainreactors/logs" + "github.com/chainreactors/utils/jev" +) + +func main() { + manifest := flag.String("manifest", "", "replay evidence manifest with labels and generation plans") + history := flag.String("history", "", "optional baseline.jsonl from a previous replay") + offline := flag.Bool("offline", false, "manifest replay using exact cached rulings only; no provider requests") + maintain := flag.Bool("maintain", false, "audit novel products and validate a native additive fingerprint library") + library := flag.String("library", "", "existing local native YAML library to load before manifest replay") + samples := flag.String("samples", "testdata/samples", "directory of raw HTTP responses (*.http)") + labels := flag.String("labels", "", "optional ground truth (see testdata/labels.json)") + provider := flag.String("provider", "jev", "judge provider: jev") + cache := flag.String("cache", "judgecache", "directory caching rulings") + out := flag.String("out", "judgereport", "output directory") + rps := flag.Float64("rps", 15, "max provider requests per second") + flag.Float64Var(&minConfidence, "min-confidence", 0, "judge MinConfidence (0 = Jev default)") + workers := flag.Int("workers", 16, "concurrent pages") + limit := flag.Int("limit", 0, "only the first N samples (0 = all)") + endpoint := flag.String("endpoint", "", "provider API endpoint (default: the provider's)") + flag.Parse() + logs.Log.SetLevel(logs.ErrorLevel) + if *manifest != "" { + if err := replay(*manifest, *history, *provider, *endpoint, *cache, *out, *rps, *offline, *maintain, *library); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + return + } + if *offline || *maintain || *library != "" { + fmt.Fprintln(os.Stderr, "-offline, -maintain and -library require -manifest") + os.Exit(1) + } + if err := run(*provider, *samples, *labels, *cache, *out, *endpoint, *rps, *workers, *limit); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +// newProvider builds a provider by name; add new providers here. tokens +// reports input tokens sent, where the provider counts them. +func newProvider(name, endpoint string, transport http.RoundTripper) (p jev.Provider, tokens func() int64, err error) { + switch name { + case "jev": + jp, err := jev.NewClient("") + if err != nil { + return nil, nil, err + } + if endpoint != "" { + jp.Endpoint = endpoint + } + jp.HTTP.Transport = transport + return jp, func() int64 { return atomic.LoadInt64(&jp.InputTokens) }, nil + } + return nil, nil, fmt.Errorf("unknown provider %q", name) +} + +func run(providerName, samples, labelsPath, cacheDir, out, endpoint string, rps float64, workers, limit int) error { + if rps <= 0 || rps > 100 || math.IsNaN(rps) || workers < 1 { + return fmt.Errorf("rps must be in (0,100] and workers positive") + } + files, err := filepath.Glob(filepath.Join(samples, "*.http")) + if err != nil { + return err + } + sort.Strings(files) + if limit > 0 && len(files) > limit { + files = files[:limit] + } + labels, err := loadLabels(labelsPath) + if err != nil { + return err + } + for _, dir := range []string{cacheDir, out} { + if err := os.MkdirAll(dir, 0700); err != nil { + return err + } + } + ticker := time.NewTicker(time.Duration(float64(time.Second) / rps)) + defer ticker.Stop() + provider, tokens, err := newProvider(providerName, endpoint, &limited{tick: ticker.C}) + if err != nil { + return err + } + cached := &recordingProvider{Provider: provider, dir: cacheDir} + j := newJudge(cached) + engine, err := fingers.NewEngine(fingers.FingersEngine, fingers.FingerPrintEngine, fingers.EHoleEngine, fingers.GobyEngine, fingers.WappalyzerEngine) + if err != nil { + return err + } + engine.EnableMatchDetail() + ledger := maintain.NewLedger() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + rows := make([]record, len(files)) + jobs := make(chan int) + var wg sync.WaitGroup + var fatal error + var once sync.Once + started := time.Now() + for w := 0; w < workers; w++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := range jobs { + r, err := evaluatePage(ctx, engine, j, files[i]) + rows[i] = r + var apiErr *jev.APIError + if errors.As(err, &apiErr) && apiErr.Status == 401 { + once.Do(func() { fatal = errors.New("unauthorized: check " + jev.EnvAPIKey); cancel() }) + } + } + }() + } +dispatch: + for i := range files { + select { + case jobs <- i: + case <-ctx.Done(): + break dispatch + } + } + close(jobs) + wg.Wait() + if fatal != nil { + return fatal + } + if err := writeRows(filepath.Join(out, "rows.jsonl"), rows); err != nil { + return err + } + for _, r := range rows { // in file order, so the ledger's samples are reproducible + ledger.Add(r.ID, r.Judged) + } + m := &replayManifest{} + for _, r := range rows { + m.Samples = append(m.Samples, replaySample{ID: r.ID, Group: r.ID, ContentSHA256: r.SHA256, Labels: labels[r.ID]}) + } + summary := summarizeReplay(m, rows, nil, nil) + summary.Model, summary.BaselineSource = provider.ID(), "current_rules_on_saved_responses" + summary.Requests, summary.CacheHits, summary.Claims = cached.stats() + summary.InputTokens, summary.Seconds = tokens(), time.Since(started).Seconds() + summary.JunkRules = ledger.Report(3, 0.5) + if err := writeJSON(filepath.Join(out, "ledger.json"), ledger.Report(1, 0)); err != nil { + return err + } + if err := writeJSON(filepath.Join(out, "metrics.json"), summary); err != nil { + return err + } + if err := writeReplayReport(filepath.Join(out, "report.md"), summary, m); err != nil { + return err + } + fmt.Printf("saved %s\n", filepath.Join(out, "report.md")) + if summary.Errors > 0 { + return fmt.Errorf("%d failed rows; see rows.jsonl", summary.Errors) + } + return nil +} + +func evaluatePage(ctx context.Context, engine *fingers.Engine, j *judge.Judge, path string) (r record, err error) { + r.ID = filepath.Base(path) + defer func() { + if p := recover(); p != nil { + err = fmt.Errorf("panic: %v", p) + } + if err != nil { + r.Err = err.Error() + } + }() + raw, err := os.ReadFile(path) + if err != nil { + return r, err + } + r.SHA256 = digest(raw) + started := time.Now() + r.Baseline, err = engine.DetectContent(raw) + r.RuleMs = ms(time.Since(started)) + if err != nil { + return r, err + } + err = judgeRecord(ctx, j, raw, &r) + return r, err +} + +// loadLabels reads ground truth: sample file name -> product labels. +func loadLabels(path string) (map[string][]productLabel, error) { + out := map[string][]productLabel{} + if path == "" { + return out, nil + } + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + if err := json.Unmarshal(data, &out); err != nil { + return nil, err + } + return out, nil +} + +func ms(d time.Duration) float64 { return float64(d.Microseconds()) / 1000 } + +type limited struct{ tick <-chan time.Time } + +func (l *limited) RoundTrip(req *http.Request) (*http.Response, error) { + select { + case <-l.tick: + case <-req.Context().Done(): + return nil, req.Context().Err() + } + return http.DefaultTransport.RoundTrip(req) +} + +var minConfidence float64 + +func newJudge(p jev.Provider) *judge.Judge { + j := judge.New(p) + if minConfidence != 0 { + j.MinConfidence = minConfidence + } + return j +} diff --git a/cmd/judgeeval/maintenance.go b/cmd/judgeeval/maintenance.go new file mode 100644 index 0000000..5b28f9a --- /dev/null +++ b/cmd/judgeeval/maintenance.go @@ -0,0 +1,427 @@ +package main + +import ( + "encoding/json" + "fmt" + "net/url" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/chainreactors/fingers" + "github.com/chainreactors/fingers/common" + fingerlib "github.com/chainreactors/fingers/fingers" + "github.com/chainreactors/fingers/judge" + "github.com/chainreactors/fingers/judge/maintain" + "github.com/chainreactors/fingers/resources" + "gopkg.in/yaml.v3" +) + +// CLI evidence records; generated SDK values remain native Finger/Frameworks. +func loadReplayLibrary(engine *fingers.Engine, path, out string) error { + // Read a local file and archive the exact bytes before using the public loader. + data, err := os.ReadFile(path) + if err != nil { + return err + } + snapshot := filepath.Join(out, "input-library.yaml") + if err := os.WriteFile(snapshot, data, 0600); err != nil { + return err + } + if err := engine.Fingers().LoadFromYAML(snapshot); err != nil { + return err + } + return engine.Compile() +} + +type catalogSource struct { + Source string `json:"source"` + SHA256 string `json:"sha256"` + Names []string `json:"names"` +} + +func embeddedCatalog() ([]catalogSource, error) { + var result []catalogSource + for _, source := range []struct { + name string + data []byte + }{ + {"fingers", resources.FingersHTTPData}, {"goby", resources.GobyData}, + {"ehole", resources.EholeData}, {"wappalyzer", resources.WappalyzerData}, + {"fingerprinthub", resources.FingerprinthubWebData}, + } { + var raw interface{} + if err := resources.UnmarshalData(source.data, &raw); err != nil { + return nil, err + } + var entries []interface{} + var names []string + switch source.name { + case "wappalyzer": + root, ok := raw.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("invalid %s catalog", source.name) + } + apps, ok := root["apps"].(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("invalid apps catalog") + } + for name := range apps { + names = append(names, name) + } + case "ehole": + root, ok := raw.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("invalid ehole catalog") + } + entries, _ = root["fingerprint"].([]interface{}) + default: + entries, _ = raw.([]interface{}) + } + for _, entry := range entries { + obj, ok := entry.(map[string]interface{}) + if !ok { + continue + } + for _, key := range []string{"name", "cms", "id"} { + if name, ok := obj[key].(string); ok { + names = append(names, name) + } + } + if info, ok := obj["info"].(map[string]interface{}); ok { + if name, ok := info["name"].(string); ok { + names = append(names, name) + } + if metadata, ok := info["metadata"].(map[string]interface{}); ok { + if name, ok := metadata["product"].(string); ok { + names = append(names, name) + } + } + } + } + if len(names) == 0 { + return nil, fmt.Errorf("empty %s catalog", source.name) + } + result = append(result, catalogSource{source.name, digest(source.data), uniqueNames(names)}) + } + var aliases []struct { + Name string `yaml:"name"` + Alias map[string][]string `yaml:"alias"` + } + if err := yaml.Unmarshal(resources.AliasesData, &aliases); err != nil { + return nil, err + } + var names []string + for _, alias := range aliases { + names = append(names, alias.Name) + for _, values := range alias.Alias { + names = append(names, values...) + } + } + result = append(result, catalogSource{"aliases", digest(resources.AliasesData), uniqueNames(names)}) + return result, nil +} + +func runtimeCatalog(engine *fingers.Engine) (catalogSource, error) { + data, err := yaml.Marshal(engine.Fingers().HTTPFingers) + if err != nil { + return catalogSource{}, err + } + var names []string + for _, f := range engine.Fingers().HTTPFingers { + if f != nil { + names = append(names, f.Name) + } + } + return catalogSource{"runtime-fingers", digest(data), uniqueNames(names)}, nil +} + +// Shared operator groups or shared hostnames connect samples into one unit. +// Different user-supplied group labels cannot turn one host into multiple votes. +func independentHostCount(samples []replaySample) int { + parent := map[string]string{} + var root func(string) string + root = func(key string) string { + if _, ok := parent[key]; !ok { + parent[key] = key + } + if parent[key] != key { + parent[key] = root(parent[key]) + } + return parent[key] + } + var hosts []string + for _, s := range samples { + u, err := url.Parse(s.URL) + if err != nil || u.Hostname() == "" || s.Group == "" { + continue + } + host := "host:" + strings.ToLower(u.Hostname()) + group := "group:" + s.Group + parent[root(host)] = root(group) + hosts = append(hosts, host) + } + groups := map[string]bool{} + for _, host := range hosts { + groups[root(host)] = true + } + return len(groups) +} + +func uniqueNames(names []string) []string { + seen := map[string]bool{} + var out []string + for _, name := range names { + if name != "" && !seen[name] { + seen[name] = true + out = append(out, name) + } + } + sort.Strings(out) + return out +} + +func catalogMatches(catalog []catalogSource, names []string) []string { + wanted := map[string]bool{} + for _, name := range names { + wanted[judge.NormalizeName(name)] = true + } + var found []string + for _, source := range catalog { + for _, name := range source.Names { + if wanted[judge.NormalizeName(name)] { + found = append(found, source.Source+":"+name) + } + } + } + return uniqueNames(found) +} + +type maintenanceProduct struct { + Plan string `json:"plan"` + Product string `json:"product"` + Status string `json:"status"` + Existing []string `json:"existing_catalog_matches"` + PositiveSamples int `json:"positive_samples"` + DiscoveredPositives int `json:"discovered_positives"` + PositiveHosts int `json:"holdout_positive_hosts"` + NegativeHosts int `json:"holdout_negative_hosts"` + Before stageScore `json:"before"` + After stageScore `json:"after"` + TestIDs []string `json:"test_ids"` +} + +type maintenanceReport struct { + Library string `json:"library,omitempty"` + CatalogFile string `json:"catalog_file"` + MinPositiveHosts int `json:"min_positive_hosts"` + MinNegativeHosts int `json:"min_negative_hosts"` + Products []maintenanceProduct `json:"products"` + LostExisting []string `json:"lost_existing_hits"` + ChangedVersions []string `json:"changed_existing_versions"` + IntegrationErrors []string `json:"integration_errors"` +} + +func maintenanceDecision(existing []string, generationStatus string, positiveHosts, negativeHosts int) string { + switch { + case len(existing) > 0: + return "already_catalogued" + case generationStatus != "passed_holdout": + return generationStatus + case positiveHosts < 2 || negativeHosts < 5: + return "insufficient_independent_hosts" + default: + return "eligible" + } +} + +func maintainReplay(m *replayManifest, clusters []*maintain.Cluster, generated []generationResult, out string, engine *fingers.Engine) error { + catalog, err := embeddedCatalog() + if err != nil { + return err + } + runtime, err := runtimeCatalog(engine) + if err != nil { + return err + } + catalog = append(catalog, runtime) + if err := writeJSON(filepath.Join(out, "catalog-before.json"), catalog); err != nil { + return err + } + report := maintenanceReport{CatalogFile: "catalog-before.json", MinPositiveHosts: 2, MinNegativeHosts: 5} + missing := missingCandidates(clusters) + plans := map[string]generationPlan{} + for _, p := range m.Generation { + plans[p.Name] = p + } + var library fingerlib.Fingers + accepted := map[string]generationPlan{} + for _, g := range generated { + p := plans[g.Name] + r := maintenanceProduct{Plan: g.Name, Product: p.Product} + names := []string{p.Product} + for _, sample := range m.Samples { + l, ok := labelFor(sample, p.Product) + if !ok { + continue + } + names = append(names, l.Aliases...) + if l.Present { + r.PositiveSamples++ + if containsProductName(missing[sample.ID], l) { + r.DiscoveredPositives++ + } + } + } + r.Existing = catalogMatches(catalog, names) + test, _, err := generationSplit(m, p) + if err != nil { + return err + } + var pos, neg []replaySample + for _, sample := range test { + l, _ := labelFor(sample, p.Product) + if l.Present { + pos = append(pos, sample) + } else { + neg = append(neg, sample) + } + r.TestIDs = append(r.TestIDs, sample.ID) + } + r.PositiveHosts, r.NegativeHosts = independentHostCount(pos), independentHostCount(neg) + _, _, status := g.assess(m, p) + r.Status = maintenanceDecision(r.Existing, status, r.PositiveHosts, r.NegativeHosts) + if r.Status == "eligible" && p.Probe != "" { + r.Status = "active_integration_not_evaluated" + } + if r.Status == "eligible" { + key := judge.NormalizeName(p.Product) + if _, exists := accepted[key]; exists { + r.Status = "duplicate_candidate" + } else { + path, err := confinedPath(out, g.File) + if err != nil { + return err + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + var f fingerlib.Finger + if err := yaml.Unmarshal(data, &f); err != nil { + return err + } + library = append(library, &f) + accepted[key] = p + } + } + report.Products = append(report.Products, r) + } + if len(library) == 0 { + return writeJSON(filepath.Join(out, "maintenance.json"), report) + } + sort.Slice(library, func(i, j int) bool { return library[i].Name < library[j].Name }) + data, err := yaml.Marshal(library) + if err != nil { + return err + } + candidatePath := filepath.Join(out, "library.candidate.yaml") + if err := os.WriteFile(candidatePath, data, 0600); err != nil { + return err + } + before := map[string]common.Frameworks{} + for _, sample := range m.Samples { + frames, err := engine.DetectContent(sample.raw) + if err != nil { + return err + } + before[sample.ID] = copyFrames(frames) + } + // Exercise the public additive loader and rebuild aliases just as SDK callers do. + if err := engine.Fingers().LoadFromYAML(candidatePath); err != nil { + return err + } + if err := engine.Compile(); err != nil { + return err + } + after := map[string]common.Frameworks{} + for _, sample := range m.Samples { + frames, err := engine.DetectContent(sample.raw) + if err != nil { + return err + } + after[sample.ID] = frames + for _, f := range before[sample.ID] { + if f == nil { + continue + } + got := findLabel(frames, productLabel{Product: f.Name}) + if got == nil { + report.LostExisting = append(report.LostExisting, sample.ID+":"+f.Name) + } else if f.Version != "" && got.Version != f.Version { + report.ChangedVersions = append(report.ChangedVersions, sample.ID+":"+f.Name) + } + } + for _, p := range accepted { + l, ok := labelFor(sample, p.Product) + if !ok { + continue + } + got := findLabel(frames, l) + if (got != nil) != l.Present || got != nil && l.Version != nil && got.Version != *l.Version { + report.IntegrationErrors = append(report.IntegrationErrors, sample.ID+":"+p.Product) + } + } + } + for i := range report.Products { + r := &report.Products[i] + if r.Status != "eligible" { + continue + } + p := plans[r.Plan] + test, _, _ := generationSplit(m, p) + for _, sample := range test { + l, _ := labelFor(sample, p.Product) + for _, stage := range []struct { + score *stageScore + frames common.Frameworks + }{{&r.Before, before[sample.ID]}, {&r.After, after[sample.ID]}} { + f := findLabel(stage.frames, l) + stage.score.add(l, f) + } + } + r.Status = "validated_library" + } + sort.Strings(report.LostExisting) + sort.Strings(report.ChangedVersions) + sort.Strings(report.IntegrationErrors) + if len(report.LostExisting)+len(report.ChangedVersions)+len(report.IntegrationErrors) > 0 { + for i := range report.Products { + if report.Products[i].Status == "validated_library" { + report.Products[i].Status = "integration_failed" + } + } + } else { + report.Library = "library.yaml" + if err := os.Rename(candidatePath, filepath.Join(out, report.Library)); err != nil { + return err + } + } + if err := writeJSON(filepath.Join(out, "maintenance.json"), report); err != nil { + return err + } + if report.Library == "" { + return fmt.Errorf("candidate library failed integration; see maintenance.json") + } + fmt.Fprintf(os.Stderr, "validated %d novel fingerprints in %s\n", len(library), report.Library) + return nil +} + +// copyFrames freezes the pre-load engine results for library validation. +func copyFrames(frames common.Frameworks) common.Frameworks { + data, _ := json.Marshal(frames) + var out common.Frameworks + _ = json.Unmarshal(data, &out) + return out +} diff --git a/cmd/judgeeval/maintenance_test.go b/cmd/judgeeval/maintenance_test.go new file mode 100644 index 0000000..b8f1b22 --- /dev/null +++ b/cmd/judgeeval/maintenance_test.go @@ -0,0 +1,93 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/chainreactors/fingers" +) + +func TestMaintenanceCountsHostAndOperatorOverlapOnce(t *testing.T) { + samples := []replaySample{ + {URL: "https://a.example/one", Group: "operator-a"}, + {URL: "https://b.example/", Group: "operator-a"}, + {URL: "https://b.example/two", Group: "operator-b"}, + {URL: "https://c.example/", Group: "operator-b"}, + {URL: "https://d.example/", Group: "operator-d"}, + } + if got := independentHostCount(samples); got != 2 { + t.Fatalf("overlapping hosts/operators counted %d times, want 2", got) + } +} + +func TestMaintenanceRecognizesPreviouslyLoadedLibrary(t *testing.T) { + engine, err := fingers.NewEngine(fingers.FingersEngine) + if err != nil { + t.Fatal(err) + } + out := t.TempDir() + path := filepath.Join(out, "fixture.yaml") + data := []byte("- name: replay-maintenance-test-app\n protocol: http\n rule:\n - regexps:\n body:\n - 'replay-maintenance-test-app'\n") + if err := os.WriteFile(path, data, 0600); err != nil { + t.Fatal(err) + } + if err := loadReplayLibrary(engine, path, out); err != nil { + t.Fatal(err) + } + archived, err := os.ReadFile(filepath.Join(out, "input-library.yaml")) + if err != nil || string(archived) != string(data) { + t.Fatalf("library bytes not preserved: %v", err) + } + catalog, err := runtimeCatalog(engine) + if err != nil { + t.Fatal(err) + } + existing := catalogMatches([]catalogSource{catalog}, []string{"replay-maintenance-test-app"}) + if got := maintenanceDecision(existing, "passed_holdout", 2, 5); got != "already_catalogued" { + t.Fatalf("would re-add existing library fingerprint: %s", got) + } +} + +func TestCatalogAuditCoversEveryEmbeddedSource(t *testing.T) { + catalog, err := embeddedCatalog() + if err != nil { + t.Fatal(err) + } + if len(catalog) != 6 { + t.Fatalf("sources: %d", len(catalog)) + } + for _, source := range catalog { + if source.SHA256 == "" || len(source.Names) == 0 { + t.Fatalf("incomplete source: %s", source.Source) + } + } + if len(catalogMatches(catalog, []string{"WordPress"})) == 0 { + t.Fatal("known product missed") + } + if len(catalogMatches(catalog, []string{"certainly-not-a-real-catalog-product-20260926"})) != 0 { + t.Fatal("invented product matched") + } +} + +func TestMaintenanceRequiresNoveltyAndIndependentHosts(t *testing.T) { + g := "passed_holdout" + for _, tc := range []struct { + existing []string + positive, negative int + want string + }{ + {[]string{"fingers:nginx"}, 3, 20, "already_catalogued"}, + {nil, 1, 20, "insufficient_independent_hosts"}, + {nil, 2, 4, "insufficient_independent_hosts"}, + {nil, 2, 5, "eligible"}, + } { + if got := maintenanceDecision(tc.existing, g, tc.positive, tc.negative); got != tc.want { + t.Fatalf("got %s want %s", got, tc.want) + } + } + g = "failed_holdout" + if got := maintenanceDecision(nil, g, 3, 20); got != "failed_holdout" { + t.Fatalf("failed candidate admitted: %s", got) + } +} diff --git a/cmd/judgeeval/provider.go b/cmd/judgeeval/provider.go new file mode 100644 index 0000000..0961100 --- /dev/null +++ b/cmd/judgeeval/provider.go @@ -0,0 +1,89 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sync" + + "github.com/chainreactors/utils/jev" +) + +// recordingProvider is the shared disk replay and accounting boundary. +// Schema 3 stores provider-neutral Claim/Ruling JSON. Provider.ID includes endpoint/model. +type recordingProvider struct { + jev.Provider + dir string + offline bool + mu sync.Mutex + calls, hits, claims int +} + +func (p *recordingProvider) stats() (calls, hits, claims int) { + p.mu.Lock() + defer p.mu.Unlock() + return p.calls, p.hits, p.claims +} + +func (p *recordingProvider) Judge(ctx context.Context, state interface{}, claims map[string]jev.Claim) (map[string]jev.Ruling, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if err := jev.ValidateClaims(claims); err != nil { + return nil, err + } + data, err := json.Marshal([]interface{}{"claim-report-v3", p.ID(), state, claims}) + if err != nil { + return nil, err + } + path := filepath.Join(p.dir, digest(data)+".json") + p.mu.Lock() + p.claims += len(claims) + p.mu.Unlock() + var rulings map[string]jev.Ruling + if cached, err := os.ReadFile(path); err == nil && json.Unmarshal(cached, &rulings) == nil && jev.ValidateRulings(claims, rulings) == nil { + p.mu.Lock() + p.hits++ + p.mu.Unlock() + return rulings, nil + } + if p.offline { + return nil, fmt.Errorf("offline cache miss: %s", filepath.Base(path)) + } + p.mu.Lock() + p.calls++ + p.mu.Unlock() + rulings, err = p.Provider.Judge(ctx, state, claims) + if err == nil { + err = jev.ValidateRulings(claims, rulings) + } + if err != nil { + return nil, err + } + if err := writeCache(path, rulings); err != nil { + return nil, err + } + return rulings, nil +} + +func writeCache(path string, value interface{}) error { + data, err := json.Marshal(value) + if err != nil { + return err + } + f, err := os.CreateTemp(filepath.Dir(path), ".claim-*.tmp") + if err != nil { + return err + } + defer os.Remove(f.Name()) + if _, err := f.Write(data); err != nil { + f.Close() + return err + } + if err := f.Close(); err != nil { + return err + } + return os.Rename(f.Name(), path) +} diff --git a/cmd/judgeeval/replay.go b/cmd/judgeeval/replay.go new file mode 100644 index 0000000..875d7d1 --- /dev/null +++ b/cmd/judgeeval/replay.go @@ -0,0 +1,472 @@ +package main + +import ( + "bufio" + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "math" + "net/http" + "net/url" + "os" + "path/filepath" + "runtime/debug" + "strings" + "time" + + "github.com/chainreactors/fingers" + "github.com/chainreactors/fingers/common" + "github.com/chainreactors/fingers/judge" + "github.com/chainreactors/fingers/judge/maintain" + "github.com/chainreactors/utils/jev" +) + +// Labels are evidence for evaluation only; they are never passed to Judge. +type productLabel struct { + Product string `json:"product"` + Aliases []string `json:"aliases,omitempty"` + Present bool `json:"present"` + Version *string `json:"version,omitempty"` // nil = not labelled, "" = not stated + Evidence []string `json:"evidence"` // literal excerpts in the saved response + Basis string `json:"basis"` // what the excerpt establishes +} +type replaySample struct { + ID string `json:"id"` + URL string `json:"url"` + Group string `json:"group"` + CapturedAt string `json:"captured_at"` + Response string `json:"response"` + SHA256 string `json:"sha256"` + ContentSHA256 string `json:"content_sha256"` + Labels []productLabel `json:"labels"` + Probes map[string]string `json:"probes,omitempty"` // send_data -> another sample ID + raw []byte +} +type generationPlan struct { + Name string `json:"name"` + Product string `json:"product"` + Positive []string `json:"positive"` + Negative []string `json:"negative"` + Probe string `json:"probe,omitempty"` +} +type replayManifest struct { + Schema int `json:"schema"` + Scope string `json:"scope"` + Samples []replaySample `json:"samples"` + Generation []generationPlan `json:"generation"` +} +type baselineRecord struct { + ID string `json:"id"` + SHA256 string `json:"sha256"` + Frames common.Frameworks `json:"frames"` +} + +// record stores source results once; accepted products and all metrics are derived. +type record struct { + ID string `json:"id"` + URL string `json:"url"` + SHA256 string `json:"sha256"` + Baseline common.Frameworks `json:"baseline"` + Judged common.Frameworks `json:"judged"` + Err string `json:"error,omitempty"` + RuleMs float64 `json:"rule_ms"` + JudgeMs float64 `json:"judge_ms"` +} + +func digest(b []byte) string { return fmt.Sprintf("%x", sha256.Sum256(b)) } +func writeJSON(path string, value interface{}) error { + b, err := json.MarshalIndent(value, "", " ") + if err != nil { + return err + } + return os.WriteFile(path, append(b, '\n'), 0600) +} +func confinedPath(root, name string) (string, error) { + if name == "" || filepath.IsAbs(name) { + return "", fmt.Errorf("expected relative response path: %q", name) + } + joined := filepath.Join(root, filepath.FromSlash(name)) + rel, err := filepath.Rel(root, joined) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("response escapes corpus: %q", name) + } + return joined, nil +} +func loadReplay(path string) (*replayManifest, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var m replayManifest + if err := json.Unmarshal(data, &m); err != nil { + return nil, err + } + if m.Schema != 1 || len(m.Samples) == 0 { + return nil, fmt.Errorf("manifest requires schema=1 and samples") + } + seen := map[string]bool{} + for i := range m.Samples { + s := &m.Samples[i] + if s.ID == "" || seen[s.ID] { + return nil, fmt.Errorf("empty or duplicate sample ID %q", s.ID) + } + seen[s.ID] = true + u, err := url.Parse(s.URL) + if err != nil || u.Hostname() == "" || s.Group == "" || s.CapturedAt == "" { + return nil, fmt.Errorf("%s: URL, group and captured_at required", s.ID) + } + filename, err := confinedPath(filepath.Dir(path), s.Response) + if err != nil { + return nil, err + } + s.raw, err = os.ReadFile(filename) + if err != nil { + return nil, err + } + if len(s.raw) > 2<<20 || digest(s.raw) != s.SHA256 { + return nil, fmt.Errorf("%s: size limit or SHA256 mismatch", s.ID) + } + resp, err := http.ReadResponse(bufio.NewReader(bytes.NewReader(s.raw)), nil) + if err != nil { + return nil, fmt.Errorf("%s: %w", s.ID, err) + } + body, err := io.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + return nil, fmt.Errorf("%s: incomplete response: %w", s.ID, err) + } + // Recompute rather than trusting the manifest for split leakage checks. + stable := http.Header{} + for k, v := range resp.Header { + switch strings.ToLower(k) { + case "date", "set-cookie", "etag", "last-modified", "connection", "content-length", "transfer-encoding", "x-request-id": + continue + } + stable[k] = v + } + head, _ := json.Marshal([]interface{}{resp.StatusCode, stable}) + s.ContentSHA256 = digest(append(head, body...)) + evidence := strings.ToLower(string(s.raw) + "\n" + string(body)) + keys := map[string]bool{} + for _, l := range s.Labels { + key := judge.NormalizeName(l.Product) + if key == "" || keys[key] || l.Basis == "" || len(l.Evidence) == 0 || (!l.Present && l.Version != nil) { + return nil, fmt.Errorf("%s: invalid/duplicate label %q", s.ID, l.Product) + } + keys[key] = true + for _, excerpt := range l.Evidence { + if strings.TrimSpace(excerpt) == "" || !strings.Contains(evidence, strings.ToLower(excerpt)) { + return nil, fmt.Errorf("%s: label evidence missing for %s", s.ID, l.Product) + } + } + } + } + for _, s := range m.Samples { + for request, id := range s.Probes { + if request == "" || !seen[id] { + return nil, fmt.Errorf("%s: invalid probe %q", s.ID, request) + } + } + } + return &m, nil +} + +func loadHistory(path string) (map[string]baselineRecord, error) { + out := map[string]baselineRecord{} + if path == "" { + return out, nil + } + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + scan := bufio.NewScanner(f) + scan.Buffer(make([]byte, 4096), 16<<20) + for scan.Scan() { + var r baselineRecord + if err := json.Unmarshal(scan.Bytes(), &r); err != nil { + return nil, err + } + if _, ok := out[r.ID]; ok { + return nil, fmt.Errorf("duplicate historical ID %s", r.ID) + } + out[r.ID] = r + } + return out, scan.Err() +} +func versionOf(f *common.Framework) string { + if f == nil || f.Attributes == nil { + return "" + } + return f.Version +} +func findLabel(frames common.Frameworks, l productLabel) *common.Framework { + names := append([]string{l.Product}, l.Aliases...) + var found *common.Framework + for _, name := range names { + for _, f := range frames { + if f != nil && judge.NormalizeName(f.Name) == judge.NormalizeName(name) { + if found == nil || (versionOf(found) == "" && versionOf(f) != "") || (versionOf(found) == versionOf(f) && f.Name < found.Name) { + found = f + } + } + } + } + return found +} +func labelFor(s replaySample, name string) (productLabel, bool) { + key := judge.NormalizeName(name) + for _, l := range s.Labels { + for _, n := range append([]string{l.Product}, l.Aliases...) { + if judge.NormalizeName(n) == key { + return l, true + } + } + } + return productLabel{}, false +} + +func replay(manifestPath, historyPath, providerName, endpoint, cacheDir, out string, rps float64, offline, audit bool, libraryPath string) error { + if rps <= 0 || rps > 100 || math.IsNaN(rps) { + return fmt.Errorf("rps must be in (0,100]") + } + m, err := loadReplay(manifestPath) + if err != nil { + return err + } + history, err := loadHistory(historyPath) + if err != nil { + return err + } + if historyPath != "" { + for _, s := range m.Samples { + if r, ok := history[s.ID]; !ok || r.SHA256 != s.SHA256 { + return fmt.Errorf("%s: history missing or response hash differs", s.ID) + } + } + } + if _, err := os.Stat(out); err == nil { + return fmt.Errorf("output directory exists: use a new run directory") + } + if err := os.MkdirAll(out, 0755); err != nil { + return err + } + if err := os.MkdirAll(cacheDir, 0700); err != nil { + return err + } + if err := saveReplaySnapshot(out, m); err != nil { + return err + } + ticker := time.NewTicker(time.Duration(float64(time.Second) / rps)) + defer ticker.Stop() + var provider jev.Provider + tokens := func() int64 { return 0 } + if offline { + if providerName != "jev" { + return fmt.Errorf("unknown provider %q", providerName) + } + // Cache misses never invoke this provider. + p, e := jev.NewClient("offline-cache-only") + err = e + if p != nil && endpoint != "" { + p.Endpoint = endpoint + } + provider = p + } else { + provider, tokens, err = newProvider(providerName, endpoint, &limited{tick: ticker.C}) + } + if err != nil { + return err + } + cached := &recordingProvider{Provider: provider, dir: cacheDir, offline: offline} + j := newJudge(cached) + engine, err := fingers.NewEngine(fingers.FingersEngine, fingers.FingerPrintEngine, fingers.EHoleEngine, fingers.GobyEngine, fingers.WappalyzerEngine) + if err != nil { + return err + } + if libraryPath != "" { + if err := loadReplayLibrary(engine, libraryPath, out); err != nil { + return err + } + } + if engine.Fingers() != nil { + engine.EnableMatchDetail() // claims quote what each rule matched + } + baselineFile, err := os.Create(filepath.Join(out, "baseline.jsonl")) + if err != nil { + return err + } + defer baselineFile.Close() + cleanedFile, err := os.Create(filepath.Join(out, "cleaned.jsonl")) + if err != nil { + return err + } + defer cleanedFile.Close() + be, ce := json.NewEncoder(baselineFile), json.NewEncoder(cleanedFile) + ledger := maintain.NewLedger() + rows := make([]record, 0, len(m.Samples)) + started := time.Now() + for _, s := range m.Samples { + r := record{ID: s.ID, URL: s.URL, SHA256: s.SHA256} + ruleStart := time.Now() + if historyPath != "" { + r.Baseline = history[s.ID].Frames + } else { + r.Baseline, err = engine.DetectContent(s.raw) + if err != nil { + return err + } + } + r.RuleMs = ms(time.Since(ruleStart)) + if err := be.Encode(baselineRecord{s.ID, s.SHA256, r.Baseline}); err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + err = judgeRecord(ctx, j, s.raw, &r) + cancel() + if err != nil { + r.Err = err.Error() + } else if err := ce.Encode(baselineRecord{s.ID, s.SHA256, r.Judged.Accepted()}); err != nil { + return err + } + ledger.Add(s.ID, r.Judged) + rows = append(rows, r) + fmt.Fprintf(os.Stderr, "replay %d/%d %s: baseline=%d accepted=%d error=%t\n", len(rows), len(m.Samples), s.ID, len(r.Baseline), len(r.Judged.Accepted()), r.Err != "") + } + if err := writeRows(filepath.Join(out, "rows.jsonl"), rows); err != nil { + return err + } + clusters, err := discoverReplay(m, j, rows, out) + if err != nil { + return err + } + generated, err := generateReplay(m, j, out) + if err != nil { + return err + } + summary := summarizeReplay(m, rows, generated, clusters) + summary.Requests, summary.CacheHits, summary.Claims = cached.stats() + summary.JunkRules = ledger.Report(3, 0.5) + if err := writeJSON(filepath.Join(out, "ledger.json"), ledger.Report(1, 0)); err != nil { + return err + } + summary.Model = provider.ID() + summary.InputTokens = tokens() + summary.Seconds = time.Since(started).Seconds() + summary.BaselineSource = "current_rules_on_saved_responses" + if historyPath != "" { + summary.BaselineSource = "historical_results" + } + if err := writeJSON(filepath.Join(out, "metrics.json"), summary); err != nil { + return err + } + if err := writeReplayReport(filepath.Join(out, "report.md"), summary, m); err != nil { + return err + } + fmt.Printf("saved %s\n", filepath.Join(out, "report.md")) + if summary.Errors > 0 { + return fmt.Errorf("%d failed rows; see rows.jsonl", summary.Errors) + } + if audit { + return maintainReplay(m, clusters, generated, out, engine) + } + return nil +} + +// Keep the exact labelled inputs together with the output for offline replay. +func saveReplaySnapshot(out string, m *replayManifest) error { + copy := *m + copy.Samples = append([]replaySample(nil), m.Samples...) + if err := os.MkdirAll(filepath.Join(out, "samples"), 0700); err != nil { + return err + } + for i := range copy.Samples { + s := ©.Samples[i] + s.Response = "samples/" + s.SHA256 + ".http" + if err := os.WriteFile(filepath.Join(out, s.Response), s.raw, 0600); err != nil { + return err + } + } + if info, ok := debug.ReadBuildInfo(); ok { + if err := writeJSON(filepath.Join(out, "build.json"), info); err != nil { + return err + } + } + return writeJSON(filepath.Join(out, "manifest.json"), copy) +} +func judgeRecord(ctx context.Context, j *judge.Judge, raw []byte, r *record) error { + started := time.Now() + var err error + r.Judged, err = j.Inspect(ctx, raw, r.Baseline) + r.JudgeMs = ms(time.Since(started)) + if err != nil { + r.Err = err.Error() + } + return err +} + +func discoverReplay(m *replayManifest, j *judge.Judge, rows []record, out string) ([]*maintain.Cluster, error) { + byID := map[string]record{} + for _, r := range rows { + byID[r.ID] = r + } + var samples []maintain.Sample + for _, s := range m.Samples { + r, ok := byID[s.ID] + if !ok || r.Err != "" { + continue + } + samples = append(samples, maintain.Sample{ID: s.ID, Host: sampleHost(s), Raw: s.raw, Accepted: r.Judged.Accepted()}) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + clusters, err := maintain.Discover(ctx, j, samples, 2) + if err != nil { + return nil, err + } + return clusters, writeJSON(filepath.Join(out, "discovery.json"), clusters) +} + +// Clusters carry sample IDs; rows do not duplicate cluster state. +func missingCandidates(clusters []*maintain.Cluster) map[string][]string { + out := map[string][]string{} + for _, c := range clusters { + if c.Outcome == jev.Refuted { + for _, id := range c.Samples { + out[id] = c.Candidates + } + } + } + return out +} + +// sampleHost is the host a sample was captured from: its group joins +// several addresses of one operator. +func sampleHost(s replaySample) string { + if s.Group != "" { + return s.Group + } + if u, err := url.Parse(s.URL); err == nil { + return strings.ToLower(u.Hostname()) + } + return s.URL +} + +func writeRows(path string, rows []record) error { + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + enc := json.NewEncoder(f) + for _, r := range rows { + if err := enc.Encode(r); err != nil { + return err + } + } + return nil +} diff --git a/cmd/judgeeval/replay_generation.go b/cmd/judgeeval/replay_generation.go new file mode 100644 index 0000000..9b1cae1 --- /dev/null +++ b/cmd/judgeeval/replay_generation.go @@ -0,0 +1,240 @@ +package main + +import ( + "context" + "fmt" + "net/url" + "os" + "path/filepath" + "strings" + "time" + + "github.com/chainreactors/fingers/common" + fingerlib "github.com/chainreactors/fingers/fingers" + "github.com/chainreactors/fingers/judge" + "github.com/chainreactors/fingers/judge/gen" + "gopkg.in/yaml.v3" +) + +// generationResult records observations only. The manifest owns the product, +// training split and truth labels; scores and status are derived from them. +type generationResult struct { + Name string `json:"name"` + File string `json:"file,omitempty"` + Training map[string]*common.Framework `json:"training"` + Holdout map[string]*common.Framework `json:"holdout"` + Excluded []string `json:"excluded"` + Error string `json:"error,omitempty"` +} + +func (r generationResult) assess(m *replayManifest, p generationPlan) (training, holdout stageScore, status string) { + test, _, err := generationSplit(m, p) + if err != nil || r.Error != "" { + return training, holdout, "failed" + } + trainingComplete := len(r.Training) == len(p.Positive)+len(p.Negative) + for _, ids := range [][]string{p.Positive, p.Negative} { + for _, id := range ids { + _, evaluated := r.Training[id] + trainingComplete = trainingComplete && evaluated + } + } + holdoutComplete := len(r.Holdout) == len(test) + for _, sample := range test { + _, evaluated := r.Holdout[sample.ID] + holdoutComplete = holdoutComplete && evaluated + } + for _, s := range m.Samples { + label, labelled := labelFor(s, p.Product) + if !labelled { + continue + } + if frame, evaluated := r.Training[s.ID]; evaluated { + training.add(label, frame) + } + if frame, evaluated := r.Holdout[s.ID]; evaluated { + holdout.add(label, frame) + } + } + switch { + case training.failed() || !trainingComplete: + status = "failed_training" + case holdout.failed(): + status = "failed_holdout" + case !holdoutComplete || holdout.Detection.TP == 0 || holdout.Detection.TN == 0: + status = "insufficient_holdout" + default: + status = "passed_holdout" + } + return +} + +func generationSplit(m *replayManifest, p generationPlan) ([]replaySample, []string, error) { + samples := map[string]replaySample{} + for _, s := range m.Samples { + samples[s.ID] = s + } + if p.Name == "" || p.Name != filepath.Base(p.Name) || strings.ContainsAny(p.Name, "/\\:") || p.Product == "" || len(p.Positive) == 0 || len(p.Negative) == 0 { + return nil, nil, fmt.Errorf("invalid generation plan") + } + trainIDs, trainGroups, trainHashes := map[string]bool{}, map[string]bool{}, map[string]bool{} + trainHosts := map[string]bool{} + host := func(s replaySample) string { + u, err := url.Parse(s.URL) + if err != nil { + return "" + } + return strings.ToLower(u.Hostname()) + } + for _, set := range []struct { + ids []string + want bool + }{{p.Positive, true}, {p.Negative, false}} { + for _, id := range set.ids { + s, ok := samples[id] + if !ok || trainIDs[id] { + return nil, nil, fmt.Errorf("missing/duplicate training sample %s", id) + } + l, ok := labelFor(s, p.Product) + if !ok || l.Present != set.want { + return nil, nil, fmt.Errorf("training label missing or contradictory: %s", id) + } + trainIDs[id] = true + trainGroups[s.Group] = true + if h := host(s); h != "" { + trainHosts[h] = true + } + trainHashes[s.ContentSHA256] = true + if p.Probe != "" { + probe, ok := samples[s.Probes[p.Probe]] + if !ok || probe.Group != s.Group { + return nil, nil, fmt.Errorf("missing or different-host probe for %s", id) + } + trainHashes[probe.ContentSHA256] = true + } + } + } + var test []replaySample + var excluded []string + seen := map[string]bool{} + for _, s := range m.Samples { + if _, ok := labelFor(s, p.Product); !ok || trainIDs[s.ID] { + continue + } + hash := s.ContentSHA256 + if p.Probe != "" { + probe, ok := samples[s.Probes[p.Probe]] + if !ok { + continue + } + hash = probe.ContentSHA256 + } + if trainGroups[s.Group] || trainHosts[host(s)] || trainHashes[hash] || seen[hash] { + excluded = append(excluded, s.ID) + continue + } + seen[hash] = true + test = append(test, s) + } + return test, excluded, nil +} +func generateReplay(m *replayManifest, j *judge.Judge, out string) ([]generationResult, error) { + samples := map[string]replaySample{} + for _, s := range m.Samples { + samples[s.ID] = s + } + results := make([]generationResult, 0, len(m.Generation)) + plans := map[string]bool{} + for _, p := range m.Generation { + if plans[p.Name] { + return nil, fmt.Errorf("duplicate generation plan %s", p.Name) + } + plans[p.Name] = true + r := generationResult{Name: p.Name, Training: map[string]*common.Framework{}, Holdout: map[string]*common.Framework{}} + test, excluded, err := generationSplit(m, p) + if err != nil { + return nil, err + } + r.Excluded = excluded + g := gen.New(j, p.Product) + for _, id := range p.Positive { + s := samples[id] + g.Positive(s.raw) + if p.Probe != "" { + g.Probe([]byte(p.Probe), samples[s.Probes[p.Probe]].raw) + } + } + for _, id := range p.Negative { + s := samples[id] + g.Negative(s.raw) + if p.Probe != "" { + g.Probe([]byte(p.Probe), samples[s.Probes[p.Probe]].raw) + } + } + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + f, err := g.Generate(ctx) + cancel() + if err != nil { + r.Error = err.Error() + results = append(results, r) + continue + } + data, err := yaml.Marshal(f) + if err != nil { + return nil, err + } + dir := filepath.Join(out, "fingerprints") + if err := os.MkdirAll(dir, 0755); err != nil { + return nil, err + } + r.File = filepath.ToSlash(filepath.Join("fingerprints", p.Name+".yaml")) + if err := os.WriteFile(filepath.Join(out, r.File), data, 0600); err != nil { + return nil, err + } + var loaded fingerlib.Finger + if err := yaml.Unmarshal(data, &loaded); err != nil { + return nil, err + } + if err := loaded.Compile(false); err != nil { + return nil, err + } + // Revalidate the exported rule on training samples before holding it out. + if err := g.Validate(&loaded); err != nil { + r.Error = err.Error() + results = append(results, r) + continue + } + // Preserve native detections, including nil for a tested non-match. + // Truth remains in the manifest and never enters generation. + for _, ids := range [][]string{p.Positive, p.Negative} { + for _, id := range ids { + r.Training[id] = evaluateGenerated(&loaded, p, samples[id], samples) + } + } + for _, sample := range test { + r.Holdout[sample.ID] = evaluateGenerated(&loaded, p, sample, samples) + } + results = append(results, r) + } + if err := writeJSON(filepath.Join(out, "generation.json"), results); err != nil { + return nil, err + } + return results, nil +} + +func evaluateGenerated(f *fingerlib.Finger, p generationPlan, s replaySample, samples map[string]replaySample) *common.Framework { + var frame *common.Framework + var matched bool + if p.Probe == "" { + frame, _, matched = f.PassiveMatch(fingerlib.NewContent(s.raw, "", true)) + } else { + frame, _, matched = f.ActiveMatch(2, func(request []byte) ([]byte, bool) { + id, ok := s.Probes[string(request)] + return samples[id].raw, ok + }) + } + if !matched { + return nil + } + return frame +} diff --git a/cmd/judgeeval/replay_generation_test.go b/cmd/judgeeval/replay_generation_test.go new file mode 100644 index 0000000..88316df --- /dev/null +++ b/cmd/judgeeval/replay_generation_test.go @@ -0,0 +1,126 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/chainreactors/fingers/common" + "github.com/chainreactors/fingers/judge" +) + +func TestGenerationRecordsNativeObservations(t *testing.T) { + plan := generationPlan{Name: "jenkins", Product: "Jenkins", Positive: []string{"train-positive"}, Negative: []string{"train-negative"}} + m := &replayManifest{Generation: []generationPlan{plan}} + for _, item := range []struct{ id, version string }{ + {"train-positive", "2.401.3"}, {"train-negative", ""}, {"test-positive", "2.402.1"}, {"test-negative", ""}, + } { + raw := "HTTP/1.1 200 OK\r\n" + if item.version != "" { + raw += "X-Jenkins: " + item.version + "\r\n" + } + raw += "\r\n" + item.id + "" + label := productLabel{Product: "Jenkins", Present: item.version != ""} + if label.Present { + label.Version = stringPtr(item.version) + } + m.Samples = append(m.Samples, replaySample{ID: item.id, Group: item.id, ContentSHA256: digest([]byte(raw)), raw: []byte(raw), Labels: []productLabel{label}}) + } + out := t.TempDir() + // Explicit response headers establish versions without any model calls. + results, err := generateReplay(m, judge.New(nil), out) + if err != nil || len(results) != 1 { + t.Fatalf("generation=%v error=%v", results, err) + } + data, err := os.ReadFile(filepath.Join(out, "generation.json")) + if err != nil { + t.Fatal(err) + } + var restored []generationResult + if err := json.Unmarshal(data, &restored); err != nil { + t.Fatal(err) + } + r := restored[0] + training, holdout, status := r.assess(m, plan) + if status != "passed_holdout" || training.Detection.TP != 1 || training.Detection.TN != 1 || training.Versions.Correct != 1 || holdout.Versions.Correct != 1 || holdout.Detection.TN != 1 { + t.Fatalf("status=%s training=%+v holdout=%+v error=%s", status, training, holdout, r.Error) + } + if frame, tested := r.Holdout["test-negative"]; !tested || frame != nil { + t.Fatal("evaluated non-match was lost") + } + if frame := r.Holdout["test-positive"]; frame == nil || frame.Name != "Jenkins" || versionOf(frame) != "2.402.1" { + t.Fatalf("native framework lost: %+v", frame) + } + if _, err := os.Stat(filepath.Join(out, r.File)); err != nil { + t.Fatal(err) + } + var fields []map[string]json.RawMessage + if err := json.Unmarshal(data, &fields); err != nil { + t.Fatal(err) + } + for _, name := range []string{"product", "status", "detection", "versions", "training_detection", "training_versions", "training_cases", "cases"} { + if _, exists := fields[0][name]; exists { + t.Fatalf("redundant generation field %s", name) + } + } + // Scores and conclusions follow the current labels and observations. + m.Samples[2].Labels[0].Version = stringPtr("9.9.9") + _, holdout, status = r.assess(m, plan) + if status != "failed_holdout" || holdout.Versions.Wrong != 1 { + t.Fatalf("stale assessment after label change: %s %+v", status, holdout) + } + r.Holdout["test-positive"].Version = "9.9.9" + _, _, status = r.assess(m, plan) + if status != "passed_holdout" { + t.Fatalf("stale assessment after observation change: %s", status) + } + report := filepath.Join(out, "report.md") + if err := writeReplayReport(report, summarizeReplay(m, nil, []generationResult{r}, nil), m); err != nil { + t.Fatal(err) + } + text, err := os.ReadFile(report) + if err != nil || !strings.Contains(string(text), "passed_holdout") { + t.Fatalf("report assessment missing: %s, %v", text, err) + } +} + +func TestGenerationAssessmentRequiresEveryExpectedObservation(t *testing.T) { + plan := generationPlan{Name: "product", Product: "Product", Positive: []string{"train-positive"}, Negative: []string{"train-negative"}} + m := &replayManifest{} + for _, id := range []string{"train-positive", "train-negative", "test-positive", "test-negative", "second-negative"} { + m.Samples = append(m.Samples, replaySample{ID: id, Group: id, ContentSHA256: id, Labels: []productLabel{{Product: plan.Product, Present: strings.HasSuffix(id, "positive")}}}) + } + for _, change := range []string{"none", "missing training", "wrong training key", "missing holdout", "wrong holdout key", "false negative", "error"} { + t.Run(change, func(t *testing.T) { + frame := common.NewFramework(plan.Product, common.FrameFromFingers) + r := generationResult{Training: map[string]*common.Framework{"train-positive": frame, "train-negative": nil}, Holdout: map[string]*common.Framework{"test-positive": frame, "test-negative": nil, "second-negative": nil}} + want := "passed_holdout" + switch change { + case "missing training", "wrong training key": + delete(r.Training, "train-negative") + if change == "wrong training key" { + r.Training["test-negative"] = nil + } + want = "failed_training" + case "missing holdout", "wrong holdout key": + delete(r.Holdout, "second-negative") + if change == "wrong holdout key" { + r.Holdout["train-negative"] = nil + } + want = "insufficient_holdout" + case "false negative": + r.Holdout["test-positive"] = nil + want = "failed_holdout" + case "error": + r.Error = "generation failed" + want = "failed" + } + _, _, status := r.assess(m, plan) + if status != want { + t.Fatalf("status=%s want=%s", status, want) + } + }) + } +} diff --git a/cmd/judgeeval/replay_metrics.go b/cmd/judgeeval/replay_metrics.go new file mode 100644 index 0000000..4674a96 --- /dev/null +++ b/cmd/judgeeval/replay_metrics.go @@ -0,0 +1,313 @@ +package main + +import ( + "fmt" + "math" + "os" + "strings" + + "github.com/chainreactors/fingers/common" + "github.com/chainreactors/fingers/judge" + "github.com/chainreactors/fingers/judge/maintain" + "github.com/chainreactors/utils/jev" +) + +type detectionScore struct { + TP int `json:"tp"` + FP int `json:"fp"` + TN int `json:"tn"` + FN int `json:"fn"` +} + +func (s *detectionScore) add(want, got bool) { + switch { + case want && got: + s.TP++ + case want: + s.FN++ + case got: + s.FP++ + default: + s.TN++ + } +} + +type versionScore struct { + Correct int `json:"correct"` + Wrong int `json:"wrong"` + Missing int `json:"missing"` + CorrectAbstentions int `json:"correct_abstentions"` + Unsupported int `json:"unsupported"` + ProductMissing int `json:"product_missing"` +} + +func (v *versionScore) add(f *common.Framework, want string) { + if f == nil { + v.ProductMissing++ + return + } + got := versionOf(f) + switch { + case want == "" && got == "": + v.CorrectAbstentions++ + case want == "": + v.Unsupported++ + case got == want: + v.Correct++ + case got == "": + v.Missing++ + default: + v.Wrong++ + } +} + +type stageScore struct { + Detection detectionScore `json:"detection"` + Versions versionScore `json:"versions"` + UnlabelledPredictions int `json:"unlabelled_predictions"` +} + +// add is the same label comparison for scan, generation and library evaluation. +func (s *stageScore) add(label productLabel, frame *common.Framework) { + s.Detection.add(label.Present, frame != nil) + if label.Version != nil { + s.Versions.add(frame, *label.Version) + } +} + +func (s stageScore) failed() bool { + return s.Detection.FP > 0 || s.Detection.FN > 0 || s.Versions.Wrong > 0 || s.Versions.Missing > 0 || s.Versions.Unsupported > 0 +} + +type replaySummary struct { + Schema int `json:"schema"` + Model string `json:"model"` + BaselineSource string `json:"baseline_source"` + Samples int `json:"samples"` + UniqueHosts int `json:"unique_hosts"` + UniqueResponses int `json:"unique_responses"` + Errors int `json:"errors"` + LabelledPairs int `json:"labelled_pairs"` + Baseline stageScore `json:"baseline"` + Accepted stageScore `json:"accepted"` + FalseHitsRemoved int `json:"false_hits_removed"` + TrueHitsRemoved int `json:"true_hits_removed"` + NaturalMissesRecovered int `json:"natural_misses_recovered"` + MissedProductsDiscovered int `json:"missed_products_discovered"` + VersionFills int `json:"version_fills"` + CorrectVersionFills int `json:"correct_version_fills"` + IncorrectVersionFills int `json:"incorrect_version_fills"` + UnlabelledVersionFills int `json:"unlabelled_version_fills"` + Generation []generationResult `json:"generation"` + Disagreements []string `json:"disagreements"` + Options map[string]int `json:"options"` + Outcomes map[jev.Outcome]int `json:"presence_outcomes"` + JunkRules []*maintain.RuleStats `json:"junk_rules"` + Clusters []*maintain.Cluster `json:"clusters"` + Claims int `json:"claims"` + Requests int `json:"provider_requests"` + CacheHits int `json:"exact_cache_hits"` + InputTokens int64 `json:"input_tokens"` + Seconds float64 `json:"seconds"` +} + +func summarizeReplay(m *replayManifest, rows []record, gen []generationResult, clusters []*maintain.Cluster) replaySummary { + out := replaySummary{Schema: 3, Clusters: clusters, Samples: len(m.Samples), Generation: gen, Options: map[string]int{}, Outcomes: map[jev.Outcome]int{}} + missing := missingCandidates(clusters) + hosts, hashes := map[string]bool{}, map[string]bool{} + byID := map[string]record{} + for _, r := range rows { + byID[r.ID] = r + } + for _, s := range m.Samples { + hosts[s.Group] = true + hashes[s.ContentSHA256] = true + r, ok := byID[s.ID] + if !ok || r.Err != "" { + out.Errors++ + continue + } + accepted := r.Judged.Accepted() + for _, f := range r.Judged { + if f != nil && f.Judge != nil && !f.Judge.Duplicate && f.Judge.Outcome != "" { + out.Options[f.Judge.Option]++ + out.Outcomes[jev.Outcome(f.Judge.Outcome)]++ + } + } + for _, stage := range []struct { + score *stageScore + frames common.Frameworks + }{{&out.Baseline, r.Baseline}, {&out.Accepted, accepted}} { + for _, l := range s.Labels { + f := findLabel(stage.frames, l) + stage.score.add(l, f) + } + for _, f := range stage.frames { + if f != nil { + if _, ok := labelFor(s, f.Name); !ok { + stage.score.UnlabelledPredictions++ + } + } + } + } + for _, l := range s.Labels { + out.LabelledPairs++ + before, after := findLabel(r.Baseline, l), findLabel(accepted, l) + if before != nil && after == nil { + if l.Present { + out.TrueHitsRemoved++ + } else { + out.FalseHitsRemoved++ + } + } + if l.Present && before == nil && after != nil { + out.NaturalMissesRecovered++ + } + if l.Present && after == nil && containsProductName(missing[r.ID], l) { + out.MissedProductsDiscovered++ + } + // Count a labelled product once, considering all baseline aliases. + // Raw per-name changes remain in rows.jsonl for audit. + if after != nil && versionOf(after) != "" && (before == nil || versionOf(before) == "") { + out.VersionFills++ + switch { + case !l.Present: + out.IncorrectVersionFills++ + case l.Version == nil: + out.UnlabelledVersionFills++ + case versionOf(after) == *l.Version: + out.CorrectVersionFills++ + default: + out.IncorrectVersionFills++ + } + } + if (after != nil) != l.Present { + out.Disagreements = append(out.Disagreements, fmt.Sprintf("%s %s: presence want=%t got=%t", s.ID, l.Product, l.Present, after != nil)) + } + if l.Version != nil && after != nil && versionOf(after) != *l.Version { + out.Disagreements = append(out.Disagreements, fmt.Sprintf("%s %s: version want=%q got=%q", s.ID, l.Product, *l.Version, versionOf(after))) + } + } + unlabelled := map[string]bool{} + for _, f := range accepted { + if f.Attributes == nil || f.Version == "" { + continue + } + name := f.Name + before := findLabel(r.Baseline, productLabel{Product: name}) + if before != nil && before.Attributes != nil && versionOf(before) != "" { + continue + } + if _, ok := labelFor(s, name); ok { + continue + } + key := judge.NormalizeName(name) + if !unlabelled[key] { + unlabelled[key] = true + out.VersionFills++ + out.UnlabelledVersionFills++ + } + } + } + out.UniqueHosts = len(hosts) + out.UniqueResponses = len(hashes) + return out +} +func containsProductName(names []string, l productLabel) bool { + for _, name := range names { + for _, expected := range append([]string{l.Product}, l.Aliases...) { + if judge.NormalizeName(name) == judge.NormalizeName(expected) { + return true + } + } + } + return false +} +func writeReplayReport(path string, s replaySummary, m *replayManifest) error { + var b strings.Builder + fmt.Fprintf(&b, "# 真实响应回放评估\n\n模型：%s；基线：%s。\n\n%d 份快照，%d 个主机组，%d 种去除易变响应头后的响应，%d 个已标注产品判断，失败 %d 份。\n\n", s.Model, s.BaselineSource, s.Samples, s.UniqueHosts, s.UniqueResponses, s.LabelledPairs, s.Errors) + b.WriteString("统计范围是保存的响应所支持的产品与版本。页面自报版本不是服务器安装审计；正负标注不代表完整技术栈。未标注的输出不计为误报，也不计为已证明正确。页面级结果含同主机响应，不能视作独立随机样本。\n\n") + b.WriteString("| 阶段 | TP | FP | FN | TN | 未标注输出 |\n|---|---:|---:|---:|---:|---:|\n") + for _, r := range []struct { + name string + s stageScore + }{{"原结果", s.Baseline}, {"Accepted", s.Accepted}} { + d := r.s.Detection + fmt.Fprintf(&b, "| %s | %d | %d | %d | %d | %d |\n", r.name, d.TP, d.FP, d.FN, d.TN, r.s.UnlabelledPredictions) + } + fmt.Fprintf(&b, "\n有标注的误报移除 %d；真实命中误删 %d；自然漏报补回 %d。模型拒绝或新增的未标注项保留在 rows.jsonl 中供复核。\n\n", s.FalseHitsRemoved, s.TrueHitsRemoved, s.NaturalMissesRecovered) + fmt.Fprintf(&b, "仍漏报的产品中，%d 项所在页面簇的 Coverage 判定为 missing，且产品名在代码给出的候选名中；不计作已识别，也不计作漏报补回。\n\n", s.MissedProductsDiscovered) + b.WriteString("| 判定 | 产品数 |\n|---|---:|\n") + for _, v := range []string{judge.OptionDeclared, judge.OptionAbsent, judge.OptionRunning, judge.OptionMentioned, judge.OptionUnrelated, jev.OptionInsufficient} { + fmt.Fprintf(&b, "| %s | %d |\n", v, s.Options[v]) + } + coverage := map[jev.Outcome]int{} + for _, c := range s.Clusters { + coverage[c.Outcome]++ + } + b.WriteString("\n| Claim | holds | refuted | insufficient |\n|---|---:|---:|---:|\n") + for _, row := range []struct { + name string + n map[jev.Outcome]int + }{{"Presence（误报，按产品）", s.Outcomes}, {"Coverage（漏报，按页面簇）", coverage}} { + fmt.Fprintf(&b, "| %s | %d | %d | %d |\n", row.name, row.n[jev.Holds], row.n[jev.Refuted], row.n[jev.Insufficient]) + } + fmt.Fprintf(&b, "\n共 %d 题（含缓存命中），平均每页 %.1f 题。\n\n", s.Claims, float64(s.Claims)/math.Max(float64(s.Samples), 1)) + b.WriteString("| 版本阶段 | 正确 | 错误 | 缺版本 | 正确留空 | 无依据填值 | 产品未识别 |\n|---|---:|---:|---:|---:|---:|---:|\n") + for _, r := range []struct { + name string + s versionScore + }{{"原结果", s.Baseline.Versions}, {"Accepted", s.Accepted.Versions}} { + v := r.s + fmt.Fprintf(&b, "| %s | %d | %d | %d | %d | %d | %d |\n", r.name, v.Correct, v.Wrong, v.Missing, v.CorrectAbstentions, v.Unsupported, v.ProductMissing) + } + fmt.Fprintf(&b, "\n按产品合并别名后的新增版本 %d：标注确认正确 %d、错误 %d、未标注 %d。已有非空版本保留，错误由报告揭示；逐名称变化见 rows.jsonl。\n\n", s.VersionFills, s.CorrectVersionFills, s.IncorrectVersionFills, s.UnlabelledVersionFills) + b.WriteString("## 指纹生成\n\n训练只使用 positive/negative 指定快照。测试排除训练主机、相同响应与重复测试响应；生成结果重新从 YAML 加载、编译后测试。此处 passed 仅代表本批独立样本通过，仍输出候选规则供评审。\n\n| 计划 | 名称 | 训练 TP/FP/FN/TN | 训练版本 正确/错误/缺失 | 独立测试 TP/FP/FN/TN | 测试版本 正确/错误/缺失 | 排除 | 结论 |\n|---|---|---|---|---|---|---:|---|\n") + plans := map[string]generationPlan{} + for _, p := range m.Generation { + plans[p.Name] = p + } + for _, g := range s.Generation { + p := plans[g.Name] + training, holdout, status := g.assess(m, p) + d, v := holdout.Detection, holdout.Versions + td, tv := training.Detection, training.Versions + fmt.Fprintf(&b, "| %s | %s | %d/%d/%d/%d | %d/%d/%d | %d/%d/%d/%d | %d/%d/%d | %d | %s |\n", g.Name, p.Product, td.TP, td.FP, td.FN, td.TN, tv.Correct, tv.Wrong, tv.Missing, d.TP, d.FP, d.FN, d.TN, v.Correct, v.Wrong, v.Missing, len(g.Excluded), status) + } + for _, g := range s.Generation { + if g.Error != "" { + fmt.Fprintf(&b, "\n%s: %s\n\n", g.Name, g.Error) + } + } + b.WriteString("\n## 垃圾规则候选\n\n至少 3 次判定命中、被否决不少于 50% 的规则。\n\n| 引擎 | 指纹 | 规则 / matcher | 命中 | 被否决 |\n|---|---|---|---:|---:|\n") + for _, r := range s.JunkRules { + matcher := "" + if r.Matcher != "" { + matcher = fmt.Sprintf("#%d `%s`", r.Rule, strings.ReplaceAll(r.Matcher, "|", "\\|")) + } + fmt.Fprintf(&b, "| %s | %s | %s | %d | %.0f%% |\n", r.Engine, r.Name, matcher, r.Hits(), 100*r.RefutedFraction()) + } + b.WriteString("\n## 漏报（Coverage）\n\n至少 2 个独立主机的页面簇；Claim：规则已报告的产品解释了这个页面。候选名由代码从页面中抽取，供人命名，不作为选项。\n\n| 主机数 | 判定 | 已报告 | 候选名（前 3） | 样例 |\n|---:|---|---|---|---|\n") + for _, c := range s.Clusters { + ids := c.Samples + if len(ids) > 3 { + ids = ids[:3] + } + cands := c.Candidates + if len(cands) > 3 { + cands = cands[:3] + } + fmt.Fprintf(&b, "| %d | %s | %s | %s | %s |\n", c.Hosts, c.Coverage.Option, strings.Join(c.Reported, ", "), strings.ReplaceAll(strings.Join(cands, ", "), "|", "\\|"), strings.Join(ids, ", ")) + } + b.WriteString("\ngeneration.json 的 training / holdout 保存逐样本原生 Framework 或 null，真值来自 manifest.json；训练正确不等于独立验证通过。\n") + b.WriteString("\n## 待核查\n\n") + if len(s.Disagreements) == 0 { + b.WriteString("已标注范围内没有分歧。\n") + } else { + for _, d := range s.Disagreements { + fmt.Fprintf(&b, "- %s\n", d) + } + } + fmt.Fprintf(&b, "\nProvider 调用 %d，完整输入精确缓存命中 %d，输入 token %d，耗时 %.1f 秒。费用未估算。\n\nbaseline.jsonl 保存输入；cleaned.jsonl 仅含成功完成的结果；rows.jsonl 保存基线、全部判定注解和错误，discovery.json 按样本 ID 保存漏报簇；generation.json 保存每个独立样本结果。\n", s.Requests, s.CacheHits, s.InputTokens, s.Seconds) + return os.WriteFile(path, []byte(b.String()), 0600) +} diff --git a/cmd/judgeeval/replay_test.go b/cmd/judgeeval/replay_test.go new file mode 100644 index 0000000..8536c01 --- /dev/null +++ b/cmd/judgeeval/replay_test.go @@ -0,0 +1,318 @@ +package main + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + + "github.com/chainreactors/utils/jev" + + "github.com/chainreactors/fingers/common" + "github.com/chainreactors/fingers/judge" + "github.com/chainreactors/fingers/judge/maintain" +) + +func stringPtr(s string) *string { return &s } +func testFrames(name, version string) common.Frameworks { + f := common.NewFrameworkWithVersion(name, common.FrameFromGUESS, version) + out := common.Frameworks{} + out.Add(f) + return out +} +func TestReplayCountsErrorsAndUnlabelledSeparately(t *testing.T) { + m := &replayManifest{Samples: []replaySample{{ID: "a", Group: "host-a", ContentSHA256: "a", Labels: []productLabel{{Product: "Apache", Present: true, Version: stringPtr("2.4.38")}, {Product: "nginx", Present: false}}}, {ID: "failed", Group: "host-b", ContentSHA256: "b", Labels: []productLabel{{Product: "Apache", Present: true}}}}} + base := testFrames("nginx", "") + after := testFrames("apache http server", "2.4.38") + after.Add(common.NewFramework("unlabelled-library", common.FrameFromGUESS)) + r := record{ID: "a", Baseline: base, Judged: after} + summary := summarizeReplay(m, []record{r, {ID: "failed", Err: "network"}}, nil, nil) + if summary.Errors != 1 || summary.LabelledPairs != 2 || summary.Baseline.Detection.FP != 1 || summary.Baseline.Detection.FN != 1 || summary.Accepted.Detection.TP != 1 || summary.Accepted.Detection.TN != 1 || summary.Accepted.UnlabelledPredictions != 1 { + t.Fatalf("metrics: %+v", summary) + } + if summary.NaturalMissesRecovered != 1 || summary.FalseHitsRemoved != 1 || summary.CorrectVersionFills != 1 { + t.Fatalf("changes: %+v", summary) + } +} +func TestReplayExactNamesDoNotCreditDifferentProducts(t *testing.T) { + frames := testFrames("wordpress-woocommerce", "10.4.4") + if findLabel(frames, productLabel{Product: "wordpress"}) != nil { + t.Fatal("substring credited a different product") + } + if findLabel(frames, productLabel{Product: "WooCommerce", Aliases: []string{"wordpress-woocommerce"}}) == nil { + t.Fatal("explicit alias not matched") + } +} +func TestReplayAbstentionRequiresIdentifiedProduct(t *testing.T) { + v := versionScore{} + v.add(nil, "") + v.add(common.NewFramework("nginx", common.FrameFromGUESS), "") + v.add(common.NewFrameworkWithVersion("nginx", common.FrameFromGUESS, "1.2.3"), "") + if v.ProductMissing != 1 || v.CorrectAbstentions != 1 || v.Unsupported != 1 { + t.Fatalf("versions: %+v", v) + } +} +func TestGenerationSplitRejectsLeakageAndContradictoryLabels(t *testing.T) { + sample := func(id, group, hash string, present bool) replaySample { + return replaySample{ID: id, Group: group, ContentSHA256: hash, Labels: []productLabel{{Product: "Caddy", Present: present}}} + } + m := &replayManifest{Samples: []replaySample{sample("train", "a", "one", true), sample("negative", "b", "two", false), sample("same-host", "a", "three", true), sample("same-response", "c", "one", true), sample("test", "d", "four", true), sample("duplicate-test", "e", "four", true), sample("test-negative", "f", "five", false)}} + p := generationPlan{Name: "caddy", Product: "Caddy", Positive: []string{"train"}, Negative: []string{"negative"}} + test, excluded, err := generationSplit(m, p) + if err != nil || len(test) != 2 || len(excluded) != 3 { + t.Fatalf("split test=%v excluded=%v err=%v", test, excluded, err) + } + p.Positive = []string{"negative"} + if _, _, err := generationSplit(m, p); err == nil { + t.Fatal("contradictory training truth accepted") + } +} +func TestReplayValidatesResponseHashAndEvidence(t *testing.T) { + dir := t.TempDir() + raw := []byte("HTTP/1.1 200 OK\r\nServer: nginx/1.21.5\r\nContent-Length: 0\r\n\r\n") + if err := os.WriteFile(filepath.Join(dir, "page.http"), raw, 0600); err != nil { + t.Fatal(err) + } + m := replayManifest{Schema: 1, Samples: []replaySample{{ID: "one", URL: "http://example.test/", Group: "example.test", CapturedAt: "2026-09-25T00:00:00Z", Response: "page.http", SHA256: digest(raw), Labels: []productLabel{{Product: "nginx", Present: true, Evidence: []string{"Server: nginx/1.21.5"}, Basis: "header"}}}}} + path := filepath.Join(dir, "manifest.json") + if err := writeJSON(path, m); err != nil { + t.Fatal(err) + } + if _, err := loadReplay(path); err != nil { + t.Fatal(err) + } + m.Samples[0].SHA256 = strings.Repeat("0", 64) + _ = writeJSON(path, m) + if _, err := loadReplay(path); err == nil { + t.Fatal("changed response accepted") + } + m.Samples[0].SHA256 = digest(raw) + m.Samples[0].Labels[0].Evidence = []string{"invented"} + _ = writeJSON(path, m) + if _, err := loadReplay(path); err == nil { + t.Fatal("missing evidence accepted") + } +} +func TestReplayConfinesResponsePaths(t *testing.T) { + if _, err := confinedPath(t.TempDir(), "../secret.http"); err == nil { + t.Fatal("outside path accepted") + } +} + +type replayTestProvider struct { + calls int + id string +} + +func (p *replayTestProvider) ID() string { return "test/replay/" + p.id } +func (p *replayTestProvider) Judge(_ context.Context, _ interface{}, qs map[string]jev.Claim) (map[string]jev.Ruling, error) { + p.calls++ + rulings := map[string]jev.Ruling{} + for key := range qs { + rulings[key] = jev.Ruling{Option: "yes", Confidence: 1} + } + return rulings, nil +} + +func TestReplayOfflineNeverCallsProvider(t *testing.T) { + provider := &replayTestProvider{} + p := &recordingProvider{Provider: provider, dir: t.TempDir(), offline: true} + qs := map[string]jev.Claim{"product": {Statement: "Is this nginx?", Options: map[string]jev.Option{"yes": {Outcome: jev.Holds}, "no": {Outcome: jev.Refuted}, jev.OptionInsufficient: {Outcome: jev.Insufficient}}}} + ctx := context.Background() + if _, err := p.Judge(ctx, "page", qs); err == nil || !strings.Contains(err.Error(), "offline cache miss") { + t.Fatalf("expected offline cache miss: %v", err) + } + if provider.calls != 0 || p.calls != 0 { + t.Fatal("offline cache miss called the provider") + } + p.offline = false + if _, err := p.Judge(ctx, "page", qs); err != nil { + t.Fatal(err) + } + p.offline = true + if rulings, err := p.Judge(ctx, "page", qs); err != nil || rulings["product"].Option != "yes" { + t.Fatalf("cached ruling: %v %v", rulings, err) + } + if _, err := p.Judge(ctx, "different page", qs); err == nil { + t.Fatal("different state reused cached ruling") + } + provider.id = "different-endpoint" + if _, err := p.Judge(ctx, "page", qs); err == nil { + t.Fatal("different endpoint reused cached ruling") + } + if provider.calls != 1 || p.calls != 1 || p.hits != 1 { + t.Fatalf("calls=%d requests=%d hits=%d", provider.calls, p.calls, p.hits) + } +} + +func TestReplayVersionFillsDeduplicateAliases(t *testing.T) { + l := productLabel{Product: "PHP", Aliases: []string{"php-runtime"}, Present: true, Version: stringPtr("5.6.40")} + m := &replayManifest{Samples: []replaySample{{ID: "a", Labels: []productLabel{l}}}} + after := testFrames("PHP", "5.6.40") + after.Add(common.NewFrameworkWithVersion("php-runtime", common.FrameFromGUESS, "5.6.40")) + r := record{ID: "a", Baseline: testFrames("PHP", "5.6.40"), Judged: after} + if s := summarizeReplay(m, []record{r}, nil, nil); s.VersionFills != 0 { + t.Fatalf("existing alias version counted as new: %+v", s) + } + r.Baseline = testFrames("PHP", "") + if s := summarizeReplay(m, []record{r}, nil, nil); s.VersionFills != 1 || s.CorrectVersionFills != 1 { + t.Fatalf("alias fills counted twice: %+v", s) + } +} + +func TestReplayDiscoveryDoesNotRecoverDetections(t *testing.T) { + m := &replayManifest{Samples: []replaySample{{ID: "a", Labels: []productLabel{{Product: "New API", Present: true}}}}} + r := record{ID: "a"} + clusters := []*maintain.Cluster{{Samples: []string{"a"}, Outcome: jev.Refuted, Candidates: []string{"Login", "New API"}}} + s := summarizeReplay(m, []record{r}, nil, clusters) + if s.MissedProductsDiscovered != 1 || s.NaturalMissesRecovered != 0 || s.Accepted.Detection.FN != 1 { + t.Fatalf("discovery credited as detections: %+v", s) + } +} + +func TestReplaySnapshotIsSelfContained(t *testing.T) { + raw := []byte("HTTP/1.1 200 OK\r\nServer: nginx/1.21.5\r\nContent-Length: 0\r\n\r\n") + m := &replayManifest{Schema: 1, Samples: []replaySample{{ID: "a", URL: "http://example.test/", Group: "example.test", CapturedAt: "2026-09-25T00:00:00Z", Response: "unavailable/page.http", SHA256: digest(raw), raw: raw}}} + dir := t.TempDir() + if err := saveReplaySnapshot(dir, m); err != nil { + t.Fatal(err) + } + loaded, err := loadReplay(filepath.Join(dir, "manifest.json")) + if err != nil { + t.Fatal(err) + } + if string(loaded.Samples[0].raw) != string(raw) || loaded.Samples[0].SHA256 != m.Samples[0].SHA256 { + t.Fatal("snapshot response changed") + } + if m.Samples[0].Response != "unavailable/page.http" { + t.Fatal("snapshot write mutated source manifest") + } +} + +func TestGenerationSplitRejectsSameHostnameAcrossGroups(t *testing.T) { + m := &replayManifest{Samples: []replaySample{ + {ID: "positive", URL: "http://example.test/", Group: "one", ContentSHA256: "1", Labels: []productLabel{{Product: "Caddy", Present: true}}}, + {ID: "negative", URL: "http://negative.test/", Group: "two", ContentSHA256: "2", Labels: []productLabel{{Product: "Caddy", Present: false}}}, + {ID: "leak", URL: "https://EXAMPLE.test:443/another-page", Group: "three", ContentSHA256: "3", Labels: []productLabel{{Product: "Caddy", Present: true}}}, + }} + plan := generationPlan{Name: "caddy", Product: "Caddy", Positive: []string{"positive"}, Negative: []string{"negative"}} + test, excluded, err := generationSplit(m, plan) + if err != nil || len(test) != 0 || len(excluded) != 1 || excluded[0] != "leak" { + t.Fatalf("hostname leakage: test=%v excluded=%v error=%v", test, excluded, err) + } +} + +// Exercise both public CLI modes against one local Jev server, then replay +// offline from the exact same cache. Reports must explain the same evaluation. +func TestDirectoryAndReplayShareRecordAndCache(t *testing.T) { + var calls int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt64(&calls, 1) + var req struct { + Questions map[string]struct{ Criteria map[string]interface{} } + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Error(err) + w.WriteHeader(400) + return + } + answers := map[string]map[string]interface{}{} + for id, q := range req.Questions { + option := jev.OptionInsufficient + if _, ok := q.Criteria[judge.OptionRunning]; ok { + option = judge.OptionRunning + } + answers[id] = map[string]interface{}{"choice": option, "confidence": 1} + } + _ = json.NewEncoder(w).Encode(map[string]interface{}{"answers": answers}) + })) + defer srv.Close() + t.Setenv(jev.EnvAPIKey, "local-test") + root := t.TempDir() + samples := filepath.Join(root, "samples") + if err := os.Mkdir(samples, 0700); err != nil { + t.Fatal(err) + } + raw := []byte("HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\nJenkinsJenkins ") + if err := os.WriteFile(filepath.Join(samples, "a.http"), raw, 0600); err != nil { + t.Fatal(err) + } + cache := filepath.Join(root, "cache") + directory := filepath.Join(root, "directory") + if err := run("jev", samples, "", cache, directory, srv.URL, 100, 2, 0); err != nil { + t.Fatal(err) + } + readRow := func(dir string) record { + t.Helper() + data, err := os.ReadFile(filepath.Join(dir, "rows.jsonl")) + if err != nil { + t.Fatal(err) + } + var r record + if err := json.Unmarshal(data, &r); err != nil { + t.Fatal(err) + } + return r + } + first := readRow(directory) + if len(first.Judged) == 0 || atomic.LoadInt64(&calls) != 1 { + t.Fatalf("expected one presence batch: frames=%d calls=%d", len(first.Judged), calls) + } + m := replayManifest{Schema: 1, Samples: []replaySample{{ID: "a.http", URL: "http://example.test/", Group: "example.test", CapturedAt: "2026-09-26T00:00:00Z", Response: "samples/a.http", SHA256: digest(raw)}}} + manifest := filepath.Join(root, "manifest.json") + if err := writeJSON(manifest, m); err != nil { + t.Fatal(err) + } + replayDir := filepath.Join(root, "replay") + if err := replay(manifest, "", "jev", srv.URL, cache, replayDir, 100, true, false, ""); err != nil { + t.Fatal(err) + } + second := readRow(replayDir) + a, _ := json.Marshal(first.Judged) + b, _ := json.Marshal(second.Judged) + if string(a) != string(b) || atomic.LoadInt64(&calls) != 1 { + t.Fatal("replay disagrees or invoked backend") + } + for _, dir := range []string{directory, replayDir} { + data, err := os.ReadFile(filepath.Join(dir, "metrics.json")) + if err != nil { + t.Fatal(err) + } + var summary replaySummary + if err := json.Unmarshal(data, &summary); err != nil { + t.Fatal(err) + } + if summary.Schema != 3 || summary.Errors != 0 { + t.Fatalf("summary %+v", summary) + } + } +} + +func TestLoadLabels(t *testing.T) { + path := filepath.Join(t.TempDir(), "labels.json") + data := []byte(`{"a":[{"product":"Orion","aliases":["Orion-console"],"present":true,"version":"1.2.3"}]}`) + if err := os.WriteFile(path, data, 0600); err != nil { + t.Fatal(err) + } + labels, err := loadLabels(path) + if err != nil || len(labels["a"]) != 1 || *labels["a"][0].Version != "1.2.3" || len(labels["a"][0].Aliases) != 1 { + t.Fatalf("labels=%v error=%v", labels, err) + } + if _, err := loadLabels("testdata/labels.json"); err != nil { + t.Fatal(err) + } +} + +func TestReplayHandlesBaselineWithoutAttributes(t *testing.T) { + frames := common.Frameworks{"nginx": {Name: "nginx"}} + m := &replayManifest{Samples: []replaySample{{ID: "a", Labels: []productLabel{{Product: "nginx", Present: true, Version: stringPtr("")}}}}} + summary := summarizeReplay(m, []record{{ID: "a", Baseline: frames, Judged: frames}}, nil, nil) + if summary.Accepted.Versions.CorrectAbstentions != 1 || summary.Baseline.Detection.TP != 1 { + t.Fatalf("summary %+v", summary) + } +} diff --git a/cmd/judgeeval/testdata/labels.json b/cmd/judgeeval/testdata/labels.json new file mode 100644 index 0000000..9038e45 --- /dev/null +++ b/cmd/judgeeval/testdata/labels.json @@ -0,0 +1,167 @@ +{ + "blog_mentions_jenkins.http": [ + {"product": "cloudflare", "present": true, "basis": "文章正文提到 Jenkins/ThinkPHP"}, + {"product": "hugo", "present": true, "basis": "文章正文提到 Jenkins/ThinkPHP"}, + {"product": "jenkins", "present": false, "basis": "文章正文提到 Jenkins/ThinkPHP"}, + {"product": "thinkphp", "present": false, "basis": "文章正文提到 Jenkins/ThinkPHP"} + ], + "confluence_login.http": [ + {"product": "confluence", "present": true, "version": "7.19.8"}, + {"product": "struts2", "present": false}, + {"product": "telerik_report_server", "present": false} + ], + "corporate_cn_portal.http": [ + {"product": "nginx", "present": true, "basis": "新闻里提到用友，规则没误报，是对照"}, + {"product": "jquery", "present": true, "basis": "新闻里提到用友，规则没误报，是对照"} + ], + "corporate_site.http": [ + {"product": "nginx", "present": true} + ], + "custom_employee_portal.http": [ + {"product": "nginx", "present": true, "basis": "单位自研登录页，不是通用产品"} + ], + "dir_listing.http": [ + {"product": "apache", "present": true, "version": "2.4.41"} + ], + "discuz_forum.http": [ + {"product": "discuz", "present": true, "version": "3.4", "basis": "基于 CMS 的单位内容页"} + ], + "docs_tomcat_howto.http": [ + {"product": "github pages", "present": true, "basis": "正文里的 Tomcat/nginx 版本号是干扰"}, + {"product": "jekyll", "present": true, "version": "3.9.3", "basis": "正文里的 Tomcat/nginx 版本号是干扰"}, + {"product": "tomcat", "present": false, "basis": "正文里的 Tomcat/nginx 版本号是干扰"}, + {"product": "apache-web-server", "present": false, "basis": "正文里的 Tomcat/nginx 版本号是干扰"} + ], + "gitlab_signin.http": [ + {"product": "gitlab", "present": true, "version": ""}, + {"product": "webp_server_go", "present": false} + ], + "grafana_login.http": [ + {"product": "grafana", "present": true, "version": "10.2.3"} + ], + "harbor_portal.http": [ + {"product": "harbor", "present": true, "version": ""} + ], + "iis_default.http": [ + {"product": "iis", "present": true, "version": "10.0"} + ], + "jenkins_login.http": [ + {"product": "jenkins", "present": true, "version": "2.401.3"}, + {"product": "log4j2", "present": false} + ], + "job_posting_cn.http": [ + {"product": "tengine", "present": true, "basis": "招聘要求里列的技术栈"}, + {"product": "elasticsearch", "present": false, "basis": "招聘要求里列的技术栈"}, + {"product": "log4j2", "present": false, "basis": "招聘要求里列的技术栈"}, + {"product": "gitlab", "present": false, "basis": "招聘要求里列的技术栈"}, + {"product": "jenkins", "present": false, "basis": "招聘要求里列的技术栈"}, + {"product": "thinkphp", "present": false, "basis": "招聘要求里列的技术栈"} + ], + "json_api.http": [ + {"product": "uvicorn", "present": true}, + {"product": "postgresql", "present": false} + ], + "kibana_home.http": [ + {"product": "kibana", "present": true, "version": "8.11.1"}, + {"product": "log4j2", "present": false} + ], + "minio_console.http": [ + {"product": "minio", "present": true, "version": ""} + ], + "nacos_console.http": [ + {"product": "nacos", "present": true, "version": "2.2.3"} + ], + "news_fortios_patch.http": [ + {"product": "cloudflare", "present": true, "basis": "新闻里的 FortiOS 版本号是干扰"}, + {"product": "ghost", "present": true, "version": "5.71", "basis": "新闻里的 FortiOS 版本号是干扰"}, + {"product": "fortianalyzer", "present": false, "basis": "新闻里的 FortiOS 版本号是干扰"}, + {"product": "vnpt-technology-igate", "present": false, "basis": "新闻里的 FortiOS 版本号是干扰"}, + {"product": "飞塔防火墙", "present": false, "basis": "新闻里的 FortiOS 版本号是干扰"}, + {"product": "fortinet sslvpn", "present": false, "basis": "新闻里的 FortiOS 版本号是干扰"} + ], + "nginx_404.http": [ + {"product": "nginx", "present": true, "version": "1.22.1"} + ], + "nginx_default.http": [ + {"product": "nginx", "present": true, "version": "1.18.0"} + ], + "pgadmin_login.http": [ + {"product": "pgadmin", "present": true, "version": "", "basis": "ver=80000 是内部编号，不是版本号"}, + {"product": "gunicorn", "present": true, "basis": "ver=80000 是内部编号，不是版本号"}, + {"product": "postgresql", "present": false, "basis": "ver=80000 是内部编号，不是版本号"} + ], + "phpmyadmin_login.http": [ + {"product": "phpmyadmin", "present": true, "version": "5.2.1", "basis": "PHP/8.1.12、Apache/2.4.54 是干扰"}, + {"product": "php", "present": true, "basis": "PHP/8.1.12、Apache/2.4.54 是干扰"} + ], + "portainer_login.http": [ + {"product": "portainer", "present": true, "version": ""} + ], + "rabbitmq_mgmt.http": [ + {"product": "rabbitmq", "present": true, "version": "", "basis": "js 文件名里的是第三方库版本"}, + {"product": "log4j2", "present": false, "basis": "js 文件名里的是第三方库版本"}, + {"product": "ipeakcms", "present": false, "basis": "js 文件名里的是第三方库版本"} + ], + "rebranded_grafana.http": [ + {"product": "grafana", "present": true, "version": "9.5.2", "basis": "改过品牌，规则漏报"} + ], + "rebranded_jenkins.http": [ + {"product": "jenkins", "present": true, "version": "", "basis": "改过品牌，规则漏报"} + ], + "rebranded_zabbix.http": [ + {"product": "nginx", "present": true, "basis": "改过品牌的中文页面，规则漏报"}, + {"product": "zabbix", "present": true, "version": "", "basis": "改过品牌的中文页面，规则漏报"} + ], + "seeyon_login.http": [ + {"product": "致远oa", "present": true, "version": "8.1"}, + {"product": "log4j2", "present": false} + ], + "shop_page.http": [ + {"product": "shopify", "present": true, "basis": "SaaS 上的单位店铺"}, + {"product": "cloudflare", "present": true, "basis": "SaaS 上的单位店铺"} + ], + "springboot_error.http": [ + {"product": "springboot", "present": true}, + {"product": "log4j2", "present": false} + ], + "status_page_mentions.http": [ + {"product": "nginx", "present": true}, + {"product": "gitlab", "present": false}, + {"product": "jenkins", "present": false}, + {"product": "grafana_labs-公司产品", "present": false} + ], + "synology_dsm.http": [ + {"product": "synology diskstation", "present": true, "version": "", "basis": "v= 是时间戳"}, + {"product": "windows", "present": false, "basis": "v= 是时间戳"} + ], + "tomcat_default.http": [ + {"product": "tomcat", "present": true, "version": "9.0.65"}, + {"product": "tomcat_jk_connector", "present": false} + ], + "tplink_router.http": [ + {"product": "tp-link无线路由器", "present": true, "version": "0.9.1"}, + {"product": "apache http server", "present": false}, + {"product": "archer-ax21", "present": false} + ], + "unknown_admin_login.http": [ + {"product": "openresty", "present": true, "basis": "长尾产品，指纹库里没有；技术支持写着厂商"}, + {"product": "zkteco", "present": false, "basis": "长尾产品，指纹库里没有；技术支持写着厂商"} + ], + "weaver_ecology_login.http": [ + {"product": "泛微 e-cology", "present": true, "version": ""} + ], + "wordpress_home.http": [ + {"product": "wordpress", "present": true, "version": "6.4.2"}, + {"product": "windows", "present": false}, + {"product": "microsoft-windows远程连接", "present": false} + ], + "wordpress_login.http": [ + {"product": "wordpress", "present": true, "version": "6.4.2", "basis": "jquery 3.7.1 是干扰"}, + {"product": "windows", "present": false, "basis": "jquery 3.7.1 是干扰"}, + {"product": "microsoft-windows远程连接", "present": false, "basis": "jquery 3.7.1 是干扰"}, + {"product": "telerik_report_server", "present": false, "basis": "jquery 3.7.1 是干扰"} + ], + "zabbix_login.http": [ + {"product": "zabbix", "present": true, "version": "", "basis": "文档链接里的 6.0 和年份是干扰"} + ] +} diff --git a/cmd/judgeeval/testdata/samples/blog_mentions_jenkins.http b/cmd/judgeeval/testdata/samples/blog_mentions_jenkins.http new file mode 100644 index 0000000..b7225b6 --- /dev/null +++ b/cmd/judgeeval/testdata/samples/blog_mentions_jenkins.http @@ -0,0 +1,5 @@ +HTTP/1.1 200 OK +Server: cloudflare +Content-Type: text/html; charset=utf-8 + +Migrating our CI from Jenkins to GitHub Actions | DevNotes

Migrating our CI from Jenkins to GitHub Actions

For five years our team ran Jenkins with a pile of Groovy pipelines. We also had a legacy ThinkPHP admin panel that depended on it. This post explains why we moved to GitHub Actions and what we learned.

Tags: ci, jenkins, devops

diff --git a/cmd/judgeeval/testdata/samples/confluence_login.http b/cmd/judgeeval/testdata/samples/confluence_login.http new file mode 100644 index 0000000..2ed01a2 --- /dev/null +++ b/cmd/judgeeval/testdata/samples/confluence_login.http @@ -0,0 +1,7 @@ +HTTP/1.1 200 OK +X-Confluence-Request-Time: 1727170000123 +X-ASEN: SEN-L1234567 +Set-Cookie: JSESSIONID=8F2C; Path=/; HttpOnly +Content-Type: text/html;charset=UTF-8 + +Log In - Confluence

Log in to Confluence

Powered by Atlassian Confluence 7.19.8

diff --git a/cmd/judgeeval/testdata/samples/corporate_cn_portal.http b/cmd/judgeeval/testdata/samples/corporate_cn_portal.http new file mode 100644 index 0000000..bf28531 --- /dev/null +++ b/cmd/judgeeval/testdata/samples/corporate_cn_portal.http @@ -0,0 +1,5 @@ +HTTP/1.1 200 OK +Server: nginx +Content-Type: text/html; charset=utf-8 + +江南精密机械集团有限公司 - 官方网站
首页 关于我们 产品中心 新闻动态 联系我们

集团与用友网络签署数字化转型战略合作协议

2026年9月，集团与用友网络科技股份有限公司签署合作协议，将在 ERP 与财务共享领域展开合作。

版权所有 © 2026 江南精密机械集团有限公司 苏ICP备12345678号
diff --git a/cmd/judgeeval/testdata/samples/corporate_site.http b/cmd/judgeeval/testdata/samples/corporate_site.http new file mode 100644 index 0000000..edf9200 --- /dev/null +++ b/cmd/judgeeval/testdata/samples/corporate_site.http @@ -0,0 +1,5 @@ +HTTP/1.1 200 OK +Server: nginx +Content-Type: text/html; charset=utf-8 + +Northwind Logistics - Freight and Warehousing

Northwind Logistics

We move freight across 40 countries. Contact our sales team for a quote.

About Careers diff --git a/cmd/judgeeval/testdata/samples/custom_employee_portal.http b/cmd/judgeeval/testdata/samples/custom_employee_portal.http new file mode 100644 index 0000000..04bc7d7 --- /dev/null +++ b/cmd/judgeeval/testdata/samples/custom_employee_portal.http @@ -0,0 +1,6 @@ +HTTP/1.1 200 OK +Server: nginx +Set-Cookie: ACME_SID=5d1e; path=/; HttpOnly +Content-Type: text/html; charset=utf-8 + +ACME Corp · Employee Portal
ACME Corp

Sign in to the ACME Employee Portal

Trouble signing in? Contact the ACME IT Service Desk at ext. 4400.

diff --git a/cmd/judgeeval/testdata/samples/dir_listing.http b/cmd/judgeeval/testdata/samples/dir_listing.http new file mode 100644 index 0000000..fc834e1 --- /dev/null +++ b/cmd/judgeeval/testdata/samples/dir_listing.http @@ -0,0 +1,5 @@ +HTTP/1.1 200 OK +Server: Apache/2.4.41 (Ubuntu) +Content-Type: text/html;charset=UTF-8 + +Index of /backup

Index of /backup

NameLast modifiedSize
Parent Directory
db_2026-09-01.sql.gz2026-09-01 03:0012M
Apache/2.4.41 (Ubuntu) Server at 10.0.0.5 Port 80
diff --git a/cmd/judgeeval/testdata/samples/discuz_forum.http b/cmd/judgeeval/testdata/samples/discuz_forum.http new file mode 100644 index 0000000..51fb0f7 --- /dev/null +++ b/cmd/judgeeval/testdata/samples/discuz_forum.http @@ -0,0 +1,7 @@ +HTTP/1.1 200 OK +Server: nginx +X-Powered-By: PHP/7.4.33 +Set-Cookie: q7Xz_2132_saltkey=Ab12Cd34; path=/; httponly +Content-Type: text/html; charset=utf-8 + +钓鱼之家论坛 - Powered by Discuz!

钓鱼之家论坛

今日: 128 | 昨日: 342 | 帖子: 1203452 | 会员: 88321

路亚交流区 · 台钓技术 · 渔具二手市场

Powered by Discuz! X3.4

© 2001-2026 Discuz! Team.

diff --git a/cmd/judgeeval/testdata/samples/docs_tomcat_howto.http b/cmd/judgeeval/testdata/samples/docs_tomcat_howto.http new file mode 100644 index 0000000..d0b39cc --- /dev/null +++ b/cmd/judgeeval/testdata/samples/docs_tomcat_howto.http @@ -0,0 +1,6 @@ +HTTP/1.1 200 OK +Server: GitHub.com +Content-Type: text/html; charset=utf-8 +X-GitHub-Request-Id: 9A2C:3B1F + +How to tune the Tomcat 10.1 NIO connector | Sam's Ops Notes

How to tune the Tomcat 10.1 NIO connector

When we upgraded from Tomcat 9.0.80 to Tomcat 10.1.13, throughput dropped. Here is the server.xml we ended up with:

<Connector port="8080" protocol="org.apache.coyote.http11.Http11NioProtocol" maxThreads="400" />

We run it behind nginx 1.24 and Spring Boot 3.1 apps.

Hosted on GitHub Pages
diff --git a/cmd/judgeeval/testdata/samples/gitlab_signin.http b/cmd/judgeeval/testdata/samples/gitlab_signin.http new file mode 100644 index 0000000..375f1da --- /dev/null +++ b/cmd/judgeeval/testdata/samples/gitlab_signin.http @@ -0,0 +1,8 @@ +HTTP/1.1 200 OK +Server: nginx +X-Gitlab-Meta: {"correlation_id":"01HB2X","version":"1"} +X-Request-Id: 01HB2X +Set-Cookie: _gitlab_session=3f1c; path=/; HttpOnly +Content-Type: text/html; charset=utf-8 + +Sign in · GitLab

GitLab Community Edition

Don't have an account yet? Register now

diff --git a/cmd/judgeeval/testdata/samples/grafana_login.http b/cmd/judgeeval/testdata/samples/grafana_login.http new file mode 100644 index 0000000..3c32a1e --- /dev/null +++ b/cmd/judgeeval/testdata/samples/grafana_login.http @@ -0,0 +1,6 @@ +HTTP/1.1 200 OK +Cache-Control: no-store +Content-Type: text/html; charset=UTF-8 +X-Content-Type-Options: nosniff + +Grafana
diff --git a/cmd/judgeeval/testdata/samples/harbor_portal.http b/cmd/judgeeval/testdata/samples/harbor_portal.http new file mode 100644 index 0000000..0c26570 --- /dev/null +++ b/cmd/judgeeval/testdata/samples/harbor_portal.http @@ -0,0 +1,6 @@ +HTTP/1.1 200 OK +Server: nginx +Set-Cookie: sid=8a7b6c5d; Path=/; HttpOnly +Content-Type: text/html + +HarborLoading... diff --git a/cmd/judgeeval/testdata/samples/iis_default.http b/cmd/judgeeval/testdata/samples/iis_default.http new file mode 100644 index 0000000..21d4d64 --- /dev/null +++ b/cmd/judgeeval/testdata/samples/iis_default.http @@ -0,0 +1,7 @@ +HTTP/1.1 200 OK +Server: Microsoft-IIS/10.0 +X-Powered-By: ASP.NET +Accept-Ranges: bytes +Content-Type: text/html + +IIS Windows Server
IIS
diff --git a/cmd/judgeeval/testdata/samples/jenkins_login.http b/cmd/judgeeval/testdata/samples/jenkins_login.http new file mode 100644 index 0000000..1f2ddee --- /dev/null +++ b/cmd/judgeeval/testdata/samples/jenkins_login.http @@ -0,0 +1,6 @@ +HTTP/1.1 200 OK +X-Jenkins: 2.401.3 +Set-Cookie: JSESSIONID.1a2b3c=node01; Path=/; HttpOnly +Content-Type: text/html;charset=utf-8 + +Sign in [Jenkins]

Welcome to Jenkins!

diff --git a/cmd/judgeeval/testdata/samples/job_posting_cn.http b/cmd/judgeeval/testdata/samples/job_posting_cn.http new file mode 100644 index 0000000..b520c89 --- /dev/null +++ b/cmd/judgeeval/testdata/samples/job_posting_cn.http @@ -0,0 +1,5 @@ +HTTP/1.1 200 OK +Server: Tengine +Content-Type: text/html; charset=utf-8 + +Java 后端开发工程师 - 华东某物流科技有限公司招聘

Java 后端开发工程师

薪资：20-35K · 上海 · 3-5年

岗位要求

  • 熟悉 Spring Boot、Spring Cloud、MyBatis 等框架；
  • 熟悉 Redis、Kafka、Elasticsearch、Nacos 等中间件；
  • 了解 ThinkPHP 或 Laravel 者优先，维护过泛微、致远 OA 二次开发者优先；
  • 熟悉 Jenkins、GitLab CI 等持续集成工具。

公司地址：上海市浦东新区张江路 88 号

diff --git a/cmd/judgeeval/testdata/samples/json_api.http b/cmd/judgeeval/testdata/samples/json_api.http new file mode 100644 index 0000000..cbaaf14 --- /dev/null +++ b/cmd/judgeeval/testdata/samples/json_api.http @@ -0,0 +1,5 @@ +HTTP/1.1 200 OK +Server: uvicorn +Content-Type: application/json + +{"service":"inventory-api","status":"ok","uptime_seconds":882133,"build":"2026.09.18-3","dependencies":{"postgres":"up","redis":"up"}} diff --git a/cmd/judgeeval/testdata/samples/kibana_home.http b/cmd/judgeeval/testdata/samples/kibana_home.http new file mode 100644 index 0000000..e2a6684 --- /dev/null +++ b/cmd/judgeeval/testdata/samples/kibana_home.http @@ -0,0 +1,7 @@ +HTTP/1.1 200 OK +kbn-name: kibana-prod-01 +kbn-license-sig: 9c3b1f +content-security-policy: script-src 'report-sample' 'self'; worker-src 'report-sample' 'self' blob: +Content-Type: text/html; charset=utf-8 + +Elastic
Loading Elastic
diff --git a/cmd/judgeeval/testdata/samples/minio_console.http b/cmd/judgeeval/testdata/samples/minio_console.http new file mode 100644 index 0000000..e3a4fb1 --- /dev/null +++ b/cmd/judgeeval/testdata/samples/minio_console.http @@ -0,0 +1,6 @@ +HTTP/1.1 200 OK +Server: MinIO Console +Content-Security-Policy: default-src 'self' 'unsafe-eval' 'unsafe-inline'; +Content-Type: text/html + +MinIO Console
diff --git a/cmd/judgeeval/testdata/samples/nacos_console.http b/cmd/judgeeval/testdata/samples/nacos_console.http new file mode 100644 index 0000000..1d3ec9b --- /dev/null +++ b/cmd/judgeeval/testdata/samples/nacos_console.http @@ -0,0 +1,5 @@ +HTTP/1.1 200 OK +Content-Type: text/html;charset=UTF-8 +Vary: Origin + +Nacos
diff --git a/cmd/judgeeval/testdata/samples/news_fortios_patch.http b/cmd/judgeeval/testdata/samples/news_fortios_patch.http new file mode 100644 index 0000000..73111c6 --- /dev/null +++ b/cmd/judgeeval/testdata/samples/news_fortios_patch.http @@ -0,0 +1,6 @@ +HTTP/1.1 200 OK +Server: cloudflare +CF-Cache-Status: HIT +Content-Type: text/html; charset=utf-8 + +Fortinet ships FortiOS 7.2.5 update for SSL VPN bug | Network Weekly

Fortinet ships FortiOS 7.2.5 update for SSL VPN bug

By Dana Reyes · 12 Sep 2026

Fortinet has released FortiOS 7.2.5 and 7.0.12, fixing an issue in the FortiGate SSL VPN portal. Administrators running FortiGate appliances should upgrade through the FortiGuard update service. Older 6.4.x branches receive the fix in 6.4.14.

The vendor said FortiManager and FortiAnalyzer are not affected.

© 2026 Network Weekly. Published with Ghost.
diff --git a/cmd/judgeeval/testdata/samples/nginx_404.http b/cmd/judgeeval/testdata/samples/nginx_404.http new file mode 100644 index 0000000..82f5fff --- /dev/null +++ b/cmd/judgeeval/testdata/samples/nginx_404.http @@ -0,0 +1,11 @@ +HTTP/1.1 404 Not Found +Server: nginx/1.22.1 +Content-Type: text/html + + +404 Not Found + +

404 Not Found

+
nginx/1.22.1
+ + diff --git a/cmd/judgeeval/testdata/samples/nginx_default.http b/cmd/judgeeval/testdata/samples/nginx_default.http new file mode 100644 index 0000000..50f7e4d --- /dev/null +++ b/cmd/judgeeval/testdata/samples/nginx_default.http @@ -0,0 +1,5 @@ +HTTP/1.1 200 OK +Server: nginx/1.18.0 (Ubuntu) +Content-Type: text/html + +Welcome to nginx!

Welcome to nginx!

If you see this page, the nginx web server is successfully installed and working. Further configuration is required.

Thank you for using nginx.

diff --git a/cmd/judgeeval/testdata/samples/pgadmin_login.http b/cmd/judgeeval/testdata/samples/pgadmin_login.http new file mode 100644 index 0000000..b84c602 --- /dev/null +++ b/cmd/judgeeval/testdata/samples/pgadmin_login.http @@ -0,0 +1,6 @@ +HTTP/1.1 200 OK +Server: gunicorn +Set-Cookie: pga4_session=0f1e2d3c!abc; HttpOnly; Path=/ +Content-Type: text/html; charset=utf-8 + +pgAdmin 4
pgAdmin 4 - PostgreSQL Tools
diff --git a/cmd/judgeeval/testdata/samples/phpmyadmin_login.http b/cmd/judgeeval/testdata/samples/phpmyadmin_login.http new file mode 100644 index 0000000..0889924 --- /dev/null +++ b/cmd/judgeeval/testdata/samples/phpmyadmin_login.http @@ -0,0 +1,8 @@ +HTTP/1.1 200 OK +Server: Apache/2.4.54 (Debian) +X-Powered-By: PHP/8.1.12 +Set-Cookie: phpMyAdmin=4b7c1d; path=/; HttpOnly +Set-Cookie: pma_lang=en; path=/ +Content-Type: text/html; charset=utf-8 + +phpMyAdmin

Welcome to phpMyAdmin

diff --git a/cmd/judgeeval/testdata/samples/portainer_login.http b/cmd/judgeeval/testdata/samples/portainer_login.http new file mode 100644 index 0000000..508de2b --- /dev/null +++ b/cmd/judgeeval/testdata/samples/portainer_login.http @@ -0,0 +1,6 @@ +HTTP/1.1 200 OK +X-Xss-Protection: 1; mode=block +X-Content-Type-Options: nosniff +Content-Type: text/html; charset=utf-8 + +Portainer
Loading Portainer...
diff --git a/cmd/judgeeval/testdata/samples/rabbitmq_mgmt.http b/cmd/judgeeval/testdata/samples/rabbitmq_mgmt.http new file mode 100644 index 0000000..d16890c --- /dev/null +++ b/cmd/judgeeval/testdata/samples/rabbitmq_mgmt.http @@ -0,0 +1,5 @@ +HTTP/1.1 200 OK +Server: Cowboy +Content-Type: text/html + +RabbitMQ Management
diff --git a/cmd/judgeeval/testdata/samples/rebranded_grafana.http b/cmd/judgeeval/testdata/samples/rebranded_grafana.http new file mode 100644 index 0000000..56b412e --- /dev/null +++ b/cmd/judgeeval/testdata/samples/rebranded_grafana.http @@ -0,0 +1,5 @@ +HTTP/1.1 200 OK +Cache-Control: no-store +Content-Type: text/html; charset=UTF-8 + +Acme Ops Monitoring
diff --git a/cmd/judgeeval/testdata/samples/rebranded_jenkins.http b/cmd/judgeeval/testdata/samples/rebranded_jenkins.http new file mode 100644 index 0000000..d58a694 --- /dev/null +++ b/cmd/judgeeval/testdata/samples/rebranded_jenkins.http @@ -0,0 +1,6 @@ +HTTP/1.1 200 OK +Set-Cookie: JSESSIONID.8f3c2a1b=node0abc; Path=/; HttpOnly +X-Content-Type-Options: nosniff +Content-Type: text/html;charset=utf-8 + +Sign in [ACME Build]

Welcome to ACME Build!

diff --git a/cmd/judgeeval/testdata/samples/rebranded_zabbix.http b/cmd/judgeeval/testdata/samples/rebranded_zabbix.http new file mode 100644 index 0000000..c994666 --- /dev/null +++ b/cmd/judgeeval/testdata/samples/rebranded_zabbix.http @@ -0,0 +1,6 @@ +HTTP/1.1 200 OK +Server: nginx +Set-Cookie: zbx_session=eyJzZXNzaW9; path=/; HttpOnly +Content-Type: text/html; charset=UTF-8 + +集团IT运维监控中心
集团信息中心 运维保障
diff --git a/cmd/judgeeval/testdata/samples/seeyon_login.http b/cmd/judgeeval/testdata/samples/seeyon_login.http new file mode 100644 index 0000000..ce93bba --- /dev/null +++ b/cmd/judgeeval/testdata/samples/seeyon_login.http @@ -0,0 +1,6 @@ +HTTP/1.1 200 OK +Server: SY8044 +Set-Cookie: JSESSIONID=A1B2C3; Path=/seeyon; HttpOnly +Content-Type: text/html;charset=UTF-8 + +协同管理软件 V8.1SP2

欢迎使用协同管理软件

北京致远互联软件股份有限公司 版权所有 Copyright © 2002-2023
diff --git a/cmd/judgeeval/testdata/samples/shop_page.http b/cmd/judgeeval/testdata/samples/shop_page.http new file mode 100644 index 0000000..2212bca --- /dev/null +++ b/cmd/judgeeval/testdata/samples/shop_page.http @@ -0,0 +1,8 @@ +HTTP/1.1 200 OK +Server: cloudflare +Set-Cookie: _shopify_y=8c1e; path=/; expires=Wed, 24 Sep 2027 11:00:00 GMT +X-ShopId: 60123456789 +X-Shardid: 214 +Content-Type: text/html; charset=utf-8 + +Handmade Ceramic Mugs – Clay & Kiln Studio

Handmade Ceramic Mugs

Each mug is thrown by hand in our Portland studio. $38.00 USD

diff --git a/cmd/judgeeval/testdata/samples/springboot_error.http b/cmd/judgeeval/testdata/samples/springboot_error.http new file mode 100644 index 0000000..a067d6a --- /dev/null +++ b/cmd/judgeeval/testdata/samples/springboot_error.http @@ -0,0 +1,4 @@ +HTTP/1.1 404 Not Found +Content-Type: text/html;charset=UTF-8 + +

Whitelabel Error Page

This application has no explicit mapping for /error, so you are seeing this as a fallback.

Tue Sep 24 10:00:00 CST 2026
There was an unexpected error (type=Not Found, status=404).
diff --git a/cmd/judgeeval/testdata/samples/status_page_mentions.http b/cmd/judgeeval/testdata/samples/status_page_mentions.http new file mode 100644 index 0000000..cd6be3f --- /dev/null +++ b/cmd/judgeeval/testdata/samples/status_page_mentions.http @@ -0,0 +1,5 @@ +HTTP/1.1 200 OK +Server: nginx +Content-Type: text/html; charset=utf-8 + +Northwind Systems Status

Northwind Systems Status

Customer PortalOperational
Order APIOperational
Internal Grafana dashboardsDegraded performance
Jenkins build farmScheduled maintenance 02:00-04:00 UTC
GitLab mirrorOperational

Last updated 2026-09-24 09:12 UTC. Subscribe to updates via RSS.

diff --git a/cmd/judgeeval/testdata/samples/synology_dsm.http b/cmd/judgeeval/testdata/samples/synology_dsm.http new file mode 100644 index 0000000..1b559ac --- /dev/null +++ b/cmd/judgeeval/testdata/samples/synology_dsm.http @@ -0,0 +1,5 @@ +HTTP/1.1 200 OK +Server: nginx +Content-Type: text/html; charset="UTF-8" + +Synology DiskStation - NAS-HOME
diff --git a/cmd/judgeeval/testdata/samples/tomcat_default.http b/cmd/judgeeval/testdata/samples/tomcat_default.http new file mode 100644 index 0000000..b996e92 --- /dev/null +++ b/cmd/judgeeval/testdata/samples/tomcat_default.http @@ -0,0 +1,4 @@ +HTTP/1.1 200 OK +Content-Type: text/html;charset=UTF-8 + +Apache Tomcat/9.0.65
Home

If you're seeing this, you've successfully installed Tomcat. Congratulations!

Server Status Manager App Host Manager diff --git a/cmd/judgeeval/testdata/samples/tplink_router.http b/cmd/judgeeval/testdata/samples/tplink_router.http new file mode 100644 index 0000000..8aa7929 --- /dev/null +++ b/cmd/judgeeval/testdata/samples/tplink_router.http @@ -0,0 +1,5 @@ +HTTP/1.1 200 OK +Server: TP-LINK HTTPD/1.0 +Content-Type: text/html + +TL-WR841N

Wireless N Router WR841N

Hardware Version: WR841N v14 00000014

Firmware Version: 0.9.1 4.19 v0001.0 Build 230516 Rel.61208n

Copyright © 2023 TP-Link Corporation Limited. All rights reserved.

diff --git a/cmd/judgeeval/testdata/samples/unknown_admin_login.http b/cmd/judgeeval/testdata/samples/unknown_admin_login.http new file mode 100644 index 0000000..4ce0bd3 --- /dev/null +++ b/cmd/judgeeval/testdata/samples/unknown_admin_login.http @@ -0,0 +1,5 @@ +HTTP/1.1 200 OK +Server: openresty +Content-Type: text/html; charset=utf-8 + +智慧园区综合管理平台

智慧园区综合管理平台 V3.2.1

技术支持：某某科技有限公司

diff --git a/cmd/judgeeval/testdata/samples/weaver_ecology_login.http b/cmd/judgeeval/testdata/samples/weaver_ecology_login.http new file mode 100644 index 0000000..bfe9cca --- /dev/null +++ b/cmd/judgeeval/testdata/samples/weaver_ecology_login.http @@ -0,0 +1,5 @@ +HTTP/1.1 200 OK +Set-Cookie: ecology_JSessionid=aaaXYZ; path=/ +Content-Type: text/html; charset=UTF-8 + +泛微协同管理应用平台 e-cology

欢迎登录 OA 系统

版权所有 泛微网络科技股份有限公司
diff --git a/cmd/judgeeval/testdata/samples/wordpress_home.http b/cmd/judgeeval/testdata/samples/wordpress_home.http new file mode 100644 index 0000000..87bd5f4 --- /dev/null +++ b/cmd/judgeeval/testdata/samples/wordpress_home.http @@ -0,0 +1,6 @@ +HTTP/1.1 200 OK +Server: Apache/2.4.57 +Link: ; rel="https://api.w.org/" +Content-Type: text/html; charset=UTF-8 + +Acme Bakery – Fresh bread daily

Acme Bakery

Fresh sourdough every morning. Visit our shop on Main Street.

diff --git a/cmd/judgeeval/testdata/samples/wordpress_login.http b/cmd/judgeeval/testdata/samples/wordpress_login.http new file mode 100644 index 0000000..557295d --- /dev/null +++ b/cmd/judgeeval/testdata/samples/wordpress_login.http @@ -0,0 +1,7 @@ +HTTP/1.1 200 OK +Server: nginx/1.24.0 +X-Frame-Options: SAMEORIGIN +Set-Cookie: wordpress_test_cookie=WP%20Cookie%20check; path=/; secure +Content-Type: text/html; charset=UTF-8 + +Log In ‹ Riverside Dental Clinic — WordPress

Powered by WordPress

Lost your password?

diff --git a/cmd/judgeeval/testdata/samples/zabbix_login.http b/cmd/judgeeval/testdata/samples/zabbix_login.http new file mode 100644 index 0000000..8d67b59 --- /dev/null +++ b/cmd/judgeeval/testdata/samples/zabbix_login.http @@ -0,0 +1,6 @@ +HTTP/1.1 200 OK +Server: nginx +Set-Cookie: zbx_session=eyJzZXNzaW9; path=/; HttpOnly +Content-Type: text/html; charset=UTF-8 + +Zabbix
Help • Support
© 2001–2023, Zabbix SIA
diff --git a/common/framework.go b/common/framework.go index 4968b21..6cc997d 100644 --- a/common/framework.go +++ b/common/framework.go @@ -27,6 +27,7 @@ type Framework = parsers.Framework type MatchDetail = parsers.MatchDetail type Frameworks = parsers.Frameworks type Attributes = parsers.Attributes +type Judgement = parsers.Judgement var FrameFromMap = parsers.FrameFromMap diff --git a/docs/jev-claim-acceptance-20260926.md b/docs/jev-claim-acceptance-20260926.md new file mode 100644 index 0000000..50d5cd4 --- /dev/null +++ b/docs/jev-claim-acceptance-20260926.md @@ -0,0 +1,92 @@ +# judge 重构验收：事实 → Claim → Jev → 动作表 + +> 历史验收记录：本文记录早期 Claim 实现，Refine 与自动校准等接口已移除。当前合约与默认值见 [judge](../judge/README.md)，当前输出见 [judgeeval](../cmd/judgeeval/README.md)；本文测量结果未由当前实现重新验证。 + +## 结论 + +**计划的全部验收标准都已通过。** 输入与 run-07-final 相同：69 份真实响应快照，682 个已标注的产品判断。模型为 jev-1.13.0，`MinConfidence` 取校准后的默认值 0.3，0 份请求失败。 + +| 验收标准 | run-07 | 本次 | 结论 | +|---|---:|---:|---| +| 有标注的误报删除数（要求 ≥24） | 24 | **24** | ✅ | +| 真实命中误删 | 0 | **0** | ✅ | +| 3 个宝塔默认页上的宝塔 | 被误删 | **全部保留** | ✅ | +| `varnish=1.1` | 出现 | **不出现** | ✅ | +| Refine 每页题目数（要求约 -35%） | 15.14 | **6.29（-58%）** | ✅ | +| 版本 正确/错误（要求不劣于 23/0） | 23/0 | **36/0**，无依据填值 0 | ✅ | +| Ledger 前几名包含 git、ipeakcms、webp_server_go | — | 第 1、2、3 名 | ✅ | +| Discover 找出 SearXNG、Wakapi、IT Tools 的簇 | — | 三个簇都找到了。不加载 library 时，Coverage 把 SearXNG、Wakapi 判为 `missing`，IT Tools 判为 `insufficient`（见"漏报"一节） | ⚠️ 2/3 | + +**计数口径** +- run-07 的误报在前、本次的在后，基线的真命中数不同（46 对 63），因为本次基线是用当前规则重新扫描保存的响应得到的。 +- 题目数的两个数字，是用同一个固定回答的 Provider，分别在旧设计（1f42439）和新设计上，对同一批命中跑 Refine 统计出来的。评估报告里的"平均每页 8.6 题"是另一个口径：它还包含 Discover 的 Coverage 题目。 + +## 超出计划的改动及原因 + +第一次真实评估只删掉了 8 个误报。原因都在"代码负责找事实"这一侧：给 Jev 的证据不足，或者给出的选项不贴切。按边界原则，修正都放在代码的事实层和题面里，没有加任何推翻或组合 Jev 回答的逻辑。 + +1. **四个引擎记录命中依据**：以前只有 fingers 引擎会写 `MatchDetail`。goby、fingerprinthub、ehole、wappalyzer 的命中没有任何证据，Jev 只能回答 insufficient。现在这四个引擎在命中时都会记录实际匹配到的文本，由 `Engine.EnableMatchDetail()` 统一打开。开关默认关闭，现有输出不变。 +2. **Evidence 带上 `matched`**：State 里除了上下文片段，还单独给出规则匹配到的文本本身。 +3. **新增判定 `unrelated`**：goby 的 `"s7"`、`"rdp"`、`"git"` 这类短串会碰巧命中 base64 数据或别的单词。原来的选项里没有贴切的一项，模型只好弃权。新增这个选项后，动作表执行删除；Ledger 里它也是垃圾规则最直接的信号。parsers 的字段注释同步更新了。 +4. **版本候选排除非版本数值**：viewport 的 `initial-scale=1.0` 和全零值（例如 NEL 头里的 `"success_fraction":0.0`）不会作为候选，与已有的 IPv4 排除属于同一类。题面也从 "a meta tag" 改成了 "a generator meta tag"。 +5. **Jev 不做指纹识别，全部判断统一为 Claim**（见下一节）。 +6. **Discover 的聚类**：通用安全头（`X-Content-Type-Options`、`X-Frame-Options` 等）不再算作结构特征。之前"请通过官方域名访问"的 403 页和 Privacy Redirect 站点只因为都带这些头就被并成了一簇。候选名排除 HTTP 状态短语（"404 Not Found"、"Forbidden"）。 + +## 统一为 Claim + +Jev 只审查规则给出的结论，不说出产品名。判定层只有一个机制：代码提出 Claim 并附上证据，Provider 选出证据支持的选项，选项的 Outcome（Holds / Refuted / Insufficient）驱动固定动作。 + +| Claim | 方向 | 选项 → Outcome | +|---|---|---| +| Presence：规则说 P 参与生成了这个响应 | 误报 | `running`→Holds；`mentioned`、`unrelated`→Refuted | +| Coverage：已报告的产品解释了这组页面 | 漏报 | `explained`、`custom_site`→Holds；`missing`→Refuted | +| Version：P 的版本是这些候选之一 | 版本 | 候选→Holds；`not_stated`→Refuted | + +- 每个 Claim 都带 `insufficient`；置信度低于 `MinConfidence` 的回答一律判为 Insufficient。代码确定的事实（declared / absent）也表达为 Ruling，和 Provider 的回答走同一张动作表。 +- 删除了 `Identify`、`Ask`、`Yes`、`Choose`、`Score`，以及 binary 和 score 题型。自定义判断用 `j.Judge(ctx, state, claims)`。 +- 规则库已有的产品名从不作为选项，只作为被审查的 Claim 出现在题干或 State 里。 +- 结论 DTO `parsers.Judgement` 与 Ruling 对齐：新增 `Outcome`，删除 `Layer`、`Primary`、`Recalled` 和 `Frameworks.Primary()`。Ledger 和评估报告都按 Outcome 统计。离线重放显示，Presence 的 holds 225 = declared 133 + running 92，refuted 135 = absent 5 + mentioned 54 + unrelated 76，insufficient 7。 +- `judge/gen` 必须由人提供 `Name`。Discover 的簇附带代码抽取的候选名，供人命名。 + +### 漏报（Coverage）结果 + +| 簇 | 不加载 library | 加载 library | +|---|---|---| +| SearXNG（10 主机） | `missing` 0.86 ✅ | `explained` ✅ | +| Wakapi（4 主机） | `missing` 0.73 ✅ | `explained` ✅ | +| IT Tools（2 主机） | `insufficient` 0.29 ⚠️ 漏判 | `missing` 0.35 ❌ 误报警 | +| nginx 默认页、通用 404、宝塔默认页、阿里 WAF 403、SoftEther | 全部 `explained` ✅ | 全部 `explained` ✅ | + +IT Tools 在两种情况下的概率分布几乎相同（missing 约 0.47–0.50），说明 Jev 对这个文字很少的单页应用，判断和"已报告里是否有 it tools"基本无关。误报警只会进入人工审核队列，不会删除任何东西。 + +## MinConfidence 校准 + +在同一输入上用 `judgeeval -min-confidence` 扫描不同阈值： + +| MinConfidence | 误报删除 | 误删 | 版本 对/错 | 宝塔 | 删除总数 | +|---:|---:|---:|---:|---|---:| +| 0.2 | 24 | 0 | 36/0 | 保留 | 138 | +| 0.3 | 24 | 0 | 36/0 | 保留 | 135 | +| 0.5 | 24 | 0 | 36/0 | 保留 | 117 | +| 0.6 | 24 | 0 | 36/0 | 保留 | 109 | +| 0.7 | 23 | 0 | 36/0 | 保留 | — | + +有标注的结果在 0.2–0.6 之间完全相同。阈值从 0.5 降到 0.3，多删的 18 个未标注命中逐一抽查都是误报：`harbor` 命中了图片 alt 文字，`webp_server_go` 命中了 `.webp` 文件名，`windows` 命中了 "Microsoft YaHei" 字体。从 0.3 降到 0.2 多删的 3 个也都是误报。默认值取 0.3：拿到几乎全部收益，同时没有降到 0.2 那么低；样本只有 69 页，留出一定余量。 + +## 遗留问题 + +- **宝塔别名**：`宝塔` 与 `宝塔-bt.cn` 归一化后不相同，所以不会去重。这需要在名称归一化或别名表里处理，不在本次范围内。宝塔默认页的簇判为 insufficient，因为页面本身没有写出产品名，这个结果是对的；宝塔在所有页面上都保留了。 +- **通用错误页**：聚类会把不同服务器的通用 404/403 页并成一簇，目前判为 insufficient，结果正确，但这类簇本身没有维护价值。 +- **样本规模**：结论只基于 69 页、59 个主机，`MinConfidence` 需要在更大的样本集上复核。 +- **网络错误**：评估中偶尔出现 `connection reset` 或 `EOF`，重跑即可恢复。 + +## 复现 + +``` +go run ./cmd/judgeeval \ + -manifest .judge-data/expansion-20260926/run-07-final/manifest.json \ + -library .judge-data/expansion-20260926/run-07-final/library.yaml \ + -cache .judge-data/cache -out .judge-data/claim-v1 +``` + +`TYPESAFE_API_KEY` 只通过环境变量传入。去掉 `-library` 即可验证 Discover 能否发现新产品（输出在 `.judge-data/claim-v1-nolib`）。`.judge-data/` 不纳入版本控制。 diff --git a/docs/jev-claim-retest-20260927.md b/docs/jev-claim-retest-20260927.md new file mode 100644 index 0000000..4b217a8 --- /dev/null +++ b/docs/jev-claim-retest-20260927.md @@ -0,0 +1,56 @@ +# judge 复测：合约下沉 utils/jev 之后（2026-09-27） + +## 结论 + +**当前代码在真实 Jev 上通过了 2026-09-26 验收的全部标准，并补上了当时遗留的两个问题。** 输入与[历史验收](jev-claim-acceptance-20260926.md)相同：69 份真实响应快照，682 个已标注的产品判断。模型为 jev-1.13.0，`MinConfidence` 为 0.3，0 份请求失败。 + +| 验收标准 | 2026-09-26 | 本次 | 结论 | +|---|---:|---:|---| +| 有标注的误报删除数（要求 ≥24） | 24 | **24** | ✅ | +| 真实命中误删 | 0 | **0** | ✅ | +| 3 个宝塔默认页上的宝塔 | 全部保留 | **全部保留**（insufficient，未删除） | ✅ | +| `varnish=1.1` | 不出现 | **不出现** | ✅ | +| 版本 正确/错误 | 36/0 | **36/0**，无依据填值 0，正确弃权 26 | ✅ | +| Ledger 前三 | git、ipeakcms、webp_server_go | **git、ipeakcms、webp_server_go** | ✅ | +| Discover，不加载 library | SearXNG、Wakapi 判 missing；IT Tools 判 insufficient | **三个都判 missing**（0.88 / 0.69 / 0.40） | ✅ 3/3 | +| Discover，加载 library | IT Tools 误报警 missing | **没有误报警**，IT Tools 判 insufficient（0.21） | ✅ | + +## 本轮修正 + +**规则引擎输出不确定。** 同一份响应跑两次，合并结果可能不同。几处都按 map 顺序遍历：引擎之间、fingers 的候选规则、fingerprinthub 命中的模板、wappalyzer 的应用及其 header / cookie / meta 模式。同一产品有多条规则命中时，保留谁的 MatchDetail 和 Attributes 取决于遍历顺序，于是证据、发给 Jev 的请求和缓存键也随之变化。现在都改为固定顺序，Ledger 的排序和样本也固定了。 + +**两类格式版本不再作为候选。** 第一次真实复测中，Jev 在 4 个 GitHub Pages 页面上把 `Via: 1.1 varnish` 的 1.1 选为 varnish 的版本（置信度 0.41–0.74），在 Caddy 默认页上选了 SVG 的 `version="1.1"`（0.48）。这些都是代码能确定的事实：Via 头每一项开头是 HTTP 协议版本，SVG / XML 声明里的 version 是格式版本。现在它们和 viewport 的 `initial-scale=1.0`、IPv4 一样，在提取候选时由代码排除，没有加任何推翻 Jev 回答的逻辑。 + +**代码精简。** 以下改动都用确定性回归验证过：用一个按请求内容哈希回答的本地 Provider，跑完 judgeeval 全链路，重构前后 189 个 Provider 请求和全部输出逐字节一致。具体改动： +- Claim 合约移到 `utils/jev`。 +- 删除重复的 Ruling 校验。 +- 代码确定的事实使用固定的 `factClaim`。 +- gen 改为 `New(j, name)`，删除 `ProbeWith`。 +- 测试按机制重排。 + +## 用量 + +| 运行 | Provider 请求 | 缓存命中 | Claim 数 | 输入 token | 耗时 | +|---|---:|---:|---:|---:|---:| +| 加载 library，冷缓存 | 101 | 4 | 340 | 218,512 | 72 秒 | +| 修正版本候选后，加载 library | 11 | 91 | 331 | 25,726 | 17 秒 | +| 修正版本候选后，不加载 library | 0 | 101 | 314 | 0 | 9 秒 | + +修正只改变了含 Via 头或 SVG 的页面的版本 Claim，其余请求全部命中缓存。 + +## 局限 + +- 样本仍是 69 页、59 个主机，`MinConfidence` 需要在更大的样本集上复核。 +- IT Tools 的 Coverage 置信度低（0.40 / 0.21），文字很少的单页应用仍是 Discover 的弱项。 +- Presence 中 insufficient 从 7 增至 12，都没有删除条目（`DropInsufficient=false`），不影响上表指标。 + +## 复现 + +``` +go run ./cmd/judgeeval \ + -manifest .judge-data/expansion-20260926/run-07-final/manifest.json \ + -library .judge-data/expansion-20260926/run-07-final/library.yaml \ + -cache .judge-data/cache-v2 -out .judge-data/claim-v2b +``` + +`TYPESAFE_API_KEY` 只通过环境变量传入。去掉 `-library` 验证 Discover（输出在 `.judge-data/claim-v2b-nolib`）。`.judge-data/` 不纳入版本控制。 diff --git a/docs/jev-harness-design-20260926.md b/docs/jev-harness-design-20260926.md new file mode 100644 index 0000000..0aa4bad --- /dev/null +++ b/docs/jev-harness-design-20260926.md @@ -0,0 +1,323 @@ +# Habit：一个 System One 系统 + +> 本文设计的是一个**独立系统**，不是 harness 的一个功能。它可以单独使用，也可以作为 +> harness 的底层 System One。Jev 是它的一种底层实现，不是它的抽象。 +> +> 指纹侧的断言在 [judge](../judge/README.md)，与本文的关系见 §9。 + +## 结论 + +传统 harness 是一个**没有 System One 的 System 2**：所有判断要么由 L2 做，要么根本没在做——被正则、阈值和字符串相等顶替。 + +System One 不是给 harness 加的一层，是**被编译的判定**：L2 在工作中反复做的那些判断，重复到形状已知，于是可以脱离 L2 运行。 + +它和记忆系统是对偶的： + +| | 记忆 | 习惯 | +|---|---|---| +| 做什么 | 把东西**放进**上下文 | 让东西**不必进**上下文 | +| 底层 | 存储 + 检索 | **两个模型**：一个写题，一个答题 | +| 对 L2 | 有感——它读到了召回的内容 | **无感**——判定没出现在它面前 | +| 方向 | 加法 | 减法 | + +所以叫 `habit`：记忆存"我知道什么"，习惯存"我不必再想什么"。 + +**三个概念、两个端口、一条规则、零个自由参数。** + +--- + +## 1. 不可约前提 + +| | 前提 | 性质 | +|---|---|---| +| P1 | 推理是序列到序列的函数；一轮里模型知道的一切都在序列里 | 机制 | +| P2 | 输出自回归（串行、最贵、无缓存）；输入按前缀匹配 | 机制 | +| P3 | thinking 是生成出来的 token，按 output 计价，且默认留在前缀里 | 机制 + 当前 API 行为 | +| P4 | 存在一种便宜的类型化判断，但它的概率**没有在你的分布上校准** | **产品事实，不是机制** | + +由 P4 的性质得一条铁律： + +> **任何依赖 P4 数值的设计，必须带一个测量，不能带一个常数。** + +### L1 / L2 按能力定义 + +| | 能做什么 | 代价 | +|---|---|---| +| **L2** | 产生**新的**符号序列 | 慢、贵、输出留在前缀里 | +| **L1** | 从**事先给定的**有限集合里选一个 | 快、约等于免费、不产生 token | + +只有这一条差别。四个推论全是强制的： + +1. L1 不能给自己出题 ⟹ **题只能由 L2 写** +2. 同一道判定两边都能答 ⟹ **一致率是免费的** +3. L1 不产生 output token ⟹ 短路省掉的正好是 L2 的输出 +4. L1 不能解释 ⟹ **审计靠记录，不靠理由** + +推论 1 是这套能自转的原因：**编译器就是 L2 自己。** + +## 2. 推导 + +**第一步（P1 + P3）。** L2 生成的 token 分两类：**新工作**，和**重新做已经做过的判断**。后者是纯开销。 + +**第二步（P2 + P3）。** 重复判断付两次钱：生成时按 output 付（最贵的一档，无缓存），之后留在前缀里，每一轮再付一次。 + +**第三步。** 重复，所以形状已知；形状已知，所以可类型化。 + +> **重复性本身就是可编译性的判据。** 不需要第二个筛子。 + +反面同样成立：**新工作天然不可类型化，因为你事先不知道它的形状。** 所以"会不会把 L2 的推理掏空"这个担心自动消解——能被编译的恰好只有开销。 + +**第四步：短路的正确形态是旁路，不是注入。** + +把 L1 的答案**告诉** L2（Brief、注入陈述），L2 是有感的——它知道有人给了它一个结论，而且会盲信这条陈述胜过自己读出来的东西。一条错的陈述就是毒。 + +正确的形态是**塑形入口**： + +``` +世界 ──→ [L1 旁路：塑形] ──→ 上下文 ──→ L2 +``` + +人不会先看到原始光子、再收到一张纸条说"System 1 认为这是一张脸"。**你就是看到一张脸。** 视觉系统的计算对你完全无感。 + +这一步的后果： + +| | 注入形态 | 旁路形态 | +|---|---|---| +| L2 | 有感 | **无感** | +| 失败模式 | 一条假陈述，L2 盲信 | 一次糟糕的检索，漏了东西 | +| 缓存 | 要设计注入信道 | 发生在入上下文**之前**，永远是追加 | +| 弃权 | 必须显式写进注入内容 | **不塑形 = 原样通过**，零代码 | + +判据： + +> **一条判定如果必须讲给 L2 听，它就不是旁路。** + +"目标满足了吗""我试过这个吗"是**任务自省**，只能讲给 L2 听——那属于记忆或提示，不属于习惯。真正的旁路关于**材料**：哪几段相关、这两个观测是不是同一个、这条命令碰什么、这一步要多深的 effort。 + +**第五步（P4）。** 概率没校准，就要测。对照物是现成的：**L2 自己声明过的答案**。免费、在你的分布上、每次运行都产出。 + +> **替换和测量是同一个操作。** + +## 3. 三个概念 + +```go +// Claim：L2 在工作中声明的一次判定——问什么、判成了什么。 +// 绑在具体场景上，不能独立运行。 +type Claim struct { + Statement string + Chose string +} + +// Reflex：从反复出现的 Claim 里抽出来的问题，给定了有限选项， +// 验证过可以脱离 L2 运行。 +type Reflex struct { + Statement string + Options map[string]Option // 必须含"判不了" +} + +// Compile：实例 → 类。由 L2 执行，因为 L1 只会选不会写。 +Compile([]Claim) → Reflex +``` + +Claim 是**实例**，Reflex 是**类**。分成两个类型而不是一个类型加状态标志，是因为这样**未验证的东西在类型上就跑不了**：`Claim` 没有 Options，L1 根本接不了它。约定变成结构。 + +**Provider 不在这张表里。** 它是底层设施（§8），不是抽象。换掉 Jev 换成 Laya 或 Kev，这三个概念一个都不动。 + +## 4. 两个端口 + +调用方只认识这两个，不需要知道 Claim、Reflex、Compile 的存在。 + +```go +// 写：注册 L2 刚做过的判定 +Register(ctx, []Claim) error + +// 读：这次观测触发了哪些反射 +Sense(ctx, state any) (map[string]string, error) // 题 → 选中的选项 +``` + +`Sense` 返回**决定**，不执行动作——动作是领域相关的（留哪几段、effort 调多少、丢哪个重复），属于调用方。系统本身完全不认识领域，这是它能独立存在的原因。 + +## 5. 两种用法 + +### 5.1 独立使用（像记忆系统一样） + +手写 Reflex，直接 `Sense`。不需要 `Register`，不需要 `Compile`。 + +用途：任何**重复出现的有限选择判断**——日志归类、告警分级、内容路由。 + +代价要说清：跳过 Compile 就跳过了 §6 那两个解出来的数。晋升和退役由你负责，而 P4 的铁律不会因为你手写就失效——阈值仍然必须是测出来的，不能是选出来的。 + +### 5.2 作为 harness 的 System One + +L2 边跑边注册，短路**自行形成**： + +``` +L2 工作中声明 ──→ Register ──→ Claim 日志（只增） + │ + Compile（离线、周期性、L2 执行） + │ ① 聚出反复出现的 + │ ② 抽成带有限选项的问题 + │ ③ 用日志里 L2 自己的答案验证 + ▼ + Reflex 集合 ◄─── 持续抽样；保持率掉了就移出 + │ +观测 ──→ Sense ──→ 塑形 ──→ 上下文 ──→ L2（无感） +``` + +**"之后不再需要 Claim"的准确机制**：不是禁止 L2 声明，是**那个判定不再出现在它面前**。L2 原来会声明"我在判这 48K 里哪几段有用"；编译之后它收到的就是 1.2K——那个判定对它来说不存在了。不是被压制，是 moot。 + +**不需要看见 thinking。** 声明是显式输出，不是内部推理。thinking display 默认 `omitted`、开了也只给摘要、raw CoT 任何模型都不外露——这条限制整个绕开了。 + +## 6. 自转：两个解出来的数 + +一旦出现"一致率到 0.95 就晋升"这种人挑的数，就不自转了。 + +但 L2 是随机函数（P1）——**它和自己也不完全一致**。同一情境重放两次，结论会变。这个自洽率免费可测，而且是唯一诚实的标尺： + +> **晋升**：L1 在历史声明上与 L2 的一致率 ≥ L2 与自己的重放一致率 +> **退役**：持续抽样，上式不再成立 + +你没有理由要求替代品比被替代品更稳。参照物始终是日志里 L2 自己写下的 `Chose`。 + +**采样从不停。** 晋升只是把双跑率从 1 降到 ε，不是切换模式。因为一直在测，测量不会过期，所以不需要时间戳、不需要"训练完成"标志。 + +**没有置信度阈值。** 弃权已经是一个选项（每条 Reflex 构造上必含"判不了"），而阈值那套建在一个**在 0.3–0.8 中段最不可靠**的数上（§10）。两套机制做同一件事，砍掉弱的那套。 + +### 自愈 + +Reflex 移出 → 塑形不再发生 → 原始观测流过去 → 判定重新出现在 L2 面前 → L2 又开始声明它 → 攒够了重新 Compile。 + +**没有恢复逻辑。** 系统的降级路径就是它的正常路径。 + +## 7. 一条规则 + +> **未答即原样。** 没有 Reflex、或弃权——`Sense` 返回空，调用方什么都不做，行为和没有这个系统时逐字节相同。 + +在旁路形态下这不是一条要维护的约定，是**物理事实**：塑形发生在观测入上下文之前，不塑形就是原样通过。因此不需要升级机制、不需要唤醒判据、不需要兜底逻辑。**失败恒等于不启用。** + +## 8. 底层设施 + +以下全部是设施，不是抽象。换掉它们，§3 的三个概念一个都不动。 + +**Provider。** L1 的实现。`ID()` 标识端点和模型——**它一变，所有 Reflex 的验证作废**，回到 Claim 日志重来。候选：Jev（托管）、Laya（421M，Apache-2.0，本机，含多语种变体）、Kev（实现同一 `/v1/systemone` 契约，SDK 可直接指过去）。 + +**只增。** 缓存按前缀匹配，中段编辑使其后全部失效；编辑过的历史现在还可能直接 400（preserved thinking 的 history-editing check，2026-08-31 之后创建的账号）。旁路形态天然满足：塑形在入上下文之前。 + +**Claude Code 的既有实践可直接引用**，不必自己论证：`` 追加而非改写历史；plan mode 用常驻工具进出；MCP schema 用 `defer_loading` 存根；官方对缓存中断按可用性级别告警。DeepSeek Harness 的 append-only 会话日志 + Trajectory 视图 + fork/replay，正好是 Claim 日志需要的仪器。 + +**模型不能换，effort 可以。** 缓存按模型隔离，无逃生口；effort 有——`{"role":"system","content":[],"output_config":{"effort":...}}` 不触发 messages 缓存失效（beta `mid-conversation-output-config-2026-07-01`，Opus 5 / Fable 5.1）。所以"这一步该想多深"是一条合法的 Reflex。 + +**经济学。** thinking 按 output 计价且留在前缀（P3），所以短路省两样：生成时的 output token（最贵一档、无缓存）+ 此后每轮的前缀增长。参考量级：Opus 5 $5/$25 每 MTok，Sonnet 5 $2/$10；缓存读约 0.1×，写 1.25×（5 分钟）/ 2×（1 小时）；Jev $0.042/M，输出免费。 + +## 9. 边界：旁路 ≠ 新权威 + +§1–§8 整套只对一种东西成立，必须和另一种分开： + +| | 旁路 | 新权威 | +|---|---|---| +| 做什么 | 替换掉**已经在发生**的判定 | 做一件**原本没人在做**的判断 | +| 有对照物吗 | 有——L2 的声明 | 没有 | +| 怎么定阈值 | 噪声底判据，免费 | 真标注，要花钱 | +| 失败时 | 退回原状 | 产生一个原本不存在的后果 | + +**`judge` 的 `presence` 是第二种。** 规则引擎命中之后，原本没有任何东西在判"这个产品是不是真的在跑"。两个后果： + +1. `judge.New` 默认的 `MinConfidence = jev.DefaultMinConfidence`（0.3）没有依据，**也不可能通过一致率取得依据**——没有"原来的做法"可比。而 0.3 恰好落在 Jev confidence 最不可靠的 0.3–0.8 中段下沿。 +2. `presence.go` 里 `Refuted → Rejected` 删掉了规则引擎的命中，**弃权或出错不退回原样，而是丢召回**——直接违反 §7。指纹场景里召回损失是静默的。 + +两条路都行：改成退回（`Rejected` 降级为"标记待审"），或者承认它是新权威、用真标注给它有依据的阈值。不能一边按 System One 设计、一边行使权威。 + +真标注已经在仓库里：`.judge-data/product-authenticity/` 的 17 份带标签部署响应，SHA256 已固定。 + +## 10. 已知未解决与风险 + +1. **Projection —— 唯一会随规模真正崩掉的一处。** 每条 Reflex 喂哪段 state：64K 总 / 32K 单问上限、每页 40 条 Claim、8 个产品版本，随运行时变化，**且没有明确归属者**。试过把它收进抽象（让 Reflex 自带 `Needs`），砍掉了——今天手写截断能跑。但它是最弱的工程点。 +2. **抓不到系统性偏差。** 噪声底判据只保证"不比 L2 更差"，不保证对。L1 和 L2 可以一致地错，而一致的错看起来像确证。 +3. **标定不可移植。** 独立测评：公开 benchmark ECE 0.024–0.032，自建分布 0.107（噪声底 0.024）；一篇 pentest harness 论文在自有评估集上测得 Jev ECE 0.246。confidence **两端可靠、0.3–0.8 中段最不可靠**。这是 §6 砍掉阈值的经验依据。 +4. **无内省。** L1 不给理由，可审计性只能靠记录（题、证据摘要、选项、动作），不能靠解释。 +5. **CJK 精度。** 本仓库目标大量是中文 web 资产，需专门留中文样本。 +6. **早期访问。** 无论文、无开放权重、payload 形状可能变。生产必须钉版本，不用 `jev-latest`。 +7. **唯一的端到端对照数据**来自一篇 pentest harness 论文（单次运行，作者自陈不具统计效力）：墙钟 32m12s → 26m53s，findings 16 → 18，**但场景命中 10/13 → 9/13**。方向上支持省墙钟，不支持完成率不降。 + +## 11. 术语 + +**启用**：`habit`（系统）、`Claim`（L2 声明的一次判定）、`Reflex`（抽出来的类）、`Compile`（实例→类）、`Register` / `Sense`（两个端口）。 + +**退役**：`Verdict`、`Is` / `Pick` / `IsMany` / `IsAll`、`Brief`（含 A1/A2 双写）、`Groove`、`Startle`、`Precedent`、`Impression`、`Floor`、`MeasuredAt`。 + +**从未启用**：Sieve、Assay、Attest、Ledger、Rung、SiftAct、Understudy、Harvest、CoDE。 + +对不等于现在就该有名字；命名纪律也需要停止条件。 + +--- + +## 附录 A：Jev API 参考 + +来源 、。复核于 2026-09-27——`api.md` 当前页面只给端点、鉴权、题型格式与返回字段，未给价格、限速、上下文上限和语言支持。 + +| 项 | 值 | 复核 | +|---|---|---| +| 端点 | `POST https://api.typesafe.ai/v1/systemone`；`GET /v1/models` | ✅ | +| 鉴权 | `Authorization: Bearer `，`TYPESAFE_API_KEY` | ✅ | +| 请求 | `state`（文本 / JSON 对象 / 文本数组）+ `model` + `questions` | ✅ | +| 响应 | `model` + `answers` + `usage{input_tokens, output_tokens}` | ✅ | +| 错误 | 401 鉴权、422 校验、429 限速、529 过载；429/529 指数退避 | ✅ | +| 价格 | $0.042/M 输入，输出免费 | ⚠️ 未复核 | +| 限速 | 250,000 tokens/s；1,200 req/min | ⚠️ 未复核 | +| 上下文 | 64K 总；state + 最长单问 ≤ 32K | ⚠️ 未复核 | +| 延迟 | 70–500 ms；问题并行 | ✅ 与实测一致（p50 236–276 ms，15 问一次调用） | +| 语言 | 英文为主，CJK 准确度较低 | ❌ 当前页面无此内容，按未证实处理 | + +| 类型 | criteria | 返回 | +|---|---|---| +| `noul` | 可选，`true`/`false` 各自的描述 | `noul`，0–1 概率，**不是布尔**；**无 `confidence`** | +| `choice` | 选项→描述的 map，**最多 255 项** | `choice` + `probabilities` + `confidence` | +| `score` | **2–10 个**有序等级描述的数组 | 加权 `score` + `legend` + `probabilities` + `confidence` | + +**本仓库只用 `choice`**（客户端只解码 `answer.Choice`，缺 `confidence` 直接报错）。这是个有意的选择：因为只用 choice，"判不了"才能是一个**真正的选项**而不是特殊返回值。 + +**已知坑**：`instructions` 与 `criteria` 自相矛盾会明显拉低质量。Compile 阶段的不一致样本最容易暴露这类规约错误。 + +**与本仓库现状的差异**： + +- Jev 客户端与合约已迁出本仓库，现为独立模块 `github.com/chainreactors/utils/jev`；`judge/jev/` 与 `judge/page.go` 均已不存在。 +- `cmd/judgeeval` 默认 `-rps 15`（上限 100），相对 1,200 req/min 偏保守。 +- 分页预算散落在 `judge/presence.go`（40 条）与 `judge/version.go`（`maxVersions` 40、`maxVersioned` 8），**没有对 64K/32K 做整体校验**——即 §10 风险 1。 +- `utils/jev` 现在把旁路侧和权威侧混在一个包里（`Ruling`、`Resolve(ruling, minConfidence)`、`Outcome` 的三个真值属于权威侧）。这是 §9 那条分界在代码里的位置。 + +## 附录 B：修订记录 + +### 2026-09-27 v4 —— System One 成为独立系统 + +**轴变了。** v3 沿着"一次调用的数据流"组织（Claim → Ruling → Resolve → Outcome）；本版沿着"一个判定从每次都做变成不必再做"组织。前者是实现细节，后者才是这套东西存在的理由。 + +**减法**： + +| v3 | 去向 | +|---|---| +| Brief（尾部全量 / 增量入历史）、`role:"system"` 注入信道、`clear_at` | **全部退役。** 旁路形态下没有要注入的陈述（§2 第四步） | +| 「弃权必须显式出现在注入里」 | 退役。不塑形 = 原样通过，物理事实 | +| `Groove` / `Startle` / `Precedent` / `Impression` | 退役。分别塌进 Reflex、弃权、Reflex、一个单字段返回值 | +| `Floor` / `MinConfidence` / `MeasuredAt` | 退役。弃权已是选项，阈值那套建在最不可靠的信号上；持续采样使时间戳多余 | +| `Provider` 作为核心概念 | 降级为设施（§8） | +| 「五态生命周期」「四个筛子」「影子模式」 | 分别塌成两个类型、一个判据、一个采样率 | + +**加法**： + +1. **L1 / L2 按能力定义**（§1）——一条差别推出四个推论，取代 v3 的散列前提。 +2. **旁路形态**（§2 第四步）——本版最大的结构改动。判定不讲给 L2 听，而是塑形它感知到的东西。连带消灭了 v3 一整套注入设计。 +3. **Claim / Reflex 分离**（§3）——实例与类。`Claim` 这个名字被释放给 L2 的声明；v3 叫 `Claim` 的东西其实一直是 `Reflex`。净效果是**不再需要挖 thinking**（raw CoT 不外露那条硬依赖消失）。 +4. **两个端口**（§4）——`Register` / `Sense`。系统不认识领域，因而可以独立存在。 +5. **两种用法**（§5）——独立使用 / 作为 harness 的 System One。 +6. **噪声底判据**（§6）——晋升和退役都从 L2 的自洽率解出，零自由参数。 +7. **记忆 / 习惯对偶**（结论）——定位这个系统在 harness 里的位置。 + +概念总账：v3 是 4 个概念 + 1 个参数 + 1 个配置结构；v4 是 **3 个概念 + 2 个端口 + 1 条规则 + 0 个自由参数**，Provider 降为设施。 + +### 2026-09-27 v3 —— 先减法后加法 + +砍掉四方案同心结构、`Verdict`/`Is`/`Pick` 系列、Brief 双写、五条固定判定清单、升级/唤醒机制；加入 P4 铁律、「重复⟹可类型化」、「替换与测量同一」、`Groove`、「未答即原样」、「短路≠新权威」。v4 保留了 P4 铁律、§9 的分界和「未答即原样」，其余重构。 + +### 2026-09-26 v2 —— KV 缓存被指出后重写 + +骨架由「一个约束」改为「两个约束」；新增只增约束对设计空间的裁剪；经济学重写（v1 按全价计算上下文节省，夸大约 10 倍）；指标分列。v4 保留只增约束（已升级为 API 硬错误），其余结构已重构。 diff --git a/docs/jev-maintenance-validation-20260926.md b/docs/jev-maintenance-validation-20260926.md new file mode 100644 index 0000000..889141f --- /dev/null +++ b/docs/jev-maintenance-validation-20260926.md @@ -0,0 +1,98 @@ +# Jev 新产品指纹维护验证：20260926 批次 + +> 历史评估记录：文中的 Refine、SuggestNames、Classify 等机制已移除，数据保留当时含义。当前实现见 [Claim 合约](../judge/README.md) 和 [judgeeval 报告格式](../cmd/judgeeval/README.md)，本文指标未由当前实现重新验证。 + +## 结论 + +本次扩大真实响应范围，核验了八种原目录没有的产品，生成八份候选；其中 **SearXNG、Wakapi、IT Tools 三种通过独立样本测试及原生库加载检查**，输出可直接加载的增量库。加载该库后再运行 Jev 清洗，也保留了全部标注真命中；重复维护识别为已有产品，不重复入库。 + +补充[产品真实性复核](jev-product-authenticity-20260926.md)：三个都是已有真实开源项目，“新”指本次库内新增。**IT Tools 的标题被原版和衍生版共用，因此本文对它的正例及 FP/FN 仅代表产品家族识别，不代表已区分 CorentinTh 原版与其他分支。** + +这验证了有证据标注和评审的辅助维护流程。`IsUnknownProduct` 对 IT Tools 的三份真实界面仍返回 false；`SuggestNames` 均能提出正确名称。因此当前自动发现入口有漏检，不能依赖单个 unknown 布尔值决定所有新产品是否进入维护流程。 + +## 交付物 + +- [原生增量库](../fingerprints/jev-maintained-20260926.yaml)：三个产品，1782 字节，SHA256 `522a85806592e10bdffa87ccb70be90e22098fdeba85cc52452235132520065a`。 +- [来源与验证记录](../fingerprints/jev-maintained-20260926.provenance.json)：目录哈希、真实样本 URL/哈希/产品标签、训练划分、测试结果、重复维护、离线复现及代码哈希。 +- [CLI 使用说明](../cmd/judgeeval/README.md)：`-manifest`、`-history`、`-maintain`、`-library`、`-offline` 的明确语义。 +- 完整本地数据：`.judge-data/expansion-20260926/`。HTTP 原文、模型答案缓存和各次运行被 Git 忽略，不混入源码或指纹文件。 + +## 数据及真值 + +新增采集尝试 53 个明确的公开 URL，成功 34 个、失败 19 个；合并上一批 35 份响应，共 **69 份响应、59 个主机/运营方组、65 份去除部分动态头后的不同响应**。失败请求单独保存，不计为负例。 + +原始快照 UTC 时间为 `2026-09-25T14:18:17Z` 至 `2026-09-25T16:05:54Z`；北京时间跨 9 月 25 日晚至 9 月 26 日凌晨，故目录沿用 `20260926` 批次名。 + +新目标包括：官方 SearXNG 实例目录使用种子 `20260926` 抽取的公开实例、项目公开演示和另外找到的公开安装。训练反例包含 nginx/WordPress 响应及官方文档/README，专门验证“页面提到产品”不会被当作该产品的运行界面。来源快照、目标列表、失败记录和选样方式均存档。这是按产品补充的验证样本，不是互联网产品分布的随机代表样本。 + +682 个产品—响应标注对由响应头、generator meta、标题、资源路径和人工复核支持。真值指**响应可支持的产品/版本**，不等于主机实际安装审计。没有证据的版本应保持空值；未标注的识别结果单列，不算真阳性。 + +一次标签修订：PairDrop 原本被标为空版本，但原 HTML 的产品标题旁明确出现 `
v1.11.2
`，已修正为 `1.11.2` 并保存修订记录。没有依据模型回答修改标签。 + +## 实际验证链路 + +1. 审计五个内置 HTTP 目录：fingers、goby、ehole、wappalyzer、fingerprinthub；再检查别名表及加载后的 fingers 运行时目录，记录 SHA256。八个候选在初始审计范围均无同名/显式别名产品。 +2. 人工依据证据组织产品正反例；Jev 从响应中提取的候选里选择名称和版本，Go 代码生成原生规则。使用普通 `Positive`，没有调用 `PositiveVersion` 向真实评估的生成器喂标签版本。 +3. 导出 YAML 后重新加载、编译，验证训练匹配及版本；用训练主机以外的正反例测试。相同主机、同一运营方组及重复内容被隔离。 +4. 候选至少具备两个独立正例主机组、五个独立反例主机组；身份和版本验证均无错漏才允许进入增量库。 +5. 调用 `engine.Fingers().LoadFromYAML(...)`、`engine.Compile()`，用真正的 `DetectContent` 重放全部 69 份响应。检查新产品、版本、已有命中和已有非空版本。 +6. 用加载后的库再运行清洗和版本补全，验证重复维护去重；最后严格离线重放，缓存未命中即失败。 + +## 新增指纹结果 + +下表为最终原生库加载后、**排除训练数据的测试集**结果。三个产品的测试集共享部分反例，不能把负例总数当作不同网站数量相加。 + +| 产品 | 训练正例 | 独立测试正例 | 测试负例 | 新增前漏报 → 加载后命中 | FP / FN | 测试版本 | +|---|---:|---:|---:|---:|---:|---| +| SearXNG | 2 | 8 | 47 | 8 → 8 | 0 / 0 | 8 个准确，保留日期与提交后缀 | +| Wakapi | 1 | 3 | 53 | 3 → 3 | 0 / 0 | 3 个准确，覆盖 2.18.0 与 2.12.0 | +| IT Tools | 1 | 2 | 54 | 2 → 2 | 0 / 0 | 2 个正确留空，未虚构版本 | + +共补回 **13 个独立测试正例**的产品识别，准确提取其中 11 个有证据的版本。原生加载在全部样本中没有丢失已有命中、改坏已有非空版本或产生新增产品的标注错误。 + +`library.yaml` 是新增的原生列表文件，需显式加载；没有重写内置 gzip 资源或发布外部模板仓库。 + +### 尚未收录 + +| 产品 | 已采集正例 | 结论 | +|---|---:|---| +| PairDrop | 1 | 能生成规则、训练版本 1.11.2 正确，但缺独立正例 | +| Docmost | 1 | 能生成规则，但缺独立正例 | +| Linkwarden | 1 | 能生成规则，但缺独立正例 | +| Karakeep | 3 | 均属同一运营方组，无法作为独立测试正例 | +| New API | 1 | 能生成规则并提取 1.0.0-rc.35，但缺独立正例 | + +五种候选最终均为 `insufficient_holdout`，保存在最终运行的 `fingerprints/` 下，未进入发布用增量文件。它们在本批测试负例上未命中，仍不能据此证明召回能力。 + +## 真实数据暴露的问题与修复 + +- **共享基础设施被学成应用指纹**：初版 IT Tools 在测试集有 6 个误报；Docmost、Linkwarden、Karakeep 分别有 9、8、6 个。规则曾选用 Cloudflare 或通用安全头。现在应用候选头必须带有目标产品身份，普通部署环境特征不能独立成为应用规则。 +- **年份版本与 generator 名称**：SearXNG 的 `2026.9.23+3cd69d30e` 被旧模式截断或混进产品名称。现在保留四位年份和后缀，以 generator 声明生成稳定识别及版本规则，可跨定制标题复用。 +- **版本远离产品名**：Wakapi 版本在静态资源参数和页脚。新增带产品身份约束的版本正则；身份与版本在同一个正则中结合，避免原生匹配器的 OR 语义造成无约束版本命中。 +- **加载后清洗回归**：最初重跑发现 IT Tools 两个真命中被删，SearXNG 文档被错误召回并填入版本。补充页面 description 证据，明确静态浏览器应用和文档主题的不同语义后，这三个具体问题均消除。 +- **重复收录与样本计数**：`-library` 归档并加载已有扩展库；运行时目录参与查重。相同主机名或运营方的交叉分组连通后只计一次。 + +初始失败保留在 `run-01`，加载后清洗失败保留在 `run-05-with-library`。这些独立主机测试集在修复中被重复使用，最终结果属于回归验证，**不是未经调试使用的外部盲测准确率**。 + +## 清洗与发现边界 + +最终未加载增量库时，`Refine + Version` 在已标注范围删除 24 个误报、误删 0 个、正确补全 20 个版本，没有自然补回产品漏报；名称建议覆盖 25 个漏报样本，建议不计为已识别。 + +加载三条新指纹后，`run-06-with-library` 的清洗结果为 **TP 63 / FP 0 / TN 610 / FN 9**；已有真命中误删 0、错误补版本 0，37 个有值版本正确、26 个应空版本正确留空。这里包含训练样本；9 个漏报仍包括尚未收录的候选以及旧批次的 Nutz、aaPanel。还有 148 个未标注产品预测，不能把当前标注范围的 FP=0 推广成整体误报率为零。 + +在没有加载新库时，`SuggestNames` 分别支持 SearXNG 10/10、Wakapi 4/4、IT Tools 3/3；`IsUnknownProduct` 分别只有 9/10、4/4、0/3。当前维护依赖证据分组及评审，不能只用 `Classify.Generic`/unknown 作为发现入口。主动探测规则、覆盖更多独立产品版本、全面纠正已有错误非空版本也不属于本批已完成的验证范围。 + +## 复现与检查 + +- `run-07-final`：最终生成、审计和原生加载；`run-06-with-library`：相同库加载后的清洗及重复维护，三种产品全部 `already_catalogued`、没有新库重复输出。 +- `run-08-offline`：0 次模型请求、483 次精确缓存命中。语义指标一致，`library.yaml`、`generation.json`、`maintenance.json`、`rows.jsonl`、`baseline.jsonl`、`cleaned.jsonl` 均与最终运行逐字节一致。 +- 每次运行保存原始输入快照、标签清单、构建信息；另存本次代码文件哈希，便于在有未提交修改的工作区复核。 + +```powershell +go build -tags goregexp -o bin/judgeeval.exe ./cmd/judgeeval +./bin/judgeeval.exe -offline -maintain -manifest .judge-data/expansion-20260926/run-07-final/manifest.json -history .judge-data/expansion-20260926/run-07-final/baseline.jsonl -cache .judge-data/cache -out .judge-data/reproduce-new-products +go test -short -tags goregexp ./judge ./judge/jev ./cmd/judgeeval . ./cmd/engine -count=1 -timeout 3m +git diff --check +``` + +受影响包测试通过。全仓库 `go test ./...` 的既有 go-re2/go.sum、nmap 和 fingerprinthub 旧接口问题未纳入修复，未声称全仓库测试通过。后续扩大验证应优先补充五个草案产品的独立部署，以及未参与调试的新一批样本。 diff --git a/docs/jev-product-authenticity-20260926.md b/docs/jev-product-authenticity-20260926.md new file mode 100644 index 0000000..c2aa44b --- /dev/null +++ b/docs/jev-product-authenticity-20260926.md @@ -0,0 +1,83 @@ +# 三个新增指纹的产品真实性复核 + +## 结论 + +**SearXNG、Wakapi、IT Tools 都是真实存在、有公开源码及实际部署的项目。** 本批“新产品”指当前已审计指纹库中的新增识别对象，不代表软件近期发布，也不代表首次在全球被识别。 + +| 产品 | 官方仓库 | 仓库创建日期（UTC） | 本批指纹的准确语义 | +|---|---|---|---| +| SearXNG | `searxng/searxng` | 2021-04-12 | SearXNG 产品识别；从 generator 声明提取版本 | +| Wakapi | `muety/wakapi` | 2019-05-21 | Wakapi 产品识别；结合标题与资源/页脚证据提取版本 | +| IT Tools | `CorentinTh/it-tools` | 2020-04-05 | **IT Tools 产品家族识别**；无法区分原版与衍生版，版本留空 | + +日期来自 GitHub 仓库 API 的 `created_at`，不是首次发布日。API 和源码复核时间为 2026-09-25 17:02–17:06 UTC，即北京时间 2026-09-26 01:02–01:06。部署证据复用上一轮已归档的真实 HTTP 响应。 + +## SearXNG：身份和生成标记相互印证 + +- 官方仓库：[searxng/searxng](https://github.com/searxng/searxng)。官方 README 将其定义为元搜索引擎。 +- 官方模板 `searx/templates/simple/base.html` 第 8 行生成 `name="generator"`、`content="searxng/{{ searxng_version }}"`；第 72 行也显示产品名及版本。 +- 官方 `searx/infopage/en/about.md` 第 49 行说明 SearXNG 源自 Searx 的分支。应保留 SearXNG 独立产品身份，不能把名字包含 Searx 的页面全部认作 SearXNG。 +- 本批 10 份 SearXNG 正例均包含对应 generator，包括定制标题的实例。规则依赖产品声明，不要求站点标题恰好叫 SearXNG。 + +固定源码证据：[模板 blob](https://api.github.com/repos/searxng/searxng/git/blobs/abad3f141daaecf0465211c463319beaac30ce39)、[项目关系说明 blob](https://api.github.com/repos/searxng/searxng/git/blobs/486b2617595ee91d50853163af983a4329ee09aa)。 + +**判定：真实产品，本次库内新增身份成立。** generator 是应用自行声明的证据，不等同于主机安装审计或不可伪造的版本证明。 + +## Wakapi：独立应用，兼容 WakaTime + +- 官方仓库：[muety/wakapi](https://github.com/muety/wakapi)。官方 README 第 14 行说明它是兼容 WakaTime 的自托管编程统计后端。 +- 官方 `views/head.tpl.html` 第 2 行使用 `Wakapi – Coding Statistics` 标题，与本批 4 份正例一致。 +- 模板第 18 行使用 `assets/css/app.dist.css?v={{ getCacheBuster }}`；`views/footer.tpl.html` 第 3 行通过 `getVersion` 输出版本。缓存参数须结合具体响应验证，不能仅因出现 `?v=` 就认定它是产品版本。 +- 本批样本包括项目主页及三个独立测试站点；版本提取结果见[原验证报告](jev-maintenance-validation-20260926.md)。 +- 内置 Wappalyzer 的 `Wakav Performance Monitoring` 是另一项监控产品，不能因前缀相似认作 Wakapi。兼容 WakaTime 也不等于 Wakapi 是 WakaTime 的别名。 + +固定源码证据：[README blob](https://api.github.com/repos/muety/wakapi/git/blobs/ce5f15f2198397dd5b41a6a1eb1be08ccab0d626)、[页面头模板 blob](https://api.github.com/repos/muety/wakapi/git/blobs/12f24762324fece08a9ccad1ba9265556c62aeaa)、[页脚模板 blob](https://api.github.com/repos/muety/wakapi/git/blobs/3d50040e195b1792925994163c6b7c0139ebe022)。 + +**判定：真实产品，本次库内新增身份成立。** + +## IT Tools：真实项目，当前指纹仅识别产品家族 + +- 原项目：[CorentinTh/it-tools](https://github.com/CorentinTh/it-tools)，开发者在线工具集合。官方 `index.html` 第 7 行即当前规则使用的完整标题。 +- 衍生项目：[sharevb/it-tools](https://github.com/sharevb/it-tools)。GitHub API 返回 `fork: true`，`parent.full_name` 为 `CorentinTh/it-tools`，仓库创建于 2023-11-30。这不是本批额外发现的第四个独立产品。 +- 衍生版 `index.html` 第 26 行保留相同标题，第 37 行的 `og:url` 指向 `https://sharevb-it-tools.vercel.app/`。 + +| 本批样本 | 直接观察 | 能够支持的结论 | +|---|---|---| +| `it-tools.tech` | 完整产品标题；`og:url` 指向项目主页 | 原项目官方站点样本 | +| `tools.code.pro.vn` | 同一标题；`og:url` 改为当前站点 | IT Tools 家族，具体分支未确定 | +| `tools.quitw.org` | 同一标题；`og:url` 指向 sharevb 主页 | 携带 sharevb 衍生版元数据，不能仅凭该信息确定精确代码分支或版本 | + +固定源码证据：[原版 index.html blob](https://api.github.com/repos/CorentinTh/it-tools/git/blobs/e8b8a60ea6f6864ea062874c4d6e84cbea608de6)、[衍生版 index.html blob](https://api.github.com/repos/sharevb/it-tools/git/blobs/7e0246142e9818a8745d758aed6bd9429c221546)。分支关系来源：[仓库 API](https://api.github.com/repos/sharevb/it-tools)。 + +**判定：真实项目，本次库内新增家族级识别成立。当前规则只匹配标题，不能宣称三个站点都是 CorentinTh 原版，也不能据此补具体版本。** 原报告的 IT Tools 正例和 FP/FN 应按家族级标签理解；若目标改为原版或指定分支，需要重新标注和验证，原指标不能直接沿用。 + +## “库内新增”的核查范围 + +复核 `run-07-final/catalog-before.json` 的五个内置 HTTP 目录、别名及运行时 fingers 目录，再对以下文件进行全文检查： + +- `resources/fingers_http.json.gz` +- `resources/goby.json.gz` +- `resources/ehole.json.gz` +- `resources/wappalyzer.json.gz` +- `resources/fingerprinthub_web.json.gz` +- `resources/aliases.yaml` + +解压后忽略大小写，检索 `searx|waka|it[\s_-]*tools|corentinth|sharevb|handy[\s_-]*online`。只有 Wappalyzer 中同一个 `Wakav Performance Monitoring` 条目的五处文本命中；其余为零。未发现三个产品的同名、显式别名或上述身份标记。 + +结合此前原生检测的新增前漏报、加载后的命中，以及重复维护时三项均成为 `already_catalogued`，支持“本次审计范围中的新增指纹”。不覆盖外部所有指纹库；关键词检查也不能排除不含这些标记的规则存在语义重合。 + +## 证据与变更 + +完整证据位于本地 `.judge-data/product-authenticity/`： + +| 文件 | 内容 | +|---|---| +| `repositories.json` 与四份仓库原始 JSON | 官方来源、创建时间、分支关系、采集时间、SHA256 | +| `source-tree-index.json` 与四份 tree JSON | 源文件定位、Git tree SHA、完整性标志 | +| `source-evidence.json` 与 `source/` | 官方源码、Git blob SHA、SHA256、证据行号 | +| `deployment-evidence.json` | 17 份正例的响应路径、URL、SHA256、身份标记 | +| `library-fulltext-audit.json` | 六个本地资源的 SHA256、检索命中及上下文 | + +17 份部署响应的 SHA256 均与原 manifest 一致。其中包含训练样本，不是新增盲测样本数量。本次复核补充官方来源和身份范围，没有增加识别准确率或覆盖率声明。 + +本次仅新增证据和文档说明；原指纹 YAML、历史标签及评估结果保持原样。YAML 的 SHA256 仍为 `522a85806592e10bdffa87ccb70be90e22098fdeba85cc52452235132520065a`。 diff --git a/docs/jev-real-validation-20260925.md b/docs/jev-real-validation-20260925.md new file mode 100644 index 0000000..b00d0b9 --- /dev/null +++ b/docs/jev-real-validation-20260925.md @@ -0,0 +1,106 @@ +# Jev 真实响应验证：2026-09-25 + +> 历史评估记录：文中的 Refine、SuggestNames 等接口已移除，数据保留当时含义。当前实现见 [Claim 合约](../judge/README.md) 和 [judgeeval 报告格式](../cmd/judgeeval/README.md)，本文指标未由当前实现重新验证。 + +## 结论 + +已经用新采集的真实 HTTP 响应跑通保存结果清洗、逐产品版本补全、名称建议、候选指纹生成、原生 YAML 导出加载和离线回放。发现的问题已修复并重新评估。 + +在本批已标注范围内，清洗移除 24 个误报，没有误删真实命中；新增 20 个正确版本。自然漏报仍有 3 个，没有被自动补回。6 项指纹生成实验中，3 项通过独立样本测试，另外 3 项独立正例不足。不能据此宣称全部功能已经验证充分，或推断全网误报率为零。 + +主报告：[run-03/report.md](../.judge-data/run-03/report.md)；机器指标：[run-03/metrics.json](../.judge-data/run-03/metrics.json)；离线复现：[run-04/report.md](../.judge-data/run-04/report.md)。原始响应和模型缓存均留在本机 Git 忽略目录 `.judge-data/`。 + +## 数据与标注 + +- 2026-09-25 22:18:17–22:18:19（北京时间）重新采集；来源为用户指定 `47.239.11.0/24` 内的 HTTP/HTTPS 页面及 3 份 `/robots.txt` 响应。 +- 共 35 份响应、30 个主机组、31 种去除易变响应头后的响应；本批明确 URL 列表采集失败 0 项。这是选定的响应集合，不是完整网段可用性普查或全网随机代表样本。 +- 138 个产品正负标注，其中 49 个包含版本标注。版本缺省表示未标注，空字符串表示该响应没有可归属的版本证据，非空字符串表示期望的完整版本。 +- 标注依据为原始响应头、generator meta、资源路径、页面产品归属及保存的原文片段。加载时检查 SHA256、HTTP 完整性和证据片段是否存在；证据是否足以支持结论仍需要复核。 +- 标注不来自 Jev 的答案，不输入清洗或版本判定。真值定义为“响应证据支持的产品和版本”，不是服务器安装审计；IP 抓取也不能覆盖域名虚拟主机的所有内容。 +- 页面自报版本、产品头和资产版本都可能与实际安装状态不同。同一主机多份响应不视作独立统计样本；未标注输出不能算作已证明正确或误报。 + +首次标注复核后修正了两项：New API 的 `X-New-Api-Version: v1.0.0-rc.35`，以及 ThinkPHP 产品归属下的 `V8.1.2`。修订前后值和原文保存在 [label-review.json](../.judge-data/real-20260925/label-review.json)。最终各阶段统一按修订后的标签计分。 + +## 清洗、误报与漏报 + +计数单位是已标注的“响应 × 产品”。产品名称按规范化名称及显式别名匹配，不使用子串判断。 + +| 阶段 | TP | FP | FN | TN | 未标注输出 | +|---|---:|---:|---:|---:|---:| +| 保存的原规则结果 | 46 | 24 | 3 | 65 | 74 | +| Refine | 46 | 0 | 3 | 89 | 28 | +| 再对各产品补版本 | 46 | 0 | 3 | 89 | 28 | + +- 已标注误报移除 24，真实命中误删 0，自然漏报恢复 0；35 份响应全部完成，无失败行。 +- 剩余漏报：Nutz（`c31d254bc1fd0230`）、aaPanel（`afac5614625bfc6f`）、New API（`05346dbcf28946f0`）。 +- 名称建议覆盖 Nutz 和 New API 两项，但建议没有成为已接受的指纹命中，因此不计作漏报恢复。 +- 清洗后仍有 28 个未标注输出，尚不能由本批标签判断其准确性。 + +本次 `-history` 使用的是本批新响应首次运行时保存的 `baseline.jsonl`。这证明“保存结果 → 重新判定 → 清洗结果”机制可用，不代表覆盖长期生产历史分布。 + +## 版本补全 + +| 阶段 | 正确 | 错误 | 缺版本 | 正确留空 | 无依据填值 | 产品未识别 | +|---|---:|---:|---:|---:|---:|---:| +| 原规则结果 | 3 | 0 | 20 | 23 | 0 | 3 | +| Refine | 9 | 0 | 14 | 23 | 0 | 3 | +| 各保留产品分别调用 Version | 23 | 0 | 0 | 23 | 0 | 3 | + +`Refine` 只补主产品的版本；独立 `Version` 调用验证剩余保留产品的空版本。正确非空版本从 3 增至 23，净新增 20 项。另有 2 项新增版本未标注，不计入正确项。按产品合并别名，基线任一别名已有版本即不算新增；逐名称变化仍保存在 `rows.jsonl`。 + +已有非空版本保留。本批没有已标注的错误历史版本，因此“修正一个已有但错误的版本值”未在本批验证，也不是这里的补空值操作。漏报产品的版本记为“产品未识别”，不能冒充正确留空。 + +## 指纹生成与加载 + +真实正反例经 `NewGenerator → Positive/Negative → Generate` 生成原生指纹。自动命名计划使用 `SuggestNames`；版本从普通 `Positive` 自动推断，不向生成器传入评估标签。导出 YAML 后重新加载、编译，并通过原生 `PassiveMatch` 或录制探测响应的 `ActiveMatch` 测试。 + +训练和测试按主机组、实际 hostname、去除易变头的响应内容隔离；测试集还排除重复响应。以下计数均为未参加训练的独立响应，TP/FP/FN/TN 顺序固定。 + +| 计划 | 独立 TP/FP/FN/TN | 独立版本结果 | 结论 | +|---|---|---|---| +| nginx-auto | 13/0/0/5 | 4 个正确版本，9 个正确留空 | passed_holdout | +| caddy-auto | 2/0/0/18 | 2 个正确留空 | passed_holdout | +| apache-auto-version | 2/0/0/17 | 2 个正确留空 | passed_holdout | +| caddy-active | 0/0/0/0 | 无独立样本 | insufficient_holdout | +| new-api-candidate | 0/0/0/3 | 无独立正例版本 | insufficient_holdout | +| softether-candidate | 0/0/0/4 | 无独立正例版本 | insufficient_holdout | + +训练版本单独统计：nginx 为 2 个正确版本，Apache 为 1 个，New API 为 1 个。New API 的候选规则已完整提取 `1.0.0-rc.35`，并采用版本头名称作为产品特征；这仅证明该训练响应上的生成机制，不能代替独立正例验证。Apache 的版本提取也只在本批训练响应得到验证。 + +Caddy 主动规则已在录制的训练探测响应上完成匹配；其剩余正例探测响应与训练内容重复，被剔除。SoftEther 的另一主机返回相同页面，也被剔除。New API 只有一个正例主机。候选文件保存在 [run-03/fingerprints](../.judge-data/run-03/fingerprints)，未直接写入发布指纹库。 + +## 真实证据推动的修复 + +1. **协议特征误保留**：HTTP Basic 等指纹原先被无条件信任。现在需要对应响应头证据；正文提及认证或错误命中不能绕过清洗。 +2. **版本被截断**：统一版本 token，保留 OpenSSL `1.0.2q`、预发布 `1.0.0-rc.35`、`8.1SP2` 等后缀，支持脚本文件名的版本边界。 +3. **版本候选被噪声挤掉**：先收集再按证据排序，避免前部大量数字挤掉后部 generator、响应头或资源版本。 +4. **生成规则绑定单一版本**：`X-*-Version` 使用头名称识别产品，并独立提取版本；生成头规则时规范化产品名。 +5. **评估夸大改进**：修正别名重复统计，区分名称建议与真正补回漏报，未识别产品不计为版本留空成功，错误行及未标注输出单列。 + +对应回归位于 [judge/real_evidence_test.go](../judge/real_evidence_test.go) 和 [cmd/judgeeval/replay_test.go](../cmd/judgeeval/replay_test.go)。最小证据回归包含受控构造，不加入真实样本数量或真实准确率统计。 + +## 回放与复现 + +每个运行目录保存原始响应副本、相对路径 manifest、SHA256、原始及清洗后 Frameworks、逐项结果、生成 YAML 和 Go 构建信息。响应快照可独立加载；严格离线的模型答案还需要保留指定的精确缓存目录。 + +| 运行 | 用途 | Provider 请求 | 精确缓存命中 | 输入 tokens | +|---|---|---:|---:|---:| +| run-01 | 首轮真实响应评估 | 180 | 41 | 313,477 | +| run-02 | 证据修复后评估 | 30 | 190 | 64,248 | +| run-03 | 最终计数及训练版本核验 | 0 | 220 | 0 | +| run-04 | 无凭据严格离线回放 run-03 快照 | 0 | 220 | 0 | + +模型标识为 `jev/jev-1.13.0`。精确缓存键包含模型、endpoint、完整状态和问题，评估禁用近似页面缓存。离线缓存缺失直接报错，测试已确认不会回退调用 Provider。最终回放检查：除耗时外指标一致；manifest、baseline、cleaned、rows、generation 的 JSON 内容一致；6 个 YAML 文件逐字节一致。复现证明输入与已保存模型回答的确定性，不等于重新远程请求模型后的稳定性试验。 + +```powershell +go build -tags goregexp -o bin/judgeeval.exe ./cmd/judgeeval +# 使用新目录，已有结果拒绝覆盖。离线无需 TYPESAFE_API_KEY。 +./bin/judgeeval.exe -offline -manifest .judge-data/run-03/manifest.json -history .judge-data/run-03/baseline.jsonl -cache .judge-data/cache -out .judge-data/run-reproduce -rps 8 + +go test -short -tags goregexp ./judge ./judge/jev ./cmd/judgeeval . ./cmd/engine -count=1 -timeout 3m +git diff --check +``` + +本次受影响包测试及 diff 检查通过。全仓库测试尚有此前存在的 benchmark go-re2 依赖和 nmap/fingerprinthub 旧接口问题，不在本次修复范围，不能声称全仓库测试通过。 + +后续验证缺口集中在：3 个自然漏报、错误非空历史版本、New API/SoftEther/Caddy 主动规则的独立正反例，以及跨网段、跨版本和跨时间的新批次。现有报告已明确区分“已执行”“本批通过”和“证据不足”。 diff --git a/docs/jev-report.html b/docs/jev-report.html new file mode 100644 index 0000000..8ee8cf6 --- /dev/null +++ b/docs/jev-report.html @@ -0,0 +1,142 @@ + + + + + +Jev 判定层效果 + + + +
+ +

Jev 判定层效果

+

同样 1070 个真实网站，传统规则指纹 vs 规则 + Jev 判定

+
传统规则规则 + Jev
+ +
+ +
+
可信度（40 页标准答案）
0 误删
误报剔除 37/37，真实结果保留 45/45
+
速度代价
+0.3 秒/页
异步执行，原有规则识别不变慢
+
成本
$3.3
每万页
+
+ +

+ 注意：改过品牌的产品 Jev 也认不出来，发现新产品仍靠规则；Jev 请求失败时自动保持传统规则的结果。
+ 数据来自 Common Crawl 公开抓取数据，每个网站只取一页；"误报"指产品名只出现在正文、不在任何页面结构中的命中；版本号以页面自带的 generator 标记为参考答案；模型 jev-1.13.0。文字版：jev-report.md +

+
+ + + diff --git a/docs/jev-report.md b/docs/jev-report.md new file mode 100644 index 0000000..8772fdf --- /dev/null +++ b/docs/jev-report.md @@ -0,0 +1,31 @@ +# Jev 判定层效果报告 + +> 历史评估记录：保留当时的接口、报告字段和测量结果。当前实现以 [Claim 合约](../judge/README.md) 和 [judgeeval 报告格式](../cmd/judgeeval/README.md) 为准；本文指标未由当前实现重新验证。 + +在 1070 个真实网站上，把**传统规则指纹**与**规则 + Jev 判定**逐页对比。 + +## 核心对比 + +| 指标 | 传统规则 | 规则 + Jev | 效果 | +|---|---|---|---| +| **误报**：只在正文里提到、实际并不存在的产品 | 138 个 | 13 个 | 减少 91% | +| **重复**：每页结果条数 | 13.4 条 | 6.5 条 | 减半，同名重复全部合并 | +| **漏报**：规则没识别、被补回的产品 | 0 个 | 467 个 | 新增 | +| **版本号**：识别正确的页面（共 274 页） | 43 页 | 200 页 | 提升 4.7 倍 | + +**代价**：每页多花约 0.3 秒（异步执行，不影响原有的规则识别速度），每万页约 3.3 美元。 + +## 可信度 + +在 40 个有标准答案的页面上： +- 误报剔除 37/37； +- 真实结果保留 45/45，没有误删。 + +## 注意 + +- 改过品牌的产品，Jev 也认不出来；发现新产品仍然靠规则。 +- Jev 请求失败时，结果自动保持为传统规则的结果。 + +--- + +测试方法：数据来自 Common Crawl 公开抓取数据，每个网站只取一页。"误报"指产品名只出现在正文、不在任何页面结构中的命中；"版本号正确"以页面自带的 generator 标记作为参考答案。模型为 `jev-1.13.0`。 diff --git a/ehole/ehole.go b/ehole/ehole.go index 8dfcd45..e9fd49a 100644 --- a/ehole/ehole.go +++ b/ehole/ehole.go @@ -37,6 +37,8 @@ func NewEHoleEngine(data []byte) (*EHoleEngine, error) { type EHoleEngine struct { Fingerprints []*Fingerprint `json:"fingerprint"` FaviconMap map[string]string + // MatchDetailEnabled records on each hit the keyword or text it matched. + MatchDetailEnabled bool `json:"-"` } func (engine *EHoleEngine) Name() string { @@ -106,6 +108,9 @@ func (engine *EHoleEngine) MatchWithHeaderAndBody(header, body string) common.Fr for _, finger := range engine.Fingerprints { frame := finger.Match(header, body) if frame != nil { + if !engine.MatchDetailEnabled { + frame.MatchDetail = nil + } frames.Add(frame) } } @@ -122,18 +127,32 @@ type Fingerprint struct { } func (finger *Fingerprint) Match(header, body string) *common.Framework { + var content string switch finger.Location { case BodyLocation, TitleLocation: - if finger.MatchMethod(body) { - return common.NewFramework(finger.Cms, common.FrameFromEhole) - } + content = body case HeaderLocation: - if finger.MatchMethod(header) { - return common.NewFramework(finger.Cms, common.FrameFromEhole) - } + content = header default: return nil } + if !finger.MatchMethod(content) { + return nil + } + frame := common.NewFramework(finger.Cms, common.FrameFromEhole) + frame.MatchDetail = finger.matched(content) + return frame +} + +// matched is what a matching fingerprint found in content: its first keyword, +// or the text its first regexp matched. +func (finger *Fingerprint) matched(content string) *common.MatchDetail { + switch { + case finger.Method == KeywordMethod && len(finger.LowerKeyword) > 0: + return &common.MatchDetail{MatcherType: "word", MatcherValue: finger.LowerKeyword[0]} + case finger.Method == RegularMethod && len(finger.compiledRegexp) > 0: + return &common.MatchDetail{MatcherType: "word", MatcherValue: finger.compiledRegexp[0].FindString(content)} + } return nil } diff --git a/engine.go b/engine.go index 2034bde..dfa48c2 100644 --- a/engine.go +++ b/engine.go @@ -3,7 +3,12 @@ package fingers import ( "bufio" "bytes" + "context" "fmt" + "net/http" + "sort" + "strings" + "github.com/chainreactors/fingers/alias" "github.com/chainreactors/fingers/common" "github.com/chainreactors/fingers/ehole" @@ -11,14 +16,15 @@ import ( "github.com/chainreactors/fingers/fingerprinthub" "github.com/chainreactors/fingers/fingers" "github.com/chainreactors/fingers/goby" + "github.com/chainreactors/fingers/judge" gonmap "github.com/chainreactors/fingers/nmap" "github.com/chainreactors/fingers/resources" wappalyzer "github.com/chainreactors/fingers/wappalyzer" xrayengine "github.com/chainreactors/fingers/xray" + "github.com/chainreactors/logs" "github.com/chainreactors/utils/httputils" + "github.com/chainreactors/utils/jev" "github.com/pkg/errors" - "net/http" - "strings" ) const ( @@ -86,6 +92,10 @@ type Engine struct { *alias.Aliases Enabled map[string]bool Capabilities map[string]common.EngineCapability // 新增：记录各引擎能力 + // Judge, when set, reviews every web match: false positives and duplicate + // spellings are dropped and versions filled. EnableJudge sets it up with + // Jev; assign your own for another provider or policy. + Judge *judge.Judge } func (engine *Engine) String() string { @@ -204,6 +214,56 @@ func (engine *Engine) Disable(name string) { engine.Enabled[name] = false } +// EnableJudge makes every web match (WebMatch, Match, DetectContent, +// DetectResponse, WebMatchWithEngines) review its rule results with Jev and +// return only those the evidence supports: false positives and duplicate +// spellings are dropped, versions filled. An empty apiKey reads +// TYPESAFE_API_KEY. It sends a compact view of each matched response to +// the Jev API; if a request fails, the rule results are returned as they are. +func (engine *Engine) EnableJudge(apiKey string) error { + c, err := jev.NewClient(apiKey) + if err != nil { + return err + } + engine.EnableMatchDetail() + engine.Judge = judge.New(jev.Cached(c, jev.DefaultCacheSize)) + return nil +} + +// review applies Judge to the rule results of one response. +func (engine *Engine) review(content []byte, frames common.Frameworks) common.Frameworks { + if engine.Judge == nil || len(frames) == 0 { + return frames + } + all, err := engine.Judge.Inspect(context.Background(), content, frames) + if err != nil { + logs.Log.Debugf("judge: %v; unjudged hits are kept", err) + } + return all.Accepted() +} + +// EnableMatchDetail makes every engine that can record what a hit matched +// (fingers, fingerprinthub, wappalyzer, ehole, goby) do so in +// Framework.MatchDetail. Off by default: it costs fingers and fingerprinthub +// their keyword fast paths. +func (engine *Engine) EnableMatchDetail() { + if e := engine.Fingers(); e != nil { + e.EnableMatchDetail() + } + if e := engine.FingerPrintHub(); e != nil { + e.MatchDetailEnabled = true + } + if e := engine.Wappalyzer(); e != nil { + e.MatchDetailEnabled = true + } + if e := engine.EHole(); e != nil { + e.MatchDetailEnabled = true + } + if e := engine.Goby(); e != nil { + e.MatchDetailEnabled = true + } +} + func (engine *Engine) Fingers() *fingers.FingersEngine { if impl, ok := engine.EnginesImpl[FingersEngine]; ok { return impl.(*fingers.FingersEngine) @@ -307,11 +367,7 @@ func (engine *Engine) WebMatch(resp *http.Response) common.Frameworks { body, header, _ := httputils.SplitHttpRaw(lower) combined := make(common.Frameworks) - for name, ok := range engine.Enabled { - if !ok { - continue - } - + for _, name := range engine.enabledNames() { // Check if engine supports web fingerprinting if !engine.Capabilities[name].SupportWeb { continue @@ -346,7 +402,28 @@ func (engine *Engine) WebMatch(resp *http.Response) common.Frameworks { combined = engine.MergeFrameworks(combined, fs) } - return combined + return engine.review(content, combined) +} + +// enabledNames lists the enabled engines in AllEngines order, then any others +// by name. Merging keeps the first engine's details for a shared product, so +// a fixed order makes results reproducible. +func (engine *Engine) enabledNames() []string { + var names, extra []string + known := map[string]bool{} + for _, name := range AllEngines { + known[name] = true + if engine.Enabled[name] { + names = append(names, name) + } + } + for name, ok := range engine.Enabled { + if ok && !known[name] { + extra = append(extra, name) + } + } + sort.Strings(extra) + return append(names, extra...) } // ServiceMatch 专门用于Service指纹识别 @@ -374,7 +451,7 @@ func (engine *Engine) WebMatchWithEngines(content []byte, engines ...string) com combined = engine.MergeFrameworks(combined, fs) } } - return combined + return engine.review(content, combined) } // MatchWithEngines (deprecated, use WebMatchWithEngines instead) @@ -397,7 +474,9 @@ func (engine *Engine) MergeFrameworks(origin, other common.Frameworks) common.Fr if aliasFrame != nil { if ok { frame.Name = aliasFrame.Name - frame.UpdateAttributes(aliasFrame.ToWFN()) + // 复制一份: alias 的 Attributes 被所有页面共享, 直接引用会让一个页面写入的版本号泄漏到之后的页面 + attrs := *aliasFrame.ToWFN() + frame.UpdateAttributes(&attrs) } if aliasFrame.IsBlocked(frame.From.String()) { continue diff --git a/fingerprinthub/ac_index_test.go b/fingerprinthub/ac_index_test.go index 81b30c1..bb77a12 100644 --- a/fingerprinthub/ac_index_test.go +++ b/fingerprinthub/ac_index_test.go @@ -15,7 +15,6 @@ import ( "github.com/chainreactors/fingers/common" "github.com/chainreactors/fingers/resources" - "github.com/chainreactors/utils/httputils" ) func TestTemplateKeywordIndex_Build(t *testing.T) { @@ -64,43 +63,17 @@ func TestTemplateKeywordIndex_Match(t *testing.T) { } } -// webMatchBaseline runs the original full-iteration WebMatch logic without AC prefiltering. +// webMatchBaseline runs the full-iteration WebMatch logic without AC prefiltering. func (engine *FingerPrintHubEngine) webMatchBaseline(content []byte) common.Frameworks { - resp := httputils.NewResponseWithRaw(content) - if resp == nil { - return make(common.Frameworks) - } - - rawBody := httputils.ReadBody(resp) - rawBodyStr := string(rawBody) - event := engine.buildInternalEvent(resp, rawBodyStr, len(content)) frames := make(common.Frameworks) - + event, ok := parseRawHTTPEvent(content, engine.CaseInsensitive) + if !ok { + return frames + } for _, tmpl := range engine.webTemplates { - requests := tmpl.GetRequests() - if len(requests) == 0 { - continue - } - for _, req := range requests { - if req.Matchers == nil || len(req.Matchers) == 0 { - continue - } - matched := engine.matchRequest(req, event) - if matched { - name := tmpl.Info.Name - if name == "" { - name = tmpl.Id - } - frame := common.NewFramework(name, common.FrameFromFingerprintHub) - if tmpl.Info.Metadata != nil { - if vendor, ok := tmpl.Info.Metadata["vendor"].(string); ok { - frame.Attributes.Vendor = vendor - } - if product, ok := tmpl.Info.Metadata["product"].(string); ok { - frame.Attributes.Product = product - } - } - frames.Add(frame) + for _, req := range tmpl.requests { + if matchPassiveRequest(req, event) { + frames.Add(engine.newFramework(tmpl)) break } } diff --git a/fingerprinthub/fingerprinthub.go b/fingerprinthub/fingerprinthub.go index 89c6cd7..b1ca4fc 100644 --- a/fingerprinthub/fingerprinthub.go +++ b/fingerprinthub/fingerprinthub.go @@ -3,6 +3,7 @@ package fingerprinthub import ( "encoding/json" "fmt" + "sort" "strings" "github.com/chainreactors/fingers/common" @@ -23,6 +24,10 @@ type FingerPrintHubEngine struct { // CaseInsensitive controls whether matching ignores case (default true). CaseInsensitive bool + // MatchDetailEnabled records on each hit the snippet its matcher matched. + // Hits then skip the keyword-index fast path, which knows no snippet. + MatchDetailEnabled bool + // active holds full neutron templates for HTTPActiveMatch/ServiceMatch. // Nil in passive_only builds. active *activeState @@ -197,18 +202,19 @@ func (engine *FingerPrintHubEngine) WebMatch(content []byte) common.Frameworks { mr := engine.webTemplateIndex.Match(lowerHeaderStr, lowerBodyStr) - if engine.CaseInsensitive { - for ti := range mr.Matched { + if engine.CaseInsensitive && !engine.MatchDetailEnabled { + for _, ti := range sortedIndices(mr.Matched) { frames.Add(engine.newFramework(engine.webTemplates[ti])) } } - if !engine.CaseInsensitive { + if !engine.CaseInsensitive || engine.MatchDetailEnabled { for ti := range mr.Matched { mr.NeedsCheck[ti] = true } } - for ti := range mr.NeedsCheck { + // Templates run in index order: the first to hit a product keeps its details. + for _, ti := range sortedIndices(mr.NeedsCheck) { tmpl := engine.webTemplates[ti] if len(tmpl.requests) == 0 { continue @@ -218,8 +224,12 @@ func (engine *FingerPrintHubEngine) WebMatch(content []byte) common.Frameworks { if len(req.Matchers) == 0 { continue } - if matchPassiveRequest(req, event) { - frames.Add(engine.newFramework(tmpl)) + if matched, detail := matchPassiveRequestDetail(req, event); matched { + frame := engine.newFramework(tmpl) + if engine.MatchDetailEnabled { + frame.MatchDetail = detail + } + frames.Add(frame) break } } @@ -258,3 +268,12 @@ func activeServiceLen(engine *FingerPrintHubEngine) int { func yamlMarshal(v interface{}) ([]byte, error) { return yaml.Marshal(v) } + +func sortedIndices(set map[int]bool) []int { + out := make([]int, 0, len(set)) + for i := range set { + out = append(out, i) + } + sort.Ints(out) + return out +} diff --git a/fingerprinthub/fingerprinthub_test.go b/fingerprinthub/fingerprinthub_test.go index d0b4fc3..5399e35 100644 --- a/fingerprinthub/fingerprinthub_test.go +++ b/fingerprinthub/fingerprinthub_test.go @@ -5,8 +5,6 @@ import ( "github.com/chainreactors/fingers/resources" "github.com/chainreactors/neutron/operators" - "github.com/chainreactors/neutron/protocols" - "github.com/chainreactors/neutron/templates" "github.com/chainreactors/utils/encode" ) @@ -71,11 +69,7 @@ func TestFingerPrintHubEngine_WebMatch(t *testing.T) { // ============================================================================ func newTestEngine(caseInsensitive bool) *FingerPrintHubEngine { - return &FingerPrintHubEngine{ - CaseInsensitive: caseInsensitive, - webTemplates: make([]*templates.Template, 0), - executerOptions: &protocols.ExecuterOptions{Options: &protocols.Options{Timeout: 10}}, - } + return &FingerPrintHubEngine{CaseInsensitive: caseInsensitive} } const testHTTPRaw = "HTTP/1.1 200 OK\r\n" + @@ -240,7 +234,7 @@ func TestCaseInsensitive_AllMatchers(t *testing.T) { engine := newTestEngine(true) tmpls := buildTestTemplates() - count, errs := engine.loadTemplates(tmpls, true) + count, errs := engine.loadPassiveTemplates(tmpls) if count != len(tmpls) { t.Fatalf("loaded %d/%d, errors: %v", count, len(tmpls), errs) } @@ -251,20 +245,20 @@ func TestCaseInsensitive_AllMatchers(t *testing.T) { // CaseInsensitive=true 时，所有字面量/keywords/patterns 只要大小写无关就能匹配 expect := map[string]bool{ - "word body mixed": true, - "word body lower": true, - "word body upper": true, - "word header mixed": true, - "word header lower": true, - "word and body": true, - "regex body version": true, - "regex body mixed": false, // regex 引擎不受 CaseInsensitive 控制，pattern "Nacos" 匹配不到小写 body + "word body mixed": true, + "word body lower": true, + "word body upper": true, + "word header mixed": true, + "word header lower": true, + "word and body": true, + "regex body version": true, + "regex body mixed": false, // regex 引擎不受 CaseInsensitive 控制，pattern "Nacos" 匹配不到小写 body "regex header version": true, - "dsl body lower": true, - "dsl body mixed": false, // body 已 ToLower，"Nacos" 匹配不到 - "dsl header": true, - "status 200": true, - "combo and": true, + "dsl body lower": true, + "dsl body mixed": false, // body 已 ToLower，"<title>Nacos" 匹配不到 + "dsl header": true, + "status 200": true, + "combo and": true, } for name, shouldMatch := range expect { @@ -280,7 +274,7 @@ func TestCaseSensitive_AllMatchers(t *testing.T) { engine := newTestEngine(false) tmpls := buildTestTemplates() - count, errs := engine.loadTemplates(tmpls, true) + count, errs := engine.loadPassiveTemplates(tmpls) if count != len(tmpls) { t.Fatalf("loaded %d/%d, errors: %v", count, len(tmpls), errs) } @@ -293,20 +287,20 @@ func TestCaseSensitive_AllMatchers(t *testing.T) { // body: "<title>Nacos...Nacos v2.1.0" // header value: "Tengine/2.3.3", "Nacos-Server/2.1.0" expect := map[string]bool{ - "word body mixed": true, // "Nacos" 在原始 body 中存在（case-sensitive word 无 ToLower） - "word body lower": false, // "nacos" 不在原始 body 中 - "word body upper": false, // "<TITLE>NACOS" 不在原始 body 中 - "word header mixed": true, // "Tengine/2.3.3" 在 header value 中 - "word header lower": false, // "tengine/2.3.3" — header key 始终小写，但 value 保留原始 "Tengine/2.3.3" - "word and body": false, // "nacos" 不在原始 body (小写 n)；即使 header 匹配，AND 也失败 - "regex body version": false, // "nacos v[\d.]+" — body 中是 "Nacos v2.1.0"，小写 n 匹配不到 - "regex body mixed": true, // "Nacos v[\d.]+" 精确匹配原始大小写 + "word body mixed": true, // "Nacos" 在原始 body 中存在（case-sensitive word 无 ToLower） + "word body lower": false, // "<title>nacos" 不在原始 body 中 + "word body upper": false, // "<TITLE>NACOS" 不在原始 body 中 + "word header mixed": true, // "Tengine/2.3.3" 在 header value 中 + "word header lower": false, // "tengine/2.3.3" — header key 始终小写，但 value 保留原始 "Tengine/2.3.3" + "word and body": false, // "nacos" 不在原始 body (小写 n)；即使 header 匹配，AND 也失败 + "regex body version": false, // "nacos v[\d.]+" — body 中是 "Nacos v2.1.0"，小写 n 匹配不到 + "regex body mixed": true, // "Nacos v[\d.]+" 精确匹配原始大小写 "regex header version": false, // "tengine/[\d.]+" — header value 是 "Tengine/2.3.3" - "dsl body lower": false, // contains(body, "<title>nacos") — body 中是 Nacos - "dsl body mixed": true, // contains(body, "Nacos") 精确匹配 - "dsl header": false, // contains(all_headers, "nacos-server") — header value 保留了 "Nacos-Server" - "status 200": true, // 不受大小写影响 - "combo and": true, // status=200 ✓, "Tengine" 在 header ✓, "console-ui" 在 body ✓ + "dsl body lower": false, // contains(body, "nacos") — body 中是 Nacos + "dsl body mixed": true, // contains(body, "Nacos") 精确匹配 + "dsl header": false, // contains(all_headers, "nacos-server") — header value 保留了 "Nacos-Server" + "status 200": true, // 不受大小写影响 + "combo and": true, // status=200 ✓, "Tengine" 在 header ✓, "console-ui" 在 body ✓ } for name, shouldMatch := range expect { @@ -397,4 +391,3 @@ func TestFaviconMatcher_MatchHashValues(t *testing.T) { t.Error("Expected no match") } } - diff --git a/fingerprinthub/passive_template.go b/fingerprinthub/passive_template.go index c392405..ed4091f 100644 --- a/fingerprinthub/passive_template.go +++ b/fingerprinthub/passive_template.go @@ -6,6 +6,7 @@ import ( "strconv" "strings" + "github.com/chainreactors/fingers/common" "github.com/chainreactors/neutron/operators" "github.com/chainreactors/neutron/protocols" "github.com/chainreactors/utils/encode" @@ -146,8 +147,15 @@ func parseRawHTTPEvent(content []byte, caseInsensitive bool) (protocols.Internal } func matchPassiveRequest(req *passiveRequest, event protocols.InternalEvent) bool { + matched, _ := matchPassiveRequestDetail(req, event) + return matched +} + +// matchPassiveRequestDetail also returns the first snippet a positive matcher +// matched. +func matchPassiveRequestDetail(req *passiveRequest, event protocols.InternalEvent) (bool, *common.MatchDetail) { if len(req.Matchers) == 0 { - return false + return false, nil } matchersCondition := req.MatchersCondition @@ -156,23 +164,29 @@ func matchPassiveRequest(req *passiveRequest, event protocols.InternalEvent) boo } matchedCount := 0 - for _, matcher := range req.Matchers { - matched, _ := protocols.HTTPMatch(event, matcher) + var detail *common.MatchDetail + for i, matcher := range req.Matchers { + matched, hits := protocols.HTTPMatch(event, matcher) if matched { matchedCount++ + for _, hit := range hits { + if detail == nil && hit.Value != "" { + detail = &common.MatchDetail{MatcherType: matcher.Type, MatcherIndex: i, MatcherValue: hit.Value} + } + } if matchersCondition == "or" { - return true + return true, detail } } else { if matchersCondition == "and" { - return false + return false, nil } } } - if matchersCondition == "and" { - return matchedCount == len(req.Matchers) + if matchersCondition == "and" && matchedCount == len(req.Matchers) { + return true, detail } - return false + return false, nil } diff --git a/fingerprints/README.md b/fingerprints/README.md new file mode 100644 index 0000000..9f58202 --- /dev/null +++ b/fingerprints/README.md @@ -0,0 +1,25 @@ +# 经真实响应验证的 Jev 增量指纹 + +本目录保存通过 `judgeeval -maintain` 生成、导出、重新编译、独立样本测试及原生加载检查的原生 YAML。每个文件附带数据来源、目录审计和验证范围；默认引擎的内置压缩资源仍按原有资源流程维护。 + +## 加载 + +```go +engine, err := fingers.NewEngine(fingers.FingersEngine) +if err != nil { + return err +} +if err := engine.Fingers().LoadFromYAML("fingerprints/jev-maintained-20260926.yaml"); err != nil { + return err +} +if err := engine.Compile(); err != nil { + return err +} +hits, err := engine.DetectContent(rawHTTPResponse) +``` + +生成规则维持原生 `Finger` / `Frameworks` 语义。SearXNG 和 Wakapi 包含版本提取；IT Tools 当前按完整标题识别产品家族，覆盖原版与保留该标题的衍生版，不能区分具体分支，版本留空。 + +三个项目均经官方仓库和源码复核；“新增”仅指本次已审计的指纹库范围，不是近期发布的新软件。身份依据和分支范围见[产品真实性复核](../docs/jev-product-authenticity-20260926.md)。 + +扩展测试报告见 [Jev 指纹库维护验证](../docs/jev-maintenance-validation-20260926.md)，运行数据和完整响应保存在本地 `.judge-data/expansion-20260926/`。这些指纹的结论仅适用于报告中列出的版本、样本和标注范围。 diff --git a/fingerprints/jev-maintained-20260926.provenance.json b/fingerprints/jev-maintained-20260926.provenance.json new file mode 100644 index 0000000..4fc8daf --- /dev/null +++ b/fingerprints/jev-maintained-20260926.provenance.json @@ -0,0 +1,4298 @@ +{ + "batch": "expansion-20260926", + "scope": "Products and versions supported by archived HTTP evidence; not an audit of installed software. Host-isolated regression set reused during development, not an untouched external benchmark.", + "model": "jev/jev-1.13.0", + "library": "jev-maintained-20260926.yaml", + "library_sha256": "522a85806592e10bdffa87ccb70be90e22098fdeba85cc52452235132520065a", + "final_run": ".judge-data/expansion-20260926/run-07-final", + "offline_run": ".judge-data/expansion-20260926/run-08-offline", + "loaded_library_run": ".judge-data/expansion-20260926/run-06-with-library", + "new_capture": { + "attempts": 53, + "successes": 34, + "failures": 19 + }, + "corpus": { + "samples": 69, + "unique_hosts": 59, + "unique_responses": 65, + "labelled_pairs": 682 + }, + "manifest_sha256": "699557708fdec2d6f0404c3c0f37978a6b01d984cdebf3e4b01eae6feba0cfb3", + "target_selection": { + "selection": "SearXNG official-directory seeded sample plus project demos and search-indexed public installations; docs are mention-only negative controls", + "seed": 20260926, + "batch1": "targets-batch1.txt", + "batch2": "targets-batch2.txt" + }, + "sources": [ + { + "name": "searxng-instances", + "url": "https://searx.space/data/instances.json", + "status": 200, + "bytes": 413354, + "file": ".judge-data\\expansion-20260926\\searxng-instances.json", + "sha256": "e17431b01ccb60b9885139be4d0ecd9f5baf0f45ce2bc169501fd4c8a3495a4f" + }, + { + "name": "wakapi-readme", + "url": "https://raw.githubusercontent.com/muety/wakapi/master/README.md", + "status": 200, + "bytes": 58370, + "file": ".judge-data\\expansion-20260926\\wakapi-readme.md", + "sha256": "626b3574cb6d1732ecd8ae76a0b0969a8a83993022dea67b1be43385eb3a38d0" + }, + { + "name": "pairdrop-readme", + "url": "https://raw.githubusercontent.com/schlagmichdoch/PairDrop/master/README.md", + "status": 200, + "bytes": 7156, + "file": ".judge-data\\expansion-20260926\\pairdrop-readme.md", + "sha256": "fe1fcf4cb27f732477160c00ec0c9313bcdfa95bed89f482a5dda12c6d24cac7" + }, + { + "name": "gatus-readme", + "url": "https://raw.githubusercontent.com/TwiN/gatus/master/README.md", + "status": 200, + "bytes": 215037, + "file": ".judge-data\\expansion-20260926\\gatus-readme.md", + "sha256": "e1156f61879dcac5dcb3cc5931f5ae542aa757cc2c4973507664fe69661121ad" + }, + { + "name": "karakeep-readme", + "url": "https://raw.githubusercontent.com/karakeep-app/karakeep/main/README.md", + "status": 200, + "bytes": 8186, + "file": ".judge-data\\expansion-20260926\\karakeep-readme.md", + "sha256": "ecc3cd2eb9a38a919bea7ccb71cd9e83c2dfc01d03601a70879433e0d3f326d9" + } + ], + "catalogs": [ + { + "source": "fingers", + "sha256": "770208a0f55ab01d6d983f1ff304e501ab8c717d7812d4ad774518b39c00d1f5", + "names": 4289 + }, + { + "source": "goby", + "sha256": "b1cdffe4eca50c0140ee27ddebf23ec007a3337b9a3f528c889d704ad0218fe1", + "names": 7083 + }, + { + "source": "ehole", + "sha256": "0e673c5569b04c3ffb3eed8410048807a8103a62a4e7e500e445ec2e8d2193af", + "names": 811 + }, + { + "source": "wappalyzer", + "sha256": "18b47fc3deec61c8260f1f1da9efe8301d292cc0f14f0f57885181482eed71b9", + "names": 7226 + }, + { + "source": "fingerprinthub", + "sha256": "e4b2faa806e032e09b72c6e17d5a63e72f9a80449d6aedee82ee88138213a4c6", + "names": 3483 + }, + { + "source": "aliases", + "sha256": "24316f010fd85c9d8d7d68d1a9241e10f770851cbfc939c2f78124ae9291dc6b", + "names": 87 + }, + { + "source": "runtime-fingers", + "sha256": "3f8303600efb5aa5e4ce668f418e4ac5c46b7e029643f2bc2d216872638c47d3", + "names": 4289 + } + ], + "maintenance": { + "library": "library.yaml", + "catalog_file": "catalog-before.json", + "min_positive_hosts": 2, + "min_negative_hosts": 5, + "products": [ + { + "plan": "searxng-new", + "product": "SearXNG", + "status": "validated_library", + "existing_catalog_matches": null, + "positive_samples": 10, + "suggested_positives": 10, + "unknown_positives": 9, + "holdout_positive_hosts": 8, + "holdout_negative_hosts": 42, + "before": { + "detection": { + "tp": 0, + "fp": 0, + "tn": 47, + "fn": 8 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 8 + }, + "unlabelled_predictions": 0 + }, + "after": { + "detection": { + "tp": 8, + "fp": 0, + "tn": 47, + "fn": 0 + }, + "versions": { + "correct": 8, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "unlabelled_predictions": 0 + }, + "test_ids": [ + "36f6645c0290996f", + "145e314623e76dfd", + "7a415cdbb6244358", + "e3651c67dbb2ad17", + "718e4c943a25a995", + "c31d254bc1fd0230", + "e7d23d9f3c44fee1", + "867e79ba27f05078", + "bc537dc669808db3", + "e13843bffacca3e0", + "6025845b2aa21eb7", + "f288433c7ec32303", + "51ccb99538406996", + "1700c6017ed82aaf", + "f7039343f41ed237", + "7c56b28279b128c0", + "afac5614625bfc6f", + "d79b4325363f7010", + "f349b9e30f493cb5", + "b8c2c3d64f1947fa", + "087a771998e8460b", + "05346dbcf28946f0", + "0a87b2d06772daf3", + "2ee69d30ad1b9ba1", + "6290a947cc8369e7", + "846e9f943798d3ab", + "86e96862d2e6618c", + "335745c8550b1062", + "ebc3a07dd3e76b6e", + "fb25cf0e86ffcd7d", + "fa1b267c4911f7a0", + "33481799a00eb89a", + "e80f19b7afe3d734", + "e173c09a7713b5b0", + "3ee248ced1fa920a", + "65b15b71fa6f508f", + "a8f7be21e03253d3", + "2b89e57a9b28bfbf", + "d9b5eaa66bdc7bec", + "b2382b7ca202c344", + "9a8481abc3453d0b", + "8e2c3affdcca9c97", + "bfba0e0e48f5eb54", + "97544c226c577151", + "68eb99ccb687369d", + "24c03d50574b79f2", + "302f5cbbc6e1596b", + "a04da6570878fb03", + "09e85a53a7b4850d", + "e331de721c086202", + "0dabb37b2a7b3687", + "91315c96894dcfd3", + "fba6e296e0acdccc", + "e693fe9346caa44c", + "28276d6b56102882" + ] + }, + { + "plan": "wakapi-new", + "product": "Wakapi", + "status": "validated_library", + "existing_catalog_matches": null, + "positive_samples": 4, + "suggested_positives": 4, + "unknown_positives": 4, + "holdout_positive_hosts": 3, + "holdout_negative_hosts": 48, + "before": { + "detection": { + "tp": 0, + "fp": 0, + "tn": 53, + "fn": 3 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 3 + }, + "unlabelled_predictions": 0 + }, + "after": { + "detection": { + "tp": 3, + "fp": 0, + "tn": 53, + "fn": 0 + }, + "versions": { + "correct": 3, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "unlabelled_predictions": 0 + }, + "test_ids": [ + "36f6645c0290996f", + "145e314623e76dfd", + "7a415cdbb6244358", + "e3651c67dbb2ad17", + "718e4c943a25a995", + "c31d254bc1fd0230", + "e7d23d9f3c44fee1", + "867e79ba27f05078", + "bc537dc669808db3", + "e13843bffacca3e0", + "6025845b2aa21eb7", + "f288433c7ec32303", + "51ccb99538406996", + "1700c6017ed82aaf", + "f7039343f41ed237", + "7c56b28279b128c0", + "afac5614625bfc6f", + "d79b4325363f7010", + "f349b9e30f493cb5", + "b8c2c3d64f1947fa", + "087a771998e8460b", + "05346dbcf28946f0", + "0a87b2d06772daf3", + "2ee69d30ad1b9ba1", + "6290a947cc8369e7", + "846e9f943798d3ab", + "86e96862d2e6618c", + "335745c8550b1062", + "ebc3a07dd3e76b6e", + "fb25cf0e86ffcd7d", + "fa1b267c4911f7a0", + "33481799a00eb89a", + "e80f19b7afe3d734", + "e173c09a7713b5b0", + "7c727a84546a6bba", + "3ee248ced1fa920a", + "65b15b71fa6f508f", + "a8f7be21e03253d3", + "2b89e57a9b28bfbf", + "d9b5eaa66bdc7bec", + "b2382b7ca202c344", + "9a8481abc3453d0b", + "a40e20d96d920f67", + "8e2c3affdcca9c97", + "bfba0e0e48f5eb54", + "97544c226c577151", + "68eb99ccb687369d", + "24c03d50574b79f2", + "a04da6570878fb03", + "09e85a53a7b4850d", + "e331de721c086202", + "0dabb37b2a7b3687", + "91315c96894dcfd3", + "fba6e296e0acdccc", + "e693fe9346caa44c", + "28276d6b56102882" + ] + }, + { + "plan": "it-tools-new", + "product": "IT Tools", + "status": "validated_library", + "existing_catalog_matches": null, + "positive_samples": 3, + "suggested_positives": 3, + "unknown_positives": 0, + "holdout_positive_hosts": 2, + "holdout_negative_hosts": 49, + "before": { + "detection": { + "tp": 0, + "fp": 0, + "tn": 54, + "fn": 2 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 2 + }, + "unlabelled_predictions": 0 + }, + "after": { + "detection": { + "tp": 2, + "fp": 0, + "tn": 54, + "fn": 0 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 2, + "unsupported": 0, + "product_missing": 0 + }, + "unlabelled_predictions": 0 + }, + "test_ids": [ + "36f6645c0290996f", + "145e314623e76dfd", + "7a415cdbb6244358", + "e3651c67dbb2ad17", + "718e4c943a25a995", + "c31d254bc1fd0230", + "e7d23d9f3c44fee1", + "867e79ba27f05078", + "bc537dc669808db3", + "e13843bffacca3e0", + "6025845b2aa21eb7", + "f288433c7ec32303", + "51ccb99538406996", + "1700c6017ed82aaf", + "f7039343f41ed237", + "7c56b28279b128c0", + "afac5614625bfc6f", + "d79b4325363f7010", + "f349b9e30f493cb5", + "b8c2c3d64f1947fa", + "087a771998e8460b", + "05346dbcf28946f0", + "0a87b2d06772daf3", + "2ee69d30ad1b9ba1", + "6290a947cc8369e7", + "846e9f943798d3ab", + "86e96862d2e6618c", + "335745c8550b1062", + "ebc3a07dd3e76b6e", + "fb25cf0e86ffcd7d", + "fa1b267c4911f7a0", + "33481799a00eb89a", + "e80f19b7afe3d734", + "7c727a84546a6bba", + "3ee248ced1fa920a", + "65b15b71fa6f508f", + "a8f7be21e03253d3", + "2b89e57a9b28bfbf", + "d9b5eaa66bdc7bec", + "b2382b7ca202c344", + "9a8481abc3453d0b", + "a40e20d96d920f67", + "8e2c3affdcca9c97", + "bfba0e0e48f5eb54", + "97544c226c577151", + "68eb99ccb687369d", + "24c03d50574b79f2", + "302f5cbbc6e1596b", + "a04da6570878fb03", + "09e85a53a7b4850d", + "e331de721c086202", + "0dabb37b2a7b3687", + "91315c96894dcfd3", + "fba6e296e0acdccc", + "e693fe9346caa44c", + "28276d6b56102882" + ] + }, + { + "plan": "pairdrop-new", + "product": "PairDrop", + "status": "insufficient_holdout", + "existing_catalog_matches": null, + "positive_samples": 1, + "suggested_positives": 1, + "unknown_positives": 0, + "holdout_positive_hosts": 0, + "holdout_negative_hosts": 51, + "before": { + "detection": { + "tp": 0, + "fp": 0, + "tn": 0, + "fn": 0 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "unlabelled_predictions": 0 + }, + "after": { + "detection": { + "tp": 0, + "fp": 0, + "tn": 0, + "fn": 0 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "unlabelled_predictions": 0 + }, + "test_ids": [ + "36f6645c0290996f", + "145e314623e76dfd", + "7a415cdbb6244358", + "e3651c67dbb2ad17", + "718e4c943a25a995", + "c31d254bc1fd0230", + "e7d23d9f3c44fee1", + "867e79ba27f05078", + "bc537dc669808db3", + "e13843bffacca3e0", + "6025845b2aa21eb7", + "f288433c7ec32303", + "51ccb99538406996", + "1700c6017ed82aaf", + "f7039343f41ed237", + "7c56b28279b128c0", + "afac5614625bfc6f", + "d79b4325363f7010", + "f349b9e30f493cb5", + "b8c2c3d64f1947fa", + "087a771998e8460b", + "05346dbcf28946f0", + "0a87b2d06772daf3", + "2ee69d30ad1b9ba1", + "6290a947cc8369e7", + "846e9f943798d3ab", + "86e96862d2e6618c", + "335745c8550b1062", + "ebc3a07dd3e76b6e", + "fb25cf0e86ffcd7d", + "fa1b267c4911f7a0", + "33481799a00eb89a", + "e80f19b7afe3d734", + "e173c09a7713b5b0", + "7c727a84546a6bba", + "3ee248ced1fa920a", + "a8f7be21e03253d3", + "2b89e57a9b28bfbf", + "d9b5eaa66bdc7bec", + "b2382b7ca202c344", + "9a8481abc3453d0b", + "a40e20d96d920f67", + "8e2c3affdcca9c97", + "bfba0e0e48f5eb54", + "97544c226c577151", + "68eb99ccb687369d", + "24c03d50574b79f2", + "302f5cbbc6e1596b", + "a04da6570878fb03", + "09e85a53a7b4850d", + "e331de721c086202", + "0dabb37b2a7b3687", + "91315c96894dcfd3", + "fba6e296e0acdccc", + "e693fe9346caa44c", + "28276d6b56102882" + ] + }, + { + "plan": "docmost-new", + "product": "Docmost", + "status": "insufficient_holdout", + "existing_catalog_matches": null, + "positive_samples": 1, + "suggested_positives": 1, + "unknown_positives": 1, + "holdout_positive_hosts": 0, + "holdout_negative_hosts": 51, + "before": { + "detection": { + "tp": 0, + "fp": 0, + "tn": 0, + "fn": 0 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "unlabelled_predictions": 0 + }, + "after": { + "detection": { + "tp": 0, + "fp": 0, + "tn": 0, + "fn": 0 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "unlabelled_predictions": 0 + }, + "test_ids": [ + "36f6645c0290996f", + "145e314623e76dfd", + "7a415cdbb6244358", + "e3651c67dbb2ad17", + "718e4c943a25a995", + "c31d254bc1fd0230", + "e7d23d9f3c44fee1", + "867e79ba27f05078", + "bc537dc669808db3", + "e13843bffacca3e0", + "6025845b2aa21eb7", + "f288433c7ec32303", + "51ccb99538406996", + "1700c6017ed82aaf", + "f7039343f41ed237", + "7c56b28279b128c0", + "afac5614625bfc6f", + "d79b4325363f7010", + "f349b9e30f493cb5", + "b8c2c3d64f1947fa", + "087a771998e8460b", + "05346dbcf28946f0", + "0a87b2d06772daf3", + "2ee69d30ad1b9ba1", + "6290a947cc8369e7", + "846e9f943798d3ab", + "86e96862d2e6618c", + "335745c8550b1062", + "ebc3a07dd3e76b6e", + "fa1b267c4911f7a0", + "33481799a00eb89a", + "e80f19b7afe3d734", + "e173c09a7713b5b0", + "7c727a84546a6bba", + "3ee248ced1fa920a", + "65b15b71fa6f508f", + "a8f7be21e03253d3", + "2b89e57a9b28bfbf", + "d9b5eaa66bdc7bec", + "b2382b7ca202c344", + "9a8481abc3453d0b", + "a40e20d96d920f67", + "8e2c3affdcca9c97", + "bfba0e0e48f5eb54", + "97544c226c577151", + "68eb99ccb687369d", + "24c03d50574b79f2", + "302f5cbbc6e1596b", + "a04da6570878fb03", + "09e85a53a7b4850d", + "e331de721c086202", + "0dabb37b2a7b3687", + "91315c96894dcfd3", + "fba6e296e0acdccc", + "e693fe9346caa44c", + "28276d6b56102882" + ] + }, + { + "plan": "linkwarden-new", + "product": "Linkwarden", + "status": "insufficient_holdout", + "existing_catalog_matches": null, + "positive_samples": 1, + "suggested_positives": 1, + "unknown_positives": 0, + "holdout_positive_hosts": 0, + "holdout_negative_hosts": 51, + "before": { + "detection": { + "tp": 0, + "fp": 0, + "tn": 0, + "fn": 0 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "unlabelled_predictions": 0 + }, + "after": { + "detection": { + "tp": 0, + "fp": 0, + "tn": 0, + "fn": 0 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "unlabelled_predictions": 0 + }, + "test_ids": [ + "36f6645c0290996f", + "145e314623e76dfd", + "7a415cdbb6244358", + "e3651c67dbb2ad17", + "718e4c943a25a995", + "c31d254bc1fd0230", + "e7d23d9f3c44fee1", + "867e79ba27f05078", + "bc537dc669808db3", + "e13843bffacca3e0", + "6025845b2aa21eb7", + "f288433c7ec32303", + "51ccb99538406996", + "1700c6017ed82aaf", + "f7039343f41ed237", + "7c56b28279b128c0", + "afac5614625bfc6f", + "d79b4325363f7010", + "f349b9e30f493cb5", + "b8c2c3d64f1947fa", + "087a771998e8460b", + "05346dbcf28946f0", + "0a87b2d06772daf3", + "2ee69d30ad1b9ba1", + "6290a947cc8369e7", + "846e9f943798d3ab", + "86e96862d2e6618c", + "335745c8550b1062", + "ebc3a07dd3e76b6e", + "fb25cf0e86ffcd7d", + "fa1b267c4911f7a0", + "33481799a00eb89a", + "e173c09a7713b5b0", + "7c727a84546a6bba", + "3ee248ced1fa920a", + "65b15b71fa6f508f", + "a8f7be21e03253d3", + "2b89e57a9b28bfbf", + "d9b5eaa66bdc7bec", + "b2382b7ca202c344", + "9a8481abc3453d0b", + "a40e20d96d920f67", + "8e2c3affdcca9c97", + "bfba0e0e48f5eb54", + "97544c226c577151", + "68eb99ccb687369d", + "24c03d50574b79f2", + "302f5cbbc6e1596b", + "a04da6570878fb03", + "09e85a53a7b4850d", + "e331de721c086202", + "0dabb37b2a7b3687", + "91315c96894dcfd3", + "fba6e296e0acdccc", + "e693fe9346caa44c", + "28276d6b56102882" + ] + }, + { + "plan": "karakeep-new", + "product": "Karakeep", + "status": "insufficient_holdout", + "existing_catalog_matches": null, + "positive_samples": 3, + "suggested_positives": 3, + "unknown_positives": 0, + "holdout_positive_hosts": 0, + "holdout_negative_hosts": 51, + "before": { + "detection": { + "tp": 0, + "fp": 0, + "tn": 0, + "fn": 0 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "unlabelled_predictions": 0 + }, + "after": { + "detection": { + "tp": 0, + "fp": 0, + "tn": 0, + "fn": 0 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "unlabelled_predictions": 0 + }, + "test_ids": [ + "36f6645c0290996f", + "145e314623e76dfd", + "7a415cdbb6244358", + "e3651c67dbb2ad17", + "718e4c943a25a995", + "c31d254bc1fd0230", + "e7d23d9f3c44fee1", + "867e79ba27f05078", + "bc537dc669808db3", + "e13843bffacca3e0", + "6025845b2aa21eb7", + "f288433c7ec32303", + "51ccb99538406996", + "1700c6017ed82aaf", + "f7039343f41ed237", + "7c56b28279b128c0", + "afac5614625bfc6f", + "d79b4325363f7010", + "f349b9e30f493cb5", + "b8c2c3d64f1947fa", + "087a771998e8460b", + "05346dbcf28946f0", + "0a87b2d06772daf3", + "2ee69d30ad1b9ba1", + "6290a947cc8369e7", + "846e9f943798d3ab", + "86e96862d2e6618c", + "335745c8550b1062", + "ebc3a07dd3e76b6e", + "fb25cf0e86ffcd7d", + "fa1b267c4911f7a0", + "33481799a00eb89a", + "e80f19b7afe3d734", + "e173c09a7713b5b0", + "7c727a84546a6bba", + "3ee248ced1fa920a", + "65b15b71fa6f508f", + "a8f7be21e03253d3", + "2b89e57a9b28bfbf", + "d9b5eaa66bdc7bec", + "b2382b7ca202c344", + "9a8481abc3453d0b", + "a40e20d96d920f67", + "8e2c3affdcca9c97", + "bfba0e0e48f5eb54", + "97544c226c577151", + "68eb99ccb687369d", + "302f5cbbc6e1596b", + "a04da6570878fb03", + "e331de721c086202", + "0dabb37b2a7b3687", + "91315c96894dcfd3", + "e693fe9346caa44c", + "28276d6b56102882" + ] + }, + { + "plan": "new-api-new", + "product": "New API", + "status": "insufficient_holdout", + "existing_catalog_matches": null, + "positive_samples": 1, + "suggested_positives": 1, + "unknown_positives": 1, + "holdout_positive_hosts": 0, + "holdout_negative_hosts": 51, + "before": { + "detection": { + "tp": 0, + "fp": 0, + "tn": 0, + "fn": 0 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "unlabelled_predictions": 0 + }, + "after": { + "detection": { + "tp": 0, + "fp": 0, + "tn": 0, + "fn": 0 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "unlabelled_predictions": 0 + }, + "test_ids": [ + "36f6645c0290996f", + "145e314623e76dfd", + "7a415cdbb6244358", + "e3651c67dbb2ad17", + "718e4c943a25a995", + "c31d254bc1fd0230", + "e7d23d9f3c44fee1", + "867e79ba27f05078", + "bc537dc669808db3", + "e13843bffacca3e0", + "6025845b2aa21eb7", + "f288433c7ec32303", + "51ccb99538406996", + "1700c6017ed82aaf", + "f7039343f41ed237", + "7c56b28279b128c0", + "afac5614625bfc6f", + "d79b4325363f7010", + "f349b9e30f493cb5", + "b8c2c3d64f1947fa", + "087a771998e8460b", + "0a87b2d06772daf3", + "2ee69d30ad1b9ba1", + "6290a947cc8369e7", + "846e9f943798d3ab", + "86e96862d2e6618c", + "335745c8550b1062", + "ebc3a07dd3e76b6e", + "fb25cf0e86ffcd7d", + "fa1b267c4911f7a0", + "33481799a00eb89a", + "e80f19b7afe3d734", + "e173c09a7713b5b0", + "7c727a84546a6bba", + "3ee248ced1fa920a", + "65b15b71fa6f508f", + "a8f7be21e03253d3", + "2b89e57a9b28bfbf", + "d9b5eaa66bdc7bec", + "b2382b7ca202c344", + "9a8481abc3453d0b", + "a40e20d96d920f67", + "8e2c3affdcca9c97", + "bfba0e0e48f5eb54", + "97544c226c577151", + "68eb99ccb687369d", + "24c03d50574b79f2", + "302f5cbbc6e1596b", + "a04da6570878fb03", + "09e85a53a7b4850d", + "e331de721c086202", + "0dabb37b2a7b3687", + "91315c96894dcfd3", + "fba6e296e0acdccc", + "e693fe9346caa44c", + "28276d6b56102882" + ] + } + ], + "lost_existing_hits": null, + "changed_existing_versions": null, + "integration_errors": null + }, + "generation": [ + { + "name": "searxng-new", + "product": "searxng", + "status": "passed_holdout", + "file": "fingerprints/searxng-new.yaml", + "detection": { + "tp": 8, + "fp": 0, + "tn": 47, + "fn": 0 + }, + "versions": { + "correct": 8, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "training_detection": { + "tp": 2, + "fp": 0, + "tn": 7, + "fn": 0 + }, + "training_versions": { + "correct": 2, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "excluded": [ + "9b0d62c13f476614", + "aaef32bb77101a86", + "45c65575b6da19ac", + "1a125b61fc8b7dd7", + "acede3e046de943e" + ] + }, + { + "name": "wakapi-new", + "product": "Wakapi", + "status": "passed_holdout", + "file": "fingerprints/wakapi-new.yaml", + "detection": { + "tp": 3, + "fp": 0, + "tn": 53, + "fn": 0 + }, + "versions": { + "correct": 3, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "training_detection": { + "tp": 1, + "fp": 0, + "tn": 7, + "fn": 0 + }, + "training_versions": { + "correct": 1, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "excluded": [ + "9b0d62c13f476614", + "aaef32bb77101a86", + "45c65575b6da19ac", + "1a125b61fc8b7dd7", + "acede3e046de943e" + ] + }, + { + "name": "it-tools-new", + "product": "IT Tools", + "status": "passed_holdout", + "file": "fingerprints/it-tools-new.yaml", + "detection": { + "tp": 2, + "fp": 0, + "tn": 54, + "fn": 0 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 2, + "unsupported": 0, + "product_missing": 0 + }, + "training_detection": { + "tp": 1, + "fp": 0, + "tn": 7, + "fn": 0 + }, + "training_versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 1, + "unsupported": 0, + "product_missing": 0 + }, + "excluded": [ + "9b0d62c13f476614", + "aaef32bb77101a86", + "45c65575b6da19ac", + "1a125b61fc8b7dd7", + "acede3e046de943e" + ] + }, + { + "name": "pairdrop-new", + "product": "PairDrop", + "status": "insufficient_holdout", + "file": "fingerprints/pairdrop-new.yaml", + "detection": { + "tp": 0, + "fp": 0, + "tn": 56, + "fn": 0 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "training_detection": { + "tp": 1, + "fp": 0, + "tn": 7, + "fn": 0 + }, + "training_versions": { + "correct": 1, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "excluded": [ + "9b0d62c13f476614", + "aaef32bb77101a86", + "45c65575b6da19ac", + "1a125b61fc8b7dd7", + "acede3e046de943e" + ] + }, + { + "name": "docmost-new", + "product": "Docmost", + "status": "insufficient_holdout", + "file": "fingerprints/docmost-new.yaml", + "detection": { + "tp": 0, + "fp": 0, + "tn": 56, + "fn": 0 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "training_detection": { + "tp": 1, + "fp": 0, + "tn": 7, + "fn": 0 + }, + "training_versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 1, + "unsupported": 0, + "product_missing": 0 + }, + "excluded": [ + "9b0d62c13f476614", + "aaef32bb77101a86", + "45c65575b6da19ac", + "1a125b61fc8b7dd7", + "acede3e046de943e" + ] + }, + { + "name": "linkwarden-new", + "product": "Linkwarden", + "status": "insufficient_holdout", + "file": "fingerprints/linkwarden-new.yaml", + "detection": { + "tp": 0, + "fp": 0, + "tn": 56, + "fn": 0 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "training_detection": { + "tp": 1, + "fp": 0, + "tn": 7, + "fn": 0 + }, + "training_versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 1, + "unsupported": 0, + "product_missing": 0 + }, + "excluded": [ + "9b0d62c13f476614", + "aaef32bb77101a86", + "45c65575b6da19ac", + "1a125b61fc8b7dd7", + "acede3e046de943e" + ] + }, + { + "name": "karakeep-new", + "product": "Karakeep", + "status": "insufficient_holdout", + "file": "fingerprints/karakeep-new.yaml", + "detection": { + "tp": 0, + "fp": 0, + "tn": 54, + "fn": 0 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "training_detection": { + "tp": 1, + "fp": 0, + "tn": 7, + "fn": 0 + }, + "training_versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 1, + "unsupported": 0, + "product_missing": 0 + }, + "excluded": [ + "9b0d62c13f476614", + "aaef32bb77101a86", + "45c65575b6da19ac", + "1a125b61fc8b7dd7", + "acede3e046de943e", + "24c03d50574b79f2", + "09e85a53a7b4850d" + ] + }, + { + "name": "new-api-new", + "product": "New API", + "status": "insufficient_holdout", + "file": "fingerprints/new-api-new.yaml", + "detection": { + "tp": 0, + "fp": 0, + "tn": 56, + "fn": 0 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "training_detection": { + "tp": 1, + "fp": 0, + "tn": 7, + "fn": 0 + }, + "training_versions": { + "correct": 1, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "excluded": [ + "9b0d62c13f476614", + "aaef32bb77101a86", + "45c65575b6da19ac", + "1a125b61fc8b7dd7", + "acede3e046de943e" + ] + } + ], + "repeated_maintenance": [ + { + "product": "SearXNG", + "status": "already_catalogued", + "existing_catalog_matches": [ + "runtime-fingers:searxng" + ] + }, + { + "product": "Wakapi", + "status": "already_catalogued", + "existing_catalog_matches": [ + "runtime-fingers:Wakapi" + ] + }, + { + "product": "IT Tools", + "status": "already_catalogued", + "existing_catalog_matches": [ + "runtime-fingers:IT Tools" + ] + }, + { + "product": "PairDrop", + "status": "insufficient_holdout", + "existing_catalog_matches": null + }, + { + "product": "Docmost", + "status": "insufficient_holdout", + "existing_catalog_matches": null + }, + { + "product": "Linkwarden", + "status": "insufficient_holdout", + "existing_catalog_matches": null + }, + { + "product": "Karakeep", + "status": "insufficient_holdout", + "existing_catalog_matches": null + }, + { + "product": "New API", + "status": "insufficient_holdout", + "existing_catalog_matches": null + } + ], + "loaded_library_cleanup": { + "model": "jev/jev-1.13.0", + "baseline_source": "current_rules_on_saved_responses", + "samples": 69, + "unique_hosts": 59, + "unique_responses": 65, + "errors": 0, + "labelled_pairs": 682, + "baseline": { + "detection": { + "tp": 63, + "fp": 24, + "tn": 586, + "fn": 9 + }, + "versions": { + "correct": 17, + "wrong": 0, + "missing": 20, + "correct_abstentions": 26, + "unsupported": 0, + "product_missing": 9 + }, + "unlabelled_predictions": 301 + }, + "refined": { + "detection": { + "tp": 63, + "fp": 0, + "tn": 610, + "fn": 9 + }, + "versions": { + "correct": 23, + "wrong": 0, + "missing": 14, + "correct_abstentions": 26, + "unsupported": 0, + "product_missing": 9 + }, + "unlabelled_predictions": 148 + }, + "completed": { + "detection": { + "tp": 63, + "fp": 0, + "tn": 610, + "fn": 9 + }, + "versions": { + "correct": 37, + "wrong": 0, + "missing": 0, + "correct_abstentions": 26, + "unsupported": 0, + "product_missing": 9 + }, + "unlabelled_predictions": 148 + }, + "false_hits_removed": 24, + "true_hits_removed": 0, + "natural_misses_recovered": 0, + "missed_products_suggested": 8, + "version_fills": 27, + "correct_version_fills": 20, + "incorrect_version_fills": 0, + "unlabelled_version_fills": 7, + "provider_requests": 306, + "exact_cache_hits": 181, + "input_tokens": 741618 + }, + "offline": { + "provider_requests": 0, + "cache_hits": 483, + "same_semantic_metrics": true, + "identical_files": { + "library.yaml": true, + "generation.json": true, + "maintenance.json": true, + "rows.jsonl": true, + "baseline.jsonl": true, + "cleaned.jsonl": true + } + }, + "label_revisions": [ + { + "product": "PairDrop", + "id": "65b15b71fa6f508f", + "previous_version": "", + "corrected_version": "1.11.2", + "basis": "Adjacent PairDrop product heading and visible release declaration in the original saved HTML; not inferred from a provider answer.", + "evidence": "
v1.11.2
" + } + ], + "samples": [ + { + "id": "36f6645c0290996f", + "url": "http://47.239.11.1/", + "group": "47.239.11.1", + "captured_at": "2026-09-25T14:18:17.927604+00:00", + "sha256": "01c9933b1ea5a0ffbb8ccfc911a548f4c6c53500d2f2fffe08aa54ad00524859", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "145e314623e76dfd", + "url": "http://47.239.11.117/", + "group": "47.239.11.117", + "captured_at": "2026-09-25T14:18:17.927604+00:00", + "sha256": "6f4b55cad227274d1acbf3575a76308a395588c51e84c41790d6e5eb9a580634", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "7a415cdbb6244358", + "url": "http://47.239.11.119/", + "group": "47.239.11.119", + "captured_at": "2026-09-25T14:18:17.928110+00:00", + "sha256": "3094c7260747f35b8f46b973da195908f64734bf5fa138bc1fe3335c7058bc7d", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "e3651c67dbb2ad17", + "url": "http://47.239.11.120/", + "group": "47.239.11.120", + "captured_at": "2026-09-25T14:18:17.928110+00:00", + "sha256": "58cf9c0f520cd99dfd8377e28181b7b26b4e35a0cdf56c93fac2613dd683b1f4", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "718e4c943a25a995", + "url": "http://47.239.11.121/", + "group": "47.239.11.121", + "captured_at": "2026-09-25T14:18:17.926596+00:00", + "sha256": "9cf03440fe1d5d60261b8ef9434d05b2dd7702ec789ff6cb289b41fa4753aa55", + "product_labels": [ + { + "product": "New API", + "present": false + }, + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + } + ] + }, + { + "id": "c31d254bc1fd0230", + "url": "http://47.239.11.134/", + "group": "47.239.11.134", + "captured_at": "2026-09-25T14:18:17.928110+00:00", + "sha256": "dffc9b4b13160201cb54898cbe83629a654b1aa91f7139f02ce0f0c44672815e", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "e7d23d9f3c44fee1", + "url": "http://47.239.11.151/", + "group": "47.239.11.151", + "captured_at": "2026-09-25T14:18:17.970662+00:00", + "sha256": "2e39b279754a9dcc613fd51f66647804701ca795715d6b3e4c7a5f6898ce9a91", + "product_labels": [ + { + "product": "New API", + "present": false + }, + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + } + ] + }, + { + "id": "867e79ba27f05078", + "url": "http://47.239.11.151/robots.txt", + "group": "47.239.11.151", + "captured_at": "2026-09-25T14:18:17.922561+00:00", + "sha256": "d920d770512d4605376b8269e3bddb42b2a301f6791829edc1f03bb0ed4009ca", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "bc537dc669808db3", + "url": "http://47.239.11.155/", + "group": "47.239.11.155", + "captured_at": "2026-09-25T14:18:18.168005+00:00", + "sha256": "8ee30984aa2b1cf9ed38f45ce449ed75cc4f09b0d51622285f3cf8f8039f6102", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "e13843bffacca3e0", + "url": "http://47.239.11.156/", + "group": "47.239.11.156", + "captured_at": "2026-09-25T14:18:18.038424+00:00", + "sha256": "073eb56c353211f2e01597d85a12496d14f1f710590442978303d7ea4a43ff55", + "product_labels": [ + { + "product": "New API", + "present": false + }, + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + } + ] + }, + { + "id": "6025845b2aa21eb7", + "url": "http://47.239.11.169/", + "group": "47.239.11.169", + "captured_at": "2026-09-25T14:18:18.052902+00:00", + "sha256": "b7905ab6f47917366f3412c5588cc7388d722a9377d7d69929affc8181438070", + "product_labels": [ + { + "product": "New API", + "present": false + }, + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + } + ] + }, + { + "id": "f288433c7ec32303", + "url": "http://47.239.11.22/", + "group": "47.239.11.22", + "captured_at": "2026-09-25T14:18:18.044381+00:00", + "sha256": "64bc64f6243ee12859cdd7428e3a9fb6543007c38889410d519308154dfb0f39", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "51ccb99538406996", + "url": "http://47.239.11.226/", + "group": "47.239.11.226", + "captured_at": "2026-09-25T14:18:18.044381+00:00", + "sha256": "e25d8751b8b9e8a4d3358a67b9f5dec50b4bb8a78203531023c084f5805f10c3", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "1700c6017ed82aaf", + "url": "http://47.239.11.25/", + "group": "47.239.11.25", + "captured_at": "2026-09-25T14:18:18.096093+00:00", + "sha256": "73a98b2721ffcbee6c24860eb35d77699b8078a44feda6bba403f431884b023f", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "9b0d62c13f476614", + "url": "http://47.239.11.25/robots.txt", + "group": "47.239.11.25", + "captured_at": "2026-09-25T14:18:18.040055+00:00", + "sha256": "d920d770512d4605376b8269e3bddb42b2a301f6791829edc1f03bb0ed4009ca", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "f7039343f41ed237", + "url": "http://47.239.11.26/", + "group": "47.239.11.26", + "captured_at": "2026-09-25T14:18:18.068707+00:00", + "sha256": "c618035e4ce2e29cd00081e9ee5c1d44a21e29e2e45bb0776b4e8a28157905fd", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "7c56b28279b128c0", + "url": "http://47.239.11.31/", + "group": "47.239.11.31", + "captured_at": "2026-09-25T14:18:18.503310+00:00", + "sha256": "8bb71aca1f485344ddcab8b24fb60d0d3024674b374c042755f7afc8afdfefcc", + "product_labels": [ + { + "product": "New API", + "present": false + }, + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + } + ] + }, + { + "id": "afac5614625bfc6f", + "url": "http://47.239.11.37/", + "group": "47.239.11.37", + "captured_at": "2026-09-25T14:18:18.227548+00:00", + "sha256": "964afc12b6a7cf669d5e5cc1c273817243e8556cdda804288f568fa527b23949", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "d79b4325363f7010", + "url": "http://47.239.11.81/", + "group": "47.239.11.81", + "captured_at": "2026-09-25T14:18:18.135189+00:00", + "sha256": "80a65f5096d6484405f30628730b819e71b006bbc076629bd219b090a8bd0210", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "aaef32bb77101a86", + "url": "http://47.239.11.88/", + "group": "47.239.11.88", + "captured_at": "2026-09-25T14:18:18.145815+00:00", + "sha256": "b6c6c656c2ab14ae11e9a193fbceec59e00f55c2dd144c8855e2a484a1c7822e", + "product_labels": [ + { + "product": "New API", + "present": false + }, + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + } + ] + }, + { + "id": "f0bd3986b1f949f9", + "url": "http://47.239.11.9/", + "group": "47.239.11.9", + "captured_at": "2026-09-25T14:18:18.360498+00:00", + "sha256": "ef6be733b856d9360d413b70c68c1e9215d721abd47719c44a42fc0c8c098ca5", + "product_labels": [ + { + "product": "New API", + "present": false + }, + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + } + ] + }, + { + "id": "45c65575b6da19ac", + "url": "http://47.239.11.9/robots.txt", + "group": "47.239.11.9", + "captured_at": "2026-09-25T14:18:18.362502+00:00", + "sha256": "1b09dc6a8fda8fc5d9542cfd51c0d117bf8920247024f967258cddc94e28ac31", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "f349b9e30f493cb5", + "url": "http://47.239.11.90/", + "group": "47.239.11.90", + "captured_at": "2026-09-25T14:18:18.204350+00:00", + "sha256": "20fbdce2fbcf59605ceaac8c5b2abd19ffe9f221d06e16492f10a406abcd3ca8", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "1a125b61fc8b7dd7", + "url": "http://47.239.11.98/", + "group": "47.239.11.98", + "captured_at": "2026-09-25T14:18:18.238181+00:00", + "sha256": "01c9933b1ea5a0ffbb8ccfc911a548f4c6c53500d2f2fffe08aa54ad00524859", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "b8c2c3d64f1947fa", + "url": "https://47.239.11.1/", + "group": "47.239.11.1", + "captured_at": "2026-09-25T14:18:18.293071+00:00", + "sha256": "0bc50587714d9b9dd614f10d14e5c189753e82c3556c52510386cddb790db595", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "087a771998e8460b", + "url": "https://47.239.11.107/", + "group": "47.239.11.107", + "captured_at": "2026-09-25T14:18:18.408116+00:00", + "sha256": "fb18c6b1cbd965735c819dd7cf481ae3a43ee56fc594653c9e9275658be337c7", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "05346dbcf28946f0", + "url": "https://47.239.11.123/", + "group": "47.239.11.123", + "captured_at": "2026-09-25T14:18:18.740314+00:00", + "sha256": "321ddf27f33ffe2d6c5a2cf46912a37532afbaa09e6ec95e836ab17cbf4d160e", + "product_labels": [ + { + "product": "New API", + "present": true, + "version": "1.0.0-rc.35" + }, + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + } + ] + }, + { + "id": "0a87b2d06772daf3", + "url": "https://47.239.11.167/", + "group": "47.239.11.167", + "captured_at": "2026-09-25T14:18:18.389966+00:00", + "sha256": "b8dc990fcf6c2da2a577a434b663d77e38e3e4121e2f81409f2e2d2678237c1c", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "2ee69d30ad1b9ba1", + "url": "https://47.239.11.189/", + "group": "47.239.11.189", + "captured_at": "2026-09-25T14:18:18.391967+00:00", + "sha256": "fa3774df099c5131e279010cb96c2405ad9504fad1f24b65e436e312e2f30def", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "6290a947cc8369e7", + "url": "https://47.239.11.196/", + "group": "47.239.11.196", + "captured_at": "2026-09-25T14:18:18.617358+00:00", + "sha256": "bcdc6b21d56ce3e1cdbb410eaa16817cafd2c308946da41a7faf001e45894689", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "877f11f543fab972", + "url": "https://47.239.11.201/", + "group": "47.239.11.201", + "captured_at": "2026-09-25T14:18:19.980167+00:00", + "sha256": "19094cdce76f6db2b41180c2b886d83fb8f4565845e01b3745016e2009d1630b", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "846e9f943798d3ab", + "url": "https://47.239.11.205/", + "group": "47.239.11.205", + "captured_at": "2026-09-25T14:18:18.571434+00:00", + "sha256": "53daeaa2e944eebe9763104e20f2f87744fb88105ab21d542fa1961611a53f53", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "86e96862d2e6618c", + "url": "https://47.239.11.31/", + "group": "47.239.11.31", + "captured_at": "2026-09-25T14:18:18.585630+00:00", + "sha256": "6610cfe1cb5781ca48d4ee66d9f770161894e181288d0eab100bc3b6ef3a5c27", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "acede3e046de943e", + "url": "https://47.239.11.42/", + "group": "47.239.11.42", + "captured_at": "2026-09-25T14:18:18.667816+00:00", + "sha256": "bcdc6b21d56ce3e1cdbb410eaa16817cafd2c308946da41a7faf001e45894689", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "335745c8550b1062", + "url": "https://47.239.11.56/", + "group": "47.239.11.56", + "captured_at": "2026-09-25T14:18:18.620892+00:00", + "sha256": "30cabb65837840a343e98d1f32fe73aeab525ca76ea5575002e0bd6be348f15e", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "ebc3a07dd3e76b6e", + "url": "https://auth.seddens.net/?rd=https://search.seddens.net/", + "group": "auth.seddens.net", + "captured_at": "2026-09-25T16:02:44.182709+00:00", + "sha256": "0a1e86f1d97559a345f2330854612ba467fca23fec7bc9fbaf4e8fb3360e383d", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "fb25cf0e86ffcd7d", + "url": "https://demo.docmost.com/", + "group": "demo.docmost.com", + "captured_at": "2026-09-25T16:02:44.705457+00:00", + "sha256": "af7efc632705f747e3678d8d5433d985d38e4265b988d1254d243cc9075fd8bc", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": true, + "version": "" + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "fa1b267c4911f7a0", + "url": "https://demo.gatus.io/", + "group": "demo.gatus.io", + "captured_at": "2026-09-25T16:02:44.046979+00:00", + "sha256": "80fca347a00ecf6b8447090e38b86772bd3e2496c1c1ebcb232e63d7860bca43", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "33481799a00eb89a", + "url": "https://demo.immich.app/", + "group": "demo.immich.app", + "captured_at": "2026-09-25T16:02:46.028922+00:00", + "sha256": "b0285892e15d87f89f5cbf0069b787bd76cfdf263c2f55fd8ba95317293062fa", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "e80f19b7afe3d734", + "url": "https://demo.linkwarden.app/", + "group": "demo.linkwarden.app", + "captured_at": "2026-09-25T16:02:44.753890+00:00", + "sha256": "f8003078dff0c362781ba26cac30b46b773bdd0cbbad92c64043ae7a380e4f04", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": true, + "version": "" + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "e173c09a7713b5b0", + "url": "https://it-tools.tech/", + "group": "it-tools.tech", + "captured_at": "2026-09-25T16:02:46.944395+00:00", + "sha256": "aa5123f269ffc06cb6c267e61d0928c812757cd79dd45644488217947d0dcf7e", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": true, + "version": "" + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "7c727a84546a6bba", + "url": "https://libresearch.space/", + "group": "libresearch.space", + "captured_at": "2026-09-25T16:02:47.592975+00:00", + "sha256": "933c9c1838412c7c2e28a93995c700dca04d59072197db72b8402b580fda848e", + "product_labels": [ + { + "product": "SearXNG", + "present": true, + "version": "2026.9.23+3cd69d30e" + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "3ee248ced1fa920a", + "url": "https://linkding.link/demo/", + "group": "linkding.link", + "captured_at": "2026-09-25T16:02:47.193117+00:00", + "sha256": "cbef1330a4de2184b8500644dfc2da1c219482b5f73bdcd96868569b48559d28", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "65b15b71fa6f508f", + "url": "https://pairdrop.net/", + "group": "pairdrop.net", + "captured_at": "2026-09-25T16:02:48.304311+00:00", + "sha256": "544859141ca31286fd7ad3396b7e4111cd40df06e37af8d6ef8c700400da753b", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": true, + "version": "1.11.2" + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "a8f7be21e03253d3", + "url": "https://privacyredirect.com/", + "group": "privacyredirect.com", + "captured_at": "2026-09-25T16:02:48.791402+00:00", + "sha256": "2862ecff85a029752274e65e36f1cd975aeb5aa882263c3c51324357e9469bb3", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "2b89e57a9b28bfbf", + "url": "https://search.liuzj.net/", + "group": "search.liuzj.net", + "captured_at": "2026-09-25T16:02:50.042863+00:00", + "sha256": "a0729317e8d6efc1761808e2bcfa6aa3d0f1dc012b59b2083e12ba273b92ef52", + "product_labels": [ + { + "product": "SearXNG", + "present": true, + "version": "2026.9.23+3cd69d30e" + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "d9b5eaa66bdc7bec", + "url": "https://search.minus27315.dev/", + "group": "search.minus27315.dev", + "captured_at": "2026-09-25T16:02:49.494654+00:00", + "sha256": "1ec95f79589ea486f8b229f5465e3eb7cfe526a7c6bb027d0cb5e46d7a8a74c6", + "product_labels": [ + { + "product": "SearXNG", + "present": true, + "version": "2026.9.23+3cd69d30e" + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "b2382b7ca202c344", + "url": "https://search.mizuki.st/", + "group": "search.mizuki.st", + "captured_at": "2026-09-25T16:02:50.068114+00:00", + "sha256": "f052900e00a4761ffeaf166c20476d3ace37e9273399fc9da4cdefffd62400d7", + "product_labels": [ + { + "product": "SearXNG", + "present": true, + "version": "2026.9.25+487f51922" + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "9a8481abc3453d0b", + "url": "https://search.serpensin.com/", + "group": "search.serpensin.com", + "captured_at": "2026-09-25T16:02:50.801354+00:00", + "sha256": "d44354e6a04543536be6657dc9cf6455545623186be9254cb337a861b04bdb3a", + "product_labels": [ + { + "product": "SearXNG", + "present": true, + "version": "2026.9.25+d8ae3abd5" + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "a40e20d96d920f67", + "url": "https://search.undertale.uk/", + "group": "search.undertale.uk", + "captured_at": "2026-09-25T16:02:51.784166+00:00", + "sha256": "12631af1a6b39d5f1ae3cdf9b4aa4375aded4402bbd51d6de97a968406ae7503", + "product_labels": [ + { + "product": "SearXNG", + "present": true, + "version": "2026.9.12+d4f00d15" + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "8e2c3affdcca9c97", + "url": "https://searx.linxx.net/", + "group": "searx.linxx.net", + "captured_at": "2026-09-25T16:02:52.064348+00:00", + "sha256": "6a1d74981c9c690fc68166125f25f7e6f1dd591701e3e3728323d62324ddc082", + "product_labels": [ + { + "product": "SearXNG", + "present": true, + "version": "2026.9.23+3cd69d30e" + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "bfba0e0e48f5eb54", + "url": "https://searx.ononoki.org/", + "group": "searx.ononoki.org", + "captured_at": "2026-09-25T16:02:52.089914+00:00", + "sha256": "1cfe7a92e63295863796002845a691a81d27ca89ed5d69812d7f8add32366b9f", + "product_labels": [ + { + "product": "SearXNG", + "present": true, + "version": "2026.9.25+d8ae3abd5" + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "97544c226c577151", + "url": "https://searxng.paralaxitaentomology.org/", + "group": "searxng.paralaxitaentomology.org", + "captured_at": "2026-09-25T16:02:54.679780+00:00", + "sha256": "6ab60af098c988d1edae89ecc9e20f209dd93169d9e0e0a656bc71a4f6234d88", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "68eb99ccb687369d", + "url": "https://searxng.website/", + "group": "searxng.website", + "captured_at": "2026-09-25T16:02:55.674788+00:00", + "sha256": "6c2450e80e7b0a6ebfad345dd75604703a27e93a0223de4410d03e4e68561720", + "product_labels": [ + { + "product": "SearXNG", + "present": true, + "version": "2026.9.25+1667fe364" + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "24c03d50574b79f2", + "url": "https://try.karakeep.app/", + "group": "karakeep.app", + "captured_at": "2026-09-25T16:02:57.656703+00:00", + "sha256": "277c85209de80220e5e51e2967bd67b7bf52b3be2f6573ef9dae1b4173bda6c8", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": true, + "version": "" + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "302f5cbbc6e1596b", + "url": "https://wakapi.dev/", + "group": "wakapi.dev", + "captured_at": "2026-09-25T16:02:56.614578+00:00", + "sha256": "94421406e8ac6431d018d9df8c32896cf57d41bf4a4df2f9c0012a504e5e90e5", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": true, + "version": "2.18.0-be6aeb4" + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "a04da6570878fb03", + "url": "https://www.isci.si/", + "group": "www.isci.si", + "captured_at": "2026-09-25T16:02:57.538684+00:00", + "sha256": "adc7023023f3c2afe2d1da333a77e5b211f738afc3625e264b9e9ad632660275", + "product_labels": [ + { + "product": "SearXNG", + "present": true, + "version": "2026.9.23+3cd69d30e" + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "09e85a53a7b4850d", + "url": "https://cloud.karakeep.app/signin", + "group": "karakeep.app", + "captured_at": "2026-09-25T16:05:50.702001+00:00", + "sha256": "77074091f1d359b4b834054eef368cd0e94bacb7363b21294b452d9d3984159b", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": true, + "version": "" + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "e331de721c086202", + "url": "https://codingstats.realzone22.de/", + "group": "codingstats.realzone22.de", + "captured_at": "2026-09-25T16:05:50.303704+00:00", + "sha256": "731deda0f5c0eb0bea2e4f97fd9ca41bbc3de1941606b0ce8c234fda4c2de8cf", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": true, + "version": "2.18.0" + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "8dd6d80787caae8a", + "url": "https://docs.searxng.org/", + "group": "docs.searxng.org", + "captured_at": "2026-09-25T16:05:51.770395+00:00", + "sha256": "66862f9d00640d603cfd43a50910f74006234fbdbeb66a30dfe92db1a5385755", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "fbd19c030c2a721b", + "url": "https://raw.githubusercontent.com/CorentinTh/it-tools/main/README.md", + "group": "raw.githubusercontent.com", + "captured_at": "2026-09-25T16:05:49.692493+00:00", + "sha256": "1a3a4c3665beaa75cbe790e4bb88fecea7353e278274290fd1df15bbb3a0bec1", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "0a5bca2c75194b89", + "url": "https://raw.githubusercontent.com/docmost/docmost/main/README.md", + "group": "raw.githubusercontent.com", + "captured_at": "2026-09-25T16:05:50.626716+00:00", + "sha256": "5dbb7ec0f3cf18b6225a366e424dff7d90de9c7fee00faf600f23b17f74dd768", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "81a3a968cecaaa27", + "url": "https://raw.githubusercontent.com/muety/wakapi/master/README.md", + "group": "raw.githubusercontent.com", + "captured_at": "2026-09-25T16:05:50.376033+00:00", + "sha256": "747e660c59224621d7ad89f29c77833d9d6ecd16b43908f81b8814b8a56a876b", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "ed7fbc949fd767e2", + "url": "https://raw.githubusercontent.com/schlagmichdoch/PairDrop/master/README.md", + "group": "raw.githubusercontent.com", + "captured_at": "2026-09-25T16:05:50.874980+00:00", + "sha256": "4e96b5c538e4d47f4b68e8c09dbc372093949f30bc0ef7e8529510fe24a49b1e", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "0dabb37b2a7b3687", + "url": "https://tools.code.pro.vn/", + "group": "tools.code.pro.vn", + "captured_at": "2026-09-25T16:05:54.693801+00:00", + "sha256": "4908ff993998339aefc32b3a5c1517a9fe83dc2e5a1e9c4988b5a8033b7cc69d", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": true, + "version": "" + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "91315c96894dcfd3", + "url": "https://tools.quitw.org/", + "group": "tools.quitw.org", + "captured_at": "2026-09-25T16:05:52.563429+00:00", + "sha256": "69d5c4c23f61a2403f372dc297964d7f5303b41568702a8c4f1fddca3df5c16f", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": true, + "version": "" + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "fba6e296e0acdccc", + "url": "https://try.karakeep.app/signin", + "group": "karakeep.app", + "captured_at": "2026-09-25T16:05:53.634556+00:00", + "sha256": "94544d1086d031186e6accc94b82c73f76d01a872c67fe98f592aaa417608bf0", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": true, + "version": "" + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "e693fe9346caa44c", + "url": "https://wakapi.anneraud.fr/", + "group": "wakapi.anneraud.fr", + "captured_at": "2026-09-25T16:05:53.080022+00:00", + "sha256": "c9f9b05fdbc2da2234c1f152dab418ae6ffeeaafdab6677e44f2f3a814166763", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": true, + "version": "2.18.0" + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "28276d6b56102882", + "url": "https://wakapi.m1000.fr/", + "group": "wakapi.m1000.fr", + "captured_at": "2026-09-25T16:05:53.920039+00:00", + "sha256": "bb456f874905c48d870d5571c445128554c3d04fa8dc8a643cd6cf22f1a61865", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": true, + "version": "2.12.0" + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + } + ], + "source_revision": "2d296928c8847bbc60dbfedad77e68bbfea00d14", + "source_worktree_dirty": true, + "code_sha256": { + "judge/cache.go": "5c80af5ec1e0fc210ce650282f9f38568b7c0323e841426d9150980753a6c345", + "judge/classify.go": "60471102fecea7e10bbeac81ec61ed65dc7f503721f7f6452986c303ffb39de5", + "judge/discovery.go": "b9281f9eceac0e09ff4d93f06557450dc17446956a7b5b686914880954bdcacd", + "judge/doc.go": "505a382e87b75d706bfd7962088fe62d4c408597bc41920bfe00e2e1d711fbae", + "judge/generator.go": "8fc8537de9c6aaf233efafc12090e9d9db387154f5ee01a0ebc263e7a3aa3745", + "judge/judge.go": "56ec5d8da51b4edf5bbaa3b2f76b7b80da1926cba7695094f1ce918e8d903535", + "judge/judge_test.go": "6ae85fd3db383235fa76b53fe18e8c5be6d8117420b6cbd68d089d9101eba8a3", + "judge/mark.go": "6c8a7be884737668f54399acb2efd7c262fb5f4bed25183c490ab12561432f56", + "judge/operations.go": "ae5fb0f3b655e008c40ceafcf9c454c4148e654b1badbceeff8454158d83b57c", + "judge/operations_test.go": "417525756c47ff90487b3184d8628b95865894363075839035d8d97d13653777", + "judge/page.go": "d935eba4d1f80d4ac9747e0c7da5090270dcb5a886c1df73ff1f363bcc4a2029", + "judge/provider.go": "722dcdc8bb7339444cb9b831aa1c6f4e2f7f89ead5f7786aa1f9de26c97c6c92", + "judge/real_evidence_test.go": "ccb2e1ee43ade119ac4229f69507ab027ab37b8bd76004c148750016b5ab586a", + "judge/recall.go": "1f27353e28455daf593aa3cacccc46a158bfe41c9f70611cc46f906386bc5b68", + "judge/refine.go": "94cbd144fe3d8694cc2d1ea2fa0770e8ae977f7fdacfc8865a9dad8f3d5915bb", + "judge/round.go": "5750ca5939311685ec7db1d16db1437d7a07cba3d953d7c9ef7381197aa22593", + "judge/stack.go": "a513102b88af5d1fa0e91bb661cb8518b03e9b0dfc4141885082b600376331bf", + "judge/verify.go": "92a1be4b91eeb3553154ec7977c3b087914112b09c06d3421e029b5c31e303f5", + "judge/version.go": "ade768f2f2c93761d2769487c106758f7139707bd548a5aaaeecdcd021f911a1", + "cmd/judgeeval/main.go": "bc401d84036b9beee951bdac2bbe38098b54ecc33eb6c36815b201070d577afa", + "cmd/judgeeval/maintenance.go": "12e80ffd99b5e91a47a1bc72e991190396731b358a1604e19151adf19e6d7574", + "cmd/judgeeval/maintenance_test.go": "0640e8764a867cb5429613df3e05347b1da8ffa717aa21887565e4d97fe6d2a1", + "cmd/judgeeval/replay.go": "1f1a470d3bef3ce1936205407db8a5d42ec893f27fb43c8824aa6340f0f5aa6f", + "cmd/judgeeval/replay_generation.go": "9792c010021381863237d7ca204df024c308339bf8b4387f949db0600cb53e66", + "cmd/judgeeval/replay_metrics.go": "1c1db0deaf9bfa9e87527e83b2cbadf6c96c770aa958f13764ccdeff39f95d5f", + "cmd/judgeeval/replay_test.go": "9a10f9f6ea907f34f2b99d283b85378a2b86f9d0ae97ff84a4e4b45e44891168" + } +} diff --git a/fingerprints/jev-maintained-20260926.yaml b/fingerprints/jev-maintained-20260926.yaml new file mode 100644 index 0000000..f6e49a6 --- /dev/null +++ b/fingerprints/jev-maintained-20260926.yaml @@ -0,0 +1,47 @@ +- name: IT Tools + part: "" + vendor: "" + product: "" + protocol: http + default_port: + - "80" + rule: + - version: _ + regexps: + body: + - it tools - handy online tools for developers +- name: Wakapi + part: "" + vendor: "" + product: "" + protocol: http + default_port: + - "80" + rule: + - version: _ + regexps: + regexp: + - (?s)wakapi – coding statistics.*?wakapi – coding statistics.*?\s+v([0-9]{1,4}(?:\.[0-9]{1,4}){1,3}(?:[A-Za-z]+[0-9]*|[-+][0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*)?)(?:[^0-9A-Za-z.+-]|$) + - (?s)wakapi – coding statistics.*?wakapi – coding statistics.*?wakapi – coding statistics +- name: searxng + part: "" + vendor: "" + product: "" + protocol: http + default_port: + - "80" + rule: + - version: _ + regexps: + regexp: + - Welcome to nginx!

Welcome to nginx!

If you see this page, the nginx web server is successfully installed.

"), }, { - Name: "wordpress-site", + Name: "wordpress-site", Content: []byte("HTTP/1.1 200 OK\r\nServer: Apache/2.4.41\r\nX-Powered-By: PHP/7.4.3\r\nLink: ; rel=\"https://api.w.org/\"\r\n\r\n

Example Blog

Hello World

Welcome to WordPress.

Powered by WordPress

"), }, { - Name: "spring-boot-app", + Name: "spring-boot-app", Content: []byte("HTTP/1.1 200 OK\r\nServer: Apache-Coyote/1.1\r\nContent-Type: application/json\r\nX-Application-Context: application:8080\r\n\r\n{\"status\":\"UP\",\"components\":{\"db\":{\"status\":\"UP\",\"details\":{\"database\":\"MySQL\",\"validationQuery\":\"isValid()\"}},\"diskSpace\":{\"status\":\"UP\",\"details\":{\"total\":107374182400,\"free\":53687091200,\"threshold\":10485760,\"path\":\"/opt/app/.\",\"exists\":true}}},\"groups\":[\"liveness\",\"readiness\"]}"), }, { - Name: "tomcat-default", + Name: "tomcat-default", Content: []byte("HTTP/1.1 200 OK\r\nServer: Apache-Coyote/1.1\r\nContent-Type: text/html;charset=UTF-8\r\n\r\nApache Tomcat/9.0.50

Apache Tomcat/9.0.50

If you're seeing this, you've successfully installed Tomcat.

  • Server Status
  • Host Manager
"), }, { - Name: "iis-aspnet", + Name: "iis-aspnet", Content: []byte("HTTP/1.1 200 OK\r\nServer: Microsoft-IIS/10.0\r\nX-Powered-By: ASP.NET\r\nX-AspNet-Version: 4.0.30319\r\n\r\nIIS Windows Server\"IIS\"

Internet Information Services

"), }, { - Name: "large-react-app", + Name: "large-react-app", Content: generateLargeResponse(64 * 1024), }, { - Name: "very-large-response", + Name: "very-large-response", Content: generateLargeResponse(256 * 1024), }, { - Name: "empty-body", + Name: "empty-body", Content: []byte("HTTP/1.1 302 Found\r\nServer: cloudflare\r\nLocation: /login\r\nCF-RAY: abc123\r\n\r\n"), }, } @@ -671,7 +669,7 @@ func TestContentSizeScaling(t *testing.T) { const iterations = 50 t.Logf("%-10s %15s %15s %10s", "Size", "Baseline", "AC", "Speedup") - t.Logf(strings.Repeat("-", 55)) + t.Log(strings.Repeat("-", 55)) for _, size := range sizes { raw := generateLargeResponse(size) @@ -746,255 +744,3 @@ func TestACBuildCost(t *testing.T) { t.Logf("Single AC match time: %s, hits: %d", matchTime, len(results)) t.Logf("Build amortized over 1000 matches: %s per match", buildTime/1000) } - -// ────────────────────────────────────────────────────────────── -// 10. Go stdlib regexp vs go-re2 (C wrapper) comparison -// ────────────────────────────────────────────────────────────── - -func TestGoRegexpVsRE2(t *testing.T) { - if testing.Short() { - t.Skip("skipping heavy performance comparison in short mode") - } - engine := newPerfEngine(t) - - type patternPair struct { - goRe *regexp.Regexp - re2Re *re2.Regexp - src string - } - - var patterns []patternPair - for _, finger := range engine.HTTPFingers { - for _, rule := range finger.Rules { - if rule.Regexps == nil { - continue - } - for _, r := range rule.Regexps.CompliedRegexp { - src := r.String() - goRe, err := regexp.Compile(src) - if err != nil { - continue - } - re2Re, err := re2.Compile(src) - if err != nil { - t.Logf("SKIP re2 compile %q: %v", src, err) - continue - } - patterns = append(patterns, patternPair{goRe: goRe, re2Re: re2Re, src: src}) - } - for _, r := range rule.Regexps.CompiledVulnRegexp { - src := r.String() - goRe, err := regexp.Compile(src) - if err != nil { - continue - } - re2Re, err := re2.Compile(src) - if err != nil { - t.Logf("SKIP re2 compile %q: %v", src, err) - continue - } - patterns = append(patterns, patternPair{goRe: goRe, re2Re: re2Re, src: src}) - } - } - } - t.Logf("Compiled %d patterns for both engines", len(patterns)) - - sizes := []int{1024, 8192, 32768, 131072} - const iterations = 20 - - for _, size := range sizes { - content := NewContent(generateLargeResponse(size), "", true) - - // Go stdlib - runtime.GC() - goStart := time.Now() - goHits := 0 - for i := 0; i < iterations; i++ { - for _, p := range patterns { - if p.goRe.Match(content.Content) { - goHits++ - } - } - } - goElapsed := time.Since(goStart) / time.Duration(iterations) - - // go-re2 - runtime.GC() - re2Start := time.Now() - re2Hits := 0 - for i := 0; i < iterations; i++ { - for _, p := range patterns { - if p.re2Re.Match(content.Content) { - re2Hits++ - } - } - } - re2Elapsed := time.Since(re2Start) / time.Duration(iterations) - - speedup := float64(goElapsed) / float64(re2Elapsed) - t.Logf("[%dB] go_regexp=%s(%d) re2=%s(%d) speedup=%.2fx", - size, goElapsed, goHits/iterations, re2Elapsed, re2Hits/iterations, speedup) - } -} - -func BenchmarkRegexp_RE2(b *testing.B) { - engine := newPerfEngine(b) - var re2Regexps []*re2.Regexp - for _, finger := range engine.HTTPFingers { - for _, rule := range finger.Rules { - if rule.Regexps == nil { - continue - } - for _, r := range rule.Regexps.CompliedRegexp { - re2r, err := re2.Compile(r.String()) - if err != nil { - continue - } - re2Regexps = append(re2Regexps, re2r) - } - for _, r := range rule.Regexps.CompiledVulnRegexp { - re2r, err := re2.Compile(r.String()) - if err != nil { - continue - } - re2Regexps = append(re2Regexps, re2r) - } - } - } - - content := NewContent(generateLargeResponse(32*1024), "", true) - - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - for _, r := range re2Regexps { - r.Match(content.Content) - } - } -} - -// ────────────────────────────────────────────────────────────── -// 11. Combined AC + RE2 full optimization -// ────────────────────────────────────────────────────────────── - -func TestCombinedOptimization(t *testing.T) { - if testing.Short() { - t.Skip("skipping heavy performance comparison in short mode") - } - engine := newPerfEngine(t) - acIdx := buildACIndex(engine.HTTPFingers) - - // Pre-compile re2 versions of all regex patterns - type re2RulePatterns struct { - regexps []*re2.Regexp - vulnRegexps []*re2.Regexp - } - re2Cache := make(map[*Rule]*re2RulePatterns) - for _, finger := range engine.HTTPFingers { - for _, rule := range finger.Rules { - if rule.Regexps == nil { - continue - } - rp := &re2RulePatterns{} - for _, r := range rule.Regexps.CompliedRegexp { - re2r, err := re2.Compile(r.String()) - if err == nil { - rp.regexps = append(rp.regexps, re2r) - } - } - for _, r := range rule.Regexps.CompiledVulnRegexp { - re2r, err := re2.Compile(r.String()) - if err == nil { - rp.vulnRegexps = append(rp.vulnRegexps, re2r) - } - } - re2Cache[rule] = rp - } - } - - sites := generateSyntheticResponses() - realSites := loadTestSites(t) - sites = append(sites, realSites...) - - iterations := 50 - for _, site := range sites { - if len(site.Content) > 300000 { - iterations = 10 - } - - content := NewContent(site.Content, "", true) - - // Baseline - runtime.GC() - baseStart := time.Now() - var baseFrames int - for i := 0; i < iterations; i++ { - fs, _ := engine.HTTPMatch(site.Content, "") - baseFrames = len(fs) - } - baseElapsed := time.Since(baseStart) / time.Duration(iterations) - - // AC + RE2 combined - runtime.GC() - optStart := time.Now() - var optFrames int - for i := 0; i < iterations; i++ { - // Phase 1: AC keyword scan - acResults := acIdx.matchKeywords(content.Header, content.Body) - matched := make(map[int]bool) - for _, r := range acResults { - matched[r.FingerIdx] = true - } - - // Phase 2: RE2 regex scan for fingerprints with regex-only rules - for fi, finger := range engine.HTTPFingers { - if matched[fi] { - continue - } - for _, rule := range finger.Rules { - if rule.Regexps == nil { - continue - } - rp := re2Cache[rule] - if rp == nil { - continue - } - hit := false - for _, r := range rp.vulnRegexps { - if r.Match(content.Content) { - hit = true - break - } - } - if !hit { - for _, r := range rp.regexps { - if r.Match(content.Content) { - hit = true - break - } - } - } - if hit { - matched[fi] = true - break - } - } - } - - count := 0 - for fi := range matched { - finger := engine.HTTPFingers[fi] - _, _, ok := finger.PassiveMatch(content) - if ok { - count++ - } - } - optFrames = count - } - optElapsed := time.Since(optStart) / time.Duration(iterations) - - speedup := float64(baseElapsed) / float64(optElapsed) - t.Logf("[%s] %dB: baseline=%s(%d) AC+RE2=%s(%d) speedup=%.2fx", - site.Name, len(site.Content), baseElapsed, baseFrames, optElapsed, optFrames, speedup) - } -} diff --git a/fingers/regexp_default.go b/fingers/regexp_default.go deleted file mode 100644 index e00c96a..0000000 --- a/fingers/regexp_default.go +++ /dev/null @@ -1,10 +0,0 @@ -//go:build goregexp -// +build goregexp - -package fingers - -import "regexp" - -func compileRegexp(s string) (CompiledRegexp, error) { - return regexp.Compile(s) -} diff --git a/fingers/regexp_iface.go b/fingers/regexp_iface.go index f2e09e5..caa348d 100644 --- a/fingers/regexp_iface.go +++ b/fingers/regexp_iface.go @@ -1,8 +1,20 @@ package fingers +import "regexp" + type CompiledRegexp interface { FindSubmatch(b []byte) [][]byte FindAllString(s string, n int) []string Match(b []byte) bool String() string } + +// RegexpCompiler compiles the regexps of fingerprint rules. It is the +// standard library's by default; importing github.com/chainreactors/fingers/re2 +// switches it to RE2, which is faster on large rule sets. Set it before +// fingerprints are loaded. +var RegexpCompiler = func(expr string) (CompiledRegexp, error) { + return regexp.Compile(expr) +} + +func compileRegexp(s string) (CompiledRegexp, error) { return RegexpCompiler(s) } diff --git a/fingers/regexp_re2.go b/fingers/regexp_re2.go deleted file mode 100644 index ae73457..0000000 --- a/fingers/regexp_re2.go +++ /dev/null @@ -1,10 +0,0 @@ -//go:build !goregexp -// +build !goregexp - -package fingers - -import re2 "github.com/wasilibs/go-re2" - -func compileRegexp(s string) (CompiledRegexp, error) { - return re2.Compile(s) -} diff --git a/go.mod b/go.mod index bad3acb..36664d4 100644 --- a/go.mod +++ b/go.mod @@ -1,48 +1,40 @@ module github.com/chainreactors/fingers -go 1.24.0 - -toolchain go1.24.3 +go 1.17 require ( github.com/chainreactors/logs v0.0.0-20260508055944-c678762ed15c github.com/chainreactors/utils v0.0.0-20260629043228-93bdd2142c9a github.com/stretchr/testify v1.10.0 - golang.org/x/net v0.21.0 + golang.org/x/net v0.17.0 ) require ( - github.com/bahlo/generic-list-go v0.2.0 // indirect - github.com/buger/jsonparser v1.1.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/facebookincubator/nvdtools v0.1.5 // indirect github.com/go-dedup/megophone v0.0.0-20170830025436-f01be21026f5 // indirect github.com/go-dedup/simhash v0.0.0-20170904020510-9ecaca7b509c // indirect github.com/go-dedup/text v0.0.0-20170907015346-8bb1b95e3cb7 // indirect github.com/hashicorp/go-version v1.6.0 // indirect - github.com/mailru/easyjson v0.7.7 // indirect + github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect + github.com/kr/text v0.2.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect - github.com/tetratelabs/wazero v1.11.0 // indirect github.com/twmb/murmur3 v1.1.8 // indirect - github.com/wasilibs/wazero-helpers v0.0.0-20250123031827-cd30c44769bb // indirect - github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect - golang.org/x/sys v0.38.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect ) require ( github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible github.com/chainreactors/neutron v0.1.1-0.20260714062907-716c6b167cb6 - github.com/chainreactors/utils/parsers v0.0.3 + github.com/chainreactors/utils/jev v0.0.0-20260927031300-6bf027367a50 + github.com/chainreactors/utils/parsers v0.0.4-0.20260927032755-fd113e94d6c0 github.com/chainreactors/words v0.0.0-20260520145736-270600e60fb4 github.com/dlclark/regexp2 v1.11.5 - github.com/invopop/jsonschema v0.13.0 - github.com/jessevdk/go-flags v1.6.1 + github.com/invopop/jsonschema v0.7.0 + github.com/jessevdk/go-flags v1.5.0 github.com/mozillazg/go-pinyin v0.20.0 github.com/pkg/errors v0.9.1 - github.com/wasilibs/go-re2 v1.6.0 gopkg.in/yaml.v3 v3.0.1 ) - -replace github.com/wasilibs/go-re2 => github.com/chainreactors/go-re2 v1.11.1-0.20260716082604-8121b6cd261e diff --git a/go.sum b/go.sum index 3f6ae0c..8db9777 100644 --- a/go.sum +++ b/go.sum @@ -64,29 +64,25 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= -github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= -github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chainreactors/go-re2 v1.11.1-0.20260716142648-7b4fcc466374 h1:PDUneeDqDAhsmJtnL4CenHMPukoUdEG1vO94puW6Gac= -github.com/chainreactors/go-re2 v1.11.1-0.20260716142648-7b4fcc466374/go.mod h1:4qC68vqWSuPTct3spuTrWBqCpm00mQ707JKLS1izVjI= github.com/chainreactors/logs v0.0.0-20260508055944-c678762ed15c h1:6Net2Mgo/qo6ADFBZJWWScKuMfZ0rbzLqSCVDuLKFdc= github.com/chainreactors/logs v0.0.0-20260508055944-c678762ed15c/go.mod h1:VrXmYPbNN5AVoo1sc5aeyPVBYqubMdb3KO/tn5rRZpo= github.com/chainreactors/neutron v0.1.1-0.20260714062907-716c6b167cb6 h1:apEDnJeZ5fe2AaEJHV6TRDldoy2t/d7E1maxmUz2tfU= github.com/chainreactors/neutron v0.1.1-0.20260714062907-716c6b167cb6/go.mod h1:zok/CDxut71iw8NQTHKPvy0f+1G5639zr9GgDsXGnOs= github.com/chainreactors/utils v0.0.0-20260629043228-93bdd2142c9a h1:4rLTKr+VlG7yGbw2ym7nhNvFglvZ4acdphxKEozF7CI= github.com/chainreactors/utils v0.0.0-20260629043228-93bdd2142c9a/go.mod h1:xwbUlFoSSxLHujyb8D48o1s2DqmEAxUNfxIy0DVUmcg= -github.com/chainreactors/utils/parsers v0.0.3 h1:3ld7xG5TSvzikVOCkQHjqjHO3otjODwSwHQDkMKbu5o= -github.com/chainreactors/utils/parsers v0.0.3/go.mod h1:bE/znJWt08n9QOORWsWu0ggB8GWfOg3+dfUMMITmwV4= +github.com/chainreactors/utils/jev v0.0.0-20260927031300-6bf027367a50 h1:XnGpBYIS4u1BUNL3KtlrrXde/vSmDon34X5AEV7WOi8= +github.com/chainreactors/utils/jev v0.0.0-20260927031300-6bf027367a50/go.mod h1:9w5fvcKrpQVIHWlKuvpVARpZUd+OKsYSIii2JR1ZRB8= +github.com/chainreactors/utils/parsers v0.0.4-0.20260927032755-fd113e94d6c0 h1:rfhEDw1QfbIgUAhGWRbovR3ur7Ug3V7j5WxqstwlnW0= +github.com/chainreactors/utils/parsers v0.0.4-0.20260927032755-fd113e94d6c0/go.mod h1:bE/znJWt08n9QOORWsWu0ggB8GWfOg3+dfUMMITmwV4= github.com/chainreactors/words v0.0.0-20260520145736-270600e60fb4 h1:lvnDYEkatmZFHP5i321qQXK9L4vKRfso/uUfr5tOeC8= github.com/chainreactors/words v0.0.0-20260520145736-270600e60fb4/go.mod h1:zfz367PUmyaX6oAqV9SktVqyRXKlEh0sel9Wsq9dd2c= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -108,6 +104,7 @@ github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWH github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -256,15 +253,16 @@ github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOn github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= +github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 h1:i462o439ZjprVSFSZLZxcsoAe592sZB1rci2Z8j4wdk= +github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA= github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/invopop/jsonschema v0.13.0 h1:KvpoAJWEjR3uD9Kbm2HWJmqsEaHt8lBUpd0qHcIi21E= -github.com/invopop/jsonschema v0.13.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= -github.com/jessevdk/go-flags v1.6.1 h1:Cvu5U8UGrLay1rZfv/zP7iLpSHGUZ/Ou68T0iX1bBK4= -github.com/jessevdk/go-flags v1.6.1/go.mod h1:Mk8T1hIAWpOiJiHa9rJASDK2UGWji0EuPGBnNLMooyc= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/invopop/jsonschema v0.7.0 h1:2vgQcBz1n256N+FpX3Jq7Y17AjYt46Ig3zIWyy770So= +github.com/invopop/jsonschema v0.7.0/go.mod h1:O9uiLokuu0+MGFlyiaqtWxwqJm41/+8Nj0lD7A36YH0= +github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= +github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -280,14 +278,12 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -345,7 +341,6 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= @@ -368,6 +363,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -375,15 +371,9 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tetratelabs/wazero v1.11.0 h1:+gKemEuKCTevU4d7ZTzlsvgd1uaToIDtlQlmNbwqYhA= -github.com/tetratelabs/wazero v1.11.0/go.mod h1:eV28rsN8Q+xwjogd7f4/Pp4xFxO7uOGbLcD/LzB1wiU= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= -github.com/wasilibs/wazero-helpers v0.0.0-20250123031827-cd30c44769bb h1:gQ+ZV4wJke/EBKYciZ2MshEouEHFuinB85dY3f5s1q8= -github.com/wasilibs/wazero-helpers v0.0.0-20250123031827-cd30c44769bb/go.mod h1:jMeV4Vpbi8osrE/pKUxRZkVaA0EX7NZN0A9/oRzgpgY= -github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= -github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -490,8 +480,8 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -585,8 +575,8 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -597,8 +587,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -808,7 +798,6 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/goby/goby.go b/goby/goby.go index 16741bf..5d8eff1 100644 --- a/goby/goby.go +++ b/goby/goby.go @@ -26,6 +26,8 @@ func NewGobyEngine(data []byte) (*GobyEngine, error) { type GobyEngine struct { Fingers []*GobyFinger + // MatchDetailEnabled records on each hit the feature it matched. + MatchDetailEnabled bool } func (engine *GobyEngine) Name() string { @@ -69,6 +71,9 @@ func (engine *GobyEngine) MatchRaw(raw string) common.Frameworks { for _, finger := range engine.Fingers { frame := finger.Match(raw) if frame != nil { + if !engine.MatchDetailEnabled { + frame.MatchDetail = nil + } frames.Add(frame) } } @@ -101,15 +106,21 @@ func (finger *GobyFinger) Compile() error { func (finger *GobyFinger) Match(raw string) *common.Framework { env := make(map[string]bool) - for _, r := range finger.Rule { + var detail *common.MatchDetail + for i, r := range finger.Rule { match := strings.Contains(raw, r.Feature) env[r.Label] = match == r.IsEquel + if match && r.IsEquel && detail == nil { + detail = &common.MatchDetail{MatcherType: "word", MatcherIndex: i, MatcherValue: r.Feature} + } } matched := logic.EvalLogic(finger.logicExpr, env) if matched { - return common.NewFramework(finger.Name, common.FrameFromGoby) + frame := common.NewFramework(finger.Name, common.FrameFromGoby) + frame.MatchDetail = detail + return frame } return nil } diff --git a/judge/README.md b/judge/README.md new file mode 100644 index 0000000..ff56f2c --- /dev/null +++ b/judge/README.md @@ -0,0 +1,59 @@ +# judge：以 Claim 审查规则结果 + +规则引擎负责检测；judge 在线只审查产品是否参与生成响应，以及响应能否确认版本。Provider 从有限选项中选择，不生成产品名或版本字符串。离线审计、聚类与生成分别放在 `judge/maintain` 和 `judge/gen`，复用同一 Claim 合约。 + +Claim 合约（Claim / Option / Outcome / Ruling / Provider）、校验、缓存和 Jev 客户端都在 [`github.com/chainreactors/utils/jev`](https://github.com/chainreactors/utils/tree/master/jev)。本包只定义指纹相关的断言：presence、version 和离线的 coverage，以及代码确定的事实如何表达成 Ruling。 + +## 代码确定的事实 + +代码确定的产品、协议和版本事实也构建合法 Claim/Ruling（置信度 1），经相同的 `Resolve` 和应用函数执行。重复写法的归并是 bookkeeping，不是 Claim。 + +## 在线入口 + +普通用户只需 `engine.EnableJudge(key)`：引擎的 Web 匹配入口自动调用 `Inspect` 并返回 `Accepted()`，失败时返回规则结果。需要全部注解（含被剔除条目）或自定义策略时，直接使用本包： + +```go +engine, _ := fingers.NewEngine() +engine.EnableMatchDetail() +c, err := jev.NewClient("") // TYPESAFE_API_KEY +if err != nil { return err } +j := judge.New(jev.Cached(c, jev.DefaultCacheSize)) // 一个扫描任务共用 + +hits, err := engine.DetectContent(raw) +if err != nil { return err } +all, err := j.Inspect(ctx, raw, hits) +accepted := all.Accepted() +// all 同时保存接受、拒绝、重复及未判定条目；出错时保留已完成的结果。 +``` + +`Inspect` 完成 presence 和 version，最多两个 Provider 批次。报告直接读这份结果，不再执行第二次判断。输入会复制，重新评估会清除旧 presence 注解。解析错误返回复制的基线和错误；Provider 失败时保留已确定事实和去重信息，尚未判断的条目没有 Outcome。输入中已有非空版本保持原值。 + +| 入口 | 用途 | +|---|---| +| `Inspect(ctx, raw, hits)` | 所有命中及 presence 注解，保留产品的版本补全 | +| `all.Accepted()` | 排除 Rejected / Duplicate 的结果视图，不调用 Provider | +| `Version(ctx, raw, f)` | 为生成工具单独确认版本，不修改 f | + +Presence 用 `running` 表示产品参与生成响应，`mentioned` / `unrelated` 表示误报。协议事实从完整响应头读取；产品确定事实限于明确的软件声明和产品专属头，Location、Cookie 中的名称只作为待审证据。版本从响应中抽取，每个产品只有一个多选 Claim：候选版本、`not_stated`、`insufficient`。按名字声明的版本走本地 Ruling；其他版本由 Provider 选择。每页最多向 Provider 提交 40 个 presence Claim，最多为 8 个产品确认版本，每个产品最多展示 15 个候选。 + +`Framework.Judge` 是输出注解，沿用现有 common 类型： + +| 字段 | 含义 | +|---|---| +| Option | 原始选项；即使置信度不足也保留 | +| Outcome | Resolve 后的 holds / refuted / insufficient；消费方按此统计 | +| Confidence / Evidence | 置信度及规则命中原文；本地确定事实置信度为 1 | +| Rejected | Refuted，或 Insufficient 且 DropInsufficient 为 true | +| Duplicate | 同一产品的其他写法；即使尚未判定也可标记 | + +`judge.New(p)` 默认 `MinConfidence=jev.DefaultMinConfidence`（0.3）、`DropInsufficient=false`。策略和 Provider 配置应在并发扫描前确定。Version 在证据不足时不填值。 + +## 离线维护 + +- `maintain.Ledger.Add(id, all)` 从既有 presence 注解汇总规则表现，不再请求 Provider。`Report` 返回深复制快照，记录 Outcomes、Options 和样本 ID；`Hits()` 与 `RefutedFraction()` 从 Outcomes 推导。Presence 选项常量统一为 `judge.OptionRunning` 等 `Option*`，原生 `Framework.Judge.Option` 输出字段继续保留。当前引擎聚合后的 MatchDetail 仍只保留首个匹配详情，未重设计完整规则来源。 +- `maintain.Discover` 按资源路径、表单、Cookie/自定义头名和页面文字等结构特征聚类。每个多主机簇提出 Coverage Claim：`explained` / `custom_site` → Holds，`missing` → Refuted，另有 insufficient。Cluster 直接可序列化，仅保存成员 ID、Coverage 原始 Ruling、解析的 Outcome、已报告产品和代码抽取的候选名。候选名供人命名，不作为 Provider 选项。 +- `gen.New(j, name)` 为人指定的产品名从正反样本生成规则，Probe 附加主动探测的请求与响应；正例未带版本时调用 Version。Validate 运行规则引擎，不调用模型，不修改内置库。 + +`cmd/judgeeval` 两种输入模式共用 `baseline` / `judged` 行记录。接受结果、拒绝项、版本差异和指标均从它们派生，聚类通过样本 ID 关联。生成评估同样保存原生 Framework，以 manifest 为唯一标签来源，分数和状态按需推导。报告 schema 为 3，统计使用 `claims/options`；旧缓存因命名空间隔离不会被复用。详见 [judgeeval](../cmd/judgeeval/README.md)。 + +Provider 收到精简响应视图、规则命中上下文和版本候选上下文；完整原始响应仍仅在本地用于解析事实与提取证据。 diff --git a/judge/doc.go b/judge/doc.go new file mode 100644 index 0000000..1661113 --- /dev/null +++ b/judge/doc.go @@ -0,0 +1,18 @@ +// Package judge reviews fingerprint rule results with jev claims +// (github.com/chainreactors/utils/jev): whether each hit's product produced +// the response, and which version the response states. It never identifies +// products itself. Deterministic facts use the same claims. Duplicate +// grouping is bookkeeping. +// +// j := judge.New(jev.Cached(client, jev.DefaultCacheSize)) +// all, err := j.Inspect(ctx, raw, hits) +// accepted := all.Accepted() +// +// Inspect returns all annotated hits after at most two provider batches. Inputs +// are copied; errors preserve established facts and leave untouched claims +// unjudged. A low-confidence ruling resolves to Insufficient without +// rewriting the selected option. +// +// Offline audit and clustering live in judge/maintain, fingerprint +// generation in judge/gen. +package judge diff --git a/judge/facts.go b/judge/facts.go new file mode 100644 index 0000000..fb4c9ec --- /dev/null +++ b/judge/facts.go @@ -0,0 +1,248 @@ +package judge + +import ( + "regexp" + "sort" + "strings" + "unicode/utf8" + + "github.com/chainreactors/fingers/common" + "github.com/chainreactors/fingers/judge/internal/evidence" +) + +// protocolFeatures are HTTP features some engines report as fingerprints +// (NormalizeName keys). The response head establishes them, or they are absent. +var protocolFeatures = map[string]bool{"hsts": true, "altsvc": true, "http3": true, "http2": true, "poweredby": true, "http基本认证": true} + +const ( + maxClaims = 40 // products judged per page; the rest stay unjudged (Judge == nil) + maxEvidence = 3 // excerpts per claim + evidenceContext = 80 // bytes of response around a match + maxExcerpt = 240 // runes per excerpt +) + +// groupProducts returns each product's spellings with the representative first. +// Names, evidence and rulings are derived from these original frameworks. +func groupProducts(frames common.Frameworks) [][]*common.Framework { + byKey := map[string][]*common.Framework{} + var keys []string + for _, f := range frames { + if f == nil { + continue + } + key := NormalizeName(f.Name) + if key == "" { + continue + } + if len(byKey[key]) == 0 { + keys = append(keys, key) + } + byKey[key] = append(byKey[key], f) + } + sort.Strings(keys) + groups := make([][]*common.Framework, 0, len(keys)) + for _, key := range keys { + group := byKey[key] + sort.SliceStable(group, func(a, b int) bool { + if qa, qb := frameworkQuality(group[a]), frameworkQuality(group[b]); qa != qb { + return qa > qb + } + return group[a].Name < group[b].Name + }) + groups = append(groups, group) + } + return groups +} + +// declares accepts explicit software declarations, not incidental URL/cookie text. +func declares(p *evidence.Page, name string) bool { + key := NormalizeName(name) + if key == "" { + return false + } + for header, values := range p.Header { + h := strings.ToLower(header) + if strings.HasPrefix(h, "x-") && NormalizeName(strings.TrimSuffix(strings.TrimPrefix(h, "x-"), "-version")) == key { + return true + } + if h != "server" && h != "x-powered-by" && h != "product" { + continue + } + for _, value := range values { + // "Apache/2.4 (Debian)", "nginx, Varnish": a token names the product... + for _, token := range strings.FieldsFunc(value, func(r rune) bool { return r == ',' || r == ';' || r == '(' || r == ')' }) { + if i := strings.IndexByte(token, '/'); i >= 0 { + token = token[:i] + } + if NormalizeName(strings.TrimSpace(token)) == key { + return true + } + } + // ...or one of several space-separated "name/version" products does. + for _, part := range strings.Fields(value) { + if i := strings.IndexByte(part, '/'); i > 0 && NormalizeName(part[:i]) == key { + return true + } + } + } + } + return false +} + +func protocolPresent(p *evidence.Page, key string) bool { + switch key { + case "hsts": + return p.Header.Get("Strict-Transport-Security") != "" + case "altsvc": + return p.Header.Get("Alt-Svc") != "" + case "poweredby": + return p.Header.Get("X-Powered-By") != "" + case "http2": + return strings.HasPrefix(string(p.Raw), "HTTP/2") || strings.Contains(strings.ToLower(strings.Join(p.Header.Values("Alt-Svc"), ",")), "h2=") + case "http3": + return strings.HasPrefix(string(p.Raw), "HTTP/3") || strings.Contains(strings.ToLower(strings.Join(p.Header.Values("Alt-Svc"), ",")), "h3=") + case "http基本认证": + for _, value := range p.Header.Values("WWW-Authenticate") { + // A comma inside a quoted realm does not start a new challenge. + start, quoted, escaped := 0, false, false + for i := 0; i <= len(value); i++ { + if i == len(value) || (value[i] == ',' && !quoted) { + words := strings.Fields(value[start:i]) + if len(words) > 0 && strings.EqualFold(words[0], "Basic") { + return true + } + start = i + 1 + } else if escaped { + escaped = false + } else if value[i] == '\\' && quoted { + escaped = true + } else if value[i] == '"' { + quoted = !quoted + } + } + } + } + return false +} + +// Prefer a representative that already carries a version and useful source +// metadata, since duplicate spellings are hidden from the accepted result. +func frameworkQuality(f *common.Framework) int { + quality := len(f.Froms) + if f.Attributes != nil { + if f.Version != "" { + quality += 100 + } + if f.Vendor != "" { + quality += 10 + } + } + if f.MatchDetail != nil { + quality++ + } + return quality +} + +// evidenceFor quotes the response where the claim comes from: what each +// spelling's rule matched (MatchDetail, when the engine records it), then +// where the product's name occurs. A rule may rest on text deep in the +// page, which the compact page view does not carry. +func evidenceFor(raw []byte, frames []*common.Framework) []matchExcerpt { + text := string(raw) + lower := strings.ToLower(text) + var out []matchExcerpt + seen := map[int]bool{} + add := func(start, end int, matched bool) { + if start < 0 || len(out) >= maxEvidence { + return + } + bucket := start / (2 * evidenceContext) + if seen[bucket] { + return + } + seen[bucket] = true + e := excerpt(text, start, end) + if matched { + e.Matched = evidence.Truncate(evidence.Clean(strings.ToValidUTF8(text[start:end], "")), maxExcerpt) + } + out = append(out, e) + } + for _, f := range frames { + d := f.MatchDetail + if d == nil || d.MatcherValue == "" { + continue + } + if strings.HasPrefix(d.MatcherType, "regexp") { + if re, err := regexp.Compile("(?i)" + d.MatcherValue); err == nil { + if loc := re.FindStringIndex(text); loc != nil { + add(loc[0], loc[1], true) + } + } + continue + } + v := strings.ToLower(d.MatcherValue) + if i := strings.Index(lower, v); i >= 0 { + add(i, i+len(v), true) + } + } + for _, f := range frames { + n := strings.ToLower(f.Name) + if utf8.RuneCountInString(n) < 3 { + continue + } + for from, found := 0, 0; found < 2; found++ { + i := indexWord(lower[from:], n) + if i < 0 { + break + } + add(from+i, from+i+len(n), false) + from += i + len(n) + } + } + return out +} + +func excerpt(text string, start, end int) matchExcerpt { + where := "body" + head := strings.Index(text, "\r\n\r\n") + if head < 0 { + head = strings.Index(text, "\n\n") + } + if head < 0 || start < head { + where = "header" + } + lo, hi := start-evidenceContext, end+evidenceContext + if lo < 0 { + lo = 0 + } + if hi > len(text) { + hi = len(text) + } + return matchExcerpt{Where: where, Text: evidence.Truncate(evidence.Clean(strings.ToValidUTF8(text[lo:hi], "")), maxExcerpt)} +} + +// indexWord finds ASCII keys on word boundaries ("acti" must not match +// "action"); CJK keys have no word boundaries and match as substrings. +func indexWord(haystack, key string) int { + if key == "" { + return -1 + } + if utf8.RuneCountInString(key) != len(key) { + return strings.Index(haystack, key) + } + for from := 0; ; { + i := strings.Index(haystack[from:], key) + if i < 0 { + return -1 + } + start, end := from+i, from+i+len(key) + if (start == 0 || !isWordByte(haystack[start-1])) && (end == len(haystack) || !isWordByte(haystack[end])) { + return start + } + from = start + 1 + } +} + +func isWordByte(b byte) bool { + return b == '_' || b >= '0' && b <= '9' || b >= 'a' && b <= 'z' || b >= 'A' && b <= 'Z' +} diff --git a/judge/gen/gen.go b/judge/gen/gen.go new file mode 100644 index 0000000..8ec5e80 --- /dev/null +++ b/judge/gen/gen.go @@ -0,0 +1,620 @@ +// Package gen builds native fingerprints from labelled HTTP responses for a +// product a person names. It uses a judge.Judge only for versions the +// positives do not carry; rules are derived and validated by code against +// the rule engine. +package gen + +import ( + "bytes" + "context" + "fmt" + "regexp" + "sort" + "strings" + + "github.com/chainreactors/fingers/common" + fingerlib "github.com/chainreactors/fingers/fingers" + "github.com/chainreactors/fingers/judge" + "github.com/chainreactors/fingers/judge/internal/evidence" +) + +var whitespace = regexp.MustCompile(`\s+`) + +// Generator builds one native Finger from labelled HTTP responses. Probe +// belongs to the most recently added positive or negative sample. +type Generator struct { + judge *judge.Judge + name string + positive []generatorSample + negative []generatorSample + current *generatorSample + err error +} + +type generatorSample struct { + raw []byte + version string + probes map[string][]byte +} + +// New creates a generator for the product a person names, using j for +// versions the positives do not carry. j may be nil when every positive +// carries its version (PositiveVersion). +func New(j *judge.Judge, name string) *Generator { + return &Generator{judge: j, name: strings.TrimSpace(name)} +} + +// Positive adds an HTTP response known to belong to the product. +func (g *Generator) Positive(raw []byte) *Generator { + g.positive = append(g.positive, generatorSample{raw: append([]byte(nil), raw...), probes: map[string][]byte{}}) + g.current = &g.positive[len(g.positive)-1] + return g +} + +// PositiveVersion adds a positive response with an authoritative version. +func (g *Generator) PositiveVersion(raw []byte, version string) *Generator { + g.Positive(raw) + g.positive[len(g.positive)-1].version = strings.TrimSpace(version) + return g +} + +// Negative adds an HTTP response known not to belong to the product. +func (g *Generator) Negative(raw []byte) *Generator { + g.negative = append(g.negative, generatorSample{raw: append([]byte(nil), raw...), probes: map[string][]byte{}}) + g.current = &g.negative[len(g.negative)-1] + return g +} + +// Probe attaches a recorded request and response to the last sample. request +// is the exact send_data used by the rule engine. +func (g *Generator) Probe(request, response []byte) *Generator { + if g.current == nil { + g.err = fmt.Errorf("gen: Probe requires a preceding Positive or Negative") + return g + } + if len(request) == 0 { + g.err = fmt.Errorf("gen: Probe requires non-empty send_data") + return g + } + g.current.probes[string(request)] = append([]byte(nil), response...) + return g +} + +// Generate returns one compiled, rule-validated fingerprint. It never writes +// a templates repository. At least one positive and one negative are required. +func (g *Generator) Generate(ctx context.Context) (*fingerlib.Finger, error) { + if g.err != nil { + return nil, g.err + } + if len(g.positive) == 0 || len(g.negative) == 0 { + return nil, fmt.Errorf("gen: Generate requires positive and negative samples") + } + if err := g.validateSamples(); err != nil { + return nil, err + } + name := g.name + if name == "" { + return nil, fmt.Errorf("gen: a person names the product; the provider never does") + } + + versions := make([]string, len(g.positive)) + for i, sample := range g.positive { + versions[i] = sample.version + if versions[i] == "" && g.judge != nil { + v, err := g.judge.Version(ctx, sample.raw, common.NewFramework(name, common.FrameFromGUESS)) + if err != nil { + return nil, err + } + versions[i] = v + } + } + + f := &fingerlib.Finger{Name: name, Protocol: fingerlib.HTTPProtocol} + f.Rules = append(f.Rules, g.rules("", name)...) + hasVersion := false + for _, version := range versions { + hasVersion = hasVersion || version != "" + } + for _, request := range g.probeRequests() { + if hasVersion { + if rule := g.versionRule(request, name, versions); rule != nil { + f.Rules = append(f.Rules, rule) + f.Rules = append(f.Rules, g.rules(request, name)...) + } + } else { + f.Rules = append(f.Rules, g.rules(request, name)...) + } + } + if len(f.Rules) == 0 { + return nil, fmt.Errorf("gen: no rule distinguishes the positive and negative samples") + } + if rule := g.versionRule("", name, versions); rule != nil { + f.Rules = append(fingerlib.Rules{rule}, f.Rules...) + } + if err := g.Validate(f); err != nil { + return nil, err + } + if err := f.Compile(false); err != nil { + return nil, err + } + return f, nil +} + +// Validate checks the generated or edited Finger against every recorded +// positive and negative using the rule engine, without calling the provider. +func (g *Generator) Validate(f *fingerlib.Finger) error { + if g.err != nil { + return g.err + } + if f == nil || len(f.Rules) == 0 { + return fmt.Errorf("gen: fingerprint has no rules") + } + if len(g.positive) == 0 || len(g.negative) == 0 { + return fmt.Errorf("gen: validation requires positive and negative samples") + } + if err := g.validateSamples(); err != nil { + return err + } + for i, rule := range f.Rules { + if rule == nil { + return fmt.Errorf("gen: rule %d is nil", i) + } + } + checking := copyFingerForValidation(f) + if err := checking.Compile(false); err != nil { + return err + } + check := func(sample generatorSample) (*common.Framework, bool) { + sender := func(request []byte) ([]byte, bool) { + response, ok := sample.probes[string(request)] + return response, ok + } + frame, _, ok := checking.Match(fingerlib.NewContent(sample.raw, "", true), 2, sender) + return frame, ok + } + for i, sample := range g.positive { + frame, ok := check(sample) + if !ok { + return fmt.Errorf("gen: positive sample %d did not match", i) + } + if sample.version != "" && frame.Version != sample.version { + return fmt.Errorf("gen: positive sample %d version = %q, want %q", i, frame.Version, sample.version) + } + } + for i, sample := range g.negative { + if _, ok := check(sample); ok { + return fmt.Errorf("gen: negative sample %d matched", i) + } + } + return nil +} + +func (g *Generator) validateSamples() error { + for _, sample := range append(append([]generatorSample(nil), g.positive...), g.negative...) { + if _, err := evidence.NewPage(sample.raw); err != nil { + return fmt.Errorf("gen: invalid sample: %w", err) + } + for request, response := range sample.probes { + if request == "" { + return fmt.Errorf("gen: probe has empty send_data") + } + if _, err := evidence.NewPage(response); err != nil { + return fmt.Errorf("gen: invalid probe response: %w", err) + } + } + } + return nil +} + +func copyFingerForValidation(f *fingerlib.Finger) *fingerlib.Finger { + copy := *f + copy.Rules = make(fingerlib.Rules, len(f.Rules)) + for i, rule := range f.Rules { + if rule == nil { + continue + } + cloned := *rule + if rule.Regexps != nil { + regexps := *rule.Regexps + regexps.Body = append([]string(nil), rule.Regexps.Body...) + regexps.Header = append([]string(nil), rule.Regexps.Header...) + regexps.CompliedRegexp = nil + regexps.CompiledVulnRegexp = nil + regexps.CompiledVersionRegexp = nil + cloned.Regexps = ®exps + } + copy.Rules[i] = &cloned + } + return © +} + +type ruleCandidate struct { + kind string + value string + weight int +} + +func candidates(raw []byte) []ruleCandidate { + p, err := evidence.NewPage(raw) + if err != nil { + return nil + } + var out []ruleCandidate + add := func(kind, value string, weight int) { + value = strings.ToLower(strings.TrimSpace(value)) + if len(value) >= 5 && len(value) <= 120 { + out = append(out, ruleCandidate{kind, value, weight}) + } + } + for k, v := range p.Headers { + if k == "Server" || strings.HasPrefix(k, "X-") || k == "Product" { + value := strings.TrimSpace(evidence.NameVersion.ReplaceAllString(strings.Split(v, "/")[0], "")) + if strings.HasPrefix(k, "X-") && (strings.HasSuffix(k, "-Version") || value == "" || value[0] >= '0' && value[0] <= '9') { + add("header", k+":", 6) + } else { + add("header", k+": "+value, 7) + } + } + } + add("body", p.Generator, 6) + // Preserve the declaration, but not a single release or customized title. + if name := evidence.NameVersion.ReplaceAllString(p.Generator, ""); name != "" { + if loc := evidence.Generator.FindSubmatchIndex(raw); loc != nil { + start := loc[2] + if start < 0 { + start = loc[4] + } + end := start + len(name) + if end < len(raw) && (raw[end] == '/' || raw[end] == ' ') { + end++ + } + add("body", string(raw[loc[0]:end]), 8) + } + } + if !evidence.GenericName(evidence.NormalizeName(p.Title)) { + if title := evidence.Title.Find(raw); len(title) > 0 { + add("body", string(title), 5) + } + add("body", p.Title, 4) + } + for _, asset := range append(append([]string(nil), p.Scripts...), p.Styles...) { + path := strings.Split(asset, "?")[0] + add("body", path, 3) + } + for _, comment := range p.Comments { + add("body", comment, 2) + } + for _, name := range evidence.Names(p.Generator, p.Title, p.Text, p.Headers, append(append([]string(nil), p.Scripts...), p.Styles...)) { + if strings.Contains(strings.ToLower(p.Text), strings.ToLower(name)) { + add("body", name, 1) + } + } + sort.SliceStable(out, func(a, b int) bool { + if out[a].weight != out[b].weight { + return out[a].weight > out[b].weight + } + return len(out[a].value) > len(out[b].value) + }) + return out +} + +func containsCandidate(raw []byte, c ruleCandidate) bool { + content := fingerlib.NewContent(raw, "", true) + var target []byte + if c.kind == "header" { + target = content.Header + } else { + target = content.Body + } + return bytes.Contains(target, []byte(c.value)) +} + +func candidateRule(c ruleCandidate) *fingerlib.Rule { + r := &fingerlib.Rule{Regexps: &fingerlib.Regexps{}} + if c.kind == "header" { + r.Regexps.Header = []string{c.value} + } else { + r.Regexps.Body = []string{c.value} + } + return r +} + +func (g *Generator) rules(request, name string) fingerlib.Rules { + positive, negative, _ := g.responses(request) + if len(positive) == 0 || len(negative) == 0 { + return nil + } + var rules fingerlib.Rules + for _, c := range selectCandidates(positive, negative, name) { + rule := candidateRule(c) + if request != "" { + rule.SendDataStr, rule.Level = request, 2 + } + rules = append(rules, rule) + } + return rules +} + +// Pick evidence that covers the most still-uncovered positives while staying +// absent from every negative. A single common matcher wins when one exists. +func selectCandidates(positive, negative [][]byte, product ...string) []ruleCandidate { + if len(positive) == 0 { + return nil + } + options := map[string]ruleCandidate{} + for _, raw := range positive { + for _, c := range candidates(raw) { + // A product must not inherit its proxy/CDN or an unrelated security + // header merely because that header is absent from a small negative set. + if c.kind == "header" && len(product) > 0 && !strings.Contains(evidence.NormalizeName(c.value), evidence.NormalizeName(product[0])) { + continue + } + key := c.kind + "\x00" + c.value + if previous, ok := options[key]; !ok || c.weight > previous.weight { + options[key] = c + } + } + } + covered := make([]bool, len(positive)) + var selected []ruleCandidate + for len(selected) < len(positive) { + best, bestCount := ruleCandidate{}, 0 + for _, c := range options { + valid := true + for _, raw := range negative { + if containsCandidate(raw, c) { + valid = false + break + } + } + if !valid { + continue + } + count := 0 + for i, raw := range positive { + if !covered[i] && containsCandidate(raw, c) { + count++ + } + } + if count > bestCount || count == bestCount && count > 0 && (c.weight > best.weight || c.weight == best.weight && (c.value < best.value || c.value == best.value && c.kind < best.kind)) { + best, bestCount = c, count + } + } + if bestCount == 0 { + break + } + selected = append(selected, best) + delete(options, best.kind+"\x00"+best.value) + for i, raw := range positive { + covered[i] = covered[i] || containsCandidate(raw, best) + } + all := true + for _, ok := range covered { + all = all && ok + } + if all { + break + } + } + return selected +} + +func (g *Generator) probeRequests() []string { + seen := map[string]bool{} + var out []string + for _, sample := range g.positive { + for request := range sample.probes { + if !seen[request] { + seen[request] = true + out = append(out, request) + } + } + } + sort.Strings(out) + return out +} + +func (g *Generator) versionRule(request, name string, versions []string) *fingerlib.Rule { + positive, negative, indices := g.responses(request) + if len(positive) == 0 || len(negative) == 0 { + return nil + } + known := make([]string, len(indices)) + for i, index := range indices { + known[i] = versions[index] + } + var patterns []string + if pattern := versionPattern(name, positive, known, negative); pattern != "" { + patterns = append(patterns, pattern) + } else { + patterns = guardedVersionPatterns(name, positive, known, negative) + } + if len(patterns) == 0 { + return nil + } + rule := &fingerlib.Rule{Regexps: &fingerlib.Regexps{Regexp: patterns}} + if request != "" { + rule.SendDataStr, rule.Level = request, 2 + } + return rule +} + +func (g *Generator) responses(request string) (positive, negative [][]byte, indices []int) { + response := func(sample generatorSample) ([]byte, bool) { + if request == "" { + return sample.raw, true + } + raw, ok := sample.probes[request] + return raw, ok + } + for i, sample := range g.positive { + if raw, ok := response(sample); ok { + positive = append(positive, raw) + indices = append(indices, i) + } + } + for _, sample := range g.negative { + if raw, ok := response(sample); ok { + negative = append(negative, raw) + } + } + return positive, negative, indices +} + +func versionPattern(name string, positive [][]byte, versions []string, negative [][]byte) string { + first := -1 + for i, v := range versions { + if v != "" { + first = i + break + } + } + if first < 0 { + return "" + } + raw := string(positive[first]) + for from := 0; from < len(raw); { + next := strings.Index(raw[from:], versions[first]) + if next < 0 { + break + } + index := from + next + from = index + len(versions[first]) + pattern := versionPatternAt(raw, index, name) + if pattern == "" { + continue + } + re, err := regexp.Compile("(?i)" + pattern) + if err != nil { + continue + } + valid := true + for i, version := range versions { + if version == "" { + continue + } + match := re.FindStringSubmatch(string(positive[i])) + if len(match) < 2 || match[1] != version { + valid = false + break + } + } + for _, response := range negative { + if re.Match(response) { + valid = false + break + } + } + if valid { + return pattern + } + } + return "" +} + +func versionPatternAt(raw string, index int, name string) string { + var pattern string + // A generator meta tag is stronger than an unscoped product mention. + for _, loc := range evidence.Generator.FindAllStringSubmatchIndex(raw, -1) { + start, end := loc[2], loc[3] + if start < 0 { + start, end = loc[4], loc[5] + } + if index >= start && index < end && strings.Contains(evidence.NormalizeName(raw[start:index]), evidence.NormalizeName(name)) { + return regexp.QuoteMeta(raw[loc[0]:index]) + `(` + evidence.VersionToken + `|[0-9]+)` + } + } + if headerEnd := strings.Index(raw, "\r\n\r\n"); headerEnd >= 0 && index < headerEnd { + lineStart := strings.LastIndex(raw[:index], "\n") + 1 + prefix := strings.TrimSpace(raw[lineStart:index]) + if len(prefix) > 48 || !strings.Contains(evidence.NormalizeName(prefix), evidence.NormalizeName(name)) { + return "" + } + pattern = `(?m)^` + regexp.QuoteMeta(prefix) + `\s*(` + evidence.VersionToken + `|[0-9]+)` + } else { + begin := index - 60 + if begin < 0 { + begin = 0 + } + window := raw[begin:index] + nameIndex := strings.LastIndex(strings.ToLower(window), strings.ToLower(name)) + if nameIndex < 0 { + return "" + } + prefix := window[nameIndex:] + if len(prefix) > 48 || strings.ContainsAny(prefix, "\r\n<>") { + return "" + } + pattern = regexp.QuoteMeta(strings.TrimSpace(prefix)) + `\s*(` + evidence.VersionToken + `|[0-9]+)` + } + return pattern +} + +// Some applications put their release only in asset queries or a footer. +// Learn literal local context, conjoined with a product declaration in ONE +// regex: native body/header/regexp matchers are alternatives, not AND gates. +func guardedVersionPatterns(name string, positive [][]byte, versions []string, negative [][]byte) []string { + var patterns []string + seen := map[string]bool{} + for _, guard := range selectCandidates(positive, negative, name) { + if guard.weight < 4 || !strings.Contains(evidence.NormalizeName(guard.value), evidence.NormalizeName(name)) { + continue + } + for i, version := range versions { + if version == "" { + continue + } + raw := string(positive[i]) + for from := 0; from < len(raw); { + next := strings.Index(raw[from:], version) + if next < 0 { + break + } + index := from + next + from = index + len(version) + begin := strings.LastIndex(raw[:index], "<") + if begin < 0 || index-begin > 120 { + continue + } + prefix := regexp.QuoteMeta(raw[begin:index]) + prefix = whitespace.ReplaceAllStringFunc(prefix, func(string) string { return `\s+` }) + pattern := `(?s)` + regexp.QuoteMeta(guard.value) + `.*?` + prefix + `(` + evidence.VersionToken + `)(?:[^0-9A-Za-z.+-]|$)` + if seen[pattern] { + continue + } + seen[pattern] = true + re, err := regexp.Compile("(?i)" + pattern) + if err != nil { + continue + } + valid := true + for j, want := range versions { + match := re.FindStringSubmatch(string(positive[j])) + if want == "" { + if match != nil { + valid = false + } + continue + } + if len(match) < 2 || match[1] != want { + valid = false + break + } + } + for _, raw := range negative { + if re.Match(raw) { + valid = false + break + } + } + if valid { + patterns = append(patterns, pattern) + if len(patterns) == 4 { + return patterns + } + } + } + } + } + return patterns +} diff --git a/judge/gen/gen_test.go b/judge/gen/gen_test.go new file mode 100644 index 0000000..baede7a --- /dev/null +++ b/judge/gen/gen_test.go @@ -0,0 +1,198 @@ +package gen + +import ( + "bytes" + "context" + "fmt" + "strings" + "testing" + + fingerlib "github.com/chainreactors/fingers/fingers" + "github.com/chainreactors/fingers/judge" + "github.com/chainreactors/utils/jev" +) + +// names picks its product whenever it is offered as an option. +type names string + +func (names) ID() string { return "names" } + +func (n names) Judge(_ context.Context, _ interface{}, qs map[string]jev.Claim) (map[string]jev.Ruling, error) { + out := map[string]jev.Ruling{} + for key, q := range qs { + picked := jev.OptionInsufficient + for option := range q.Options { + if strings.EqualFold(option, string(n)) { + picked = option + } + } + out[key] = jev.Ruling{Option: picked, Confidence: 0.95} + } + return out, nil +} + +func TestGeneratorPassiveVersionAndValidation(t *testing.T) { + positive1 := []byte("HTTP/1.1 200 OK\r\nServer: nginx/2.401.3\r\nX-Jenkins: 2.401.3\r\n\r\nJenkins") + positive2 := []byte("HTTP/1.1 200 OK\r\nServer: nginx/2.402.1\r\nX-Jenkins: 2.402.1\r\n\r\nJenkins") + negative := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nBlog

X-Jenkins: 9.9.9 release notes

") + g := New(nil, "Jenkins").PositiveVersion(positive1, "2.401.3").PositiveVersion(positive2, "2.402.1").Negative(negative) + f, err := g.Generate(context.Background()) + if err != nil { + t.Fatal(err) + } + if f.Name != "Jenkins" || len(f.Rules) < 2 { + t.Fatalf("fingerprint: %+v", f) + } + for _, sample := range []struct { + raw []byte + version string + }{{positive1, "2.401.3"}, {positive2, "2.402.1"}} { + frame, _, ok := f.PassiveMatch(fingerlib.NewContent(sample.raw, "", true)) + if !ok || frame.Version != sample.version { + t.Fatalf("version matcher = %+v, %v", frame, ok) + } + } + compiled := len(f.Rules[0].Regexps.CompliedRegexp) + if err := g.Validate(f); err != nil || len(f.Rules[0].Regexps.CompliedRegexp) != compiled { + t.Fatalf("Validate mutated compiled matcher: %v", err) + } + if err := g.Validate(&fingerlib.Finger{Name: "Jenkins", Rules: fingerlib.Rules{&fingerlib.Rule{Regexps: &fingerlib.Regexps{Body: []string{"jenkins"}}}}}); err == nil { + t.Fatal("validator accepted rule matching negative sample") + } +} + +func TestGeneratorActive(t *testing.T) { + base := []byte("HTTP/1.1 200 OK\r\n\r\nLogin") + request := []byte("/admin") + positive := []byte("HTTP/1.1 200 OK\r\nX-Orion: console\r\n\r\nOrion console") + negative := []byte("HTTP/1.1 404 Not Found\r\n\r\nmissing") + g := New(nil, "Orion").Positive(base).Probe(request, positive).Negative(base).Probe(request, negative) + f, err := g.Generate(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(f.Rules) != 1 || f.Rules[0].SendDataStr != string(request) || !f.IsActive { + t.Fatalf("active rule: %+v", f) + } + if err := g.Validate(f); err != nil { + t.Fatal(err) + } + if len(f.Rules[0].Regexps.CompliedRegexp) != 0 { + t.Fatal("Validate changed compiled rules") + } + if _, err := New(nil, "Orion").Positive(base).Probe(request, positive).Negative(base).Generate(context.Background()); err == nil { + t.Fatal("generated active rule without a negative probe") + } +} + +// A person names the product: without a name, Generate fails rather than ask +// the provider for one. +func TestGeneratorRequiresName(t *testing.T) { + positive := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nOrion

Orion console

") + negative := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nOther") + j := judge.New(names("Orion")) + if f, err := New(j, "").Positive(positive).Negative(negative).Generate(context.Background()); err == nil { + t.Fatalf("generated %+v without a name", f) + } +} + +func TestGeneratorActiveVersion(t *testing.T) { + base := []byte("HTTP/1.1 200 OK\r\n\r\nLogin") + request := []byte("/version") + positive := []byte("HTTP/1.1 200 OK\r\nX-Orion: 3.1.2\r\n\r\nversion") + negative := []byte("HTTP/1.1 200 OK\r\n\r\nnot installed") + g := New(nil, "Orion").PositiveVersion(base, "3.1.2").Probe(request, positive).Negative(base).Probe(request, negative) + f, err := g.Generate(context.Background()) + if err != nil { + t.Fatal(err) + } + frame, _, ok := f.ActiveMatch(2, func(got []byte) ([]byte, bool) { return positive, bytes.Equal(got, request) }) + if !ok || frame.Version != "3.1.2" { + t.Fatalf("active version = %+v, %v", frame, ok) + } +} + +func TestGeneratorCoversDifferentPositiveVariants(t *testing.T) { + base := "HTTP/1.1 200 OK\r\n%s\r\nWelcome" + first := []byte(fmt.Sprintf(base, "X-Orion-Instance: ready\r\n")) + second := []byte(fmt.Sprintf(base, "X-Orion-Service: yes\r\n")) + other := []byte(fmt.Sprintf(base, "Server: nginx\r\n")) + g := New(nil, "Orion").Positive(first).Positive(second).Negative(other) + f, err := g.Generate(context.Background()) + if err != nil || len(f.Rules) != 2 { + t.Fatalf("variant rules = %+v, %v", f, err) + } +} + +func TestGeneratedVersionHandlesHyphenatedHeaderAndReleaseSuffix(t *testing.T) { + raw := func(v string) []byte { + return []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\nX-New-Api-Version: v" + v + "\r\n\r\nNew API") + } + neg := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nWelcome") + g := New(nil, "New API").PositiveVersion(raw("1.0.0-rc.35"), "1.0.0-rc.35").Negative(neg) + f, err := g.Generate(context.Background()) + if err != nil { + t.Fatal(err) + } + for _, version := range []string{"1.0.0-rc.35", "1.2.0-rc.2", "2.0.1"} { + frame, _, matched := f.PassiveMatch(fingerlib.NewContent(raw(version), "", true)) + if !matched || frame.Version != version { + t.Fatalf("version=%s matched=%t frame=%+v", version, matched, frame) + } + } +} + +func TestGeneratedMetaRuleSurvivesTitleAndCalendarVersionChanges(t *testing.T) { + page := func(title, version string) []byte { + return []byte("HTTP/1.1 200 OK\r\n\r\n" + title + "") + } + negative := []byte("HTTP/1.1 200 OK\r\n\r\nSearXNG Documentation

Install SearXNG

") + f, err := New(nil, "SearXNG").PositiveVersion(page("Search One", "2026.9.23+3cd69d30e"), "2026.9.23+3cd69d30e").Negative(negative).Generate(context.Background()) + if err != nil { + t.Fatal(err) + } + got, _, matched := f.PassiveMatch(fingerlib.NewContent(page("Different Search", "2026.9.25+d8ae3abd5"), "", true)) + if !matched || got.Version != "2026.9.25+d8ae3abd5" { + t.Fatalf("matched=%t got=%+v", matched, got) + } + if _, _, matched := f.PassiveMatch(fingerlib.NewContent(negative, "", true)); matched { + t.Fatal("documentation matched") + } +} + +func TestGeneratedFooterVersionRequiresProductDeclaration(t *testing.T) { + page := func(title, version string) []byte { + return []byte("HTTP/1.1 200 OK\r\n\r\n" + title + "
Log in
\n v" + version + " @ mysql\n
") + } + negative := page("Different Application", "2.18.0-be6aeb4") + f, err := New(nil, "Wakapi").PositiveVersion(page("Wakapi – Coding Statistics", "2.18.0-be6aeb4"), "2.18.0-be6aeb4").Negative(negative).Generate(context.Background()) + if err != nil { + t.Fatal(err) + } + got, _, matched := f.PassiveMatch(fingerlib.NewContent(page("Wakapi – Coding Statistics", "2.12.0"), "", true)) + if !matched || got.Version != "2.12.0" { + t.Fatalf("matched=%t got=%+v", matched, got) + } + if _, _, matched := f.PassiveMatch(fingerlib.NewContent(negative, "", true)); matched { + t.Fatal("unguarded footer matched another product") + } +} + +func TestGeneratedApplicationDoesNotLearnSharedInfrastructure(t *testing.T) { + for _, name := range []string{"IT Tools", "Docmost"} { + positive := []byte("HTTP/1.1 200 OK\r\nServer: cloudflare\r\nX-Frame-Options: SAMEORIGIN\r\n\r\n" + name + "") + negative := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nDifferent Application") + f, err := New(nil, name).Positive(positive).Negative(negative).Generate(context.Background()) + if err != nil { + t.Fatal(err) + } + sameProxy := []byte("HTTP/1.1 200 OK\r\nServer: cloudflare\r\nX-Frame-Options: SAMEORIGIN\r\n\r\nUnrelated Application") + if _, _, matched := f.PassiveMatch(fingerlib.NewContent(sameProxy, "", true)); matched { + t.Fatalf("%s learned shared infrastructure", name) + } + otherProxy := []byte("HTTP/1.1 200 OK\r\nServer: caddy\r\n\r\n" + name + "") + if _, _, matched := f.PassiveMatch(fingerlib.NewContent(otherProxy, "", true)); !matched { + t.Fatalf("%s requires original infrastructure", name) + } + } +} diff --git a/judge/inspect.go b/judge/inspect.go new file mode 100644 index 0000000..859500c --- /dev/null +++ b/judge/inspect.go @@ -0,0 +1,115 @@ +package judge + +import ( + "context" + "sort" + + "github.com/chainreactors/fingers/common" + "github.com/chainreactors/fingers/judge/internal/evidence" + "github.com/chainreactors/utils/jev" +) + +// Inspect evaluates presence and versions once and returns every annotated hit. +// Call Accepted on the result to obtain the reporting view. Inputs are never changed. +// On error, the copied baseline and all facts established so far remain available; +// unjudged claims have no outcome. Each provider batch is validated before application. +func (j *Judge) Inspect(ctx context.Context, raw []byte, hits common.Frameworks) (common.Frameworks, error) { + working := cloneFrameworks(hits) + for _, f := range working { + if f != nil { + f.Judge = nil + } + } + if err := j.ready(ctx); err != nil { + return working, err + } + p, err := evidence.NewPage(raw) + if err != nil { + return working, err + } + if err := j.presence(ctx, p, working); err != nil { + return working, err + } + return working, j.versions(ctx, p, versionOrder(working.Accepted())...) +} + +// Version selects one extracted version using the same claims as Inspect. +func (j *Judge) Version(ctx context.Context, raw []byte, f *common.Framework) (string, error) { + if f == nil { + return "", nil + } + if err := j.ready(ctx); err != nil { + return "", err + } + if f.Attributes != nil && f.Version != "" { + return f.Version, nil + } + p, err := evidence.NewPage(raw) + if err != nil { + return "", err + } + copied := cloneFramework(f) + err = j.versions(ctx, p, copied) + if copied.Attributes == nil { + return "", err + } + return copied.Version, err +} + +// versionOrder is the order products are versioned in, as maxVersioned caps +// them: confirmed products first, then undecided, then unjudged, by name. +func versionOrder(frames common.Frameworks) []*common.Framework { + rank := func(f *common.Framework) int { + switch { + case f.Judge == nil: + return 2 + case f.Judge.Outcome == jev.Holds.String(): + return 0 + } + return 1 + } + list := frames.List() + sort.Slice(list, func(a, b int) bool { + if ra, rb := rank(list[a]), rank(list[b]); ra != rb { + return ra < rb + } + return list[a].Name < list[b].Name + }) + return list +} + +func cloneFrameworks(frames common.Frameworks) common.Frameworks { + working := make(common.Frameworks, len(frames)) + for name, f := range frames { + working[name] = cloneFramework(f) + } + return working +} + +func cloneFramework(f *common.Framework) *common.Framework { + if f == nil { + return nil + } + copy := *f + copy.Tags = append([]string(nil), f.Tags...) + if f.Froms != nil { + copy.Froms = make(map[common.From]bool, len(f.Froms)) + for from, present := range f.Froms { + copy.Froms[from] = present + } + } + if f.Attributes != nil { + attrs := *f.Attributes + copy.Attributes = &attrs + } + if f.Judge != nil { + judgement := *f.Judge + judgement.Evidence = append([]string(nil), f.Judge.Evidence...) + copy.Judge = &judgement + } + if f.MatchDetail != nil { + detail := *f.MatchDetail + copy.MatchDetail = &detail + } + return © +} diff --git a/judge/inspect_test.go b/judge/inspect_test.go new file mode 100644 index 0000000..d3b4ec5 --- /dev/null +++ b/judge/inspect_test.go @@ -0,0 +1,288 @@ +package judge + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "github.com/chainreactors/fingers/common" + "github.com/chainreactors/utils/jev" +) + +func TestInspect(t *testing.T) { + m := newMock() + j := New(jev.Cached(m, jev.DefaultCacheSize)) + all, err := j.Inspect(context.Background(), []byte(jenkinsRaw), testFrames()) + if err != nil { + t.Fatal(err) + } + if w := all["wordpress"].Judge; w == nil || !w.Rejected || w.Option != OptionMentioned || w.Outcome != "refuted" || len(w.Evidence) == 0 || !strings.Contains(w.Evidence[0], "WordPress") { + t.Errorf("text-only wordpress not rejected on its evidence: %+v", w) + } + for _, name := range []string{"nginx", "jenkins"} { + if v := all[name].Judge; v.Rejected || v.Option != OptionDeclared || v.Outcome != "holds" { + t.Errorf("%s named in a header must be declared: %+v", name, v) + } + } + if h := all["hsts"].Judge; !h.Rejected || h.Option != OptionAbsent || h.Outcome != "refuted" { + t.Errorf("hsts without the header must be absent: %+v", h) + } + if all["apache tomcat"].Judge.Duplicate == all["apache-tomcat"].Judge.Duplicate { + t.Errorf("exactly one tomcat spelling must be a dup") + } + // Facts are not asked: only wordpress and tomcat are claims. + if len(m.requests) != 1 || strings.Join(m.requests[0], ",") != "presence_apachetomcat,presence_wordpress" { + t.Fatalf("requests: %v", m.requests) + } + claims, _ := json.Marshal(m.lastState["matches"]) + if !strings.Contains(string(claims), "We moved here from WordPress") { + t.Errorf("claim evidence missing from the state: %s", claims) + } + + accepted, err := j.Inspect(context.Background(), []byte(jenkinsRaw), testFrames()) + accepted = accepted.Accepted() + if err != nil { + t.Fatal(err) + } + if len(accepted) != 3 { // nginx, jenkins, tomcat + t.Errorf("accepted: %v", accepted) + } + // Both versions are bound by name in the headers: taken without asking. + if accepted["jenkins"].Version != "2.401.3" || accepted["nginx"].Version != "1.24.0" { + t.Errorf("versions: jenkins %q nginx %q", accepted["jenkins"].Version, accepted["nginx"].Version) + } + // The presence come from the cache; tomcat is offered no version, since + // both strings are bound by name to other products. + if len(m.requests) != 1 { + t.Errorf("cached presence were asked again: %v", m.requests) + } +} + +// DropInsufficient decides what an undecided claim does; a ruling below +// MinConfidence is undecided whatever it chose. +func TestInsufficient(t *testing.T) { + for _, c := range []struct { + on bool + confidence float64 + option string + rejected bool + }{ + {false, 0.95, OptionRunning, false}, + {false, 0.2, jev.OptionInsufficient, false}, + {true, 0.2, jev.OptionInsufficient, true}, + {true, 0.95, OptionRunning, false}, + } { + m := newMock() + m.confidence = c.confidence + j := New(jev.Cached(m, jev.DefaultCacheSize)) + j.DropInsufficient = c.on + all, err := j.Inspect(context.Background(), []byte(jenkinsRaw), testFrames()) + if err != nil { + t.Fatal(err) + } + tomcat := all["apache tomcat"].Judge + if all["apache-tomcat"].Judge.Duplicate { + tomcat = all["apache-tomcat"].Judge + } + outcome := "holds" + if c.option == jev.OptionInsufficient { + outcome = "insufficient" + } + if tomcat.Option != OptionRunning || tomcat.Outcome != outcome || tomcat.Rejected != c.rejected { + t.Errorf("%+v: tomcat %+v", c, tomcat) + } + } +} + +// A failed provider leaves the input untouched, and the result still holds +// what code established: duplicates merged, header facts decided, the +// undecided hits kept unjudged. +func TestFailedProviderKeepsFacts(t *testing.T) { + m := newMock() + m.fail = true + frames := testFrames() + before, _ := json.Marshal(frames) + accepted, err := New(m).Inspect(context.Background(), []byte(jenkinsRaw), frames) + accepted = accepted.Accepted() + if err == nil { + t.Fatal("expected error") + } + if after, _ := json.Marshal(frames); string(after) != string(before) { + t.Fatalf("frames changed on failure:\n%s\n%s", before, after) + } + // nginx, jenkins (declared), wordpress and one tomcat (unjudged); hsts absent, one tomcat a duplicate. + if len(accepted) != 4 || accepted["hsts"] != nil || accepted["wordpress"] == nil || accepted["wordpress"].Judge != nil || + accepted["jenkins"].Judge.Option != OptionDeclared { + t.Fatalf("accepted on failure: %v", accepted) + } +} + +func TestRepeatedInspectIsCacheable(t *testing.T) { + m := newMock() + m.presence["jenkins"] = OptionRunning + j := New(jev.Cached(m, jev.DefaultCacheSize)) + for i := 0; i < 2; i++ { + accepted, err := j.Inspect(context.Background(), []byte(bodyRaw), testFrames()) + accepted = accepted.Accepted() + if err != nil { + t.Fatal(err) + } + if accepted["jenkins"].Version != "2.401.3" { + t.Fatalf("run %d: cached rulings not applied", i) + } + } + if m.calls != 2 { + t.Fatalf("calls=%d", m.calls) + } +} + +// Versions are ordered by their resolved outcome. +func TestVersionOrder(t *testing.T) { + frames := common.Frameworks{} + for name, j := range map[string]*common.Judgement{"jquery": {Option: jev.OptionInsufficient}, "apache": {Option: OptionDeclared, Outcome: jev.Holds.String()}, + "gitlab": {Option: OptionRunning, Outcome: jev.Holds.String()}, "unjudged": nil} { + f := common.NewFramework(name, common.FrameFromFingers) + f.Judge = j + frames.Add(f) + } + var got []string + for _, f := range versionOrder(frames) { + got = append(got, f.Name) + } + if strings.Join(got, ",") != "apache,gitlab,jquery,unjudged" { + t.Fatalf("order: %v", got) + } +} + +func TestPublicResultsDoNotMutateHits(t *testing.T) { + m := newMock() + m.presence["jenkins"] = OptionRunning + j := New(m) + frames := testFrames() + before, _ := json.Marshal(frames) + inspected, err := j.Inspect(context.Background(), []byte(bodyRaw), frames) + if err != nil || !inspected["wordpress"].Judge.Rejected { + t.Fatalf("inspect = %v, %v", inspected, err) + } + accepted, err := j.Inspect(context.Background(), []byte(bodyRaw), frames) + accepted = accepted.Accepted() + if err != nil || accepted["wordpress"] != nil || accepted["jenkins"].Version != "2.401.3" { + t.Fatalf("accepted = %v, %v", accepted, err) + } + // The judgement is copied, not shared with the Inspect result. + accepted["nginx"].Judge.Option = "changed" + if inspected["nginx"].Judge.Option == "changed" { + t.Fatal("Independent Inspect calls share a Judgement") + } + if after, _ := json.Marshal(frames); string(after) != string(before) { + t.Fatalf("hits changed:\n%s\n%s", before, after) + } + version, err := j.Version(context.Background(), []byte(bodyRaw), frames["jenkins"]) + if err != nil || version != "2.401.3" || frames["jenkins"].Version != "" { + t.Fatalf("version = %q, %v; input = %q", version, err, frames["jenkins"].Version) + } +} + +// A failed version round keeps the presence and leaves the input untouched. +func TestInspectVersionFailureKeepsPresence(t *testing.T) { + frames := testFrames() + m := newMock() + m.presence["jenkins"] = OptionRunning + accepted, err := New(failVersionProvider{m}).Inspect(context.Background(), []byte(bodyRaw), frames) + accepted = accepted.Accepted() + if err == nil || accepted["wordpress"] != nil || accepted["jenkins"] == nil || accepted["jenkins"].Version != "" { + t.Fatalf("failed version round = %v, %v", accepted, err) + } + for _, f := range frames { + if f.Judge != nil || f.Version != "" { + t.Fatalf("input changed after the version round failed: %+v", f) + } + } +} + +func TestInspectInvalidBatchIsNotPartlyApplied(t *testing.T) { + j := New(rulingProvider(func(cs map[string]jev.Claim) map[string]jev.Ruling { + out := map[string]jev.Ruling{} + for id := range cs { + out[id] = jev.Ruling{Option: OptionRunning, Confidence: 1} + } + out["presence_wordpress"] = jev.Ruling{Option: "invented", Confidence: 1} + return out + })) + all, err := j.Inspect(context.Background(), []byte(jenkinsRaw), testFrames()) + if err == nil || all["wordpress"].Judge != nil || (all["apache tomcat"].Judge != nil && all["apache tomcat"].Judge.Outcome != "") || all["nginx"].Judge.Outcome != jev.Holds.String() { + t.Fatalf("partially applied invalid batch: %v %v", all, err) + } +} + +func TestInspectReevaluatesAndCopiesInputs(t *testing.T) { + m := newMock() + j := New(m) + frames := testFrames() + f := frames["wordpress"] + f.MatchDetail = &common.MatchDetail{MatcherValue: "WordPress"} + f.Judge = &common.Judgement{Option: OptionRunning, Outcome: jev.Holds.String(), Evidence: []string{"old"}} + before, _ := json.Marshal(frames) + all, err := j.Inspect(context.Background(), []byte(jenkinsRaw), frames) + if err != nil || !all["wordpress"].Judge.Rejected { + t.Fatalf("old judgement reused: %v %v", all, err) + } + all["wordpress"].MatchDetail.MatcherValue = "changed" + all["wordpress"].Tags = append(all["wordpress"].Tags, "changed") + all["wordpress"].Froms[common.FrameFromFingers] = true + all["wordpress"].Version = "changed" + all["wordpress"].Judge.Evidence[0] = "changed" + after, _ := json.Marshal(frames) + if string(before) != string(after) { + t.Fatal("result mutated input") + } + partial, err := j.Inspect(context.Background(), []byte("not HTTP"), frames) + if err == nil || len(partial) != len(frames) || partial["wordpress"].Judge != nil { + t.Fatalf("parse failure lost baseline: %v %v", partial, err) + } + partial["wordpress"].MatchDetail.MatcherValue = "also changed" + if frames["wordpress"].MatchDetail.MatcherValue != "WordPress" { + t.Fatal("parse-error baseline shares pointers") + } + copied := cloneFramework(f) + copied.Judge.Evidence[0] = "changed" + if f.Judge.Evidence[0] != "old" { + t.Fatal("clone shares judgement evidence") + } +} + +// Versions stated only in the body are picked by the provider, all kept +// products in one request, each from its own candidates. +func TestInspectVersionsEveryProduct(t *testing.T) { + m := newMock() + m.presence["jenkins"] = OptionRunning + accepted, err := New(m).Inspect(context.Background(), []byte(bodyRaw), testFrames()) + accepted = accepted.Accepted() + if err != nil { + t.Fatal(err) + } + if accepted["jenkins"].Version != "2.401.3" || accepted["nginx"].Version != "1.24.0" { + t.Fatalf("versions: jenkins %q nginx %q", accepted["jenkins"].Version, accepted["nginx"].Version) + } + if len(m.requests) != 2 || !strings.HasPrefix(strings.Join(m.requests[1], ","), "version_apachetomcat,version_jenkins") { + t.Fatalf("requests: %v", m.requests) + } + if _, ok := m.lastState["version_strings"]; !ok { + t.Errorf("version strings not in the state: %v", m.lastState) + } +} + +func TestVersionNeedsConfidence(t *testing.T) { + m := newMock() + m.presence["jenkins"] = OptionRunning + m.confidence = 0.2 + accepted, err := New(m).Inspect(context.Background(), []byte(bodyRaw), testFrames()) + accepted = accepted.Accepted() + if err != nil { + t.Fatal(err) + } + if v := accepted["jenkins"].Version; v != "" { + t.Fatalf("low-confidence version written: %q", v) + } +} diff --git a/judge/internal/evidence/evidence.go b/judge/internal/evidence/evidence.go new file mode 100644 index 0000000..d30f223 --- /dev/null +++ b/judge/internal/evidence/evidence.go @@ -0,0 +1,154 @@ +// Package evidence holds the text primitives shared by the judge and the +// fingerprint generator: product name folding, name candidates found in a +// response, and the patterns for titles, generator metas and versions. +package evidence + +import ( + "net/http" + "regexp" + "sort" + "strconv" + "strings" + "unicode" + "unicode/utf8" +) + +// VersionToken matches a version, keeping release suffixes and calendar +// versions (e.g. 1.0.2q, 1.0.0-rc.35, 2026.9.23+3cd69d30e). +const VersionToken = `[0-9]{1,4}(?:\.[0-9]{1,4}){1,3}(?:[A-Za-z]+[0-9]*|[-+][0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*)?` + +var ( + Title = regexp.MustCompile(`(?is)]*>(.*?)`) + Generator = regexp.MustCompile(`(?is)]+name=["']generator["'][^>]*content=["']([^"']+)|]+content=["']([^"']+)["'][^>]*name=["']generator["']`) + // NameVersion is a trailing version after a product name ("nginx/1.24", "Jenkins v2"). + NameVersion = regexp.MustCompile(`(?i)[/\s]+[vV]?(?:` + VersionToken + `|[0-9]+)\s*$`) + + space = regexp.MustCompile(`\s+`) + nameWords = regexp.MustCompile(`[A-Z][A-Za-z0-9!+._]{2,}(?:[ -][A-Z][A-Za-z0-9!+._]{2,}){0,2}|[\p{Han}]{2,20}`) +) + +// Clean decodes common entities and collapses whitespace. +func Clean(s string) string { + s = strings.NewReplacer(" ", " ", "&", "&", "<", "<", ">", ">", """, `"`).Replace(s) + return strings.TrimSpace(space.ReplaceAllString(s, " ")) +} + +var genericSuffixes = []string{"companyproducts", "公司产品", "product", "products", "产品", "operatingsystem", "操作系统", "system"} + +// genericWords are trailing words engines add to a product name ("Apache +// HTTP Server", "apache-web-server", "Discuz! X"). They are dropped only as +// whole words, so "lighttpd" keeps its "httpd". +var genericWords = map[string]bool{"http": true, "httpd": true, "server": true, "web": true, "webserver": true, "x": true, "cms": true, "oa": true} + +// NormalizeName folds the spellings different engines use for one product +// ("Apache-Tomcat", "apache tomcat", "Apache HTTP Server", "apache") into one +// key. It never merges names whose remaining letters differ. +func NormalizeName(name string) string { + words := strings.FieldsFunc(strings.ToLower(name), func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) }) + for len(words) > 1 && genericWords[words[len(words)-1]] { + words = words[:len(words)-1] + } + key := strings.Join(words, "") + for _, s := range genericSuffixes { + if len(key) > len(s)+2 && strings.HasSuffix(key, s) { + key = strings.TrimSuffix(key, s) + } + } + return key +} + +// usableKey drops keys too short to be meaningful evidence ("oa", "cms"). +func usableKey(k string) bool { + n := utf8.RuneCountInString(k) + if n == len(k) { // ascii + return n >= 4 + } + return n >= 2 +} + +// GenericName reports whether a normalized name is a generic interface word +// ("login", "控制台") or an HTTP status ("404notfound") rather than a product. +func GenericName(key string) bool { + switch key { + case "login", "signin", "welcome", "dashboard", "admin", "console", "home", "index", "server", "error", "notfound", "password", "username", "management", "system", "登录", "首页", "管理", "用户", "密码", "控制台": + return true + } + if len(key) > 3 { + if _, err := strconv.Atoi(key[:3]); err == nil { + key = key[3:] + } + } + return statusTexts[key] +} + +// statusTexts are the normalized HTTP reason phrases ("notfound"). +var statusTexts = func() map[string]bool { + m := map[string]bool{} + for code := 100; code < 600; code++ { + if text := http.StatusText(code); text != "" { + m[NormalizeName(text)] = true + } + } + return m +}() + +// Names returns up to 20 product name candidates found in a response, most +// telling source first: generator meta, product headers, title parts, asset +// path words, then capitalized words and CJK runs of the visible text. +func Names(generator, title, text string, headers map[string]string, assets []string) []string { + type candidate struct { + name string + weight int + } + byKey := map[string]candidate{} + add := func(text string, weight int) { + text = strings.Trim(strings.TrimSpace(NameVersion.ReplaceAllString(Clean(text), "")), "-:|/ ") + if text == "" || utf8.RuneCountInString(text) > 40 || !usableKey(strings.ToLower(text)) { + return + } + key := NormalizeName(text) + if key == "" || GenericName(key) { + return + } + if prev, ok := byKey[key]; !ok || weight > prev.weight { + byKey[key] = candidate{text, weight} + } + } + add(generator, 5) + for k, v := range headers { + if strings.HasPrefix(k, "X-") || k == "Server" || k == "Product" { + add(strings.Split(v, "/")[0], 4) + } + } + for _, part := range strings.FieldsFunc(title, func(r rune) bool { return r == '|' || r == '-' || r == ':' || r == '–' }) { + add(part, 4) + } + for _, asset := range assets { + for _, word := range strings.FieldsFunc(asset, func(r rune) bool { return r == '/' || r == '_' }) { + add(word, 2) + } + } + for _, segment := range strings.FieldsFunc(text, func(r rune) bool { return r == '.' || r == ':' || r == '。' || r == '：' || r == '|' || r == '，' }) { + for _, match := range nameWords.FindAllString(segment, 20) { + add(match, 1) + } + } + list := make([]candidate, 0, len(byKey)) + for _, c := range byKey { + list = append(list, c) + } + sort.Slice(list, func(a, b int) bool { + if list[a].weight != list[b].weight { + return list[a].weight > list[b].weight + } + return list[a].name < list[b].name + }) + if len(list) > 20 { + list = list[:20] + } + out := make([]string, len(list)) + for i, c := range list { + out[i] = c.name + } + return out +} diff --git a/judge/internal/evidence/evidence_test.go b/judge/internal/evidence/evidence_test.go new file mode 100644 index 0000000..062d859 --- /dev/null +++ b/judge/internal/evidence/evidence_test.go @@ -0,0 +1,19 @@ +package evidence + +import "testing" + +func TestGenericName(t *testing.T) { + for key, generic := range map[string]bool{ + NormalizeName("404 Not Found"): true, + NormalizeName("403 Forbidden"): true, + NormalizeName("502 Bad Gateway"): true, + NormalizeName("Forbidden"): true, + NormalizeName("login"): true, + NormalizeName("SearXNG"): false, + NormalizeName("404 Tools"): false, + } { + if GenericName(key) != generic { + t.Errorf("GenericName(%q) = %v", key, !generic) + } + } +} diff --git a/judge/internal/evidence/page.go b/judge/internal/evidence/page.go new file mode 100644 index 0000000..a2ccc0c --- /dev/null +++ b/judge/internal/evidence/page.go @@ -0,0 +1,134 @@ +package evidence + +import ( + "bufio" + "bytes" + "io" + "net/http" + "regexp" + "strings" + "unicode/utf8" +) + +// Page is the compact, named view of one HTTP response that providers judge +// (serialized as the request state). A provider only judges the evidence it +// is given and loses accuracy on large states full of irrelevant detail, so +// the fields below are the ones that carry fingerprint signal. +type Page struct { + Status string `json:"status"` + Headers map[string]string `json:"headers,omitempty"` + Cookies []string `json:"cookie_names,omitempty"` + Title string `json:"title,omitempty"` + Generator string `json:"meta_generator,omitempty"` + Description string `json:"meta_description,omitempty"` + Scripts []string `json:"script_src,omitempty"` + Styles []string `json:"stylesheet_href,omitempty"` + InlineHints []string `json:"inline_script_starts,omitempty"` // first chars of inline scripts, e.g. "(function(w,d,s,l,i){ ... GTM" + Comments []string `json:"html_comments,omitempty"` + Forms []string `json:"form_inputs,omitempty"` + Text string `json:"visible_text,omitempty"` + + Raw []byte `json:"-"` + Header http.Header `json:"-"` +} + +var ( + // Headers that carry no fingerprint signal, or only noise; every other + // header is kept, since products announce themselves in custom headers + // ("product: Z-BlogPHP 1.7.4", "X-Jenkins", "kbn-name"). + skipHeaders = map[string]bool{"Date": true, "Content-Length": true, "Content-Type": true, "Expires": true, "Cache-Control": true, + "Pragma": true, "Etag": true, "Last-Modified": true, "Age": true, "Connection": true, "Keep-Alive": true, "Accept-Ranges": true, + "Vary": true, "Set-Cookie": true, "Content-Security-Policy": true, "Content-Security-Policy-Report-Only": true, "Report-To": true, + "Nel": true, "Permissions-Policy": true, + // per-request values: no signal, and they would defeat the cache + "X-Request-Id": true, "X-Amzn-Trace-Id": true, "Cf-Ray": true, "X-Runtime": true, "Server-Timing": true} + + reDescription = regexp.MustCompile(`(?is)]+name=["']description["'][^>]*content=["']([^"']+)|]+content=["']([^"']+)["'][^>]*name=["']description["']`) + reScript = regexp.MustCompile(`(?is)]+src=["']([^"']+)`) + reStyle = regexp.MustCompile(`(?is)]+rel=["']?stylesheet[^>]*href=["']([^"']+)|]+href=["']([^"']+)["'][^>]*rel=["']?stylesheet`) + reInline = regexp.MustCompile(`(?is)]*)?>(.*?)`) + reComment = regexp.MustCompile(`(?s)`) + reInput = regexp.MustCompile(`(?is)]*>`) + reAttrName = regexp.MustCompile(`(?is)\b(?:name|id)=["']([^"']+)`) + reAttrType = regexp.MustCompile(`(?is)\btype=["']([^"']+)`) + reDropBlock = regexp.MustCompile(`(?is)<(script|style|noscript)[^>]*>.*?`) + reTag = regexp.MustCompile(`(?s)<[^>]+>`) +) + +const ( + maxHeaderValue = 160 + maxText = 1500 // visible text runes +) + +// NewPage parses a raw HTTP response (head and body, as rule engines take it). +func NewPage(raw []byte) (*Page, error) { + resp, err := http.ReadResponse(bufio.NewReader(bytes.NewReader(raw)), nil) + if err != nil { + return nil, err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + s := &Page{Status: resp.Status, Headers: map[string]string{}, Raw: raw, Header: resp.Header.Clone()} + for k, vs := range resp.Header { + if skipHeaders[k] || strings.HasPrefix(k, "X-Crawler-") || len(vs) == 0 { + continue + } + s.Headers[k] = Truncate(strings.Join(vs, ", "), maxHeaderValue) + } + for _, c := range resp.Cookies() { + s.Cookies = append(s.Cookies, c.Name) + } + html := string(body) + if m := Title.FindStringSubmatch(html); m != nil { + s.Title = clean(m[1]) + } + if m := Generator.FindStringSubmatch(html); m != nil { + s.Generator = m[1] + m[2] + } + if m := reDescription.FindStringSubmatch(html); m != nil { + s.Description = Truncate(clean(m[1]+m[2]), 300) + } + for _, m := range reScript.FindAllStringSubmatch(html, 8) { + s.Scripts = append(s.Scripts, m[1]) + } + for _, m := range reStyle.FindAllStringSubmatch(html, 8) { + s.Styles = append(s.Styles, m[1]+m[2]) + } + for _, m := range reInline.FindAllStringSubmatch(html, -1) { + if code := clean(m[1]); len(code) > 20 && len(s.InlineHints) < 5 { + s.InlineHints = append(s.InlineHints, Truncate(code, 100)) + } + } + for _, m := range reComment.FindAllStringSubmatch(html, -1) { + if c := clean(m[1]); len(c) > 3 && len(s.Comments) < 5 && !strings.HasPrefix(c, "[if") { + s.Comments = append(s.Comments, Truncate(c, 80)) + } + } + for _, in := range reInput.FindAllString(html, 12) { + name, typ := "", "text" + if m := reAttrName.FindStringSubmatch(in); m != nil { + name = m[1] + } + if m := reAttrType.FindStringSubmatch(in); m != nil { + typ = m[1] + } + if name != "" { + s.Forms = append(s.Forms, name+":"+typ) + } + } + text := clean(reTag.ReplaceAllString(reDropBlock.ReplaceAllString(html, " "), " ")) + s.Text = Truncate(text, maxText) + return s, nil +} + +func clean(s string) string { return Clean(s) } + +func Truncate(s string, max int) string { + if utf8.RuneCountInString(s) <= max { + return s + } + return string([]rune(s)[:max]) + "…" +} diff --git a/judge/judge.go b/judge/judge.go new file mode 100644 index 0000000..97e5d69 --- /dev/null +++ b/judge/judge.go @@ -0,0 +1,44 @@ +package judge + +import ( + "context" + "fmt" + "math" + + "github.com/chainreactors/fingers/judge/internal/evidence" + "github.com/chainreactors/utils/jev" +) + +// Judge reviews fingerprint hits through jev claims. Configure it before +// sharing it across a scan. Caching, request accounting and rate limiting +// wrap its Provider (see jev.Cached). +type Judge struct { + Provider jev.Provider + MinConfidence float64 // below it a ruling resolves to Insufficient + DropInsufficient bool // also reject hits the evidence decides neither way +} + +// New reviews hits with p at jev.DefaultMinConfidence. +func New(p jev.Provider) *Judge { + return &Judge{Provider: p, MinConfidence: jev.DefaultMinConfidence} +} + +func NormalizeName(name string) string { return evidence.NormalizeName(name) } + +func (j *Judge) judge(ctx context.Context, state interface{}, claims map[string]jev.Claim) (map[string]jev.Ruling, error) { + return jev.Judge(ctx, j.Provider, state, claims) +} + +// ready rejects a canceled context or an invalid configuration up front. +func (j *Judge) ready(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + if math.IsNaN(j.MinConfidence) || j.MinConfidence < 0 || j.MinConfidence > 1 { + return fmt.Errorf("judge: invalid MinConfidence %v", j.MinConfidence) + } + return nil +} + +// insufficientDescription describes the abstention every claim offers. +const insufficientDescription = "The evidence decides neither way." diff --git a/judge/judge_test.go b/judge/judge_test.go new file mode 100644 index 0000000..f64ff89 --- /dev/null +++ b/judge/judge_test.go @@ -0,0 +1,130 @@ +package judge + +import ( + "context" + "encoding/json" + "errors" + "sort" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/chainreactors/fingers/common" + "github.com/chainreactors/utils/jev" +) + +const jenkinsRaw = "HTTP/1.1 200 OK\r\nServer: nginx/1.24.0\r\nX-Jenkins: 2.401.3\r\nSet-Cookie: JSESSIONID.1=x; Path=/\r\nContent-Type: text/html\r\n\r\n" + + `Sign in [Jenkins]` + + `

We moved here from WordPress.

` + + `` + +// mock is a Provider ruling from a table keyed by claim kind and the +// product named in backticks; it records the claim keys of each call. +type mock struct { + mu sync.Mutex + delay time.Duration + calls int64 + requests [][]string + fail bool + confidence float64 + presence map[string]string // product -> presence option + versions map[string]string // product -> version picked, if among the options + coverage string + lastState map[string]interface{} +} + +func newMock() *mock { + return &mock{ + confidence: 0.95, + presence: map[string]string{"wordpress": OptionMentioned, "apache tomcat": OptionRunning, "prototype": OptionRunning}, + versions: map[string]string{"jenkins": "2.401.3", "nginx": "1.24.0"}, + } +} + +// named is the first product named in backticks, skipping state field names. +func named(q jev.Claim) string { + parts := strings.Split(q.Statement, "`") + for i := 1; i < len(parts); i += 2 { + if parts[i] != "version_strings" && !strings.HasPrefix(parts[i], "matches.") { + return parts[i] + } + } + return "" +} + +func (m *mock) ID() string { return "mock" } + +func (m *mock) Judge(ctx context.Context, state interface{}, claims map[string]jev.Claim) (map[string]jev.Ruling, error) { + atomic.AddInt64(&m.calls, 1) + time.Sleep(m.delay) + m.mu.Lock() + defer m.mu.Unlock() + if m.fail { + return nil, errors.New("provider down") + } + data, _ := json.Marshal(state) + _ = json.Unmarshal(data, &m.lastState) + rulings := map[string]jev.Ruling{} + var keys []string + for k, q := range claims { + keys = append(keys, k) + name := strings.ToLower(named(q)) + switch { + case strings.HasPrefix(k, "presence_"): + v := m.presence[name] + if v == "" { + v = jev.OptionInsufficient + } + rulings[k] = jev.Ruling{Option: v, Confidence: m.confidence} + case strings.HasPrefix(k, "version_"): + option := notStated + if v, ok := m.versions[name]; ok { + if _, offered := q.Options[v]; offered { + option = v + } + } + rulings[k] = jev.Ruling{Option: option, Confidence: m.confidence} + case k == "coverage": + rulings[k] = jev.Ruling{Option: m.coverage, Confidence: m.confidence} + default: + rulings[k] = jev.Ruling{Option: jev.OptionInsufficient, Confidence: m.confidence} + } + } + sort.Strings(keys) + m.requests = append(m.requests, keys) + return rulings, nil +} + +func testFrames() common.Frameworks { + fs := common.Frameworks{} + fs.Add(common.NewFramework("nginx", common.FrameFromFingers)) + fs.Add(common.NewFramework("wordpress", common.FrameFromGoby)) + fs.Add(common.NewFramework("jenkins", common.FrameFromFingerprintHub)) + fs.Add(common.NewFramework("apache-tomcat", common.FrameFromFingers)) + fs.Add(common.NewFramework("apache tomcat", common.FrameFromWappalyzer)) + fs.Add(common.NewFramework("hsts", common.FrameFromWappalyzer)) + return fs +} + +// bodyRaw states versions only in the body, so the provider has to pick them. +const bodyRaw = "HTTP/1.1 200 OK\r\nServer: nginx\r\nSet-Cookie: JSESSIONID.1=x; Path=/\r\n\r\n" + + `Sign in [Jenkins]` + + `

We moved here from WordPress.

Jenkins 2.401.3, nginx 1.24.0
` + +type rulingProvider func(map[string]jev.Claim) map[string]jev.Ruling + +func (rulingProvider) ID() string { return "ruling-provider" } + +func (p rulingProvider) Judge(_ context.Context, _ interface{}, claims map[string]jev.Claim) (map[string]jev.Ruling, error) { + return p(claims), nil +} + +type failVersionProvider struct{ *mock } + +func (p failVersionProvider) Judge(ctx context.Context, state interface{}, claims map[string]jev.Claim) (map[string]jev.Ruling, error) { + if _, ok := claims["version_jenkins"]; ok { + return nil, errors.New("version unavailable") + } + return p.mock.Judge(ctx, state, claims) +} diff --git a/judge/maintain/coverage.go b/judge/maintain/coverage.go new file mode 100644 index 0000000..85d60c9 --- /dev/null +++ b/judge/maintain/coverage.go @@ -0,0 +1,68 @@ +package maintain + +import ( + "context" + "fmt" + "sort" + + "github.com/chainreactors/fingers/judge" + "github.com/chainreactors/fingers/judge/internal/evidence" + "github.com/chainreactors/utils/jev" +) + +// Coverage options: what the samples of one page show beside the products +// the rules report on them. +const ( + CoverageExplained = "explained" // holds: the reported products account for the page + CoverageCustom = "custom_site" // holds: content one organization wrote; no packaged product to fingerprint + CoverageMissing = "missing" // refuted: the stock page of a packaged product the rules miss +) + +const maxCoverageSamples = 3 + +// Coverage rules on the claim that the products the rules report account +// for samples: responses of one page, ideally from unrelated hosts. It is +// the offline counterpart of presence claims. A Refuted ruling +// (CoverageMissing) is a missed product for a person to name, e.g. from +// Cluster candidates; the provider never names one, and reported +// are the claim under review, never options to pick from. +func coverage(ctx context.Context, j *judge.Judge, samples [][]byte, reported []string) (jev.Ruling, jev.Outcome, error) { + var pages []*evidence.Page + for _, raw := range samples { + if len(pages) == maxCoverageSamples { + break + } + p, err := evidence.NewPage(raw) + if err != nil { + return jev.Ruling{}, jev.Insufficient, err + } + pages = append(pages, p) + } + if len(pages) == 0 { + return jev.Ruling{}, jev.Insufficient, fmt.Errorf("coverage: no samples") + } + names := append([]string{}, reported...) + sort.Strings(names) + state := map[string]interface{}{"samples": pages, "reported": names} + if len(pages) > 1 { + state["note"] = "the samples are the same page served by unrelated hosts: a page many unrelated operators serve alike is usually a packaged product, not content one organization wrote" + } + claim := jev.Claim{ + Statement: "`reported` lists the products fingerprint rules report on these `samples`. " + + "Claim: they account for the page, that is, the page is not the stock interface of a packaged product missing from `reported`. " + + "A stock interface is a product's own login, console, default, error or application page, " + + "which looks the same on every deployment of it except for branding, host names and data. " + + "Servers, CDNs, frameworks, libraries and databases underneath an application do not account for the application's own page.", + Options: map[string]jev.Option{ + jev.OptionInsufficient: {Description: "The evidence does not decide the claim.", Outcome: jev.Insufficient}, + CoverageExplained: {Description: "The page is the stock interface of a product in `reported`, or shows no packaged product beyond them, such as a bare server default or error page.", Outcome: jev.Holds}, + CoverageCustom: {Description: "Content one organization wrote for its own purpose: articles, company pages, shops, forums, custom-built portals.", Outcome: jev.Holds}, + CoverageMissing: {Description: "The page is the stock interface of a packaged product that is not in `reported`.", Outcome: jev.Refuted}, + }, + } + rulings, err := jev.Judge(ctx, j.Provider, state, map[string]jev.Claim{"coverage": claim}) + if err != nil { + return jev.Ruling{}, jev.Insufficient, err + } + return rulings["coverage"], claim.Resolve(rulings["coverage"], j.MinConfidence), nil +} diff --git a/judge/maintain/discover.go b/judge/maintain/discover.go new file mode 100644 index 0000000..91d89ad --- /dev/null +++ b/judge/maintain/discover.go @@ -0,0 +1,231 @@ +package maintain + +import ( + "context" + "regexp" + "sort" + "strings" + + "github.com/chainreactors/fingers/common" + "github.com/chainreactors/fingers/judge" + "github.com/chainreactors/fingers/judge/internal/evidence" + "github.com/chainreactors/utils/jev" +) + +// Sample is one scanned response for discovery. +type Sample struct { + ID, Host string + Raw []byte + Accepted common.Frameworks // Inspect(Raw).Accepted() for Raw +} + +// Cluster is one page served by several hosts, as Discover found it. +type Cluster struct { + Samples []string `json:"samples"` + Hosts int `json:"hosts"` + // Reported are the products the samples' rules report. + Reported []string `json:"reported,omitempty"` + // Coverage is the ruling on the claim that Reported account for the + // page: Refuted (CoverageMissing) marks a missed product. + Coverage jev.Ruling `json:"coverage"` + Outcome jev.Outcome `json:"outcome"` + // Candidates are names code finds in the samples, for the person who + // names a missed product; they are never offered to the provider. + Candidates []string `json:"candidates,omitempty"` +} + +// MinSimilarity is the structural overlap (Jaccard of asset paths, form +// fields, cookie and custom header names, generator and title words) at +// which two pages are one page. On the 2026-09-26 corpus it links 52 of 54 +// cross-host pairs of one product and 22 of 2278 other pairs, mostly stock +// pages of one known product. +const MinSimilarity = 0.4 + +// Discover clusters samples that are one page on different hosts and, once +// per cluster of at least minHosts hosts, evaluates whether +// the products the rules report account for it. Code does the clustering +// and the host count: a page served the same by unrelated hosts is what a +// fingerprint can describe. +func Discover(ctx context.Context, j *judge.Judge, samples []Sample, minHosts int) ([]*Cluster, error) { + pages := make([]map[string]bool, len(samples)) + for i, s := range samples { + p, err := evidence.NewPage(s.Raw) + if err != nil { + continue + } + pages[i] = features(p) + } + parent := make([]int, len(samples)) + for i := range parent { + parent[i] = i + } + var find func(int) int + find = func(i int) int { + if parent[i] != i { + parent[i] = find(parent[i]) + } + return parent[i] + } + for a := range samples { + for b := a + 1; b < len(samples); b++ { + if pages[a] != nil && pages[b] != nil && jaccard(pages[a], pages[b]) >= MinSimilarity { + parent[find(a)] = find(b) + } + } + } + groups := map[int][]Sample{} + var roots []int + for i, s := range samples { + if pages[i] == nil { + continue + } + r := find(i) + if groups[r] == nil { + roots = append(roots, r) + } + groups[r] = append(groups[r], s) + } + var out []*Cluster + for _, r := range roots { + c := &Cluster{} + var raws [][]byte + hosts := map[string]bool{} + for _, s := range groups[r] { + c.Samples = append(c.Samples, s.ID) + if !hosts[s.Host] { + hosts[s.Host] = true + raws = append(raws, s.Raw) + } + } + c.Hosts = len(hosts) + if c.Hosts < minHosts { + continue + } + c.Reported = reported(groups[r]) + ruling, outcome, err := coverage(ctx, j, raws, c.Reported) + if err != nil { + return out, err + } + c.Coverage, c.Outcome = ruling, outcome + c.Candidates = candidates(raws) + out = append(out, c) + } + sort.SliceStable(out, func(a, b int) bool { return out[a].Hosts > out[b].Hosts }) + return out, nil +} + +// reported are the products any sample reports, one spelling each. +func reported(samples []Sample) []string { + seen := map[string]bool{} + var out []string + for _, s := range samples { + for _, f := range s.Accepted { + if key := judge.NormalizeName(f.Name); key != "" && !seen[key] { + seen[key] = true + out = append(out, f.Name) + } + } + } + sort.Strings(out) + return out +} + +const maxCandidates = 20 + +// candidates are the names that occur in two samples (or the only one), +// most telling source first: generator meta, product headers, title parts, +// asset path words, then capitalized words of the visible text. +func candidates(raws [][]byte) []string { + need := 2 + if len(raws) < need { + need = len(raws) + } + count := map[string]int{} + label := map[string]string{} + var order []string + for _, raw := range raws { + p, err := evidence.NewPage(raw) + if err != nil { + continue + } + seen := map[string]bool{} + for _, n := range evidence.Names(p.Generator, p.Title, p.Text, p.Headers, append(append([]string(nil), p.Scripts...), p.Styles...)) { + key := judge.NormalizeName(n) + if seen[key] { + continue + } + seen[key] = true + if count[key] == 0 { + label[key] = n + order = append(order, key) + } + count[key]++ + } + } + var out []string + for _, key := range order { + if count[key] >= need && len(out) < maxCandidates { + out = append(out, label[key]) + } + } + return out +} + +var reVolatile = regexp.MustCompile(`[0-9a-f]{6,}|[0-9]+`) + +// commonHeaders are X- headers any stack sets: they tell pages of one +// product apart from nothing. +var commonHeaders = map[string]bool{ + "x-content-type-options": true, "x-frame-options": true, "x-xss-protection": true, "x-request-id": true, + "x-download-options": true, "x-permitted-cross-domain-policies": true, "x-dns-prefetch-control": true, + "x-robots-tag": true, "x-cache": true, "x-served-by": true, "x-forwarded-for": true, "x-forwarded-proto": true, +} + +// features are the structural traits of a page that its deployments share: +// asset paths without queries or hashes, form fields, cookie and custom +// header names, the generator and title words. +func features(p *evidence.Page) map[string]bool { + out := map[string]bool{} + fold := func(s string) string { + if i := strings.IndexAny(s, "?#"); i >= 0 { + s = s[:i] + } + return reVolatile.ReplaceAllString(strings.ToLower(s), "") + } + for _, x := range append(append([]string(nil), p.Scripts...), p.Styles...) { + out["asset:"+fold(x)] = true + } + for _, x := range p.Forms { + out["form:"+strings.ToLower(x)] = true + } + for _, x := range p.Cookies { + out["cookie:"+fold(x)] = true + } + for k := range p.Headers { + if k := strings.ToLower(k); strings.HasPrefix(k, "x-") && !commonHeaders[k] { + out["header:"+k] = true + } + } + if p.Generator != "" { + out["generator:"+judge.NormalizeName(reVolatile.ReplaceAllString(p.Generator, ""))] = true + } + for _, w := range strings.FieldsFunc(strings.ToLower(p.Title), func(r rune) bool { return strings.ContainsRune(" -|:.·–", r) }) { + if len(w) > 2 { + out["title:"+w] = true + } + } + return out +} + +func jaccard(a, b map[string]bool) float64 { + if len(a) == 0 || len(b) == 0 { + return 0 + } + n := 0 + for k := range a { + if b[k] { + n++ + } + } + return float64(n) / float64(len(a)+len(b)-n) +} diff --git a/judge/maintain/ledger.go b/judge/maintain/ledger.go new file mode 100644 index 0000000..04fa198 --- /dev/null +++ b/judge/maintain/ledger.go @@ -0,0 +1,151 @@ +// Package maintain audits a fingerprint library offline from judged scan +// results. Ledger asks the provider nothing new: a rule whose presence +// claims are refuted again and again is a junk rule. Discover clusters one +// page served by several hosts and asks one coverage claim per cluster; a +// refuted one is a missed product, named by a person from code's +// candidates and turned into rules by judge/gen. +package maintain + +import ( + "sort" + "sync" + + "github.com/chainreactors/fingers/common" + "github.com/chainreactors/utils/jev" +) + +// RuleKey identifies the rule behind a hit as precisely as the engine +// reports it: engines that record MatchDetail (the native fingers engine) +// down to the matcher, the others by fingerprint name. +type RuleKey struct { + Engine string `json:"engine"` + Name string `json:"name"` + Rule int `json:"rule,omitempty"` + Matcher string `json:"matcher,omitempty"` +} + +// RuleStats are the rulings on the presence claims of one rule's hits. +type RuleStats struct { + RuleKey + Outcomes map[jev.Outcome]int `json:"outcomes"` // holds, refuted, insufficient + Options map[string]int `json:"options"` // the options behind them + Samples map[jev.Outcome][]string `json:"samples"` // outcome -> page ids, up to maxSamples +} + +const maxSamples = 5 + +func (s *RuleStats) Hits() int { + total := 0 + for _, count := range s.Outcomes { + total += count + } + return total +} + +// RefutedFraction is the share of judged hits whose presence claim is refuted. +func (s *RuleStats) RefutedFraction() float64 { + if s.Hits() == 0 { + return 0 + } + return float64(s.Outcomes[jev.Refuted]) / float64(s.Hits()) +} + +// Ledger accumulates rulings per rule over a scan or corpus. It is safe +// for concurrent use. +type Ledger struct { + mu sync.Mutex + rules map[RuleKey]*RuleStats +} + +func NewLedger() *Ledger { return &Ledger{rules: map[RuleKey]*RuleStats{}} } + +// Add records the rulings of one page's Inspect result. Unjudged hits are +// skipped; duplicate spellings count for their own rule with the ruling of +// the product they spell. +func (l *Ledger) Add(page string, inspected common.Frameworks) { + l.mu.Lock() + defer l.mu.Unlock() + for _, f := range inspected { + if f == nil || f.Judge == nil || f.Judge.Outcome == "" { + continue + } + o := jev.Outcome(f.Judge.Outcome) + for _, key := range ruleKeys(f) { + s := l.rules[key] + if s == nil { + s = &RuleStats{RuleKey: key, Outcomes: map[jev.Outcome]int{}, Options: map[string]int{}, Samples: map[jev.Outcome][]string{}} + l.rules[key] = s + } + s.Outcomes[o]++ + s.Options[f.Judge.Option]++ + if len(s.Samples[o]) < maxSamples { + s.Samples[o] = append(s.Samples[o], page) + } + } + } +} + +// Report lists the rules with at least minHits judged hits whose refuted +// share is at least minRefuted, most refuted first: the junk rule +// candidates, each with the pages to review. +func (l *Ledger) Report(minHits int, minRefuted float64) []*RuleStats { + l.mu.Lock() + defer l.mu.Unlock() + var out []*RuleStats + for _, s := range l.rules { + if s.Hits() >= minHits && s.RefutedFraction() >= minRefuted { + snapshot := *s + snapshot.Outcomes, snapshot.Options, snapshot.Samples = map[jev.Outcome]int{}, map[string]int{}, map[jev.Outcome][]string{} + for k, v := range s.Outcomes { + snapshot.Outcomes[k] = v + } + for k, v := range s.Options { + snapshot.Options[k] = v + } + for k, v := range s.Samples { + snapshot.Samples[k] = append([]string(nil), v...) + } + out = append(out, &snapshot) + } + } + sort.Slice(out, func(a, b int) bool { + ra, rb := out[a].Outcomes[jev.Refuted], out[b].Outcomes[jev.Refuted] + if ra != rb { + return ra > rb + } + ka, kb := out[a].RuleKey, out[b].RuleKey + if ka.Name != kb.Name { + return ka.Name < kb.Name + } + if ka.Engine != kb.Engine { + return ka.Engine < kb.Engine + } + if ka.Rule != kb.Rule { + return ka.Rule < kb.Rule + } + return ka.Matcher < kb.Matcher + }) + return out +} + +func ruleKeys(f *common.Framework) []RuleKey { + var engines []string + for from := range f.Froms { + if from >= common.FrameFromFingers { + engines = append(engines, from.String()) + } + } + if len(engines) == 0 { + engines = []string{"unknown"} + } + sort.Strings(engines) + var keys []RuleKey + for _, e := range engines { + key := RuleKey{Engine: e, Name: f.Name} + if d := f.MatchDetail; d != nil && e == common.FrameFromFingers.String() { + key.Rule, key.Matcher = d.RuleIndex, d.MatcherValue + } + keys = append(keys, key) + } + return keys +} diff --git a/judge/maintain/maintain_test.go b/judge/maintain/maintain_test.go new file mode 100644 index 0000000..e140e48 --- /dev/null +++ b/judge/maintain/maintain_test.go @@ -0,0 +1,148 @@ +package maintain + +import ( + "context" + "encoding/json" + "fmt" + "testing" + + "github.com/chainreactors/fingers/common" + "github.com/chainreactors/fingers/judge" + "github.com/chainreactors/utils/jev" +) + +func judged(name string, from common.From, option string, outcome jev.Outcome, detail *common.MatchDetail) *common.Framework { + f := common.NewFramework(name, from) + f.Judge = &common.Judgement{Option: option, Outcome: outcome.String()} + f.MatchDetail = detail + return f +} + +func TestLedgerReportsJunkRules(t *testing.T) { + l := NewLedger() + webp := &common.MatchDetail{RuleIndex: 2, MatcherValue: "webp"} + for i := 0; i < 4; i++ { + fs := common.Frameworks{} + fs.Add(judged("webp_server_go", common.FrameFromFingers, judge.OptionUnrelated, jev.Refuted, webp)) + fs.Add(judged("nginx", common.FrameFromGoby, judge.OptionDeclared, jev.Holds, nil)) + option, outcome := judge.OptionMentioned, jev.Refuted + if i == 0 { + option, outcome = judge.OptionRunning, jev.Holds + } + fs.Add(judged("git", common.FrameFromEhole, option, outcome, nil)) + fs.Add(common.NewFramework("unjudged", common.FrameFromGoby)) + l.Add(fmt.Sprintf("page%d", i), fs) + } + got := l.Report(3, 0.5) + if len(got) != 2 || got[0].Name != "webp_server_go" || got[0].Matcher != "webp" || got[0].Rule != 2 || got[0].Engine != "fingers" || + got[1].Name != "git" || got[1].Engine != "ehole" || got[1].RefutedFraction() != 0.75 || len(got[1].Samples[jev.Refuted]) != 3 || got[1].Options[judge.OptionMentioned] != 3 { + t.Fatalf("report: %+v", got) + } +} + +// coverer rules every coverage claim with one option. +type coverer string + +func (coverer) ID() string { return "coverer" } + +func (c coverer) Judge(_ context.Context, _ interface{}, qs map[string]jev.Claim) (map[string]jev.Ruling, error) { + out := map[string]jev.Ruling{} + for k := range qs { + out[k] = jev.Ruling{Option: string(c), Confidence: 0.9} + } + return out, nil +} + +func TestDiscoverClaimsCoverageOfPagesSharedByHosts(t *testing.T) { + orion := func(host string) []byte { + return []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\nX-Orion-Node: " + host + "\r\n\r\nOrion Console" + + `` + + `

Welcome to Orion on ` + host + `

`) + } + blog := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nMy Blog

Hello

") + nginx := common.Frameworks{} + nginx.Add(common.NewFramework("nginx", common.FrameFromFingers)) + var samples []Sample + for _, h := range []string{"a.example", "b.example", "c.example"} { + samples = append(samples, Sample{ID: h, Host: h, Raw: orion(h), Accepted: nginx}) + } + samples = append(samples, Sample{ID: "blog", Host: "blog.example", Raw: blog, Accepted: nginx}) + + clusters, err := Discover(context.Background(), judge.New(coverer(CoverageMissing)), samples, 2) + if err != nil { + t.Fatal(err) + } + c := clusters[0] + if len(clusters) != 1 || c.Hosts != 3 || c.Outcome != jev.Refuted || len(c.Reported) != 1 || c.Reported[0] != "nginx" { + t.Fatalf("clusters: %+v", clusters) + } + // Code, not the provider, offers the names a person picks from. + found := false + for _, n := range c.Candidates { + found = found || n == "Orion Console" + } + if !found { + t.Fatalf("candidates: %v", c.Candidates) + } + clusters, err = Discover(context.Background(), judge.New(coverer(CoverageExplained)), samples, 2) + if err != nil || len(clusters) != 1 || clusters[0].Outcome != jev.Holds { + t.Fatalf("explained: %+v %v", clusters, err) + } +} + +// Pages that share only the security headers any stack sets are not one page. +func TestDiscoverIgnoresCommonHeaders(t *testing.T) { + head := "HTTP/1.1 200 OK\r\nX-Content-Type-Options: nosniff\r\nX-Frame-Options: DENY\r\nX-Request-Id: 1\r\n\r\n" + samples := []Sample{ + {ID: "a", Host: "a.example", Raw: []byte(head + "请通过官方域名访问")}, + {ID: "b", Host: "b.example", Raw: []byte(head + "Privacy Redirect")}, + } + clusters, err := Discover(context.Background(), judge.New(coverer(CoverageMissing)), samples, 2) + if err != nil || len(clusters) != 0 { + t.Fatalf("clusters: %+v %v", clusters, err) + } +} + +func TestLedgerReportIsDeepSnapshot(t *testing.T) { + l := NewLedger() + f := judged("one", common.FrameFromFingers, judge.OptionRunning, jev.Holds, nil) + fs := common.Frameworks{"one": f} + l.Add("a", fs) + first := l.Report(1, 0) + l.Add("b", fs) + if first[0].Hits() != 1 || first[0].Outcomes["holds"] != 1 || len(first[0].Samples["holds"]) != 1 { + t.Fatal("snapshot changed after Add") + } + first[0].Samples["holds"][0] = "changed" + first[0].Options[judge.OptionRunning] = 99 + first[0].Outcomes["holds"] = 99 + second := l.Report(1, 0) + if second[0].Samples["holds"][0] != "a" || second[0].Outcomes["holds"] != 2 || second[0].Options[judge.OptionRunning] != 2 { + t.Fatal("snapshot mutation changed ledger") + } +} + +func TestClusterSnapshotKeepsRawOptionAndResolvedOutcome(t *testing.T) { + raw := []byte("HTTP/1.1 200 OK\r\n\r\nOrion Console") + j := judge.New(coverer(CoverageMissing)) + j.MinConfidence = 1 + clusters, err := Discover(context.Background(), j, []Sample{{ID: "a", Host: "one", Raw: raw}, {ID: "b", Host: "two", Raw: raw}}, 2) + if err != nil || len(clusters) != 1 { + t.Fatalf("%v %v", clusters, err) + } + c := clusters[0] + if c.Coverage.Option != CoverageMissing || c.Outcome != jev.Insufficient { + t.Fatalf("raw option lost: %+v", c) + } + data, err := json.Marshal(c) + if err != nil { + t.Fatal(err) + } + var restored Cluster + if err := json.Unmarshal(data, &restored); err != nil { + t.Fatal(err) + } + if restored.Samples[0] != "a" || restored.Samples[1] != "b" || restored.Coverage != c.Coverage || restored.Outcome != c.Outcome { + t.Fatalf("snapshot: %+v", restored) + } +} diff --git a/judge/presence.go b/judge/presence.go new file mode 100644 index 0000000..656269c --- /dev/null +++ b/judge/presence.go @@ -0,0 +1,125 @@ +package judge + +import ( + "context" + "fmt" + "strconv" + + "github.com/chainreactors/fingers/common" + "github.com/chainreactors/fingers/judge/internal/evidence" + "github.com/chainreactors/utils/jev" +) + +// Presence options, as written to Framework.Judge.Option: the option a +// presence claim was ruled with. Declared and absent are rulings code makes +// from the response head; the rest are the provider's. +const ( + OptionDeclared = "declared" // holds: the response names the product in an explicit header declaration, or shows the protocol feature + OptionAbsent = "absent" // refuted: a protocol feature the response head does not show + OptionRunning = "running" // holds: the evidence shows the product produced the response + OptionMentioned = "mentioned" // refuted: the product only appears in the page's content + OptionUnrelated = "unrelated" // refuted: the rule matched text that has nothing to do with the product +) + +// matchExcerpt is one excerpt of the response a claim rests on, as the provider +// reads it. +type matchExcerpt struct { + Where string `json:"where"` // "header" or "body" + Matched string `json:"matched,omitempty"` // the text the rule matched, when the engine records it + Text string `json:"text"` // the matched text with its context +} + +func (e matchExcerpt) String() string { + if e.Matched != "" { + return e.Where + " (matched " + strconv.Quote(e.Matched) + "): " + e.Text + } + return e.Where + ": " + e.Text +} + +// factClaim carries presence facts code establishes from the response head: +// explicit product declarations and protocol features. +var factClaim = jev.Claim{Statement: "The response head establishes the product or protocol feature.", Options: map[string]jev.Option{ + OptionDeclared: {Description: "Explicit product declaration or protocol feature", Outcome: jev.Holds}, + OptionAbsent: {Description: "The protocol feature is absent", Outcome: jev.Refuted}, + jev.OptionInsufficient: {Description: insufficientDescription, Outcome: jev.Insufficient}, +}} + +// presenceClaim is a rule's claim that name is part of the software stack +// that produced the response; state.matches[id] quotes where it matched. +func presenceClaim(id, name string) jev.Claim { + return jev.Claim{ + Statement: fmt.Sprintf("`matches.%s` contains evidence for the fingerprint rule's claim that `%s` is part of the software stack "+ + "that produced this HTTP response; it quotes the response where the rule matched: `matched` is the text "+ + "the rule matched, `text` that text in its context; evidence without `matched` is where the product's name occurs. "+ + "Judge the claim from the evidence and the response. On documentation, README or tutorial pages the documented "+ + "product is mentioned, not running; the documentation generator is running.", id, name), + Options: map[string]jev.Option{ + OptionRunning: {Description: "`" + name + "` served, generated or is loaded by this response: headers, cookies, asset paths, markup, " + + "or the stock text of its own login, console, default or error page. A packaged browser-side application counts.", Outcome: jev.Holds}, + OptionMentioned: {Description: "`" + name + "` only appears in content: an article, README, documentation, link or list.", Outcome: jev.Refuted}, + OptionUnrelated: {Description: "The matched text has nothing to do with `" + name + "`: it is a fragment of encoded or random data, " + + "part of another word, or another product's asset.", Outcome: jev.Refuted}, + jev.OptionInsufficient: {Description: "The evidence shows neither.", Outcome: jev.Insufficient}, + }, + } +} + +// presence applies local facts and asks one batch for the remaining products. +func (j *Judge) presence(ctx context.Context, p *evidence.Page, frames common.Frameworks) error { + claims := map[string]jev.Claim{} + targets := map[string][]*common.Framework{} + matches := map[string][]matchExcerpt{} + for _, group := range groupProducts(frames) { + name := group[0].Name + for _, f := range group[1:] { + f.Judge = &common.Judgement{Duplicate: true} + } + key := NormalizeName(name) + excerpts := evidenceFor(p.Raw, group) + option := "" + if protocolFeatures[key] { + option = OptionAbsent + if protocolPresent(p, key) { + option = OptionDeclared + } + } else if declares(p, name) { + option = OptionDeclared + } + if option != "" { + j.applyPresence(group, excerpts, factClaim, jev.Ruling{Option: option, Confidence: 1}) + continue + } + if len(claims) >= maxClaims { + continue + } + id := "presence_" + key + claims[id], targets[id], matches[id] = presenceClaim(id, name), group, excerpts + } + rulings, err := j.judge(ctx, map[string]interface{}{"response": p, "matches": matches}, claims) + if err != nil { + return err + } + for id, ruling := range rulings { + j.applyPresence(targets[id], matches[id], claims[id], ruling) + } + return nil +} + +func (j *Judge) rejects(o jev.Outcome) bool { + return o == jev.Refuted || (o == jev.Insufficient && j.DropInsufficient) +} + +// applyPresence annotates a product's spellings with a ruling, whether code +// or the provider made it; spellings after the first are duplicates. +func (j *Judge) applyPresence(frames []*common.Framework, evidence []matchExcerpt, claim jev.Claim, ruling jev.Ruling) { + outcome := claim.Resolve(ruling, j.MinConfidence) + var excerpts []string + for _, e := range evidence { + excerpts = append(excerpts, e.String()) + } + for i, f := range frames { + f.Judge = &common.Judgement{Option: ruling.Option, Outcome: outcome.String(), + Evidence: append([]string(nil), excerpts...), Confidence: ruling.Confidence, + Rejected: j.rejects(outcome), Duplicate: i > 0} + } +} diff --git a/judge/presence_test.go b/judge/presence_test.go new file mode 100644 index 0000000..ac51f95 --- /dev/null +++ b/judge/presence_test.go @@ -0,0 +1,165 @@ +package judge + +import ( + "context" + "strings" + "testing" + + "github.com/chainreactors/fingers/common" + "github.com/chainreactors/fingers/judge/internal/evidence" + "github.com/chainreactors/utils/jev" +) + +func TestPageDeclarations(t *testing.T) { + s, err := evidence.NewPage([]byte(jenkinsRaw)) + if err != nil { + t.Fatal(err) + } + if s.Title != "Sign in [Jenkins]" || s.Headers["X-Jenkins"] != "2.401.3" { + t.Fatalf("title/header: %+v", s) + } + if len(s.Cookies) != 1 || len(s.Scripts) != 1 || len(s.Forms) != 2 { + t.Fatalf("cookies/scripts/forms: %+v", s) + } + if got := extractVersions([]byte(jenkinsRaw), 5); len(got) != 2 || got[0].Value != "1.24.0" || got[1].Value != "2.401.3" { + t.Fatalf("versions: %v", got) + } + if !declares(s, "jenkins") || !declares(s, "nginx") || declares(s, "wordpress") || declares(s, "jenk") { + t.Fatal("declares") + } +} + +// A rule that matched text unrelated to its product is a false positive, +// removed like a mere mention, and the matched text reaches the provider. +func TestUnrelatedIsRejected(t *testing.T) { + m := newMock() + m.presence["apache tomcat"] = OptionUnrelated + frames := testFrames() + for _, f := range frames { + if f.Name == "apache tomcat" || f.Name == "apache-tomcat" { + f.MatchDetail = &common.MatchDetail{MatcherType: "word", MatcherValue: "jenkins"} + } + } + all, err := New(m).Inspect(context.Background(), []byte(jenkinsRaw), frames) + if err != nil { + t.Fatal(err) + } + tomcat := all["apache tomcat"].Judge + if all["apache-tomcat"].Judge.Duplicate { + tomcat = all["apache-tomcat"].Judge + } + if tomcat.Option != OptionUnrelated || !tomcat.Rejected || len(tomcat.Evidence) == 0 || !strings.Contains(tomcat.Evidence[0], `matched "Jenkins"`) { + t.Fatalf("tomcat %+v", tomcat) + } +} + +func TestWordBoundary(t *testing.T) { + if indexWord("take action now", "acti") >= 0 || indexWord("x-github-request-id: 1", "git") >= 0 { + t.Fatal("substring matched") + } + if indexWord("sign in [jenkins]", "jenkins") < 0 || indexWord("泛微oa", "泛微") < 0 { + t.Fatal("word missed") + } +} + +func TestNormalizeName(t *testing.T) { + same := [][]string{{"Apache HTTP Server", "apache-http", "apache-web-server", "Apache"}, {"Discuz! X", "discuz"}, {"Apache-Tomcat", "apache tomcat"}, {"泛微 OA", "泛微"}} + for _, names := range same { + for _, n := range names[1:] { + if NormalizeName(n) != NormalizeName(names[0]) { + t.Errorf("%q and %q not folded: %q %q", names[0], n, NormalizeName(names[0]), NormalizeName(n)) + } + } + } + for _, pair := range [][2]string{{"lighttpd", "lig"}, {"Apache Tomcat", "Apache"}, {"nginx", "ngin"}} { + if NormalizeName(pair[0]) == NormalizeName(pair[1]) { + t.Errorf("%q and %q folded", pair[0], pair[1]) + } + } +} + +// Rule excerpts include matches deep in the response. +func TestEvidenceQuotesTheMatcher(t *testing.T) { + raw := "HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\n没有找到站点" + strings.Repeat("

filler

", 300) + + "

您的请求在Web服务器中没有找到对应的站点！

" + p, _ := evidence.NewPage([]byte(raw)) + f := common.NewFramework("宝塔", common.FrameFromFingers) + f.MatchDetail = &common.MatchDetail{MatcherType: "regexp", MatcherValue: "没有找到对应的站点"} + frames := common.Frameworks{} + frames.Add(f) + groups := groupProducts(frames) + excerpts := evidenceFor(p.Raw, groups[0]) + if len(groups) != 1 || len(excerpts) == 0 || + !strings.Contains(excerpts[0].Text, "您的请求在Web服务器中没有找到对应的站点") || excerpts[0].Where != "body" { + t.Fatalf("evidence: %+v", groups[0]) + } + if strings.Contains(p.Text, "您的请求") { + t.Fatal("test page too short: the compact view already carries the match") + } +} + +// Minimal public evidence retained from the 2026-09-25 corpus, not full pages. +func TestStaticApplicationDescriptionIsPreservedAsEvidence(t *testing.T) { + description := "IT Tools is a free and open-source collection of handy online tools for developers." + for _, meta := range []string{ + ``, + ``, + } { + p, err := evidence.NewPage([]byte("HTTP/1.1 200 OK\r\n\r\nIT Tools - Handy online tools for developers" + meta)) + if err != nil || p.Description != description { + t.Fatalf("description evidence missing: %+v err=%v", p, err) + } + if declares(p, "it tools") { + t.Fatal("description incorrectly treated as deterministic header evidence") + } + } +} + +// Protocol features are facts of the response head: decided by code, never +// asked, whatever a provider would rule. +func TestProtocolHitRequiresResponseEvidence(t *testing.T) { + for _, tc := range []struct { + name, header string + want bool + }{{"http基本认证", "", false}, {"hsts", "", false}, {"http基本认证", "WWW-Authenticate: Basic realm=\"test\"\r\n", true}, {"hsts", "Strict-Transport-Security: max-age=31536000\r\n", true}} { + asked := false + j := New(rulingProvider(func(qs map[string]jev.Claim) map[string]jev.Ruling { + asked = true + return nil + })) + hits := common.Frameworks{} + hits.Add(common.NewFramework(tc.name, common.FrameFromGUESS)) + all, err := j.Inspect(context.Background(), []byte("HTTP/1.1 200 OK\r\n"+tc.header+"\r\n"), hits) + got := all.Accepted() + if err != nil || asked || (len(got) > 0) != tc.want { + t.Errorf("%s evidence=%q got=%v asked=%v err=%v", tc.name, tc.header, got, asked, err) + } + } +} + +func TestBasicChallengeUsesFullHeaderAndQuotedRealms(t *testing.T) { + for _, tc := range []struct { + header string + want bool + }{ + {"WWW-Authenticate: Digest realm=\"" + strings.Repeat("x", 200) + "\", Basic realm=\"private\"\r\n", true}, + {"WWW-Authenticate: Digest realm=\"x, Basic fake\"\r\n", false}, + {"WWW-Authenticate: Digest realm=\"private\"\r\nWWW-Authenticate: Basic realm=\"private\"\r\n", true}, + } { + p, err := evidence.NewPage([]byte("HTTP/1.1 401 Unauthorized\r\n" + tc.header + "\r\n")) + if err != nil || protocolPresent(p, "http基本认证") != tc.want { + t.Fatalf("header %q: %v", tc.header, err) + } + } +} + +func TestIncidentalHeaderNamesAreNotDeclarations(t *testing.T) { + raw := []byte("HTTP/1.1 302 Found\r\nLocation: https://docs.test/jenkins/2.401.3\r\nSet-Cookie: jenkins=x\r\nX-Github-Request-Id: jenkins\r\n\r\n") + p, err := evidence.NewPage(raw) + if err != nil { + t.Fatal(err) + } + if declares(p, "jenkins") || declaredVersion(declarations(raw), "jenkins") != "" { + t.Fatal("incidental URL/cookie text became a fact") + } +} diff --git a/judge/version.go b/judge/version.go new file mode 100644 index 0000000..9c4852d --- /dev/null +++ b/judge/version.go @@ -0,0 +1,369 @@ +package judge + +import ( + "context" + "fmt" + "regexp" + "sort" + "strconv" + "strings" + + "github.com/chainreactors/fingers/common" + "github.com/chainreactors/fingers/judge/internal/evidence" + "github.com/chainreactors/utils/jev" +) + +var ( + // Allows a v/V/x/X prefix ("X3.4", "V8.1SP2") and a letter suffix; rejects + // digits glued to other digits or dots so IPs and long builds stay out. + reGenMajor = regexp.MustCompile(`(?i)generator["'][^>]*content=["'][A-Za-z][^"']*?\s[vV]?(\d{1,3})(?:[\s"'(]|$)|content=["'][A-Za-z][^"']*?\s[vV]?(\d{1,3})(?:[\s(][^"']*)?["'][^>]*name=["']generator`) + reVersion = regexp.MustCompile(`(?:^|[^0-9A-Za-z.])[vVxX]?(` + evidence.VersionToken + `)(?:[^0-9A-Za-z.]|\.[A-Za-z]|$)`) + // What precedes numbers reVersion catches that are never versions: the + // viewport's "initial-scale=1.0". + reNotVersion = regexp.MustCompile(`(?i)(?:initial|minimum|maximum)-scale\s*=\s*$`) + // A Via header entry starts with the HTTP version it was received with + // ("Via: 1.1 varnish", "1.0 fred, 1.1 p.example.net"), never a product's. + reViaProtocol = regexp.MustCompile(`(?i)^via:\s*(?:[^,\r\n]*,\s*)*(?:HTTP/)?$`) + // The format version of an SVG or XML declaration: . + reMarkupVersion = regexp.MustCompile(`(?i)^<(?:svg|\?xml)\b[^>]*\bversion\s*=\s*["']?$`) +) + +// formatVersion reports whether the number at i in text is a protocol or +// markup format version rather than a product version. +func formatVersion(text string, i int) bool { + line := text[strings.LastIndexByte(text[:i], '\n')+1 : i] + if reViaProtocol.MatchString(line) { + return true + } + if tag := strings.LastIndexByte(text[:i], '<'); tag >= 0 && !strings.Contains(text[tag:i], ">") { + return reMarkupVersion.MatchString(text[tag:i]) + } + return false +} + +const ( + notStated = "not_stated" + + maxVersions = 40 // extracted from the raw response + maxVersionsShown = 15 // options per product after ranking + maxVersioned = 8 // products versioned per page +) + +// versions uses one finite-option claim per product, including local declarations. +func (j *Judge) versions(ctx context.Context, p *evidence.Page, frames ...*common.Framework) error { + decls := declarations(p.Raw) + extracted := extractVersions(p.Raw, maxVersions) + claims := map[string]jev.Claim{} + targets := map[string]*common.Framework{} + var shown []versionString + seen := map[string]bool{} + count := 0 + for _, f := range frames { + if f == nil || protocolFeatures[NormalizeName(f.Name)] || (f.Attributes != nil && f.Version != "") { + continue + } + if count >= maxVersioned { + break + } + count++ + claim := jev.Claim{Statement: fmt.Sprintf("`version_strings` lists version-like strings with their context. Which is the version of `%s`? A product's asset URL parameter, generator meta or embedded build info counts. Ignore third-party libraries, other products, protocols, years, timestamps and build numbers.", f.Name), Options: map[string]jev.Option{ + notStated: {Description: "The response does not state the version of " + f.Name, Outcome: jev.Refuted}, + jev.OptionInsufficient: {Description: insufficientDescription, Outcome: jev.Insufficient}, + }} + if v := declaredVersion(decls, f.Name); v != "" { + claim.Options[v] = jev.Option{Description: "The response explicitly binds this version to " + f.Name, Outcome: jev.Holds} + j.applyVersion(f, claim, jev.Ruling{Option: v, Confidence: 1}) + continue + } + key := NormalizeName(f.Name) + var own []versionString + for _, c := range extracted { + if c.count == 1 && claimedByOther(decls, c.Value, key) { + continue + } + own = append(own, c) + } + cands := rankVersions(own, f.Name, maxVersionsShown) + if len(cands) == 0 { + continue + } + for _, c := range cands { + claim.Options[c.Value] = jev.Option{Outcome: jev.Holds} + if !seen[c.Value] { + seen[c.Value] = true + shown = append(shown, c) + } + } + id := "version_" + key + claims[id], targets[id] = claim, f + } + rulings, err := j.judge(ctx, map[string]interface{}{"response": p, "version_strings": shown}, claims) + if err != nil { + return err + } + for id, ruling := range rulings { + j.applyVersion(targets[id], claims[id], ruling) + } + return nil +} + +// applyVersion writes the picked version when the ruling holds. +func (j *Judge) applyVersion(f *common.Framework, claim jev.Claim, ruling jev.Ruling) { + if claim.Resolve(ruling, j.MinConfidence) == jev.Holds { + if f.Attributes == nil { + f.Attributes = common.NewAttributesWithAny() + } + f.Version = ruling.Option + } +} + +// declaration is a version the response binds to a product name. +type declaration struct{ name, version string } + +var ( + reDeclared = regexp.MustCompile(`([A-Za-z][A-Za-z0-9._-]*(?: [A-Za-z][A-Za-z0-9._-]*){0,2})(?:/| +[vV]?)(` + evidence.VersionToken + `)(?:[^0-9A-Za-z.+-]|$)`) + reLeadingVersion = regexp.MustCompile(`^[vV]?(` + evidence.VersionToken + `)(?:[\s;,(]|$)`) +) + +// declarations finds "name/version" and "name version" pairs in response +// headers and generator metas, and headers whose name is the product +// ("X-Jenkins: 2.401.3", "X-AspNet-Version: 4.0.30319"). +func declarations(raw []byte) []declaration { + text := string(raw) + head, body := text, "" + if i := strings.Index(text, "\r\n\r\n"); i >= 0 { + head, body = text[:i], text[i+4:] + } else if i := strings.Index(text, "\n\n"); i >= 0 { + head, body = text[:i], text[i+2:] + } + var out []declaration + pairs := func(value string) { + for _, m := range reDeclared.FindAllStringSubmatch(value, -1) { + out = append(out, declaration{m[1], m[2]}) + } + } + lines := strings.Split(head, "\n") + for _, line := range lines[1:] { // skip the status line + line = strings.TrimSpace(line) + colon := strings.IndexByte(line, ':') + if colon < 0 { + continue + } + key, value := line[:colon], strings.TrimSpace(line[colon+1:]) + if m := reLeadingVersion.FindStringSubmatch(value); m != nil && strings.HasPrefix(strings.ToLower(key), "x-") { + name := strings.TrimSuffix(strings.TrimPrefix(strings.ToLower(key), "x-"), "-version") + out = append(out, declaration{name, m[1]}) + } + if strings.EqualFold(key, "Server") || strings.EqualFold(key, "X-Powered-By") || strings.EqualFold(key, "Product") { + pairs(value) + } + if strings.HasPrefix(strings.ToLower(key), "x-") { + product := NormalizeName(strings.TrimSuffix(strings.TrimPrefix(strings.ToLower(key), "x-"), "-version")) + for _, m := range reDeclared.FindAllStringSubmatch(value, -1) { + d := declaration{m[1], m[2]} + if d.names(product) { + out = append(out, d) + } + } + } + } + for _, m := range evidence.Generator.FindAllStringSubmatch(body, 4) { + pairs(m[1] + m[2]) + } + return out +} + +// names reports whether a declared name is the product with key: the whole +// name or its trailing words, so "Apache Tomcat/9.0" names tomcat, not apache. +func (d declaration) names(key string) bool { + if key == "" { + return false + } + words := strings.Fields(d.name) + for i := range words { + if NormalizeName(strings.Join(words[i:], " ")) == key { + return true + } + } + return false +} + +// declaredVersion is the version the response binds to product, if exactly one. +func declaredVersion(decls []declaration, product string) string { + key := NormalizeName(product) + found := "" + for _, d := range decls { + if d.names(key) { + if found != "" && found != d.version { + return "" + } + found = d.version + } + } + return found +} + +// claimedByOther reports whether value is bound by name to a product other than key. +func claimedByOther(decls []declaration, value, key string) bool { + for _, d := range decls { + if d.version == value && !d.names(key) { + return true + } + } + return false +} + +// versionString is a version-like string with the raw text around its first +// occurrence, so the provider can tell a product version from a library version. The +// JSON names are what the provider reads. +type versionString struct { + Value string `json:"value"` + Context string `json:"found_in"` + count int // occurrences in the response +} + +// extractVersions scans the raw response (headers and body, including +// inline scripts and meta tags that Page drops). Providers do not +// generate strings: versions are extracted by regex and one is picked. +func extractVersions(raw []byte, max int) []versionString { + text := string(raw) + if i := strings.IndexByte(text, '\n'); i >= 0 { // skip the "HTTP/1.1 200" status line + text = text[i+1:] + } + seen := map[string]int{} // value -> index in out + var out []versionString + for _, loc := range reVersion.FindAllStringSubmatchIndex(text, -1) { + v := text[loc[2]:loc[3]] + before := loc[2] - 20 + if before < 0 { + before = 0 + } + if looksLikeIPv4(v) || strings.Trim(v, "0.") == "" || reNotVersion.MatchString(text[before:loc[2]]) || formatVersion(text, loc[2]) { + continue + } + start, end := loc[0]-60, loc[1]+30 + if start < 0 { + start = 0 + } + if end > len(text) { + end = len(text) + } + ctx := evidence.Clean(strings.ToValidUTF8(text[start:end], "")) + // A value repeats across a page ("?ver=7.1" on every asset, then + // the generator meta); keep the context that best states a version. + if i, ok := seen[v]; ok { + out[i].count++ + if contextWeight(ctx) > contextWeight(out[i].Context) { + out[i].Context = ctx + } + continue + } + seen[v] = len(out) + out = append(out, versionString{Value: v, Context: ctx, count: 1}) + } + // "Drupal 10": a generator meta that states only a major version. + for _, m := range reGenMajor.FindAllStringSubmatchIndex(text, -1) { + lo, hi := m[2], m[3] + if lo < 0 { + lo, hi = m[4], m[5] + } + if v := text[lo:hi]; v != "" { + if _, ok := seen[v]; ok { + continue + } + seen[v] = len(out) + out = append(out, versionString{Value: v, Context: evidence.Clean(strings.ToValidUTF8(text[m[0]:m[1]], "")), count: 1}) + } + } + // Select evidence after scanning: SVG numbers or early library assets must + // not crowd later generator/header/script versions out of the shortlist. + if len(out) > max { + sort.SliceStable(out, func(i, j int) bool { return contextWeight(out[i].Context) > contextWeight(out[j].Context) }) + out = out[:max] + } + return out +} + +// contextWeight prefers contexts that declare a version (generator meta, +// product headers) over asset URLs that merely carry one. +func contextWeight(ctx string) int { + c := strings.ToLower(ctx) + switch { + case strings.Contains(c, "generator"): + return 5 + case strings.Contains(c, "server:") || strings.Contains(c, "x-powered-by:") || strings.Contains(c, "product:"): + return 4 + case strings.Contains(c, " 255 || (i == 0 && n < 10) { + return false + } + } + return true +} + +// rankVersions orders candidates by how likely their context states the +// version of product, and keeps the first max. Pages often carry dozens of +// "?ver=" strings, so without ranking the one real version can fall outside +// the list the provider sees. +func rankVersions(cands []versionString, product string, max int) []versionString { + key := NormalizeName(product) + out := append([]versionString(nil), cands...) + sort.SliceStable(out, func(a, b int) bool { return versionScore(out[a], key) > versionScore(out[b], key) }) + if len(out) > max { + out = out[:max] + } + return out +} + +// versionScore rates how well c's context ties it to the product with key. +// Naming the product counts most, the more so right before the value; words +// that declare some version (generator, Server:) count less, since they do +// not say whose; library and plugin paths count against. +func versionScore(c versionString, key string) int { + ctx := strings.ToLower(c.Context) + s := 0 + if key != "" { + if strings.Contains(NormalizeName(ctx), key) { + s += 3 + } + before := ctx + if i := strings.Index(ctx, strings.ToLower(c.Value)); i >= 0 { + before = ctx[:i] + } + if len(before) > 24 { + before = before[len(before)-24:] + } + if strings.Contains(NormalizeName(before), key) { + s += 3 + } + } + for _, h := range []string{"generator", "server:", "x-powered-by:", "product:", "version"} { + if strings.Contains(ctx, h) { + s += 2 + break + } + } + for _, noisy := range []string{"/plugins/", "/themes/", "/vendor/", "jquery", "bootstrap", "schema_version"} { + if strings.Contains(ctx, noisy) && !strings.Contains(key, NormalizeName(noisy)) { + s -= 2 + break + } + } + return s +} diff --git a/judge/version_test.go b/judge/version_test.go new file mode 100644 index 0000000..83bf775 --- /dev/null +++ b/judge/version_test.go @@ -0,0 +1,125 @@ +package judge + +import ( + "fmt" + "strings" + "testing" + + "github.com/chainreactors/fingers/judge/internal/evidence" +) + +func TestDeclarations(t *testing.T) { + raw := []byte("HTTP/1.1 200 OK\r\nServer: Apache/2.4.38 (Debian) OpenSSL/1.0.2q PHP/5.6.40\r\nX-Jenkins: 2.401.3\r\n" + + "X-Gitea-Version: 1.21.4\r\nX-Tomcat: Apache Tomcat/9.0.1\r\n\r\n") + decls := declarations(raw) + for product, want := range map[string]string{"Apache": "2.4.38", "openssl": "1.0.2q", "PHP": "5.6.40", "jenkins": "2.401.3", + "Gitea": "1.21.4", "Apache Tomcat": "9.0.1", "WordPress": "7.0.3", "nginx": ""} { + if got := declaredVersion(decls, product); got != want { + t.Errorf("%s: got %q want %q", product, got, want) + } + } + if !claimedByOther(decls, "5.6.40", NormalizeName("ThinkPHP")) || claimedByOther(decls, "5.6.40", NormalizeName("php")) { + t.Error("claims") + } +} + +func TestExtractVersionsPrefixes(t *testing.T) { + raw := "HTTP/1.1 200 OK\r\n\r\nV8.1SP2 10.0.0.5 jquery.min.js?ver=3.7.1{\"success_fraction\":0.0}" + var got []string + for _, c := range extractVersions([]byte(raw), 10) { + got = append(got, c.Value) + } + if strings.Join(got, ",") != "3.4,8.1SP2,3.7.1" { + t.Fatalf("got %v", got) + } +} + +func TestRankVersions(t *testing.T) { + raw := "HTTP/1.1 200 OK\r\n\r\n" + for i := 0; i < 15; i++ { + raw += fmt.Sprintf(``, i, i) + } + raw += `` + cands := extractVersions([]byte(raw), 40) + got := rankVersions(cands, "wordpress", 3) + if got[0].Value != "7.0.3" || !strings.Contains(got[0].Context, "generator") { + t.Fatalf("wordpress generator not ranked first: %+v", got) + } + var drupal bool + for _, c := range cands { + drupal = drupal || c.Value == "10" + } + if !drupal { + t.Fatalf("major-only generator version missing: %+v", cands) + } +} + +func TestVersionCandidatesFromRealResponseEvidence(t *testing.T) { + raw := []byte("HTTP/1.1 200 OK\r\nServer: Apache/2.4.38 OpenSSL/1.0.2q PHP/5.6.40\r\nX-New-Api-Version: v1.0.0-rc.35\r\n\r\n") + got := map[string]bool{} + for _, c := range extractVersions(raw, 40) { + got[c.Value] = true + } + for _, want := range []string{"2.4.38", "1.0.2q", "5.6.40", "1.0.0-rc.35", "3.6.3"} { + if !got[want] { + t.Errorf("missing %s in %v", want, got) + } + } + if got["1.0.2"] { + t.Fatal("truncated OpenSSL version") + } +} + +func TestVersionCandidatesDoNotDiscardLateProductEvidence(t *testing.T) { + var b strings.Builder + b.WriteString("HTTP/1.1 200 OK\r\n\r\n") + for i := 0; i < 80; i++ { + b.WriteString(" " + string(rune('a'+i%26)) + " ") + b.WriteString(strings.Repeat("1", i/10+1) + ".25 ") + } + // Add distinct numbers to exceed the shortlist. + for _, n := range []string{"10.11", "11.12", "12.13", "13.14", "14.15", "15.16", "16.17", "17.18", "18.19", "19.20"} { + b.WriteString(n + " ") + } + b.WriteString("") + found := false + for _, c := range extractVersions([]byte(b.String()), 5) { + found = found || c.Value == "3.7.1" + } + if !found { + t.Fatal("early numeric noise evicted late library version") + } +} + +func TestCalendarVersionAndGeneratorName(t *testing.T) { + raw := []byte("HTTP/1.1 200 OK\r\n\r\nCustom Search") + p, err := evidence.NewPage(raw) + if err != nil { + t.Fatal(err) + } + foundName, foundVersion := false, false + for _, name := range evidence.Names(p.Generator, p.Title, p.Text, p.Headers, p.Scripts) { + foundName = foundName || NormalizeName(name) == "searxng" + } + for _, v := range extractVersions(raw, 40) { + foundVersion = foundVersion || v.Value == "2026.9.23+3cd69d30e" + } + if !foundName || !foundVersion { + t.Fatalf("name=%t version=%t", foundName, foundVersion) + } +} + +// Protocol and markup format versions are facts code excludes: the provider +// is never offered them as a product's version. +func TestFormatVersionsAreNotCandidates(t *testing.T) { + raw := "HTTP/1.1 200 OK\r\nServer: GitHub.com\r\nVia: 1.1 varnish, HTTP/1.0 proxy\r\n\r\n" + + `` + + `

Varnish 7.4.2

` + var got []string + for _, c := range extractVersions([]byte(raw), 10) { + got = append(got, c.Value) + } + if strings.Join(got, ",") != "2.4.1,7.4.2" { + t.Fatalf("got %v", got) + } +} diff --git a/judge_test.go b/judge_test.go new file mode 100644 index 0000000..b6ddc96 --- /dev/null +++ b/judge_test.go @@ -0,0 +1,123 @@ +package fingers + +import ( + "context" + "strings" + "testing" + + "github.com/chainreactors/fingers/judge" + "github.com/chainreactors/utils/jev" + "github.com/pkg/errors" +) + +// With a Judge, the engine's own entry points return reviewed results: +// declared products are kept, a product the page only mentions is dropped, +// and a failing provider leaves the rule results in place. +func TestEngineJudge(t *testing.T) { + engine, err := NewEngine(FingersEngine, WappalyzerEngine) + if err != nil { + t.Fatal(err) + } + raw := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\nContent-Type: text/html\r\n\r\n" + + `Welcome to nginx!

Migrating from WordPress? wp-login.php

`) + rules, _ := engine.DetectContent(raw) + if rules["nginx"] == nil || len(rules) < 2 { + t.Fatalf("rule results %v", rules) + } + engine.EnableMatchDetail() + engine.Judge = judge.New(verdict(judge.OptionMentioned)) + reviewed, _ := engine.DetectContent(raw) + if nginx := reviewed["nginx"]; nginx == nil || nginx.Judge == nil || nginx.Judge.Option != judge.OptionDeclared { + t.Fatalf("declared nginx: %v", reviewed["nginx"]) + } + for name, f := range reviewed { + if f.Judge != nil && f.Judge.Rejected { + t.Errorf("rejected %s returned", name) + } + } + if len(reviewed) >= len(rules) { + t.Fatalf("nothing dropped: %d reviewed of %d", len(reviewed), len(rules)) + } + engine.Judge = judge.New(failing{}) + if fallback, _ := engine.DetectContent(raw); len(fallback) != len(rules) { + t.Fatalf("failed provider changed results: %d of %d", len(fallback), len(rules)) + } +} + +func TestEnableJudge(t *testing.T) { + engine, err := NewEngine(FingersEngine) + if err != nil { + t.Fatal(err) + } + t.Setenv(jev.EnvAPIKey, "") + if err := engine.EnableJudge(""); err == nil || engine.Judge != nil { + t.Fatal("enabled without a key") + } + if err := engine.EnableJudge("local-test"); err != nil || engine.Judge == nil || !engine.Fingers().MatchDetailEnabled { + t.Fatalf("EnableJudge: %v", err) + } +} + +// verdict is a provider that rules every presence claim with one option and +// states no version. +type verdict string + +func (verdict) ID() string { return "test" } + +func (v verdict) Judge(ctx context.Context, state interface{}, claims map[string]jev.Claim) (map[string]jev.Ruling, error) { + rulings := map[string]jev.Ruling{} + for k := range claims { + switch { + case strings.HasPrefix(k, "presence_"): + rulings[k] = jev.Ruling{Option: string(v), Confidence: 0.9} + default: + rulings[k] = jev.Ruling{Option: "not_stated", Confidence: 0.9} + } + } + return rulings, nil +} + +type failing struct{} + +func (failing) ID() string { return "failing" } + +func (failing) Judge(context.Context, interface{}, map[string]jev.Claim) (map[string]jev.Ruling, error) { + return nil, errors.New("unavailable") +} + +// Every engine that can record what a hit matched does so once enabled, and +// quotes text the response contains; disabled, results carry no detail. +func TestEnableMatchDetail(t *testing.T) { + raw := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\nX-Powered-By: PHP/7.4.3\r\nContent-Type: text/html\r\n\r\n" + + `Blog` + + ``) + engines := []string{FingerPrintEngine, WappalyzerEngine, EHoleEngine, GobyEngine} + for _, enabled := range []bool{false, true} { + engine, err := NewEngine(engines...) + if err != nil { + t.Fatal(err) + } + if enabled { + engine.EnableMatchDetail() + } + frames, _ := engine.DetectContent(raw) + seen := map[string]bool{} + for _, f := range frames { + for from := range f.Froms { + name := from.String() + seen[name] = true + if !enabled && f.MatchDetail != nil { + t.Errorf("%s from %s: detail %+v while disabled", f.Name, name, f.MatchDetail) + } + if enabled && len(f.Froms) == 1 && (f.MatchDetail == nil || !strings.Contains(strings.ToLower(string(raw)), strings.ToLower(f.MatchDetail.MatcherValue))) { + t.Errorf("%s from %s: detail %+v is not in the response", f.Name, name, f.MatchDetail) + } + } + } + for _, name := range []string{"fingerprinthub", "wappalyzer", "goby"} { + if !seen[name] { + t.Errorf("no hit from %s", name) + } + } + } +} diff --git a/nmap/gonmap_test.go b/nmap/gonmap_test.go index 86d29d2..2fd3cf8 100644 --- a/nmap/gonmap_test.go +++ b/nmap/gonmap_test.go @@ -10,6 +10,7 @@ import ( "github.com/chainreactors/fingers/common" "github.com/chainreactors/fingers/resources" + "github.com/chainreactors/utils/parsers" ) // TestParseLogic 测试解析逻辑 @@ -148,14 +149,14 @@ func TestNoGuessFlag(t *testing.T) { fmt.Println("\n=== Testing NoGuess Flag Behavior ===") // Save original NoGuess value and restore after test - originalNoGuess := common.NoGuess + originalNoGuess := parsers.NoGuess defer func() { - common.NoGuess = originalNoGuess + parsers.NoGuess = originalNoGuess }() // Test 1: NoGuess = false (default) - guess should work fmt.Println("\n1. Testing with NoGuess = false (guess enabled)") - common.NoGuess = false + parsers.NoGuess = false // Test against a high port that's unlikely to be running but might be guessable // Using port 9999 which is unlikely to have a service but has a guess entry @@ -168,7 +169,7 @@ func TestNoGuessFlag(t *testing.T) { // Test 2: NoGuess = true - guess should be skipped fmt.Println("\n2. Testing with NoGuess = true (guess disabled)") - common.NoGuess = true + parsers.NoGuess = true res2 := engine.ServiceMatch("127.0.0.1", "9999", 1, common.NewServiceSender(time.Second*2), nil) if res2 == nil { diff --git a/re2/go.mod b/re2/go.mod new file mode 100644 index 0000000..24921c4 --- /dev/null +++ b/re2/go.mod @@ -0,0 +1,40 @@ +module github.com/chainreactors/fingers/re2 + +go 1.24.0 + +require ( + github.com/chainreactors/fingers v1.2.2-0.20260925174709-350638e91234 + github.com/wasilibs/go-re2 v1.6.0 +) + +require ( + github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible // indirect + github.com/chainreactors/logs v0.0.0-20260508055944-c678762ed15c // indirect + github.com/chainreactors/neutron v0.1.1-0.20260714062907-716c6b167cb6 // indirect + github.com/chainreactors/utils v0.0.0-20260629043228-93bdd2142c9a // indirect + github.com/chainreactors/utils/jev v0.0.0-20260927031300-6bf027367a50 // indirect + github.com/chainreactors/utils/parsers v0.0.4-0.20260927032755-fd113e94d6c0 // indirect + github.com/chainreactors/words v0.0.0-20260520145736-270600e60fb4 // indirect + github.com/dlclark/regexp2 v1.11.5 // indirect + github.com/facebookincubator/nvdtools v0.1.5 // indirect + github.com/go-dedup/megophone v0.0.0-20170830025436-f01be21026f5 // indirect + github.com/go-dedup/simhash v0.0.0-20170904020510-9ecaca7b509c // indirect + github.com/go-dedup/text v0.0.0-20170907015346-8bb1b95e3cb7 // indirect + github.com/hashicorp/go-version v1.6.0 // indirect + github.com/mozillazg/go-pinyin v0.20.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/tetratelabs/wazero v1.11.0 // indirect + github.com/twmb/murmur3 v1.1.8 // indirect + github.com/wasilibs/wazero-helpers v0.0.0-20250123031827-cd30c44769bb // indirect + golang.org/x/net v0.21.0 // indirect + golang.org/x/sys v0.38.0 // indirect + golang.org/x/text v0.14.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) + +replace github.com/wasilibs/go-re2 => github.com/chainreactors/go-re2 v1.11.1-0.20260716152604-8121b6cd261e + +// Development and CI use the checkout; downstream builds use the version +// required above. +replace github.com/chainreactors/fingers => ../ diff --git a/re2/go.sum b/re2/go.sum new file mode 100644 index 0000000..2baeaaf --- /dev/null +++ b/re2/go.sum @@ -0,0 +1,824 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= +cloud.google.com/go v0.98.0/go.mod h1:ua6Ush4NALrHk5QXDWnjvZHN93OuF0HfuEPq9I1X0cM= +cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible h1:1G1pk05UrOh0NlF1oeaaix1x8XzrfjIDK47TY0Zehcw= +github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chainreactors/go-re2 v1.11.1-0.20260716152604-8121b6cd261e h1:8SMCCo85OKxOM0NtD8sPhmnhGzigU2i4YPUZHgWDVq8= +github.com/chainreactors/go-re2 v1.11.1-0.20260716152604-8121b6cd261e/go.mod h1:4qC68vqWSuPTct3spuTrWBqCpm00mQ707JKLS1izVjI= +github.com/chainreactors/logs v0.0.0-20260508055944-c678762ed15c h1:6Net2Mgo/qo6ADFBZJWWScKuMfZ0rbzLqSCVDuLKFdc= +github.com/chainreactors/logs v0.0.0-20260508055944-c678762ed15c/go.mod h1:VrXmYPbNN5AVoo1sc5aeyPVBYqubMdb3KO/tn5rRZpo= +github.com/chainreactors/neutron v0.1.1-0.20260714062907-716c6b167cb6 h1:apEDnJeZ5fe2AaEJHV6TRDldoy2t/d7E1maxmUz2tfU= +github.com/chainreactors/neutron v0.1.1-0.20260714062907-716c6b167cb6/go.mod h1:zok/CDxut71iw8NQTHKPvy0f+1G5639zr9GgDsXGnOs= +github.com/chainreactors/utils v0.0.0-20260629043228-93bdd2142c9a h1:4rLTKr+VlG7yGbw2ym7nhNvFglvZ4acdphxKEozF7CI= +github.com/chainreactors/utils v0.0.0-20260629043228-93bdd2142c9a/go.mod h1:xwbUlFoSSxLHujyb8D48o1s2DqmEAxUNfxIy0DVUmcg= +github.com/chainreactors/utils/jev v0.0.0-20260927031300-6bf027367a50 h1:XnGpBYIS4u1BUNL3KtlrrXde/vSmDon34X5AEV7WOi8= +github.com/chainreactors/utils/jev v0.0.0-20260927031300-6bf027367a50/go.mod h1:9w5fvcKrpQVIHWlKuvpVARpZUd+OKsYSIii2JR1ZRB8= +github.com/chainreactors/utils/parsers v0.0.4-0.20260927032755-fd113e94d6c0 h1:rfhEDw1QfbIgUAhGWRbovR3ur7Ug3V7j5WxqstwlnW0= +github.com/chainreactors/utils/parsers v0.0.4-0.20260927032755-fd113e94d6c0/go.mod h1:bE/znJWt08n9QOORWsWu0ggB8GWfOg3+dfUMMITmwV4= +github.com/chainreactors/words v0.0.0-20260520145736-270600e60fb4 h1:lvnDYEkatmZFHP5i321qQXK9L4vKRfso/uUfr5tOeC8= +github.com/chainreactors/words v0.0.0-20260520145736-270600e60fb4/go.mod h1:zfz367PUmyaX6oAqV9SktVqyRXKlEh0sel9Wsq9dd2c= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1/go.mod h1:+hnT3ywWDTAFrW5aE+u2Sa/wT555ZqwoCS+pk3p6ry4= +github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= +github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= +github.com/facebookincubator/flog v0.0.0-20190930132826-d2511d0ce33c/go.mod h1:QGzNH9ujQ2ZUr/CjDGZGWeDAVStrWNjHeEcjJL96Nuk= +github.com/facebookincubator/nvdtools v0.1.5 h1:jbmDT1nd6+k+rlvKhnkgMokrCAzHoASWE5LtHbX2qFQ= +github.com/facebookincubator/nvdtools v0.1.5/go.mod h1:Kh55SAWnjckS96TBSrXI99KrEKH4iB0OJby3N8GRJO4= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-dedup/megophone v0.0.0-20170830025436-f01be21026f5 h1:4U+x+EB1P66zwYgTjxWXSOT8vF+651Ksr1lojiCZnT8= +github.com/go-dedup/megophone v0.0.0-20170830025436-f01be21026f5/go.mod h1:poR/Cp00iqtqu9ltFwl6C00sKC0HY13u/Gh05ZBmP54= +github.com/go-dedup/simhash v0.0.0-20170904020510-9ecaca7b509c h1:mucYYQn+sMGNSxidhleonzAdwL203RxhjJGnxQU4NWU= +github.com/go-dedup/simhash v0.0.0-20170904020510-9ecaca7b509c/go.mod h1:gO3u2bjRAgUaLdQd2XK+3oooxrheOAx1BzS7WmPzw1s= +github.com/go-dedup/text v0.0.0-20170907015346-8bb1b95e3cb7 h1:11wFcswN+37U+ByjxdKzsRY5KzNqqq5Uk5ztxnLOc7w= +github.com/go-dedup/text v0.0.0-20170907015346-8bb1b95e3cb7/go.mod h1:wSsK4VOECOSfSYTzkBFw+iGY7wj59e7X96ABtNj9aCQ= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= +github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-hclog v1.0.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= +github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= +github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= +github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= +github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= +github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mozillazg/go-pinyin v0.20.0 h1:BtR3DsxpApHfKReaPO1fCqF4pThRwH9uwvXzm+GnMFQ= +github.com/mozillazg/go-pinyin v0.20.0/go.mod h1:iR4EnMMRXkfpFVV5FMi4FNB6wGq9NV6uDWbUuPhP4Yc= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tetratelabs/wazero v1.11.0 h1:+gKemEuKCTevU4d7ZTzlsvgd1uaToIDtlQlmNbwqYhA= +github.com/tetratelabs/wazero v1.11.0/go.mod h1:eV28rsN8Q+xwjogd7f4/Pp4xFxO7uOGbLcD/LzB1wiU= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= +github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= +github.com/wasilibs/wazero-helpers v0.0.0-20250123031827-cd30c44769bb h1:gQ+ZV4wJke/EBKYciZ2MshEouEHFuinB85dY3f5s1q8= +github.com/wasilibs/wazero-helpers v0.0.0-20250123031827-cd30c44769bb/go.mod h1:jMeV4Vpbi8osrE/pKUxRZkVaA0EX7NZN0A9/oRzgpgY= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU= +google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= +google.golang.org/api v0.62.0/go.mod h1:dKmwPCydfsad4qCH08MSdgWjfHOyfpd4VtDGgRFdavw= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/re2/re2.go b/re2/re2.go new file mode 100644 index 0000000..bdde320 --- /dev/null +++ b/re2/re2.go @@ -0,0 +1,19 @@ +// Package re2 switches the fingers rule engine to RE2 +// (github.com/wasilibs/go-re2), which matches large rule sets faster than +// the standard library. Import it for its side effect: +// +// import _ "github.com/chainreactors/fingers/re2" +// +// It is a separate module because RE2 needs Go 1.24, while fingers itself +// supports Go 1.17. Build with -tags re2_cgo to use a native libre2. +package re2 + +import ( + "github.com/chainreactors/fingers/fingers" + re2 "github.com/wasilibs/go-re2" +) + +func init() { fingers.RegexpCompiler = Compile } + +// Compile compiles expr with RE2. +func Compile(expr string) (fingers.CompiledRegexp, error) { return re2.Compile(expr) } diff --git a/re2/re2_test.go b/re2/re2_test.go new file mode 100644 index 0000000..5a1c28e --- /dev/null +++ b/re2/re2_test.go @@ -0,0 +1,55 @@ +package re2 + +import ( + "regexp" + "testing" + + fingerlib "github.com/chainreactors/fingers" + "github.com/chainreactors/fingers/fingers" + re2 "github.com/wasilibs/go-re2" +) + +const nginx = "HTTP/1.1 200 OK\r\nServer: nginx/1.24.0\r\n\r\nWelcome to nginx!" + +func TestEngineUsesRE2(t *testing.T) { + if _, err := fingers.RegexpCompiler("a+"); err != nil { + t.Fatal(err) + } + if c, _ := fingers.RegexpCompiler("a+"); c == nil { + t.Fatal("no compiler") + } else if _, ok := c.(*re2.Regexp); !ok { + t.Fatalf("compiler is %T, want *re2.Regexp", c) + } + engine, err := fingerlib.NewEngine(fingerlib.FingersEngine) + if err != nil { + t.Fatal(err) + } + frames, err := engine.DetectContent([]byte(nginx)) + if err != nil || frames["nginx"] == nil { + t.Fatalf("frames %v, %v", frames, err) + } +} + +// BenchmarkDetect compares the fingers engine on the standard library and RE2. +func BenchmarkDetect(b *testing.B) { + for _, c := range []struct { + name string + compile func(string) (fingers.CompiledRegexp, error) + }{ + {"regexp", func(s string) (fingers.CompiledRegexp, error) { return regexp.Compile(s) }}, + {"re2", Compile}, + } { + b.Run(c.name, func(b *testing.B) { + fingers.RegexpCompiler = c.compile + defer func() { fingers.RegexpCompiler = Compile }() + engine, err := fingerlib.NewEngine(fingerlib.FingersEngine) + if err != nil { + b.Fatal(err) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + engine.DetectContent([]byte(nginx)) + } + }) + } +} diff --git a/scripts/capture_judge.py b/scripts/capture_judge.py new file mode 100644 index 0000000..bc454f8 --- /dev/null +++ b/scripts/capture_judge.py @@ -0,0 +1,88 @@ +"""Capture explicit URLs as a repeatable, local judgeeval corpus (Python stdlib).""" +import argparse +import concurrent.futures +import datetime +import hashlib +import http.client +import io +import json +import pathlib +import subprocess +import urllib.parse + + +class ResponseSocket: + def __init__(self, data): + self.data = data + + def makefile(self, *_): + return io.BytesIO(self.data) + + +def capture(url, destination, insecure): + parsed = urllib.parse.urlsplit(url) + if parsed.scheme not in ("http", "https") or not parsed.hostname or parsed.username: + return {"url": url, "error": "expected an HTTP(S) URL without credentials"} + sample_id = hashlib.sha256(url.encode()).hexdigest()[:16] + command = ["curl.exe" if __import__("os").name == "nt" else "curl", + "--noproxy", "*", "--http1.1", "--raw", "--silent", "--show-error", + "--include", "--connect-timeout", "3", "--max-time", "8", + "--max-filesize", "524288", "--user-agent", "fingers-validation/1.0"] + if insecure: + command.append("--insecure") + try: + result = subprocess.run(command + [url], capture_output=True, timeout=10) + if result.returncode: + return {"url": url, "curl_exit": result.returncode, + "error": result.stderr.decode(errors="replace").strip()} + response = http.client.HTTPResponse(ResponseSocket(result.stdout)) + response.begin() + body = response.read(524289) + if len(body) > 524288: + raise ValueError("body exceeds capture limit") + raw_path = destination / "samples" / (sample_id + ".http") + raw_path.write_bytes(result.stdout) + # Ignore volatile header values for duplicate detection; retain product evidence. + ignored = {"date", "set-cookie", "etag", "last-modified", "connection", + "content-length", "transfer-encoding", "x-request-id"} + stable_headers = sorted((k.lower(), v) for k, v in response.getheaders() if k.lower() not in ignored) + semantic = json.dumps([response.status, stable_headers]).encode() + b"\n" + body + return {"id": sample_id, "url": url, "group": parsed.hostname, + "captured_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "response": "samples/" + raw_path.name, + "sha256": hashlib.sha256(result.stdout).hexdigest(), + "content_sha256": hashlib.sha256(semantic).hexdigest(), + "status": response.status, "body_bytes": len(body), + "tls_verified": parsed.scheme == "https" and not insecure, + "labels": []} + except (OSError, ValueError, subprocess.TimeoutExpired, http.client.HTTPException) as error: + return {"url": url, "error": str(error)} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--urls", type=pathlib.Path, required=True, help="one explicit URL per line") + parser.add_argument("--out", type=pathlib.Path, required=True, help="new corpus directory") + parser.add_argument("--workers", type=int, default=6) + parser.add_argument("--insecure", action="store_true", help="allow certificate mismatch for IP captures") + args = parser.parse_args() + if not 1 <= args.workers <= 24: + parser.error("workers must be between 1 and 24") + if args.out.exists(): + parser.error("output already exists; use a new directory to preserve the snapshot") + (args.out / "samples").mkdir(parents=True) + urls = sorted(set(line.strip() for line in args.urls.read_text(encoding="utf-8-sig").splitlines() + if line.strip() and not line.startswith("#"))) + with concurrent.futures.ThreadPoolExecutor(max_workers=args.workers) as pool: + rows = list(pool.map(lambda url: capture(url, args.out, args.insecure), urls)) + manifest = {"schema": 1, "scope": "products and versions supported by captured response evidence", + "samples": [row for row in rows if "error" not in row], "generation": []} + (args.out / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8") + failures = [row for row in rows if "error" in row] + (args.out / "fetch-errors.json").write_text(json.dumps(failures, ensure_ascii=False, indent=2), encoding="utf-8") + print(json.dumps({"requests": len(urls), "captured": len(manifest["samples"]), + "failed": len(failures), "manifest": str(args.out / "manifest.json")})) + + +if __name__ == "__main__": + main() diff --git a/wappalyzer/fingerprints.go b/wappalyzer/fingerprints.go index a9a7f8d..bd9e27c 100644 --- a/wappalyzer/fingerprints.go +++ b/wappalyzer/fingerprints.go @@ -3,6 +3,7 @@ package wappalyzer import ( "github.com/chainreactors/fingers/common" "regexp" + "sort" "strconv" "strings" ) @@ -34,6 +35,21 @@ type Fingerprint struct { type CompiledFingerprints struct { // Apps is organized as Apps map[string]*CompiledFingerprint + // ordered holds Apps by name, so apps whose names differ only in case + // ("LiteSpeed Cache", "Litespeed Cache") merge the same way every run. + ordered []*CompiledFingerprint +} + +func (f *CompiledFingerprints) order() { + names := make([]string, 0, len(f.Apps)) + for name := range f.Apps { + names = append(names, name) + } + sort.Strings(names) + f.ordered = make([]*CompiledFingerprint, len(names)) + for i, name := range names { + f.ordered[i] = f.Apps[name] + } } // CompiledFingerprint contains the compiled fingerprints from the tech json @@ -63,6 +79,9 @@ type CompiledFingerprint struct { meta map[string][]*versionRegex // cpe contains the cpe for a fingerpritn cpe string + // cookieKeys, headerKeys and metaKeys list the map keys in a fixed + // order, so the evidence recorded for a hit does not vary between runs. + cookieKeys, headerKeys, metaKeys []string } func (finger *CompiledFingerprint) NewFrame(version string) *common.Framework { @@ -142,6 +161,17 @@ func (v *versionRegex) MatchString(value string) (bool, string) { return true, version } +// matchedText is the text a matching pattern found in value, or key when the +// pattern only requires the key to be present. +func (v *versionRegex) matchedText(value, key string) string { + if !v.skipRegex { + if text := v.regex.FindString(value); text != "" { + return text + } + } + return key +} + // part is the part of the fingerprint to match type part int @@ -236,14 +266,27 @@ func compileFingerprint(app string, fingerprint *Fingerprint) *CompiledFingerpri } compiled.meta[meta] = compiledList } + for k := range compiled.cookies { + compiled.cookieKeys = append(compiled.cookieKeys, k) + } + for k := range compiled.headers { + compiled.headerKeys = append(compiled.headerKeys, k) + } + for k := range compiled.meta { + compiled.metaKeys = append(compiled.metaKeys, k) + } + sort.Strings(compiled.cookieKeys) + sort.Strings(compiled.headerKeys) + sort.Strings(compiled.metaKeys) return compiled } // matchString matches a string for the fingerprints func (f *CompiledFingerprints) matchString(data string, part part) common.Frameworks { var matched bool + var evidence string technologies := make(common.Frameworks) - for _, fingerprint := range f.Apps { + for _, fingerprint := range f.ordered { var version string switch part { @@ -251,6 +294,7 @@ func (f *CompiledFingerprints) matchString(data string, part part) common.Framew for _, pattern := range fingerprint.js { if valid, versionString := pattern.MatchString(data); valid { matched = true + evidence = pattern.matchedText(data, "") version = versionString } } @@ -258,6 +302,7 @@ func (f *CompiledFingerprints) matchString(data string, part part) common.Framew for _, pattern := range fingerprint.scriptSrc { if valid, versionString := pattern.MatchString(data); valid { matched = true + evidence = pattern.matchedText(data, "") version = versionString } } @@ -265,6 +310,7 @@ func (f *CompiledFingerprints) matchString(data string, part part) common.Framew for _, pattern := range fingerprint.html { if valid, versionString := pattern.MatchString(data); valid { matched = true + evidence = pattern.matchedText(data, "") version = versionString } } @@ -278,8 +324,12 @@ func (f *CompiledFingerprints) matchString(data string, part part) common.Framew } frame := fingerprint.NewFrame(version) + if evidence != "" { + frame.MatchDetail = &common.MatchDetail{MatcherType: "word", MatcherValue: evidence} + } technologies.Add(frame) matched = false + evidence = "" } return technologies } @@ -287,9 +337,10 @@ func (f *CompiledFingerprints) matchString(data string, part part) common.Framew // matchKeyValue matches a key-value store map for the fingerprints func (f *CompiledFingerprints) matchKeyValueString(key, value string, part part) common.Frameworks { var matched bool + var evidence string var technologies = make(common.Frameworks) - for _, fingerprint := range f.Apps { + for _, fingerprint := range f.ordered { var version string switch part { @@ -301,6 +352,7 @@ func (f *CompiledFingerprints) matchKeyValueString(key, value string, part part) if valid, versionString := pattern.MatchString(value); valid { matched = true + evidence = pattern.matchedText(value, key) version = versionString break } @@ -313,6 +365,7 @@ func (f *CompiledFingerprints) matchKeyValueString(key, value string, part part) if valid, versionString := pattern.MatchString(value); valid { matched = true + evidence = pattern.matchedText(value, key) version = versionString break } @@ -326,6 +379,7 @@ func (f *CompiledFingerprints) matchKeyValueString(key, value string, part part) for _, pattern := range patterns { if valid, versionString := pattern.MatchString(value); valid { matched = true + evidence = pattern.matchedText(value, key) version = versionString break } @@ -338,8 +392,12 @@ func (f *CompiledFingerprints) matchKeyValueString(key, value string, part part) continue } frame := fingerprint.NewFrame(version) + if evidence != "" { + frame.MatchDetail = &common.MatchDetail{MatcherType: "word", MatcherValue: evidence} + } technologies.Add(frame) matched = false + evidence = "" } return technologies } @@ -347,29 +405,34 @@ func (f *CompiledFingerprints) matchKeyValueString(key, value string, part part) // matchMapString matches a key-value store map for the fingerprints func (f *CompiledFingerprints) matchMapString(keyValue map[string]string, part part) common.Frameworks { var matched bool + var evidence string technologies := make(common.Frameworks) - for _, fingerprint := range f.Apps { + for _, fingerprint := range f.ordered { var version string switch part { case cookiesPart: - for data, pattern := range fingerprint.cookies { + for _, data := range fingerprint.cookieKeys { + pattern := fingerprint.cookies[data] value, ok := keyValue[data] if !ok { continue } if pattern == nil { matched = true + evidence = data } if valid, versionString := pattern.MatchString(value); valid { matched = true + evidence = pattern.matchedText(value, data) version = versionString break } } case headersPart: - for data, pattern := range fingerprint.headers { + for _, data := range fingerprint.headerKeys { + pattern := fingerprint.headers[data] value, ok := keyValue[data] if !ok { continue @@ -377,12 +440,14 @@ func (f *CompiledFingerprints) matchMapString(keyValue map[string]string, part p if valid, versionString := pattern.MatchString(value); valid { matched = true + evidence = pattern.matchedText(value, data) version = versionString break } } case metaPart: - for data, patterns := range fingerprint.meta { + for _, data := range fingerprint.metaKeys { + patterns := fingerprint.meta[data] value, ok := keyValue[data] if !ok { continue @@ -391,6 +456,7 @@ func (f *CompiledFingerprints) matchMapString(keyValue map[string]string, part p for _, pattern := range patterns { if valid, versionString := pattern.MatchString(value); valid { matched = true + evidence = pattern.matchedText(value, data) version = versionString break } @@ -406,8 +472,12 @@ func (f *CompiledFingerprints) matchMapString(keyValue map[string]string, part p // Append the technologies as well as implied ones frame := fingerprint.NewFrame(version) + if evidence != "" { + frame.MatchDetail = &common.MatchDetail{MatcherType: "word", MatcherValue: evidence} + } technologies.Add(frame) matched = false + evidence = "" } return technologies } diff --git a/wappalyzer/wappalyzer.go b/wappalyzer/wappalyzer.go index 31d88ef..3da8fe1 100644 --- a/wappalyzer/wappalyzer.go +++ b/wappalyzer/wappalyzer.go @@ -11,6 +11,8 @@ import ( // Wappalyze is a client for working with tech detection type Wappalyze struct { fingerprints *CompiledFingerprints + // MatchDetailEnabled records on each hit the text its pattern matched. + MatchDetailEnabled bool } // NewWappalyzeEngine creates a new tech detection instance @@ -56,6 +58,7 @@ func (engine *Wappalyze) loadFingerprints(data []byte) error { for app, fingerprint := range fingerprintsStruct.Apps { engine.fingerprints.Apps[app] = compileFingerprint(app, fingerprint) } + engine.fingerprints.order() return nil } @@ -105,7 +108,7 @@ func (engine *Wappalyze) Fingerprint(headers map[string][]string, body []byte) c // Check for stuff in the body finally uniqueFingerprints.Merge(engine.checkBody(normalizedBody)) - return uniqueFingerprints + return engine.withDetail(uniqueFingerprints) } // FingerprintWithTitle identifies technologies on a target, @@ -137,9 +140,9 @@ func (engine *Wappalyze) FingerprintWithTitle(headers map[string][]string, body bodyTech := engine.checkBody(normalizedBody) uniqueFingerprints.Merge(bodyTech) title := engine.getTitle(body) - return uniqueFingerprints, title + return engine.withDetail(uniqueFingerprints), title } - return uniqueFingerprints, "" + return engine.withDetail(uniqueFingerprints), "" } // FingerprintWithInfo identifies technologies on a target, @@ -185,3 +188,13 @@ func (engine *Wappalyze) FingerprintWithCats(headers map[string][]string, body [ return result } + +// withDetail drops the matched text from hits unless MatchDetailEnabled. +func (engine *Wappalyze) withDetail(frames common.Frameworks) common.Frameworks { + if !engine.MatchDetailEnabled { + for _, frame := range frames { + frame.MatchDetail = nil + } + } + return frames +}