From bfd56d5656f00cd139ea6c1a95c1ae0b1f88167a Mon Sep 17 00:00:00 2001 From: M09Ic Date: Thu, 24 Sep 2026 21:48:14 -0700 Subject: [PATCH 01/16] feat(jev): add the Jev judgement layer on top of the rule engines Jev judges the rule result instead of being another engine: rules and code recall candidates, Jev only judges evidence, code makes the final call. - jev.Page is the page entry, jev.Round is one request of questions with callbacks; Verify / Classify / Version are the capabilities - results annotate common.Frameworks in place with jev:* tags; a failed round leaves frames untouched - Client cache keyed by exact questions plus a page simhash, so similar pages can share answers - Engine.AttachJev / Engine.Refine; WebMatch and DetectContent unchanged - cmd/jevbench compares rules vs rules+Jev; cmd/engine gets --jev Also fix MergeFrameworks sharing alias Attributes across pages: a version written on one page leaked into every later hit of the same alias. Refs #34 Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 44 ++ alias_attributes_test.go | 26 + cmd/engine/example.go | 463 ++++++------ cmd/jevbench/README.md | 25 + cmd/jevbench/main.go | 657 ++++++++++++++++++ cmd/jevbench/testdata/labels.json | 78 +++ .../samples/blog_mentions_jenkins.http | 5 + .../testdata/samples/confluence_login.http | 7 + .../testdata/samples/corporate_cn_portal.http | 5 + .../testdata/samples/corporate_site.http | 5 + .../samples/custom_employee_portal.http | 6 + .../testdata/samples/dir_listing.http | 5 + .../testdata/samples/discuz_forum.http | 7 + .../testdata/samples/docs_tomcat_howto.http | 6 + .../testdata/samples/gitlab_signin.http | 8 + .../testdata/samples/grafana_login.http | 6 + .../testdata/samples/harbor_portal.http | 6 + .../testdata/samples/iis_default.http | 7 + .../testdata/samples/jenkins_login.http | 6 + .../testdata/samples/job_posting_cn.http | 5 + cmd/jevbench/testdata/samples/json_api.http | 5 + .../testdata/samples/kibana_home.http | 7 + .../testdata/samples/minio_console.http | 6 + .../testdata/samples/nacos_console.http | 5 + .../testdata/samples/news_fortios_patch.http | 6 + cmd/jevbench/testdata/samples/nginx_404.http | 11 + .../testdata/samples/nginx_default.http | 5 + .../testdata/samples/pgadmin_login.http | 6 + .../testdata/samples/phpmyadmin_login.http | 8 + .../testdata/samples/portainer_login.http | 6 + .../testdata/samples/rabbitmq_mgmt.http | 5 + .../testdata/samples/rebranded_grafana.http | 5 + .../testdata/samples/rebranded_jenkins.http | 6 + .../testdata/samples/rebranded_zabbix.http | 6 + .../testdata/samples/seeyon_login.http | 6 + cmd/jevbench/testdata/samples/shop_page.http | 8 + .../testdata/samples/springboot_error.http | 4 + .../samples/status_page_mentions.http | 5 + .../testdata/samples/synology_dsm.http | 5 + .../testdata/samples/tomcat_default.http | 4 + .../testdata/samples/tplink_router.http | 5 + .../testdata/samples/unknown_admin_login.http | 5 + .../samples/weaver_ecology_login.http | 5 + .../testdata/samples/wordpress_home.http | 6 + .../testdata/samples/wordpress_login.http | 7 + .../testdata/samples/zabbix_login.http | 6 + engine.go | 9 +- jev.go | 40 ++ jev/cache.go | 91 +++ jev/classify.go | 30 + jev/client.go | 200 ++++++ jev/jev_test.go | 407 +++++++++++ jev/page.go | 235 +++++++ jev/recall.go | 81 +++ jev/refine.go | 65 ++ jev/round.go | 87 +++ jev/stack.go | 59 ++ jev/verify.go | 153 ++++ jev/version.go | 188 +++++ jev_test.go | 52 ++ 60 files changed, 3010 insertions(+), 217 deletions(-) create mode 100644 alias_attributes_test.go create mode 100644 cmd/jevbench/README.md create mode 100644 cmd/jevbench/main.go create mode 100644 cmd/jevbench/testdata/labels.json create mode 100644 cmd/jevbench/testdata/samples/blog_mentions_jenkins.http create mode 100644 cmd/jevbench/testdata/samples/confluence_login.http create mode 100644 cmd/jevbench/testdata/samples/corporate_cn_portal.http create mode 100644 cmd/jevbench/testdata/samples/corporate_site.http create mode 100644 cmd/jevbench/testdata/samples/custom_employee_portal.http create mode 100644 cmd/jevbench/testdata/samples/dir_listing.http create mode 100644 cmd/jevbench/testdata/samples/discuz_forum.http create mode 100644 cmd/jevbench/testdata/samples/docs_tomcat_howto.http create mode 100644 cmd/jevbench/testdata/samples/gitlab_signin.http create mode 100644 cmd/jevbench/testdata/samples/grafana_login.http create mode 100644 cmd/jevbench/testdata/samples/harbor_portal.http create mode 100644 cmd/jevbench/testdata/samples/iis_default.http create mode 100644 cmd/jevbench/testdata/samples/jenkins_login.http create mode 100644 cmd/jevbench/testdata/samples/job_posting_cn.http create mode 100644 cmd/jevbench/testdata/samples/json_api.http create mode 100644 cmd/jevbench/testdata/samples/kibana_home.http create mode 100644 cmd/jevbench/testdata/samples/minio_console.http create mode 100644 cmd/jevbench/testdata/samples/nacos_console.http create mode 100644 cmd/jevbench/testdata/samples/news_fortios_patch.http create mode 100644 cmd/jevbench/testdata/samples/nginx_404.http create mode 100644 cmd/jevbench/testdata/samples/nginx_default.http create mode 100644 cmd/jevbench/testdata/samples/pgadmin_login.http create mode 100644 cmd/jevbench/testdata/samples/phpmyadmin_login.http create mode 100644 cmd/jevbench/testdata/samples/portainer_login.http create mode 100644 cmd/jevbench/testdata/samples/rabbitmq_mgmt.http create mode 100644 cmd/jevbench/testdata/samples/rebranded_grafana.http create mode 100644 cmd/jevbench/testdata/samples/rebranded_jenkins.http create mode 100644 cmd/jevbench/testdata/samples/rebranded_zabbix.http create mode 100644 cmd/jevbench/testdata/samples/seeyon_login.http create mode 100644 cmd/jevbench/testdata/samples/shop_page.http create mode 100644 cmd/jevbench/testdata/samples/springboot_error.http create mode 100644 cmd/jevbench/testdata/samples/status_page_mentions.http create mode 100644 cmd/jevbench/testdata/samples/synology_dsm.http create mode 100644 cmd/jevbench/testdata/samples/tomcat_default.http create mode 100644 cmd/jevbench/testdata/samples/tplink_router.http create mode 100644 cmd/jevbench/testdata/samples/unknown_admin_login.http create mode 100644 cmd/jevbench/testdata/samples/weaver_ecology_login.http create mode 100644 cmd/jevbench/testdata/samples/wordpress_home.http create mode 100644 cmd/jevbench/testdata/samples/wordpress_login.http create mode 100644 cmd/jevbench/testdata/samples/zabbix_login.http create mode 100644 jev.go create mode 100644 jev/cache.go create mode 100644 jev/classify.go create mode 100644 jev/client.go create mode 100644 jev/jev_test.go create mode 100644 jev/page.go create mode 100644 jev/recall.go create mode 100644 jev/refine.go create mode 100644 jev/round.go create mode 100644 jev/stack.go create mode 100644 jev/verify.go create mode 100644 jev/version.go create mode 100644 jev_test.go diff --git a/README.md b/README.md index 4538fb0..1300cf3 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,50 @@ func TestFavicon(t *testing.T) { 更多用法请见: https://chainreactors.github.io/wiki/libs/fingers/sdk/ +## Jev 判定层 + +规则引擎召回能力强，但会有误报、多个引擎的同名重复，也分不清主应用和底层组件，版本号更是难以拿准。`jev` 包在规则结果之上接入 [TypeSafe Jev](https://docs.typesafe.ai/api.md)，做一层判定：由规则和代码负责召回，Jev 只对证据做判断，最终决策由代码做出。它不是新的引擎，`WebMatch` 和 `DetectContent` 的行为保持不变。设计讨论见 [#34](https://github.com/chainreactors/fingers/issues/34)。 + +```golang +client, _ := jev.NewClient("") // 读取 TYPESAFE_API_KEY +client.Cache = jev.NewMemoryCache(10000) // 可选，按请求内容缓存 +engine.AttachJev(client) + +frames, _ := engine.DetectContent(raw) // 同步，纯规则 +page, err := engine.Refine(ctx, raw, frames) // 通常异步执行；原地标注 frames +if err == nil { + fmt.Println(page.Kind, jev.Accepted(frames)) // 页面类型；去掉误报和重复后的结果 +} +``` + +`Refine` 在 frames 上原地打标签，不删除任何条目： + +| 标签 | 含义 | +|---|---| +| `jev:rejected` | 误报：不在技术栈里，或者只是正文里提到。名字出现在 header/cookie 中的命中永远不会被否决 | +| `jev:dup` | 另一个引擎对同一产品的另一种写法 | +| `jev:layer=` | 所属层级：`cdn_or_waf`、`web_server`、`language_runtime`、`web_framework`、`application`、`frontend_library`、`os_or_device` | +| `jev:primary` | 本页所属的主应用。主应用没有版本号时，由 Jev 从页面上的版本字符串中选出，置信度 ≥0.9 才写入 | +| `jev:recall` | 漏报：规则没有命中，但指纹名在页面中出现且经 Jev 确认，以 `guess` 来源加入 | + +每页最多 2 次请求。请求失败时 frames 保持规则结果不变。 + +各项能力都是可以单独使用的函数，组合在一个 `Round`（即一次请求）里： + +```golang +page, _ := jev.NewPage(raw) +r := page.Round() +jev.Classify(r) // 只做页面类型 / 是否通用页面，例如用于 404 识别 +r.Add("honeypot", jev.Noul("Is this a honeypot?"), func(a jev.Answer) { /* 自定义问题 */ }) +err := r.Ask(ctx, client) + +r = page.Round() +jev.Version(r, frame) // 只给某个已确定的产品补版本号 +err = r.Ask(ctx, client) +``` + +边界：改过品牌的产品 Jev 认不出来，召回仍然靠规则；阈值针对 `jev.DefaultModel` 调过，换模型后要用 `cmd/jevbench` 重新评测。 + ## fingers 引擎 fingers指纹引擎是目前特性最丰富, 性能最强的指纹规则库. diff --git a/alias_attributes_test.go b/alias_attributes_test.go new file mode 100644 index 0000000..1a42b88 --- /dev/null +++ b/alias_attributes_test.go @@ -0,0 +1,26 @@ +package fingers + +import "testing" + +// A version written on one page's frameworks must not reach the next page: +// alias attributes are shared by every page. +func TestMergeFrameworksCopiesAliasAttributes(t *testing.T) { + e, err := NewEngine(FingersEngine, WappalyzerEngine, GobyEngine, EHoleEngine, FingerPrintEngine) + if err != nil { + t.Fatal(err) + } + raw := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\n" + + "") + for i := 0; i < 30; i++ { // which pointer survives depends on map order + a, _ := e.DetectContent(raw) + for _, f := range a { + f.Attributes.Version = "9.9.9" + } + b, _ := e.DetectContent(raw) + for _, f := range b { + if f.Version == "9.9.9" { + t.Fatalf("version leaked into the next page: %s", f.Name) + } + } + } +} diff --git a/cmd/engine/example.go b/cmd/engine/example.go index 472311e..48c6771 100644 --- a/cmd/engine/example.go +++ b/cmd/engine/example.go @@ -1,216 +1,247 @@ -package main - -import ( - "crypto/tls" - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "os" - "path/filepath" - "strings" - - "github.com/chainreactors/fingers" - "github.com/chainreactors/fingers/common" - "github.com/chainreactors/fingers/resources" - "github.com/chainreactors/utils/encode" - "github.com/chainreactors/utils/httputils" - "github.com/jessevdk/go-flags" - "gopkg.in/yaml.v3" -) - -var opts struct { - // 指定要使用的引擎，多个引擎用逗号分隔 - Engines string `short:"e" long:"engines" description:"Specify engines to use (comma separated)" default:"fingers,fingerprinthub,wappalyzer,ehole,goby"` - - // 是否忽略SSL证书验证 - InsecureSSL bool `short:"k" long:"insecure" description:"Skip SSL certificate verification"` - - // 目标URL - URL string `short:"u" long:"url" description:"Target URL to fingerprint" required:"true"` - - // 是否显示详细信息 - Verbose bool `short:"v" long:"verbose" description:"Show verbose debug information"` - - // 是否只检测favicon - FaviconOnly bool `short:"f" long:"favicon" description:"Only detect favicon"` - - // 资源文件覆盖 - GobyFile string `long:"goby" description:"Override goby.json.gz with custom file"` - FingerprintHubWebFile string `long:"fingerprinthub-web" description:"Override fingerprinthub_web.json.gz with custom file"` - FingerprintHubServiceFile string `long:"fingerprinthub-service" description:"Override fingerprinthub_service.json.gz with custom file"` - EholeFile string `long:"ehole" description:"Override ehole.json.gz with custom file"` - FingersFile string `long:"fingers" description:"Override fingers_http.json.gz with custom file"` - WappalyzerFile string `long:"wappalyzer" description:"Override wappalyzer.json.gz with custom file"` - AliasesFile string `long:"aliases" description:"Override aliases.yaml with custom file"` -} - -// 处理资源文件覆盖 -func processResourceFile(filePath string) ([]byte, error) { - if filePath == "" { - return nil, nil - } - - data, err := ioutil.ReadFile(filePath) - if err != nil { - return nil, fmt.Errorf("failed to read file %s: %v", filePath, err) - } - - ext := strings.ToLower(filepath.Ext(filePath)) - - // 如果是JSON文件，转换为YAML - if ext == ".json" { - var jsonData interface{} - if err := json.Unmarshal(data, &jsonData); err != nil { - return nil, fmt.Errorf("failed to parse JSON file %s: %v", filePath, err) - } - - data, err = yaml.Marshal(jsonData) - if err != nil { - return nil, fmt.Errorf("failed to convert JSON to YAML for file %s: %v", filePath, err) - } - } - - // 如果文件不是.gz结尾，进行gzip压缩 - if !strings.HasSuffix(filePath, ".gz") { - compressedData, err := encode.GzipCompress(data) - if err != nil { - return nil, fmt.Errorf("failed to compress file %s: %v", filePath, err) - } - data = compressedData - } - - return data, nil -} - -func main() { - // 解析命令行参数 - _, err := flags.Parse(&opts) - if err != nil { - os.Exit(1) - } - - // 处理资源文件覆盖 - if opts.GobyFile != "" { - if data, err := processResourceFile(opts.GobyFile); err != nil { - fmt.Println(err) - os.Exit(1) - } else { - resources.GobyData = data - } - } - - if opts.FingerprintHubWebFile != "" { - if data, err := processResourceFile(opts.FingerprintHubWebFile); err != nil { - fmt.Println(err) - os.Exit(1) - } else { - resources.FingerprinthubWebData = data - } - } - - if opts.FingerprintHubServiceFile != "" { - if data, err := processResourceFile(opts.FingerprintHubServiceFile); err != nil { - fmt.Println(err) - os.Exit(1) - } else { - resources.FingerprinthubServiceData = data - } - } - - if opts.EholeFile != "" { - if data, err := processResourceFile(opts.EholeFile); err != nil { - fmt.Println(err) - os.Exit(1) - } else { - resources.EholeData = data - } - } - - if opts.FingersFile != "" { - if data, err := processResourceFile(opts.FingersFile); err != nil { - fmt.Println(err) - os.Exit(1) - } else { - resources.FingersHTTPData = data - } - } - - if opts.WappalyzerFile != "" { - if data, err := processResourceFile(opts.WappalyzerFile); err != nil { - fmt.Println(err) - os.Exit(1) - } else { - resources.WappalyzerData = data - } - } - - if opts.AliasesFile != "" { - if data, err := processResourceFile(opts.AliasesFile); err != nil { - fmt.Println(err) - os.Exit(1) - } else { - resources.AliasesData = data - } - } - - // 创建HTTP客户端 - client := &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: opts.InsecureSSL, - }, - }, - } - - // 创建引擎实例 - var engineNames []string - if opts.Engines != "" { - engineNames = strings.Split(opts.Engines, ",") - } - - engine, err := fingers.NewEngine(engineNames...) - if err != nil { - fmt.Printf("Failed to create engine: %v\n", err) - os.Exit(1) - } - - if opts.Verbose { - fmt.Printf("Loaded engines: %s\n", engine.String()) - } - - // 发送HTTP请求 - resp, err := client.Get(opts.URL) - if err != nil { - fmt.Printf("Failed to request URL %s: %v\n", opts.URL, err) - os.Exit(1) - } - defer resp.Body.Close() - - // 检测指纹 - var frames common.Frameworks - if opts.FaviconOnly { - content := httputils.ReadBody(resp) - frame := engine.DetectFavicon(content) - if frame != nil { - frames.Add(frame) - } - } else { - frames = engine.Match(resp) - } - - // 输出结果 - if opts.Verbose { - fmt.Printf("\nDetected frameworks for %s:\n", opts.URL) - for _, frame := range frames { - fmt.Printf("Name: %s\n", frame.Name) - fmt.Printf("Vendor: %s\n", frame.Vendor) - fmt.Printf("Product: %s\n", frame.Product) - fmt.Printf("Version: %s\n", frame.Version) - fmt.Printf("CPE: %s\n", frame.CPE()) - fmt.Printf("---\n") - } - } else { - fmt.Println(frames.String()) - } -} +package main + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/chainreactors/fingers" + "github.com/chainreactors/fingers/common" + "github.com/chainreactors/fingers/jev" + "github.com/chainreactors/fingers/resources" + "github.com/chainreactors/utils/encode" + "github.com/chainreactors/utils/httputils" + "github.com/jessevdk/go-flags" + "gopkg.in/yaml.v3" +) + +var opts struct { + // 指定要使用的引擎，多个引擎用逗号分隔 + Engines string `short:"e" long:"engines" description:"Specify engines to use (comma separated)" default:"fingers,fingerprinthub,wappalyzer,ehole,goby"` + + // 是否忽略SSL证书验证 + InsecureSSL bool `short:"k" long:"insecure" description:"Skip SSL certificate verification"` + + // 目标URL + URL string `short:"u" long:"url" description:"Target URL to fingerprint" required:"true"` + + // 是否显示详细信息 + Verbose bool `short:"v" long:"verbose" description:"Show verbose debug information"` + + // 是否只检测favicon + FaviconOnly bool `short:"f" long:"favicon" description:"Only detect favicon"` + + // 在规则结果之上运行 Jev 判定层, 需要 TYPESAFE_API_KEY + Jev bool `long:"jev" description:"Judge the rule result with Jev (needs TYPESAFE_API_KEY)"` + + // 资源文件覆盖 + GobyFile string `long:"goby" description:"Override goby.json.gz with custom file"` + FingerprintHubWebFile string `long:"fingerprinthub-web" description:"Override fingerprinthub_web.json.gz with custom file"` + FingerprintHubServiceFile string `long:"fingerprinthub-service" description:"Override fingerprinthub_service.json.gz with custom file"` + EholeFile string `long:"ehole" description:"Override ehole.json.gz with custom file"` + FingersFile string `long:"fingers" description:"Override fingers_http.json.gz with custom file"` + WappalyzerFile string `long:"wappalyzer" description:"Override wappalyzer.json.gz with custom file"` + AliasesFile string `long:"aliases" description:"Override aliases.yaml with custom file"` +} + +// 处理资源文件覆盖 +func processResourceFile(filePath string) ([]byte, error) { + if filePath == "" { + return nil, nil + } + + data, err := ioutil.ReadFile(filePath) + if err != nil { + return nil, fmt.Errorf("failed to read file %s: %v", filePath, err) + } + + ext := strings.ToLower(filepath.Ext(filePath)) + + // 如果是JSON文件，转换为YAML + if ext == ".json" { + var jsonData interface{} + if err := json.Unmarshal(data, &jsonData); err != nil { + return nil, fmt.Errorf("failed to parse JSON file %s: %v", filePath, err) + } + + data, err = yaml.Marshal(jsonData) + if err != nil { + return nil, fmt.Errorf("failed to convert JSON to YAML for file %s: %v", filePath, err) + } + } + + // 如果文件不是.gz结尾，进行gzip压缩 + if !strings.HasSuffix(filePath, ".gz") { + compressedData, err := encode.GzipCompress(data) + if err != nil { + return nil, fmt.Errorf("failed to compress file %s: %v", filePath, err) + } + data = compressedData + } + + return data, nil +} + +func main() { + // 解析命令行参数 + _, err := flags.Parse(&opts) + if err != nil { + os.Exit(1) + } + + // 处理资源文件覆盖 + if opts.GobyFile != "" { + if data, err := processResourceFile(opts.GobyFile); err != nil { + fmt.Println(err) + os.Exit(1) + } else { + resources.GobyData = data + } + } + + if opts.FingerprintHubWebFile != "" { + if data, err := processResourceFile(opts.FingerprintHubWebFile); err != nil { + fmt.Println(err) + os.Exit(1) + } else { + resources.FingerprinthubWebData = data + } + } + + if opts.FingerprintHubServiceFile != "" { + if data, err := processResourceFile(opts.FingerprintHubServiceFile); err != nil { + fmt.Println(err) + os.Exit(1) + } else { + resources.FingerprinthubServiceData = data + } + } + + if opts.EholeFile != "" { + if data, err := processResourceFile(opts.EholeFile); err != nil { + fmt.Println(err) + os.Exit(1) + } else { + resources.EholeData = data + } + } + + if opts.FingersFile != "" { + if data, err := processResourceFile(opts.FingersFile); err != nil { + fmt.Println(err) + os.Exit(1) + } else { + resources.FingersHTTPData = data + } + } + + if opts.WappalyzerFile != "" { + if data, err := processResourceFile(opts.WappalyzerFile); err != nil { + fmt.Println(err) + os.Exit(1) + } else { + resources.WappalyzerData = data + } + } + + if opts.AliasesFile != "" { + if data, err := processResourceFile(opts.AliasesFile); err != nil { + fmt.Println(err) + os.Exit(1) + } else { + resources.AliasesData = data + } + } + + // 创建HTTP客户端 + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: opts.InsecureSSL, + }, + }, + } + + // 创建引擎实例 + var engineNames []string + if opts.Engines != "" { + engineNames = strings.Split(opts.Engines, ",") + } + + engine, err := fingers.NewEngine(engineNames...) + if err != nil { + fmt.Printf("Failed to create engine: %v\n", err) + os.Exit(1) + } + + if opts.Verbose { + fmt.Printf("Loaded engines: %s\n", engine.String()) + } + + // 发送HTTP请求 + resp, err := client.Get(opts.URL) + if err != nil { + fmt.Printf("Failed to request URL %s: %v\n", opts.URL, err) + os.Exit(1) + } + defer resp.Body.Close() + + // 检测指纹 + var frames common.Frameworks + if opts.FaviconOnly { + content := httputils.ReadBody(resp) + frame := engine.DetectFavicon(content) + if frame != nil { + frames.Add(frame) + } + } else { + var content []byte + if opts.Jev { // Refine needs the raw response; put the body back for Match + content = httputils.ReadRaw(resp) + body, _, _ := httputils.SplitHttpRaw(content) + resp.Body = ioutil.NopCloser(bytes.NewReader(body)) + } + frames = engine.Match(resp) + if opts.Jev { + client, err := jev.NewClient("") + if err != nil { + fmt.Println(err) + os.Exit(1) + } + engine.AttachJev(client) + page, err := engine.Refine(context.Background(), content, frames) + if err != nil { + fmt.Printf("jev failed, showing the rule result: %v\n", err) + } else { + fmt.Printf("page: %s, generic: %v\n", page.Kind, page.Generic) + for _, frame := range frames { + fmt.Printf(" %-40s %-10s %v\n", frame.Name, frame.Version, frame.Tags) + } + fmt.Printf("accepted: %s\n", jev.Accepted(frames).String()) + return + } + } + } + + // 输出结果 + if opts.Verbose { + fmt.Printf("\nDetected frameworks for %s:\n", opts.URL) + for _, frame := range frames { + fmt.Printf("Name: %s\n", frame.Name) + fmt.Printf("Vendor: %s\n", frame.Vendor) + fmt.Printf("Product: %s\n", frame.Product) + fmt.Printf("Version: %s\n", frame.Version) + fmt.Printf("CPE: %s\n", frame.CPE()) + fmt.Printf("---\n") + } + } else { + fmt.Println(frames.String()) + } +} diff --git a/cmd/jevbench/README.md b/cmd/jevbench/README.md new file mode 100644 index 0000000..9e66779 --- /dev/null +++ b/cmd/jevbench/README.md @@ -0,0 +1,25 @@ +# jevbench：纯规则 vs 规则 + Jev + +在一批原始 HTTP 响应上，用和 SDK 调用方完全相同的路径（`engine.DetectContent` → `engine.Refine`）跑一遍，逐项对比纯规则结果和经过 Jev 判定之后的结果。 + +```bash +go build -buildvcs=false -tags goregexp -o jevbench ./cmd/jevbench + +# 标注集（仓库自带 40 页，含真值） +TYPESAFE_API_KEY=... ./jevbench -samples testdata/samples -labels testdata/labels.json + +# 任意真实页面目录（*.http，完整的原始响应） +TYPESAFE_API_KEY=... ./jevbench -samples cc/samples -cache jevcache -out jevreport -rps 15 +``` + +- Jev 的回答按请求内容缓存在 `-cache` 目录（实现了 `jev.Cache` 的文件缓存），重跑不花钱，中断后可以接着跑。 +- `jevreport/report.md`：对比表，另附页面类型分布、各引擎的否决率、否决率最高的规则（待修的规则），以及新指纹候选。 +- `jevreport/rows.jsonl`：每页一行，列出规则命中、被接受的、被否决的、召回的、层级、主应用和版本号，用于人工复核。 +- 给了 `-labels` 时，额外按真值统计误报剔除、真实命中保留、页面类型、通用页面和版本号的正确率。 + +没有真值的数据集用这两个代理指标： + +- **仅正文出现的命中**：产品名只出现在可见正文里、不在任何结构字段中，是由代码判断出的疑似误报； +- **版本号**：以页面的 generator meta 作为参考答案。 + +真实网站的页面属于第三方内容，不要提交进仓库。 diff --git a/cmd/jevbench/main.go b/cmd/jevbench/main.go new file mode 100644 index 0000000..a4e842c --- /dev/null +++ b/cmd/jevbench/main.go @@ -0,0 +1,657 @@ +// jevbench measures the Jev judgement layer, exactly as SDK callers run it +// (fingers.Engine.Refine), on a directory of raw HTTP responses. +// +// TYPESAFE_API_KEY=... jevbench -samples testdata/samples -labels testdata/labels.json +// TYPESAFE_API_KEY=... jevbench -samples cclogin/samples -cache cache -out report +// +// Jev answers are cached on disk by request content, so reruns are free and +// an interrupted run resumes. Build with -tags goregexp. +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "net/http" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "sync" + "time" + "unicode" + + "github.com/chainreactors/fingers" + "github.com/chainreactors/fingers/common" + "github.com/chainreactors/fingers/jev" + "github.com/chainreactors/logs" +) + +// row is the outcome for one page, written to rows.jsonl for review. +type row struct { + ID string `json:"id"` + Rules []string `json:"rules"` // name@engine of every rule hit + Accepted []string `json:"accepted"` + Rejected []string `json:"rejected,omitempty"` + Recall []string `json:"recall,omitempty"` + Layers map[string]string `json:"layers,omitempty"` + Primary string `json:"primary,omitempty"` + Version string `json:"version,omitempty"` + JevVersion bool `json:"jev_version,omitempty"` // Version written by Jev, not by a rule + Kind string `json:"kind"` + Generic bool `json:"generic"` + NewFinger bool `json:"new_fingerprint_candidate,omitempty"` + Err string `json:"error,omitempty"` + + // rules vs rules+Jev on the same page + RuleProducts int `json:"rule_products"` // rule hits after folding spellings + TextOnly []string `json:"text_only,omitempty"` // rule hits named only in visible text: likely false positives + TextOnlyKept int `json:"text_only_kept"` // of those, accepted by Jev + Generator string `json:"generator,omitempty"` // "wordpress 6.4.2": the reference version + RuleGenVer string `json:"rule_gen_version,omitempty"` + JevGenVer string `json:"jev_gen_version,omitempty"` + RuleMs float64 `json:"rule_ms"` + JevMs float64 `json:"jev_ms"` + + frames common.Frameworks // annotated + ruleEngines map[string][]string // rule name -> engines + err error +} + +func main() { + samples := flag.String("samples", "testdata/samples", "directory of raw HTTP responses (*.http)") + labels := flag.String("labels", "", "optional ground truth (see testdata/labels.json)") + cache := flag.String("cache", "jevcache", "directory caching Jev answers") + out := flag.String("out", "jevreport", "output directory") + rps := flag.Float64("rps", 15, "max Jev requests per second") + workers := flag.Int("workers", 16, "concurrent pages") + limit := flag.Int("limit", 0, "only the first N samples (0 = all)") + endpoint := flag.String("endpoint", jev.DefaultEndpoint, "Jev API endpoint") + flag.Parse() + logs.Log.SetLevel(logs.ErrorLevel) + if err := run(*samples, *labels, *cache, *out, *endpoint, *rps, *workers, *limit); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run(samples, labelsPath, cacheDir, out, endpoint string, rps float64, workers, limit int) error { + files, err := filepath.Glob(filepath.Join(samples, "*.http")) + if err != nil { + return err + } + sort.Strings(files) + if limit > 0 && len(files) > limit { + files = files[:limit] + } + for _, d := range []string{cacheDir, out} { + if err := os.MkdirAll(d, 0o755); err != nil { + return err + } + } + client, err := jev.NewClient("") + if err != nil { + return err + } + client.Endpoint = endpoint + sim := newSimCheck(cacheDir) + client.Cache = sim + client.HTTP.Transport = &limited{tick: time.NewTicker(time.Duration(float64(time.Second) / rps)).C} + + engine, err := fingers.NewEngine(fingers.FingersEngine, fingers.FingerPrintEngine, fingers.EHoleEngine, fingers.GobyEngine, fingers.WappalyzerEngine) + if err != nil { + return err + } + engine.AttachJev(client) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + rows := make([]*row, len(files)) + jobs := make(chan int) + var wg sync.WaitGroup + var fatal error + var once sync.Once + for w := 0; w < workers; w++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := range jobs { + rows[i] = judge(ctx, engine, files[i]) + var apiErr *jev.APIError + if rows[i].err != nil && errors.As(rows[i].err, &apiErr) && apiErr.Status == 401 { + once.Do(func() { fatal = errors.New("unauthorized: check " + jev.EnvAPIKey); cancel() }) + } + } + }() + } + start := time.Now() + for i := range files { + if ctx.Err() != nil { + break + } + jobs <- i + } + close(jobs) + wg.Wait() + if fatal != nil { + return fatal + } + + f, err := os.Create(filepath.Join(out, "rows.jsonl")) + if err != nil { + return err + } + enc := json.NewEncoder(f) + enc.SetEscapeHTML(false) + for _, r := range rows { + if r != nil { + enc.Encode(r) + } + } + f.Close() + + var b strings.Builder + report(&b, rows, client, time.Since(start)) + fmt.Fprintf(&b, "\n## 相似度缓存（SimilarDistance=%d，模拟）\n\n", jev.SimilarDistance) + fmt.Fprintf(&b, "- 请求 %d 次，其中 %d 次（%.1f%%）可以直接复用相似页面的答案；复用答案与真实答案在所有决策上完全一致的占 %d/%d（%.1f%%）\n", + sim.requests, sim.hits, 100*div(sim.hits, sim.requests), sim.agree, sim.hits, 100*div(sim.agree, sim.hits)) + for _, k := range sortedKeys(sim.diffs, func(k string) int { return sim.diffs[k] }) { + fmt.Fprintf(&b, " - 不一致的题目 %s: %d\n", k, sim.diffs[k]) + } + if labelsPath != "" { + if err := evaluate(&b, rows, labelsPath); err != nil { + return err + } + } + fmt.Print(b.String()) + return os.WriteFile(filepath.Join(out, "report.md"), []byte(b.String()), 0o644) +} + +func judge(ctx context.Context, engine *fingers.Engine, path string) (r *row) { + r = &row{ID: filepath.Base(path), ruleEngines: map[string][]string{}} + defer func() { + if p := recover(); p != nil { // one malformed page must not stop a long run + r.Err = fmt.Sprintf("panic: %v", p) + } + }() + raw, err := os.ReadFile(path) + var page *jev.Page + if err == nil { + page, err = jev.NewPage(raw) // for code-only evidence; Refine builds its own + } + if err == nil { + t := time.Now() + r.frames, err = engine.DetectContent(raw) + r.RuleMs = ms(time.Since(t)) + } + if err != nil { + r.Err, r.err = err.Error(), err + return r + } + product, want := generatorVersion(page.Generator) + if want != "" { + r.Generator = product + " " + want + } + keys := map[string]bool{} + for _, f := range r.frames { + keys[jev.NormalizeName(f.Name)] = true + if ev := page.Evidence(f.Name); len(ev) == 1 && ev[0] == "text" { + r.TextOnly = append(r.TextOnly, f.Name) + } + } + r.RuleProducts = len(keys) + r.RuleGenVer = versionOf(r.frames, product) + versions := map[string]string{} + for _, f := range r.frames { + for from := range f.Froms { + r.Rules = append(r.Rules, f.Name+"@"+from.String()) + r.ruleEngines[f.Name] = append(r.ruleEngines[f.Name], from.String()) + } + versions[f.Name] = f.Version + } + sort.Strings(r.Rules) + t := time.Now() + page, err = engine.Refine(ctx, raw, r.frames) + r.JevMs = ms(time.Since(t)) + if err != nil { + r.Err, r.err = err.Error(), err + return r + } + r.JevGenVer = versionOf(jev.Accepted(r.frames), product) + for _, name := range r.TextOnly { + if f := r.frames[name]; f != nil && !f.HasTag(jev.TagRejected) { + r.TextOnlyKept++ + } + } + r.Kind, r.Generic, r.Layers = page.Kind, page.Generic, map[string]string{} + identified := false // an accepted application or device: the page's product is known + for _, f := range r.frames { + for _, t := range f.Tags { + if strings.HasPrefix(t, jev.TagLayer) { + r.Layers[f.Name] = strings.TrimPrefix(t, jev.TagLayer) + } + } + switch { + case f.HasTag(jev.TagRejected): + r.Rejected = append(r.Rejected, f.Name) + case f.HasTag(jev.TagDup): + default: + r.Accepted = append(r.Accepted, f.Name) + switch jev.Layer(r.Layers[f.Name]) { + case jev.LayerApplication, jev.LayerDevice: + identified = true + } + } + if f.HasTag(jev.TagRecall) { + r.Recall = append(r.Recall, f.Name) + } + } + if p := jev.Primary(r.frames); p != nil { + r.Primary, r.Version = p.Name, p.Version + r.JevVersion = p.Version != "" && p.Version != versions[p.Name] + } + // A stock page nobody identified is material for a new rule; a server's + // own default, error or index page is identified by the server. + serverPage := r.Kind == "default_install" || r.Kind == "error_page" || r.Kind == "directory_listing" + r.NewFinger = r.Generic && r.Primary == "" && !identified && !serverPage + sort.Strings(r.Accepted) + sort.Strings(r.Rejected) + return r +} + +func report(b *strings.Builder, rows []*row, c *jev.Client, took time.Duration) { + w := func(format string, a ...interface{}) { fmt.Fprintf(b, format+"\n", a...) } + var pages, errs, rules, accepted, recall, primary, generic, newFinger int + var ruleProducts, textOnly, textOnlyKept, rejected, genPages int + var genRule, genJev [3]int // correct, wrong, missing + var perRule, perJev []int + var ruleMs, jevMs []float64 + kinds := map[string]int{} + type stat struct{ hits, rejected int } + byEngine, byRule := map[string]*stat{}, map[string]*stat{} + add := func(m map[string]*stat, k string, rejected bool) { + if m[k] == nil { + m[k] = &stat{} + } + m[k].hits++ + if rejected { + m[k].rejected++ + } + } + var newFingers []string + for _, r := range rows { + if r == nil || r.Err != "" { + errs++ + continue + } + pages++ + rules += len(r.frames) - len(r.Recall) + accepted += len(r.Accepted) + recall += len(r.Recall) + ruleProducts += r.RuleProducts + textOnly += len(r.TextOnly) + textOnlyKept += r.TextOnlyKept + rejected += len(r.Rejected) + perRule = append(perRule, len(r.frames)-len(r.Recall)) + perJev = append(perJev, len(r.Accepted)) + ruleMs = append(ruleMs, r.RuleMs) + jevMs = append(jevMs, r.JevMs) + kinds[r.Kind]++ + if r.Primary != "" { + primary++ + } + if r.Generator != "" { + genPages++ + want := r.Generator[strings.LastIndex(r.Generator, " ")+1:] + grade(&genRule, r.RuleGenVer, want) + grade(&genJev, r.JevGenVer, want) + } + if r.Generic { + generic++ + } + if r.NewFinger { + newFinger++ + newFingers = append(newFingers, r.ID) + } + for _, f := range r.frames { + rejected := f.HasTag(jev.TagRejected) + for _, e := range r.ruleEngines[f.Name] { + add(byEngine, e, rejected) + add(byRule, e+"/"+f.Name, rejected) + } + } + } + w("# 纯规则 vs 规则 + Jev\n") + w("pages %d, errors %d, took %s\n", pages, errs, took.Round(time.Second)) + w("| 指标 | 纯规则 | 规则 + Jev |\n|---|---|---|") + w("| 每页结果条数 mean / p90 | %.2f / %d | %.2f / %d |", mean(perRule), p90(perRule), mean(perJev), p90(perJev)) + w("| 每页不同产品数（按名字归并） | %.2f | %.2f |", div(ruleProducts, pages), div(accepted, pages)) + w("| 同名重复条目 | %d | 0（标 jev:dup） |", rules-ruleProducts) + w("| 仅正文出现的命中（疑似误报） | %d | %d |", textOnly, textOnlyKept) + w("| 被否决的规则命中 | — | %d / %d（%.1f%%） |", rejected, rules, 100*div(rejected, rules)) + w("| 召回（规则漏掉、Jev 确认） | 0 | %d |", recall) + w("| 主应用 | — | %d 页 |", primary) + w("| 版本号 vs generator：正确 / 错误 / 缺失（%d 页） | %d / %d / %d | %d / %d / %d |", genPages, + genRule[0], genRule[1], genRule[2], genJev[0], genJev[1], genJev[2]) + w("| 页面类型 / 是否通用页面 | — | ✓ / 通用 %d 页 |", generic) + w("| 新指纹候选 | — | %d |", newFinger) + w("| 每页耗时 p50 / p95 | %.1fms / %.1fms | +%.0fms / +%.0fms |", pct(ruleMs, 50), pct(ruleMs, 95), pct(jevMs, 50), pct(jevMs, 95)) + w("| 成本 | 0 | %d 次请求（缓存命中 %d），%d input tokens，≈ $%.3f，折合每万页 $%.2f |", + c.Requests, c.CacheHits, c.InputTokens, float64(c.InputTokens)*0.042/1e6, float64(c.InputTokens)*0.042/1e6/float64(max(pages, 1))*1e4) + w("\n## page kind\n") + for _, k := range sortedKeys(kinds, func(k string) int { return kinds[k] }) { + w("- %s: %d", k, kinds[k]) + } + w("\n## rejections per engine\n\n| engine | hits | rejected | rate |\n|---|---|---|---|") + for _, e := range sortedKeys(byEngine, func(k string) int { return byEngine[k].hits }) { + s := byEngine[e] + w("| %s | %d | %d | %.1f%% |", e, s.hits, s.rejected, 100*div(s.rejected, s.hits)) + } + w("\n## worst rules (>= 5 hits): candidates to fix\n\n| rule | hits | rejected |\n|---|---|---|") + worst := sortedKeys(byRule, func(k string) int { return byRule[k].rejected*1000 + byRule[k].hits }) + for i, k := range worst { + if s := byRule[k]; i < 25 && s.hits >= 5 && s.rejected*2 > s.hits { + w("| %s | %d | %d |", k, s.hits, s.rejected) + } + } + if len(newFingers) > 0 { + w("\n## new fingerprint candidates\n") + for _, id := range newFingers { + w("- %s", id) + } + } +} + +// label is the ground truth for one sample: Keep are real products, Drop are +// false positives the rules report. +type label struct { + PageKind string `json:"page_kind"` + Generic bool `json:"generic"` + Keep []string `json:"keep"` + Drop []string `json:"drop"` + Version *struct { + Product string `json:"product"` + Value string `json:"value"` // "" = the page does not show it + } `json:"version"` +} + +func evaluate(b *strings.Builder, rows []*row, path string) error { + data, err := os.ReadFile(path) + if err != nil { + return err + } + var labels map[string]label + if err := json.Unmarshal(data, &labels); err != nil { + return err + } + w := func(format string, a ...interface{}) { fmt.Fprintf(b, format+"\n", a...) } + var kept, keep, dropped, drop, kind, generic, pages, version, versions int + var misses []string + for _, r := range rows { + l, ok := labels[r.ID] + if !ok || r.Err != "" { + continue + } + pages++ + for _, name := range l.Keep { + keep++ + if find(r.frames, name, false) != nil { + kept++ + } else { + misses = append(misses, r.ID+": lost "+name) + } + } + for _, name := range l.Drop { + drop++ + if find(r.frames, name, true) == nil { + dropped++ + } else { + misses = append(misses, r.ID+": kept "+name) + } + } + if r.Kind == l.PageKind { + kind++ + } + if r.Generic == l.Generic { + generic++ + } + if l.Version != nil { + versions++ + got := "" + if f := find(r.frames, l.Version.Product, false); f != nil { + got = f.Version + } + if got == l.Version.Value { + version++ + } else { + misses = append(misses, fmt.Sprintf("%s: version of %s %q, want %q", r.ID, l.Version.Product, got, l.Version.Value)) + } + } + } + w("\n## against labels (%d pages)\n", pages) + w("- false positives removed %d/%d, real products kept %d/%d", dropped, drop, kept, keep) + w("- page kind %d/%d, generic %d/%d, version %d/%d", kind, pages, generic, pages, version, versions) + if len(misses) > 0 { + w("\n### misses\n") + for _, m := range misses { + w("- %s", m) + } + } + return nil +} + +// find returns the accepted framework matching a label name, preferring one +// with a version. Label names are loose ("tomcat" for "apache-tomcat"), so a +// match is containment either way; strict (for false positive labels such as +// "tomcat_jk_connector") only lets the framework name contain the label. +func find(frames common.Frameworks, name string, strict bool) *common.Framework { + want := jev.NormalizeName(name) + var found *common.Framework + for _, f := range jev.Accepted(frames) { + got := jev.NormalizeName(f.Name) + if got == "" || !(strings.Contains(got, want) || !strict && strings.Contains(want, got)) { + continue + } + if found == nil || found.Version == "" { + found = f + } + } + return found +} + +var reVersion = regexp.MustCompile(`\d+(?:\.\d+)+`) + +// generatorVersion reads the reference answer from a generator meta such as +// "WordPress 6.4.2": the product's first word and the version. +func generatorVersion(gen string) (string, string) { + v := reVersion.FindString(gen) + words := strings.FieldsFunc(strings.ToLower(gen), func(r rune) bool { return !unicode.IsLetter(r) }) + if v == "" || len(words) == 0 { + return "", "" + } + return words[0], v +} + +// versionOf is the version reported for product, preferring an exact name. +func versionOf(frames common.Frameworks, product string) string { + if product == "" { + return "" + } + var got string + for _, f := range frames { + key := jev.NormalizeName(f.Name) + if f.Version == "" || !strings.HasPrefix(key, product) { + continue + } + if key == product || got == "" { + got = f.Version + } + } + return got +} + +func grade(g *[3]int, got, want string) { + switch { + case got == want: + g[0]++ + case got != "": + g[1]++ + default: + g[2]++ + } +} + +func ms(d time.Duration) float64 { return float64(d.Microseconds()) / 1000 } + +func mean(xs []int) float64 { + sum := 0 + for _, x := range xs { + sum += x + } + return div(sum, len(xs)) +} + +func p90(xs []int) int { + if len(xs) == 0 { + return 0 + } + s := append([]int(nil), xs...) + sort.Ints(s) + return s[len(s)*9/10] +} + +func pct(xs []float64, p int) float64 { + if len(xs) == 0 { + return 0 + } + s := append([]float64(nil), xs...) + sort.Float64s(s) + return s[len(s)*p/100] +} + +func div(a, b int) float64 { + if b == 0 { + return 0 + } + return float64(a) / float64(b) +} + +func sortedKeys[V any](m map[string]V, weight func(string) int) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Slice(keys, func(a, b int) bool { + if wa, wb := weight(keys[a]), weight(keys[b]); wa != wb { + return wa > wb + } + return keys[a] < keys[b] + }) + return keys +} + +// fileCache is an exact jev.Cache on disk, one file per request. +type fileCache string + +func (d fileCache) path(key string, sig uint64) string { + return filepath.Join(string(d), fmt.Sprintf("%s_%016x.json", key, sig)) +} + +func (d fileCache) Get(key string, sig uint64) ([]byte, bool) { + data, err := os.ReadFile(d.path(key, sig)) + return data, err == nil +} + +func (d fileCache) Put(key string, sig uint64, value []byte) { + path := d.path(key, sig) + if os.WriteFile(path+".tmp", value, 0o644) == nil { + os.Rename(path+".tmp", path) + } +} + +// simCheck measures the similarity cache without trusting it: every page is +// still answered exactly, and whenever an earlier similar page would have +// been served instead, the two answers are compared decision by decision. +type simCheck struct { + fileCache + mu sync.Mutex + seen map[string][]simEntry // key -> earlier answers + pending map[string][]byte // key+sig -> answer a similarity cache would have served + requests, hits, agree int + diffs map[string]int // question key prefix -> disagreements +} + +type simEntry struct { + sig uint64 + value []byte +} + +func newSimCheck(dir string) *simCheck { + return &simCheck{fileCache: fileCache(dir), seen: map[string][]simEntry{}, pending: map[string][]byte{}, diffs: map[string]int{}} +} + +func (c *simCheck) Get(key string, sig uint64) ([]byte, bool) { + value, ok := c.fileCache.Get(key, sig) + c.mu.Lock() + defer c.mu.Unlock() + var near []byte + for _, e := range c.seen[key] { + if e.sig != sig && jev.Similar(e.sig, sig) { + near = e.value + break + } + } + if ok { + c.observe(key, sig, near, value) + } else if near != nil { + c.pending[fmt.Sprintf("%s_%x", key, sig)] = near + } + return value, ok +} + +func (c *simCheck) Put(key string, sig uint64, value []byte) { + c.fileCache.Put(key, sig, value) + c.mu.Lock() + defer c.mu.Unlock() + id := fmt.Sprintf("%s_%x", key, sig) + c.observe(key, sig, c.pending[id], value) + delete(c.pending, id) +} + +func (c *simCheck) observe(key string, sig uint64, near, value []byte) { + c.requests++ + c.seen[key] = append(c.seen[key], simEntry{sig, value}) + if near == nil { + return + } + c.hits++ + var a, b jev.Response + json.Unmarshal(near, &a) + json.Unmarshal(value, &b) + same := true + for k, x := range b.Answers { + y := a.Answers[k] + if x.Choice != y.Choice || (x.Noul >= jev.Threshold) != (y.Noul >= jev.Threshold) { + same = false + c.diffs[strings.TrimRight(k, "0123456789")]++ + } + } + if same { + c.agree++ + } +} + +// limited rate-limits Jev requests; cache hits never reach it. +type limited struct{ tick <-chan time.Time } + +func (l *limited) RoundTrip(req *http.Request) (*http.Response, error) { + select { + case <-l.tick: + case <-req.Context().Done(): + return nil, req.Context().Err() + } + return http.DefaultTransport.RoundTrip(req) +} diff --git a/cmd/jevbench/testdata/labels.json b/cmd/jevbench/testdata/labels.json new file mode 100644 index 0000000..940cdde --- /dev/null +++ b/cmd/jevbench/testdata/labels.json @@ -0,0 +1,78 @@ +{ + "blog_mentions_jenkins.http": {"page_kind": "normal_content", "generic": false, "keep": ["cloudflare", "hugo"], "drop": ["jenkins", "thinkphp"], + "note": "文章正文提到 Jenkins/ThinkPHP"}, + "confluence_login.http": {"page_kind": "login", "generic": true, "keep": ["confluence"], "drop": ["struts2", "telerik_report_server"], + "version": {"product": "confluence", "value": "7.19.8"}}, + "corporate_cn_portal.http": {"page_kind": "normal_content", "generic": false, "keep": ["nginx", "jquery"], + "note": "新闻里提到用友，规则没误报，是对照"}, + "corporate_site.http": {"page_kind": "normal_content", "generic": false, "keep": ["nginx"]}, + "custom_employee_portal.http": {"page_kind": "login", "generic": false, "keep": ["nginx"], + "note": "单位自研登录页，不是通用产品"}, + "dir_listing.http": {"page_kind": "directory_listing", "generic": true, "keep": ["apache"], + "version": {"product": "apache", "value": "2.4.41"}}, + "discuz_forum.http": {"page_kind": "normal_content", "generic": false, "keep": ["discuz"], + "version": {"product": "discuz", "value": "3.4"}, "note": "基于 CMS 的单位内容页"}, + "docs_tomcat_howto.http": {"page_kind": "normal_content", "generic": false, "keep": ["github pages", "jekyll"], "drop": ["tomcat", "apache-web-server"], + "version": {"product": "jekyll", "value": "3.9.3"}, "note": "正文里的 Tomcat/nginx 版本号是干扰"}, + "gitlab_signin.http": {"page_kind": "login", "generic": true, "keep": ["gitlab"], "drop": ["webp_server_go"], + "version": {"product": "gitlab", "value": ""}}, + "grafana_login.http": {"page_kind": "login", "generic": true, "keep": ["grafana"], + "version": {"product": "grafana", "value": "10.2.3"}}, + "harbor_portal.http": {"page_kind": "app_console", "generic": true, "keep": ["harbor"], + "version": {"product": "harbor", "value": ""}}, + "iis_default.http": {"page_kind": "default_install", "generic": true, "keep": ["iis"], + "version": {"product": "iis", "value": "10.0"}}, + "jenkins_login.http": {"page_kind": "login", "generic": true, "keep": ["jenkins"], "drop": ["log4j2"], + "version": {"product": "jenkins", "value": "2.401.3"}}, + "job_posting_cn.http": {"page_kind": "normal_content", "generic": false, "keep": ["tengine"], "drop": ["elasticsearch", "log4j2", "gitlab", "jenkins", "thinkphp"], + "note": "招聘要求里列的技术栈"}, + "json_api.http": {"page_kind": "api_response", "generic": false, "keep": ["uvicorn"], "drop": ["postgresql"]}, + "kibana_home.http": {"page_kind": "app_console", "generic": true, "keep": ["kibana"], "drop": ["log4j2"], + "version": {"product": "kibana", "value": "8.11.1"}}, + "minio_console.http": {"page_kind": "app_console", "generic": true, "keep": ["minio"], + "version": {"product": "minio", "value": ""}}, + "nacos_console.http": {"page_kind": "app_console", "generic": true, "keep": ["nacos"], + "version": {"product": "nacos", "value": "2.2.3"}}, + "news_fortios_patch.http": {"page_kind": "normal_content", "generic": false, "keep": ["cloudflare", "ghost"], "drop": ["fortianalyzer", "vnpt-technology-igate", "飞塔防火墙", "fortinet sslvpn"], + "version": {"product": "ghost", "value": "5.71"}, "note": "新闻里的 FortiOS 版本号是干扰"}, + "nginx_404.http": {"page_kind": "error_page", "generic": true, "keep": ["nginx"], + "version": {"product": "nginx", "value": "1.22.1"}}, + "nginx_default.http": {"page_kind": "default_install", "generic": true, "keep": ["nginx"], + "version": {"product": "nginx", "value": "1.18.0"}}, + "pgadmin_login.http": {"page_kind": "login", "generic": true, "keep": ["pgadmin", "gunicorn"], "drop": ["postgresql"], + "version": {"product": "pgadmin", "value": ""}, "note": "ver=80000 是内部编号，不是版本号"}, + "phpmyadmin_login.http": {"page_kind": "login", "generic": true, "keep": ["phpmyadmin", "php"], + "version": {"product": "phpmyadmin", "value": "5.2.1"}, "note": "PHP/8.1.12、Apache/2.4.54 是干扰"}, + "portainer_login.http": {"page_kind": "app_console", "generic": true, "keep": ["portainer"], + "version": {"product": "portainer", "value": ""}}, + "rabbitmq_mgmt.http": {"page_kind": "app_console", "generic": true, "keep": ["rabbitmq"], "drop": ["log4j2", "ipeakcms"], + "version": {"product": "rabbitmq", "value": ""}, "note": "js 文件名里的是第三方库版本"}, + "rebranded_grafana.http": {"page_kind": "login", "generic": true, "miss": ["grafana"], + "version": {"product": "grafana", "value": "9.5.2"}, "note": "改过品牌，规则漏报"}, + "rebranded_jenkins.http": {"page_kind": "login", "generic": true, "miss": ["jenkins"], + "version": {"product": "jenkins", "value": ""}, "note": "改过品牌，规则漏报"}, + "rebranded_zabbix.http": {"page_kind": "login", "generic": true, "keep": ["nginx"], "miss": ["zabbix"], + "version": {"product": "zabbix", "value": ""}, "note": "改过品牌的中文页面，规则漏报"}, + "seeyon_login.http": {"page_kind": "login", "generic": true, "keep": ["致远oa"], "drop": ["log4j2"], + "version": {"product": "致远oa", "value": "8.1"}}, + "shop_page.http": {"page_kind": "normal_content", "generic": false, "keep": ["shopify", "cloudflare"], + "note": "SaaS 上的单位店铺"}, + "springboot_error.http": {"page_kind": "error_page", "generic": true, "keep": ["springboot"], "drop": ["log4j2"]}, + "status_page_mentions.http": {"page_kind": "normal_content", "generic": false, "keep": ["nginx"], "drop": ["gitlab", "jenkins", "grafana_labs-公司产品"]}, + "synology_dsm.http": {"page_kind": "login", "generic": true, "keep": ["synology diskstation"], "drop": ["windows"], + "version": {"product": "synology diskstation", "value": ""}, "note": "v= 是时间戳"}, + "tomcat_default.http": {"page_kind": "default_install", "generic": true, "keep": ["tomcat"], "drop": ["tomcat_jk_connector"], + "version": {"product": "tomcat", "value": "9.0.65"}}, + "tplink_router.http": {"page_kind": "login", "generic": true, "keep": ["tp-link无线路由器"], "drop": ["apache http server", "archer-ax21"], + "version": {"product": "tp-link无线路由器", "value": "0.9.1"}}, + "unknown_admin_login.http": {"page_kind": "login", "generic": true, "keep": ["openresty"], "drop": ["zkteco"], + "note": "长尾产品，指纹库里没有；技术支持写着厂商"}, + "weaver_ecology_login.http": {"page_kind": "login", "generic": true, "keep": ["泛微 e-cology"], + "version": {"product": "泛微 e-cology", "value": ""}}, + "wordpress_home.http": {"page_kind": "normal_content", "generic": false, "keep": ["wordpress"], "drop": ["windows", "microsoft-windows远程连接"], + "version": {"product": "wordpress", "value": "6.4.2"}}, + "wordpress_login.http": {"page_kind": "login", "generic": true, "keep": ["wordpress"], "drop": ["windows", "microsoft-windows远程连接", "telerik_report_server"], + "version": {"product": "wordpress", "value": "6.4.2"}, "note": "jquery 3.7.1 是干扰"}, + "zabbix_login.http": {"page_kind": "login", "generic": true, "keep": ["zabbix"], + "version": {"product": "zabbix", "value": ""}, "note": "文档链接里的 6.0 和年份是干扰"} +} diff --git a/cmd/jevbench/testdata/samples/blog_mentions_jenkins.http b/cmd/jevbench/testdata/samples/blog_mentions_jenkins.http new file mode 100644 index 0000000..b7225b6 --- /dev/null +++ b/cmd/jevbench/testdata/samples/blog_mentions_jenkins.http @@ -0,0 +1,5 @@ +HTTP/1.1 200 OK +Server: cloudflare +Content-Type: text/html; charset=utf-8 + +Migrating our CI from Jenkins to GitHub Actions | DevNotes

Migrating our CI from Jenkins to GitHub Actions

For five years our team ran Jenkins with a pile of Groovy pipelines. We also had a legacy ThinkPHP admin panel that depended on it. This post explains why we moved to GitHub Actions and what we learned.

Tags: ci, jenkins, devops

diff --git a/cmd/jevbench/testdata/samples/confluence_login.http b/cmd/jevbench/testdata/samples/confluence_login.http new file mode 100644 index 0000000..2ed01a2 --- /dev/null +++ b/cmd/jevbench/testdata/samples/confluence_login.http @@ -0,0 +1,7 @@ +HTTP/1.1 200 OK +X-Confluence-Request-Time: 1727170000123 +X-ASEN: SEN-L1234567 +Set-Cookie: JSESSIONID=8F2C; Path=/; HttpOnly +Content-Type: text/html;charset=UTF-8 + +Log In - Confluence

Log in to Confluence

Powered by Atlassian Confluence 7.19.8

diff --git a/cmd/jevbench/testdata/samples/corporate_cn_portal.http b/cmd/jevbench/testdata/samples/corporate_cn_portal.http new file mode 100644 index 0000000..bf28531 --- /dev/null +++ b/cmd/jevbench/testdata/samples/corporate_cn_portal.http @@ -0,0 +1,5 @@ +HTTP/1.1 200 OK +Server: nginx +Content-Type: text/html; charset=utf-8 + +江南精密机械集团有限公司 - 官方网站
首页 关于我们 产品中心 新闻动态 联系我们

集团与用友网络签署数字化转型战略合作协议

2026年9月，集团与用友网络科技股份有限公司签署合作协议，将在 ERP 与财务共享领域展开合作。

版权所有 © 2026 江南精密机械集团有限公司 苏ICP备12345678号
diff --git a/cmd/jevbench/testdata/samples/corporate_site.http b/cmd/jevbench/testdata/samples/corporate_site.http new file mode 100644 index 0000000..edf9200 --- /dev/null +++ b/cmd/jevbench/testdata/samples/corporate_site.http @@ -0,0 +1,5 @@ +HTTP/1.1 200 OK +Server: nginx +Content-Type: text/html; charset=utf-8 + +Northwind Logistics - Freight and Warehousing

Northwind Logistics

We move freight across 40 countries. Contact our sales team for a quote.

About Careers diff --git a/cmd/jevbench/testdata/samples/custom_employee_portal.http b/cmd/jevbench/testdata/samples/custom_employee_portal.http new file mode 100644 index 0000000..04bc7d7 --- /dev/null +++ b/cmd/jevbench/testdata/samples/custom_employee_portal.http @@ -0,0 +1,6 @@ +HTTP/1.1 200 OK +Server: nginx +Set-Cookie: ACME_SID=5d1e; path=/; HttpOnly +Content-Type: text/html; charset=utf-8 + +ACME Corp · Employee Portal
ACME Corp

Sign in to the ACME Employee Portal

Trouble signing in? Contact the ACME IT Service Desk at ext. 4400.

diff --git a/cmd/jevbench/testdata/samples/dir_listing.http b/cmd/jevbench/testdata/samples/dir_listing.http new file mode 100644 index 0000000..fc834e1 --- /dev/null +++ b/cmd/jevbench/testdata/samples/dir_listing.http @@ -0,0 +1,5 @@ +HTTP/1.1 200 OK +Server: Apache/2.4.41 (Ubuntu) +Content-Type: text/html;charset=UTF-8 + +Index of /backup

Index of /backup

NameLast modifiedSize
Parent Directory
db_2026-09-01.sql.gz2026-09-01 03:0012M
Apache/2.4.41 (Ubuntu) Server at 10.0.0.5 Port 80
diff --git a/cmd/jevbench/testdata/samples/discuz_forum.http b/cmd/jevbench/testdata/samples/discuz_forum.http new file mode 100644 index 0000000..51fb0f7 --- /dev/null +++ b/cmd/jevbench/testdata/samples/discuz_forum.http @@ -0,0 +1,7 @@ +HTTP/1.1 200 OK +Server: nginx +X-Powered-By: PHP/7.4.33 +Set-Cookie: q7Xz_2132_saltkey=Ab12Cd34; path=/; httponly +Content-Type: text/html; charset=utf-8 + +钓鱼之家论坛 - Powered by Discuz!

钓鱼之家论坛

今日: 128 | 昨日: 342 | 帖子: 1203452 | 会员: 88321

路亚交流区 · 台钓技术 · 渔具二手市场

Powered by Discuz! X3.4

© 2001-2026 Discuz! Team.

diff --git a/cmd/jevbench/testdata/samples/docs_tomcat_howto.http b/cmd/jevbench/testdata/samples/docs_tomcat_howto.http new file mode 100644 index 0000000..d0b39cc --- /dev/null +++ b/cmd/jevbench/testdata/samples/docs_tomcat_howto.http @@ -0,0 +1,6 @@ +HTTP/1.1 200 OK +Server: GitHub.com +Content-Type: text/html; charset=utf-8 +X-GitHub-Request-Id: 9A2C:3B1F + +How to tune the Tomcat 10.1 NIO connector | Sam's Ops Notes

How to tune the Tomcat 10.1 NIO connector

When we upgraded from Tomcat 9.0.80 to Tomcat 10.1.13, throughput dropped. Here is the server.xml we ended up with:

<Connector port="8080" protocol="org.apache.coyote.http11.Http11NioProtocol" maxThreads="400" />

We run it behind nginx 1.24 and Spring Boot 3.1 apps.

Hosted on GitHub Pages
diff --git a/cmd/jevbench/testdata/samples/gitlab_signin.http b/cmd/jevbench/testdata/samples/gitlab_signin.http new file mode 100644 index 0000000..375f1da --- /dev/null +++ b/cmd/jevbench/testdata/samples/gitlab_signin.http @@ -0,0 +1,8 @@ +HTTP/1.1 200 OK +Server: nginx +X-Gitlab-Meta: {"correlation_id":"01HB2X","version":"1"} +X-Request-Id: 01HB2X +Set-Cookie: _gitlab_session=3f1c; path=/; HttpOnly +Content-Type: text/html; charset=utf-8 + +Sign in · GitLab

GitLab Community Edition

Don't have an account yet? Register now

diff --git a/cmd/jevbench/testdata/samples/grafana_login.http b/cmd/jevbench/testdata/samples/grafana_login.http new file mode 100644 index 0000000..3c32a1e --- /dev/null +++ b/cmd/jevbench/testdata/samples/grafana_login.http @@ -0,0 +1,6 @@ +HTTP/1.1 200 OK +Cache-Control: no-store +Content-Type: text/html; charset=UTF-8 +X-Content-Type-Options: nosniff + +Grafana
diff --git a/cmd/jevbench/testdata/samples/harbor_portal.http b/cmd/jevbench/testdata/samples/harbor_portal.http new file mode 100644 index 0000000..0c26570 --- /dev/null +++ b/cmd/jevbench/testdata/samples/harbor_portal.http @@ -0,0 +1,6 @@ +HTTP/1.1 200 OK +Server: nginx +Set-Cookie: sid=8a7b6c5d; Path=/; HttpOnly +Content-Type: text/html + +HarborLoading... diff --git a/cmd/jevbench/testdata/samples/iis_default.http b/cmd/jevbench/testdata/samples/iis_default.http new file mode 100644 index 0000000..21d4d64 --- /dev/null +++ b/cmd/jevbench/testdata/samples/iis_default.http @@ -0,0 +1,7 @@ +HTTP/1.1 200 OK +Server: Microsoft-IIS/10.0 +X-Powered-By: ASP.NET +Accept-Ranges: bytes +Content-Type: text/html + +IIS Windows Server
IIS
diff --git a/cmd/jevbench/testdata/samples/jenkins_login.http b/cmd/jevbench/testdata/samples/jenkins_login.http new file mode 100644 index 0000000..1f2ddee --- /dev/null +++ b/cmd/jevbench/testdata/samples/jenkins_login.http @@ -0,0 +1,6 @@ +HTTP/1.1 200 OK +X-Jenkins: 2.401.3 +Set-Cookie: JSESSIONID.1a2b3c=node01; Path=/; HttpOnly +Content-Type: text/html;charset=utf-8 + +Sign in [Jenkins]

Welcome to Jenkins!

diff --git a/cmd/jevbench/testdata/samples/job_posting_cn.http b/cmd/jevbench/testdata/samples/job_posting_cn.http new file mode 100644 index 0000000..b520c89 --- /dev/null +++ b/cmd/jevbench/testdata/samples/job_posting_cn.http @@ -0,0 +1,5 @@ +HTTP/1.1 200 OK +Server: Tengine +Content-Type: text/html; charset=utf-8 + +Java 后端开发工程师 - 华东某物流科技有限公司招聘

Java 后端开发工程师

薪资：20-35K · 上海 · 3-5年

岗位要求

  • 熟悉 Spring Boot、Spring Cloud、MyBatis 等框架；
  • 熟悉 Redis、Kafka、Elasticsearch、Nacos 等中间件；
  • 了解 ThinkPHP 或 Laravel 者优先，维护过泛微、致远 OA 二次开发者优先；
  • 熟悉 Jenkins、GitLab CI 等持续集成工具。

公司地址：上海市浦东新区张江路 88 号

diff --git a/cmd/jevbench/testdata/samples/json_api.http b/cmd/jevbench/testdata/samples/json_api.http new file mode 100644 index 0000000..cbaaf14 --- /dev/null +++ b/cmd/jevbench/testdata/samples/json_api.http @@ -0,0 +1,5 @@ +HTTP/1.1 200 OK +Server: uvicorn +Content-Type: application/json + +{"service":"inventory-api","status":"ok","uptime_seconds":882133,"build":"2026.09.18-3","dependencies":{"postgres":"up","redis":"up"}} diff --git a/cmd/jevbench/testdata/samples/kibana_home.http b/cmd/jevbench/testdata/samples/kibana_home.http new file mode 100644 index 0000000..e2a6684 --- /dev/null +++ b/cmd/jevbench/testdata/samples/kibana_home.http @@ -0,0 +1,7 @@ +HTTP/1.1 200 OK +kbn-name: kibana-prod-01 +kbn-license-sig: 9c3b1f +content-security-policy: script-src 'report-sample' 'self'; worker-src 'report-sample' 'self' blob: +Content-Type: text/html; charset=utf-8 + +Elastic
Loading Elastic
diff --git a/cmd/jevbench/testdata/samples/minio_console.http b/cmd/jevbench/testdata/samples/minio_console.http new file mode 100644 index 0000000..e3a4fb1 --- /dev/null +++ b/cmd/jevbench/testdata/samples/minio_console.http @@ -0,0 +1,6 @@ +HTTP/1.1 200 OK +Server: MinIO Console +Content-Security-Policy: default-src 'self' 'unsafe-eval' 'unsafe-inline'; +Content-Type: text/html + +MinIO Console
diff --git a/cmd/jevbench/testdata/samples/nacos_console.http b/cmd/jevbench/testdata/samples/nacos_console.http new file mode 100644 index 0000000..1d3ec9b --- /dev/null +++ b/cmd/jevbench/testdata/samples/nacos_console.http @@ -0,0 +1,5 @@ +HTTP/1.1 200 OK +Content-Type: text/html;charset=UTF-8 +Vary: Origin + +Nacos
diff --git a/cmd/jevbench/testdata/samples/news_fortios_patch.http b/cmd/jevbench/testdata/samples/news_fortios_patch.http new file mode 100644 index 0000000..73111c6 --- /dev/null +++ b/cmd/jevbench/testdata/samples/news_fortios_patch.http @@ -0,0 +1,6 @@ +HTTP/1.1 200 OK +Server: cloudflare +CF-Cache-Status: HIT +Content-Type: text/html; charset=utf-8 + +Fortinet ships FortiOS 7.2.5 update for SSL VPN bug | Network Weekly

Fortinet ships FortiOS 7.2.5 update for SSL VPN bug

By Dana Reyes · 12 Sep 2026

Fortinet has released FortiOS 7.2.5 and 7.0.12, fixing an issue in the FortiGate SSL VPN portal. Administrators running FortiGate appliances should upgrade through the FortiGuard update service. Older 6.4.x branches receive the fix in 6.4.14.

The vendor said FortiManager and FortiAnalyzer are not affected.

© 2026 Network Weekly. Published with Ghost.
diff --git a/cmd/jevbench/testdata/samples/nginx_404.http b/cmd/jevbench/testdata/samples/nginx_404.http new file mode 100644 index 0000000..82f5fff --- /dev/null +++ b/cmd/jevbench/testdata/samples/nginx_404.http @@ -0,0 +1,11 @@ +HTTP/1.1 404 Not Found +Server: nginx/1.22.1 +Content-Type: text/html + + +404 Not Found + +

404 Not Found

+
nginx/1.22.1
+ + diff --git a/cmd/jevbench/testdata/samples/nginx_default.http b/cmd/jevbench/testdata/samples/nginx_default.http new file mode 100644 index 0000000..50f7e4d --- /dev/null +++ b/cmd/jevbench/testdata/samples/nginx_default.http @@ -0,0 +1,5 @@ +HTTP/1.1 200 OK +Server: nginx/1.18.0 (Ubuntu) +Content-Type: text/html + +Welcome to nginx!

Welcome to nginx!

If you see this page, the nginx web server is successfully installed and working. Further configuration is required.

Thank you for using nginx.

diff --git a/cmd/jevbench/testdata/samples/pgadmin_login.http b/cmd/jevbench/testdata/samples/pgadmin_login.http new file mode 100644 index 0000000..b84c602 --- /dev/null +++ b/cmd/jevbench/testdata/samples/pgadmin_login.http @@ -0,0 +1,6 @@ +HTTP/1.1 200 OK +Server: gunicorn +Set-Cookie: pga4_session=0f1e2d3c!abc; HttpOnly; Path=/ +Content-Type: text/html; charset=utf-8 + +pgAdmin 4
pgAdmin 4 - PostgreSQL Tools
diff --git a/cmd/jevbench/testdata/samples/phpmyadmin_login.http b/cmd/jevbench/testdata/samples/phpmyadmin_login.http new file mode 100644 index 0000000..0889924 --- /dev/null +++ b/cmd/jevbench/testdata/samples/phpmyadmin_login.http @@ -0,0 +1,8 @@ +HTTP/1.1 200 OK +Server: Apache/2.4.54 (Debian) +X-Powered-By: PHP/8.1.12 +Set-Cookie: phpMyAdmin=4b7c1d; path=/; HttpOnly +Set-Cookie: pma_lang=en; path=/ +Content-Type: text/html; charset=utf-8 + +phpMyAdmin

Welcome to phpMyAdmin

diff --git a/cmd/jevbench/testdata/samples/portainer_login.http b/cmd/jevbench/testdata/samples/portainer_login.http new file mode 100644 index 0000000..508de2b --- /dev/null +++ b/cmd/jevbench/testdata/samples/portainer_login.http @@ -0,0 +1,6 @@ +HTTP/1.1 200 OK +X-Xss-Protection: 1; mode=block +X-Content-Type-Options: nosniff +Content-Type: text/html; charset=utf-8 + +Portainer
Loading Portainer...
diff --git a/cmd/jevbench/testdata/samples/rabbitmq_mgmt.http b/cmd/jevbench/testdata/samples/rabbitmq_mgmt.http new file mode 100644 index 0000000..d16890c --- /dev/null +++ b/cmd/jevbench/testdata/samples/rabbitmq_mgmt.http @@ -0,0 +1,5 @@ +HTTP/1.1 200 OK +Server: Cowboy +Content-Type: text/html + +RabbitMQ Management
diff --git a/cmd/jevbench/testdata/samples/rebranded_grafana.http b/cmd/jevbench/testdata/samples/rebranded_grafana.http new file mode 100644 index 0000000..56b412e --- /dev/null +++ b/cmd/jevbench/testdata/samples/rebranded_grafana.http @@ -0,0 +1,5 @@ +HTTP/1.1 200 OK +Cache-Control: no-store +Content-Type: text/html; charset=UTF-8 + +Acme Ops Monitoring
diff --git a/cmd/jevbench/testdata/samples/rebranded_jenkins.http b/cmd/jevbench/testdata/samples/rebranded_jenkins.http new file mode 100644 index 0000000..d58a694 --- /dev/null +++ b/cmd/jevbench/testdata/samples/rebranded_jenkins.http @@ -0,0 +1,6 @@ +HTTP/1.1 200 OK +Set-Cookie: JSESSIONID.8f3c2a1b=node0abc; Path=/; HttpOnly +X-Content-Type-Options: nosniff +Content-Type: text/html;charset=utf-8 + +Sign in [ACME Build]

Welcome to ACME Build!

diff --git a/cmd/jevbench/testdata/samples/rebranded_zabbix.http b/cmd/jevbench/testdata/samples/rebranded_zabbix.http new file mode 100644 index 0000000..c994666 --- /dev/null +++ b/cmd/jevbench/testdata/samples/rebranded_zabbix.http @@ -0,0 +1,6 @@ +HTTP/1.1 200 OK +Server: nginx +Set-Cookie: zbx_session=eyJzZXNzaW9; path=/; HttpOnly +Content-Type: text/html; charset=UTF-8 + +集团IT运维监控中心
集团信息中心 运维保障
diff --git a/cmd/jevbench/testdata/samples/seeyon_login.http b/cmd/jevbench/testdata/samples/seeyon_login.http new file mode 100644 index 0000000..ce93bba --- /dev/null +++ b/cmd/jevbench/testdata/samples/seeyon_login.http @@ -0,0 +1,6 @@ +HTTP/1.1 200 OK +Server: SY8044 +Set-Cookie: JSESSIONID=A1B2C3; Path=/seeyon; HttpOnly +Content-Type: text/html;charset=UTF-8 + +协同管理软件 V8.1SP2

欢迎使用协同管理软件

北京致远互联软件股份有限公司 版权所有 Copyright © 2002-2023
diff --git a/cmd/jevbench/testdata/samples/shop_page.http b/cmd/jevbench/testdata/samples/shop_page.http new file mode 100644 index 0000000..2212bca --- /dev/null +++ b/cmd/jevbench/testdata/samples/shop_page.http @@ -0,0 +1,8 @@ +HTTP/1.1 200 OK +Server: cloudflare +Set-Cookie: _shopify_y=8c1e; path=/; expires=Wed, 24 Sep 2027 11:00:00 GMT +X-ShopId: 60123456789 +X-Shardid: 214 +Content-Type: text/html; charset=utf-8 + +Handmade Ceramic Mugs – Clay & Kiln Studio

Handmade Ceramic Mugs

Each mug is thrown by hand in our Portland studio. $38.00 USD

diff --git a/cmd/jevbench/testdata/samples/springboot_error.http b/cmd/jevbench/testdata/samples/springboot_error.http new file mode 100644 index 0000000..a067d6a --- /dev/null +++ b/cmd/jevbench/testdata/samples/springboot_error.http @@ -0,0 +1,4 @@ +HTTP/1.1 404 Not Found +Content-Type: text/html;charset=UTF-8 + +

Whitelabel Error Page

This application has no explicit mapping for /error, so you are seeing this as a fallback.

Tue Sep 24 10:00:00 CST 2026
There was an unexpected error (type=Not Found, status=404).
diff --git a/cmd/jevbench/testdata/samples/status_page_mentions.http b/cmd/jevbench/testdata/samples/status_page_mentions.http new file mode 100644 index 0000000..cd6be3f --- /dev/null +++ b/cmd/jevbench/testdata/samples/status_page_mentions.http @@ -0,0 +1,5 @@ +HTTP/1.1 200 OK +Server: nginx +Content-Type: text/html; charset=utf-8 + +Northwind Systems Status

Northwind Systems Status

Customer PortalOperational
Order APIOperational
Internal Grafana dashboardsDegraded performance
Jenkins build farmScheduled maintenance 02:00-04:00 UTC
GitLab mirrorOperational

Last updated 2026-09-24 09:12 UTC. Subscribe to updates via RSS.

diff --git a/cmd/jevbench/testdata/samples/synology_dsm.http b/cmd/jevbench/testdata/samples/synology_dsm.http new file mode 100644 index 0000000..1b559ac --- /dev/null +++ b/cmd/jevbench/testdata/samples/synology_dsm.http @@ -0,0 +1,5 @@ +HTTP/1.1 200 OK +Server: nginx +Content-Type: text/html; charset="UTF-8" + +Synology DiskStation - NAS-HOME
diff --git a/cmd/jevbench/testdata/samples/tomcat_default.http b/cmd/jevbench/testdata/samples/tomcat_default.http new file mode 100644 index 0000000..b996e92 --- /dev/null +++ b/cmd/jevbench/testdata/samples/tomcat_default.http @@ -0,0 +1,4 @@ +HTTP/1.1 200 OK +Content-Type: text/html;charset=UTF-8 + +Apache Tomcat/9.0.65
Home

If you're seeing this, you've successfully installed Tomcat. Congratulations!

Server Status Manager App Host Manager diff --git a/cmd/jevbench/testdata/samples/tplink_router.http b/cmd/jevbench/testdata/samples/tplink_router.http new file mode 100644 index 0000000..8aa7929 --- /dev/null +++ b/cmd/jevbench/testdata/samples/tplink_router.http @@ -0,0 +1,5 @@ +HTTP/1.1 200 OK +Server: TP-LINK HTTPD/1.0 +Content-Type: text/html + +TL-WR841N

Wireless N Router WR841N

Hardware Version: WR841N v14 00000014

Firmware Version: 0.9.1 4.19 v0001.0 Build 230516 Rel.61208n

Copyright © 2023 TP-Link Corporation Limited. All rights reserved.

diff --git a/cmd/jevbench/testdata/samples/unknown_admin_login.http b/cmd/jevbench/testdata/samples/unknown_admin_login.http new file mode 100644 index 0000000..4ce0bd3 --- /dev/null +++ b/cmd/jevbench/testdata/samples/unknown_admin_login.http @@ -0,0 +1,5 @@ +HTTP/1.1 200 OK +Server: openresty +Content-Type: text/html; charset=utf-8 + +智慧园区综合管理平台

智慧园区综合管理平台 V3.2.1

技术支持：某某科技有限公司

diff --git a/cmd/jevbench/testdata/samples/weaver_ecology_login.http b/cmd/jevbench/testdata/samples/weaver_ecology_login.http new file mode 100644 index 0000000..bfe9cca --- /dev/null +++ b/cmd/jevbench/testdata/samples/weaver_ecology_login.http @@ -0,0 +1,5 @@ +HTTP/1.1 200 OK +Set-Cookie: ecology_JSessionid=aaaXYZ; path=/ +Content-Type: text/html; charset=UTF-8 + +泛微协同管理应用平台 e-cology

欢迎登录 OA 系统

版权所有 泛微网络科技股份有限公司
diff --git a/cmd/jevbench/testdata/samples/wordpress_home.http b/cmd/jevbench/testdata/samples/wordpress_home.http new file mode 100644 index 0000000..87bd5f4 --- /dev/null +++ b/cmd/jevbench/testdata/samples/wordpress_home.http @@ -0,0 +1,6 @@ +HTTP/1.1 200 OK +Server: Apache/2.4.57 +Link: ; rel="https://api.w.org/" +Content-Type: text/html; charset=UTF-8 + +Acme Bakery – Fresh bread daily

Acme Bakery

Fresh sourdough every morning. Visit our shop on Main Street.

diff --git a/cmd/jevbench/testdata/samples/wordpress_login.http b/cmd/jevbench/testdata/samples/wordpress_login.http new file mode 100644 index 0000000..557295d --- /dev/null +++ b/cmd/jevbench/testdata/samples/wordpress_login.http @@ -0,0 +1,7 @@ +HTTP/1.1 200 OK +Server: nginx/1.24.0 +X-Frame-Options: SAMEORIGIN +Set-Cookie: wordpress_test_cookie=WP%20Cookie%20check; path=/; secure +Content-Type: text/html; charset=UTF-8 + +Log In ‹ Riverside Dental Clinic — WordPress

Powered by WordPress

Lost your password?

diff --git a/cmd/jevbench/testdata/samples/zabbix_login.http b/cmd/jevbench/testdata/samples/zabbix_login.http new file mode 100644 index 0000000..8d67b59 --- /dev/null +++ b/cmd/jevbench/testdata/samples/zabbix_login.http @@ -0,0 +1,6 @@ +HTTP/1.1 200 OK +Server: nginx +Set-Cookie: zbx_session=eyJzZXNzaW9; path=/; HttpOnly +Content-Type: text/html; charset=UTF-8 + +Zabbix
Help • Support
© 2001–2023, Zabbix SIA
diff --git a/engine.go b/engine.go index 2034bde..9b1dea3 100644 --- a/engine.go +++ b/engine.go @@ -11,6 +11,7 @@ import ( "github.com/chainreactors/fingers/fingerprinthub" "github.com/chainreactors/fingers/fingers" "github.com/chainreactors/fingers/goby" + "github.com/chainreactors/fingers/jev" gonmap "github.com/chainreactors/fingers/nmap" "github.com/chainreactors/fingers/resources" wappalyzer "github.com/chainreactors/fingers/wappalyzer" @@ -86,6 +87,10 @@ type Engine struct { *alias.Aliases Enabled map[string]bool Capabilities map[string]common.EngineCapability // 新增：记录各引擎能力 + + // 可选的 Jev 判定层, 见 AttachJev + jevClient *jev.Client + jevRecall *jev.Retriever } func (engine *Engine) String() string { @@ -397,7 +402,9 @@ func (engine *Engine) MergeFrameworks(origin, other common.Frameworks) common.Fr if aliasFrame != nil { if ok { frame.Name = aliasFrame.Name - frame.UpdateAttributes(aliasFrame.ToWFN()) + // 复制一份: alias 的 Attributes 被所有页面共享, 直接引用会让一个页面写入的版本号泄漏到之后的页面 + attrs := *aliasFrame.ToWFN() + frame.UpdateAttributes(&attrs) } if aliasFrame.IsBlocked(frame.From.String()) { continue diff --git a/jev.go b/jev.go new file mode 100644 index 0000000..f1aead0 --- /dev/null +++ b/jev.go @@ -0,0 +1,40 @@ +package fingers + +import ( + "context" + "errors" + + "github.com/chainreactors/fingers/common" + "github.com/chainreactors/fingers/jev" +) + +var ErrJevNotAttached = errors.New("jev is not attached, call AttachJev first") + +// AttachJev adds the Jev judgement layer on top of the enabled engines. The +// rule engines are untouched: WebMatch/DetectContent stay synchronous and +// offline; Refine judges their result afterwards, typically asynchronously. +func (engine *Engine) AttachJev(client *jev.Client) { + var names []string + if engine.Aliases != nil { + for name := range engine.Aliases.Aliases { + names = append(names, name) + } + } + engine.jevClient, engine.jevRecall = client, jev.NewRetriever(names) +} + +// Refine judges frames, the rule result for the raw HTTP response content, +// and annotates them in place (see the jev.Tag* constants); fingerprint +// names found in the page but missed by the rules are recall candidates. +// The returned page carries the page kind. Refine writes to frames: do not +// read them concurrently. On error, frames hold the rule result. +func (engine *Engine) Refine(ctx context.Context, content []byte, frames common.Frameworks) (*jev.Page, error) { + if engine.jevClient == nil { + return nil, ErrJevNotAttached + } + page, err := jev.NewPage(content) + if err != nil { + return nil, err + } + return page, jev.Refine(ctx, engine.jevClient, page, frames, engine.jevRecall.Find(page.Haystack(), 24)) +} diff --git a/jev/cache.go b/jev/cache.go new file mode 100644 index 0000000..4b2bca9 --- /dev/null +++ b/jev/cache.go @@ -0,0 +1,91 @@ +package jev + +import ( + "container/list" + "math/bits" + "sync" +) + +// SimilarDistance is how many of the 64 signature bits two pages may differ +// in to share cached answers; 0 shares answers only between identical pages. +var SimilarDistance = 3 + +// Cache stores successful responses. key covers everything that must match +// exactly: model, questions and named evidence such as version strings. sig +// is the page's Signature: a cache may return the answers of a page whose +// signature is within SimilarDistance bits, so near-identical pages (the +// same product's login page on many hosts) cost one request. Jev answers an +// identical request identically, so an exact hit is always valid. +// Implementations must be safe for concurrent use. +type Cache interface { + Get(key string, sig uint64) ([]byte, bool) + Put(key string, sig uint64, value []byte) +} + +// Similar reports whether two signatures are within SimilarDistance. +func Similar(a, b uint64) bool { return bits.OnesCount64(a^b) <= SimilarDistance } + +// NewMemoryCache is an LRU cache holding up to size responses. +func NewMemoryCache(size int) Cache { + return &memoryCache{size: size, order: list.New(), buckets: map[string][]*list.Element{}} +} + +type memoryCache struct { + mu sync.Mutex + size int + order *list.List // front = most recent; values are *memoryEntry + buckets map[string][]*list.Element +} + +type memoryEntry struct { + key string + sig uint64 + value []byte +} + +func (c *memoryCache) Get(key string, sig uint64) ([]byte, bool) { + c.mu.Lock() + defer c.mu.Unlock() + var best *list.Element + bestDist := 65 + for _, e := range c.buckets[key] { + if d := bits.OnesCount64(e.Value.(*memoryEntry).sig ^ sig); d < bestDist { + best, bestDist = e, d + } + } + if best == nil || bestDist > SimilarDistance { + return nil, false + } + c.order.MoveToFront(best) + return best.Value.(*memoryEntry).value, true +} + +func (c *memoryCache) Put(key string, sig uint64, value []byte) { + c.mu.Lock() + defer c.mu.Unlock() + for _, e := range c.buckets[key] { + if me := e.Value.(*memoryEntry); me.sig == sig { + me.value = value + c.order.MoveToFront(e) + return + } + } + c.buckets[key] = append(c.buckets[key], c.order.PushFront(&memoryEntry{key, sig, value})) + for c.order.Len() > c.size { + last := c.order.Back() + c.order.Remove(last) + me := last.Value.(*memoryEntry) + bucket := c.buckets[me.key] + for i, e := range bucket { + if e == last { + bucket = append(bucket[:i], bucket[i+1:]...) + break + } + } + if len(bucket) == 0 { + delete(c.buckets, me.key) + } else { + c.buckets[me.key] = bucket + } + } +} diff --git a/jev/classify.go b/jev/classify.go new file mode 100644 index 0000000..94bc2aa --- /dev/null +++ b/jev/classify.go @@ -0,0 +1,30 @@ +package jev + +// PageKinds is the Choice criteria for Page.Kind. +var PageKinds = map[string]interface{}{ + "login": "A sign-in form or authentication page", + "default_install": "The stock page a web server or product shows right after installation", + "error_page": "A framework or server error page such as 404 or 500", + "directory_listing": "An auto-generated file index of a directory", + "app_console": "The main interface or loading screen of a web application or management console", + "normal_content": "An ordinary website page: article, company site, shop, forum, blog, status page", + "api_response": "A machine-readable API response rather than a web page", +} + +// Classify judges the page itself and writes Page.Kind and Page.Generic. It +// needs no fingerprints: spray can use it alone for 404 detection and +// prioritisation. A generic page without an accepted application is a +// candidate for a new fingerprint. +func Classify(r *Round) { + p := r.page + r.Add("page_kind", Choice("What kind of page is this HTTP response?", PageKinds), func(a Answer) { p.Kind = a.Choice }) + r.Add("generic", Question{ + Type: "noul", + Instructions: "Is this page the stock interface of a packaged software product, framework or device, " + + "so that the same page would appear on many unrelated deployments?", + Criteria: map[string]string{ + "true": "Login, console, default or error page shipped by a product; only branding, host names or data differ between installs", + "false": "Content written by one organization for its own purpose: articles, company pages, shops, forums, custom-built portals", + }, + }, func(a Answer) { p.Generic = a.Noul >= Threshold }) +} diff --git a/jev/client.go b/jev/client.go new file mode 100644 index 0000000..f2ce3ac --- /dev/null +++ b/jev/client.go @@ -0,0 +1,200 @@ +// Package jev is the judgement layer on top of the rule engines, backed by +// TypeSafe's Jev (System One) model. It is not an engine: rules and code +// recall candidates, Jev only verifies and chooses (false positives, stack +// layer, primary application, page kind, generic page, version), and code +// makes the final call. Each capability is a Task; the tasks of a round share +// one request, so a page costs at most two. +// +// API reference: https://docs.typesafe.ai/api.md +package jev + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "sync/atomic" + "time" +) + +const ( + DefaultEndpoint = "https://api.typesafe.ai/v1/systemone" + // Pin a versioned model: thresholds tuned against one version do not carry over. + DefaultModel = "jev-1.13.0" + EnvAPIKey = "TYPESAFE_API_KEY" +) + +// Question is one of noul / choice / score. +type Question struct { + Type string `json:"type"` + Instructions interface{} `json:"instructions"` + Criteria interface{} `json:"criteria,omitempty"` +} + +func Noul(instructions interface{}) Question { + return Question{Type: "noul", Instructions: instructions} +} + +// Choice criteria maps option -> description (nil for none). Max 255 options. +func Choice(instructions interface{}, criteria map[string]interface{}) Question { + return Question{Type: "choice", Instructions: instructions, Criteria: criteria} +} + +// Score criteria is an ordered list of 2..10 level descriptions. +func Score(instructions interface{}, levels []string) Question { + return Question{Type: "score", Instructions: instructions, Criteria: levels} +} + +type request struct { + State interface{} `json:"state"` + Model string `json:"model"` + Questions map[string]Question `json:"questions"` +} + +type Answer struct { + Type string `json:"type"` + Noul float64 `json:"noul,omitempty"` + Choice string `json:"choice,omitempty"` + Score float64 `json:"score,omitempty"` + Probabilities map[string]float64 `json:"probabilities,omitempty"` + Confidence float64 `json:"confidence,omitempty"` +} + +type Usage struct { + InputTokens int `json:"input_tokens"` + OutputTokens int `json:"output_tokens"` +} + +type Response struct { + Model string `json:"model"` + Answers map[string]Answer `json:"answers"` + Usage Usage `json:"usage"` + Latency time.Duration `json:"-"` + Cached bool `json:"-"` +} + +type Client struct { + Endpoint string + Model string + APIKey string + HTTP *http.Client + MaxRetries int + Cache Cache // nil disables caching + + // Totals since creation, for cost monitoring; read with atomic.LoadInt64. + Requests, CacheHits, InputTokens int64 +} + +// NewClient reads the API key from TYPESAFE_API_KEY when apiKey is empty. +func NewClient(apiKey string) (*Client, error) { + if apiKey == "" { + apiKey = os.Getenv(EnvAPIKey) + } + if apiKey == "" { + return nil, fmt.Errorf("%s is not set", EnvAPIKey) + } + return &Client{ + Endpoint: DefaultEndpoint, + Model: DefaultModel, + APIKey: apiKey, + HTTP: &http.Client{Timeout: 30 * time.Second}, + MaxRetries: 3, + }, nil +} + +type APIError struct { + Status int + Body string +} + +func (e *APIError) Error() string { return fmt.Sprintf("typesafe: http %d: %s", e.Status, e.Body) } + +// Ask evaluates every question against one state in a single request. +// 429 / 529 are retried with exponential backoff. Most callers use Round, +// which also lets the cache match similar pages. +func (c *Client) Ask(ctx context.Context, state interface{}, questions map[string]Question) (*Response, error) { + return c.ask(ctx, state, questions, nil, 0) +} + +// ask caches under the hash of exact (the parts that must match exactly) and +// the page signature sig. With exact nil the whole request must match. +func (c *Client) ask(ctx context.Context, state interface{}, questions map[string]Question, exact []byte, sig uint64) (*Response, error) { + // encoding/json sorts map keys, so equal requests marshal to equal bytes. + body, err := json.Marshal(request{State: state, Model: c.Model, Questions: questions}) + if err != nil { + return nil, err + } + if exact == nil { + exact = body + } else { + exact = append([]byte(c.Model+"\x00"), exact...) + } + sum := sha256.Sum256(exact) + key := hex.EncodeToString(sum[:]) + if c.Cache != nil { + if raw, ok := c.Cache.Get(key, sig); ok { + var out Response + if json.Unmarshal(raw, &out) == nil { + atomic.AddInt64(&c.CacheHits, 1) + out.Cached = true + return &out, nil + } + } + } + backoff := time.Second + for attempt := 0; ; attempt++ { + resp, raw, err := c.do(ctx, body) + if err == nil { + atomic.AddInt64(&c.Requests, 1) + atomic.AddInt64(&c.InputTokens, int64(resp.Usage.InputTokens)) + if c.Cache != nil { + c.Cache.Put(key, sig, raw) + } + return resp, nil + } + apiErr, ok := err.(*APIError) + if !ok || (apiErr.Status != 429 && apiErr.Status != 529) || attempt >= c.MaxRetries { + return nil, err + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(backoff): + } + backoff *= 2 + } +} + +func (c *Client) do(ctx context.Context, body []byte) (*Response, []byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.Endpoint, bytes.NewReader(body)) + if err != nil { + return nil, nil, err + } + req.Header.Set("Authorization", "Bearer "+c.APIKey) + req.Header.Set("Content-Type", "application/json") + + start := time.Now() + resp, err := c.HTTP.Do(req) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + raw, err := io.ReadAll(resp.Body) + if err != nil { + return nil, nil, err + } + if resp.StatusCode != http.StatusOK { + return nil, nil, &APIError{Status: resp.StatusCode, Body: string(raw)} + } + var out Response + if err := json.Unmarshal(raw, &out); err != nil { + return nil, nil, fmt.Errorf("typesafe: decode response: %w", err) + } + out.Latency = time.Since(start) + return &out, raw, nil +} diff --git a/jev/jev_test.go b/jev/jev_test.go new file mode 100644 index 0000000..5414a83 --- /dev/null +++ b/jev/jev_test.go @@ -0,0 +1,407 @@ +package jev + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "sort" + "strings" + "sync/atomic" + "testing" + + "github.com/chainreactors/fingers/common" +) + +const jenkinsRaw = "HTTP/1.1 200 OK\r\nServer: nginx/1.24.0\r\nX-Jenkins: 2.401.3\r\nSet-Cookie: JSESSIONID.1=x; Path=/\r\nContent-Type: text/html\r\n\r\n" + + `Sign in [Jenkins]` + + `

We moved here from WordPress.

` + + `` + +func TestNewPage(t *testing.T) { + s, err := NewPage([]byte(jenkinsRaw)) + if err != nil { + t.Fatal(err) + } + if s.Title != "Sign in [Jenkins]" || s.Headers["X-Jenkins"] != "2.401.3" { + t.Fatalf("title/header: %+v", s) + } + if len(s.Cookies) != 1 || len(s.Scripts) != 1 || len(s.Forms) != 2 { + t.Fatalf("cookies/scripts/forms: %+v", s) + } + if got := extractVersions([]byte(jenkinsRaw), 5); len(got) != 2 || got[0].Value != "1.24.0" || got[1].Value != "2.401.3" { + t.Fatalf("versions: %v", got) + } + if got := NewRetriever([]string{"jenkins", "oa", "tomcat"}).Find(s.Haystack(), 5); len(got) != 1 || got[0] != "jenkins" { + t.Fatalf("retriever: %v", got) + } + if b, _ := json.Marshal(s); strings.Contains(string(b), "Kind") || strings.Contains(string(b), "Generic") { + t.Fatalf("conclusions leak into the state: %s", b) + } +} + +// mock answers every question from a table keyed by question kind and the +// product named in backticks, and records the question keys of each request. +type mock struct { + calls int64 + requests [][]string + status int + versionP float64 + present map[string]float64 + layer map[string]Layer + primary string + lastState map[string]interface{} +} + +func newMock() *mock { + return &mock{ + versionP: 0.95, + present: map[string]float64{"nginx": 0.1, "wordpress": 0.05, "jenkins": 0.97, "apache tomcat": 0.8, "Prototype": 0.9}, + layer: map[string]Layer{"nginx": LayerServer, "wordpress": LayerNotPresent, "jenkins": LayerApplication, + "apache tomcat": LayerServer, "Prototype": LayerFrontend}, + primary: "jenkins", + } +} + +func named(q Question) string { + s, _ := q.Instructions.(string) + if i := strings.Index(s, "`"); i >= 0 { + if j := strings.Index(s[i+1:], "`"); j >= 0 { + return s[i+1 : i+1+j] + } + } + return "" +} + +func (m *mock) client(t *testing.T) (*Client, func()) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt64(&m.calls, 1) + if m.status != 0 { + w.WriteHeader(m.status) + return + } + var req request + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Errorf("decode: %v", err) + } + m.lastState, _ = req.State.(map[string]interface{}) + answers := map[string]Answer{} + var keys []string + for k, q := range req.Questions { + keys = append(keys, k) + name := named(q) + switch { + case strings.HasPrefix(k, "is_"): + answers[k] = Answer{Type: "noul", Noul: m.present[name]} + case strings.HasPrefix(k, "layer_"): + answers[k] = Answer{Type: "choice", Choice: string(m.layer[name]), Confidence: 0.9} + case k == "primary": + answers[k] = Answer{Type: "choice", Choice: m.primary, Confidence: 0.9} + case k == "page_kind": + answers[k] = Answer{Type: "choice", Choice: "login", Confidence: 0.9} + case k == "generic": + answers[k] = Answer{Type: "noul", Noul: 0.9} + case k == "version": + answers[k] = Answer{Type: "choice", Choice: "2.401.3", Confidence: m.versionP} + default: + answers[k] = Answer{Type: "noul", Noul: 0.7} + } + } + sort.Strings(keys) + m.requests = append(m.requests, keys) + json.NewEncoder(w).Encode(Response{Model: DefaultModel, Answers: answers, Usage: Usage{InputTokens: 100}}) + })) + c, _ := NewClient("k") + c.Endpoint = srv.URL + c.MaxRetries = 0 + return c, srv.Close +} + +func testFrames() common.Frameworks { + fs := common.Frameworks{} + fs.Add(common.NewFramework("nginx", common.FrameFromFingers)) + fs.Add(common.NewFramework("wordpress", common.FrameFromGoby)) + fs.Add(common.NewFramework("jenkins", common.FrameFromFingerprintHub)) + fs.Add(common.NewFramework("apache-tomcat", common.FrameFromFingers)) + fs.Add(common.NewFramework("apache tomcat", common.FrameFromWappalyzer)) + return fs +} + +func refine(t *testing.T, c *Client, frames common.Frameworks) (*Page, error) { + p, err := NewPage([]byte(jenkinsRaw)) + if err != nil { + t.Fatal(err) + } + return p, Refine(context.Background(), c, p, frames, []string{"Jenkins", "Prototype"}) +} + +func TestRefine(t *testing.T) { + m := newMock() + c, stop := m.client(t) + defer stop() + frames := testFrames() + p, err := refine(t, c, frames) + if err != nil { + t.Fatal(err) + } + if !frames["wordpress"].HasTag(TagRejected) { + t.Errorf("text-only wordpress not rejected: %v", frames["wordpress"].Tags) + } + if nginx := frames["nginx"]; nginx.HasTag(TagRejected) || !nginx.HasTag(TagLayer+string(LayerServer)) { + t.Errorf("nginx from the Server header must be kept: %v", nginx.Tags) + } + if frames["apache tomcat"].HasTag(TagDup) == frames["apache-tomcat"].HasTag(TagDup) { + t.Errorf("exactly one tomcat spelling must be a dup") + } + j := frames["jenkins"] + if !j.HasTag(TagPrimary) || j.Version != "2.401.3" || Primary(frames) != j { + t.Errorf("jenkins primary/version: %v %q", j.Tags, j.Version) + } + if f := frames["prototype"]; f == nil || !f.HasTag(TagRecall) || !f.IsGuess() { + t.Errorf("confirmed recall not added: %+v", f) + } + if got := Accepted(frames); len(got) != 4 { // nginx, tomcat, jenkins, prototype + t.Errorf("accepted: %v", got) + } + if p.Kind != "login" || !p.Generic { + t.Errorf("page: %q %v", p.Kind, p.Generic) + } + // Jenkins is both a rule hit and a recall name: asked once. 5 products x 2 + primary + page_kind + generic. + if len(m.requests) != 2 || len(m.requests[0]) != 13 || strings.Join(m.requests[1], ",") != "version" { + t.Errorf("requests: %v", m.requests) + } + if _, ok := m.lastState["version_strings"]; !ok { + t.Errorf("version strings not in the state: %v", m.lastState) + } + if c.Requests != 2 || c.InputTokens != 200 { + t.Errorf("usage: %d %d", c.Requests, c.InputTokens) + } +} + +func TestVersionNeedsConfidence(t *testing.T) { + m := newMock() + m.versionP = 0.6 + c, stop := m.client(t) + defer stop() + frames := testFrames() + if _, err := refine(t, c, frames); err != nil { + t.Fatal(err) + } + if v := frames["jenkins"].Version; v != "" { + t.Fatalf("low-confidence version written: %q", v) + } +} + +func TestFailedRoundLeavesFrames(t *testing.T) { + m := newMock() + m.status = 500 + c, stop := m.client(t) + defer stop() + frames := testFrames() + before, _ := json.Marshal(frames) + p, err := refine(t, c, frames) + if err == nil { + t.Fatal("expected error") + } + if after, _ := json.Marshal(frames); string(after) != string(before) || p.Kind != "" { + t.Fatalf("frames changed on failure:\n%s\n%s", before, after) + } +} + +func TestCache(t *testing.T) { + m := newMock() + c, stop := m.client(t) + defer stop() + c.Cache = NewMemoryCache(16) + for i := 0; i < 2; i++ { + frames := testFrames() + if _, err := refine(t, c, frames); err != nil { + t.Fatal(err) + } + if frames["jenkins"].Version != "2.401.3" { + t.Fatalf("run %d: cached answers not applied", i) + } + } + if m.calls != 2 || c.CacheHits != 2 { + t.Fatalf("calls=%d hits=%d", m.calls, c.CacheHits) + } +} + +func TestClassifyAlone(t *testing.T) { + m := newMock() + c, stop := m.client(t) + defer stop() + p, _ := NewPage([]byte(jenkinsRaw)) + r := p.Round() + Classify(r) + var custom float64 + r.Add("honeypot", Noul("Is this a honeypot?"), func(a Answer) { custom = a.Noul }) + if err := r.Ask(context.Background(), c); err != nil { + t.Fatal(err) + } + if strings.Join(m.requests[0], ",") != "generic,honeypot,page_kind" || p.Kind != "login" || custom != 0.7 { + t.Fatalf("requests=%v kind=%q custom=%v", m.requests, p.Kind, custom) + } + if err := p.Round().Ask(context.Background(), c); err != nil || m.calls != 1 { + t.Fatalf("empty round sent a request: %v %d", err, m.calls) + } + r = p.Round() + r.Add("x", Noul("?"), nil) + r.Add("x", Noul("?"), nil) + if err := r.Ask(context.Background(), c); err == nil { + t.Fatal("duplicate key accepted") + } +} + +func TestMemoryCacheEvicts(t *testing.T) { + c := NewMemoryCache(2) + c.Put("a", 0, []byte("1")) + c.Put("b", 0, []byte("2")) + c.Get("a", 0) + c.Put("c", 0, []byte("3")) + if _, ok := c.Get("b", 0); ok { + t.Fatal("least recently used entry kept") + } + if v, ok := c.Get("a", 0); !ok || string(v) != "1" { + t.Fatal("recent entry evicted") + } + if _, ok := c.Get("a", 0xff); ok { + t.Fatal("signature 8 bits away matched") + } + if v, ok := c.Get("a", 0x3); !ok || string(v) != "1" { + t.Fatal("signature 2 bits away missed") + } +} + +// The same product page on another host (other token, date, host name) reuses +// the answers; a different page does not. +func TestSimilarPagesShareAnswers(t *testing.T) { + m := newMock() + c, stop := m.client(t) + defer stop() + c.Cache = NewMemoryCache(16) + variant := strings.NewReplacer("JSESSIONID.1=x", "JSESSIONID.1=9f8e7d", "Sign in [Jenkins]", "Sign in [Jenkins]", + "We moved here from WordPress.", "We moved here from WordPress on 2026-09-24.").Replace(jenkinsRaw) + a, _ := NewPage([]byte(jenkinsRaw)) + b, _ := NewPage([]byte(variant)) + other, _ := NewPage([]byte("HTTP/1.1 200 OK\r\nServer: Apache\r\n\r\nIndex of /a.txt")) + if !Similar(a.Signature(), b.Signature()) || Similar(a.Signature(), other.Signature()) { + t.Fatalf("signatures: %x %x %x", a.Signature(), b.Signature(), other.Signature()) + } + for _, raw := range []string{jenkinsRaw, variant} { + p, _ := NewPage([]byte(raw)) + r := p.Round() + Classify(r) + if err := r.Ask(context.Background(), c); err != nil || p.Kind != "login" { + t.Fatalf("kind %q err %v", p.Kind, err) + } + } + if m.calls != 1 || c.CacheHits != 1 { + t.Fatalf("similar page not served from cache: calls=%d hits=%d", m.calls, c.CacheHits) + } +} + +func TestAskRetriesOverload(t *testing.T) { + calls := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + if r.Header.Get("Authorization") != "Bearer k" { + t.Errorf("auth header: %q", r.Header.Get("Authorization")) + } + if calls == 1 { + w.WriteHeader(529) + return + } + var req request + json.NewDecoder(r.Body).Decode(&req) + if req.Model != DefaultModel || req.Questions["q"].Type != "noul" { + t.Errorf("request: %+v", req) + } + w.Write([]byte(`{"model":"jev-1.13.0","answers":{"q":{"type":"noul","noul":0.9}},"usage":{"input_tokens":10,"output_tokens":1}}`)) + })) + defer srv.Close() + + c, _ := NewClient("k") + c.Endpoint = srv.URL + resp, err := c.Ask(context.Background(), "state", map[string]Question{"q": Noul("?")}) + if err != nil { + t.Fatal(err) + } + if calls != 2 || resp.Answers["q"].Noul != 0.9 { + t.Fatalf("calls=%d resp=%+v", calls, resp) + } +} + +func TestExtractVersionsPrefixes(t *testing.T) { + raw := "HTTP/1.1 200 OK\r\n\r\nV8.1SP2 10.0.0.5 jquery.min.js?ver=3.7.1" + var got []string + for _, c := range extractVersions([]byte(raw), 10) { + got = append(got, c.Value) + } + if strings.Join(got, ",") != "3.4,8.1,3.7.1" { + t.Fatalf("got %v", got) + } +} + +func TestRetrieverWordBoundary(t *testing.T) { + r := NewRetriever([]string{"acti", "jenkins", "泛微"}) + if got := r.Find("take action now", 5); len(got) != 0 { + t.Fatalf("substring matched: %v", got) + } + if got := r.Find("sign in [jenkins] 泛微oa", 5); len(got) != 2 { + t.Fatalf("got %v", got) + } +} + +func TestRankVersions(t *testing.T) { + raw := "HTTP/1.1 200 OK\r\n\r\n" + for i := 0; i < 15; i++ { + raw += fmt.Sprintf(``, i, i) + } + raw += `` + cands := extractVersions([]byte(raw), 40) + got := rankVersions(cands, "wordpress", 3) + if got[0].Value != "7.0.3" || !strings.Contains(got[0].Context, "generator") { + t.Fatalf("wordpress generator not ranked first: %+v", got) + } + var drupal bool + for _, c := range cands { + drupal = drupal || c.Value == "10" + } + if !drupal { + t.Fatalf("major-only generator version missing: %+v", cands) + } +} + +func TestNormalizeName(t *testing.T) { + same := [][]string{{"Apache HTTP Server", "apache-http", "apache-web-server", "Apache"}, {"Discuz! X", "discuz"}, {"Apache-Tomcat", "apache tomcat"}, {"泛微 OA", "泛微"}} + for _, names := range same { + for _, n := range names[1:] { + if NormalizeName(n) != NormalizeName(names[0]) { + t.Errorf("%q and %q not folded: %q %q", names[0], n, NormalizeName(names[0]), NormalizeName(n)) + } + } + } + for _, pair := range [][2]string{{"lighttpd", "lig"}, {"Apache Tomcat", "Apache"}, {"nginx", "ngin"}} { + if NormalizeName(pair[0]) == NormalizeName(pair[1]) { + t.Errorf("%q and %q folded", pair[0], pair[1]) + } + } +} + +// A server's own page has no primary application; the only server is the subject. +func TestVersionWithoutPrimary(t *testing.T) { + m := newMock() + m.primary = NoneOfThem + c, stop := m.client(t) + defer stop() + frames := common.Frameworks{} + frames.Add(common.NewFramework("nginx", common.FrameFromFingers)) + if _, err := refine(t, c, frames); err != nil { + t.Fatal(err) + } + if Primary(frames) != nil || frames["nginx"].Version == "" { + t.Fatalf("subject version not asked: %+v", frames["nginx"]) + } +} diff --git a/jev/page.go b/jev/page.go new file mode 100644 index 0000000..d264c22 --- /dev/null +++ b/jev/page.go @@ -0,0 +1,235 @@ +package jev + +import ( + "bufio" + "bytes" + "hash/fnv" + "io" + "net/http" + "regexp" + "strings" + "unicode/utf8" +) + +// Page is the entry point of the judgement layer: the compact, named view of +// one HTTP response that Jev judges (serialized as the request state), plus +// the page-level conclusions written back by Classify. Jev only judges the +// evidence it is given and loses accuracy on large states full of irrelevant +// detail, so the fields below are the ones that carry fingerprint signal. +type Page struct { + Status string `json:"status"` + Headers map[string]string `json:"headers,omitempty"` + Cookies []string `json:"cookie_names,omitempty"` + Title string `json:"title,omitempty"` + Generator string `json:"meta_generator,omitempty"` + Scripts []string `json:"script_src,omitempty"` + Styles []string `json:"stylesheet_href,omitempty"` + InlineHints []string `json:"inline_script_starts,omitempty"` // first chars of inline scripts, e.g. "(function(w,d,s,l,i){ ... GTM" + Comments []string `json:"html_comments,omitempty"` + Forms []string `json:"form_inputs,omitempty"` + Text string `json:"visible_text,omitempty"` + + Kind string `json:"-"` // one of PageKinds, set by Classify + Generic bool `json:"-"` // stock page of a packaged product, set by Classify + + raw []byte +} + +var ( + // Headers that carry no fingerprint signal, or only noise; every other + // header is kept, since products announce themselves in custom headers + // ("product: Z-BlogPHP 1.7.4", "X-Jenkins", "kbn-name"). + skipHeaders = map[string]bool{"Date": true, "Content-Length": true, "Content-Type": true, "Expires": true, "Cache-Control": true, + "Pragma": true, "Etag": true, "Last-Modified": true, "Age": true, "Connection": true, "Keep-Alive": true, "Accept-Ranges": true, + "Vary": true, "Set-Cookie": true, "Content-Security-Policy": true, "Content-Security-Policy-Report-Only": true, "Report-To": true, + "Nel": true, "Permissions-Policy": true, + // per-request values: no signal, and they would defeat the cache + "X-Request-Id": true, "X-Amzn-Trace-Id": true, "Cf-Ray": true, "X-Runtime": true, "Server-Timing": true} + + reTitle = regexp.MustCompile(`(?is)]*>(.*?)`) + reGenerator = regexp.MustCompile(`(?is)]+name=["']generator["'][^>]*content=["']([^"']+)|]+content=["']([^"']+)["'][^>]*name=["']generator["']`) + reScript = regexp.MustCompile(`(?is)]+src=["']([^"']+)`) + reStyle = regexp.MustCompile(`(?is)]+rel=["']?stylesheet[^>]*href=["']([^"']+)|]+href=["']([^"']+)["'][^>]*rel=["']?stylesheet`) + reInline = regexp.MustCompile(`(?is)]*)?>(.*?)`) + reComment = regexp.MustCompile(`(?s)`) + reInput = regexp.MustCompile(`(?is)]*>`) + reAttrName = regexp.MustCompile(`(?is)\b(?:name|id)=["']([^"']+)`) + reAttrType = regexp.MustCompile(`(?is)\btype=["']([^"']+)`) + reDropBlock = regexp.MustCompile(`(?is)<(script|style|noscript)[^>]*>.*?`) + reTag = regexp.MustCompile(`(?s)<[^>]+>`) + reSpace = regexp.MustCompile(`\s+`) + // Allows a v/V/x/X prefix ("X3.4", "V8.1SP2") and a letter suffix; rejects + // digits glued to other digits or dots so IPs and long builds stay out. + reGenMajor = regexp.MustCompile(`(?i)generator["'][^>]*content=["'][A-Za-z][^"']*?\s[vV]?(\d{1,3})(?:[\s"'(]|$)|content=["'][A-Za-z][^"']*?\s[vV]?(\d{1,3})(?:[\s(][^"']*)?["'][^>]*name=["']generator`) + reVersion = regexp.MustCompile(`(?:^|[^0-9A-Za-z.])[vVxX]?(\d{1,3}\.\d{1,3}(?:\.\d{1,4}){0,2})(?:[^0-9.]|$)`) +) + +const ( + maxHeaderValue = 160 + maxText = 1500 // visible text runes +) + +// NewPage parses a raw HTTP response (head and body, as rule engines take it). +func NewPage(raw []byte) (*Page, error) { + resp, err := http.ReadResponse(bufio.NewReader(bytes.NewReader(raw)), nil) + if err != nil { + return nil, err + } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + s := &Page{Status: resp.Status, Headers: map[string]string{}, raw: raw} + for k, vs := range resp.Header { + if skipHeaders[k] || strings.HasPrefix(k, "X-Crawler-") || len(vs) == 0 { + continue + } + s.Headers[k] = truncate(strings.Join(vs, ", "), maxHeaderValue) + } + for _, c := range resp.Cookies() { + s.Cookies = append(s.Cookies, c.Name) + } + html := string(body) + if m := reTitle.FindStringSubmatch(html); m != nil { + s.Title = clean(m[1]) + } + if m := reGenerator.FindStringSubmatch(html); m != nil { + s.Generator = m[1] + m[2] + } + for _, m := range reScript.FindAllStringSubmatch(html, 8) { + s.Scripts = append(s.Scripts, m[1]) + } + for _, m := range reStyle.FindAllStringSubmatch(html, 8) { + s.Styles = append(s.Styles, m[1]+m[2]) + } + for _, m := range reInline.FindAllStringSubmatch(html, -1) { + if code := clean(m[1]); len(code) > 20 && len(s.InlineHints) < 5 { + s.InlineHints = append(s.InlineHints, truncate(code, 100)) + } + } + for _, m := range reComment.FindAllStringSubmatch(html, -1) { + if c := clean(m[1]); len(c) > 3 && len(s.Comments) < 5 && !strings.HasPrefix(c, "[if") { + s.Comments = append(s.Comments, truncate(c, 80)) + } + } + for _, in := range reInput.FindAllString(html, 12) { + name, typ := "", "text" + if m := reAttrName.FindStringSubmatch(in); m != nil { + name = m[1] + } + if m := reAttrType.FindStringSubmatch(in); m != nil { + typ = m[1] + } + if name != "" { + s.Forms = append(s.Forms, name+":"+typ) + } + } + text := clean(reTag.ReplaceAllString(reDropBlock.ReplaceAllString(html, " "), " ")) + s.Text = truncate(text, maxText) + return s, nil +} + +// Haystack is the lowercase text that candidate names are searched in. +func (s *Page) Haystack() string { + var b strings.Builder + for k, v := range s.Headers { + b.WriteString(k + ": " + v + "\n") + } + b.WriteString(strings.Join(s.Cookies, " ") + "\n") + b.WriteString(s.Title + "\n" + s.Generator + "\n") + b.WriteString(strings.Join(s.Scripts, " ") + "\n") + b.WriteString(strings.Join(s.Styles, " ") + "\n") + b.WriteString(strings.Join(s.InlineHints, " ") + "\n") + b.WriteString(strings.Join(s.Comments, " ") + "\n") + b.WriteString(s.Text) + return strings.ToLower(b.String()) +} + +func clean(s string) string { + s = strings.NewReplacer(" ", " ", "&", "&", "<", "<", ">", ">", """, `"`).Replace(s) + return strings.TrimSpace(reSpace.ReplaceAllString(s, " ")) +} + +func truncate(s string, max int) string { + if utf8.RuneCountInString(s) <= max { + return s + } + return string([]rune(s)[:max]) + "…" +} + +// Evidence reports where a candidate name literally occurs in the page. It is +// a cheap code-only baseline for false positive detection: a name found only +// in visible text is a likely false positive. +func (s *Page) Evidence(name string) []string { + n := strings.ToLower(name) + if len(n) < 3 { + return nil + } + var where []string + check := func(label, text string) { + if strings.Contains(strings.ToLower(text), n) { + where = append(where, label) + } + } + var headers strings.Builder + for k, v := range s.Headers { + headers.WriteString(k + ":" + v + " ") + } + check("header", headers.String()) + check("cookie", strings.Join(s.Cookies, " ")) + check("title", s.Title) + check("meta", s.Generator) + check("script", strings.Join(s.Scripts, " ")+" "+strings.Join(s.InlineHints, " ")) + check("style", strings.Join(s.Styles, " ")) + check("comment", strings.Join(s.Comments, " ")) + check("form", strings.Join(s.Forms, " ")) + check("text", s.Text) + return where +} + +var reDigits = regexp.MustCompile(`[0-9]+`) + +// Signature is a 64-bit simhash of the page's evidence. Digits are folded so +// timestamps, build ids and tokens do not matter; structure (headers, +// assets, forms) weighs more than visible text. Pages of one product on +// different hosts get signatures a few bits apart; see SimilarDistance. +func (s *Page) Signature() uint64 { + var v [64]int + add := func(kind, feature string, weight int) { + h := fnv.New64a() + h.Write([]byte(kind)) + h.Write([]byte(reDigits.ReplaceAllString(strings.ToLower(feature), "0"))) + x := h.Sum64() + for i := 0; i < 64; i++ { + if x&(1<= 0 { + f = f[:i] + } + add("struct", f, 3) + } + } + add("generator", s.Generator, 3) + for _, w := range strings.Fields(s.Title) { + add("title", w, 2) + } + for _, w := range strings.Fields(s.Text) { + add("text", w, 1) + } + var sig uint64 + for i := 0; i < 64; i++ { + if v[i] > 0 { + sig |= 1 << uint(i) + } + } + return sig +} diff --git a/jev/recall.go b/jev/recall.go new file mode 100644 index 0000000..88f952f --- /dev/null +++ b/jev/recall.go @@ -0,0 +1,81 @@ +package jev + +import ( + "sort" + "strings" + "unicode/utf8" +) + +// Retriever finds fingerprint names that literally occur in a page. Every +// candidate handed to Jev is therefore anchored to evidence found by code, +// so Jev only ranks and verifies, it never invents a product. +type Retriever struct { + names []string // display names + keys []string // lowercase match keys, same index as names +} + +func NewRetriever(names []string) *Retriever { + r := &Retriever{} + seen := map[string]bool{} + for _, n := range names { + k := strings.ToLower(strings.TrimSpace(n)) + if seen[k] || !usableKey(k) { + continue + } + seen[k] = true + r.names = append(r.names, n) + r.keys = append(r.keys, k) + } + return r +} + +// usableKey drops keys too short to be meaningful evidence ("oa", "cms"). +func usableKey(k string) bool { + n := utf8.RuneCountInString(k) + if n == len(k) { // ascii + return n >= 4 + } + return n >= 2 +} + +// containsWord matches ASCII keys on word boundaries ("acti" must not match +// "action"); CJK keys have no word boundaries and match as substrings. +func containsWord(haystack, key string) bool { + if utf8.RuneCountInString(key) != len(key) { + return strings.Contains(haystack, key) + } + for from := 0; ; { + i := strings.Index(haystack[from:], key) + if i < 0 { + return false + } + start, end := from+i, from+i+len(key) + if (start == 0 || !isWordByte(haystack[start-1])) && (end == len(haystack) || !isWordByte(haystack[end])) { + return true + } + from = start + 1 + } +} + +func isWordByte(b byte) bool { + return b == '_' || b >= '0' && b <= '9' || b >= 'a' && b <= 'z' || b >= 'A' && b <= 'Z' +} + +// Find returns up to max names occurring in haystack, longest match first. +func (r *Retriever) Find(haystack string, max int) []string { + var hits []int + for i, k := range r.keys { + if containsWord(haystack, k) { + hits = append(hits, i) + } + } + sort.Slice(hits, func(a, b int) bool { return len(r.keys[hits[a]]) > len(r.keys[hits[b]]) }) + var out []string + for _, i := range hits { + out = append(out, r.names[i]) + if len(out) >= max { + break + } + } + return out +} diff --git a/jev/refine.go b/jev/refine.go new file mode 100644 index 0000000..cbf3b64 --- /dev/null +++ b/jev/refine.go @@ -0,0 +1,65 @@ +package jev + +import ( + "context" + + "github.com/chainreactors/fingers/common" +) + +// Refine is the default pipeline: Verify and Classify in one request, then +// the version of the page's subject in a second one. frames and p are +// annotated in place; on error the second round's annotations are missing, +// the first round's are complete or absent. +func Refine(ctx context.Context, c *Client, p *Page, frames common.Frameworks, recall []string) error { + r := p.Round() + Verify(r, frames, recall) + Classify(r) + if err := r.Ask(ctx, c); err != nil { + return err + } + r = p.Round() + Version(r, subject(frames)) + return r.Ask(ctx, c) +} + +// subject is the product whose version the page most likely shows: the +// primary application, or on pages without one (a server's default or error +// page) the only accepted server, application or device. +func subject(frames common.Frameworks) *common.Framework { + if f := Primary(frames); f != nil { + return f + } + var only *common.Framework + for _, f := range Accepted(frames) { + for _, l := range []Layer{LayerServer, LayerApplication, LayerDevice} { + if f.HasTag(TagLayer + string(l)) { + if only != nil { + return nil + } + only = f + } + } + } + return only +} + +// Primary returns the framework Verify tagged as the page's application. +func Primary(frames common.Frameworks) *common.Framework { + for _, f := range frames { + if f.HasTag(TagPrimary) { + return f + } + } + return nil +} + +// Accepted returns frames without rejected hits and duplicate spellings. +func Accepted(frames common.Frameworks) common.Frameworks { + out := common.Frameworks{} + for k, f := range frames { + if !f.HasTag(TagRejected) && !f.HasTag(TagDup) { + out[k] = f + } + } + return out +} diff --git a/jev/round.go b/jev/round.go new file mode 100644 index 0000000..15e19f5 --- /dev/null +++ b/jev/round.go @@ -0,0 +1,87 @@ +package jev + +import ( + "context" + "encoding/json" + "fmt" +) + +// Round is the Jev primitive: questions about one page, each with a callback, +// sent as a single request. Capabilities (Verify, Classify, Version) are +// functions that add to a Round; callers may add their own questions too. +// Callbacks run only after the whole request succeeded, so a failed Round +// leaves every annotated value untouched. +type Round struct { + page *Page + questions map[string]Question + apply map[string]func(Answer) + evidence map[string]interface{} + done []func() + err error +} + +func (p *Page) Round() *Round { + return &Round{page: p, questions: map[string]Question{}, apply: map[string]func(Answer){}} +} + +func (r *Round) Page() *Page { return r.page } + +// Add asks q under key; apply receives its answer. Keys must be unique in a Round. +func (r *Round) Add(key string, q Question, apply func(Answer)) { + if _, dup := r.questions[key]; dup && r.err == nil { + r.err = fmt.Errorf("jev: duplicate question key %q", key) + } + r.questions[key] = q + r.apply[key] = apply +} + +// Done registers f to run after every answer callback, in registration +// order: for decisions that combine several answers. +func (r *Round) Done(f func()) { r.done = append(r.done, f) } + +// Evidence shows Jev a named piece of evidence besides the page; the state +// becomes {"response": page, name: value}. Jev reads literally: evidence it is +// asked about by name weighs more than the same text in option descriptions. +func (r *Round) Evidence(name string, value interface{}) { + if r.evidence == nil { + r.evidence = map[string]interface{}{} + } + r.evidence[name] = value +} + +// Ask sends the round; a round without questions sends nothing. +func (r *Round) Ask(ctx context.Context, c *Client) error { + if r.err != nil { + return r.err + } + if len(r.questions) == 0 { + return nil + } + var state interface{} = r.page + if len(r.evidence) > 0 { + m := map[string]interface{}{"response": r.page} + for k, v := range r.evidence { + m[k] = v + } + state = m + } + // Similar pages share answers: the cache matches the page by signature, + // everything else (questions, named evidence such as version strings) exactly. + exact, err := json.Marshal([]interface{}{r.questions, r.evidence}) + if err != nil { + return err + } + resp, err := c.ask(ctx, state, r.questions, exact, r.page.Signature()) + if err != nil { + return err + } + for key, apply := range r.apply { + if a, ok := resp.Answers[key]; ok && apply != nil { + apply(a) + } + } + for _, f := range r.done { + f() + } + return nil +} diff --git a/jev/stack.go b/jev/stack.go new file mode 100644 index 0000000..17d8a92 --- /dev/null +++ b/jev/stack.go @@ -0,0 +1,59 @@ +package jev + +import ( + "strings" + "unicode" +) + +// Layer is the role a fingerprint plays in the software stack behind one +// response. A page normally carries several fingerprints at different layers +// (CDN + web server + framework + application + libraries); each candidate is +// judged independently, and the layer decides how results combine. +type Layer string + +const ( + LayerCDN Layer = "cdn_or_waf" + LayerServer Layer = "web_server" + LayerRuntime Layer = "language_runtime" + LayerFramework Layer = "web_framework" + LayerApplication Layer = "application" + LayerFrontend Layer = "frontend_library" + LayerDevice Layer = "os_or_device" + LayerNotPresent Layer = "not_present" +) + +// LayerCriteria is the Choice criteria for classifying one candidate's layer. +var LayerCriteria = map[string]interface{}{ + string(LayerCDN): "A CDN, reverse proxy cloud or web application firewall in front of the site", + string(LayerServer): "The HTTP server or proxy software, such as nginx, Apache httpd, IIS, Tomcat", + string(LayerRuntime): "A programming language or runtime, such as PHP, Java, ASP.NET, Node.js", + string(LayerFramework): "A backend web framework, such as Spring, Django, Laravel, ThinkPHP", + string(LayerApplication): "The application or product the page belongs to: CMS, OA, admin console, SaaS app, forum, monitoring tool", + string(LayerFrontend): "A browser-side library, UI kit, analytics or tag script, such as jQuery, Bootstrap, Google Analytics", + string(LayerDevice): "An operating system or the firmware of a hardware device such as a router, camera, NAS or firewall", + string(LayerNotPresent): "Not part of the software serving this response: only mentioned in text, or absent", +} + +var genericSuffixes = []string{"companyproducts", "公司产品", "product", "products", "产品", "operatingsystem", "操作系统", "system"} + +// genericWords are trailing words engines add to a product name ("Apache +// HTTP Server", "apache-web-server", "Discuz! X"). They are dropped only as +// whole words, so "lighttpd" keeps its "httpd". +var genericWords = map[string]bool{"http": true, "httpd": true, "server": true, "web": true, "webserver": true, "x": true, "cms": true, "oa": true} + +// NormalizeName folds the spellings different engines use for one product +// ("Apache-Tomcat", "apache tomcat", "Apache HTTP Server", "apache") into one +// key. It never merges names whose remaining letters differ. +func NormalizeName(name string) string { + words := strings.FieldsFunc(strings.ToLower(name), func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) }) + for len(words) > 1 && genericWords[words[len(words)-1]] { + words = words[:len(words)-1] + } + key := strings.Join(words, "") + for _, s := range genericSuffixes { + if len(key) > len(s)+2 && strings.HasSuffix(key, s) { + key = strings.TrimSuffix(key, s) + } + } + return key +} diff --git a/jev/verify.go b/jev/verify.go new file mode 100644 index 0000000..ae8c7b1 --- /dev/null +++ b/jev/verify.go @@ -0,0 +1,153 @@ +package jev + +import ( + "fmt" + "sort" + + "github.com/chainreactors/fingers/common" +) + +// Threshold is the Noul at or above which Jev's "yes" counts. Tuned for DefaultModel. +var Threshold = 0.5 + +// Tags written onto frameworks by Verify. Frameworks are annotated in place, +// never removed: callers decide whether to drop what Jev rejected. +const ( + TagRejected = "jev:rejected" // rule hit Jev judged absent or only mentioned in text + TagDup = "jev:dup" // another spelling of a product already in frames + TagPrimary = "jev:primary" // the application the page belongs to + TagRecall = "jev:recall" // added: no rule hit, found by name and confirmed by Jev + TagLayer = "jev:layer=" // prefix; followed by a Layer +) + +// NoneOfThem is the primary option for pages that belong to no listed product. +const NoneOfThem = "none_of_these" + +// ProtocolFeatures are HTTP features some engines report as fingerprints +// (NormalizeName keys). They are facts of the response head: never rejected. +var ProtocolFeatures = map[string]bool{"hsts": true, "altsvc": true, "http3": true, "http2": true, "poweredby": true, "http基本认证": true} + +const ( + maxGroups = 40 // products judged per page + maxRecall = 8 // recall names judged per page +) + +// product is one product claimed for the page: every engine spelling of it +// (frames), or a recall name without a rule hit. +type product struct { + name string + frames []*common.Framework + strong bool // named in a header or cookie: code accepts it, Jev under-trusts it +} + +// Verify is the engine result filter. It folds the spellings of all engines +// into products, asks Jev per product whether it is really in the stack (a +// Noul, so several products can be present) and at which layer, and which +// product is the page's application. recall names (e.g. fingerprint names +// found in the page) that no rule reported are judged the same way; confirmed +// ones are added to frames. Code decides: a product named in a header or +// cookie is never rejected. +func Verify(r *Round, frames common.Frameworks, recall []string) { + page := r.page + list := frames.List() + sort.Slice(list, func(a, b int) bool { return list[a].Name < list[b].Name }) + byKey := map[string]*product{} + var products []*product + for _, f := range list { + key := NormalizeName(f.Name) + if key == "" { + continue + } + p, ok := byKey[key] + if !ok { + if len(products) >= maxGroups { + continue + } + p = &product{name: f.Name, strong: ProtocolFeatures[key]} + byKey[key] = p + products = append(products, p) + } + p.frames = append(p.frames, f) + for _, e := range page.Evidence(f.Name) { + p.strong = p.strong || e == "header" || e == "cookie" + } + } + recalled := 0 + for _, name := range recall { + key := NormalizeName(name) + if _, ok := byKey[key]; ok || key == "" { + continue + } + if recalled >= maxRecall { + break + } + p := &product{name: name} + byKey[key] = p + products = append(products, p) + recalled++ + } + if len(products) == 0 { + return + } + + type answer struct { + present float64 + answered bool + layer Layer + } + answers := make([]answer, len(products)) + var primary string + primaryOpts := map[string]interface{}{NoneOfThem: "The page belongs to none of the listed products, e.g. a custom-built site"} + for i, p := range products { + i, name := i, p.name + primaryOpts[name] = nil + r.Add(fmt.Sprintf("is_%d", i), Question{ + Type: "noul", + Instructions: fmt.Sprintf("Is `%s` part of the software stack that produced this HTTP response "+ + "(seen in its headers, cookies, asset paths, page structure or title), rather than only mentioned in the page text?", name), + Criteria: map[string]string{ + "true": name + " served, generated or is loaded by this response", + "false": name + " only appears in the page's text, or is not present", + }, + }, func(a Answer) { answers[i].present, answers[i].answered = a.Noul, true }) + r.Add(fmt.Sprintf("layer_%d", i), Choice(fmt.Sprintf("What role does `%s` play in producing this HTTP response?", name), LayerCriteria), + func(a Answer) { answers[i].layer = Layer(a.Choice) }) + } + r.Add("primary", Choice("Which listed product is the main application this page belongs to "+ + "(the product whose login, console or content this is), as opposed to servers, frameworks and libraries underneath it?", primaryOpts), + func(a Answer) { + if a.Choice != NoneOfThem { + primary = a.Choice + } + }) + + r.Done(func() { + for i, p := range products { + a := answers[i] + rejected := !p.strong && a.answered && (a.layer == LayerNotPresent || a.present < Threshold) + if p.frames == nil { // recall: added only when Jev confirms it + if !a.answered || rejected { + continue + } + f := common.NewFramework(p.name, common.FrameFromGUESS) + f.AddTag(TagRecall) + frames.Add(f) + p.frames = []*common.Framework{f} + } + for j, f := range p.frames { + if a.layer != "" { + f.AddTag(TagLayer + string(a.layer)) + } + if rejected { + f.AddTag(TagRejected) + } + if j > 0 { + f.AddTag(TagDup) + } + } + if p.name == primary && !rejected { + p.frames[0].AddTag(TagPrimary) + } + } + }) +} diff --git a/jev/version.go b/jev/version.go new file mode 100644 index 0000000..9868147 --- /dev/null +++ b/jev/version.go @@ -0,0 +1,188 @@ +package jev + +import ( + "fmt" + "sort" + "strconv" + "strings" + + "github.com/chainreactors/fingers/common" +) + +// VersionConfidence is the confidence at or above which a version is written. +// Tuned for DefaultModel: 42 of 46 such answers were right on real pages. +var VersionConfidence = 0.9 + +const ( + NotStated = "not_stated" + + maxVersions = 40 // extracted from the raw response + maxVersionsShown = 15 // shown to Jev after ranking +) + +// Version adds to r the question which extracted version string is the +// version of f, and writes it to f when Jev is confident. It does nothing when +// f is nil, already has a version, or the page shows no version-like string. +// Jev cannot generate strings: code extracts, Jev only picks. +func Version(r *Round, f *common.Framework) { + if f == nil || (f.Attributes != nil && f.Attributes.Version != "") { + return + } + cands := rankVersions(extractVersions(r.page.raw, maxVersions), f.Name, maxVersionsShown) + if len(cands) == 0 { + return + } + // The candidates go into the state as a named list: Jev reads literally, + // and picked 100% right this way against 78% with them in the options. + opts := map[string]interface{}{NotStated: "The response does not show the version of " + f.Name} + for _, c := range cands { + opts[c.Value] = nil + } + r.Evidence("version_strings", cands) + r.Add("version", Choice(fmt.Sprintf("`version_strings` lists every version-like string found in the raw response, with the text around it. "+ + "Which one is the version of `%s`? A version in an asset URL parameter, a meta tag or embedded build info of %s counts. "+ + "Ignore versions of third-party libraries, other products mentioned in text, years, timestamps and build numbers.", f.Name, f.Name), opts), + func(a Answer) { + if a.Choice == "" || a.Choice == NotStated || a.Confidence < VersionConfidence { + return + } + // copy on write: Attributes may be shared with other frameworks + attrs := common.NewAttributesWithAny() + if f.Attributes != nil { + copied := *f.Attributes + attrs = &copied + } + attrs.Version = a.Choice + f.Attributes = attrs + }) +} + +// versionString is a version-like string with the raw text around its first +// occurrence, so Jev can tell a product version from a library version. The +// JSON names are what Jev reads. +type versionString struct { + Value string `json:"value"` + Context string `json:"found_in"` +} + +// extractVersions scans the raw response (headers and body, including +// inline scripts and meta tags that Page drops). Jev cannot generate +// strings, so versions are extracted by regex and Jev only picks one. +func extractVersions(raw []byte, max int) []versionString { + text := string(raw) + if i := strings.IndexByte(text, '\n'); i >= 0 { // skip the "HTTP/1.1 200" status line + text = text[i+1:] + } + seen := map[string]int{} // value -> index in out + var out []versionString + for _, loc := range reVersion.FindAllStringSubmatchIndex(text, -1) { + v := text[loc[2]:loc[3]] + if looksLikeIPv4(v) { + continue + } + start, end := loc[0]-60, loc[1]+30 + if start < 0 { + start = 0 + } + if end > len(text) { + end = len(text) + } + ctx := clean(strings.ToValidUTF8(text[start:end], "")) + // A value repeats across a page ("?ver=7.1" on every asset, then + // the generator meta); keep the context that best states a version. + if i, ok := seen[v]; ok { + if contextWeight(ctx) > contextWeight(out[i].Context) { + out[i].Context = ctx + } + continue + } + if len(out) >= max { + continue + } + seen[v] = len(out) + out = append(out, versionString{Value: v, Context: ctx}) + } + // "Drupal 10": a generator meta that states only a major version. + for _, m := range reGenMajor.FindAllStringSubmatchIndex(text, -1) { + lo, hi := m[2], m[3] + if lo < 0 { + lo, hi = m[4], m[5] + } + if v := text[lo:hi]; len(out) < max { + if _, ok := seen[v]; ok { + continue + } + seen[v] = len(out) + out = append(out, versionString{Value: v, Context: clean(strings.ToValidUTF8(text[m[0]:m[1]], ""))}) + } + } + return out +} + +// contextWeight prefers contexts that declare a version (generator meta, +// product headers) over asset URLs that merely carry one. +func contextWeight(ctx string) int { + c := strings.ToLower(ctx) + switch { + case strings.Contains(c, "generator"): + return 3 + case strings.Contains(c, "server:") || strings.Contains(c, "x-powered-by:") || strings.Contains(c, "product:"): + return 2 + case strings.Contains(c, "/plugins/") || strings.Contains(c, "/themes/"): + return 0 + } + return 1 +} + +// looksLikeIPv4 filters "10.0.0.5"; real four-part versions rarely start at 10+ +// with every part <= 255. +func looksLikeIPv4(v string) bool { + parts := strings.Split(v, ".") + if len(parts) != 4 { + return false + } + for i, p := range parts { + n, err := strconv.Atoi(p) + if err != nil || n > 255 || (i == 0 && n < 10) { + return false + } + } + return true +} + +// rankVersions orders candidates by how likely their context names +// the version of product, and keeps the first max. Pages often carry dozens +// of "?ver=" strings, so without ranking the one real version (a generator +// meta or a header) can fall outside the list Jev sees. +func rankVersions(cands []versionString, product string, max int) []versionString { + key := NormalizeName(product) + score := func(c versionString) int { + ctx := strings.ToLower(c.Context) + s := 0 + if strings.Contains(ctx, "generator") { + s += 4 + } + for _, h := range []string{"server:", "x-powered-by:", "product:", "x-generator:", "version:"} { + if strings.Contains(ctx, h) { + s += 3 + break + } + } + if key != "" && strings.Contains(NormalizeName(ctx), key) { + s += 2 + } + for _, noisy := range []string{"/plugins/", "/themes/", "/vendor/", "jquery", "bootstrap", "schema_version"} { + if strings.Contains(ctx, noisy) { + s -= 2 + break + } + } + return s + } + out := append([]versionString(nil), cands...) + sort.SliceStable(out, func(a, b int) bool { return score(out[a]) > score(out[b]) }) + if len(out) > max { + out = out[:max] + } + return out +} diff --git a/jev_test.go b/jev_test.go new file mode 100644 index 0000000..9dfe3aa --- /dev/null +++ b/jev_test.go @@ -0,0 +1,52 @@ +package fingers + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/chainreactors/fingers/jev" +) + +func TestRefineWithJev(t *testing.T) { + engine, err := NewEngine(FingersEngine, WappalyzerEngine) + if err != nil { + t.Fatal(err) + } + raw := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nWelcome to nginx!") + frames, _ := engine.DetectContent(raw) + if _, err := engine.Refine(context.Background(), raw, frames); err != ErrJevNotAttached { + t.Fatalf("want ErrJevNotAttached, got %v", err) + } + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req struct{ Questions map[string]jev.Question } + json.NewDecoder(r.Body).Decode(&req) + answers := map[string]jev.Answer{} + for k := range req.Questions { + switch { + case strings.HasPrefix(k, "layer_"): + answers[k] = jev.Answer{Choice: string(jev.LayerServer)} + case k == "page_kind": + answers[k] = jev.Answer{Choice: "default_install"} + default: + answers[k] = jev.Answer{Noul: 0.9, Choice: jev.NotStated} + } + } + json.NewEncoder(w).Encode(jev.Response{Answers: answers}) + })) + defer srv.Close() + client, _ := jev.NewClient("k") + client.Endpoint = srv.URL + engine.AttachJev(client) + page, err := engine.Refine(context.Background(), raw, frames) + if err != nil { + t.Fatal(err) + } + if page.Kind != "default_install" || !frames["nginx"].HasTag(jev.TagLayer+string(jev.LayerServer)) { + t.Fatalf("kind %q, nginx %v", page.Kind, frames["nginx"].Tags) + } +} From 2d296928c8847bbc60dbfedad77e68bbfea00d14 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Thu, 24 Sep 2026 22:19:46 -0700 Subject: [PATCH 02/16] refactor(judge): make the judgement layer provider-neutral Jev is one provider of a generic judgement and rerank layer, not the layer itself. - package judge: Provider interface (ID + Judge), Judge (cache, request merging, thresholds, optional provider Calibration), neutral question types Binary / Choice / Score - package judge/jev: the TypeSafe Jev provider, protocol and transport only - marks are one enum (Rejected, Duplicate, Primary, Recalled) read with judge.Is / LayerOf / Accepted, stored as judge:* tags; page Kind is an enum - cache is on by default and per question: any grouping of questions hits, partial hits send only the rest, concurrent rounds on similar pages merge - similar pages must share a title and differ by <= 1 signature bit; text is shingled by two runes. On 1070 real pages every answer reused at distance <= 1 matched (159/159), 96.9% at the old default of 3, and unrelated sparse CJK pages collided without the title guard - Verify skips judged frames, so Refine on the same frames is free - version subject without a primary: the only application, else server, else device (fixes directory listings next to an OS hit) - Engine.AttachJudge; cmd/judgeeval -provider; cmd/engine --judge jev - docs: judge/README.md Refs #34 Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 417 ++++++++---------- cmd/engine/example.go | 32 +- cmd/jevbench/README.md | 25 -- cmd/judgeeval/README.md | 25 ++ cmd/{jevbench => judgeeval}/main.go | 282 ++++++------ .../testdata/labels.json | 0 .../samples/blog_mentions_jenkins.http | 0 .../testdata/samples/confluence_login.http | 0 .../testdata/samples/corporate_cn_portal.http | 0 .../testdata/samples/corporate_site.http | 0 .../samples/custom_employee_portal.http | 0 .../testdata/samples/dir_listing.http | 0 .../testdata/samples/discuz_forum.http | 0 .../testdata/samples/docs_tomcat_howto.http | 0 .../testdata/samples/gitlab_signin.http | 0 .../testdata/samples/grafana_login.http | 0 .../testdata/samples/harbor_portal.http | 0 .../testdata/samples/iis_default.http | 0 .../testdata/samples/jenkins_login.http | 0 .../testdata/samples/job_posting_cn.http | 0 .../testdata/samples/json_api.http | 0 .../testdata/samples/kibana_home.http | 0 .../testdata/samples/minio_console.http | 0 .../testdata/samples/nacos_console.http | 0 .../testdata/samples/news_fortios_patch.http | 0 .../testdata/samples/nginx_404.http | 0 .../testdata/samples/nginx_default.http | 0 .../testdata/samples/pgadmin_login.http | 0 .../testdata/samples/phpmyadmin_login.http | 0 .../testdata/samples/portainer_login.http | 0 .../testdata/samples/rabbitmq_mgmt.http | 0 .../testdata/samples/rebranded_grafana.http | 0 .../testdata/samples/rebranded_jenkins.http | 0 .../testdata/samples/rebranded_zabbix.http | 0 .../testdata/samples/seeyon_login.http | 0 .../testdata/samples/shop_page.http | 0 .../testdata/samples/springboot_error.http | 0 .../samples/status_page_mentions.http | 0 .../testdata/samples/synology_dsm.http | 0 .../testdata/samples/tomcat_default.http | 0 .../testdata/samples/tplink_router.http | 0 .../testdata/samples/unknown_admin_login.http | 0 .../samples/weaver_ecology_login.http | 0 .../testdata/samples/wordpress_home.http | 0 .../testdata/samples/wordpress_login.http | 0 .../testdata/samples/zabbix_login.http | 0 engine.go | 8 +- jev.go | 40 -- jev/classify.go | 30 -- jev/client.go | 200 --------- jev/refine.go | 65 --- jev_test.go | 52 --- judge.go | 43 ++ judge/README.md | 141 ++++++ {jev => judge}/cache.go | 27 +- judge/classify.go | 38 ++ judge/doc.go | 23 + judge/jev/jev.go | 194 ++++++++ judge/jev/jev_test.go | 53 +++ judge/judge.go | 186 ++++++++ jev/jev_test.go => judge/judge_test.go | 281 ++++++++---- judge/mark.go | 77 ++++ {jev => judge}/page.go | 45 +- judge/provider.go | 70 +++ {jev => judge}/recall.go | 6 +- judge/refine.go | 47 ++ {jev => judge}/round.go | 27 +- {jev => judge}/stack.go | 6 +- {jev => judge}/verify.go | 84 ++-- {jev => judge}/version.go | 34 +- judge_test.go | 50 +++ 71 files changed, 1637 insertions(+), 971 deletions(-) delete mode 100644 cmd/jevbench/README.md create mode 100644 cmd/judgeeval/README.md rename cmd/{jevbench => judgeeval}/main.go (64%) rename cmd/{jevbench => judgeeval}/testdata/labels.json (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/blog_mentions_jenkins.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/confluence_login.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/corporate_cn_portal.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/corporate_site.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/custom_employee_portal.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/dir_listing.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/discuz_forum.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/docs_tomcat_howto.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/gitlab_signin.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/grafana_login.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/harbor_portal.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/iis_default.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/jenkins_login.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/job_posting_cn.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/json_api.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/kibana_home.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/minio_console.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/nacos_console.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/news_fortios_patch.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/nginx_404.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/nginx_default.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/pgadmin_login.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/phpmyadmin_login.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/portainer_login.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/rabbitmq_mgmt.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/rebranded_grafana.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/rebranded_jenkins.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/rebranded_zabbix.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/seeyon_login.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/shop_page.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/springboot_error.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/status_page_mentions.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/synology_dsm.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/tomcat_default.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/tplink_router.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/unknown_admin_login.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/weaver_ecology_login.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/wordpress_home.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/wordpress_login.http (100%) rename cmd/{jevbench => judgeeval}/testdata/samples/zabbix_login.http (100%) delete mode 100644 jev.go delete mode 100644 jev/classify.go delete mode 100644 jev/client.go delete mode 100644 jev/refine.go delete mode 100644 jev_test.go create mode 100644 judge.go create mode 100644 judge/README.md rename {jev => judge}/cache.go (66%) create mode 100644 judge/classify.go create mode 100644 judge/doc.go create mode 100644 judge/jev/jev.go create mode 100644 judge/jev/jev_test.go create mode 100644 judge/judge.go rename jev/jev_test.go => judge/judge_test.go (58%) create mode 100644 judge/mark.go rename {jev => judge}/page.go (85%) create mode 100644 judge/provider.go rename {jev => judge}/recall.go (91%) create mode 100644 judge/refine.go rename {jev => judge}/round.go (66%) rename {jev => judge}/stack.go (95%) rename {jev => judge}/verify.go (54%) rename {jev => judge}/version.go (82%) create mode 100644 judge_test.go diff --git a/README.md b/README.md index 1300cf3..9f74ad4 100644 --- a/README.md +++ b/README.md @@ -1,221 +1,196 @@ - -## Introduce - -多指纹库聚合识别引擎. 当前支持`fingers(主指纹库)` `wappalyzer`, `fingerprinthub`, `ehole`, `goby` 指纹 - -不用再挑选指纹识别的工具, AllInOne一站式实现 - -使用了fingers的工具: - -* ⭐ [spray](https://github.com/chainreactors/spray) **最佳实践**, 集合了目录爆破, 指纹识别, 信息收集等等功能的超强性能的http fuzz工具 -* [gogo](https://github.com/chainreactors/gogo), 使用了fingers原生指纹库, 红队向的自动化扫描引擎 -* [zombie](https://github.com/chainreactors/zombie), 在爆破前使用fingers进行指纹验证, 提高爆破效率 - -(任何使用了fingers的工具欢迎在issue中告诉我, 我会将你的工具添加到这里) - -## Features - -* 支持多指纹库聚合识别 - * ✅ fingers 原生指纹库 - * ✅ [wappalyzer](https://github.com/projectdiscovery/wappalyzergo) - * ✅ [fingerprinthub](https://github.com/0x727/FingerprintHub) - * ✅ [ehole](https://github.com/EdgeSecurityTeam/EHole) - * ✅ goby -* 支持多指纹源favicon识别 -* 超强性能, 单个站点识别 <100ms. 重写了各指纹库的引擎, 并极大优化了性能 -* 聚合输出, 多指纹库的结果将会自动整合 -* 支持CPE的URI, FSB, WFN格式输出 - -### morefingers - -https://github.com/chainreactors/morefingers - -fingers的拓展引擎, 有更全更大的指纹库. - -从对闭源工具的逆向得到的指纹库, 为了避免可能存在的纠纷, 不提供开源版本. - -## QuickStart - -`go get github.com/chainreactors/fingers@master` - -### Example - -document: https://chainreactors.github.io/wiki/libs/fingers/ - -调用内置所有进行指纹引擎识别, 示例: - -```golang -func TestEngine(t *testing.T) { - engine, err := NewEngine() - if err != nil { - panic(err) - } - resp, err := http.Get("http://127.0.0.1:8080/") - if err != nil { - return - } - content := httputils.ReadRaw(resp) - frames, err := engine.DetectContent(content) - if err != nil { - return - } - fmt.Println(frames.String()) -} -``` - -调用SDK识别Favicon指纹, 示例: - -```golang -func TestFavicon(t *testing.T) { - engine, err := NewEngine() - if err != nil { - panic(err) - } - resp, err := http.Get("http://baidu.com/favicon.ico") - if err != nil { - return - } - content := httputils.ReadRaw(resp) - body, _, _ := httputils.SplitHttpRaw(content) - frame := engine.DetectFavicon(body) - fmt.Println(frame.String()) -} -``` - -更多用法请见: https://chainreactors.github.io/wiki/libs/fingers/sdk/ - -## Jev 判定层 - -规则引擎召回能力强，但会有误报、多个引擎的同名重复，也分不清主应用和底层组件，版本号更是难以拿准。`jev` 包在规则结果之上接入 [TypeSafe Jev](https://docs.typesafe.ai/api.md)，做一层判定：由规则和代码负责召回，Jev 只对证据做判断，最终决策由代码做出。它不是新的引擎，`WebMatch` 和 `DetectContent` 的行为保持不变。设计讨论见 [#34](https://github.com/chainreactors/fingers/issues/34)。 - -```golang -client, _ := jev.NewClient("") // 读取 TYPESAFE_API_KEY -client.Cache = jev.NewMemoryCache(10000) // 可选，按请求内容缓存 -engine.AttachJev(client) - -frames, _ := engine.DetectContent(raw) // 同步，纯规则 -page, err := engine.Refine(ctx, raw, frames) // 通常异步执行；原地标注 frames -if err == nil { - fmt.Println(page.Kind, jev.Accepted(frames)) // 页面类型；去掉误报和重复后的结果 -} -``` - -`Refine` 在 frames 上原地打标签，不删除任何条目： - -| 标签 | 含义 | -|---|---| -| `jev:rejected` | 误报：不在技术栈里，或者只是正文里提到。名字出现在 header/cookie 中的命中永远不会被否决 | -| `jev:dup` | 另一个引擎对同一产品的另一种写法 | -| `jev:layer=` | 所属层级：`cdn_or_waf`、`web_server`、`language_runtime`、`web_framework`、`application`、`frontend_library`、`os_or_device` | -| `jev:primary` | 本页所属的主应用。主应用没有版本号时，由 Jev 从页面上的版本字符串中选出，置信度 ≥0.9 才写入 | -| `jev:recall` | 漏报：规则没有命中，但指纹名在页面中出现且经 Jev 确认，以 `guess` 来源加入 | - -每页最多 2 次请求。请求失败时 frames 保持规则结果不变。 - -各项能力都是可以单独使用的函数，组合在一个 `Round`（即一次请求）里： - -```golang -page, _ := jev.NewPage(raw) -r := page.Round() -jev.Classify(r) // 只做页面类型 / 是否通用页面，例如用于 404 识别 -r.Add("honeypot", jev.Noul("Is this a honeypot?"), func(a jev.Answer) { /* 自定义问题 */ }) -err := r.Ask(ctx, client) - -r = page.Round() -jev.Version(r, frame) // 只给某个已确定的产品补版本号 -err = r.Ask(ctx, client) -``` - -边界：改过品牌的产品 Jev 认不出来，召回仍然靠规则；阈值针对 `jev.DefaultModel` 调过，换模型后要用 `cmd/jevbench` 重新评测。 - -## fingers 引擎 - -fingers指纹引擎是目前特性最丰富, 性能最强的指纹规则库. - -* 支持多种方式规则配置 -* 支持多种方式的版本号匹配 -* 404/favicon/waf/cdn/供应链指纹识别 -* 主动指纹识别 -* 超强性能, 采用了缓存,正则预编译,默认端口,优先级等等算法提高引擎性能 -* 重点指纹,指纹来源与tag标记 - - -### 内置指纹库 - -指纹库位于: https://github.com/chainreactors/templates/tree/master/fingers - -文档: https://chainreactors.github.io/wiki/libs/fingers/rule/ - -tcp指纹与http指纹为同一格式, 但通过不同的文件进行管理 - -### 完整的配置 - -fingers设计的核心思路是命中一个指纹仅需要一条规则, 因此配置的多条规则中, 只需要任意一条命中即标记为命中, 需要在编写指纹的时候注意找到最能匹配目标框架的那条规则. - -一个完整的配置: - -```yaml -- name: frame # 指纹名字, 匹配到的时候输出的值 - default_port: # 指纹的默认端口, 加速匹配. tcp指纹如果匹配到第一个就会结束指纹匹配, http则会继续匹配, 所以默认端口对http没有特殊优化 - - '1111' - protocol: http # tcp/http, 默认为http - rule: - - version: v1.1.1 # 可不填, 默认为空, 表示无具体版本 - regexps: # 匹配的方式 - vuln: # 匹配到vuln的正则, 如果匹配到, 会输出framework为name的同时, 还会添加vuln为vuln的漏洞信息 - - version:(.*) # vuln只支持正则, 同时支持版本号匹配, 使用括号的正则分组. 只支持第一组 - regexp: # 匹配指纹正则 - - "finger.*test" - # 除了正则, 还支持其他类型的匹配, 包括以下方式 - header: # 仅http协议可用, 匹配header中包含的数据 - - string - body: # 包含匹配, 非正则表达式 - - string - md5: # 匹配body的md5hash - - [md5] - mmh3: # 匹配body的mmh3hash - - [mmh3] - - # 只有上面规则中的至少一条命中才会执行version - version: - - version:(.*) # 某些情况下难以同时编写指纹的正则与关于版本的正则, 可以特地为version写一条正则 - - favicon: # favicon的hash值, 仅http生效 - md5: - - f7e3d97f404e71d302b3239eef48d5f2 - mmh3: - - '516963061' - level: 1 # 0代表不需要主动发包, 1代表需要额外主动发起请求. 如果当前level为0则不会发送数据, 但是依旧会进行被动的指纹匹配. - send_data: "info\n" # 匹配指纹需要主动发送的数据 - vuln: frame_unauthorized # 如果regexps中的vuln命中, 则会输出漏洞名称. 某些漏洞也可以通过匹配关键字识别, 因此一些简单的poc使用指纹的方式实现, 复杂的poc请使用-e下的nuclei yaml配置 - -``` - -为了压缩体积, 没有特别指定的参数可以留空会使用默认值。 - -在两个配置文件中包含大量案例可供参考。 - -但实际上大部分字段都不需要配置, 仅作为特殊情况下的能力储备。 - -每个指纹都可以有多个rule, 每个rule中都有一个regexps, 每个regexps有多条不同种类的字符串/正则/hash - - -## TODO - -- [x] 指纹名重定向, 统一多指纹库的同一指纹不同名问题 -- [x] 指纹黑名单, 用于过滤指纹库中的垃圾指纹 -- [x] 更丰富的CPE相关特性支持 -- [ ] 更优雅的与nuclei或其他漏洞库联动 -- 支持更多引擎 - - [ ] [nuclei technologies](https://github.com/projectdiscovery/nuclei-templates/tree/main/http/technologies) 实现 - - [ ] fingerprinthub v4 - - [ ] tidefinger - - [ ] kscan - - [ ] nmap - -## Thanks - -* [wappalyzer](https://github.com/projectdiscovery/wappalyzergo) -* [fingerprinthub](https://github.com/0x727/FingerprintHub) -* [ehole](https://github.com/EdgeSecurityTeam/EHole) -* goby @XiaoliChan @9bie + +## Introduce + +多指纹库聚合识别引擎. 当前支持`fingers(主指纹库)` `wappalyzer`, `fingerprinthub`, `ehole`, `goby` 指纹 + +不用再挑选指纹识别的工具, AllInOne一站式实现 + +使用了fingers的工具: + +* ⭐ [spray](https://github.com/chainreactors/spray) **最佳实践**, 集合了目录爆破, 指纹识别, 信息收集等等功能的超强性能的http fuzz工具 +* [gogo](https://github.com/chainreactors/gogo), 使用了fingers原生指纹库, 红队向的自动化扫描引擎 +* [zombie](https://github.com/chainreactors/zombie), 在爆破前使用fingers进行指纹验证, 提高爆破效率 + +(任何使用了fingers的工具欢迎在issue中告诉我, 我会将你的工具添加到这里) + +## Features + +* 支持多指纹库聚合识别 + * ✅ fingers 原生指纹库 + * ✅ [wappalyzer](https://github.com/projectdiscovery/wappalyzergo) + * ✅ [fingerprinthub](https://github.com/0x727/FingerprintHub) + * ✅ [ehole](https://github.com/EdgeSecurityTeam/EHole) + * ✅ goby +* 支持多指纹源favicon识别 +* 超强性能, 单个站点识别 <100ms. 重写了各指纹库的引擎, 并极大优化了性能 +* 聚合输出, 多指纹库的结果将会自动整合 +* 支持CPE的URI, FSB, WFN格式输出 + +### morefingers + +https://github.com/chainreactors/morefingers + +fingers的拓展引擎, 有更全更大的指纹库. + +从对闭源工具的逆向得到的指纹库, 为了避免可能存在的纠纷, 不提供开源版本. + +## QuickStart + +`go get github.com/chainreactors/fingers@master` + +### Example + +document: https://chainreactors.github.io/wiki/libs/fingers/ + +调用内置所有进行指纹引擎识别, 示例: + +```golang +func TestEngine(t *testing.T) { + engine, err := NewEngine() + if err != nil { + panic(err) + } + resp, err := http.Get("http://127.0.0.1:8080/") + if err != nil { + return + } + content := httputils.ReadRaw(resp) + frames, err := engine.DetectContent(content) + if err != nil { + return + } + fmt.Println(frames.String()) +} +``` + +调用SDK识别Favicon指纹, 示例: + +```golang +func TestFavicon(t *testing.T) { + engine, err := NewEngine() + if err != nil { + panic(err) + } + resp, err := http.Get("http://baidu.com/favicon.ico") + if err != nil { + return + } + content := httputils.ReadRaw(resp) + body, _, _ := httputils.SplitHttpRaw(content) + frame := engine.DetectFavicon(body) + fmt.Println(frame.String()) +} +``` + +更多用法请见: https://chainreactors.github.io/wiki/libs/fingers/sdk/ + +## 判定层 judge + +规则引擎召回能力强，但会有误报、多个引擎的同名重复，也分不清主应用和底层组件，版本号更是难以拿准。`judge` 包在规则结果之上加了一层判定（判别 + rerank）：规则和代码负责召回，Provider（模型）只对证据做判断，最终决策由代码做出。它不是新的引擎，`WebMatch` 和 `DetectContent` 的行为保持不变。Provider 可以替换，目前内置 [TypeSafe Jev](https://docs.typesafe.ai/api.md)（`judge/jev`）。 + +```golang +j, _ := jev.NewJudge("") // 读取 TYPESAFE_API_KEY；自带缓存和相似页面去重 +engine.AttachJudge(j) + +frames, _ := engine.DetectContent(raw) // 同步，纯规则 +page, err := engine.Refine(ctx, raw, frames) // 通常异步执行；原地标注 frames，失败时保持规则结果 +if err == nil { + fmt.Println(page.Kind, judge.Accepted(frames)) // 页面类型；去掉误报和重复后的结果 +} +``` + +在 1070 个真实页面上：每页结果从 13.4 条降到 6.5 条，同名重复全部归并，版本号对照 generator 从 43 个正确提升到 191 个，并额外召回了 468 个规则漏掉的产品。成本约每万页 $3.4。 + +完整的能力、标注方式（`judge.Is(f, judge.Rejected)` 等）、缓存、自定义问题以及如何接入新的 Provider，见 [judge/README.md](judge/README.md)；设计讨论见 [#34](https://github.com/chainreactors/fingers/issues/34)。 + +## fingers 引擎 + +fingers指纹引擎是目前特性最丰富, 性能最强的指纹规则库. + +* 支持多种方式规则配置 +* 支持多种方式的版本号匹配 +* 404/favicon/waf/cdn/供应链指纹识别 +* 主动指纹识别 +* 超强性能, 采用了缓存,正则预编译,默认端口,优先级等等算法提高引擎性能 +* 重点指纹,指纹来源与tag标记 + + +### 内置指纹库 + +指纹库位于: https://github.com/chainreactors/templates/tree/master/fingers + +文档: https://chainreactors.github.io/wiki/libs/fingers/rule/ + +tcp指纹与http指纹为同一格式, 但通过不同的文件进行管理 + +### 完整的配置 + +fingers设计的核心思路是命中一个指纹仅需要一条规则, 因此配置的多条规则中, 只需要任意一条命中即标记为命中, 需要在编写指纹的时候注意找到最能匹配目标框架的那条规则. + +一个完整的配置: + +```yaml +- name: frame # 指纹名字, 匹配到的时候输出的值 + default_port: # 指纹的默认端口, 加速匹配. tcp指纹如果匹配到第一个就会结束指纹匹配, http则会继续匹配, 所以默认端口对http没有特殊优化 + - '1111' + protocol: http # tcp/http, 默认为http + rule: + - version: v1.1.1 # 可不填, 默认为空, 表示无具体版本 + regexps: # 匹配的方式 + vuln: # 匹配到vuln的正则, 如果匹配到, 会输出framework为name的同时, 还会添加vuln为vuln的漏洞信息 + - version:(.*) # vuln只支持正则, 同时支持版本号匹配, 使用括号的正则分组. 只支持第一组 + regexp: # 匹配指纹正则 + - "finger.*test" + # 除了正则, 还支持其他类型的匹配, 包括以下方式 + header: # 仅http协议可用, 匹配header中包含的数据 + - string + body: # 包含匹配, 非正则表达式 + - string + md5: # 匹配body的md5hash + - [md5] + mmh3: # 匹配body的mmh3hash + - [mmh3] + + # 只有上面规则中的至少一条命中才会执行version + version: + - version:(.*) # 某些情况下难以同时编写指纹的正则与关于版本的正则, 可以特地为version写一条正则 + + favicon: # favicon的hash值, 仅http生效 + md5: + - f7e3d97f404e71d302b3239eef48d5f2 + mmh3: + - '516963061' + level: 1 # 0代表不需要主动发包, 1代表需要额外主动发起请求. 如果当前level为0则不会发送数据, 但是依旧会进行被动的指纹匹配. + send_data: "info\n" # 匹配指纹需要主动发送的数据 + vuln: frame_unauthorized # 如果regexps中的vuln命中, 则会输出漏洞名称. 某些漏洞也可以通过匹配关键字识别, 因此一些简单的poc使用指纹的方式实现, 复杂的poc请使用-e下的nuclei yaml配置 + +``` + +为了压缩体积, 没有特别指定的参数可以留空会使用默认值。 + +在两个配置文件中包含大量案例可供参考。 + +但实际上大部分字段都不需要配置, 仅作为特殊情况下的能力储备。 + +每个指纹都可以有多个rule, 每个rule中都有一个regexps, 每个regexps有多条不同种类的字符串/正则/hash + + +## TODO + +- [x] 指纹名重定向, 统一多指纹库的同一指纹不同名问题 +- [x] 指纹黑名单, 用于过滤指纹库中的垃圾指纹 +- [x] 更丰富的CPE相关特性支持 +- [ ] 更优雅的与nuclei或其他漏洞库联动 +- 支持更多引擎 + - [ ] [nuclei technologies](https://github.com/projectdiscovery/nuclei-templates/tree/main/http/technologies) 实现 + - [ ] fingerprinthub v4 + - [ ] tidefinger + - [ ] kscan + - [ ] nmap + +## Thanks + +* [wappalyzer](https://github.com/projectdiscovery/wappalyzergo) +* [fingerprinthub](https://github.com/0x727/FingerprintHub) +* [ehole](https://github.com/EdgeSecurityTeam/EHole) +* goby @XiaoliChan @9bie diff --git a/cmd/engine/example.go b/cmd/engine/example.go index 48c6771..b4c4a58 100644 --- a/cmd/engine/example.go +++ b/cmd/engine/example.go @@ -14,7 +14,8 @@ import ( "github.com/chainreactors/fingers" "github.com/chainreactors/fingers/common" - "github.com/chainreactors/fingers/jev" + "github.com/chainreactors/fingers/judge" + "github.com/chainreactors/fingers/judge/jev" "github.com/chainreactors/fingers/resources" "github.com/chainreactors/utils/encode" "github.com/chainreactors/utils/httputils" @@ -38,8 +39,8 @@ var opts struct { // 是否只检测favicon FaviconOnly bool `short:"f" long:"favicon" description:"Only detect favicon"` - // 在规则结果之上运行 Jev 判定层, 需要 TYPESAFE_API_KEY - Jev bool `long:"jev" description:"Judge the rule result with Jev (needs TYPESAFE_API_KEY)"` + // 在规则结果之上运行判定层, 目前支持的 provider: jev (需要 TYPESAFE_API_KEY) + Judge string `long:"judge" description:"Judge the rule result with a provider: jev (needs TYPESAFE_API_KEY)"` // 资源文件覆盖 GobyFile string `long:"goby" description:"Override goby.json.gz with custom file"` @@ -203,28 +204,41 @@ func main() { } } else { var content []byte - if opts.Jev { // Refine needs the raw response; put the body back for Match + if opts.Judge != "" { // Refine needs the raw response; put the body back for Match content = httputils.ReadRaw(resp) body, _, _ := httputils.SplitHttpRaw(content) resp.Body = ioutil.NopCloser(bytes.NewReader(body)) } frames = engine.Match(resp) - if opts.Jev { - client, err := jev.NewClient("") + if opts.Judge != "" { + if opts.Judge != "jev" { + fmt.Printf("unknown judge provider %q\n", opts.Judge) + os.Exit(1) + } + j, err := jev.NewJudge("") if err != nil { fmt.Println(err) os.Exit(1) } - engine.AttachJev(client) + engine.AttachJudge(j) page, err := engine.Refine(context.Background(), content, frames) if err != nil { fmt.Printf("jev failed, showing the rule result: %v\n", err) } else { fmt.Printf("page: %s, generic: %v\n", page.Kind, page.Generic) for _, frame := range frames { - fmt.Printf(" %-40s %-10s %v\n", frame.Name, frame.Version, frame.Tags) + verdict := "accepted" + for _, m := range []judge.Mark{judge.Rejected, judge.Duplicate} { + if judge.Is(frame, m) { + verdict = string(m) + } + } + if judge.Is(frame, judge.Primary) { + verdict += ",primary" + } + fmt.Printf(" %-40s %-10s %-18s %s\n", frame.Name, frame.Version, judge.LayerOf(frame), verdict) } - fmt.Printf("accepted: %s\n", jev.Accepted(frames).String()) + fmt.Printf("accepted: %s\n", judge.Accepted(frames).String()) return } } diff --git a/cmd/jevbench/README.md b/cmd/jevbench/README.md deleted file mode 100644 index 9e66779..0000000 --- a/cmd/jevbench/README.md +++ /dev/null @@ -1,25 +0,0 @@ -# jevbench：纯规则 vs 规则 + Jev - -在一批原始 HTTP 响应上，用和 SDK 调用方完全相同的路径（`engine.DetectContent` → `engine.Refine`）跑一遍，逐项对比纯规则结果和经过 Jev 判定之后的结果。 - -```bash -go build -buildvcs=false -tags goregexp -o jevbench ./cmd/jevbench - -# 标注集（仓库自带 40 页，含真值） -TYPESAFE_API_KEY=... ./jevbench -samples testdata/samples -labels testdata/labels.json - -# 任意真实页面目录（*.http，完整的原始响应） -TYPESAFE_API_KEY=... ./jevbench -samples cc/samples -cache jevcache -out jevreport -rps 15 -``` - -- Jev 的回答按请求内容缓存在 `-cache` 目录（实现了 `jev.Cache` 的文件缓存），重跑不花钱，中断后可以接着跑。 -- `jevreport/report.md`：对比表，另附页面类型分布、各引擎的否决率、否决率最高的规则（待修的规则），以及新指纹候选。 -- `jevreport/rows.jsonl`：每页一行，列出规则命中、被接受的、被否决的、召回的、层级、主应用和版本号，用于人工复核。 -- 给了 `-labels` 时，额外按真值统计误报剔除、真实命中保留、页面类型、通用页面和版本号的正确率。 - -没有真值的数据集用这两个代理指标： - -- **仅正文出现的命中**：产品名只出现在可见正文里、不在任何结构字段中，是由代码判断出的疑似误报； -- **版本号**：以页面的 generator meta 作为参考答案。 - -真实网站的页面属于第三方内容，不要提交进仓库。 diff --git a/cmd/judgeeval/README.md b/cmd/judgeeval/README.md new file mode 100644 index 0000000..4b6cbdd --- /dev/null +++ b/cmd/judgeeval/README.md @@ -0,0 +1,25 @@ +# judgeeval：纯规则 vs 规则 + judge + +在一批原始 HTTP 响应上，用和 SDK 调用方完全相同的路径（`engine.DetectContent` → `engine.Refine`）跑一遍，逐项对比纯规则结果和经过判定之后的结果。它也用来校准新的 Provider。 + +```bash +go build -buildvcs=false -tags goregexp -o judgeeval ./cmd/judgeeval + +# 标注集（仓库自带 40 页合成样本，含真值） +TYPESAFE_API_KEY=... ./judgeeval -provider jev -samples testdata/samples -labels testdata/labels.json + +# 任意真实页面目录（*.http，完整的原始响应） +TYPESAFE_API_KEY=... ./judgeeval -provider jev -samples cc/samples -cache judgecache -out judgereport -rps 15 +``` + +- 答案缓存在 `-cache` 目录，重跑不花钱，中断后可以接着跑。这个目录用的是精确缓存，另外会模拟相似度缓存：按签名距离分档，统计每一档能复用多少答案、与真实答案的一致率，用来确定 `judge.SimilarDistance` 的取值。 +- `report.md`：对比表，另附页面类型分布、各引擎的否决率、否决率最高的规则（需要修复），以及新指纹候选。 +- `rows.jsonl`：每页一行，用于人工复核。 +- 给了 `-labels` 时，额外按真值统计误报剔除、真实命中保留、页面类型、通用页面和版本号的正确率。 +- 新 Provider：在 `main.go` 的 `newProvider` 里注册。 + +没有真值的数据集用两个代理指标： +- **仅正文出现的命中**：代码判断出的疑似误报； +- **版本号**：以页面上的 generator meta 作为参考答案。 + +真实网站的页面属于第三方内容，不要提交进仓库。 diff --git a/cmd/jevbench/main.go b/cmd/judgeeval/main.go similarity index 64% rename from cmd/jevbench/main.go rename to cmd/judgeeval/main.go index a4e842c..bd870fe 100644 --- a/cmd/jevbench/main.go +++ b/cmd/judgeeval/main.go @@ -1,11 +1,12 @@ -// jevbench measures the Jev judgement layer, exactly as SDK callers run it -// (fingers.Engine.Refine), on a directory of raw HTTP responses. +// judgeeval compares the rule engines alone with rules plus a judge provider, +// exactly as SDK callers run it (fingers.Engine.Refine), on a directory of +// raw HTTP responses; it is also how a new provider is calibrated. // -// TYPESAFE_API_KEY=... jevbench -samples testdata/samples -labels testdata/labels.json -// TYPESAFE_API_KEY=... jevbench -samples cclogin/samples -cache cache -out report +// TYPESAFE_API_KEY=... judgeeval -provider jev -samples testdata/samples -labels testdata/labels.json +// TYPESAFE_API_KEY=... judgeeval -provider jev -samples cc/samples -cache cache -out report // -// Jev answers are cached on disk by request content, so reruns are free and -// an interrupted run resumes. Build with -tags goregexp. +// Answers are cached on disk, so reruns are free and an interrupted run +// resumes. Build with -tags goregexp. package main import ( @@ -14,6 +15,7 @@ import ( "errors" "flag" "fmt" + "math/bits" "net/http" "os" "path/filepath" @@ -21,40 +23,42 @@ import ( "sort" "strings" "sync" + "sync/atomic" "time" "unicode" "github.com/chainreactors/fingers" "github.com/chainreactors/fingers/common" - "github.com/chainreactors/fingers/jev" + "github.com/chainreactors/fingers/judge" + "github.com/chainreactors/fingers/judge/jev" "github.com/chainreactors/logs" ) // row is the outcome for one page, written to rows.jsonl for review. type row struct { - ID string `json:"id"` - Rules []string `json:"rules"` // name@engine of every rule hit - Accepted []string `json:"accepted"` - Rejected []string `json:"rejected,omitempty"` - Recall []string `json:"recall,omitempty"` - Layers map[string]string `json:"layers,omitempty"` - Primary string `json:"primary,omitempty"` - Version string `json:"version,omitempty"` - JevVersion bool `json:"jev_version,omitempty"` // Version written by Jev, not by a rule - Kind string `json:"kind"` - Generic bool `json:"generic"` - NewFinger bool `json:"new_fingerprint_candidate,omitempty"` - Err string `json:"error,omitempty"` - - // rules vs rules+Jev on the same page + ID string `json:"id"` + Rules []string `json:"rules"` // name@engine of every rule hit + Accepted []string `json:"accepted"` + Rejected []string `json:"rejected,omitempty"` + Recall []string `json:"recall,omitempty"` + Layers map[string]string `json:"layers,omitempty"` + Primary string `json:"primary,omitempty"` + Version string `json:"version,omitempty"` + JudgeVersion bool `json:"judge_version,omitempty"` // Version written by the judge, not by a rule + Kind judge.Kind `json:"kind"` + Generic bool `json:"generic"` + NewFinger bool `json:"new_fingerprint_candidate,omitempty"` + Err string `json:"error,omitempty"` + + // rules vs rules+judge on the same page RuleProducts int `json:"rule_products"` // rule hits after folding spellings TextOnly []string `json:"text_only,omitempty"` // rule hits named only in visible text: likely false positives - TextOnlyKept int `json:"text_only_kept"` // of those, accepted by Jev + TextOnlyKept int `json:"text_only_kept"` // of those, accepted by the judge Generator string `json:"generator,omitempty"` // "wordpress 6.4.2": the reference version RuleGenVer string `json:"rule_gen_version,omitempty"` - JevGenVer string `json:"jev_gen_version,omitempty"` + JudgeGenVer string `json:"judge_gen_version,omitempty"` RuleMs float64 `json:"rule_ms"` - JevMs float64 `json:"jev_ms"` + JudgeMs float64 `json:"judge_ms"` frames common.Frameworks // annotated ruleEngines map[string][]string // rule name -> engines @@ -64,21 +68,40 @@ type row struct { func main() { samples := flag.String("samples", "testdata/samples", "directory of raw HTTP responses (*.http)") labels := flag.String("labels", "", "optional ground truth (see testdata/labels.json)") - cache := flag.String("cache", "jevcache", "directory caching Jev answers") - out := flag.String("out", "jevreport", "output directory") - rps := flag.Float64("rps", 15, "max Jev requests per second") + provider := flag.String("provider", "jev", "judge provider: jev") + cache := flag.String("cache", "judgecache", "directory caching answers") + out := flag.String("out", "judgereport", "output directory") + rps := flag.Float64("rps", 15, "max provider requests per second") workers := flag.Int("workers", 16, "concurrent pages") limit := flag.Int("limit", 0, "only the first N samples (0 = all)") - endpoint := flag.String("endpoint", jev.DefaultEndpoint, "Jev API endpoint") + endpoint := flag.String("endpoint", "", "provider API endpoint (default: the provider's)") flag.Parse() logs.Log.SetLevel(logs.ErrorLevel) - if err := run(*samples, *labels, *cache, *out, *endpoint, *rps, *workers, *limit); err != nil { + if err := run(*provider, *samples, *labels, *cache, *out, *endpoint, *rps, *workers, *limit); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } } -func run(samples, labelsPath, cacheDir, out, endpoint string, rps float64, workers, limit int) error { +// newProvider builds a provider by name; add new providers here. tokens +// reports input tokens sent, where the provider counts them. +func newProvider(name, endpoint string, transport http.RoundTripper) (p judge.Provider, tokens func() int64, err error) { + switch name { + case "jev": + jp, err := jev.New("") + if err != nil { + return nil, nil, err + } + if endpoint != "" { + jp.Endpoint = endpoint + } + jp.HTTP.Transport = transport + return jp, func() int64 { return atomic.LoadInt64(&jp.InputTokens) }, nil + } + return nil, nil, fmt.Errorf("unknown provider %q", name) +} + +func run(providerName, samples, labelsPath, cacheDir, out, endpoint string, rps float64, workers, limit int) error { files, err := filepath.Glob(filepath.Join(samples, "*.http")) if err != nil { return err @@ -92,20 +115,22 @@ func run(samples, labelsPath, cacheDir, out, endpoint string, rps float64, worke return err } } - client, err := jev.NewClient("") + provider, tokens, err := newProvider(providerName, endpoint, &limited{tick: time.NewTicker(time.Duration(float64(time.Second) / rps)).C}) if err != nil { return err } - client.Endpoint = endpoint + j := judge.New(provider) + // Measure similarity rather than use it: every page gets its own answers. + defaultDistance := judge.SimilarDistance + judge.SimilarDistance = 0 sim := newSimCheck(cacheDir) - client.Cache = sim - client.HTTP.Transport = &limited{tick: time.NewTicker(time.Duration(float64(time.Second) / rps)).C} + j.Cache = sim engine, err := fingers.NewEngine(fingers.FingersEngine, fingers.FingerPrintEngine, fingers.EHoleEngine, fingers.GobyEngine, fingers.WappalyzerEngine) if err != nil { return err } - engine.AttachJev(client) + engine.AttachJudge(j) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -119,7 +144,7 @@ func run(samples, labelsPath, cacheDir, out, endpoint string, rps float64, worke go func() { defer wg.Done() for i := range jobs { - rows[i] = judge(ctx, engine, files[i]) + rows[i] = evaluatePage(ctx, engine, files[i]) var apiErr *jev.APIError if rows[i].err != nil && errors.As(rows[i].err, &apiErr) && apiErr.Status == 401 { once.Do(func() { fatal = errors.New("unauthorized: check " + jev.EnvAPIKey); cancel() }) @@ -154,13 +179,8 @@ func run(samples, labelsPath, cacheDir, out, endpoint string, rps float64, worke f.Close() var b strings.Builder - report(&b, rows, client, time.Since(start)) - fmt.Fprintf(&b, "\n## 相似度缓存（SimilarDistance=%d，模拟）\n\n", jev.SimilarDistance) - fmt.Fprintf(&b, "- 请求 %d 次，其中 %d 次（%.1f%%）可以直接复用相似页面的答案；复用答案与真实答案在所有决策上完全一致的占 %d/%d（%.1f%%）\n", - sim.requests, sim.hits, 100*div(sim.hits, sim.requests), sim.agree, sim.hits, 100*div(sim.agree, sim.hits)) - for _, k := range sortedKeys(sim.diffs, func(k string) int { return sim.diffs[k] }) { - fmt.Fprintf(&b, " - 不一致的题目 %s: %d\n", k, sim.diffs[k]) - } + report(&b, rows, provider.ID(), j, tokens(), time.Since(start)) + sim.report(&b, defaultDistance) if labelsPath != "" { if err := evaluate(&b, rows, labelsPath); err != nil { return err @@ -170,7 +190,7 @@ func run(samples, labelsPath, cacheDir, out, endpoint string, rps float64, worke return os.WriteFile(filepath.Join(out, "report.md"), []byte(b.String()), 0o644) } -func judge(ctx context.Context, engine *fingers.Engine, path string) (r *row) { +func evaluatePage(ctx context.Context, engine *fingers.Engine, path string) (r *row) { r = &row{ID: filepath.Base(path), ruleEngines: map[string][]string{}} defer func() { if p := recover(); p != nil { // one malformed page must not stop a long run @@ -178,9 +198,9 @@ func judge(ctx context.Context, engine *fingers.Engine, path string) (r *row) { } }() raw, err := os.ReadFile(path) - var page *jev.Page + var page *judge.Page if err == nil { - page, err = jev.NewPage(raw) // for code-only evidence; Refine builds its own + page, err = judge.NewPage(raw) // for code-only evidence; Refine builds its own } if err == nil { t := time.Now() @@ -197,7 +217,7 @@ func judge(ctx context.Context, engine *fingers.Engine, path string) (r *row) { } keys := map[string]bool{} for _, f := range r.frames { - keys[jev.NormalizeName(f.Name)] = true + keys[judge.NormalizeName(f.Name)] = true if ev := page.Evidence(f.Name); len(ev) == 1 && ev[0] == "text" { r.TextOnly = append(r.TextOnly, f.Name) } @@ -215,60 +235,58 @@ func judge(ctx context.Context, engine *fingers.Engine, path string) (r *row) { sort.Strings(r.Rules) t := time.Now() page, err = engine.Refine(ctx, raw, r.frames) - r.JevMs = ms(time.Since(t)) + r.JudgeMs = ms(time.Since(t)) if err != nil { r.Err, r.err = err.Error(), err return r } - r.JevGenVer = versionOf(jev.Accepted(r.frames), product) + r.JudgeGenVer = versionOf(judge.Accepted(r.frames), product) for _, name := range r.TextOnly { - if f := r.frames[name]; f != nil && !f.HasTag(jev.TagRejected) { + if f := r.frames[name]; f != nil && !judge.Is(f, judge.Rejected) { r.TextOnlyKept++ } } r.Kind, r.Generic, r.Layers = page.Kind, page.Generic, map[string]string{} identified := false // an accepted application or device: the page's product is known for _, f := range r.frames { - for _, t := range f.Tags { - if strings.HasPrefix(t, jev.TagLayer) { - r.Layers[f.Name] = strings.TrimPrefix(t, jev.TagLayer) - } + if l := judge.LayerOf(f); l != "" { + r.Layers[f.Name] = string(l) } switch { - case f.HasTag(jev.TagRejected): + case judge.Is(f, judge.Rejected): r.Rejected = append(r.Rejected, f.Name) - case f.HasTag(jev.TagDup): + case judge.Is(f, judge.Duplicate): default: r.Accepted = append(r.Accepted, f.Name) - switch jev.Layer(r.Layers[f.Name]) { - case jev.LayerApplication, jev.LayerDevice: + switch judge.Layer(r.Layers[f.Name]) { + case judge.LayerApplication, judge.LayerDevice: identified = true } } - if f.HasTag(jev.TagRecall) { + if judge.Is(f, judge.Recalled) { r.Recall = append(r.Recall, f.Name) } } - if p := jev.Primary(r.frames); p != nil { + if p := judge.PrimaryOf(r.frames); p != nil { r.Primary, r.Version = p.Name, p.Version - r.JevVersion = p.Version != "" && p.Version != versions[p.Name] + r.JudgeVersion = p.Version != "" && p.Version != versions[p.Name] } // A stock page nobody identified is material for a new rule; a server's // own default, error or index page is identified by the server. - serverPage := r.Kind == "default_install" || r.Kind == "error_page" || r.Kind == "directory_listing" + serverPage := r.Kind == judge.KindDefault || r.Kind == judge.KindError || r.Kind == judge.KindDirListing r.NewFinger = r.Generic && r.Primary == "" && !identified && !serverPage sort.Strings(r.Accepted) sort.Strings(r.Rejected) return r } -func report(b *strings.Builder, rows []*row, c *jev.Client, took time.Duration) { +func report(b *strings.Builder, rows []*row, providerID string, j *judge.Judge, tokens int64, took time.Duration) { w := func(format string, a ...interface{}) { fmt.Fprintf(b, format+"\n", a...) } var pages, errs, rules, accepted, recall, primary, generic, newFinger int var ruleProducts, textOnly, textOnlyKept, rejected, genPages int - var genRule, genJev [3]int // correct, wrong, missing - var perRule, perJev []int - var ruleMs, jevMs []float64 + var genRule, genJudge [3]int // correct, wrong, missing + var perRule, perJudge []int + var ruleMs, judgeMs []float64 kinds := map[string]int{} type stat struct{ hits, rejected int } byEngine, byRule := map[string]*stat{}, map[string]*stat{} @@ -296,10 +314,10 @@ func report(b *strings.Builder, rows []*row, c *jev.Client, took time.Duration) textOnlyKept += r.TextOnlyKept rejected += len(r.Rejected) perRule = append(perRule, len(r.frames)-len(r.Recall)) - perJev = append(perJev, len(r.Accepted)) + perJudge = append(perJudge, len(r.Accepted)) ruleMs = append(ruleMs, r.RuleMs) - jevMs = append(jevMs, r.JevMs) - kinds[r.Kind]++ + judgeMs = append(judgeMs, r.JudgeMs) + kinds[string(r.Kind)]++ if r.Primary != "" { primary++ } @@ -307,7 +325,7 @@ func report(b *strings.Builder, rows []*row, c *jev.Client, took time.Duration) genPages++ want := r.Generator[strings.LastIndex(r.Generator, " ")+1:] grade(&genRule, r.RuleGenVer, want) - grade(&genJev, r.JevGenVer, want) + grade(&genJudge, r.JudgeGenVer, want) } if r.Generic { generic++ @@ -317,30 +335,30 @@ func report(b *strings.Builder, rows []*row, c *jev.Client, took time.Duration) newFingers = append(newFingers, r.ID) } for _, f := range r.frames { - rejected := f.HasTag(jev.TagRejected) + rejected := judge.Is(f, judge.Rejected) for _, e := range r.ruleEngines[f.Name] { add(byEngine, e, rejected) add(byRule, e+"/"+f.Name, rejected) } } } - w("# 纯规则 vs 规则 + Jev\n") + w("# 纯规则 vs 规则 + judge（%s）\n", providerID) w("pages %d, errors %d, took %s\n", pages, errs, took.Round(time.Second)) - w("| 指标 | 纯规则 | 规则 + Jev |\n|---|---|---|") - w("| 每页结果条数 mean / p90 | %.2f / %d | %.2f / %d |", mean(perRule), p90(perRule), mean(perJev), p90(perJev)) + w("| 指标 | 纯规则 | 规则 + judge |\n|---|---|---|") + w("| 每页结果条数 mean / p90 | %.2f / %d | %.2f / %d |", mean(perRule), p90(perRule), mean(perJudge), p90(perJudge)) w("| 每页不同产品数（按名字归并） | %.2f | %.2f |", div(ruleProducts, pages), div(accepted, pages)) - w("| 同名重复条目 | %d | 0（标 jev:dup） |", rules-ruleProducts) + w("| 同名重复条目 | %d | 0（标为 Duplicate） |", rules-ruleProducts) w("| 仅正文出现的命中（疑似误报） | %d | %d |", textOnly, textOnlyKept) w("| 被否决的规则命中 | — | %d / %d（%.1f%%） |", rejected, rules, 100*div(rejected, rules)) - w("| 召回（规则漏掉、Jev 确认） | 0 | %d |", recall) + w("| 召回（规则漏掉、judge 确认） | 0 | %d |", recall) w("| 主应用 | — | %d 页 |", primary) w("| 版本号 vs generator：正确 / 错误 / 缺失（%d 页） | %d / %d / %d | %d / %d / %d |", genPages, - genRule[0], genRule[1], genRule[2], genJev[0], genJev[1], genJev[2]) + genRule[0], genRule[1], genRule[2], genJudge[0], genJudge[1], genJudge[2]) w("| 页面类型 / 是否通用页面 | — | ✓ / 通用 %d 页 |", generic) w("| 新指纹候选 | — | %d |", newFinger) - w("| 每页耗时 p50 / p95 | %.1fms / %.1fms | +%.0fms / +%.0fms |", pct(ruleMs, 50), pct(ruleMs, 95), pct(jevMs, 50), pct(jevMs, 95)) - w("| 成本 | 0 | %d 次请求（缓存命中 %d），%d input tokens，≈ $%.3f，折合每万页 $%.2f |", - c.Requests, c.CacheHits, c.InputTokens, float64(c.InputTokens)*0.042/1e6, float64(c.InputTokens)*0.042/1e6/float64(max(pages, 1))*1e4) + w("| 每页耗时 p50 / p95 | %.1fms / %.1fms | +%.0fms / +%.0fms |", pct(ruleMs, 50), pct(ruleMs, 95), pct(judgeMs, 50), pct(judgeMs, 95)) + w("| 成本 | 0 | %d 次请求（缓存直接答完 %d 轮），%d input tokens（jev 按 $0.042/Mtok ≈ $%.3f，折合每万页 $%.2f） |", + j.Requests, j.CacheHits, tokens, float64(tokens)*0.042/1e6, float64(tokens)*0.042/1e6/float64(max(pages, 1))*1e4) w("\n## page kind\n") for _, k := range sortedKeys(kinds, func(k string) int { return kinds[k] }) { w("- %s: %d", k, kinds[k]) @@ -412,7 +430,7 @@ func evaluate(b *strings.Builder, rows []*row, path string) error { misses = append(misses, r.ID+": kept "+name) } } - if r.Kind == l.PageKind { + if string(r.Kind) == l.PageKind { kind++ } if r.Generic == l.Generic { @@ -448,10 +466,10 @@ func evaluate(b *strings.Builder, rows []*row, path string) error { // match is containment either way; strict (for false positive labels such as // "tomcat_jk_connector") only lets the framework name contain the label. func find(frames common.Frameworks, name string, strict bool) *common.Framework { - want := jev.NormalizeName(name) + want := judge.NormalizeName(name) var found *common.Framework - for _, f := range jev.Accepted(frames) { - got := jev.NormalizeName(f.Name) + for _, f := range judge.Accepted(frames) { + got := judge.NormalizeName(f.Name) if got == "" || !(strings.Contains(got, want) || !strict && strings.Contains(want, got)) { continue } @@ -482,7 +500,7 @@ func versionOf(frames common.Frameworks, product string) string { } var got string for _, f := range frames { - key := jev.NormalizeName(f.Name) + key := judge.NormalizeName(f.Name) if f.Version == "" || !strings.HasPrefix(key, product) { continue } @@ -553,7 +571,7 @@ func sortedKeys[V any](m map[string]V, weight func(string) int) []string { return keys } -// fileCache is an exact jev.Cache on disk, one file per request. +// fileCache is an exact judge.Cache on disk, one file per request. type fileCache string func (d fileCache) path(key string, sig uint64) string { @@ -572,41 +590,54 @@ func (d fileCache) Put(key string, sig uint64, value []byte) { } } -// simCheck measures the similarity cache without trusting it: every page is -// still answered exactly, and whenever an earlier similar page would have -// been served instead, the two answers are compared decision by decision. +// simCheck measures the similarity cache without trusting it: every question +// is still answered for its exact page (by the file cache), and whenever an +// earlier page with the same key had a signature within maxProbe bits, its +// answer is compared with the real one, bucketed by distance. That shows +// which SimilarDistance is safe. type simCheck struct { fileCache - mu sync.Mutex - seen map[string][]simEntry // key -> earlier answers - pending map[string][]byte // key+sig -> answer a similarity cache would have served - requests, hits, agree int - diffs map[string]int // question key prefix -> disagreements + mu sync.Mutex + seen map[string][]simEntry // key -> earlier answers + pending map[string]simNear // key+sig -> nearest earlier answer, until the real one arrives + answers int + byDist [maxProbe + 1]struct{ reusable, agree int } } +const maxProbe = 10 + type simEntry struct { sig uint64 value []byte } +type simNear struct { + dist int + value []byte +} + func newSimCheck(dir string) *simCheck { - return &simCheck{fileCache: fileCache(dir), seen: map[string][]simEntry{}, pending: map[string][]byte{}, diffs: map[string]int{}} + return &simCheck{fileCache: fileCache(dir), seen: map[string][]simEntry{}, pending: map[string]simNear{}} +} + +func (c *simCheck) nearest(key string, sig uint64) (simNear, bool) { + best, ok := simNear{dist: maxProbe + 1}, false + for _, e := range c.seen[key] { + if d := bits.OnesCount64(e.sig ^ sig); d < best.dist { + best, ok = simNear{d, e.value}, true + } + } + return best, ok } func (c *simCheck) Get(key string, sig uint64) ([]byte, bool) { value, ok := c.fileCache.Get(key, sig) c.mu.Lock() defer c.mu.Unlock() - var near []byte - for _, e := range c.seen[key] { - if e.sig != sig && jev.Similar(e.sig, sig) { - near = e.value - break - } - } + near, found := c.nearest(key, sig) if ok { - c.observe(key, sig, near, value) - } else if near != nil { + c.observe(key, sig, near, found, value) + } else if found { c.pending[fmt.Sprintf("%s_%x", key, sig)] = near } return value, ok @@ -617,34 +648,39 @@ func (c *simCheck) Put(key string, sig uint64, value []byte) { c.mu.Lock() defer c.mu.Unlock() id := fmt.Sprintf("%s_%x", key, sig) - c.observe(key, sig, c.pending[id], value) + near, found := c.pending[id] delete(c.pending, id) + c.observe(key, sig, near, found, value) } -func (c *simCheck) observe(key string, sig uint64, near, value []byte) { - c.requests++ +func (c *simCheck) observe(key string, sig uint64, near simNear, found bool, value []byte) { + c.answers++ c.seen[key] = append(c.seen[key], simEntry{sig, value}) - if near == nil { + if !found || near.dist > maxProbe { return } - c.hits++ - var a, b jev.Response - json.Unmarshal(near, &a) - json.Unmarshal(value, &b) - same := true - for k, x := range b.Answers { - y := a.Answers[k] - if x.Choice != y.Choice || (x.Noul >= jev.Threshold) != (y.Noul >= jev.Threshold) { - same = false - c.diffs[strings.TrimRight(k, "0123456789")]++ - } + var x, y judge.Answer + json.Unmarshal(near.value, &x) + json.Unmarshal(value, &y) + c.byDist[near.dist].reusable++ + if x.Choice == y.Choice && (x.Yes >= 0.5) == (y.Yes >= 0.5) { + c.byDist[near.dist].agree++ } - if same { - c.agree++ +} + +func (c *simCheck) report(b *strings.Builder, current int) { + fmt.Fprintf(b, "\n## 相似度缓存（模拟，当前默认 SimilarDistance=%d）\n\n", current) + fmt.Fprintf(b, "共 %d 个答案。下表：若 SimilarDistance 取 d，能由相似页面（同标题）复用的答案数，以及其中与真实答案一致的比例。\n\n", c.answers) + fmt.Fprintf(b, "| d | 可复用 | 一致 | 一致率 |\n|---|---|---|---|\n") + reusable, agree := 0, 0 + for d := 0; d <= maxProbe; d++ { + reusable += c.byDist[d].reusable + agree += c.byDist[d].agree + fmt.Fprintf(b, "| %d | %d | %d | %.1f%% |\n", d, reusable, agree, 100*div(agree, reusable)) } } -// limited rate-limits Jev requests; cache hits never reach it. +// limited rate-limits provider requests; cache hits never reach it. type limited struct{ tick <-chan time.Time } func (l *limited) RoundTrip(req *http.Request) (*http.Response, error) { diff --git a/cmd/jevbench/testdata/labels.json b/cmd/judgeeval/testdata/labels.json similarity index 100% rename from cmd/jevbench/testdata/labels.json rename to cmd/judgeeval/testdata/labels.json diff --git a/cmd/jevbench/testdata/samples/blog_mentions_jenkins.http b/cmd/judgeeval/testdata/samples/blog_mentions_jenkins.http similarity index 100% rename from cmd/jevbench/testdata/samples/blog_mentions_jenkins.http rename to cmd/judgeeval/testdata/samples/blog_mentions_jenkins.http diff --git a/cmd/jevbench/testdata/samples/confluence_login.http b/cmd/judgeeval/testdata/samples/confluence_login.http similarity index 100% rename from cmd/jevbench/testdata/samples/confluence_login.http rename to cmd/judgeeval/testdata/samples/confluence_login.http diff --git a/cmd/jevbench/testdata/samples/corporate_cn_portal.http b/cmd/judgeeval/testdata/samples/corporate_cn_portal.http similarity index 100% rename from cmd/jevbench/testdata/samples/corporate_cn_portal.http rename to cmd/judgeeval/testdata/samples/corporate_cn_portal.http diff --git a/cmd/jevbench/testdata/samples/corporate_site.http b/cmd/judgeeval/testdata/samples/corporate_site.http similarity index 100% rename from cmd/jevbench/testdata/samples/corporate_site.http rename to cmd/judgeeval/testdata/samples/corporate_site.http diff --git a/cmd/jevbench/testdata/samples/custom_employee_portal.http b/cmd/judgeeval/testdata/samples/custom_employee_portal.http similarity index 100% rename from cmd/jevbench/testdata/samples/custom_employee_portal.http rename to cmd/judgeeval/testdata/samples/custom_employee_portal.http diff --git a/cmd/jevbench/testdata/samples/dir_listing.http b/cmd/judgeeval/testdata/samples/dir_listing.http similarity index 100% rename from cmd/jevbench/testdata/samples/dir_listing.http rename to cmd/judgeeval/testdata/samples/dir_listing.http diff --git a/cmd/jevbench/testdata/samples/discuz_forum.http b/cmd/judgeeval/testdata/samples/discuz_forum.http similarity index 100% rename from cmd/jevbench/testdata/samples/discuz_forum.http rename to cmd/judgeeval/testdata/samples/discuz_forum.http diff --git a/cmd/jevbench/testdata/samples/docs_tomcat_howto.http b/cmd/judgeeval/testdata/samples/docs_tomcat_howto.http similarity index 100% rename from cmd/jevbench/testdata/samples/docs_tomcat_howto.http rename to cmd/judgeeval/testdata/samples/docs_tomcat_howto.http diff --git a/cmd/jevbench/testdata/samples/gitlab_signin.http b/cmd/judgeeval/testdata/samples/gitlab_signin.http similarity index 100% rename from cmd/jevbench/testdata/samples/gitlab_signin.http rename to cmd/judgeeval/testdata/samples/gitlab_signin.http diff --git a/cmd/jevbench/testdata/samples/grafana_login.http b/cmd/judgeeval/testdata/samples/grafana_login.http similarity index 100% rename from cmd/jevbench/testdata/samples/grafana_login.http rename to cmd/judgeeval/testdata/samples/grafana_login.http diff --git a/cmd/jevbench/testdata/samples/harbor_portal.http b/cmd/judgeeval/testdata/samples/harbor_portal.http similarity index 100% rename from cmd/jevbench/testdata/samples/harbor_portal.http rename to cmd/judgeeval/testdata/samples/harbor_portal.http diff --git a/cmd/jevbench/testdata/samples/iis_default.http b/cmd/judgeeval/testdata/samples/iis_default.http similarity index 100% rename from cmd/jevbench/testdata/samples/iis_default.http rename to cmd/judgeeval/testdata/samples/iis_default.http diff --git a/cmd/jevbench/testdata/samples/jenkins_login.http b/cmd/judgeeval/testdata/samples/jenkins_login.http similarity index 100% rename from cmd/jevbench/testdata/samples/jenkins_login.http rename to cmd/judgeeval/testdata/samples/jenkins_login.http diff --git a/cmd/jevbench/testdata/samples/job_posting_cn.http b/cmd/judgeeval/testdata/samples/job_posting_cn.http similarity index 100% rename from cmd/jevbench/testdata/samples/job_posting_cn.http rename to cmd/judgeeval/testdata/samples/job_posting_cn.http diff --git a/cmd/jevbench/testdata/samples/json_api.http b/cmd/judgeeval/testdata/samples/json_api.http similarity index 100% rename from cmd/jevbench/testdata/samples/json_api.http rename to cmd/judgeeval/testdata/samples/json_api.http diff --git a/cmd/jevbench/testdata/samples/kibana_home.http b/cmd/judgeeval/testdata/samples/kibana_home.http similarity index 100% rename from cmd/jevbench/testdata/samples/kibana_home.http rename to cmd/judgeeval/testdata/samples/kibana_home.http diff --git a/cmd/jevbench/testdata/samples/minio_console.http b/cmd/judgeeval/testdata/samples/minio_console.http similarity index 100% rename from cmd/jevbench/testdata/samples/minio_console.http rename to cmd/judgeeval/testdata/samples/minio_console.http diff --git a/cmd/jevbench/testdata/samples/nacos_console.http b/cmd/judgeeval/testdata/samples/nacos_console.http similarity index 100% rename from cmd/jevbench/testdata/samples/nacos_console.http rename to cmd/judgeeval/testdata/samples/nacos_console.http diff --git a/cmd/jevbench/testdata/samples/news_fortios_patch.http b/cmd/judgeeval/testdata/samples/news_fortios_patch.http similarity index 100% rename from cmd/jevbench/testdata/samples/news_fortios_patch.http rename to cmd/judgeeval/testdata/samples/news_fortios_patch.http diff --git a/cmd/jevbench/testdata/samples/nginx_404.http b/cmd/judgeeval/testdata/samples/nginx_404.http similarity index 100% rename from cmd/jevbench/testdata/samples/nginx_404.http rename to cmd/judgeeval/testdata/samples/nginx_404.http diff --git a/cmd/jevbench/testdata/samples/nginx_default.http b/cmd/judgeeval/testdata/samples/nginx_default.http similarity index 100% rename from cmd/jevbench/testdata/samples/nginx_default.http rename to cmd/judgeeval/testdata/samples/nginx_default.http diff --git a/cmd/jevbench/testdata/samples/pgadmin_login.http b/cmd/judgeeval/testdata/samples/pgadmin_login.http similarity index 100% rename from cmd/jevbench/testdata/samples/pgadmin_login.http rename to cmd/judgeeval/testdata/samples/pgadmin_login.http diff --git a/cmd/jevbench/testdata/samples/phpmyadmin_login.http b/cmd/judgeeval/testdata/samples/phpmyadmin_login.http similarity index 100% rename from cmd/jevbench/testdata/samples/phpmyadmin_login.http rename to cmd/judgeeval/testdata/samples/phpmyadmin_login.http diff --git a/cmd/jevbench/testdata/samples/portainer_login.http b/cmd/judgeeval/testdata/samples/portainer_login.http similarity index 100% rename from cmd/jevbench/testdata/samples/portainer_login.http rename to cmd/judgeeval/testdata/samples/portainer_login.http diff --git a/cmd/jevbench/testdata/samples/rabbitmq_mgmt.http b/cmd/judgeeval/testdata/samples/rabbitmq_mgmt.http similarity index 100% rename from cmd/jevbench/testdata/samples/rabbitmq_mgmt.http rename to cmd/judgeeval/testdata/samples/rabbitmq_mgmt.http diff --git a/cmd/jevbench/testdata/samples/rebranded_grafana.http b/cmd/judgeeval/testdata/samples/rebranded_grafana.http similarity index 100% rename from cmd/jevbench/testdata/samples/rebranded_grafana.http rename to cmd/judgeeval/testdata/samples/rebranded_grafana.http diff --git a/cmd/jevbench/testdata/samples/rebranded_jenkins.http b/cmd/judgeeval/testdata/samples/rebranded_jenkins.http similarity index 100% rename from cmd/jevbench/testdata/samples/rebranded_jenkins.http rename to cmd/judgeeval/testdata/samples/rebranded_jenkins.http diff --git a/cmd/jevbench/testdata/samples/rebranded_zabbix.http b/cmd/judgeeval/testdata/samples/rebranded_zabbix.http similarity index 100% rename from cmd/jevbench/testdata/samples/rebranded_zabbix.http rename to cmd/judgeeval/testdata/samples/rebranded_zabbix.http diff --git a/cmd/jevbench/testdata/samples/seeyon_login.http b/cmd/judgeeval/testdata/samples/seeyon_login.http similarity index 100% rename from cmd/jevbench/testdata/samples/seeyon_login.http rename to cmd/judgeeval/testdata/samples/seeyon_login.http diff --git a/cmd/jevbench/testdata/samples/shop_page.http b/cmd/judgeeval/testdata/samples/shop_page.http similarity index 100% rename from cmd/jevbench/testdata/samples/shop_page.http rename to cmd/judgeeval/testdata/samples/shop_page.http diff --git a/cmd/jevbench/testdata/samples/springboot_error.http b/cmd/judgeeval/testdata/samples/springboot_error.http similarity index 100% rename from cmd/jevbench/testdata/samples/springboot_error.http rename to cmd/judgeeval/testdata/samples/springboot_error.http diff --git a/cmd/jevbench/testdata/samples/status_page_mentions.http b/cmd/judgeeval/testdata/samples/status_page_mentions.http similarity index 100% rename from cmd/jevbench/testdata/samples/status_page_mentions.http rename to cmd/judgeeval/testdata/samples/status_page_mentions.http diff --git a/cmd/jevbench/testdata/samples/synology_dsm.http b/cmd/judgeeval/testdata/samples/synology_dsm.http similarity index 100% rename from cmd/jevbench/testdata/samples/synology_dsm.http rename to cmd/judgeeval/testdata/samples/synology_dsm.http diff --git a/cmd/jevbench/testdata/samples/tomcat_default.http b/cmd/judgeeval/testdata/samples/tomcat_default.http similarity index 100% rename from cmd/jevbench/testdata/samples/tomcat_default.http rename to cmd/judgeeval/testdata/samples/tomcat_default.http diff --git a/cmd/jevbench/testdata/samples/tplink_router.http b/cmd/judgeeval/testdata/samples/tplink_router.http similarity index 100% rename from cmd/jevbench/testdata/samples/tplink_router.http rename to cmd/judgeeval/testdata/samples/tplink_router.http diff --git a/cmd/jevbench/testdata/samples/unknown_admin_login.http b/cmd/judgeeval/testdata/samples/unknown_admin_login.http similarity index 100% rename from cmd/jevbench/testdata/samples/unknown_admin_login.http rename to cmd/judgeeval/testdata/samples/unknown_admin_login.http diff --git a/cmd/jevbench/testdata/samples/weaver_ecology_login.http b/cmd/judgeeval/testdata/samples/weaver_ecology_login.http similarity index 100% rename from cmd/jevbench/testdata/samples/weaver_ecology_login.http rename to cmd/judgeeval/testdata/samples/weaver_ecology_login.http diff --git a/cmd/jevbench/testdata/samples/wordpress_home.http b/cmd/judgeeval/testdata/samples/wordpress_home.http similarity index 100% rename from cmd/jevbench/testdata/samples/wordpress_home.http rename to cmd/judgeeval/testdata/samples/wordpress_home.http diff --git a/cmd/jevbench/testdata/samples/wordpress_login.http b/cmd/judgeeval/testdata/samples/wordpress_login.http similarity index 100% rename from cmd/jevbench/testdata/samples/wordpress_login.http rename to cmd/judgeeval/testdata/samples/wordpress_login.http diff --git a/cmd/jevbench/testdata/samples/zabbix_login.http b/cmd/judgeeval/testdata/samples/zabbix_login.http similarity index 100% rename from cmd/jevbench/testdata/samples/zabbix_login.http rename to cmd/judgeeval/testdata/samples/zabbix_login.http diff --git a/engine.go b/engine.go index 9b1dea3..30a0a72 100644 --- a/engine.go +++ b/engine.go @@ -11,7 +11,7 @@ import ( "github.com/chainreactors/fingers/fingerprinthub" "github.com/chainreactors/fingers/fingers" "github.com/chainreactors/fingers/goby" - "github.com/chainreactors/fingers/jev" + "github.com/chainreactors/fingers/judge" gonmap "github.com/chainreactors/fingers/nmap" "github.com/chainreactors/fingers/resources" wappalyzer "github.com/chainreactors/fingers/wappalyzer" @@ -88,9 +88,9 @@ type Engine struct { Enabled map[string]bool Capabilities map[string]common.EngineCapability // 新增：记录各引擎能力 - // 可选的 Jev 判定层, 见 AttachJev - jevClient *jev.Client - jevRecall *jev.Retriever + // 可选的判定层, 见 AttachJudge + judge *judge.Judge + recall *judge.Retriever } func (engine *Engine) String() string { diff --git a/jev.go b/jev.go deleted file mode 100644 index f1aead0..0000000 --- a/jev.go +++ /dev/null @@ -1,40 +0,0 @@ -package fingers - -import ( - "context" - "errors" - - "github.com/chainreactors/fingers/common" - "github.com/chainreactors/fingers/jev" -) - -var ErrJevNotAttached = errors.New("jev is not attached, call AttachJev first") - -// AttachJev adds the Jev judgement layer on top of the enabled engines. The -// rule engines are untouched: WebMatch/DetectContent stay synchronous and -// offline; Refine judges their result afterwards, typically asynchronously. -func (engine *Engine) AttachJev(client *jev.Client) { - var names []string - if engine.Aliases != nil { - for name := range engine.Aliases.Aliases { - names = append(names, name) - } - } - engine.jevClient, engine.jevRecall = client, jev.NewRetriever(names) -} - -// Refine judges frames, the rule result for the raw HTTP response content, -// and annotates them in place (see the jev.Tag* constants); fingerprint -// names found in the page but missed by the rules are recall candidates. -// The returned page carries the page kind. Refine writes to frames: do not -// read them concurrently. On error, frames hold the rule result. -func (engine *Engine) Refine(ctx context.Context, content []byte, frames common.Frameworks) (*jev.Page, error) { - if engine.jevClient == nil { - return nil, ErrJevNotAttached - } - page, err := jev.NewPage(content) - if err != nil { - return nil, err - } - return page, jev.Refine(ctx, engine.jevClient, page, frames, engine.jevRecall.Find(page.Haystack(), 24)) -} diff --git a/jev/classify.go b/jev/classify.go deleted file mode 100644 index 94bc2aa..0000000 --- a/jev/classify.go +++ /dev/null @@ -1,30 +0,0 @@ -package jev - -// PageKinds is the Choice criteria for Page.Kind. -var PageKinds = map[string]interface{}{ - "login": "A sign-in form or authentication page", - "default_install": "The stock page a web server or product shows right after installation", - "error_page": "A framework or server error page such as 404 or 500", - "directory_listing": "An auto-generated file index of a directory", - "app_console": "The main interface or loading screen of a web application or management console", - "normal_content": "An ordinary website page: article, company site, shop, forum, blog, status page", - "api_response": "A machine-readable API response rather than a web page", -} - -// Classify judges the page itself and writes Page.Kind and Page.Generic. It -// needs no fingerprints: spray can use it alone for 404 detection and -// prioritisation. A generic page without an accepted application is a -// candidate for a new fingerprint. -func Classify(r *Round) { - p := r.page - r.Add("page_kind", Choice("What kind of page is this HTTP response?", PageKinds), func(a Answer) { p.Kind = a.Choice }) - r.Add("generic", Question{ - Type: "noul", - Instructions: "Is this page the stock interface of a packaged software product, framework or device, " + - "so that the same page would appear on many unrelated deployments?", - Criteria: map[string]string{ - "true": "Login, console, default or error page shipped by a product; only branding, host names or data differ between installs", - "false": "Content written by one organization for its own purpose: articles, company pages, shops, forums, custom-built portals", - }, - }, func(a Answer) { p.Generic = a.Noul >= Threshold }) -} diff --git a/jev/client.go b/jev/client.go deleted file mode 100644 index f2ce3ac..0000000 --- a/jev/client.go +++ /dev/null @@ -1,200 +0,0 @@ -// Package jev is the judgement layer on top of the rule engines, backed by -// TypeSafe's Jev (System One) model. It is not an engine: rules and code -// recall candidates, Jev only verifies and chooses (false positives, stack -// layer, primary application, page kind, generic page, version), and code -// makes the final call. Each capability is a Task; the tasks of a round share -// one request, so a page costs at most two. -// -// API reference: https://docs.typesafe.ai/api.md -package jev - -import ( - "bytes" - "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "fmt" - "io" - "net/http" - "os" - "sync/atomic" - "time" -) - -const ( - DefaultEndpoint = "https://api.typesafe.ai/v1/systemone" - // Pin a versioned model: thresholds tuned against one version do not carry over. - DefaultModel = "jev-1.13.0" - EnvAPIKey = "TYPESAFE_API_KEY" -) - -// Question is one of noul / choice / score. -type Question struct { - Type string `json:"type"` - Instructions interface{} `json:"instructions"` - Criteria interface{} `json:"criteria,omitempty"` -} - -func Noul(instructions interface{}) Question { - return Question{Type: "noul", Instructions: instructions} -} - -// Choice criteria maps option -> description (nil for none). Max 255 options. -func Choice(instructions interface{}, criteria map[string]interface{}) Question { - return Question{Type: "choice", Instructions: instructions, Criteria: criteria} -} - -// Score criteria is an ordered list of 2..10 level descriptions. -func Score(instructions interface{}, levels []string) Question { - return Question{Type: "score", Instructions: instructions, Criteria: levels} -} - -type request struct { - State interface{} `json:"state"` - Model string `json:"model"` - Questions map[string]Question `json:"questions"` -} - -type Answer struct { - Type string `json:"type"` - Noul float64 `json:"noul,omitempty"` - Choice string `json:"choice,omitempty"` - Score float64 `json:"score,omitempty"` - Probabilities map[string]float64 `json:"probabilities,omitempty"` - Confidence float64 `json:"confidence,omitempty"` -} - -type Usage struct { - InputTokens int `json:"input_tokens"` - OutputTokens int `json:"output_tokens"` -} - -type Response struct { - Model string `json:"model"` - Answers map[string]Answer `json:"answers"` - Usage Usage `json:"usage"` - Latency time.Duration `json:"-"` - Cached bool `json:"-"` -} - -type Client struct { - Endpoint string - Model string - APIKey string - HTTP *http.Client - MaxRetries int - Cache Cache // nil disables caching - - // Totals since creation, for cost monitoring; read with atomic.LoadInt64. - Requests, CacheHits, InputTokens int64 -} - -// NewClient reads the API key from TYPESAFE_API_KEY when apiKey is empty. -func NewClient(apiKey string) (*Client, error) { - if apiKey == "" { - apiKey = os.Getenv(EnvAPIKey) - } - if apiKey == "" { - return nil, fmt.Errorf("%s is not set", EnvAPIKey) - } - return &Client{ - Endpoint: DefaultEndpoint, - Model: DefaultModel, - APIKey: apiKey, - HTTP: &http.Client{Timeout: 30 * time.Second}, - MaxRetries: 3, - }, nil -} - -type APIError struct { - Status int - Body string -} - -func (e *APIError) Error() string { return fmt.Sprintf("typesafe: http %d: %s", e.Status, e.Body) } - -// Ask evaluates every question against one state in a single request. -// 429 / 529 are retried with exponential backoff. Most callers use Round, -// which also lets the cache match similar pages. -func (c *Client) Ask(ctx context.Context, state interface{}, questions map[string]Question) (*Response, error) { - return c.ask(ctx, state, questions, nil, 0) -} - -// ask caches under the hash of exact (the parts that must match exactly) and -// the page signature sig. With exact nil the whole request must match. -func (c *Client) ask(ctx context.Context, state interface{}, questions map[string]Question, exact []byte, sig uint64) (*Response, error) { - // encoding/json sorts map keys, so equal requests marshal to equal bytes. - body, err := json.Marshal(request{State: state, Model: c.Model, Questions: questions}) - if err != nil { - return nil, err - } - if exact == nil { - exact = body - } else { - exact = append([]byte(c.Model+"\x00"), exact...) - } - sum := sha256.Sum256(exact) - key := hex.EncodeToString(sum[:]) - if c.Cache != nil { - if raw, ok := c.Cache.Get(key, sig); ok { - var out Response - if json.Unmarshal(raw, &out) == nil { - atomic.AddInt64(&c.CacheHits, 1) - out.Cached = true - return &out, nil - } - } - } - backoff := time.Second - for attempt := 0; ; attempt++ { - resp, raw, err := c.do(ctx, body) - if err == nil { - atomic.AddInt64(&c.Requests, 1) - atomic.AddInt64(&c.InputTokens, int64(resp.Usage.InputTokens)) - if c.Cache != nil { - c.Cache.Put(key, sig, raw) - } - return resp, nil - } - apiErr, ok := err.(*APIError) - if !ok || (apiErr.Status != 429 && apiErr.Status != 529) || attempt >= c.MaxRetries { - return nil, err - } - select { - case <-ctx.Done(): - return nil, ctx.Err() - case <-time.After(backoff): - } - backoff *= 2 - } -} - -func (c *Client) do(ctx context.Context, body []byte) (*Response, []byte, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.Endpoint, bytes.NewReader(body)) - if err != nil { - return nil, nil, err - } - req.Header.Set("Authorization", "Bearer "+c.APIKey) - req.Header.Set("Content-Type", "application/json") - - start := time.Now() - resp, err := c.HTTP.Do(req) - if err != nil { - return nil, nil, err - } - defer resp.Body.Close() - raw, err := io.ReadAll(resp.Body) - if err != nil { - return nil, nil, err - } - if resp.StatusCode != http.StatusOK { - return nil, nil, &APIError{Status: resp.StatusCode, Body: string(raw)} - } - var out Response - if err := json.Unmarshal(raw, &out); err != nil { - return nil, nil, fmt.Errorf("typesafe: decode response: %w", err) - } - out.Latency = time.Since(start) - return &out, raw, nil -} diff --git a/jev/refine.go b/jev/refine.go deleted file mode 100644 index cbf3b64..0000000 --- a/jev/refine.go +++ /dev/null @@ -1,65 +0,0 @@ -package jev - -import ( - "context" - - "github.com/chainreactors/fingers/common" -) - -// Refine is the default pipeline: Verify and Classify in one request, then -// the version of the page's subject in a second one. frames and p are -// annotated in place; on error the second round's annotations are missing, -// the first round's are complete or absent. -func Refine(ctx context.Context, c *Client, p *Page, frames common.Frameworks, recall []string) error { - r := p.Round() - Verify(r, frames, recall) - Classify(r) - if err := r.Ask(ctx, c); err != nil { - return err - } - r = p.Round() - Version(r, subject(frames)) - return r.Ask(ctx, c) -} - -// subject is the product whose version the page most likely shows: the -// primary application, or on pages without one (a server's default or error -// page) the only accepted server, application or device. -func subject(frames common.Frameworks) *common.Framework { - if f := Primary(frames); f != nil { - return f - } - var only *common.Framework - for _, f := range Accepted(frames) { - for _, l := range []Layer{LayerServer, LayerApplication, LayerDevice} { - if f.HasTag(TagLayer + string(l)) { - if only != nil { - return nil - } - only = f - } - } - } - return only -} - -// Primary returns the framework Verify tagged as the page's application. -func Primary(frames common.Frameworks) *common.Framework { - for _, f := range frames { - if f.HasTag(TagPrimary) { - return f - } - } - return nil -} - -// Accepted returns frames without rejected hits and duplicate spellings. -func Accepted(frames common.Frameworks) common.Frameworks { - out := common.Frameworks{} - for k, f := range frames { - if !f.HasTag(TagRejected) && !f.HasTag(TagDup) { - out[k] = f - } - } - return out -} diff --git a/jev_test.go b/jev_test.go deleted file mode 100644 index 9dfe3aa..0000000 --- a/jev_test.go +++ /dev/null @@ -1,52 +0,0 @@ -package fingers - -import ( - "context" - "encoding/json" - "net/http" - "net/http/httptest" - "strings" - "testing" - - "github.com/chainreactors/fingers/jev" -) - -func TestRefineWithJev(t *testing.T) { - engine, err := NewEngine(FingersEngine, WappalyzerEngine) - if err != nil { - t.Fatal(err) - } - raw := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nWelcome to nginx!") - frames, _ := engine.DetectContent(raw) - if _, err := engine.Refine(context.Background(), raw, frames); err != ErrJevNotAttached { - t.Fatalf("want ErrJevNotAttached, got %v", err) - } - - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - var req struct{ Questions map[string]jev.Question } - json.NewDecoder(r.Body).Decode(&req) - answers := map[string]jev.Answer{} - for k := range req.Questions { - switch { - case strings.HasPrefix(k, "layer_"): - answers[k] = jev.Answer{Choice: string(jev.LayerServer)} - case k == "page_kind": - answers[k] = jev.Answer{Choice: "default_install"} - default: - answers[k] = jev.Answer{Noul: 0.9, Choice: jev.NotStated} - } - } - json.NewEncoder(w).Encode(jev.Response{Answers: answers}) - })) - defer srv.Close() - client, _ := jev.NewClient("k") - client.Endpoint = srv.URL - engine.AttachJev(client) - page, err := engine.Refine(context.Background(), raw, frames) - if err != nil { - t.Fatal(err) - } - if page.Kind != "default_install" || !frames["nginx"].HasTag(jev.TagLayer+string(jev.LayerServer)) { - t.Fatalf("kind %q, nginx %v", page.Kind, frames["nginx"].Tags) - } -} diff --git a/judge.go b/judge.go new file mode 100644 index 0000000..a3ad53e --- /dev/null +++ b/judge.go @@ -0,0 +1,43 @@ +package fingers + +import ( + "context" + "errors" + + "github.com/chainreactors/fingers/common" + "github.com/chainreactors/fingers/judge" +) + +var ErrNoJudge = errors.New("no judge attached, call AttachJudge first") + +// AttachJudge adds a judgement layer (see package judge) on top of the +// enabled engines, e.g. engine.AttachJudge(jev.NewJudge("")). The rule +// engines are untouched: WebMatch/DetectContent stay synchronous and +// offline; Refine judges their result afterwards, typically asynchronously. +// Every alias name becomes a recall candidate for false negatives. +func (engine *Engine) AttachJudge(j *judge.Judge) { + var names []string + if engine.Aliases != nil { + for name := range engine.Aliases.Aliases { + names = append(names, name) + } + } + engine.judge, engine.recall = j, judge.NewRetriever(names) +} + +// Refine judges frames, the rule result for the raw HTTP response content, +// and marks them in place (read with judge.Is, judge.LayerOf, +// judge.Accepted); fingerprint names found in the page but missed by the +// rules are recall candidates. The returned page carries the page kind. +// Refine writes to frames: do not read them concurrently. On error, frames +// hold the rule result. +func (engine *Engine) Refine(ctx context.Context, content []byte, frames common.Frameworks) (*judge.Page, error) { + if engine.judge == nil { + return nil, ErrNoJudge + } + page, err := judge.NewPage(content) + if err != nil { + return nil, err + } + return page, judge.Refine(ctx, engine.judge, page, frames, engine.recall.Find(page.Haystack(), 24)) +} diff --git a/judge/README.md b/judge/README.md new file mode 100644 index 0000000..2cf6537 --- /dev/null +++ b/judge/README.md @@ -0,0 +1,141 @@ +# judge：指纹判定与重排层 + +`judge` 是 fingers 在规则引擎之上的判定（判别 + rerank）层。它不是指纹引擎： + +- **召回**由规则引擎和代码负责：规则命中、在页面里按指纹名检索、从页面中抽取版本字符串； +- **判断**交给 Provider（一个模型）：只回答关于这些证据的类型化问题，例如"是否存在"、"从下面几项里选一个"； +- **决策**由代码做出：阈值、强证据保留、同名归并、结果标注。 + +这种分工让效果不依赖某个具体模型。[TypeSafe Jev](https://docs.typesafe.ai/api.md) 是目前唯一的 Provider（`judge/jev`）；接入 LLM、本地分类器或 reranker，只需要实现一个接口。 + +## 快速开始 + +```go +import ( + "github.com/chainreactors/fingers" + "github.com/chainreactors/fingers/judge" + "github.com/chainreactors/fingers/judge/jev" +) + +engine, _ := fingers.NewEngine() +j, err := jev.NewJudge("") // 读取 TYPESAFE_API_KEY；自带缓存，整个扫描共用一个 +if err != nil { ... } +engine.AttachJudge(j) + +frames, _ := engine.DetectContent(raw) // 同步、离线，与之前完全相同 +page, err := engine.Refine(ctx, raw, frames) // 判定并原地标注 frames；通常异步执行 +if err != nil { + // 判定失败：frames 保持规则结果不变，照常使用即可 +} +for _, f := range judge.Accepted(frames) { // 去掉误报和同名重复之后的结果 + fmt.Println(f.Name, f.Version, judge.LayerOf(f), judge.Is(f, judge.Primary)) +} +fmt.Println(page.Kind, page.Generic) // 页面类型；是否为某个产品的标准页面 +``` + +`Refine` 每页最多发 2 次请求：第一次判断页面和所有候选，第二次挑选版本号。命中缓存的问题不会再发。 + +## 能力 + +| 能力 | 函数 | 结果 | 实测（jev-1.13.0，1070 个真实页面） | +|---|---|---|---| +| 多引擎同名去重 | `Verify` | `Duplicate` | 重复条目 851 → 0 | +| 去误报 | `Verify` | `Rejected`；名字出现在 header/cookie 中的命中永不否决 | 否决 49% 的规则命中；仅正文出现的疑似误报 138 → 13 | +| 漏报召回 | `Verify` 的 `recall` 参数 | `Recalled`，以 `guess` 来源加入 | 新增 468 个 | +| 技术栈分层 | `Verify` | `LayerOf(f)` | — | +| 主应用 | `Verify` | `Primary`，用 `PrimaryOf(frames)` 读取 | 433 页 | +| 版本号 | `Version(r, f)` | 写入 `f.Version`（只在置信度达标、且原来没有版本号时） | 对照 generator：191 对 / 6 错 / 77 缺失（纯规则：43 / 3 / 228） | +| 页面类型 | `Classify` | `page.Kind`：`KindLogin` `KindConsole` `KindError` `KindDefault` `KindDirListing` `KindAPI` `KindContent` | 标注集 36/40 | +| 通用页面 / 新指纹候选 | `Classify` | `page.Generic`；判为通用、但没有主应用的页面就是新指纹的候选 | 标注集 38/40 | +| 规则体检 | `cmd/judgeeval` | 按引擎、按规则统计否决率 | goby 65%，fingerprinthub 55% | + +做不到的： +- 认出改过品牌的产品（召回仍靠规则）； +- 凭空给出版本号（只能从代码抽取的候选里选）。 + +## 标注：Mark + +结果直接写在 `common.Frameworks` 上，**不删除任何条目**。存储形式是 `Framework.Tags` 里的 `judge:` 和 `judge:layer=`，所以 gogo、spray 现有的输出和序列化不需要改动。读取时请用函数，不要解析 tag： + +| API | 含义 | +|---|---| +| `judge.Is(f, judge.Rejected)` | 误报：不在技术栈里，或者只是正文里提到 | +| `judge.Is(f, judge.Duplicate)` | 另一个引擎对同一产品的另一种写法 | +| `judge.Is(f, judge.Primary)` | 本页所属的主应用 | +| `judge.Is(f, judge.Recalled)` | 规则漏掉、经判定确认后补入 | +| `judge.LayerOf(f)` | 所属层级：`LayerCDN` `LayerServer` `LayerRuntime` `LayerFramework` `LayerApplication` `LayerFrontend` `LayerDevice` `LayerNotPresent` | +| `judge.Judged(f)` | 是否已经判定过 | +| `judge.Accepted(frames)` | 去掉 `Rejected` 和 `Duplicate` 之后的结果 | +| `judge.PrimaryOf(frames)` | 主应用 | + +`Verify` 会跳过已经判定过的条目，所以对同一份 frames 重复调用 `Refine` 不会产生额外请求。 + +## 缓存与去重 + +默认开启，不需要任何配置： + +- **按问题缓存**：每个问题的缓存 key 由问题内容、具名证据、页面标题（数字归一化）和 Provider ID 组成，再加上页面签名（`Page.Signature()`：64 位 simhash；数字归一化，文本按双字切分，结构特征权重更高）。标题相同、签名相差不超过 `SimilarDistance`（默认 1）位的页面直接复用答案。问题怎么组合都能命中；部分命中时，只发送没命中的问题。 +- **并发合并**：多个 worker 同时判定相似页面、问同样的问题时，只有一个请求真正发出去，其余的等它的答案。 +- `Judge.Cache` 可以换成共享实现，比如 redis 或磁盘，只需要实现 `Get(key, sig)` / `Put(key, sig, value)`。设为 nil 就关闭缓存，并发合并仍然生效。 +- 统计：`Judge.Requests`（实际发出的请求数）、`Judge.CacheHits`（不发请求就答完的轮次数）。 + +`SimilarDistance` 的取值来自实测（`cmd/judgeeval`，1070 个真实页面，32284 个答案）：复用相似页面的答案，与该页面自己的答案对比： + +| 距离 ≤ | 可复用答案 | 一致率 | +|---|---|---| +| 0 | 136 | 100% | +| 1 | 159 | 100% | +| 2 | 194 | 98.5% | +| 3 | 225 | 96.9% | +| 5 | 293 | 94.9% | + +所以默认值取 1，只在实测零误差的范围内复用。标题一致这个前提同样必要：没有它的话，彼此无关、特征稀疏的页面（例如中文站点，都带 `Server: nginx` 和 jquery）会出现签名距离 0–3 的碰撞。 + +随机抽取的互联网站点之间，能复用的答案不到 1%。收益主要在扫描场景：同一主机的大量 404 页或登录页、同一网段里的同型号设备、托管商的人机验证页。完全相同的问题，只要页面相同或相似，都会走缓存。 + +## 自定义问题 + +每项能力都是往 `Round` 里加问题的函数，调用方也可以加自己的问题，和内置问题合并在同一次请求里： + +```go +page, _ := judge.NewPage(raw) +r := page.Round() +judge.Classify(r) +r.Add("honeypot", judge.Binary("Is this page a honeypot?"), func(a judge.Answer) { + honeypot = a.Yes >= j.Threshold +}) +err := r.Ask(ctx, j) +``` + +- 问题类型： + - `Binary`（`Answer.Yes` 为"是"的概率，每个问题独立判断，所以一页上可以同时认定多个产品存在）； + - `Choice`（`Answer.Choice` 和 `Confidence`，即在选项之间做 rerank）； + - `Score`（`Answer.Score`）。 +- `BinaryWith` 可以分别写明"是"和"否"的含义，判断会更准。 +- `r.Evidence(name, v)` 可以把一份具名证据放进 state（例如版本号候选列表）。模型读得很字面：具名给出的证据，比写在选项描述里的效果好得多。 +- 只有整轮请求成功后才会执行回调；请求失败时，页面和 frames 都不会被修改。 + +## 接入新的 Provider + +```go +type Provider interface { + ID() string // provider/模型版本；它是缓存 key 的一部分，模型换了就要换 ID + Judge(ctx context.Context, state interface{}, questions map[string]judge.Question) (map[string]judge.Answer, error) +} +``` + +- Provider 只负责把问题翻译成自己的协议并发送出去；缓存、合并、阈值和决策都不需要自己实现。可以参考 `judge/jev`（约 200 行）。 +- 可选实现 `Calibrated`，返回在标注集上测出来的阈值：`Calibration() (threshold, versionConfidence)`。`judge.New` 会采用它，否则使用 0.5 / 0.9。 +- 校准和对比：先在 `cmd/judgeeval` 的 `newProvider` 里注册，然后 + + ```bash + judgeeval -provider -samples testdata/samples -labels testdata/labels.json + ``` + + 用真值集调好阈值，再在真实页面上和纯规则做对比。 + +## 稳定性约定 + +- 对外 API：`New`、`Judge`、`Provider`、`Calibrated`、`Question`/`Answer` 及其构造函数、`Page`/`NewPage`、`Round`、`Verify`/`Classify`/`Version`/`Refine`、`Mark`/`Is`/`LayerOf`/`Judged`/`Accepted`/`PrimaryOf`、`Kind`、`Layer`、`Cache`/`NewMemoryCache`。 +- tag 的写法（`judge:*`）和 `Kind`、`Layer` 的取值属于输出格式，不会随意变更。 +- 问题的措辞、候选数量上限、版本号排序规则属于实现细节，可能随评测结果调整；每次调整都要用 `cmd/judgeeval` 回归。 diff --git a/jev/cache.go b/judge/cache.go similarity index 66% rename from jev/cache.go rename to judge/cache.go index 4b2bca9..cb20e5f 100644 --- a/jev/cache.go +++ b/judge/cache.go @@ -1,4 +1,4 @@ -package jev +package judge import ( "container/list" @@ -6,17 +6,22 @@ import ( "sync" ) -// SimilarDistance is how many of the 64 signature bits two pages may differ -// in to share cached answers; 0 shares answers only between identical pages. -var SimilarDistance = 3 +// SimilarDistance is how many of the 64 signature bits two pages with the +// same title may differ in to share cached answers; 0 shares answers only +// between pages with identical signatures. Measured with cmd/judgeeval on +// 1070 real pages: every answer reused at distance <= 1 matched the page's +// own answer (159/159), at 2 it was 98.5%, at 3 96.9%. +var SimilarDistance = 1 -// Cache stores successful responses. key covers everything that must match -// exactly: model, questions and named evidence such as version strings. sig -// is the page's Signature: a cache may return the answers of a page whose -// signature is within SimilarDistance bits, so near-identical pages (the -// same product's login page on many hosts) cost one request. Jev answers an -// identical request identically, so an exact hit is always valid. -// Implementations must be safe for concurrent use. +// Cache stores answers, one per question. key covers everything that must +// match exactly: model, question and named evidence such as version strings. +// sig is the page's Signature: Get may return the answer stored for a page +// whose signature is within SimilarDistance bits, so near-identical pages +// (one product's login page on many hosts, one site's 404 page on every +// path) are asked once. Providers answer identical input identically, so an exact +// hit is always valid. Implementations must be safe for concurrent use; the +// in-memory NewMemoryCache is the default, a shared one (redis, disk) lets +// several scanners share answers. type Cache interface { Get(key string, sig uint64) ([]byte, bool) Put(key string, sig uint64, value []byte) diff --git a/judge/classify.go b/judge/classify.go new file mode 100644 index 0000000..3219ab1 --- /dev/null +++ b/judge/classify.go @@ -0,0 +1,38 @@ +package judge + +// Kind is what a page is, set by Classify. +type Kind string + +const ( + KindLogin Kind = "login" + KindDefault Kind = "default_install" + KindError Kind = "error_page" + KindDirListing Kind = "directory_listing" + KindConsole Kind = "app_console" + KindContent Kind = "normal_content" + KindAPI Kind = "api_response" +) + +var pageKinds = map[string]string{ + string(KindLogin): "A sign-in form or authentication page", + string(KindDefault): "The stock page a web server or product shows right after installation", + string(KindError): "A framework or server error page such as 404 or 500", + string(KindDirListing): "An auto-generated file index of a directory", + string(KindConsole): "The main interface or loading screen of a web application or management console", + string(KindContent): "An ordinary website page: article, company site, shop, forum, blog, status page", + string(KindAPI): "A machine-readable API response rather than a web page", +} + +// Classify judges the page itself and writes Page.Kind and Page.Generic. It +// needs no fingerprints: spray can use it alone for 404 detection and +// prioritisation. A generic page without an accepted application is a +// candidate for a new fingerprint. +func Classify(r *Round) { + p := r.page + r.Add("page_kind", Choice("What kind of page is this HTTP response?", pageKinds), func(a Answer) { p.Kind = Kind(a.Choice) }) + r.Add("generic", BinaryWith("Is this page the stock interface of a packaged software product, framework or device, "+ + "so that the same page would appear on many unrelated deployments?", + "Login, console, default or error page shipped by a product; only branding, host names or data differ between installs", + "Content written by one organization for its own purpose: articles, company pages, shops, forums, custom-built portals"), + func(a Answer) { p.Generic = a.Yes >= r.judge.Threshold }) +} diff --git a/judge/doc.go b/judge/doc.go new file mode 100644 index 0000000..1d067e1 --- /dev/null +++ b/judge/doc.go @@ -0,0 +1,23 @@ +// Package judge is the judgement and rerank layer of fingers. It sits on top +// of the rule engines instead of being one: rules and code recall candidates +// with evidence, a Provider (a model such as TypeSafe Jev in judge/jev) +// answers typed questions about that evidence, and code makes every final +// decision. +// +// j := judge.New(provider) // shared by a whole scan: cache, merging, thresholds +// page, _ := judge.NewPage(raw) // page data entry +// r := page.Round() // one request +// judge.Verify(r, frames, recall) // false positives / negatives, duplicates, layer, primary +// judge.Classify(r) // page kind, generic page +// err := r.Ask(ctx, j) // answers mark frames and page in place +// r = page.Round() +// judge.Version(r, judge.PrimaryOf(frames)) // version of the primary application +// err = r.Ask(ctx, j) +// +// Refine runs exactly this; fingers.Engine.Refine adds recall from the +// fingerprint names. Results are Marks on the frameworks (Is, LayerOf, +// Accepted) and Kind / Generic on the page. Answers are cached per question, +// and similar pages (Page.Signature) share them. +// +// See README.md in this directory. +package judge diff --git a/judge/jev/jev.go b/judge/jev/jev.go new file mode 100644 index 0000000..7fc7374 --- /dev/null +++ b/judge/jev/jev.go @@ -0,0 +1,194 @@ +// Package jev is the TypeSafe Jev (System One) provider for package judge. +// It translates judge questions to Jev's wire format (binary -> noul) and +// sends them; everything else lives in judge. +// +// API reference: https://docs.typesafe.ai/api.md +package jev + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "sync/atomic" + "time" + + "github.com/chainreactors/fingers/judge" +) + +const ( + DefaultEndpoint = "https://api.typesafe.ai/v1/systemone" + // Pin a versioned model: the calibration below was measured on this one. + DefaultModel = "jev-1.13.0" + EnvAPIKey = "TYPESAFE_API_KEY" +) + +// Provider sends judge questions to Jev. +type Provider struct { + Endpoint string + Model string + APIKey string + HTTP *http.Client + MaxRetries int // on 429 / 529, with exponential backoff + + // InputTokens sent so far; read with atomic.LoadInt64. + InputTokens int64 +} + +// New reads the API key from TYPESAFE_API_KEY when apiKey is empty. +func New(apiKey string) (*Provider, error) { + if apiKey == "" { + apiKey = os.Getenv(EnvAPIKey) + } + if apiKey == "" { + return nil, fmt.Errorf("%s is not set", EnvAPIKey) + } + return &Provider{ + Endpoint: DefaultEndpoint, + Model: DefaultModel, + APIKey: apiKey, + HTTP: &http.Client{Timeout: 30 * time.Second}, + MaxRetries: 3, + }, nil +} + +// NewJudge is judge.New(New(apiKey)). +func NewJudge(apiKey string) (*judge.Judge, error) { + p, err := New(apiKey) + if err != nil { + return nil, err + } + return judge.New(p), nil +} + +func (p *Provider) ID() string { return "jev/" + p.Model } + +// Calibration: on jev-1.13.0, a Noul of 0.5 separated real from false hits +// on labelled pages, and 42 of 46 versions picked at >= 0.9 were right. +func (p *Provider) Calibration() (float64, float64) { return 0.5, 0.9 } + +type question struct { + Type string `json:"type"` + Instructions string `json:"instructions"` + Criteria interface{} `json:"criteria,omitempty"` +} + +type request struct { + State interface{} `json:"state"` + Model string `json:"model"` + Questions map[string]question `json:"questions"` +} + +type answer struct { + Type string `json:"type"` + Noul float64 `json:"noul,omitempty"` + Choice string `json:"choice,omitempty"` + Score float64 `json:"score,omitempty"` + Probabilities map[string]float64 `json:"probabilities,omitempty"` + Confidence float64 `json:"confidence,omitempty"` +} + +type response struct { + Model string `json:"model"` + Answers map[string]answer `json:"answers"` + Usage struct { + InputTokens int `json:"input_tokens"` + OutputTokens int `json:"output_tokens"` + } `json:"usage"` +} + +func toJev(q judge.Question) question { + out := question{Instructions: q.Instructions} + switch q.Type { + case judge.TypeBinary: + out.Type = "noul" + if len(q.Options) > 0 { + out.Criteria = q.Options + } + case judge.TypeChoice: + out.Type = "choice" + opts := make(map[string]interface{}, len(q.Options)) + for k, v := range q.Options { + if v == "" { + opts[k] = nil // Jev's "no description" + } else { + opts[k] = v + } + } + out.Criteria = opts + case judge.TypeScore: + out.Type = "score" + out.Criteria = q.Levels + } + return out +} + +type APIError struct { + Status int + Body string +} + +func (e *APIError) Error() string { return fmt.Sprintf("typesafe: http %d: %s", e.Status, e.Body) } + +func (p *Provider) Judge(ctx context.Context, state interface{}, questions map[string]judge.Question) (map[string]judge.Answer, error) { + req := request{State: state, Model: p.Model, Questions: make(map[string]question, len(questions))} + for k, q := range questions { + req.Questions[k] = toJev(q) + } + body, err := json.Marshal(req) + if err != nil { + return nil, err + } + backoff := time.Second + for attempt := 0; ; attempt++ { + resp, err := p.do(ctx, body) + if err == nil { + atomic.AddInt64(&p.InputTokens, int64(resp.Usage.InputTokens)) + out := make(map[string]judge.Answer, len(resp.Answers)) + for k, a := range resp.Answers { + out[k] = judge.Answer{Yes: a.Noul, Choice: a.Choice, Score: a.Score, Probabilities: a.Probabilities, Confidence: a.Confidence} + } + return out, nil + } + apiErr, ok := err.(*APIError) + if !ok || (apiErr.Status != 429 && apiErr.Status != 529) || attempt >= p.MaxRetries { + return nil, err + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(backoff): + } + backoff *= 2 + } +} + +func (p *Provider) do(ctx context.Context, body []byte) (*response, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, p.Endpoint, bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", "Bearer "+p.APIKey) + req.Header.Set("Content-Type", "application/json") + + resp, err := p.HTTP.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + raw, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + return nil, &APIError{Status: resp.StatusCode, Body: string(raw)} + } + var out response + if err := json.Unmarshal(raw, &out); err != nil { + return nil, fmt.Errorf("typesafe: decode response: %w", err) + } + return &out, nil +} diff --git a/judge/jev/jev_test.go b/judge/jev/jev_test.go new file mode 100644 index 0000000..e062986 --- /dev/null +++ b/judge/jev/jev_test.go @@ -0,0 +1,53 @@ +package jev + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/chainreactors/fingers/judge" +) + +func TestProviderWireFormatAndRetry(t *testing.T) { + calls := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + if r.Header.Get("Authorization") != "Bearer k" { + t.Errorf("auth header: %q", r.Header.Get("Authorization")) + } + if calls == 1 { + w.WriteHeader(529) + return + } + var req struct { + Model string + Questions map[string]map[string]interface{} + } + json.NewDecoder(r.Body).Decode(&req) + b, c := req.Questions["b"], req.Questions["c"] + criteria, _ := c["criteria"].(map[string]interface{}) + if req.Model != DefaultModel || b["type"] != "noul" || c["type"] != "choice" || criteria["x"] != nil || criteria["y"] != "why" { + t.Errorf("request: %+v", req) + } + w.Write([]byte(`{"model":"jev-1.13.0","answers":{"b":{"type":"noul","noul":0.9},"c":{"type":"choice","choice":"y","confidence":0.8}},"usage":{"input_tokens":10,"output_tokens":1}}`)) + })) + defer srv.Close() + + p, _ := New("k") + p.Endpoint = srv.URL + answers, err := p.Judge(context.Background(), "state", map[string]judge.Question{ + "b": judge.Binary("?"), + "c": judge.Choice("?", map[string]string{"x": "", "y": "why"}), + }) + if err != nil { + t.Fatal(err) + } + if calls != 2 || answers["b"].Yes != 0.9 || answers["c"].Choice != "y" || p.InputTokens != 10 { + t.Fatalf("calls=%d answers=%+v tokens=%d", calls, answers, p.InputTokens) + } + if j := judge.New(p); j.Threshold != 0.5 || j.VersionConfidence != 0.9 || p.ID() != "jev/jev-1.13.0" { + t.Fatalf("calibration %v %v id %s", j.Threshold, j.VersionConfidence, p.ID()) + } +} diff --git a/judge/judge.go b/judge/judge.go new file mode 100644 index 0000000..1427c7e --- /dev/null +++ b/judge/judge.go @@ -0,0 +1,186 @@ +package judge + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "sort" + "strings" + "sync" + "sync/atomic" +) + +// Judge puts a Provider to work: it answers questions from the cache where +// it can, merges concurrent rounds asking the same questions about similar +// pages, and holds the thresholds the capabilities decide with. One Judge is +// meant to be shared by every page of a scan. +type Judge struct { + Provider Provider + // Cache keeps answers per question, so a question asked again about the + // same or a similar page (see SimilarDistance) is not sent again, however + // questions are grouped into rounds. New sets an in-memory cache; nil + // disables it. Concurrent rounds are merged either way. + Cache Cache + // Threshold is the Yes probability that counts as "yes"; VersionConfidence + // the confidence at which a picked version is adopted. + Threshold, VersionConfidence float64 + + // Totals, for cost monitoring; read with atomic.LoadInt64. Requests went + // to the provider; CacheHits are rounds answered without one. + Requests, CacheHits int64 + + mu sync.Mutex + flights map[string][]*flight +} + +// DefaultCacheSize is the number of answers the cache New creates holds. +const DefaultCacheSize = 1 << 16 + +// New returns a Judge with an in-memory cache and the provider's +// calibration, or 0.5 / 0.9 for providers that report none. +func New(p Provider) *Judge { + j := &Judge{Provider: p, Cache: NewMemoryCache(DefaultCacheSize), Threshold: 0.5, VersionConfidence: 0.9} + if c, ok := p.(Calibrated); ok { + j.Threshold, j.VersionConfidence = c.Calibration() + } + return j +} + +// Ask asks questions about state, which must match exactly for cached +// answers to apply. Most callers use Round, which also matches similar pages. +func (j *Judge) Ask(ctx context.Context, state interface{}, questions map[string]Question) (map[string]Answer, error) { + scope, err := json.Marshal(state) + if err != nil { + return nil, err + } + return j.ask(ctx, state, questions, scope, 0) +} + +// flight is a request in progress; rounds asking the same questions about a +// similar page wait for it instead of sending their own. +type flight struct { + sig uint64 + done chan struct{} + answers map[string]Answer // by cache key; nil if the request failed +} + +// ask answers questions from the cache where it can and sends the rest in +// one request. A cached answer applies when the question, the scope (what +// must match exactly besides the question) and the model are equal and the +// page signature sig is similar. +func (j *Judge) ask(ctx context.Context, state interface{}, questions map[string]Question, scope []byte, sig uint64) (map[string]Answer, error) { + answers := make(map[string]Answer, len(questions)) + keys := make(map[string]string, len(questions)) // question key -> cache key + missing := map[string]Question{} + var pending []string + for k, q := range questions { + qj, err := json.Marshal(q) + if err != nil { + return nil, err + } + h := sha256.New() + for _, part := range [][]byte{[]byte(j.Provider.ID()), scope, qj} { + h.Write(part) + h.Write([]byte{0}) + } + ck := hex.EncodeToString(h.Sum(nil)) + keys[k] = ck + if a, ok := j.cached(ck, sig); ok { + answers[k] = a + continue + } + missing[k] = q + pending = append(pending, ck) + } + if len(missing) == 0 { + atomic.AddInt64(&j.CacheHits, 1) + return answers, nil + } + sort.Strings(pending) + fk := strings.Join(pending, ",") + for { + f, leader := j.join(fk, sig) + if !leader { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-f.done: + } + if f.answers == nil { + continue // the request we waited on failed: try again, possibly as the leader + } + for k := range missing { + answers[k] = f.answers[keys[k]] + } + atomic.AddInt64(&j.CacheHits, 1) + return answers, nil + } + got, err := j.Provider.Judge(ctx, state, missing) + var byKey map[string]Answer + if err == nil { + atomic.AddInt64(&j.Requests, 1) + byKey = make(map[string]Answer, len(got)) + for k, a := range got { + answers[k] = a + if ck, ok := keys[k]; ok { + byKey[ck] = a + if j.Cache != nil { + if v, err := json.Marshal(a); err == nil { + j.Cache.Put(ck, sig, v) + } + } + } + } + } + j.land(fk, f, byKey) + if err != nil { + return nil, err + } + return answers, nil + } +} + +func (j *Judge) cached(key string, sig uint64) (Answer, bool) { + var a Answer + if j.Cache == nil { + return a, false + } + v, ok := j.Cache.Get(key, sig) + return a, ok && json.Unmarshal(v, &a) == nil +} + +func (j *Judge) join(key string, sig uint64) (*flight, bool) { + j.mu.Lock() + defer j.mu.Unlock() + for _, f := range j.flights[key] { + if Similar(f.sig, sig) { + return f, false + } + } + if j.flights == nil { + j.flights = map[string][]*flight{} + } + f := &flight{sig: sig, done: make(chan struct{})} + j.flights[key] = append(j.flights[key], f) + return f, true +} + +func (j *Judge) land(key string, f *flight, answers map[string]Answer) { + j.mu.Lock() + fs := j.flights[key] + for i, x := range fs { + if x == f { + fs = append(fs[:i], fs[i+1:]...) + break + } + } + if len(fs) == 0 { + delete(j.flights, key) + } else { + j.flights[key] = fs + } + j.mu.Unlock() + f.answers = answers + close(f.done) +} diff --git a/jev/jev_test.go b/judge/judge_test.go similarity index 58% rename from jev/jev_test.go rename to judge/judge_test.go index 5414a83..84cca55 100644 --- a/jev/jev_test.go +++ b/judge/judge_test.go @@ -1,15 +1,16 @@ -package jev +package judge import ( "context" "encoding/json" + "errors" "fmt" - "net/http" - "net/http/httptest" "sort" "strings" + "sync" "sync/atomic" "testing" + "time" "github.com/chainreactors/fingers/common" ) @@ -41,12 +42,14 @@ func TestNewPage(t *testing.T) { } } -// mock answers every question from a table keyed by question kind and the -// product named in backticks, and records the question keys of each request. +// mock is a Provider answering from a table keyed by question kind and the +// product named in backticks; it records the question keys of each call. type mock struct { + mu sync.Mutex + delay time.Duration calls int64 requests [][]string - status int + fail bool versionP float64 present map[string]float64 layer map[string]Layer @@ -65,7 +68,7 @@ func newMock() *mock { } func named(q Question) string { - s, _ := q.Instructions.(string) + s := q.Instructions if i := strings.Index(s, "`"); i >= 0 { if j := strings.Index(s[i+1:], "`"); j >= 0 { return s[i+1 : i+1+j] @@ -74,50 +77,47 @@ func named(q Question) string { return "" } -func (m *mock) client(t *testing.T) (*Client, func()) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - atomic.AddInt64(&m.calls, 1) - if m.status != 0 { - w.WriteHeader(m.status) - return - } - var req request - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - t.Errorf("decode: %v", err) - } - m.lastState, _ = req.State.(map[string]interface{}) - answers := map[string]Answer{} - var keys []string - for k, q := range req.Questions { - keys = append(keys, k) - name := named(q) - switch { - case strings.HasPrefix(k, "is_"): - answers[k] = Answer{Type: "noul", Noul: m.present[name]} - case strings.HasPrefix(k, "layer_"): - answers[k] = Answer{Type: "choice", Choice: string(m.layer[name]), Confidence: 0.9} - case k == "primary": - answers[k] = Answer{Type: "choice", Choice: m.primary, Confidence: 0.9} - case k == "page_kind": - answers[k] = Answer{Type: "choice", Choice: "login", Confidence: 0.9} - case k == "generic": - answers[k] = Answer{Type: "noul", Noul: 0.9} - case k == "version": - answers[k] = Answer{Type: "choice", Choice: "2.401.3", Confidence: m.versionP} - default: - answers[k] = Answer{Type: "noul", Noul: 0.7} - } +func (m *mock) ID() string { return "mock" } + +func (m *mock) Judge(ctx context.Context, state interface{}, questions map[string]Question) (map[string]Answer, error) { + atomic.AddInt64(&m.calls, 1) + time.Sleep(m.delay) + m.mu.Lock() + defer m.mu.Unlock() + if m.fail { + return nil, errors.New("provider down") + } + m.lastState, _ = state.(map[string]interface{}) + answers := map[string]Answer{} + var keys []string + for k, q := range questions { + keys = append(keys, k) + name := named(q) + switch { + case strings.HasPrefix(k, "is_"): + answers[k] = Answer{Yes: m.present[name]} + case strings.HasPrefix(k, "layer_"): + answers[k] = Answer{Choice: string(m.layer[name]), Confidence: 0.9} + case k == "primary": + answers[k] = Answer{Choice: m.primary, Confidence: 0.9} + case k == "page_kind": + answers[k] = Answer{Choice: "login", Confidence: 0.9} + case k == "generic": + answers[k] = Answer{Yes: 0.9} + case k == "version": + answers[k] = Answer{Choice: "2.401.3", Confidence: m.versionP} + default: + answers[k] = Answer{Yes: 0.7} } - sort.Strings(keys) - m.requests = append(m.requests, keys) - json.NewEncoder(w).Encode(Response{Model: DefaultModel, Answers: answers, Usage: Usage{InputTokens: 100}}) - })) - c, _ := NewClient("k") - c.Endpoint = srv.URL - c.MaxRetries = 0 - return c, srv.Close + } + sort.Strings(keys) + m.requests = append(m.requests, keys) + return answers, nil } +// client keeps the old test shape: a Judge over the mock. +func (m *mock) client(t *testing.T) (*Judge, func()) { return New(m), func() {} } + func testFrames() common.Frameworks { fs := common.Frameworks{} fs.Add(common.NewFramework("nginx", common.FrameFromFingers)) @@ -128,7 +128,7 @@ func testFrames() common.Frameworks { return fs } -func refine(t *testing.T, c *Client, frames common.Frameworks) (*Page, error) { +func refine(t *testing.T, c *Judge, frames common.Frameworks) (*Page, error) { p, err := NewPage([]byte(jenkinsRaw)) if err != nil { t.Fatal(err) @@ -145,26 +145,26 @@ func TestRefine(t *testing.T) { if err != nil { t.Fatal(err) } - if !frames["wordpress"].HasTag(TagRejected) { + if !Is(frames["wordpress"], Rejected) { t.Errorf("text-only wordpress not rejected: %v", frames["wordpress"].Tags) } - if nginx := frames["nginx"]; nginx.HasTag(TagRejected) || !nginx.HasTag(TagLayer+string(LayerServer)) { + if nginx := frames["nginx"]; Is(nginx, Rejected) || LayerOf(nginx) != LayerServer { t.Errorf("nginx from the Server header must be kept: %v", nginx.Tags) } - if frames["apache tomcat"].HasTag(TagDup) == frames["apache-tomcat"].HasTag(TagDup) { + if Is(frames["apache tomcat"], Duplicate) == Is(frames["apache-tomcat"], Duplicate) { t.Errorf("exactly one tomcat spelling must be a dup") } j := frames["jenkins"] - if !j.HasTag(TagPrimary) || j.Version != "2.401.3" || Primary(frames) != j { + if !Is(j, Primary) || j.Version != "2.401.3" || PrimaryOf(frames) != j { t.Errorf("jenkins primary/version: %v %q", j.Tags, j.Version) } - if f := frames["prototype"]; f == nil || !f.HasTag(TagRecall) || !f.IsGuess() { + if f := frames["prototype"]; f == nil || !Is(f, Recalled) || !f.IsGuess() { t.Errorf("confirmed recall not added: %+v", f) } if got := Accepted(frames); len(got) != 4 { // nginx, tomcat, jenkins, prototype t.Errorf("accepted: %v", got) } - if p.Kind != "login" || !p.Generic { + if p.Kind != KindLogin || !p.Generic { t.Errorf("page: %q %v", p.Kind, p.Generic) } // Jenkins is both a rule hit and a recall name: asked once. 5 products x 2 + primary + page_kind + generic. @@ -174,8 +174,8 @@ func TestRefine(t *testing.T) { if _, ok := m.lastState["version_strings"]; !ok { t.Errorf("version strings not in the state: %v", m.lastState) } - if c.Requests != 2 || c.InputTokens != 200 { - t.Errorf("usage: %d %d", c.Requests, c.InputTokens) + if c.Requests != 2 { + t.Errorf("requests: %d", c.Requests) } } @@ -195,7 +195,7 @@ func TestVersionNeedsConfidence(t *testing.T) { func TestFailedRoundLeavesFrames(t *testing.T) { m := newMock() - m.status = 500 + m.fail = true c, stop := m.client(t) defer stop() frames := testFrames() @@ -236,19 +236,19 @@ func TestClassifyAlone(t *testing.T) { r := p.Round() Classify(r) var custom float64 - r.Add("honeypot", Noul("Is this a honeypot?"), func(a Answer) { custom = a.Noul }) + r.Add("honeypot", Binary("Is this a honeypot?"), func(a Answer) { custom = a.Yes }) if err := r.Ask(context.Background(), c); err != nil { t.Fatal(err) } - if strings.Join(m.requests[0], ",") != "generic,honeypot,page_kind" || p.Kind != "login" || custom != 0.7 { + if strings.Join(m.requests[0], ",") != "generic,honeypot,page_kind" || p.Kind != KindLogin || custom != 0.7 { t.Fatalf("requests=%v kind=%q custom=%v", m.requests, p.Kind, custom) } if err := p.Round().Ask(context.Background(), c); err != nil || m.calls != 1 { t.Fatalf("empty round sent a request: %v %d", err, m.calls) } r = p.Round() - r.Add("x", Noul("?"), nil) - r.Add("x", Noul("?"), nil) + r.Add("x", Binary("?"), nil) + r.Add("x", Binary("?"), nil) if err := r.Ask(context.Background(), c); err == nil { t.Fatal("duplicate key accepted") } @@ -269,8 +269,11 @@ func TestMemoryCacheEvicts(t *testing.T) { if _, ok := c.Get("a", 0xff); ok { t.Fatal("signature 8 bits away matched") } - if v, ok := c.Get("a", 0x3); !ok || string(v) != "1" { - t.Fatal("signature 2 bits away missed") + if v, ok := c.Get("a", 0x1); !ok || string(v) != "1" { + t.Fatal("signature 1 bit away missed") + } + if _, ok := c.Get("a", 0x3); ok { + t.Fatal("signature 2 bits away matched") } } @@ -281,19 +284,22 @@ func TestSimilarPagesShareAnswers(t *testing.T) { c, stop := m.client(t) defer stop() c.Cache = NewMemoryCache(16) - variant := strings.NewReplacer("JSESSIONID.1=x", "JSESSIONID.1=9f8e7d", "Sign in [Jenkins]", "Sign in [Jenkins]", - "We moved here from WordPress.", "We moved here from WordPress on 2026-09-24.").Replace(jenkinsRaw) - a, _ := NewPage([]byte(jenkinsRaw)) + // Same page on another host: other session, build number and date. + page := func(build, date string) string { + return strings.Replace(jenkinsRaw, "We moved here from WordPress.", "We moved here from WordPress. Build "+build+", "+date+".", 1) + } + orig, variant := page("101", "2026-09-24"), page("2087", "2025-01-03") + a, _ := NewPage([]byte(orig)) b, _ := NewPage([]byte(variant)) other, _ := NewPage([]byte("HTTP/1.1 200 OK\r\nServer: Apache\r\n\r\nIndex of /a.txt")) if !Similar(a.Signature(), b.Signature()) || Similar(a.Signature(), other.Signature()) { t.Fatalf("signatures: %x %x %x", a.Signature(), b.Signature(), other.Signature()) } - for _, raw := range []string{jenkinsRaw, variant} { + for _, raw := range []string{orig, variant} { p, _ := NewPage([]byte(raw)) r := p.Round() Classify(r) - if err := r.Ask(context.Background(), c); err != nil || p.Kind != "login" { + if err := r.Ask(context.Background(), c); err != nil || p.Kind != KindLogin { t.Fatalf("kind %q err %v", p.Kind, err) } } @@ -302,34 +308,17 @@ func TestSimilarPagesShareAnswers(t *testing.T) { } } -func TestAskRetriesOverload(t *testing.T) { - calls := 0 - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - calls++ - if r.Header.Get("Authorization") != "Bearer k" { - t.Errorf("auth header: %q", r.Header.Get("Authorization")) - } - if calls == 1 { - w.WriteHeader(529) - return - } - var req request - json.NewDecoder(r.Body).Decode(&req) - if req.Model != DefaultModel || req.Questions["q"].Type != "noul" { - t.Errorf("request: %+v", req) - } - w.Write([]byte(`{"model":"jev-1.13.0","answers":{"q":{"type":"noul","noul":0.9}},"usage":{"input_tokens":10,"output_tokens":1}}`)) - })) - defer srv.Close() +// A provider that reports a calibration sets the Judge's thresholds. +type calibrated struct{ mock } - c, _ := NewClient("k") - c.Endpoint = srv.URL - resp, err := c.Ask(context.Background(), "state", map[string]Question{"q": Noul("?")}) - if err != nil { - t.Fatal(err) +func (*calibrated) Calibration() (float64, float64) { return 0.7, 0.8 } + +func TestCalibration(t *testing.T) { + if j := New(newMock()); j.Threshold != 0.5 || j.VersionConfidence != 0.9 { + t.Fatalf("defaults: %v %v", j.Threshold, j.VersionConfidence) } - if calls != 2 || resp.Answers["q"].Noul != 0.9 { - t.Fatalf("calls=%d resp=%+v", calls, resp) + if j := New(&calibrated{}); j.Threshold != 0.7 || j.VersionConfidence != 0.8 { + t.Fatalf("calibration: %v %v", j.Threshold, j.VersionConfidence) } } @@ -393,7 +382,7 @@ func TestNormalizeName(t *testing.T) { // A server's own page has no primary application; the only server is the subject. func TestVersionWithoutPrimary(t *testing.T) { m := newMock() - m.primary = NoneOfThem + m.primary = noneOfThem c, stop := m.client(t) defer stop() frames := common.Frameworks{} @@ -401,7 +390,105 @@ func TestVersionWithoutPrimary(t *testing.T) { if _, err := refine(t, c, frames); err != nil { t.Fatal(err) } - if Primary(frames) != nil || frames["nginx"].Version == "" { + if PrimaryOf(frames) != nil || frames["nginx"].Version == "" { t.Fatalf("subject version not asked: %+v", frames["nginx"]) } } + +// Refine on frames it already judged sends nothing: Verify skips judged hits, +// Classify is answered by the cache, and the version is already set. +func TestRefineIsIdempotent(t *testing.T) { + m := newMock() + c, stop := m.client(t) + defer stop() + frames := testFrames() + for i := 0; i < 2; i++ { + if _, err := refine(t, c, frames); err != nil { + t.Fatal(err) + } + } + if m.calls != 2 || !Is(frames["wordpress"], Rejected) || len(frames["wordpress"].Tags) != len(testFrames()["wordpress"].Tags)+2 { + t.Fatalf("calls=%d wordpress=%v requests=%v", m.calls, frames["wordpress"].Tags, m.requests) + } +} + +// Many workers judging the same page at once (a scan hitting one product +// everywhere) cost one request. +func TestConcurrentSimilarPagesMerge(t *testing.T) { + m := newMock() + m.delay = 100 * time.Millisecond + c, stop := m.client(t) + defer stop() + var wg sync.WaitGroup + kinds := make([]Kind, 20) + for i := range kinds { + wg.Add(1) + go func(i int) { + defer wg.Done() + raw := strings.Replace(jenkinsRaw, "JSESSIONID.1=x", fmt.Sprintf("JSESSIONID.1=%d", i), 1) + p, _ := NewPage([]byte(raw)) + r := p.Round() + Classify(r) + if err := r.Ask(context.Background(), c); err != nil { + t.Error(err) + } + kinds[i] = p.Kind + }(i) + } + wg.Wait() + for i, k := range kinds { + if k != KindLogin { + t.Fatalf("worker %d: kind %q", i, k) + } + } + if m.calls != 1 || c.CacheHits != 19 { + t.Fatalf("calls=%d hits=%d", m.calls, c.CacheHits) + } +} + +// A round partly answered by the cache sends only the rest. +func TestPartialCacheHit(t *testing.T) { + m := newMock() + c, stop := m.client(t) + defer stop() + p, _ := NewPage([]byte(jenkinsRaw)) + r := p.Round() + Classify(r) + if err := r.Ask(context.Background(), c); err != nil { + t.Fatal(err) + } + p, _ = NewPage([]byte(jenkinsRaw)) + r = p.Round() + Classify(r) + r.Add("extra", Binary("Is this page served over a CDN?"), nil) + if err := r.Ask(context.Background(), c); err != nil || p.Kind != KindLogin { + t.Fatalf("err %v kind %q", err, p.Kind) + } + if len(m.requests) != 2 || strings.Join(m.requests[1], ",") != "extra" { + t.Fatalf("requests: %v", m.requests) + } +} + +// Without a primary, the only server is the subject even next to an OS. +func TestVersionSubjectPrefersServer(t *testing.T) { + frames := common.Frameworks{} + for name, l := range map[string]Layer{"apache": LayerServer, "ubuntu": LayerDevice} { + f := common.NewFramework(name, common.FrameFromFingers) + setLayer(f, l) + frames.Add(f) + } + if f := subject(frames); f == nil || f.Name != "apache" { + t.Fatalf("subject: %+v", f) + } +} + +// Sparse pages with common headers must not look alike: CJK text counts, and +// similar pages must share a title. +func TestUnrelatedSparsePagesDoNotShare(t *testing.T) { + head := "HTTP/1.1 200 OK\r\nServer: nginx\r\nX-Frame-Options: SAMEORIGIN\r\n\r\n" + a, _ := NewPage([]byte(head + "职业规划咨询

生涯规划师与高考志愿规划，帮助学生找到方向

")) + b, _ := NewPage([]byte(head + "官方网站

在线娱乐平台，注册即送体验金

")) + if Similar(a.Signature(), b.Signature()) && a.similarityScope() == b.similarityScope() { + t.Fatalf("unrelated pages share answers: %x %x", a.Signature(), b.Signature()) + } +} diff --git a/judge/mark.go b/judge/mark.go new file mode 100644 index 0000000..a73b3fc --- /dev/null +++ b/judge/mark.go @@ -0,0 +1,77 @@ +package judge + +import ( + "strings" + + "github.com/chainreactors/fingers/common" +) + +// Mark is a judgement written onto a framework. Marks live in Framework.Tags +// as "judge:" (and the layer as "judge:layer="), so they serialize +// with every existing output and need no new field; read them with Is and +// LayerOf rather than parsing tags. +type Mark string + +const ( + Rejected Mark = "rejected" // not in the stack, or only mentioned in text: a false positive + Duplicate Mark = "dup" // another engine's spelling of a product already in the result + Primary Mark = "primary" // the application the page belongs to + Recalled Mark = "recall" // missed by the rules, found by name and confirmed by the judge +) + +// TagPrefix starts every tag this package writes. +const TagPrefix = "judge:" + +const layerTag = TagPrefix + "layer=" + +// Tag is the Framework.Tags spelling of m. +func (m Mark) Tag() string { return TagPrefix + string(m) } + +// Is reports whether f carries m. +func Is(f *common.Framework, m Mark) bool { return f != nil && f.HasTag(m.Tag()) } + +// LayerOf returns the layer f was judged to be at, or "" if f was not judged. +func LayerOf(f *common.Framework) Layer { + if f == nil { + return "" + } + for _, t := range f.Tags { + if strings.HasPrefix(t, layerTag) { + return Layer(strings.TrimPrefix(t, layerTag)) + } + } + return "" +} + +// Judged reports whether f went through Verify. +func Judged(f *common.Framework) bool { return LayerOf(f) != "" } + +func mark(f *common.Framework, m Mark) { f.AddTag(m.Tag()) } + +func setLayer(f *common.Framework, l Layer) { + if l != "" && LayerOf(f) == "" { + f.AddTag(layerTag + string(l)) + } +} + +// PrimaryOf returns the framework marked Primary. +func PrimaryOf(frames common.Frameworks) *common.Framework { + for _, f := range frames { + if Is(f, Primary) { + return f + } + } + return nil +} + +// Accepted returns frames without Rejected hits and Duplicate spellings: one +// framework per product that is really there. +func Accepted(frames common.Frameworks) common.Frameworks { + out := common.Frameworks{} + for k, f := range frames { + if !Is(f, Rejected) && !Is(f, Duplicate) { + out[k] = f + } + } + return out +} diff --git a/jev/page.go b/judge/page.go similarity index 85% rename from jev/page.go rename to judge/page.go index d264c22..284fe35 100644 --- a/jev/page.go +++ b/judge/page.go @@ -1,4 +1,4 @@ -package jev +package judge import ( "bufio" @@ -12,8 +12,9 @@ import ( ) // Page is the entry point of the judgement layer: the compact, named view of -// one HTTP response that Jev judges (serialized as the request state), plus -// the page-level conclusions written back by Classify. Jev only judges the +// one HTTP response that providers judge (serialized as the request state), +// plus the page-level conclusions written back by Classify. A provider only +// judges the // evidence it is given and loses accuracy on large states full of irrelevant // detail, so the fields below are the ones that carry fingerprint signal. type Page struct { @@ -29,8 +30,8 @@ type Page struct { Forms []string `json:"form_inputs,omitempty"` Text string `json:"visible_text,omitempty"` - Kind string `json:"-"` // one of PageKinds, set by Classify - Generic bool `json:"-"` // stock page of a packaged product, set by Classify + Kind Kind `json:"-"` // set by Classify + Generic bool `json:"-"` // stock page of a packaged product, set by Classify raw []byte } @@ -187,10 +188,32 @@ func (s *Page) Evidence(name string) []string { var reDigits = regexp.MustCompile(`[0-9]+`) +// bigrams are overlapping two-rune shingles of text with spaces removed, so +// text without word separators (Chinese, Japanese) weighs like any other. +func bigrams(text string) []string { + rs := []rune(strings.Join(strings.Fields(strings.ToLower(text)), " ")) + var out []string + for i := 0; i+1 < len(rs); i++ { + if rs[i] != ' ' && rs[i+1] != ' ' { + out = append(out, string(rs[i:i+2])) + } + } + return out +} + +// similarityScope is what similar pages must share exactly: their title +// (digits folded). Signatures alone collide on sparse pages dominated by +// common headers and assets; pages that really are one product's stock page +// share a title. +func (s *Page) similarityScope() string { + return reDigits.ReplaceAllString(strings.ToLower(strings.TrimSpace(s.Title)), "0") +} + // Signature is a 64-bit simhash of the page's evidence. Digits are folded so // timestamps, build ids and tokens do not matter; structure (headers, -// assets, forms) weighs more than visible text. Pages of one product on -// different hosts get signatures a few bits apart; see SimilarDistance. +// assets, forms) weighs more than visible text, which counts by two-rune +// shingles. Near-identical pages get signatures a few bits apart; see +// SimilarDistance. The cache also requires equal titles. func (s *Page) Signature() uint64 { var v [64]int add := func(kind, feature string, weight int) { @@ -219,11 +242,11 @@ func (s *Page) Signature() uint64 { } } add("generator", s.Generator, 3) - for _, w := range strings.Fields(s.Title) { - add("title", w, 2) + for _, g := range bigrams(s.Title) { + add("title", g, 2) } - for _, w := range strings.Fields(s.Text) { - add("text", w, 1) + for _, g := range bigrams(s.Text) { + add("text", g, 1) } var sig uint64 for i := 0; i < 64; i++ { diff --git a/judge/provider.go b/judge/provider.go new file mode 100644 index 0000000..48a539a --- /dev/null +++ b/judge/provider.go @@ -0,0 +1,70 @@ +package judge + +import "context" + +// Provider answers typed questions about a state. It is how a model plugs in: +// TypeSafe Jev (package judge/jev), an LLM, a local classifier or a +// cross-encoder reranker. A provider only transports questions and answers; +// caching, request merging, thresholds and every decision stay in Judge and +// the capabilities, so they behave the same whichever provider answers. +type Provider interface { + // ID names the provider and model version. It scopes cached answers: + // change it whenever the same question may be answered differently. + ID() string + // Judge answers every question in one call, keyed as questions are. + // A missing answer is treated as no answer, not as an error. + Judge(ctx context.Context, state interface{}, questions map[string]Question) (map[string]Answer, error) +} + +// Calibrated is implemented by providers whose answers were measured on a +// labelled set (see cmd/judgeeval): New then uses their thresholds. +type Calibrated interface { + // Calibration returns the Yes probability that counts as "yes" and the + // confidence at which a picked version is adopted. + Calibration() (threshold, versionConfidence float64) +} + +// QuestionType is the answer shape a question asks for. +type QuestionType string + +const ( + TypeBinary QuestionType = "binary" // Answer.Yes: probability of "yes"; independent per question + TypeChoice QuestionType = "choice" // Answer.Choice: one of Options, with Confidence: a rerank over options + TypeScore QuestionType = "score" // Answer.Score: position on Levels, from 0 to 1 +) + +// Question is provider-neutral: plain instructions plus the options or levels. +type Question struct { + Type QuestionType `json:"type"` + Instructions string `json:"instructions"` + // Options: for TypeChoice, option -> description ("" for none); for + // TypeBinary, optional descriptions of "true" and "false". + Options map[string]string `json:"options,omitempty"` + Levels []string `json:"levels,omitempty"` // TypeScore: 2..10 ordered level descriptions +} + +func Binary(instructions string) Question { + return Question{Type: TypeBinary, Instructions: instructions} +} + +// BinaryWith describes what "yes" and "no" mean, which sharpens the answer. +func BinaryWith(instructions, yes, no string) Question { + return Question{Type: TypeBinary, Instructions: instructions, Options: map[string]string{"true": yes, "false": no}} +} + +func Choice(instructions string, options map[string]string) Question { + return Question{Type: TypeChoice, Instructions: instructions, Options: options} +} + +func Score(instructions string, levels []string) Question { + return Question{Type: TypeScore, Instructions: instructions, Levels: levels} +} + +// Answer is one answer; only the fields of the question's type are set. +type Answer struct { + Yes float64 `json:"yes,omitempty"` + Choice string `json:"choice,omitempty"` + Score float64 `json:"score,omitempty"` + Probabilities map[string]float64 `json:"probabilities,omitempty"` // choice: per option + Confidence float64 `json:"confidence,omitempty"` +} diff --git a/jev/recall.go b/judge/recall.go similarity index 91% rename from jev/recall.go rename to judge/recall.go index 88f952f..c1cdbe0 100644 --- a/jev/recall.go +++ b/judge/recall.go @@ -1,4 +1,4 @@ -package jev +package judge import ( "sort" @@ -7,8 +7,8 @@ import ( ) // Retriever finds fingerprint names that literally occur in a page. Every -// candidate handed to Jev is therefore anchored to evidence found by code, -// so Jev only ranks and verifies, it never invents a product. +// candidate handed to the provider is therefore anchored to evidence found +// by code, so the provider only ranks and verifies, it never invents a product. type Retriever struct { names []string // display names keys []string // lowercase match keys, same index as names diff --git a/judge/refine.go b/judge/refine.go new file mode 100644 index 0000000..f206206 --- /dev/null +++ b/judge/refine.go @@ -0,0 +1,47 @@ +package judge + +import ( + "context" + + "github.com/chainreactors/fingers/common" +) + +// Refine is the default pipeline: Verify and Classify in one request, then +// the version of the page's subject in a second one. frames and p are +// annotated in place; on error the second round's annotations are missing, +// the first round's are complete or absent. +func Refine(ctx context.Context, j *Judge, p *Page, frames common.Frameworks, recall []string) error { + r := p.Round() + Verify(r, frames, recall) + Classify(r) + if err := r.Ask(ctx, j); err != nil { + return err + } + r = p.Round() + Version(r, subject(frames)) + return r.Ask(ctx, j) +} + +// subject is the product whose version the page most likely shows: the +// primary application or, on pages without one (a server's default, error +// or index page), the only accepted application, else the only server, else +// the only device. +func subject(frames common.Frameworks) *common.Framework { + if f := PrimaryOf(frames); f != nil { + return f + } + accepted := Accepted(frames) + for _, l := range []Layer{LayerApplication, LayerServer, LayerDevice} { + var only *common.Framework + n := 0 + for _, f := range accepted { + if LayerOf(f) == l { + only, n = f, n+1 + } + } + if n == 1 { + return only + } + } + return nil +} diff --git a/jev/round.go b/judge/round.go similarity index 66% rename from jev/round.go rename to judge/round.go index 15e19f5..579b18e 100644 --- a/jev/round.go +++ b/judge/round.go @@ -1,4 +1,4 @@ -package jev +package judge import ( "context" @@ -6,7 +6,7 @@ import ( "fmt" ) -// Round is the Jev primitive: questions about one page, each with a callback, +// Round is the judge primitive: questions about one page, each with a callback, // sent as a single request. Capabilities (Verify, Classify, Version) are // functions that add to a Round; callers may add their own questions too. // Callbacks run only after the whole request succeeded, so a failed Round @@ -17,6 +17,7 @@ type Round struct { apply map[string]func(Answer) evidence map[string]interface{} done []func() + judge *Judge // set by Ask, for thresholds in callbacks err error } @@ -29,7 +30,7 @@ func (r *Round) Page() *Page { return r.page } // Add asks q under key; apply receives its answer. Keys must be unique in a Round. func (r *Round) Add(key string, q Question, apply func(Answer)) { if _, dup := r.questions[key]; dup && r.err == nil { - r.err = fmt.Errorf("jev: duplicate question key %q", key) + r.err = fmt.Errorf("judge: duplicate question key %q", key) } r.questions[key] = q r.apply[key] = apply @@ -39,9 +40,10 @@ func (r *Round) Add(key string, q Question, apply func(Answer)) { // order: for decisions that combine several answers. func (r *Round) Done(f func()) { r.done = append(r.done, f) } -// Evidence shows Jev a named piece of evidence besides the page; the state -// becomes {"response": page, name: value}. Jev reads literally: evidence it is -// asked about by name weighs more than the same text in option descriptions. +// Evidence shows the provider a named piece of evidence besides the page; the state +// becomes {"response": page, name: value}. Providers read literally: evidence +// they are asked about by name weighs more than the same text in option +// descriptions. func (r *Round) Evidence(name string, value interface{}) { if r.evidence == nil { r.evidence = map[string]interface{}{} @@ -50,7 +52,7 @@ func (r *Round) Evidence(name string, value interface{}) { } // Ask sends the round; a round without questions sends nothing. -func (r *Round) Ask(ctx context.Context, c *Client) error { +func (r *Round) Ask(ctx context.Context, j *Judge) error { if r.err != nil { return r.err } @@ -65,18 +67,19 @@ func (r *Round) Ask(ctx context.Context, c *Client) error { } state = m } - // Similar pages share answers: the cache matches the page by signature, - // everything else (questions, named evidence such as version strings) exactly. - exact, err := json.Marshal([]interface{}{r.questions, r.evidence}) + // Answers are cached per question: the page matches by signature (similar + // pages share answers) plus its title, named evidence exactly. + scope, err := json.Marshal([]interface{}{r.page.similarityScope(), r.evidence}) if err != nil { return err } - resp, err := c.ask(ctx, state, r.questions, exact, r.page.Signature()) + answers, err := j.ask(ctx, state, r.questions, scope, r.page.Signature()) if err != nil { return err } + r.judge = j for key, apply := range r.apply { - if a, ok := resp.Answers[key]; ok && apply != nil { + if a, ok := answers[key]; ok && apply != nil { apply(a) } } diff --git a/jev/stack.go b/judge/stack.go similarity index 95% rename from jev/stack.go rename to judge/stack.go index 17d8a92..6f18bd8 100644 --- a/jev/stack.go +++ b/judge/stack.go @@ -1,4 +1,4 @@ -package jev +package judge import ( "strings" @@ -22,8 +22,8 @@ const ( LayerNotPresent Layer = "not_present" ) -// LayerCriteria is the Choice criteria for classifying one candidate's layer. -var LayerCriteria = map[string]interface{}{ +// layerCriteria is the Choice criteria for classifying one candidate's layer. +var layerCriteria = map[string]string{ string(LayerCDN): "A CDN, reverse proxy cloud or web application firewall in front of the site", string(LayerServer): "The HTTP server or proxy software, such as nginx, Apache httpd, IIS, Tomcat", string(LayerRuntime): "A programming language or runtime, such as PHP, Java, ASP.NET, Node.js", diff --git a/jev/verify.go b/judge/verify.go similarity index 54% rename from jev/verify.go rename to judge/verify.go index ae8c7b1..fe89d74 100644 --- a/jev/verify.go +++ b/judge/verify.go @@ -1,4 +1,4 @@ -package jev +package judge import ( "fmt" @@ -7,25 +7,11 @@ import ( "github.com/chainreactors/fingers/common" ) -// Threshold is the Noul at or above which Jev's "yes" counts. Tuned for DefaultModel. -var Threshold = 0.5 +const noneOfThem = "none_of_these" // primary option for pages of no listed product -// Tags written onto frameworks by Verify. Frameworks are annotated in place, -// never removed: callers decide whether to drop what Jev rejected. -const ( - TagRejected = "jev:rejected" // rule hit Jev judged absent or only mentioned in text - TagDup = "jev:dup" // another spelling of a product already in frames - TagPrimary = "jev:primary" // the application the page belongs to - TagRecall = "jev:recall" // added: no rule hit, found by name and confirmed by Jev - TagLayer = "jev:layer=" // prefix; followed by a Layer -) - -// NoneOfThem is the primary option for pages that belong to no listed product. -const NoneOfThem = "none_of_these" - -// ProtocolFeatures are HTTP features some engines report as fingerprints +// protocolFeatures are HTTP features some engines report as fingerprints // (NormalizeName keys). They are facts of the response head: never rejected. -var ProtocolFeatures = map[string]bool{"hsts": true, "altsvc": true, "http3": true, "http2": true, "poweredby": true, "http基本认证": true} +var protocolFeatures = map[string]bool{"hsts": true, "altsvc": true, "http3": true, "http2": true, "poweredby": true, "http基本认证": true} const ( maxGroups = 40 // products judged per page @@ -37,19 +23,31 @@ const ( type product struct { name string frames []*common.Framework - strong bool // named in a header or cookie: code accepts it, Jev under-trusts it + strong bool // named in a header or cookie: code accepts it, models under-trust it } // Verify is the engine result filter. It folds the spellings of all engines -// into products, asks Jev per product whether it is really in the stack (a -// Noul, so several products can be present) and at which layer, and which +// into products, asks the provider per product whether it is really in the stack (a +// binary question, so several products can be present) and at which layer (a choice), and which // product is the page's application. recall names (e.g. fingerprint names // found in the page) that no rule reported are judged the same way; confirmed // ones are added to frames. Code decides: a product named in a header or // cookie is never rejected. +// +// Results are Marks and a layer on frames (see Is, LayerOf, Accepted). +// Frameworks judged before are skipped, so calling Verify again on the same +// frames asks only about new hits. func Verify(r *Round, frames common.Frameworks, recall []string) { page := r.page - list := frames.List() + var list []*common.Framework + judged := map[string]bool{} // keys judged before: not asked again, not recalled again + for _, f := range frames { + if Judged(f) { + judged[NormalizeName(f.Name)] = true + } else { + list = append(list, f) + } + } sort.Slice(list, func(a, b int) bool { return list[a].Name < list[b].Name }) byKey := map[string]*product{} var products []*product @@ -63,7 +61,7 @@ func Verify(r *Round, frames common.Frameworks, recall []string) { if len(products) >= maxGroups { continue } - p = &product{name: f.Name, strong: ProtocolFeatures[key]} + p = &product{name: f.Name, strong: protocolFeatures[key]} byKey[key] = p products = append(products, p) } @@ -75,7 +73,7 @@ func Verify(r *Round, frames common.Frameworks, recall []string) { recalled := 0 for _, name := range recall { key := NormalizeName(name) - if _, ok := byKey[key]; ok || key == "" { + if _, ok := byKey[key]; ok || key == "" || judged[key] { continue } if recalled >= maxRecall { @@ -97,26 +95,22 @@ func Verify(r *Round, frames common.Frameworks, recall []string) { } answers := make([]answer, len(products)) var primary string - primaryOpts := map[string]interface{}{NoneOfThem: "The page belongs to none of the listed products, e.g. a custom-built site"} + primaryOpts := map[string]string{noneOfThem: "The page belongs to none of the listed products, e.g. a custom-built site"} for i, p := range products { i, name := i, p.name - primaryOpts[name] = nil - r.Add(fmt.Sprintf("is_%d", i), Question{ - Type: "noul", - Instructions: fmt.Sprintf("Is `%s` part of the software stack that produced this HTTP response "+ - "(seen in its headers, cookies, asset paths, page structure or title), rather than only mentioned in the page text?", name), - Criteria: map[string]string{ - "true": name + " served, generated or is loaded by this response", - "false": name + " only appears in the page's text, or is not present", - }, - }, func(a Answer) { answers[i].present, answers[i].answered = a.Noul, true }) - r.Add(fmt.Sprintf("layer_%d", i), Choice(fmt.Sprintf("What role does `%s` play in producing this HTTP response?", name), LayerCriteria), + primaryOpts[name] = "" + r.Add(fmt.Sprintf("is_%d", i), BinaryWith(fmt.Sprintf("Is `%s` part of the software stack that produced this HTTP response "+ + "(seen in its headers, cookies, asset paths, page structure or title), rather than only mentioned in the page text?", name), + name+" served, generated or is loaded by this response", + name+" only appears in the page's text, or is not present"), + func(a Answer) { answers[i].present, answers[i].answered = a.Yes, true }) + r.Add(fmt.Sprintf("layer_%d", i), Choice(fmt.Sprintf("What role does `%s` play in producing this HTTP response?", name), layerCriteria), func(a Answer) { answers[i].layer = Layer(a.Choice) }) } r.Add("primary", Choice("Which listed product is the main application this page belongs to "+ "(the product whose login, console or content this is), as opposed to servers, frameworks and libraries underneath it?", primaryOpts), func(a Answer) { - if a.Choice != NoneOfThem { + if a.Choice != noneOfThem { primary = a.Choice } }) @@ -124,29 +118,27 @@ func Verify(r *Round, frames common.Frameworks, recall []string) { r.Done(func() { for i, p := range products { a := answers[i] - rejected := !p.strong && a.answered && (a.layer == LayerNotPresent || a.present < Threshold) - if p.frames == nil { // recall: added only when Jev confirms it + rejected := !p.strong && a.answered && (a.layer == LayerNotPresent || a.present < r.judge.Threshold) + if p.frames == nil { // recall: added only when the provider confirms it if !a.answered || rejected { continue } f := common.NewFramework(p.name, common.FrameFromGUESS) - f.AddTag(TagRecall) + mark(f, Recalled) frames.Add(f) p.frames = []*common.Framework{f} } for j, f := range p.frames { - if a.layer != "" { - f.AddTag(TagLayer + string(a.layer)) - } + setLayer(f, a.layer) if rejected { - f.AddTag(TagRejected) + mark(f, Rejected) } if j > 0 { - f.AddTag(TagDup) + mark(f, Duplicate) } } if p.name == primary && !rejected { - p.frames[0].AddTag(TagPrimary) + mark(p.frames[0], Primary) } } }) diff --git a/jev/version.go b/judge/version.go similarity index 82% rename from jev/version.go rename to judge/version.go index 9868147..af5e055 100644 --- a/jev/version.go +++ b/judge/version.go @@ -1,4 +1,4 @@ -package jev +package judge import ( "fmt" @@ -9,21 +9,17 @@ import ( "github.com/chainreactors/fingers/common" ) -// VersionConfidence is the confidence at or above which a version is written. -// Tuned for DefaultModel: 42 of 46 such answers were right on real pages. -var VersionConfidence = 0.9 - const ( - NotStated = "not_stated" + notStated = "not_stated" maxVersions = 40 // extracted from the raw response - maxVersionsShown = 15 // shown to Jev after ranking + maxVersionsShown = 15 // shown to the provider after ranking ) // Version adds to r the question which extracted version string is the -// version of f, and writes it to f when Jev is confident. It does nothing when +// version of f, and writes it to f when the provider is confident. It does nothing when // f is nil, already has a version, or the page shows no version-like string. -// Jev cannot generate strings: code extracts, Jev only picks. +// Code extracts, the provider only picks: it never has to generate a string. func Version(r *Round, f *common.Framework) { if f == nil || (f.Attributes != nil && f.Attributes.Version != "") { return @@ -32,18 +28,18 @@ func Version(r *Round, f *common.Framework) { if len(cands) == 0 { return } - // The candidates go into the state as a named list: Jev reads literally, - // and picked 100% right this way against 78% with them in the options. - opts := map[string]interface{}{NotStated: "The response does not show the version of " + f.Name} + // The candidates go into the state as a named list: Jev picked 100% right + // this way against 78% with them in the option descriptions. + opts := map[string]string{notStated: "The response does not show the version of " + f.Name} for _, c := range cands { - opts[c.Value] = nil + opts[c.Value] = "" } r.Evidence("version_strings", cands) r.Add("version", Choice(fmt.Sprintf("`version_strings` lists every version-like string found in the raw response, with the text around it. "+ "Which one is the version of `%s`? A version in an asset URL parameter, a meta tag or embedded build info of %s counts. "+ "Ignore versions of third-party libraries, other products mentioned in text, years, timestamps and build numbers.", f.Name, f.Name), opts), func(a Answer) { - if a.Choice == "" || a.Choice == NotStated || a.Confidence < VersionConfidence { + if a.Choice == "" || a.Choice == notStated || a.Confidence < r.judge.VersionConfidence { return } // copy on write: Attributes may be shared with other frameworks @@ -58,16 +54,16 @@ func Version(r *Round, f *common.Framework) { } // versionString is a version-like string with the raw text around its first -// occurrence, so Jev can tell a product version from a library version. The -// JSON names are what Jev reads. +// occurrence, so the provider can tell a product version from a library version. The +// JSON names are what the provider reads. type versionString struct { Value string `json:"value"` Context string `json:"found_in"` } // extractVersions scans the raw response (headers and body, including -// inline scripts and meta tags that Page drops). Jev cannot generate -// strings, so versions are extracted by regex and Jev only picks one. +// inline scripts and meta tags that Page drops). Providers do not +// generate strings: versions are extracted by regex and one is picked. func extractVersions(raw []byte, max int) []versionString { text := string(raw) if i := strings.IndexByte(text, '\n'); i >= 0 { // skip the "HTTP/1.1 200" status line @@ -153,7 +149,7 @@ func looksLikeIPv4(v string) bool { // rankVersions orders candidates by how likely their context names // the version of product, and keeps the first max. Pages often carry dozens // of "?ver=" strings, so without ranking the one real version (a generator -// meta or a header) can fall outside the list Jev sees. +// meta or a header) can fall outside the list the provider sees. func rankVersions(cands []versionString, product string, max int) []versionString { key := NormalizeName(product) score := func(c versionString) int { diff --git a/judge_test.go b/judge_test.go new file mode 100644 index 0000000..60e28bf --- /dev/null +++ b/judge_test.go @@ -0,0 +1,50 @@ +package fingers + +import ( + "context" + "strings" + "testing" + + "github.com/chainreactors/fingers/judge" +) + +func TestRefineWithJudge(t *testing.T) { + engine, err := NewEngine(FingersEngine, WappalyzerEngine) + if err != nil { + t.Fatal(err) + } + raw := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nWelcome to nginx!") + frames, _ := engine.DetectContent(raw) + if _, err := engine.Refine(context.Background(), raw, frames); err != ErrNoJudge { + t.Fatalf("want ErrNoJudge, got %v", err) + } + + engine.AttachJudge(judge.New(server{})) + page, err := engine.Refine(context.Background(), raw, frames) + if err != nil { + t.Fatal(err) + } + if page.Kind != judge.KindDefault || judge.LayerOf(frames["nginx"]) != judge.LayerServer { + t.Fatalf("kind %q, nginx %v", page.Kind, frames["nginx"].Tags) + } +} + +// server is a provider that judges every product present at the server layer. +type server struct{} + +func (server) ID() string { return "test" } + +func (server) Judge(ctx context.Context, state interface{}, questions map[string]judge.Question) (map[string]judge.Answer, error) { + answers := map[string]judge.Answer{} + for k := range questions { + switch { + case strings.HasPrefix(k, "layer_"): + answers[k] = judge.Answer{Choice: string(judge.LayerServer)} + case k == "page_kind": + answers[k] = judge.Answer{Choice: string(judge.KindDefault)} + default: + answers[k] = judge.Answer{Yes: 0.9, Choice: "not_stated"} + } + } + return answers, nil +} From 6c1a6f254f2cc42a21bc038c1d8e1e4eaaf33b58 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Fri, 25 Sep 2026 09:48:22 -0700 Subject: [PATCH 03/16] wip(judge): checkpoint explicit judge API, replay evaluation and generator Snapshot of the working tree before the judge layer cleanup: explicit Judge methods (Refine/Inspect/Classify/Version/...), replayable judgeeval (-manifest/-history/-maintain/-offline), the fingerprint generator and the validation reports. Co-Authored-By: Claude Opus 5.5 (1M context) --- .gitignore | 2 + README.md | 7 +- cmd/engine/example.go | 14 +- cmd/judgeeval/README.md | 80 +- cmd/judgeeval/main.go | 61 +- cmd/judgeeval/maintenance.go | 427 ++ cmd/judgeeval/maintenance_test.go | 93 + cmd/judgeeval/replay.go | 494 ++ cmd/judgeeval/replay_generation.go | 239 + cmd/judgeeval/replay_metrics.go | 235 + cmd/judgeeval/replay_test.go | 198 + docs/jev-maintenance-validation-20260926.md | 94 + docs/jev-real-validation-20260925.md | 104 + docs/jev-report.html | 142 + docs/jev-report.md | 29 + engine.go | 4 - fingerprints/README.md | 23 + .../jev-maintained-20260926.provenance.json | 4298 +++++++++++++++++ fingerprints/jev-maintained-20260926.yaml | 47 + judge.go | 43 - judge/README.md | 160 +- judge/classify.go | 4 +- judge/discovery.go | 146 + judge/doc.go | 28 +- judge/generator.go | 669 +++ judge/judge.go | 45 +- judge/judge_test.go | 35 +- judge/mark.go | 2 +- judge/operations.go | 130 + judge/operations_test.go | 282 ++ judge/page.go | 37 +- judge/provider.go | 2 +- judge/real_evidence_test.go | 183 + judge/refine.go | 52 +- judge/stack.go | 2 +- judge/verify.go | 79 +- judge/version.go | 17 +- judge_test.go | 14 +- scripts/capture_judge.py | 88 + 39 files changed, 8336 insertions(+), 273 deletions(-) create mode 100644 cmd/judgeeval/maintenance.go create mode 100644 cmd/judgeeval/maintenance_test.go create mode 100644 cmd/judgeeval/replay.go create mode 100644 cmd/judgeeval/replay_generation.go create mode 100644 cmd/judgeeval/replay_metrics.go create mode 100644 cmd/judgeeval/replay_test.go create mode 100644 docs/jev-maintenance-validation-20260926.md create mode 100644 docs/jev-real-validation-20260925.md create mode 100644 docs/jev-report.html create mode 100644 docs/jev-report.md create mode 100644 fingerprints/README.md create mode 100644 fingerprints/jev-maintained-20260926.provenance.json create mode 100644 fingerprints/jev-maintained-20260926.yaml delete mode 100644 judge.go create mode 100644 judge/discovery.go create mode 100644 judge/generator.go create mode 100644 judge/operations.go create mode 100644 judge/operations_test.go create mode 100644 judge/real_evidence_test.go create mode 100644 scripts/capture_judge.py diff --git a/.gitignore b/.gitignore index a7c8889..c132f3b 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,5 @@ resources/nmap-service-probes.txt /CLAUDE.md /MIGRATION.md vendor/ +/.judge-data/ +/scripts/__pycache__/ diff --git a/README.md b/README.md index 9f74ad4..881b263 100644 --- a/README.md +++ b/README.md @@ -90,18 +90,17 @@ func TestFavicon(t *testing.T) { ```golang j, _ := jev.NewJudge("") // 读取 TYPESAFE_API_KEY；自带缓存和相似页面去重 -engine.AttachJudge(j) frames, _ := engine.DetectContent(raw) // 同步，纯规则 -page, err := engine.Refine(ctx, raw, frames) // 通常异步执行；原地标注 frames，失败时保持规则结果 +accepted, kind, generic, err := j.Refine(ctx, raw, frames) if err == nil { - fmt.Println(page.Kind, judge.Accepted(frames)) // 页面类型；去掉误报和重复后的结果 + fmt.Println(kind, generic, accepted) // frames 保持原始规则命中 } ``` 在 1070 个真实页面上：每页结果从 13.4 条降到 6.5 条，同名重复全部归并，版本号对照 generator 从 43 个正确提升到 191 个，并额外召回了 468 个规则漏掉的产品。成本约每万页 $3.4。 -完整的能力、标注方式（`judge.Is(f, judge.Rejected)` 等）、缓存、自定义问题以及如何接入新的 Provider，见 [judge/README.md](judge/README.md)；设计讨论见 [#34](https://github.com/chainreactors/fingers/issues/34)。 +完整的能力、`Inspect` 诊断、未知产品判定、指纹生成器、缓存和 Provider 接口，见 [judge/README.md](judge/README.md)；设计讨论见 [#34](https://github.com/chainreactors/fingers/issues/34)。 ## fingers 引擎 diff --git a/cmd/engine/example.go b/cmd/engine/example.go index b4c4a58..f523e92 100644 --- a/cmd/engine/example.go +++ b/cmd/engine/example.go @@ -220,25 +220,19 @@ func main() { fmt.Println(err) os.Exit(1) } - engine.AttachJudge(j) - page, err := engine.Refine(context.Background(), content, frames) + accepted, kind, generic, err := j.Refine(context.Background(), content, frames) if err != nil { fmt.Printf("jev failed, showing the rule result: %v\n", err) } else { - fmt.Printf("page: %s, generic: %v\n", page.Kind, page.Generic) - for _, frame := range frames { + fmt.Printf("page: %s, generic: %v\n", kind, generic) + for _, frame := range accepted { verdict := "accepted" - for _, m := range []judge.Mark{judge.Rejected, judge.Duplicate} { - if judge.Is(frame, m) { - verdict = string(m) - } - } if judge.Is(frame, judge.Primary) { verdict += ",primary" } fmt.Printf(" %-40s %-10s %-18s %s\n", frame.Name, frame.Version, judge.LayerOf(frame), verdict) } - fmt.Printf("accepted: %s\n", judge.Accepted(frames).String()) + fmt.Printf("accepted: %s\n", accepted.String()) return } } diff --git a/cmd/judgeeval/README.md b/cmd/judgeeval/README.md index 4b6cbdd..a9325d5 100644 --- a/cmd/judgeeval/README.md +++ b/cmd/judgeeval/README.md @@ -1,6 +1,84 @@ # judgeeval：纯规则 vs 规则 + judge -在一批原始 HTTP 响应上，用和 SDK 调用方完全相同的路径（`engine.DetectContent` → `engine.Refine`）跑一遍，逐项对比纯规则结果和经过判定之后的结果。它也用来校准新的 Provider。 +## 可回放真实数据评估 + +`-manifest` 模式覆盖保存结果清洗、逐产品补全版本、名称召回和原生指纹生成。SDK 接口不变，结果写入独立目录。 + +```powershell +# urls.txt 每行一个 URL；新采集用新目录。IP 证书不匹配时显式加 --insecure。 +python scripts/capture_judge.py --urls urls.txt --out .judge-data/corpus --workers 6 +go build -tags goregexp -o bin/judgeeval.exe ./cmd/judgeeval +# TYPESAFE_API_KEY 仅通过进程环境提供。 +./bin/judgeeval.exe -manifest .judge-data/corpus/manifest.json -cache .judge-data/cache -out .judge-data/run-01 -rps 8 +# 同一快照及原结果用于验证历史清洗机制。 +./bin/judgeeval.exe -manifest .judge-data/corpus/manifest.json -history .judge-data/run-01/baseline.jsonl -cache .judge-data/cache -out .judge-data/run-02 +# 用运行目录内归档的响应严格离线复现；无须 API key，缓存未命中直接失败。 +./bin/judgeeval.exe -offline -manifest .judge-data/run-02/manifest.json -history .judge-data/run-02/baseline.jsonl -cache .judge-data/cache -out .judge-data/run-offline +``` + +采集器保存完整响应、来源、时间、SHA256 和失败记录，不自动跳转。第三方原文保存在 git 忽略的 `.judge-data/`。输出目录不能复用覆盖。 + +### 标注 + +为 manifest 的每份快照填写 `labels`：`product`、`present`、原文片段 `evidence` 和结论依据 `basis`。`version` 缺省表示未标注；空字符串表示未声明版本；非空字符串为准确版本。`aliases` 显式列出同义名，计分不使用子串匹配。加载时验证响应哈希及证据片段，片段是否足以支持结论仍需评审。 + +格式示例（仅示意，不作为真实数据）： + +```json +{"product":"Apache","aliases":["apache http server"],"present":true,"version":"2.4.38","evidence":["Server: Apache/2.4.38"],"basis":"响应头声明产品和版本"} +``` + +真值范围是响应证据支持的产品与版本，不等价于主机安装审计。标注来自响应头、generator meta、资源路径和人工评审；不以 Jev 回答作真值，不向清洗和版本判定传入标签。 + +### 生成与独立验证 + +`generation` 每项填写 `name`（输出文件名）、`product`、`auto_name`、`positive`/`negative`（训练快照 ID）。其他有该产品标注的快照用于测试；排除训练主机、相同响应和重复测试响应。使用普通 `Positive` 自动推断版本，测试标签只打分。 + +主动规则另填 `probe`（请求字符串），各样本 `probes` 将请求映射到同主机已采集响应 ID。回放不发送网络请求。导出 YAML 后重新加载、编译并验证。结论为 `passed_holdout`、`failed_holdout`、`insufficient_holdout`；至少需要独立正反例且版本无错漏才能在本批次通过。单主机新产品仅能产生待验证草案。生成文件供审阅，不自动写入指纹库。 + +### 输出 + +- `manifest.json` / `samples/*.http` / `build.json`：本次实际输入及 Go 构建信息，响应可独立加载；离线模型答案仍需同时保留 `-cache` 目录。 +- `baseline.jsonl`：原生 Frameworks，来源是当前规则或 `-history`。历史记录按 ID、SHA256 严格对齐，缺失即报错。 +- `cleaned.jsonl`：成功清洗并逐产品补版本的原生 Frameworks。 +- `rows.jsonl`：基线、Refine、补版本及拒绝/合并/新增/错误。 +- `metrics.json` / `report.md`：标注范围内的 TP/FP/FN/TN、误删、自然漏报恢复、版本正确/错误/缺失/无依据填值。失败和未标注输出单列。 +- `generation.json` / `fingerprints/*.yaml`：候选指纹、独立验证结果及排除的样本。 + +`Refine` 只补主产品版本；回放另对保留产品的空版本调用 `Version`，比较两阶段。已有非空版本保留，错误由报告揭示。缓存按模型、endpoint、完整 state 和问题精确复用，评估不使用近似页面缓存。 + +新增版本按产品及显式别名合并计数，原基线任一别名已有版本即不算新增。名称建议单独统计，不能算作已补回漏报。生成器训练样本的自动版本结果在 `training_versions` / `training_cases` 单列，错误时标记 `failed_training`；训练标签只在导出后打分。 + +新采集快照的历史回放验证的是清洗机制，不代表已验证生产历史分布。样本量、同主机依赖和标签覆盖范围必须随指标报告。 + +### 维护原生指纹库 + +`-maintain` 在同一次 manifest 回放中核验新产品、生成原生 YAML、验证候选，再通过实际引擎加载新增指纹。`-library` 明确指定本地已有扩展库，先加载再评估；不要求新增 SDK 请求/响应结构。 + +```powershell +# 审核新增产品并输出本次通过验证的增量库。 +./bin/judgeeval.exe -maintain -manifest .judge-data/corpus/manifest.json -cache .judge-data/cache -out .judge-data/maintain-01 +# 下一轮维护使用已有库作为基线，防止重复收录。 +./bin/judgeeval.exe -maintain -library .judge-data/maintain-01/library.yaml -manifest .judge-data/corpus/manifest.json -cache .judge-data/cache -out .judge-data/maintain-02 +# 完整离线复现；如果原运行用了扩展库，也必须传入归档的 input-library.yaml。 +./bin/judgeeval.exe -offline -maintain -library .judge-data/maintain-02/input-library.yaml -manifest .judge-data/maintain-02/manifest.json -history .judge-data/maintain-02/baseline.jsonl -cache .judge-data/cache -out .judge-data/maintain-offline +``` + +新增被动指纹的收录条件： + +1. 在五个内置 HTTP 引擎目录、别名表和实际加载的 fingers 运行时目录中均无同名产品。按规范名及显式别名审计，记录各目录 SHA256；并非覆盖所有外部指纹源。 +2. 普通 `Positive` 自动推断名称和版本，通过训练集、独立正反例、版本验证；不把标签版本传给生成器。 +3. 至少两个独立正例主机组、五个独立反例主机组。相同主机名或同属一个 `group` 的样本合并计数，且排除训练主机和重复内容。 +4. 调用 `engine.Fingers().LoadFromYAML(...)` 和 `engine.Compile()` 后重放全部样本。新增产品的标注识别/版本必须全部正确，已有命中和非空版本必须保留。 + +输出 `catalog-before.json`、`maintenance.json`；通过时再将 `library.candidate.yaml` 提升为 `library.yaml`。已有产品为 `already_catalogued`；没有足够独立样本的候选保留为 `insufficient_holdout` 或 `insufficient_independent_hosts`，不会混入输出库。主动规则暂不进入此被动库维护流程。`-library` 的原文另存 `input-library.yaml`，便于准确复现。 + +此收录检查验证**原生规则加载**。加载后再由 Jev 清洗是否误删、错误召回或错误补版本，要用上述第二轮当前规则基线单独验证；见 `rows.jsonl` / `metrics.json`，不能把去重成功当成整个清洗链路准确。扩展库需要调用方显式加载，命令不重写内置压缩资源或发布外部仓库。 + + +## 目录模式 + +在一批原始 HTTP 响应上，用和 SDK 调用方完全相同的路径（`engine.DetectContent` → `judge.Refine`）跑一遍，逐项对比纯规则结果和经过判定之后的结果。它也用来校准新的 Provider。 ```bash go build -buildvcs=false -tags goregexp -o judgeeval ./cmd/judgeeval diff --git a/cmd/judgeeval/main.go b/cmd/judgeeval/main.go index bd870fe..72a1fbf 100644 --- a/cmd/judgeeval/main.go +++ b/cmd/judgeeval/main.go @@ -1,5 +1,5 @@ // judgeeval compares the rule engines alone with rules plus a judge provider, -// exactly as SDK callers run it (fingers.Engine.Refine), on a directory of +// exactly as SDK callers run it (judge.Judge.Refine), on a directory of // raw HTTP responses; it is also how a new provider is calibrated. // // TYPESAFE_API_KEY=... judgeeval -provider jev -samples testdata/samples -labels testdata/labels.json @@ -66,6 +66,11 @@ type row struct { } func main() { + manifest := flag.String("manifest", "", "replay evidence manifest with labels and generation plans") + history := flag.String("history", "", "optional baseline.jsonl from a previous replay") + offline := flag.Bool("offline", false, "manifest replay using exact cached answers only; no provider requests") + maintain := flag.Bool("maintain", false, "audit novel products and validate a native additive fingerprint library") + library := flag.String("library", "", "existing local native YAML library to load before manifest replay") samples := flag.String("samples", "testdata/samples", "directory of raw HTTP responses (*.http)") labels := flag.String("labels", "", "optional ground truth (see testdata/labels.json)") provider := flag.String("provider", "jev", "judge provider: jev") @@ -77,6 +82,17 @@ func main() { endpoint := flag.String("endpoint", "", "provider API endpoint (default: the provider's)") flag.Parse() logs.Log.SetLevel(logs.ErrorLevel) + if *manifest != "" { + if err := replay(*manifest, *history, *provider, *endpoint, *cache, *out, *rps, *offline, *maintain, *library); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + return + } + if *offline || *maintain || *library != "" { + fmt.Fprintln(os.Stderr, "-offline, -maintain and -library require -manifest") + os.Exit(1) + } if err := run(*provider, *samples, *labels, *cache, *out, *endpoint, *rps, *workers, *limit); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) @@ -130,7 +146,13 @@ func run(providerName, samples, labelsPath, cacheDir, out, endpoint string, rps if err != nil { return err } - engine.AttachJudge(j) + var names []string + if engine.Aliases != nil { + for name := range engine.Aliases.Aliases { + names = append(names, name) + } + } + retriever := judge.NewRetriever(names) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -144,7 +166,7 @@ func run(providerName, samples, labelsPath, cacheDir, out, endpoint string, rps go func() { defer wg.Done() for i := range jobs { - rows[i] = evaluatePage(ctx, engine, files[i]) + rows[i] = evaluatePage(ctx, engine, j, retriever, files[i]) var apiErr *jev.APIError if rows[i].err != nil && errors.As(rows[i].err, &apiErr) && apiErr.Status == 401 { once.Do(func() { fatal = errors.New("unauthorized: check " + jev.EnvAPIKey); cancel() }) @@ -190,7 +212,7 @@ func run(providerName, samples, labelsPath, cacheDir, out, endpoint string, rps return os.WriteFile(filepath.Join(out, "report.md"), []byte(b.String()), 0o644) } -func evaluatePage(ctx context.Context, engine *fingers.Engine, path string) (r *row) { +func evaluatePage(ctx context.Context, engine *fingers.Engine, j *judge.Judge, retriever *judge.Retriever, path string) (r *row) { r = &row{ID: filepath.Base(path), ruleEngines: map[string][]string{}} defer func() { if p := recover(); p != nil { // one malformed page must not stop a long run @@ -234,20 +256,30 @@ func evaluatePage(ctx context.Context, engine *fingers.Engine, path string) (r * } sort.Strings(r.Rules) t := time.Now() - page, err = engine.Refine(ctx, raw, r.frames) + accepted, kind, generic, err := j.Refine(ctx, raw, r.frames, retriever.Find(page.Haystack(), 24)...) r.JudgeMs = ms(time.Since(t)) if err != nil { r.Err, r.err = err.Error(), err return r } - r.JudgeGenVer = versionOf(judge.Accepted(r.frames), product) + inspected, err := j.Inspect(ctx, raw, r.frames, retriever.Find(page.Haystack(), 24)...) + if err != nil { + r.Err, r.err = err.Error(), err + return r + } + for name, f := range accepted { + if target := inspected[name]; target != nil { + target.Attributes = f.Attributes + } + } + r.frames = inspected + r.JudgeGenVer = versionOf(accepted, product) for _, name := range r.TextOnly { if f := r.frames[name]; f != nil && !judge.Is(f, judge.Rejected) { r.TextOnlyKept++ } } - r.Kind, r.Generic, r.Layers = page.Kind, page.Generic, map[string]string{} - identified := false // an accepted application or device: the page's product is known + r.Kind, r.Generic, r.Layers = kind, generic, map[string]string{} for _, f := range r.frames { if l := judge.LayerOf(f); l != "" { r.Layers[f.Name] = string(l) @@ -258,10 +290,6 @@ func evaluatePage(ctx context.Context, engine *fingers.Engine, path string) (r * case judge.Is(f, judge.Duplicate): default: r.Accepted = append(r.Accepted, f.Name) - switch judge.Layer(r.Layers[f.Name]) { - case judge.LayerApplication, judge.LayerDevice: - identified = true - } } if judge.Is(f, judge.Recalled) { r.Recall = append(r.Recall, f.Name) @@ -271,10 +299,11 @@ func evaluatePage(ctx context.Context, engine *fingers.Engine, path string) (r * r.Primary, r.Version = p.Name, p.Version r.JudgeVersion = p.Version != "" && p.Version != versions[p.Name] } - // A stock page nobody identified is material for a new rule; a server's - // own default, error or index page is identified by the server. - serverPage := r.Kind == judge.KindDefault || r.Kind == judge.KindError || r.Kind == judge.KindDirListing - r.NewFinger = r.Generic && r.Primary == "" && !identified && !serverPage + r.NewFinger, err = j.IsUnknownProduct(ctx, raw, accepted) + if err != nil { + r.Err, r.err = err.Error(), err + return r + } sort.Strings(r.Accepted) sort.Strings(r.Rejected) return r diff --git a/cmd/judgeeval/maintenance.go b/cmd/judgeeval/maintenance.go new file mode 100644 index 0000000..c0106cb --- /dev/null +++ b/cmd/judgeeval/maintenance.go @@ -0,0 +1,427 @@ +package main + +import ( + "fmt" + "net/url" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/chainreactors/fingers" + "github.com/chainreactors/fingers/common" + fingerlib "github.com/chainreactors/fingers/fingers" + "github.com/chainreactors/fingers/judge" + "github.com/chainreactors/fingers/resources" + "gopkg.in/yaml.v3" +) + +// CLI evidence records; generated SDK values remain native Finger/Frameworks. +func loadReplayLibrary(engine *fingers.Engine, path, out string) error { + // Read a local file and archive the exact bytes before using the public loader. + data, err := os.ReadFile(path) + if err != nil { + return err + } + snapshot := filepath.Join(out, "input-library.yaml") + if err := os.WriteFile(snapshot, data, 0600); err != nil { + return err + } + if err := engine.Fingers().LoadFromYAML(snapshot); err != nil { + return err + } + return engine.Compile() +} + +type catalogSource struct { + Source string `json:"source"` + SHA256 string `json:"sha256"` + Names []string `json:"names"` +} + +func embeddedCatalog() ([]catalogSource, error) { + var result []catalogSource + for _, source := range []struct { + name string + data []byte + }{ + {"fingers", resources.FingersHTTPData}, {"goby", resources.GobyData}, + {"ehole", resources.EholeData}, {"wappalyzer", resources.WappalyzerData}, + {"fingerprinthub", resources.FingerprinthubWebData}, + } { + var raw any + if err := resources.UnmarshalData(source.data, &raw); err != nil { + return nil, err + } + var entries []any + var names []string + switch source.name { + case "wappalyzer": + root, ok := raw.(map[string]any) + if !ok { + return nil, fmt.Errorf("invalid %s catalog", source.name) + } + apps, ok := root["apps"].(map[string]any) + if !ok { + return nil, fmt.Errorf("invalid apps catalog") + } + for name := range apps { + names = append(names, name) + } + case "ehole": + root, ok := raw.(map[string]any) + if !ok { + return nil, fmt.Errorf("invalid ehole catalog") + } + entries, _ = root["fingerprint"].([]any) + default: + entries, _ = raw.([]any) + } + for _, entry := range entries { + obj, ok := entry.(map[string]any) + if !ok { + continue + } + for _, key := range []string{"name", "cms", "id"} { + if name, ok := obj[key].(string); ok { + names = append(names, name) + } + } + if info, ok := obj["info"].(map[string]any); ok { + if name, ok := info["name"].(string); ok { + names = append(names, name) + } + if metadata, ok := info["metadata"].(map[string]any); ok { + if name, ok := metadata["product"].(string); ok { + names = append(names, name) + } + } + } + } + if len(names) == 0 { + return nil, fmt.Errorf("empty %s catalog", source.name) + } + result = append(result, catalogSource{source.name, digest(source.data), uniqueNames(names)}) + } + var aliases []struct { + Name string `yaml:"name"` + Alias map[string][]string `yaml:"alias"` + } + if err := yaml.Unmarshal(resources.AliasesData, &aliases); err != nil { + return nil, err + } + var names []string + for _, alias := range aliases { + names = append(names, alias.Name) + for _, values := range alias.Alias { + names = append(names, values...) + } + } + result = append(result, catalogSource{"aliases", digest(resources.AliasesData), uniqueNames(names)}) + return result, nil +} + +func runtimeCatalog(engine *fingers.Engine) (catalogSource, error) { + data, err := yaml.Marshal(engine.Fingers().HTTPFingers) + if err != nil { + return catalogSource{}, err + } + var names []string + for _, f := range engine.Fingers().HTTPFingers { + if f != nil { + names = append(names, f.Name) + } + } + return catalogSource{"runtime-fingers", digest(data), uniqueNames(names)}, nil +} + +// Shared operator groups or shared hostnames connect samples into one unit. +// Different user-supplied group labels cannot turn one host into multiple votes. +func independentHostCount(samples []replaySample) int { + parent := map[string]string{} + var root func(string) string + root = func(key string) string { + if _, ok := parent[key]; !ok { + parent[key] = key + } + if parent[key] != key { + parent[key] = root(parent[key]) + } + return parent[key] + } + var hosts []string + for _, s := range samples { + u, err := url.Parse(s.URL) + if err != nil || u.Hostname() == "" || s.Group == "" { + continue + } + host := "host:" + strings.ToLower(u.Hostname()) + group := "group:" + s.Group + parent[root(host)] = root(group) + hosts = append(hosts, host) + } + groups := map[string]bool{} + for _, host := range hosts { + groups[root(host)] = true + } + return len(groups) +} + +func uniqueNames(names []string) []string { + seen := map[string]bool{} + var out []string + for _, name := range names { + if name != "" && !seen[name] { + seen[name] = true + out = append(out, name) + } + } + sort.Strings(out) + return out +} + +func catalogMatches(catalog []catalogSource, names []string) []string { + wanted := map[string]bool{} + for _, name := range names { + wanted[judge.NormalizeName(name)] = true + } + var found []string + for _, source := range catalog { + for _, name := range source.Names { + if wanted[judge.NormalizeName(name)] { + found = append(found, source.Source+":"+name) + } + } + } + return uniqueNames(found) +} + +type maintenanceProduct struct { + Plan string `json:"plan"` + Product string `json:"product"` + Status string `json:"status"` + Existing []string `json:"existing_catalog_matches"` + PositiveSamples int `json:"positive_samples"` + SuggestedPositives int `json:"suggested_positives"` + UnknownPositives int `json:"unknown_positives"` + PositiveHosts int `json:"holdout_positive_hosts"` + NegativeHosts int `json:"holdout_negative_hosts"` + Before stageScore `json:"before"` + After stageScore `json:"after"` + TestIDs []string `json:"test_ids"` +} + +type maintenanceReport struct { + Library string `json:"library,omitempty"` + CatalogFile string `json:"catalog_file"` + MinPositiveHosts int `json:"min_positive_hosts"` + MinNegativeHosts int `json:"min_negative_hosts"` + Products []maintenanceProduct `json:"products"` + LostExisting []string `json:"lost_existing_hits"` + ChangedVersions []string `json:"changed_existing_versions"` + IntegrationErrors []string `json:"integration_errors"` +} + +func maintenanceDecision(existing []string, g generationResult, positiveHosts, negativeHosts int) string { + switch { + case len(existing) > 0: + return "already_catalogued" + case g.Status != "passed_holdout": + return g.Status + case positiveHosts < 2 || negativeHosts < 5: + return "insufficient_independent_hosts" + default: + return "eligible" + } +} + +func maintainReplay(m *replayManifest, rows []replayRow, generated []generationResult, out string, engine *fingers.Engine) error { + catalog, err := embeddedCatalog() + if err != nil { + return err + } + runtime, err := runtimeCatalog(engine) + if err != nil { + return err + } + catalog = append(catalog, runtime) + if err := writeJSON(filepath.Join(out, "catalog-before.json"), catalog); err != nil { + return err + } + report := maintenanceReport{CatalogFile: "catalog-before.json", MinPositiveHosts: 2, MinNegativeHosts: 5} + byID := map[string]replayRow{} + for _, row := range rows { + byID[row.ID] = row + } + plans := map[string]generationPlan{} + for _, p := range m.Generation { + plans[p.Name] = p + } + var library fingerlib.Fingers + accepted := map[string]generationPlan{} + for _, g := range generated { + p := plans[g.Name] + r := maintenanceProduct{Plan: g.Name, Product: p.Product} + names := []string{p.Product, g.Product} + for _, sample := range m.Samples { + l, ok := labelFor(sample, p.Product) + if !ok { + continue + } + names = append(names, l.Aliases...) + if l.Present { + r.PositiveSamples++ + row := byID[sample.ID] + if containsProductName(row.Suggestions, l) { + r.SuggestedPositives++ + } + if row.Unknown { + r.UnknownPositives++ + } + } + } + r.Existing = catalogMatches(catalog, names) + test, _, err := generationSplit(m, p) + if err != nil { + return err + } + var pos, neg []replaySample + for _, sample := range test { + l, _ := labelFor(sample, p.Product) + if l.Present { + pos = append(pos, sample) + } else { + neg = append(neg, sample) + } + r.TestIDs = append(r.TestIDs, sample.ID) + } + r.PositiveHosts, r.NegativeHosts = independentHostCount(pos), independentHostCount(neg) + r.Status = maintenanceDecision(r.Existing, g, r.PositiveHosts, r.NegativeHosts) + if r.Status == "eligible" && p.Probe != "" { + r.Status = "active_integration_not_evaluated" + } + if r.Status == "eligible" { + key := judge.NormalizeName(p.Product) + if _, exists := accepted[key]; exists { + r.Status = "duplicate_candidate" + } else { + path, err := confinedPath(out, g.File) + if err != nil { + return err + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + var f fingerlib.Finger + if err := yaml.Unmarshal(data, &f); err != nil { + return err + } + library = append(library, &f) + accepted[key] = p + } + } + report.Products = append(report.Products, r) + } + if len(library) == 0 { + return writeJSON(filepath.Join(out, "maintenance.json"), report) + } + sort.Slice(library, func(i, j int) bool { return library[i].Name < library[j].Name }) + data, err := yaml.Marshal(library) + if err != nil { + return err + } + candidatePath := filepath.Join(out, "library.candidate.yaml") + if err := os.WriteFile(candidatePath, data, 0600); err != nil { + return err + } + before := map[string]common.Frameworks{} + for _, sample := range m.Samples { + frames, err := engine.DetectContent(sample.raw) + if err != nil { + return err + } + before[sample.ID] = copyFrames(frames) + } + // Exercise the public additive loader and rebuild aliases just as SDK callers do. + if err := engine.Fingers().LoadFromYAML(candidatePath); err != nil { + return err + } + if err := engine.Compile(); err != nil { + return err + } + after := map[string]common.Frameworks{} + for _, sample := range m.Samples { + frames, err := engine.DetectContent(sample.raw) + if err != nil { + return err + } + after[sample.ID] = frames + for _, f := range before[sample.ID] { + if f == nil { + continue + } + got := findLabel(frames, productLabel{Product: f.Name}) + if got == nil { + report.LostExisting = append(report.LostExisting, sample.ID+":"+f.Name) + } else if f.Version != "" && got.Version != f.Version { + report.ChangedVersions = append(report.ChangedVersions, sample.ID+":"+f.Name) + } + } + for _, p := range accepted { + l, ok := labelFor(sample, p.Product) + if !ok { + continue + } + got := findLabel(frames, l) + if (got != nil) != l.Present || got != nil && l.Version != nil && got.Version != *l.Version { + report.IntegrationErrors = append(report.IntegrationErrors, sample.ID+":"+p.Product) + } + } + } + for i := range report.Products { + r := &report.Products[i] + if r.Status != "eligible" { + continue + } + p := plans[r.Plan] + test, _, _ := generationSplit(m, p) + for _, sample := range test { + l, _ := labelFor(sample, p.Product) + for _, stage := range []struct { + score *stageScore + frames common.Frameworks + }{{&r.Before, before[sample.ID]}, {&r.After, after[sample.ID]}} { + f := findLabel(stage.frames, l) + stage.score.Detection.add(l.Present, f != nil) + if l.Version != nil { + stage.score.Versions.add(f, *l.Version) + } + } + } + r.Status = "validated_library" + } + sort.Strings(report.LostExisting) + sort.Strings(report.ChangedVersions) + sort.Strings(report.IntegrationErrors) + if len(report.LostExisting)+len(report.ChangedVersions)+len(report.IntegrationErrors) > 0 { + for i := range report.Products { + if report.Products[i].Status == "validated_library" { + report.Products[i].Status = "integration_failed" + } + } + } else { + report.Library = "library.yaml" + if err := os.Rename(candidatePath, filepath.Join(out, report.Library)); err != nil { + return err + } + } + if err := writeJSON(filepath.Join(out, "maintenance.json"), report); err != nil { + return err + } + if report.Library == "" { + return fmt.Errorf("candidate library failed integration; see maintenance.json") + } + fmt.Fprintf(os.Stderr, "validated %d novel fingerprints in %s\n", len(library), report.Library) + return nil +} diff --git a/cmd/judgeeval/maintenance_test.go b/cmd/judgeeval/maintenance_test.go new file mode 100644 index 0000000..814f8d9 --- /dev/null +++ b/cmd/judgeeval/maintenance_test.go @@ -0,0 +1,93 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/chainreactors/fingers" +) + +func TestMaintenanceCountsHostAndOperatorOverlapOnce(t *testing.T) { + samples := []replaySample{ + {URL: "https://a.example/one", Group: "operator-a"}, + {URL: "https://b.example/", Group: "operator-a"}, + {URL: "https://b.example/two", Group: "operator-b"}, + {URL: "https://c.example/", Group: "operator-b"}, + {URL: "https://d.example/", Group: "operator-d"}, + } + if got := independentHostCount(samples); got != 2 { + t.Fatalf("overlapping hosts/operators counted %d times, want 2", got) + } +} + +func TestMaintenanceRecognizesPreviouslyLoadedLibrary(t *testing.T) { + engine, err := fingers.NewEngine(fingers.FingersEngine) + if err != nil { + t.Fatal(err) + } + out := t.TempDir() + path := filepath.Join(out, "fixture.yaml") + data := []byte("- name: replay-maintenance-test-app\n protocol: http\n rule:\n - regexps:\n body:\n - 'replay-maintenance-test-app'\n") + if err := os.WriteFile(path, data, 0600); err != nil { + t.Fatal(err) + } + if err := loadReplayLibrary(engine, path, out); err != nil { + t.Fatal(err) + } + archived, err := os.ReadFile(filepath.Join(out, "input-library.yaml")) + if err != nil || string(archived) != string(data) { + t.Fatalf("library bytes not preserved: %v", err) + } + catalog, err := runtimeCatalog(engine) + if err != nil { + t.Fatal(err) + } + existing := catalogMatches([]catalogSource{catalog}, []string{"replay-maintenance-test-app"}) + if got := maintenanceDecision(existing, generationResult{Status: "passed_holdout"}, 2, 5); got != "already_catalogued" { + t.Fatalf("would re-add existing library fingerprint: %s", got) + } +} + +func TestCatalogAuditCoversEveryEmbeddedSource(t *testing.T) { + catalog, err := embeddedCatalog() + if err != nil { + t.Fatal(err) + } + if len(catalog) != 6 { + t.Fatalf("sources: %d", len(catalog)) + } + for _, source := range catalog { + if source.SHA256 == "" || len(source.Names) == 0 { + t.Fatalf("incomplete source: %s", source.Source) + } + } + if len(catalogMatches(catalog, []string{"WordPress"})) == 0 { + t.Fatal("known product missed") + } + if len(catalogMatches(catalog, []string{"certainly-not-a-real-catalog-product-20260926"})) != 0 { + t.Fatal("invented product matched") + } +} + +func TestMaintenanceRequiresNoveltyAndIndependentHosts(t *testing.T) { + g := generationResult{Status: "passed_holdout"} + for _, tc := range []struct { + existing []string + positive, negative int + want string + }{ + {[]string{"fingers:nginx"}, 3, 20, "already_catalogued"}, + {nil, 1, 20, "insufficient_independent_hosts"}, + {nil, 2, 4, "insufficient_independent_hosts"}, + {nil, 2, 5, "eligible"}, + } { + if got := maintenanceDecision(tc.existing, g, tc.positive, tc.negative); got != tc.want { + t.Fatalf("got %s want %s", got, tc.want) + } + } + g.Status = "failed_holdout" + if got := maintenanceDecision(nil, g, 3, 20); got != "failed_holdout" { + t.Fatalf("failed candidate admitted: %s", got) + } +} diff --git a/cmd/judgeeval/replay.go b/cmd/judgeeval/replay.go new file mode 100644 index 0000000..62f919e --- /dev/null +++ b/cmd/judgeeval/replay.go @@ -0,0 +1,494 @@ +package main + +import ( + "bufio" + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "math" + "net/http" + "net/url" + "os" + "path/filepath" + "runtime/debug" + "sort" + "strings" + "time" + + "github.com/chainreactors/fingers" + "github.com/chainreactors/fingers/common" + "github.com/chainreactors/fingers/judge" + "github.com/chainreactors/fingers/judge/jev" +) + +// Labels are evidence for evaluation only; they are never passed to Judge. +type productLabel struct { + Product string `json:"product"` + Aliases []string `json:"aliases,omitempty"` + Present bool `json:"present"` + Version *string `json:"version,omitempty"` // nil = not labelled, "" = not stated + Evidence []string `json:"evidence"` // literal excerpts in the saved response + Basis string `json:"basis"` // what the excerpt establishes +} +type replaySample struct { + ID string `json:"id"` + URL string `json:"url"` + Group string `json:"group"` + CapturedAt string `json:"captured_at"` + Response string `json:"response"` + SHA256 string `json:"sha256"` + ContentSHA256 string `json:"content_sha256"` + Labels []productLabel `json:"labels"` + Probes map[string]string `json:"probes,omitempty"` // send_data -> another sample ID + raw []byte +} +type generationPlan struct { + Name string `json:"name"` + Product string `json:"product"` + AutoName bool `json:"auto_name"` + Positive []string `json:"positive"` + Negative []string `json:"negative"` + Probe string `json:"probe,omitempty"` +} +type replayManifest struct { + Schema int `json:"schema"` + Scope string `json:"scope"` + Samples []replaySample `json:"samples"` + Generation []generationPlan `json:"generation"` +} +type baselineRecord struct { + ID string `json:"id"` + SHA256 string `json:"sha256"` + Frames common.Frameworks `json:"frames"` +} +type replayRow struct { + ID string `json:"id"` + URL string `json:"url"` + SHA256 string `json:"sha256"` + Baseline common.Frameworks `json:"baseline"` + Refined common.Frameworks `json:"refined,omitempty"` + Completed common.Frameworks `json:"completed,omitempty"` + Rejected []string `json:"rejected,omitempty"` + Duplicates []string `json:"duplicates,omitempty"` + Added []string `json:"added,omitempty"` + Filled map[string]string `json:"filled_versions,omitempty"` + Kind judge.Kind `json:"kind,omitempty"` + Generic bool `json:"generic"` + Unknown bool `json:"unknown"` + Suggestions []string `json:"suggestions,omitempty"` + Err string `json:"error,omitempty"` +} + +func digest(b []byte) string { return fmt.Sprintf("%x", sha256.Sum256(b)) } +func writeJSON(path string, value any) error { + b, err := json.MarshalIndent(value, "", " ") + if err != nil { + return err + } + return os.WriteFile(path, append(b, '\n'), 0600) +} +func confinedPath(root, name string) (string, error) { + if name == "" || filepath.IsAbs(name) { + return "", fmt.Errorf("expected relative response path: %q", name) + } + joined := filepath.Join(root, filepath.FromSlash(name)) + rel, err := filepath.Rel(root, joined) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("response escapes corpus: %q", name) + } + return joined, nil +} +func loadReplay(path string) (*replayManifest, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var m replayManifest + if err := json.Unmarshal(data, &m); err != nil { + return nil, err + } + if m.Schema != 1 || len(m.Samples) == 0 { + return nil, fmt.Errorf("manifest requires schema=1 and samples") + } + seen := map[string]bool{} + for i := range m.Samples { + s := &m.Samples[i] + if s.ID == "" || seen[s.ID] { + return nil, fmt.Errorf("empty or duplicate sample ID %q", s.ID) + } + seen[s.ID] = true + u, err := url.Parse(s.URL) + if err != nil || u.Hostname() == "" || s.Group == "" || s.CapturedAt == "" { + return nil, fmt.Errorf("%s: URL, group and captured_at required", s.ID) + } + filename, err := confinedPath(filepath.Dir(path), s.Response) + if err != nil { + return nil, err + } + s.raw, err = os.ReadFile(filename) + if err != nil { + return nil, err + } + if len(s.raw) > 2<<20 || digest(s.raw) != s.SHA256 { + return nil, fmt.Errorf("%s: size limit or SHA256 mismatch", s.ID) + } + resp, err := http.ReadResponse(bufio.NewReader(bytes.NewReader(s.raw)), nil) + if err != nil { + return nil, fmt.Errorf("%s: %w", s.ID, err) + } + body, err := io.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + return nil, fmt.Errorf("%s: incomplete response: %w", s.ID, err) + } + // Recompute rather than trusting the manifest for split leakage checks. + stable := http.Header{} + for k, v := range resp.Header { + switch strings.ToLower(k) { + case "date", "set-cookie", "etag", "last-modified", "connection", "content-length", "transfer-encoding", "x-request-id": + continue + } + stable[k] = v + } + head, _ := json.Marshal([]any{resp.StatusCode, stable}) + s.ContentSHA256 = digest(append(head, body...)) + evidence := strings.ToLower(string(s.raw) + "\n" + string(body)) + keys := map[string]bool{} + for _, l := range s.Labels { + key := judge.NormalizeName(l.Product) + if key == "" || keys[key] || l.Basis == "" || len(l.Evidence) == 0 || (!l.Present && l.Version != nil) { + return nil, fmt.Errorf("%s: invalid/duplicate label %q", s.ID, l.Product) + } + keys[key] = true + for _, excerpt := range l.Evidence { + if strings.TrimSpace(excerpt) == "" || !strings.Contains(evidence, strings.ToLower(excerpt)) { + return nil, fmt.Errorf("%s: label evidence missing for %s", s.ID, l.Product) + } + } + } + } + for _, s := range m.Samples { + for request, id := range s.Probes { + if request == "" || !seen[id] { + return nil, fmt.Errorf("%s: invalid probe %q", s.ID, request) + } + } + } + return &m, nil +} + +// Cache the full provider input, not a near-page signature, for evaluation. +type replayProvider struct { + judge.Provider + dir string + namespace string + offline bool + calls, hits int +} + +func (p *replayProvider) Calibration() (float64, float64) { + if c, ok := p.Provider.(judge.Calibrated); ok { + return c.Calibration() + } + return .5, .9 +} +func (p *replayProvider) Judge(ctx context.Context, state any, qs map[string]judge.Question) (map[string]judge.Answer, error) { + data, err := json.Marshal([]any{p.ID(), p.namespace, state, qs}) + if err != nil { + return nil, err + } + path := filepath.Join(p.dir, digest(data)+".json") + var answers map[string]judge.Answer + if cached, err := os.ReadFile(path); err == nil && json.Unmarshal(cached, &answers) == nil { + p.hits++ + return answers, nil + } + if p.offline { + return nil, fmt.Errorf("offline cache miss: %s", filepath.Base(path)) + } + p.calls++ + answers, err = p.Provider.Judge(ctx, state, qs) + if err == nil { + if e := writeJSON(path, answers); e != nil { + return nil, e + } + } + return answers, err +} +func loadHistory(path string) (map[string]baselineRecord, error) { + out := map[string]baselineRecord{} + if path == "" { + return out, nil + } + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + scan := bufio.NewScanner(f) + scan.Buffer(make([]byte, 4096), 16<<20) + for scan.Scan() { + var r baselineRecord + if err := json.Unmarshal(scan.Bytes(), &r); err != nil { + return nil, err + } + if _, ok := out[r.ID]; ok { + return nil, fmt.Errorf("duplicate historical ID %s", r.ID) + } + out[r.ID] = r + } + return out, scan.Err() +} +func copyFrames(frames common.Frameworks) common.Frameworks { + data, _ := json.Marshal(frames) + var out common.Frameworks + _ = json.Unmarshal(data, &out) + return out +} +func findLabel(frames common.Frameworks, l productLabel) *common.Framework { + names := append([]string{l.Product}, l.Aliases...) + var found *common.Framework + for _, name := range names { + for _, f := range frames { + if f != nil && judge.NormalizeName(f.Name) == judge.NormalizeName(name) { + if found == nil || (found.Version == "" && f.Version != "") || (found.Version == f.Version && f.Name < found.Name) { + found = f + } + } + } + } + return found +} +func labelFor(s replaySample, name string) (productLabel, bool) { + key := judge.NormalizeName(name) + for _, l := range s.Labels { + for _, n := range append([]string{l.Product}, l.Aliases...) { + if judge.NormalizeName(n) == key { + return l, true + } + } + } + return productLabel{}, false +} + +func replay(manifestPath, historyPath, providerName, endpoint, cacheDir, out string, rps float64, offline, maintain bool, libraryPath string) error { + if rps <= 0 || rps > 100 || math.IsNaN(rps) { + return fmt.Errorf("rps must be in (0,100]") + } + m, err := loadReplay(manifestPath) + if err != nil { + return err + } + history, err := loadHistory(historyPath) + if err != nil { + return err + } + if historyPath != "" { + for _, s := range m.Samples { + if r, ok := history[s.ID]; !ok || r.SHA256 != s.SHA256 { + return fmt.Errorf("%s: history missing or response hash differs", s.ID) + } + } + } + if _, err := os.Stat(out); err == nil { + return fmt.Errorf("output directory exists: use a new run directory") + } + if err := os.MkdirAll(out, 0755); err != nil { + return err + } + if err := os.MkdirAll(cacheDir, 0700); err != nil { + return err + } + if err := saveReplaySnapshot(out, m); err != nil { + return err + } + ticker := time.NewTicker(time.Duration(float64(time.Second) / rps)) + defer ticker.Stop() + var provider judge.Provider + tokens := func() int64 { return 0 } + if offline { + if providerName != "jev" { + return fmt.Errorf("unknown provider %q", providerName) + } + // Only ID and calibration are used; cache misses never invoke this provider. + provider, err = jev.New("offline-cache-only") + } else { + provider, tokens, err = newProvider(providerName, endpoint, &limited{tick: ticker.C}) + } + if err != nil { + return err + } + cached := &replayProvider{Provider: provider, dir: cacheDir, namespace: endpoint, offline: offline} + j := judge.New(cached) + j.Cache = nil + engine, err := fingers.NewEngine(fingers.FingersEngine, fingers.FingerPrintEngine, fingers.EHoleEngine, fingers.GobyEngine, fingers.WappalyzerEngine) + if err != nil { + return err + } + if libraryPath != "" { + if err := loadReplayLibrary(engine, libraryPath, out); err != nil { + return err + } + } + var names []string + if engine.Aliases != nil { + for name := range engine.Aliases.Aliases { + names = append(names, name) + } + } + sort.Strings(names) + retriever := judge.NewRetriever(names) + baselineFile, err := os.Create(filepath.Join(out, "baseline.jsonl")) + if err != nil { + return err + } + defer baselineFile.Close() + cleanedFile, err := os.Create(filepath.Join(out, "cleaned.jsonl")) + if err != nil { + return err + } + defer cleanedFile.Close() + rowFile, err := os.Create(filepath.Join(out, "rows.jsonl")) + if err != nil { + return err + } + defer rowFile.Close() + be, ce, re := json.NewEncoder(baselineFile), json.NewEncoder(cleanedFile), json.NewEncoder(rowFile) + rows := make([]replayRow, 0, len(m.Samples)) + started := time.Now() + for _, s := range m.Samples { + r := replayRow{ID: s.ID, URL: s.URL, SHA256: s.SHA256} + if historyPath != "" { + r.Baseline = copyFrames(history[s.ID].Frames) + } else { + r.Baseline, err = engine.DetectContent(s.raw) + if err != nil { + return err + } + } + if err := be.Encode(baselineRecord{s.ID, s.SHA256, r.Baseline}); err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + err = cleanReplay(ctx, j, retriever, s, &r) + cancel() + if err != nil { + r.Err = err.Error() + } else if err := ce.Encode(baselineRecord{s.ID, s.SHA256, r.Completed}); err != nil { + return err + } + if err := re.Encode(r); err != nil { + return err + } + rows = append(rows, r) + fmt.Fprintf(os.Stderr, "replay %d/%d %s: baseline=%d completed=%d error=%t\n", len(rows), len(m.Samples), s.ID, len(r.Baseline), len(r.Completed), r.Err != "") + } + generated, err := generateReplay(m, j, out) + if err != nil { + return err + } + summary := summarizeReplay(m, rows, generated) + summary.Model = provider.ID() + summary.Requests = cached.calls + summary.CacheHits = cached.hits + summary.InputTokens = tokens() + summary.Seconds = time.Since(started).Seconds() + summary.BaselineSource = "current_rules_on_saved_responses" + if historyPath != "" { + summary.BaselineSource = "historical_results" + } + if err := writeJSON(filepath.Join(out, "metrics.json"), summary); err != nil { + return err + } + if err := writeReplayReport(filepath.Join(out, "report.md"), summary); err != nil { + return err + } + fmt.Printf("saved %s\n", filepath.Join(out, "report.md")) + if summary.Errors > 0 { + return fmt.Errorf("%d failed rows; see rows.jsonl", summary.Errors) + } + if maintain { + return maintainReplay(m, rows, generated, out, engine) + } + return nil +} + +// Keep the exact labelled inputs together with the output for offline replay. +func saveReplaySnapshot(out string, m *replayManifest) error { + copy := *m + copy.Samples = append([]replaySample(nil), m.Samples...) + if err := os.MkdirAll(filepath.Join(out, "samples"), 0700); err != nil { + return err + } + for i := range copy.Samples { + s := ©.Samples[i] + s.Response = "samples/" + s.SHA256 + ".http" + if err := os.WriteFile(filepath.Join(out, s.Response), s.raw, 0600); err != nil { + return err + } + } + if info, ok := debug.ReadBuildInfo(); ok { + if err := writeJSON(filepath.Join(out, "build.json"), info); err != nil { + return err + } + } + return writeJSON(filepath.Join(out, "manifest.json"), copy) +} +func cleanReplay(ctx context.Context, j *judge.Judge, retriever *judge.Retriever, s replaySample, r *replayRow) error { + p, err := judge.NewPage(s.raw) + if err != nil { + return err + } + known := retriever.Find(p.Haystack(), 24) + r.Refined, r.Kind, r.Generic, err = j.Refine(ctx, s.raw, r.Baseline, known...) + if err != nil { + return err + } + inspected, err := j.Inspect(ctx, s.raw, r.Baseline, known...) + if err != nil { + return err + } + for _, f := range inspected { + if judge.Is(f, judge.Rejected) { + r.Rejected = append(r.Rejected, f.Name) + } + if judge.Is(f, judge.Duplicate) { + r.Duplicates = append(r.Duplicates, f.Name) + } + } + r.Completed = copyFrames(r.Refined) + r.Filled = map[string]string{} + for _, f := range r.Completed { + before := findLabel(r.Baseline, productLabel{Product: f.Name}) + if before == nil { + r.Added = append(r.Added, f.Name) + } + if f.Version == "" { + v, err := j.Version(ctx, s.raw, f) + if err != nil { + return err + } + if v != "" { + if f.Attributes == nil { + f.Attributes = common.NewAttributesWithAny() + } + f.Version = v + } + } + if f.Version != "" && (before == nil || before.Version == "") { + r.Filled[f.Name] = f.Version + } + } + r.Unknown, err = j.IsUnknownProduct(ctx, s.raw, r.Completed) + if err != nil { + return err + } + r.Suggestions, err = j.SuggestNames(ctx, s.raw) + sort.Strings(r.Rejected) + sort.Strings(r.Duplicates) + sort.Strings(r.Added) + return err +} diff --git a/cmd/judgeeval/replay_generation.go b/cmd/judgeeval/replay_generation.go new file mode 100644 index 0000000..2ff56ce --- /dev/null +++ b/cmd/judgeeval/replay_generation.go @@ -0,0 +1,239 @@ +package main + +import ( + "context" + "fmt" + "net/url" + "os" + "path/filepath" + "strings" + "time" + + "github.com/chainreactors/fingers/common" + fingerlib "github.com/chainreactors/fingers/fingers" + "github.com/chainreactors/fingers/judge" + "gopkg.in/yaml.v3" +) + +type generationCase struct { + ID string `json:"id"` + Present bool `json:"present"` + Matched bool `json:"matched"` + Version string `json:"version,omitempty"` + ExpectedVersion *string `json:"expected_version,omitempty"` +} +type generationResult struct { + Name string `json:"name"` + Product string `json:"product"` + Status string `json:"status"` + File string `json:"file,omitempty"` + Detection detectionScore `json:"detection"` + Versions versionScore `json:"versions"` + TrainingDetection detectionScore `json:"training_detection"` + TrainingVersions versionScore `json:"training_versions"` + TrainingCases []generationCase `json:"training_cases"` + Cases []generationCase `json:"cases"` + Excluded []string `json:"excluded"` + Error string `json:"error,omitempty"` +} + +func generationSplit(m *replayManifest, p generationPlan) ([]replaySample, []string, error) { + samples := map[string]replaySample{} + for _, s := range m.Samples { + samples[s.ID] = s + } + if p.Name == "" || p.Name != filepath.Base(p.Name) || strings.ContainsAny(p.Name, "/\\:") || p.Product == "" || len(p.Positive) == 0 || len(p.Negative) == 0 { + return nil, nil, fmt.Errorf("invalid generation plan") + } + trainIDs, trainGroups, trainHashes := map[string]bool{}, map[string]bool{}, map[string]bool{} + trainHosts := map[string]bool{} + host := func(s replaySample) string { + u, err := url.Parse(s.URL) + if err != nil { + return "" + } + return strings.ToLower(u.Hostname()) + } + for _, set := range []struct { + ids []string + want bool + }{{p.Positive, true}, {p.Negative, false}} { + for _, id := range set.ids { + s, ok := samples[id] + if !ok || trainIDs[id] { + return nil, nil, fmt.Errorf("missing/duplicate training sample %s", id) + } + l, ok := labelFor(s, p.Product) + if !ok || l.Present != set.want { + return nil, nil, fmt.Errorf("training label missing or contradictory: %s", id) + } + trainIDs[id] = true + trainGroups[s.Group] = true + if h := host(s); h != "" { + trainHosts[h] = true + } + trainHashes[s.ContentSHA256] = true + if p.Probe != "" { + probe, ok := samples[s.Probes[p.Probe]] + if !ok || probe.Group != s.Group { + return nil, nil, fmt.Errorf("missing or different-host probe for %s", id) + } + trainHashes[probe.ContentSHA256] = true + } + } + } + var test []replaySample + var excluded []string + seen := map[string]bool{} + for _, s := range m.Samples { + if _, ok := labelFor(s, p.Product); !ok || trainIDs[s.ID] { + continue + } + hash := s.ContentSHA256 + if p.Probe != "" { + probe, ok := samples[s.Probes[p.Probe]] + if !ok { + continue + } + hash = probe.ContentSHA256 + } + if trainGroups[s.Group] || trainHosts[host(s)] || trainHashes[hash] || seen[hash] { + excluded = append(excluded, s.ID) + continue + } + seen[hash] = true + test = append(test, s) + } + return test, excluded, nil +} +func generateReplay(m *replayManifest, j *judge.Judge, out string) ([]generationResult, error) { + samples := map[string]replaySample{} + for _, s := range m.Samples { + samples[s.ID] = s + } + results := make([]generationResult, 0, len(m.Generation)) + plans := map[string]bool{} + for _, p := range m.Generation { + if plans[p.Name] { + return nil, fmt.Errorf("duplicate generation plan %s", p.Name) + } + plans[p.Name] = true + r := generationResult{Name: p.Name, Product: p.Product, Status: "failed"} + test, excluded, err := generationSplit(m, p) + if err != nil { + return nil, err + } + r.Excluded = excluded + g := judge.NewGenerator(j) + if !p.AutoName { + g.Name(p.Product) + } + for _, id := range p.Positive { + s := samples[id] + g.Positive(s.raw) + if p.Probe != "" { + g.Probe([]byte(p.Probe), samples[s.Probes[p.Probe]].raw) + } + } + for _, id := range p.Negative { + s := samples[id] + g.Negative(s.raw) + if p.Probe != "" { + g.Probe([]byte(p.Probe), samples[s.Probes[p.Probe]].raw) + } + } + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + f, err := g.Generate(ctx) + cancel() + if err != nil { + r.Error = err.Error() + results = append(results, r) + continue + } + r.Product = f.Name + if judge.NormalizeName(f.Name) != judge.NormalizeName(p.Product) { + r.Error = "automatic name differs from labelled product" + results = append(results, r) + continue + } + data, err := yaml.Marshal(f) + if err != nil { + return nil, err + } + dir := filepath.Join(out, "fingerprints") + if err := os.MkdirAll(dir, 0755); err != nil { + return nil, err + } + r.File = filepath.ToSlash(filepath.Join("fingerprints", p.Name+".yaml")) + if err := os.WriteFile(filepath.Join(out, r.File), data, 0600); err != nil { + return nil, err + } + var loaded fingerlib.Finger + if err := yaml.Unmarshal(data, &loaded); err != nil { + return nil, err + } + if err := loaded.Compile(false); err != nil { + return nil, err + } + // Revalidate the exported rule on training samples before holding it out. + if err := g.Validate(&loaded); err != nil { + r.Error = err.Error() + results = append(results, r) + continue + } + // Score inferred training versions only after generation and export. + // Positive deliberately receives no labelled version or other truth. + for _, ids := range [][]string{p.Positive, p.Negative} { + for _, id := range ids { + c, frame := evaluateGenerated(&loaded, p, samples[id], samples) + r.TrainingCases = append(r.TrainingCases, c) + r.TrainingDetection.add(c.Present, c.Matched) + if c.ExpectedVersion != nil { + r.TrainingVersions.add(frame, *c.ExpectedVersion) + } + } + } + for _, s := range test { + c, frame := evaluateGenerated(&loaded, p, s, samples) + r.Cases = append(r.Cases, c) + r.Detection.add(c.Present, c.Matched) + if c.ExpectedVersion != nil { + r.Versions.add(frame, *c.ExpectedVersion) + } + } + switch { + case r.TrainingDetection.FP > 0 || r.TrainingDetection.FN > 0 || r.TrainingVersions.Wrong > 0 || r.TrainingVersions.Missing > 0 || r.TrainingVersions.Unsupported > 0: + r.Status = "failed_training" + case r.Detection.FP > 0 || r.Detection.FN > 0 || r.Versions.Wrong > 0 || r.Versions.Missing > 0 || r.Versions.Unsupported > 0: + r.Status = "failed_holdout" + case r.Detection.TP == 0 || r.Detection.TN == 0: + r.Status = "insufficient_holdout" + default: + r.Status = "passed_holdout" + } + results = append(results, r) + } + if err := writeJSON(filepath.Join(out, "generation.json"), results); err != nil { + return nil, err + } + return results, nil +} + +func evaluateGenerated(f *fingerlib.Finger, p generationPlan, s replaySample, samples map[string]replaySample) (generationCase, *common.Framework) { + l, _ := labelFor(s, p.Product) + var frame *common.Framework + var matched bool + if p.Probe == "" { + frame, _, matched = f.PassiveMatch(fingerlib.NewContent(s.raw, "", true)) + } else { + frame, _, matched = f.ActiveMatch(2, func(request []byte) ([]byte, bool) { + id, ok := s.Probes[string(request)] + return samples[id].raw, ok + }) + } + c := generationCase{ID: s.ID, Present: l.Present, Matched: matched, ExpectedVersion: l.Version} + if matched && frame != nil { + c.Version = frame.Version + } + return c, frame +} diff --git a/cmd/judgeeval/replay_metrics.go b/cmd/judgeeval/replay_metrics.go new file mode 100644 index 0000000..c69bb83 --- /dev/null +++ b/cmd/judgeeval/replay_metrics.go @@ -0,0 +1,235 @@ +package main + +import ( + "fmt" + "os" + "strings" + + "github.com/chainreactors/fingers/common" + "github.com/chainreactors/fingers/judge" +) + +type detectionScore struct { + TP int `json:"tp"` + FP int `json:"fp"` + TN int `json:"tn"` + FN int `json:"fn"` +} + +func (s *detectionScore) add(want, got bool) { + switch { + case want && got: + s.TP++ + case want: + s.FN++ + case got: + s.FP++ + default: + s.TN++ + } +} + +type versionScore struct { + Correct int `json:"correct"` + Wrong int `json:"wrong"` + Missing int `json:"missing"` + CorrectAbstentions int `json:"correct_abstentions"` + Unsupported int `json:"unsupported"` + ProductMissing int `json:"product_missing"` +} + +func (v *versionScore) add(f *common.Framework, want string) { + if f == nil { + v.ProductMissing++ + return + } + got := f.Version + switch { + case want == "" && got == "": + v.CorrectAbstentions++ + case want == "": + v.Unsupported++ + case got == want: + v.Correct++ + case got == "": + v.Missing++ + default: + v.Wrong++ + } +} + +type stageScore struct { + Detection detectionScore `json:"detection"` + Versions versionScore `json:"versions"` + UnlabelledPredictions int `json:"unlabelled_predictions"` +} +type replaySummary struct { + Model string `json:"model"` + BaselineSource string `json:"baseline_source"` + Samples int `json:"samples"` + UniqueHosts int `json:"unique_hosts"` + UniqueResponses int `json:"unique_responses"` + Errors int `json:"errors"` + LabelledPairs int `json:"labelled_pairs"` + Baseline stageScore `json:"baseline"` + Refined stageScore `json:"refined"` + Completed stageScore `json:"completed"` + FalseHitsRemoved int `json:"false_hits_removed"` + TrueHitsRemoved int `json:"true_hits_removed"` + NaturalMissesRecovered int `json:"natural_misses_recovered"` + MissedProductsSuggested int `json:"missed_products_suggested"` + VersionFills int `json:"version_fills"` + CorrectVersionFills int `json:"correct_version_fills"` + IncorrectVersionFills int `json:"incorrect_version_fills"` + UnlabelledVersionFills int `json:"unlabelled_version_fills"` + Generation []generationResult `json:"generation"` + Disagreements []string `json:"disagreements"` + Requests int `json:"provider_requests"` + CacheHits int `json:"exact_cache_hits"` + InputTokens int64 `json:"input_tokens"` + Seconds float64 `json:"seconds"` +} + +func summarizeReplay(m *replayManifest, rows []replayRow, gen []generationResult) replaySummary { + out := replaySummary{Samples: len(m.Samples), Generation: gen} + hosts, hashes := map[string]bool{}, map[string]bool{} + byID := map[string]replayRow{} + for _, r := range rows { + byID[r.ID] = r + } + for _, s := range m.Samples { + hosts[s.Group] = true + hashes[s.ContentSHA256] = true + r, ok := byID[s.ID] + if !ok || r.Err != "" { + out.Errors++ + continue + } + for _, stage := range []struct { + score *stageScore + frames common.Frameworks + }{{&out.Baseline, r.Baseline}, {&out.Refined, r.Refined}, {&out.Completed, r.Completed}} { + for _, l := range s.Labels { + f := findLabel(stage.frames, l) + stage.score.Detection.add(l.Present, f != nil) + if l.Version != nil { + stage.score.Versions.add(f, *l.Version) + } + } + for _, f := range stage.frames { + if f != nil { + if _, ok := labelFor(s, f.Name); !ok { + stage.score.UnlabelledPredictions++ + } + } + } + } + for _, l := range s.Labels { + out.LabelledPairs++ + before, after := findLabel(r.Baseline, l), findLabel(r.Completed, l) + if before != nil && after == nil { + if l.Present { + out.TrueHitsRemoved++ + } else { + out.FalseHitsRemoved++ + } + } + if l.Present && before == nil && after != nil { + out.NaturalMissesRecovered++ + } + if l.Present && after == nil && containsProductName(r.Suggestions, l) { + out.MissedProductsSuggested++ + } + // Count a labelled product once, considering all baseline aliases. + // Raw per-name changes remain in rows.jsonl for audit. + if after != nil && after.Version != "" && (before == nil || before.Version == "") { + out.VersionFills++ + switch { + case !l.Present: + out.IncorrectVersionFills++ + case l.Version == nil: + out.UnlabelledVersionFills++ + case after.Version == *l.Version: + out.CorrectVersionFills++ + default: + out.IncorrectVersionFills++ + } + } + if (after != nil) != l.Present { + out.Disagreements = append(out.Disagreements, fmt.Sprintf("%s %s: presence want=%t got=%t", s.ID, l.Product, l.Present, after != nil)) + } + if l.Version != nil && after != nil && after.Version != *l.Version { + out.Disagreements = append(out.Disagreements, fmt.Sprintf("%s %s: version want=%q got=%q", s.ID, l.Product, *l.Version, after.Version)) + } + } + unlabelled := map[string]bool{} + for name := range r.Filled { + if _, ok := labelFor(s, name); ok { + continue + } + key := judge.NormalizeName(name) + if !unlabelled[key] { + unlabelled[key] = true + out.VersionFills++ + out.UnlabelledVersionFills++ + } + } + } + out.UniqueHosts = len(hosts) + out.UniqueResponses = len(hashes) + return out +} +func containsProductName(names []string, l productLabel) bool { + for _, name := range names { + for _, expected := range append([]string{l.Product}, l.Aliases...) { + if judge.NormalizeName(name) == judge.NormalizeName(expected) { + return true + } + } + } + return false +} +func writeReplayReport(path string, s replaySummary) error { + var b strings.Builder + fmt.Fprintf(&b, "# 真实响应回放评估\n\n模型：%s；基线：%s。\n\n%d 份快照，%d 个主机组，%d 种去除易变响应头后的响应，%d 个已标注产品判断，失败 %d 份。\n\n", s.Model, s.BaselineSource, s.Samples, s.UniqueHosts, s.UniqueResponses, s.LabelledPairs, s.Errors) + b.WriteString("统计范围是保存的响应所支持的产品与版本。页面自报版本不是服务器安装审计；正负标注不代表完整技术栈。未标注的输出不计为误报，也不计为已证明正确。页面级结果含同主机响应，不能视作独立随机样本。\n\n") + b.WriteString("| 阶段 | TP | FP | FN | TN | 未标注输出 |\n|---|---:|---:|---:|---:|---:|\n") + for _, r := range []struct { + name string + s stageScore + }{{"原结果", s.Baseline}, {"Refine", s.Refined}, {"逐产品补版本", s.Completed}} { + d := r.s.Detection + fmt.Fprintf(&b, "| %s | %d | %d | %d | %d | %d |\n", r.name, d.TP, d.FP, d.FN, d.TN, r.s.UnlabelledPredictions) + } + fmt.Fprintf(&b, "\n有标注的误报移除 %d；真实命中误删 %d；自然漏报补回 %d。模型拒绝或新增的未标注项保留在 rows.jsonl 中供复核。\n\n", s.FalseHitsRemoved, s.TrueHitsRemoved, s.NaturalMissesRecovered) + fmt.Fprintf(&b, "仍漏报的产品中，名称建议覆盖 %d 项；建议不计作已识别，也不计作漏报补回。\n\n", s.MissedProductsSuggested) + b.WriteString("| 版本阶段 | 正确 | 错误 | 缺版本 | 正确留空 | 无依据填值 | 产品未识别 |\n|---|---:|---:|---:|---:|---:|---:|\n") + for _, r := range []struct { + name string + s versionScore + }{{"原结果", s.Baseline.Versions}, {"Refine", s.Refined.Versions}, {"逐产品补版本", s.Completed.Versions}} { + v := r.s + fmt.Fprintf(&b, "| %s | %d | %d | %d | %d | %d | %d |\n", r.name, v.Correct, v.Wrong, v.Missing, v.CorrectAbstentions, v.Unsupported, v.ProductMissing) + } + fmt.Fprintf(&b, "\n按产品合并别名后的新增版本 %d：标注确认正确 %d、错误 %d、未标注 %d。已有非空版本保留，错误由报告揭示；逐名称变化见 rows.jsonl。\n\n", s.VersionFills, s.CorrectVersionFills, s.IncorrectVersionFills, s.UnlabelledVersionFills) + b.WriteString("## 指纹生成\n\n训练只使用 positive/negative 指定快照。测试排除训练主机、相同响应与重复测试响应；生成结果重新从 YAML 加载、编译后测试。此处 passed 仅代表本批独立样本通过，仍输出候选规则供评审。\n\n| 计划 | 名称 | 独立测试 TP/FP/FN/TN | 版本 正确/错误/缺失 | 排除 | 结论 |\n|---|---|---|---|---:|---|\n") + for _, g := range s.Generation { + d := g.Detection + v := g.Versions + fmt.Fprintf(&b, "| %s | %s | %d/%d/%d/%d | %d/%d/%d | %d | %s |\n", g.Name, g.Product, d.TP, d.FP, d.FN, d.TN, v.Correct, v.Wrong, v.Missing, len(g.Excluded), g.Status) + if g.Error != "" { + fmt.Fprintf(&b, "\n%s: %s\n\n", g.Name, g.Error) + } + } + b.WriteString("\n训练样本的自动版本推断另外记录在 generation.json 的 training_versions / training_cases；训练正确不等于独立验证通过。\n") + b.WriteString("\n## 待核查\n\n") + if len(s.Disagreements) == 0 { + b.WriteString("已标注范围内没有分歧。\n") + } else { + for _, d := range s.Disagreements { + fmt.Fprintf(&b, "- %s\n", d) + } + } + fmt.Fprintf(&b, "\nProvider 调用 %d，完整输入精确缓存命中 %d，输入 token %d，耗时 %.1f 秒。费用未估算。\n\nbaseline.jsonl 保存输入；cleaned.jsonl 仅含成功完成的结果；rows.jsonl 保存拒绝、补回、补版本、名称建议及错误；generation.json 保存每个独立样本结果。\n", s.Requests, s.CacheHits, s.InputTokens, s.Seconds) + return os.WriteFile(path, []byte(b.String()), 0600) +} diff --git a/cmd/judgeeval/replay_test.go b/cmd/judgeeval/replay_test.go new file mode 100644 index 0000000..a17abc7 --- /dev/null +++ b/cmd/judgeeval/replay_test.go @@ -0,0 +1,198 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/chainreactors/fingers/common" + "github.com/chainreactors/fingers/judge" +) + +func stringPtr(s string) *string { return &s } +func testFrames(name, version string) common.Frameworks { + f := common.NewFrameworkWithVersion(name, common.FrameFromGUESS, version) + out := common.Frameworks{} + out.Add(f) + return out +} +func TestReplayCountsErrorsAndUnlabelledSeparately(t *testing.T) { + m := &replayManifest{Samples: []replaySample{{ID: "a", Group: "host-a", ContentSHA256: "a", Labels: []productLabel{{Product: "Apache", Present: true, Version: stringPtr("2.4.38")}, {Product: "nginx", Present: false}}}, {ID: "failed", Group: "host-b", ContentSHA256: "b", Labels: []productLabel{{Product: "Apache", Present: true}}}}} + base := testFrames("nginx", "") + after := testFrames("apache http server", "2.4.38") + after.Add(common.NewFramework("unlabelled-library", common.FrameFromGUESS)) + r := replayRow{ID: "a", Baseline: base, Refined: after, Completed: after, Filled: map[string]string{"apache http server": "2.4.38"}} + summary := summarizeReplay(m, []replayRow{r, {ID: "failed", Err: "network"}}, nil) + if summary.Errors != 1 || summary.LabelledPairs != 2 || summary.Baseline.Detection.FP != 1 || summary.Baseline.Detection.FN != 1 || summary.Completed.Detection.TP != 1 || summary.Completed.Detection.TN != 1 || summary.Completed.UnlabelledPredictions != 1 { + t.Fatalf("metrics: %+v", summary) + } + if summary.NaturalMissesRecovered != 1 || summary.FalseHitsRemoved != 1 || summary.CorrectVersionFills != 1 { + t.Fatalf("changes: %+v", summary) + } +} +func TestReplayExactNamesDoNotCreditDifferentProducts(t *testing.T) { + frames := testFrames("wordpress-woocommerce", "10.4.4") + if findLabel(frames, productLabel{Product: "wordpress"}) != nil { + t.Fatal("substring credited a different product") + } + if findLabel(frames, productLabel{Product: "WooCommerce", Aliases: []string{"wordpress-woocommerce"}}) == nil { + t.Fatal("explicit alias not matched") + } +} +func TestReplayAbstentionRequiresIdentifiedProduct(t *testing.T) { + v := versionScore{} + v.add(nil, "") + v.add(common.NewFramework("nginx", common.FrameFromGUESS), "") + v.add(common.NewFrameworkWithVersion("nginx", common.FrameFromGUESS, "1.2.3"), "") + if v.ProductMissing != 1 || v.CorrectAbstentions != 1 || v.Unsupported != 1 { + t.Fatalf("versions: %+v", v) + } +} +func TestGenerationSplitRejectsLeakageAndContradictoryLabels(t *testing.T) { + sample := func(id, group, hash string, present bool) replaySample { + return replaySample{ID: id, Group: group, ContentSHA256: hash, Labels: []productLabel{{Product: "Caddy", Present: present}}} + } + m := &replayManifest{Samples: []replaySample{sample("train", "a", "one", true), sample("negative", "b", "two", false), sample("same-host", "a", "three", true), sample("same-response", "c", "one", true), sample("test", "d", "four", true), sample("duplicate-test", "e", "four", true), sample("test-negative", "f", "five", false)}} + p := generationPlan{Name: "caddy", Product: "Caddy", Positive: []string{"train"}, Negative: []string{"negative"}} + test, excluded, err := generationSplit(m, p) + if err != nil || len(test) != 2 || len(excluded) != 3 { + t.Fatalf("split test=%v excluded=%v err=%v", test, excluded, err) + } + p.Positive = []string{"negative"} + if _, _, err := generationSplit(m, p); err == nil { + t.Fatal("contradictory training truth accepted") + } +} +func TestReplayValidatesResponseHashAndEvidence(t *testing.T) { + dir := t.TempDir() + raw := []byte("HTTP/1.1 200 OK\r\nServer: nginx/1.21.5\r\nContent-Length: 0\r\n\r\n") + if err := os.WriteFile(filepath.Join(dir, "page.http"), raw, 0600); err != nil { + t.Fatal(err) + } + m := replayManifest{Schema: 1, Samples: []replaySample{{ID: "one", URL: "http://example.test/", Group: "example.test", CapturedAt: "2026-09-25T00:00:00Z", Response: "page.http", SHA256: digest(raw), Labels: []productLabel{{Product: "nginx", Present: true, Evidence: []string{"Server: nginx/1.21.5"}, Basis: "header"}}}}} + path := filepath.Join(dir, "manifest.json") + if err := writeJSON(path, m); err != nil { + t.Fatal(err) + } + if _, err := loadReplay(path); err != nil { + t.Fatal(err) + } + m.Samples[0].SHA256 = strings.Repeat("0", 64) + _ = writeJSON(path, m) + if _, err := loadReplay(path); err == nil { + t.Fatal("changed response accepted") + } + m.Samples[0].SHA256 = digest(raw) + m.Samples[0].Labels[0].Evidence = []string{"invented"} + _ = writeJSON(path, m) + if _, err := loadReplay(path); err == nil { + t.Fatal("missing evidence accepted") + } +} +func TestReplayConfinesResponsePaths(t *testing.T) { + if _, err := confinedPath(t.TempDir(), "../secret.http"); err == nil { + t.Fatal("outside path accepted") + } +} + +type replayTestProvider struct{ calls int } + +func (*replayTestProvider) ID() string { return "test/replay" } +func (p *replayTestProvider) Judge(_ context.Context, _ any, qs map[string]judge.Question) (map[string]judge.Answer, error) { + p.calls++ + answers := map[string]judge.Answer{} + for key := range qs { + answers[key] = judge.Answer{Yes: 1} + } + return answers, nil +} + +func TestReplayOfflineNeverCallsProvider(t *testing.T) { + provider := &replayTestProvider{} + p := &replayProvider{Provider: provider, dir: t.TempDir(), offline: true} + qs := map[string]judge.Question{"product": judge.Binary("Is this nginx?")} + ctx := context.Background() + if _, err := p.Judge(ctx, "page", qs); err == nil || !strings.Contains(err.Error(), "offline cache miss") { + t.Fatalf("expected offline cache miss: %v", err) + } + if provider.calls != 0 || p.calls != 0 { + t.Fatal("offline cache miss called the provider") + } + p.offline = false + if _, err := p.Judge(ctx, "page", qs); err != nil { + t.Fatal(err) + } + p.offline = true + if answers, err := p.Judge(ctx, "page", qs); err != nil || answers["product"].Yes != 1 { + t.Fatalf("cached answer: %v %v", answers, err) + } + if _, err := p.Judge(ctx, "different page", qs); err == nil { + t.Fatal("different state reused cached answer") + } + p.namespace = "different-endpoint" + if _, err := p.Judge(ctx, "page", qs); err == nil { + t.Fatal("different endpoint reused cached answer") + } + if provider.calls != 1 || p.calls != 1 || p.hits != 1 { + t.Fatalf("calls=%d requests=%d hits=%d", provider.calls, p.calls, p.hits) + } +} + +func TestReplayVersionFillsDeduplicateAliases(t *testing.T) { + l := productLabel{Product: "PHP", Aliases: []string{"php-runtime"}, Present: true, Version: stringPtr("5.6.40")} + m := &replayManifest{Samples: []replaySample{{ID: "a", Labels: []productLabel{l}}}} + after := testFrames("PHP", "5.6.40") + after.Add(common.NewFrameworkWithVersion("php-runtime", common.FrameFromGUESS, "5.6.40")) + r := replayRow{ID: "a", Baseline: testFrames("PHP", "5.6.40"), Refined: after, Completed: after, Filled: map[string]string{"php-runtime": "5.6.40"}} + if s := summarizeReplay(m, []replayRow{r}, nil); s.VersionFills != 0 { + t.Fatalf("existing alias version counted as new: %+v", s) + } + r.Baseline = testFrames("PHP", "") + r.Filled["PHP"] = "5.6.40" + if s := summarizeReplay(m, []replayRow{r}, nil); s.VersionFills != 1 || s.CorrectVersionFills != 1 { + t.Fatalf("alias fills counted twice: %+v", s) + } +} + +func TestReplaySuggestionsDoNotRecoverDetections(t *testing.T) { + m := &replayManifest{Samples: []replaySample{{ID: "a", Labels: []productLabel{{Product: "New API", Present: true}}}}} + r := replayRow{ID: "a", Suggestions: []string{"new-api", "New API"}} + s := summarizeReplay(m, []replayRow{r}, nil) + if s.MissedProductsSuggested != 1 || s.NaturalMissesRecovered != 0 || s.Completed.Detection.FN != 1 { + t.Fatalf("suggestions credited as detections: %+v", s) + } +} + +func TestReplaySnapshotIsSelfContained(t *testing.T) { + raw := []byte("HTTP/1.1 200 OK\r\nServer: nginx/1.21.5\r\nContent-Length: 0\r\n\r\n") + m := &replayManifest{Schema: 1, Samples: []replaySample{{ID: "a", URL: "http://example.test/", Group: "example.test", CapturedAt: "2026-09-25T00:00:00Z", Response: "unavailable/page.http", SHA256: digest(raw), raw: raw}}} + dir := t.TempDir() + if err := saveReplaySnapshot(dir, m); err != nil { + t.Fatal(err) + } + loaded, err := loadReplay(filepath.Join(dir, "manifest.json")) + if err != nil { + t.Fatal(err) + } + if string(loaded.Samples[0].raw) != string(raw) || loaded.Samples[0].SHA256 != m.Samples[0].SHA256 { + t.Fatal("snapshot response changed") + } + if m.Samples[0].Response != "unavailable/page.http" { + t.Fatal("snapshot write mutated source manifest") + } +} + +func TestGenerationSplitRejectsSameHostnameAcrossGroups(t *testing.T) { + m := &replayManifest{Samples: []replaySample{ + {ID: "positive", URL: "http://example.test/", Group: "one", ContentSHA256: "1", Labels: []productLabel{{Product: "Caddy", Present: true}}}, + {ID: "negative", URL: "http://negative.test/", Group: "two", ContentSHA256: "2", Labels: []productLabel{{Product: "Caddy", Present: false}}}, + {ID: "leak", URL: "https://EXAMPLE.test:443/another-page", Group: "three", ContentSHA256: "3", Labels: []productLabel{{Product: "Caddy", Present: true}}}, + }} + plan := generationPlan{Name: "caddy", Product: "Caddy", Positive: []string{"positive"}, Negative: []string{"negative"}} + test, excluded, err := generationSplit(m, plan) + if err != nil || len(test) != 0 || len(excluded) != 1 || excluded[0] != "leak" { + t.Fatalf("hostname leakage: test=%v excluded=%v error=%v", test, excluded, err) + } +} diff --git a/docs/jev-maintenance-validation-20260926.md b/docs/jev-maintenance-validation-20260926.md new file mode 100644 index 0000000..fd4b795 --- /dev/null +++ b/docs/jev-maintenance-validation-20260926.md @@ -0,0 +1,94 @@ +# Jev 新产品指纹维护验证：20260926 批次 + +## 结论 + +本次扩大真实响应范围，核验了八种原目录没有的产品，生成八份候选；其中 **SearXNG、Wakapi、IT Tools 三种通过独立样本测试及原生库加载检查**，输出可直接加载的增量库。加载该库后再运行 Jev 清洗，也保留了全部标注真命中；重复维护识别为已有产品，不重复入库。 + +这验证了有证据标注和评审的辅助维护流程。`IsUnknownProduct` 对 IT Tools 的三份真实界面仍返回 false；`SuggestNames` 均能提出正确名称。因此当前自动发现入口有漏检，不能依赖单个 unknown 布尔值决定所有新产品是否进入维护流程。 + +## 交付物 + +- [原生增量库](../fingerprints/jev-maintained-20260926.yaml)：三个产品，1782 字节，SHA256 `522a85806592e10bdffa87ccb70be90e22098fdeba85cc52452235132520065a`。 +- [来源与验证记录](../fingerprints/jev-maintained-20260926.provenance.json)：目录哈希、真实样本 URL/哈希/产品标签、训练划分、测试结果、重复维护、离线复现及代码哈希。 +- [CLI 使用说明](../cmd/judgeeval/README.md)：`-manifest`、`-history`、`-maintain`、`-library`、`-offline` 的明确语义。 +- 完整本地数据：`.judge-data/expansion-20260926/`。HTTP 原文、模型答案缓存和各次运行被 Git 忽略，不混入源码或指纹文件。 + +## 数据及真值 + +新增采集尝试 53 个明确的公开 URL，成功 34 个、失败 19 个；合并上一批 35 份响应，共 **69 份响应、59 个主机/运营方组、65 份去除部分动态头后的不同响应**。失败请求单独保存，不计为负例。 + +原始快照 UTC 时间为 `2026-09-25T14:18:17Z` 至 `2026-09-25T16:05:54Z`；北京时间跨 9 月 25 日晚至 9 月 26 日凌晨，故目录沿用 `20260926` 批次名。 + +新目标包括：官方 SearXNG 实例目录使用种子 `20260926` 抽取的公开实例、项目公开演示和另外找到的公开安装。训练反例包含 nginx/WordPress 响应及官方文档/README，专门验证“页面提到产品”不会被当作该产品的运行界面。来源快照、目标列表、失败记录和选样方式均存档。这是按产品补充的验证样本，不是互联网产品分布的随机代表样本。 + +682 个产品—响应标注对由响应头、generator meta、标题、资源路径和人工复核支持。真值指**响应可支持的产品/版本**，不等于主机实际安装审计。没有证据的版本应保持空值；未标注的识别结果单列，不算真阳性。 + +一次标签修订：PairDrop 原本被标为空版本，但原 HTML 的产品标题旁明确出现 `
v1.11.2
`，已修正为 `1.11.2` 并保存修订记录。没有依据模型回答修改标签。 + +## 实际验证链路 + +1. 审计五个内置 HTTP 目录：fingers、goby、ehole、wappalyzer、fingerprinthub；再检查别名表及加载后的 fingers 运行时目录，记录 SHA256。八个候选在初始审计范围均无同名/显式别名产品。 +2. 人工依据证据组织产品正反例；Jev 从响应中提取的候选里选择名称和版本，Go 代码生成原生规则。使用普通 `Positive`，没有调用 `PositiveVersion` 向真实评估的生成器喂标签版本。 +3. 导出 YAML 后重新加载、编译，验证训练匹配及版本；用训练主机以外的正反例测试。相同主机、同一运营方组及重复内容被隔离。 +4. 候选至少具备两个独立正例主机组、五个独立反例主机组；身份和版本验证均无错漏才允许进入增量库。 +5. 调用 `engine.Fingers().LoadFromYAML(...)`、`engine.Compile()`，用真正的 `DetectContent` 重放全部 69 份响应。检查新产品、版本、已有命中和已有非空版本。 +6. 用加载后的库再运行清洗和版本补全，验证重复维护去重；最后严格离线重放，缓存未命中即失败。 + +## 新增指纹结果 + +下表为最终原生库加载后、**排除训练数据的测试集**结果。三个产品的测试集共享部分反例，不能把负例总数当作不同网站数量相加。 + +| 产品 | 训练正例 | 独立测试正例 | 测试负例 | 新增前漏报 → 加载后命中 | FP / FN | 测试版本 | +|---|---:|---:|---:|---:|---:|---| +| SearXNG | 2 | 8 | 47 | 8 → 8 | 0 / 0 | 8 个准确，保留日期与提交后缀 | +| Wakapi | 1 | 3 | 53 | 3 → 3 | 0 / 0 | 3 个准确，覆盖 2.18.0 与 2.12.0 | +| IT Tools | 1 | 2 | 54 | 2 → 2 | 0 / 0 | 2 个正确留空，未虚构版本 | + +共补回 **13 个独立测试正例**的产品识别，准确提取其中 11 个有证据的版本。原生加载在全部样本中没有丢失已有命中、改坏已有非空版本或产生新增产品的标注错误。 + +`library.yaml` 是新增的原生列表文件，需显式加载；没有重写内置 gzip 资源或发布外部模板仓库。 + +### 尚未收录 + +| 产品 | 已采集正例 | 结论 | +|---|---:|---| +| PairDrop | 1 | 能生成规则、训练版本 1.11.2 正确，但缺独立正例 | +| Docmost | 1 | 能生成规则，但缺独立正例 | +| Linkwarden | 1 | 能生成规则，但缺独立正例 | +| Karakeep | 3 | 均属同一运营方组，无法作为独立测试正例 | +| New API | 1 | 能生成规则并提取 1.0.0-rc.35，但缺独立正例 | + +五种候选最终均为 `insufficient_holdout`，保存在最终运行的 `fingerprints/` 下，未进入发布用增量文件。它们在本批测试负例上未命中，仍不能据此证明召回能力。 + +## 真实数据暴露的问题与修复 + +- **共享基础设施被学成应用指纹**：初版 IT Tools 在测试集有 6 个误报；Docmost、Linkwarden、Karakeep 分别有 9、8、6 个。规则曾选用 Cloudflare 或通用安全头。现在应用候选头必须带有目标产品身份，普通部署环境特征不能独立成为应用规则。 +- **年份版本与 generator 名称**：SearXNG 的 `2026.9.23+3cd69d30e` 被旧模式截断或混进产品名称。现在保留四位年份和后缀，以 generator 声明生成稳定识别及版本规则，可跨定制标题复用。 +- **版本远离产品名**：Wakapi 版本在静态资源参数和页脚。新增带产品身份约束的版本正则；身份与版本在同一个正则中结合，避免原生匹配器的 OR 语义造成无约束版本命中。 +- **加载后清洗回归**：最初重跑发现 IT Tools 两个真命中被删，SearXNG 文档被错误召回并填入版本。补充页面 description 证据，明确静态浏览器应用和文档主题的不同语义后，这三个具体问题均消除。 +- **重复收录与样本计数**：`-library` 归档并加载已有扩展库；运行时目录参与查重。相同主机名或运营方的交叉分组连通后只计一次。 + +初始失败保留在 `run-01`，加载后清洗失败保留在 `run-05-with-library`。这些独立主机测试集在修复中被重复使用，最终结果属于回归验证，**不是未经调试使用的外部盲测准确率**。 + +## 清洗与发现边界 + +最终未加载增量库时，`Refine + Version` 在已标注范围删除 24 个误报、误删 0 个、正确补全 20 个版本，没有自然补回产品漏报；名称建议覆盖 25 个漏报样本，建议不计为已识别。 + +加载三条新指纹后，`run-06-with-library` 的清洗结果为 **TP 63 / FP 0 / TN 610 / FN 9**；已有真命中误删 0、错误补版本 0，37 个有值版本正确、26 个应空版本正确留空。这里包含训练样本；9 个漏报仍包括尚未收录的候选以及旧批次的 Nutz、aaPanel。还有 148 个未标注产品预测，不能把当前标注范围的 FP=0 推广成整体误报率为零。 + +在没有加载新库时，`SuggestNames` 分别支持 SearXNG 10/10、Wakapi 4/4、IT Tools 3/3；`IsUnknownProduct` 分别只有 9/10、4/4、0/3。当前维护依赖证据分组及评审，不能只用 `Classify.Generic`/unknown 作为发现入口。主动探测规则、覆盖更多独立产品版本、全面纠正已有错误非空版本也不属于本批已完成的验证范围。 + +## 复现与检查 + +- `run-07-final`：最终生成、审计和原生加载；`run-06-with-library`：相同库加载后的清洗及重复维护，三种产品全部 `already_catalogued`、没有新库重复输出。 +- `run-08-offline`：0 次模型请求、483 次精确缓存命中。语义指标一致，`library.yaml`、`generation.json`、`maintenance.json`、`rows.jsonl`、`baseline.jsonl`、`cleaned.jsonl` 均与最终运行逐字节一致。 +- 每次运行保存原始输入快照、标签清单、构建信息；另存本次代码文件哈希，便于在有未提交修改的工作区复核。 + +```powershell +go build -tags goregexp -o bin/judgeeval.exe ./cmd/judgeeval +./bin/judgeeval.exe -offline -maintain -manifest .judge-data/expansion-20260926/run-07-final/manifest.json -history .judge-data/expansion-20260926/run-07-final/baseline.jsonl -cache .judge-data/cache -out .judge-data/reproduce-new-products +go test -short -tags goregexp ./judge ./judge/jev ./cmd/judgeeval . ./cmd/engine -count=1 -timeout 3m +git diff --check +``` + +受影响包测试通过。全仓库 `go test ./...` 的既有 go-re2/go.sum、nmap 和 fingerprinthub 旧接口问题未纳入修复，未声称全仓库测试通过。后续扩大验证应优先补充五个草案产品的独立部署，以及未参与调试的新一批样本。 diff --git a/docs/jev-real-validation-20260925.md b/docs/jev-real-validation-20260925.md new file mode 100644 index 0000000..91fb8ec --- /dev/null +++ b/docs/jev-real-validation-20260925.md @@ -0,0 +1,104 @@ +# Jev 真实响应验证：2026-09-25 + +## 结论 + +已经用新采集的真实 HTTP 响应跑通保存结果清洗、逐产品版本补全、名称建议、候选指纹生成、原生 YAML 导出加载和离线回放。发现的问题已修复并重新评估。 + +在本批已标注范围内，清洗移除 24 个误报，没有误删真实命中；新增 20 个正确版本。自然漏报仍有 3 个，没有被自动补回。6 项指纹生成实验中，3 项通过独立样本测试，另外 3 项独立正例不足。不能据此宣称全部功能已经验证充分，或推断全网误报率为零。 + +主报告：[run-03/report.md](../.judge-data/run-03/report.md)；机器指标：[run-03/metrics.json](../.judge-data/run-03/metrics.json)；离线复现：[run-04/report.md](../.judge-data/run-04/report.md)。原始响应和模型缓存均留在本机 Git 忽略目录 `.judge-data/`。 + +## 数据与标注 + +- 2026-09-25 22:18:17–22:18:19（北京时间）重新采集；来源为用户指定 `47.239.11.0/24` 内的 HTTP/HTTPS 页面及 3 份 `/robots.txt` 响应。 +- 共 35 份响应、30 个主机组、31 种去除易变响应头后的响应；本批明确 URL 列表采集失败 0 项。这是选定的响应集合，不是完整网段可用性普查或全网随机代表样本。 +- 138 个产品正负标注，其中 49 个包含版本标注。版本缺省表示未标注，空字符串表示该响应没有可归属的版本证据，非空字符串表示期望的完整版本。 +- 标注依据为原始响应头、generator meta、资源路径、页面产品归属及保存的原文片段。加载时检查 SHA256、HTTP 完整性和证据片段是否存在；证据是否足以支持结论仍需要复核。 +- 标注不来自 Jev 的答案，不输入清洗或版本判定。真值定义为“响应证据支持的产品和版本”，不是服务器安装审计；IP 抓取也不能覆盖域名虚拟主机的所有内容。 +- 页面自报版本、产品头和资产版本都可能与实际安装状态不同。同一主机多份响应不视作独立统计样本；未标注输出不能算作已证明正确或误报。 + +首次标注复核后修正了两项：New API 的 `X-New-Api-Version: v1.0.0-rc.35`，以及 ThinkPHP 产品归属下的 `V8.1.2`。修订前后值和原文保存在 [label-review.json](../.judge-data/real-20260925/label-review.json)。最终各阶段统一按修订后的标签计分。 + +## 清洗、误报与漏报 + +计数单位是已标注的“响应 × 产品”。产品名称按规范化名称及显式别名匹配，不使用子串判断。 + +| 阶段 | TP | FP | FN | TN | 未标注输出 | +|---|---:|---:|---:|---:|---:| +| 保存的原规则结果 | 46 | 24 | 3 | 65 | 74 | +| Refine | 46 | 0 | 3 | 89 | 28 | +| 再对各产品补版本 | 46 | 0 | 3 | 89 | 28 | + +- 已标注误报移除 24，真实命中误删 0，自然漏报恢复 0；35 份响应全部完成，无失败行。 +- 剩余漏报：Nutz（`c31d254bc1fd0230`）、aaPanel（`afac5614625bfc6f`）、New API（`05346dbcf28946f0`）。 +- 名称建议覆盖 Nutz 和 New API 两项，但建议没有成为已接受的指纹命中，因此不计作漏报恢复。 +- 清洗后仍有 28 个未标注输出，尚不能由本批标签判断其准确性。 + +本次 `-history` 使用的是本批新响应首次运行时保存的 `baseline.jsonl`。这证明“保存结果 → 重新判定 → 清洗结果”机制可用，不代表覆盖长期生产历史分布。 + +## 版本补全 + +| 阶段 | 正确 | 错误 | 缺版本 | 正确留空 | 无依据填值 | 产品未识别 | +|---|---:|---:|---:|---:|---:|---:| +| 原规则结果 | 3 | 0 | 20 | 23 | 0 | 3 | +| Refine | 9 | 0 | 14 | 23 | 0 | 3 | +| 各保留产品分别调用 Version | 23 | 0 | 0 | 23 | 0 | 3 | + +`Refine` 只补主产品的版本；独立 `Version` 调用验证剩余保留产品的空版本。正确非空版本从 3 增至 23，净新增 20 项。另有 2 项新增版本未标注，不计入正确项。按产品合并别名，基线任一别名已有版本即不算新增；逐名称变化仍保存在 `rows.jsonl`。 + +已有非空版本保留。本批没有已标注的错误历史版本，因此“修正一个已有但错误的版本值”未在本批验证，也不是这里的补空值操作。漏报产品的版本记为“产品未识别”，不能冒充正确留空。 + +## 指纹生成与加载 + +真实正反例经 `NewGenerator → Positive/Negative → Generate` 生成原生指纹。自动命名计划使用 `SuggestNames`；版本从普通 `Positive` 自动推断，不向生成器传入评估标签。导出 YAML 后重新加载、编译，并通过原生 `PassiveMatch` 或录制探测响应的 `ActiveMatch` 测试。 + +训练和测试按主机组、实际 hostname、去除易变头的响应内容隔离；测试集还排除重复响应。以下计数均为未参加训练的独立响应，TP/FP/FN/TN 顺序固定。 + +| 计划 | 独立 TP/FP/FN/TN | 独立版本结果 | 结论 | +|---|---|---|---| +| nginx-auto | 13/0/0/5 | 4 个正确版本，9 个正确留空 | passed_holdout | +| caddy-auto | 2/0/0/18 | 2 个正确留空 | passed_holdout | +| apache-auto-version | 2/0/0/17 | 2 个正确留空 | passed_holdout | +| caddy-active | 0/0/0/0 | 无独立样本 | insufficient_holdout | +| new-api-candidate | 0/0/0/3 | 无独立正例版本 | insufficient_holdout | +| softether-candidate | 0/0/0/4 | 无独立正例版本 | insufficient_holdout | + +训练版本单独统计：nginx 为 2 个正确版本，Apache 为 1 个，New API 为 1 个。New API 的候选规则已完整提取 `1.0.0-rc.35`，并采用版本头名称作为产品特征；这仅证明该训练响应上的生成机制，不能代替独立正例验证。Apache 的版本提取也只在本批训练响应得到验证。 + +Caddy 主动规则已在录制的训练探测响应上完成匹配；其剩余正例探测响应与训练内容重复，被剔除。SoftEther 的另一主机返回相同页面，也被剔除。New API 只有一个正例主机。候选文件保存在 [run-03/fingerprints](../.judge-data/run-03/fingerprints)，未直接写入发布指纹库。 + +## 真实证据推动的修复 + +1. **协议特征误保留**：HTTP Basic 等指纹原先被无条件信任。现在需要对应响应头证据；正文提及认证或错误命中不能绕过清洗。 +2. **版本被截断**：统一版本 token，保留 OpenSSL `1.0.2q`、预发布 `1.0.0-rc.35`、`8.1SP2` 等后缀，支持脚本文件名的版本边界。 +3. **版本候选被噪声挤掉**：先收集再按证据排序，避免前部大量数字挤掉后部 generator、响应头或资源版本。 +4. **生成规则绑定单一版本**：`X-*-Version` 使用头名称识别产品，并独立提取版本；生成头规则时规范化产品名。 +5. **评估夸大改进**：修正别名重复统计，区分名称建议与真正补回漏报，未识别产品不计为版本留空成功，错误行及未标注输出单列。 + +对应回归位于 [judge/real_evidence_test.go](../judge/real_evidence_test.go) 和 [cmd/judgeeval/replay_test.go](../cmd/judgeeval/replay_test.go)。最小证据回归包含受控构造，不加入真实样本数量或真实准确率统计。 + +## 回放与复现 + +每个运行目录保存原始响应副本、相对路径 manifest、SHA256、原始及清洗后 Frameworks、逐项结果、生成 YAML 和 Go 构建信息。响应快照可独立加载；严格离线的模型答案还需要保留指定的精确缓存目录。 + +| 运行 | 用途 | Provider 请求 | 精确缓存命中 | 输入 tokens | +|---|---|---:|---:|---:| +| run-01 | 首轮真实响应评估 | 180 | 41 | 313,477 | +| run-02 | 证据修复后评估 | 30 | 190 | 64,248 | +| run-03 | 最终计数及训练版本核验 | 0 | 220 | 0 | +| run-04 | 无凭据严格离线回放 run-03 快照 | 0 | 220 | 0 | + +模型标识为 `jev/jev-1.13.0`。精确缓存键包含模型、endpoint、完整状态和问题，评估禁用近似页面缓存。离线缓存缺失直接报错，测试已确认不会回退调用 Provider。最终回放检查：除耗时外指标一致；manifest、baseline、cleaned、rows、generation 的 JSON 内容一致；6 个 YAML 文件逐字节一致。复现证明输入与已保存模型回答的确定性，不等于重新远程请求模型后的稳定性试验。 + +```powershell +go build -tags goregexp -o bin/judgeeval.exe ./cmd/judgeeval +# 使用新目录，已有结果拒绝覆盖。离线无需 TYPESAFE_API_KEY。 +./bin/judgeeval.exe -offline -manifest .judge-data/run-03/manifest.json -history .judge-data/run-03/baseline.jsonl -cache .judge-data/cache -out .judge-data/run-reproduce -rps 8 + +go test -short -tags goregexp ./judge ./judge/jev ./cmd/judgeeval . ./cmd/engine -count=1 -timeout 3m +git diff --check +``` + +本次受影响包测试及 diff 检查通过。全仓库测试尚有此前存在的 benchmark go-re2 依赖和 nmap/fingerprinthub 旧接口问题，不在本次修复范围，不能声称全仓库测试通过。 + +后续验证缺口集中在：3 个自然漏报、错误非空历史版本、New API/SoftEther/Caddy 主动规则的独立正反例，以及跨网段、跨版本和跨时间的新批次。现有报告已明确区分“已执行”“本批通过”和“证据不足”。 diff --git a/docs/jev-report.html b/docs/jev-report.html new file mode 100644 index 0000000..8ee8cf6 --- /dev/null +++ b/docs/jev-report.html @@ -0,0 +1,142 @@ + + + + + +Jev 判定层效果 + + + +
+ +

Jev 判定层效果

+

同样 1070 个真实网站，传统规则指纹 vs 规则 + Jev 判定

+
传统规则规则 + Jev
+ +
+ +
+
可信度（40 页标准答案）
0 误删
误报剔除 37/37，真实结果保留 45/45
+
速度代价
+0.3 秒/页
异步执行，原有规则识别不变慢
+
成本
$3.3
每万页
+
+ +

+ 注意：改过品牌的产品 Jev 也认不出来，发现新产品仍靠规则；Jev 请求失败时自动保持传统规则的结果。
+ 数据来自 Common Crawl 公开抓取数据，每个网站只取一页；"误报"指产品名只出现在正文、不在任何页面结构中的命中；版本号以页面自带的 generator 标记为参考答案；模型 jev-1.13.0。文字版：jev-report.md +

+
+ + + diff --git a/docs/jev-report.md b/docs/jev-report.md new file mode 100644 index 0000000..c135d21 --- /dev/null +++ b/docs/jev-report.md @@ -0,0 +1,29 @@ +# Jev 判定层效果报告 + +在 1070 个真实网站上，把**传统规则指纹**与**规则 + Jev 判定**逐页对比。 + +## 核心对比 + +| 指标 | 传统规则 | 规则 + Jev | 效果 | +|---|---|---|---| +| **误报**：只在正文里提到、实际并不存在的产品 | 138 个 | 13 个 | 减少 91% | +| **重复**：每页结果条数 | 13.4 条 | 6.5 条 | 减半，同名重复全部合并 | +| **漏报**：规则没识别、被补回的产品 | 0 个 | 467 个 | 新增 | +| **版本号**：识别正确的页面（共 274 页） | 43 页 | 200 页 | 提升 4.7 倍 | + +**代价**：每页多花约 0.3 秒（异步执行，不影响原有的规则识别速度），每万页约 3.3 美元。 + +## 可信度 + +在 40 个有标准答案的页面上： +- 误报剔除 37/37； +- 真实结果保留 45/45，没有误删。 + +## 注意 + +- 改过品牌的产品，Jev 也认不出来；发现新产品仍然靠规则。 +- Jev 请求失败时，结果自动保持为传统规则的结果。 + +--- + +测试方法：数据来自 Common Crawl 公开抓取数据，每个网站只取一页。"误报"指产品名只出现在正文、不在任何页面结构中的命中；"版本号正确"以页面自带的 generator 标记作为参考答案。模型为 `jev-1.13.0`。 diff --git a/engine.go b/engine.go index 30a0a72..c89bd87 100644 --- a/engine.go +++ b/engine.go @@ -11,7 +11,6 @@ import ( "github.com/chainreactors/fingers/fingerprinthub" "github.com/chainreactors/fingers/fingers" "github.com/chainreactors/fingers/goby" - "github.com/chainreactors/fingers/judge" gonmap "github.com/chainreactors/fingers/nmap" "github.com/chainreactors/fingers/resources" wappalyzer "github.com/chainreactors/fingers/wappalyzer" @@ -88,9 +87,6 @@ type Engine struct { Enabled map[string]bool Capabilities map[string]common.EngineCapability // 新增：记录各引擎能力 - // 可选的判定层, 见 AttachJudge - judge *judge.Judge - recall *judge.Retriever } func (engine *Engine) String() string { diff --git a/fingerprints/README.md b/fingerprints/README.md new file mode 100644 index 0000000..ae11268 --- /dev/null +++ b/fingerprints/README.md @@ -0,0 +1,23 @@ +# 经真实响应验证的 Jev 增量指纹 + +本目录保存通过 `judgeeval -maintain` 生成、导出、重新编译、独立样本测试及原生加载检查的原生 YAML。每个文件附带数据来源、目录审计和验证范围；默认引擎的内置压缩资源仍按原有资源流程维护。 + +## 加载 + +```go +engine, err := fingers.NewEngine(fingers.FingersEngine) +if err != nil { + return err +} +if err := engine.Fingers().LoadFromYAML("fingerprints/jev-maintained-20260926.yaml"); err != nil { + return err +} +if err := engine.Compile(); err != nil { + return err +} +hits, err := engine.DetectContent(rawHTTPResponse) +``` + +生成规则维持原生 `Finger` / `Frameworks` 语义。SearXNG 和 Wakapi 包含版本提取；IT Tools 在当前真实样本没有明确版本证据时只识别产品。 + +扩展测试报告见 [Jev 指纹库维护验证](../docs/jev-maintenance-validation-20260926.md)，运行数据和完整响应保存在本地 `.judge-data/expansion-20260926/`。这些指纹的结论仅适用于报告中列出的版本、样本和标注范围。 diff --git a/fingerprints/jev-maintained-20260926.provenance.json b/fingerprints/jev-maintained-20260926.provenance.json new file mode 100644 index 0000000..4fc8daf --- /dev/null +++ b/fingerprints/jev-maintained-20260926.provenance.json @@ -0,0 +1,4298 @@ +{ + "batch": "expansion-20260926", + "scope": "Products and versions supported by archived HTTP evidence; not an audit of installed software. Host-isolated regression set reused during development, not an untouched external benchmark.", + "model": "jev/jev-1.13.0", + "library": "jev-maintained-20260926.yaml", + "library_sha256": "522a85806592e10bdffa87ccb70be90e22098fdeba85cc52452235132520065a", + "final_run": ".judge-data/expansion-20260926/run-07-final", + "offline_run": ".judge-data/expansion-20260926/run-08-offline", + "loaded_library_run": ".judge-data/expansion-20260926/run-06-with-library", + "new_capture": { + "attempts": 53, + "successes": 34, + "failures": 19 + }, + "corpus": { + "samples": 69, + "unique_hosts": 59, + "unique_responses": 65, + "labelled_pairs": 682 + }, + "manifest_sha256": "699557708fdec2d6f0404c3c0f37978a6b01d984cdebf3e4b01eae6feba0cfb3", + "target_selection": { + "selection": "SearXNG official-directory seeded sample plus project demos and search-indexed public installations; docs are mention-only negative controls", + "seed": 20260926, + "batch1": "targets-batch1.txt", + "batch2": "targets-batch2.txt" + }, + "sources": [ + { + "name": "searxng-instances", + "url": "https://searx.space/data/instances.json", + "status": 200, + "bytes": 413354, + "file": ".judge-data\\expansion-20260926\\searxng-instances.json", + "sha256": "e17431b01ccb60b9885139be4d0ecd9f5baf0f45ce2bc169501fd4c8a3495a4f" + }, + { + "name": "wakapi-readme", + "url": "https://raw.githubusercontent.com/muety/wakapi/master/README.md", + "status": 200, + "bytes": 58370, + "file": ".judge-data\\expansion-20260926\\wakapi-readme.md", + "sha256": "626b3574cb6d1732ecd8ae76a0b0969a8a83993022dea67b1be43385eb3a38d0" + }, + { + "name": "pairdrop-readme", + "url": "https://raw.githubusercontent.com/schlagmichdoch/PairDrop/master/README.md", + "status": 200, + "bytes": 7156, + "file": ".judge-data\\expansion-20260926\\pairdrop-readme.md", + "sha256": "fe1fcf4cb27f732477160c00ec0c9313bcdfa95bed89f482a5dda12c6d24cac7" + }, + { + "name": "gatus-readme", + "url": "https://raw.githubusercontent.com/TwiN/gatus/master/README.md", + "status": 200, + "bytes": 215037, + "file": ".judge-data\\expansion-20260926\\gatus-readme.md", + "sha256": "e1156f61879dcac5dcb3cc5931f5ae542aa757cc2c4973507664fe69661121ad" + }, + { + "name": "karakeep-readme", + "url": "https://raw.githubusercontent.com/karakeep-app/karakeep/main/README.md", + "status": 200, + "bytes": 8186, + "file": ".judge-data\\expansion-20260926\\karakeep-readme.md", + "sha256": "ecc3cd2eb9a38a919bea7ccb71cd9e83c2dfc01d03601a70879433e0d3f326d9" + } + ], + "catalogs": [ + { + "source": "fingers", + "sha256": "770208a0f55ab01d6d983f1ff304e501ab8c717d7812d4ad774518b39c00d1f5", + "names": 4289 + }, + { + "source": "goby", + "sha256": "b1cdffe4eca50c0140ee27ddebf23ec007a3337b9a3f528c889d704ad0218fe1", + "names": 7083 + }, + { + "source": "ehole", + "sha256": "0e673c5569b04c3ffb3eed8410048807a8103a62a4e7e500e445ec2e8d2193af", + "names": 811 + }, + { + "source": "wappalyzer", + "sha256": "18b47fc3deec61c8260f1f1da9efe8301d292cc0f14f0f57885181482eed71b9", + "names": 7226 + }, + { + "source": "fingerprinthub", + "sha256": "e4b2faa806e032e09b72c6e17d5a63e72f9a80449d6aedee82ee88138213a4c6", + "names": 3483 + }, + { + "source": "aliases", + "sha256": "24316f010fd85c9d8d7d68d1a9241e10f770851cbfc939c2f78124ae9291dc6b", + "names": 87 + }, + { + "source": "runtime-fingers", + "sha256": "3f8303600efb5aa5e4ce668f418e4ac5c46b7e029643f2bc2d216872638c47d3", + "names": 4289 + } + ], + "maintenance": { + "library": "library.yaml", + "catalog_file": "catalog-before.json", + "min_positive_hosts": 2, + "min_negative_hosts": 5, + "products": [ + { + "plan": "searxng-new", + "product": "SearXNG", + "status": "validated_library", + "existing_catalog_matches": null, + "positive_samples": 10, + "suggested_positives": 10, + "unknown_positives": 9, + "holdout_positive_hosts": 8, + "holdout_negative_hosts": 42, + "before": { + "detection": { + "tp": 0, + "fp": 0, + "tn": 47, + "fn": 8 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 8 + }, + "unlabelled_predictions": 0 + }, + "after": { + "detection": { + "tp": 8, + "fp": 0, + "tn": 47, + "fn": 0 + }, + "versions": { + "correct": 8, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "unlabelled_predictions": 0 + }, + "test_ids": [ + "36f6645c0290996f", + "145e314623e76dfd", + "7a415cdbb6244358", + "e3651c67dbb2ad17", + "718e4c943a25a995", + "c31d254bc1fd0230", + "e7d23d9f3c44fee1", + "867e79ba27f05078", + "bc537dc669808db3", + "e13843bffacca3e0", + "6025845b2aa21eb7", + "f288433c7ec32303", + "51ccb99538406996", + "1700c6017ed82aaf", + "f7039343f41ed237", + "7c56b28279b128c0", + "afac5614625bfc6f", + "d79b4325363f7010", + "f349b9e30f493cb5", + "b8c2c3d64f1947fa", + "087a771998e8460b", + "05346dbcf28946f0", + "0a87b2d06772daf3", + "2ee69d30ad1b9ba1", + "6290a947cc8369e7", + "846e9f943798d3ab", + "86e96862d2e6618c", + "335745c8550b1062", + "ebc3a07dd3e76b6e", + "fb25cf0e86ffcd7d", + "fa1b267c4911f7a0", + "33481799a00eb89a", + "e80f19b7afe3d734", + "e173c09a7713b5b0", + "3ee248ced1fa920a", + "65b15b71fa6f508f", + "a8f7be21e03253d3", + "2b89e57a9b28bfbf", + "d9b5eaa66bdc7bec", + "b2382b7ca202c344", + "9a8481abc3453d0b", + "8e2c3affdcca9c97", + "bfba0e0e48f5eb54", + "97544c226c577151", + "68eb99ccb687369d", + "24c03d50574b79f2", + "302f5cbbc6e1596b", + "a04da6570878fb03", + "09e85a53a7b4850d", + "e331de721c086202", + "0dabb37b2a7b3687", + "91315c96894dcfd3", + "fba6e296e0acdccc", + "e693fe9346caa44c", + "28276d6b56102882" + ] + }, + { + "plan": "wakapi-new", + "product": "Wakapi", + "status": "validated_library", + "existing_catalog_matches": null, + "positive_samples": 4, + "suggested_positives": 4, + "unknown_positives": 4, + "holdout_positive_hosts": 3, + "holdout_negative_hosts": 48, + "before": { + "detection": { + "tp": 0, + "fp": 0, + "tn": 53, + "fn": 3 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 3 + }, + "unlabelled_predictions": 0 + }, + "after": { + "detection": { + "tp": 3, + "fp": 0, + "tn": 53, + "fn": 0 + }, + "versions": { + "correct": 3, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "unlabelled_predictions": 0 + }, + "test_ids": [ + "36f6645c0290996f", + "145e314623e76dfd", + "7a415cdbb6244358", + "e3651c67dbb2ad17", + "718e4c943a25a995", + "c31d254bc1fd0230", + "e7d23d9f3c44fee1", + "867e79ba27f05078", + "bc537dc669808db3", + "e13843bffacca3e0", + "6025845b2aa21eb7", + "f288433c7ec32303", + "51ccb99538406996", + "1700c6017ed82aaf", + "f7039343f41ed237", + "7c56b28279b128c0", + "afac5614625bfc6f", + "d79b4325363f7010", + "f349b9e30f493cb5", + "b8c2c3d64f1947fa", + "087a771998e8460b", + "05346dbcf28946f0", + "0a87b2d06772daf3", + "2ee69d30ad1b9ba1", + "6290a947cc8369e7", + "846e9f943798d3ab", + "86e96862d2e6618c", + "335745c8550b1062", + "ebc3a07dd3e76b6e", + "fb25cf0e86ffcd7d", + "fa1b267c4911f7a0", + "33481799a00eb89a", + "e80f19b7afe3d734", + "e173c09a7713b5b0", + "7c727a84546a6bba", + "3ee248ced1fa920a", + "65b15b71fa6f508f", + "a8f7be21e03253d3", + "2b89e57a9b28bfbf", + "d9b5eaa66bdc7bec", + "b2382b7ca202c344", + "9a8481abc3453d0b", + "a40e20d96d920f67", + "8e2c3affdcca9c97", + "bfba0e0e48f5eb54", + "97544c226c577151", + "68eb99ccb687369d", + "24c03d50574b79f2", + "a04da6570878fb03", + "09e85a53a7b4850d", + "e331de721c086202", + "0dabb37b2a7b3687", + "91315c96894dcfd3", + "fba6e296e0acdccc", + "e693fe9346caa44c", + "28276d6b56102882" + ] + }, + { + "plan": "it-tools-new", + "product": "IT Tools", + "status": "validated_library", + "existing_catalog_matches": null, + "positive_samples": 3, + "suggested_positives": 3, + "unknown_positives": 0, + "holdout_positive_hosts": 2, + "holdout_negative_hosts": 49, + "before": { + "detection": { + "tp": 0, + "fp": 0, + "tn": 54, + "fn": 2 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 2 + }, + "unlabelled_predictions": 0 + }, + "after": { + "detection": { + "tp": 2, + "fp": 0, + "tn": 54, + "fn": 0 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 2, + "unsupported": 0, + "product_missing": 0 + }, + "unlabelled_predictions": 0 + }, + "test_ids": [ + "36f6645c0290996f", + "145e314623e76dfd", + "7a415cdbb6244358", + "e3651c67dbb2ad17", + "718e4c943a25a995", + "c31d254bc1fd0230", + "e7d23d9f3c44fee1", + "867e79ba27f05078", + "bc537dc669808db3", + "e13843bffacca3e0", + "6025845b2aa21eb7", + "f288433c7ec32303", + "51ccb99538406996", + "1700c6017ed82aaf", + "f7039343f41ed237", + "7c56b28279b128c0", + "afac5614625bfc6f", + "d79b4325363f7010", + "f349b9e30f493cb5", + "b8c2c3d64f1947fa", + "087a771998e8460b", + "05346dbcf28946f0", + "0a87b2d06772daf3", + "2ee69d30ad1b9ba1", + "6290a947cc8369e7", + "846e9f943798d3ab", + "86e96862d2e6618c", + "335745c8550b1062", + "ebc3a07dd3e76b6e", + "fb25cf0e86ffcd7d", + "fa1b267c4911f7a0", + "33481799a00eb89a", + "e80f19b7afe3d734", + "7c727a84546a6bba", + "3ee248ced1fa920a", + "65b15b71fa6f508f", + "a8f7be21e03253d3", + "2b89e57a9b28bfbf", + "d9b5eaa66bdc7bec", + "b2382b7ca202c344", + "9a8481abc3453d0b", + "a40e20d96d920f67", + "8e2c3affdcca9c97", + "bfba0e0e48f5eb54", + "97544c226c577151", + "68eb99ccb687369d", + "24c03d50574b79f2", + "302f5cbbc6e1596b", + "a04da6570878fb03", + "09e85a53a7b4850d", + "e331de721c086202", + "0dabb37b2a7b3687", + "91315c96894dcfd3", + "fba6e296e0acdccc", + "e693fe9346caa44c", + "28276d6b56102882" + ] + }, + { + "plan": "pairdrop-new", + "product": "PairDrop", + "status": "insufficient_holdout", + "existing_catalog_matches": null, + "positive_samples": 1, + "suggested_positives": 1, + "unknown_positives": 0, + "holdout_positive_hosts": 0, + "holdout_negative_hosts": 51, + "before": { + "detection": { + "tp": 0, + "fp": 0, + "tn": 0, + "fn": 0 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "unlabelled_predictions": 0 + }, + "after": { + "detection": { + "tp": 0, + "fp": 0, + "tn": 0, + "fn": 0 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "unlabelled_predictions": 0 + }, + "test_ids": [ + "36f6645c0290996f", + "145e314623e76dfd", + "7a415cdbb6244358", + "e3651c67dbb2ad17", + "718e4c943a25a995", + "c31d254bc1fd0230", + "e7d23d9f3c44fee1", + "867e79ba27f05078", + "bc537dc669808db3", + "e13843bffacca3e0", + "6025845b2aa21eb7", + "f288433c7ec32303", + "51ccb99538406996", + "1700c6017ed82aaf", + "f7039343f41ed237", + "7c56b28279b128c0", + "afac5614625bfc6f", + "d79b4325363f7010", + "f349b9e30f493cb5", + "b8c2c3d64f1947fa", + "087a771998e8460b", + "05346dbcf28946f0", + "0a87b2d06772daf3", + "2ee69d30ad1b9ba1", + "6290a947cc8369e7", + "846e9f943798d3ab", + "86e96862d2e6618c", + "335745c8550b1062", + "ebc3a07dd3e76b6e", + "fb25cf0e86ffcd7d", + "fa1b267c4911f7a0", + "33481799a00eb89a", + "e80f19b7afe3d734", + "e173c09a7713b5b0", + "7c727a84546a6bba", + "3ee248ced1fa920a", + "a8f7be21e03253d3", + "2b89e57a9b28bfbf", + "d9b5eaa66bdc7bec", + "b2382b7ca202c344", + "9a8481abc3453d0b", + "a40e20d96d920f67", + "8e2c3affdcca9c97", + "bfba0e0e48f5eb54", + "97544c226c577151", + "68eb99ccb687369d", + "24c03d50574b79f2", + "302f5cbbc6e1596b", + "a04da6570878fb03", + "09e85a53a7b4850d", + "e331de721c086202", + "0dabb37b2a7b3687", + "91315c96894dcfd3", + "fba6e296e0acdccc", + "e693fe9346caa44c", + "28276d6b56102882" + ] + }, + { + "plan": "docmost-new", + "product": "Docmost", + "status": "insufficient_holdout", + "existing_catalog_matches": null, + "positive_samples": 1, + "suggested_positives": 1, + "unknown_positives": 1, + "holdout_positive_hosts": 0, + "holdout_negative_hosts": 51, + "before": { + "detection": { + "tp": 0, + "fp": 0, + "tn": 0, + "fn": 0 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "unlabelled_predictions": 0 + }, + "after": { + "detection": { + "tp": 0, + "fp": 0, + "tn": 0, + "fn": 0 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "unlabelled_predictions": 0 + }, + "test_ids": [ + "36f6645c0290996f", + "145e314623e76dfd", + "7a415cdbb6244358", + "e3651c67dbb2ad17", + "718e4c943a25a995", + "c31d254bc1fd0230", + "e7d23d9f3c44fee1", + "867e79ba27f05078", + "bc537dc669808db3", + "e13843bffacca3e0", + "6025845b2aa21eb7", + "f288433c7ec32303", + "51ccb99538406996", + "1700c6017ed82aaf", + "f7039343f41ed237", + "7c56b28279b128c0", + "afac5614625bfc6f", + "d79b4325363f7010", + "f349b9e30f493cb5", + "b8c2c3d64f1947fa", + "087a771998e8460b", + "05346dbcf28946f0", + "0a87b2d06772daf3", + "2ee69d30ad1b9ba1", + "6290a947cc8369e7", + "846e9f943798d3ab", + "86e96862d2e6618c", + "335745c8550b1062", + "ebc3a07dd3e76b6e", + "fa1b267c4911f7a0", + "33481799a00eb89a", + "e80f19b7afe3d734", + "e173c09a7713b5b0", + "7c727a84546a6bba", + "3ee248ced1fa920a", + "65b15b71fa6f508f", + "a8f7be21e03253d3", + "2b89e57a9b28bfbf", + "d9b5eaa66bdc7bec", + "b2382b7ca202c344", + "9a8481abc3453d0b", + "a40e20d96d920f67", + "8e2c3affdcca9c97", + "bfba0e0e48f5eb54", + "97544c226c577151", + "68eb99ccb687369d", + "24c03d50574b79f2", + "302f5cbbc6e1596b", + "a04da6570878fb03", + "09e85a53a7b4850d", + "e331de721c086202", + "0dabb37b2a7b3687", + "91315c96894dcfd3", + "fba6e296e0acdccc", + "e693fe9346caa44c", + "28276d6b56102882" + ] + }, + { + "plan": "linkwarden-new", + "product": "Linkwarden", + "status": "insufficient_holdout", + "existing_catalog_matches": null, + "positive_samples": 1, + "suggested_positives": 1, + "unknown_positives": 0, + "holdout_positive_hosts": 0, + "holdout_negative_hosts": 51, + "before": { + "detection": { + "tp": 0, + "fp": 0, + "tn": 0, + "fn": 0 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "unlabelled_predictions": 0 + }, + "after": { + "detection": { + "tp": 0, + "fp": 0, + "tn": 0, + "fn": 0 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "unlabelled_predictions": 0 + }, + "test_ids": [ + "36f6645c0290996f", + "145e314623e76dfd", + "7a415cdbb6244358", + "e3651c67dbb2ad17", + "718e4c943a25a995", + "c31d254bc1fd0230", + "e7d23d9f3c44fee1", + "867e79ba27f05078", + "bc537dc669808db3", + "e13843bffacca3e0", + "6025845b2aa21eb7", + "f288433c7ec32303", + "51ccb99538406996", + "1700c6017ed82aaf", + "f7039343f41ed237", + "7c56b28279b128c0", + "afac5614625bfc6f", + "d79b4325363f7010", + "f349b9e30f493cb5", + "b8c2c3d64f1947fa", + "087a771998e8460b", + "05346dbcf28946f0", + "0a87b2d06772daf3", + "2ee69d30ad1b9ba1", + "6290a947cc8369e7", + "846e9f943798d3ab", + "86e96862d2e6618c", + "335745c8550b1062", + "ebc3a07dd3e76b6e", + "fb25cf0e86ffcd7d", + "fa1b267c4911f7a0", + "33481799a00eb89a", + "e173c09a7713b5b0", + "7c727a84546a6bba", + "3ee248ced1fa920a", + "65b15b71fa6f508f", + "a8f7be21e03253d3", + "2b89e57a9b28bfbf", + "d9b5eaa66bdc7bec", + "b2382b7ca202c344", + "9a8481abc3453d0b", + "a40e20d96d920f67", + "8e2c3affdcca9c97", + "bfba0e0e48f5eb54", + "97544c226c577151", + "68eb99ccb687369d", + "24c03d50574b79f2", + "302f5cbbc6e1596b", + "a04da6570878fb03", + "09e85a53a7b4850d", + "e331de721c086202", + "0dabb37b2a7b3687", + "91315c96894dcfd3", + "fba6e296e0acdccc", + "e693fe9346caa44c", + "28276d6b56102882" + ] + }, + { + "plan": "karakeep-new", + "product": "Karakeep", + "status": "insufficient_holdout", + "existing_catalog_matches": null, + "positive_samples": 3, + "suggested_positives": 3, + "unknown_positives": 0, + "holdout_positive_hosts": 0, + "holdout_negative_hosts": 51, + "before": { + "detection": { + "tp": 0, + "fp": 0, + "tn": 0, + "fn": 0 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "unlabelled_predictions": 0 + }, + "after": { + "detection": { + "tp": 0, + "fp": 0, + "tn": 0, + "fn": 0 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "unlabelled_predictions": 0 + }, + "test_ids": [ + "36f6645c0290996f", + "145e314623e76dfd", + "7a415cdbb6244358", + "e3651c67dbb2ad17", + "718e4c943a25a995", + "c31d254bc1fd0230", + "e7d23d9f3c44fee1", + "867e79ba27f05078", + "bc537dc669808db3", + "e13843bffacca3e0", + "6025845b2aa21eb7", + "f288433c7ec32303", + "51ccb99538406996", + "1700c6017ed82aaf", + "f7039343f41ed237", + "7c56b28279b128c0", + "afac5614625bfc6f", + "d79b4325363f7010", + "f349b9e30f493cb5", + "b8c2c3d64f1947fa", + "087a771998e8460b", + "05346dbcf28946f0", + "0a87b2d06772daf3", + "2ee69d30ad1b9ba1", + "6290a947cc8369e7", + "846e9f943798d3ab", + "86e96862d2e6618c", + "335745c8550b1062", + "ebc3a07dd3e76b6e", + "fb25cf0e86ffcd7d", + "fa1b267c4911f7a0", + "33481799a00eb89a", + "e80f19b7afe3d734", + "e173c09a7713b5b0", + "7c727a84546a6bba", + "3ee248ced1fa920a", + "65b15b71fa6f508f", + "a8f7be21e03253d3", + "2b89e57a9b28bfbf", + "d9b5eaa66bdc7bec", + "b2382b7ca202c344", + "9a8481abc3453d0b", + "a40e20d96d920f67", + "8e2c3affdcca9c97", + "bfba0e0e48f5eb54", + "97544c226c577151", + "68eb99ccb687369d", + "302f5cbbc6e1596b", + "a04da6570878fb03", + "e331de721c086202", + "0dabb37b2a7b3687", + "91315c96894dcfd3", + "e693fe9346caa44c", + "28276d6b56102882" + ] + }, + { + "plan": "new-api-new", + "product": "New API", + "status": "insufficient_holdout", + "existing_catalog_matches": null, + "positive_samples": 1, + "suggested_positives": 1, + "unknown_positives": 1, + "holdout_positive_hosts": 0, + "holdout_negative_hosts": 51, + "before": { + "detection": { + "tp": 0, + "fp": 0, + "tn": 0, + "fn": 0 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "unlabelled_predictions": 0 + }, + "after": { + "detection": { + "tp": 0, + "fp": 0, + "tn": 0, + "fn": 0 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "unlabelled_predictions": 0 + }, + "test_ids": [ + "36f6645c0290996f", + "145e314623e76dfd", + "7a415cdbb6244358", + "e3651c67dbb2ad17", + "718e4c943a25a995", + "c31d254bc1fd0230", + "e7d23d9f3c44fee1", + "867e79ba27f05078", + "bc537dc669808db3", + "e13843bffacca3e0", + "6025845b2aa21eb7", + "f288433c7ec32303", + "51ccb99538406996", + "1700c6017ed82aaf", + "f7039343f41ed237", + "7c56b28279b128c0", + "afac5614625bfc6f", + "d79b4325363f7010", + "f349b9e30f493cb5", + "b8c2c3d64f1947fa", + "087a771998e8460b", + "0a87b2d06772daf3", + "2ee69d30ad1b9ba1", + "6290a947cc8369e7", + "846e9f943798d3ab", + "86e96862d2e6618c", + "335745c8550b1062", + "ebc3a07dd3e76b6e", + "fb25cf0e86ffcd7d", + "fa1b267c4911f7a0", + "33481799a00eb89a", + "e80f19b7afe3d734", + "e173c09a7713b5b0", + "7c727a84546a6bba", + "3ee248ced1fa920a", + "65b15b71fa6f508f", + "a8f7be21e03253d3", + "2b89e57a9b28bfbf", + "d9b5eaa66bdc7bec", + "b2382b7ca202c344", + "9a8481abc3453d0b", + "a40e20d96d920f67", + "8e2c3affdcca9c97", + "bfba0e0e48f5eb54", + "97544c226c577151", + "68eb99ccb687369d", + "24c03d50574b79f2", + "302f5cbbc6e1596b", + "a04da6570878fb03", + "09e85a53a7b4850d", + "e331de721c086202", + "0dabb37b2a7b3687", + "91315c96894dcfd3", + "fba6e296e0acdccc", + "e693fe9346caa44c", + "28276d6b56102882" + ] + } + ], + "lost_existing_hits": null, + "changed_existing_versions": null, + "integration_errors": null + }, + "generation": [ + { + "name": "searxng-new", + "product": "searxng", + "status": "passed_holdout", + "file": "fingerprints/searxng-new.yaml", + "detection": { + "tp": 8, + "fp": 0, + "tn": 47, + "fn": 0 + }, + "versions": { + "correct": 8, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "training_detection": { + "tp": 2, + "fp": 0, + "tn": 7, + "fn": 0 + }, + "training_versions": { + "correct": 2, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "excluded": [ + "9b0d62c13f476614", + "aaef32bb77101a86", + "45c65575b6da19ac", + "1a125b61fc8b7dd7", + "acede3e046de943e" + ] + }, + { + "name": "wakapi-new", + "product": "Wakapi", + "status": "passed_holdout", + "file": "fingerprints/wakapi-new.yaml", + "detection": { + "tp": 3, + "fp": 0, + "tn": 53, + "fn": 0 + }, + "versions": { + "correct": 3, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "training_detection": { + "tp": 1, + "fp": 0, + "tn": 7, + "fn": 0 + }, + "training_versions": { + "correct": 1, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "excluded": [ + "9b0d62c13f476614", + "aaef32bb77101a86", + "45c65575b6da19ac", + "1a125b61fc8b7dd7", + "acede3e046de943e" + ] + }, + { + "name": "it-tools-new", + "product": "IT Tools", + "status": "passed_holdout", + "file": "fingerprints/it-tools-new.yaml", + "detection": { + "tp": 2, + "fp": 0, + "tn": 54, + "fn": 0 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 2, + "unsupported": 0, + "product_missing": 0 + }, + "training_detection": { + "tp": 1, + "fp": 0, + "tn": 7, + "fn": 0 + }, + "training_versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 1, + "unsupported": 0, + "product_missing": 0 + }, + "excluded": [ + "9b0d62c13f476614", + "aaef32bb77101a86", + "45c65575b6da19ac", + "1a125b61fc8b7dd7", + "acede3e046de943e" + ] + }, + { + "name": "pairdrop-new", + "product": "PairDrop", + "status": "insufficient_holdout", + "file": "fingerprints/pairdrop-new.yaml", + "detection": { + "tp": 0, + "fp": 0, + "tn": 56, + "fn": 0 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "training_detection": { + "tp": 1, + "fp": 0, + "tn": 7, + "fn": 0 + }, + "training_versions": { + "correct": 1, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "excluded": [ + "9b0d62c13f476614", + "aaef32bb77101a86", + "45c65575b6da19ac", + "1a125b61fc8b7dd7", + "acede3e046de943e" + ] + }, + { + "name": "docmost-new", + "product": "Docmost", + "status": "insufficient_holdout", + "file": "fingerprints/docmost-new.yaml", + "detection": { + "tp": 0, + "fp": 0, + "tn": 56, + "fn": 0 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "training_detection": { + "tp": 1, + "fp": 0, + "tn": 7, + "fn": 0 + }, + "training_versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 1, + "unsupported": 0, + "product_missing": 0 + }, + "excluded": [ + "9b0d62c13f476614", + "aaef32bb77101a86", + "45c65575b6da19ac", + "1a125b61fc8b7dd7", + "acede3e046de943e" + ] + }, + { + "name": "linkwarden-new", + "product": "Linkwarden", + "status": "insufficient_holdout", + "file": "fingerprints/linkwarden-new.yaml", + "detection": { + "tp": 0, + "fp": 0, + "tn": 56, + "fn": 0 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "training_detection": { + "tp": 1, + "fp": 0, + "tn": 7, + "fn": 0 + }, + "training_versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 1, + "unsupported": 0, + "product_missing": 0 + }, + "excluded": [ + "9b0d62c13f476614", + "aaef32bb77101a86", + "45c65575b6da19ac", + "1a125b61fc8b7dd7", + "acede3e046de943e" + ] + }, + { + "name": "karakeep-new", + "product": "Karakeep", + "status": "insufficient_holdout", + "file": "fingerprints/karakeep-new.yaml", + "detection": { + "tp": 0, + "fp": 0, + "tn": 54, + "fn": 0 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "training_detection": { + "tp": 1, + "fp": 0, + "tn": 7, + "fn": 0 + }, + "training_versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 1, + "unsupported": 0, + "product_missing": 0 + }, + "excluded": [ + "9b0d62c13f476614", + "aaef32bb77101a86", + "45c65575b6da19ac", + "1a125b61fc8b7dd7", + "acede3e046de943e", + "24c03d50574b79f2", + "09e85a53a7b4850d" + ] + }, + { + "name": "new-api-new", + "product": "New API", + "status": "insufficient_holdout", + "file": "fingerprints/new-api-new.yaml", + "detection": { + "tp": 0, + "fp": 0, + "tn": 56, + "fn": 0 + }, + "versions": { + "correct": 0, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "training_detection": { + "tp": 1, + "fp": 0, + "tn": 7, + "fn": 0 + }, + "training_versions": { + "correct": 1, + "wrong": 0, + "missing": 0, + "correct_abstentions": 0, + "unsupported": 0, + "product_missing": 0 + }, + "excluded": [ + "9b0d62c13f476614", + "aaef32bb77101a86", + "45c65575b6da19ac", + "1a125b61fc8b7dd7", + "acede3e046de943e" + ] + } + ], + "repeated_maintenance": [ + { + "product": "SearXNG", + "status": "already_catalogued", + "existing_catalog_matches": [ + "runtime-fingers:searxng" + ] + }, + { + "product": "Wakapi", + "status": "already_catalogued", + "existing_catalog_matches": [ + "runtime-fingers:Wakapi" + ] + }, + { + "product": "IT Tools", + "status": "already_catalogued", + "existing_catalog_matches": [ + "runtime-fingers:IT Tools" + ] + }, + { + "product": "PairDrop", + "status": "insufficient_holdout", + "existing_catalog_matches": null + }, + { + "product": "Docmost", + "status": "insufficient_holdout", + "existing_catalog_matches": null + }, + { + "product": "Linkwarden", + "status": "insufficient_holdout", + "existing_catalog_matches": null + }, + { + "product": "Karakeep", + "status": "insufficient_holdout", + "existing_catalog_matches": null + }, + { + "product": "New API", + "status": "insufficient_holdout", + "existing_catalog_matches": null + } + ], + "loaded_library_cleanup": { + "model": "jev/jev-1.13.0", + "baseline_source": "current_rules_on_saved_responses", + "samples": 69, + "unique_hosts": 59, + "unique_responses": 65, + "errors": 0, + "labelled_pairs": 682, + "baseline": { + "detection": { + "tp": 63, + "fp": 24, + "tn": 586, + "fn": 9 + }, + "versions": { + "correct": 17, + "wrong": 0, + "missing": 20, + "correct_abstentions": 26, + "unsupported": 0, + "product_missing": 9 + }, + "unlabelled_predictions": 301 + }, + "refined": { + "detection": { + "tp": 63, + "fp": 0, + "tn": 610, + "fn": 9 + }, + "versions": { + "correct": 23, + "wrong": 0, + "missing": 14, + "correct_abstentions": 26, + "unsupported": 0, + "product_missing": 9 + }, + "unlabelled_predictions": 148 + }, + "completed": { + "detection": { + "tp": 63, + "fp": 0, + "tn": 610, + "fn": 9 + }, + "versions": { + "correct": 37, + "wrong": 0, + "missing": 0, + "correct_abstentions": 26, + "unsupported": 0, + "product_missing": 9 + }, + "unlabelled_predictions": 148 + }, + "false_hits_removed": 24, + "true_hits_removed": 0, + "natural_misses_recovered": 0, + "missed_products_suggested": 8, + "version_fills": 27, + "correct_version_fills": 20, + "incorrect_version_fills": 0, + "unlabelled_version_fills": 7, + "provider_requests": 306, + "exact_cache_hits": 181, + "input_tokens": 741618 + }, + "offline": { + "provider_requests": 0, + "cache_hits": 483, + "same_semantic_metrics": true, + "identical_files": { + "library.yaml": true, + "generation.json": true, + "maintenance.json": true, + "rows.jsonl": true, + "baseline.jsonl": true, + "cleaned.jsonl": true + } + }, + "label_revisions": [ + { + "product": "PairDrop", + "id": "65b15b71fa6f508f", + "previous_version": "", + "corrected_version": "1.11.2", + "basis": "Adjacent PairDrop product heading and visible release declaration in the original saved HTML; not inferred from a provider answer.", + "evidence": "
v1.11.2
" + } + ], + "samples": [ + { + "id": "36f6645c0290996f", + "url": "http://47.239.11.1/", + "group": "47.239.11.1", + "captured_at": "2026-09-25T14:18:17.927604+00:00", + "sha256": "01c9933b1ea5a0ffbb8ccfc911a548f4c6c53500d2f2fffe08aa54ad00524859", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "145e314623e76dfd", + "url": "http://47.239.11.117/", + "group": "47.239.11.117", + "captured_at": "2026-09-25T14:18:17.927604+00:00", + "sha256": "6f4b55cad227274d1acbf3575a76308a395588c51e84c41790d6e5eb9a580634", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "7a415cdbb6244358", + "url": "http://47.239.11.119/", + "group": "47.239.11.119", + "captured_at": "2026-09-25T14:18:17.928110+00:00", + "sha256": "3094c7260747f35b8f46b973da195908f64734bf5fa138bc1fe3335c7058bc7d", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "e3651c67dbb2ad17", + "url": "http://47.239.11.120/", + "group": "47.239.11.120", + "captured_at": "2026-09-25T14:18:17.928110+00:00", + "sha256": "58cf9c0f520cd99dfd8377e28181b7b26b4e35a0cdf56c93fac2613dd683b1f4", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "718e4c943a25a995", + "url": "http://47.239.11.121/", + "group": "47.239.11.121", + "captured_at": "2026-09-25T14:18:17.926596+00:00", + "sha256": "9cf03440fe1d5d60261b8ef9434d05b2dd7702ec789ff6cb289b41fa4753aa55", + "product_labels": [ + { + "product": "New API", + "present": false + }, + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + } + ] + }, + { + "id": "c31d254bc1fd0230", + "url": "http://47.239.11.134/", + "group": "47.239.11.134", + "captured_at": "2026-09-25T14:18:17.928110+00:00", + "sha256": "dffc9b4b13160201cb54898cbe83629a654b1aa91f7139f02ce0f0c44672815e", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "e7d23d9f3c44fee1", + "url": "http://47.239.11.151/", + "group": "47.239.11.151", + "captured_at": "2026-09-25T14:18:17.970662+00:00", + "sha256": "2e39b279754a9dcc613fd51f66647804701ca795715d6b3e4c7a5f6898ce9a91", + "product_labels": [ + { + "product": "New API", + "present": false + }, + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + } + ] + }, + { + "id": "867e79ba27f05078", + "url": "http://47.239.11.151/robots.txt", + "group": "47.239.11.151", + "captured_at": "2026-09-25T14:18:17.922561+00:00", + "sha256": "d920d770512d4605376b8269e3bddb42b2a301f6791829edc1f03bb0ed4009ca", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "bc537dc669808db3", + "url": "http://47.239.11.155/", + "group": "47.239.11.155", + "captured_at": "2026-09-25T14:18:18.168005+00:00", + "sha256": "8ee30984aa2b1cf9ed38f45ce449ed75cc4f09b0d51622285f3cf8f8039f6102", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "e13843bffacca3e0", + "url": "http://47.239.11.156/", + "group": "47.239.11.156", + "captured_at": "2026-09-25T14:18:18.038424+00:00", + "sha256": "073eb56c353211f2e01597d85a12496d14f1f710590442978303d7ea4a43ff55", + "product_labels": [ + { + "product": "New API", + "present": false + }, + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + } + ] + }, + { + "id": "6025845b2aa21eb7", + "url": "http://47.239.11.169/", + "group": "47.239.11.169", + "captured_at": "2026-09-25T14:18:18.052902+00:00", + "sha256": "b7905ab6f47917366f3412c5588cc7388d722a9377d7d69929affc8181438070", + "product_labels": [ + { + "product": "New API", + "present": false + }, + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + } + ] + }, + { + "id": "f288433c7ec32303", + "url": "http://47.239.11.22/", + "group": "47.239.11.22", + "captured_at": "2026-09-25T14:18:18.044381+00:00", + "sha256": "64bc64f6243ee12859cdd7428e3a9fb6543007c38889410d519308154dfb0f39", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "51ccb99538406996", + "url": "http://47.239.11.226/", + "group": "47.239.11.226", + "captured_at": "2026-09-25T14:18:18.044381+00:00", + "sha256": "e25d8751b8b9e8a4d3358a67b9f5dec50b4bb8a78203531023c084f5805f10c3", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "1700c6017ed82aaf", + "url": "http://47.239.11.25/", + "group": "47.239.11.25", + "captured_at": "2026-09-25T14:18:18.096093+00:00", + "sha256": "73a98b2721ffcbee6c24860eb35d77699b8078a44feda6bba403f431884b023f", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "9b0d62c13f476614", + "url": "http://47.239.11.25/robots.txt", + "group": "47.239.11.25", + "captured_at": "2026-09-25T14:18:18.040055+00:00", + "sha256": "d920d770512d4605376b8269e3bddb42b2a301f6791829edc1f03bb0ed4009ca", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "f7039343f41ed237", + "url": "http://47.239.11.26/", + "group": "47.239.11.26", + "captured_at": "2026-09-25T14:18:18.068707+00:00", + "sha256": "c618035e4ce2e29cd00081e9ee5c1d44a21e29e2e45bb0776b4e8a28157905fd", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "7c56b28279b128c0", + "url": "http://47.239.11.31/", + "group": "47.239.11.31", + "captured_at": "2026-09-25T14:18:18.503310+00:00", + "sha256": "8bb71aca1f485344ddcab8b24fb60d0d3024674b374c042755f7afc8afdfefcc", + "product_labels": [ + { + "product": "New API", + "present": false + }, + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + } + ] + }, + { + "id": "afac5614625bfc6f", + "url": "http://47.239.11.37/", + "group": "47.239.11.37", + "captured_at": "2026-09-25T14:18:18.227548+00:00", + "sha256": "964afc12b6a7cf669d5e5cc1c273817243e8556cdda804288f568fa527b23949", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "d79b4325363f7010", + "url": "http://47.239.11.81/", + "group": "47.239.11.81", + "captured_at": "2026-09-25T14:18:18.135189+00:00", + "sha256": "80a65f5096d6484405f30628730b819e71b006bbc076629bd219b090a8bd0210", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "aaef32bb77101a86", + "url": "http://47.239.11.88/", + "group": "47.239.11.88", + "captured_at": "2026-09-25T14:18:18.145815+00:00", + "sha256": "b6c6c656c2ab14ae11e9a193fbceec59e00f55c2dd144c8855e2a484a1c7822e", + "product_labels": [ + { + "product": "New API", + "present": false + }, + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + } + ] + }, + { + "id": "f0bd3986b1f949f9", + "url": "http://47.239.11.9/", + "group": "47.239.11.9", + "captured_at": "2026-09-25T14:18:18.360498+00:00", + "sha256": "ef6be733b856d9360d413b70c68c1e9215d721abd47719c44a42fc0c8c098ca5", + "product_labels": [ + { + "product": "New API", + "present": false + }, + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + } + ] + }, + { + "id": "45c65575b6da19ac", + "url": "http://47.239.11.9/robots.txt", + "group": "47.239.11.9", + "captured_at": "2026-09-25T14:18:18.362502+00:00", + "sha256": "1b09dc6a8fda8fc5d9542cfd51c0d117bf8920247024f967258cddc94e28ac31", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "f349b9e30f493cb5", + "url": "http://47.239.11.90/", + "group": "47.239.11.90", + "captured_at": "2026-09-25T14:18:18.204350+00:00", + "sha256": "20fbdce2fbcf59605ceaac8c5b2abd19ffe9f221d06e16492f10a406abcd3ca8", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "1a125b61fc8b7dd7", + "url": "http://47.239.11.98/", + "group": "47.239.11.98", + "captured_at": "2026-09-25T14:18:18.238181+00:00", + "sha256": "01c9933b1ea5a0ffbb8ccfc911a548f4c6c53500d2f2fffe08aa54ad00524859", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "b8c2c3d64f1947fa", + "url": "https://47.239.11.1/", + "group": "47.239.11.1", + "captured_at": "2026-09-25T14:18:18.293071+00:00", + "sha256": "0bc50587714d9b9dd614f10d14e5c189753e82c3556c52510386cddb790db595", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "087a771998e8460b", + "url": "https://47.239.11.107/", + "group": "47.239.11.107", + "captured_at": "2026-09-25T14:18:18.408116+00:00", + "sha256": "fb18c6b1cbd965735c819dd7cf481ae3a43ee56fc594653c9e9275658be337c7", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "05346dbcf28946f0", + "url": "https://47.239.11.123/", + "group": "47.239.11.123", + "captured_at": "2026-09-25T14:18:18.740314+00:00", + "sha256": "321ddf27f33ffe2d6c5a2cf46912a37532afbaa09e6ec95e836ab17cbf4d160e", + "product_labels": [ + { + "product": "New API", + "present": true, + "version": "1.0.0-rc.35" + }, + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + } + ] + }, + { + "id": "0a87b2d06772daf3", + "url": "https://47.239.11.167/", + "group": "47.239.11.167", + "captured_at": "2026-09-25T14:18:18.389966+00:00", + "sha256": "b8dc990fcf6c2da2a577a434b663d77e38e3e4121e2f81409f2e2d2678237c1c", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "2ee69d30ad1b9ba1", + "url": "https://47.239.11.189/", + "group": "47.239.11.189", + "captured_at": "2026-09-25T14:18:18.391967+00:00", + "sha256": "fa3774df099c5131e279010cb96c2405ad9504fad1f24b65e436e312e2f30def", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "6290a947cc8369e7", + "url": "https://47.239.11.196/", + "group": "47.239.11.196", + "captured_at": "2026-09-25T14:18:18.617358+00:00", + "sha256": "bcdc6b21d56ce3e1cdbb410eaa16817cafd2c308946da41a7faf001e45894689", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "877f11f543fab972", + "url": "https://47.239.11.201/", + "group": "47.239.11.201", + "captured_at": "2026-09-25T14:18:19.980167+00:00", + "sha256": "19094cdce76f6db2b41180c2b886d83fb8f4565845e01b3745016e2009d1630b", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "846e9f943798d3ab", + "url": "https://47.239.11.205/", + "group": "47.239.11.205", + "captured_at": "2026-09-25T14:18:18.571434+00:00", + "sha256": "53daeaa2e944eebe9763104e20f2f87744fb88105ab21d542fa1961611a53f53", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "86e96862d2e6618c", + "url": "https://47.239.11.31/", + "group": "47.239.11.31", + "captured_at": "2026-09-25T14:18:18.585630+00:00", + "sha256": "6610cfe1cb5781ca48d4ee66d9f770161894e181288d0eab100bc3b6ef3a5c27", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "acede3e046de943e", + "url": "https://47.239.11.42/", + "group": "47.239.11.42", + "captured_at": "2026-09-25T14:18:18.667816+00:00", + "sha256": "bcdc6b21d56ce3e1cdbb410eaa16817cafd2c308946da41a7faf001e45894689", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "335745c8550b1062", + "url": "https://47.239.11.56/", + "group": "47.239.11.56", + "captured_at": "2026-09-25T14:18:18.620892+00:00", + "sha256": "30cabb65837840a343e98d1f32fe73aeab525ca76ea5575002e0bd6be348f15e", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "ebc3a07dd3e76b6e", + "url": "https://auth.seddens.net/?rd=https://search.seddens.net/", + "group": "auth.seddens.net", + "captured_at": "2026-09-25T16:02:44.182709+00:00", + "sha256": "0a1e86f1d97559a345f2330854612ba467fca23fec7bc9fbaf4e8fb3360e383d", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "fb25cf0e86ffcd7d", + "url": "https://demo.docmost.com/", + "group": "demo.docmost.com", + "captured_at": "2026-09-25T16:02:44.705457+00:00", + "sha256": "af7efc632705f747e3678d8d5433d985d38e4265b988d1254d243cc9075fd8bc", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": true, + "version": "" + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "fa1b267c4911f7a0", + "url": "https://demo.gatus.io/", + "group": "demo.gatus.io", + "captured_at": "2026-09-25T16:02:44.046979+00:00", + "sha256": "80fca347a00ecf6b8447090e38b86772bd3e2496c1c1ebcb232e63d7860bca43", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "33481799a00eb89a", + "url": "https://demo.immich.app/", + "group": "demo.immich.app", + "captured_at": "2026-09-25T16:02:46.028922+00:00", + "sha256": "b0285892e15d87f89f5cbf0069b787bd76cfdf263c2f55fd8ba95317293062fa", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "e80f19b7afe3d734", + "url": "https://demo.linkwarden.app/", + "group": "demo.linkwarden.app", + "captured_at": "2026-09-25T16:02:44.753890+00:00", + "sha256": "f8003078dff0c362781ba26cac30b46b773bdd0cbbad92c64043ae7a380e4f04", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": true, + "version": "" + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "e173c09a7713b5b0", + "url": "https://it-tools.tech/", + "group": "it-tools.tech", + "captured_at": "2026-09-25T16:02:46.944395+00:00", + "sha256": "aa5123f269ffc06cb6c267e61d0928c812757cd79dd45644488217947d0dcf7e", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": true, + "version": "" + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "7c727a84546a6bba", + "url": "https://libresearch.space/", + "group": "libresearch.space", + "captured_at": "2026-09-25T16:02:47.592975+00:00", + "sha256": "933c9c1838412c7c2e28a93995c700dca04d59072197db72b8402b580fda848e", + "product_labels": [ + { + "product": "SearXNG", + "present": true, + "version": "2026.9.23+3cd69d30e" + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "3ee248ced1fa920a", + "url": "https://linkding.link/demo/", + "group": "linkding.link", + "captured_at": "2026-09-25T16:02:47.193117+00:00", + "sha256": "cbef1330a4de2184b8500644dfc2da1c219482b5f73bdcd96868569b48559d28", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "65b15b71fa6f508f", + "url": "https://pairdrop.net/", + "group": "pairdrop.net", + "captured_at": "2026-09-25T16:02:48.304311+00:00", + "sha256": "544859141ca31286fd7ad3396b7e4111cd40df06e37af8d6ef8c700400da753b", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": true, + "version": "1.11.2" + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "a8f7be21e03253d3", + "url": "https://privacyredirect.com/", + "group": "privacyredirect.com", + "captured_at": "2026-09-25T16:02:48.791402+00:00", + "sha256": "2862ecff85a029752274e65e36f1cd975aeb5aa882263c3c51324357e9469bb3", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "2b89e57a9b28bfbf", + "url": "https://search.liuzj.net/", + "group": "search.liuzj.net", + "captured_at": "2026-09-25T16:02:50.042863+00:00", + "sha256": "a0729317e8d6efc1761808e2bcfa6aa3d0f1dc012b59b2083e12ba273b92ef52", + "product_labels": [ + { + "product": "SearXNG", + "present": true, + "version": "2026.9.23+3cd69d30e" + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "d9b5eaa66bdc7bec", + "url": "https://search.minus27315.dev/", + "group": "search.minus27315.dev", + "captured_at": "2026-09-25T16:02:49.494654+00:00", + "sha256": "1ec95f79589ea486f8b229f5465e3eb7cfe526a7c6bb027d0cb5e46d7a8a74c6", + "product_labels": [ + { + "product": "SearXNG", + "present": true, + "version": "2026.9.23+3cd69d30e" + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "b2382b7ca202c344", + "url": "https://search.mizuki.st/", + "group": "search.mizuki.st", + "captured_at": "2026-09-25T16:02:50.068114+00:00", + "sha256": "f052900e00a4761ffeaf166c20476d3ace37e9273399fc9da4cdefffd62400d7", + "product_labels": [ + { + "product": "SearXNG", + "present": true, + "version": "2026.9.25+487f51922" + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "9a8481abc3453d0b", + "url": "https://search.serpensin.com/", + "group": "search.serpensin.com", + "captured_at": "2026-09-25T16:02:50.801354+00:00", + "sha256": "d44354e6a04543536be6657dc9cf6455545623186be9254cb337a861b04bdb3a", + "product_labels": [ + { + "product": "SearXNG", + "present": true, + "version": "2026.9.25+d8ae3abd5" + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "a40e20d96d920f67", + "url": "https://search.undertale.uk/", + "group": "search.undertale.uk", + "captured_at": "2026-09-25T16:02:51.784166+00:00", + "sha256": "12631af1a6b39d5f1ae3cdf9b4aa4375aded4402bbd51d6de97a968406ae7503", + "product_labels": [ + { + "product": "SearXNG", + "present": true, + "version": "2026.9.12+d4f00d15" + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "8e2c3affdcca9c97", + "url": "https://searx.linxx.net/", + "group": "searx.linxx.net", + "captured_at": "2026-09-25T16:02:52.064348+00:00", + "sha256": "6a1d74981c9c690fc68166125f25f7e6f1dd591701e3e3728323d62324ddc082", + "product_labels": [ + { + "product": "SearXNG", + "present": true, + "version": "2026.9.23+3cd69d30e" + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "bfba0e0e48f5eb54", + "url": "https://searx.ononoki.org/", + "group": "searx.ononoki.org", + "captured_at": "2026-09-25T16:02:52.089914+00:00", + "sha256": "1cfe7a92e63295863796002845a691a81d27ca89ed5d69812d7f8add32366b9f", + "product_labels": [ + { + "product": "SearXNG", + "present": true, + "version": "2026.9.25+d8ae3abd5" + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "97544c226c577151", + "url": "https://searxng.paralaxitaentomology.org/", + "group": "searxng.paralaxitaentomology.org", + "captured_at": "2026-09-25T16:02:54.679780+00:00", + "sha256": "6ab60af098c988d1edae89ecc9e20f209dd93169d9e0e0a656bc71a4f6234d88", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "68eb99ccb687369d", + "url": "https://searxng.website/", + "group": "searxng.website", + "captured_at": "2026-09-25T16:02:55.674788+00:00", + "sha256": "6c2450e80e7b0a6ebfad345dd75604703a27e93a0223de4410d03e4e68561720", + "product_labels": [ + { + "product": "SearXNG", + "present": true, + "version": "2026.9.25+1667fe364" + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "24c03d50574b79f2", + "url": "https://try.karakeep.app/", + "group": "karakeep.app", + "captured_at": "2026-09-25T16:02:57.656703+00:00", + "sha256": "277c85209de80220e5e51e2967bd67b7bf52b3be2f6573ef9dae1b4173bda6c8", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": true, + "version": "" + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "302f5cbbc6e1596b", + "url": "https://wakapi.dev/", + "group": "wakapi.dev", + "captured_at": "2026-09-25T16:02:56.614578+00:00", + "sha256": "94421406e8ac6431d018d9df8c32896cf57d41bf4a4df2f9c0012a504e5e90e5", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": true, + "version": "2.18.0-be6aeb4" + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "a04da6570878fb03", + "url": "https://www.isci.si/", + "group": "www.isci.si", + "captured_at": "2026-09-25T16:02:57.538684+00:00", + "sha256": "adc7023023f3c2afe2d1da333a77e5b211f738afc3625e264b9e9ad632660275", + "product_labels": [ + { + "product": "SearXNG", + "present": true, + "version": "2026.9.23+3cd69d30e" + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "09e85a53a7b4850d", + "url": "https://cloud.karakeep.app/signin", + "group": "karakeep.app", + "captured_at": "2026-09-25T16:05:50.702001+00:00", + "sha256": "77074091f1d359b4b834054eef368cd0e94bacb7363b21294b452d9d3984159b", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": true, + "version": "" + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "e331de721c086202", + "url": "https://codingstats.realzone22.de/", + "group": "codingstats.realzone22.de", + "captured_at": "2026-09-25T16:05:50.303704+00:00", + "sha256": "731deda0f5c0eb0bea2e4f97fd9ca41bbc3de1941606b0ce8c234fda4c2de8cf", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": true, + "version": "2.18.0" + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "8dd6d80787caae8a", + "url": "https://docs.searxng.org/", + "group": "docs.searxng.org", + "captured_at": "2026-09-25T16:05:51.770395+00:00", + "sha256": "66862f9d00640d603cfd43a50910f74006234fbdbeb66a30dfe92db1a5385755", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "fbd19c030c2a721b", + "url": "https://raw.githubusercontent.com/CorentinTh/it-tools/main/README.md", + "group": "raw.githubusercontent.com", + "captured_at": "2026-09-25T16:05:49.692493+00:00", + "sha256": "1a3a4c3665beaa75cbe790e4bb88fecea7353e278274290fd1df15bbb3a0bec1", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "0a5bca2c75194b89", + "url": "https://raw.githubusercontent.com/docmost/docmost/main/README.md", + "group": "raw.githubusercontent.com", + "captured_at": "2026-09-25T16:05:50.626716+00:00", + "sha256": "5dbb7ec0f3cf18b6225a366e424dff7d90de9c7fee00faf600f23b17f74dd768", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "81a3a968cecaaa27", + "url": "https://raw.githubusercontent.com/muety/wakapi/master/README.md", + "group": "raw.githubusercontent.com", + "captured_at": "2026-09-25T16:05:50.376033+00:00", + "sha256": "747e660c59224621d7ad89f29c77833d9d6ecd16b43908f81b8814b8a56a876b", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "ed7fbc949fd767e2", + "url": "https://raw.githubusercontent.com/schlagmichdoch/PairDrop/master/README.md", + "group": "raw.githubusercontent.com", + "captured_at": "2026-09-25T16:05:50.874980+00:00", + "sha256": "4e96b5c538e4d47f4b68e8c09dbc372093949f30bc0ef7e8529510fe24a49b1e", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "0dabb37b2a7b3687", + "url": "https://tools.code.pro.vn/", + "group": "tools.code.pro.vn", + "captured_at": "2026-09-25T16:05:54.693801+00:00", + "sha256": "4908ff993998339aefc32b3a5c1517a9fe83dc2e5a1e9c4988b5a8033b7cc69d", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": true, + "version": "" + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "91315c96894dcfd3", + "url": "https://tools.quitw.org/", + "group": "tools.quitw.org", + "captured_at": "2026-09-25T16:05:52.563429+00:00", + "sha256": "69d5c4c23f61a2403f372dc297964d7f5303b41568702a8c4f1fddca3df5c16f", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": true, + "version": "" + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "fba6e296e0acdccc", + "url": "https://try.karakeep.app/signin", + "group": "karakeep.app", + "captured_at": "2026-09-25T16:05:53.634556+00:00", + "sha256": "94544d1086d031186e6accc94b82c73f76d01a872c67fe98f592aaa417608bf0", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": false + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": true, + "version": "" + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "e693fe9346caa44c", + "url": "https://wakapi.anneraud.fr/", + "group": "wakapi.anneraud.fr", + "captured_at": "2026-09-25T16:05:53.080022+00:00", + "sha256": "c9f9b05fdbc2da2234c1f152dab418ae6ffeeaafdab6677e44f2f3a814166763", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": true, + "version": "2.18.0" + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + }, + { + "id": "28276d6b56102882", + "url": "https://wakapi.m1000.fr/", + "group": "wakapi.m1000.fr", + "captured_at": "2026-09-25T16:05:53.920039+00:00", + "sha256": "bb456f874905c48d870d5571c445128554c3d04fa8dc8a643cd6cf22f1a61865", + "product_labels": [ + { + "product": "SearXNG", + "present": false + }, + { + "product": "Wakapi", + "present": true, + "version": "2.12.0" + }, + { + "product": "IT Tools", + "present": false + }, + { + "product": "PairDrop", + "present": false + }, + { + "product": "Docmost", + "present": false + }, + { + "product": "Linkwarden", + "present": false + }, + { + "product": "Karakeep", + "present": false + }, + { + "product": "New API", + "present": false + } + ] + } + ], + "source_revision": "2d296928c8847bbc60dbfedad77e68bbfea00d14", + "source_worktree_dirty": true, + "code_sha256": { + "judge/cache.go": "5c80af5ec1e0fc210ce650282f9f38568b7c0323e841426d9150980753a6c345", + "judge/classify.go": "60471102fecea7e10bbeac81ec61ed65dc7f503721f7f6452986c303ffb39de5", + "judge/discovery.go": "b9281f9eceac0e09ff4d93f06557450dc17446956a7b5b686914880954bdcacd", + "judge/doc.go": "505a382e87b75d706bfd7962088fe62d4c408597bc41920bfe00e2e1d711fbae", + "judge/generator.go": "8fc8537de9c6aaf233efafc12090e9d9db387154f5ee01a0ebc263e7a3aa3745", + "judge/judge.go": "56ec5d8da51b4edf5bbaa3b2f76b7b80da1926cba7695094f1ce918e8d903535", + "judge/judge_test.go": "6ae85fd3db383235fa76b53fe18e8c5be6d8117420b6cbd68d089d9101eba8a3", + "judge/mark.go": "6c8a7be884737668f54399acb2efd7c262fb5f4bed25183c490ab12561432f56", + "judge/operations.go": "ae5fb0f3b655e008c40ceafcf9c454c4148e654b1badbceeff8454158d83b57c", + "judge/operations_test.go": "417525756c47ff90487b3184d8628b95865894363075839035d8d97d13653777", + "judge/page.go": "d935eba4d1f80d4ac9747e0c7da5090270dcb5a886c1df73ff1f363bcc4a2029", + "judge/provider.go": "722dcdc8bb7339444cb9b831aa1c6f4e2f7f89ead5f7786aa1f9de26c97c6c92", + "judge/real_evidence_test.go": "ccb2e1ee43ade119ac4229f69507ab027ab37b8bd76004c148750016b5ab586a", + "judge/recall.go": "1f27353e28455daf593aa3cacccc46a158bfe41c9f70611cc46f906386bc5b68", + "judge/refine.go": "94cbd144fe3d8694cc2d1ea2fa0770e8ae977f7fdacfc8865a9dad8f3d5915bb", + "judge/round.go": "5750ca5939311685ec7db1d16db1437d7a07cba3d953d7c9ef7381197aa22593", + "judge/stack.go": "a513102b88af5d1fa0e91bb661cb8518b03e9b0dfc4141885082b600376331bf", + "judge/verify.go": "92a1be4b91eeb3553154ec7977c3b087914112b09c06d3421e029b5c31e303f5", + "judge/version.go": "ade768f2f2c93761d2769487c106758f7139707bd548a5aaaeecdcd021f911a1", + "cmd/judgeeval/main.go": "bc401d84036b9beee951bdac2bbe38098b54ecc33eb6c36815b201070d577afa", + "cmd/judgeeval/maintenance.go": "12e80ffd99b5e91a47a1bc72e991190396731b358a1604e19151adf19e6d7574", + "cmd/judgeeval/maintenance_test.go": "0640e8764a867cb5429613df3e05347b1da8ffa717aa21887565e4d97fe6d2a1", + "cmd/judgeeval/replay.go": "1f1a470d3bef3ce1936205407db8a5d42ec893f27fb43c8824aa6340f0f5aa6f", + "cmd/judgeeval/replay_generation.go": "9792c010021381863237d7ca204df024c308339bf8b4387f949db0600cb53e66", + "cmd/judgeeval/replay_metrics.go": "1c1db0deaf9bfa9e87527e83b2cbadf6c96c770aa958f13764ccdeff39f95d5f", + "cmd/judgeeval/replay_test.go": "9a10f9f6ea907f34f2b99d283b85378a2b86f9d0ae97ff84a4e4b45e44891168" + } +} diff --git a/fingerprints/jev-maintained-20260926.yaml b/fingerprints/jev-maintained-20260926.yaml new file mode 100644 index 0000000..f6e49a6 --- /dev/null +++ b/fingerprints/jev-maintained-20260926.yaml @@ -0,0 +1,47 @@ +- name: IT Tools + part: "" + vendor: "" + product: "" + protocol: http + default_port: + - "80" + rule: + - version: _ + regexps: + body: + - it tools - handy online tools for developers +- name: Wakapi + part: "" + vendor: "" + product: "" + protocol: http + default_port: + - "80" + rule: + - version: _ + regexps: + regexp: + - (?s)wakapi – coding statistics.*?wakapi – coding statistics.*?\s+v([0-9]{1,4}(?:\.[0-9]{1,4}){1,3}(?:[A-Za-z]+[0-9]*|[-+][0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*)?)(?:[^0-9A-Za-z.+-]|$) + - (?s)wakapi – coding statistics.*?wakapi – coding statistics.*?wakapi – coding statistics +- name: searxng + part: "" + vendor: "" + product: "" + protocol: http + default_port: + - "80" + rule: + - version: _ + regexps: + regexp: + - ` 和 `judge:layer=`，所以 gogo、spray 现有的输出和序列化不需要改动。读取时请用函数，不要解析 tag： +## 能力与语义 | API | 含义 | |---|---| -| `judge.Is(f, judge.Rejected)` | 误报：不在技术栈里，或者只是正文里提到 | -| `judge.Is(f, judge.Duplicate)` | 另一个引擎对同一产品的另一种写法 | -| `judge.Is(f, judge.Primary)` | 本页所属的主应用 | -| `judge.Is(f, judge.Recalled)` | 规则漏掉、经判定确认后补入 | -| `judge.LayerOf(f)` | 所属层级：`LayerCDN` `LayerServer` `LayerRuntime` `LayerFramework` `LayerApplication` `LayerFrontend` `LayerDevice` `LayerNotPresent` | -| `judge.Judged(f)` | 是否已经判定过 | -| `judge.Accepted(frames)` | 去掉 `Rejected` 和 `Duplicate` 之后的结果 | -| `judge.PrimaryOf(frames)` | 主应用 | - -`Verify` 会跳过已经判定过的条目，所以对同一份 frames 重复调用 `Refine` 不会产生额外请求。 - -## 缓存与去重 - -默认开启，不需要任何配置： +| `j.Yes(ctx, state, question)` | 返回“是”的概率，`float64`，不是布尔判决 | +| `j.Choose(ctx, state, question, options)` | 从给定选项中选一个，返回选项和置信度 | +| `j.Score(ctx, state, question, levels...)` | 按有序等级返回 0 到 1 的分数 | +| `j.Verify(ctx, raw, hits, knownNames...)` | 核验产品是否存在、归并同名、判断层级和主应用，返回接受的指纹 | +| `j.Inspect(ctx, raw, hits, knownNames...)` | 返回所有规则命中及拒绝、重复、层级、主应用等诊断标记 | +| `j.Classify(ctx, raw)` | 返回页面类型 `Kind` 和是否为产品标准页面 | +| `j.Version(ctx, raw, framework)` | 在响应中的版本候选里选择该产品的版本；不修改输入 | +| `j.Refine(ctx, raw, hits, knownNames...)` | 核验、分类并为主产品选择版本；返回指纹、类型和标准页面判断 | +| `j.IsUnknownProduct(ctx, raw, accepted)` | 标准产品页面但没有已确认的应用或设备时返回 true | +| `j.SuggestNames(ctx, raw)` | 从本页头、标题、资源和正文提取候选，并判断哪些是产品名 | +| `judge.NewGenerator(j)` | 用正反样本生成原生 `*fingers.Finger` | -- **按问题缓存**：每个问题的缓存 key 由问题内容、具名证据、页面标题（数字归一化）和 Provider ID 组成，再加上页面签名（`Page.Signature()`：64 位 simhash；数字归一化，文本按双字切分，结构特征权重更高）。标题相同、签名相差不超过 `SimilarDistance`（默认 1）位的页面直接复用答案。问题怎么组合都能命中；部分命中时，只发送没命中的问题。 -- **并发合并**：多个 worker 同时判定相似页面、问同样的问题时，只有一个请求真正发出去，其余的等它的答案。 -- `Judge.Cache` 可以换成共享实现，比如 redis 或磁盘，只需要实现 `Get(key, sig)` / `Put(key, sig, value)`。设为 nil 就关闭缓存，并发合并仍然生效。 -- 统计：`Judge.Requests`（实际发出的请求数）、`Judge.CacheHits`（不发请求就答完的轮次数）。 +`state` 是任意可 JSON 编码的值。判断 HTTP 页面时，可以传 `judge.NewPage(raw)` 的结果。`Verify` 与 `Refine` 不修改 `hits`，失败时也不会返回部分判定结果。需要看到被剔除的条目时用 `Inspect`，再以 `judge.Is`、`judge.LayerOf`、`judge.PrimaryOf` 读取标记。 -`SimilarDistance` 的取值来自实测（`cmd/judgeeval`，1070 个真实页面，32284 个答案）：复用相似页面的答案，与该页面自己的答案对比： +`knownNames` 是调用方明确给出的**已有**产品名候选。`judge.NewRetriever(names).Find(page.Haystack(), limit)` 可在页面中查找现有别名。它只做已知名称召回；`IsUnknownProduct` 和 `SuggestNames` 是独立能力。后者只返回本页可提取的名字，不能还原已彻底改名且没有原名证据的产品。 -| 距离 ≤ | 可复用答案 | 一致率 | -|---|---|---| -| 0 | 136 | 100% | -| 1 | 159 | 100% | -| 2 | 194 | 98.5% | -| 3 | 225 | 96.9% | -| 5 | 293 | 94.9% | +`Refine` 通常发两轮请求：第一轮核验与页面分类，第二轮为已确认的主产品挑版本。答案按问题缓存，相似页面可共享。Provider 缺少任何被请求的答案时，整轮失败。 -所以默认值取 1，只在实测零误差的范围内复用。标题一致这个前提同样必要：没有它的话，彼此无关、特征稀疏的页面（例如中文站点，都带 `Server: nginx` 和 jquery）会出现签名距离 0–3 的碰撞。 - -随机抽取的互联网站点之间，能复用的答案不到 1%。收益主要在扫描场景：同一主机的大量 404 页或登录页、同一网段里的同型号设备、托管商的人机验证页。完全相同的问题，只要页面相同或相似，都会走缓存。 - -## 自定义问题 - -每项能力都是往 `Round` 里加问题的函数，调用方也可以加自己的问题，和内置问题合并在同一次请求里： +## 生成指纹 ```go -page, _ := judge.NewPage(raw) -r := page.Round() -judge.Classify(r) -r.Add("honeypot", judge.Binary("Is this page a honeypot?"), func(a judge.Answer) { - honeypot = a.Yes >= j.Threshold -}) -err := r.Ask(ctx, j) +g := judge.NewGenerator(j). + PositiveVersion(jenkinsA, "2.401.3"). + PositiveVersion(jenkinsB, "2.402.1"). + Negative(otherProduct) + +finger, err := g.Generate(ctx) // *fingers.Finger，可直接用于现有规则引擎 +if err != nil { return err } +err = g.Validate(finger) // 编辑规则后，再按全部正反样本验证 ``` -- 问题类型： - - `Binary`（`Answer.Yes` 为"是"的概率，每个问题独立判断，所以一页上可以同时认定多个产品存在）； - - `Choice`（`Answer.Choice` 和 `Confidence`，即在选项之间做 rerank）； - - `Score`（`Answer.Score`）。 -- `BinaryWith` 可以分别写明"是"和"否"的含义，判断会更准。 -- `r.Evidence(name, v)` 可以把一份具名证据放进 state（例如版本号候选列表）。模型读得很字面：具名给出的证据，比写在选项描述里的效果好得多。 -- 只有整轮请求成功后才会执行回调；请求失败时，页面和 frames 都不会被修改。 +至少需要一个正样本和一个反样本。`Name("Jenkins")` 可以给出已知名称；缺省时从正样本调用 `SuggestNames` 自动选择。`PositiveVersion` 提供版本真值；普通 `Positive` 在有 Judge 时会尝试从页面候选中高置信度选版本。版本规则只有在样本证据足以构造并验证提取式时才会生成。 -## 接入新的 Provider +主动探测以原始 `send_data` 和对应的完整 HTTP 响应记录： ```go -type Provider interface { - ID() string // provider/模型版本；它是缓存 key 的一部分，模型换了就要换 ID - Judge(ctx context.Context, state interface{}, questions map[string]judge.Question) (map[string]judge.Answer, error) -} +g := judge.NewGenerator(j).Name("Orion"). + Positive(home).Probe([]byte("/admin"), adminResponse). + Negative(otherHome).Probe([]byte("/admin"), otherAdminResponse) +finger, err := g.Generate(ctx) ``` -- Provider 只负责把问题翻译成自己的协议并发送出去；缓存、合并、阈值和决策都不需要自己实现。可以参考 `judge/jev`（约 200 行）。 -- 可选实现 `Calibrated`，返回在标注集上测出来的阈值：`Calibration() (threshold, versionConfidence)`。`judge.New` 会采用它，否则使用 0.5 / 0.9。 -- 校准和对比：先在 `cmd/judgeeval` 的 `newProvider` 里注册，然后 - - ```bash - judgeeval -provider -samples testdata/samples -labels testdata/labels.json - ``` +`ProbeWith(ctx, request, send)` 可由调用方提供在线发送函数。主动规则需要同一请求的正反响应；`Validate` 通过现有 `Finger.Match` 执行被动与主动规则，不调用模型，也不会写入模板仓库。无法区分正反样本时，`Generate` 返回错误。 - 用真值集调好阈值，再在真实页面上和纯规则做对比。 +## 自定义 Provider 与批量问题 -## 稳定性约定 +Provider 只需实现 `ID() string` 和 `Judge(ctx, state, map[string]judge.Question) (map[string]judge.Answer, error)`。可选实现 `Calibration() (threshold, versionConfidence float64)`。Jev 适配器把二元问题映射到 `noul`，把选择和评分映射到 Jev 对应类型；缓存、并发合并和阈值决策在 `Judge` 中完成。 -- 对外 API：`New`、`Judge`、`Provider`、`Calibrated`、`Question`/`Answer` 及其构造函数、`Page`/`NewPage`、`Round`、`Verify`/`Classify`/`Version`/`Refine`、`Mark`/`Is`/`LayerOf`/`Judged`/`Accepted`/`PrimaryOf`、`Kind`、`Layer`、`Cache`/`NewMemoryCache`。 -- tag 的写法（`judge:*`）和 `Kind`、`Layer` 的取值属于输出格式，不会随意变更。 -- 问题的措辞、候选数量上限、版本号排序规则属于实现细节，可能随评测结果调整；每次调整都要用 `cmd/judgeeval` 回归。 +自定义问题可通过 `j.Yes`、`j.Choose`、`j.Score` 单独调用。需要把多个问题合并为一次请求时，使用 `page.Round()`、`Round.Add` 和 `Round.Ask`；它是高级组合接口，`Question`/`Answer` 仅服务于 Provider 和批量扩展。 diff --git a/judge/classify.go b/judge/classify.go index 3219ab1..e9a66cd 100644 --- a/judge/classify.go +++ b/judge/classify.go @@ -18,7 +18,7 @@ var pageKinds = map[string]string{ string(KindDefault): "The stock page a web server or product shows right after installation", string(KindError): "A framework or server error page such as 404 or 500", string(KindDirListing): "An auto-generated file index of a directory", - string(KindConsole): "The main interface or loading screen of a web application or management console", + string(KindConsole): "The main interface or loading screen of a web application or management console, including packaged static/browser-side tools and search applications", string(KindContent): "An ordinary website page: article, company site, shop, forum, blog, status page", string(KindAPI): "A machine-readable API response rather than a web page", } @@ -32,7 +32,7 @@ func Classify(r *Round) { r.Add("page_kind", Choice("What kind of page is this HTTP response?", pageKinds), func(a Answer) { p.Kind = Kind(a.Choice) }) r.Add("generic", BinaryWith("Is this page the stock interface of a packaged software product, framework or device, "+ "so that the same page would appear on many unrelated deployments?", - "Login, console, default or error page shipped by a product; only branding, host names or data differ between installs", + "Login, console, default or error page, packaged search application or static/browser-side tools shipped by a product; no login or backend is required; only branding, host names or data differ between installs", "Content written by one organization for its own purpose: articles, company pages, shops, forums, custom-built portals"), func(a Answer) { p.Generic = a.Yes >= r.judge.Threshold }) } diff --git a/judge/discovery.go b/judge/discovery.go new file mode 100644 index 0000000..1d1b47b --- /dev/null +++ b/judge/discovery.go @@ -0,0 +1,146 @@ +package judge + +import ( + "context" + "fmt" + "regexp" + "sort" + "strings" + "unicode/utf8" + + "github.com/chainreactors/fingers/common" +) + +var ( + nameWords = regexp.MustCompile(`[A-Z][A-Za-z0-9!+._]{2,}(?:[ -][A-Z][A-Za-z0-9!+._]{2,}){0,2}|[\p{Han}]{2,20}`) + nameVersion = regexp.MustCompile(`(?i)[/\s]+[vV]?(?:` + versionToken + `|[0-9]+)\s*$`) +) + +// IsUnknownProduct reports whether this looks like a reusable product page +// with no identified product. Pass accepted fingerprints from Verify or +// Refine; known infrastructure alone does not identify an application. +func (j *Judge) IsUnknownProduct(ctx context.Context, raw []byte, accepted common.Frameworks) (bool, error) { + kind, generic, err := j.Classify(ctx, raw) + if err != nil || !generic { + return false, err + } + for _, f := range accepted { + if Is(f, Rejected) || Is(f, Duplicate) { + continue + } + switch LayerOf(f) { + case LayerApplication, LayerDevice: + return false, nil + case LayerServer, LayerCDN: + if kind == KindDefault || kind == KindError || kind == KindDirListing { + return false, nil + } + } + } + return true, nil +} + +// SuggestNames returns product names supported by page evidence. Candidates +// are extracted from this response, including visible body text; no name is +// invented by the provider or drawn from an installed fingerprint catalog. +func (j *Judge) SuggestNames(ctx context.Context, raw []byte) ([]string, error) { + p, err := NewPage(raw) + if err != nil { + return nil, err + } + candidates := pageNames(p) + if len(candidates) == 0 { + return nil, nil + } + r := p.Round() + prob := make([]float64, len(candidates)) + for i, name := range candidates { + i, name := i, name + r.Add(fmt.Sprintf("name_%d", i), BinaryWith( + fmt.Sprintf("Is `%s` the product whose own interface, login, console or default page this response shows, rather than infrastructure underneath another product, generic wording or an article mention?", name), + name+" owns this page", + name+" is infrastructure or merely mentioned"), + func(a Answer) { prob[i] = a.Yes }) + } + if err := r.Ask(ctx, j); err != nil { + return nil, err + } + var indices []int + for i, v := range prob { + if v >= j.Threshold { + indices = append(indices, i) + } + } + sort.SliceStable(indices, func(a, b int) bool { return prob[indices[a]] > prob[indices[b]] }) + out := make([]string, 0, len(indices)) + for _, i := range indices { + out = append(out, candidates[i]) + } + return out, nil +} + +func pageNames(p *Page) []string { + type candidate struct { + name string + weight int + } + byKey := map[string]candidate{} + add := func(text string, weight int) { + text = strings.Trim(strings.TrimSpace(nameVersion.ReplaceAllString(clean(text), "")), "-:|/ ") + if text == "" || utf8.RuneCountInString(text) > 40 || !usableKey(strings.ToLower(text)) { + return + } + key := NormalizeName(text) + if key == "" || genericName(key) { + return + } + if prev, ok := byKey[key]; !ok || weight > prev.weight { + byKey[key] = candidate{text, weight} + } + } + add(p.Generator, 5) + for k, v := range p.Headers { + if strings.HasPrefix(k, "X-") || k == "Server" || k == "X-Powered-By" || k == "Product" { + add(strings.Split(v, "/")[0], 4) + } + } + for _, part := range strings.FieldsFunc(p.Title, func(r rune) bool { return r == '|' || r == '-' || r == ':' || r == '–' }) { + add(part, 4) + } + for _, asset := range append(append([]string(nil), p.Scripts...), p.Styles...) { + for _, word := range strings.FieldsFunc(asset, func(r rune) bool { return r == '/' || r == '_' }) { + add(word, 2) + } + } + for _, segment := range strings.FieldsFunc(p.Text, func(r rune) bool { return r == '.' || r == ':' || r == '。' || r == '：' || r == '|' || r == '，' }) { + for _, match := range nameWords.FindAllString(segment, 20) { + add(match, 1) + } + } + list := make([]candidate, 0, len(byKey)) + for _, c := range byKey { + list = append(list, c) + } + sort.Slice(list, func(a, b int) bool { + if list[a].weight != list[b].weight { + return list[a].weight > list[b].weight + } + return list[a].name < list[b].name + }) + if len(list) > 20 { + list = list[:20] + } + out := make([]string, len(list)) + for i, c := range list { + out[i] = c.name + } + return out +} + +func genericName(key string) bool { + switch key { + case "login", "signin", "welcome", "dashboard", "admin", "console", "home", "index", "server", "error", "notfound", "password", "username", "management", "system", "登录", "首页", "管理", "用户", "密码", "控制台": + return true + } + return false +} diff --git a/judge/doc.go b/judge/doc.go index 1d067e1..7e7c89e 100644 --- a/judge/doc.go +++ b/judge/doc.go @@ -1,23 +1,15 @@ -// Package judge is the judgement and rerank layer of fingers. It sits on top -// of the rule engines instead of being one: rules and code recall candidates -// with evidence, a Provider (a model such as TypeSafe Jev in judge/jev) -// answers typed questions about that evidence, and code makes every final -// decision. +// Package judge exposes evidence-based judgement operations for HTTP +// fingerprints. A Provider (such as TypeSafe Jev in judge/jev) answers typed +// questions; Judge applies thresholds, caching and fingerprint semantics. // -// j := judge.New(provider) // shared by a whole scan: cache, merging, thresholds -// page, _ := judge.NewPage(raw) // page data entry -// r := page.Round() // one request -// judge.Verify(r, frames, recall) // false positives / negatives, duplicates, layer, primary -// judge.Classify(r) // page kind, generic page -// err := r.Ask(ctx, j) // answers mark frames and page in place -// r = page.Round() -// judge.Version(r, judge.PrimaryOf(frames)) // version of the primary application -// err = r.Ask(ctx, j) +// j := judge.New(provider) +// accepted, kind, generic, err := j.Refine(ctx, raw, ruleHits) +// all, err := j.Inspect(ctx, raw, ruleHits) // includes rejected hits +// finger, err := judge.NewGenerator(j).Positive(good).Negative(other).Generate(ctx) // -// Refine runs exactly this; fingers.Engine.Refine adds recall from the -// fingerprint names. Results are Marks on the frameworks (Is, LayerOf, -// Accepted) and Kind / Generic on the page. Answers are cached per question, -// and similar pages (Page.Signature) share them. +// Judge methods do not mutate rule hits. Round remains available for callers +// composing custom questions into one provider request. Similar pages can +// share cached answers through Page.Signature. // // See README.md in this directory. package judge diff --git a/judge/generator.go b/judge/generator.go new file mode 100644 index 0000000..565cdd9 --- /dev/null +++ b/judge/generator.go @@ -0,0 +1,669 @@ +package judge + +import ( + "bytes" + "context" + "fmt" + "regexp" + "sort" + "strings" + + "github.com/chainreactors/fingers/common" + fingerlib "github.com/chainreactors/fingers/fingers" +) + +// Generator builds one native Finger from labelled HTTP responses. Probe +// belongs to the most recently added positive or negative sample. +type Generator struct { + judge *Judge + name string + positive []generatorSample + negative []generatorSample + current *generatorSample + err error +} + +type generatorSample struct { + raw []byte + version string + probes map[string][]byte +} + +// NewGenerator creates a fingerprint generator using j for name and version +// selection. A Name hint lets rule generation work without a provider. +func NewGenerator(j *Judge) *Generator { return &Generator{judge: j} } + +// Name supplies a known product name. Without it Generate selects a name +// from the positive responses. +func (g *Generator) Name(name string) *Generator { + g.name = strings.TrimSpace(name) + return g +} + +// Positive adds an HTTP response known to belong to the product. +func (g *Generator) Positive(raw []byte) *Generator { + g.positive = append(g.positive, generatorSample{raw: bytes.Clone(raw), probes: map[string][]byte{}}) + g.current = &g.positive[len(g.positive)-1] + return g +} + +// PositiveVersion adds a positive response with an authoritative version. +func (g *Generator) PositiveVersion(raw []byte, version string) *Generator { + g.Positive(raw) + g.positive[len(g.positive)-1].version = strings.TrimSpace(version) + return g +} + +// Negative adds an HTTP response known not to belong to the product. +func (g *Generator) Negative(raw []byte) *Generator { + g.negative = append(g.negative, generatorSample{raw: bytes.Clone(raw), probes: map[string][]byte{}}) + g.current = &g.negative[len(g.negative)-1] + return g +} + +// Probe attaches a recorded request and response to the last sample. request +// is the exact send_data used by the rule engine. +func (g *Generator) Probe(request, response []byte) *Generator { + if g.current == nil { + g.err = fmt.Errorf("judge: Probe requires a preceding Positive or Negative") + return g + } + if len(request) == 0 { + g.err = fmt.Errorf("judge: Probe requires non-empty send_data") + return g + } + g.current.probes[string(request)] = bytes.Clone(response) + return g +} + +// ProbeWith sends request and records the response for the last sample. The +// caller controls network behavior through send. +func (g *Generator) ProbeWith(ctx context.Context, request []byte, send func(context.Context, []byte) ([]byte, error)) error { + if g.current == nil { + return fmt.Errorf("judge: ProbeWith requires a preceding Positive or Negative") + } + if send == nil || len(request) == 0 { + return fmt.Errorf("judge: ProbeWith requires a sender and non-empty send_data") + } + response, err := send(ctx, request) + if err != nil { + return err + } + g.Probe(request, response) + return nil +} + +// Generate returns one compiled, rule-validated fingerprint. It never writes +// a templates repository. At least one positive and one negative are required. +func (g *Generator) Generate(ctx context.Context) (*fingerlib.Finger, error) { + if g.err != nil { + return nil, g.err + } + if len(g.positive) == 0 || len(g.negative) == 0 { + return nil, fmt.Errorf("judge: Generate requires positive and negative samples") + } + if err := g.validateSamples(); err != nil { + return nil, err + } + name := g.name + if name == "" { + if g.judge == nil { + return nil, fmt.Errorf("judge: a Judge or Name is required to select a product name") + } + counts := map[string]int{} + ranks := map[string]int{} + labels := map[string]string{} + for _, sample := range g.positive { + names, err := g.judge.SuggestNames(ctx, sample.raw) + if err != nil { + return nil, err + } + for rank, candidate := range names { + key := NormalizeName(candidate) + counts[key]++ + ranks[key] += len(names) - rank + if labels[key] == "" { + labels[key] = candidate + } + } + } + var keys []string + for key := range counts { + keys = append(keys, key) + } + sort.Slice(keys, func(a, b int) bool { + if counts[keys[a]] != counts[keys[b]] { + return counts[keys[a]] > counts[keys[b]] + } + if ranks[keys[a]] != ranks[keys[b]] { + return ranks[keys[a]] > ranks[keys[b]] + } + return keys[a] < keys[b] + }) + if len(keys) == 0 || counts[keys[0]] != len(g.positive) { + return nil, fmt.Errorf("judge: no product name supported by every positive sample; provide Name") + } + name = labels[keys[0]] + } + + versions := make([]string, len(g.positive)) + for i, sample := range g.positive { + versions[i] = sample.version + if versions[i] == "" && g.judge != nil { + v, err := g.judge.Version(ctx, sample.raw, common.NewFramework(name, common.FrameFromGUESS)) + if err != nil { + return nil, err + } + versions[i] = v + } + } + + f := &fingerlib.Finger{Name: name, Protocol: fingerlib.HTTPProtocol} + f.Rules = append(f.Rules, g.rules("", name)...) + hasVersion := false + for _, version := range versions { + hasVersion = hasVersion || version != "" + } + for _, request := range g.probeRequests() { + if hasVersion { + if rule := g.versionRule(request, name, versions); rule != nil { + f.Rules = append(f.Rules, rule) + f.Rules = append(f.Rules, g.rules(request, name)...) + } + } else { + f.Rules = append(f.Rules, g.rules(request, name)...) + } + } + if len(f.Rules) == 0 { + return nil, fmt.Errorf("judge: no rule distinguishes the positive and negative samples") + } + if rule := g.versionRule("", name, versions); rule != nil { + f.Rules = append(fingerlib.Rules{rule}, f.Rules...) + } + if err := g.Validate(f); err != nil { + return nil, err + } + if err := f.Compile(false); err != nil { + return nil, err + } + return f, nil +} + +// Validate checks the generated or edited Finger against every recorded +// positive and negative using the rule engine, without calling the provider. +func (g *Generator) Validate(f *fingerlib.Finger) error { + if g.err != nil { + return g.err + } + if f == nil || len(f.Rules) == 0 { + return fmt.Errorf("judge: fingerprint has no rules") + } + if len(g.positive) == 0 || len(g.negative) == 0 { + return fmt.Errorf("judge: validation requires positive and negative samples") + } + if err := g.validateSamples(); err != nil { + return err + } + for i, rule := range f.Rules { + if rule == nil { + return fmt.Errorf("judge: rule %d is nil", i) + } + } + checking := copyFingerForValidation(f) + if err := checking.Compile(false); err != nil { + return err + } + check := func(sample generatorSample) (*common.Framework, bool) { + sender := func(request []byte) ([]byte, bool) { + response, ok := sample.probes[string(request)] + return response, ok + } + frame, _, ok := checking.Match(fingerlib.NewContent(sample.raw, "", true), 2, sender) + return frame, ok + } + for i, sample := range g.positive { + frame, ok := check(sample) + if !ok { + return fmt.Errorf("judge: positive sample %d did not match", i) + } + if sample.version != "" && frame.Version != sample.version { + return fmt.Errorf("judge: positive sample %d version = %q, want %q", i, frame.Version, sample.version) + } + } + for i, sample := range g.negative { + if _, ok := check(sample); ok { + return fmt.Errorf("judge: negative sample %d matched", i) + } + } + return nil +} + +func (g *Generator) validateSamples() error { + for _, sample := range append(append([]generatorSample(nil), g.positive...), g.negative...) { + if _, err := NewPage(sample.raw); err != nil { + return fmt.Errorf("judge: invalid sample: %w", err) + } + for request, response := range sample.probes { + if request == "" { + return fmt.Errorf("judge: probe has empty send_data") + } + if _, err := NewPage(response); err != nil { + return fmt.Errorf("judge: invalid probe response: %w", err) + } + } + } + return nil +} + +func copyFingerForValidation(f *fingerlib.Finger) *fingerlib.Finger { + copy := *f + copy.Rules = make(fingerlib.Rules, len(f.Rules)) + for i, rule := range f.Rules { + if rule == nil { + continue + } + cloned := *rule + if rule.Regexps != nil { + regexps := *rule.Regexps + regexps.Body = append([]string(nil), rule.Regexps.Body...) + regexps.Header = append([]string(nil), rule.Regexps.Header...) + regexps.CompliedRegexp = nil + regexps.CompiledVulnRegexp = nil + regexps.CompiledVersionRegexp = nil + cloned.Regexps = ®exps + } + copy.Rules[i] = &cloned + } + return © +} + +type ruleCandidate struct { + kind string + value string + weight int +} + +func candidates(raw []byte) []ruleCandidate { + p, err := NewPage(raw) + if err != nil { + return nil + } + var out []ruleCandidate + add := func(kind, value string, weight int) { + value = strings.ToLower(strings.TrimSpace(value)) + if len(value) >= 5 && len(value) <= 120 { + out = append(out, ruleCandidate{kind, value, weight}) + } + } + for k, v := range p.Headers { + if k == "Server" || strings.HasPrefix(k, "X-") || k == "Product" { + value := strings.TrimSpace(nameVersion.ReplaceAllString(strings.Split(v, "/")[0], "")) + if strings.HasPrefix(k, "X-") && (strings.HasSuffix(k, "-Version") || value == "" || value[0] >= '0' && value[0] <= '9') { + add("header", k+":", 6) + } else { + add("header", k+": "+value, 7) + } + } + } + add("body", p.Generator, 6) + // Preserve the declaration, but not a single release or customized title. + if name := nameVersion.ReplaceAllString(p.Generator, ""); name != "" { + if loc := reGenerator.FindSubmatchIndex(raw); loc != nil { + start := loc[2] + if start < 0 { + start = loc[4] + } + end := start + len(name) + if end < len(raw) && (raw[end] == '/' || raw[end] == ' ') { + end++ + } + add("body", string(raw[loc[0]:end]), 8) + } + } + if !genericName(NormalizeName(p.Title)) { + if title := reTitle.Find(raw); len(title) > 0 { + add("body", string(title), 5) + } + add("body", p.Title, 4) + } + for _, asset := range append(append([]string(nil), p.Scripts...), p.Styles...) { + path := strings.Split(asset, "?")[0] + add("body", path, 3) + } + for _, comment := range p.Comments { + add("body", comment, 2) + } + for _, name := range pageNames(p) { + if strings.Contains(strings.ToLower(p.Text), strings.ToLower(name)) { + add("body", name, 1) + } + } + sort.SliceStable(out, func(a, b int) bool { + if out[a].weight != out[b].weight { + return out[a].weight > out[b].weight + } + return len(out[a].value) > len(out[b].value) + }) + return out +} + +func containsCandidate(raw []byte, c ruleCandidate) bool { + content := fingerlib.NewContent(raw, "", true) + var target []byte + if c.kind == "header" { + target = content.Header + } else { + target = content.Body + } + return bytes.Contains(target, []byte(c.value)) +} + +func candidateRule(c ruleCandidate) *fingerlib.Rule { + r := &fingerlib.Rule{Regexps: &fingerlib.Regexps{}} + if c.kind == "header" { + r.Regexps.Header = []string{c.value} + } else { + r.Regexps.Body = []string{c.value} + } + return r +} + +func (g *Generator) rules(request, name string) fingerlib.Rules { + positive, negative, _ := g.responses(request) + if len(positive) == 0 || len(negative) == 0 { + return nil + } + var rules fingerlib.Rules + for _, c := range selectCandidates(positive, negative, name) { + rule := candidateRule(c) + if request != "" { + rule.SendDataStr, rule.Level = request, 2 + } + rules = append(rules, rule) + } + return rules +} + +// Pick evidence that covers the most still-uncovered positives while staying +// absent from every negative. A single common matcher wins when one exists. +func selectCandidates(positive, negative [][]byte, product ...string) []ruleCandidate { + if len(positive) == 0 { + return nil + } + options := map[string]ruleCandidate{} + for _, raw := range positive { + for _, c := range candidates(raw) { + // A product must not inherit its proxy/CDN or an unrelated security + // header merely because that header is absent from a small negative set. + if c.kind == "header" && len(product) > 0 && !strings.Contains(NormalizeName(c.value), NormalizeName(product[0])) { + continue + } + key := c.kind + "\x00" + c.value + if previous, ok := options[key]; !ok || c.weight > previous.weight { + options[key] = c + } + } + } + covered := make([]bool, len(positive)) + var selected []ruleCandidate + for len(selected) < len(positive) { + best, bestCount := ruleCandidate{}, 0 + for _, c := range options { + valid := true + for _, raw := range negative { + if containsCandidate(raw, c) { + valid = false + break + } + } + if !valid { + continue + } + count := 0 + for i, raw := range positive { + if !covered[i] && containsCandidate(raw, c) { + count++ + } + } + if count > bestCount || count == bestCount && count > 0 && (c.weight > best.weight || c.weight == best.weight && (c.value < best.value || c.value == best.value && c.kind < best.kind)) { + best, bestCount = c, count + } + } + if bestCount == 0 { + break + } + selected = append(selected, best) + delete(options, best.kind+"\x00"+best.value) + for i, raw := range positive { + covered[i] = covered[i] || containsCandidate(raw, best) + } + all := true + for _, ok := range covered { + all = all && ok + } + if all { + break + } + } + return selected +} + +func (g *Generator) probeRequests() []string { + seen := map[string]bool{} + var out []string + for _, sample := range g.positive { + for request := range sample.probes { + if !seen[request] { + seen[request] = true + out = append(out, request) + } + } + } + sort.Strings(out) + return out +} + +func (g *Generator) versionRule(request, name string, versions []string) *fingerlib.Rule { + positive, negative, indices := g.responses(request) + if len(positive) == 0 || len(negative) == 0 { + return nil + } + known := make([]string, len(indices)) + for i, index := range indices { + known[i] = versions[index] + } + var patterns []string + if pattern := versionPattern(name, positive, known, negative); pattern != "" { + patterns = append(patterns, pattern) + } else { + patterns = guardedVersionPatterns(name, positive, known, negative) + } + if len(patterns) == 0 { + return nil + } + rule := &fingerlib.Rule{Regexps: &fingerlib.Regexps{Regexp: patterns}} + if request != "" { + rule.SendDataStr, rule.Level = request, 2 + } + return rule +} + +func (g *Generator) responses(request string) (positive, negative [][]byte, indices []int) { + response := func(sample generatorSample) ([]byte, bool) { + if request == "" { + return sample.raw, true + } + raw, ok := sample.probes[request] + return raw, ok + } + for i, sample := range g.positive { + if raw, ok := response(sample); ok { + positive = append(positive, raw) + indices = append(indices, i) + } + } + for _, sample := range g.negative { + if raw, ok := response(sample); ok { + negative = append(negative, raw) + } + } + return positive, negative, indices +} + +func versionPattern(name string, positive [][]byte, versions []string, negative [][]byte) string { + first := -1 + for i, v := range versions { + if v != "" { + first = i + break + } + } + if first < 0 { + return "" + } + raw := string(positive[first]) + for from := 0; from < len(raw); { + next := strings.Index(raw[from:], versions[first]) + if next < 0 { + break + } + index := from + next + from = index + len(versions[first]) + pattern := versionPatternAt(raw, index, name) + if pattern == "" { + continue + } + re, err := regexp.Compile("(?i)" + pattern) + if err != nil { + continue + } + valid := true + for i, version := range versions { + if version == "" { + continue + } + match := re.FindStringSubmatch(string(positive[i])) + if len(match) < 2 || match[1] != version { + valid = false + break + } + } + for _, response := range negative { + if re.Match(response) { + valid = false + break + } + } + if valid { + return pattern + } + } + return "" +} + +func versionPatternAt(raw string, index int, name string) string { + var pattern string + // A generator meta tag is stronger than an unscoped product mention. + for _, loc := range reGenerator.FindAllStringSubmatchIndex(raw, -1) { + start, end := loc[2], loc[3] + if start < 0 { + start, end = loc[4], loc[5] + } + if index >= start && index < end && strings.Contains(NormalizeName(raw[start:index]), NormalizeName(name)) { + return regexp.QuoteMeta(raw[loc[0]:index]) + `(` + versionToken + `|[0-9]+)` + } + } + if headerEnd := strings.Index(raw, "\r\n\r\n"); headerEnd >= 0 && index < headerEnd { + lineStart := strings.LastIndex(raw[:index], "\n") + 1 + prefix := strings.TrimSpace(raw[lineStart:index]) + if len(prefix) > 48 || !strings.Contains(NormalizeName(prefix), NormalizeName(name)) { + return "" + } + pattern = `(?m)^` + regexp.QuoteMeta(prefix) + `\s*(` + versionToken + `|[0-9]+)` + } else { + begin := index - 60 + if begin < 0 { + begin = 0 + } + window := raw[begin:index] + nameIndex := strings.LastIndex(strings.ToLower(window), strings.ToLower(name)) + if nameIndex < 0 { + return "" + } + prefix := window[nameIndex:] + if len(prefix) > 48 || strings.ContainsAny(prefix, "\r\n<>") { + return "" + } + pattern = regexp.QuoteMeta(strings.TrimSpace(prefix)) + `\s*(` + versionToken + `|[0-9]+)` + } + return pattern +} + +// Some applications put their release only in asset queries or a footer. +// Learn literal local context, conjoined with a product declaration in ONE +// regex: native body/header/regexp matchers are alternatives, not AND gates. +func guardedVersionPatterns(name string, positive [][]byte, versions []string, negative [][]byte) []string { + var patterns []string + seen := map[string]bool{} + for _, guard := range selectCandidates(positive, negative, name) { + if guard.weight < 4 || !strings.Contains(NormalizeName(guard.value), NormalizeName(name)) { + continue + } + for i, version := range versions { + if version == "" { + continue + } + raw := string(positive[i]) + for from := 0; from < len(raw); { + next := strings.Index(raw[from:], version) + if next < 0 { + break + } + index := from + next + from = index + len(version) + begin := strings.LastIndex(raw[:index], "<") + if begin < 0 || index-begin > 120 { + continue + } + prefix := regexp.QuoteMeta(raw[begin:index]) + prefix = reSpace.ReplaceAllStringFunc(prefix, func(string) string { return `\s+` }) + pattern := `(?s)` + regexp.QuoteMeta(guard.value) + `.*?` + prefix + `(` + versionToken + `)(?:[^0-9A-Za-z.+-]|$)` + if seen[pattern] { + continue + } + seen[pattern] = true + re, err := regexp.Compile("(?i)" + pattern) + if err != nil { + continue + } + valid := true + for j, want := range versions { + match := re.FindStringSubmatch(string(positive[j])) + if want == "" { + if match != nil { + valid = false + } + continue + } + if len(match) < 2 || match[1] != want { + valid = false + break + } + } + for _, raw := range negative { + if re.Match(raw) { + valid = false + break + } + } + if valid { + patterns = append(patterns, pattern) + if len(patterns) == 4 { + return patterns + } + } + } + } + } + return patterns +} diff --git a/judge/judge.go b/judge/judge.go index 1427c7e..ac58171 100644 --- a/judge/judge.go +++ b/judge/judge.go @@ -5,6 +5,8 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "fmt" + "math" "sort" "strings" "sync" @@ -111,12 +113,27 @@ func (j *Judge) ask(ctx context.Context, state interface{}, questions map[string continue // the request we waited on failed: try again, possibly as the leader } for k := range missing { - answers[k] = f.answers[keys[k]] + if a, ok := f.answers[keys[k]]; ok { + answers[k] = a + } } atomic.AddInt64(&j.CacheHits, 1) return answers, nil } got, err := j.Provider.Judge(ctx, state, missing) + if err == nil { + for k, q := range missing { + a, ok := got[k] + if !ok { + err = fmt.Errorf("judge: provider omitted answer %q", k) + break + } + if err = validateAnswer(q, a); err != nil { + err = fmt.Errorf("judge: answer %q: %w", k, err) + break + } + } + } var byKey map[string]Answer if err == nil { atomic.AddInt64(&j.Requests, 1) @@ -141,6 +158,32 @@ func (j *Judge) ask(ctx context.Context, state interface{}, questions map[string } } +func validateAnswer(q Question, a Answer) error { + validProbability := func(value float64) bool { + return !math.IsNaN(value) && !math.IsInf(value, 0) && value >= 0 && value <= 1 + } + switch q.Type { + case TypeBinary: + if !validProbability(a.Yes) { + return fmt.Errorf("invalid yes probability %v", a.Yes) + } + case TypeChoice: + if _, ok := q.Options[a.Choice]; !ok { + return fmt.Errorf("unknown choice %q", a.Choice) + } + if !validProbability(a.Confidence) { + return fmt.Errorf("invalid confidence %v", a.Confidence) + } + case TypeScore: + if !validProbability(a.Score) { + return fmt.Errorf("invalid score %v", a.Score) + } + default: + return fmt.Errorf("unknown question type %q", q.Type) + } + return nil +} + func (j *Judge) cached(key string, sig uint64) (Answer, bool) { var a Answer if j.Cache == nil { diff --git a/judge/judge_test.go b/judge/judge_test.go index 84cca55..8def9e0 100644 --- a/judge/judge_test.go +++ b/judge/judge_test.go @@ -60,9 +60,9 @@ type mock struct { func newMock() *mock { return &mock{ versionP: 0.95, - present: map[string]float64{"nginx": 0.1, "wordpress": 0.05, "jenkins": 0.97, "apache tomcat": 0.8, "Prototype": 0.9}, + present: map[string]float64{"nginx": 0.1, "wordpress": 0.05, "jenkins": 0.97, "apache tomcat": 0.8, "prototype": 0.9}, layer: map[string]Layer{"nginx": LayerServer, "wordpress": LayerNotPresent, "jenkins": LayerApplication, - "apache tomcat": LayerServer, "Prototype": LayerFrontend}, + "apache tomcat": LayerServer, "prototype": LayerFrontend}, primary: "jenkins", } } @@ -92,7 +92,7 @@ func (m *mock) Judge(ctx context.Context, state interface{}, questions map[strin var keys []string for k, q := range questions { keys = append(keys, k) - name := named(q) + name := strings.ToLower(named(q)) switch { case strings.HasPrefix(k, "is_"): answers[k] = Answer{Yes: m.present[name]} @@ -105,7 +105,16 @@ func (m *mock) Judge(ctx context.Context, state interface{}, questions map[strin case k == "generic": answers[k] = Answer{Yes: 0.9} case k == "version": - answers[k] = Answer{Choice: "2.401.3", Confidence: m.versionP} + choice := "2.401.3" + if _, ok := q.Options[choice]; !ok { + for candidate := range q.Options { + if candidate != notStated { + choice = candidate + break + } + } + } + answers[k] = Answer{Choice: choice, Confidence: m.versionP} default: answers[k] = Answer{Yes: 0.7} } @@ -133,7 +142,13 @@ func refine(t *testing.T, c *Judge, frames common.Frameworks) (*Page, error) { if err != nil { t.Fatal(err) } - return p, Refine(context.Background(), c, p, frames, []string{"Jenkins", "Prototype"}) + out, err := runRefine(context.Background(), c, p, frames, []string{"Jenkins", "Prototype"}) + if err == nil { + for name, f := range out { + frames[name] = f + } + } + return p, err } func TestRefine(t *testing.T) { @@ -328,7 +343,7 @@ func TestExtractVersionsPrefixes(t *testing.T) { for _, c := range extractVersions([]byte(raw), 10) { got = append(got, c.Value) } - if strings.Join(got, ",") != "3.4,8.1,3.7.1" { + if strings.Join(got, ",") != "3.4,8.1SP2,3.7.1" { t.Fatalf("got %v", got) } } @@ -387,11 +402,13 @@ func TestVersionWithoutPrimary(t *testing.T) { defer stop() frames := common.Frameworks{} frames.Add(common.NewFramework("nginx", common.FrameFromFingers)) - if _, err := refine(t, c, frames); err != nil { + p, _ := NewPage([]byte(jenkinsRaw)) + result, err := runRefine(context.Background(), c, p, frames, nil) + if err != nil { t.Fatal(err) } - if PrimaryOf(frames) != nil || frames["nginx"].Version == "" { - t.Fatalf("subject version not asked: %+v", frames["nginx"]) + if PrimaryOf(result) != nil || result["nginx"].Version == "" { + t.Fatalf("subject version not asked: %+v", result["nginx"]) } } diff --git a/judge/mark.go b/judge/mark.go index a73b3fc..dca6c98 100644 --- a/judge/mark.go +++ b/judge/mark.go @@ -69,7 +69,7 @@ func PrimaryOf(frames common.Frameworks) *common.Framework { func Accepted(frames common.Frameworks) common.Frameworks { out := common.Frameworks{} for k, f := range frames { - if !Is(f, Rejected) && !Is(f, Duplicate) { + if f != nil && !Is(f, Rejected) && !Is(f, Duplicate) { out[k] = f } } diff --git a/judge/operations.go b/judge/operations.go new file mode 100644 index 0000000..9065c39 --- /dev/null +++ b/judge/operations.go @@ -0,0 +1,130 @@ +package judge + +import ( + "context" + "fmt" + "math" + + "github.com/chainreactors/fingers/common" +) + +// Yes returns the probability that the answer to question is yes. State can +// be any JSON-serializable value, including a Page returned by NewPage. +func (j *Judge) Yes(ctx context.Context, state any, question string) (float64, error) { + answers, err := j.Ask(ctx, state, map[string]Question{"yes": Binary(question)}) + if err != nil { + return 0, err + } + return answers["yes"].Yes, nil +} + +// Choose selects one option and returns its confidence. +func (j *Judge) Choose(ctx context.Context, state any, question string, options map[string]string) (string, float64, error) { + if len(options) < 2 { + return "", 0, fmt.Errorf("judge: choose needs at least two options") + } + answers, err := j.Ask(ctx, state, map[string]Question{"choice": Choice(question, options)}) + if err != nil { + return "", 0, err + } + a := answers["choice"] + if _, ok := options[a.Choice]; !ok { + return "", 0, fmt.Errorf("judge: provider chose unknown option %q", a.Choice) + } + return a.Choice, a.Confidence, nil +} + +// Score returns a value from 0 to 1 on the ordered levels. +func (j *Judge) Score(ctx context.Context, state any, question string, levels ...string) (float64, error) { + if len(levels) < 2 || len(levels) > 10 { + return 0, fmt.Errorf("judge: score needs 2 to 10 levels") + } + answers, err := j.Ask(ctx, state, map[string]Question{"score": Score(question, levels)}) + if err != nil { + return 0, err + } + score := answers["score"].Score + if math.IsNaN(score) || score < 0 || score > 1 { + return 0, fmt.Errorf("judge: provider returned invalid score %v", score) + } + return score, nil +} + +// Inspect returns all rule hits with judgement marks, including rejected and +// duplicate hits. The input map and its frameworks are never changed. +func (j *Judge) Inspect(ctx context.Context, raw []byte, frames common.Frameworks, knownNames ...string) (common.Frameworks, error) { + p, err := NewPage(raw) + if err != nil { + return nil, err + } + working := cloneFrameworks(frames) + r := p.Round() + Verify(r, working, knownNames) + if err := r.Ask(ctx, j); err != nil { + return nil, err + } + return working, nil +} + +// Verify returns the accepted fingerprints after judging rule hits and +// optional known-name candidates. Use Inspect for rejected-hit diagnostics. +func (j *Judge) Verify(ctx context.Context, raw []byte, frames common.Frameworks, knownNames ...string) (common.Frameworks, error) { + all, err := j.Inspect(ctx, raw, frames, knownNames...) + if err != nil { + return nil, err + } + return Accepted(all), nil +} + +// Classify reports the page kind and whether it is a stock product page. +func (j *Judge) Classify(ctx context.Context, raw []byte) (Kind, bool, error) { + p, err := NewPage(raw) + if err != nil { + return "", false, err + } + r := p.Round() + Classify(r) + if err := r.Ask(ctx, j); err != nil { + return "", false, err + } + return p.Kind, p.Generic, nil +} + +// Version selects a version for f when the response contains a supported +// version string. It does not change f. +func (j *Judge) Version(ctx context.Context, raw []byte, f *common.Framework) (string, error) { + if f == nil { + return "", nil + } + if f.Attributes != nil && f.Version != "" { + return f.Version, nil + } + p, err := NewPage(raw) + if err != nil { + return "", err + } + copy := cloneFramework(f) + r := p.Round() + Version(r, copy) + if err := r.Ask(ctx, j); err != nil { + return "", err + } + if copy.Attributes == nil { + return "", nil + } + return copy.Version, nil +} + +// Refine verifies hits, classifies the page and resolves the primary product's +// version. Both rounds must succeed; input frames are never changed. +func (j *Judge) Refine(ctx context.Context, raw []byte, frames common.Frameworks, knownNames ...string) (common.Frameworks, Kind, bool, error) { + p, err := NewPage(raw) + if err != nil { + return nil, "", false, err + } + all, err := runRefine(ctx, j, p, frames, knownNames) + if err != nil { + return nil, "", false, err + } + return Accepted(all), p.Kind, p.Generic, nil +} diff --git a/judge/operations_test.go b/judge/operations_test.go new file mode 100644 index 0000000..8fbebb3 --- /dev/null +++ b/judge/operations_test.go @@ -0,0 +1,282 @@ +package judge + +import ( + "bytes" + "context" + "errors" + "fmt" + "strings" + "testing" + + "github.com/chainreactors/fingers/common" + "github.com/chainreactors/fingers/fingers" +) + +type answerProvider func(map[string]Question) map[string]Answer + +func (answerProvider) ID() string { return "answer-provider" } + +func (p answerProvider) Judge(_ context.Context, _ interface{}, questions map[string]Question) (map[string]Answer, error) { + return p(questions), nil +} + +type failVersionProvider struct{ *mock } + +func (p failVersionProvider) Judge(ctx context.Context, state interface{}, questions map[string]Question) (map[string]Answer, error) { + if _, ok := questions["version"]; ok { + return nil, errors.New("version unavailable") + } + return p.mock.Judge(ctx, state, questions) +} + +func TestTypedPrimitives(t *testing.T) { + j := New(answerProvider(func(qs map[string]Question) map[string]Answer { + out := map[string]Answer{} + for key, q := range qs { + switch q.Type { + case TypeBinary: + out[key] = Answer{Yes: 0.82} + case TypeChoice: + out[key] = Answer{Choice: "b", Confidence: 0.93} + case TypeScore: + out[key] = Answer{Score: 0.4} + } + } + return out + })) + state := map[string]string{"title": "Example"} + if p, err := j.Yes(context.Background(), state, "Is this a product?"); err != nil || p != 0.82 { + t.Fatalf("yes = %v, %v", p, err) + } + if choice, confidence, err := j.Choose(context.Background(), state, "Pick", map[string]string{"a": "A", "b": "B"}); err != nil || choice != "b" || confidence != 0.93 { + t.Fatalf("choose = %q %v, %v", choice, confidence, err) + } + if score, err := j.Score(context.Background(), state, "Rate", "low", "high"); err != nil || score != 0.4 { + t.Fatalf("score = %v, %v", score, err) + } + if _, _, err := j.Choose(context.Background(), state, "Pick", map[string]string{"one": "only"}); err == nil { + t.Fatal("accepted a choice with one option") + } + if _, err := New(answerProvider(func(map[string]Question) map[string]Answer { return nil })).Yes(context.Background(), state, "Missing"); err == nil { + t.Fatal("missing provider answer was accepted") + } + if _, _, err := New(answerProvider(func(map[string]Question) map[string]Answer { + return map[string]Answer{"choice": {Choice: "outside", Confidence: 0.9}} + })).Choose(context.Background(), state, "Pick", map[string]string{"a": "A", "b": "B"}); err == nil { + t.Fatal("unknown provider choice was accepted") + } +} + +func TestPublicResultsDoNotMutateHits(t *testing.T) { + j := New(newMock()) + frames := testFrames() + inputJenkins := frames["jenkins"] + accepted, err := j.Verify(context.Background(), []byte(jenkinsRaw), frames, "Prototype") + if err != nil { + t.Fatal(err) + } + if accepted["wordpress"] != nil || accepted["jenkins"] == nil || Judged(inputJenkins) { + t.Fatalf("verify result/input: %v / %v", accepted, frames) + } + inspected, err := j.Inspect(context.Background(), []byte(jenkinsRaw), frames, "Prototype") + if err != nil || !Is(inspected["wordpress"], Rejected) || Judged(frames["wordpress"]) { + t.Fatalf("inspect result/input: %v / %v / %v", inspected, frames, err) + } + refined, kind, generic, err := j.Refine(context.Background(), []byte(jenkinsRaw), frames, "Prototype") + if err != nil || kind != KindLogin || !generic || refined["jenkins"].Version != "2.401.3" { + t.Fatalf("refine = %v %q %v, %v", refined, kind, generic, err) + } + if frames["jenkins"] != inputJenkins || inputJenkins.Version != "" || Judged(inputJenkins) { + t.Fatal("Refine changed a rule hit") + } + version, err := j.Version(context.Background(), []byte(jenkinsRaw), inputJenkins) + if err != nil || version != "2.401.3" || inputJenkins.Version != "" { + t.Fatalf("version = %q, %v; input = %q", version, err, inputJenkins.Version) + } +} + +func TestRefineSecondRoundFailureIsAtomic(t *testing.T) { + frames := testFrames() + j := New(failVersionProvider{newMock()}) + accepted, kind, generic, err := j.Refine(context.Background(), []byte(jenkinsRaw), frames) + if err == nil || accepted != nil || kind != "" || generic { + t.Fatalf("failed refinement = %v %q %v, %v", accepted, kind, generic, err) + } + for _, f := range frames { + if Judged(f) || f.Version != "" { + t.Fatalf("input changed after second round failed: %+v", f) + } + } +} + +func TestUnknownAndBodyNameSuggestion(t *testing.T) { + raw := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nSign in

QuantumGate Portal account access

") + j := New(answerProvider(func(qs map[string]Question) map[string]Answer { + out := map[string]Answer{} + for key, q := range qs { + switch key { + case "page_kind": + out[key] = Answer{Choice: string(KindLogin)} + case "generic": + out[key] = Answer{Yes: 0.9} + default: + if strings.Contains(q.Instructions, "`QuantumGate Portal`") { + out[key] = Answer{Yes: 0.95} + } else { + out[key] = Answer{Yes: 0.05} + } + } + } + return out + })) + names, err := j.SuggestNames(context.Background(), raw) + if err != nil || len(names) != 1 || names[0] != "QuantumGate Portal" { + t.Fatalf("body suggestions = %v, %v", names, err) + } + unknown, err := j.IsUnknownProduct(context.Background(), raw, nil) + if err != nil || !unknown { + t.Fatalf("unknown = %v, %v", unknown, err) + } + known := common.Frameworks{} + f := common.NewFramework("QuantumGate Portal", common.FrameFromGUESS) + setLayer(f, LayerApplication) + known.Add(f) + unknown, err = j.IsUnknownProduct(context.Background(), raw, known) + if err != nil || unknown { + t.Fatalf("known application = %v, %v", unknown, err) + } +} + +func TestKnownCDNErrorPageIsNotUnknown(t *testing.T) { + raw := []byte("HTTP/1.1 403 Forbidden\r\nServer: cloudflare\r\n\r\nJust a moment...") + j := New(answerProvider(func(qs map[string]Question) map[string]Answer { + out := map[string]Answer{} + for key := range qs { + if key == "page_kind" { + out[key] = Answer{Choice: string(KindError)} + } else { + out[key] = Answer{Yes: 0.95} + } + } + return out + })) + f := common.NewFramework("cloudflare", common.FrameFromFingers) + setLayer(f, LayerCDN) + accepted := common.Frameworks{} + accepted.Add(f) + unknown, err := j.IsUnknownProduct(context.Background(), raw, accepted) + if err != nil || unknown { + t.Fatalf("known CDN error = %t, %v", unknown, err) + } +} + +func TestGeneratorPassiveVersionAndValidation(t *testing.T) { + positive1 := []byte("HTTP/1.1 200 OK\r\nServer: nginx/2.401.3\r\nX-Jenkins: 2.401.3\r\n\r\nJenkins") + positive2 := []byte("HTTP/1.1 200 OK\r\nServer: nginx/2.402.1\r\nX-Jenkins: 2.402.1\r\n\r\nJenkins") + negative := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nBlog

X-Jenkins: 9.9.9 release notes

") + g := NewGenerator(nil).Name("Jenkins").PositiveVersion(positive1, "2.401.3").PositiveVersion(positive2, "2.402.1").Negative(negative) + f, err := g.Generate(context.Background()) + if err != nil { + t.Fatal(err) + } + if f.Name != "Jenkins" || len(f.Rules) < 2 { + t.Fatalf("fingerprint: %+v", f) + } + for _, sample := range []struct { + raw []byte + version string + }{{positive1, "2.401.3"}, {positive2, "2.402.1"}} { + frame, _, ok := f.PassiveMatch(fingers.NewContent(sample.raw, "", true)) + if !ok || frame.Version != sample.version { + t.Fatalf("version matcher = %+v, %v", frame, ok) + } + } + compiled := len(f.Rules[0].Regexps.CompliedRegexp) + if err := g.Validate(f); err != nil || len(f.Rules[0].Regexps.CompliedRegexp) != compiled { + t.Fatalf("Validate mutated compiled matcher: %v", err) + } + if err := g.Validate(&fingers.Finger{Name: "Jenkins", Rules: fingers.Rules{&fingers.Rule{Regexps: &fingers.Regexps{Body: []string{"jenkins"}}}}}); err == nil { + t.Fatal("validator accepted rule matching negative sample") + } +} + +func TestGeneratorActiveAndProbeWith(t *testing.T) { + base := []byte("HTTP/1.1 200 OK\r\n\r\nLogin") + request := []byte("/admin") + positive := []byte("HTTP/1.1 200 OK\r\nX-Orion: console\r\n\r\nOrion console") + negative := []byte("HTTP/1.1 404 Not Found\r\n\r\nmissing") + g := NewGenerator(nil).Name("Orion").Positive(base) + if err := g.ProbeWith(context.Background(), request, func(_ context.Context, got []byte) ([]byte, error) { + if !bytes.Equal(got, request) { + return nil, errors.New("wrong request") + } + return positive, nil + }); err != nil { + t.Fatal(err) + } + g.Negative(base).Probe(request, negative) + f, err := g.Generate(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(f.Rules) != 1 || f.Rules[0].SendDataStr != string(request) || !f.IsActive { + t.Fatalf("active rule: %+v", f) + } + if err := g.Validate(f); err != nil { + t.Fatal(err) + } + if len(f.Rules[0].Regexps.CompliedRegexp) != 0 { + t.Fatal("Validate changed compiled rules") + } + if _, err := NewGenerator(nil).Name("Orion").Positive(base).Probe(request, positive).Negative(base).Generate(context.Background()); err == nil { + t.Fatal("generated active rule without a negative probe") + } +} + +func TestGeneratorSelectsNameByDefault(t *testing.T) { + positive := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nOrion

Orion console

") + negative := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nOther") + j := New(answerProvider(func(qs map[string]Question) map[string]Answer { + out := map[string]Answer{} + for key, q := range qs { + if strings.Contains(q.Instructions, "`Orion`") { + out[key] = Answer{Yes: 0.95} + } else { + out[key] = Answer{Yes: 0.05} + } + } + return out + })) + f, err := NewGenerator(j).Positive(positive).Negative(negative).Generate(context.Background()) + if err != nil || f.Name != "Orion" { + t.Fatalf("auto name = %+v, %v", f, err) + } +} + +func TestGeneratorActiveVersion(t *testing.T) { + base := []byte("HTTP/1.1 200 OK\r\n\r\nLogin") + request := []byte("/version") + positive := []byte("HTTP/1.1 200 OK\r\nX-Orion: 3.1.2\r\n\r\nversion") + negative := []byte("HTTP/1.1 200 OK\r\n\r\nnot installed") + g := NewGenerator(nil).Name("Orion").PositiveVersion(base, "3.1.2").Probe(request, positive).Negative(base).Probe(request, negative) + f, err := g.Generate(context.Background()) + if err != nil { + t.Fatal(err) + } + frame, _, ok := f.ActiveMatch(2, func(got []byte) ([]byte, bool) { return positive, bytes.Equal(got, request) }) + if !ok || frame.Version != "3.1.2" { + t.Fatalf("active version = %+v, %v", frame, ok) + } +} + +func TestGeneratorCoversDifferentPositiveVariants(t *testing.T) { + base := "HTTP/1.1 200 OK\r\n%s\r\nWelcome" + first := []byte(fmt.Sprintf(base, "X-Orion-Instance: ready\r\n")) + second := []byte(fmt.Sprintf(base, "X-Orion-Service: yes\r\n")) + other := []byte(fmt.Sprintf(base, "Server: nginx\r\n")) + g := NewGenerator(nil).Name("Orion").Positive(first).Positive(second).Negative(other) + f, err := g.Generate(context.Background()) + if err != nil || len(f.Rules) != 2 { + t.Fatalf("variant rules = %+v, %v", f, err) + } +} diff --git a/judge/page.go b/judge/page.go index 284fe35..319ed51 100644 --- a/judge/page.go +++ b/judge/page.go @@ -23,6 +23,7 @@ type Page struct { Cookies []string `json:"cookie_names,omitempty"` Title string `json:"title,omitempty"` Generator string `json:"meta_generator,omitempty"` + Description string `json:"meta_description,omitempty"` Scripts []string `json:"script_src,omitempty"` Styles []string `json:"stylesheet_href,omitempty"` InlineHints []string `json:"inline_script_starts,omitempty"` // first chars of inline scripts, e.g. "(function(w,d,s,l,i){ ... GTM" @@ -47,24 +48,28 @@ var ( // per-request values: no signal, and they would defeat the cache "X-Request-Id": true, "X-Amzn-Trace-Id": true, "Cf-Ray": true, "X-Runtime": true, "Server-Timing": true} - reTitle = regexp.MustCompile(`(?is)]*>(.*?)`) - reGenerator = regexp.MustCompile(`(?is)]+name=["']generator["'][^>]*content=["']([^"']+)|]+content=["']([^"']+)["'][^>]*name=["']generator["']`) - reScript = regexp.MustCompile(`(?is)]+src=["']([^"']+)`) - reStyle = regexp.MustCompile(`(?is)]+rel=["']?stylesheet[^>]*href=["']([^"']+)|]+href=["']([^"']+)["'][^>]*rel=["']?stylesheet`) - reInline = regexp.MustCompile(`(?is)]*)?>(.*?)`) - reComment = regexp.MustCompile(`(?s)`) - reInput = regexp.MustCompile(`(?is)]*>`) - reAttrName = regexp.MustCompile(`(?is)\b(?:name|id)=["']([^"']+)`) - reAttrType = regexp.MustCompile(`(?is)\btype=["']([^"']+)`) - reDropBlock = regexp.MustCompile(`(?is)<(script|style|noscript)[^>]*>.*?`) - reTag = regexp.MustCompile(`(?s)<[^>]+>`) - reSpace = regexp.MustCompile(`\s+`) + reTitle = regexp.MustCompile(`(?is)]*>(.*?)`) + reGenerator = regexp.MustCompile(`(?is)]+name=["']generator["'][^>]*content=["']([^"']+)|]+content=["']([^"']+)["'][^>]*name=["']generator["']`) + reDescription = regexp.MustCompile(`(?is)]+name=["']description["'][^>]*content=["']([^"']+)|]+content=["']([^"']+)["'][^>]*name=["']description["']`) + reScript = regexp.MustCompile(`(?is)]+src=["']([^"']+)`) + reStyle = regexp.MustCompile(`(?is)]+rel=["']?stylesheet[^>]*href=["']([^"']+)|]+href=["']([^"']+)["'][^>]*rel=["']?stylesheet`) + reInline = regexp.MustCompile(`(?is)]*)?>(.*?)`) + reComment = regexp.MustCompile(`(?s)`) + reInput = regexp.MustCompile(`(?is)]*>`) + reAttrName = regexp.MustCompile(`(?is)\b(?:name|id)=["']([^"']+)`) + reAttrType = regexp.MustCompile(`(?is)\btype=["']([^"']+)`) + reDropBlock = regexp.MustCompile(`(?is)<(script|style|noscript)[^>]*>.*?`) + reTag = regexp.MustCompile(`(?s)<[^>]+>`) + reSpace = regexp.MustCompile(`\s+`) // Allows a v/V/x/X prefix ("X3.4", "V8.1SP2") and a letter suffix; rejects // digits glued to other digits or dots so IPs and long builds stay out. reGenMajor = regexp.MustCompile(`(?i)generator["'][^>]*content=["'][A-Za-z][^"']*?\s[vV]?(\d{1,3})(?:[\s"'(]|$)|content=["'][A-Za-z][^"']*?\s[vV]?(\d{1,3})(?:[\s(][^"']*)?["'][^>]*name=["']generator`) - reVersion = regexp.MustCompile(`(?:^|[^0-9A-Za-z.])[vVxX]?(\d{1,3}\.\d{1,3}(?:\.\d{1,4}){0,2})(?:[^0-9.]|$)`) + reVersion = regexp.MustCompile(`(?:^|[^0-9A-Za-z.])[vVxX]?(` + versionToken + `)(?:[^0-9A-Za-z.]|\.[A-Za-z]|$)`) ) +// Keep release suffixes and calendar versions (e.g. 2026.9.23+3cd69d30e). +const versionToken = `[0-9]{1,4}(?:\.[0-9]{1,4}){1,3}(?:[A-Za-z]+[0-9]*|[-+][0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*)?` + const ( maxHeaderValue = 160 maxText = 1500 // visible text runes @@ -95,6 +100,9 @@ func NewPage(raw []byte) (*Page, error) { if m := reGenerator.FindStringSubmatch(html); m != nil { s.Generator = m[1] + m[2] } + if m := reDescription.FindStringSubmatch(html); m != nil { + s.Description = truncate(clean(m[1]+m[2]), 300) + } for _, m := range reScript.FindAllStringSubmatch(html, 8) { s.Scripts = append(s.Scripts, m[1]) } @@ -136,6 +144,7 @@ func (s *Page) Haystack() string { } b.WriteString(strings.Join(s.Cookies, " ") + "\n") b.WriteString(s.Title + "\n" + s.Generator + "\n") + b.WriteString(s.Description + "\n") b.WriteString(strings.Join(s.Scripts, " ") + "\n") b.WriteString(strings.Join(s.Styles, " ") + "\n") b.WriteString(strings.Join(s.InlineHints, " ") + "\n") @@ -178,6 +187,7 @@ func (s *Page) Evidence(name string) []string { check("cookie", strings.Join(s.Cookies, " ")) check("title", s.Title) check("meta", s.Generator) + check("description", s.Description) check("script", strings.Join(s.Scripts, " ")+" "+strings.Join(s.InlineHints, " ")) check("style", strings.Join(s.Styles, " ")) check("comment", strings.Join(s.Comments, " ")) @@ -230,6 +240,7 @@ func (s *Page) Signature() uint64 { } } add("status", s.Status, 3) + add("description", s.Description, 3) for k, val := range s.Headers { add("header", k+": "+val, 3) } diff --git a/judge/provider.go b/judge/provider.go index 48a539a..2be13dc 100644 --- a/judge/provider.go +++ b/judge/provider.go @@ -12,7 +12,7 @@ type Provider interface { // change it whenever the same question may be answered differently. ID() string // Judge answers every question in one call, keyed as questions are. - // A missing answer is treated as no answer, not as an error. + // Every requested key must have an answer; missing answers fail the round. Judge(ctx context.Context, state interface{}, questions map[string]Question) (map[string]Answer, error) } diff --git a/judge/real_evidence_test.go b/judge/real_evidence_test.go new file mode 100644 index 0000000..335ec8f --- /dev/null +++ b/judge/real_evidence_test.go @@ -0,0 +1,183 @@ +package judge + +import ( + "context" + "github.com/chainreactors/fingers/common" + fingerlib "github.com/chainreactors/fingers/fingers" + "strings" + "testing" +) + +// Minimal public evidence retained from the 2026-09-25 corpus, not full pages. +func TestStaticApplicationDescriptionIsPreservedAsEvidence(t *testing.T) { + description := "IT Tools is a free and open-source collection of handy online tools for developers." + for _, meta := range []string{ + ``, + ``, + } { + p, err := NewPage([]byte("HTTP/1.1 200 OK\r\n\r\nIT Tools - Handy online tools for developers" + meta)) + if err != nil || p.Description != description || !strings.Contains(p.Haystack(), strings.ToLower(description)) { + t.Fatalf("description evidence missing: %+v err=%v", p, err) + } + if strings.Contains(strings.Join(p.Evidence("it tools"), ","), "header") { + t.Fatal("description incorrectly treated as deterministic header evidence") + } + } +} + +func TestVersionCandidatesFromRealResponseEvidence(t *testing.T) { + raw := []byte("HTTP/1.1 200 OK\r\nServer: Apache/2.4.38 OpenSSL/1.0.2q PHP/5.6.40\r\nX-New-Api-Version: v1.0.0-rc.35\r\n\r\n") + got := map[string]bool{} + for _, c := range extractVersions(raw, 40) { + got[c.Value] = true + } + for _, want := range []string{"2.4.38", "1.0.2q", "5.6.40", "1.0.0-rc.35", "3.6.3"} { + if !got[want] { + t.Errorf("missing %s in %v", want, got) + } + } + if got["1.0.2"] { + t.Fatal("truncated OpenSSL version") + } +} +func TestVersionCandidatesDoNotDiscardLateProductEvidence(t *testing.T) { + var b strings.Builder + b.WriteString("HTTP/1.1 200 OK\r\n\r\n") + for i := 0; i < 80; i++ { + b.WriteString(" " + string(rune('a'+i%26)) + " ") + b.WriteString(strings.Repeat("1", i/10+1) + ".25 ") + } + // Add distinct numbers to exceed the shortlist. + for _, n := range []string{"10.11", "11.12", "12.13", "13.14", "14.15", "15.16", "16.17", "17.18", "18.19", "19.20"} { + b.WriteString(n + " ") + } + b.WriteString("") + found := false + for _, c := range extractVersions([]byte(b.String()), 5) { + found = found || c.Value == "3.7.1" + } + if !found { + t.Fatal("early numeric noise evicted late library version") + } +} +func TestProtocolHitRequiresResponseEvidence(t *testing.T) { + for _, tc := range []struct { + name, header string + yes bool + want bool + }{{"http基本认证", "", true, false}, {"hsts", "", true, false}, {"http基本认证", "WWW-Authenticate: Basic realm=\"test\"\r\n", false, true}, {"hsts", "Strict-Transport-Security: max-age=31536000\r\n", false, true}} { + j := New(answerProvider(func(qs map[string]Question) map[string]Answer { + out := map[string]Answer{} + for k, q := range qs { + if q.Type == TypeBinary { + v := .01 + if tc.yes { + v = .99 + } + out[k] = Answer{Yes: v} + } else if k == "primary" { + out[k] = Answer{Choice: noneOfThem} + } else { + out[k] = Answer{Choice: string(LayerNotPresent)} + } + } + return out + })) + hits := common.Frameworks{} + hits.Add(common.NewFramework(tc.name, common.FrameFromGUESS)) + got, err := j.Verify(context.Background(), []byte("HTTP/1.1 200 OK\r\n"+tc.header+"\r\n"), hits) + if err != nil || (len(got) > 0) != tc.want { + t.Errorf("%s evidence=%q got=%v err=%v", tc.name, tc.header, got, err) + } + } +} +func TestGeneratedVersionHandlesHyphenatedHeaderAndReleaseSuffix(t *testing.T) { + raw := func(v string) []byte { + return []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\nX-New-Api-Version: v" + v + "\r\n\r\nNew API") + } + neg := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nWelcome") + g := NewGenerator(nil).Name("New API").PositiveVersion(raw("1.0.0-rc.35"), "1.0.0-rc.35").Negative(neg) + f, err := g.Generate(context.Background()) + if err != nil { + t.Fatal(err) + } + for _, version := range []string{"1.0.0-rc.35", "1.2.0-rc.2", "2.0.1"} { + frame, _, matched := f.PassiveMatch(fingerlib.NewContent(raw(version), "", true)) + if !matched || frame.Version != version { + t.Fatalf("version=%s matched=%t frame=%+v", version, matched, frame) + } + } +} + +func TestCalendarVersionAndGeneratorName(t *testing.T) { + raw := []byte("HTTP/1.1 200 OK\r\n\r\nCustom Search") + p, err := NewPage(raw) + if err != nil { + t.Fatal(err) + } + foundName, foundVersion := false, false + for _, name := range pageNames(p) { + foundName = foundName || NormalizeName(name) == "searxng" + } + for _, v := range extractVersions(raw, 40) { + foundVersion = foundVersion || v.Value == "2026.9.23+3cd69d30e" + } + if !foundName || !foundVersion { + t.Fatalf("name=%t version=%t", foundName, foundVersion) + } +} + +func TestGeneratedMetaRuleSurvivesTitleAndCalendarVersionChanges(t *testing.T) { + page := func(title, version string) []byte { + return []byte("HTTP/1.1 200 OK\r\n\r\n" + title + "") + } + negative := []byte("HTTP/1.1 200 OK\r\n\r\nSearXNG Documentation

Install SearXNG

") + f, err := NewGenerator(nil).Name("SearXNG").PositiveVersion(page("Search One", "2026.9.23+3cd69d30e"), "2026.9.23+3cd69d30e").Negative(negative).Generate(context.Background()) + if err != nil { + t.Fatal(err) + } + got, _, matched := f.PassiveMatch(fingerlib.NewContent(page("Different Search", "2026.9.25+d8ae3abd5"), "", true)) + if !matched || got.Version != "2026.9.25+d8ae3abd5" { + t.Fatalf("matched=%t got=%+v", matched, got) + } + if _, _, matched := f.PassiveMatch(fingerlib.NewContent(negative, "", true)); matched { + t.Fatal("documentation matched") + } +} + +func TestGeneratedFooterVersionRequiresProductDeclaration(t *testing.T) { + page := func(title, version string) []byte { + return []byte("HTTP/1.1 200 OK\r\n\r\n" + title + "
Log in
\n v" + version + " @ mysql\n
") + } + negative := page("Different Application", "2.18.0-be6aeb4") + f, err := NewGenerator(nil).Name("Wakapi").PositiveVersion(page("Wakapi – Coding Statistics", "2.18.0-be6aeb4"), "2.18.0-be6aeb4").Negative(negative).Generate(context.Background()) + if err != nil { + t.Fatal(err) + } + got, _, matched := f.PassiveMatch(fingerlib.NewContent(page("Wakapi – Coding Statistics", "2.12.0"), "", true)) + if !matched || got.Version != "2.12.0" { + t.Fatalf("matched=%t got=%+v", matched, got) + } + if _, _, matched := f.PassiveMatch(fingerlib.NewContent(negative, "", true)); matched { + t.Fatal("unguarded footer matched another product") + } +} + +func TestGeneratedApplicationDoesNotLearnSharedInfrastructure(t *testing.T) { + for _, name := range []string{"IT Tools", "Docmost"} { + positive := []byte("HTTP/1.1 200 OK\r\nServer: cloudflare\r\nX-Frame-Options: SAMEORIGIN\r\n\r\n" + name + "") + negative := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nDifferent Application") + f, err := NewGenerator(nil).Name(name).Positive(positive).Negative(negative).Generate(context.Background()) + if err != nil { + t.Fatal(err) + } + sameProxy := []byte("HTTP/1.1 200 OK\r\nServer: cloudflare\r\nX-Frame-Options: SAMEORIGIN\r\n\r\nUnrelated Application") + if _, _, matched := f.PassiveMatch(fingerlib.NewContent(sameProxy, "", true)); matched { + t.Fatalf("%s learned shared infrastructure", name) + } + otherProxy := []byte("HTTP/1.1 200 OK\r\nServer: caddy\r\n\r\n" + name + "") + if _, _, matched := f.PassiveMatch(fingerlib.NewContent(otherProxy, "", true)); !matched { + t.Fatalf("%s requires original infrastructure", name) + } + } +} diff --git a/judge/refine.go b/judge/refine.go index f206206..20da8b0 100644 --- a/judge/refine.go +++ b/judge/refine.go @@ -6,20 +6,50 @@ import ( "github.com/chainreactors/fingers/common" ) -// Refine is the default pipeline: Verify and Classify in one request, then -// the version of the page's subject in a second one. frames and p are -// annotated in place; on error the second round's annotations are missing, -// the first round's are complete or absent. -func Refine(ctx context.Context, j *Judge, p *Page, frames common.Frameworks, recall []string) error { - r := p.Round() - Verify(r, frames, recall) +// refine keeps the first round's output private until version resolution ends. +func runRefine(ctx context.Context, j *Judge, p *Page, frames common.Frameworks, recall []string) (common.Frameworks, error) { + working := cloneFrameworks(frames) + page := *p + r := page.Round() + Verify(r, working, recall) Classify(r) if err := r.Ask(ctx, j); err != nil { - return err + return nil, err } - r = p.Round() - Version(r, subject(frames)) - return r.Ask(ctx, j) + r = page.Round() + Version(r, subject(working)) + if err := r.Ask(ctx, j); err != nil { + return nil, err + } + p.Kind, p.Generic = page.Kind, page.Generic + return working, nil +} + +func cloneFrameworks(frames common.Frameworks) common.Frameworks { + working := make(common.Frameworks, len(frames)) + for name, f := range frames { + working[name] = cloneFramework(f) + } + return working +} + +func cloneFramework(f *common.Framework) *common.Framework { + if f == nil { + return nil + } + copy := *f + copy.Tags = append([]string(nil), f.Tags...) + if f.Froms != nil { + copy.Froms = make(map[common.From]bool, len(f.Froms)) + for from, present := range f.Froms { + copy.Froms[from] = present + } + } + if f.Attributes != nil { + attrs := *f.Attributes + copy.Attributes = &attrs + } + return © } // subject is the product whose version the page most likely shows: the diff --git a/judge/stack.go b/judge/stack.go index 6f18bd8..f3bf3a2 100644 --- a/judge/stack.go +++ b/judge/stack.go @@ -28,7 +28,7 @@ var layerCriteria = map[string]string{ string(LayerServer): "The HTTP server or proxy software, such as nginx, Apache httpd, IIS, Tomcat", string(LayerRuntime): "A programming language or runtime, such as PHP, Java, ASP.NET, Node.js", string(LayerFramework): "A backend web framework, such as Spring, Django, Laravel, ThinkPHP", - string(LayerApplication): "The application or product the page belongs to: CMS, OA, admin console, SaaS app, forum, monitoring tool", + string(LayerApplication): "The application whose interface this is: CMS, OA, admin console, SaaS app, forum, monitoring tool, search engine, or packaged static/browser-side tools; a documentation subject is not the application serving the docs", string(LayerFrontend): "A browser-side library, UI kit, analytics or tag script, such as jQuery, Bootstrap, Google Analytics", string(LayerDevice): "An operating system or the firmware of a hardware device such as a router, camera, NAS or firewall", string(LayerNotPresent): "Not part of the software serving this response: only mentioned in text, or absent", diff --git a/judge/verify.go b/judge/verify.go index fe89d74..f0d583a 100644 --- a/judge/verify.go +++ b/judge/verify.go @@ -3,6 +3,7 @@ package judge import ( "fmt" "sort" + "strings" "github.com/chainreactors/fingers/common" ) @@ -10,7 +11,7 @@ import ( const noneOfThem = "none_of_these" // primary option for pages of no listed product // protocolFeatures are HTTP features some engines report as fingerprints -// (NormalizeName keys). They are facts of the response head: never rejected. +// (NormalizeName keys). They must be established by this response's head. var protocolFeatures = map[string]bool{"hsts": true, "altsvc": true, "http3": true, "http2": true, "poweredby": true, "http基本认证": true} const ( @@ -21,9 +22,10 @@ const ( // product is one product claimed for the page: every engine spelling of it // (frames), or a recall name without a rule hit. type product struct { - name string - frames []*common.Framework - strong bool // named in a header or cookie: code accepts it, models under-trust it + name string + frames []*common.Framework + strong bool // named in a header or cookie: code accepts it, models under-trust it + feature bool // protocol features have deterministic response-head evidence } // Verify is the engine result filter. It folds the spellings of all engines @@ -42,6 +44,9 @@ func Verify(r *Round, frames common.Frameworks, recall []string) { var list []*common.Framework judged := map[string]bool{} // keys judged before: not asked again, not recalled again for _, f := range frames { + if f == nil { + continue + } if Judged(f) { judged[NormalizeName(f.Name)] = true } else { @@ -61,15 +66,30 @@ func Verify(r *Round, frames common.Frameworks, recall []string) { if len(products) >= maxGroups { continue } - p = &product{name: f.Name, strong: protocolFeatures[key]} + p = &product{name: f.Name, feature: protocolFeatures[key]} + if p.feature { + p.strong = protocolPresent(page, key) + } byKey[key] = p products = append(products, p) } p.frames = append(p.frames, f) for _, e := range page.Evidence(f.Name) { - p.strong = p.strong || e == "header" || e == "cookie" + if !p.feature { + p.strong = p.strong || e == "header" || e == "cookie" + } } } + for _, p := range products { + sort.SliceStable(p.frames, func(a, b int) bool { + qa, qb := frameworkQuality(p.frames[a]), frameworkQuality(p.frames[b]) + if qa != qb { + return qa > qb + } + return p.frames[a].Name < p.frames[b].Name + }) + p.name = p.frames[0].Name + } recalled := 0 for _, name := range recall { key := NormalizeName(name) @@ -100,7 +120,9 @@ func Verify(r *Round, frames common.Frameworks, recall []string) { i, name := i, p.name primaryOpts[name] = "" r.Add(fmt.Sprintf("is_%d", i), BinaryWith(fmt.Sprintf("Is `%s` part of the software stack that produced this HTTP response "+ - "(seen in its headers, cookies, asset paths, page structure or title), rather than only mentioned in the page text?", name), + "(seen in its headers, cookies, asset paths, page structure or title), rather than only mentioned in the page text? "+ + "A packaged browser-side or static web application counts even without a login or server backend. "+ + "On documentation, README or tutorial pages, the product being documented does not count as running; distinguish it from the documentation generator.", name), name+" served, generated or is loaded by this response", name+" only appears in the page's text, or is not present"), func(a Answer) { answers[i].present, answers[i].answered = a.Yes, true }) @@ -118,7 +140,7 @@ func Verify(r *Round, frames common.Frameworks, recall []string) { r.Done(func() { for i, p := range products { a := answers[i] - rejected := !p.strong && a.answered && (a.layer == LayerNotPresent || a.present < r.judge.Threshold) + rejected := !p.strong && (p.feature || a.answered && (a.layer == LayerNotPresent || a.present < r.judge.Threshold)) if p.frames == nil { // recall: added only when the provider confirms it if !a.answered || rejected { continue @@ -143,3 +165,44 @@ func Verify(r *Round, frames common.Frameworks, recall []string) { } }) } + +func protocolPresent(p *Page, key string) bool { + switch key { + case "hsts": + return p.Headers["Strict-Transport-Security"] != "" + case "altsvc": + return p.Headers["Alt-Svc"] != "" + case "poweredby": + return p.Headers["X-Powered-By"] != "" + case "http2": + return strings.HasPrefix(string(p.raw), "HTTP/2") || strings.Contains(strings.ToLower(p.Headers["Alt-Svc"]), "h2=") + case "http3": + return strings.HasPrefix(string(p.raw), "HTTP/3") || strings.Contains(strings.ToLower(p.Headers["Alt-Svc"]), "h3=") + case "http基本认证": + for _, challenge := range strings.Split(p.Headers["Www-Authenticate"], ",") { + words := strings.Fields(challenge) + if len(words) > 0 && strings.EqualFold(words[0], "Basic") { + return true + } + } + } + return false +} + +// Prefer a representative that already carries a version and useful source +// metadata, since duplicate spellings are hidden from the accepted result. +func frameworkQuality(f *common.Framework) int { + quality := len(f.Froms) + if f.Attributes != nil { + if f.Version != "" { + quality += 100 + } + if f.Vendor != "" { + quality += 10 + } + } + if f.MatchDetail != nil { + quality++ + } + return quality +} diff --git a/judge/version.go b/judge/version.go index af5e055..990eada 100644 --- a/judge/version.go +++ b/judge/version.go @@ -92,9 +92,6 @@ func extractVersions(raw []byte, max int) []versionString { } continue } - if len(out) >= max { - continue - } seen[v] = len(out) out = append(out, versionString{Value: v, Context: ctx}) } @@ -104,7 +101,7 @@ func extractVersions(raw []byte, max int) []versionString { if lo < 0 { lo, hi = m[4], m[5] } - if v := text[lo:hi]; len(out) < max { + if v := text[lo:hi]; v != "" { if _, ok := seen[v]; ok { continue } @@ -112,6 +109,12 @@ func extractVersions(raw []byte, max int) []versionString { out = append(out, versionString{Value: v, Context: clean(strings.ToValidUTF8(text[m[0]:m[1]], ""))}) } } + // Select evidence after scanning: SVG numbers or early library assets must + // not crowd later generator/header/script versions out of the shortlist. + if len(out) > max { + sort.SliceStable(out, func(i, j int) bool { return contextWeight(out[i].Context) > contextWeight(out[j].Context) }) + out = out[:max] + } return out } @@ -121,9 +124,11 @@ func contextWeight(ctx string) int { c := strings.ToLower(ctx) switch { case strings.Contains(c, "generator"): - return 3 + return 5 case strings.Contains(c, "server:") || strings.Contains(c, "x-powered-by:") || strings.Contains(c, "product:"): - return 2 + return 4 + case strings.Contains(c, "Welcome to nginx!") frames, _ := engine.DetectContent(raw) - if _, err := engine.Refine(context.Background(), raw, frames); err != ErrNoJudge { - t.Fatalf("want ErrNoJudge, got %v", err) - } - - engine.AttachJudge(judge.New(server{})) - page, err := engine.Refine(context.Background(), raw, frames) + j := judge.New(server{}) + accepted, kind, _, err := j.Refine(context.Background(), raw, frames) if err != nil { t.Fatal(err) } - if page.Kind != judge.KindDefault || judge.LayerOf(frames["nginx"]) != judge.LayerServer { - t.Fatalf("kind %q, nginx %v", page.Kind, frames["nginx"].Tags) + if kind != judge.KindDefault || judge.LayerOf(accepted["nginx"]) != judge.LayerServer || judge.Judged(frames["nginx"]) { + t.Fatalf("kind %q, accepted %v, input %v", kind, accepted["nginx"], frames["nginx"]) } } @@ -42,6 +38,8 @@ func (server) Judge(ctx context.Context, state interface{}, questions map[string answers[k] = judge.Answer{Choice: string(judge.LayerServer)} case k == "page_kind": answers[k] = judge.Answer{Choice: string(judge.KindDefault)} + case k == "primary": + answers[k] = judge.Answer{Choice: "none_of_these"} default: answers[k] = judge.Answer{Yes: 0.9, Choice: "not_stated"} } diff --git a/scripts/capture_judge.py b/scripts/capture_judge.py new file mode 100644 index 0000000..bc454f8 --- /dev/null +++ b/scripts/capture_judge.py @@ -0,0 +1,88 @@ +"""Capture explicit URLs as a repeatable, local judgeeval corpus (Python stdlib).""" +import argparse +import concurrent.futures +import datetime +import hashlib +import http.client +import io +import json +import pathlib +import subprocess +import urllib.parse + + +class ResponseSocket: + def __init__(self, data): + self.data = data + + def makefile(self, *_): + return io.BytesIO(self.data) + + +def capture(url, destination, insecure): + parsed = urllib.parse.urlsplit(url) + if parsed.scheme not in ("http", "https") or not parsed.hostname or parsed.username: + return {"url": url, "error": "expected an HTTP(S) URL without credentials"} + sample_id = hashlib.sha256(url.encode()).hexdigest()[:16] + command = ["curl.exe" if __import__("os").name == "nt" else "curl", + "--noproxy", "*", "--http1.1", "--raw", "--silent", "--show-error", + "--include", "--connect-timeout", "3", "--max-time", "8", + "--max-filesize", "524288", "--user-agent", "fingers-validation/1.0"] + if insecure: + command.append("--insecure") + try: + result = subprocess.run(command + [url], capture_output=True, timeout=10) + if result.returncode: + return {"url": url, "curl_exit": result.returncode, + "error": result.stderr.decode(errors="replace").strip()} + response = http.client.HTTPResponse(ResponseSocket(result.stdout)) + response.begin() + body = response.read(524289) + if len(body) > 524288: + raise ValueError("body exceeds capture limit") + raw_path = destination / "samples" / (sample_id + ".http") + raw_path.write_bytes(result.stdout) + # Ignore volatile header values for duplicate detection; retain product evidence. + ignored = {"date", "set-cookie", "etag", "last-modified", "connection", + "content-length", "transfer-encoding", "x-request-id"} + stable_headers = sorted((k.lower(), v) for k, v in response.getheaders() if k.lower() not in ignored) + semantic = json.dumps([response.status, stable_headers]).encode() + b"\n" + body + return {"id": sample_id, "url": url, "group": parsed.hostname, + "captured_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "response": "samples/" + raw_path.name, + "sha256": hashlib.sha256(result.stdout).hexdigest(), + "content_sha256": hashlib.sha256(semantic).hexdigest(), + "status": response.status, "body_bytes": len(body), + "tls_verified": parsed.scheme == "https" and not insecure, + "labels": []} + except (OSError, ValueError, subprocess.TimeoutExpired, http.client.HTTPException) as error: + return {"url": url, "error": str(error)} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--urls", type=pathlib.Path, required=True, help="one explicit URL per line") + parser.add_argument("--out", type=pathlib.Path, required=True, help="new corpus directory") + parser.add_argument("--workers", type=int, default=6) + parser.add_argument("--insecure", action="store_true", help="allow certificate mismatch for IP captures") + args = parser.parse_args() + if not 1 <= args.workers <= 24: + parser.error("workers must be between 1 and 24") + if args.out.exists(): + parser.error("output already exists; use a new directory to preserve the snapshot") + (args.out / "samples").mkdir(parents=True) + urls = sorted(set(line.strip() for line in args.urls.read_text(encoding="utf-8-sig").splitlines() + if line.strip() and not line.startswith("#"))) + with concurrent.futures.ThreadPoolExecutor(max_workers=args.workers) as pool: + rows = list(pool.map(lambda url: capture(url, args.out, args.insecure), urls)) + manifest = {"schema": 1, "scope": "products and versions supported by captured response evidence", + "samples": [row for row in rows if "error" not in row], "generation": []} + (args.out / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8") + failures = [row for row in rows if "error" in row] + (args.out / "fetch-errors.json").write_text(json.dumps(failures, ensure_ascii=False, indent=2), encoding="utf-8") + print(json.dumps({"requests": len(urls), "captured": len(manifest["samples"]), + "failed": len(failures), "manifest": str(args.out / "manifest.json")})) + + +if __name__ == "__main__": + main() From 898e9d3548700fffbb50dd5f9389c2628cd0a6de Mon Sep 17 00:00:00 2001 From: M09Ic Date: Fri, 25 Sep 2026 10:25:30 -0700 Subject: [PATCH 04/16] build: fix the go-re2 fork pseudo-version and require parsers Judgement The replace pinned github.com/chainreactors/go-re2 with a pseudo-version whose timestamp did not match the commit, so go.sum carried another commit and builds without -tags goregexp failed with a missing go.sum entry. Pin commit 8121b6cd261e with its real pseudo-version. Require utils/parsers at the commit adding Framework.Judge (chainreactors/utils#11), to be re-pinned to parsers/v0.0.4 once tagged. Co-Authored-By: Claude Opus 5.5 (1M context) --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index bad3acb..b67c2af 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,7 @@ require ( require ( github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible github.com/chainreactors/neutron v0.1.1-0.20260714062907-716c6b167cb6 - github.com/chainreactors/utils/parsers v0.0.3 + github.com/chainreactors/utils/parsers v0.0.4-0.20260925164932-39c659c84c64 github.com/chainreactors/words v0.0.0-20260520145736-270600e60fb4 github.com/dlclark/regexp2 v1.11.5 github.com/invopop/jsonschema v0.13.0 @@ -45,4 +45,4 @@ require ( gopkg.in/yaml.v3 v3.0.1 ) -replace github.com/wasilibs/go-re2 => github.com/chainreactors/go-re2 v1.11.1-0.20260716082604-8121b6cd261e +replace github.com/wasilibs/go-re2 => github.com/chainreactors/go-re2 v1.11.1-0.20260716152604-8121b6cd261e diff --git a/go.sum b/go.sum index 3f6ae0c..3f818bb 100644 --- a/go.sum +++ b/go.sum @@ -77,16 +77,16 @@ github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chainreactors/go-re2 v1.11.1-0.20260716142648-7b4fcc466374 h1:PDUneeDqDAhsmJtnL4CenHMPukoUdEG1vO94puW6Gac= -github.com/chainreactors/go-re2 v1.11.1-0.20260716142648-7b4fcc466374/go.mod h1:4qC68vqWSuPTct3spuTrWBqCpm00mQ707JKLS1izVjI= +github.com/chainreactors/go-re2 v1.11.1-0.20260716152604-8121b6cd261e h1:8SMCCo85OKxOM0NtD8sPhmnhGzigU2i4YPUZHgWDVq8= +github.com/chainreactors/go-re2 v1.11.1-0.20260716152604-8121b6cd261e/go.mod h1:4qC68vqWSuPTct3spuTrWBqCpm00mQ707JKLS1izVjI= github.com/chainreactors/logs v0.0.0-20260508055944-c678762ed15c h1:6Net2Mgo/qo6ADFBZJWWScKuMfZ0rbzLqSCVDuLKFdc= github.com/chainreactors/logs v0.0.0-20260508055944-c678762ed15c/go.mod h1:VrXmYPbNN5AVoo1sc5aeyPVBYqubMdb3KO/tn5rRZpo= github.com/chainreactors/neutron v0.1.1-0.20260714062907-716c6b167cb6 h1:apEDnJeZ5fe2AaEJHV6TRDldoy2t/d7E1maxmUz2tfU= github.com/chainreactors/neutron v0.1.1-0.20260714062907-716c6b167cb6/go.mod h1:zok/CDxut71iw8NQTHKPvy0f+1G5639zr9GgDsXGnOs= github.com/chainreactors/utils v0.0.0-20260629043228-93bdd2142c9a h1:4rLTKr+VlG7yGbw2ym7nhNvFglvZ4acdphxKEozF7CI= github.com/chainreactors/utils v0.0.0-20260629043228-93bdd2142c9a/go.mod h1:xwbUlFoSSxLHujyb8D48o1s2DqmEAxUNfxIy0DVUmcg= -github.com/chainreactors/utils/parsers v0.0.3 h1:3ld7xG5TSvzikVOCkQHjqjHO3otjODwSwHQDkMKbu5o= -github.com/chainreactors/utils/parsers v0.0.3/go.mod h1:bE/znJWt08n9QOORWsWu0ggB8GWfOg3+dfUMMITmwV4= +github.com/chainreactors/utils/parsers v0.0.4-0.20260925164932-39c659c84c64 h1:MgEJ0vr7png0bKb5nYL1pK5lTUMDBRbigYt74K+UwYw= +github.com/chainreactors/utils/parsers v0.0.4-0.20260925164932-39c659c84c64/go.mod h1:bE/znJWt08n9QOORWsWu0ggB8GWfOg3+dfUMMITmwV4= github.com/chainreactors/words v0.0.0-20260520145736-270600e60fb4 h1:lvnDYEkatmZFHP5i321qQXK9L4vKRfso/uUfr5tOeC8= github.com/chainreactors/words v0.0.0-20260520145736-270600e60fb4/go.mod h1:zfz367PUmyaX6oAqV9SktVqyRXKlEh0sel9Wsq9dd2c= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= From 53816c2e53d5e0ac20df2ad4b97fd7d2563d0b22 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Fri, 25 Sep 2026 10:25:31 -0700 Subject: [PATCH 05/16] refactor(judge): verdicts on Framework.Judge, one Refine, generator in judge/gen - Verdicts (layer, confidence, rejected, duplicate, primary, recalled) are written to parsers.Framework.Judge instead of judge:* tags; callers use Frameworks.Accepted/Primary without importing judge. - Public API: Refine (report), Inspect (explain), Classify (page), Version, IsUnknownProduct, SuggestNames, Yes/Choose/Score. Judge.Verify, the knownNames variadics, the Round layer and Page.Kind/Generic are gone. - Judge.Known recalls fingerprint names (Engine.Names) automatically; SimilarDistance is a Judge field instead of a mutable global. - Refine resolves the version of every kept product. Versions a response binds by name (Server: nginx/1.24.0, X-Jenkins: 2.401.3, generator meta) are taken without asking; the provider picks among candidates ranked by closeness to the product name, versions bound to another product are not offered, and a string occurring once goes to one product only. - The fingerprint generator moves to judge/gen, so package judge depends only on common; shared text primitives live in judge/internal/evidence. - judgeeval replay drops the separate per-product Version stage. - Code in judge, gen, jev and judgeeval is Go 1.17 compatible. Co-Authored-By: Claude Opus 5.5 (1M context) --- cmd/engine/example.go | 23 +- cmd/judgeeval/main.go | 91 ++++--- cmd/judgeeval/maintenance.go | 20 +- cmd/judgeeval/replay.go | 56 ++--- cmd/judgeeval/replay_generation.go | 3 +- cmd/judgeeval/replay_metrics.go | 9 +- cmd/judgeeval/replay_test.go | 10 +- common/framework.go | 1 + engine.go | 20 +- judge/cache.go | 26 +- judge/classify.go | 14 +- judge/discovery.go | 89 +------ judge/{generator.go => gen/gen.go} | 98 ++++---- judge/gen/gen_test.go | 204 ++++++++++++++++ judge/internal/evidence/evidence.go | 136 +++++++++++ judge/judge.go | 33 ++- judge/judge_test.go | 352 ++++++++++++++++------------ judge/mark.go | 77 ------ judge/operations.go | 75 +++--- judge/operations_test.go | 165 ++----------- judge/page.go | 47 ++-- judge/real_evidence_test.go | 85 +------ judge/recall.go | 13 +- judge/refine.go | 85 ++++--- judge/round.go | 44 ++-- judge/stack.go | 68 ++---- judge/verify.go | 53 ++--- judge/version.go | 304 +++++++++++++++++++----- judge_test.go | 14 +- 29 files changed, 1235 insertions(+), 980 deletions(-) rename judge/{generator.go => gen/gen.go} (80%) create mode 100644 judge/gen/gen_test.go create mode 100644 judge/internal/evidence/evidence.go delete mode 100644 judge/mark.go diff --git a/cmd/engine/example.go b/cmd/engine/example.go index f523e92..5722483 100644 --- a/cmd/engine/example.go +++ b/cmd/engine/example.go @@ -220,17 +220,32 @@ func main() { fmt.Println(err) os.Exit(1) } - accepted, kind, generic, err := j.Refine(context.Background(), content, frames) + j.Known = judge.NewRetriever(engine.Names()) + ctx := context.Background() + accepted, err := j.Refine(ctx, content, frames) + if err == nil { + var kind judge.Kind + var generic bool + if kind, generic, err = j.Classify(ctx, content); err == nil { + fmt.Printf("page: %s, generic: %v\n", kind, generic) + } + } if err != nil { fmt.Printf("jev failed, showing the rule result: %v\n", err) } else { - fmt.Printf("page: %s, generic: %v\n", kind, generic) for _, frame := range accepted { verdict := "accepted" - if judge.Is(frame, judge.Primary) { + if frame.Judge != nil && frame.Judge.Primary { verdict += ",primary" } - fmt.Printf(" %-40s %-10s %-18s %s\n", frame.Name, frame.Version, judge.LayerOf(frame), verdict) + if frame.Judge != nil && frame.Judge.Recalled { + verdict += ",recalled" + } + layer := "" + if frame.Judge != nil { + layer = frame.Judge.Layer + } + fmt.Printf(" %-40s %-10s %-18s %s\n", frame.Name, frame.Version, layer, verdict) } fmt.Printf("accepted: %s\n", accepted.String()) return diff --git a/cmd/judgeeval/main.go b/cmd/judgeeval/main.go index 72a1fbf..18aea73 100644 --- a/cmd/judgeeval/main.go +++ b/cmd/judgeeval/main.go @@ -15,10 +15,12 @@ import ( "errors" "flag" "fmt" + "math" "math/bits" "net/http" "os" "path/filepath" + "reflect" "regexp" "sort" "strings" @@ -137,8 +139,7 @@ func run(providerName, samples, labelsPath, cacheDir, out, endpoint string, rps } j := judge.New(provider) // Measure similarity rather than use it: every page gets its own answers. - defaultDistance := judge.SimilarDistance - judge.SimilarDistance = 0 + j.SimilarDistance = 0 sim := newSimCheck(cacheDir) j.Cache = sim @@ -146,13 +147,7 @@ func run(providerName, samples, labelsPath, cacheDir, out, endpoint string, rps if err != nil { return err } - var names []string - if engine.Aliases != nil { - for name := range engine.Aliases.Aliases { - names = append(names, name) - } - } - retriever := judge.NewRetriever(names) + j.Known = judge.NewRetriever(engine.Names()) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -166,7 +161,7 @@ func run(providerName, samples, labelsPath, cacheDir, out, endpoint string, rps go func() { defer wg.Done() for i := range jobs { - rows[i] = evaluatePage(ctx, engine, j, retriever, files[i]) + rows[i] = evaluatePage(ctx, engine, j, files[i]) var apiErr *jev.APIError if rows[i].err != nil && errors.As(rows[i].err, &apiErr) && apiErr.Status == 401 { once.Do(func() { fatal = errors.New("unauthorized: check " + jev.EnvAPIKey); cancel() }) @@ -202,7 +197,7 @@ func run(providerName, samples, labelsPath, cacheDir, out, endpoint string, rps var b strings.Builder report(&b, rows, provider.ID(), j, tokens(), time.Since(start)) - sim.report(&b, defaultDistance) + sim.report(&b, judge.DefaultSimilarDistance) if labelsPath != "" { if err := evaluate(&b, rows, labelsPath); err != nil { return err @@ -212,7 +207,7 @@ func run(providerName, samples, labelsPath, cacheDir, out, endpoint string, rps return os.WriteFile(filepath.Join(out, "report.md"), []byte(b.String()), 0o644) } -func evaluatePage(ctx context.Context, engine *fingers.Engine, j *judge.Judge, retriever *judge.Retriever, path string) (r *row) { +func evaluatePage(ctx context.Context, engine *fingers.Engine, j *judge.Judge, path string) (r *row) { r = &row{ID: filepath.Base(path), ruleEngines: map[string][]string{}} defer func() { if p := recover(); p != nil { // one malformed page must not stop a long run @@ -240,7 +235,7 @@ func evaluatePage(ctx context.Context, engine *fingers.Engine, j *judge.Judge, r keys := map[string]bool{} for _, f := range r.frames { keys[judge.NormalizeName(f.Name)] = true - if ev := page.Evidence(f.Name); len(ev) == 1 && ev[0] == "text" { + if textOnly(page, f.Name) { r.TextOnly = append(r.TextOnly, f.Name) } } @@ -256,13 +251,18 @@ func evaluatePage(ctx context.Context, engine *fingers.Engine, j *judge.Judge, r } sort.Strings(r.Rules) t := time.Now() - accepted, kind, generic, err := j.Refine(ctx, raw, r.frames, retriever.Find(page.Haystack(), 24)...) + accepted, err := j.Refine(ctx, raw, r.frames) r.JudgeMs = ms(time.Since(t)) if err != nil { r.Err, r.err = err.Error(), err return r } - inspected, err := j.Inspect(ctx, raw, r.frames, retriever.Find(page.Haystack(), 24)...) + inspected, err := j.Inspect(ctx, raw, r.frames) + if err != nil { + r.Err, r.err = err.Error(), err + return r + } + kind, generic, err := j.Classify(ctx, raw) if err != nil { r.Err, r.err = err.Error(), err return r @@ -275,27 +275,32 @@ func evaluatePage(ctx context.Context, engine *fingers.Engine, j *judge.Judge, r r.frames = inspected r.JudgeGenVer = versionOf(accepted, product) for _, name := range r.TextOnly { - if f := r.frames[name]; f != nil && !judge.Is(f, judge.Rejected) { + if f := r.frames[name]; f != nil && (f.Judge == nil || !f.Judge.Rejected) { r.TextOnlyKept++ } } r.Kind, r.Generic, r.Layers = kind, generic, map[string]string{} for _, f := range r.frames { - if l := judge.LayerOf(f); l != "" { - r.Layers[f.Name] = string(l) + v := f.Judge + if v == nil { + r.Accepted = append(r.Accepted, f.Name) + continue + } + if v.Layer != "" { + r.Layers[f.Name] = v.Layer } switch { - case judge.Is(f, judge.Rejected): + case v.Rejected: r.Rejected = append(r.Rejected, f.Name) - case judge.Is(f, judge.Duplicate): + case v.Duplicate: default: r.Accepted = append(r.Accepted, f.Name) } - if judge.Is(f, judge.Recalled) { + if v.Recalled { r.Recall = append(r.Recall, f.Name) } } - if p := judge.PrimaryOf(r.frames); p != nil { + if p := r.frames.Primary(); p != nil { r.Primary, r.Version = p.Name, p.Version r.JudgeVersion = p.Version != "" && p.Version != versions[p.Name] } @@ -309,6 +314,24 @@ func evaluatePage(ctx context.Context, engine *fingers.Engine, j *judge.Judge, r return r } +// textOnly reports whether name occurs in the page's visible text and in no +// structural evidence: the code-only proxy for a false positive. +func textOnly(p *judge.Page, name string) bool { + n := strings.ToLower(name) + if len(n) < 3 || !strings.Contains(strings.ToLower(p.Text), n) { + return false + } + var structural []string + for k, v := range p.Headers { + structural = append(structural, k+":"+v) + } + structural = append(structural, p.Title, p.Generator, p.Description) + for _, list := range [][]string{p.Cookies, p.Scripts, p.Styles, p.InlineHints, p.Comments, p.Forms} { + structural = append(structural, list...) + } + return !strings.Contains(strings.ToLower(strings.Join(structural, " ")), n) +} + func report(b *strings.Builder, rows []*row, providerID string, j *judge.Judge, tokens int64, took time.Duration) { w := func(format string, a ...interface{}) { fmt.Fprintf(b, format+"\n", a...) } var pages, errs, rules, accepted, recall, primary, generic, newFinger int @@ -364,7 +387,7 @@ func report(b *strings.Builder, rows []*row, providerID string, j *judge.Judge, newFingers = append(newFingers, r.ID) } for _, f := range r.frames { - rejected := judge.Is(f, judge.Rejected) + rejected := f.Judge != nil && f.Judge.Rejected for _, e := range r.ruleEngines[f.Name] { add(byEngine, e, rejected) add(byRule, e+"/"+f.Name, rejected) @@ -387,7 +410,7 @@ func report(b *strings.Builder, rows []*row, providerID string, j *judge.Judge, w("| 新指纹候选 | — | %d |", newFinger) w("| 每页耗时 p50 / p95 | %.1fms / %.1fms | +%.0fms / +%.0fms |", pct(ruleMs, 50), pct(ruleMs, 95), pct(judgeMs, 50), pct(judgeMs, 95)) w("| 成本 | 0 | %d 次请求（缓存直接答完 %d 轮），%d input tokens（jev 按 $0.042/Mtok ≈ $%.3f，折合每万页 $%.2f） |", - j.Requests, j.CacheHits, tokens, float64(tokens)*0.042/1e6, float64(tokens)*0.042/1e6/float64(max(pages, 1))*1e4) + j.Requests, j.CacheHits, tokens, float64(tokens)*0.042/1e6, float64(tokens)*0.042/1e6/math.Max(float64(pages), 1)*1e4) w("\n## page kind\n") for _, k := range sortedKeys(kinds, func(k string) int { return kinds[k] }) { w("- %s: %d", k, kinds[k]) @@ -497,7 +520,7 @@ func evaluate(b *strings.Builder, rows []*row, path string) error { func find(frames common.Frameworks, name string, strict bool) *common.Framework { want := judge.NormalizeName(name) var found *common.Framework - for _, f := range judge.Accepted(frames) { + for _, f := range frames.Accepted() { got := judge.NormalizeName(f.Name) if got == "" || !(strings.Contains(got, want) || !strict && strings.Contains(want, got)) { continue @@ -586,10 +609,11 @@ func div(a, b int) float64 { return float64(a) / float64(b) } -func sortedKeys[V any](m map[string]V, weight func(string) int) []string { - keys := make([]string, 0, len(m)) - for k := range m { - keys = append(keys, k) +// sortedKeys returns the keys of m, a map with string keys, by descending weight. +func sortedKeys(m interface{}, weight func(string) int) []string { + var keys []string + for _, k := range reflect.ValueOf(m).MapKeys() { + keys = append(keys, k.String()) } sort.Slice(keys, func(a, b int) bool { if wa, wb := weight(keys[a]), weight(keys[b]); wa != wb { @@ -600,14 +624,15 @@ func sortedKeys[V any](m map[string]V, weight func(string) int) []string { return keys } -// fileCache is an exact judge.Cache on disk, one file per request. +// fileCache is an exact judge.Cache on disk, one file per answer: the +// signature is part of the file name, so distance is ignored. type fileCache string func (d fileCache) path(key string, sig uint64) string { return filepath.Join(string(d), fmt.Sprintf("%s_%016x.json", key, sig)) } -func (d fileCache) Get(key string, sig uint64) ([]byte, bool) { +func (d fileCache) Get(key string, sig uint64, _ int) ([]byte, bool) { data, err := os.ReadFile(d.path(key, sig)) return data, err == nil } @@ -659,8 +684,8 @@ func (c *simCheck) nearest(key string, sig uint64) (simNear, bool) { return best, ok } -func (c *simCheck) Get(key string, sig uint64) ([]byte, bool) { - value, ok := c.fileCache.Get(key, sig) +func (c *simCheck) Get(key string, sig uint64, distance int) ([]byte, bool) { + value, ok := c.fileCache.Get(key, sig, distance) c.mu.Lock() defer c.mu.Unlock() near, found := c.nearest(key, sig) diff --git a/cmd/judgeeval/maintenance.go b/cmd/judgeeval/maintenance.go index c0106cb..277b394 100644 --- a/cmd/judgeeval/maintenance.go +++ b/cmd/judgeeval/maintenance.go @@ -49,19 +49,19 @@ func embeddedCatalog() ([]catalogSource, error) { {"ehole", resources.EholeData}, {"wappalyzer", resources.WappalyzerData}, {"fingerprinthub", resources.FingerprinthubWebData}, } { - var raw any + var raw interface{} if err := resources.UnmarshalData(source.data, &raw); err != nil { return nil, err } - var entries []any + var entries []interface{} var names []string switch source.name { case "wappalyzer": - root, ok := raw.(map[string]any) + root, ok := raw.(map[string]interface{}) if !ok { return nil, fmt.Errorf("invalid %s catalog", source.name) } - apps, ok := root["apps"].(map[string]any) + apps, ok := root["apps"].(map[string]interface{}) if !ok { return nil, fmt.Errorf("invalid apps catalog") } @@ -69,16 +69,16 @@ func embeddedCatalog() ([]catalogSource, error) { names = append(names, name) } case "ehole": - root, ok := raw.(map[string]any) + root, ok := raw.(map[string]interface{}) if !ok { return nil, fmt.Errorf("invalid ehole catalog") } - entries, _ = root["fingerprint"].([]any) + entries, _ = root["fingerprint"].([]interface{}) default: - entries, _ = raw.([]any) + entries, _ = raw.([]interface{}) } for _, entry := range entries { - obj, ok := entry.(map[string]any) + obj, ok := entry.(map[string]interface{}) if !ok { continue } @@ -87,11 +87,11 @@ func embeddedCatalog() ([]catalogSource, error) { names = append(names, name) } } - if info, ok := obj["info"].(map[string]any); ok { + if info, ok := obj["info"].(map[string]interface{}); ok { if name, ok := info["name"].(string); ok { names = append(names, name) } - if metadata, ok := info["metadata"].(map[string]any); ok { + if metadata, ok := info["metadata"].(map[string]interface{}); ok { if name, ok := metadata["product"].(string); ok { names = append(names, name) } diff --git a/cmd/judgeeval/replay.go b/cmd/judgeeval/replay.go index 62f919e..f6ce63d 100644 --- a/cmd/judgeeval/replay.go +++ b/cmd/judgeeval/replay.go @@ -70,7 +70,6 @@ type replayRow struct { SHA256 string `json:"sha256"` Baseline common.Frameworks `json:"baseline"` Refined common.Frameworks `json:"refined,omitempty"` - Completed common.Frameworks `json:"completed,omitempty"` Rejected []string `json:"rejected,omitempty"` Duplicates []string `json:"duplicates,omitempty"` Added []string `json:"added,omitempty"` @@ -83,7 +82,7 @@ type replayRow struct { } func digest(b []byte) string { return fmt.Sprintf("%x", sha256.Sum256(b)) } -func writeJSON(path string, value any) error { +func writeJSON(path string, value interface{}) error { b, err := json.MarshalIndent(value, "", " ") if err != nil { return err @@ -153,7 +152,7 @@ func loadReplay(path string) (*replayManifest, error) { } stable[k] = v } - head, _ := json.Marshal([]any{resp.StatusCode, stable}) + head, _ := json.Marshal([]interface{}{resp.StatusCode, stable}) s.ContentSHA256 = digest(append(head, body...)) evidence := strings.ToLower(string(s.raw) + "\n" + string(body)) keys := map[string]bool{} @@ -195,8 +194,8 @@ func (p *replayProvider) Calibration() (float64, float64) { } return .5, .9 } -func (p *replayProvider) Judge(ctx context.Context, state any, qs map[string]judge.Question) (map[string]judge.Answer, error) { - data, err := json.Marshal([]any{p.ID(), p.namespace, state, qs}) +func (p *replayProvider) Judge(ctx context.Context, state interface{}, qs map[string]judge.Question) (map[string]judge.Answer, error) { + data, err := json.Marshal([]interface{}{p.ID(), p.namespace, state, qs}) if err != nil { return nil, err } @@ -333,14 +332,7 @@ func replay(manifestPath, historyPath, providerName, endpoint, cacheDir, out str return err } } - var names []string - if engine.Aliases != nil { - for name := range engine.Aliases.Aliases { - names = append(names, name) - } - } - sort.Strings(names) - retriever := judge.NewRetriever(names) + j.Known = judge.NewRetriever(engine.Names()) baselineFile, err := os.Create(filepath.Join(out, "baseline.jsonl")) if err != nil { return err @@ -373,18 +365,18 @@ func replay(manifestPath, historyPath, providerName, endpoint, cacheDir, out str return err } ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) - err = cleanReplay(ctx, j, retriever, s, &r) + err = cleanReplay(ctx, j, s, &r) cancel() if err != nil { r.Err = err.Error() - } else if err := ce.Encode(baselineRecord{s.ID, s.SHA256, r.Completed}); err != nil { + } else if err := ce.Encode(baselineRecord{s.ID, s.SHA256, r.Refined}); err != nil { return err } if err := re.Encode(r); err != nil { return err } rows = append(rows, r) - fmt.Fprintf(os.Stderr, "replay %d/%d %s: baseline=%d completed=%d error=%t\n", len(rows), len(m.Samples), s.ID, len(r.Baseline), len(r.Completed), r.Err != "") + fmt.Fprintf(os.Stderr, "replay %d/%d %s: baseline=%d refined=%d error=%t\n", len(rows), len(m.Samples), s.ID, len(r.Baseline), len(r.Refined), r.Err != "") } generated, err := generateReplay(m, j, out) if err != nil { @@ -437,52 +429,38 @@ func saveReplaySnapshot(out string, m *replayManifest) error { } return writeJSON(filepath.Join(out, "manifest.json"), copy) } -func cleanReplay(ctx context.Context, j *judge.Judge, retriever *judge.Retriever, s replaySample, r *replayRow) error { - p, err := judge.NewPage(s.raw) +func cleanReplay(ctx context.Context, j *judge.Judge, s replaySample, r *replayRow) error { + var err error + r.Refined, err = j.Refine(ctx, s.raw, r.Baseline) if err != nil { return err } - known := retriever.Find(p.Haystack(), 24) - r.Refined, r.Kind, r.Generic, err = j.Refine(ctx, s.raw, r.Baseline, known...) - if err != nil { + if r.Kind, r.Generic, err = j.Classify(ctx, s.raw); err != nil { return err } - inspected, err := j.Inspect(ctx, s.raw, r.Baseline, known...) + inspected, err := j.Inspect(ctx, s.raw, r.Baseline) if err != nil { return err } for _, f := range inspected { - if judge.Is(f, judge.Rejected) { + if f.Judge != nil && f.Judge.Rejected { r.Rejected = append(r.Rejected, f.Name) } - if judge.Is(f, judge.Duplicate) { + if f.Judge != nil && f.Judge.Duplicate { r.Duplicates = append(r.Duplicates, f.Name) } } - r.Completed = copyFrames(r.Refined) r.Filled = map[string]string{} - for _, f := range r.Completed { + for _, f := range r.Refined { before := findLabel(r.Baseline, productLabel{Product: f.Name}) if before == nil { r.Added = append(r.Added, f.Name) } - if f.Version == "" { - v, err := j.Version(ctx, s.raw, f) - if err != nil { - return err - } - if v != "" { - if f.Attributes == nil { - f.Attributes = common.NewAttributesWithAny() - } - f.Version = v - } - } if f.Version != "" && (before == nil || before.Version == "") { r.Filled[f.Name] = f.Version } } - r.Unknown, err = j.IsUnknownProduct(ctx, s.raw, r.Completed) + r.Unknown, err = j.IsUnknownProduct(ctx, s.raw, r.Refined) if err != nil { return err } diff --git a/cmd/judgeeval/replay_generation.go b/cmd/judgeeval/replay_generation.go index 2ff56ce..e231579 100644 --- a/cmd/judgeeval/replay_generation.go +++ b/cmd/judgeeval/replay_generation.go @@ -12,6 +12,7 @@ import ( "github.com/chainreactors/fingers/common" fingerlib "github.com/chainreactors/fingers/fingers" "github.com/chainreactors/fingers/judge" + "github.com/chainreactors/fingers/judge/gen" "gopkg.in/yaml.v3" ) @@ -124,7 +125,7 @@ func generateReplay(m *replayManifest, j *judge.Judge, out string) ([]generation return nil, err } r.Excluded = excluded - g := judge.NewGenerator(j) + g := gen.New(j) if !p.AutoName { g.Name(p.Product) } diff --git a/cmd/judgeeval/replay_metrics.go b/cmd/judgeeval/replay_metrics.go index c69bb83..d85e385 100644 --- a/cmd/judgeeval/replay_metrics.go +++ b/cmd/judgeeval/replay_metrics.go @@ -73,7 +73,6 @@ type replaySummary struct { LabelledPairs int `json:"labelled_pairs"` Baseline stageScore `json:"baseline"` Refined stageScore `json:"refined"` - Completed stageScore `json:"completed"` FalseHitsRemoved int `json:"false_hits_removed"` TrueHitsRemoved int `json:"true_hits_removed"` NaturalMissesRecovered int `json:"natural_misses_recovered"` @@ -108,7 +107,7 @@ func summarizeReplay(m *replayManifest, rows []replayRow, gen []generationResult for _, stage := range []struct { score *stageScore frames common.Frameworks - }{{&out.Baseline, r.Baseline}, {&out.Refined, r.Refined}, {&out.Completed, r.Completed}} { + }{{&out.Baseline, r.Baseline}, {&out.Refined, r.Refined}} { for _, l := range s.Labels { f := findLabel(stage.frames, l) stage.score.Detection.add(l.Present, f != nil) @@ -126,7 +125,7 @@ func summarizeReplay(m *replayManifest, rows []replayRow, gen []generationResult } for _, l := range s.Labels { out.LabelledPairs++ - before, after := findLabel(r.Baseline, l), findLabel(r.Completed, l) + before, after := findLabel(r.Baseline, l), findLabel(r.Refined, l) if before != nil && after == nil { if l.Present { out.TrueHitsRemoved++ @@ -197,7 +196,7 @@ func writeReplayReport(path string, s replaySummary) error { for _, r := range []struct { name string s stageScore - }{{"原结果", s.Baseline}, {"Refine", s.Refined}, {"逐产品补版本", s.Completed}} { + }{{"原结果", s.Baseline}, {"Refine", s.Refined}} { d := r.s.Detection fmt.Fprintf(&b, "| %s | %d | %d | %d | %d | %d |\n", r.name, d.TP, d.FP, d.FN, d.TN, r.s.UnlabelledPredictions) } @@ -207,7 +206,7 @@ func writeReplayReport(path string, s replaySummary) error { for _, r := range []struct { name string s versionScore - }{{"原结果", s.Baseline.Versions}, {"Refine", s.Refined.Versions}, {"逐产品补版本", s.Completed.Versions}} { + }{{"原结果", s.Baseline.Versions}, {"Refine", s.Refined.Versions}} { v := r.s fmt.Fprintf(&b, "| %s | %d | %d | %d | %d | %d | %d |\n", r.name, v.Correct, v.Wrong, v.Missing, v.CorrectAbstentions, v.Unsupported, v.ProductMissing) } diff --git a/cmd/judgeeval/replay_test.go b/cmd/judgeeval/replay_test.go index a17abc7..9d87d9f 100644 --- a/cmd/judgeeval/replay_test.go +++ b/cmd/judgeeval/replay_test.go @@ -23,9 +23,9 @@ func TestReplayCountsErrorsAndUnlabelledSeparately(t *testing.T) { base := testFrames("nginx", "") after := testFrames("apache http server", "2.4.38") after.Add(common.NewFramework("unlabelled-library", common.FrameFromGUESS)) - r := replayRow{ID: "a", Baseline: base, Refined: after, Completed: after, Filled: map[string]string{"apache http server": "2.4.38"}} + r := replayRow{ID: "a", Baseline: base, Refined: after, Filled: map[string]string{"apache http server": "2.4.38"}} summary := summarizeReplay(m, []replayRow{r, {ID: "failed", Err: "network"}}, nil) - if summary.Errors != 1 || summary.LabelledPairs != 2 || summary.Baseline.Detection.FP != 1 || summary.Baseline.Detection.FN != 1 || summary.Completed.Detection.TP != 1 || summary.Completed.Detection.TN != 1 || summary.Completed.UnlabelledPredictions != 1 { + if summary.Errors != 1 || summary.LabelledPairs != 2 || summary.Baseline.Detection.FP != 1 || summary.Baseline.Detection.FN != 1 || summary.Refined.Detection.TP != 1 || summary.Refined.Detection.TN != 1 || summary.Refined.UnlabelledPredictions != 1 { t.Fatalf("metrics: %+v", summary) } if summary.NaturalMissesRecovered != 1 || summary.FalseHitsRemoved != 1 || summary.CorrectVersionFills != 1 { @@ -100,7 +100,7 @@ func TestReplayConfinesResponsePaths(t *testing.T) { type replayTestProvider struct{ calls int } func (*replayTestProvider) ID() string { return "test/replay" } -func (p *replayTestProvider) Judge(_ context.Context, _ any, qs map[string]judge.Question) (map[string]judge.Answer, error) { +func (p *replayTestProvider) Judge(_ context.Context, _ interface{}, qs map[string]judge.Question) (map[string]judge.Answer, error) { p.calls++ answers := map[string]judge.Answer{} for key := range qs { @@ -145,7 +145,7 @@ func TestReplayVersionFillsDeduplicateAliases(t *testing.T) { m := &replayManifest{Samples: []replaySample{{ID: "a", Labels: []productLabel{l}}}} after := testFrames("PHP", "5.6.40") after.Add(common.NewFrameworkWithVersion("php-runtime", common.FrameFromGUESS, "5.6.40")) - r := replayRow{ID: "a", Baseline: testFrames("PHP", "5.6.40"), Refined: after, Completed: after, Filled: map[string]string{"php-runtime": "5.6.40"}} + r := replayRow{ID: "a", Baseline: testFrames("PHP", "5.6.40"), Refined: after, Filled: map[string]string{"php-runtime": "5.6.40"}} if s := summarizeReplay(m, []replayRow{r}, nil); s.VersionFills != 0 { t.Fatalf("existing alias version counted as new: %+v", s) } @@ -160,7 +160,7 @@ func TestReplaySuggestionsDoNotRecoverDetections(t *testing.T) { m := &replayManifest{Samples: []replaySample{{ID: "a", Labels: []productLabel{{Product: "New API", Present: true}}}}} r := replayRow{ID: "a", Suggestions: []string{"new-api", "New API"}} s := summarizeReplay(m, []replayRow{r}, nil) - if s.MissedProductsSuggested != 1 || s.NaturalMissesRecovered != 0 || s.Completed.Detection.FN != 1 { + if s.MissedProductsSuggested != 1 || s.NaturalMissesRecovered != 0 || s.Refined.Detection.FN != 1 { t.Fatalf("suggestions credited as detections: %+v", s) } } diff --git a/common/framework.go b/common/framework.go index 4968b21..6cc997d 100644 --- a/common/framework.go +++ b/common/framework.go @@ -27,6 +27,7 @@ type Framework = parsers.Framework type MatchDetail = parsers.MatchDetail type Frameworks = parsers.Frameworks type Attributes = parsers.Attributes +type Judgement = parsers.Judgement var FrameFromMap = parsers.FrameFromMap diff --git a/engine.go b/engine.go index c89bd87..47c39ba 100644 --- a/engine.go +++ b/engine.go @@ -4,6 +4,10 @@ import ( "bufio" "bytes" "fmt" + "net/http" + "sort" + "strings" + "github.com/chainreactors/fingers/alias" "github.com/chainreactors/fingers/common" "github.com/chainreactors/fingers/ehole" @@ -17,8 +21,6 @@ import ( xrayengine "github.com/chainreactors/fingers/xray" "github.com/chainreactors/utils/httputils" "github.com/pkg/errors" - "net/http" - "strings" ) const ( @@ -97,6 +99,20 @@ func (engine *Engine) String() string { return strings.TrimSpace(s.String()) } +// Names returns the canonical names of every fingerprint in the alias +// table, sorted: what judge.NewRetriever takes to recall missed products. +func (engine *Engine) Names() []string { + if engine.Aliases == nil { + return nil + } + names := make([]string, 0, len(engine.Aliases.Aliases)) + for name := range engine.Aliases.Aliases { + names = append(names, name) + } + sort.Strings(names) + return names +} + func (engine *Engine) Compile() error { // 从所有引擎中填充Favicon引擎的数据 if impl := engine.Fingers(); impl != nil { diff --git a/judge/cache.go b/judge/cache.go index cb20e5f..49a23b6 100644 --- a/judge/cache.go +++ b/judge/cache.go @@ -6,30 +6,20 @@ import ( "sync" ) -// SimilarDistance is how many of the 64 signature bits two pages with the -// same title may differ in to share cached answers; 0 shares answers only -// between pages with identical signatures. Measured with cmd/judgeeval on -// 1070 real pages: every answer reused at distance <= 1 matched the page's -// own answer (159/159), at 2 it was 98.5%, at 3 96.9%. -var SimilarDistance = 1 - // Cache stores answers, one per question. key covers everything that must // match exactly: model, question and named evidence such as version strings. -// sig is the page's Signature: Get may return the answer stored for a page -// whose signature is within SimilarDistance bits, so near-identical pages -// (one product's login page on many hosts, one site's 404 page on every -// path) are asked once. Providers answer identical input identically, so an exact -// hit is always valid. Implementations must be safe for concurrent use; the +// sig is the page's signature: Get may return the answer stored for a page +// whose signature is within distance bits (Judge.SimilarDistance), so +// near-identical pages (one product's login page on many hosts, one site's +// 404 page on every path) are asked once. Providers answer identical input +// identically, so an exact hit is always valid. Implementations must be safe for concurrent use; the // in-memory NewMemoryCache is the default, a shared one (redis, disk) lets // several scanners share answers. type Cache interface { - Get(key string, sig uint64) ([]byte, bool) + Get(key string, sig uint64, distance int) ([]byte, bool) Put(key string, sig uint64, value []byte) } -// Similar reports whether two signatures are within SimilarDistance. -func Similar(a, b uint64) bool { return bits.OnesCount64(a^b) <= SimilarDistance } - // NewMemoryCache is an LRU cache holding up to size responses. func NewMemoryCache(size int) Cache { return &memoryCache{size: size, order: list.New(), buckets: map[string][]*list.Element{}} @@ -48,7 +38,7 @@ type memoryEntry struct { value []byte } -func (c *memoryCache) Get(key string, sig uint64) ([]byte, bool) { +func (c *memoryCache) Get(key string, sig uint64, distance int) ([]byte, bool) { c.mu.Lock() defer c.mu.Unlock() var best *list.Element @@ -58,7 +48,7 @@ func (c *memoryCache) Get(key string, sig uint64) ([]byte, bool) { best, bestDist = e, d } } - if best == nil || bestDist > SimilarDistance { + if best == nil || bestDist > distance { return nil, false } c.order.MoveToFront(best) diff --git a/judge/classify.go b/judge/classify.go index e9a66cd..572b565 100644 --- a/judge/classify.go +++ b/judge/classify.go @@ -23,16 +23,16 @@ var pageKinds = map[string]string{ string(KindAPI): "A machine-readable API response rather than a web page", } -// Classify judges the page itself and writes Page.Kind and Page.Generic. It -// needs no fingerprints: spray can use it alone for 404 detection and +// classifyRound asks what the page is and whether it is the stock page of a +// packaged product, writing the answers to kind and generic. It needs no +// fingerprints: spray can use it alone for 404 detection and // prioritisation. A generic page without an accepted application is a // candidate for a new fingerprint. -func Classify(r *Round) { - p := r.page - r.Add("page_kind", Choice("What kind of page is this HTTP response?", pageKinds), func(a Answer) { p.Kind = Kind(a.Choice) }) - r.Add("generic", BinaryWith("Is this page the stock interface of a packaged software product, framework or device, "+ +func classifyRound(r *round, kind *Kind, generic *bool) { + r.add("page_kind", Choice("What kind of page is this HTTP response?", pageKinds), func(a Answer) { *kind = Kind(a.Choice) }) + r.add("generic", BinaryWith("Is this page the stock interface of a packaged software product, framework or device, "+ "so that the same page would appear on many unrelated deployments?", "Login, console, default or error page, packaged search application or static/browser-side tools shipped by a product; no login or backend is required; only branding, host names or data differ between installs", "Content written by one organization for its own purpose: articles, company pages, shops, forums, custom-built portals"), - func(a Answer) { p.Generic = a.Yes >= r.judge.Threshold }) + func(a Answer) { *generic = a.Yes >= r.judge.Threshold }) } diff --git a/judge/discovery.go b/judge/discovery.go index 1d1b47b..b51da8f 100644 --- a/judge/discovery.go +++ b/judge/discovery.go @@ -3,32 +3,25 @@ package judge import ( "context" "fmt" - "regexp" "sort" - "strings" - "unicode/utf8" "github.com/chainreactors/fingers/common" -) - -var ( - nameWords = regexp.MustCompile(`[A-Z][A-Za-z0-9!+._]{2,}(?:[ -][A-Z][A-Za-z0-9!+._]{2,}){0,2}|[\p{Han}]{2,20}`) - nameVersion = regexp.MustCompile(`(?i)[/\s]+[vV]?(?:` + versionToken + `|[0-9]+)\s*$`) + "github.com/chainreactors/fingers/judge/internal/evidence" ) // IsUnknownProduct reports whether this looks like a reusable product page -// with no identified product. Pass accepted fingerprints from Verify or -// Refine; known infrastructure alone does not identify an application. +// with no identified product. Pass the result of Refine; known +// infrastructure alone does not identify an application. func (j *Judge) IsUnknownProduct(ctx context.Context, raw []byte, accepted common.Frameworks) (bool, error) { kind, generic, err := j.Classify(ctx, raw) if err != nil || !generic { return false, err } - for _, f := range accepted { - if Is(f, Rejected) || Is(f, Duplicate) { + for _, f := range accepted.Accepted() { + if f.Judge == nil { continue } - switch LayerOf(f) { + switch f.Judge.Layer { case LayerApplication, LayerDevice: return false, nil case LayerServer, LayerCDN: @@ -52,17 +45,17 @@ func (j *Judge) SuggestNames(ctx context.Context, raw []byte) ([]string, error) if len(candidates) == 0 { return nil, nil } - r := p.Round() + r := newRound(p) prob := make([]float64, len(candidates)) for i, name := range candidates { i, name := i, name - r.Add(fmt.Sprintf("name_%d", i), BinaryWith( + r.add(fmt.Sprintf("name_%d", i), BinaryWith( fmt.Sprintf("Is `%s` the product whose own interface, login, console or default page this response shows, rather than infrastructure underneath another product, generic wording or an article mention?", name), name+" owns this page", name+" is infrastructure or merely mentioned"), func(a Answer) { prob[i] = a.Yes }) } - if err := r.Ask(ctx, j); err != nil { + if err := r.ask(ctx, j); err != nil { return nil, err } var indices []int @@ -80,67 +73,5 @@ func (j *Judge) SuggestNames(ctx context.Context, raw []byte) ([]string, error) } func pageNames(p *Page) []string { - type candidate struct { - name string - weight int - } - byKey := map[string]candidate{} - add := func(text string, weight int) { - text = strings.Trim(strings.TrimSpace(nameVersion.ReplaceAllString(clean(text), "")), "-:|/ ") - if text == "" || utf8.RuneCountInString(text) > 40 || !usableKey(strings.ToLower(text)) { - return - } - key := NormalizeName(text) - if key == "" || genericName(key) { - return - } - if prev, ok := byKey[key]; !ok || weight > prev.weight { - byKey[key] = candidate{text, weight} - } - } - add(p.Generator, 5) - for k, v := range p.Headers { - if strings.HasPrefix(k, "X-") || k == "Server" || k == "X-Powered-By" || k == "Product" { - add(strings.Split(v, "/")[0], 4) - } - } - for _, part := range strings.FieldsFunc(p.Title, func(r rune) bool { return r == '|' || r == '-' || r == ':' || r == '–' }) { - add(part, 4) - } - for _, asset := range append(append([]string(nil), p.Scripts...), p.Styles...) { - for _, word := range strings.FieldsFunc(asset, func(r rune) bool { return r == '/' || r == '_' }) { - add(word, 2) - } - } - for _, segment := range strings.FieldsFunc(p.Text, func(r rune) bool { return r == '.' || r == ':' || r == '。' || r == '：' || r == '|' || r == '，' }) { - for _, match := range nameWords.FindAllString(segment, 20) { - add(match, 1) - } - } - list := make([]candidate, 0, len(byKey)) - for _, c := range byKey { - list = append(list, c) - } - sort.Slice(list, func(a, b int) bool { - if list[a].weight != list[b].weight { - return list[a].weight > list[b].weight - } - return list[a].name < list[b].name - }) - if len(list) > 20 { - list = list[:20] - } - out := make([]string, len(list)) - for i, c := range list { - out[i] = c.name - } - return out -} - -func genericName(key string) bool { - switch key { - case "login", "signin", "welcome", "dashboard", "admin", "console", "home", "index", "server", "error", "notfound", "password", "username", "management", "system", "登录", "首页", "管理", "用户", "密码", "控制台": - return true - } - return false + return evidence.Names(p.Generator, p.Title, p.Text, p.Headers, append(append([]string(nil), p.Scripts...), p.Styles...)) } diff --git a/judge/generator.go b/judge/gen/gen.go similarity index 80% rename from judge/generator.go rename to judge/gen/gen.go index 565cdd9..e7d1a2b 100644 --- a/judge/generator.go +++ b/judge/gen/gen.go @@ -1,4 +1,7 @@ -package judge +// Package gen builds native fingerprints from labelled HTTP responses. It +// uses a judge.Judge only to pick a product name and versions; rules are +// derived and validated by code against the rule engine. +package gen import ( "bytes" @@ -10,12 +13,16 @@ import ( "github.com/chainreactors/fingers/common" fingerlib "github.com/chainreactors/fingers/fingers" + "github.com/chainreactors/fingers/judge" + "github.com/chainreactors/fingers/judge/internal/evidence" ) +var whitespace = regexp.MustCompile(`\s+`) + // Generator builds one native Finger from labelled HTTP responses. Probe // belongs to the most recently added positive or negative sample. type Generator struct { - judge *Judge + judge *judge.Judge name string positive []generatorSample negative []generatorSample @@ -29,9 +36,10 @@ type generatorSample struct { probes map[string][]byte } -// NewGenerator creates a fingerprint generator using j for name and version -// selection. A Name hint lets rule generation work without a provider. -func NewGenerator(j *Judge) *Generator { return &Generator{judge: j} } +// New creates a fingerprint generator using j for name and version +// selection. j may be nil when Name is given and every positive carries its +// version (PositiveVersion). +func New(j *judge.Judge) *Generator { return &Generator{judge: j} } // Name supplies a known product name. Without it Generate selects a name // from the positive responses. @@ -42,7 +50,7 @@ func (g *Generator) Name(name string) *Generator { // Positive adds an HTTP response known to belong to the product. func (g *Generator) Positive(raw []byte) *Generator { - g.positive = append(g.positive, generatorSample{raw: bytes.Clone(raw), probes: map[string][]byte{}}) + g.positive = append(g.positive, generatorSample{raw: append([]byte(nil), raw...), probes: map[string][]byte{}}) g.current = &g.positive[len(g.positive)-1] return g } @@ -56,7 +64,7 @@ func (g *Generator) PositiveVersion(raw []byte, version string) *Generator { // Negative adds an HTTP response known not to belong to the product. func (g *Generator) Negative(raw []byte) *Generator { - g.negative = append(g.negative, generatorSample{raw: bytes.Clone(raw), probes: map[string][]byte{}}) + g.negative = append(g.negative, generatorSample{raw: append([]byte(nil), raw...), probes: map[string][]byte{}}) g.current = &g.negative[len(g.negative)-1] return g } @@ -65,14 +73,14 @@ func (g *Generator) Negative(raw []byte) *Generator { // is the exact send_data used by the rule engine. func (g *Generator) Probe(request, response []byte) *Generator { if g.current == nil { - g.err = fmt.Errorf("judge: Probe requires a preceding Positive or Negative") + g.err = fmt.Errorf("gen: Probe requires a preceding Positive or Negative") return g } if len(request) == 0 { - g.err = fmt.Errorf("judge: Probe requires non-empty send_data") + g.err = fmt.Errorf("gen: Probe requires non-empty send_data") return g } - g.current.probes[string(request)] = bytes.Clone(response) + g.current.probes[string(request)] = append([]byte(nil), response...) return g } @@ -80,10 +88,10 @@ func (g *Generator) Probe(request, response []byte) *Generator { // caller controls network behavior through send. func (g *Generator) ProbeWith(ctx context.Context, request []byte, send func(context.Context, []byte) ([]byte, error)) error { if g.current == nil { - return fmt.Errorf("judge: ProbeWith requires a preceding Positive or Negative") + return fmt.Errorf("gen: ProbeWith requires a preceding Positive or Negative") } if send == nil || len(request) == 0 { - return fmt.Errorf("judge: ProbeWith requires a sender and non-empty send_data") + return fmt.Errorf("gen: ProbeWith requires a sender and non-empty send_data") } response, err := send(ctx, request) if err != nil { @@ -100,7 +108,7 @@ func (g *Generator) Generate(ctx context.Context) (*fingerlib.Finger, error) { return nil, g.err } if len(g.positive) == 0 || len(g.negative) == 0 { - return nil, fmt.Errorf("judge: Generate requires positive and negative samples") + return nil, fmt.Errorf("gen: Generate requires positive and negative samples") } if err := g.validateSamples(); err != nil { return nil, err @@ -108,7 +116,7 @@ func (g *Generator) Generate(ctx context.Context) (*fingerlib.Finger, error) { name := g.name if name == "" { if g.judge == nil { - return nil, fmt.Errorf("judge: a Judge or Name is required to select a product name") + return nil, fmt.Errorf("gen: a Judge or Name is required to select a product name") } counts := map[string]int{} ranks := map[string]int{} @@ -119,7 +127,7 @@ func (g *Generator) Generate(ctx context.Context) (*fingerlib.Finger, error) { return nil, err } for rank, candidate := range names { - key := NormalizeName(candidate) + key := evidence.NormalizeName(candidate) counts[key]++ ranks[key] += len(names) - rank if labels[key] == "" { @@ -141,7 +149,7 @@ func (g *Generator) Generate(ctx context.Context) (*fingerlib.Finger, error) { return keys[a] < keys[b] }) if len(keys) == 0 || counts[keys[0]] != len(g.positive) { - return nil, fmt.Errorf("judge: no product name supported by every positive sample; provide Name") + return nil, fmt.Errorf("gen: no product name supported by every positive sample; provide Name") } name = labels[keys[0]] } @@ -175,7 +183,7 @@ func (g *Generator) Generate(ctx context.Context) (*fingerlib.Finger, error) { } } if len(f.Rules) == 0 { - return nil, fmt.Errorf("judge: no rule distinguishes the positive and negative samples") + return nil, fmt.Errorf("gen: no rule distinguishes the positive and negative samples") } if rule := g.versionRule("", name, versions); rule != nil { f.Rules = append(fingerlib.Rules{rule}, f.Rules...) @@ -196,17 +204,17 @@ func (g *Generator) Validate(f *fingerlib.Finger) error { return g.err } if f == nil || len(f.Rules) == 0 { - return fmt.Errorf("judge: fingerprint has no rules") + return fmt.Errorf("gen: fingerprint has no rules") } if len(g.positive) == 0 || len(g.negative) == 0 { - return fmt.Errorf("judge: validation requires positive and negative samples") + return fmt.Errorf("gen: validation requires positive and negative samples") } if err := g.validateSamples(); err != nil { return err } for i, rule := range f.Rules { if rule == nil { - return fmt.Errorf("judge: rule %d is nil", i) + return fmt.Errorf("gen: rule %d is nil", i) } } checking := copyFingerForValidation(f) @@ -224,15 +232,15 @@ func (g *Generator) Validate(f *fingerlib.Finger) error { for i, sample := range g.positive { frame, ok := check(sample) if !ok { - return fmt.Errorf("judge: positive sample %d did not match", i) + return fmt.Errorf("gen: positive sample %d did not match", i) } if sample.version != "" && frame.Version != sample.version { - return fmt.Errorf("judge: positive sample %d version = %q, want %q", i, frame.Version, sample.version) + return fmt.Errorf("gen: positive sample %d version = %q, want %q", i, frame.Version, sample.version) } } for i, sample := range g.negative { if _, ok := check(sample); ok { - return fmt.Errorf("judge: negative sample %d matched", i) + return fmt.Errorf("gen: negative sample %d matched", i) } } return nil @@ -240,15 +248,15 @@ func (g *Generator) Validate(f *fingerlib.Finger) error { func (g *Generator) validateSamples() error { for _, sample := range append(append([]generatorSample(nil), g.positive...), g.negative...) { - if _, err := NewPage(sample.raw); err != nil { - return fmt.Errorf("judge: invalid sample: %w", err) + if _, err := judge.NewPage(sample.raw); err != nil { + return fmt.Errorf("gen: invalid sample: %w", err) } for request, response := range sample.probes { if request == "" { - return fmt.Errorf("judge: probe has empty send_data") + return fmt.Errorf("gen: probe has empty send_data") } - if _, err := NewPage(response); err != nil { - return fmt.Errorf("judge: invalid probe response: %w", err) + if _, err := judge.NewPage(response); err != nil { + return fmt.Errorf("gen: invalid probe response: %w", err) } } } @@ -284,7 +292,7 @@ type ruleCandidate struct { } func candidates(raw []byte) []ruleCandidate { - p, err := NewPage(raw) + p, err := judge.NewPage(raw) if err != nil { return nil } @@ -297,7 +305,7 @@ func candidates(raw []byte) []ruleCandidate { } for k, v := range p.Headers { if k == "Server" || strings.HasPrefix(k, "X-") || k == "Product" { - value := strings.TrimSpace(nameVersion.ReplaceAllString(strings.Split(v, "/")[0], "")) + value := strings.TrimSpace(evidence.NameVersion.ReplaceAllString(strings.Split(v, "/")[0], "")) if strings.HasPrefix(k, "X-") && (strings.HasSuffix(k, "-Version") || value == "" || value[0] >= '0' && value[0] <= '9') { add("header", k+":", 6) } else { @@ -307,8 +315,8 @@ func candidates(raw []byte) []ruleCandidate { } add("body", p.Generator, 6) // Preserve the declaration, but not a single release or customized title. - if name := nameVersion.ReplaceAllString(p.Generator, ""); name != "" { - if loc := reGenerator.FindSubmatchIndex(raw); loc != nil { + if name := evidence.NameVersion.ReplaceAllString(p.Generator, ""); name != "" { + if loc := evidence.Generator.FindSubmatchIndex(raw); loc != nil { start := loc[2] if start < 0 { start = loc[4] @@ -320,8 +328,8 @@ func candidates(raw []byte) []ruleCandidate { add("body", string(raw[loc[0]:end]), 8) } } - if !genericName(NormalizeName(p.Title)) { - if title := reTitle.Find(raw); len(title) > 0 { + if !evidence.GenericName(evidence.NormalizeName(p.Title)) { + if title := evidence.Title.Find(raw); len(title) > 0 { add("body", string(title), 5) } add("body", p.Title, 4) @@ -333,7 +341,7 @@ func candidates(raw []byte) []ruleCandidate { for _, comment := range p.Comments { add("body", comment, 2) } - for _, name := range pageNames(p) { + for _, name := range evidence.Names(p.Generator, p.Title, p.Text, p.Headers, append(append([]string(nil), p.Scripts...), p.Styles...)) { if strings.Contains(strings.ToLower(p.Text), strings.ToLower(name)) { add("body", name, 1) } @@ -395,7 +403,7 @@ func selectCandidates(positive, negative [][]byte, product ...string) []ruleCand for _, c := range candidates(raw) { // A product must not inherit its proxy/CDN or an unrelated security // header merely because that header is absent from a small negative set. - if c.kind == "header" && len(product) > 0 && !strings.Contains(NormalizeName(c.value), NormalizeName(product[0])) { + if c.kind == "header" && len(product) > 0 && !strings.Contains(evidence.NormalizeName(c.value), evidence.NormalizeName(product[0])) { continue } key := c.kind + "\x00" + c.value @@ -564,22 +572,22 @@ func versionPattern(name string, positive [][]byte, versions []string, negative func versionPatternAt(raw string, index int, name string) string { var pattern string // A generator meta tag is stronger than an unscoped product mention. - for _, loc := range reGenerator.FindAllStringSubmatchIndex(raw, -1) { + for _, loc := range evidence.Generator.FindAllStringSubmatchIndex(raw, -1) { start, end := loc[2], loc[3] if start < 0 { start, end = loc[4], loc[5] } - if index >= start && index < end && strings.Contains(NormalizeName(raw[start:index]), NormalizeName(name)) { - return regexp.QuoteMeta(raw[loc[0]:index]) + `(` + versionToken + `|[0-9]+)` + if index >= start && index < end && strings.Contains(evidence.NormalizeName(raw[start:index]), evidence.NormalizeName(name)) { + return regexp.QuoteMeta(raw[loc[0]:index]) + `(` + evidence.VersionToken + `|[0-9]+)` } } if headerEnd := strings.Index(raw, "\r\n\r\n"); headerEnd >= 0 && index < headerEnd { lineStart := strings.LastIndex(raw[:index], "\n") + 1 prefix := strings.TrimSpace(raw[lineStart:index]) - if len(prefix) > 48 || !strings.Contains(NormalizeName(prefix), NormalizeName(name)) { + if len(prefix) > 48 || !strings.Contains(evidence.NormalizeName(prefix), evidence.NormalizeName(name)) { return "" } - pattern = `(?m)^` + regexp.QuoteMeta(prefix) + `\s*(` + versionToken + `|[0-9]+)` + pattern = `(?m)^` + regexp.QuoteMeta(prefix) + `\s*(` + evidence.VersionToken + `|[0-9]+)` } else { begin := index - 60 if begin < 0 { @@ -594,7 +602,7 @@ func versionPatternAt(raw string, index int, name string) string { if len(prefix) > 48 || strings.ContainsAny(prefix, "\r\n<>") { return "" } - pattern = regexp.QuoteMeta(strings.TrimSpace(prefix)) + `\s*(` + versionToken + `|[0-9]+)` + pattern = regexp.QuoteMeta(strings.TrimSpace(prefix)) + `\s*(` + evidence.VersionToken + `|[0-9]+)` } return pattern } @@ -606,7 +614,7 @@ func guardedVersionPatterns(name string, positive [][]byte, versions []string, n var patterns []string seen := map[string]bool{} for _, guard := range selectCandidates(positive, negative, name) { - if guard.weight < 4 || !strings.Contains(NormalizeName(guard.value), NormalizeName(name)) { + if guard.weight < 4 || !strings.Contains(evidence.NormalizeName(guard.value), evidence.NormalizeName(name)) { continue } for i, version := range versions { @@ -626,8 +634,8 @@ func guardedVersionPatterns(name string, positive [][]byte, versions []string, n continue } prefix := regexp.QuoteMeta(raw[begin:index]) - prefix = reSpace.ReplaceAllStringFunc(prefix, func(string) string { return `\s+` }) - pattern := `(?s)` + regexp.QuoteMeta(guard.value) + `.*?` + prefix + `(` + versionToken + `)(?:[^0-9A-Za-z.+-]|$)` + prefix = whitespace.ReplaceAllStringFunc(prefix, func(string) string { return `\s+` }) + pattern := `(?s)` + regexp.QuoteMeta(guard.value) + `.*?` + prefix + `(` + evidence.VersionToken + `)(?:[^0-9A-Za-z.+-]|$)` if seen[pattern] { continue } diff --git a/judge/gen/gen_test.go b/judge/gen/gen_test.go new file mode 100644 index 0000000..9299d6c --- /dev/null +++ b/judge/gen/gen_test.go @@ -0,0 +1,204 @@ +package gen + +import ( + "bytes" + "context" + "errors" + "fmt" + "strings" + "testing" + + fingerlib "github.com/chainreactors/fingers/fingers" + "github.com/chainreactors/fingers/judge" +) + +// names answers yes for the product named in backticks, no otherwise. +type names string + +func (names) ID() string { return "names" } + +func (n names) Judge(_ context.Context, _ interface{}, qs map[string]judge.Question) (map[string]judge.Answer, error) { + out := map[string]judge.Answer{} + for key, q := range qs { + if strings.Contains(q.Instructions, "`"+string(n)+"`") { + out[key] = judge.Answer{Yes: 0.95} + } else { + out[key] = judge.Answer{Yes: 0.05} + } + } + return out, nil +} + +func TestGeneratorPassiveVersionAndValidation(t *testing.T) { + positive1 := []byte("HTTP/1.1 200 OK\r\nServer: nginx/2.401.3\r\nX-Jenkins: 2.401.3\r\n\r\nJenkins") + positive2 := []byte("HTTP/1.1 200 OK\r\nServer: nginx/2.402.1\r\nX-Jenkins: 2.402.1\r\n\r\nJenkins") + negative := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nBlog

X-Jenkins: 9.9.9 release notes

") + g := New(nil).Name("Jenkins").PositiveVersion(positive1, "2.401.3").PositiveVersion(positive2, "2.402.1").Negative(negative) + f, err := g.Generate(context.Background()) + if err != nil { + t.Fatal(err) + } + if f.Name != "Jenkins" || len(f.Rules) < 2 { + t.Fatalf("fingerprint: %+v", f) + } + for _, sample := range []struct { + raw []byte + version string + }{{positive1, "2.401.3"}, {positive2, "2.402.1"}} { + frame, _, ok := f.PassiveMatch(fingerlib.NewContent(sample.raw, "", true)) + if !ok || frame.Version != sample.version { + t.Fatalf("version matcher = %+v, %v", frame, ok) + } + } + compiled := len(f.Rules[0].Regexps.CompliedRegexp) + if err := g.Validate(f); err != nil || len(f.Rules[0].Regexps.CompliedRegexp) != compiled { + t.Fatalf("Validate mutated compiled matcher: %v", err) + } + if err := g.Validate(&fingerlib.Finger{Name: "Jenkins", Rules: fingerlib.Rules{&fingerlib.Rule{Regexps: &fingerlib.Regexps{Body: []string{"jenkins"}}}}}); err == nil { + t.Fatal("validator accepted rule matching negative sample") + } +} + +func TestGeneratorActiveAndProbeWith(t *testing.T) { + base := []byte("HTTP/1.1 200 OK\r\n\r\nLogin") + request := []byte("/admin") + positive := []byte("HTTP/1.1 200 OK\r\nX-Orion: console\r\n\r\nOrion console") + negative := []byte("HTTP/1.1 404 Not Found\r\n\r\nmissing") + g := New(nil).Name("Orion").Positive(base) + if err := g.ProbeWith(context.Background(), request, func(_ context.Context, got []byte) ([]byte, error) { + if !bytes.Equal(got, request) { + return nil, errors.New("wrong request") + } + return positive, nil + }); err != nil { + t.Fatal(err) + } + g.Negative(base).Probe(request, negative) + f, err := g.Generate(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(f.Rules) != 1 || f.Rules[0].SendDataStr != string(request) || !f.IsActive { + t.Fatalf("active rule: %+v", f) + } + if err := g.Validate(f); err != nil { + t.Fatal(err) + } + if len(f.Rules[0].Regexps.CompliedRegexp) != 0 { + t.Fatal("Validate changed compiled rules") + } + if _, err := New(nil).Name("Orion").Positive(base).Probe(request, positive).Negative(base).Generate(context.Background()); err == nil { + t.Fatal("generated active rule without a negative probe") + } +} + +func TestGeneratorSelectsNameByDefault(t *testing.T) { + positive := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nOrion

Orion console

") + negative := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nOther") + j := judge.New(names("Orion")) + f, err := New(j).Positive(positive).Negative(negative).Generate(context.Background()) + if err != nil || f.Name != "Orion" { + t.Fatalf("auto name = %+v, %v", f, err) + } +} + +func TestGeneratorActiveVersion(t *testing.T) { + base := []byte("HTTP/1.1 200 OK\r\n\r\nLogin") + request := []byte("/version") + positive := []byte("HTTP/1.1 200 OK\r\nX-Orion: 3.1.2\r\n\r\nversion") + negative := []byte("HTTP/1.1 200 OK\r\n\r\nnot installed") + g := New(nil).Name("Orion").PositiveVersion(base, "3.1.2").Probe(request, positive).Negative(base).Probe(request, negative) + f, err := g.Generate(context.Background()) + if err != nil { + t.Fatal(err) + } + frame, _, ok := f.ActiveMatch(2, func(got []byte) ([]byte, bool) { return positive, bytes.Equal(got, request) }) + if !ok || frame.Version != "3.1.2" { + t.Fatalf("active version = %+v, %v", frame, ok) + } +} + +func TestGeneratorCoversDifferentPositiveVariants(t *testing.T) { + base := "HTTP/1.1 200 OK\r\n%s\r\nWelcome" + first := []byte(fmt.Sprintf(base, "X-Orion-Instance: ready\r\n")) + second := []byte(fmt.Sprintf(base, "X-Orion-Service: yes\r\n")) + other := []byte(fmt.Sprintf(base, "Server: nginx\r\n")) + g := New(nil).Name("Orion").Positive(first).Positive(second).Negative(other) + f, err := g.Generate(context.Background()) + if err != nil || len(f.Rules) != 2 { + t.Fatalf("variant rules = %+v, %v", f, err) + } +} + +func TestGeneratedVersionHandlesHyphenatedHeaderAndReleaseSuffix(t *testing.T) { + raw := func(v string) []byte { + return []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\nX-New-Api-Version: v" + v + "\r\n\r\nNew API") + } + neg := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nWelcome") + g := New(nil).Name("New API").PositiveVersion(raw("1.0.0-rc.35"), "1.0.0-rc.35").Negative(neg) + f, err := g.Generate(context.Background()) + if err != nil { + t.Fatal(err) + } + for _, version := range []string{"1.0.0-rc.35", "1.2.0-rc.2", "2.0.1"} { + frame, _, matched := f.PassiveMatch(fingerlib.NewContent(raw(version), "", true)) + if !matched || frame.Version != version { + t.Fatalf("version=%s matched=%t frame=%+v", version, matched, frame) + } + } +} + +func TestGeneratedMetaRuleSurvivesTitleAndCalendarVersionChanges(t *testing.T) { + page := func(title, version string) []byte { + return []byte("HTTP/1.1 200 OK\r\n\r\n" + title + "") + } + negative := []byte("HTTP/1.1 200 OK\r\n\r\nSearXNG Documentation

Install SearXNG

") + f, err := New(nil).Name("SearXNG").PositiveVersion(page("Search One", "2026.9.23+3cd69d30e"), "2026.9.23+3cd69d30e").Negative(negative).Generate(context.Background()) + if err != nil { + t.Fatal(err) + } + got, _, matched := f.PassiveMatch(fingerlib.NewContent(page("Different Search", "2026.9.25+d8ae3abd5"), "", true)) + if !matched || got.Version != "2026.9.25+d8ae3abd5" { + t.Fatalf("matched=%t got=%+v", matched, got) + } + if _, _, matched := f.PassiveMatch(fingerlib.NewContent(negative, "", true)); matched { + t.Fatal("documentation matched") + } +} + +func TestGeneratedFooterVersionRequiresProductDeclaration(t *testing.T) { + page := func(title, version string) []byte { + return []byte("HTTP/1.1 200 OK\r\n\r\n" + title + "
Log in
\n v" + version + " @ mysql\n
") + } + negative := page("Different Application", "2.18.0-be6aeb4") + f, err := New(nil).Name("Wakapi").PositiveVersion(page("Wakapi – Coding Statistics", "2.18.0-be6aeb4"), "2.18.0-be6aeb4").Negative(negative).Generate(context.Background()) + if err != nil { + t.Fatal(err) + } + got, _, matched := f.PassiveMatch(fingerlib.NewContent(page("Wakapi – Coding Statistics", "2.12.0"), "", true)) + if !matched || got.Version != "2.12.0" { + t.Fatalf("matched=%t got=%+v", matched, got) + } + if _, _, matched := f.PassiveMatch(fingerlib.NewContent(negative, "", true)); matched { + t.Fatal("unguarded footer matched another product") + } +} + +func TestGeneratedApplicationDoesNotLearnSharedInfrastructure(t *testing.T) { + for _, name := range []string{"IT Tools", "Docmost"} { + positive := []byte("HTTP/1.1 200 OK\r\nServer: cloudflare\r\nX-Frame-Options: SAMEORIGIN\r\n\r\n" + name + "") + negative := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nDifferent Application") + f, err := New(nil).Name(name).Positive(positive).Negative(negative).Generate(context.Background()) + if err != nil { + t.Fatal(err) + } + sameProxy := []byte("HTTP/1.1 200 OK\r\nServer: cloudflare\r\nX-Frame-Options: SAMEORIGIN\r\n\r\nUnrelated Application") + if _, _, matched := f.PassiveMatch(fingerlib.NewContent(sameProxy, "", true)); matched { + t.Fatalf("%s learned shared infrastructure", name) + } + otherProxy := []byte("HTTP/1.1 200 OK\r\nServer: caddy\r\n\r\n" + name + "") + if _, _, matched := f.PassiveMatch(fingerlib.NewContent(otherProxy, "", true)); !matched { + t.Fatalf("%s requires original infrastructure", name) + } + } +} diff --git a/judge/internal/evidence/evidence.go b/judge/internal/evidence/evidence.go new file mode 100644 index 0000000..15c359c --- /dev/null +++ b/judge/internal/evidence/evidence.go @@ -0,0 +1,136 @@ +// Package evidence holds the text primitives shared by the judge and the +// fingerprint generator: product name folding, name candidates found in a +// response, and the patterns for titles, generator metas and versions. +package evidence + +import ( + "regexp" + "sort" + "strings" + "unicode" + "unicode/utf8" +) + +// VersionToken matches a version, keeping release suffixes and calendar +// versions (e.g. 1.0.2q, 1.0.0-rc.35, 2026.9.23+3cd69d30e). +const VersionToken = `[0-9]{1,4}(?:\.[0-9]{1,4}){1,3}(?:[A-Za-z]+[0-9]*|[-+][0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*)?` + +var ( + Title = regexp.MustCompile(`(?is)]*>(.*?)`) + Generator = regexp.MustCompile(`(?is)]+name=["']generator["'][^>]*content=["']([^"']+)|]+content=["']([^"']+)["'][^>]*name=["']generator["']`) + // NameVersion is a trailing version after a product name ("nginx/1.24", "Jenkins v2"). + NameVersion = regexp.MustCompile(`(?i)[/\s]+[vV]?(?:` + VersionToken + `|[0-9]+)\s*$`) + + space = regexp.MustCompile(`\s+`) + nameWords = regexp.MustCompile(`[A-Z][A-Za-z0-9!+._]{2,}(?:[ -][A-Z][A-Za-z0-9!+._]{2,}){0,2}|[\p{Han}]{2,20}`) +) + +// Clean decodes common entities and collapses whitespace. +func Clean(s string) string { + s = strings.NewReplacer(" ", " ", "&", "&", "<", "<", ">", ">", """, `"`).Replace(s) + return strings.TrimSpace(space.ReplaceAllString(s, " ")) +} + +var genericSuffixes = []string{"companyproducts", "公司产品", "product", "products", "产品", "operatingsystem", "操作系统", "system"} + +// genericWords are trailing words engines add to a product name ("Apache +// HTTP Server", "apache-web-server", "Discuz! X"). They are dropped only as +// whole words, so "lighttpd" keeps its "httpd". +var genericWords = map[string]bool{"http": true, "httpd": true, "server": true, "web": true, "webserver": true, "x": true, "cms": true, "oa": true} + +// NormalizeName folds the spellings different engines use for one product +// ("Apache-Tomcat", "apache tomcat", "Apache HTTP Server", "apache") into one +// key. It never merges names whose remaining letters differ. +func NormalizeName(name string) string { + words := strings.FieldsFunc(strings.ToLower(name), func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) }) + for len(words) > 1 && genericWords[words[len(words)-1]] { + words = words[:len(words)-1] + } + key := strings.Join(words, "") + for _, s := range genericSuffixes { + if len(key) > len(s)+2 && strings.HasSuffix(key, s) { + key = strings.TrimSuffix(key, s) + } + } + return key +} + +// UsableKey drops keys too short to be meaningful evidence ("oa", "cms"). +func UsableKey(k string) bool { + n := utf8.RuneCountInString(k) + if n == len(k) { // ascii + return n >= 4 + } + return n >= 2 +} + +// GenericName reports whether a normalized name is a generic interface word +// ("login", "控制台") rather than a product. +func GenericName(key string) bool { + switch key { + case "login", "signin", "welcome", "dashboard", "admin", "console", "home", "index", "server", "error", "notfound", "password", "username", "management", "system", "登录", "首页", "管理", "用户", "密码", "控制台": + return true + } + return false +} + +// Names returns up to 20 product name candidates found in a response, most +// telling source first: generator meta, product headers, title parts, asset +// path words, then capitalized words and CJK runs of the visible text. +func Names(generator, title, text string, headers map[string]string, assets []string) []string { + type candidate struct { + name string + weight int + } + byKey := map[string]candidate{} + add := func(text string, weight int) { + text = strings.Trim(strings.TrimSpace(NameVersion.ReplaceAllString(Clean(text), "")), "-:|/ ") + if text == "" || utf8.RuneCountInString(text) > 40 || !UsableKey(strings.ToLower(text)) { + return + } + key := NormalizeName(text) + if key == "" || GenericName(key) { + return + } + if prev, ok := byKey[key]; !ok || weight > prev.weight { + byKey[key] = candidate{text, weight} + } + } + add(generator, 5) + for k, v := range headers { + if strings.HasPrefix(k, "X-") || k == "Server" || k == "Product" { + add(strings.Split(v, "/")[0], 4) + } + } + for _, part := range strings.FieldsFunc(title, func(r rune) bool { return r == '|' || r == '-' || r == ':' || r == '–' }) { + add(part, 4) + } + for _, asset := range assets { + for _, word := range strings.FieldsFunc(asset, func(r rune) bool { return r == '/' || r == '_' }) { + add(word, 2) + } + } + for _, segment := range strings.FieldsFunc(text, func(r rune) bool { return r == '.' || r == ':' || r == '。' || r == '：' || r == '|' || r == '，' }) { + for _, match := range nameWords.FindAllString(segment, 20) { + add(match, 1) + } + } + list := make([]candidate, 0, len(byKey)) + for _, c := range byKey { + list = append(list, c) + } + sort.Slice(list, func(a, b int) bool { + if list[a].weight != list[b].weight { + return list[a].weight > list[b].weight + } + return list[a].name < list[b].name + }) + if len(list) > 20 { + list = list[:20] + } + out := make([]string, len(list)) + for i, c := range list { + out[i] = c.name + } + return out +} diff --git a/judge/judge.go b/judge/judge.go index ac58171..f45d2d4 100644 --- a/judge/judge.go +++ b/judge/judge.go @@ -7,6 +7,7 @@ import ( "encoding/json" "fmt" "math" + "math/bits" "sort" "strings" "sync" @@ -20,13 +21,21 @@ import ( type Judge struct { Provider Provider // Cache keeps answers per question, so a question asked again about the - // same or a similar page (see SimilarDistance) is not sent again, however - // questions are grouped into rounds. New sets an in-memory cache; nil - // disables it. Concurrent rounds are merged either way. + // same or a similar page is not sent again, however questions are grouped + // into requests. New sets an in-memory cache; nil disables it. Concurrent + // requests are merged either way. Cache Cache + // Known finds fingerprint names that occur in a page (typically + // NewRetriever(engine.Names())). Refine and Inspect judge those the rules + // did not report, and add the confirmed ones as Recalled. nil disables recall. + Known *Retriever // Threshold is the Yes probability that counts as "yes"; VersionConfidence // the confidence at which a picked version is adopted. Threshold, VersionConfidence float64 + // SimilarDistance is how many of the 64 signature bits two pages with the + // same title may differ in to share answers; 0 shares answers only between + // identical signatures. See DefaultSimilarDistance. + SimilarDistance int // Totals, for cost monitoring; read with atomic.LoadInt64. Requests went // to the provider; CacheHits are rounds answered without one. @@ -36,13 +45,19 @@ type Judge struct { flights map[string][]*flight } -// DefaultCacheSize is the number of answers the cache New creates holds. -const DefaultCacheSize = 1 << 16 +const ( + // DefaultCacheSize is the number of answers the cache New creates holds. + DefaultCacheSize = 1 << 16 + // DefaultSimilarDistance was measured with cmd/judgeeval on 1070 real + // pages: every answer reused at distance <= 1 matched the page's own + // answer (159/159), at 2 it was 98.5%, at 3 96.9%. + DefaultSimilarDistance = 1 +) // New returns a Judge with an in-memory cache and the provider's // calibration, or 0.5 / 0.9 for providers that report none. func New(p Provider) *Judge { - j := &Judge{Provider: p, Cache: NewMemoryCache(DefaultCacheSize), Threshold: 0.5, VersionConfidence: 0.9} + j := &Judge{Provider: p, Cache: NewMemoryCache(DefaultCacheSize), Threshold: 0.5, VersionConfidence: 0.9, SimilarDistance: DefaultSimilarDistance} if c, ok := p.(Calibrated); ok { j.Threshold, j.VersionConfidence = c.Calibration() } @@ -50,7 +65,7 @@ func New(p Provider) *Judge { } // Ask asks questions about state, which must match exactly for cached -// answers to apply. Most callers use Round, which also matches similar pages. +// answers to apply. Yes, Choose and Score are the one-question forms. func (j *Judge) Ask(ctx context.Context, state interface{}, questions map[string]Question) (map[string]Answer, error) { scope, err := json.Marshal(state) if err != nil { @@ -189,7 +204,7 @@ func (j *Judge) cached(key string, sig uint64) (Answer, bool) { if j.Cache == nil { return a, false } - v, ok := j.Cache.Get(key, sig) + v, ok := j.Cache.Get(key, sig, j.SimilarDistance) return a, ok && json.Unmarshal(v, &a) == nil } @@ -197,7 +212,7 @@ func (j *Judge) join(key string, sig uint64) (*flight, bool) { j.mu.Lock() defer j.mu.Unlock() for _, f := range j.flights[key] { - if Similar(f.sig, sig) { + if bits.OnesCount64(f.sig^sig) <= j.SimilarDistance { return f, false } } diff --git a/judge/judge_test.go b/judge/judge_test.go index 8def9e0..1f73f32 100644 --- a/judge/judge_test.go +++ b/judge/judge_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "math/bits" "sort" "strings" "sync" @@ -34,12 +35,9 @@ func TestNewPage(t *testing.T) { if got := extractVersions([]byte(jenkinsRaw), 5); len(got) != 2 || got[0].Value != "1.24.0" || got[1].Value != "2.401.3" { t.Fatalf("versions: %v", got) } - if got := NewRetriever([]string{"jenkins", "oa", "tomcat"}).Find(s.Haystack(), 5); len(got) != 1 || got[0] != "jenkins" { + if got := NewRetriever([]string{"jenkins", "oa", "tomcat"}).Find(s.haystack(), 5); len(got) != 1 || got[0] != "jenkins" { t.Fatalf("retriever: %v", got) } - if b, _ := json.Marshal(s); strings.Contains(string(b), "Kind") || strings.Contains(string(b), "Generic") { - t.Fatalf("conclusions leak into the state: %s", b) - } } // mock is a Provider answering from a table keyed by question kind and the @@ -52,7 +50,8 @@ type mock struct { fail bool versionP float64 present map[string]float64 - layer map[string]Layer + layer map[string]string + versions map[string]string // product -> version picked, if among the options primary string lastState map[string]interface{} } @@ -61,17 +60,19 @@ func newMock() *mock { return &mock{ versionP: 0.95, present: map[string]float64{"nginx": 0.1, "wordpress": 0.05, "jenkins": 0.97, "apache tomcat": 0.8, "prototype": 0.9}, - layer: map[string]Layer{"nginx": LayerServer, "wordpress": LayerNotPresent, "jenkins": LayerApplication, + versions: map[string]string{"jenkins": "2.401.3", "nginx": "1.24.0"}, + layer: map[string]string{"nginx": LayerServer, "wordpress": LayerNotPresent, "jenkins": LayerApplication, "apache tomcat": LayerServer, "prototype": LayerFrontend}, primary: "jenkins", } } +// named is the first product named in backticks, skipping evidence names. func named(q Question) string { - s := q.Instructions - if i := strings.Index(s, "`"); i >= 0 { - if j := strings.Index(s[i+1:], "`"); j >= 0 { - return s[i+1 : i+1+j] + parts := strings.Split(q.Instructions, "`") + for i := 1; i < len(parts); i += 2 { + if parts[i] != "version_strings" { + return parts[i] } } return "" @@ -104,14 +105,11 @@ func (m *mock) Judge(ctx context.Context, state interface{}, questions map[strin answers[k] = Answer{Choice: "login", Confidence: 0.9} case k == "generic": answers[k] = Answer{Yes: 0.9} - case k == "version": - choice := "2.401.3" - if _, ok := q.Options[choice]; !ok { - for candidate := range q.Options { - if candidate != notStated { - choice = candidate - break - } + case strings.HasPrefix(k, "version_"): + choice := notStated + if v, ok := m.versions[name]; ok { + if _, offered := q.Options[v]; offered { + choice = v } } answers[k] = Answer{Choice: choice, Confidence: m.versionP} @@ -124,9 +122,6 @@ func (m *mock) Judge(ctx context.Context, state interface{}, questions map[strin return answers, nil } -// client keeps the old test shape: a Judge over the mock. -func (m *mock) client(t *testing.T) (*Judge, func()) { return New(m), func() {} } - func testFrames() common.Frameworks { fs := common.Frameworks{} fs.Add(common.NewFramework("nginx", common.FrameFromFingers)) @@ -137,73 +132,124 @@ func testFrames() common.Frameworks { return fs } -func refine(t *testing.T, c *Judge, frames common.Frameworks) (*Page, error) { - p, err := NewPage([]byte(jenkinsRaw)) - if err != nil { - t.Fatal(err) - } - out, err := runRefine(context.Background(), c, p, frames, []string{"Jenkins", "Prototype"}) - if err == nil { - for name, f := range out { - frames[name] = f - } - } - return p, err +// newJudge is a Judge over m that recalls Jenkins and Prototype by name. +func newJudge(m *mock) *Judge { + j := New(m) + j.Known = NewRetriever([]string{"Jenkins", "Prototype"}) + return j } +// bodyRaw states versions only in the body, so the provider has to pick them. +const bodyRaw = "HTTP/1.1 200 OK\r\nServer: nginx\r\nSet-Cookie: JSESSIONID.1=x; Path=/\r\n\r\n" + + `Sign in [Jenkins]` + + `

We moved here from WordPress.

Jenkins 2.401.3, nginx 1.24.0
` + func TestRefine(t *testing.T) { m := newMock() - c, stop := m.client(t) - defer stop() - frames := testFrames() - p, err := refine(t, c, frames) + j := newJudge(m) + all, err := j.Inspect(context.Background(), []byte(jenkinsRaw), testFrames()) if err != nil { t.Fatal(err) } - if !Is(frames["wordpress"], Rejected) { - t.Errorf("text-only wordpress not rejected: %v", frames["wordpress"].Tags) + if w := all["wordpress"]; w.Judge == nil || !w.Judge.Rejected { + t.Errorf("text-only wordpress not rejected: %+v", w.Judge) } - if nginx := frames["nginx"]; Is(nginx, Rejected) || LayerOf(nginx) != LayerServer { - t.Errorf("nginx from the Server header must be kept: %v", nginx.Tags) + if nginx := all["nginx"].Judge; nginx.Rejected || nginx.Layer != LayerServer || nginx.Confidence != 0.1 { + t.Errorf("nginx from the Server header must be kept: %+v", nginx) } - if Is(frames["apache tomcat"], Duplicate) == Is(frames["apache-tomcat"], Duplicate) { + if all["apache tomcat"].Judge.Duplicate == all["apache-tomcat"].Judge.Duplicate { t.Errorf("exactly one tomcat spelling must be a dup") } - j := frames["jenkins"] - if !Is(j, Primary) || j.Version != "2.401.3" || PrimaryOf(frames) != j { - t.Errorf("jenkins primary/version: %v %q", j.Tags, j.Version) + if p := all["prototype"]; p == nil || !p.Judge.Recalled || !p.IsGuess() { + t.Errorf("confirmed recall not added: %+v", p) } - if f := frames["prototype"]; f == nil || !Is(f, Recalled) || !f.IsGuess() { - t.Errorf("confirmed recall not added: %+v", f) + // Jenkins is both a rule hit and a recall name: asked once. 5 products x 2 + primary + page_kind + generic. + if len(m.requests) != 1 || len(m.requests[0]) != 13 { + t.Fatalf("requests: %v", m.requests) } - if got := Accepted(frames); len(got) != 4 { // nginx, tomcat, jenkins, prototype - t.Errorf("accepted: %v", got) + + accepted, err := j.Refine(context.Background(), []byte(jenkinsRaw), testFrames()) + if err != nil { + t.Fatal(err) } - if p.Kind != KindLogin || !p.Generic { - t.Errorf("page: %q %v", p.Kind, p.Generic) + if len(accepted) != 4 { // nginx, tomcat, jenkins, prototype + t.Errorf("accepted: %v", accepted) } - // Jenkins is both a rule hit and a recall name: asked once. 5 products x 2 + primary + page_kind + generic. - if len(m.requests) != 2 || len(m.requests[0]) != 13 || strings.Join(m.requests[1], ",") != "version" { - t.Errorf("requests: %v", m.requests) + jenkins := accepted["jenkins"] + if !jenkins.Judge.Primary || accepted.Primary() != jenkins { + t.Errorf("jenkins primary: %+v", jenkins.Judge) + } + // Both versions are bound by name in the headers: taken without asking, + // and not offered to the products they do not name. + if jenkins.Version != "2.401.3" || accepted["nginx"].Version != "1.24.0" || accepted["prototype"].Version != "" { + t.Errorf("versions: jenkins %q nginx %q prototype %q", jenkins.Version, accepted["nginx"].Version, accepted["prototype"].Version) + } + if len(m.requests) != 1 || j.CacheHits != 1 { + t.Errorf("header versions or cached verdicts were asked again: %v hits=%d", m.requests, j.CacheHits) + } + if kind, generic, err := j.Classify(context.Background(), []byte(jenkinsRaw)); err != nil || kind != KindLogin || !generic || len(m.requests) != 1 { + t.Errorf("classify after refine: %q %v %v, requests %v", kind, generic, err, m.requests) + } +} + +// Versions stated only in the body are picked by the provider, all kept +// products in one request, each from its own candidates. +func TestRefineVersionsEveryProduct(t *testing.T) { + m := newMock() + j := newJudge(m) + accepted, err := j.Refine(context.Background(), []byte(bodyRaw), testFrames()) + if err != nil { + t.Fatal(err) + } + if accepted["jenkins"].Version != "2.401.3" || accepted["nginx"].Version != "1.24.0" { + t.Fatalf("versions: jenkins %q nginx %q", accepted["jenkins"].Version, accepted["nginx"].Version) + } + // jenkins (primary), apache tomcat and nginx (servers), prototype (frontend). + if len(m.requests) != 2 || strings.Join(m.requests[1], ",") != "version_0,version_1,version_2,version_3" { + t.Fatalf("requests: %v", m.requests) } if _, ok := m.lastState["version_strings"]; !ok { t.Errorf("version strings not in the state: %v", m.lastState) } - if c.Requests != 2 { - t.Errorf("requests: %d", c.Requests) +} + +// One version string occurring once goes to one product only. +func TestVersionGoesToOneProduct(t *testing.T) { + m := newMock() + m.versions = map[string]string{"jenkins": "2.401.3", "nginx": "2.401.3"} + raw := strings.Replace(bodyRaw, "Jenkins 2.401.3, nginx 1.24.0", "Jenkins 2.401.3", 1) + accepted, err := newJudge(m).Refine(context.Background(), []byte(raw), testFrames()) + if err != nil { + t.Fatal(err) + } + if accepted["jenkins"].Version != "2.401.3" || accepted["nginx"].Version != "" { + t.Fatalf("versions: jenkins %q nginx %q", accepted["jenkins"].Version, accepted["nginx"].Version) + } +} + +func TestDeclarations(t *testing.T) { + raw := []byte("HTTP/1.1 200 OK\r\nServer: Apache/2.4.38 (Debian) OpenSSL/1.0.2q PHP/5.6.40\r\nX-Jenkins: 2.401.3\r\n" + + "X-Gitea-Version: 1.21.4\r\nX-Tomcat: Apache Tomcat/9.0.1\r\n\r\n") + decls := declarations(raw) + for product, want := range map[string]string{"Apache": "2.4.38", "openssl": "1.0.2q", "PHP": "5.6.40", "jenkins": "2.401.3", + "Gitea": "1.21.4", "Apache Tomcat": "9.0.1", "WordPress": "7.0.3", "nginx": ""} { + if got := declaredVersion(decls, product); got != want { + t.Errorf("%s: got %q want %q", product, got, want) + } + } + if !claimedByOther(decls, "5.6.40", NormalizeName("ThinkPHP")) || claimedByOther(decls, "5.6.40", NormalizeName("php")) { + t.Error("claims") } } func TestVersionNeedsConfidence(t *testing.T) { m := newMock() m.versionP = 0.6 - c, stop := m.client(t) - defer stop() - frames := testFrames() - if _, err := refine(t, c, frames); err != nil { + accepted, err := newJudge(m).Refine(context.Background(), []byte(bodyRaw), testFrames()) + if err != nil { t.Fatal(err) } - if v := frames["jenkins"].Version; v != "" { + if v := accepted["jenkins"].Version; v != "" { t.Fatalf("low-confidence version written: %q", v) } } @@ -211,60 +257,57 @@ func TestVersionNeedsConfidence(t *testing.T) { func TestFailedRoundLeavesFrames(t *testing.T) { m := newMock() m.fail = true - c, stop := m.client(t) - defer stop() frames := testFrames() before, _ := json.Marshal(frames) - p, err := refine(t, c, frames) - if err == nil { + accepted, err := newJudge(m).Refine(context.Background(), []byte(jenkinsRaw), frames) + if err == nil || accepted != nil { t.Fatal("expected error") } - if after, _ := json.Marshal(frames); string(after) != string(before) || p.Kind != "" { + if after, _ := json.Marshal(frames); string(after) != string(before) { t.Fatalf("frames changed on failure:\n%s\n%s", before, after) } } func TestCache(t *testing.T) { m := newMock() - c, stop := m.client(t) - defer stop() - c.Cache = NewMemoryCache(16) + j := newJudge(m) for i := 0; i < 2; i++ { - frames := testFrames() - if _, err := refine(t, c, frames); err != nil { + accepted, err := j.Refine(context.Background(), []byte(bodyRaw), testFrames()) + if err != nil { t.Fatal(err) } - if frames["jenkins"].Version != "2.401.3" { + if accepted["jenkins"].Version != "2.401.3" { t.Fatalf("run %d: cached answers not applied", i) } } - if m.calls != 2 || c.CacheHits != 2 { - t.Fatalf("calls=%d hits=%d", m.calls, c.CacheHits) + if m.calls != 2 || j.CacheHits != 2 { + t.Fatalf("calls=%d hits=%d", m.calls, j.CacheHits) } } func TestClassifyAlone(t *testing.T) { m := newMock() - c, stop := m.client(t) - defer stop() + j := New(m) p, _ := NewPage([]byte(jenkinsRaw)) - r := p.Round() - Classify(r) + r := newRound(p) + var kind Kind + var generic bool + classifyRound(r, &kind, &generic) var custom float64 - r.Add("honeypot", Binary("Is this a honeypot?"), func(a Answer) { custom = a.Yes }) - if err := r.Ask(context.Background(), c); err != nil { + r.add("honeypot", Binary("Is this a honeypot?"), func(a Answer) { custom = a.Yes }) + if err := r.ask(context.Background(), j); err != nil { t.Fatal(err) } - if strings.Join(m.requests[0], ",") != "generic,honeypot,page_kind" || p.Kind != KindLogin || custom != 0.7 { - t.Fatalf("requests=%v kind=%q custom=%v", m.requests, p.Kind, custom) + if strings.Join(m.requests[0], ",") != "generic,honeypot,page_kind" || kind != KindLogin || !generic || custom != 0.7 { + t.Fatalf("requests=%v kind=%q custom=%v", m.requests, kind, custom) } - if err := p.Round().Ask(context.Background(), c); err != nil || m.calls != 1 { + if err := newRound(p).ask(context.Background(), j); err != nil || m.calls != 1 { t.Fatalf("empty round sent a request: %v %d", err, m.calls) } - r = p.Round() - r.Add("x", Binary("?"), nil) - r.Add("x", Binary("?"), nil) - if err := r.Ask(context.Background(), c); err == nil { + r = newRound(p) + r.add("x", Binary("?"), nil) + r.add("x", Binary("?"), nil) + if err := r.ask(context.Background(), j); err == nil { t.Fatal("duplicate key accepted") } } @@ -273,21 +316,24 @@ func TestMemoryCacheEvicts(t *testing.T) { c := NewMemoryCache(2) c.Put("a", 0, []byte("1")) c.Put("b", 0, []byte("2")) - c.Get("a", 0) + c.Get("a", 0, 0) c.Put("c", 0, []byte("3")) - if _, ok := c.Get("b", 0); ok { + if _, ok := c.Get("b", 0, 0); ok { t.Fatal("least recently used entry kept") } - if v, ok := c.Get("a", 0); !ok || string(v) != "1" { + if v, ok := c.Get("a", 0, 0); !ok || string(v) != "1" { t.Fatal("recent entry evicted") } - if _, ok := c.Get("a", 0xff); ok { + if _, ok := c.Get("a", 0xff, 1); ok { t.Fatal("signature 8 bits away matched") } - if v, ok := c.Get("a", 0x1); !ok || string(v) != "1" { + if v, ok := c.Get("a", 0x1, 1); !ok || string(v) != "1" { t.Fatal("signature 1 bit away missed") } - if _, ok := c.Get("a", 0x3); ok { + if _, ok := c.Get("a", 0x1, 0); ok { + t.Fatal("signature 1 bit away matched at distance 0") + } + if _, ok := c.Get("a", 0x3, 1); ok { t.Fatal("signature 2 bits away matched") } } @@ -295,10 +341,6 @@ func TestMemoryCacheEvicts(t *testing.T) { // The same product page on another host (other token, date, host name) reuses // the answers; a different page does not. func TestSimilarPagesShareAnswers(t *testing.T) { - m := newMock() - c, stop := m.client(t) - defer stop() - c.Cache = NewMemoryCache(16) // Same page on another host: other session, build number and date. page := func(build, date string) string { return strings.Replace(jenkinsRaw, "We moved here from WordPress.", "We moved here from WordPress. Build "+build+", "+date+".", 1) @@ -307,19 +349,22 @@ func TestSimilarPagesShareAnswers(t *testing.T) { a, _ := NewPage([]byte(orig)) b, _ := NewPage([]byte(variant)) other, _ := NewPage([]byte("HTTP/1.1 200 OK\r\nServer: Apache\r\n\r\nIndex of /a.txt")) - if !Similar(a.Signature(), b.Signature()) || Similar(a.Signature(), other.Signature()) { - t.Fatalf("signatures: %x %x %x", a.Signature(), b.Signature(), other.Signature()) - } - for _, raw := range []string{orig, variant} { - p, _ := NewPage([]byte(raw)) - r := p.Round() - Classify(r) - if err := r.Ask(context.Background(), c); err != nil || p.Kind != KindLogin { - t.Fatalf("kind %q err %v", p.Kind, err) + near := func(x, y *Page) bool { return bits.OnesCount64(x.signature()^y.signature()) <= DefaultSimilarDistance } + if !near(a, b) || near(a, other) { + t.Fatalf("signatures: %x %x %x", a.signature(), b.signature(), other.signature()) + } + for _, distance := range []int{DefaultSimilarDistance, 0} { + m := newMock() + j := New(m) + j.SimilarDistance = distance + for _, raw := range []string{orig, variant} { + if kind, _, err := j.Classify(context.Background(), []byte(raw)); err != nil || kind != KindLogin { + t.Fatalf("kind %q err %v", kind, err) + } + } + if want := map[int]int64{DefaultSimilarDistance: 1, 0: 2}[distance]; m.calls != want { + t.Fatalf("distance %d: calls=%d, want %d", distance, m.calls, want) } - } - if m.calls != 1 || c.CacheHits != 1 { - t.Fatalf("similar page not served from cache: calls=%d hits=%d", m.calls, c.CacheHits) } } @@ -394,38 +439,36 @@ func TestNormalizeName(t *testing.T) { } } -// A server's own page has no primary application; the only server is the subject. +// A server's own page has no primary application; its version is still resolved. func TestVersionWithoutPrimary(t *testing.T) { m := newMock() m.primary = noneOfThem - c, stop := m.client(t) - defer stop() frames := common.Frameworks{} frames.Add(common.NewFramework("nginx", common.FrameFromFingers)) - p, _ := NewPage([]byte(jenkinsRaw)) - result, err := runRefine(context.Background(), c, p, frames, nil) + accepted, err := New(m).Refine(context.Background(), []byte(bodyRaw), frames) if err != nil { t.Fatal(err) } - if PrimaryOf(result) != nil || result["nginx"].Version == "" { - t.Fatalf("subject version not asked: %+v", result["nginx"]) + if accepted.Primary() != nil || accepted["nginx"].Version != "1.24.0" { + t.Fatalf("server version not resolved: %+v", accepted["nginx"]) } } -// Refine on frames it already judged sends nothing: Verify skips judged hits, -// Classify is answered by the cache, and the version is already set. +// Refine of its own result sends nothing: judged hits are skipped and the +// remaining questions are answered by the cache. func TestRefineIsIdempotent(t *testing.T) { m := newMock() - c, stop := m.client(t) - defer stop() - frames := testFrames() - for i := 0; i < 2; i++ { - if _, err := refine(t, c, frames); err != nil { - t.Fatal(err) - } + j := newJudge(m) + first, err := j.Refine(context.Background(), []byte(bodyRaw), testFrames()) + if err != nil { + t.Fatal(err) } - if m.calls != 2 || !Is(frames["wordpress"], Rejected) || len(frames["wordpress"].Tags) != len(testFrames()["wordpress"].Tags)+2 { - t.Fatalf("calls=%d wordpress=%v requests=%v", m.calls, frames["wordpress"].Tags, m.requests) + second, err := j.Refine(context.Background(), []byte(bodyRaw), first) + if err != nil { + t.Fatal(err) + } + if m.calls != 2 || len(second) != len(first) || second["jenkins"].Version != "2.401.3" { + t.Fatalf("calls=%d first=%v second=%v requests=%v", m.calls, first, second, m.requests) } } @@ -434,8 +477,7 @@ func TestRefineIsIdempotent(t *testing.T) { func TestConcurrentSimilarPagesMerge(t *testing.T) { m := newMock() m.delay = 100 * time.Millisecond - c, stop := m.client(t) - defer stop() + c := New(m) var wg sync.WaitGroup kinds := make([]Kind, 20) for i := range kinds { @@ -443,13 +485,11 @@ func TestConcurrentSimilarPagesMerge(t *testing.T) { go func(i int) { defer wg.Done() raw := strings.Replace(jenkinsRaw, "JSESSIONID.1=x", fmt.Sprintf("JSESSIONID.1=%d", i), 1) - p, _ := NewPage([]byte(raw)) - r := p.Round() - Classify(r) - if err := r.Ask(context.Background(), c); err != nil { + kind, _, err := c.Classify(context.Background(), []byte(raw)) + if err != nil { t.Error(err) } - kinds[i] = p.Kind + kinds[i] = kind }(i) } wg.Wait() @@ -466,36 +506,40 @@ func TestConcurrentSimilarPagesMerge(t *testing.T) { // A round partly answered by the cache sends only the rest. func TestPartialCacheHit(t *testing.T) { m := newMock() - c, stop := m.client(t) - defer stop() - p, _ := NewPage([]byte(jenkinsRaw)) - r := p.Round() - Classify(r) - if err := r.Ask(context.Background(), c); err != nil { + c := New(m) + if _, _, err := c.Classify(context.Background(), []byte(jenkinsRaw)); err != nil { t.Fatal(err) } - p, _ = NewPage([]byte(jenkinsRaw)) - r = p.Round() - Classify(r) - r.Add("extra", Binary("Is this page served over a CDN?"), nil) - if err := r.Ask(context.Background(), c); err != nil || p.Kind != KindLogin { - t.Fatalf("err %v kind %q", err, p.Kind) + p, _ := NewPage([]byte(jenkinsRaw)) + r := newRound(p) + var kind Kind + var generic bool + classifyRound(r, &kind, &generic) + r.add("extra", Binary("Is this page served over a CDN?"), nil) + if err := r.ask(context.Background(), c); err != nil || kind != KindLogin { + t.Fatalf("err %v kind %q", err, kind) } if len(m.requests) != 2 || strings.Join(m.requests[1], ",") != "extra" { t.Fatalf("requests: %v", m.requests) } } -// Without a primary, the only server is the subject even next to an OS. -func TestVersionSubjectPrefersServer(t *testing.T) { +// Versions are resolved for the primary application first, then other +// applications, servers and libraries; unjudged hits last. +func TestByImportance(t *testing.T) { frames := common.Frameworks{} - for name, l := range map[string]Layer{"apache": LayerServer, "ubuntu": LayerDevice} { + for name, j := range map[string]*common.Judgement{"jquery": {Layer: LayerFrontend}, "apache": {Layer: LayerServer}, + "ubuntu": {Layer: LayerDevice}, "gitlab": {Layer: LayerApplication, Primary: true}, "unjudged": nil} { f := common.NewFramework(name, common.FrameFromFingers) - setLayer(f, l) + f.Judge = j frames.Add(f) } - if f := subject(frames); f == nil || f.Name != "apache" { - t.Fatalf("subject: %+v", f) + var got []string + for _, f := range byImportance(frames) { + got = append(got, f.Name) + } + if strings.Join(got, ",") != "gitlab,ubuntu,apache,jquery,unjudged" { + t.Fatalf("order: %v", got) } } @@ -505,7 +549,7 @@ func TestUnrelatedSparsePagesDoNotShare(t *testing.T) { head := "HTTP/1.1 200 OK\r\nServer: nginx\r\nX-Frame-Options: SAMEORIGIN\r\n\r\n" a, _ := NewPage([]byte(head + "职业规划咨询

生涯规划师与高考志愿规划，帮助学生找到方向

")) b, _ := NewPage([]byte(head + "官方网站

在线娱乐平台，注册即送体验金

")) - if Similar(a.Signature(), b.Signature()) && a.similarityScope() == b.similarityScope() { - t.Fatalf("unrelated pages share answers: %x %x", a.Signature(), b.Signature()) + if bits.OnesCount64(a.signature()^b.signature()) <= DefaultSimilarDistance && a.similarityScope() == b.similarityScope() { + t.Fatalf("unrelated pages share answers: %x %x", a.signature(), b.signature()) } } diff --git a/judge/mark.go b/judge/mark.go deleted file mode 100644 index dca6c98..0000000 --- a/judge/mark.go +++ /dev/null @@ -1,77 +0,0 @@ -package judge - -import ( - "strings" - - "github.com/chainreactors/fingers/common" -) - -// Mark is a judgement written onto a framework. Marks live in Framework.Tags -// as "judge:" (and the layer as "judge:layer="), so they serialize -// with every existing output and need no new field; read them with Is and -// LayerOf rather than parsing tags. -type Mark string - -const ( - Rejected Mark = "rejected" // not in the stack, or only mentioned in text: a false positive - Duplicate Mark = "dup" // another engine's spelling of a product already in the result - Primary Mark = "primary" // the application the page belongs to - Recalled Mark = "recall" // missed by the rules, found by name and confirmed by the judge -) - -// TagPrefix starts every tag this package writes. -const TagPrefix = "judge:" - -const layerTag = TagPrefix + "layer=" - -// Tag is the Framework.Tags spelling of m. -func (m Mark) Tag() string { return TagPrefix + string(m) } - -// Is reports whether f carries m. -func Is(f *common.Framework, m Mark) bool { return f != nil && f.HasTag(m.Tag()) } - -// LayerOf returns the layer f was judged to be at, or "" if f was not judged. -func LayerOf(f *common.Framework) Layer { - if f == nil { - return "" - } - for _, t := range f.Tags { - if strings.HasPrefix(t, layerTag) { - return Layer(strings.TrimPrefix(t, layerTag)) - } - } - return "" -} - -// Judged reports whether f went through Verify. -func Judged(f *common.Framework) bool { return LayerOf(f) != "" } - -func mark(f *common.Framework, m Mark) { f.AddTag(m.Tag()) } - -func setLayer(f *common.Framework, l Layer) { - if l != "" && LayerOf(f) == "" { - f.AddTag(layerTag + string(l)) - } -} - -// PrimaryOf returns the framework marked Primary. -func PrimaryOf(frames common.Frameworks) *common.Framework { - for _, f := range frames { - if Is(f, Primary) { - return f - } - } - return nil -} - -// Accepted returns frames without Rejected hits and Duplicate spellings: one -// framework per product that is really there. -func Accepted(frames common.Frameworks) common.Frameworks { - out := common.Frameworks{} - for k, f := range frames { - if f != nil && !Is(f, Rejected) && !Is(f, Duplicate) { - out[k] = f - } - } - return out -} diff --git a/judge/operations.go b/judge/operations.go index 9065c39..ae119d4 100644 --- a/judge/operations.go +++ b/judge/operations.go @@ -10,7 +10,7 @@ import ( // Yes returns the probability that the answer to question is yes. State can // be any JSON-serializable value, including a Page returned by NewPage. -func (j *Judge) Yes(ctx context.Context, state any, question string) (float64, error) { +func (j *Judge) Yes(ctx context.Context, state interface{}, question string) (float64, error) { answers, err := j.Ask(ctx, state, map[string]Question{"yes": Binary(question)}) if err != nil { return 0, err @@ -19,7 +19,7 @@ func (j *Judge) Yes(ctx context.Context, state any, question string) (float64, e } // Choose selects one option and returns its confidence. -func (j *Judge) Choose(ctx context.Context, state any, question string, options map[string]string) (string, float64, error) { +func (j *Judge) Choose(ctx context.Context, state interface{}, question string, options map[string]string) (string, float64, error) { if len(options) < 2 { return "", 0, fmt.Errorf("judge: choose needs at least two options") } @@ -35,7 +35,7 @@ func (j *Judge) Choose(ctx context.Context, state any, question string, options } // Score returns a value from 0 to 1 on the ordered levels. -func (j *Judge) Score(ctx context.Context, state any, question string, levels ...string) (float64, error) { +func (j *Judge) Score(ctx context.Context, state interface{}, question string, levels ...string) (float64, error) { if len(levels) < 2 || len(levels) > 10 { return 0, fmt.Errorf("judge: score needs 2 to 10 levels") } @@ -50,44 +50,55 @@ func (j *Judge) Score(ctx context.Context, state any, question string, levels .. return score, nil } -// Inspect returns all rule hits with judgement marks, including rejected and -// duplicate hits. The input map and its frameworks are never changed. -func (j *Judge) Inspect(ctx context.Context, raw []byte, frames common.Frameworks, knownNames ...string) (common.Frameworks, error) { +// Refine is the scan-time entry point: it returns the fingerprints to report +// for a response. False positives and duplicate spellings are dropped, known +// names the rules missed are recalled (see Judge.Known) and every kept +// product without a version gets one when the response shows it. Each +// returned framework carries its verdict in Framework.Judge. hits are never +// changed; on error, use them as the rule-only result. +func (j *Judge) Refine(ctx context.Context, raw []byte, hits common.Frameworks) (common.Frameworks, error) { p, err := NewPage(raw) if err != nil { return nil, err } - working := cloneFrameworks(frames) - r := p.Round() - Verify(r, working, knownNames) - if err := r.Ask(ctx, j); err != nil { + all, err := j.inspect(ctx, p, hits) + if err != nil { + return nil, err + } + accepted := all.Accepted() + r := newRound(p) + versionRound(r, byImportance(accepted)...) + if err := r.ask(ctx, j); err != nil { return nil, err } - return working, nil + return accepted, nil } -// Verify returns the accepted fingerprints after judging rule hits and -// optional known-name candidates. Use Inspect for rejected-hit diagnostics. -func (j *Judge) Verify(ctx context.Context, raw []byte, frames common.Frameworks, knownNames ...string) (common.Frameworks, error) { - all, err := j.Inspect(ctx, raw, frames, knownNames...) +// Inspect explains Refine: it returns every hit and recalled name with its +// verdict in Framework.Judge, rejected and duplicate ones included, and +// resolves no versions. hits are never changed. +func (j *Judge) Inspect(ctx context.Context, raw []byte, hits common.Frameworks) (common.Frameworks, error) { + p, err := NewPage(raw) if err != nil { return nil, err } - return Accepted(all), nil + return j.inspect(ctx, p, hits) } -// Classify reports the page kind and whether it is a stock product page. -func (j *Judge) Classify(ctx context.Context, raw []byte) (Kind, bool, error) { +// Classify reports the page kind and whether it is the stock page of a +// packaged product. After Refine of the same response it is answered from +// the cache. +func (j *Judge) Classify(ctx context.Context, raw []byte) (kind Kind, generic bool, err error) { p, err := NewPage(raw) if err != nil { return "", false, err } - r := p.Round() - Classify(r) - if err := r.Ask(ctx, j); err != nil { + r := newRound(p) + classifyRound(r, &kind, &generic) + if err := r.ask(ctx, j); err != nil { return "", false, err } - return p.Kind, p.Generic, nil + return kind, generic, nil } // Version selects a version for f when the response contains a supported @@ -104,9 +115,9 @@ func (j *Judge) Version(ctx context.Context, raw []byte, f *common.Framework) (s return "", err } copy := cloneFramework(f) - r := p.Round() - Version(r, copy) - if err := r.Ask(ctx, j); err != nil { + r := newRound(p) + versionRound(r, copy) + if err := r.ask(ctx, j); err != nil { return "", err } if copy.Attributes == nil { @@ -114,17 +125,3 @@ func (j *Judge) Version(ctx context.Context, raw []byte, f *common.Framework) (s } return copy.Version, nil } - -// Refine verifies hits, classifies the page and resolves the primary product's -// version. Both rounds must succeed; input frames are never changed. -func (j *Judge) Refine(ctx context.Context, raw []byte, frames common.Frameworks, knownNames ...string) (common.Frameworks, Kind, bool, error) { - p, err := NewPage(raw) - if err != nil { - return nil, "", false, err - } - all, err := runRefine(ctx, j, p, frames, knownNames) - if err != nil { - return nil, "", false, err - } - return Accepted(all), p.Kind, p.Generic, nil -} diff --git a/judge/operations_test.go b/judge/operations_test.go index 8fbebb3..5d327f5 100644 --- a/judge/operations_test.go +++ b/judge/operations_test.go @@ -1,15 +1,13 @@ package judge import ( - "bytes" "context" + "encoding/json" "errors" - "fmt" "strings" "testing" "github.com/chainreactors/fingers/common" - "github.com/chainreactors/fingers/fingers" ) type answerProvider func(map[string]Question) map[string]Answer @@ -23,7 +21,7 @@ func (p answerProvider) Judge(_ context.Context, _ interface{}, questions map[st type failVersionProvider struct{ *mock } func (p failVersionProvider) Judge(ctx context.Context, state interface{}, questions map[string]Question) (map[string]Answer, error) { - if _, ok := questions["version"]; ok { + if _, ok := questions["version_0"]; ok { return nil, errors.New("version unavailable") } return p.mock.Judge(ctx, state, questions) @@ -68,42 +66,40 @@ func TestTypedPrimitives(t *testing.T) { } func TestPublicResultsDoNotMutateHits(t *testing.T) { - j := New(newMock()) + j := newJudge(newMock()) frames := testFrames() - inputJenkins := frames["jenkins"] - accepted, err := j.Verify(context.Background(), []byte(jenkinsRaw), frames, "Prototype") - if err != nil { - t.Fatal(err) + before, _ := json.Marshal(frames) + inspected, err := j.Inspect(context.Background(), []byte(bodyRaw), frames) + if err != nil || !inspected["wordpress"].Judge.Rejected || inspected["prototype"] == nil { + t.Fatalf("inspect = %v, %v", inspected, err) } - if accepted["wordpress"] != nil || accepted["jenkins"] == nil || Judged(inputJenkins) { - t.Fatalf("verify result/input: %v / %v", accepted, frames) + refined, err := j.Refine(context.Background(), []byte(bodyRaw), frames) + if err != nil || refined["wordpress"] != nil || refined["jenkins"].Version != "2.401.3" { + t.Fatalf("refine = %v, %v", refined, err) } - inspected, err := j.Inspect(context.Background(), []byte(jenkinsRaw), frames, "Prototype") - if err != nil || !Is(inspected["wordpress"], Rejected) || Judged(frames["wordpress"]) { - t.Fatalf("inspect result/input: %v / %v / %v", inspected, frames, err) + // The judgement is copied, not shared with the Inspect result. + refined["nginx"].Judge.Primary = true + if inspected["nginx"].Judge.Primary { + t.Fatal("Refine and Inspect share a Judgement") } - refined, kind, generic, err := j.Refine(context.Background(), []byte(jenkinsRaw), frames, "Prototype") - if err != nil || kind != KindLogin || !generic || refined["jenkins"].Version != "2.401.3" { - t.Fatalf("refine = %v %q %v, %v", refined, kind, generic, err) + if after, _ := json.Marshal(frames); string(after) != string(before) { + t.Fatalf("hits changed:\n%s\n%s", before, after) } - if frames["jenkins"] != inputJenkins || inputJenkins.Version != "" || Judged(inputJenkins) { - t.Fatal("Refine changed a rule hit") - } - version, err := j.Version(context.Background(), []byte(jenkinsRaw), inputJenkins) - if err != nil || version != "2.401.3" || inputJenkins.Version != "" { - t.Fatalf("version = %q, %v; input = %q", version, err, inputJenkins.Version) + version, err := j.Version(context.Background(), []byte(bodyRaw), frames["jenkins"]) + if err != nil || version != "2.401.3" || frames["jenkins"].Version != "" { + t.Fatalf("version = %q, %v; input = %q", version, err, frames["jenkins"].Version) } } func TestRefineSecondRoundFailureIsAtomic(t *testing.T) { frames := testFrames() j := New(failVersionProvider{newMock()}) - accepted, kind, generic, err := j.Refine(context.Background(), []byte(jenkinsRaw), frames) - if err == nil || accepted != nil || kind != "" || generic { - t.Fatalf("failed refinement = %v %q %v, %v", accepted, kind, generic, err) + accepted, err := j.Refine(context.Background(), []byte(bodyRaw), frames) + if err == nil || accepted != nil { + t.Fatalf("failed refinement = %v, %v", accepted, err) } for _, f := range frames { - if Judged(f) || f.Version != "" { + if f.Judge != nil || f.Version != "" { t.Fatalf("input changed after second round failed: %+v", f) } } @@ -139,7 +135,7 @@ func TestUnknownAndBodyNameSuggestion(t *testing.T) { } known := common.Frameworks{} f := common.NewFramework("QuantumGate Portal", common.FrameFromGUESS) - setLayer(f, LayerApplication) + f.Judge = &common.Judgement{Layer: LayerApplication} known.Add(f) unknown, err = j.IsUnknownProduct(context.Background(), raw, known) if err != nil || unknown { @@ -161,7 +157,7 @@ func TestKnownCDNErrorPageIsNotUnknown(t *testing.T) { return out })) f := common.NewFramework("cloudflare", common.FrameFromFingers) - setLayer(f, LayerCDN) + f.Judge = &common.Judgement{Layer: LayerCDN} accepted := common.Frameworks{} accepted.Add(f) unknown, err := j.IsUnknownProduct(context.Background(), raw, accepted) @@ -169,114 +165,3 @@ func TestKnownCDNErrorPageIsNotUnknown(t *testing.T) { t.Fatalf("known CDN error = %t, %v", unknown, err) } } - -func TestGeneratorPassiveVersionAndValidation(t *testing.T) { - positive1 := []byte("HTTP/1.1 200 OK\r\nServer: nginx/2.401.3\r\nX-Jenkins: 2.401.3\r\n\r\nJenkins") - positive2 := []byte("HTTP/1.1 200 OK\r\nServer: nginx/2.402.1\r\nX-Jenkins: 2.402.1\r\n\r\nJenkins") - negative := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nBlog

X-Jenkins: 9.9.9 release notes

") - g := NewGenerator(nil).Name("Jenkins").PositiveVersion(positive1, "2.401.3").PositiveVersion(positive2, "2.402.1").Negative(negative) - f, err := g.Generate(context.Background()) - if err != nil { - t.Fatal(err) - } - if f.Name != "Jenkins" || len(f.Rules) < 2 { - t.Fatalf("fingerprint: %+v", f) - } - for _, sample := range []struct { - raw []byte - version string - }{{positive1, "2.401.3"}, {positive2, "2.402.1"}} { - frame, _, ok := f.PassiveMatch(fingers.NewContent(sample.raw, "", true)) - if !ok || frame.Version != sample.version { - t.Fatalf("version matcher = %+v, %v", frame, ok) - } - } - compiled := len(f.Rules[0].Regexps.CompliedRegexp) - if err := g.Validate(f); err != nil || len(f.Rules[0].Regexps.CompliedRegexp) != compiled { - t.Fatalf("Validate mutated compiled matcher: %v", err) - } - if err := g.Validate(&fingers.Finger{Name: "Jenkins", Rules: fingers.Rules{&fingers.Rule{Regexps: &fingers.Regexps{Body: []string{"jenkins"}}}}}); err == nil { - t.Fatal("validator accepted rule matching negative sample") - } -} - -func TestGeneratorActiveAndProbeWith(t *testing.T) { - base := []byte("HTTP/1.1 200 OK\r\n\r\nLogin") - request := []byte("/admin") - positive := []byte("HTTP/1.1 200 OK\r\nX-Orion: console\r\n\r\nOrion console") - negative := []byte("HTTP/1.1 404 Not Found\r\n\r\nmissing") - g := NewGenerator(nil).Name("Orion").Positive(base) - if err := g.ProbeWith(context.Background(), request, func(_ context.Context, got []byte) ([]byte, error) { - if !bytes.Equal(got, request) { - return nil, errors.New("wrong request") - } - return positive, nil - }); err != nil { - t.Fatal(err) - } - g.Negative(base).Probe(request, negative) - f, err := g.Generate(context.Background()) - if err != nil { - t.Fatal(err) - } - if len(f.Rules) != 1 || f.Rules[0].SendDataStr != string(request) || !f.IsActive { - t.Fatalf("active rule: %+v", f) - } - if err := g.Validate(f); err != nil { - t.Fatal(err) - } - if len(f.Rules[0].Regexps.CompliedRegexp) != 0 { - t.Fatal("Validate changed compiled rules") - } - if _, err := NewGenerator(nil).Name("Orion").Positive(base).Probe(request, positive).Negative(base).Generate(context.Background()); err == nil { - t.Fatal("generated active rule without a negative probe") - } -} - -func TestGeneratorSelectsNameByDefault(t *testing.T) { - positive := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nOrion

Orion console

") - negative := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nOther") - j := New(answerProvider(func(qs map[string]Question) map[string]Answer { - out := map[string]Answer{} - for key, q := range qs { - if strings.Contains(q.Instructions, "`Orion`") { - out[key] = Answer{Yes: 0.95} - } else { - out[key] = Answer{Yes: 0.05} - } - } - return out - })) - f, err := NewGenerator(j).Positive(positive).Negative(negative).Generate(context.Background()) - if err != nil || f.Name != "Orion" { - t.Fatalf("auto name = %+v, %v", f, err) - } -} - -func TestGeneratorActiveVersion(t *testing.T) { - base := []byte("HTTP/1.1 200 OK\r\n\r\nLogin") - request := []byte("/version") - positive := []byte("HTTP/1.1 200 OK\r\nX-Orion: 3.1.2\r\n\r\nversion") - negative := []byte("HTTP/1.1 200 OK\r\n\r\nnot installed") - g := NewGenerator(nil).Name("Orion").PositiveVersion(base, "3.1.2").Probe(request, positive).Negative(base).Probe(request, negative) - f, err := g.Generate(context.Background()) - if err != nil { - t.Fatal(err) - } - frame, _, ok := f.ActiveMatch(2, func(got []byte) ([]byte, bool) { return positive, bytes.Equal(got, request) }) - if !ok || frame.Version != "3.1.2" { - t.Fatalf("active version = %+v, %v", frame, ok) - } -} - -func TestGeneratorCoversDifferentPositiveVariants(t *testing.T) { - base := "HTTP/1.1 200 OK\r\n%s\r\nWelcome" - first := []byte(fmt.Sprintf(base, "X-Orion-Instance: ready\r\n")) - second := []byte(fmt.Sprintf(base, "X-Orion-Service: yes\r\n")) - other := []byte(fmt.Sprintf(base, "Server: nginx\r\n")) - g := NewGenerator(nil).Name("Orion").Positive(first).Positive(second).Negative(other) - f, err := g.Generate(context.Background()) - if err != nil || len(f.Rules) != 2 { - t.Fatalf("variant rules = %+v, %v", f, err) - } -} diff --git a/judge/page.go b/judge/page.go index 319ed51..9ee7c08 100644 --- a/judge/page.go +++ b/judge/page.go @@ -9,14 +9,15 @@ import ( "regexp" "strings" "unicode/utf8" + + "github.com/chainreactors/fingers/judge/internal/evidence" ) -// Page is the entry point of the judgement layer: the compact, named view of -// one HTTP response that providers judge (serialized as the request state), -// plus the page-level conclusions written back by Classify. A provider only -// judges the -// evidence it is given and loses accuracy on large states full of irrelevant -// detail, so the fields below are the ones that carry fingerprint signal. +// Page is the compact, named view of one HTTP response that providers judge +// (serialized as the request state). A provider only judges the evidence it +// is given and loses accuracy on large states full of irrelevant detail, so +// the fields below are the ones that carry fingerprint signal. Pass one to +// Yes, Choose or Score to ask custom questions about a response. type Page struct { Status string `json:"status"` Headers map[string]string `json:"headers,omitempty"` @@ -31,9 +32,6 @@ type Page struct { Forms []string `json:"form_inputs,omitempty"` Text string `json:"visible_text,omitempty"` - Kind Kind `json:"-"` // set by Classify - Generic bool `json:"-"` // stock page of a packaged product, set by Classify - raw []byte } @@ -48,8 +46,6 @@ var ( // per-request values: no signal, and they would defeat the cache "X-Request-Id": true, "X-Amzn-Trace-Id": true, "Cf-Ray": true, "X-Runtime": true, "Server-Timing": true} - reTitle = regexp.MustCompile(`(?is)]*>(.*?)`) - reGenerator = regexp.MustCompile(`(?is)]+name=["']generator["'][^>]*content=["']([^"']+)|]+content=["']([^"']+)["'][^>]*name=["']generator["']`) reDescription = regexp.MustCompile(`(?is)]+name=["']description["'][^>]*content=["']([^"']+)|]+content=["']([^"']+)["'][^>]*name=["']description["']`) reScript = regexp.MustCompile(`(?is)]+src=["']([^"']+)`) reStyle = regexp.MustCompile(`(?is)]+rel=["']?stylesheet[^>]*href=["']([^"']+)|]+href=["']([^"']+)["'][^>]*rel=["']?stylesheet`) @@ -60,16 +56,12 @@ var ( reAttrType = regexp.MustCompile(`(?is)\btype=["']([^"']+)`) reDropBlock = regexp.MustCompile(`(?is)<(script|style|noscript)[^>]*>.*?`) reTag = regexp.MustCompile(`(?s)<[^>]+>`) - reSpace = regexp.MustCompile(`\s+`) // Allows a v/V/x/X prefix ("X3.4", "V8.1SP2") and a letter suffix; rejects // digits glued to other digits or dots so IPs and long builds stay out. reGenMajor = regexp.MustCompile(`(?i)generator["'][^>]*content=["'][A-Za-z][^"']*?\s[vV]?(\d{1,3})(?:[\s"'(]|$)|content=["'][A-Za-z][^"']*?\s[vV]?(\d{1,3})(?:[\s(][^"']*)?["'][^>]*name=["']generator`) - reVersion = regexp.MustCompile(`(?:^|[^0-9A-Za-z.])[vVxX]?(` + versionToken + `)(?:[^0-9A-Za-z.]|\.[A-Za-z]|$)`) + reVersion = regexp.MustCompile(`(?:^|[^0-9A-Za-z.])[vVxX]?(` + evidence.VersionToken + `)(?:[^0-9A-Za-z.]|\.[A-Za-z]|$)`) ) -// Keep release suffixes and calendar versions (e.g. 2026.9.23+3cd69d30e). -const versionToken = `[0-9]{1,4}(?:\.[0-9]{1,4}){1,3}(?:[A-Za-z]+[0-9]*|[-+][0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*)?` - const ( maxHeaderValue = 160 maxText = 1500 // visible text runes @@ -94,10 +86,10 @@ func NewPage(raw []byte) (*Page, error) { s.Cookies = append(s.Cookies, c.Name) } html := string(body) - if m := reTitle.FindStringSubmatch(html); m != nil { + if m := evidence.Title.FindStringSubmatch(html); m != nil { s.Title = clean(m[1]) } - if m := reGenerator.FindStringSubmatch(html); m != nil { + if m := evidence.Generator.FindStringSubmatch(html); m != nil { s.Generator = m[1] + m[2] } if m := reDescription.FindStringSubmatch(html); m != nil { @@ -136,8 +128,8 @@ func NewPage(raw []byte) (*Page, error) { return s, nil } -// Haystack is the lowercase text that candidate names are searched in. -func (s *Page) Haystack() string { +// haystack is the lowercase text that candidate names are searched in. +func (s *Page) haystack() string { var b strings.Builder for k, v := range s.Headers { b.WriteString(k + ": " + v + "\n") @@ -153,10 +145,7 @@ func (s *Page) Haystack() string { return strings.ToLower(b.String()) } -func clean(s string) string { - s = strings.NewReplacer(" ", " ", "&", "&", "<", "<", ">", ">", """, `"`).Replace(s) - return strings.TrimSpace(reSpace.ReplaceAllString(s, " ")) -} +func clean(s string) string { return evidence.Clean(s) } func truncate(s string, max int) string { if utf8.RuneCountInString(s) <= max { @@ -165,10 +154,10 @@ func truncate(s string, max int) string { return string([]rune(s)[:max]) + "…" } -// Evidence reports where a candidate name literally occurs in the page. It is +// where reports where a candidate name literally occurs in the page. It is // a cheap code-only baseline for false positive detection: a name found only // in visible text is a likely false positive. -func (s *Page) Evidence(name string) []string { +func (s *Page) where(name string) []string { n := strings.ToLower(name) if len(n) < 3 { return nil @@ -219,12 +208,12 @@ func (s *Page) similarityScope() string { return reDigits.ReplaceAllString(strings.ToLower(strings.TrimSpace(s.Title)), "0") } -// Signature is a 64-bit simhash of the page's evidence. Digits are folded so +// signature is a 64-bit simhash of the page's evidence. Digits are folded so // timestamps, build ids and tokens do not matter; structure (headers, // assets, forms) weighs more than visible text, which counts by two-rune // shingles. Near-identical pages get signatures a few bits apart; see -// SimilarDistance. The cache also requires equal titles. -func (s *Page) Signature() uint64 { +// Judge.SimilarDistance. The cache also requires equal titles. +func (s *Page) signature() uint64 { var v [64]int add := func(kind, feature string, weight int) { h := fnv.New64a() diff --git a/judge/real_evidence_test.go b/judge/real_evidence_test.go index 335ec8f..3f5e4c7 100644 --- a/judge/real_evidence_test.go +++ b/judge/real_evidence_test.go @@ -2,10 +2,10 @@ package judge import ( "context" - "github.com/chainreactors/fingers/common" - fingerlib "github.com/chainreactors/fingers/fingers" "strings" "testing" + + "github.com/chainreactors/fingers/common" ) // Minimal public evidence retained from the 2026-09-25 corpus, not full pages. @@ -16,10 +16,10 @@ func TestStaticApplicationDescriptionIsPreservedAsEvidence(t *testing.T) { ``, } { p, err := NewPage([]byte("HTTP/1.1 200 OK\r\n\r\nIT Tools - Handy online tools for developers" + meta)) - if err != nil || p.Description != description || !strings.Contains(p.Haystack(), strings.ToLower(description)) { + if err != nil || p.Description != description || !strings.Contains(p.haystack(), strings.ToLower(description)) { t.Fatalf("description evidence missing: %+v err=%v", p, err) } - if strings.Contains(strings.Join(p.Evidence("it tools"), ","), "header") { + if strings.Contains(strings.Join(p.where("it tools"), ","), "header") { t.Fatal("description incorrectly treated as deterministic header evidence") } } @@ -75,6 +75,8 @@ func TestProtocolHitRequiresResponseEvidence(t *testing.T) { v = .99 } out[k] = Answer{Yes: v} + } else if k == "page_kind" { + out[k] = Answer{Choice: string(KindLogin)} } else if k == "primary" { out[k] = Answer{Choice: noneOfThem} } else { @@ -85,29 +87,13 @@ func TestProtocolHitRequiresResponseEvidence(t *testing.T) { })) hits := common.Frameworks{} hits.Add(common.NewFramework(tc.name, common.FrameFromGUESS)) - got, err := j.Verify(context.Background(), []byte("HTTP/1.1 200 OK\r\n"+tc.header+"\r\n"), hits) + all, err := j.Inspect(context.Background(), []byte("HTTP/1.1 200 OK\r\n"+tc.header+"\r\n"), hits) + got := all.Accepted() if err != nil || (len(got) > 0) != tc.want { t.Errorf("%s evidence=%q got=%v err=%v", tc.name, tc.header, got, err) } } } -func TestGeneratedVersionHandlesHyphenatedHeaderAndReleaseSuffix(t *testing.T) { - raw := func(v string) []byte { - return []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\nX-New-Api-Version: v" + v + "\r\n\r\nNew API") - } - neg := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nWelcome") - g := NewGenerator(nil).Name("New API").PositiveVersion(raw("1.0.0-rc.35"), "1.0.0-rc.35").Negative(neg) - f, err := g.Generate(context.Background()) - if err != nil { - t.Fatal(err) - } - for _, version := range []string{"1.0.0-rc.35", "1.2.0-rc.2", "2.0.1"} { - frame, _, matched := f.PassiveMatch(fingerlib.NewContent(raw(version), "", true)) - if !matched || frame.Version != version { - t.Fatalf("version=%s matched=%t frame=%+v", version, matched, frame) - } - } -} func TestCalendarVersionAndGeneratorName(t *testing.T) { raw := []byte("HTTP/1.1 200 OK\r\n\r\nCustom Search") @@ -126,58 +112,3 @@ func TestCalendarVersionAndGeneratorName(t *testing.T) { t.Fatalf("name=%t version=%t", foundName, foundVersion) } } - -func TestGeneratedMetaRuleSurvivesTitleAndCalendarVersionChanges(t *testing.T) { - page := func(title, version string) []byte { - return []byte("HTTP/1.1 200 OK\r\n\r\n" + title + "") - } - negative := []byte("HTTP/1.1 200 OK\r\n\r\nSearXNG Documentation

Install SearXNG

") - f, err := NewGenerator(nil).Name("SearXNG").PositiveVersion(page("Search One", "2026.9.23+3cd69d30e"), "2026.9.23+3cd69d30e").Negative(negative).Generate(context.Background()) - if err != nil { - t.Fatal(err) - } - got, _, matched := f.PassiveMatch(fingerlib.NewContent(page("Different Search", "2026.9.25+d8ae3abd5"), "", true)) - if !matched || got.Version != "2026.9.25+d8ae3abd5" { - t.Fatalf("matched=%t got=%+v", matched, got) - } - if _, _, matched := f.PassiveMatch(fingerlib.NewContent(negative, "", true)); matched { - t.Fatal("documentation matched") - } -} - -func TestGeneratedFooterVersionRequiresProductDeclaration(t *testing.T) { - page := func(title, version string) []byte { - return []byte("HTTP/1.1 200 OK\r\n\r\n" + title + "
Log in
\n v" + version + " @ mysql\n
") - } - negative := page("Different Application", "2.18.0-be6aeb4") - f, err := NewGenerator(nil).Name("Wakapi").PositiveVersion(page("Wakapi – Coding Statistics", "2.18.0-be6aeb4"), "2.18.0-be6aeb4").Negative(negative).Generate(context.Background()) - if err != nil { - t.Fatal(err) - } - got, _, matched := f.PassiveMatch(fingerlib.NewContent(page("Wakapi – Coding Statistics", "2.12.0"), "", true)) - if !matched || got.Version != "2.12.0" { - t.Fatalf("matched=%t got=%+v", matched, got) - } - if _, _, matched := f.PassiveMatch(fingerlib.NewContent(negative, "", true)); matched { - t.Fatal("unguarded footer matched another product") - } -} - -func TestGeneratedApplicationDoesNotLearnSharedInfrastructure(t *testing.T) { - for _, name := range []string{"IT Tools", "Docmost"} { - positive := []byte("HTTP/1.1 200 OK\r\nServer: cloudflare\r\nX-Frame-Options: SAMEORIGIN\r\n\r\n" + name + "") - negative := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nDifferent Application") - f, err := NewGenerator(nil).Name(name).Positive(positive).Negative(negative).Generate(context.Background()) - if err != nil { - t.Fatal(err) - } - sameProxy := []byte("HTTP/1.1 200 OK\r\nServer: cloudflare\r\nX-Frame-Options: SAMEORIGIN\r\n\r\nUnrelated Application") - if _, _, matched := f.PassiveMatch(fingerlib.NewContent(sameProxy, "", true)); matched { - t.Fatalf("%s learned shared infrastructure", name) - } - otherProxy := []byte("HTTP/1.1 200 OK\r\nServer: caddy\r\n\r\n" + name + "") - if _, _, matched := f.PassiveMatch(fingerlib.NewContent(otherProxy, "", true)); !matched { - t.Fatalf("%s requires original infrastructure", name) - } - } -} diff --git a/judge/recall.go b/judge/recall.go index c1cdbe0..15de17c 100644 --- a/judge/recall.go +++ b/judge/recall.go @@ -4,6 +4,8 @@ import ( "sort" "strings" "unicode/utf8" + + "github.com/chainreactors/fingers/judge/internal/evidence" ) // Retriever finds fingerprint names that literally occur in a page. Every @@ -19,7 +21,7 @@ func NewRetriever(names []string) *Retriever { seen := map[string]bool{} for _, n := range names { k := strings.ToLower(strings.TrimSpace(n)) - if seen[k] || !usableKey(k) { + if seen[k] || !evidence.UsableKey(k) { continue } seen[k] = true @@ -29,15 +31,6 @@ func NewRetriever(names []string) *Retriever { return r } -// usableKey drops keys too short to be meaningful evidence ("oa", "cms"). -func usableKey(k string) bool { - n := utf8.RuneCountInString(k) - if n == len(k) { // ascii - return n >= 4 - } - return n >= 2 -} - // containsWord matches ASCII keys on word boundaries ("acti" must not match // "action"); CJK keys have no word boundaries and match as substrings. func containsWord(haystack, key string) bool { diff --git a/judge/refine.go b/judge/refine.go index 20da8b0..5dc16ab 100644 --- a/judge/refine.go +++ b/judge/refine.go @@ -2,29 +2,62 @@ package judge import ( "context" + "sort" "github.com/chainreactors/fingers/common" ) -// refine keeps the first round's output private until version resolution ends. -func runRefine(ctx context.Context, j *Judge, p *Page, frames common.Frameworks, recall []string) (common.Frameworks, error) { +// inspect judges a copy of frames plus the known names found in the page. +// When the Judge caches answers, the page is classified in the same request, +// so a later Classify of this response costs nothing. +func (j *Judge) inspect(ctx context.Context, p *Page, frames common.Frameworks) (common.Frameworks, error) { working := cloneFrameworks(frames) - page := *p - r := page.Round() - Verify(r, working, recall) - Classify(r) - if err := r.Ask(ctx, j); err != nil { - return nil, err + r := newRound(p) + verifyRound(r, working, j.known(p)) + if j.Cache != nil { + var kind Kind + var generic bool + classifyRound(r, &kind, &generic) } - r = page.Round() - Version(r, subject(working)) - if err := r.Ask(ctx, j); err != nil { + if err := r.ask(ctx, j); err != nil { return nil, err } - p.Kind, p.Generic = page.Kind, page.Generic return working, nil } +func (j *Judge) known(p *Page) []string { + if j.Known == nil { + return nil + } + return j.Known.Find(p.haystack(), maxKnown) +} + +// byImportance orders frames for versioning: the primary application first, +// then applications, servers and the rest, each by name. +func byImportance(frames common.Frameworks) []*common.Framework { + rank := func(f *common.Framework) int { + switch { + case f.Judge == nil: + return 3 + case f.Judge.Primary: + return 0 + case f.Judge.Layer == LayerApplication || f.Judge.Layer == LayerDevice: + return 1 + case f.Judge.Layer == LayerFrontend: + return 3 + } + return 2 + } + list := frames.List() + sort.Slice(list, func(a, b int) bool { + if ra, rb := rank(list[a]), rank(list[b]); ra != rb { + return ra < rb + } + return list[a].Name < list[b].Name + }) + return list +} + func cloneFrameworks(frames common.Frameworks) common.Frameworks { working := make(common.Frameworks, len(frames)) for name, f := range frames { @@ -49,29 +82,9 @@ func cloneFramework(f *common.Framework) *common.Framework { attrs := *f.Attributes copy.Attributes = &attrs } - return © -} - -// subject is the product whose version the page most likely shows: the -// primary application or, on pages without one (a server's default, error -// or index page), the only accepted application, else the only server, else -// the only device. -func subject(frames common.Frameworks) *common.Framework { - if f := PrimaryOf(frames); f != nil { - return f - } - accepted := Accepted(frames) - for _, l := range []Layer{LayerApplication, LayerServer, LayerDevice} { - var only *common.Framework - n := 0 - for _, f := range accepted { - if LayerOf(f) == l { - only, n = f, n+1 - } - } - if n == 1 { - return only - } + if f.Judge != nil { + judgement := *f.Judge + copy.Judge = &judgement } - return nil + return © } diff --git a/judge/round.go b/judge/round.go index 579b18e..485b6dc 100644 --- a/judge/round.go +++ b/judge/round.go @@ -6,29 +6,27 @@ import ( "fmt" ) -// Round is the judge primitive: questions about one page, each with a callback, -// sent as a single request. Capabilities (Verify, Classify, Version) are -// functions that add to a Round; callers may add their own questions too. -// Callbacks run only after the whole request succeeded, so a failed Round -// leaves every annotated value untouched. -type Round struct { +// round is the judge primitive: questions about one page, each with a +// callback, sent as a single request. Capabilities (verifyRound, +// classifyRound, versionRound) add to a round. Callbacks run only after the +// whole request succeeded, so a failed round leaves every annotated value +// untouched. +type round struct { page *Page questions map[string]Question apply map[string]func(Answer) evidence map[string]interface{} done []func() - judge *Judge // set by Ask, for thresholds in callbacks + judge *Judge // set by ask, for thresholds in callbacks err error } -func (p *Page) Round() *Round { - return &Round{page: p, questions: map[string]Question{}, apply: map[string]func(Answer){}} +func newRound(p *Page) *round { + return &round{page: p, questions: map[string]Question{}, apply: map[string]func(Answer){}} } -func (r *Round) Page() *Page { return r.page } - -// Add asks q under key; apply receives its answer. Keys must be unique in a Round. -func (r *Round) Add(key string, q Question, apply func(Answer)) { +// add asks q under key; apply receives its answer. Keys must be unique in a round. +func (r *round) add(key string, q Question, apply func(Answer)) { if _, dup := r.questions[key]; dup && r.err == nil { r.err = fmt.Errorf("judge: duplicate question key %q", key) } @@ -36,23 +34,23 @@ func (r *Round) Add(key string, q Question, apply func(Answer)) { r.apply[key] = apply } -// Done registers f to run after every answer callback, in registration +// then registers f to run after every answer callback, in registration // order: for decisions that combine several answers. -func (r *Round) Done(f func()) { r.done = append(r.done, f) } +func (r *round) then(f func()) { r.done = append(r.done, f) } -// Evidence shows the provider a named piece of evidence besides the page; the state -// becomes {"response": page, name: value}. Providers read literally: evidence -// they are asked about by name weighs more than the same text in option -// descriptions. -func (r *Round) Evidence(name string, value interface{}) { +// show gives the provider a named piece of evidence besides the page; the +// state becomes {"response": page, name: value}. Providers read literally: +// evidence they are asked about by name weighs more than the same text in +// option descriptions. +func (r *round) show(name string, value interface{}) { if r.evidence == nil { r.evidence = map[string]interface{}{} } r.evidence[name] = value } -// Ask sends the round; a round without questions sends nothing. -func (r *Round) Ask(ctx context.Context, j *Judge) error { +// ask sends the round; a round without questions sends nothing. +func (r *round) ask(ctx context.Context, j *Judge) error { if r.err != nil { return r.err } @@ -73,7 +71,7 @@ func (r *Round) Ask(ctx context.Context, j *Judge) error { if err != nil { return err } - answers, err := j.ask(ctx, state, r.questions, scope, r.page.Signature()) + answers, err := j.ask(ctx, state, r.questions, scope, r.page.signature()) if err != nil { return err } diff --git a/judge/stack.go b/judge/stack.go index f3bf3a2..d2becdb 100644 --- a/judge/stack.go +++ b/judge/stack.go @@ -1,59 +1,35 @@ package judge -import ( - "strings" - "unicode" -) - -// Layer is the role a fingerprint plays in the software stack behind one -// response. A page normally carries several fingerprints at different layers -// (CDN + web server + framework + application + libraries); each candidate is -// judged independently, and the layer decides how results combine. -type Layer string +import "github.com/chainreactors/fingers/judge/internal/evidence" +// Layers are the roles a fingerprint plays in the software stack behind one +// response, as written to Framework.Judge.Layer. A page normally carries +// several fingerprints at different layers (CDN + web server + framework + +// application + libraries); each candidate is judged independently. const ( - LayerCDN Layer = "cdn_or_waf" - LayerServer Layer = "web_server" - LayerRuntime Layer = "language_runtime" - LayerFramework Layer = "web_framework" - LayerApplication Layer = "application" - LayerFrontend Layer = "frontend_library" - LayerDevice Layer = "os_or_device" - LayerNotPresent Layer = "not_present" + LayerCDN = "cdn_or_waf" + LayerServer = "web_server" + LayerRuntime = "language_runtime" + LayerFramework = "web_framework" + LayerApplication = "application" + LayerFrontend = "frontend_library" + LayerDevice = "os_or_device" + LayerNotPresent = "not_present" ) // layerCriteria is the Choice criteria for classifying one candidate's layer. var layerCriteria = map[string]string{ - string(LayerCDN): "A CDN, reverse proxy cloud or web application firewall in front of the site", - string(LayerServer): "The HTTP server or proxy software, such as nginx, Apache httpd, IIS, Tomcat", - string(LayerRuntime): "A programming language or runtime, such as PHP, Java, ASP.NET, Node.js", - string(LayerFramework): "A backend web framework, such as Spring, Django, Laravel, ThinkPHP", - string(LayerApplication): "The application whose interface this is: CMS, OA, admin console, SaaS app, forum, monitoring tool, search engine, or packaged static/browser-side tools; a documentation subject is not the application serving the docs", - string(LayerFrontend): "A browser-side library, UI kit, analytics or tag script, such as jQuery, Bootstrap, Google Analytics", - string(LayerDevice): "An operating system or the firmware of a hardware device such as a router, camera, NAS or firewall", - string(LayerNotPresent): "Not part of the software serving this response: only mentioned in text, or absent", + LayerCDN: "A CDN, reverse proxy cloud or web application firewall in front of the site", + LayerServer: "The HTTP server or proxy software, such as nginx, Apache httpd, IIS, Tomcat", + LayerRuntime: "A programming language or runtime, such as PHP, Java, ASP.NET, Node.js", + LayerFramework: "A backend web framework, such as Spring, Django, Laravel, ThinkPHP", + LayerApplication: "The application whose interface this is: CMS, OA, admin console, SaaS app, forum, monitoring tool, search engine, or packaged static/browser-side tools; a documentation subject is not the application serving the docs", + LayerFrontend: "A browser-side library, UI kit, analytics or tag script, such as jQuery, Bootstrap, Google Analytics", + LayerDevice: "An operating system or the firmware of a hardware device such as a router, camera, NAS or firewall", + LayerNotPresent: "Not part of the software serving this response: only mentioned in text, or absent", } -var genericSuffixes = []string{"companyproducts", "公司产品", "product", "products", "产品", "operatingsystem", "操作系统", "system"} - -// genericWords are trailing words engines add to a product name ("Apache -// HTTP Server", "apache-web-server", "Discuz! X"). They are dropped only as -// whole words, so "lighttpd" keeps its "httpd". -var genericWords = map[string]bool{"http": true, "httpd": true, "server": true, "web": true, "webserver": true, "x": true, "cms": true, "oa": true} - // NormalizeName folds the spellings different engines use for one product // ("Apache-Tomcat", "apache tomcat", "Apache HTTP Server", "apache") into one // key. It never merges names whose remaining letters differ. -func NormalizeName(name string) string { - words := strings.FieldsFunc(strings.ToLower(name), func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) }) - for len(words) > 1 && genericWords[words[len(words)-1]] { - words = words[:len(words)-1] - } - key := strings.Join(words, "") - for _, s := range genericSuffixes { - if len(key) > len(s)+2 && strings.HasSuffix(key, s) { - key = strings.TrimSuffix(key, s) - } - } - return key -} +func NormalizeName(name string) string { return evidence.NormalizeName(name) } diff --git a/judge/verify.go b/judge/verify.go index f0d583a..cc0ef1d 100644 --- a/judge/verify.go +++ b/judge/verify.go @@ -17,6 +17,7 @@ var protocolFeatures = map[string]bool{"hsts": true, "altsvc": true, "http3": tr const ( maxGroups = 40 // products judged per page maxRecall = 8 // recall names judged per page + maxKnown = 24 // known names looked up per page ) // product is one product claimed for the page: every engine spelling of it @@ -28,18 +29,18 @@ type product struct { feature bool // protocol features have deterministic response-head evidence } -// Verify is the engine result filter. It folds the spellings of all engines -// into products, asks the provider per product whether it is really in the stack (a -// binary question, so several products can be present) and at which layer (a choice), and which -// product is the page's application. recall names (e.g. fingerprint names -// found in the page) that no rule reported are judged the same way; confirmed -// ones are added to frames. Code decides: a product named in a header or -// cookie is never rejected. +// verifyRound is the engine result filter. It folds the spellings of all +// engines into products, asks the provider per product whether it is really +// in the stack (a binary question, so several products can be present) and at +// which layer (a choice), and which product is the page's application. +// recall names (fingerprint names found in the page) that no rule reported +// are judged the same way; confirmed ones are added to frames. Code decides: +// a product named in a header or cookie is never rejected. // -// Results are Marks and a layer on frames (see Is, LayerOf, Accepted). -// Frameworks judged before are skipped, so calling Verify again on the same -// frames asks only about new hits. -func Verify(r *Round, frames common.Frameworks, recall []string) { +// The verdict is written to Framework.Judge. Frameworks that already carry +// one are not asked again. Past maxGroups products, hits are left unjudged +// (Judge == nil), which Frameworks.Accepted keeps. +func verifyRound(r *round, frames common.Frameworks, recall []string) { page := r.page var list []*common.Framework judged := map[string]bool{} // keys judged before: not asked again, not recalled again @@ -47,7 +48,7 @@ func Verify(r *Round, frames common.Frameworks, recall []string) { if f == nil { continue } - if Judged(f) { + if f.Judge != nil { judged[NormalizeName(f.Name)] = true } else { list = append(list, f) @@ -74,7 +75,7 @@ func Verify(r *Round, frames common.Frameworks, recall []string) { products = append(products, p) } p.frames = append(p.frames, f) - for _, e := range page.Evidence(f.Name) { + for _, e := range page.where(f.Name) { if !p.feature { p.strong = p.strong || e == "header" || e == "cookie" } @@ -111,7 +112,7 @@ func Verify(r *Round, frames common.Frameworks, recall []string) { type answer struct { present float64 answered bool - layer Layer + layer string } answers := make([]answer, len(products)) var primary string @@ -119,17 +120,17 @@ func Verify(r *Round, frames common.Frameworks, recall []string) { for i, p := range products { i, name := i, p.name primaryOpts[name] = "" - r.Add(fmt.Sprintf("is_%d", i), BinaryWith(fmt.Sprintf("Is `%s` part of the software stack that produced this HTTP response "+ + r.add(fmt.Sprintf("is_%d", i), BinaryWith(fmt.Sprintf("Is `%s` part of the software stack that produced this HTTP response "+ "(seen in its headers, cookies, asset paths, page structure or title), rather than only mentioned in the page text? "+ "A packaged browser-side or static web application counts even without a login or server backend. "+ "On documentation, README or tutorial pages, the product being documented does not count as running; distinguish it from the documentation generator.", name), name+" served, generated or is loaded by this response", name+" only appears in the page's text, or is not present"), func(a Answer) { answers[i].present, answers[i].answered = a.Yes, true }) - r.Add(fmt.Sprintf("layer_%d", i), Choice(fmt.Sprintf("What role does `%s` play in producing this HTTP response?", name), layerCriteria), - func(a Answer) { answers[i].layer = Layer(a.Choice) }) + r.add(fmt.Sprintf("layer_%d", i), Choice(fmt.Sprintf("What role does `%s` play in producing this HTTP response?", name), layerCriteria), + func(a Answer) { answers[i].layer = a.Choice }) } - r.Add("primary", Choice("Which listed product is the main application this page belongs to "+ + r.add("primary", Choice("Which listed product is the main application this page belongs to "+ "(the product whose login, console or content this is), as opposed to servers, frameworks and libraries underneath it?", primaryOpts), func(a Answer) { if a.Choice != noneOfThem { @@ -137,30 +138,24 @@ func Verify(r *Round, frames common.Frameworks, recall []string) { } }) - r.Done(func() { + r.then(func() { for i, p := range products { a := answers[i] rejected := !p.strong && (p.feature || a.answered && (a.layer == LayerNotPresent || a.present < r.judge.Threshold)) - if p.frames == nil { // recall: added only when the provider confirms it + recalled := p.frames == nil + if recalled { // recall: added only when the provider confirms it if !a.answered || rejected { continue } f := common.NewFramework(p.name, common.FrameFromGUESS) - mark(f, Recalled) frames.Add(f) p.frames = []*common.Framework{f} } for j, f := range p.frames { - setLayer(f, a.layer) - if rejected { - mark(f, Rejected) - } - if j > 0 { - mark(f, Duplicate) - } + f.Judge = &common.Judgement{Layer: a.layer, Confidence: a.present, Rejected: rejected, Duplicate: j > 0, Recalled: recalled} } if p.name == primary && !rejected { - mark(p.frames[0], Primary) + p.frames[0].Judge.Primary = true } } }) diff --git a/judge/version.go b/judge/version.go index 990eada..9d2c06d 100644 --- a/judge/version.go +++ b/judge/version.go @@ -2,55 +2,223 @@ package judge import ( "fmt" + "regexp" "sort" "strconv" "strings" "github.com/chainreactors/fingers/common" + "github.com/chainreactors/fingers/judge/internal/evidence" ) const ( notStated = "not_stated" maxVersions = 40 // extracted from the raw response - maxVersionsShown = 15 // shown to the provider after ranking + maxVersionsShown = 15 // options per product after ranking + maxVersioned = 8 // products versioned per page ) -// Version adds to r the question which extracted version string is the -// version of f, and writes it to f when the provider is confident. It does nothing when -// f is nil, already has a version, or the page shows no version-like string. -// Code extracts, the provider only picks: it never has to generate a string. -func Version(r *Round, f *common.Framework) { - if f == nil || (f.Attributes != nil && f.Attributes.Version != "") { +// versionRound resolves the versions of frames that have none. Code decides +// first: a version the response binds to the product by name (a header such +// as "Server: nginx/1.24.0" or "X-Jenkins: 2.401.3", a generator meta such +// as "WordPress 7.0.3") is taken as is. For the rest the provider picks +// among extracted candidates, never generating a string: the candidates go +// into the state once, as a named list (Jev picked 100% right this way +// against 78% with them in the option descriptions), and each product's +// options are its own best-ranked ones. A version bound by name to another +// product is not offered, and one string is not given to two products. +func versionRound(r *round, frames ...*common.Framework) { + var targets []*common.Framework + for _, f := range frames { + if f != nil && (f.Attributes == nil || f.Attributes.Version == "") && len(targets) < maxVersioned { + targets = append(targets, f) + } + } + if len(targets) == 0 { return } - cands := rankVersions(extractVersions(r.page.raw, maxVersions), f.Name, maxVersionsShown) - if len(cands) == 0 { + decls := declarations(r.page.raw) + var asked []*common.Framework + for _, f := range targets { + if v := declaredVersion(decls, f.Name); v != "" { + setVersion(f, v) + } else { + asked = append(asked, f) + } + } + extracted := extractVersions(r.page.raw, maxVersions) + if len(asked) == 0 || len(extracted) == 0 { return } - // The candidates go into the state as a named list: Jev picked 100% right - // this way against 78% with them in the option descriptions. - opts := map[string]string{notStated: "The response does not show the version of " + f.Name} - for _, c := range cands { - opts[c.Value] = "" - } - r.Evidence("version_strings", cands) - r.Add("version", Choice(fmt.Sprintf("`version_strings` lists every version-like string found in the raw response, with the text around it. "+ - "Which one is the version of `%s`? A version in an asset URL parameter, a meta tag or embedded build info of %s counts. "+ - "Ignore versions of third-party libraries, other products mentioned in text, years, timestamps and build numbers.", f.Name, f.Name), opts), - func(a Answer) { - if a.Choice == "" || a.Choice == notStated || a.Confidence < r.judge.VersionConfidence { - return + type pick struct { + f *common.Framework + value string + score int + confidence float64 + } + picks := make([]pick, len(asked)) + var shown []versionString + seen := map[string]bool{} + for i, f := range asked { + i, f := i, f + key := NormalizeName(f.Name) + var own []versionString + for _, c := range extracted { + if c.count == 1 && claimedByOther(decls, c.Value, key) { + continue + } + own = append(own, c) + } + cands := rankVersions(own, f.Name, maxVersionsShown) + if len(cands) == 0 { + continue + } + scores := map[string]int{} + opts := map[string]string{notStated: "The response does not show the version of " + f.Name} + for _, c := range cands { + opts[c.Value] = "" + scores[c.Value] = versionScore(c, key) + if !seen[c.Value] { + seen[c.Value] = true + shown = append(shown, c) + } + } + r.add(fmt.Sprintf("version_%d", i), Choice(fmt.Sprintf("`version_strings` lists every version-like string found in the raw response, with the text around it. "+ + "Which one is the version of `%s`? A version in an asset URL parameter, a meta tag or embedded build info of %s counts. "+ + "Ignore versions of third-party libraries, other products mentioned in text, years, timestamps and build numbers.", f.Name, f.Name), opts), + func(a Answer) { + if a.Choice != "" && a.Choice != notStated && a.Confidence >= r.judge.VersionConfidence { + picks[i] = pick{f: f, value: a.Choice, score: scores[a.Choice], confidence: a.Confidence} + } + }) + } + if len(shown) == 0 { + return + } + r.show("version_strings", shown) + count := map[string]int{} + for _, c := range extracted { + count[c.Value] = c.count + } + r.then(func() { + // A string that occurs once belongs to one product: the one whose + // context names it best, then the more confident pick. + best := map[string]int{} + for i, p := range picks { + if p.f == nil || count[p.value] > 1 { + continue + } + if j, ok := best[p.value]; !ok || p.score > picks[j].score || p.score == picks[j].score && p.confidence > picks[j].confidence { + best[p.value] = i + } + } + for i, p := range picks { + if p.f == nil { + continue } - // copy on write: Attributes may be shared with other frameworks - attrs := common.NewAttributesWithAny() - if f.Attributes != nil { - copied := *f.Attributes - attrs = &copied + if j, ok := best[p.value]; ok && j != i { + continue + } + setVersion(p.f, p.value) + } + }) +} + +// setVersion copies on write: Attributes may be shared with other frameworks. +func setVersion(f *common.Framework, version string) { + attrs := common.NewAttributesWithAny() + if f.Attributes != nil { + copied := *f.Attributes + attrs = &copied + } + attrs.Version = version + f.Attributes = attrs +} + +// declaration is a version the response binds to a product name. +type declaration struct{ name, version string } + +var ( + reDeclared = regexp.MustCompile(`([A-Za-z][A-Za-z0-9._-]*(?: [A-Za-z][A-Za-z0-9._-]*){0,2})(?:/| +[vV]?)(` + evidence.VersionToken + `)(?:[^0-9A-Za-z.+-]|$)`) + reLeadingVersion = regexp.MustCompile(`^[vV]?(` + evidence.VersionToken + `)(?:[\s;,(]|$)`) +) + +// declarations finds "name/version" and "name version" pairs in response +// headers and generator metas, and headers whose name is the product +// ("X-Jenkins: 2.401.3", "X-AspNet-Version: 4.0.30319"). +func declarations(raw []byte) []declaration { + text := string(raw) + head, body := text, "" + if i := strings.Index(text, "\r\n\r\n"); i >= 0 { + head, body = text[:i], text[i+4:] + } else if i := strings.Index(text, "\n\n"); i >= 0 { + head, body = text[:i], text[i+2:] + } + var out []declaration + pairs := func(value string) { + for _, m := range reDeclared.FindAllStringSubmatch(value, -1) { + out = append(out, declaration{m[1], m[2]}) + } + } + lines := strings.Split(head, "\n") + for _, line := range lines[1:] { // skip the status line + line = strings.TrimSpace(line) + colon := strings.IndexByte(line, ':') + if colon < 0 { + continue + } + key, value := line[:colon], strings.TrimSpace(line[colon+1:]) + if m := reLeadingVersion.FindStringSubmatch(value); m != nil { + name := strings.TrimSuffix(strings.TrimPrefix(strings.ToLower(key), "x-"), "-version") + out = append(out, declaration{name, m[1]}) + } + pairs(value) + } + for _, m := range evidence.Generator.FindAllStringSubmatch(body, 4) { + pairs(m[1] + m[2]) + } + return out +} + +// names reports whether a declared name is the product with key: the whole +// name or its trailing words, so "Apache Tomcat/9.0" names tomcat, not apache. +func (d declaration) names(key string) bool { + if key == "" { + return false + } + words := strings.Fields(d.name) + for i := range words { + if NormalizeName(strings.Join(words[i:], " ")) == key { + return true + } + } + return false +} + +// declaredVersion is the version the response binds to product, if exactly one. +func declaredVersion(decls []declaration, product string) string { + key := NormalizeName(product) + found := "" + for _, d := range decls { + if d.names(key) { + if found != "" && found != d.version { + return "" } - attrs.Version = a.Choice - f.Attributes = attrs - }) + found = d.version + } + } + return found +} + +// claimedByOther reports whether value is bound by name to a product other than key. +func claimedByOther(decls []declaration, value, key string) bool { + for _, d := range decls { + if d.version == value && !d.names(key) { + return true + } + } + return false } // versionString is a version-like string with the raw text around its first @@ -59,6 +227,7 @@ func Version(r *Round, f *common.Framework) { type versionString struct { Value string `json:"value"` Context string `json:"found_in"` + count int // occurrences in the response } // extractVersions scans the raw response (headers and body, including @@ -87,13 +256,14 @@ func extractVersions(raw []byte, max int) []versionString { // A value repeats across a page ("?ver=7.1" on every asset, then // the generator meta); keep the context that best states a version. if i, ok := seen[v]; ok { + out[i].count++ if contextWeight(ctx) > contextWeight(out[i].Context) { out[i].Context = ctx } continue } seen[v] = len(out) - out = append(out, versionString{Value: v, Context: ctx}) + out = append(out, versionString{Value: v, Context: ctx, count: 1}) } // "Drupal 10": a generator meta that states only a major version. for _, m := range reGenMajor.FindAllStringSubmatchIndex(text, -1) { @@ -106,7 +276,7 @@ func extractVersions(raw []byte, max int) []versionString { continue } seen[v] = len(out) - out = append(out, versionString{Value: v, Context: clean(strings.ToValidUTF8(text[m[0]:m[1]], ""))}) + out = append(out, versionString{Value: v, Context: clean(strings.ToValidUTF8(text[m[0]:m[1]], "")), count: 1}) } } // Select evidence after scanning: SVG numbers or early library assets must @@ -151,39 +321,53 @@ func looksLikeIPv4(v string) bool { return true } -// rankVersions orders candidates by how likely their context names -// the version of product, and keeps the first max. Pages often carry dozens -// of "?ver=" strings, so without ranking the one real version (a generator -// meta or a header) can fall outside the list the provider sees. +// rankVersions orders candidates by how likely their context states the +// version of product, and keeps the first max. Pages often carry dozens of +// "?ver=" strings, so without ranking the one real version can fall outside +// the list the provider sees. func rankVersions(cands []versionString, product string, max int) []versionString { key := NormalizeName(product) - score := func(c versionString) int { - ctx := strings.ToLower(c.Context) - s := 0 - if strings.Contains(ctx, "generator") { - s += 4 - } - for _, h := range []string{"server:", "x-powered-by:", "product:", "x-generator:", "version:"} { - if strings.Contains(ctx, h) { - s += 3 - break - } - } - if key != "" && strings.Contains(NormalizeName(ctx), key) { - s += 2 - } - for _, noisy := range []string{"/plugins/", "/themes/", "/vendor/", "jquery", "bootstrap", "schema_version"} { - if strings.Contains(ctx, noisy) { - s -= 2 - break - } - } - return s - } out := append([]versionString(nil), cands...) - sort.SliceStable(out, func(a, b int) bool { return score(out[a]) > score(out[b]) }) + sort.SliceStable(out, func(a, b int) bool { return versionScore(out[a], key) > versionScore(out[b], key) }) if len(out) > max { out = out[:max] } return out } + +// versionScore rates how well c's context ties it to the product with key. +// Naming the product counts most, the more so right before the value; words +// that declare some version (generator, Server:) count less, since they do +// not say whose; library and plugin paths count against. +func versionScore(c versionString, key string) int { + ctx := strings.ToLower(c.Context) + s := 0 + if key != "" { + if strings.Contains(NormalizeName(ctx), key) { + s += 3 + } + before := ctx + if i := strings.Index(ctx, strings.ToLower(c.Value)); i >= 0 { + before = ctx[:i] + } + if len(before) > 24 { + before = before[len(before)-24:] + } + if strings.Contains(NormalizeName(before), key) { + s += 3 + } + } + for _, h := range []string{"generator", "server:", "x-powered-by:", "product:", "version"} { + if strings.Contains(ctx, h) { + s += 2 + break + } + } + for _, noisy := range []string{"/plugins/", "/themes/", "/vendor/", "jquery", "bootstrap", "schema_version"} { + if strings.Contains(ctx, noisy) && !strings.Contains(key, NormalizeName(noisy)) { + s -= 2 + break + } + } + return s +} diff --git a/judge_test.go b/judge_test.go index 1bf4472..d861692 100644 --- a/judge_test.go +++ b/judge_test.go @@ -16,12 +16,20 @@ func TestRefineWithJudge(t *testing.T) { raw := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nWelcome to nginx!") frames, _ := engine.DetectContent(raw) j := judge.New(server{}) - accepted, kind, _, err := j.Refine(context.Background(), raw, frames) + j.Known = judge.NewRetriever(engine.Names()) + accepted, err := j.Refine(context.Background(), raw, frames) if err != nil { t.Fatal(err) } - if kind != judge.KindDefault || judge.LayerOf(accepted["nginx"]) != judge.LayerServer || judge.Judged(frames["nginx"]) { - t.Fatalf("kind %q, accepted %v, input %v", kind, accepted["nginx"], frames["nginx"]) + kind, _, err := j.Classify(context.Background(), raw) + if err != nil || kind != judge.KindDefault { + t.Fatalf("kind %q, %v", kind, err) + } + if nginx := accepted["nginx"]; nginx == nil || nginx.Judge == nil || nginx.Judge.Layer != judge.LayerServer || frames["nginx"].Judge != nil { + t.Fatalf("accepted %v, input %v", accepted["nginx"], frames["nginx"]) + } + if len(engine.Names()) == 0 { + t.Fatal("engine has no names") } } From 348018fc8590b9b2a9200fa5598a03798d3204ab Mon Sep 17 00:00:00 2001 From: M09Ic Date: Fri, 25 Sep 2026 10:25:31 -0700 Subject: [PATCH 06/16] test: repair tests left behind by earlier refactors - fingerprinthub: baseline and case tests use the passive template loader and matcher (buildInternalEvent/loadTemplates were removed). - nmap: NoGuess moved to parsers. - fingers bench: constant format string for vet. go vet ./... and go test -short ./... pass with and without -tags goregexp. Co-Authored-By: Claude Opus 5.5 (1M context) --- fingerprinthub/ac_index_test.go | 43 ++++-------------- fingerprinthub/fingerprinthub_test.go | 65 ++++++++++++--------------- fingers/optimization_bench_test.go | 2 +- nmap/gonmap_test.go | 9 ++-- 4 files changed, 43 insertions(+), 76 deletions(-) diff --git a/fingerprinthub/ac_index_test.go b/fingerprinthub/ac_index_test.go index 81b30c1..bb77a12 100644 --- a/fingerprinthub/ac_index_test.go +++ b/fingerprinthub/ac_index_test.go @@ -15,7 +15,6 @@ import ( "github.com/chainreactors/fingers/common" "github.com/chainreactors/fingers/resources" - "github.com/chainreactors/utils/httputils" ) func TestTemplateKeywordIndex_Build(t *testing.T) { @@ -64,43 +63,17 @@ func TestTemplateKeywordIndex_Match(t *testing.T) { } } -// webMatchBaseline runs the original full-iteration WebMatch logic without AC prefiltering. +// webMatchBaseline runs the full-iteration WebMatch logic without AC prefiltering. func (engine *FingerPrintHubEngine) webMatchBaseline(content []byte) common.Frameworks { - resp := httputils.NewResponseWithRaw(content) - if resp == nil { - return make(common.Frameworks) - } - - rawBody := httputils.ReadBody(resp) - rawBodyStr := string(rawBody) - event := engine.buildInternalEvent(resp, rawBodyStr, len(content)) frames := make(common.Frameworks) - + event, ok := parseRawHTTPEvent(content, engine.CaseInsensitive) + if !ok { + return frames + } for _, tmpl := range engine.webTemplates { - requests := tmpl.GetRequests() - if len(requests) == 0 { - continue - } - for _, req := range requests { - if req.Matchers == nil || len(req.Matchers) == 0 { - continue - } - matched := engine.matchRequest(req, event) - if matched { - name := tmpl.Info.Name - if name == "" { - name = tmpl.Id - } - frame := common.NewFramework(name, common.FrameFromFingerprintHub) - if tmpl.Info.Metadata != nil { - if vendor, ok := tmpl.Info.Metadata["vendor"].(string); ok { - frame.Attributes.Vendor = vendor - } - if product, ok := tmpl.Info.Metadata["product"].(string); ok { - frame.Attributes.Product = product - } - } - frames.Add(frame) + for _, req := range tmpl.requests { + if matchPassiveRequest(req, event) { + frames.Add(engine.newFramework(tmpl)) break } } diff --git a/fingerprinthub/fingerprinthub_test.go b/fingerprinthub/fingerprinthub_test.go index d0b4fc3..5399e35 100644 --- a/fingerprinthub/fingerprinthub_test.go +++ b/fingerprinthub/fingerprinthub_test.go @@ -5,8 +5,6 @@ import ( "github.com/chainreactors/fingers/resources" "github.com/chainreactors/neutron/operators" - "github.com/chainreactors/neutron/protocols" - "github.com/chainreactors/neutron/templates" "github.com/chainreactors/utils/encode" ) @@ -71,11 +69,7 @@ func TestFingerPrintHubEngine_WebMatch(t *testing.T) { // ============================================================================ func newTestEngine(caseInsensitive bool) *FingerPrintHubEngine { - return &FingerPrintHubEngine{ - CaseInsensitive: caseInsensitive, - webTemplates: make([]*templates.Template, 0), - executerOptions: &protocols.ExecuterOptions{Options: &protocols.Options{Timeout: 10}}, - } + return &FingerPrintHubEngine{CaseInsensitive: caseInsensitive} } const testHTTPRaw = "HTTP/1.1 200 OK\r\n" + @@ -240,7 +234,7 @@ func TestCaseInsensitive_AllMatchers(t *testing.T) { engine := newTestEngine(true) tmpls := buildTestTemplates() - count, errs := engine.loadTemplates(tmpls, true) + count, errs := engine.loadPassiveTemplates(tmpls) if count != len(tmpls) { t.Fatalf("loaded %d/%d, errors: %v", count, len(tmpls), errs) } @@ -251,20 +245,20 @@ func TestCaseInsensitive_AllMatchers(t *testing.T) { // CaseInsensitive=true 时，所有字面量/keywords/patterns 只要大小写无关就能匹配 expect := map[string]bool{ - "word body mixed": true, - "word body lower": true, - "word body upper": true, - "word header mixed": true, - "word header lower": true, - "word and body": true, - "regex body version": true, - "regex body mixed": false, // regex 引擎不受 CaseInsensitive 控制，pattern "Nacos" 匹配不到小写 body + "word body mixed": true, + "word body lower": true, + "word body upper": true, + "word header mixed": true, + "word header lower": true, + "word and body": true, + "regex body version": true, + "regex body mixed": false, // regex 引擎不受 CaseInsensitive 控制，pattern "Nacos" 匹配不到小写 body "regex header version": true, - "dsl body lower": true, - "dsl body mixed": false, // body 已 ToLower，"Nacos" 匹配不到 - "dsl header": true, - "status 200": true, - "combo and": true, + "dsl body lower": true, + "dsl body mixed": false, // body 已 ToLower，"<title>Nacos" 匹配不到 + "dsl header": true, + "status 200": true, + "combo and": true, } for name, shouldMatch := range expect { @@ -280,7 +274,7 @@ func TestCaseSensitive_AllMatchers(t *testing.T) { engine := newTestEngine(false) tmpls := buildTestTemplates() - count, errs := engine.loadTemplates(tmpls, true) + count, errs := engine.loadPassiveTemplates(tmpls) if count != len(tmpls) { t.Fatalf("loaded %d/%d, errors: %v", count, len(tmpls), errs) } @@ -293,20 +287,20 @@ func TestCaseSensitive_AllMatchers(t *testing.T) { // body: "<title>Nacos...Nacos v2.1.0" // header value: "Tengine/2.3.3", "Nacos-Server/2.1.0" expect := map[string]bool{ - "word body mixed": true, // "Nacos" 在原始 body 中存在（case-sensitive word 无 ToLower） - "word body lower": false, // "nacos" 不在原始 body 中 - "word body upper": false, // "<TITLE>NACOS" 不在原始 body 中 - "word header mixed": true, // "Tengine/2.3.3" 在 header value 中 - "word header lower": false, // "tengine/2.3.3" — header key 始终小写，但 value 保留原始 "Tengine/2.3.3" - "word and body": false, // "nacos" 不在原始 body (小写 n)；即使 header 匹配，AND 也失败 - "regex body version": false, // "nacos v[\d.]+" — body 中是 "Nacos v2.1.0"，小写 n 匹配不到 - "regex body mixed": true, // "Nacos v[\d.]+" 精确匹配原始大小写 + "word body mixed": true, // "Nacos" 在原始 body 中存在（case-sensitive word 无 ToLower） + "word body lower": false, // "<title>nacos" 不在原始 body 中 + "word body upper": false, // "<TITLE>NACOS" 不在原始 body 中 + "word header mixed": true, // "Tengine/2.3.3" 在 header value 中 + "word header lower": false, // "tengine/2.3.3" — header key 始终小写，但 value 保留原始 "Tengine/2.3.3" + "word and body": false, // "nacos" 不在原始 body (小写 n)；即使 header 匹配，AND 也失败 + "regex body version": false, // "nacos v[\d.]+" — body 中是 "Nacos v2.1.0"，小写 n 匹配不到 + "regex body mixed": true, // "Nacos v[\d.]+" 精确匹配原始大小写 "regex header version": false, // "tengine/[\d.]+" — header value 是 "Tengine/2.3.3" - "dsl body lower": false, // contains(body, "<title>nacos") — body 中是 Nacos - "dsl body mixed": true, // contains(body, "Nacos") 精确匹配 - "dsl header": false, // contains(all_headers, "nacos-server") — header value 保留了 "Nacos-Server" - "status 200": true, // 不受大小写影响 - "combo and": true, // status=200 ✓, "Tengine" 在 header ✓, "console-ui" 在 body ✓ + "dsl body lower": false, // contains(body, "nacos") — body 中是 Nacos + "dsl body mixed": true, // contains(body, "Nacos") 精确匹配 + "dsl header": false, // contains(all_headers, "nacos-server") — header value 保留了 "Nacos-Server" + "status 200": true, // 不受大小写影响 + "combo and": true, // status=200 ✓, "Tengine" 在 header ✓, "console-ui" 在 body ✓ } for name, shouldMatch := range expect { @@ -397,4 +391,3 @@ func TestFaviconMatcher_MatchHashValues(t *testing.T) { t.Error("Expected no match") } } - diff --git a/fingers/optimization_bench_test.go b/fingers/optimization_bench_test.go index 01db700..283562f 100644 --- a/fingers/optimization_bench_test.go +++ b/fingers/optimization_bench_test.go @@ -671,7 +671,7 @@ func TestContentSizeScaling(t *testing.T) { const iterations = 50 t.Logf("%-10s %15s %15s %10s", "Size", "Baseline", "AC", "Speedup") - t.Logf(strings.Repeat("-", 55)) + t.Log(strings.Repeat("-", 55)) for _, size := range sizes { raw := generateLargeResponse(size) diff --git a/nmap/gonmap_test.go b/nmap/gonmap_test.go index 86d29d2..2fd3cf8 100644 --- a/nmap/gonmap_test.go +++ b/nmap/gonmap_test.go @@ -10,6 +10,7 @@ import ( "github.com/chainreactors/fingers/common" "github.com/chainreactors/fingers/resources" + "github.com/chainreactors/utils/parsers" ) // TestParseLogic 测试解析逻辑 @@ -148,14 +149,14 @@ func TestNoGuessFlag(t *testing.T) { fmt.Println("\n=== Testing NoGuess Flag Behavior ===") // Save original NoGuess value and restore after test - originalNoGuess := common.NoGuess + originalNoGuess := parsers.NoGuess defer func() { - common.NoGuess = originalNoGuess + parsers.NoGuess = originalNoGuess }() // Test 1: NoGuess = false (default) - guess should work fmt.Println("\n1. Testing with NoGuess = false (guess enabled)") - common.NoGuess = false + parsers.NoGuess = false // Test against a high port that's unlikely to be running but might be guessable // Using port 9999 which is unlikely to have a service but has a guess entry @@ -168,7 +169,7 @@ func TestNoGuessFlag(t *testing.T) { // Test 2: NoGuess = true - guess should be skipped fmt.Println("\n2. Testing with NoGuess = true (guess disabled)") - common.NoGuess = true + parsers.NoGuess = true res2 := engine.ServiceMatch("127.0.0.1", "9999", 1, common.NewServiceSender(time.Second*2), nil) if res2 == nil { From 12939f6b6693bcb1b90b91f98c2a7b7242744d16 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Fri, 25 Sep 2026 10:47:08 -0700 Subject: [PATCH 07/16] build: support Go 1.17 again; RE2 moves to the fingers/re2 module go.mod had drifted to go 1.24.0 with a toolchain line, which Go 1.17-1.20 cannot build ("invalid go version", "cannot compile Go 1.24 code") and which raises the go line of every downstream module. The cause was go-re2/wazero; build tags keep it out of compilation but not out of the module graph, so go mod tidy refused go 1.17. - fingers.RegexpCompiler (standard library by default) replaces the goregexp/re2 build-tag pair; -tags goregexp is now a no-op. - New module github.com/chainreactors/fingers/re2 (go 1.24) sets RE2 on import: import _ "github.com/chainreactors/fingers/re2". The RE2 comparison benchmark moves there, on the public API. - Downgrade to versions declaring go <= 1.17: x/net v0.17.0, x/text v0.13.0, x/sys v0.13.0, go-flags v1.5.0, jsonschema v0.7.0; pin kr/text v0.2.0. go mod tidy -go=1.17 -compat=1.17 is clean. - CI: tidy must keep go 1.17; vet/test on stable Go; a real Go 1.17 module-mode build and test replaces the GOPATH job; the re2 module is tested on Go 1.24 with the cgo tags. Verified with go1.17.13 (build, vet, go test ./...), go1.20.14 and go1.26.5 as downstream toolchains: downstream go.mod stays at go 1.17 and no wazero enters its module graph. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/ci.yml | 73 +-- fingers/optimization_bench_test.go | 270 +--------- fingers/regexp_default.go | 10 - fingers/regexp_iface.go | 12 + fingers/regexp_re2.go | 10 - go.mod | 25 +- go.sum | 45 +- re2/go.mod | 39 ++ re2/go.sum | 822 +++++++++++++++++++++++++++++ re2/re2.go | 19 + re2/re2_test.go | 55 ++ 11 files changed, 1017 insertions(+), 363 deletions(-) delete mode 100644 fingers/regexp_default.go delete mode 100644 fingers/regexp_re2.go create mode 100644 re2/go.mod create mode 100644 re2/go.sum create mode 100644 re2/re2.go create mode 100644 re2/re2_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ffd9969..51239e6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,21 +9,6 @@ on: permissions: contents: read -env: - COMPAT_PKGS: >- - github.com/chainreactors/fingers - github.com/chainreactors/fingers/alias - github.com/chainreactors/fingers/common - github.com/chainreactors/fingers/ehole - github.com/chainreactors/fingers/favicon - github.com/chainreactors/fingers/fingerprinthub - github.com/chainreactors/fingers/fingers - github.com/chainreactors/fingers/goby - github.com/chainreactors/fingers/nmap - github.com/chainreactors/fingers/resources - github.com/chainreactors/fingers/wappalyzer - github.com/chainreactors/fingers/xray - jobs: test: name: test @@ -36,52 +21,70 @@ jobs: - name: Setup Go uses: actions/setup-go@v5 with: - go-version: '1.22.x' + go-version: 'stable' cache: true + # go.mod declares go 1.17: vet's stdversion check flags any newer + # standard library API, and tidy must not raise the go line. + - name: Tidy + run: | + go mod tidy -go=1.17 -compat=1.17 + git diff --exit-code go.mod go.sum - name: Vet - run: go vet -tags "re2_cgo re2_static" ./... + run: go vet ./... - name: Build - run: go build -tags "re2_cgo re2_static" ./... + run: go build ./... - name: Test - run: go test -tags "re2_cgo re2_static" -short -timeout 300s -race -count=1 ./... + run: go test -short -timeout 300s -race -count=1 ./... - go117-build: - name: Go 1.17 compatibility build + re2: + name: re2 module runs-on: ubuntu-latest + defaults: + run: + working-directory: re2 steps: - name: Checkout uses: actions/checkout@v4 - - name: Setup Go for vendoring + - name: Setup Go uses: actions/setup-go@v5 with: - go-version: '1.22.x' + go-version: '1.24.x' cache: true - - name: Prepare vendor tree - run: go mod vendor + - name: Vet + run: go vet -tags "re2_cgo re2_static" ./... + + - name: Test + run: go test -tags "re2_cgo re2_static" -count=1 ./... + + go117: + name: Go 1.17 module build and test + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 - name: Setup Go 1.17 uses: actions/setup-go@v5 with: go-version: '1.17.x' - - name: Build with Go 1.17 + - name: Build env: - GO111MODULE: off - CGO_ENABLED: '0' - run: | - export GOPATH="${RUNNER_TEMP}/gopath" - src="${GOPATH}/src/github.com/chainreactors/fingers" - mkdir -p "$(dirname "${src}")" - cp -a . "${src}" - cd "${src}" - go build -tags="noembed goregexp" ${COMPAT_PKGS} + GOTOOLCHAIN: local + run: go build ./... + + - name: Test + env: + GOTOOLCHAIN: local + run: go test -short -timeout 300s -count=1 ./... go111-build: name: Go 1.11 compatibility build diff --git a/fingers/optimization_bench_test.go b/fingers/optimization_bench_test.go index 283562f..eb4e066 100644 --- a/fingers/optimization_bench_test.go +++ b/fingers/optimization_bench_test.go @@ -8,14 +8,12 @@ import ( "io" "net/http" "os" - "regexp" "runtime" "strings" "testing" "time" "github.com/chainreactors/utils/ahocorasick" - re2 "github.com/wasilibs/go-re2" ) // ────────────────────────────────────────────────────────────── @@ -168,35 +166,35 @@ func loadTestSites(t *testing.T) []realSiteData { func generateSyntheticResponses() []realSiteData { return []realSiteData{ { - Name: "typical-nginx", + Name: "typical-nginx", Content: []byte("HTTP/1.1 200 OK\r\nServer: nginx/1.18.0\r\nContent-Type: text/html\r\nX-Powered-By: PHP/7.4.3\r\n\r\nWelcome to nginx!

Welcome to nginx!

If you see this page, the nginx web server is successfully installed.

"), }, { - Name: "wordpress-site", + Name: "wordpress-site", Content: []byte("HTTP/1.1 200 OK\r\nServer: Apache/2.4.41\r\nX-Powered-By: PHP/7.4.3\r\nLink: ; rel=\"https://api.w.org/\"\r\n\r\n

Example Blog

Hello World

Welcome to WordPress.

Powered by WordPress

"), }, { - Name: "spring-boot-app", + Name: "spring-boot-app", Content: []byte("HTTP/1.1 200 OK\r\nServer: Apache-Coyote/1.1\r\nContent-Type: application/json\r\nX-Application-Context: application:8080\r\n\r\n{\"status\":\"UP\",\"components\":{\"db\":{\"status\":\"UP\",\"details\":{\"database\":\"MySQL\",\"validationQuery\":\"isValid()\"}},\"diskSpace\":{\"status\":\"UP\",\"details\":{\"total\":107374182400,\"free\":53687091200,\"threshold\":10485760,\"path\":\"/opt/app/.\",\"exists\":true}}},\"groups\":[\"liveness\",\"readiness\"]}"), }, { - Name: "tomcat-default", + Name: "tomcat-default", Content: []byte("HTTP/1.1 200 OK\r\nServer: Apache-Coyote/1.1\r\nContent-Type: text/html;charset=UTF-8\r\n\r\nApache Tomcat/9.0.50

Apache Tomcat/9.0.50

If you're seeing this, you've successfully installed Tomcat.

  • Server Status
  • Host Manager
"), }, { - Name: "iis-aspnet", + Name: "iis-aspnet", Content: []byte("HTTP/1.1 200 OK\r\nServer: Microsoft-IIS/10.0\r\nX-Powered-By: ASP.NET\r\nX-AspNet-Version: 4.0.30319\r\n\r\nIIS Windows Server\"IIS\"

Internet Information Services

"), }, { - Name: "large-react-app", + Name: "large-react-app", Content: generateLargeResponse(64 * 1024), }, { - Name: "very-large-response", + Name: "very-large-response", Content: generateLargeResponse(256 * 1024), }, { - Name: "empty-body", + Name: "empty-body", Content: []byte("HTTP/1.1 302 Found\r\nServer: cloudflare\r\nLocation: /login\r\nCF-RAY: abc123\r\n\r\n"), }, } @@ -746,255 +744,3 @@ func TestACBuildCost(t *testing.T) { t.Logf("Single AC match time: %s, hits: %d", matchTime, len(results)) t.Logf("Build amortized over 1000 matches: %s per match", buildTime/1000) } - -// ────────────────────────────────────────────────────────────── -// 10. Go stdlib regexp vs go-re2 (C wrapper) comparison -// ────────────────────────────────────────────────────────────── - -func TestGoRegexpVsRE2(t *testing.T) { - if testing.Short() { - t.Skip("skipping heavy performance comparison in short mode") - } - engine := newPerfEngine(t) - - type patternPair struct { - goRe *regexp.Regexp - re2Re *re2.Regexp - src string - } - - var patterns []patternPair - for _, finger := range engine.HTTPFingers { - for _, rule := range finger.Rules { - if rule.Regexps == nil { - continue - } - for _, r := range rule.Regexps.CompliedRegexp { - src := r.String() - goRe, err := regexp.Compile(src) - if err != nil { - continue - } - re2Re, err := re2.Compile(src) - if err != nil { - t.Logf("SKIP re2 compile %q: %v", src, err) - continue - } - patterns = append(patterns, patternPair{goRe: goRe, re2Re: re2Re, src: src}) - } - for _, r := range rule.Regexps.CompiledVulnRegexp { - src := r.String() - goRe, err := regexp.Compile(src) - if err != nil { - continue - } - re2Re, err := re2.Compile(src) - if err != nil { - t.Logf("SKIP re2 compile %q: %v", src, err) - continue - } - patterns = append(patterns, patternPair{goRe: goRe, re2Re: re2Re, src: src}) - } - } - } - t.Logf("Compiled %d patterns for both engines", len(patterns)) - - sizes := []int{1024, 8192, 32768, 131072} - const iterations = 20 - - for _, size := range sizes { - content := NewContent(generateLargeResponse(size), "", true) - - // Go stdlib - runtime.GC() - goStart := time.Now() - goHits := 0 - for i := 0; i < iterations; i++ { - for _, p := range patterns { - if p.goRe.Match(content.Content) { - goHits++ - } - } - } - goElapsed := time.Since(goStart) / time.Duration(iterations) - - // go-re2 - runtime.GC() - re2Start := time.Now() - re2Hits := 0 - for i := 0; i < iterations; i++ { - for _, p := range patterns { - if p.re2Re.Match(content.Content) { - re2Hits++ - } - } - } - re2Elapsed := time.Since(re2Start) / time.Duration(iterations) - - speedup := float64(goElapsed) / float64(re2Elapsed) - t.Logf("[%dB] go_regexp=%s(%d) re2=%s(%d) speedup=%.2fx", - size, goElapsed, goHits/iterations, re2Elapsed, re2Hits/iterations, speedup) - } -} - -func BenchmarkRegexp_RE2(b *testing.B) { - engine := newPerfEngine(b) - var re2Regexps []*re2.Regexp - for _, finger := range engine.HTTPFingers { - for _, rule := range finger.Rules { - if rule.Regexps == nil { - continue - } - for _, r := range rule.Regexps.CompliedRegexp { - re2r, err := re2.Compile(r.String()) - if err != nil { - continue - } - re2Regexps = append(re2Regexps, re2r) - } - for _, r := range rule.Regexps.CompiledVulnRegexp { - re2r, err := re2.Compile(r.String()) - if err != nil { - continue - } - re2Regexps = append(re2Regexps, re2r) - } - } - } - - content := NewContent(generateLargeResponse(32*1024), "", true) - - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - for _, r := range re2Regexps { - r.Match(content.Content) - } - } -} - -// ────────────────────────────────────────────────────────────── -// 11. Combined AC + RE2 full optimization -// ────────────────────────────────────────────────────────────── - -func TestCombinedOptimization(t *testing.T) { - if testing.Short() { - t.Skip("skipping heavy performance comparison in short mode") - } - engine := newPerfEngine(t) - acIdx := buildACIndex(engine.HTTPFingers) - - // Pre-compile re2 versions of all regex patterns - type re2RulePatterns struct { - regexps []*re2.Regexp - vulnRegexps []*re2.Regexp - } - re2Cache := make(map[*Rule]*re2RulePatterns) - for _, finger := range engine.HTTPFingers { - for _, rule := range finger.Rules { - if rule.Regexps == nil { - continue - } - rp := &re2RulePatterns{} - for _, r := range rule.Regexps.CompliedRegexp { - re2r, err := re2.Compile(r.String()) - if err == nil { - rp.regexps = append(rp.regexps, re2r) - } - } - for _, r := range rule.Regexps.CompiledVulnRegexp { - re2r, err := re2.Compile(r.String()) - if err == nil { - rp.vulnRegexps = append(rp.vulnRegexps, re2r) - } - } - re2Cache[rule] = rp - } - } - - sites := generateSyntheticResponses() - realSites := loadTestSites(t) - sites = append(sites, realSites...) - - iterations := 50 - for _, site := range sites { - if len(site.Content) > 300000 { - iterations = 10 - } - - content := NewContent(site.Content, "", true) - - // Baseline - runtime.GC() - baseStart := time.Now() - var baseFrames int - for i := 0; i < iterations; i++ { - fs, _ := engine.HTTPMatch(site.Content, "") - baseFrames = len(fs) - } - baseElapsed := time.Since(baseStart) / time.Duration(iterations) - - // AC + RE2 combined - runtime.GC() - optStart := time.Now() - var optFrames int - for i := 0; i < iterations; i++ { - // Phase 1: AC keyword scan - acResults := acIdx.matchKeywords(content.Header, content.Body) - matched := make(map[int]bool) - for _, r := range acResults { - matched[r.FingerIdx] = true - } - - // Phase 2: RE2 regex scan for fingerprints with regex-only rules - for fi, finger := range engine.HTTPFingers { - if matched[fi] { - continue - } - for _, rule := range finger.Rules { - if rule.Regexps == nil { - continue - } - rp := re2Cache[rule] - if rp == nil { - continue - } - hit := false - for _, r := range rp.vulnRegexps { - if r.Match(content.Content) { - hit = true - break - } - } - if !hit { - for _, r := range rp.regexps { - if r.Match(content.Content) { - hit = true - break - } - } - } - if hit { - matched[fi] = true - break - } - } - } - - count := 0 - for fi := range matched { - finger := engine.HTTPFingers[fi] - _, _, ok := finger.PassiveMatch(content) - if ok { - count++ - } - } - optFrames = count - } - optElapsed := time.Since(optStart) / time.Duration(iterations) - - speedup := float64(baseElapsed) / float64(optElapsed) - t.Logf("[%s] %dB: baseline=%s(%d) AC+RE2=%s(%d) speedup=%.2fx", - site.Name, len(site.Content), baseElapsed, baseFrames, optElapsed, optFrames, speedup) - } -} diff --git a/fingers/regexp_default.go b/fingers/regexp_default.go deleted file mode 100644 index e00c96a..0000000 --- a/fingers/regexp_default.go +++ /dev/null @@ -1,10 +0,0 @@ -//go:build goregexp -// +build goregexp - -package fingers - -import "regexp" - -func compileRegexp(s string) (CompiledRegexp, error) { - return regexp.Compile(s) -} diff --git a/fingers/regexp_iface.go b/fingers/regexp_iface.go index f2e09e5..caa348d 100644 --- a/fingers/regexp_iface.go +++ b/fingers/regexp_iface.go @@ -1,8 +1,20 @@ package fingers +import "regexp" + type CompiledRegexp interface { FindSubmatch(b []byte) [][]byte FindAllString(s string, n int) []string Match(b []byte) bool String() string } + +// RegexpCompiler compiles the regexps of fingerprint rules. It is the +// standard library's by default; importing github.com/chainreactors/fingers/re2 +// switches it to RE2, which is faster on large rule sets. Set it before +// fingerprints are loaded. +var RegexpCompiler = func(expr string) (CompiledRegexp, error) { + return regexp.Compile(expr) +} + +func compileRegexp(s string) (CompiledRegexp, error) { return RegexpCompiler(s) } diff --git a/fingers/regexp_re2.go b/fingers/regexp_re2.go deleted file mode 100644 index ae73457..0000000 --- a/fingers/regexp_re2.go +++ /dev/null @@ -1,10 +0,0 @@ -//go:build !goregexp -// +build !goregexp - -package fingers - -import re2 "github.com/wasilibs/go-re2" - -func compileRegexp(s string) (CompiledRegexp, error) { - return re2.Compile(s) -} diff --git a/go.mod b/go.mod index b67c2af..e90890b 100644 --- a/go.mod +++ b/go.mod @@ -1,34 +1,28 @@ module github.com/chainreactors/fingers -go 1.24.0 - -toolchain go1.24.3 +go 1.17 require ( github.com/chainreactors/logs v0.0.0-20260508055944-c678762ed15c github.com/chainreactors/utils v0.0.0-20260629043228-93bdd2142c9a github.com/stretchr/testify v1.10.0 - golang.org/x/net v0.21.0 + golang.org/x/net v0.17.0 ) require ( - github.com/bahlo/generic-list-go v0.2.0 // indirect - github.com/buger/jsonparser v1.1.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/facebookincubator/nvdtools v0.1.5 // indirect github.com/go-dedup/megophone v0.0.0-20170830025436-f01be21026f5 // indirect github.com/go-dedup/simhash v0.0.0-20170904020510-9ecaca7b509c // indirect github.com/go-dedup/text v0.0.0-20170907015346-8bb1b95e3cb7 // indirect github.com/hashicorp/go-version v1.6.0 // indirect - github.com/mailru/easyjson v0.7.7 // indirect + github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect + github.com/kr/text v0.2.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect - github.com/tetratelabs/wazero v1.11.0 // indirect github.com/twmb/murmur3 v1.1.8 // indirect - github.com/wasilibs/wazero-helpers v0.0.0-20250123031827-cd30c44769bb // indirect - github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect - golang.org/x/sys v0.38.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect ) require ( @@ -37,12 +31,9 @@ require ( github.com/chainreactors/utils/parsers v0.0.4-0.20260925164932-39c659c84c64 github.com/chainreactors/words v0.0.0-20260520145736-270600e60fb4 github.com/dlclark/regexp2 v1.11.5 - github.com/invopop/jsonschema v0.13.0 - github.com/jessevdk/go-flags v1.6.1 + github.com/invopop/jsonschema v0.7.0 + github.com/jessevdk/go-flags v1.5.0 github.com/mozillazg/go-pinyin v0.20.0 github.com/pkg/errors v0.9.1 - github.com/wasilibs/go-re2 v1.6.0 gopkg.in/yaml.v3 v3.0.1 ) - -replace github.com/wasilibs/go-re2 => github.com/chainreactors/go-re2 v1.11.1-0.20260716152604-8121b6cd261e diff --git a/go.sum b/go.sum index 3f818bb..1830173 100644 --- a/go.sum +++ b/go.sum @@ -64,21 +64,15 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= -github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= -github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chainreactors/go-re2 v1.11.1-0.20260716152604-8121b6cd261e h1:8SMCCo85OKxOM0NtD8sPhmnhGzigU2i4YPUZHgWDVq8= -github.com/chainreactors/go-re2 v1.11.1-0.20260716152604-8121b6cd261e/go.mod h1:4qC68vqWSuPTct3spuTrWBqCpm00mQ707JKLS1izVjI= github.com/chainreactors/logs v0.0.0-20260508055944-c678762ed15c h1:6Net2Mgo/qo6ADFBZJWWScKuMfZ0rbzLqSCVDuLKFdc= github.com/chainreactors/logs v0.0.0-20260508055944-c678762ed15c/go.mod h1:VrXmYPbNN5AVoo1sc5aeyPVBYqubMdb3KO/tn5rRZpo= github.com/chainreactors/neutron v0.1.1-0.20260714062907-716c6b167cb6 h1:apEDnJeZ5fe2AaEJHV6TRDldoy2t/d7E1maxmUz2tfU= @@ -108,6 +102,7 @@ github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWH github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -256,15 +251,16 @@ github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOn github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= +github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 h1:i462o439ZjprVSFSZLZxcsoAe592sZB1rci2Z8j4wdk= +github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA= github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/invopop/jsonschema v0.13.0 h1:KvpoAJWEjR3uD9Kbm2HWJmqsEaHt8lBUpd0qHcIi21E= -github.com/invopop/jsonschema v0.13.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= -github.com/jessevdk/go-flags v1.6.1 h1:Cvu5U8UGrLay1rZfv/zP7iLpSHGUZ/Ou68T0iX1bBK4= -github.com/jessevdk/go-flags v1.6.1/go.mod h1:Mk8T1hIAWpOiJiHa9rJASDK2UGWji0EuPGBnNLMooyc= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/invopop/jsonschema v0.7.0 h1:2vgQcBz1n256N+FpX3Jq7Y17AjYt46Ig3zIWyy770So= +github.com/invopop/jsonschema v0.7.0/go.mod h1:O9uiLokuu0+MGFlyiaqtWxwqJm41/+8Nj0lD7A36YH0= +github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= +github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -280,14 +276,12 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -345,7 +339,6 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= @@ -368,6 +361,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -375,15 +369,9 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tetratelabs/wazero v1.11.0 h1:+gKemEuKCTevU4d7ZTzlsvgd1uaToIDtlQlmNbwqYhA= -github.com/tetratelabs/wazero v1.11.0/go.mod h1:eV28rsN8Q+xwjogd7f4/Pp4xFxO7uOGbLcD/LzB1wiU= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= -github.com/wasilibs/wazero-helpers v0.0.0-20250123031827-cd30c44769bb h1:gQ+ZV4wJke/EBKYciZ2MshEouEHFuinB85dY3f5s1q8= -github.com/wasilibs/wazero-helpers v0.0.0-20250123031827-cd30c44769bb/go.mod h1:jMeV4Vpbi8osrE/pKUxRZkVaA0EX7NZN0A9/oRzgpgY= -github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= -github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -490,8 +478,8 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -585,8 +573,8 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -597,8 +585,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -808,7 +796,6 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/re2/go.mod b/re2/go.mod new file mode 100644 index 0000000..b089598 --- /dev/null +++ b/re2/go.mod @@ -0,0 +1,39 @@ +module github.com/chainreactors/fingers/re2 + +go 1.24.0 + +require ( + github.com/chainreactors/fingers v0.0.0 + github.com/wasilibs/go-re2 v1.6.0 +) + +require ( + github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible // indirect + github.com/chainreactors/logs v0.0.0-20260508055944-c678762ed15c // indirect + github.com/chainreactors/neutron v0.1.1-0.20260714062907-716c6b167cb6 // indirect + github.com/chainreactors/utils v0.0.0-20260629043228-93bdd2142c9a // indirect + github.com/chainreactors/utils/parsers v0.0.4-0.20260925164932-39c659c84c64 // indirect + github.com/chainreactors/words v0.0.0-20260520145736-270600e60fb4 // indirect + github.com/dlclark/regexp2 v1.11.5 // indirect + github.com/facebookincubator/nvdtools v0.1.5 // indirect + github.com/go-dedup/megophone v0.0.0-20170830025436-f01be21026f5 // indirect + github.com/go-dedup/simhash v0.0.0-20170904020510-9ecaca7b509c // indirect + github.com/go-dedup/text v0.0.0-20170907015346-8bb1b95e3cb7 // indirect + github.com/hashicorp/go-version v1.6.0 // indirect + github.com/mozillazg/go-pinyin v0.20.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/tetratelabs/wazero v1.11.0 // indirect + github.com/twmb/murmur3 v1.1.8 // indirect + github.com/wasilibs/wazero-helpers v0.0.0-20250123031827-cd30c44769bb // indirect + golang.org/x/net v0.21.0 // indirect + golang.org/x/sys v0.38.0 // indirect + golang.org/x/text v0.14.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) + +// Development against the fingers checkout; downstream builds use the +// required version. +replace github.com/chainreactors/fingers => ../ + +replace github.com/wasilibs/go-re2 => github.com/chainreactors/go-re2 v1.11.1-0.20260716152604-8121b6cd261e diff --git a/re2/go.sum b/re2/go.sum new file mode 100644 index 0000000..4274373 --- /dev/null +++ b/re2/go.sum @@ -0,0 +1,822 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= +cloud.google.com/go v0.98.0/go.mod h1:ua6Ush4NALrHk5QXDWnjvZHN93OuF0HfuEPq9I1X0cM= +cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible h1:1G1pk05UrOh0NlF1oeaaix1x8XzrfjIDK47TY0Zehcw= +github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chainreactors/go-re2 v1.11.1-0.20260716152604-8121b6cd261e h1:8SMCCo85OKxOM0NtD8sPhmnhGzigU2i4YPUZHgWDVq8= +github.com/chainreactors/go-re2 v1.11.1-0.20260716152604-8121b6cd261e/go.mod h1:4qC68vqWSuPTct3spuTrWBqCpm00mQ707JKLS1izVjI= +github.com/chainreactors/logs v0.0.0-20260508055944-c678762ed15c h1:6Net2Mgo/qo6ADFBZJWWScKuMfZ0rbzLqSCVDuLKFdc= +github.com/chainreactors/logs v0.0.0-20260508055944-c678762ed15c/go.mod h1:VrXmYPbNN5AVoo1sc5aeyPVBYqubMdb3KO/tn5rRZpo= +github.com/chainreactors/neutron v0.1.1-0.20260714062907-716c6b167cb6 h1:apEDnJeZ5fe2AaEJHV6TRDldoy2t/d7E1maxmUz2tfU= +github.com/chainreactors/neutron v0.1.1-0.20260714062907-716c6b167cb6/go.mod h1:zok/CDxut71iw8NQTHKPvy0f+1G5639zr9GgDsXGnOs= +github.com/chainreactors/utils v0.0.0-20260629043228-93bdd2142c9a h1:4rLTKr+VlG7yGbw2ym7nhNvFglvZ4acdphxKEozF7CI= +github.com/chainreactors/utils v0.0.0-20260629043228-93bdd2142c9a/go.mod h1:xwbUlFoSSxLHujyb8D48o1s2DqmEAxUNfxIy0DVUmcg= +github.com/chainreactors/utils/parsers v0.0.4-0.20260925164932-39c659c84c64 h1:MgEJ0vr7png0bKb5nYL1pK5lTUMDBRbigYt74K+UwYw= +github.com/chainreactors/utils/parsers v0.0.4-0.20260925164932-39c659c84c64/go.mod h1:bE/znJWt08n9QOORWsWu0ggB8GWfOg3+dfUMMITmwV4= +github.com/chainreactors/words v0.0.0-20260520145736-270600e60fb4 h1:lvnDYEkatmZFHP5i321qQXK9L4vKRfso/uUfr5tOeC8= +github.com/chainreactors/words v0.0.0-20260520145736-270600e60fb4/go.mod h1:zfz367PUmyaX6oAqV9SktVqyRXKlEh0sel9Wsq9dd2c= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1/go.mod h1:+hnT3ywWDTAFrW5aE+u2Sa/wT555ZqwoCS+pk3p6ry4= +github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= +github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= +github.com/facebookincubator/flog v0.0.0-20190930132826-d2511d0ce33c/go.mod h1:QGzNH9ujQ2ZUr/CjDGZGWeDAVStrWNjHeEcjJL96Nuk= +github.com/facebookincubator/nvdtools v0.1.5 h1:jbmDT1nd6+k+rlvKhnkgMokrCAzHoASWE5LtHbX2qFQ= +github.com/facebookincubator/nvdtools v0.1.5/go.mod h1:Kh55SAWnjckS96TBSrXI99KrEKH4iB0OJby3N8GRJO4= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-dedup/megophone v0.0.0-20170830025436-f01be21026f5 h1:4U+x+EB1P66zwYgTjxWXSOT8vF+651Ksr1lojiCZnT8= +github.com/go-dedup/megophone v0.0.0-20170830025436-f01be21026f5/go.mod h1:poR/Cp00iqtqu9ltFwl6C00sKC0HY13u/Gh05ZBmP54= +github.com/go-dedup/simhash v0.0.0-20170904020510-9ecaca7b509c h1:mucYYQn+sMGNSxidhleonzAdwL203RxhjJGnxQU4NWU= +github.com/go-dedup/simhash v0.0.0-20170904020510-9ecaca7b509c/go.mod h1:gO3u2bjRAgUaLdQd2XK+3oooxrheOAx1BzS7WmPzw1s= +github.com/go-dedup/text v0.0.0-20170907015346-8bb1b95e3cb7 h1:11wFcswN+37U+ByjxdKzsRY5KzNqqq5Uk5ztxnLOc7w= +github.com/go-dedup/text v0.0.0-20170907015346-8bb1b95e3cb7/go.mod h1:wSsK4VOECOSfSYTzkBFw+iGY7wj59e7X96ABtNj9aCQ= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= +github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-hclog v1.0.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= +github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= +github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= +github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= +github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= +github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mozillazg/go-pinyin v0.20.0 h1:BtR3DsxpApHfKReaPO1fCqF4pThRwH9uwvXzm+GnMFQ= +github.com/mozillazg/go-pinyin v0.20.0/go.mod h1:iR4EnMMRXkfpFVV5FMi4FNB6wGq9NV6uDWbUuPhP4Yc= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tetratelabs/wazero v1.11.0 h1:+gKemEuKCTevU4d7ZTzlsvgd1uaToIDtlQlmNbwqYhA= +github.com/tetratelabs/wazero v1.11.0/go.mod h1:eV28rsN8Q+xwjogd7f4/Pp4xFxO7uOGbLcD/LzB1wiU= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= +github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= +github.com/wasilibs/wazero-helpers v0.0.0-20250123031827-cd30c44769bb h1:gQ+ZV4wJke/EBKYciZ2MshEouEHFuinB85dY3f5s1q8= +github.com/wasilibs/wazero-helpers v0.0.0-20250123031827-cd30c44769bb/go.mod h1:jMeV4Vpbi8osrE/pKUxRZkVaA0EX7NZN0A9/oRzgpgY= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU= +google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= +google.golang.org/api v0.62.0/go.mod h1:dKmwPCydfsad4qCH08MSdgWjfHOyfpd4VtDGgRFdavw= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/re2/re2.go b/re2/re2.go new file mode 100644 index 0000000..bdde320 --- /dev/null +++ b/re2/re2.go @@ -0,0 +1,19 @@ +// Package re2 switches the fingers rule engine to RE2 +// (github.com/wasilibs/go-re2), which matches large rule sets faster than +// the standard library. Import it for its side effect: +// +// import _ "github.com/chainreactors/fingers/re2" +// +// It is a separate module because RE2 needs Go 1.24, while fingers itself +// supports Go 1.17. Build with -tags re2_cgo to use a native libre2. +package re2 + +import ( + "github.com/chainreactors/fingers/fingers" + re2 "github.com/wasilibs/go-re2" +) + +func init() { fingers.RegexpCompiler = Compile } + +// Compile compiles expr with RE2. +func Compile(expr string) (fingers.CompiledRegexp, error) { return re2.Compile(expr) } diff --git a/re2/re2_test.go b/re2/re2_test.go new file mode 100644 index 0000000..5a1c28e --- /dev/null +++ b/re2/re2_test.go @@ -0,0 +1,55 @@ +package re2 + +import ( + "regexp" + "testing" + + fingerlib "github.com/chainreactors/fingers" + "github.com/chainreactors/fingers/fingers" + re2 "github.com/wasilibs/go-re2" +) + +const nginx = "HTTP/1.1 200 OK\r\nServer: nginx/1.24.0\r\n\r\nWelcome to nginx!" + +func TestEngineUsesRE2(t *testing.T) { + if _, err := fingers.RegexpCompiler("a+"); err != nil { + t.Fatal(err) + } + if c, _ := fingers.RegexpCompiler("a+"); c == nil { + t.Fatal("no compiler") + } else if _, ok := c.(*re2.Regexp); !ok { + t.Fatalf("compiler is %T, want *re2.Regexp", c) + } + engine, err := fingerlib.NewEngine(fingerlib.FingersEngine) + if err != nil { + t.Fatal(err) + } + frames, err := engine.DetectContent([]byte(nginx)) + if err != nil || frames["nginx"] == nil { + t.Fatalf("frames %v, %v", frames, err) + } +} + +// BenchmarkDetect compares the fingers engine on the standard library and RE2. +func BenchmarkDetect(b *testing.B) { + for _, c := range []struct { + name string + compile func(string) (fingers.CompiledRegexp, error) + }{ + {"regexp", func(s string) (fingers.CompiledRegexp, error) { return regexp.Compile(s) }}, + {"re2", Compile}, + } { + b.Run(c.name, func(b *testing.B) { + fingers.RegexpCompiler = c.compile + defer func() { fingers.RegexpCompiler = Compile }() + engine, err := fingerlib.NewEngine(fingerlib.FingersEngine) + if err != nil { + b.Fatal(err) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + engine.DetectContent([]byte(nginx)) + } + }) + } +} From 350638e912347d24c94b70043dcedf74de500827 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Fri, 25 Sep 2026 10:47:09 -0700 Subject: [PATCH 08/16] docs(judge): document Refine/Inspect/Classify, Framework.Judge and data egress Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 33 ++++++++++---- cmd/judgeeval/README.md | 16 +++---- cmd/judgeeval/main.go | 2 +- judge/README.md | 97 ++++++++++++++++++++++------------------- judge/doc.go | 19 ++++---- 5 files changed, 95 insertions(+), 72 deletions(-) diff --git a/README.md b/README.md index 881b263..364d77b 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,12 @@ fingers的拓展引擎, 有更全更大的指纹库. `go get github.com/chainreactors/fingers@master` +支持 Go 1.17 及以上。规则正则默认使用标准库；在 Go 1.24 及以上可以换成更快的 RE2，只需额外引入一行： + +```golang +import _ "github.com/chainreactors/fingers/re2" // go get github.com/chainreactors/fingers/re2 +``` + ### Example document: https://chainreactors.github.io/wiki/libs/fingers/ @@ -86,21 +92,32 @@ func TestFavicon(t *testing.T) { ## 判定层 judge -规则引擎召回能力强，但会有误报、多个引擎的同名重复，也分不清主应用和底层组件，版本号更是难以拿准。`judge` 包在规则结果之上加了一层判定（判别 + rerank）：规则和代码负责召回，Provider（模型）只对证据做判断，最终决策由代码做出。它不是新的引擎，`WebMatch` 和 `DetectContent` 的行为保持不变。Provider 可以替换，目前内置 [TypeSafe Jev](https://docs.typesafe.ai/api.md)（`judge/jev`）。 +规则引擎召回能力强，但会有误报、多个引擎的同名重复，也分不清主应用和底层组件，版本号更是难以拿准。`judge` 包在规则结果之上加了一层判定：规则和代码负责召回与抽取，Provider（模型）只对证据做判断，最终决策由代码做出。它不是新的引擎，`WebMatch` 和 `DetectContent` 的行为保持不变。Provider 可以替换，目前内置 [TypeSafe Jev](https://docs.typesafe.ai/api.md)（`judge/jev`）。 ```golang -j, _ := jev.NewJudge("") // 读取 TYPESAFE_API_KEY；自带缓存和相似页面去重 +j, _ := jev.NewJudge("") // 读取 TYPESAFE_API_KEY；一个扫描任务共用一个 Judge +j.Known = judge.NewRetriever(engine.Names()) // 召回规则漏掉、但页面里出现的已知产品 -frames, _ := engine.DetectContent(raw) // 同步，纯规则 -accepted, kind, generic, err := j.Refine(ctx, raw, frames) -if err == nil { - fmt.Println(kind, generic, accepted) // frames 保持原始规则命中 +hits, _ := engine.DetectContent(raw) // 同步，纯规则 +accepted, err := j.Refine(ctx, raw, hits) // 去误报、去重复、召回，并给每个产品补版本 +if err != nil { + accepted = hits // 判定失败时退回纯规则结果；hits 从不被修改 } +for _, f := range accepted { + if f.Judge != nil { // nil：未经判定（如超过每页 40 个产品的上限） + fmt.Println(f.Name, f.Version, f.Judge.Layer, f.Judge.Primary) + } +} +kind, generic, _ := j.Classify(ctx, raw) // 页面类型；Refine 之后命中缓存，不再请求 ``` -在 1070 个真实页面上：每页结果从 13.4 条降到 6.5 条，同名重复全部归并，版本号对照 generator 从 43 个正确提升到 191 个，并额外召回了 468 个规则漏掉的产品。成本约每万页 $3.4。 +判定结论是 `Framework.Judge` 字段（层级、置信度、误报、重复、主应用、召回），随现有输出一起序列化；下游不 import judge 也能用 `frames.Accepted()` / `frames.Primary()` 读取。需要看被剔除的条目和原因时用 `j.Inspect`。 + +**数据外发**：判定时会把响应的精简视图发给 Provider，包括响应头（去掉 Date、Set-Cookie 值等无关头）、Cookie 名、标题、generator/description、脚本和样式路径、内联脚本开头、HTML 注释、表单字段名，以及正文前 1500 字。扫描授权范围内的目标前，请确认允许把这些内容发送给第三方服务。 + +2026-09-25 在一批已标注的真实响应上：误报 24 → 0，真实命中误删 0，正确版本 3 → 23、错误 0，见 [验证报告](docs/jev-real-validation-20260925.md)。这些数字是本次版本算法调整之前测得的，新算法需要用 `cmd/judgeeval` 重新验证。 -完整的能力、`Inspect` 诊断、未知产品判定、指纹生成器、缓存和 Provider 接口，见 [judge/README.md](judge/README.md)；设计讨论见 [#34](https://github.com/chainreactors/fingers/issues/34)。 +完整的能力、缓存和 Provider 接口见 [judge/README.md](judge/README.md)；用正反样本生成原生指纹见 `judge/gen`；设计讨论见 [#34](https://github.com/chainreactors/fingers/issues/34)。 ## fingers 引擎 diff --git a/cmd/judgeeval/README.md b/cmd/judgeeval/README.md index a9325d5..edc3df2 100644 --- a/cmd/judgeeval/README.md +++ b/cmd/judgeeval/README.md @@ -2,12 +2,12 @@ ## 可回放真实数据评估 -`-manifest` 模式覆盖保存结果清洗、逐产品补全版本、名称召回和原生指纹生成。SDK 接口不变，结果写入独立目录。 +`-manifest` 模式覆盖保存结果清洗、逐产品补全版本、名称召回和原生指纹生成，结果写入独立目录。 ```powershell # urls.txt 每行一个 URL；新采集用新目录。IP 证书不匹配时显式加 --insecure。 python scripts/capture_judge.py --urls urls.txt --out .judge-data/corpus --workers 6 -go build -tags goregexp -o bin/judgeeval.exe ./cmd/judgeeval +go build -o bin/judgeeval.exe ./cmd/judgeeval # TYPESAFE_API_KEY 仅通过进程环境提供。 ./bin/judgeeval.exe -manifest .judge-data/corpus/manifest.json -cache .judge-data/cache -out .judge-data/run-01 -rps 8 # 同一快照及原结果用于验证历史清洗机制。 @@ -40,12 +40,12 @@ go build -tags goregexp -o bin/judgeeval.exe ./cmd/judgeeval - `manifest.json` / `samples/*.http` / `build.json`：本次实际输入及 Go 构建信息，响应可独立加载；离线模型答案仍需同时保留 `-cache` 目录。 - `baseline.jsonl`：原生 Frameworks，来源是当前规则或 `-history`。历史记录按 ID、SHA256 严格对齐，缺失即报错。 -- `cleaned.jsonl`：成功清洗并逐产品补版本的原生 Frameworks。 -- `rows.jsonl`：基线、Refine、补版本及拒绝/合并/新增/错误。 +- `cleaned.jsonl`：成功清洗并补版本的原生 Frameworks。 +- `rows.jsonl`：基线、Refine 及拒绝/合并/新增/补版本/错误。 - `metrics.json` / `report.md`：标注范围内的 TP/FP/FN/TN、误删、自然漏报恢复、版本正确/错误/缺失/无依据填值。失败和未标注输出单列。 - `generation.json` / `fingerprints/*.yaml`：候选指纹、独立验证结果及排除的样本。 -`Refine` 只补主产品版本；回放另对保留产品的空版本调用 `Version`，比较两阶段。已有非空版本保留，错误由报告揭示。缓存按模型、endpoint、完整 state 和问题精确复用，评估不使用近似页面缓存。 +`Refine` 给每个保留产品补版本；报告比较原结果和 Refine 两个阶段。已有非空版本保留，错误由报告揭示。缓存按模型、endpoint、完整 state 和问题精确复用，评估不使用近似页面缓存。 新增版本按产品及显式别名合并计数，原基线任一别名已有版本即不算新增。名称建议单独统计，不能算作已补回漏报。生成器训练样本的自动版本结果在 `training_versions` / `training_cases` 单列，错误时标记 `failed_training`；训练标签只在导出后打分。 @@ -78,10 +78,10 @@ go build -tags goregexp -o bin/judgeeval.exe ./cmd/judgeeval ## 目录模式 -在一批原始 HTTP 响应上，用和 SDK 调用方完全相同的路径（`engine.DetectContent` → `judge.Refine`）跑一遍，逐项对比纯规则结果和经过判定之后的结果。它也用来校准新的 Provider。 +在一批原始 HTTP 响应上，用和 SDK 调用方完全相同的路径（`engine.DetectContent` → `j.Refine`）跑一遍，逐项对比纯规则结果和经过判定之后的结果。它也用来校准新的 Provider。 ```bash -go build -buildvcs=false -tags goregexp -o judgeeval ./cmd/judgeeval +go build -buildvcs=false -o judgeeval ./cmd/judgeeval # 标注集（仓库自带 40 页合成样本，含真值） TYPESAFE_API_KEY=... ./judgeeval -provider jev -samples testdata/samples -labels testdata/labels.json @@ -90,7 +90,7 @@ TYPESAFE_API_KEY=... ./judgeeval -provider jev -samples testdata/samples -labels TYPESAFE_API_KEY=... ./judgeeval -provider jev -samples cc/samples -cache judgecache -out judgereport -rps 15 ``` -- 答案缓存在 `-cache` 目录，重跑不花钱，中断后可以接着跑。这个目录用的是精确缓存，另外会模拟相似度缓存：按签名距离分档，统计每一档能复用多少答案、与真实答案的一致率，用来确定 `judge.SimilarDistance` 的取值。 +- 答案缓存在 `-cache` 目录，重跑不花钱，中断后可以接着跑。这个目录用的是精确缓存，另外会模拟相似度缓存：按签名距离分档，统计每一档能复用多少答案、与真实答案的一致率，用来确定 `Judge.SimilarDistance` 的取值。 - `report.md`：对比表，另附页面类型分布、各引擎的否决率、否决率最高的规则（需要修复），以及新指纹候选。 - `rows.jsonl`：每页一行，用于人工复核。 - 给了 `-labels` 时，额外按真值统计误报剔除、真实命中保留、页面类型、通用页面和版本号的正确率。 diff --git a/cmd/judgeeval/main.go b/cmd/judgeeval/main.go index 18aea73..ce1cfcc 100644 --- a/cmd/judgeeval/main.go +++ b/cmd/judgeeval/main.go @@ -6,7 +6,7 @@ // TYPESAFE_API_KEY=... judgeeval -provider jev -samples cc/samples -cache cache -out report // // Answers are cached on disk, so reruns are free and an interrupted run -// resumes. Build with -tags goregexp. +// resumes. Build with any Go >= 1.17. package main import ( diff --git a/judge/README.md b/judge/README.md index e4305b8..e615906 100644 --- a/judge/README.md +++ b/judge/README.md @@ -1,69 +1,74 @@ -# judge: 显式判定 API +# judge：规则结果的判定层 -`fingers.Engine` 只做规则检测。`judge.Judge` 接受规则命中和原始 HTTP 响应，调用 Provider（当前提供 `judge/jev`），以代码确定最终语义。判定结果是新对象，原始规则命中保持不变。 +`fingers.Engine` 只做规则检测。`judge.Judge` 接收规则命中和原始 HTTP 响应，由代码抽取候选、由 Provider（当前是 `judge/jev`）回答判断题，最后由代码做决定。输入的规则命中从不被修改。 ```go engine, _ := fingers.NewEngine() -j, err := jev.NewJudge("") // TYPESAFE_API_KEY；一个扫描任务共用一个 Judge +j, err := jev.NewJudge("") // 读取 TYPESAFE_API_KEY；一个扫描任务共用一个 Judge if err != nil { return err } +j.Known = judge.NewRetriever(engine.Names()) // 可选：召回规则漏掉、但名字出现在页面里的产品 -hits, err := engine.DetectContent(raw) -if err != nil { return err } -accepted, kind, generic, err := j.Refine(ctx, raw, hits) -if err != nil { return err } // hits 仍可作为纯规则结果使用 -fmt.Println(accepted, kind, generic) +hits, _ := engine.DetectContent(raw) +accepted, err := j.Refine(ctx, raw, hits) +if err != nil { accepted = hits } // 判定失败时退回纯规则结果 ``` -## 能力与语义 +## 三个入口 -| API | 含义 | +| 方法 | 用途 | |---|---| -| `j.Yes(ctx, state, question)` | 返回“是”的概率，`float64`，不是布尔判决 | -| `j.Choose(ctx, state, question, options)` | 从给定选项中选一个，返回选项和置信度 | -| `j.Score(ctx, state, question, levels...)` | 按有序等级返回 0 到 1 的分数 | -| `j.Verify(ctx, raw, hits, knownNames...)` | 核验产品是否存在、归并同名、判断层级和主应用，返回接受的指纹 | -| `j.Inspect(ctx, raw, hits, knownNames...)` | 返回所有规则命中及拒绝、重复、层级、主应用等诊断标记 | -| `j.Classify(ctx, raw)` | 返回页面类型 `Kind` 和是否为产品标准页面 | -| `j.Version(ctx, raw, framework)` | 在响应中的版本候选里选择该产品的版本；不修改输入 | -| `j.Refine(ctx, raw, hits, knownNames...)` | 核验、分类并为主产品选择版本；返回指纹、类型和标准页面判断 | -| `j.IsUnknownProduct(ctx, raw, accepted)` | 标准产品页面但没有已确认的应用或设备时返回 true | -| `j.SuggestNames(ctx, raw)` | 从本页头、标题、资源和正文提取候选，并判断哪些是产品名 | -| `judge.NewGenerator(j)` | 用正反样本生成原生 `*fingers.Finger` | +| `j.Refine(ctx, raw, hits)` | **要上报的结果**：去掉误报和重复写法，加入召回的产品，给每个保留的产品补版本 | +| `j.Inspect(ctx, raw, hits)` | **为什么**：全部命中及判定结论，包括被拒绝和重复的；不补版本 | +| `j.Classify(ctx, raw)` | **这是什么页面**：`Kind`（登录、错误、默认页……）和是否为产品标准页；Refine 之后命中缓存 | -`state` 是任意可 JSON 编码的值。判断 HTTP 页面时，可以传 `judge.NewPage(raw)` 的结果。`Verify` 与 `Refine` 不修改 `hits`，失败时也不会返回部分判定结果。需要看到被剔除的条目时用 `Inspect`，再以 `judge.Is`、`judge.LayerOf`、`judge.PrimaryOf` 读取标记。 +其余能力：`j.Version(ctx, raw, f)` 单独给一个产品选版本；`j.IsUnknownProduct(ctx, raw, accepted)` 判断是否为尚无指纹的产品标准页；`j.SuggestNames(ctx, raw)` 从页面证据中提出产品名；`j.Yes/Choose/Score` 用来问自定义问题，state 可以是任意可 JSON 编码的值，比如 `judge.NewPage(raw)`。 -`knownNames` 是调用方明确给出的**已有**产品名候选。`judge.NewRetriever(names).Find(page.Haystack(), limit)` 可在页面中查找现有别名。它只做已知名称召回；`IsUnknownProduct` 和 `SuggestNames` 是独立能力。后者只返回本页可提取的名字，不能还原已彻底改名且没有原名证据的产品。 +## 判定结论 -`Refine` 通常发两轮请求：第一轮核验与页面分类，第二轮为已确认的主产品挑版本。答案按问题缓存，相似页面可共享。Provider 缺少任何被请求的答案时，整轮失败。 +结论写在 `Framework.Judge`（定义在 `utils/parsers`）上，随现有输出一起序列化： -## 生成指纹 +| 字段 | 含义 | +|---|---| +| `Layer` | 在技术栈中的层级：`judge.LayerCDN`、`LayerServer`、`LayerApplication` …… | +| `Confidence` | Provider 给出的"该产品确实参与生成此响应"的概率 | +| `Rejected` | 误报：只在页面文字中出现，或不存在。响应头或 Cookie 中出现的产品不会被拒绝 | +| `Duplicate` | 另一个引擎对已保留产品的不同写法 | +| `Primary` | 页面所属的主应用 | +| `Recalled` | 规则没报、由 `j.Known` 在页面中找到并经确认 | -```go -g := judge.NewGenerator(j). - PositiveVersion(jenkinsA, "2.401.3"). - PositiveVersion(jenkinsB, "2.402.1"). - Negative(otherProduct) +`Judge == nil` 表示未经判定（每页超过 40 个产品时，多出的部分不判定），`frames.Accepted()` 会保留它们。 -finger, err := g.Generate(ctx) // *fingers.Finger，可直接用于现有规则引擎 -if err != nil { return err } -err = g.Validate(finger) // 编辑规则后，再按全部正反样本验证 -``` +## 版本号 -至少需要一个正样本和一个反样本。`Name("Jenkins")` 可以给出已知名称；缺省时从正样本调用 `SuggestNames` 自动选择。`PositiveVersion` 提供版本真值；普通 `Positive` 在有 Judge 时会尝试从页面候选中高置信度选版本。版本规则只有在样本证据足以构造并验证提取式时才会生成。 +版本字符串一律由代码从原始响应中抽取，Provider 只做选择，不生成字符串： -主动探测以原始 `send_data` 和对应的完整 HTTP 响应记录： +1. 响应**按名字绑定**的版本直接采用，不提问：`Server: nginx/1.24.0`、`X-Jenkins: 2.401.3`、`X-Gitea-Version: 1.21.4`、``。名字按末尾词匹配，所以 `Apache Tomcat/9.0` 属于 Tomcat 而不是 Apache。 +2. 其余产品在同一个请求里各问一题。候选按与产品名的接近程度排序（上下文提到产品名，紧挨版本前更好），插件、主题、第三方库路径降权；已按名字绑定给别的产品、且只出现一次的版本不会提供给它。 +3. 只出现一次的版本字符串只归一个产品：上下文更贴近的优先，其次置信度更高的。置信度低于 `VersionConfidence` 的选择不采用。 -```go -g := judge.NewGenerator(j).Name("Orion"). - Positive(home).Probe([]byte("/admin"), adminResponse). - Negative(otherHome).Probe([]byte("/admin"), otherAdminResponse) -finger, err := g.Generate(ctx) -``` +## 缓存与成本 -`ProbeWith(ctx, request, send)` 可由调用方提供在线发送函数。主动规则需要同一请求的正反响应；`Validate` 通过现有 `Finger.Match` 执行被动与主动规则，不调用模型，也不会写入模板仓库。无法区分正反样本时，`Generate` 返回错误。 +答案按问题缓存，同一问题不会重复发送，无论它被放进哪次请求。标题相同、签名相差不超过 `j.SimilarDistance` 位（默认 1）的页面共享答案，同一产品在不同主机上的登录页只问一次；并发的相同请求会合并。`j.Cache` 默认是内存 LRU，实现 `judge.Cache` 即可换成共享存储。`j.Requests`、`j.CacheHits` 用于成本监控。 -## 自定义 Provider 与批量问题 +## 自定义 Provider -Provider 只需实现 `ID() string` 和 `Judge(ctx, state, map[string]judge.Question) (map[string]judge.Answer, error)`。可选实现 `Calibration() (threshold, versionConfidence float64)`。Jev 适配器把二元问题映射到 `noul`，把选择和评分映射到 Jev 对应类型；缓存、并发合并和阈值决策在 `Judge` 中完成。 +实现 `ID() string` 和 `Judge(ctx, state, map[string]judge.Question) (map[string]judge.Answer, error)` 即可，可选实现 `Calibration() (threshold, versionConfidence float64)`。问题分三类：`TypeBinary`（是的概率）、`TypeChoice`（从选项中选一个）、`TypeScore`（有序等级上的分数）。缓存、并发合并和阈值决策都在 `Judge` 中完成，因此不同 Provider 的行为一致。新 Provider 需在 `cmd/judgeeval` 的 `newProvider` 注册并跑一遍评估来校准阈值。 + +## 数据外发 + +发给 Provider 的是响应的精简视图：响应头（去掉 Date、Set-Cookie 等无关或易变的头）、Cookie 名、标题、generator/description、脚本和样式路径、内联脚本开头、HTML 注释、表单字段名，以及正文前 1500 字。扫描前请确认允许把这些内容发送给第三方服务。 + +## 生成指纹 + +`judge/gen` 用正反样本生成原生 `*fingers.Finger`： + +```go +g := gen.New(j). + PositiveVersion(jenkinsA, "2.401.3"). + PositiveVersion(jenkinsB, "2.402.1"). + Negative(otherProduct) +finger, err := g.Generate(ctx) // 可直接用于现有规则引擎 +err = g.Validate(finger) // 编辑规则后，再按全部正反样本验证 +``` -自定义问题可通过 `j.Yes`、`j.Choose`、`j.Score` 单独调用。需要把多个问题合并为一次请求时，使用 `page.Round()`、`Round.Add` 和 `Round.Ask`；它是高级组合接口，`Question`/`Answer` 仅服务于 Provider 和批量扩展。 +至少需要一个正样本和一个反样本。不给 `Name` 时，由 `SuggestNames` 从正样本中选名字；`Probe`/`ProbeWith` 记录主动探测的请求和响应。`Validate` 只运行规则引擎、不调用模型，也不会写入模板仓库。 diff --git a/judge/doc.go b/judge/doc.go index 7e7c89e..0c569e9 100644 --- a/judge/doc.go +++ b/judge/doc.go @@ -1,15 +1,16 @@ -// Package judge exposes evidence-based judgement operations for HTTP -// fingerprints. A Provider (such as TypeSafe Jev in judge/jev) answers typed -// questions; Judge applies thresholds, caching and fingerprint semantics. +// Package judge filters and completes rule-engine results with a judgement +// provider. Code extracts candidates and decides; a Provider (such as +// TypeSafe Jev in judge/jev) only answers typed questions about the evidence. // // j := judge.New(provider) -// accepted, kind, generic, err := j.Refine(ctx, raw, ruleHits) -// all, err := j.Inspect(ctx, raw, ruleHits) // includes rejected hits -// finger, err := judge.NewGenerator(j).Positive(good).Negative(other).Generate(ctx) +// j.Known = judge.NewRetriever(engine.Names()) +// accepted, err := j.Refine(ctx, raw, hits) // what to report +// all, err := j.Inspect(ctx, raw, hits) // why: rejected and duplicate hits too +// kind, generic, err := j.Classify(ctx, raw) // -// Judge methods do not mutate rule hits. Round remains available for callers -// composing custom questions into one provider request. Similar pages can -// share cached answers through Page.Signature. +// Verdicts are written to Framework.Judge. Hits passed in are never changed. +// Answers are cached per question, and similar pages share them. Fingerprint +// generation from labelled responses lives in judge/gen. // // See README.md in this directory. package judge From 1f4243915799d97c350432d33753eefe0ec96005 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Fri, 25 Sep 2026 10:48:23 -0700 Subject: [PATCH 09/16] build(re2): require a published fingers version Replace directives of a dependency are ignored downstream, so the re2 module must require a fingers version that exists; the local replace stays for development and CI. Co-Authored-By: Claude Opus 5.5 (1M context) --- re2/go.mod | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/re2/go.mod b/re2/go.mod index b089598..db18a83 100644 --- a/re2/go.mod +++ b/re2/go.mod @@ -3,7 +3,7 @@ module github.com/chainreactors/fingers/re2 go 1.24.0 require ( - github.com/chainreactors/fingers v0.0.0 + github.com/chainreactors/fingers v1.2.2-0.20260925174709-350638e91234 github.com/wasilibs/go-re2 v1.6.0 ) @@ -32,8 +32,8 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect ) -// Development against the fingers checkout; downstream builds use the -// required version. -replace github.com/chainreactors/fingers => ../ - replace github.com/wasilibs/go-re2 => github.com/chainreactors/go-re2 v1.11.1-0.20260716152604-8121b6cd261e + +// Development and CI use the checkout; downstream builds use the version +// required above. +replace github.com/chainreactors/fingers => ../ From 12229287bf574ea0acaab75a5c0b7414444fb8b9 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Sat, 26 Sep 2026 20:16:35 -0700 Subject: [PATCH 10/16] refactor(judge): fingerprint claims only; the contract moves to utils/jev Every judgement is a Claim: a statement with finite options, each mapped to holds / refuted / insufficient, and an explicit abstention. The generic part (Claim/Option/Outcome/Ruling, validation, Provider, the exact batch cache and the Jev HTTP client) now lives in github.com/chainreactors/utils/jev; fingers keeps only what is about fingerprints: - judge: Judge{Provider, MinConfidence, DropInsufficient}, New, Inspect (presence + version in at most two batches), Version, and the presence options declared/absent/running/mentioned/unrelated. Code-established facts are rulings of the same claims. - judge/maintain: Ledger (junk-rule audit from presence annotations) and Discover (clusters pages shared by hosts, Coverage claim per cluster). - judge/gen: rule generation for a human-given name. - Engine.EnableMatchDetail: fingers, fingerprinthub, wappalyzer, ehole and goby record the matched text so claims can quote it. Removed: judge/jev, the generic j.Judge, Refine/Classify/Recall and the Identify/Ask/Yes/Choose/Score question types. judge.New now defaults to jev.DefaultMinConfidence (0.3). Requires utils/parsers with the Judgement {Verdict, Outcome, Confidence, Evidence, Rejected, Duplicate}. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 21 +- cmd/engine/example.go | 45 +- cmd/judgeeval/README.md | 32 +- cmd/judgeeval/main.go | 701 ++++------------------- cmd/judgeeval/maintenance.go | 64 +-- cmd/judgeeval/maintenance_test.go | 6 +- cmd/judgeeval/provider.go | 89 +++ cmd/judgeeval/replay.go | 238 ++++---- cmd/judgeeval/replay_generation.go | 122 ++-- cmd/judgeeval/replay_generation_test.go | 126 ++++ cmd/judgeeval/replay_metrics.go | 179 ++++-- cmd/judgeeval/replay_test.go | 166 +++++- docs/jev-claim-acceptance-20260926.md | 92 +++ ehole/ehole.go | 31 +- engine.go | 37 +- fingerprinthub/fingerprinthub.go | 16 +- fingerprinthub/passive_template.go | 30 +- go.mod | 3 +- go.sum | 6 +- goby/goby.go | 15 +- judge/README.md | 91 ++- judge/cache.go | 86 --- judge/classify.go | 38 -- judge/discovery.go | 77 --- judge/doc.go | 26 +- judge/facts.go | 254 ++++++++ judge/gen/gen.go | 59 +- judge/gen/gen_test.go | 26 +- judge/inspect.go | 122 ++++ judge/internal/evidence/evidence.go | 22 +- judge/internal/evidence/evidence_test.go | 19 + judge/internal/evidence/page.go | 134 +++++ judge/jev/jev.go | 194 ------- judge/jev/jev_test.go | 53 -- judge/judge.go | 247 +------- judge/judge_test.go | 481 ++++++---------- judge/maintain/coverage.go | 68 +++ judge/maintain/discover.go | 231 ++++++++ judge/maintain/ledger.go | 141 +++++ judge/maintain/maintain_test.go | 148 +++++ judge/operations.go | 127 ---- judge/operations_test.go | 172 +++--- judge/page.go | 258 --------- judge/presence.go | 129 +++++ judge/provider.go | 70 --- judge/real_evidence_test.go | 71 ++- judge/recall.go | 74 --- judge/refine.go | 90 --- judge/round.go | 88 --- judge/stack.go | 35 -- judge/verify.go | 203 ------- judge/version.go | 169 +++--- judge_test.go | 74 ++- wappalyzer/fingerprints.go | 36 ++ wappalyzer/wappalyzer.go | 18 +- 55 files changed, 2889 insertions(+), 3261 deletions(-) create mode 100644 cmd/judgeeval/provider.go create mode 100644 cmd/judgeeval/replay_generation_test.go create mode 100644 docs/jev-claim-acceptance-20260926.md delete mode 100644 judge/cache.go delete mode 100644 judge/classify.go delete mode 100644 judge/discovery.go create mode 100644 judge/facts.go create mode 100644 judge/inspect.go create mode 100644 judge/internal/evidence/evidence_test.go create mode 100644 judge/internal/evidence/page.go delete mode 100644 judge/jev/jev.go delete mode 100644 judge/jev/jev_test.go create mode 100644 judge/maintain/coverage.go create mode 100644 judge/maintain/discover.go create mode 100644 judge/maintain/ledger.go create mode 100644 judge/maintain/maintain_test.go delete mode 100644 judge/operations.go delete mode 100644 judge/page.go create mode 100644 judge/presence.go delete mode 100644 judge/provider.go delete mode 100644 judge/recall.go delete mode 100644 judge/refine.go delete mode 100644 judge/round.go delete mode 100644 judge/stack.go delete mode 100644 judge/verify.go diff --git a/README.md b/README.md index 364d77b..cc3f82d 100644 --- a/README.md +++ b/README.md @@ -92,32 +92,35 @@ func TestFavicon(t *testing.T) { ## 判定层 judge -规则引擎召回能力强，但会有误报、多个引擎的同名重复，也分不清主应用和底层组件，版本号更是难以拿准。`judge` 包在规则结果之上加了一层判定：规则和代码负责召回与抽取，Provider（模型）只对证据做判断，最终决策由代码做出。它不是新的引擎，`WebMatch` 和 `DetectContent` 的行为保持不变。Provider 可以替换，目前内置 [TypeSafe Jev](https://docs.typesafe.ai/api.md)（`judge/jev`）。 +规则引擎召回能力强，但会有误报、多个引擎的同名重复，也会漏掉规则库里没有的产品，版本号更是难以拿准。`judge` 包在规则结果之上加了一层审查：**它只审查规则给出的结论，不做指纹识别，模型从不说出产品名。** 它不是新的引擎，`WebMatch` 和 `DetectContent` 的行为保持不变。Claim 合约、缓存和 [TypeSafe Jev](https://docs.typesafe.ai/api.md) 客户端在 [`github.com/chainreactors/utils/jev`](https://github.com/chainreactors/utils/tree/master/jev)，fingers 只保留指纹相关的断言。 + +判定层只有一个机制，即 Claim：代码提出断言并附上证据，Provider 选出证据支持的选项，选项的 Outcome（holds / refuted / insufficient）驱动固定动作。去误报（Presence）、找漏报（Coverage）、确认版本（Version）都是 Claim；与指纹无关的自定义 Claim 直接用 `jev.Judge`。 ```golang -j, _ := jev.NewJudge("") // 读取 TYPESAFE_API_KEY；一个扫描任务共用一个 Judge -j.Known = judge.NewRetriever(engine.Names()) // 召回规则漏掉、但页面里出现的已知产品 +engine.EnableMatchDetail() // 让 Claim 能引用规则命中的原文 +c, _ := jev.NewClient("") // 读取 TYPESAFE_API_KEY +j := judge.New(jev.Cached(c, jev.DefaultCacheSize)) // 一个扫描任务共用一个 Judge hits, _ := engine.DetectContent(raw) // 同步，纯规则 -accepted, err := j.Refine(ctx, raw, hits) // 去误报、去重复、召回，并给每个产品补版本 +all, err := j.Inspect(ctx, raw, hits) // 一次完成 presence 与 version，保留全部注解 +accepted := all.Accepted() // 上报视图：排除拒绝和重复条目 if err != nil { accepted = hits // 判定失败时退回纯规则结果；hits 从不被修改 } for _, f := range accepted { if f.Judge != nil { // nil：未经判定（如超过每页 40 个产品的上限） - fmt.Println(f.Name, f.Version, f.Judge.Layer, f.Judge.Primary) + fmt.Println(f.Name, f.Version, f.Judge.Verdict, f.Judge.Outcome) } } -kind, generic, _ := j.Classify(ctx, raw) // 页面类型；Refine 之后命中缓存，不再请求 ``` -判定结论是 `Framework.Judge` 字段（层级、置信度、误报、重复、主应用、召回），随现有输出一起序列化；下游不 import judge 也能用 `frames.Accepted()` / `frames.Primary()` 读取。需要看被剔除的条目和原因时用 `j.Inspect`。 +判定结论是 `Framework.Judge` 字段（选项、Outcome、置信度、证据、是否剔除、是否重复），随现有输出一起序列化；下游不 import judge 也能用 `frames.Accepted()` 读取。被剔除的条目和原因已经包含在 `all` 中，无须再次请求 Provider。漏报在离线维护中处理：`judge/maintain.Discover` 把多个主机上的同一页面聚成簇，逐簇提出 Coverage Claim，判为 missing 的簇连同候选名交给人命名，再用 `judge/gen` 生成规则。 **数据外发**：判定时会把响应的精简视图发给 Provider，包括响应头（去掉 Date、Set-Cookie 值等无关头）、Cookie 名、标题、generator/description、脚本和样式路径、内联脚本开头、HTML 注释、表单字段名，以及正文前 1500 字。扫描授权范围内的目标前，请确认允许把这些内容发送给第三方服务。 -2026-09-25 在一批已标注的真实响应上：误报 24 → 0，真实命中误删 0，正确版本 3 → 23、错误 0，见 [验证报告](docs/jev-real-validation-20260925.md)。这些数字是本次版本算法调整之前测得的，新算法需要用 `cmd/judgeeval` 重新验证。 +历史 Claim 实现于 2026-09-26 在 69 份已标注的真实响应上：误报 24 → 0，真实命中误删 0，版本正确 36、错误 0，见 [历史验收报告](docs/jev-claim-acceptance-20260926.md)。这些指标尚未用本次简化实现重新请求 Jev 验证。 -完整的能力、缓存和 Provider 接口见 [judge/README.md](judge/README.md)；用正反样本生成原生指纹见 `judge/gen`；设计讨论见 [#34](https://github.com/chainreactors/fingers/issues/34)。 +指纹判定的完整能力见 [judge/README.md](judge/README.md)；用正反样本生成原生指纹见 `judge/gen`；设计讨论见 [#34](https://github.com/chainreactors/fingers/issues/34)。 ## fingers 引擎 diff --git a/cmd/engine/example.go b/cmd/engine/example.go index 5722483..2ff3737 100644 --- a/cmd/engine/example.go +++ b/cmd/engine/example.go @@ -15,10 +15,10 @@ import ( "github.com/chainreactors/fingers" "github.com/chainreactors/fingers/common" "github.com/chainreactors/fingers/judge" - "github.com/chainreactors/fingers/judge/jev" "github.com/chainreactors/fingers/resources" "github.com/chainreactors/utils/encode" "github.com/chainreactors/utils/httputils" + "github.com/chainreactors/utils/jev" "github.com/jessevdk/go-flags" "gopkg.in/yaml.v3" ) @@ -181,6 +181,9 @@ func main() { fmt.Printf("Failed to create engine: %v\n", err) os.Exit(1) } + if opts.Judge != "" && engine.Fingers() != nil { + engine.EnableMatchDetail() // the judge quotes what each rule matched + } if opts.Verbose { fmt.Printf("Loaded engines: %s\n", engine.String()) @@ -204,7 +207,7 @@ func main() { } } else { var content []byte - if opts.Judge != "" { // Refine needs the raw response; put the body back for Match + if opts.Judge != "" { // Inspect needs the raw response; put the body back for Match content = httputils.ReadRaw(resp) body, _, _ := httputils.SplitHttpRaw(content) resp.Body = ioutil.NopCloser(bytes.NewReader(body)) @@ -215,38 +218,26 @@ func main() { fmt.Printf("unknown judge provider %q\n", opts.Judge) os.Exit(1) } - j, err := jev.NewJudge("") + c, err := jev.NewClient("") if err != nil { fmt.Println(err) os.Exit(1) } - j.Known = judge.NewRetriever(engine.Names()) + j := judge.New(jev.Cached(c, jev.DefaultCacheSize)) ctx := context.Background() - accepted, err := j.Refine(ctx, content, frames) - if err == nil { - var kind judge.Kind - var generic bool - if kind, generic, err = j.Classify(ctx, content); err == nil { - fmt.Printf("page: %s, generic: %v\n", kind, generic) - } - } + judged, err := j.Inspect(ctx, content, frames) + accepted := judged.Accepted() if err != nil { - fmt.Printf("jev failed, showing the rule result: %v\n", err) - } else { - for _, frame := range accepted { - verdict := "accepted" - if frame.Judge != nil && frame.Judge.Primary { - verdict += ",primary" - } - if frame.Judge != nil && frame.Judge.Recalled { - verdict += ",recalled" - } - layer := "" - if frame.Judge != nil { - layer = frame.Judge.Layer - } - fmt.Printf(" %-40s %-10s %-18s %s\n", frame.Name, frame.Version, layer, verdict) + fmt.Printf("jev failed, code-established verdicts only: %v\n", err) + } + for _, frame := range accepted { + verdict := "unjudged" + if frame.Judge != nil { + verdict = frame.Judge.Verdict } + fmt.Printf(" %-40s %-10s %s\n", frame.Name, frame.Version, verdict) + } + if err == nil { fmt.Printf("accepted: %s\n", accepted.String()) return } diff --git a/cmd/judgeeval/README.md b/cmd/judgeeval/README.md index edc3df2..695a410 100644 --- a/cmd/judgeeval/README.md +++ b/cmd/judgeeval/README.md @@ -2,7 +2,7 @@ ## 可回放真实数据评估 -`-manifest` 模式覆盖保存结果清洗、逐产品补全版本、名称召回和原生指纹生成，结果写入独立目录。 +`-manifest` 模式覆盖保存结果清洗、逐产品补全版本、漏报聚类和原生指纹生成，结果写入独立目录。 ```powershell # urls.txt 每行一个 URL；新采集用新目录。IP 证书不匹配时显式加 --insecure。 @@ -32,7 +32,7 @@ go build -o bin/judgeeval.exe ./cmd/judgeeval ### 生成与独立验证 -`generation` 每项填写 `name`（输出文件名）、`product`、`auto_name`、`positive`/`negative`（训练快照 ID）。其他有该产品标注的快照用于测试；排除训练主机、相同响应和重复测试响应。使用普通 `Positive` 自动推断版本，测试标签只打分。 +`generation` 每项填写 `name`（输出文件名）、`product`、`positive`/`negative`（训练快照 ID）。其他有该产品标注的快照用于测试；排除训练主机、相同响应和重复测试响应。使用普通 `Positive` 自动推断版本，测试标签只打分。 主动规则另填 `probe`（请求字符串），各样本 `probes` 将请求映射到同主机已采集响应 ID。回放不发送网络请求。导出 YAML 后重新加载、编译并验证。结论为 `passed_holdout`、`failed_holdout`、`insufficient_holdout`；至少需要独立正反例且版本无错漏才能在本批次通过。单主机新产品仅能产生待验证草案。生成文件供审阅，不自动写入指纹库。 @@ -41,13 +41,13 @@ go build -o bin/judgeeval.exe ./cmd/judgeeval - `manifest.json` / `samples/*.http` / `build.json`：本次实际输入及 Go 构建信息，响应可独立加载；离线模型答案仍需同时保留 `-cache` 目录。 - `baseline.jsonl`：原生 Frameworks，来源是当前规则或 `-history`。历史记录按 ID、SHA256 严格对齐，缺失即报错。 - `cleaned.jsonl`：成功清洗并补版本的原生 Frameworks。 -- `rows.jsonl`：基线、Refine 及拒绝/合并/新增/补版本/错误。 +- `rows.jsonl`：`id/url/sha256/baseline/judged/error/rule_ms/judge_ms`，两种输入模式相同；派生状态不重复存储。 - `metrics.json` / `report.md`：标注范围内的 TP/FP/FN/TN、误删、自然漏报恢复、版本正确/错误/缺失/无依据填值。失败和未标注输出单列。 -- `generation.json` / `fingerprints/*.yaml`：候选指纹、独立验证结果及排除的样本。 +- `generation.json` / `fingerprints/*.yaml`：候选指纹、逐样本原生 Framework 结果及排除的样本。`training` / `holdout` 按样本 ID 索引，null 表示已评估且未命中，缺失 key 表示未评估。产品、标签、训练计划统一读取 manifest；状态与分数由报告推导，不再写入另一套逐样本 DTO。 -`Refine` 给每个保留产品补版本；报告比较原结果和 Refine 两个阶段。已有非空版本保留，错误由报告揭示。缓存按模型、endpoint、完整 state 和问题精确复用，评估不使用近似页面缓存。 +`Inspect` 一次完成存在性和版本判断；报告比较 Baseline 与 Judged.Accepted() 两个阶段。已有非空版本保留，错误由报告揭示。缓存按模型、endpoint、完整 state 和 Claim 精确复用，使用 `claim-report-v3` 命名空间隔离旧缓存，Ruling 保存为 `option/confidence`。metrics.json 的 schema 为 3，统计使用 `claims/options`；manifest、旧 labels 和 baseline 输入格式仍支持。 -新增版本按产品及显式别名合并计数，原基线任一别名已有版本即不算新增。名称建议单独统计，不能算作已补回漏报。生成器训练样本的自动版本结果在 `training_versions` / `training_cases` 单列，错误时标记 `failed_training`；训练标签只在导出后打分。 +新增版本按产品及显式别名合并计数，原基线任一别名已有版本即不算新增。漏报簇的候选名称单独统计，不能算作已补回漏报。生成器训练样本的自动版本保存在 `training` 内的 Framework；训练标签只在导出后打分。训练结果错误或缺失得到 `failed_training`，测试有错得到 `failed_holdout`，未完成全部测试或缺少独立正反例得到 `insufficient_holdout`。报告与维护收录共用这一评估逻辑。 新采集快照的历史回放验证的是清洗机制，不代表已验证生产历史分布。样本量、同主机依赖和标签覆盖范围必须随指标报告。 @@ -67,7 +67,7 @@ go build -o bin/judgeeval.exe ./cmd/judgeeval 新增被动指纹的收录条件： 1. 在五个内置 HTTP 引擎目录、别名表和实际加载的 fingers 运行时目录中均无同名产品。按规范名及显式别名审计，记录各目录 SHA256；并非覆盖所有外部指纹源。 -2. 普通 `Positive` 自动推断名称和版本，通过训练集、独立正反例、版本验证；不把标签版本传给生成器。 +2. 普通 `Positive` 为指定名称自动确认版本，通过训练集、独立正反例、版本验证；不把标签版本传给生成器。 3. 至少两个独立正例主机组、五个独立反例主机组。相同主机名或同属一个 `group` 的样本合并计数，且排除训练主机和重复内容。 4. 调用 `engine.Fingers().LoadFromYAML(...)` 和 `engine.Compile()` 后重放全部样本。新增产品的标注识别/版本必须全部正确，已有命中和非空版本必须保留。 @@ -78,7 +78,7 @@ go build -o bin/judgeeval.exe ./cmd/judgeeval ## 目录模式 -在一批原始 HTTP 响应上，用和 SDK 调用方完全相同的路径（`engine.DetectContent` → `j.Refine`）跑一遍，逐项对比纯规则结果和经过判定之后的结果。它也用来校准新的 Provider。 +在一批原始 HTTP 响应上，用和 SDK 调用方完全相同的路径（`engine.DetectContent` → `j.Inspect` → `all.Accepted()`）跑一遍，逐项对比纯规则结果和经过判定之后的结果。它也用来校准新的 Provider。 ```bash go build -buildvcs=false -o judgeeval ./cmd/judgeeval @@ -90,14 +90,8 @@ TYPESAFE_API_KEY=... ./judgeeval -provider jev -samples testdata/samples -labels TYPESAFE_API_KEY=... ./judgeeval -provider jev -samples cc/samples -cache judgecache -out judgereport -rps 15 ``` -- 答案缓存在 `-cache` 目录，重跑不花钱，中断后可以接着跑。这个目录用的是精确缓存，另外会模拟相似度缓存：按签名距离分档，统计每一档能复用多少答案、与真实答案的一致率，用来确定 `Judge.SimilarDistance` 的取值。 -- `report.md`：对比表，另附页面类型分布、各引擎的否决率、否决率最高的规则（需要修复），以及新指纹候选。 -- `rows.jsonl`：每页一行，用于人工复核。 -- 给了 `-labels` 时，额外按真值统计误报剔除、真实命中保留、页面类型、通用页面和版本号的正确率。 -- 新 Provider：在 `main.go` 的 `newProvider` 里注册。 - -没有真值的数据集用两个代理指标： -- **仅正文出现的命中**：代码判断出的疑似误报； -- **版本号**：以页面上的 generator meta 作为参考答案。 - -真实网站的页面属于第三方内容，不要提交进仓库。 +- 两种模式共用一个磁盘 Provider 包装器，保存完整批次的有效 Ruling、统计调用和缓存命中。缓存写入使用临时文件再重命名；失败与非法回答不落盘。 +- `rows.jsonl` 保存唯一的 Baseline/Judged 记录；`metrics.json` / `report.md` 从同一结果统计，未标注预测单列，不使用正文提及或 generator 代理真值。 +- 目录模式的 `-labels` 支持旧 keep/drop/version 对象，也支持每个样本的 productLabel 数组。旧格式只在输入边界转换；统一使用规范名和显式 aliases 打分。 +- `ledger.json` 保存规则审计快照；manifest 模式的 `discovery.json` 直接保存 maintain.Cluster，以样本 ID 关联，不在每行复制漏报候选。 +- 新 Provider 在 `main.go` 的 `newProvider` 注册，并实现 Claim/Ruling 合约。 diff --git a/cmd/judgeeval/main.go b/cmd/judgeeval/main.go index ce1cfcc..496c6f1 100644 --- a/cmd/judgeeval/main.go +++ b/cmd/judgeeval/main.go @@ -1,12 +1,4 @@ -// judgeeval compares the rule engines alone with rules plus a judge provider, -// exactly as SDK callers run it (judge.Judge.Refine), on a directory of -// raw HTTP responses; it is also how a new provider is calibrated. -// -// TYPESAFE_API_KEY=... judgeeval -provider jev -samples testdata/samples -labels testdata/labels.json -// TYPESAFE_API_KEY=... judgeeval -provider jev -samples cc/samples -cache cache -out report -// -// Answers are cached on disk, so reruns are free and an interrupted run -// resumes. Build with any Go >= 1.17. +// judgeeval evaluates saved HTTP responses through the same Inspect pipeline as SDK callers. package main import ( @@ -16,57 +8,22 @@ import ( "flag" "fmt" "math" - "math/bits" "net/http" "os" "path/filepath" - "reflect" - "regexp" "sort" "strings" "sync" "sync/atomic" "time" - "unicode" "github.com/chainreactors/fingers" - "github.com/chainreactors/fingers/common" "github.com/chainreactors/fingers/judge" - "github.com/chainreactors/fingers/judge/jev" + "github.com/chainreactors/fingers/judge/maintain" "github.com/chainreactors/logs" + "github.com/chainreactors/utils/jev" ) -// row is the outcome for one page, written to rows.jsonl for review. -type row struct { - ID string `json:"id"` - Rules []string `json:"rules"` // name@engine of every rule hit - Accepted []string `json:"accepted"` - Rejected []string `json:"rejected,omitempty"` - Recall []string `json:"recall,omitempty"` - Layers map[string]string `json:"layers,omitempty"` - Primary string `json:"primary,omitempty"` - Version string `json:"version,omitempty"` - JudgeVersion bool `json:"judge_version,omitempty"` // Version written by the judge, not by a rule - Kind judge.Kind `json:"kind"` - Generic bool `json:"generic"` - NewFinger bool `json:"new_fingerprint_candidate,omitempty"` - Err string `json:"error,omitempty"` - - // rules vs rules+judge on the same page - RuleProducts int `json:"rule_products"` // rule hits after folding spellings - TextOnly []string `json:"text_only,omitempty"` // rule hits named only in visible text: likely false positives - TextOnlyKept int `json:"text_only_kept"` // of those, accepted by the judge - Generator string `json:"generator,omitempty"` // "wordpress 6.4.2": the reference version - RuleGenVer string `json:"rule_gen_version,omitempty"` - JudgeGenVer string `json:"judge_gen_version,omitempty"` - RuleMs float64 `json:"rule_ms"` - JudgeMs float64 `json:"judge_ms"` - - frames common.Frameworks // annotated - ruleEngines map[string][]string // rule name -> engines - err error -} - func main() { manifest := flag.String("manifest", "", "replay evidence manifest with labels and generation plans") history := flag.String("history", "", "optional baseline.jsonl from a previous replay") @@ -79,6 +36,7 @@ func main() { cache := flag.String("cache", "judgecache", "directory caching answers") out := flag.String("out", "judgereport", "output directory") rps := flag.Float64("rps", 15, "max provider requests per second") + flag.Float64Var(&minConfidence, "min-confidence", 0, "judge MinConfidence (0 = Jev default)") workers := flag.Int("workers", 16, "concurrent pages") limit := flag.Int("limit", 0, "only the first N samples (0 = all)") endpoint := flag.String("endpoint", "", "provider API endpoint (default: the provider's)") @@ -103,10 +61,10 @@ func main() { // newProvider builds a provider by name; add new providers here. tokens // reports input tokens sent, where the provider counts them. -func newProvider(name, endpoint string, transport http.RoundTripper) (p judge.Provider, tokens func() int64, err error) { +func newProvider(name, endpoint string, transport http.RoundTripper) (p jev.Provider, tokens func() int64, err error) { switch name { case "jev": - jp, err := jev.New("") + jp, err := jev.NewClient("") if err != nil { return nil, nil, err } @@ -120,6 +78,9 @@ func newProvider(name, endpoint string, transport http.RoundTripper) (p judge.Pr } func run(providerName, samples, labelsPath, cacheDir, out, endpoint string, rps float64, workers, limit int) error { + if rps <= 0 || rps > 100 || math.IsNaN(rps) || workers < 1 { + return fmt.Errorf("rps must be in (0,100] and workers positive") + } files, err := filepath.Glob(filepath.Join(samples, "*.http")) if err != nil { return err @@ -128,613 +89,175 @@ func run(providerName, samples, labelsPath, cacheDir, out, endpoint string, rps if limit > 0 && len(files) > limit { files = files[:limit] } - for _, d := range []string{cacheDir, out} { - if err := os.MkdirAll(d, 0o755); err != nil { + labels, err := loadLabels(labelsPath) + if err != nil { + return err + } + for _, dir := range []string{cacheDir, out} { + if err := os.MkdirAll(dir, 0700); err != nil { return err } } - provider, tokens, err := newProvider(providerName, endpoint, &limited{tick: time.NewTicker(time.Duration(float64(time.Second) / rps)).C}) + ticker := time.NewTicker(time.Duration(float64(time.Second) / rps)) + defer ticker.Stop() + provider, tokens, err := newProvider(providerName, endpoint, &limited{tick: ticker.C}) if err != nil { return err } - j := judge.New(provider) - // Measure similarity rather than use it: every page gets its own answers. - j.SimilarDistance = 0 - sim := newSimCheck(cacheDir) - j.Cache = sim - + cached := &recordingProvider{Provider: provider, dir: cacheDir} + j := newJudge(cached) engine, err := fingers.NewEngine(fingers.FingersEngine, fingers.FingerPrintEngine, fingers.EHoleEngine, fingers.GobyEngine, fingers.WappalyzerEngine) if err != nil { return err } - j.Known = judge.NewRetriever(engine.Names()) - + engine.EnableMatchDetail() + ledger := maintain.NewLedger() ctx, cancel := context.WithCancel(context.Background()) defer cancel() - rows := make([]*row, len(files)) + rows := make([]record, len(files)) jobs := make(chan int) var wg sync.WaitGroup var fatal error var once sync.Once + started := time.Now() for w := 0; w < workers; w++ { wg.Add(1) go func() { defer wg.Done() for i := range jobs { - rows[i] = evaluatePage(ctx, engine, j, files[i]) + r, err := evaluatePage(ctx, engine, j, files[i]) + rows[i] = r + ledger.Add(r.ID, r.Judged) var apiErr *jev.APIError - if rows[i].err != nil && errors.As(rows[i].err, &apiErr) && apiErr.Status == 401 { + if errors.As(err, &apiErr) && apiErr.Status == 401 { once.Do(func() { fatal = errors.New("unauthorized: check " + jev.EnvAPIKey); cancel() }) } } }() } - start := time.Now() +dispatch: for i := range files { - if ctx.Err() != nil { - break + select { + case jobs <- i: + case <-ctx.Done(): + break dispatch } - jobs <- i } close(jobs) wg.Wait() if fatal != nil { return fatal } - - f, err := os.Create(filepath.Join(out, "rows.jsonl")) - if err != nil { + if err := writeRows(filepath.Join(out, "rows.jsonl"), rows); err != nil { return err } - enc := json.NewEncoder(f) - enc.SetEscapeHTML(false) + m := &replayManifest{} for _, r := range rows { - if r != nil { - enc.Encode(r) - } + m.Samples = append(m.Samples, replaySample{ID: r.ID, Group: r.ID, ContentSHA256: r.SHA256, Labels: labels[r.ID]}) + } + summary := summarizeReplay(m, rows, nil, nil) + summary.Model, summary.BaselineSource = provider.ID(), "current_rules_on_saved_responses" + summary.Requests, summary.CacheHits, summary.Claims = cached.stats() + summary.InputTokens, summary.Seconds = tokens(), time.Since(started).Seconds() + summary.JunkRules = ledger.Report(3, 0.5) + if err := writeJSON(filepath.Join(out, "ledger.json"), ledger.Report(1, 0)); err != nil { + return err } - f.Close() - - var b strings.Builder - report(&b, rows, provider.ID(), j, tokens(), time.Since(start)) - sim.report(&b, judge.DefaultSimilarDistance) - if labelsPath != "" { - if err := evaluate(&b, rows, labelsPath); err != nil { - return err - } + if err := writeJSON(filepath.Join(out, "metrics.json"), summary); err != nil { + return err + } + if err := writeReplayReport(filepath.Join(out, "report.md"), summary, m); err != nil { + return err } - fmt.Print(b.String()) - return os.WriteFile(filepath.Join(out, "report.md"), []byte(b.String()), 0o644) + fmt.Printf("saved %s\n", filepath.Join(out, "report.md")) + if summary.Errors > 0 { + return fmt.Errorf("%d failed rows; see rows.jsonl", summary.Errors) + } + return nil } -func evaluatePage(ctx context.Context, engine *fingers.Engine, j *judge.Judge, path string) (r *row) { - r = &row{ID: filepath.Base(path), ruleEngines: map[string][]string{}} +func evaluatePage(ctx context.Context, engine *fingers.Engine, j *judge.Judge, path string) (r record, err error) { + r.ID = filepath.Base(path) defer func() { - if p := recover(); p != nil { // one malformed page must not stop a long run - r.Err = fmt.Sprintf("panic: %v", p) + if p := recover(); p != nil { + err = fmt.Errorf("panic: %v", p) + } + if err != nil { + r.Err = err.Error() } }() raw, err := os.ReadFile(path) - var page *judge.Page - if err == nil { - page, err = judge.NewPage(raw) // for code-only evidence; Refine builds its own - } - if err == nil { - t := time.Now() - r.frames, err = engine.DetectContent(raw) - r.RuleMs = ms(time.Since(t)) - } - if err != nil { - r.Err, r.err = err.Error(), err - return r - } - product, want := generatorVersion(page.Generator) - if want != "" { - r.Generator = product + " " + want - } - keys := map[string]bool{} - for _, f := range r.frames { - keys[judge.NormalizeName(f.Name)] = true - if textOnly(page, f.Name) { - r.TextOnly = append(r.TextOnly, f.Name) - } - } - r.RuleProducts = len(keys) - r.RuleGenVer = versionOf(r.frames, product) - versions := map[string]string{} - for _, f := range r.frames { - for from := range f.Froms { - r.Rules = append(r.Rules, f.Name+"@"+from.String()) - r.ruleEngines[f.Name] = append(r.ruleEngines[f.Name], from.String()) - } - versions[f.Name] = f.Version - } - sort.Strings(r.Rules) - t := time.Now() - accepted, err := j.Refine(ctx, raw, r.frames) - r.JudgeMs = ms(time.Since(t)) - if err != nil { - r.Err, r.err = err.Error(), err - return r - } - inspected, err := j.Inspect(ctx, raw, r.frames) if err != nil { - r.Err, r.err = err.Error(), err - return r - } - kind, generic, err := j.Classify(ctx, raw) - if err != nil { - r.Err, r.err = err.Error(), err - return r - } - for name, f := range accepted { - if target := inspected[name]; target != nil { - target.Attributes = f.Attributes - } - } - r.frames = inspected - r.JudgeGenVer = versionOf(accepted, product) - for _, name := range r.TextOnly { - if f := r.frames[name]; f != nil && (f.Judge == nil || !f.Judge.Rejected) { - r.TextOnlyKept++ - } - } - r.Kind, r.Generic, r.Layers = kind, generic, map[string]string{} - for _, f := range r.frames { - v := f.Judge - if v == nil { - r.Accepted = append(r.Accepted, f.Name) - continue - } - if v.Layer != "" { - r.Layers[f.Name] = v.Layer - } - switch { - case v.Rejected: - r.Rejected = append(r.Rejected, f.Name) - case v.Duplicate: - default: - r.Accepted = append(r.Accepted, f.Name) - } - if v.Recalled { - r.Recall = append(r.Recall, f.Name) - } - } - if p := r.frames.Primary(); p != nil { - r.Primary, r.Version = p.Name, p.Version - r.JudgeVersion = p.Version != "" && p.Version != versions[p.Name] + return r, err } - r.NewFinger, err = j.IsUnknownProduct(ctx, raw, accepted) + r.SHA256 = digest(raw) + started := time.Now() + r.Baseline, err = engine.DetectContent(raw) + r.RuleMs = ms(time.Since(started)) if err != nil { - r.Err, r.err = err.Error(), err - return r - } - sort.Strings(r.Accepted) - sort.Strings(r.Rejected) - return r -} - -// textOnly reports whether name occurs in the page's visible text and in no -// structural evidence: the code-only proxy for a false positive. -func textOnly(p *judge.Page, name string) bool { - n := strings.ToLower(name) - if len(n) < 3 || !strings.Contains(strings.ToLower(p.Text), n) { - return false - } - var structural []string - for k, v := range p.Headers { - structural = append(structural, k+":"+v) - } - structural = append(structural, p.Title, p.Generator, p.Description) - for _, list := range [][]string{p.Cookies, p.Scripts, p.Styles, p.InlineHints, p.Comments, p.Forms} { - structural = append(structural, list...) + return r, err } - return !strings.Contains(strings.ToLower(strings.Join(structural, " ")), n) + err = judgeRecord(ctx, j, raw, &r) + return r, err } -func report(b *strings.Builder, rows []*row, providerID string, j *judge.Judge, tokens int64, took time.Duration) { - w := func(format string, a ...interface{}) { fmt.Fprintf(b, format+"\n", a...) } - var pages, errs, rules, accepted, recall, primary, generic, newFinger int - var ruleProducts, textOnly, textOnlyKept, rejected, genPages int - var genRule, genJudge [3]int // correct, wrong, missing - var perRule, perJudge []int - var ruleMs, judgeMs []float64 - kinds := map[string]int{} - type stat struct{ hits, rejected int } - byEngine, byRule := map[string]*stat{}, map[string]*stat{} - add := func(m map[string]*stat, k string, rejected bool) { - if m[k] == nil { - m[k] = &stat{} - } - m[k].hits++ - if rejected { - m[k].rejected++ - } - } - var newFingers []string - for _, r := range rows { - if r == nil || r.Err != "" { - errs++ - continue - } - pages++ - rules += len(r.frames) - len(r.Recall) - accepted += len(r.Accepted) - recall += len(r.Recall) - ruleProducts += r.RuleProducts - textOnly += len(r.TextOnly) - textOnlyKept += r.TextOnlyKept - rejected += len(r.Rejected) - perRule = append(perRule, len(r.frames)-len(r.Recall)) - perJudge = append(perJudge, len(r.Accepted)) - ruleMs = append(ruleMs, r.RuleMs) - judgeMs = append(judgeMs, r.JudgeMs) - kinds[string(r.Kind)]++ - if r.Primary != "" { - primary++ - } - if r.Generator != "" { - genPages++ - want := r.Generator[strings.LastIndex(r.Generator, " ")+1:] - grade(&genRule, r.RuleGenVer, want) - grade(&genJudge, r.JudgeGenVer, want) - } - if r.Generic { - generic++ - } - if r.NewFinger { - newFinger++ - newFingers = append(newFingers, r.ID) - } - for _, f := range r.frames { - rejected := f.Judge != nil && f.Judge.Rejected - for _, e := range r.ruleEngines[f.Name] { - add(byEngine, e, rejected) - add(byRule, e+"/"+f.Name, rejected) - } - } - } - w("# 纯规则 vs 规则 + judge（%s）\n", providerID) - w("pages %d, errors %d, took %s\n", pages, errs, took.Round(time.Second)) - w("| 指标 | 纯规则 | 规则 + judge |\n|---|---|---|") - w("| 每页结果条数 mean / p90 | %.2f / %d | %.2f / %d |", mean(perRule), p90(perRule), mean(perJudge), p90(perJudge)) - w("| 每页不同产品数（按名字归并） | %.2f | %.2f |", div(ruleProducts, pages), div(accepted, pages)) - w("| 同名重复条目 | %d | 0（标为 Duplicate） |", rules-ruleProducts) - w("| 仅正文出现的命中（疑似误报） | %d | %d |", textOnly, textOnlyKept) - w("| 被否决的规则命中 | — | %d / %d（%.1f%%） |", rejected, rules, 100*div(rejected, rules)) - w("| 召回（规则漏掉、judge 确认） | 0 | %d |", recall) - w("| 主应用 | — | %d 页 |", primary) - w("| 版本号 vs generator：正确 / 错误 / 缺失（%d 页） | %d / %d / %d | %d / %d / %d |", genPages, - genRule[0], genRule[1], genRule[2], genJudge[0], genJudge[1], genJudge[2]) - w("| 页面类型 / 是否通用页面 | — | ✓ / 通用 %d 页 |", generic) - w("| 新指纹候选 | — | %d |", newFinger) - w("| 每页耗时 p50 / p95 | %.1fms / %.1fms | +%.0fms / +%.0fms |", pct(ruleMs, 50), pct(ruleMs, 95), pct(judgeMs, 50), pct(judgeMs, 95)) - w("| 成本 | 0 | %d 次请求（缓存直接答完 %d 轮），%d input tokens（jev 按 $0.042/Mtok ≈ $%.3f，折合每万页 $%.2f） |", - j.Requests, j.CacheHits, tokens, float64(tokens)*0.042/1e6, float64(tokens)*0.042/1e6/math.Max(float64(pages), 1)*1e4) - w("\n## page kind\n") - for _, k := range sortedKeys(kinds, func(k string) int { return kinds[k] }) { - w("- %s: %d", k, kinds[k]) - } - w("\n## rejections per engine\n\n| engine | hits | rejected | rate |\n|---|---|---|---|") - for _, e := range sortedKeys(byEngine, func(k string) int { return byEngine[k].hits }) { - s := byEngine[e] - w("| %s | %d | %d | %.1f%% |", e, s.hits, s.rejected, 100*div(s.rejected, s.hits)) - } - w("\n## worst rules (>= 5 hits): candidates to fix\n\n| rule | hits | rejected |\n|---|---|---|") - worst := sortedKeys(byRule, func(k string) int { return byRule[k].rejected*1000 + byRule[k].hits }) - for i, k := range worst { - if s := byRule[k]; i < 25 && s.hits >= 5 && s.rejected*2 > s.hits { - w("| %s | %d | %d |", k, s.hits, s.rejected) - } - } - if len(newFingers) > 0 { - w("\n## new fingerprint candidates\n") - for _, id := range newFingers { - w("- %s", id) - } +// loadLabels converts the old keep/drop/version input once at the boundary. +// All scoring thereafter uses productLabel and exact names or explicit aliases. +func loadLabels(path string) (map[string][]productLabel, error) { + out := map[string][]productLabel{} + if path == "" { + return out, nil } -} - -// label is the ground truth for one sample: Keep are real products, Drop are -// false positives the rules report. -type label struct { - PageKind string `json:"page_kind"` - Generic bool `json:"generic"` - Keep []string `json:"keep"` - Drop []string `json:"drop"` - Version *struct { - Product string `json:"product"` - Value string `json:"value"` // "" = the page does not show it - } `json:"version"` -} - -func evaluate(b *strings.Builder, rows []*row, path string) error { data, err := os.ReadFile(path) if err != nil { - return err + return nil, err } - var labels map[string]label - if err := json.Unmarshal(data, &labels); err != nil { - return err + var values map[string]json.RawMessage + if err := json.Unmarshal(data, &values); err != nil { + return nil, err } - w := func(format string, a ...interface{}) { fmt.Fprintf(b, format+"\n", a...) } - var kept, keep, dropped, drop, kind, generic, pages, version, versions int - var misses []string - for _, r := range rows { - l, ok := labels[r.ID] - if !ok || r.Err != "" { + for id, raw := range values { + if strings.HasPrefix(strings.TrimSpace(string(raw)), "[") { + var labels []productLabel + if err := json.Unmarshal(raw, &labels); err != nil { + return nil, err + } + out[id] = labels continue } - pages++ - for _, name := range l.Keep { - keep++ - if find(r.frames, name, false) != nil { - kept++ - } else { - misses = append(misses, r.ID+": lost "+name) - } + var old struct { + Keep []string + Drop []string + Version *struct{ Product, Value string } } - for _, name := range l.Drop { - drop++ - if find(r.frames, name, true) == nil { - dropped++ - } else { - misses = append(misses, r.ID+": kept "+name) - } + if err := json.Unmarshal(raw, &old); err != nil { + return nil, err } - if string(r.Kind) == l.PageKind { - kind++ + for _, name := range old.Keep { + out[id] = append(out[id], productLabel{Product: name, Present: true}) } - if r.Generic == l.Generic { - generic++ + for _, name := range old.Drop { + out[id] = append(out[id], productLabel{Product: name}) } - if l.Version != nil { - versions++ - got := "" - if f := find(r.frames, l.Version.Product, false); f != nil { - got = f.Version + if old.Version != nil { + found := false + for i := range out[id] { + if judge.NormalizeName(out[id][i].Product) == judge.NormalizeName(old.Version.Product) { + out[id][i].Version = &old.Version.Value + found = true + break + } } - if got == l.Version.Value { - version++ - } else { - misses = append(misses, fmt.Sprintf("%s: version of %s %q, want %q", r.ID, l.Version.Product, got, l.Version.Value)) + if !found { + out[id] = append(out[id], productLabel{Product: old.Version.Product, Present: true, Version: &old.Version.Value}) } } } - w("\n## against labels (%d pages)\n", pages) - w("- false positives removed %d/%d, real products kept %d/%d", dropped, drop, kept, keep) - w("- page kind %d/%d, generic %d/%d, version %d/%d", kind, pages, generic, pages, version, versions) - if len(misses) > 0 { - w("\n### misses\n") - for _, m := range misses { - w("- %s", m) - } - } - return nil -} - -// find returns the accepted framework matching a label name, preferring one -// with a version. Label names are loose ("tomcat" for "apache-tomcat"), so a -// match is containment either way; strict (for false positive labels such as -// "tomcat_jk_connector") only lets the framework name contain the label. -func find(frames common.Frameworks, name string, strict bool) *common.Framework { - want := judge.NormalizeName(name) - var found *common.Framework - for _, f := range frames.Accepted() { - got := judge.NormalizeName(f.Name) - if got == "" || !(strings.Contains(got, want) || !strict && strings.Contains(want, got)) { - continue - } - if found == nil || found.Version == "" { - found = f - } - } - return found -} - -var reVersion = regexp.MustCompile(`\d+(?:\.\d+)+`) - -// generatorVersion reads the reference answer from a generator meta such as -// "WordPress 6.4.2": the product's first word and the version. -func generatorVersion(gen string) (string, string) { - v := reVersion.FindString(gen) - words := strings.FieldsFunc(strings.ToLower(gen), func(r rune) bool { return !unicode.IsLetter(r) }) - if v == "" || len(words) == 0 { - return "", "" - } - return words[0], v -} - -// versionOf is the version reported for product, preferring an exact name. -func versionOf(frames common.Frameworks, product string) string { - if product == "" { - return "" - } - var got string - for _, f := range frames { - key := judge.NormalizeName(f.Name) - if f.Version == "" || !strings.HasPrefix(key, product) { - continue - } - if key == product || got == "" { - got = f.Version - } - } - return got -} - -func grade(g *[3]int, got, want string) { - switch { - case got == want: - g[0]++ - case got != "": - g[1]++ - default: - g[2]++ - } + return out, nil } func ms(d time.Duration) float64 { return float64(d.Microseconds()) / 1000 } -func mean(xs []int) float64 { - sum := 0 - for _, x := range xs { - sum += x - } - return div(sum, len(xs)) -} - -func p90(xs []int) int { - if len(xs) == 0 { - return 0 - } - s := append([]int(nil), xs...) - sort.Ints(s) - return s[len(s)*9/10] -} - -func pct(xs []float64, p int) float64 { - if len(xs) == 0 { - return 0 - } - s := append([]float64(nil), xs...) - sort.Float64s(s) - return s[len(s)*p/100] -} - -func div(a, b int) float64 { - if b == 0 { - return 0 - } - return float64(a) / float64(b) -} - -// sortedKeys returns the keys of m, a map with string keys, by descending weight. -func sortedKeys(m interface{}, weight func(string) int) []string { - var keys []string - for _, k := range reflect.ValueOf(m).MapKeys() { - keys = append(keys, k.String()) - } - sort.Slice(keys, func(a, b int) bool { - if wa, wb := weight(keys[a]), weight(keys[b]); wa != wb { - return wa > wb - } - return keys[a] < keys[b] - }) - return keys -} - -// fileCache is an exact judge.Cache on disk, one file per answer: the -// signature is part of the file name, so distance is ignored. -type fileCache string - -func (d fileCache) path(key string, sig uint64) string { - return filepath.Join(string(d), fmt.Sprintf("%s_%016x.json", key, sig)) -} - -func (d fileCache) Get(key string, sig uint64, _ int) ([]byte, bool) { - data, err := os.ReadFile(d.path(key, sig)) - return data, err == nil -} - -func (d fileCache) Put(key string, sig uint64, value []byte) { - path := d.path(key, sig) - if os.WriteFile(path+".tmp", value, 0o644) == nil { - os.Rename(path+".tmp", path) - } -} - -// simCheck measures the similarity cache without trusting it: every question -// is still answered for its exact page (by the file cache), and whenever an -// earlier page with the same key had a signature within maxProbe bits, its -// answer is compared with the real one, bucketed by distance. That shows -// which SimilarDistance is safe. -type simCheck struct { - fileCache - mu sync.Mutex - seen map[string][]simEntry // key -> earlier answers - pending map[string]simNear // key+sig -> nearest earlier answer, until the real one arrives - answers int - byDist [maxProbe + 1]struct{ reusable, agree int } -} - -const maxProbe = 10 - -type simEntry struct { - sig uint64 - value []byte -} - -type simNear struct { - dist int - value []byte -} - -func newSimCheck(dir string) *simCheck { - return &simCheck{fileCache: fileCache(dir), seen: map[string][]simEntry{}, pending: map[string]simNear{}} -} - -func (c *simCheck) nearest(key string, sig uint64) (simNear, bool) { - best, ok := simNear{dist: maxProbe + 1}, false - for _, e := range c.seen[key] { - if d := bits.OnesCount64(e.sig ^ sig); d < best.dist { - best, ok = simNear{d, e.value}, true - } - } - return best, ok -} - -func (c *simCheck) Get(key string, sig uint64, distance int) ([]byte, bool) { - value, ok := c.fileCache.Get(key, sig, distance) - c.mu.Lock() - defer c.mu.Unlock() - near, found := c.nearest(key, sig) - if ok { - c.observe(key, sig, near, found, value) - } else if found { - c.pending[fmt.Sprintf("%s_%x", key, sig)] = near - } - return value, ok -} - -func (c *simCheck) Put(key string, sig uint64, value []byte) { - c.fileCache.Put(key, sig, value) - c.mu.Lock() - defer c.mu.Unlock() - id := fmt.Sprintf("%s_%x", key, sig) - near, found := c.pending[id] - delete(c.pending, id) - c.observe(key, sig, near, found, value) -} - -func (c *simCheck) observe(key string, sig uint64, near simNear, found bool, value []byte) { - c.answers++ - c.seen[key] = append(c.seen[key], simEntry{sig, value}) - if !found || near.dist > maxProbe { - return - } - var x, y judge.Answer - json.Unmarshal(near.value, &x) - json.Unmarshal(value, &y) - c.byDist[near.dist].reusable++ - if x.Choice == y.Choice && (x.Yes >= 0.5) == (y.Yes >= 0.5) { - c.byDist[near.dist].agree++ - } -} - -func (c *simCheck) report(b *strings.Builder, current int) { - fmt.Fprintf(b, "\n## 相似度缓存（模拟，当前默认 SimilarDistance=%d）\n\n", current) - fmt.Fprintf(b, "共 %d 个答案。下表：若 SimilarDistance 取 d，能由相似页面（同标题）复用的答案数，以及其中与真实答案一致的比例。\n\n", c.answers) - fmt.Fprintf(b, "| d | 可复用 | 一致 | 一致率 |\n|---|---|---|---|\n") - reusable, agree := 0, 0 - for d := 0; d <= maxProbe; d++ { - reusable += c.byDist[d].reusable - agree += c.byDist[d].agree - fmt.Fprintf(b, "| %d | %d | %d | %.1f%% |\n", d, reusable, agree, 100*div(agree, reusable)) - } -} - -// limited rate-limits provider requests; cache hits never reach it. type limited struct{ tick <-chan time.Time } func (l *limited) RoundTrip(req *http.Request) (*http.Response, error) { @@ -745,3 +268,13 @@ func (l *limited) RoundTrip(req *http.Request) (*http.Response, error) { } return http.DefaultTransport.RoundTrip(req) } + +var minConfidence float64 + +func newJudge(p jev.Provider) *judge.Judge { + j := judge.New(p) + if minConfidence != 0 { + j.MinConfidence = minConfidence + } + return j +} diff --git a/cmd/judgeeval/maintenance.go b/cmd/judgeeval/maintenance.go index 277b394..5b28f9a 100644 --- a/cmd/judgeeval/maintenance.go +++ b/cmd/judgeeval/maintenance.go @@ -1,6 +1,7 @@ package main import ( + "encoding/json" "fmt" "net/url" "os" @@ -12,6 +13,7 @@ import ( "github.com/chainreactors/fingers/common" fingerlib "github.com/chainreactors/fingers/fingers" "github.com/chainreactors/fingers/judge" + "github.com/chainreactors/fingers/judge/maintain" "github.com/chainreactors/fingers/resources" "gopkg.in/yaml.v3" ) @@ -197,18 +199,17 @@ func catalogMatches(catalog []catalogSource, names []string) []string { } type maintenanceProduct struct { - Plan string `json:"plan"` - Product string `json:"product"` - Status string `json:"status"` - Existing []string `json:"existing_catalog_matches"` - PositiveSamples int `json:"positive_samples"` - SuggestedPositives int `json:"suggested_positives"` - UnknownPositives int `json:"unknown_positives"` - PositiveHosts int `json:"holdout_positive_hosts"` - NegativeHosts int `json:"holdout_negative_hosts"` - Before stageScore `json:"before"` - After stageScore `json:"after"` - TestIDs []string `json:"test_ids"` + Plan string `json:"plan"` + Product string `json:"product"` + Status string `json:"status"` + Existing []string `json:"existing_catalog_matches"` + PositiveSamples int `json:"positive_samples"` + DiscoveredPositives int `json:"discovered_positives"` + PositiveHosts int `json:"holdout_positive_hosts"` + NegativeHosts int `json:"holdout_negative_hosts"` + Before stageScore `json:"before"` + After stageScore `json:"after"` + TestIDs []string `json:"test_ids"` } type maintenanceReport struct { @@ -222,12 +223,12 @@ type maintenanceReport struct { IntegrationErrors []string `json:"integration_errors"` } -func maintenanceDecision(existing []string, g generationResult, positiveHosts, negativeHosts int) string { +func maintenanceDecision(existing []string, generationStatus string, positiveHosts, negativeHosts int) string { switch { case len(existing) > 0: return "already_catalogued" - case g.Status != "passed_holdout": - return g.Status + case generationStatus != "passed_holdout": + return generationStatus case positiveHosts < 2 || negativeHosts < 5: return "insufficient_independent_hosts" default: @@ -235,7 +236,7 @@ func maintenanceDecision(existing []string, g generationResult, positiveHosts, n } } -func maintainReplay(m *replayManifest, rows []replayRow, generated []generationResult, out string, engine *fingers.Engine) error { +func maintainReplay(m *replayManifest, clusters []*maintain.Cluster, generated []generationResult, out string, engine *fingers.Engine) error { catalog, err := embeddedCatalog() if err != nil { return err @@ -249,10 +250,7 @@ func maintainReplay(m *replayManifest, rows []replayRow, generated []generationR return err } report := maintenanceReport{CatalogFile: "catalog-before.json", MinPositiveHosts: 2, MinNegativeHosts: 5} - byID := map[string]replayRow{} - for _, row := range rows { - byID[row.ID] = row - } + missing := missingCandidates(clusters) plans := map[string]generationPlan{} for _, p := range m.Generation { plans[p.Name] = p @@ -262,7 +260,7 @@ func maintainReplay(m *replayManifest, rows []replayRow, generated []generationR for _, g := range generated { p := plans[g.Name] r := maintenanceProduct{Plan: g.Name, Product: p.Product} - names := []string{p.Product, g.Product} + names := []string{p.Product} for _, sample := range m.Samples { l, ok := labelFor(sample, p.Product) if !ok { @@ -271,12 +269,8 @@ func maintainReplay(m *replayManifest, rows []replayRow, generated []generationR names = append(names, l.Aliases...) if l.Present { r.PositiveSamples++ - row := byID[sample.ID] - if containsProductName(row.Suggestions, l) { - r.SuggestedPositives++ - } - if row.Unknown { - r.UnknownPositives++ + if containsProductName(missing[sample.ID], l) { + r.DiscoveredPositives++ } } } @@ -296,7 +290,8 @@ func maintainReplay(m *replayManifest, rows []replayRow, generated []generationR r.TestIDs = append(r.TestIDs, sample.ID) } r.PositiveHosts, r.NegativeHosts = independentHostCount(pos), independentHostCount(neg) - r.Status = maintenanceDecision(r.Existing, g, r.PositiveHosts, r.NegativeHosts) + _, _, status := g.assess(m, p) + r.Status = maintenanceDecision(r.Existing, status, r.PositiveHosts, r.NegativeHosts) if r.Status == "eligible" && p.Probe != "" { r.Status = "active_integration_not_evaluated" } @@ -393,10 +388,7 @@ func maintainReplay(m *replayManifest, rows []replayRow, generated []generationR frames common.Frameworks }{{&r.Before, before[sample.ID]}, {&r.After, after[sample.ID]}} { f := findLabel(stage.frames, l) - stage.score.Detection.add(l.Present, f != nil) - if l.Version != nil { - stage.score.Versions.add(f, *l.Version) - } + stage.score.add(l, f) } } r.Status = "validated_library" @@ -425,3 +417,11 @@ func maintainReplay(m *replayManifest, rows []replayRow, generated []generationR fmt.Fprintf(os.Stderr, "validated %d novel fingerprints in %s\n", len(library), report.Library) return nil } + +// copyFrames freezes the pre-load engine results for library validation. +func copyFrames(frames common.Frameworks) common.Frameworks { + data, _ := json.Marshal(frames) + var out common.Frameworks + _ = json.Unmarshal(data, &out) + return out +} diff --git a/cmd/judgeeval/maintenance_test.go b/cmd/judgeeval/maintenance_test.go index 814f8d9..b8f1b22 100644 --- a/cmd/judgeeval/maintenance_test.go +++ b/cmd/judgeeval/maintenance_test.go @@ -44,7 +44,7 @@ func TestMaintenanceRecognizesPreviouslyLoadedLibrary(t *testing.T) { t.Fatal(err) } existing := catalogMatches([]catalogSource{catalog}, []string{"replay-maintenance-test-app"}) - if got := maintenanceDecision(existing, generationResult{Status: "passed_holdout"}, 2, 5); got != "already_catalogued" { + if got := maintenanceDecision(existing, "passed_holdout", 2, 5); got != "already_catalogued" { t.Fatalf("would re-add existing library fingerprint: %s", got) } } @@ -71,7 +71,7 @@ func TestCatalogAuditCoversEveryEmbeddedSource(t *testing.T) { } func TestMaintenanceRequiresNoveltyAndIndependentHosts(t *testing.T) { - g := generationResult{Status: "passed_holdout"} + g := "passed_holdout" for _, tc := range []struct { existing []string positive, negative int @@ -86,7 +86,7 @@ func TestMaintenanceRequiresNoveltyAndIndependentHosts(t *testing.T) { t.Fatalf("got %s want %s", got, tc.want) } } - g.Status = "failed_holdout" + g = "failed_holdout" if got := maintenanceDecision(nil, g, 3, 20); got != "failed_holdout" { t.Fatalf("failed candidate admitted: %s", got) } diff --git a/cmd/judgeeval/provider.go b/cmd/judgeeval/provider.go new file mode 100644 index 0000000..0961100 --- /dev/null +++ b/cmd/judgeeval/provider.go @@ -0,0 +1,89 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sync" + + "github.com/chainreactors/utils/jev" +) + +// recordingProvider is the shared disk replay and accounting boundary. +// Schema 3 stores provider-neutral Claim/Ruling JSON. Provider.ID includes endpoint/model. +type recordingProvider struct { + jev.Provider + dir string + offline bool + mu sync.Mutex + calls, hits, claims int +} + +func (p *recordingProvider) stats() (calls, hits, claims int) { + p.mu.Lock() + defer p.mu.Unlock() + return p.calls, p.hits, p.claims +} + +func (p *recordingProvider) Judge(ctx context.Context, state interface{}, claims map[string]jev.Claim) (map[string]jev.Ruling, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if err := jev.ValidateClaims(claims); err != nil { + return nil, err + } + data, err := json.Marshal([]interface{}{"claim-report-v3", p.ID(), state, claims}) + if err != nil { + return nil, err + } + path := filepath.Join(p.dir, digest(data)+".json") + p.mu.Lock() + p.claims += len(claims) + p.mu.Unlock() + var rulings map[string]jev.Ruling + if cached, err := os.ReadFile(path); err == nil && json.Unmarshal(cached, &rulings) == nil && jev.ValidateRulings(claims, rulings) == nil { + p.mu.Lock() + p.hits++ + p.mu.Unlock() + return rulings, nil + } + if p.offline { + return nil, fmt.Errorf("offline cache miss: %s", filepath.Base(path)) + } + p.mu.Lock() + p.calls++ + p.mu.Unlock() + rulings, err = p.Provider.Judge(ctx, state, claims) + if err == nil { + err = jev.ValidateRulings(claims, rulings) + } + if err != nil { + return nil, err + } + if err := writeCache(path, rulings); err != nil { + return nil, err + } + return rulings, nil +} + +func writeCache(path string, value interface{}) error { + data, err := json.Marshal(value) + if err != nil { + return err + } + f, err := os.CreateTemp(filepath.Dir(path), ".claim-*.tmp") + if err != nil { + return err + } + defer os.Remove(f.Name()) + if _, err := f.Write(data); err != nil { + f.Close() + return err + } + if err := f.Close(); err != nil { + return err + } + return os.Rename(f.Name(), path) +} diff --git a/cmd/judgeeval/replay.go b/cmd/judgeeval/replay.go index f6ce63d..875d7d1 100644 --- a/cmd/judgeeval/replay.go +++ b/cmd/judgeeval/replay.go @@ -14,14 +14,14 @@ import ( "os" "path/filepath" "runtime/debug" - "sort" "strings" "time" "github.com/chainreactors/fingers" "github.com/chainreactors/fingers/common" "github.com/chainreactors/fingers/judge" - "github.com/chainreactors/fingers/judge/jev" + "github.com/chainreactors/fingers/judge/maintain" + "github.com/chainreactors/utils/jev" ) // Labels are evidence for evaluation only; they are never passed to Judge. @@ -48,7 +48,6 @@ type replaySample struct { type generationPlan struct { Name string `json:"name"` Product string `json:"product"` - AutoName bool `json:"auto_name"` Positive []string `json:"positive"` Negative []string `json:"negative"` Probe string `json:"probe,omitempty"` @@ -64,21 +63,17 @@ type baselineRecord struct { SHA256 string `json:"sha256"` Frames common.Frameworks `json:"frames"` } -type replayRow struct { - ID string `json:"id"` - URL string `json:"url"` - SHA256 string `json:"sha256"` - Baseline common.Frameworks `json:"baseline"` - Refined common.Frameworks `json:"refined,omitempty"` - Rejected []string `json:"rejected,omitempty"` - Duplicates []string `json:"duplicates,omitempty"` - Added []string `json:"added,omitempty"` - Filled map[string]string `json:"filled_versions,omitempty"` - Kind judge.Kind `json:"kind,omitempty"` - Generic bool `json:"generic"` - Unknown bool `json:"unknown"` - Suggestions []string `json:"suggestions,omitempty"` - Err string `json:"error,omitempty"` + +// record stores source results once; accepted products and all metrics are derived. +type record struct { + ID string `json:"id"` + URL string `json:"url"` + SHA256 string `json:"sha256"` + Baseline common.Frameworks `json:"baseline"` + Judged common.Frameworks `json:"judged"` + Err string `json:"error,omitempty"` + RuleMs float64 `json:"rule_ms"` + JudgeMs float64 `json:"judge_ms"` } func digest(b []byte) string { return fmt.Sprintf("%x", sha256.Sum256(b)) } @@ -179,44 +174,6 @@ func loadReplay(path string) (*replayManifest, error) { return &m, nil } -// Cache the full provider input, not a near-page signature, for evaluation. -type replayProvider struct { - judge.Provider - dir string - namespace string - offline bool - calls, hits int -} - -func (p *replayProvider) Calibration() (float64, float64) { - if c, ok := p.Provider.(judge.Calibrated); ok { - return c.Calibration() - } - return .5, .9 -} -func (p *replayProvider) Judge(ctx context.Context, state interface{}, qs map[string]judge.Question) (map[string]judge.Answer, error) { - data, err := json.Marshal([]interface{}{p.ID(), p.namespace, state, qs}) - if err != nil { - return nil, err - } - path := filepath.Join(p.dir, digest(data)+".json") - var answers map[string]judge.Answer - if cached, err := os.ReadFile(path); err == nil && json.Unmarshal(cached, &answers) == nil { - p.hits++ - return answers, nil - } - if p.offline { - return nil, fmt.Errorf("offline cache miss: %s", filepath.Base(path)) - } - p.calls++ - answers, err = p.Provider.Judge(ctx, state, qs) - if err == nil { - if e := writeJSON(path, answers); e != nil { - return nil, e - } - } - return answers, err -} func loadHistory(path string) (map[string]baselineRecord, error) { out := map[string]baselineRecord{} if path == "" { @@ -241,11 +198,11 @@ func loadHistory(path string) (map[string]baselineRecord, error) { } return out, scan.Err() } -func copyFrames(frames common.Frameworks) common.Frameworks { - data, _ := json.Marshal(frames) - var out common.Frameworks - _ = json.Unmarshal(data, &out) - return out +func versionOf(f *common.Framework) string { + if f == nil || f.Attributes == nil { + return "" + } + return f.Version } func findLabel(frames common.Frameworks, l productLabel) *common.Framework { names := append([]string{l.Product}, l.Aliases...) @@ -253,7 +210,7 @@ func findLabel(frames common.Frameworks, l productLabel) *common.Framework { for _, name := range names { for _, f := range frames { if f != nil && judge.NormalizeName(f.Name) == judge.NormalizeName(name) { - if found == nil || (found.Version == "" && f.Version != "") || (found.Version == f.Version && f.Name < found.Name) { + if found == nil || (versionOf(found) == "" && versionOf(f) != "") || (versionOf(found) == versionOf(f) && f.Name < found.Name) { found = f } } @@ -273,7 +230,7 @@ func labelFor(s replaySample, name string) (productLabel, bool) { return productLabel{}, false } -func replay(manifestPath, historyPath, providerName, endpoint, cacheDir, out string, rps float64, offline, maintain bool, libraryPath string) error { +func replay(manifestPath, historyPath, providerName, endpoint, cacheDir, out string, rps float64, offline, audit bool, libraryPath string) error { if rps <= 0 || rps > 100 || math.IsNaN(rps) { return fmt.Errorf("rps must be in (0,100]") } @@ -306,23 +263,27 @@ func replay(manifestPath, historyPath, providerName, endpoint, cacheDir, out str } ticker := time.NewTicker(time.Duration(float64(time.Second) / rps)) defer ticker.Stop() - var provider judge.Provider + var provider jev.Provider tokens := func() int64 { return 0 } if offline { if providerName != "jev" { return fmt.Errorf("unknown provider %q", providerName) } - // Only ID and calibration are used; cache misses never invoke this provider. - provider, err = jev.New("offline-cache-only") + // Cache misses never invoke this provider. + p, e := jev.NewClient("offline-cache-only") + err = e + if p != nil && endpoint != "" { + p.Endpoint = endpoint + } + provider = p } else { provider, tokens, err = newProvider(providerName, endpoint, &limited{tick: ticker.C}) } if err != nil { return err } - cached := &replayProvider{Provider: provider, dir: cacheDir, namespace: endpoint, offline: offline} - j := judge.New(cached) - j.Cache = nil + cached := &recordingProvider{Provider: provider, dir: cacheDir, offline: offline} + j := newJudge(cached) engine, err := fingers.NewEngine(fingers.FingersEngine, fingers.FingerPrintEngine, fingers.EHoleEngine, fingers.GobyEngine, fingers.WappalyzerEngine) if err != nil { return err @@ -332,7 +293,9 @@ func replay(manifestPath, historyPath, providerName, endpoint, cacheDir, out str return err } } - j.Known = judge.NewRetriever(engine.Names()) + if engine.Fingers() != nil { + engine.EnableMatchDetail() // claims quote what each rule matched + } baselineFile, err := os.Create(filepath.Join(out, "baseline.jsonl")) if err != nil { return err @@ -343,49 +306,55 @@ func replay(manifestPath, historyPath, providerName, endpoint, cacheDir, out str return err } defer cleanedFile.Close() - rowFile, err := os.Create(filepath.Join(out, "rows.jsonl")) - if err != nil { - return err - } - defer rowFile.Close() - be, ce, re := json.NewEncoder(baselineFile), json.NewEncoder(cleanedFile), json.NewEncoder(rowFile) - rows := make([]replayRow, 0, len(m.Samples)) + be, ce := json.NewEncoder(baselineFile), json.NewEncoder(cleanedFile) + ledger := maintain.NewLedger() + rows := make([]record, 0, len(m.Samples)) started := time.Now() for _, s := range m.Samples { - r := replayRow{ID: s.ID, URL: s.URL, SHA256: s.SHA256} + r := record{ID: s.ID, URL: s.URL, SHA256: s.SHA256} + ruleStart := time.Now() if historyPath != "" { - r.Baseline = copyFrames(history[s.ID].Frames) + r.Baseline = history[s.ID].Frames } else { r.Baseline, err = engine.DetectContent(s.raw) if err != nil { return err } } + r.RuleMs = ms(time.Since(ruleStart)) if err := be.Encode(baselineRecord{s.ID, s.SHA256, r.Baseline}); err != nil { return err } ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) - err = cleanReplay(ctx, j, s, &r) + err = judgeRecord(ctx, j, s.raw, &r) cancel() if err != nil { r.Err = err.Error() - } else if err := ce.Encode(baselineRecord{s.ID, s.SHA256, r.Refined}); err != nil { - return err - } - if err := re.Encode(r); err != nil { + } else if err := ce.Encode(baselineRecord{s.ID, s.SHA256, r.Judged.Accepted()}); err != nil { return err } + ledger.Add(s.ID, r.Judged) rows = append(rows, r) - fmt.Fprintf(os.Stderr, "replay %d/%d %s: baseline=%d refined=%d error=%t\n", len(rows), len(m.Samples), s.ID, len(r.Baseline), len(r.Refined), r.Err != "") + fmt.Fprintf(os.Stderr, "replay %d/%d %s: baseline=%d accepted=%d error=%t\n", len(rows), len(m.Samples), s.ID, len(r.Baseline), len(r.Judged.Accepted()), r.Err != "") + } + if err := writeRows(filepath.Join(out, "rows.jsonl"), rows); err != nil { + return err + } + clusters, err := discoverReplay(m, j, rows, out) + if err != nil { + return err } generated, err := generateReplay(m, j, out) if err != nil { return err } - summary := summarizeReplay(m, rows, generated) + summary := summarizeReplay(m, rows, generated, clusters) + summary.Requests, summary.CacheHits, summary.Claims = cached.stats() + summary.JunkRules = ledger.Report(3, 0.5) + if err := writeJSON(filepath.Join(out, "ledger.json"), ledger.Report(1, 0)); err != nil { + return err + } summary.Model = provider.ID() - summary.Requests = cached.calls - summary.CacheHits = cached.hits summary.InputTokens = tokens() summary.Seconds = time.Since(started).Seconds() summary.BaselineSource = "current_rules_on_saved_responses" @@ -395,15 +364,15 @@ func replay(manifestPath, historyPath, providerName, endpoint, cacheDir, out str if err := writeJSON(filepath.Join(out, "metrics.json"), summary); err != nil { return err } - if err := writeReplayReport(filepath.Join(out, "report.md"), summary); err != nil { + if err := writeReplayReport(filepath.Join(out, "report.md"), summary, m); err != nil { return err } fmt.Printf("saved %s\n", filepath.Join(out, "report.md")) if summary.Errors > 0 { return fmt.Errorf("%d failed rows; see rows.jsonl", summary.Errors) } - if maintain { - return maintainReplay(m, rows, generated, out, engine) + if audit { + return maintainReplay(m, clusters, generated, out, engine) } return nil } @@ -429,44 +398,75 @@ func saveReplaySnapshot(out string, m *replayManifest) error { } return writeJSON(filepath.Join(out, "manifest.json"), copy) } -func cleanReplay(ctx context.Context, j *judge.Judge, s replaySample, r *replayRow) error { +func judgeRecord(ctx context.Context, j *judge.Judge, raw []byte, r *record) error { + started := time.Now() var err error - r.Refined, err = j.Refine(ctx, s.raw, r.Baseline) + r.Judged, err = j.Inspect(ctx, raw, r.Baseline) + r.JudgeMs = ms(time.Since(started)) if err != nil { - return err + r.Err = err.Error() } - if r.Kind, r.Generic, err = j.Classify(ctx, s.raw); err != nil { - return err + return err +} + +func discoverReplay(m *replayManifest, j *judge.Judge, rows []record, out string) ([]*maintain.Cluster, error) { + byID := map[string]record{} + for _, r := range rows { + byID[r.ID] = r } - inspected, err := j.Inspect(ctx, s.raw, r.Baseline) + var samples []maintain.Sample + for _, s := range m.Samples { + r, ok := byID[s.ID] + if !ok || r.Err != "" { + continue + } + samples = append(samples, maintain.Sample{ID: s.ID, Host: sampleHost(s), Raw: s.raw, Accepted: r.Judged.Accepted()}) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + clusters, err := maintain.Discover(ctx, j, samples, 2) if err != nil { - return err + return nil, err } - for _, f := range inspected { - if f.Judge != nil && f.Judge.Rejected { - r.Rejected = append(r.Rejected, f.Name) - } - if f.Judge != nil && f.Judge.Duplicate { - r.Duplicates = append(r.Duplicates, f.Name) + return clusters, writeJSON(filepath.Join(out, "discovery.json"), clusters) +} + +// Clusters carry sample IDs; rows do not duplicate cluster state. +func missingCandidates(clusters []*maintain.Cluster) map[string][]string { + out := map[string][]string{} + for _, c := range clusters { + if c.Outcome == jev.Refuted { + for _, id := range c.Samples { + out[id] = c.Candidates + } } } - r.Filled = map[string]string{} - for _, f := range r.Refined { - before := findLabel(r.Baseline, productLabel{Product: f.Name}) - if before == nil { - r.Added = append(r.Added, f.Name) - } - if f.Version != "" && (before == nil || before.Version == "") { - r.Filled[f.Name] = f.Version - } + return out +} + +// sampleHost is the host a sample was captured from: its group joins +// several addresses of one operator. +func sampleHost(s replaySample) string { + if s.Group != "" { + return s.Group + } + if u, err := url.Parse(s.URL); err == nil { + return strings.ToLower(u.Hostname()) } - r.Unknown, err = j.IsUnknownProduct(ctx, s.raw, r.Refined) + return s.URL +} + +func writeRows(path string, rows []record) error { + f, err := os.Create(path) if err != nil { return err } - r.Suggestions, err = j.SuggestNames(ctx, s.raw) - sort.Strings(r.Rejected) - sort.Strings(r.Duplicates) - sort.Strings(r.Added) - return err + defer f.Close() + enc := json.NewEncoder(f) + for _, r := range rows { + if err := enc.Encode(r); err != nil { + return err + } + } + return nil } diff --git a/cmd/judgeeval/replay_generation.go b/cmd/judgeeval/replay_generation.go index e231579..2f9ffa8 100644 --- a/cmd/judgeeval/replay_generation.go +++ b/cmd/judgeeval/replay_generation.go @@ -16,26 +16,57 @@ import ( "gopkg.in/yaml.v3" ) -type generationCase struct { - ID string `json:"id"` - Present bool `json:"present"` - Matched bool `json:"matched"` - Version string `json:"version,omitempty"` - ExpectedVersion *string `json:"expected_version,omitempty"` -} +// generationResult records observations only. The manifest owns the product, +// training split and truth labels; scores and status are derived from them. type generationResult struct { - Name string `json:"name"` - Product string `json:"product"` - Status string `json:"status"` - File string `json:"file,omitempty"` - Detection detectionScore `json:"detection"` - Versions versionScore `json:"versions"` - TrainingDetection detectionScore `json:"training_detection"` - TrainingVersions versionScore `json:"training_versions"` - TrainingCases []generationCase `json:"training_cases"` - Cases []generationCase `json:"cases"` - Excluded []string `json:"excluded"` - Error string `json:"error,omitempty"` + Name string `json:"name"` + File string `json:"file,omitempty"` + Training map[string]*common.Framework `json:"training"` + Holdout map[string]*common.Framework `json:"holdout"` + Excluded []string `json:"excluded"` + Error string `json:"error,omitempty"` +} + +func (r generationResult) assess(m *replayManifest, p generationPlan) (training, holdout stageScore, status string) { + test, _, err := generationSplit(m, p) + if err != nil || r.Error != "" { + return training, holdout, "failed" + } + trainingComplete := len(r.Training) == len(p.Positive)+len(p.Negative) + for _, ids := range [][]string{p.Positive, p.Negative} { + for _, id := range ids { + _, evaluated := r.Training[id] + trainingComplete = trainingComplete && evaluated + } + } + holdoutComplete := len(r.Holdout) == len(test) + for _, sample := range test { + _, evaluated := r.Holdout[sample.ID] + holdoutComplete = holdoutComplete && evaluated + } + for _, s := range m.Samples { + label, labelled := labelFor(s, p.Product) + if !labelled { + continue + } + if frame, evaluated := r.Training[s.ID]; evaluated { + training.add(label, frame) + } + if frame, evaluated := r.Holdout[s.ID]; evaluated { + holdout.add(label, frame) + } + } + switch { + case training.failed() || !trainingComplete: + status = "failed_training" + case holdout.failed(): + status = "failed_holdout" + case !holdoutComplete || holdout.Detection.TP == 0 || holdout.Detection.TN == 0: + status = "insufficient_holdout" + default: + status = "passed_holdout" + } + return } func generationSplit(m *replayManifest, p generationPlan) ([]replaySample, []string, error) { @@ -119,16 +150,13 @@ func generateReplay(m *replayManifest, j *judge.Judge, out string) ([]generation return nil, fmt.Errorf("duplicate generation plan %s", p.Name) } plans[p.Name] = true - r := generationResult{Name: p.Name, Product: p.Product, Status: "failed"} + r := generationResult{Name: p.Name, Training: map[string]*common.Framework{}, Holdout: map[string]*common.Framework{}} test, excluded, err := generationSplit(m, p) if err != nil { return nil, err } r.Excluded = excluded - g := gen.New(j) - if !p.AutoName { - g.Name(p.Product) - } + g := gen.New(j).Name(p.Product) for _, id := range p.Positive { s := samples[id] g.Positive(s.raw) @@ -151,12 +179,6 @@ func generateReplay(m *replayManifest, j *judge.Judge, out string) ([]generation results = append(results, r) continue } - r.Product = f.Name - if judge.NormalizeName(f.Name) != judge.NormalizeName(p.Product) { - r.Error = "automatic name differs from labelled product" - results = append(results, r) - continue - } data, err := yaml.Marshal(f) if err != nil { return nil, err @@ -182,35 +204,15 @@ func generateReplay(m *replayManifest, j *judge.Judge, out string) ([]generation results = append(results, r) continue } - // Score inferred training versions only after generation and export. - // Positive deliberately receives no labelled version or other truth. + // Preserve native detections, including nil for a tested non-match. + // Truth remains in the manifest and never enters generation. for _, ids := range [][]string{p.Positive, p.Negative} { for _, id := range ids { - c, frame := evaluateGenerated(&loaded, p, samples[id], samples) - r.TrainingCases = append(r.TrainingCases, c) - r.TrainingDetection.add(c.Present, c.Matched) - if c.ExpectedVersion != nil { - r.TrainingVersions.add(frame, *c.ExpectedVersion) - } - } - } - for _, s := range test { - c, frame := evaluateGenerated(&loaded, p, s, samples) - r.Cases = append(r.Cases, c) - r.Detection.add(c.Present, c.Matched) - if c.ExpectedVersion != nil { - r.Versions.add(frame, *c.ExpectedVersion) + r.Training[id] = evaluateGenerated(&loaded, p, samples[id], samples) } } - switch { - case r.TrainingDetection.FP > 0 || r.TrainingDetection.FN > 0 || r.TrainingVersions.Wrong > 0 || r.TrainingVersions.Missing > 0 || r.TrainingVersions.Unsupported > 0: - r.Status = "failed_training" - case r.Detection.FP > 0 || r.Detection.FN > 0 || r.Versions.Wrong > 0 || r.Versions.Missing > 0 || r.Versions.Unsupported > 0: - r.Status = "failed_holdout" - case r.Detection.TP == 0 || r.Detection.TN == 0: - r.Status = "insufficient_holdout" - default: - r.Status = "passed_holdout" + for _, sample := range test { + r.Holdout[sample.ID] = evaluateGenerated(&loaded, p, sample, samples) } results = append(results, r) } @@ -220,8 +222,7 @@ func generateReplay(m *replayManifest, j *judge.Judge, out string) ([]generation return results, nil } -func evaluateGenerated(f *fingerlib.Finger, p generationPlan, s replaySample, samples map[string]replaySample) (generationCase, *common.Framework) { - l, _ := labelFor(s, p.Product) +func evaluateGenerated(f *fingerlib.Finger, p generationPlan, s replaySample, samples map[string]replaySample) *common.Framework { var frame *common.Framework var matched bool if p.Probe == "" { @@ -232,9 +233,8 @@ func evaluateGenerated(f *fingerlib.Finger, p generationPlan, s replaySample, sa return samples[id].raw, ok }) } - c := generationCase{ID: s.ID, Present: l.Present, Matched: matched, ExpectedVersion: l.Version} - if matched && frame != nil { - c.Version = frame.Version + if !matched { + return nil } - return c, frame + return frame } diff --git a/cmd/judgeeval/replay_generation_test.go b/cmd/judgeeval/replay_generation_test.go new file mode 100644 index 0000000..88316df --- /dev/null +++ b/cmd/judgeeval/replay_generation_test.go @@ -0,0 +1,126 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/chainreactors/fingers/common" + "github.com/chainreactors/fingers/judge" +) + +func TestGenerationRecordsNativeObservations(t *testing.T) { + plan := generationPlan{Name: "jenkins", Product: "Jenkins", Positive: []string{"train-positive"}, Negative: []string{"train-negative"}} + m := &replayManifest{Generation: []generationPlan{plan}} + for _, item := range []struct{ id, version string }{ + {"train-positive", "2.401.3"}, {"train-negative", ""}, {"test-positive", "2.402.1"}, {"test-negative", ""}, + } { + raw := "HTTP/1.1 200 OK\r\n" + if item.version != "" { + raw += "X-Jenkins: " + item.version + "\r\n" + } + raw += "\r\n" + item.id + "" + label := productLabel{Product: "Jenkins", Present: item.version != ""} + if label.Present { + label.Version = stringPtr(item.version) + } + m.Samples = append(m.Samples, replaySample{ID: item.id, Group: item.id, ContentSHA256: digest([]byte(raw)), raw: []byte(raw), Labels: []productLabel{label}}) + } + out := t.TempDir() + // Explicit response headers establish versions without any model calls. + results, err := generateReplay(m, judge.New(nil), out) + if err != nil || len(results) != 1 { + t.Fatalf("generation=%v error=%v", results, err) + } + data, err := os.ReadFile(filepath.Join(out, "generation.json")) + if err != nil { + t.Fatal(err) + } + var restored []generationResult + if err := json.Unmarshal(data, &restored); err != nil { + t.Fatal(err) + } + r := restored[0] + training, holdout, status := r.assess(m, plan) + if status != "passed_holdout" || training.Detection.TP != 1 || training.Detection.TN != 1 || training.Versions.Correct != 1 || holdout.Versions.Correct != 1 || holdout.Detection.TN != 1 { + t.Fatalf("status=%s training=%+v holdout=%+v error=%s", status, training, holdout, r.Error) + } + if frame, tested := r.Holdout["test-negative"]; !tested || frame != nil { + t.Fatal("evaluated non-match was lost") + } + if frame := r.Holdout["test-positive"]; frame == nil || frame.Name != "Jenkins" || versionOf(frame) != "2.402.1" { + t.Fatalf("native framework lost: %+v", frame) + } + if _, err := os.Stat(filepath.Join(out, r.File)); err != nil { + t.Fatal(err) + } + var fields []map[string]json.RawMessage + if err := json.Unmarshal(data, &fields); err != nil { + t.Fatal(err) + } + for _, name := range []string{"product", "status", "detection", "versions", "training_detection", "training_versions", "training_cases", "cases"} { + if _, exists := fields[0][name]; exists { + t.Fatalf("redundant generation field %s", name) + } + } + // Scores and conclusions follow the current labels and observations. + m.Samples[2].Labels[0].Version = stringPtr("9.9.9") + _, holdout, status = r.assess(m, plan) + if status != "failed_holdout" || holdout.Versions.Wrong != 1 { + t.Fatalf("stale assessment after label change: %s %+v", status, holdout) + } + r.Holdout["test-positive"].Version = "9.9.9" + _, _, status = r.assess(m, plan) + if status != "passed_holdout" { + t.Fatalf("stale assessment after observation change: %s", status) + } + report := filepath.Join(out, "report.md") + if err := writeReplayReport(report, summarizeReplay(m, nil, []generationResult{r}, nil), m); err != nil { + t.Fatal(err) + } + text, err := os.ReadFile(report) + if err != nil || !strings.Contains(string(text), "passed_holdout") { + t.Fatalf("report assessment missing: %s, %v", text, err) + } +} + +func TestGenerationAssessmentRequiresEveryExpectedObservation(t *testing.T) { + plan := generationPlan{Name: "product", Product: "Product", Positive: []string{"train-positive"}, Negative: []string{"train-negative"}} + m := &replayManifest{} + for _, id := range []string{"train-positive", "train-negative", "test-positive", "test-negative", "second-negative"} { + m.Samples = append(m.Samples, replaySample{ID: id, Group: id, ContentSHA256: id, Labels: []productLabel{{Product: plan.Product, Present: strings.HasSuffix(id, "positive")}}}) + } + for _, change := range []string{"none", "missing training", "wrong training key", "missing holdout", "wrong holdout key", "false negative", "error"} { + t.Run(change, func(t *testing.T) { + frame := common.NewFramework(plan.Product, common.FrameFromFingers) + r := generationResult{Training: map[string]*common.Framework{"train-positive": frame, "train-negative": nil}, Holdout: map[string]*common.Framework{"test-positive": frame, "test-negative": nil, "second-negative": nil}} + want := "passed_holdout" + switch change { + case "missing training", "wrong training key": + delete(r.Training, "train-negative") + if change == "wrong training key" { + r.Training["test-negative"] = nil + } + want = "failed_training" + case "missing holdout", "wrong holdout key": + delete(r.Holdout, "second-negative") + if change == "wrong holdout key" { + r.Holdout["train-negative"] = nil + } + want = "insufficient_holdout" + case "false negative": + r.Holdout["test-positive"] = nil + want = "failed_holdout" + case "error": + r.Error = "generation failed" + want = "failed" + } + _, _, status := r.assess(m, plan) + if status != want { + t.Fatalf("status=%s want=%s", status, want) + } + }) + } +} diff --git a/cmd/judgeeval/replay_metrics.go b/cmd/judgeeval/replay_metrics.go index d85e385..d1175ab 100644 --- a/cmd/judgeeval/replay_metrics.go +++ b/cmd/judgeeval/replay_metrics.go @@ -2,11 +2,14 @@ package main import ( "fmt" + "math" "os" "strings" "github.com/chainreactors/fingers/common" "github.com/chainreactors/fingers/judge" + "github.com/chainreactors/fingers/judge/maintain" + "github.com/chainreactors/utils/jev" ) type detectionScore struct { @@ -43,7 +46,7 @@ func (v *versionScore) add(f *common.Framework, want string) { v.ProductMissing++ return } - got := f.Version + got := versionOf(f) switch { case want == "" && got == "": v.CorrectAbstentions++ @@ -63,36 +66,56 @@ type stageScore struct { Versions versionScore `json:"versions"` UnlabelledPredictions int `json:"unlabelled_predictions"` } + +// add is the same label comparison for scan, generation and library evaluation. +func (s *stageScore) add(label productLabel, frame *common.Framework) { + s.Detection.add(label.Present, frame != nil) + if label.Version != nil { + s.Versions.add(frame, *label.Version) + } +} + +func (s stageScore) failed() bool { + return s.Detection.FP > 0 || s.Detection.FN > 0 || s.Versions.Wrong > 0 || s.Versions.Missing > 0 || s.Versions.Unsupported > 0 +} + type replaySummary struct { - Model string `json:"model"` - BaselineSource string `json:"baseline_source"` - Samples int `json:"samples"` - UniqueHosts int `json:"unique_hosts"` - UniqueResponses int `json:"unique_responses"` - Errors int `json:"errors"` - LabelledPairs int `json:"labelled_pairs"` - Baseline stageScore `json:"baseline"` - Refined stageScore `json:"refined"` - FalseHitsRemoved int `json:"false_hits_removed"` - TrueHitsRemoved int `json:"true_hits_removed"` - NaturalMissesRecovered int `json:"natural_misses_recovered"` - MissedProductsSuggested int `json:"missed_products_suggested"` - VersionFills int `json:"version_fills"` - CorrectVersionFills int `json:"correct_version_fills"` - IncorrectVersionFills int `json:"incorrect_version_fills"` - UnlabelledVersionFills int `json:"unlabelled_version_fills"` - Generation []generationResult `json:"generation"` - Disagreements []string `json:"disagreements"` - Requests int `json:"provider_requests"` - CacheHits int `json:"exact_cache_hits"` - InputTokens int64 `json:"input_tokens"` - Seconds float64 `json:"seconds"` + Schema int `json:"schema"` + Model string `json:"model"` + BaselineSource string `json:"baseline_source"` + Samples int `json:"samples"` + UniqueHosts int `json:"unique_hosts"` + UniqueResponses int `json:"unique_responses"` + Errors int `json:"errors"` + LabelledPairs int `json:"labelled_pairs"` + Baseline stageScore `json:"baseline"` + Accepted stageScore `json:"accepted"` + FalseHitsRemoved int `json:"false_hits_removed"` + TrueHitsRemoved int `json:"true_hits_removed"` + NaturalMissesRecovered int `json:"natural_misses_recovered"` + MissedProductsDiscovered int `json:"missed_products_discovered"` + VersionFills int `json:"version_fills"` + CorrectVersionFills int `json:"correct_version_fills"` + IncorrectVersionFills int `json:"incorrect_version_fills"` + UnlabelledVersionFills int `json:"unlabelled_version_fills"` + Generation []generationResult `json:"generation"` + Disagreements []string `json:"disagreements"` + Options map[string]int `json:"options"` + Outcomes map[jev.Outcome]int `json:"presence_outcomes"` + JunkRules []*maintain.RuleStats `json:"junk_rules"` + Clusters []*maintain.Cluster `json:"clusters"` + Claims int `json:"claims"` + Requests int `json:"provider_requests"` + CacheHits int `json:"exact_cache_hits"` + InputTokens int64 `json:"input_tokens"` + Seconds float64 `json:"seconds"` } -func summarizeReplay(m *replayManifest, rows []replayRow, gen []generationResult) replaySummary { - out := replaySummary{Samples: len(m.Samples), Generation: gen} +func summarizeReplay(m *replayManifest, rows []record, gen []generationResult, clusters []*maintain.Cluster) replaySummary { + out := replaySummary{Schema: 3, Clusters: clusters, Samples: len(m.Samples), Generation: gen, Options: map[string]int{}, Outcomes: map[jev.Outcome]int{}} + missing := missingCandidates(clusters) hosts, hashes := map[string]bool{}, map[string]bool{} - byID := map[string]replayRow{} + byID := map[string]record{} for _, r := range rows { byID[r.ID] = r } @@ -104,16 +127,20 @@ func summarizeReplay(m *replayManifest, rows []replayRow, gen []generationResult out.Errors++ continue } + accepted := r.Judged.Accepted() + for _, f := range r.Judged { + if f != nil && f.Judge != nil && !f.Judge.Duplicate && f.Judge.Outcome != "" { + out.Options[f.Judge.Verdict]++ + out.Outcomes[jev.Outcome(f.Judge.Outcome)]++ + } + } for _, stage := range []struct { score *stageScore frames common.Frameworks - }{{&out.Baseline, r.Baseline}, {&out.Refined, r.Refined}} { + }{{&out.Baseline, r.Baseline}, {&out.Accepted, accepted}} { for _, l := range s.Labels { f := findLabel(stage.frames, l) - stage.score.Detection.add(l.Present, f != nil) - if l.Version != nil { - stage.score.Versions.add(f, *l.Version) - } + stage.score.add(l, f) } for _, f := range stage.frames { if f != nil { @@ -125,7 +152,7 @@ func summarizeReplay(m *replayManifest, rows []replayRow, gen []generationResult } for _, l := range s.Labels { out.LabelledPairs++ - before, after := findLabel(r.Baseline, l), findLabel(r.Refined, l) + before, after := findLabel(r.Baseline, l), findLabel(accepted, l) if before != nil && after == nil { if l.Present { out.TrueHitsRemoved++ @@ -136,19 +163,19 @@ func summarizeReplay(m *replayManifest, rows []replayRow, gen []generationResult if l.Present && before == nil && after != nil { out.NaturalMissesRecovered++ } - if l.Present && after == nil && containsProductName(r.Suggestions, l) { - out.MissedProductsSuggested++ + if l.Present && after == nil && containsProductName(missing[r.ID], l) { + out.MissedProductsDiscovered++ } // Count a labelled product once, considering all baseline aliases. // Raw per-name changes remain in rows.jsonl for audit. - if after != nil && after.Version != "" && (before == nil || before.Version == "") { + if after != nil && versionOf(after) != "" && (before == nil || versionOf(before) == "") { out.VersionFills++ switch { case !l.Present: out.IncorrectVersionFills++ case l.Version == nil: out.UnlabelledVersionFills++ - case after.Version == *l.Version: + case versionOf(after) == *l.Version: out.CorrectVersionFills++ default: out.IncorrectVersionFills++ @@ -157,12 +184,20 @@ func summarizeReplay(m *replayManifest, rows []replayRow, gen []generationResult if (after != nil) != l.Present { out.Disagreements = append(out.Disagreements, fmt.Sprintf("%s %s: presence want=%t got=%t", s.ID, l.Product, l.Present, after != nil)) } - if l.Version != nil && after != nil && after.Version != *l.Version { - out.Disagreements = append(out.Disagreements, fmt.Sprintf("%s %s: version want=%q got=%q", s.ID, l.Product, *l.Version, after.Version)) + if l.Version != nil && after != nil && versionOf(after) != *l.Version { + out.Disagreements = append(out.Disagreements, fmt.Sprintf("%s %s: version want=%q got=%q", s.ID, l.Product, *l.Version, versionOf(after))) } } unlabelled := map[string]bool{} - for name := range r.Filled { + for _, f := range accepted { + if f.Attributes == nil || f.Version == "" { + continue + } + name := f.Name + before := findLabel(r.Baseline, productLabel{Product: name}) + if before != nil && before.Attributes != nil && versionOf(before) != "" { + continue + } if _, ok := labelFor(s, name); ok { continue } @@ -188,7 +223,7 @@ func containsProductName(names []string, l productLabel) bool { } return false } -func writeReplayReport(path string, s replaySummary) error { +func writeReplayReport(path string, s replaySummary, m *replayManifest) error { var b strings.Builder fmt.Fprintf(&b, "# 真实响应回放评估\n\n模型：%s；基线：%s。\n\n%d 份快照，%d 个主机组，%d 种去除易变响应头后的响应，%d 个已标注产品判断，失败 %d 份。\n\n", s.Model, s.BaselineSource, s.Samples, s.UniqueHosts, s.UniqueResponses, s.LabelledPairs, s.Errors) b.WriteString("统计范围是保存的响应所支持的产品与版本。页面自报版本不是服务器安装审计；正负标注不代表完整技术栈。未标注的输出不计为误报，也不计为已证明正确。页面级结果含同主机响应，不能视作独立随机样本。\n\n") @@ -196,31 +231,75 @@ func writeReplayReport(path string, s replaySummary) error { for _, r := range []struct { name string s stageScore - }{{"原结果", s.Baseline}, {"Refine", s.Refined}} { + }{{"原结果", s.Baseline}, {"Accepted", s.Accepted}} { d := r.s.Detection fmt.Fprintf(&b, "| %s | %d | %d | %d | %d | %d |\n", r.name, d.TP, d.FP, d.FN, d.TN, r.s.UnlabelledPredictions) } fmt.Fprintf(&b, "\n有标注的误报移除 %d；真实命中误删 %d；自然漏报补回 %d。模型拒绝或新增的未标注项保留在 rows.jsonl 中供复核。\n\n", s.FalseHitsRemoved, s.TrueHitsRemoved, s.NaturalMissesRecovered) - fmt.Fprintf(&b, "仍漏报的产品中，名称建议覆盖 %d 项；建议不计作已识别，也不计作漏报补回。\n\n", s.MissedProductsSuggested) + fmt.Fprintf(&b, "仍漏报的产品中，%d 项所在页面簇的 Coverage 判定为 missing，且产品名在代码给出的候选名中；不计作已识别，也不计作漏报补回。\n\n", s.MissedProductsDiscovered) + b.WriteString("| 判定 | 产品数 |\n|---|---:|\n") + for _, v := range []string{judge.OptionDeclared, judge.OptionAbsent, judge.OptionRunning, judge.OptionMentioned, judge.OptionUnrelated, jev.OptionInsufficient} { + fmt.Fprintf(&b, "| %s | %d |\n", v, s.Options[v]) + } + coverage := map[jev.Outcome]int{} + for _, c := range s.Clusters { + coverage[c.Outcome]++ + } + b.WriteString("\n| Claim | holds | refuted | insufficient |\n|---|---:|---:|---:|\n") + for _, row := range []struct { + name string + n map[jev.Outcome]int + }{{"Presence（误报，按产品）", s.Outcomes}, {"Coverage（漏报，按页面簇）", coverage}} { + fmt.Fprintf(&b, "| %s | %d | %d | %d |\n", row.name, row.n[jev.Holds], row.n[jev.Refuted], row.n[jev.Insufficient]) + } + fmt.Fprintf(&b, "\n共 %d 题（含缓存命中），平均每页 %.1f 题。\n\n", s.Claims, float64(s.Claims)/math.Max(float64(s.Samples), 1)) b.WriteString("| 版本阶段 | 正确 | 错误 | 缺版本 | 正确留空 | 无依据填值 | 产品未识别 |\n|---|---:|---:|---:|---:|---:|---:|\n") for _, r := range []struct { name string s versionScore - }{{"原结果", s.Baseline.Versions}, {"Refine", s.Refined.Versions}} { + }{{"原结果", s.Baseline.Versions}, {"Accepted", s.Accepted.Versions}} { v := r.s fmt.Fprintf(&b, "| %s | %d | %d | %d | %d | %d | %d |\n", r.name, v.Correct, v.Wrong, v.Missing, v.CorrectAbstentions, v.Unsupported, v.ProductMissing) } fmt.Fprintf(&b, "\n按产品合并别名后的新增版本 %d：标注确认正确 %d、错误 %d、未标注 %d。已有非空版本保留，错误由报告揭示；逐名称变化见 rows.jsonl。\n\n", s.VersionFills, s.CorrectVersionFills, s.IncorrectVersionFills, s.UnlabelledVersionFills) - b.WriteString("## 指纹生成\n\n训练只使用 positive/negative 指定快照。测试排除训练主机、相同响应与重复测试响应；生成结果重新从 YAML 加载、编译后测试。此处 passed 仅代表本批独立样本通过，仍输出候选规则供评审。\n\n| 计划 | 名称 | 独立测试 TP/FP/FN/TN | 版本 正确/错误/缺失 | 排除 | 结论 |\n|---|---|---|---|---:|---|\n") + b.WriteString("## 指纹生成\n\n训练只使用 positive/negative 指定快照。测试排除训练主机、相同响应与重复测试响应；生成结果重新从 YAML 加载、编译后测试。此处 passed 仅代表本批独立样本通过，仍输出候选规则供评审。\n\n| 计划 | 名称 | 训练 TP/FP/FN/TN | 训练版本 正确/错误/缺失 | 独立测试 TP/FP/FN/TN | 测试版本 正确/错误/缺失 | 排除 | 结论 |\n|---|---|---|---|---|---|---:|---|\n") + plans := map[string]generationPlan{} + for _, p := range m.Generation { + plans[p.Name] = p + } + for _, g := range s.Generation { + p := plans[g.Name] + training, holdout, status := g.assess(m, p) + d, v := holdout.Detection, holdout.Versions + td, tv := training.Detection, training.Versions + fmt.Fprintf(&b, "| %s | %s | %d/%d/%d/%d | %d/%d/%d | %d/%d/%d/%d | %d/%d/%d | %d | %s |\n", g.Name, p.Product, td.TP, td.FP, td.FN, td.TN, tv.Correct, tv.Wrong, tv.Missing, d.TP, d.FP, d.FN, d.TN, v.Correct, v.Wrong, v.Missing, len(g.Excluded), status) + } for _, g := range s.Generation { - d := g.Detection - v := g.Versions - fmt.Fprintf(&b, "| %s | %s | %d/%d/%d/%d | %d/%d/%d | %d | %s |\n", g.Name, g.Product, d.TP, d.FP, d.FN, d.TN, v.Correct, v.Wrong, v.Missing, len(g.Excluded), g.Status) if g.Error != "" { fmt.Fprintf(&b, "\n%s: %s\n\n", g.Name, g.Error) } } - b.WriteString("\n训练样本的自动版本推断另外记录在 generation.json 的 training_versions / training_cases；训练正确不等于独立验证通过。\n") + b.WriteString("\n## 垃圾规则候选\n\n至少 3 次判定命中、被否决不少于 50% 的规则。\n\n| 引擎 | 指纹 | 规则 / matcher | 命中 | 被否决 |\n|---|---|---|---:|---:|\n") + for _, r := range s.JunkRules { + matcher := "" + if r.Matcher != "" { + matcher = fmt.Sprintf("#%d `%s`", r.Rule, strings.ReplaceAll(r.Matcher, "|", "\\|")) + } + fmt.Fprintf(&b, "| %s | %s | %s | %d | %.0f%% |\n", r.Engine, r.Name, matcher, r.Hits(), 100*r.RefutedFraction()) + } + b.WriteString("\n## 漏报（Coverage）\n\n至少 2 个独立主机的页面簇；Claim：规则已报告的产品解释了这个页面。候选名由代码从页面中抽取，供人命名，不作为选项。\n\n| 主机数 | 判定 | 已报告 | 候选名（前 3） | 样例 |\n|---:|---|---|---|---|\n") + for _, c := range s.Clusters { + ids := c.Samples + if len(ids) > 3 { + ids = ids[:3] + } + cands := c.Candidates + if len(cands) > 3 { + cands = cands[:3] + } + fmt.Fprintf(&b, "| %d | %s | %s | %s | %s |\n", c.Hosts, c.Coverage.Option, strings.Join(c.Reported, ", "), strings.ReplaceAll(strings.Join(cands, ", "), "|", "\\|"), strings.Join(ids, ", ")) + } + b.WriteString("\ngeneration.json 的 training / holdout 保存逐样本原生 Framework 或 null，真值来自 manifest.json；训练正确不等于独立验证通过。\n") b.WriteString("\n## 待核查\n\n") if len(s.Disagreements) == 0 { b.WriteString("已标注范围内没有分歧。\n") @@ -229,6 +308,6 @@ func writeReplayReport(path string, s replaySummary) error { fmt.Fprintf(&b, "- %s\n", d) } } - fmt.Fprintf(&b, "\nProvider 调用 %d，完整输入精确缓存命中 %d，输入 token %d，耗时 %.1f 秒。费用未估算。\n\nbaseline.jsonl 保存输入；cleaned.jsonl 仅含成功完成的结果；rows.jsonl 保存拒绝、补回、补版本、名称建议及错误；generation.json 保存每个独立样本结果。\n", s.Requests, s.CacheHits, s.InputTokens, s.Seconds) + fmt.Fprintf(&b, "\nProvider 调用 %d，完整输入精确缓存命中 %d，输入 token %d，耗时 %.1f 秒。费用未估算。\n\nbaseline.jsonl 保存输入；cleaned.jsonl 仅含成功完成的结果；rows.jsonl 保存基线、全部判定注解和错误，discovery.json 按样本 ID 保存漏报簇；generation.json 保存每个独立样本结果。\n", s.Requests, s.CacheHits, s.InputTokens, s.Seconds) return os.WriteFile(path, []byte(b.String()), 0600) } diff --git a/cmd/judgeeval/replay_test.go b/cmd/judgeeval/replay_test.go index 9d87d9f..b9f921b 100644 --- a/cmd/judgeeval/replay_test.go +++ b/cmd/judgeeval/replay_test.go @@ -2,13 +2,20 @@ package main import ( "context" + "encoding/json" + "net/http" + "net/http/httptest" "os" "path/filepath" "strings" + "sync/atomic" "testing" + "github.com/chainreactors/utils/jev" + "github.com/chainreactors/fingers/common" "github.com/chainreactors/fingers/judge" + "github.com/chainreactors/fingers/judge/maintain" ) func stringPtr(s string) *string { return &s } @@ -23,9 +30,9 @@ func TestReplayCountsErrorsAndUnlabelledSeparately(t *testing.T) { base := testFrames("nginx", "") after := testFrames("apache http server", "2.4.38") after.Add(common.NewFramework("unlabelled-library", common.FrameFromGUESS)) - r := replayRow{ID: "a", Baseline: base, Refined: after, Filled: map[string]string{"apache http server": "2.4.38"}} - summary := summarizeReplay(m, []replayRow{r, {ID: "failed", Err: "network"}}, nil) - if summary.Errors != 1 || summary.LabelledPairs != 2 || summary.Baseline.Detection.FP != 1 || summary.Baseline.Detection.FN != 1 || summary.Refined.Detection.TP != 1 || summary.Refined.Detection.TN != 1 || summary.Refined.UnlabelledPredictions != 1 { + r := record{ID: "a", Baseline: base, Judged: after} + summary := summarizeReplay(m, []record{r, {ID: "failed", Err: "network"}}, nil, nil) + if summary.Errors != 1 || summary.LabelledPairs != 2 || summary.Baseline.Detection.FP != 1 || summary.Baseline.Detection.FN != 1 || summary.Accepted.Detection.TP != 1 || summary.Accepted.Detection.TN != 1 || summary.Accepted.UnlabelledPredictions != 1 { t.Fatalf("metrics: %+v", summary) } if summary.NaturalMissesRecovered != 1 || summary.FalseHitsRemoved != 1 || summary.CorrectVersionFills != 1 { @@ -97,22 +104,25 @@ func TestReplayConfinesResponsePaths(t *testing.T) { } } -type replayTestProvider struct{ calls int } +type replayTestProvider struct { + calls int + id string +} -func (*replayTestProvider) ID() string { return "test/replay" } -func (p *replayTestProvider) Judge(_ context.Context, _ interface{}, qs map[string]judge.Question) (map[string]judge.Answer, error) { +func (p *replayTestProvider) ID() string { return "test/replay/" + p.id } +func (p *replayTestProvider) Judge(_ context.Context, _ interface{}, qs map[string]jev.Claim) (map[string]jev.Ruling, error) { p.calls++ - answers := map[string]judge.Answer{} + answers := map[string]jev.Ruling{} for key := range qs { - answers[key] = judge.Answer{Yes: 1} + answers[key] = jev.Ruling{Option: "yes", Confidence: 1} } return answers, nil } func TestReplayOfflineNeverCallsProvider(t *testing.T) { provider := &replayTestProvider{} - p := &replayProvider{Provider: provider, dir: t.TempDir(), offline: true} - qs := map[string]judge.Question{"product": judge.Binary("Is this nginx?")} + p := &recordingProvider{Provider: provider, dir: t.TempDir(), offline: true} + qs := map[string]jev.Claim{"product": {Statement: "Is this nginx?", Options: map[string]jev.Option{"yes": {Outcome: jev.Holds}, "no": {Outcome: jev.Refuted}, jev.OptionInsufficient: {Outcome: jev.Insufficient}}}} ctx := context.Background() if _, err := p.Judge(ctx, "page", qs); err == nil || !strings.Contains(err.Error(), "offline cache miss") { t.Fatalf("expected offline cache miss: %v", err) @@ -125,13 +135,13 @@ func TestReplayOfflineNeverCallsProvider(t *testing.T) { t.Fatal(err) } p.offline = true - if answers, err := p.Judge(ctx, "page", qs); err != nil || answers["product"].Yes != 1 { + if answers, err := p.Judge(ctx, "page", qs); err != nil || answers["product"].Option != "yes" { t.Fatalf("cached answer: %v %v", answers, err) } if _, err := p.Judge(ctx, "different page", qs); err == nil { t.Fatal("different state reused cached answer") } - p.namespace = "different-endpoint" + provider.id = "different-endpoint" if _, err := p.Judge(ctx, "page", qs); err == nil { t.Fatal("different endpoint reused cached answer") } @@ -145,23 +155,23 @@ func TestReplayVersionFillsDeduplicateAliases(t *testing.T) { m := &replayManifest{Samples: []replaySample{{ID: "a", Labels: []productLabel{l}}}} after := testFrames("PHP", "5.6.40") after.Add(common.NewFrameworkWithVersion("php-runtime", common.FrameFromGUESS, "5.6.40")) - r := replayRow{ID: "a", Baseline: testFrames("PHP", "5.6.40"), Refined: after, Filled: map[string]string{"php-runtime": "5.6.40"}} - if s := summarizeReplay(m, []replayRow{r}, nil); s.VersionFills != 0 { + r := record{ID: "a", Baseline: testFrames("PHP", "5.6.40"), Judged: after} + if s := summarizeReplay(m, []record{r}, nil, nil); s.VersionFills != 0 { t.Fatalf("existing alias version counted as new: %+v", s) } r.Baseline = testFrames("PHP", "") - r.Filled["PHP"] = "5.6.40" - if s := summarizeReplay(m, []replayRow{r}, nil); s.VersionFills != 1 || s.CorrectVersionFills != 1 { + if s := summarizeReplay(m, []record{r}, nil, nil); s.VersionFills != 1 || s.CorrectVersionFills != 1 { t.Fatalf("alias fills counted twice: %+v", s) } } -func TestReplaySuggestionsDoNotRecoverDetections(t *testing.T) { +func TestReplayDiscoveryDoesNotRecoverDetections(t *testing.T) { m := &replayManifest{Samples: []replaySample{{ID: "a", Labels: []productLabel{{Product: "New API", Present: true}}}}} - r := replayRow{ID: "a", Suggestions: []string{"new-api", "New API"}} - s := summarizeReplay(m, []replayRow{r}, nil) - if s.MissedProductsSuggested != 1 || s.NaturalMissesRecovered != 0 || s.Refined.Detection.FN != 1 { - t.Fatalf("suggestions credited as detections: %+v", s) + r := record{ID: "a"} + clusters := []*maintain.Cluster{{Samples: []string{"a"}, Outcome: jev.Refuted, Candidates: []string{"Login", "New API"}}} + s := summarizeReplay(m, []record{r}, nil, clusters) + if s.MissedProductsDiscovered != 1 || s.NaturalMissesRecovered != 0 || s.Accepted.Detection.FN != 1 { + t.Fatalf("discovery credited as detections: %+v", s) } } @@ -196,3 +206,117 @@ func TestGenerationSplitRejectsSameHostnameAcrossGroups(t *testing.T) { t.Fatalf("hostname leakage: test=%v excluded=%v error=%v", test, excluded, err) } } + +// Exercise both public CLI modes against one local Jev server, then replay +// offline from the exact same cache. Reports must explain the same evaluation. +func TestDirectoryAndReplayShareRecordAndCache(t *testing.T) { + var calls int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt64(&calls, 1) + var req struct { + Questions map[string]struct{ Criteria map[string]interface{} } + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Error(err) + w.WriteHeader(400) + return + } + answers := map[string]map[string]interface{}{} + for id, q := range req.Questions { + option := jev.OptionInsufficient + if _, ok := q.Criteria[judge.OptionRunning]; ok { + option = judge.OptionRunning + } + answers[id] = map[string]interface{}{"choice": option, "confidence": 1} + } + _ = json.NewEncoder(w).Encode(map[string]interface{}{"answers": answers}) + })) + defer srv.Close() + t.Setenv(jev.EnvAPIKey, "local-test") + root := t.TempDir() + samples := filepath.Join(root, "samples") + if err := os.Mkdir(samples, 0700); err != nil { + t.Fatal(err) + } + raw := []byte("HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\nJenkinsJenkins ") + if err := os.WriteFile(filepath.Join(samples, "a.http"), raw, 0600); err != nil { + t.Fatal(err) + } + cache := filepath.Join(root, "cache") + directory := filepath.Join(root, "directory") + if err := run("jev", samples, "", cache, directory, srv.URL, 100, 2, 0); err != nil { + t.Fatal(err) + } + readRow := func(dir string) record { + t.Helper() + data, err := os.ReadFile(filepath.Join(dir, "rows.jsonl")) + if err != nil { + t.Fatal(err) + } + var r record + if err := json.Unmarshal(data, &r); err != nil { + t.Fatal(err) + } + var fields map[string]json.RawMessage + _ = json.Unmarshal(data, &fields) + for _, old := range []string{"refined", "filled_versions", "verdicts", "outcomes", "evidence", "rejected", "duplicates", "missing_candidates"} { + if _, ok := fields[old]; ok { + t.Fatalf("legacy field %s remains", old) + } + } + return r + } + first := readRow(directory) + if len(first.Judged) == 0 || atomic.LoadInt64(&calls) != 1 { + t.Fatalf("expected one presence batch: frames=%d calls=%d", len(first.Judged), calls) + } + m := replayManifest{Schema: 1, Samples: []replaySample{{ID: "a.http", URL: "http://example.test/", Group: "example.test", CapturedAt: "2026-09-26T00:00:00Z", Response: "samples/a.http", SHA256: digest(raw)}}} + manifest := filepath.Join(root, "manifest.json") + if err := writeJSON(manifest, m); err != nil { + t.Fatal(err) + } + replayDir := filepath.Join(root, "replay") + if err := replay(manifest, "", "jev", srv.URL, cache, replayDir, 100, true, false, ""); err != nil { + t.Fatal(err) + } + second := readRow(replayDir) + a, _ := json.Marshal(first.Judged) + b, _ := json.Marshal(second.Judged) + if string(a) != string(b) || atomic.LoadInt64(&calls) != 1 { + t.Fatal("replay disagrees or invoked backend") + } + for _, dir := range []string{directory, replayDir} { + data, err := os.ReadFile(filepath.Join(dir, "metrics.json")) + if err != nil { + t.Fatal(err) + } + var summary replaySummary + if err := json.Unmarshal(data, &summary); err != nil { + t.Fatal(err) + } + if summary.Schema != 3 || summary.Errors != 0 { + t.Fatalf("summary %+v", summary) + } + } +} + +func TestLabelInputConvertsOnce(t *testing.T) { + path := filepath.Join(t.TempDir(), "labels.json") + data := []byte(`{"old":{"keep":["nginx"],"drop":["wordpress"],"version":{"product":"nginx","value":"1.2.3"}},"new":[{"product":"Orion","aliases":["Orion-console"],"present":true}]}`) + if err := os.WriteFile(path, data, 0600); err != nil { + t.Fatal(err) + } + labels, err := loadLabels(path) + if err != nil || len(labels["old"]) != 2 || labels["old"][0].Version == nil || *labels["old"][0].Version != "1.2.3" || len(labels["new"][0].Aliases) != 1 { + t.Fatalf("labels=%v error=%v", labels, err) + } +} + +func TestReplayHandlesBaselineWithoutAttributes(t *testing.T) { + frames := common.Frameworks{"nginx": {Name: "nginx"}} + m := &replayManifest{Samples: []replaySample{{ID: "a", Labels: []productLabel{{Product: "nginx", Present: true, Version: stringPtr("")}}}}} + summary := summarizeReplay(m, []record{{ID: "a", Baseline: frames, Judged: frames}}, nil, nil) + if summary.Accepted.Versions.CorrectAbstentions != 1 || summary.Baseline.Detection.TP != 1 { + t.Fatalf("summary %+v", summary) + } +} diff --git a/docs/jev-claim-acceptance-20260926.md b/docs/jev-claim-acceptance-20260926.md new file mode 100644 index 0000000..50d5cd4 --- /dev/null +++ b/docs/jev-claim-acceptance-20260926.md @@ -0,0 +1,92 @@ +# judge 重构验收：事实 → Claim → Jev → 动作表 + +> 历史验收记录：本文记录早期 Claim 实现，Refine 与自动校准等接口已移除。当前合约与默认值见 [judge](../judge/README.md)，当前输出见 [judgeeval](../cmd/judgeeval/README.md)；本文测量结果未由当前实现重新验证。 + +## 结论 + +**计划的全部验收标准都已通过。** 输入与 run-07-final 相同：69 份真实响应快照，682 个已标注的产品判断。模型为 jev-1.13.0，`MinConfidence` 取校准后的默认值 0.3，0 份请求失败。 + +| 验收标准 | run-07 | 本次 | 结论 | +|---|---:|---:|---| +| 有标注的误报删除数（要求 ≥24） | 24 | **24** | ✅ | +| 真实命中误删 | 0 | **0** | ✅ | +| 3 个宝塔默认页上的宝塔 | 被误删 | **全部保留** | ✅ | +| `varnish=1.1` | 出现 | **不出现** | ✅ | +| Refine 每页题目数（要求约 -35%） | 15.14 | **6.29（-58%）** | ✅ | +| 版本 正确/错误（要求不劣于 23/0） | 23/0 | **36/0**，无依据填值 0 | ✅ | +| Ledger 前几名包含 git、ipeakcms、webp_server_go | — | 第 1、2、3 名 | ✅ | +| Discover 找出 SearXNG、Wakapi、IT Tools 的簇 | — | 三个簇都找到了。不加载 library 时，Coverage 把 SearXNG、Wakapi 判为 `missing`，IT Tools 判为 `insufficient`（见"漏报"一节） | ⚠️ 2/3 | + +**计数口径** +- run-07 的误报在前、本次的在后，基线的真命中数不同（46 对 63），因为本次基线是用当前规则重新扫描保存的响应得到的。 +- 题目数的两个数字，是用同一个固定回答的 Provider，分别在旧设计（1f42439）和新设计上，对同一批命中跑 Refine 统计出来的。评估报告里的"平均每页 8.6 题"是另一个口径：它还包含 Discover 的 Coverage 题目。 + +## 超出计划的改动及原因 + +第一次真实评估只删掉了 8 个误报。原因都在"代码负责找事实"这一侧：给 Jev 的证据不足，或者给出的选项不贴切。按边界原则，修正都放在代码的事实层和题面里，没有加任何推翻或组合 Jev 回答的逻辑。 + +1. **四个引擎记录命中依据**：以前只有 fingers 引擎会写 `MatchDetail`。goby、fingerprinthub、ehole、wappalyzer 的命中没有任何证据，Jev 只能回答 insufficient。现在这四个引擎在命中时都会记录实际匹配到的文本，由 `Engine.EnableMatchDetail()` 统一打开。开关默认关闭，现有输出不变。 +2. **Evidence 带上 `matched`**：State 里除了上下文片段，还单独给出规则匹配到的文本本身。 +3. **新增判定 `unrelated`**：goby 的 `"s7"`、`"rdp"`、`"git"` 这类短串会碰巧命中 base64 数据或别的单词。原来的选项里没有贴切的一项，模型只好弃权。新增这个选项后，动作表执行删除；Ledger 里它也是垃圾规则最直接的信号。parsers 的字段注释同步更新了。 +4. **版本候选排除非版本数值**：viewport 的 `initial-scale=1.0` 和全零值（例如 NEL 头里的 `"success_fraction":0.0`）不会作为候选，与已有的 IPv4 排除属于同一类。题面也从 "a meta tag" 改成了 "a generator meta tag"。 +5. **Jev 不做指纹识别，全部判断统一为 Claim**（见下一节）。 +6. **Discover 的聚类**：通用安全头（`X-Content-Type-Options`、`X-Frame-Options` 等）不再算作结构特征。之前"请通过官方域名访问"的 403 页和 Privacy Redirect 站点只因为都带这些头就被并成了一簇。候选名排除 HTTP 状态短语（"404 Not Found"、"Forbidden"）。 + +## 统一为 Claim + +Jev 只审查规则给出的结论，不说出产品名。判定层只有一个机制：代码提出 Claim 并附上证据，Provider 选出证据支持的选项，选项的 Outcome（Holds / Refuted / Insufficient）驱动固定动作。 + +| Claim | 方向 | 选项 → Outcome | +|---|---|---| +| Presence：规则说 P 参与生成了这个响应 | 误报 | `running`→Holds；`mentioned`、`unrelated`→Refuted | +| Coverage：已报告的产品解释了这组页面 | 漏报 | `explained`、`custom_site`→Holds；`missing`→Refuted | +| Version：P 的版本是这些候选之一 | 版本 | 候选→Holds；`not_stated`→Refuted | + +- 每个 Claim 都带 `insufficient`；置信度低于 `MinConfidence` 的回答一律判为 Insufficient。代码确定的事实（declared / absent）也表达为 Ruling，和 Provider 的回答走同一张动作表。 +- 删除了 `Identify`、`Ask`、`Yes`、`Choose`、`Score`，以及 binary 和 score 题型。自定义判断用 `j.Judge(ctx, state, claims)`。 +- 规则库已有的产品名从不作为选项，只作为被审查的 Claim 出现在题干或 State 里。 +- 结论 DTO `parsers.Judgement` 与 Ruling 对齐：新增 `Outcome`，删除 `Layer`、`Primary`、`Recalled` 和 `Frameworks.Primary()`。Ledger 和评估报告都按 Outcome 统计。离线重放显示，Presence 的 holds 225 = declared 133 + running 92，refuted 135 = absent 5 + mentioned 54 + unrelated 76，insufficient 7。 +- `judge/gen` 必须由人提供 `Name`。Discover 的簇附带代码抽取的候选名，供人命名。 + +### 漏报（Coverage）结果 + +| 簇 | 不加载 library | 加载 library | +|---|---|---| +| SearXNG（10 主机） | `missing` 0.86 ✅ | `explained` ✅ | +| Wakapi（4 主机） | `missing` 0.73 ✅ | `explained` ✅ | +| IT Tools（2 主机） | `insufficient` 0.29 ⚠️ 漏判 | `missing` 0.35 ❌ 误报警 | +| nginx 默认页、通用 404、宝塔默认页、阿里 WAF 403、SoftEther | 全部 `explained` ✅ | 全部 `explained` ✅ | + +IT Tools 在两种情况下的概率分布几乎相同（missing 约 0.47–0.50），说明 Jev 对这个文字很少的单页应用，判断和"已报告里是否有 it tools"基本无关。误报警只会进入人工审核队列，不会删除任何东西。 + +## MinConfidence 校准 + +在同一输入上用 `judgeeval -min-confidence` 扫描不同阈值： + +| MinConfidence | 误报删除 | 误删 | 版本 对/错 | 宝塔 | 删除总数 | +|---:|---:|---:|---:|---|---:| +| 0.2 | 24 | 0 | 36/0 | 保留 | 138 | +| 0.3 | 24 | 0 | 36/0 | 保留 | 135 | +| 0.5 | 24 | 0 | 36/0 | 保留 | 117 | +| 0.6 | 24 | 0 | 36/0 | 保留 | 109 | +| 0.7 | 23 | 0 | 36/0 | 保留 | — | + +有标注的结果在 0.2–0.6 之间完全相同。阈值从 0.5 降到 0.3，多删的 18 个未标注命中逐一抽查都是误报：`harbor` 命中了图片 alt 文字，`webp_server_go` 命中了 `.webp` 文件名，`windows` 命中了 "Microsoft YaHei" 字体。从 0.3 降到 0.2 多删的 3 个也都是误报。默认值取 0.3：拿到几乎全部收益，同时没有降到 0.2 那么低；样本只有 69 页，留出一定余量。 + +## 遗留问题 + +- **宝塔别名**：`宝塔` 与 `宝塔-bt.cn` 归一化后不相同，所以不会去重。这需要在名称归一化或别名表里处理，不在本次范围内。宝塔默认页的簇判为 insufficient，因为页面本身没有写出产品名，这个结果是对的；宝塔在所有页面上都保留了。 +- **通用错误页**：聚类会把不同服务器的通用 404/403 页并成一簇，目前判为 insufficient，结果正确，但这类簇本身没有维护价值。 +- **样本规模**：结论只基于 69 页、59 个主机，`MinConfidence` 需要在更大的样本集上复核。 +- **网络错误**：评估中偶尔出现 `connection reset` 或 `EOF`，重跑即可恢复。 + +## 复现 + +``` +go run ./cmd/judgeeval \ + -manifest .judge-data/expansion-20260926/run-07-final/manifest.json \ + -library .judge-data/expansion-20260926/run-07-final/library.yaml \ + -cache .judge-data/cache -out .judge-data/claim-v1 +``` + +`TYPESAFE_API_KEY` 只通过环境变量传入。去掉 `-library` 即可验证 Discover 能否发现新产品（输出在 `.judge-data/claim-v1-nolib`）。`.judge-data/` 不纳入版本控制。 diff --git a/ehole/ehole.go b/ehole/ehole.go index 8dfcd45..e9fd49a 100644 --- a/ehole/ehole.go +++ b/ehole/ehole.go @@ -37,6 +37,8 @@ func NewEHoleEngine(data []byte) (*EHoleEngine, error) { type EHoleEngine struct { Fingerprints []*Fingerprint `json:"fingerprint"` FaviconMap map[string]string + // MatchDetailEnabled records on each hit the keyword or text it matched. + MatchDetailEnabled bool `json:"-"` } func (engine *EHoleEngine) Name() string { @@ -106,6 +108,9 @@ func (engine *EHoleEngine) MatchWithHeaderAndBody(header, body string) common.Fr for _, finger := range engine.Fingerprints { frame := finger.Match(header, body) if frame != nil { + if !engine.MatchDetailEnabled { + frame.MatchDetail = nil + } frames.Add(frame) } } @@ -122,18 +127,32 @@ type Fingerprint struct { } func (finger *Fingerprint) Match(header, body string) *common.Framework { + var content string switch finger.Location { case BodyLocation, TitleLocation: - if finger.MatchMethod(body) { - return common.NewFramework(finger.Cms, common.FrameFromEhole) - } + content = body case HeaderLocation: - if finger.MatchMethod(header) { - return common.NewFramework(finger.Cms, common.FrameFromEhole) - } + content = header default: return nil } + if !finger.MatchMethod(content) { + return nil + } + frame := common.NewFramework(finger.Cms, common.FrameFromEhole) + frame.MatchDetail = finger.matched(content) + return frame +} + +// matched is what a matching fingerprint found in content: its first keyword, +// or the text its first regexp matched. +func (finger *Fingerprint) matched(content string) *common.MatchDetail { + switch { + case finger.Method == KeywordMethod && len(finger.LowerKeyword) > 0: + return &common.MatchDetail{MatcherType: "word", MatcherValue: finger.LowerKeyword[0]} + case finger.Method == RegularMethod && len(finger.compiledRegexp) > 0: + return &common.MatchDetail{MatcherType: "word", MatcherValue: finger.compiledRegexp[0].FindString(content)} + } return nil } diff --git a/engine.go b/engine.go index 47c39ba..1c94bf3 100644 --- a/engine.go +++ b/engine.go @@ -5,7 +5,6 @@ import ( "bytes" "fmt" "net/http" - "sort" "strings" "github.com/chainreactors/fingers/alias" @@ -99,20 +98,6 @@ func (engine *Engine) String() string { return strings.TrimSpace(s.String()) } -// Names returns the canonical names of every fingerprint in the alias -// table, sorted: what judge.NewRetriever takes to recall missed products. -func (engine *Engine) Names() []string { - if engine.Aliases == nil { - return nil - } - names := make([]string, 0, len(engine.Aliases.Aliases)) - for name := range engine.Aliases.Aliases { - names = append(names, name) - } - sort.Strings(names) - return names -} - func (engine *Engine) Compile() error { // 从所有引擎中填充Favicon引擎的数据 if impl := engine.Fingers(); impl != nil { @@ -221,6 +206,28 @@ func (engine *Engine) Disable(name string) { engine.Enabled[name] = false } +// EnableMatchDetail makes every engine that can record what a hit matched +// (fingers, fingerprinthub, wappalyzer, ehole, goby) do so in +// Framework.MatchDetail. Off by default: it costs fingers and fingerprinthub +// their keyword fast paths. +func (engine *Engine) EnableMatchDetail() { + if e := engine.Fingers(); e != nil { + e.EnableMatchDetail() + } + if e := engine.FingerPrintHub(); e != nil { + e.MatchDetailEnabled = true + } + if e := engine.Wappalyzer(); e != nil { + e.MatchDetailEnabled = true + } + if e := engine.EHole(); e != nil { + e.MatchDetailEnabled = true + } + if e := engine.Goby(); e != nil { + e.MatchDetailEnabled = true + } +} + func (engine *Engine) Fingers() *fingers.FingersEngine { if impl, ok := engine.EnginesImpl[FingersEngine]; ok { return impl.(*fingers.FingersEngine) diff --git a/fingerprinthub/fingerprinthub.go b/fingerprinthub/fingerprinthub.go index 89c6cd7..078ae8d 100644 --- a/fingerprinthub/fingerprinthub.go +++ b/fingerprinthub/fingerprinthub.go @@ -23,6 +23,10 @@ type FingerPrintHubEngine struct { // CaseInsensitive controls whether matching ignores case (default true). CaseInsensitive bool + // MatchDetailEnabled records on each hit the snippet its matcher matched. + // Hits then skip the keyword-index fast path, which knows no snippet. + MatchDetailEnabled bool + // active holds full neutron templates for HTTPActiveMatch/ServiceMatch. // Nil in passive_only builds. active *activeState @@ -197,13 +201,13 @@ func (engine *FingerPrintHubEngine) WebMatch(content []byte) common.Frameworks { mr := engine.webTemplateIndex.Match(lowerHeaderStr, lowerBodyStr) - if engine.CaseInsensitive { + if engine.CaseInsensitive && !engine.MatchDetailEnabled { for ti := range mr.Matched { frames.Add(engine.newFramework(engine.webTemplates[ti])) } } - if !engine.CaseInsensitive { + if !engine.CaseInsensitive || engine.MatchDetailEnabled { for ti := range mr.Matched { mr.NeedsCheck[ti] = true } @@ -218,8 +222,12 @@ func (engine *FingerPrintHubEngine) WebMatch(content []byte) common.Frameworks { if len(req.Matchers) == 0 { continue } - if matchPassiveRequest(req, event) { - frames.Add(engine.newFramework(tmpl)) + if matched, detail := matchPassiveRequestDetail(req, event); matched { + frame := engine.newFramework(tmpl) + if engine.MatchDetailEnabled { + frame.MatchDetail = detail + } + frames.Add(frame) break } } diff --git a/fingerprinthub/passive_template.go b/fingerprinthub/passive_template.go index c392405..ed4091f 100644 --- a/fingerprinthub/passive_template.go +++ b/fingerprinthub/passive_template.go @@ -6,6 +6,7 @@ import ( "strconv" "strings" + "github.com/chainreactors/fingers/common" "github.com/chainreactors/neutron/operators" "github.com/chainreactors/neutron/protocols" "github.com/chainreactors/utils/encode" @@ -146,8 +147,15 @@ func parseRawHTTPEvent(content []byte, caseInsensitive bool) (protocols.Internal } func matchPassiveRequest(req *passiveRequest, event protocols.InternalEvent) bool { + matched, _ := matchPassiveRequestDetail(req, event) + return matched +} + +// matchPassiveRequestDetail also returns the first snippet a positive matcher +// matched. +func matchPassiveRequestDetail(req *passiveRequest, event protocols.InternalEvent) (bool, *common.MatchDetail) { if len(req.Matchers) == 0 { - return false + return false, nil } matchersCondition := req.MatchersCondition @@ -156,23 +164,29 @@ func matchPassiveRequest(req *passiveRequest, event protocols.InternalEvent) boo } matchedCount := 0 - for _, matcher := range req.Matchers { - matched, _ := protocols.HTTPMatch(event, matcher) + var detail *common.MatchDetail + for i, matcher := range req.Matchers { + matched, hits := protocols.HTTPMatch(event, matcher) if matched { matchedCount++ + for _, hit := range hits { + if detail == nil && hit.Value != "" { + detail = &common.MatchDetail{MatcherType: matcher.Type, MatcherIndex: i, MatcherValue: hit.Value} + } + } if matchersCondition == "or" { - return true + return true, detail } } else { if matchersCondition == "and" { - return false + return false, nil } } } - if matchersCondition == "and" { - return matchedCount == len(req.Matchers) + if matchersCondition == "and" && matchedCount == len(req.Matchers) { + return true, detail } - return false + return false, nil } diff --git a/go.mod b/go.mod index e90890b..c4b09b4 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,8 @@ require ( require ( github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible github.com/chainreactors/neutron v0.1.1-0.20260714062907-716c6b167cb6 - github.com/chainreactors/utils/parsers v0.0.4-0.20260925164932-39c659c84c64 + github.com/chainreactors/utils/jev v0.0.0-20260927031300-6bf027367a50 + github.com/chainreactors/utils/parsers v0.0.4-0.20260926072629-6752abe35765 github.com/chainreactors/words v0.0.0-20260520145736-270600e60fb4 github.com/dlclark/regexp2 v1.11.5 github.com/invopop/jsonschema v0.7.0 diff --git a/go.sum b/go.sum index 1830173..917e0d8 100644 --- a/go.sum +++ b/go.sum @@ -79,8 +79,10 @@ github.com/chainreactors/neutron v0.1.1-0.20260714062907-716c6b167cb6 h1:apEDnJe github.com/chainreactors/neutron v0.1.1-0.20260714062907-716c6b167cb6/go.mod h1:zok/CDxut71iw8NQTHKPvy0f+1G5639zr9GgDsXGnOs= github.com/chainreactors/utils v0.0.0-20260629043228-93bdd2142c9a h1:4rLTKr+VlG7yGbw2ym7nhNvFglvZ4acdphxKEozF7CI= github.com/chainreactors/utils v0.0.0-20260629043228-93bdd2142c9a/go.mod h1:xwbUlFoSSxLHujyb8D48o1s2DqmEAxUNfxIy0DVUmcg= -github.com/chainreactors/utils/parsers v0.0.4-0.20260925164932-39c659c84c64 h1:MgEJ0vr7png0bKb5nYL1pK5lTUMDBRbigYt74K+UwYw= -github.com/chainreactors/utils/parsers v0.0.4-0.20260925164932-39c659c84c64/go.mod h1:bE/znJWt08n9QOORWsWu0ggB8GWfOg3+dfUMMITmwV4= +github.com/chainreactors/utils/jev v0.0.0-20260927031300-6bf027367a50 h1:XnGpBYIS4u1BUNL3KtlrrXde/vSmDon34X5AEV7WOi8= +github.com/chainreactors/utils/jev v0.0.0-20260927031300-6bf027367a50/go.mod h1:9w5fvcKrpQVIHWlKuvpVARpZUd+OKsYSIii2JR1ZRB8= +github.com/chainreactors/utils/parsers v0.0.4-0.20260926072629-6752abe35765 h1:5SDaMwwQfXHdkzUvFEjK4/blZ/EifEKZGtRhwzdnZ24= +github.com/chainreactors/utils/parsers v0.0.4-0.20260926072629-6752abe35765/go.mod h1:bE/znJWt08n9QOORWsWu0ggB8GWfOg3+dfUMMITmwV4= github.com/chainreactors/words v0.0.0-20260520145736-270600e60fb4 h1:lvnDYEkatmZFHP5i321qQXK9L4vKRfso/uUfr5tOeC8= github.com/chainreactors/words v0.0.0-20260520145736-270600e60fb4/go.mod h1:zfz367PUmyaX6oAqV9SktVqyRXKlEh0sel9Wsq9dd2c= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= diff --git a/goby/goby.go b/goby/goby.go index 16741bf..5d8eff1 100644 --- a/goby/goby.go +++ b/goby/goby.go @@ -26,6 +26,8 @@ func NewGobyEngine(data []byte) (*GobyEngine, error) { type GobyEngine struct { Fingers []*GobyFinger + // MatchDetailEnabled records on each hit the feature it matched. + MatchDetailEnabled bool } func (engine *GobyEngine) Name() string { @@ -69,6 +71,9 @@ func (engine *GobyEngine) MatchRaw(raw string) common.Frameworks { for _, finger := range engine.Fingers { frame := finger.Match(raw) if frame != nil { + if !engine.MatchDetailEnabled { + frame.MatchDetail = nil + } frames.Add(frame) } } @@ -101,15 +106,21 @@ func (finger *GobyFinger) Compile() error { func (finger *GobyFinger) Match(raw string) *common.Framework { env := make(map[string]bool) - for _, r := range finger.Rule { + var detail *common.MatchDetail + for i, r := range finger.Rule { match := strings.Contains(raw, r.Feature) env[r.Label] = match == r.IsEquel + if match && r.IsEquel && detail == nil { + detail = &common.MatchDetail{MatcherType: "word", MatcherIndex: i, MatcherValue: r.Feature} + } } matched := logic.EvalLogic(finger.logicExpr, env) if matched { - return common.NewFramework(finger.Name, common.FrameFromGoby) + frame := common.NewFramework(finger.Name, common.FrameFromGoby) + frame.MatchDetail = detail + return frame } return nil } diff --git a/judge/README.md b/judge/README.md index e615906..7158527 100644 --- a/judge/README.md +++ b/judge/README.md @@ -1,74 +1,57 @@ -# judge：规则结果的判定层 +# judge：以 Claim 审查规则结果 -`fingers.Engine` 只做规则检测。`judge.Judge` 接收规则命中和原始 HTTP 响应，由代码抽取候选、由 Provider（当前是 `judge/jev`）回答判断题，最后由代码做决定。输入的规则命中从不被修改。 +规则引擎负责检测；judge 在线只审查产品是否参与生成响应，以及响应能否确认版本。Provider 从有限选项中选择，不生成产品名或版本字符串。离线审计、聚类与生成分别放在 `judge/maintain` 和 `judge/gen`，复用同一 Claim 合约。 + +Claim 合约（Claim / Option / Outcome / Ruling / Provider）、校验、缓存和 Jev 客户端都在 [`github.com/chainreactors/utils/jev`](https://github.com/chainreactors/utils/tree/master/jev)。本包只定义指纹相关的断言：presence、version 和离线的 coverage，以及代码确定的事实如何表达成 Ruling。 + +## 代码确定的事实 + +代码确定的产品、协议和版本事实也构建合法 Claim/Ruling（置信度 1），经相同的 `Resolve` 和应用函数执行。重复写法的归并是 bookkeeping，不是 Claim。 + +## 在线入口 ```go engine, _ := fingers.NewEngine() -j, err := jev.NewJudge("") // 读取 TYPESAFE_API_KEY；一个扫描任务共用一个 Judge +engine.EnableMatchDetail() +c, err := jev.NewClient("") // TYPESAFE_API_KEY if err != nil { return err } -j.Known = judge.NewRetriever(engine.Names()) // 可选：召回规则漏掉、但名字出现在页面里的产品 +j := judge.New(jev.Cached(c, jev.DefaultCacheSize)) // 一个扫描任务共用 -hits, _ := engine.DetectContent(raw) -accepted, err := j.Refine(ctx, raw, hits) -if err != nil { accepted = hits } // 判定失败时退回纯规则结果 +hits, err := engine.DetectContent(raw) +if err != nil { return err } +all, err := j.Inspect(ctx, raw, hits) +accepted := all.Accepted() +// all 同时保存接受、拒绝、重复及未判定条目；出错时保留已完成的结果。 ``` -## 三个入口 +`Inspect` 完成 presence 和 version，最多两个 Provider 批次。报告直接读这份结果，不再执行第二次判断。输入会复制，重新评估会清除旧 presence 注解。解析错误返回复制的基线和错误；Provider 失败时保留已确定事实和去重信息，尚未判断的条目没有 Outcome。输入中已有非空版本保持原值。 -| 方法 | 用途 | +| 入口 | 用途 | |---|---| -| `j.Refine(ctx, raw, hits)` | **要上报的结果**：去掉误报和重复写法，加入召回的产品，给每个保留的产品补版本 | -| `j.Inspect(ctx, raw, hits)` | **为什么**：全部命中及判定结论，包括被拒绝和重复的；不补版本 | -| `j.Classify(ctx, raw)` | **这是什么页面**：`Kind`（登录、错误、默认页……）和是否为产品标准页；Refine 之后命中缓存 | +| `Inspect(ctx, raw, hits)` | 所有命中及 presence 注解，保留产品的版本补全 | +| `all.Accepted()` | 排除 Rejected / Duplicate 的结果视图，不调用 Provider | +| `Version(ctx, raw, f)` | 为生成工具单独确认版本，不修改 f | -其余能力：`j.Version(ctx, raw, f)` 单独给一个产品选版本；`j.IsUnknownProduct(ctx, raw, accepted)` 判断是否为尚无指纹的产品标准页；`j.SuggestNames(ctx, raw)` 从页面证据中提出产品名；`j.Yes/Choose/Score` 用来问自定义问题，state 可以是任意可 JSON 编码的值，比如 `judge.NewPage(raw)`。 +Presence 用 `running` 表示产品参与生成响应，`mentioned` / `unrelated` 表示误报。协议事实从完整响应头读取；产品确定事实限于明确的软件声明和产品专属头，Location、Cookie 中的名称只作为待审证据。版本从响应中抽取，每个产品只有一个多选 Claim：候选版本、`not_stated`、`insufficient`。按名字声明的版本走本地 Ruling；其他版本由 Provider 选择。每页最多向 Provider 提交 40 个 presence Claim，最多为 8 个产品确认版本，每个产品最多展示 15 个候选。 -## 判定结论 - -结论写在 `Framework.Judge`（定义在 `utils/parsers`）上，随现有输出一起序列化： +`Framework.Judge` 是输出注解，沿用现有 common 类型： | 字段 | 含义 | |---|---| -| `Layer` | 在技术栈中的层级：`judge.LayerCDN`、`LayerServer`、`LayerApplication` …… | -| `Confidence` | Provider 给出的"该产品确实参与生成此响应"的概率 | -| `Rejected` | 误报：只在页面文字中出现，或不存在。响应头或 Cookie 中出现的产品不会被拒绝 | -| `Duplicate` | 另一个引擎对已保留产品的不同写法 | -| `Primary` | 页面所属的主应用 | -| `Recalled` | 规则没报、由 `j.Known` 在页面中找到并经确认 | - -`Judge == nil` 表示未经判定（每页超过 40 个产品时，多出的部分不判定），`frames.Accepted()` 会保留它们。 - -## 版本号 - -版本字符串一律由代码从原始响应中抽取，Provider 只做选择，不生成字符串： +| Verdict | 原始选项；即使置信度不足也保留 | +| Outcome | Resolve 后的 holds / refuted / insufficient；消费方按此统计 | +| Confidence / Evidence | 置信度及规则命中原文；本地确定事实置信度为 1 | +| Rejected | Refuted，或 Insufficient 且 DropInsufficient 为 true | +| Duplicate | 同一产品的其他写法；即使尚未判定也可标记 | -1. 响应**按名字绑定**的版本直接采用，不提问：`Server: nginx/1.24.0`、`X-Jenkins: 2.401.3`、`X-Gitea-Version: 1.21.4`、``。名字按末尾词匹配，所以 `Apache Tomcat/9.0` 属于 Tomcat 而不是 Apache。 -2. 其余产品在同一个请求里各问一题。候选按与产品名的接近程度排序（上下文提到产品名，紧挨版本前更好），插件、主题、第三方库路径降权；已按名字绑定给别的产品、且只出现一次的版本不会提供给它。 -3. 只出现一次的版本字符串只归一个产品：上下文更贴近的优先，其次置信度更高的。置信度低于 `VersionConfidence` 的选择不采用。 +`judge.New(p)` 默认 `MinConfidence=jev.DefaultMinConfidence`（0.3）、`DropInsufficient=false`。策略和 Provider 配置应在并发扫描前确定。Version 在证据不足时不填值。 -## 缓存与成本 +## 离线维护 -答案按问题缓存，同一问题不会重复发送，无论它被放进哪次请求。标题相同、签名相差不超过 `j.SimilarDistance` 位（默认 1）的页面共享答案，同一产品在不同主机上的登录页只问一次；并发的相同请求会合并。`j.Cache` 默认是内存 LRU，实现 `judge.Cache` 即可换成共享存储。`j.Requests`、`j.CacheHits` 用于成本监控。 +- `maintain.Ledger.Add(id, all)` 从既有 presence 注解汇总规则表现，不再请求 Provider。`Report` 返回深复制快照，记录 Outcomes、Options 和样本 ID；`Hits()` 与 `RefutedFraction()` 从 Outcomes 推导。Presence 选项常量统一为 `judge.OptionRunning` 等 `Option*`，原生 `Framework.Judge.Verdict` 输出字段继续保留。当前引擎聚合后的 MatchDetail 仍只保留首个匹配详情，未重设计完整规则来源。 +- `maintain.Discover` 按资源路径、表单、Cookie/自定义头名和页面文字等结构特征聚类。每个多主机簇提出 Coverage Claim：`explained` / `custom_site` → Holds，`missing` → Refuted，另有 insufficient。Cluster 直接可序列化，仅保存成员 ID、Coverage 原始 Ruling、解析的 Outcome、已报告产品和代码抽取的候选名。候选名供人命名，不作为 Provider 选项。 +- `gen.Generator` 为人指定的 Name 从正反样本生成规则。需要时调用 Version。Validate 运行规则引擎，不调用模型，不修改内置库。 -## 自定义 Provider - -实现 `ID() string` 和 `Judge(ctx, state, map[string]judge.Question) (map[string]judge.Answer, error)` 即可，可选实现 `Calibration() (threshold, versionConfidence float64)`。问题分三类：`TypeBinary`（是的概率）、`TypeChoice`（从选项中选一个）、`TypeScore`（有序等级上的分数）。缓存、并发合并和阈值决策都在 `Judge` 中完成，因此不同 Provider 的行为一致。新 Provider 需在 `cmd/judgeeval` 的 `newProvider` 注册并跑一遍评估来校准阈值。 - -## 数据外发 - -发给 Provider 的是响应的精简视图：响应头（去掉 Date、Set-Cookie 等无关或易变的头）、Cookie 名、标题、generator/description、脚本和样式路径、内联脚本开头、HTML 注释、表单字段名，以及正文前 1500 字。扫描前请确认允许把这些内容发送给第三方服务。 - -## 生成指纹 - -`judge/gen` 用正反样本生成原生 `*fingers.Finger`： - -```go -g := gen.New(j). - PositiveVersion(jenkinsA, "2.401.3"). - PositiveVersion(jenkinsB, "2.402.1"). - Negative(otherProduct) -finger, err := g.Generate(ctx) // 可直接用于现有规则引擎 -err = g.Validate(finger) // 编辑规则后，再按全部正反样本验证 -``` +`cmd/judgeeval` 两种输入模式共用 `baseline` / `judged` 行记录。接受结果、拒绝项、版本差异和指标均从它们派生，聚类通过样本 ID 关联。生成评估同样保存原生 Framework，以 manifest 为唯一标签来源，分数和状态按需推导。报告 schema 为 3，统计使用 `claims/options`；旧 manifest、labels 和 baseline 输入仍可读取，旧缓存因命名空间隔离不会被复用。详见 [judgeeval](../cmd/judgeeval/README.md)。 -至少需要一个正样本和一个反样本。不给 `Name` 时，由 `SuggestNames` 从正样本中选名字；`Probe`/`ProbeWith` 记录主动探测的请求和响应。`Validate` 只运行规则引擎、不调用模型，也不会写入模板仓库。 +Provider 收到精简响应视图、规则命中上下文和版本候选上下文；完整原始响应仍仅在本地用于解析事实与提取证据。 diff --git a/judge/cache.go b/judge/cache.go deleted file mode 100644 index 49a23b6..0000000 --- a/judge/cache.go +++ /dev/null @@ -1,86 +0,0 @@ -package judge - -import ( - "container/list" - "math/bits" - "sync" -) - -// Cache stores answers, one per question. key covers everything that must -// match exactly: model, question and named evidence such as version strings. -// sig is the page's signature: Get may return the answer stored for a page -// whose signature is within distance bits (Judge.SimilarDistance), so -// near-identical pages (one product's login page on many hosts, one site's -// 404 page on every path) are asked once. Providers answer identical input -// identically, so an exact hit is always valid. Implementations must be safe for concurrent use; the -// in-memory NewMemoryCache is the default, a shared one (redis, disk) lets -// several scanners share answers. -type Cache interface { - Get(key string, sig uint64, distance int) ([]byte, bool) - Put(key string, sig uint64, value []byte) -} - -// NewMemoryCache is an LRU cache holding up to size responses. -func NewMemoryCache(size int) Cache { - return &memoryCache{size: size, order: list.New(), buckets: map[string][]*list.Element{}} -} - -type memoryCache struct { - mu sync.Mutex - size int - order *list.List // front = most recent; values are *memoryEntry - buckets map[string][]*list.Element -} - -type memoryEntry struct { - key string - sig uint64 - value []byte -} - -func (c *memoryCache) Get(key string, sig uint64, distance int) ([]byte, bool) { - c.mu.Lock() - defer c.mu.Unlock() - var best *list.Element - bestDist := 65 - for _, e := range c.buckets[key] { - if d := bits.OnesCount64(e.Value.(*memoryEntry).sig ^ sig); d < bestDist { - best, bestDist = e, d - } - } - if best == nil || bestDist > distance { - return nil, false - } - c.order.MoveToFront(best) - return best.Value.(*memoryEntry).value, true -} - -func (c *memoryCache) Put(key string, sig uint64, value []byte) { - c.mu.Lock() - defer c.mu.Unlock() - for _, e := range c.buckets[key] { - if me := e.Value.(*memoryEntry); me.sig == sig { - me.value = value - c.order.MoveToFront(e) - return - } - } - c.buckets[key] = append(c.buckets[key], c.order.PushFront(&memoryEntry{key, sig, value})) - for c.order.Len() > c.size { - last := c.order.Back() - c.order.Remove(last) - me := last.Value.(*memoryEntry) - bucket := c.buckets[me.key] - for i, e := range bucket { - if e == last { - bucket = append(bucket[:i], bucket[i+1:]...) - break - } - } - if len(bucket) == 0 { - delete(c.buckets, me.key) - } else { - c.buckets[me.key] = bucket - } - } -} diff --git a/judge/classify.go b/judge/classify.go deleted file mode 100644 index 572b565..0000000 --- a/judge/classify.go +++ /dev/null @@ -1,38 +0,0 @@ -package judge - -// Kind is what a page is, set by Classify. -type Kind string - -const ( - KindLogin Kind = "login" - KindDefault Kind = "default_install" - KindError Kind = "error_page" - KindDirListing Kind = "directory_listing" - KindConsole Kind = "app_console" - KindContent Kind = "normal_content" - KindAPI Kind = "api_response" -) - -var pageKinds = map[string]string{ - string(KindLogin): "A sign-in form or authentication page", - string(KindDefault): "The stock page a web server or product shows right after installation", - string(KindError): "A framework or server error page such as 404 or 500", - string(KindDirListing): "An auto-generated file index of a directory", - string(KindConsole): "The main interface or loading screen of a web application or management console, including packaged static/browser-side tools and search applications", - string(KindContent): "An ordinary website page: article, company site, shop, forum, blog, status page", - string(KindAPI): "A machine-readable API response rather than a web page", -} - -// classifyRound asks what the page is and whether it is the stock page of a -// packaged product, writing the answers to kind and generic. It needs no -// fingerprints: spray can use it alone for 404 detection and -// prioritisation. A generic page without an accepted application is a -// candidate for a new fingerprint. -func classifyRound(r *round, kind *Kind, generic *bool) { - r.add("page_kind", Choice("What kind of page is this HTTP response?", pageKinds), func(a Answer) { *kind = Kind(a.Choice) }) - r.add("generic", BinaryWith("Is this page the stock interface of a packaged software product, framework or device, "+ - "so that the same page would appear on many unrelated deployments?", - "Login, console, default or error page, packaged search application or static/browser-side tools shipped by a product; no login or backend is required; only branding, host names or data differ between installs", - "Content written by one organization for its own purpose: articles, company pages, shops, forums, custom-built portals"), - func(a Answer) { *generic = a.Yes >= r.judge.Threshold }) -} diff --git a/judge/discovery.go b/judge/discovery.go deleted file mode 100644 index b51da8f..0000000 --- a/judge/discovery.go +++ /dev/null @@ -1,77 +0,0 @@ -package judge - -import ( - "context" - "fmt" - "sort" - - "github.com/chainreactors/fingers/common" - "github.com/chainreactors/fingers/judge/internal/evidence" -) - -// IsUnknownProduct reports whether this looks like a reusable product page -// with no identified product. Pass the result of Refine; known -// infrastructure alone does not identify an application. -func (j *Judge) IsUnknownProduct(ctx context.Context, raw []byte, accepted common.Frameworks) (bool, error) { - kind, generic, err := j.Classify(ctx, raw) - if err != nil || !generic { - return false, err - } - for _, f := range accepted.Accepted() { - if f.Judge == nil { - continue - } - switch f.Judge.Layer { - case LayerApplication, LayerDevice: - return false, nil - case LayerServer, LayerCDN: - if kind == KindDefault || kind == KindError || kind == KindDirListing { - return false, nil - } - } - } - return true, nil -} - -// SuggestNames returns product names supported by page evidence. Candidates -// are extracted from this response, including visible body text; no name is -// invented by the provider or drawn from an installed fingerprint catalog. -func (j *Judge) SuggestNames(ctx context.Context, raw []byte) ([]string, error) { - p, err := NewPage(raw) - if err != nil { - return nil, err - } - candidates := pageNames(p) - if len(candidates) == 0 { - return nil, nil - } - r := newRound(p) - prob := make([]float64, len(candidates)) - for i, name := range candidates { - i, name := i, name - r.add(fmt.Sprintf("name_%d", i), BinaryWith( - fmt.Sprintf("Is `%s` the product whose own interface, login, console or default page this response shows, rather than infrastructure underneath another product, generic wording or an article mention?", name), - name+" owns this page", - name+" is infrastructure or merely mentioned"), - func(a Answer) { prob[i] = a.Yes }) - } - if err := r.ask(ctx, j); err != nil { - return nil, err - } - var indices []int - for i, v := range prob { - if v >= j.Threshold { - indices = append(indices, i) - } - } - sort.SliceStable(indices, func(a, b int) bool { return prob[indices[a]] > prob[indices[b]] }) - out := make([]string, 0, len(indices)) - for _, i := range indices { - out = append(out, candidates[i]) - } - return out, nil -} - -func pageNames(p *Page) []string { - return evidence.Names(p.Generator, p.Title, p.Text, p.Headers, append(append([]string(nil), p.Scripts...), p.Styles...)) -} diff --git a/judge/doc.go b/judge/doc.go index 0c569e9..1661113 100644 --- a/judge/doc.go +++ b/judge/doc.go @@ -1,16 +1,18 @@ -// Package judge filters and completes rule-engine results with a judgement -// provider. Code extracts candidates and decides; a Provider (such as -// TypeSafe Jev in judge/jev) only answers typed questions about the evidence. +// Package judge reviews fingerprint rule results with jev claims +// (github.com/chainreactors/utils/jev): whether each hit's product produced +// the response, and which version the response states. It never identifies +// products itself. Deterministic facts use the same claims. Duplicate +// grouping is bookkeeping. // -// j := judge.New(provider) -// j.Known = judge.NewRetriever(engine.Names()) -// accepted, err := j.Refine(ctx, raw, hits) // what to report -// all, err := j.Inspect(ctx, raw, hits) // why: rejected and duplicate hits too -// kind, generic, err := j.Classify(ctx, raw) +// j := judge.New(jev.Cached(client, jev.DefaultCacheSize)) +// all, err := j.Inspect(ctx, raw, hits) +// accepted := all.Accepted() // -// Verdicts are written to Framework.Judge. Hits passed in are never changed. -// Answers are cached per question, and similar pages share them. Fingerprint -// generation from labelled responses lives in judge/gen. +// Inspect returns all annotated hits after at most two provider batches. Inputs +// are copied; errors preserve established facts and leave untouched claims +// unjudged. A low-confidence ruling resolves to Insufficient without +// rewriting the selected option. // -// See README.md in this directory. +// Offline audit and clustering live in judge/maintain, fingerprint +// generation in judge/gen. package judge diff --git a/judge/facts.go b/judge/facts.go new file mode 100644 index 0000000..6d9ca25 --- /dev/null +++ b/judge/facts.go @@ -0,0 +1,254 @@ +package judge + +import ( + "regexp" + "sort" + "strings" + "unicode/utf8" + + "github.com/chainreactors/fingers/common" + "github.com/chainreactors/fingers/judge/internal/evidence" +) + +// protocolFeatures are HTTP features some engines report as fingerprints +// (NormalizeName keys). The response head establishes them, or they are absent. +var protocolFeatures = map[string]bool{"hsts": true, "altsvc": true, "http3": true, "http2": true, "poweredby": true, "http基本认证": true} + +const ( + maxClaims = 40 // products judged per page; the rest stay unjudged (Judge == nil) + maxEvidence = 3 // excerpts per claim + evidenceContext = 80 // bytes of response around a match + maxExcerpt = 240 // runes per excerpt +) + +// groupProducts returns each product's spellings with the representative first. +// Names, evidence and rulings are derived from these original frameworks. +func groupProducts(frames common.Frameworks) [][]*common.Framework { + byKey := map[string][]*common.Framework{} + var keys []string + for _, f := range frames { + if f == nil { + continue + } + key := NormalizeName(f.Name) + if key == "" { + continue + } + if len(byKey[key]) == 0 { + keys = append(keys, key) + } + byKey[key] = append(byKey[key], f) + } + sort.Strings(keys) + groups := make([][]*common.Framework, 0, len(keys)) + for _, key := range keys { + group := byKey[key] + sort.SliceStable(group, func(a, b int) bool { + if qa, qb := frameworkQuality(group[a]), frameworkQuality(group[b]); qa != qb { + return qa > qb + } + return group[a].Name < group[b].Name + }) + groups = append(groups, group) + } + return groups +} + +// declares accepts explicit software declarations, not incidental URL/cookie text. +func declares(p *evidence.Page, name string) bool { + key := NormalizeName(name) + if key == "" { + return false + } + for header, values := range p.Header { + h := strings.ToLower(header) + if strings.HasPrefix(h, "x-") && NormalizeName(strings.TrimSuffix(strings.TrimPrefix(h, "x-"), "-version")) == key { + return true + } + if h != "server" && h != "x-powered-by" && h != "product" { + continue + } + for _, value := range values { + for _, token := range strings.FieldsFunc(value, func(r rune) bool { return r == ',' || r == ';' || r == '(' || r == ')' }) { + token = strings.TrimSpace(token) + if i := strings.IndexByte(token, '/'); i >= 0 { + token = strings.TrimSpace(token[:i]) + } + if NormalizeName(token) == key { + return true + } + // A header can list several products separated by spaces. + for _, part := range strings.Fields(value) { + if !strings.Contains(part, "/") { + continue + } + part = strings.SplitN(part, "/", 2)[0] + if NormalizeName(part) == key { + return true + } + } + } + } + } + return false +} + +func protocolPresent(p *evidence.Page, key string) bool { + switch key { + case "hsts": + return p.Header.Get("Strict-Transport-Security") != "" + case "altsvc": + return p.Header.Get("Alt-Svc") != "" + case "poweredby": + return p.Header.Get("X-Powered-By") != "" + case "http2": + return strings.HasPrefix(string(p.Raw), "HTTP/2") || strings.Contains(strings.ToLower(strings.Join(p.Header.Values("Alt-Svc"), ",")), "h2=") + case "http3": + return strings.HasPrefix(string(p.Raw), "HTTP/3") || strings.Contains(strings.ToLower(strings.Join(p.Header.Values("Alt-Svc"), ",")), "h3=") + case "http基本认证": + for _, value := range p.Header.Values("WWW-Authenticate") { + // A comma inside a quoted realm does not start a new challenge. + start, quoted, escaped := 0, false, false + for i := 0; i <= len(value); i++ { + if i == len(value) || (value[i] == ',' && !quoted) { + words := strings.Fields(value[start:i]) + if len(words) > 0 && strings.EqualFold(words[0], "Basic") { + return true + } + start = i + 1 + } else if escaped { + escaped = false + } else if value[i] == '\\' && quoted { + escaped = true + } else if value[i] == '"' { + quoted = !quoted + } + } + } + } + return false +} + +// Prefer a representative that already carries a version and useful source +// metadata, since duplicate spellings are hidden from the accepted result. +func frameworkQuality(f *common.Framework) int { + quality := len(f.Froms) + if f.Attributes != nil { + if f.Version != "" { + quality += 100 + } + if f.Vendor != "" { + quality += 10 + } + } + if f.MatchDetail != nil { + quality++ + } + return quality +} + +// evidenceFor quotes the response where the claim comes from: what each +// spelling's rule matched (MatchDetail, when the engine records it), then +// where the product's name occurs. A rule may rest on text deep in the +// page, which the compact page view does not carry. +func evidenceFor(raw []byte, frames []*common.Framework) []matchExcerpt { + text := string(raw) + lower := strings.ToLower(text) + var out []matchExcerpt + seen := map[int]bool{} + add := func(start, end int, matched bool) { + if start < 0 || len(out) >= maxEvidence { + return + } + bucket := start / (2 * evidenceContext) + if seen[bucket] { + return + } + seen[bucket] = true + e := excerpt(text, start, end) + if matched { + e.Matched = evidence.Truncate(evidence.Clean(strings.ToValidUTF8(text[start:end], "")), maxExcerpt) + } + out = append(out, e) + } + for _, f := range frames { + d := f.MatchDetail + if d == nil || d.MatcherValue == "" { + continue + } + if strings.HasPrefix(d.MatcherType, "regexp") { + if re, err := regexp.Compile("(?i)" + d.MatcherValue); err == nil { + if loc := re.FindStringIndex(text); loc != nil { + add(loc[0], loc[1], true) + } + } + continue + } + v := strings.ToLower(d.MatcherValue) + if i := strings.Index(lower, v); i >= 0 { + add(i, i+len(v), true) + } + } + for _, f := range frames { + n := strings.ToLower(f.Name) + if utf8.RuneCountInString(n) < 3 { + continue + } + for from, found := 0, 0; found < 2; found++ { + i := indexWord(lower[from:], n) + if i < 0 { + break + } + add(from+i, from+i+len(n), false) + from += i + len(n) + } + } + return out +} + +func excerpt(text string, start, end int) matchExcerpt { + where := "body" + head := strings.Index(text, "\r\n\r\n") + if head < 0 { + head = strings.Index(text, "\n\n") + } + if head < 0 || start < head { + where = "header" + } + lo, hi := start-evidenceContext, end+evidenceContext + if lo < 0 { + lo = 0 + } + if hi > len(text) { + hi = len(text) + } + return matchExcerpt{Where: where, Text: evidence.Truncate(evidence.Clean(strings.ToValidUTF8(text[lo:hi], "")), maxExcerpt)} +} + +// containsWord matches ASCII keys on word boundaries ("acti" must not match +// "action"); CJK keys have no word boundaries and match as substrings. +func containsWord(haystack, key string) bool { return indexWord(haystack, key) >= 0 } + +func indexWord(haystack, key string) int { + if key == "" { + return -1 + } + if utf8.RuneCountInString(key) != len(key) { + return strings.Index(haystack, key) + } + for from := 0; ; { + i := strings.Index(haystack[from:], key) + if i < 0 { + return -1 + } + start, end := from+i, from+i+len(key) + if (start == 0 || !isWordByte(haystack[start-1])) && (end == len(haystack) || !isWordByte(haystack[end])) { + return start + } + from = start + 1 + } +} + +func isWordByte(b byte) bool { + return b == '_' || b >= '0' && b <= '9' || b >= 'a' && b <= 'z' || b >= 'A' && b <= 'Z' +} diff --git a/judge/gen/gen.go b/judge/gen/gen.go index e7d1a2b..b0584a8 100644 --- a/judge/gen/gen.go +++ b/judge/gen/gen.go @@ -1,6 +1,7 @@ -// Package gen builds native fingerprints from labelled HTTP responses. It -// uses a judge.Judge only to pick a product name and versions; rules are -// derived and validated by code against the rule engine. +// Package gen builds native fingerprints from labelled HTTP responses for a +// product a person names. It uses a judge.Judge only for versions the +// positives do not carry; rules are derived and validated by code against +// the rule engine. package gen import ( @@ -36,13 +37,11 @@ type generatorSample struct { probes map[string][]byte } -// New creates a fingerprint generator using j for name and version -// selection. j may be nil when Name is given and every positive carries its -// version (PositiveVersion). +// New creates a fingerprint generator using j for version selection. j may +// be nil when every positive carries its version (PositiveVersion). func New(j *judge.Judge) *Generator { return &Generator{judge: j} } -// Name supplies a known product name. Without it Generate selects a name -// from the positive responses. +// Name sets the product the rules detect; Generate requires it. func (g *Generator) Name(name string) *Generator { g.name = strings.TrimSpace(name) return g @@ -115,43 +114,7 @@ func (g *Generator) Generate(ctx context.Context) (*fingerlib.Finger, error) { } name := g.name if name == "" { - if g.judge == nil { - return nil, fmt.Errorf("gen: a Judge or Name is required to select a product name") - } - counts := map[string]int{} - ranks := map[string]int{} - labels := map[string]string{} - for _, sample := range g.positive { - names, err := g.judge.SuggestNames(ctx, sample.raw) - if err != nil { - return nil, err - } - for rank, candidate := range names { - key := evidence.NormalizeName(candidate) - counts[key]++ - ranks[key] += len(names) - rank - if labels[key] == "" { - labels[key] = candidate - } - } - } - var keys []string - for key := range counts { - keys = append(keys, key) - } - sort.Slice(keys, func(a, b int) bool { - if counts[keys[a]] != counts[keys[b]] { - return counts[keys[a]] > counts[keys[b]] - } - if ranks[keys[a]] != ranks[keys[b]] { - return ranks[keys[a]] > ranks[keys[b]] - } - return keys[a] < keys[b] - }) - if len(keys) == 0 || counts[keys[0]] != len(g.positive) { - return nil, fmt.Errorf("gen: no product name supported by every positive sample; provide Name") - } - name = labels[keys[0]] + return nil, fmt.Errorf("gen: Name is required: a person names the product, the provider never does") } versions := make([]string, len(g.positive)) @@ -248,14 +211,14 @@ func (g *Generator) Validate(f *fingerlib.Finger) error { func (g *Generator) validateSamples() error { for _, sample := range append(append([]generatorSample(nil), g.positive...), g.negative...) { - if _, err := judge.NewPage(sample.raw); err != nil { + if _, err := evidence.NewPage(sample.raw); err != nil { return fmt.Errorf("gen: invalid sample: %w", err) } for request, response := range sample.probes { if request == "" { return fmt.Errorf("gen: probe has empty send_data") } - if _, err := judge.NewPage(response); err != nil { + if _, err := evidence.NewPage(response); err != nil { return fmt.Errorf("gen: invalid probe response: %w", err) } } @@ -292,7 +255,7 @@ type ruleCandidate struct { } func candidates(raw []byte) []ruleCandidate { - p, err := judge.NewPage(raw) + p, err := evidence.NewPage(raw) if err != nil { return nil } diff --git a/judge/gen/gen_test.go b/judge/gen/gen_test.go index 9299d6c..130aa53 100644 --- a/judge/gen/gen_test.go +++ b/judge/gen/gen_test.go @@ -10,21 +10,24 @@ import ( fingerlib "github.com/chainreactors/fingers/fingers" "github.com/chainreactors/fingers/judge" + "github.com/chainreactors/utils/jev" ) -// names answers yes for the product named in backticks, no otherwise. +// names picks its product whenever it is offered as a choice. type names string func (names) ID() string { return "names" } -func (n names) Judge(_ context.Context, _ interface{}, qs map[string]judge.Question) (map[string]judge.Answer, error) { - out := map[string]judge.Answer{} +func (n names) Judge(_ context.Context, _ interface{}, qs map[string]jev.Claim) (map[string]jev.Ruling, error) { + out := map[string]jev.Ruling{} for key, q := range qs { - if strings.Contains(q.Instructions, "`"+string(n)+"`") { - out[key] = judge.Answer{Yes: 0.95} - } else { - out[key] = judge.Answer{Yes: 0.05} + choice := jev.OptionInsufficient + for option := range q.Options { + if strings.EqualFold(option, string(n)) { + choice = option + } } + out[key] = jev.Ruling{Option: choice, Confidence: 0.95} } return out, nil } @@ -92,13 +95,14 @@ func TestGeneratorActiveAndProbeWith(t *testing.T) { } } -func TestGeneratorSelectsNameByDefault(t *testing.T) { +// A person names the product: without Name, Generate fails rather than ask +// the provider for one. +func TestGeneratorRequiresName(t *testing.T) { positive := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nOrion

Orion console

") negative := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nOther") j := judge.New(names("Orion")) - f, err := New(j).Positive(positive).Negative(negative).Generate(context.Background()) - if err != nil || f.Name != "Orion" { - t.Fatalf("auto name = %+v, %v", f, err) + if f, err := New(j).Positive(positive).Negative(negative).Generate(context.Background()); err == nil { + t.Fatalf("generated %+v without a name", f) } } diff --git a/judge/inspect.go b/judge/inspect.go new file mode 100644 index 0000000..1746ad7 --- /dev/null +++ b/judge/inspect.go @@ -0,0 +1,122 @@ +package judge + +import ( + "context" + "fmt" + "sort" + + "github.com/chainreactors/fingers/common" + "github.com/chainreactors/fingers/judge/internal/evidence" + "github.com/chainreactors/utils/jev" +) + +// Inspect evaluates presence and versions once and returns every annotated hit. +// Call Accepted on the result to obtain the reporting view. Inputs are never changed. +// On error, the copied baseline and all facts established so far remain available; +// unjudged claims have no outcome. Each provider batch is validated before application. +func (j *Judge) Inspect(ctx context.Context, raw []byte, hits common.Frameworks) (common.Frameworks, error) { + working := cloneFrameworks(hits) + for _, f := range working { + if f != nil { + f.Judge = nil + } + } + if err := ctx.Err(); err != nil { + return working, err + } + if !probability(j.MinConfidence) { + return working, fmt.Errorf("invalid minimum confidence") + } + p, err := evidence.NewPage(raw) + if err != nil { + return working, err + } + if err := j.presence(ctx, p, working); err != nil { + return working, err + } + return working, j.versions(ctx, p, versionOrder(working.Accepted())...) +} + +// Version selects one extracted version using the same claims as Inspect. +func (j *Judge) Version(ctx context.Context, raw []byte, f *common.Framework) (string, error) { + if f == nil { + return "", nil + } + if err := ctx.Err(); err != nil { + return "", err + } + if !probability(j.MinConfidence) { + return "", fmt.Errorf("invalid minimum confidence") + } + if f.Attributes != nil && f.Version != "" { + return f.Version, nil + } + p, err := evidence.NewPage(raw) + if err != nil { + return "", err + } + copied := cloneFramework(f) + err = j.versions(ctx, p, copied) + if copied.Attributes == nil { + return "", err + } + return copied.Version, err +} + +// versionOrder is the order products are versioned in, as maxVersioned caps +// them: confirmed products first, then undecided, then unjudged, by name. +func versionOrder(frames common.Frameworks) []*common.Framework { + rank := func(f *common.Framework) int { + switch { + case f.Judge == nil: + return 2 + case f.Judge.Outcome == jev.Holds.String(): + return 0 + } + return 1 + } + list := frames.List() + sort.Slice(list, func(a, b int) bool { + if ra, rb := rank(list[a]), rank(list[b]); ra != rb { + return ra < rb + } + return list[a].Name < list[b].Name + }) + return list +} + +func cloneFrameworks(frames common.Frameworks) common.Frameworks { + working := make(common.Frameworks, len(frames)) + for name, f := range frames { + working[name] = cloneFramework(f) + } + return working +} + +func cloneFramework(f *common.Framework) *common.Framework { + if f == nil { + return nil + } + copy := *f + copy.Tags = append([]string(nil), f.Tags...) + if f.Froms != nil { + copy.Froms = make(map[common.From]bool, len(f.Froms)) + for from, present := range f.Froms { + copy.Froms[from] = present + } + } + if f.Attributes != nil { + attrs := *f.Attributes + copy.Attributes = &attrs + } + if f.Judge != nil { + judgement := *f.Judge + judgement.Evidence = append([]string(nil), f.Judge.Evidence...) + copy.Judge = &judgement + } + if f.MatchDetail != nil { + detail := *f.MatchDetail + copy.MatchDetail = &detail + } + return © +} diff --git a/judge/internal/evidence/evidence.go b/judge/internal/evidence/evidence.go index 15c359c..543b9db 100644 --- a/judge/internal/evidence/evidence.go +++ b/judge/internal/evidence/evidence.go @@ -4,8 +4,10 @@ package evidence import ( + "net/http" "regexp" "sort" + "strconv" "strings" "unicode" "unicode/utf8" @@ -65,15 +67,31 @@ func UsableKey(k string) bool { } // GenericName reports whether a normalized name is a generic interface word -// ("login", "控制台") rather than a product. +// ("login", "控制台") or an HTTP status ("404notfound") rather than a product. func GenericName(key string) bool { switch key { case "login", "signin", "welcome", "dashboard", "admin", "console", "home", "index", "server", "error", "notfound", "password", "username", "management", "system", "登录", "首页", "管理", "用户", "密码", "控制台": return true } - return false + if len(key) > 3 { + if _, err := strconv.Atoi(key[:3]); err == nil { + key = key[3:] + } + } + return statusTexts[key] } +// statusTexts are the normalized HTTP reason phrases ("notfound"). +var statusTexts = func() map[string]bool { + m := map[string]bool{} + for code := 100; code < 600; code++ { + if text := http.StatusText(code); text != "" { + m[NormalizeName(text)] = true + } + } + return m +}() + // Names returns up to 20 product name candidates found in a response, most // telling source first: generator meta, product headers, title parts, asset // path words, then capitalized words and CJK runs of the visible text. diff --git a/judge/internal/evidence/evidence_test.go b/judge/internal/evidence/evidence_test.go new file mode 100644 index 0000000..062d859 --- /dev/null +++ b/judge/internal/evidence/evidence_test.go @@ -0,0 +1,19 @@ +package evidence + +import "testing" + +func TestGenericName(t *testing.T) { + for key, generic := range map[string]bool{ + NormalizeName("404 Not Found"): true, + NormalizeName("403 Forbidden"): true, + NormalizeName("502 Bad Gateway"): true, + NormalizeName("Forbidden"): true, + NormalizeName("login"): true, + NormalizeName("SearXNG"): false, + NormalizeName("404 Tools"): false, + } { + if GenericName(key) != generic { + t.Errorf("GenericName(%q) = %v", key, !generic) + } + } +} diff --git a/judge/internal/evidence/page.go b/judge/internal/evidence/page.go new file mode 100644 index 0000000..a2ccc0c --- /dev/null +++ b/judge/internal/evidence/page.go @@ -0,0 +1,134 @@ +package evidence + +import ( + "bufio" + "bytes" + "io" + "net/http" + "regexp" + "strings" + "unicode/utf8" +) + +// Page is the compact, named view of one HTTP response that providers judge +// (serialized as the request state). A provider only judges the evidence it +// is given and loses accuracy on large states full of irrelevant detail, so +// the fields below are the ones that carry fingerprint signal. +type Page struct { + Status string `json:"status"` + Headers map[string]string `json:"headers,omitempty"` + Cookies []string `json:"cookie_names,omitempty"` + Title string `json:"title,omitempty"` + Generator string `json:"meta_generator,omitempty"` + Description string `json:"meta_description,omitempty"` + Scripts []string `json:"script_src,omitempty"` + Styles []string `json:"stylesheet_href,omitempty"` + InlineHints []string `json:"inline_script_starts,omitempty"` // first chars of inline scripts, e.g. "(function(w,d,s,l,i){ ... GTM" + Comments []string `json:"html_comments,omitempty"` + Forms []string `json:"form_inputs,omitempty"` + Text string `json:"visible_text,omitempty"` + + Raw []byte `json:"-"` + Header http.Header `json:"-"` +} + +var ( + // Headers that carry no fingerprint signal, or only noise; every other + // header is kept, since products announce themselves in custom headers + // ("product: Z-BlogPHP 1.7.4", "X-Jenkins", "kbn-name"). + skipHeaders = map[string]bool{"Date": true, "Content-Length": true, "Content-Type": true, "Expires": true, "Cache-Control": true, + "Pragma": true, "Etag": true, "Last-Modified": true, "Age": true, "Connection": true, "Keep-Alive": true, "Accept-Ranges": true, + "Vary": true, "Set-Cookie": true, "Content-Security-Policy": true, "Content-Security-Policy-Report-Only": true, "Report-To": true, + "Nel": true, "Permissions-Policy": true, + // per-request values: no signal, and they would defeat the cache + "X-Request-Id": true, "X-Amzn-Trace-Id": true, "Cf-Ray": true, "X-Runtime": true, "Server-Timing": true} + + reDescription = regexp.MustCompile(`(?is)]+name=["']description["'][^>]*content=["']([^"']+)|]+content=["']([^"']+)["'][^>]*name=["']description["']`) + reScript = regexp.MustCompile(`(?is)]+src=["']([^"']+)`) + reStyle = regexp.MustCompile(`(?is)]+rel=["']?stylesheet[^>]*href=["']([^"']+)|]+href=["']([^"']+)["'][^>]*rel=["']?stylesheet`) + reInline = regexp.MustCompile(`(?is)]*)?>(.*?)`) + reComment = regexp.MustCompile(`(?s)`) + reInput = regexp.MustCompile(`(?is)]*>`) + reAttrName = regexp.MustCompile(`(?is)\b(?:name|id)=["']([^"']+)`) + reAttrType = regexp.MustCompile(`(?is)\btype=["']([^"']+)`) + reDropBlock = regexp.MustCompile(`(?is)<(script|style|noscript)[^>]*>.*?`) + reTag = regexp.MustCompile(`(?s)<[^>]+>`) +) + +const ( + maxHeaderValue = 160 + maxText = 1500 // visible text runes +) + +// NewPage parses a raw HTTP response (head and body, as rule engines take it). +func NewPage(raw []byte) (*Page, error) { + resp, err := http.ReadResponse(bufio.NewReader(bytes.NewReader(raw)), nil) + if err != nil { + return nil, err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + s := &Page{Status: resp.Status, Headers: map[string]string{}, Raw: raw, Header: resp.Header.Clone()} + for k, vs := range resp.Header { + if skipHeaders[k] || strings.HasPrefix(k, "X-Crawler-") || len(vs) == 0 { + continue + } + s.Headers[k] = Truncate(strings.Join(vs, ", "), maxHeaderValue) + } + for _, c := range resp.Cookies() { + s.Cookies = append(s.Cookies, c.Name) + } + html := string(body) + if m := Title.FindStringSubmatch(html); m != nil { + s.Title = clean(m[1]) + } + if m := Generator.FindStringSubmatch(html); m != nil { + s.Generator = m[1] + m[2] + } + if m := reDescription.FindStringSubmatch(html); m != nil { + s.Description = Truncate(clean(m[1]+m[2]), 300) + } + for _, m := range reScript.FindAllStringSubmatch(html, 8) { + s.Scripts = append(s.Scripts, m[1]) + } + for _, m := range reStyle.FindAllStringSubmatch(html, 8) { + s.Styles = append(s.Styles, m[1]+m[2]) + } + for _, m := range reInline.FindAllStringSubmatch(html, -1) { + if code := clean(m[1]); len(code) > 20 && len(s.InlineHints) < 5 { + s.InlineHints = append(s.InlineHints, Truncate(code, 100)) + } + } + for _, m := range reComment.FindAllStringSubmatch(html, -1) { + if c := clean(m[1]); len(c) > 3 && len(s.Comments) < 5 && !strings.HasPrefix(c, "[if") { + s.Comments = append(s.Comments, Truncate(c, 80)) + } + } + for _, in := range reInput.FindAllString(html, 12) { + name, typ := "", "text" + if m := reAttrName.FindStringSubmatch(in); m != nil { + name = m[1] + } + if m := reAttrType.FindStringSubmatch(in); m != nil { + typ = m[1] + } + if name != "" { + s.Forms = append(s.Forms, name+":"+typ) + } + } + text := clean(reTag.ReplaceAllString(reDropBlock.ReplaceAllString(html, " "), " ")) + s.Text = Truncate(text, maxText) + return s, nil +} + +func clean(s string) string { return Clean(s) } + +func Truncate(s string, max int) string { + if utf8.RuneCountInString(s) <= max { + return s + } + return string([]rune(s)[:max]) + "…" +} diff --git a/judge/jev/jev.go b/judge/jev/jev.go deleted file mode 100644 index 7fc7374..0000000 --- a/judge/jev/jev.go +++ /dev/null @@ -1,194 +0,0 @@ -// Package jev is the TypeSafe Jev (System One) provider for package judge. -// It translates judge questions to Jev's wire format (binary -> noul) and -// sends them; everything else lives in judge. -// -// API reference: https://docs.typesafe.ai/api.md -package jev - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "os" - "sync/atomic" - "time" - - "github.com/chainreactors/fingers/judge" -) - -const ( - DefaultEndpoint = "https://api.typesafe.ai/v1/systemone" - // Pin a versioned model: the calibration below was measured on this one. - DefaultModel = "jev-1.13.0" - EnvAPIKey = "TYPESAFE_API_KEY" -) - -// Provider sends judge questions to Jev. -type Provider struct { - Endpoint string - Model string - APIKey string - HTTP *http.Client - MaxRetries int // on 429 / 529, with exponential backoff - - // InputTokens sent so far; read with atomic.LoadInt64. - InputTokens int64 -} - -// New reads the API key from TYPESAFE_API_KEY when apiKey is empty. -func New(apiKey string) (*Provider, error) { - if apiKey == "" { - apiKey = os.Getenv(EnvAPIKey) - } - if apiKey == "" { - return nil, fmt.Errorf("%s is not set", EnvAPIKey) - } - return &Provider{ - Endpoint: DefaultEndpoint, - Model: DefaultModel, - APIKey: apiKey, - HTTP: &http.Client{Timeout: 30 * time.Second}, - MaxRetries: 3, - }, nil -} - -// NewJudge is judge.New(New(apiKey)). -func NewJudge(apiKey string) (*judge.Judge, error) { - p, err := New(apiKey) - if err != nil { - return nil, err - } - return judge.New(p), nil -} - -func (p *Provider) ID() string { return "jev/" + p.Model } - -// Calibration: on jev-1.13.0, a Noul of 0.5 separated real from false hits -// on labelled pages, and 42 of 46 versions picked at >= 0.9 were right. -func (p *Provider) Calibration() (float64, float64) { return 0.5, 0.9 } - -type question struct { - Type string `json:"type"` - Instructions string `json:"instructions"` - Criteria interface{} `json:"criteria,omitempty"` -} - -type request struct { - State interface{} `json:"state"` - Model string `json:"model"` - Questions map[string]question `json:"questions"` -} - -type answer struct { - Type string `json:"type"` - Noul float64 `json:"noul,omitempty"` - Choice string `json:"choice,omitempty"` - Score float64 `json:"score,omitempty"` - Probabilities map[string]float64 `json:"probabilities,omitempty"` - Confidence float64 `json:"confidence,omitempty"` -} - -type response struct { - Model string `json:"model"` - Answers map[string]answer `json:"answers"` - Usage struct { - InputTokens int `json:"input_tokens"` - OutputTokens int `json:"output_tokens"` - } `json:"usage"` -} - -func toJev(q judge.Question) question { - out := question{Instructions: q.Instructions} - switch q.Type { - case judge.TypeBinary: - out.Type = "noul" - if len(q.Options) > 0 { - out.Criteria = q.Options - } - case judge.TypeChoice: - out.Type = "choice" - opts := make(map[string]interface{}, len(q.Options)) - for k, v := range q.Options { - if v == "" { - opts[k] = nil // Jev's "no description" - } else { - opts[k] = v - } - } - out.Criteria = opts - case judge.TypeScore: - out.Type = "score" - out.Criteria = q.Levels - } - return out -} - -type APIError struct { - Status int - Body string -} - -func (e *APIError) Error() string { return fmt.Sprintf("typesafe: http %d: %s", e.Status, e.Body) } - -func (p *Provider) Judge(ctx context.Context, state interface{}, questions map[string]judge.Question) (map[string]judge.Answer, error) { - req := request{State: state, Model: p.Model, Questions: make(map[string]question, len(questions))} - for k, q := range questions { - req.Questions[k] = toJev(q) - } - body, err := json.Marshal(req) - if err != nil { - return nil, err - } - backoff := time.Second - for attempt := 0; ; attempt++ { - resp, err := p.do(ctx, body) - if err == nil { - atomic.AddInt64(&p.InputTokens, int64(resp.Usage.InputTokens)) - out := make(map[string]judge.Answer, len(resp.Answers)) - for k, a := range resp.Answers { - out[k] = judge.Answer{Yes: a.Noul, Choice: a.Choice, Score: a.Score, Probabilities: a.Probabilities, Confidence: a.Confidence} - } - return out, nil - } - apiErr, ok := err.(*APIError) - if !ok || (apiErr.Status != 429 && apiErr.Status != 529) || attempt >= p.MaxRetries { - return nil, err - } - select { - case <-ctx.Done(): - return nil, ctx.Err() - case <-time.After(backoff): - } - backoff *= 2 - } -} - -func (p *Provider) do(ctx context.Context, body []byte) (*response, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodPost, p.Endpoint, bytes.NewReader(body)) - if err != nil { - return nil, err - } - req.Header.Set("Authorization", "Bearer "+p.APIKey) - req.Header.Set("Content-Type", "application/json") - - resp, err := p.HTTP.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - raw, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - if resp.StatusCode != http.StatusOK { - return nil, &APIError{Status: resp.StatusCode, Body: string(raw)} - } - var out response - if err := json.Unmarshal(raw, &out); err != nil { - return nil, fmt.Errorf("typesafe: decode response: %w", err) - } - return &out, nil -} diff --git a/judge/jev/jev_test.go b/judge/jev/jev_test.go deleted file mode 100644 index e062986..0000000 --- a/judge/jev/jev_test.go +++ /dev/null @@ -1,53 +0,0 @@ -package jev - -import ( - "context" - "encoding/json" - "net/http" - "net/http/httptest" - "testing" - - "github.com/chainreactors/fingers/judge" -) - -func TestProviderWireFormatAndRetry(t *testing.T) { - calls := 0 - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - calls++ - if r.Header.Get("Authorization") != "Bearer k" { - t.Errorf("auth header: %q", r.Header.Get("Authorization")) - } - if calls == 1 { - w.WriteHeader(529) - return - } - var req struct { - Model string - Questions map[string]map[string]interface{} - } - json.NewDecoder(r.Body).Decode(&req) - b, c := req.Questions["b"], req.Questions["c"] - criteria, _ := c["criteria"].(map[string]interface{}) - if req.Model != DefaultModel || b["type"] != "noul" || c["type"] != "choice" || criteria["x"] != nil || criteria["y"] != "why" { - t.Errorf("request: %+v", req) - } - w.Write([]byte(`{"model":"jev-1.13.0","answers":{"b":{"type":"noul","noul":0.9},"c":{"type":"choice","choice":"y","confidence":0.8}},"usage":{"input_tokens":10,"output_tokens":1}}`)) - })) - defer srv.Close() - - p, _ := New("k") - p.Endpoint = srv.URL - answers, err := p.Judge(context.Background(), "state", map[string]judge.Question{ - "b": judge.Binary("?"), - "c": judge.Choice("?", map[string]string{"x": "", "y": "why"}), - }) - if err != nil { - t.Fatal(err) - } - if calls != 2 || answers["b"].Yes != 0.9 || answers["c"].Choice != "y" || p.InputTokens != 10 { - t.Fatalf("calls=%d answers=%+v tokens=%d", calls, answers, p.InputTokens) - } - if j := judge.New(p); j.Threshold != 0.5 || j.VersionConfidence != 0.9 || p.ID() != "jev/jev-1.13.0" { - t.Fatalf("calibration %v %v id %s", j.Threshold, j.VersionConfidence, p.ID()) - } -} diff --git a/judge/judge.go b/judge/judge.go index f45d2d4..d08f5af 100644 --- a/judge/judge.go +++ b/judge/judge.go @@ -2,243 +2,40 @@ package judge import ( "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "fmt" "math" - "math/bits" - "sort" - "strings" - "sync" - "sync/atomic" -) - -// Judge puts a Provider to work: it answers questions from the cache where -// it can, merges concurrent rounds asking the same questions about similar -// pages, and holds the thresholds the capabilities decide with. One Judge is -// meant to be shared by every page of a scan. -type Judge struct { - Provider Provider - // Cache keeps answers per question, so a question asked again about the - // same or a similar page is not sent again, however questions are grouped - // into requests. New sets an in-memory cache; nil disables it. Concurrent - // requests are merged either way. - Cache Cache - // Known finds fingerprint names that occur in a page (typically - // NewRetriever(engine.Names())). Refine and Inspect judge those the rules - // did not report, and add the confirmed ones as Recalled. nil disables recall. - Known *Retriever - // Threshold is the Yes probability that counts as "yes"; VersionConfidence - // the confidence at which a picked version is adopted. - Threshold, VersionConfidence float64 - // SimilarDistance is how many of the 64 signature bits two pages with the - // same title may differ in to share answers; 0 shares answers only between - // identical signatures. See DefaultSimilarDistance. - SimilarDistance int - - // Totals, for cost monitoring; read with atomic.LoadInt64. Requests went - // to the provider; CacheHits are rounds answered without one. - Requests, CacheHits int64 - - mu sync.Mutex - flights map[string][]*flight -} -const ( - // DefaultCacheSize is the number of answers the cache New creates holds. - DefaultCacheSize = 1 << 16 - // DefaultSimilarDistance was measured with cmd/judgeeval on 1070 real - // pages: every answer reused at distance <= 1 matched the page's own - // answer (159/159), at 2 it was 98.5%, at 3 96.9%. - DefaultSimilarDistance = 1 + "github.com/chainreactors/fingers/judge/internal/evidence" + "github.com/chainreactors/utils/jev" ) -// New returns a Judge with an in-memory cache and the provider's -// calibration, or 0.5 / 0.9 for providers that report none. -func New(p Provider) *Judge { - j := &Judge{Provider: p, Cache: NewMemoryCache(DefaultCacheSize), Threshold: 0.5, VersionConfidence: 0.9, SimilarDistance: DefaultSimilarDistance} - if c, ok := p.(Calibrated); ok { - j.Threshold, j.VersionConfidence = c.Calibration() - } - return j +// Judge reviews fingerprint hits through jev claims. Configure it before +// sharing it across a scan. Caching, request accounting and rate limiting +// wrap its Provider (see jev.Cached). +type Judge struct { + Provider jev.Provider + MinConfidence float64 // below it a ruling resolves to Insufficient + DropInsufficient bool // also reject hits the evidence decides neither way } -// Ask asks questions about state, which must match exactly for cached -// answers to apply. Yes, Choose and Score are the one-question forms. -func (j *Judge) Ask(ctx context.Context, state interface{}, questions map[string]Question) (map[string]Answer, error) { - scope, err := json.Marshal(state) - if err != nil { - return nil, err - } - return j.ask(ctx, state, questions, scope, 0) +// New reviews hits with p at jev.DefaultMinConfidence. +func New(p jev.Provider) *Judge { + return &Judge{Provider: p, MinConfidence: jev.DefaultMinConfidence} } -// flight is a request in progress; rounds asking the same questions about a -// similar page wait for it instead of sending their own. -type flight struct { - sig uint64 - done chan struct{} - answers map[string]Answer // by cache key; nil if the request failed -} +func NormalizeName(name string) string { return evidence.NormalizeName(name) } -// ask answers questions from the cache where it can and sends the rest in -// one request. A cached answer applies when the question, the scope (what -// must match exactly besides the question) and the model are equal and the -// page signature sig is similar. -func (j *Judge) ask(ctx context.Context, state interface{}, questions map[string]Question, scope []byte, sig uint64) (map[string]Answer, error) { - answers := make(map[string]Answer, len(questions)) - keys := make(map[string]string, len(questions)) // question key -> cache key - missing := map[string]Question{} - var pending []string - for k, q := range questions { - qj, err := json.Marshal(q) - if err != nil { - return nil, err - } - h := sha256.New() - for _, part := range [][]byte{[]byte(j.Provider.ID()), scope, qj} { - h.Write(part) - h.Write([]byte{0}) - } - ck := hex.EncodeToString(h.Sum(nil)) - keys[k] = ck - if a, ok := j.cached(ck, sig); ok { - answers[k] = a - continue - } - missing[k] = q - pending = append(pending, ck) - } - if len(missing) == 0 { - atomic.AddInt64(&j.CacheHits, 1) - return answers, nil - } - sort.Strings(pending) - fk := strings.Join(pending, ",") - for { - f, leader := j.join(fk, sig) - if !leader { - select { - case <-ctx.Done(): - return nil, ctx.Err() - case <-f.done: - } - if f.answers == nil { - continue // the request we waited on failed: try again, possibly as the leader - } - for k := range missing { - if a, ok := f.answers[keys[k]]; ok { - answers[k] = a - } - } - atomic.AddInt64(&j.CacheHits, 1) - return answers, nil - } - got, err := j.Provider.Judge(ctx, state, missing) - if err == nil { - for k, q := range missing { - a, ok := got[k] - if !ok { - err = fmt.Errorf("judge: provider omitted answer %q", k) - break - } - if err = validateAnswer(q, a); err != nil { - err = fmt.Errorf("judge: answer %q: %w", k, err) - break - } - } - } - var byKey map[string]Answer - if err == nil { - atomic.AddInt64(&j.Requests, 1) - byKey = make(map[string]Answer, len(got)) - for k, a := range got { - answers[k] = a - if ck, ok := keys[k]; ok { - byKey[ck] = a - if j.Cache != nil { - if v, err := json.Marshal(a); err == nil { - j.Cache.Put(ck, sig, v) - } - } - } - } - } - j.land(fk, f, byKey) - if err != nil { - return nil, err - } - return answers, nil - } +func (j *Judge) judge(ctx context.Context, state interface{}, claims map[string]jev.Claim) (map[string]jev.Ruling, error) { + return jev.Judge(ctx, j.Provider, state, claims) } -func validateAnswer(q Question, a Answer) error { - validProbability := func(value float64) bool { - return !math.IsNaN(value) && !math.IsInf(value, 0) && value >= 0 && value <= 1 - } - switch q.Type { - case TypeBinary: - if !validProbability(a.Yes) { - return fmt.Errorf("invalid yes probability %v", a.Yes) - } - case TypeChoice: - if _, ok := q.Options[a.Choice]; !ok { - return fmt.Errorf("unknown choice %q", a.Choice) - } - if !validProbability(a.Confidence) { - return fmt.Errorf("invalid confidence %v", a.Confidence) - } - case TypeScore: - if !validProbability(a.Score) { - return fmt.Errorf("invalid score %v", a.Score) - } - default: - return fmt.Errorf("unknown question type %q", q.Type) - } - return nil +// validRuling checks a single claim and ruling, including code-built ones. +func validRuling(c jev.Claim, r jev.Ruling) error { + return jev.ValidateRulings(map[string]jev.Claim{"claim": c}, map[string]jev.Ruling{"claim": r}) } -func (j *Judge) cached(key string, sig uint64) (Answer, bool) { - var a Answer - if j.Cache == nil { - return a, false - } - v, ok := j.Cache.Get(key, sig, j.SimilarDistance) - return a, ok && json.Unmarshal(v, &a) == nil -} - -func (j *Judge) join(key string, sig uint64) (*flight, bool) { - j.mu.Lock() - defer j.mu.Unlock() - for _, f := range j.flights[key] { - if bits.OnesCount64(f.sig^sig) <= j.SimilarDistance { - return f, false - } - } - if j.flights == nil { - j.flights = map[string][]*flight{} - } - f := &flight{sig: sig, done: make(chan struct{})} - j.flights[key] = append(j.flights[key], f) - return f, true -} +// insufficientDescription describes the abstention every claim offers. +const insufficientDescription = "The evidence decides neither way." -func (j *Judge) land(key string, f *flight, answers map[string]Answer) { - j.mu.Lock() - fs := j.flights[key] - for i, x := range fs { - if x == f { - fs = append(fs[:i], fs[i+1:]...) - break - } - } - if len(fs) == 0 { - delete(j.flights, key) - } else { - j.flights[key] = fs - } - j.mu.Unlock() - f.answers = answers - close(f.done) +func probability(v float64) bool { + return !math.IsNaN(v) && !math.IsInf(v, 0) && v >= 0 && v <= 1 } diff --git a/judge/judge_test.go b/judge/judge_test.go index 1f73f32..4443a45 100644 --- a/judge/judge_test.go +++ b/judge/judge_test.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "math/bits" "sort" "strings" "sync" @@ -14,6 +13,8 @@ import ( "time" "github.com/chainreactors/fingers/common" + "github.com/chainreactors/fingers/judge/internal/evidence" + "github.com/chainreactors/utils/jev" ) const jenkinsRaw = "HTTP/1.1 200 OK\r\nServer: nginx/1.24.0\r\nX-Jenkins: 2.401.3\r\nSet-Cookie: JSESSIONID.1=x; Path=/\r\nContent-Type: text/html\r\n\r\n" + @@ -22,7 +23,7 @@ const jenkinsRaw = "HTTP/1.1 200 OK\r\nServer: nginx/1.24.0\r\nX-Jenkins: 2.401. `` func TestNewPage(t *testing.T) { - s, err := NewPage([]byte(jenkinsRaw)) + s, err := evidence.NewPage([]byte(jenkinsRaw)) if err != nil { t.Fatal(err) } @@ -35,43 +36,39 @@ func TestNewPage(t *testing.T) { if got := extractVersions([]byte(jenkinsRaw), 5); len(got) != 2 || got[0].Value != "1.24.0" || got[1].Value != "2.401.3" { t.Fatalf("versions: %v", got) } - if got := NewRetriever([]string{"jenkins", "oa", "tomcat"}).Find(s.haystack(), 5); len(got) != 1 || got[0] != "jenkins" { - t.Fatalf("retriever: %v", got) + if !declares(s, "jenkins") || !declares(s, "nginx") || declares(s, "wordpress") || declares(s, "jenk") { + t.Fatal("declares") } } // mock is a Provider answering from a table keyed by question kind and the // product named in backticks; it records the question keys of each call. type mock struct { - mu sync.Mutex - delay time.Duration - calls int64 - requests [][]string - fail bool - versionP float64 - present map[string]float64 - layer map[string]string - versions map[string]string // product -> version picked, if among the options - primary string - lastState map[string]interface{} + mu sync.Mutex + delay time.Duration + calls int64 + requests [][]string + fail bool + confidence float64 + verdicts map[string]string // product -> presence verdict + versions map[string]string // product -> version picked, if among the options + coverage string + lastState map[string]interface{} } func newMock() *mock { return &mock{ - versionP: 0.95, - present: map[string]float64{"nginx": 0.1, "wordpress": 0.05, "jenkins": 0.97, "apache tomcat": 0.8, "prototype": 0.9}, - versions: map[string]string{"jenkins": "2.401.3", "nginx": "1.24.0"}, - layer: map[string]string{"nginx": LayerServer, "wordpress": LayerNotPresent, "jenkins": LayerApplication, - "apache tomcat": LayerServer, "prototype": LayerFrontend}, - primary: "jenkins", + confidence: 0.95, + verdicts: map[string]string{"wordpress": OptionMentioned, "apache tomcat": OptionRunning, "prototype": OptionRunning}, + versions: map[string]string{"jenkins": "2.401.3", "nginx": "1.24.0"}, } } -// named is the first product named in backticks, skipping evidence names. -func named(q Question) string { - parts := strings.Split(q.Instructions, "`") +// named is the first product named in backticks, skipping state field names. +func named(q jev.Claim) string { + parts := strings.Split(q.Statement, "`") for i := 1; i < len(parts); i += 2 { - if parts[i] != "version_strings" { + if parts[i] != "version_strings" && !strings.HasPrefix(parts[i], "matches.") { return parts[i] } } @@ -80,7 +77,7 @@ func named(q Question) string { func (m *mock) ID() string { return "mock" } -func (m *mock) Judge(ctx context.Context, state interface{}, questions map[string]Question) (map[string]Answer, error) { +func (m *mock) Judge(ctx context.Context, state interface{}, questions map[string]jev.Claim) (map[string]jev.Ruling, error) { atomic.AddInt64(&m.calls, 1) time.Sleep(m.delay) m.mu.Lock() @@ -88,23 +85,20 @@ func (m *mock) Judge(ctx context.Context, state interface{}, questions map[strin if m.fail { return nil, errors.New("provider down") } - m.lastState, _ = state.(map[string]interface{}) - answers := map[string]Answer{} + data, _ := json.Marshal(state) + _ = json.Unmarshal(data, &m.lastState) + answers := map[string]jev.Ruling{} var keys []string for k, q := range questions { keys = append(keys, k) name := strings.ToLower(named(q)) switch { - case strings.HasPrefix(k, "is_"): - answers[k] = Answer{Yes: m.present[name]} - case strings.HasPrefix(k, "layer_"): - answers[k] = Answer{Choice: string(m.layer[name]), Confidence: 0.9} - case k == "primary": - answers[k] = Answer{Choice: m.primary, Confidence: 0.9} - case k == "page_kind": - answers[k] = Answer{Choice: "login", Confidence: 0.9} - case k == "generic": - answers[k] = Answer{Yes: 0.9} + case strings.HasPrefix(k, "presence_"): + v := m.verdicts[name] + if v == "" { + v = jev.OptionInsufficient + } + answers[k] = jev.Ruling{Option: v, Confidence: m.confidence} case strings.HasPrefix(k, "version_"): choice := notStated if v, ok := m.versions[name]; ok { @@ -112,9 +106,11 @@ func (m *mock) Judge(ctx context.Context, state interface{}, questions map[strin choice = v } } - answers[k] = Answer{Choice: choice, Confidence: m.versionP} + answers[k] = jev.Ruling{Option: choice, Confidence: m.confidence} + case k == "coverage": + answers[k] = jev.Ruling{Option: m.coverage, Confidence: m.confidence} default: - answers[k] = Answer{Yes: 0.7} + answers[k] = jev.Ruling{Option: jev.OptionInsufficient, Confidence: m.confidence} } } sort.Strings(keys) @@ -129,83 +125,138 @@ func testFrames() common.Frameworks { fs.Add(common.NewFramework("jenkins", common.FrameFromFingerprintHub)) fs.Add(common.NewFramework("apache-tomcat", common.FrameFromFingers)) fs.Add(common.NewFramework("apache tomcat", common.FrameFromWappalyzer)) + fs.Add(common.NewFramework("hsts", common.FrameFromWappalyzer)) return fs } -// newJudge is a Judge over m that recalls Jenkins and Prototype by name. -func newJudge(m *mock) *Judge { - j := New(m) - j.Known = NewRetriever([]string{"Jenkins", "Prototype"}) - return j -} - // bodyRaw states versions only in the body, so the provider has to pick them. const bodyRaw = "HTTP/1.1 200 OK\r\nServer: nginx\r\nSet-Cookie: JSESSIONID.1=x; Path=/\r\n\r\n" + `Sign in [Jenkins]` + `

We moved here from WordPress.

Jenkins 2.401.3, nginx 1.24.0
` -func TestRefine(t *testing.T) { +func TestInspect(t *testing.T) { m := newMock() - j := newJudge(m) + j := New(jev.Cached(m, jev.DefaultCacheSize)) all, err := j.Inspect(context.Background(), []byte(jenkinsRaw), testFrames()) if err != nil { t.Fatal(err) } - if w := all["wordpress"]; w.Judge == nil || !w.Judge.Rejected { - t.Errorf("text-only wordpress not rejected: %+v", w.Judge) + if w := all["wordpress"].Judge; w == nil || !w.Rejected || w.Verdict != OptionMentioned || w.Outcome != "refuted" || len(w.Evidence) == 0 || !strings.Contains(w.Evidence[0], "WordPress") { + t.Errorf("text-only wordpress not rejected on its evidence: %+v", w) } - if nginx := all["nginx"].Judge; nginx.Rejected || nginx.Layer != LayerServer || nginx.Confidence != 0.1 { - t.Errorf("nginx from the Server header must be kept: %+v", nginx) + for _, name := range []string{"nginx", "jenkins"} { + if v := all[name].Judge; v.Rejected || v.Verdict != OptionDeclared || v.Outcome != "holds" { + t.Errorf("%s named in a header must be declared: %+v", name, v) + } + } + if h := all["hsts"].Judge; !h.Rejected || h.Verdict != OptionAbsent || h.Outcome != "refuted" { + t.Errorf("hsts without the header must be absent: %+v", h) } if all["apache tomcat"].Judge.Duplicate == all["apache-tomcat"].Judge.Duplicate { t.Errorf("exactly one tomcat spelling must be a dup") } - if p := all["prototype"]; p == nil || !p.Judge.Recalled || !p.IsGuess() { - t.Errorf("confirmed recall not added: %+v", p) - } - // Jenkins is both a rule hit and a recall name: asked once. 5 products x 2 + primary + page_kind + generic. - if len(m.requests) != 1 || len(m.requests[0]) != 13 { + // Facts are not asked: only wordpress and tomcat are claims. + if len(m.requests) != 1 || strings.Join(m.requests[0], ",") != "presence_apachetomcat,presence_wordpress" { t.Fatalf("requests: %v", m.requests) } + claims, _ := json.Marshal(m.lastState["matches"]) + if !strings.Contains(string(claims), "We moved here from WordPress") { + t.Errorf("claim evidence missing from the state: %s", claims) + } - accepted, err := j.Refine(context.Background(), []byte(jenkinsRaw), testFrames()) + accepted, err := j.Inspect(context.Background(), []byte(jenkinsRaw), testFrames()) + accepted = accepted.Accepted() if err != nil { t.Fatal(err) } - if len(accepted) != 4 { // nginx, tomcat, jenkins, prototype + if len(accepted) != 3 { // nginx, jenkins, tomcat t.Errorf("accepted: %v", accepted) } - jenkins := accepted["jenkins"] - if !jenkins.Judge.Primary || accepted.Primary() != jenkins { - t.Errorf("jenkins primary: %+v", jenkins.Judge) + // Both versions are bound by name in the headers: taken without asking. + if accepted["jenkins"].Version != "2.401.3" || accepted["nginx"].Version != "1.24.0" { + t.Errorf("versions: jenkins %q nginx %q", accepted["jenkins"].Version, accepted["nginx"].Version) + } + // The verdicts come from the cache; tomcat is offered no version, since + // both strings are bound by name to other products. + if len(m.requests) != 1 { + t.Errorf("cached verdicts were asked again: %v", m.requests) + } +} + +// DropInsufficient decides what an undecided claim does; an answer below +// MinConfidence is undecided whatever it chose. +func TestInsufficient(t *testing.T) { + for _, c := range []struct { + on bool + confidence float64 + verdict string + rejected bool + }{ + {false, 0.95, OptionRunning, false}, + {false, 0.2, jev.OptionInsufficient, false}, + {true, 0.2, jev.OptionInsufficient, true}, + {true, 0.95, OptionRunning, false}, + } { + m := newMock() + m.confidence = c.confidence + j := New(jev.Cached(m, jev.DefaultCacheSize)) + j.DropInsufficient = c.on + all, err := j.Inspect(context.Background(), []byte(jenkinsRaw), testFrames()) + if err != nil { + t.Fatal(err) + } + tomcat := all["apache tomcat"].Judge + if all["apache-tomcat"].Judge.Duplicate { + tomcat = all["apache-tomcat"].Judge + } + outcome := "holds" + if c.verdict == jev.OptionInsufficient { + outcome = "insufficient" + } + if tomcat.Verdict != OptionRunning || tomcat.Outcome != outcome || tomcat.Rejected != c.rejected { + t.Errorf("%+v: tomcat %+v", c, tomcat) + } + } +} + +// A rule that matched text unrelated to its product is a false positive, +// removed like a mere mention, and the matched text reaches the provider. +func TestUnrelatedIsRejected(t *testing.T) { + m := newMock() + m.verdicts["apache tomcat"] = OptionUnrelated + frames := testFrames() + for _, f := range frames { + if f.Name == "apache tomcat" || f.Name == "apache-tomcat" { + f.MatchDetail = &common.MatchDetail{MatcherType: "word", MatcherValue: "jenkins"} + } } - // Both versions are bound by name in the headers: taken without asking, - // and not offered to the products they do not name. - if jenkins.Version != "2.401.3" || accepted["nginx"].Version != "1.24.0" || accepted["prototype"].Version != "" { - t.Errorf("versions: jenkins %q nginx %q prototype %q", jenkins.Version, accepted["nginx"].Version, accepted["prototype"].Version) + all, err := New(m).Inspect(context.Background(), []byte(jenkinsRaw), frames) + if err != nil { + t.Fatal(err) } - if len(m.requests) != 1 || j.CacheHits != 1 { - t.Errorf("header versions or cached verdicts were asked again: %v hits=%d", m.requests, j.CacheHits) + tomcat := all["apache tomcat"].Judge + if all["apache-tomcat"].Judge.Duplicate { + tomcat = all["apache-tomcat"].Judge } - if kind, generic, err := j.Classify(context.Background(), []byte(jenkinsRaw)); err != nil || kind != KindLogin || !generic || len(m.requests) != 1 { - t.Errorf("classify after refine: %q %v %v, requests %v", kind, generic, err, m.requests) + if tomcat.Verdict != OptionUnrelated || !tomcat.Rejected || len(tomcat.Evidence) == 0 || !strings.Contains(tomcat.Evidence[0], `matched "Jenkins"`) { + t.Fatalf("tomcat %+v", tomcat) } } // Versions stated only in the body are picked by the provider, all kept // products in one request, each from its own candidates. -func TestRefineVersionsEveryProduct(t *testing.T) { +func TestInspectVersionsEveryProduct(t *testing.T) { m := newMock() - j := newJudge(m) - accepted, err := j.Refine(context.Background(), []byte(bodyRaw), testFrames()) + m.verdicts["jenkins"] = OptionRunning + accepted, err := New(m).Inspect(context.Background(), []byte(bodyRaw), testFrames()) + accepted = accepted.Accepted() if err != nil { t.Fatal(err) } if accepted["jenkins"].Version != "2.401.3" || accepted["nginx"].Version != "1.24.0" { t.Fatalf("versions: jenkins %q nginx %q", accepted["jenkins"].Version, accepted["nginx"].Version) } - // jenkins (primary), apache tomcat and nginx (servers), prototype (frontend). - if len(m.requests) != 2 || strings.Join(m.requests[1], ",") != "version_0,version_1,version_2,version_3" { + if len(m.requests) != 2 || !strings.HasPrefix(strings.Join(m.requests[1], ","), "version_apachetomcat,version_jenkins") { t.Fatalf("requests: %v", m.requests) } if _, ok := m.lastState["version_strings"]; !ok { @@ -213,20 +264,6 @@ func TestRefineVersionsEveryProduct(t *testing.T) { } } -// One version string occurring once goes to one product only. -func TestVersionGoesToOneProduct(t *testing.T) { - m := newMock() - m.versions = map[string]string{"jenkins": "2.401.3", "nginx": "2.401.3"} - raw := strings.Replace(bodyRaw, "Jenkins 2.401.3, nginx 1.24.0", "Jenkins 2.401.3", 1) - accepted, err := newJudge(m).Refine(context.Background(), []byte(raw), testFrames()) - if err != nil { - t.Fatal(err) - } - if accepted["jenkins"].Version != "2.401.3" || accepted["nginx"].Version != "" { - t.Fatalf("versions: jenkins %q nginx %q", accepted["jenkins"].Version, accepted["nginx"].Version) - } -} - func TestDeclarations(t *testing.T) { raw := []byte("HTTP/1.1 200 OK\r\nServer: Apache/2.4.38 (Debian) OpenSSL/1.0.2q PHP/5.6.40\r\nX-Jenkins: 2.401.3\r\n" + "X-Gitea-Version: 1.21.4\r\nX-Tomcat: Apache Tomcat/9.0.1\r\n\r\n") @@ -244,8 +281,10 @@ func TestDeclarations(t *testing.T) { func TestVersionNeedsConfidence(t *testing.T) { m := newMock() - m.versionP = 0.6 - accepted, err := newJudge(m).Refine(context.Background(), []byte(bodyRaw), testFrames()) + m.verdicts["jenkins"] = OptionRunning + m.confidence = 0.2 + accepted, err := New(m).Inspect(context.Background(), []byte(bodyRaw), testFrames()) + accepted = accepted.Accepted() if err != nil { t.Fatal(err) } @@ -254,25 +293,36 @@ func TestVersionNeedsConfidence(t *testing.T) { } } -func TestFailedRoundLeavesFrames(t *testing.T) { +// A failed provider leaves the input untouched, and the result still holds +// what code established: duplicates merged, header facts decided, the +// undecided hits kept unjudged. +func TestFailedProviderKeepsFacts(t *testing.T) { m := newMock() m.fail = true frames := testFrames() before, _ := json.Marshal(frames) - accepted, err := newJudge(m).Refine(context.Background(), []byte(jenkinsRaw), frames) - if err == nil || accepted != nil { + accepted, err := New(m).Inspect(context.Background(), []byte(jenkinsRaw), frames) + accepted = accepted.Accepted() + if err == nil { t.Fatal("expected error") } if after, _ := json.Marshal(frames); string(after) != string(before) { t.Fatalf("frames changed on failure:\n%s\n%s", before, after) } + // nginx, jenkins (declared), wordpress and one tomcat (unjudged); hsts absent, one tomcat a duplicate. + if len(accepted) != 4 || accepted["hsts"] != nil || accepted["wordpress"] == nil || accepted["wordpress"].Judge != nil || + accepted["jenkins"].Judge.Verdict != OptionDeclared { + t.Fatalf("accepted on failure: %v", accepted) + } } func TestCache(t *testing.T) { m := newMock() - j := newJudge(m) + m.verdicts["jenkins"] = OptionRunning + j := New(jev.Cached(m, jev.DefaultCacheSize)) for i := 0; i < 2; i++ { - accepted, err := j.Refine(context.Background(), []byte(bodyRaw), testFrames()) + accepted, err := j.Inspect(context.Background(), []byte(bodyRaw), testFrames()) + accepted = accepted.Accepted() if err != nil { t.Fatal(err) } @@ -280,110 +330,17 @@ func TestCache(t *testing.T) { t.Fatalf("run %d: cached answers not applied", i) } } - if m.calls != 2 || j.CacheHits != 2 { - t.Fatalf("calls=%d hits=%d", m.calls, j.CacheHits) + if m.calls != 2 { + t.Fatalf("calls=%d", m.calls) } } -func TestClassifyAlone(t *testing.T) { - m := newMock() - j := New(m) - p, _ := NewPage([]byte(jenkinsRaw)) - r := newRound(p) - var kind Kind - var generic bool - classifyRound(r, &kind, &generic) - var custom float64 - r.add("honeypot", Binary("Is this a honeypot?"), func(a Answer) { custom = a.Yes }) - if err := r.ask(context.Background(), j); err != nil { - t.Fatal(err) - } - if strings.Join(m.requests[0], ",") != "generic,honeypot,page_kind" || kind != KindLogin || !generic || custom != 0.7 { - t.Fatalf("requests=%v kind=%q custom=%v", m.requests, kind, custom) - } - if err := newRound(p).ask(context.Background(), j); err != nil || m.calls != 1 { - t.Fatalf("empty round sent a request: %v %d", err, m.calls) - } - r = newRound(p) - r.add("x", Binary("?"), nil) - r.add("x", Binary("?"), nil) - if err := r.ask(context.Background(), j); err == nil { - t.Fatal("duplicate key accepted") - } -} - -func TestMemoryCacheEvicts(t *testing.T) { - c := NewMemoryCache(2) - c.Put("a", 0, []byte("1")) - c.Put("b", 0, []byte("2")) - c.Get("a", 0, 0) - c.Put("c", 0, []byte("3")) - if _, ok := c.Get("b", 0, 0); ok { - t.Fatal("least recently used entry kept") - } - if v, ok := c.Get("a", 0, 0); !ok || string(v) != "1" { - t.Fatal("recent entry evicted") - } - if _, ok := c.Get("a", 0xff, 1); ok { - t.Fatal("signature 8 bits away matched") - } - if v, ok := c.Get("a", 0x1, 1); !ok || string(v) != "1" { - t.Fatal("signature 1 bit away missed") - } - if _, ok := c.Get("a", 0x1, 0); ok { - t.Fatal("signature 1 bit away matched at distance 0") - } - if _, ok := c.Get("a", 0x3, 1); ok { - t.Fatal("signature 2 bits away matched") - } -} - -// The same product page on another host (other token, date, host name) reuses -// the answers; a different page does not. -func TestSimilarPagesShareAnswers(t *testing.T) { - // Same page on another host: other session, build number and date. - page := func(build, date string) string { - return strings.Replace(jenkinsRaw, "We moved here from WordPress.", "We moved here from WordPress. Build "+build+", "+date+".", 1) - } - orig, variant := page("101", "2026-09-24"), page("2087", "2025-01-03") - a, _ := NewPage([]byte(orig)) - b, _ := NewPage([]byte(variant)) - other, _ := NewPage([]byte("HTTP/1.1 200 OK\r\nServer: Apache\r\n\r\nIndex of /a.txt")) - near := func(x, y *Page) bool { return bits.OnesCount64(x.signature()^y.signature()) <= DefaultSimilarDistance } - if !near(a, b) || near(a, other) { - t.Fatalf("signatures: %x %x %x", a.signature(), b.signature(), other.signature()) - } - for _, distance := range []int{DefaultSimilarDistance, 0} { - m := newMock() - j := New(m) - j.SimilarDistance = distance - for _, raw := range []string{orig, variant} { - if kind, _, err := j.Classify(context.Background(), []byte(raw)); err != nil || kind != KindLogin { - t.Fatalf("kind %q err %v", kind, err) - } - } - if want := map[int]int64{DefaultSimilarDistance: 1, 0: 2}[distance]; m.calls != want { - t.Fatalf("distance %d: calls=%d, want %d", distance, m.calls, want) - } - } -} - -// A provider that reports a calibration sets the Judge's thresholds. -type calibrated struct{ mock } - -func (*calibrated) Calibration() (float64, float64) { return 0.7, 0.8 } - -func TestCalibration(t *testing.T) { - if j := New(newMock()); j.Threshold != 0.5 || j.VersionConfidence != 0.9 { - t.Fatalf("defaults: %v %v", j.Threshold, j.VersionConfidence) - } - if j := New(&calibrated{}); j.Threshold != 0.7 || j.VersionConfidence != 0.8 { - t.Fatalf("calibration: %v %v", j.Threshold, j.VersionConfidence) - } +func testClaim(statement string) jev.Claim { + return jev.Claim{Statement: statement, Options: map[string]jev.Option{"yes": {Description: "", Outcome: jev.Holds}, "no": {Description: "", Outcome: jev.Refuted}, jev.OptionInsufficient: {Description: insufficientDescription, Outcome: jev.Insufficient}}} } func TestExtractVersionsPrefixes(t *testing.T) { - raw := "HTTP/1.1 200 OK\r\n\r\nV8.1SP2 10.0.0.5 jquery.min.js?ver=3.7.1" + raw := "HTTP/1.1 200 OK\r\n\r\nV8.1SP2 10.0.0.5 jquery.min.js?ver=3.7.1{\"success_fraction\":0.0}" var got []string for _, c := range extractVersions([]byte(raw), 10) { got = append(got, c.Value) @@ -393,13 +350,12 @@ func TestExtractVersionsPrefixes(t *testing.T) { } } -func TestRetrieverWordBoundary(t *testing.T) { - r := NewRetriever([]string{"acti", "jenkins", "泛微"}) - if got := r.Find("take action now", 5); len(got) != 0 { - t.Fatalf("substring matched: %v", got) +func TestWordBoundary(t *testing.T) { + if containsWord("take action now", "acti") || containsWord("x-github-request-id: 1", "git") { + t.Fatal("substring matched") } - if got := r.Find("sign in [jenkins] 泛微oa", 5); len(got) != 2 { - t.Fatalf("got %v", got) + if !containsWord("sign in [jenkins]", "jenkins") || !containsWord("泛微oa", "泛微") { + t.Fatal("word missed") } } @@ -439,117 +395,40 @@ func TestNormalizeName(t *testing.T) { } } -// A server's own page has no primary application; its version is still resolved. -func TestVersionWithoutPrimary(t *testing.T) { - m := newMock() - m.primary = noneOfThem +// Versions are ordered by their resolved outcome. +func TestVersionOrder(t *testing.T) { frames := common.Frameworks{} - frames.Add(common.NewFramework("nginx", common.FrameFromFingers)) - accepted, err := New(m).Refine(context.Background(), []byte(bodyRaw), frames) - if err != nil { - t.Fatal(err) - } - if accepted.Primary() != nil || accepted["nginx"].Version != "1.24.0" { - t.Fatalf("server version not resolved: %+v", accepted["nginx"]) - } -} - -// Refine of its own result sends nothing: judged hits are skipped and the -// remaining questions are answered by the cache. -func TestRefineIsIdempotent(t *testing.T) { - m := newMock() - j := newJudge(m) - first, err := j.Refine(context.Background(), []byte(bodyRaw), testFrames()) - if err != nil { - t.Fatal(err) - } - second, err := j.Refine(context.Background(), []byte(bodyRaw), first) - if err != nil { - t.Fatal(err) - } - if m.calls != 2 || len(second) != len(first) || second["jenkins"].Version != "2.401.3" { - t.Fatalf("calls=%d first=%v second=%v requests=%v", m.calls, first, second, m.requests) - } -} - -// Many workers judging the same page at once (a scan hitting one product -// everywhere) cost one request. -func TestConcurrentSimilarPagesMerge(t *testing.T) { - m := newMock() - m.delay = 100 * time.Millisecond - c := New(m) - var wg sync.WaitGroup - kinds := make([]Kind, 20) - for i := range kinds { - wg.Add(1) - go func(i int) { - defer wg.Done() - raw := strings.Replace(jenkinsRaw, "JSESSIONID.1=x", fmt.Sprintf("JSESSIONID.1=%d", i), 1) - kind, _, err := c.Classify(context.Background(), []byte(raw)) - if err != nil { - t.Error(err) - } - kinds[i] = kind - }(i) - } - wg.Wait() - for i, k := range kinds { - if k != KindLogin { - t.Fatalf("worker %d: kind %q", i, k) - } - } - if m.calls != 1 || c.CacheHits != 19 { - t.Fatalf("calls=%d hits=%d", m.calls, c.CacheHits) - } -} - -// A round partly answered by the cache sends only the rest. -func TestPartialCacheHit(t *testing.T) { - m := newMock() - c := New(m) - if _, _, err := c.Classify(context.Background(), []byte(jenkinsRaw)); err != nil { - t.Fatal(err) - } - p, _ := NewPage([]byte(jenkinsRaw)) - r := newRound(p) - var kind Kind - var generic bool - classifyRound(r, &kind, &generic) - r.add("extra", Binary("Is this page served over a CDN?"), nil) - if err := r.ask(context.Background(), c); err != nil || kind != KindLogin { - t.Fatalf("err %v kind %q", err, kind) - } - if len(m.requests) != 2 || strings.Join(m.requests[1], ",") != "extra" { - t.Fatalf("requests: %v", m.requests) - } -} - -// Versions are resolved for the primary application first, then other -// applications, servers and libraries; unjudged hits last. -func TestByImportance(t *testing.T) { - frames := common.Frameworks{} - for name, j := range map[string]*common.Judgement{"jquery": {Layer: LayerFrontend}, "apache": {Layer: LayerServer}, - "ubuntu": {Layer: LayerDevice}, "gitlab": {Layer: LayerApplication, Primary: true}, "unjudged": nil} { + for name, j := range map[string]*common.Judgement{"jquery": {Verdict: jev.OptionInsufficient}, "apache": {Verdict: OptionDeclared, Outcome: jev.Holds.String()}, + "gitlab": {Verdict: OptionRunning, Outcome: jev.Holds.String()}, "unjudged": nil} { f := common.NewFramework(name, common.FrameFromFingers) f.Judge = j frames.Add(f) } var got []string - for _, f := range byImportance(frames) { + for _, f := range versionOrder(frames) { got = append(got, f.Name) } - if strings.Join(got, ",") != "gitlab,ubuntu,apache,jquery,unjudged" { + if strings.Join(got, ",") != "apache,gitlab,jquery,unjudged" { t.Fatalf("order: %v", got) } } -// Sparse pages with common headers must not look alike: CJK text counts, and -// similar pages must share a title. -func TestUnrelatedSparsePagesDoNotShare(t *testing.T) { - head := "HTTP/1.1 200 OK\r\nServer: nginx\r\nX-Frame-Options: SAMEORIGIN\r\n\r\n" - a, _ := NewPage([]byte(head + "职业规划咨询

生涯规划师与高考志愿规划，帮助学生找到方向

")) - b, _ := NewPage([]byte(head + "官方网站

在线娱乐平台，注册即送体验金

")) - if bits.OnesCount64(a.signature()^b.signature()) <= DefaultSimilarDistance && a.similarityScope() == b.similarityScope() { - t.Fatalf("unrelated pages share answers: %x %x", a.signature(), b.signature()) +// Rule excerpts include matches deep in the response. +func TestEvidenceQuotesTheMatcher(t *testing.T) { + raw := "HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\n没有找到站点" + strings.Repeat("

filler

", 300) + + "

您的请求在Web服务器中没有找到对应的站点！

" + p, _ := evidence.NewPage([]byte(raw)) + f := common.NewFramework("宝塔", common.FrameFromFingers) + f.MatchDetail = &common.MatchDetail{MatcherType: "regexp", MatcherValue: "没有找到对应的站点"} + frames := common.Frameworks{} + frames.Add(f) + groups := groupProducts(frames) + excerpts := evidenceFor(p.Raw, groups[0]) + if len(groups) != 1 || len(excerpts) == 0 || + !strings.Contains(excerpts[0].Text, "您的请求在Web服务器中没有找到对应的站点") || excerpts[0].Where != "body" { + t.Fatalf("evidence: %+v", groups[0]) + } + if strings.Contains(p.Text, "您的请求") { + t.Fatal("test page too short: the compact view already carries the match") } } diff --git a/judge/maintain/coverage.go b/judge/maintain/coverage.go new file mode 100644 index 0000000..85d60c9 --- /dev/null +++ b/judge/maintain/coverage.go @@ -0,0 +1,68 @@ +package maintain + +import ( + "context" + "fmt" + "sort" + + "github.com/chainreactors/fingers/judge" + "github.com/chainreactors/fingers/judge/internal/evidence" + "github.com/chainreactors/utils/jev" +) + +// Coverage options: what the samples of one page show beside the products +// the rules report on them. +const ( + CoverageExplained = "explained" // holds: the reported products account for the page + CoverageCustom = "custom_site" // holds: content one organization wrote; no packaged product to fingerprint + CoverageMissing = "missing" // refuted: the stock page of a packaged product the rules miss +) + +const maxCoverageSamples = 3 + +// Coverage rules on the claim that the products the rules report account +// for samples: responses of one page, ideally from unrelated hosts. It is +// the offline counterpart of presence claims. A Refuted ruling +// (CoverageMissing) is a missed product for a person to name, e.g. from +// Cluster candidates; the provider never names one, and reported +// are the claim under review, never options to pick from. +func coverage(ctx context.Context, j *judge.Judge, samples [][]byte, reported []string) (jev.Ruling, jev.Outcome, error) { + var pages []*evidence.Page + for _, raw := range samples { + if len(pages) == maxCoverageSamples { + break + } + p, err := evidence.NewPage(raw) + if err != nil { + return jev.Ruling{}, jev.Insufficient, err + } + pages = append(pages, p) + } + if len(pages) == 0 { + return jev.Ruling{}, jev.Insufficient, fmt.Errorf("coverage: no samples") + } + names := append([]string{}, reported...) + sort.Strings(names) + state := map[string]interface{}{"samples": pages, "reported": names} + if len(pages) > 1 { + state["note"] = "the samples are the same page served by unrelated hosts: a page many unrelated operators serve alike is usually a packaged product, not content one organization wrote" + } + claim := jev.Claim{ + Statement: "`reported` lists the products fingerprint rules report on these `samples`. " + + "Claim: they account for the page, that is, the page is not the stock interface of a packaged product missing from `reported`. " + + "A stock interface is a product's own login, console, default, error or application page, " + + "which looks the same on every deployment of it except for branding, host names and data. " + + "Servers, CDNs, frameworks, libraries and databases underneath an application do not account for the application's own page.", + Options: map[string]jev.Option{ + jev.OptionInsufficient: {Description: "The evidence does not decide the claim.", Outcome: jev.Insufficient}, + CoverageExplained: {Description: "The page is the stock interface of a product in `reported`, or shows no packaged product beyond them, such as a bare server default or error page.", Outcome: jev.Holds}, + CoverageCustom: {Description: "Content one organization wrote for its own purpose: articles, company pages, shops, forums, custom-built portals.", Outcome: jev.Holds}, + CoverageMissing: {Description: "The page is the stock interface of a packaged product that is not in `reported`.", Outcome: jev.Refuted}, + }, + } + rulings, err := jev.Judge(ctx, j.Provider, state, map[string]jev.Claim{"coverage": claim}) + if err != nil { + return jev.Ruling{}, jev.Insufficient, err + } + return rulings["coverage"], claim.Resolve(rulings["coverage"], j.MinConfidence), nil +} diff --git a/judge/maintain/discover.go b/judge/maintain/discover.go new file mode 100644 index 0000000..91d89ad --- /dev/null +++ b/judge/maintain/discover.go @@ -0,0 +1,231 @@ +package maintain + +import ( + "context" + "regexp" + "sort" + "strings" + + "github.com/chainreactors/fingers/common" + "github.com/chainreactors/fingers/judge" + "github.com/chainreactors/fingers/judge/internal/evidence" + "github.com/chainreactors/utils/jev" +) + +// Sample is one scanned response for discovery. +type Sample struct { + ID, Host string + Raw []byte + Accepted common.Frameworks // Inspect(Raw).Accepted() for Raw +} + +// Cluster is one page served by several hosts, as Discover found it. +type Cluster struct { + Samples []string `json:"samples"` + Hosts int `json:"hosts"` + // Reported are the products the samples' rules report. + Reported []string `json:"reported,omitempty"` + // Coverage is the ruling on the claim that Reported account for the + // page: Refuted (CoverageMissing) marks a missed product. + Coverage jev.Ruling `json:"coverage"` + Outcome jev.Outcome `json:"outcome"` + // Candidates are names code finds in the samples, for the person who + // names a missed product; they are never offered to the provider. + Candidates []string `json:"candidates,omitempty"` +} + +// MinSimilarity is the structural overlap (Jaccard of asset paths, form +// fields, cookie and custom header names, generator and title words) at +// which two pages are one page. On the 2026-09-26 corpus it links 52 of 54 +// cross-host pairs of one product and 22 of 2278 other pairs, mostly stock +// pages of one known product. +const MinSimilarity = 0.4 + +// Discover clusters samples that are one page on different hosts and, once +// per cluster of at least minHosts hosts, evaluates whether +// the products the rules report account for it. Code does the clustering +// and the host count: a page served the same by unrelated hosts is what a +// fingerprint can describe. +func Discover(ctx context.Context, j *judge.Judge, samples []Sample, minHosts int) ([]*Cluster, error) { + pages := make([]map[string]bool, len(samples)) + for i, s := range samples { + p, err := evidence.NewPage(s.Raw) + if err != nil { + continue + } + pages[i] = features(p) + } + parent := make([]int, len(samples)) + for i := range parent { + parent[i] = i + } + var find func(int) int + find = func(i int) int { + if parent[i] != i { + parent[i] = find(parent[i]) + } + return parent[i] + } + for a := range samples { + for b := a + 1; b < len(samples); b++ { + if pages[a] != nil && pages[b] != nil && jaccard(pages[a], pages[b]) >= MinSimilarity { + parent[find(a)] = find(b) + } + } + } + groups := map[int][]Sample{} + var roots []int + for i, s := range samples { + if pages[i] == nil { + continue + } + r := find(i) + if groups[r] == nil { + roots = append(roots, r) + } + groups[r] = append(groups[r], s) + } + var out []*Cluster + for _, r := range roots { + c := &Cluster{} + var raws [][]byte + hosts := map[string]bool{} + for _, s := range groups[r] { + c.Samples = append(c.Samples, s.ID) + if !hosts[s.Host] { + hosts[s.Host] = true + raws = append(raws, s.Raw) + } + } + c.Hosts = len(hosts) + if c.Hosts < minHosts { + continue + } + c.Reported = reported(groups[r]) + ruling, outcome, err := coverage(ctx, j, raws, c.Reported) + if err != nil { + return out, err + } + c.Coverage, c.Outcome = ruling, outcome + c.Candidates = candidates(raws) + out = append(out, c) + } + sort.SliceStable(out, func(a, b int) bool { return out[a].Hosts > out[b].Hosts }) + return out, nil +} + +// reported are the products any sample reports, one spelling each. +func reported(samples []Sample) []string { + seen := map[string]bool{} + var out []string + for _, s := range samples { + for _, f := range s.Accepted { + if key := judge.NormalizeName(f.Name); key != "" && !seen[key] { + seen[key] = true + out = append(out, f.Name) + } + } + } + sort.Strings(out) + return out +} + +const maxCandidates = 20 + +// candidates are the names that occur in two samples (or the only one), +// most telling source first: generator meta, product headers, title parts, +// asset path words, then capitalized words of the visible text. +func candidates(raws [][]byte) []string { + need := 2 + if len(raws) < need { + need = len(raws) + } + count := map[string]int{} + label := map[string]string{} + var order []string + for _, raw := range raws { + p, err := evidence.NewPage(raw) + if err != nil { + continue + } + seen := map[string]bool{} + for _, n := range evidence.Names(p.Generator, p.Title, p.Text, p.Headers, append(append([]string(nil), p.Scripts...), p.Styles...)) { + key := judge.NormalizeName(n) + if seen[key] { + continue + } + seen[key] = true + if count[key] == 0 { + label[key] = n + order = append(order, key) + } + count[key]++ + } + } + var out []string + for _, key := range order { + if count[key] >= need && len(out) < maxCandidates { + out = append(out, label[key]) + } + } + return out +} + +var reVolatile = regexp.MustCompile(`[0-9a-f]{6,}|[0-9]+`) + +// commonHeaders are X- headers any stack sets: they tell pages of one +// product apart from nothing. +var commonHeaders = map[string]bool{ + "x-content-type-options": true, "x-frame-options": true, "x-xss-protection": true, "x-request-id": true, + "x-download-options": true, "x-permitted-cross-domain-policies": true, "x-dns-prefetch-control": true, + "x-robots-tag": true, "x-cache": true, "x-served-by": true, "x-forwarded-for": true, "x-forwarded-proto": true, +} + +// features are the structural traits of a page that its deployments share: +// asset paths without queries or hashes, form fields, cookie and custom +// header names, the generator and title words. +func features(p *evidence.Page) map[string]bool { + out := map[string]bool{} + fold := func(s string) string { + if i := strings.IndexAny(s, "?#"); i >= 0 { + s = s[:i] + } + return reVolatile.ReplaceAllString(strings.ToLower(s), "") + } + for _, x := range append(append([]string(nil), p.Scripts...), p.Styles...) { + out["asset:"+fold(x)] = true + } + for _, x := range p.Forms { + out["form:"+strings.ToLower(x)] = true + } + for _, x := range p.Cookies { + out["cookie:"+fold(x)] = true + } + for k := range p.Headers { + if k := strings.ToLower(k); strings.HasPrefix(k, "x-") && !commonHeaders[k] { + out["header:"+k] = true + } + } + if p.Generator != "" { + out["generator:"+judge.NormalizeName(reVolatile.ReplaceAllString(p.Generator, ""))] = true + } + for _, w := range strings.FieldsFunc(strings.ToLower(p.Title), func(r rune) bool { return strings.ContainsRune(" -|:.·–", r) }) { + if len(w) > 2 { + out["title:"+w] = true + } + } + return out +} + +func jaccard(a, b map[string]bool) float64 { + if len(a) == 0 || len(b) == 0 { + return 0 + } + n := 0 + for k := range a { + if b[k] { + n++ + } + } + return float64(n) / float64(len(a)+len(b)-n) +} diff --git a/judge/maintain/ledger.go b/judge/maintain/ledger.go new file mode 100644 index 0000000..fc71a4c --- /dev/null +++ b/judge/maintain/ledger.go @@ -0,0 +1,141 @@ +// Package maintain audits a fingerprint library offline from judged scan +// results. Ledger asks the provider nothing new: a rule whose presence +// claims are refuted again and again is a junk rule. Discover clusters one +// page served by several hosts and asks one coverage claim per cluster; a +// refuted one is a missed product, named by a person from code's +// candidates and turned into rules by judge/gen. +package maintain + +import ( + "sort" + "sync" + + "github.com/chainreactors/fingers/common" + "github.com/chainreactors/utils/jev" +) + +// RuleKey identifies the rule behind a hit as precisely as the engine +// reports it: engines that record MatchDetail (the native fingers engine) +// down to the matcher, the others by fingerprint name. +type RuleKey struct { + Engine string `json:"engine"` + Name string `json:"name"` + Rule int `json:"rule,omitempty"` + Matcher string `json:"matcher,omitempty"` +} + +// RuleStats are the rulings on the presence claims of one rule's hits. +type RuleStats struct { + RuleKey + Outcomes map[jev.Outcome]int `json:"outcomes"` // holds, refuted, insufficient + Options map[string]int `json:"options"` // the options behind them + Samples map[jev.Outcome][]string `json:"samples"` // outcome -> page ids, up to maxSamples +} + +const maxSamples = 5 + +func (s *RuleStats) Hits() int { + total := 0 + for _, count := range s.Outcomes { + total += count + } + return total +} + +// RefutedFraction is the share of judged hits whose presence claim is refuted. +func (s *RuleStats) RefutedFraction() float64 { + if s.Hits() == 0 { + return 0 + } + return float64(s.Outcomes[jev.Refuted]) / float64(s.Hits()) +} + +// Ledger accumulates rulings per rule over a scan or corpus. It is safe +// for concurrent use. +type Ledger struct { + mu sync.Mutex + rules map[RuleKey]*RuleStats +} + +func NewLedger() *Ledger { return &Ledger{rules: map[RuleKey]*RuleStats{}} } + +// Add records the rulings of one page's Inspect result. Unjudged hits are +// skipped; duplicate spellings count for their own rule with the ruling of +// the product they spell. +func (l *Ledger) Add(page string, inspected common.Frameworks) { + l.mu.Lock() + defer l.mu.Unlock() + for _, f := range inspected { + if f == nil || f.Judge == nil || f.Judge.Outcome == "" { + continue + } + o := jev.Outcome(f.Judge.Outcome) + for _, key := range ruleKeys(f) { + s := l.rules[key] + if s == nil { + s = &RuleStats{RuleKey: key, Outcomes: map[jev.Outcome]int{}, Options: map[string]int{}, Samples: map[jev.Outcome][]string{}} + l.rules[key] = s + } + s.Outcomes[o]++ + s.Options[f.Judge.Verdict]++ + if len(s.Samples[o]) < maxSamples { + s.Samples[o] = append(s.Samples[o], page) + } + } + } +} + +// Report lists the rules with at least minHits judged hits whose refuted +// share is at least minRefuted, most refuted first: the junk rule +// candidates, each with the pages to review. +func (l *Ledger) Report(minHits int, minRefuted float64) []*RuleStats { + l.mu.Lock() + defer l.mu.Unlock() + var out []*RuleStats + for _, s := range l.rules { + if s.Hits() >= minHits && s.RefutedFraction() >= minRefuted { + snapshot := *s + snapshot.Outcomes, snapshot.Options, snapshot.Samples = map[jev.Outcome]int{}, map[string]int{}, map[jev.Outcome][]string{} + for k, v := range s.Outcomes { + snapshot.Outcomes[k] = v + } + for k, v := range s.Options { + snapshot.Options[k] = v + } + for k, v := range s.Samples { + snapshot.Samples[k] = append([]string(nil), v...) + } + out = append(out, &snapshot) + } + } + sort.Slice(out, func(a, b int) bool { + ra, rb := out[a].Outcomes[jev.Refuted], out[b].Outcomes[jev.Refuted] + if ra != rb { + return ra > rb + } + return out[a].Name < out[b].Name + }) + return out +} + +func ruleKeys(f *common.Framework) []RuleKey { + var engines []string + for from := range f.Froms { + if from >= common.FrameFromFingers { + engines = append(engines, from.String()) + } + } + if len(engines) == 0 { + engines = []string{"unknown"} + } + sort.Strings(engines) + var keys []RuleKey + for _, e := range engines { + key := RuleKey{Engine: e, Name: f.Name} + if d := f.MatchDetail; d != nil && e == common.FrameFromFingers.String() { + key.Rule, key.Matcher = d.RuleIndex, d.MatcherValue + } + keys = append(keys, key) + } + return keys +} diff --git a/judge/maintain/maintain_test.go b/judge/maintain/maintain_test.go new file mode 100644 index 0000000..df69125 --- /dev/null +++ b/judge/maintain/maintain_test.go @@ -0,0 +1,148 @@ +package maintain + +import ( + "context" + "encoding/json" + "fmt" + "testing" + + "github.com/chainreactors/fingers/common" + "github.com/chainreactors/fingers/judge" + "github.com/chainreactors/utils/jev" +) + +func judged(name string, from common.From, verdict string, outcome jev.Outcome, detail *common.MatchDetail) *common.Framework { + f := common.NewFramework(name, from) + f.Judge = &common.Judgement{Verdict: verdict, Outcome: outcome.String()} + f.MatchDetail = detail + return f +} + +func TestLedgerReportsJunkRules(t *testing.T) { + l := NewLedger() + webp := &common.MatchDetail{RuleIndex: 2, MatcherValue: "webp"} + for i := 0; i < 4; i++ { + fs := common.Frameworks{} + fs.Add(judged("webp_server_go", common.FrameFromFingers, judge.OptionUnrelated, jev.Refuted, webp)) + fs.Add(judged("nginx", common.FrameFromGoby, judge.OptionDeclared, jev.Holds, nil)) + verdict, outcome := judge.OptionMentioned, jev.Refuted + if i == 0 { + verdict, outcome = judge.OptionRunning, jev.Holds + } + fs.Add(judged("git", common.FrameFromEhole, verdict, outcome, nil)) + fs.Add(common.NewFramework("unjudged", common.FrameFromGoby)) + l.Add(fmt.Sprintf("page%d", i), fs) + } + got := l.Report(3, 0.5) + if len(got) != 2 || got[0].Name != "webp_server_go" || got[0].Matcher != "webp" || got[0].Rule != 2 || got[0].Engine != "fingers" || + got[1].Name != "git" || got[1].Engine != "ehole" || got[1].RefutedFraction() != 0.75 || len(got[1].Samples[jev.Refuted]) != 3 || got[1].Options[judge.OptionMentioned] != 3 { + t.Fatalf("report: %+v", got) + } +} + +// coverer rules every coverage claim with one option. +type coverer string + +func (coverer) ID() string { return "coverer" } + +func (c coverer) Judge(_ context.Context, _ interface{}, qs map[string]jev.Claim) (map[string]jev.Ruling, error) { + out := map[string]jev.Ruling{} + for k := range qs { + out[k] = jev.Ruling{Option: string(c), Confidence: 0.9} + } + return out, nil +} + +func TestDiscoverClaimsCoverageOfPagesSharedByHosts(t *testing.T) { + orion := func(host string) []byte { + return []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\nX-Orion-Node: " + host + "\r\n\r\nOrion Console" + + `` + + `

Welcome to Orion on ` + host + `

`) + } + blog := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nMy Blog

Hello

") + nginx := common.Frameworks{} + nginx.Add(common.NewFramework("nginx", common.FrameFromFingers)) + var samples []Sample + for _, h := range []string{"a.example", "b.example", "c.example"} { + samples = append(samples, Sample{ID: h, Host: h, Raw: orion(h), Accepted: nginx}) + } + samples = append(samples, Sample{ID: "blog", Host: "blog.example", Raw: blog, Accepted: nginx}) + + clusters, err := Discover(context.Background(), judge.New(coverer(CoverageMissing)), samples, 2) + if err != nil { + t.Fatal(err) + } + c := clusters[0] + if len(clusters) != 1 || c.Hosts != 3 || c.Outcome != jev.Refuted || len(c.Reported) != 1 || c.Reported[0] != "nginx" { + t.Fatalf("clusters: %+v", clusters) + } + // Code, not the provider, offers the names a person picks from. + found := false + for _, n := range c.Candidates { + found = found || n == "Orion Console" + } + if !found { + t.Fatalf("candidates: %v", c.Candidates) + } + clusters, err = Discover(context.Background(), judge.New(coverer(CoverageExplained)), samples, 2) + if err != nil || len(clusters) != 1 || clusters[0].Outcome != jev.Holds { + t.Fatalf("explained: %+v %v", clusters, err) + } +} + +// Pages that share only the security headers any stack sets are not one page. +func TestDiscoverIgnoresCommonHeaders(t *testing.T) { + head := "HTTP/1.1 200 OK\r\nX-Content-Type-Options: nosniff\r\nX-Frame-Options: DENY\r\nX-Request-Id: 1\r\n\r\n" + samples := []Sample{ + {ID: "a", Host: "a.example", Raw: []byte(head + "请通过官方域名访问")}, + {ID: "b", Host: "b.example", Raw: []byte(head + "Privacy Redirect")}, + } + clusters, err := Discover(context.Background(), judge.New(coverer(CoverageMissing)), samples, 2) + if err != nil || len(clusters) != 0 { + t.Fatalf("clusters: %+v %v", clusters, err) + } +} + +func TestLedgerReportIsDeepSnapshot(t *testing.T) { + l := NewLedger() + f := judged("one", common.FrameFromFingers, judge.OptionRunning, jev.Holds, nil) + fs := common.Frameworks{"one": f} + l.Add("a", fs) + first := l.Report(1, 0) + l.Add("b", fs) + if first[0].Hits() != 1 || first[0].Outcomes["holds"] != 1 || len(first[0].Samples["holds"]) != 1 { + t.Fatal("snapshot changed after Add") + } + first[0].Samples["holds"][0] = "changed" + first[0].Options[judge.OptionRunning] = 99 + first[0].Outcomes["holds"] = 99 + second := l.Report(1, 0) + if second[0].Samples["holds"][0] != "a" || second[0].Outcomes["holds"] != 2 || second[0].Options[judge.OptionRunning] != 2 { + t.Fatal("snapshot mutation changed ledger") + } +} + +func TestClusterSnapshotKeepsRawChoiceAndResolvedOutcome(t *testing.T) { + raw := []byte("HTTP/1.1 200 OK\r\n\r\nOrion Console") + j := judge.New(coverer(CoverageMissing)) + j.MinConfidence = 1 + clusters, err := Discover(context.Background(), j, []Sample{{ID: "a", Host: "one", Raw: raw}, {ID: "b", Host: "two", Raw: raw}}, 2) + if err != nil || len(clusters) != 1 { + t.Fatalf("%v %v", clusters, err) + } + c := clusters[0] + if c.Coverage.Option != CoverageMissing || c.Outcome != jev.Insufficient { + t.Fatalf("raw choice lost: %+v", c) + } + data, err := json.Marshal(c) + if err != nil { + t.Fatal(err) + } + var restored Cluster + if err := json.Unmarshal(data, &restored); err != nil { + t.Fatal(err) + } + if restored.Samples[0] != "a" || restored.Samples[1] != "b" || restored.Coverage != c.Coverage || restored.Outcome != c.Outcome { + t.Fatalf("snapshot: %+v", restored) + } +} diff --git a/judge/operations.go b/judge/operations.go deleted file mode 100644 index ae119d4..0000000 --- a/judge/operations.go +++ /dev/null @@ -1,127 +0,0 @@ -package judge - -import ( - "context" - "fmt" - "math" - - "github.com/chainreactors/fingers/common" -) - -// Yes returns the probability that the answer to question is yes. State can -// be any JSON-serializable value, including a Page returned by NewPage. -func (j *Judge) Yes(ctx context.Context, state interface{}, question string) (float64, error) { - answers, err := j.Ask(ctx, state, map[string]Question{"yes": Binary(question)}) - if err != nil { - return 0, err - } - return answers["yes"].Yes, nil -} - -// Choose selects one option and returns its confidence. -func (j *Judge) Choose(ctx context.Context, state interface{}, question string, options map[string]string) (string, float64, error) { - if len(options) < 2 { - return "", 0, fmt.Errorf("judge: choose needs at least two options") - } - answers, err := j.Ask(ctx, state, map[string]Question{"choice": Choice(question, options)}) - if err != nil { - return "", 0, err - } - a := answers["choice"] - if _, ok := options[a.Choice]; !ok { - return "", 0, fmt.Errorf("judge: provider chose unknown option %q", a.Choice) - } - return a.Choice, a.Confidence, nil -} - -// Score returns a value from 0 to 1 on the ordered levels. -func (j *Judge) Score(ctx context.Context, state interface{}, question string, levels ...string) (float64, error) { - if len(levels) < 2 || len(levels) > 10 { - return 0, fmt.Errorf("judge: score needs 2 to 10 levels") - } - answers, err := j.Ask(ctx, state, map[string]Question{"score": Score(question, levels)}) - if err != nil { - return 0, err - } - score := answers["score"].Score - if math.IsNaN(score) || score < 0 || score > 1 { - return 0, fmt.Errorf("judge: provider returned invalid score %v", score) - } - return score, nil -} - -// Refine is the scan-time entry point: it returns the fingerprints to report -// for a response. False positives and duplicate spellings are dropped, known -// names the rules missed are recalled (see Judge.Known) and every kept -// product without a version gets one when the response shows it. Each -// returned framework carries its verdict in Framework.Judge. hits are never -// changed; on error, use them as the rule-only result. -func (j *Judge) Refine(ctx context.Context, raw []byte, hits common.Frameworks) (common.Frameworks, error) { - p, err := NewPage(raw) - if err != nil { - return nil, err - } - all, err := j.inspect(ctx, p, hits) - if err != nil { - return nil, err - } - accepted := all.Accepted() - r := newRound(p) - versionRound(r, byImportance(accepted)...) - if err := r.ask(ctx, j); err != nil { - return nil, err - } - return accepted, nil -} - -// Inspect explains Refine: it returns every hit and recalled name with its -// verdict in Framework.Judge, rejected and duplicate ones included, and -// resolves no versions. hits are never changed. -func (j *Judge) Inspect(ctx context.Context, raw []byte, hits common.Frameworks) (common.Frameworks, error) { - p, err := NewPage(raw) - if err != nil { - return nil, err - } - return j.inspect(ctx, p, hits) -} - -// Classify reports the page kind and whether it is the stock page of a -// packaged product. After Refine of the same response it is answered from -// the cache. -func (j *Judge) Classify(ctx context.Context, raw []byte) (kind Kind, generic bool, err error) { - p, err := NewPage(raw) - if err != nil { - return "", false, err - } - r := newRound(p) - classifyRound(r, &kind, &generic) - if err := r.ask(ctx, j); err != nil { - return "", false, err - } - return kind, generic, nil -} - -// Version selects a version for f when the response contains a supported -// version string. It does not change f. -func (j *Judge) Version(ctx context.Context, raw []byte, f *common.Framework) (string, error) { - if f == nil { - return "", nil - } - if f.Attributes != nil && f.Version != "" { - return f.Version, nil - } - p, err := NewPage(raw) - if err != nil { - return "", err - } - copy := cloneFramework(f) - r := newRound(p) - versionRound(r, copy) - if err := r.ask(ctx, j); err != nil { - return "", err - } - if copy.Attributes == nil { - return "", nil - } - return copy.Version, nil -} diff --git a/judge/operations_test.go b/judge/operations_test.go index 5d327f5..bd956fc 100644 --- a/judge/operations_test.go +++ b/judge/operations_test.go @@ -4,83 +4,48 @@ import ( "context" "encoding/json" "errors" - "strings" "testing" "github.com/chainreactors/fingers/common" + "github.com/chainreactors/utils/jev" ) -type answerProvider func(map[string]Question) map[string]Answer +type answerProvider func(map[string]jev.Claim) map[string]jev.Ruling func (answerProvider) ID() string { return "answer-provider" } -func (p answerProvider) Judge(_ context.Context, _ interface{}, questions map[string]Question) (map[string]Answer, error) { +func (p answerProvider) Judge(_ context.Context, _ interface{}, questions map[string]jev.Claim) (map[string]jev.Ruling, error) { return p(questions), nil } type failVersionProvider struct{ *mock } -func (p failVersionProvider) Judge(ctx context.Context, state interface{}, questions map[string]Question) (map[string]Answer, error) { - if _, ok := questions["version_0"]; ok { +func (p failVersionProvider) Judge(ctx context.Context, state interface{}, questions map[string]jev.Claim) (map[string]jev.Ruling, error) { + if _, ok := questions["version_jenkins"]; ok { return nil, errors.New("version unavailable") } return p.mock.Judge(ctx, state, questions) } -func TestTypedPrimitives(t *testing.T) { - j := New(answerProvider(func(qs map[string]Question) map[string]Answer { - out := map[string]Answer{} - for key, q := range qs { - switch q.Type { - case TypeBinary: - out[key] = Answer{Yes: 0.82} - case TypeChoice: - out[key] = Answer{Choice: "b", Confidence: 0.93} - case TypeScore: - out[key] = Answer{Score: 0.4} - } - } - return out - })) - state := map[string]string{"title": "Example"} - if p, err := j.Yes(context.Background(), state, "Is this a product?"); err != nil || p != 0.82 { - t.Fatalf("yes = %v, %v", p, err) - } - if choice, confidence, err := j.Choose(context.Background(), state, "Pick", map[string]string{"a": "A", "b": "B"}); err != nil || choice != "b" || confidence != 0.93 { - t.Fatalf("choose = %q %v, %v", choice, confidence, err) - } - if score, err := j.Score(context.Background(), state, "Rate", "low", "high"); err != nil || score != 0.4 { - t.Fatalf("score = %v, %v", score, err) - } - if _, _, err := j.Choose(context.Background(), state, "Pick", map[string]string{"one": "only"}); err == nil { - t.Fatal("accepted a choice with one option") - } - if _, err := New(answerProvider(func(map[string]Question) map[string]Answer { return nil })).Yes(context.Background(), state, "Missing"); err == nil { - t.Fatal("missing provider answer was accepted") - } - if _, _, err := New(answerProvider(func(map[string]Question) map[string]Answer { - return map[string]Answer{"choice": {Choice: "outside", Confidence: 0.9}} - })).Choose(context.Background(), state, "Pick", map[string]string{"a": "A", "b": "B"}); err == nil { - t.Fatal("unknown provider choice was accepted") - } -} - func TestPublicResultsDoNotMutateHits(t *testing.T) { - j := newJudge(newMock()) + m := newMock() + m.verdicts["jenkins"] = OptionRunning + j := New(m) frames := testFrames() before, _ := json.Marshal(frames) inspected, err := j.Inspect(context.Background(), []byte(bodyRaw), frames) - if err != nil || !inspected["wordpress"].Judge.Rejected || inspected["prototype"] == nil { + if err != nil || !inspected["wordpress"].Judge.Rejected { t.Fatalf("inspect = %v, %v", inspected, err) } - refined, err := j.Refine(context.Background(), []byte(bodyRaw), frames) + refined, err := j.Inspect(context.Background(), []byte(bodyRaw), frames) + refined = refined.Accepted() if err != nil || refined["wordpress"] != nil || refined["jenkins"].Version != "2.401.3" { t.Fatalf("refine = %v, %v", refined, err) } // The judgement is copied, not shared with the Inspect result. - refined["nginx"].Judge.Primary = true - if inspected["nginx"].Judge.Primary { - t.Fatal("Refine and Inspect share a Judgement") + refined["nginx"].Judge.Verdict = "changed" + if inspected["nginx"].Judge.Verdict == "changed" { + t.Fatal("Independent Inspect calls share a Judgement") } if after, _ := json.Marshal(frames); string(after) != string(before) { t.Fatalf("hits changed:\n%s\n%s", before, after) @@ -91,77 +56,70 @@ func TestPublicResultsDoNotMutateHits(t *testing.T) { } } -func TestRefineSecondRoundFailureIsAtomic(t *testing.T) { +// A failed version round keeps the verdicts and leaves the input untouched. +func TestInspectVersionFailureKeepsVerdicts(t *testing.T) { frames := testFrames() - j := New(failVersionProvider{newMock()}) - accepted, err := j.Refine(context.Background(), []byte(bodyRaw), frames) - if err == nil || accepted != nil { - t.Fatalf("failed refinement = %v, %v", accepted, err) + m := newMock() + m.verdicts["jenkins"] = OptionRunning + accepted, err := New(failVersionProvider{m}).Inspect(context.Background(), []byte(bodyRaw), frames) + accepted = accepted.Accepted() + if err == nil || accepted["wordpress"] != nil || accepted["jenkins"] == nil || accepted["jenkins"].Version != "" { + t.Fatalf("failed version round = %v, %v", accepted, err) } for _, f := range frames { if f.Judge != nil || f.Version != "" { - t.Fatalf("input changed after second round failed: %+v", f) + t.Fatalf("input changed after the version round failed: %+v", f) } } } -func TestUnknownAndBodyNameSuggestion(t *testing.T) { - raw := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nSign in

QuantumGate Portal account access

") - j := New(answerProvider(func(qs map[string]Question) map[string]Answer { - out := map[string]Answer{} - for key, q := range qs { - switch key { - case "page_kind": - out[key] = Answer{Choice: string(KindLogin)} - case "generic": - out[key] = Answer{Yes: 0.9} - default: - if strings.Contains(q.Instructions, "`QuantumGate Portal`") { - out[key] = Answer{Yes: 0.95} - } else { - out[key] = Answer{Yes: 0.05} - } - } +func TestInspectInvalidBatchIsNotPartlyApplied(t *testing.T) { + j := New(answerProvider(func(cs map[string]jev.Claim) map[string]jev.Ruling { + out := map[string]jev.Ruling{} + for id := range cs { + out[id] = jev.Ruling{Option: OptionRunning, Confidence: 1} } + out["presence_wordpress"] = jev.Ruling{Option: "invented", Confidence: 1} return out })) - names, err := j.SuggestNames(context.Background(), raw) - if err != nil || len(names) != 1 || names[0] != "QuantumGate Portal" { - t.Fatalf("body suggestions = %v, %v", names, err) - } - unknown, err := j.IsUnknownProduct(context.Background(), raw, nil) - if err != nil || !unknown { - t.Fatalf("unknown = %v, %v", unknown, err) - } - known := common.Frameworks{} - f := common.NewFramework("QuantumGate Portal", common.FrameFromGUESS) - f.Judge = &common.Judgement{Layer: LayerApplication} - known.Add(f) - unknown, err = j.IsUnknownProduct(context.Background(), raw, known) - if err != nil || unknown { - t.Fatalf("known application = %v, %v", unknown, err) + all, err := j.Inspect(context.Background(), []byte(jenkinsRaw), testFrames()) + if err == nil || all["wordpress"].Judge != nil || (all["apache tomcat"].Judge != nil && all["apache tomcat"].Judge.Outcome != "") || all["nginx"].Judge.Outcome != jev.Holds.String() { + t.Fatalf("partially applied invalid batch: %v %v", all, err) } } -func TestKnownCDNErrorPageIsNotUnknown(t *testing.T) { - raw := []byte("HTTP/1.1 403 Forbidden\r\nServer: cloudflare\r\n\r\nJust a moment...") - j := New(answerProvider(func(qs map[string]Question) map[string]Answer { - out := map[string]Answer{} - for key := range qs { - if key == "page_kind" { - out[key] = Answer{Choice: string(KindError)} - } else { - out[key] = Answer{Yes: 0.95} - } - } - return out - })) - f := common.NewFramework("cloudflare", common.FrameFromFingers) - f.Judge = &common.Judgement{Layer: LayerCDN} - accepted := common.Frameworks{} - accepted.Add(f) - unknown, err := j.IsUnknownProduct(context.Background(), raw, accepted) - if err != nil || unknown { - t.Fatalf("known CDN error = %t, %v", unknown, err) +func TestInspectReevaluatesAndCopiesInputs(t *testing.T) { + m := newMock() + j := New(m) + frames := testFrames() + f := frames["wordpress"] + f.MatchDetail = &common.MatchDetail{MatcherValue: "WordPress"} + f.Judge = &common.Judgement{Verdict: OptionRunning, Outcome: jev.Holds.String(), Evidence: []string{"old"}} + before, _ := json.Marshal(frames) + all, err := j.Inspect(context.Background(), []byte(jenkinsRaw), frames) + if err != nil || !all["wordpress"].Judge.Rejected { + t.Fatalf("old judgement reused: %v %v", all, err) + } + all["wordpress"].MatchDetail.MatcherValue = "changed" + all["wordpress"].Tags = append(all["wordpress"].Tags, "changed") + all["wordpress"].Froms[common.FrameFromFingers] = true + all["wordpress"].Version = "changed" + all["wordpress"].Judge.Evidence[0] = "changed" + after, _ := json.Marshal(frames) + if string(before) != string(after) { + t.Fatal("result mutated input") + } + partial, err := j.Inspect(context.Background(), []byte("not HTTP"), frames) + if err == nil || len(partial) != len(frames) || partial["wordpress"].Judge != nil { + t.Fatalf("parse failure lost baseline: %v %v", partial, err) + } + partial["wordpress"].MatchDetail.MatcherValue = "also changed" + if frames["wordpress"].MatchDetail.MatcherValue != "WordPress" { + t.Fatal("parse-error baseline shares pointers") + } + copied := cloneFramework(f) + copied.Judge.Evidence[0] = "changed" + if f.Judge.Evidence[0] != "old" { + t.Fatal("clone shares judgement evidence") } } diff --git a/judge/page.go b/judge/page.go deleted file mode 100644 index 9ee7c08..0000000 --- a/judge/page.go +++ /dev/null @@ -1,258 +0,0 @@ -package judge - -import ( - "bufio" - "bytes" - "hash/fnv" - "io" - "net/http" - "regexp" - "strings" - "unicode/utf8" - - "github.com/chainreactors/fingers/judge/internal/evidence" -) - -// Page is the compact, named view of one HTTP response that providers judge -// (serialized as the request state). A provider only judges the evidence it -// is given and loses accuracy on large states full of irrelevant detail, so -// the fields below are the ones that carry fingerprint signal. Pass one to -// Yes, Choose or Score to ask custom questions about a response. -type Page struct { - Status string `json:"status"` - Headers map[string]string `json:"headers,omitempty"` - Cookies []string `json:"cookie_names,omitempty"` - Title string `json:"title,omitempty"` - Generator string `json:"meta_generator,omitempty"` - Description string `json:"meta_description,omitempty"` - Scripts []string `json:"script_src,omitempty"` - Styles []string `json:"stylesheet_href,omitempty"` - InlineHints []string `json:"inline_script_starts,omitempty"` // first chars of inline scripts, e.g. "(function(w,d,s,l,i){ ... GTM" - Comments []string `json:"html_comments,omitempty"` - Forms []string `json:"form_inputs,omitempty"` - Text string `json:"visible_text,omitempty"` - - raw []byte -} - -var ( - // Headers that carry no fingerprint signal, or only noise; every other - // header is kept, since products announce themselves in custom headers - // ("product: Z-BlogPHP 1.7.4", "X-Jenkins", "kbn-name"). - skipHeaders = map[string]bool{"Date": true, "Content-Length": true, "Content-Type": true, "Expires": true, "Cache-Control": true, - "Pragma": true, "Etag": true, "Last-Modified": true, "Age": true, "Connection": true, "Keep-Alive": true, "Accept-Ranges": true, - "Vary": true, "Set-Cookie": true, "Content-Security-Policy": true, "Content-Security-Policy-Report-Only": true, "Report-To": true, - "Nel": true, "Permissions-Policy": true, - // per-request values: no signal, and they would defeat the cache - "X-Request-Id": true, "X-Amzn-Trace-Id": true, "Cf-Ray": true, "X-Runtime": true, "Server-Timing": true} - - reDescription = regexp.MustCompile(`(?is)]+name=["']description["'][^>]*content=["']([^"']+)|]+content=["']([^"']+)["'][^>]*name=["']description["']`) - reScript = regexp.MustCompile(`(?is)]+src=["']([^"']+)`) - reStyle = regexp.MustCompile(`(?is)]+rel=["']?stylesheet[^>]*href=["']([^"']+)|]+href=["']([^"']+)["'][^>]*rel=["']?stylesheet`) - reInline = regexp.MustCompile(`(?is)]*)?>(.*?)`) - reComment = regexp.MustCompile(`(?s)`) - reInput = regexp.MustCompile(`(?is)]*>`) - reAttrName = regexp.MustCompile(`(?is)\b(?:name|id)=["']([^"']+)`) - reAttrType = regexp.MustCompile(`(?is)\btype=["']([^"']+)`) - reDropBlock = regexp.MustCompile(`(?is)<(script|style|noscript)[^>]*>.*?`) - reTag = regexp.MustCompile(`(?s)<[^>]+>`) - // Allows a v/V/x/X prefix ("X3.4", "V8.1SP2") and a letter suffix; rejects - // digits glued to other digits or dots so IPs and long builds stay out. - reGenMajor = regexp.MustCompile(`(?i)generator["'][^>]*content=["'][A-Za-z][^"']*?\s[vV]?(\d{1,3})(?:[\s"'(]|$)|content=["'][A-Za-z][^"']*?\s[vV]?(\d{1,3})(?:[\s(][^"']*)?["'][^>]*name=["']generator`) - reVersion = regexp.MustCompile(`(?:^|[^0-9A-Za-z.])[vVxX]?(` + evidence.VersionToken + `)(?:[^0-9A-Za-z.]|\.[A-Za-z]|$)`) -) - -const ( - maxHeaderValue = 160 - maxText = 1500 // visible text runes -) - -// NewPage parses a raw HTTP response (head and body, as rule engines take it). -func NewPage(raw []byte) (*Page, error) { - resp, err := http.ReadResponse(bufio.NewReader(bytes.NewReader(raw)), nil) - if err != nil { - return nil, err - } - defer resp.Body.Close() - body, _ := io.ReadAll(resp.Body) - s := &Page{Status: resp.Status, Headers: map[string]string{}, raw: raw} - for k, vs := range resp.Header { - if skipHeaders[k] || strings.HasPrefix(k, "X-Crawler-") || len(vs) == 0 { - continue - } - s.Headers[k] = truncate(strings.Join(vs, ", "), maxHeaderValue) - } - for _, c := range resp.Cookies() { - s.Cookies = append(s.Cookies, c.Name) - } - html := string(body) - if m := evidence.Title.FindStringSubmatch(html); m != nil { - s.Title = clean(m[1]) - } - if m := evidence.Generator.FindStringSubmatch(html); m != nil { - s.Generator = m[1] + m[2] - } - if m := reDescription.FindStringSubmatch(html); m != nil { - s.Description = truncate(clean(m[1]+m[2]), 300) - } - for _, m := range reScript.FindAllStringSubmatch(html, 8) { - s.Scripts = append(s.Scripts, m[1]) - } - for _, m := range reStyle.FindAllStringSubmatch(html, 8) { - s.Styles = append(s.Styles, m[1]+m[2]) - } - for _, m := range reInline.FindAllStringSubmatch(html, -1) { - if code := clean(m[1]); len(code) > 20 && len(s.InlineHints) < 5 { - s.InlineHints = append(s.InlineHints, truncate(code, 100)) - } - } - for _, m := range reComment.FindAllStringSubmatch(html, -1) { - if c := clean(m[1]); len(c) > 3 && len(s.Comments) < 5 && !strings.HasPrefix(c, "[if") { - s.Comments = append(s.Comments, truncate(c, 80)) - } - } - for _, in := range reInput.FindAllString(html, 12) { - name, typ := "", "text" - if m := reAttrName.FindStringSubmatch(in); m != nil { - name = m[1] - } - if m := reAttrType.FindStringSubmatch(in); m != nil { - typ = m[1] - } - if name != "" { - s.Forms = append(s.Forms, name+":"+typ) - } - } - text := clean(reTag.ReplaceAllString(reDropBlock.ReplaceAllString(html, " "), " ")) - s.Text = truncate(text, maxText) - return s, nil -} - -// haystack is the lowercase text that candidate names are searched in. -func (s *Page) haystack() string { - var b strings.Builder - for k, v := range s.Headers { - b.WriteString(k + ": " + v + "\n") - } - b.WriteString(strings.Join(s.Cookies, " ") + "\n") - b.WriteString(s.Title + "\n" + s.Generator + "\n") - b.WriteString(s.Description + "\n") - b.WriteString(strings.Join(s.Scripts, " ") + "\n") - b.WriteString(strings.Join(s.Styles, " ") + "\n") - b.WriteString(strings.Join(s.InlineHints, " ") + "\n") - b.WriteString(strings.Join(s.Comments, " ") + "\n") - b.WriteString(s.Text) - return strings.ToLower(b.String()) -} - -func clean(s string) string { return evidence.Clean(s) } - -func truncate(s string, max int) string { - if utf8.RuneCountInString(s) <= max { - return s - } - return string([]rune(s)[:max]) + "…" -} - -// where reports where a candidate name literally occurs in the page. It is -// a cheap code-only baseline for false positive detection: a name found only -// in visible text is a likely false positive. -func (s *Page) where(name string) []string { - n := strings.ToLower(name) - if len(n) < 3 { - return nil - } - var where []string - check := func(label, text string) { - if strings.Contains(strings.ToLower(text), n) { - where = append(where, label) - } - } - var headers strings.Builder - for k, v := range s.Headers { - headers.WriteString(k + ":" + v + " ") - } - check("header", headers.String()) - check("cookie", strings.Join(s.Cookies, " ")) - check("title", s.Title) - check("meta", s.Generator) - check("description", s.Description) - check("script", strings.Join(s.Scripts, " ")+" "+strings.Join(s.InlineHints, " ")) - check("style", strings.Join(s.Styles, " ")) - check("comment", strings.Join(s.Comments, " ")) - check("form", strings.Join(s.Forms, " ")) - check("text", s.Text) - return where -} - -var reDigits = regexp.MustCompile(`[0-9]+`) - -// bigrams are overlapping two-rune shingles of text with spaces removed, so -// text without word separators (Chinese, Japanese) weighs like any other. -func bigrams(text string) []string { - rs := []rune(strings.Join(strings.Fields(strings.ToLower(text)), " ")) - var out []string - for i := 0; i+1 < len(rs); i++ { - if rs[i] != ' ' && rs[i+1] != ' ' { - out = append(out, string(rs[i:i+2])) - } - } - return out -} - -// similarityScope is what similar pages must share exactly: their title -// (digits folded). Signatures alone collide on sparse pages dominated by -// common headers and assets; pages that really are one product's stock page -// share a title. -func (s *Page) similarityScope() string { - return reDigits.ReplaceAllString(strings.ToLower(strings.TrimSpace(s.Title)), "0") -} - -// signature is a 64-bit simhash of the page's evidence. Digits are folded so -// timestamps, build ids and tokens do not matter; structure (headers, -// assets, forms) weighs more than visible text, which counts by two-rune -// shingles. Near-identical pages get signatures a few bits apart; see -// Judge.SimilarDistance. The cache also requires equal titles. -func (s *Page) signature() uint64 { - var v [64]int - add := func(kind, feature string, weight int) { - h := fnv.New64a() - h.Write([]byte(kind)) - h.Write([]byte(reDigits.ReplaceAllString(strings.ToLower(feature), "0"))) - x := h.Sum64() - for i := 0; i < 64; i++ { - if x&(1<= 0 { - f = f[:i] - } - add("struct", f, 3) - } - } - add("generator", s.Generator, 3) - for _, g := range bigrams(s.Title) { - add("title", g, 2) - } - for _, g := range bigrams(s.Text) { - add("text", g, 1) - } - var sig uint64 - for i := 0; i < 64; i++ { - if v[i] > 0 { - sig |= 1 << uint(i) - } - } - return sig -} diff --git a/judge/presence.go b/judge/presence.go new file mode 100644 index 0000000..51b0ed0 --- /dev/null +++ b/judge/presence.go @@ -0,0 +1,129 @@ +package judge + +import ( + "context" + "fmt" + "strconv" + + "github.com/chainreactors/fingers/common" + "github.com/chainreactors/fingers/judge/internal/evidence" + "github.com/chainreactors/utils/jev" +) + +// Presence options, as written to Framework.Judge.Verdict: the option a +// presence claim was ruled with. Declared and absent are rulings code makes +// from the response head; the rest are the provider's. +const ( + OptionDeclared = "declared" // holds: the response names the product in an explicit header declaration, or shows the protocol feature + OptionAbsent = "absent" // refuted: a protocol feature the response head does not show + OptionRunning = "running" // holds: the evidence shows the product produced the response + OptionMentioned = "mentioned" // refuted: the product only appears in the page's content + OptionUnrelated = "unrelated" // refuted: the rule matched text that has nothing to do with the product +) + +// matchExcerpt is one excerpt of the response a claim rests on, as the provider +// reads it. +type matchExcerpt struct { + Where string `json:"where"` // "header" or "body" + Matched string `json:"matched,omitempty"` // the text the rule matched, when the engine records it + Text string `json:"text"` // the matched text with its context +} + +func (e matchExcerpt) String() string { + if e.Matched != "" { + return e.Where + " (matched " + strconv.Quote(e.Matched) + "): " + e.Text + } + return e.Where + ": " + e.Text +} + +// presenceClaim is a rule's claim that name is part of the software stack +// that produced the response; state.matches[id] quotes where it matched. +func presenceClaim(id, name string) jev.Claim { + return jev.Claim{ + Statement: fmt.Sprintf("`matches.%s` contains evidence for the fingerprint rule's claim that `%s` is part of the software stack "+ + "that produced this HTTP response; it quotes the response where the rule matched: `matched` is the text "+ + "the rule matched, `text` that text in its context; evidence without `matched` is where the product's name occurs. "+ + "Judge the claim from the evidence and the response. On documentation, README or tutorial pages the documented "+ + "product is mentioned, not running; the documentation generator is running.", id, name), + Options: map[string]jev.Option{ + OptionRunning: {Description: "`" + name + "` served, generated or is loaded by this response: headers, cookies, asset paths, markup, " + + "or the stock text of its own login, console, default or error page. A packaged browser-side application counts.", Outcome: jev.Holds}, + OptionMentioned: {Description: "`" + name + "` only appears in content: an article, README, documentation, link or list.", Outcome: jev.Refuted}, + OptionUnrelated: {Description: "The matched text has nothing to do with `" + name + "`: it is a fragment of encoded or random data, " + + "part of another word, or another product's asset.", Outcome: jev.Refuted}, + jev.OptionInsufficient: {Description: "The evidence shows neither.", Outcome: jev.Insufficient}, + }, + } +} + +// presence applies local facts and asks one batch for the remaining products. +func (j *Judge) presence(ctx context.Context, p *evidence.Page, frames common.Frameworks) error { + claims := map[string]jev.Claim{} + targets := map[string][]*common.Framework{} + matches := map[string][]matchExcerpt{} + for _, group := range groupProducts(frames) { + name := group[0].Name + for _, f := range group[1:] { + f.Judge = &common.Judgement{Duplicate: true} + } + key := NormalizeName(name) + excerpts := evidenceFor(p.Raw, group) + option := "" + if protocolFeatures[key] { + option = OptionAbsent + if protocolPresent(p, key) { + option = OptionDeclared + } + } else if declares(p, name) { + option = OptionDeclared + } + if option != "" { + claim := jev.Claim{Statement: "The response declares " + name, Options: map[string]jev.Option{ + OptionDeclared: {Description: "Explicit product declaration or protocol feature", Outcome: jev.Holds}, + OptionAbsent: {Description: "The protocol feature is absent", Outcome: jev.Refuted}, + jev.OptionInsufficient: {Description: insufficientDescription, Outcome: jev.Insufficient}, + }} + if err := j.applyPresence(group, excerpts, claim, jev.Ruling{Option: option, Confidence: 1}); err != nil { + return err + } + continue + } + if len(claims) >= maxClaims { + continue + } + id := "presence_" + key + claims[id], targets[id], matches[id] = presenceClaim(id, name), group, excerpts + } + rulings, err := j.judge(ctx, map[string]interface{}{"response": p, "matches": matches}, claims) + if err != nil { + return err + } + for id, ruling := range rulings { + if err := j.applyPresence(targets[id], matches[id], claims[id], ruling); err != nil { + return err + } + } + return nil +} + +func (j *Judge) rejects(o jev.Outcome) bool { + return o == jev.Refuted || (o == jev.Insufficient && j.DropInsufficient) +} + +// applyPresence is shared by deterministic and provider presence rulings. +func (j *Judge) applyPresence(frames []*common.Framework, evidence []matchExcerpt, claim jev.Claim, ruling jev.Ruling) error { + if err := validRuling(claim, ruling); err != nil { + return err + } + outcome := claim.Resolve(ruling, j.MinConfidence) + var excerpts []string + for _, e := range evidence { + excerpts = append(excerpts, e.String()) + } + for i, f := range frames { + f.Judge = &common.Judgement{Verdict: ruling.Option, Outcome: outcome.String(), + Evidence: append([]string(nil), excerpts...), Confidence: ruling.Confidence, + Rejected: j.rejects(outcome), Duplicate: i > 0} + } + return nil +} diff --git a/judge/provider.go b/judge/provider.go deleted file mode 100644 index 2be13dc..0000000 --- a/judge/provider.go +++ /dev/null @@ -1,70 +0,0 @@ -package judge - -import "context" - -// Provider answers typed questions about a state. It is how a model plugs in: -// TypeSafe Jev (package judge/jev), an LLM, a local classifier or a -// cross-encoder reranker. A provider only transports questions and answers; -// caching, request merging, thresholds and every decision stay in Judge and -// the capabilities, so they behave the same whichever provider answers. -type Provider interface { - // ID names the provider and model version. It scopes cached answers: - // change it whenever the same question may be answered differently. - ID() string - // Judge answers every question in one call, keyed as questions are. - // Every requested key must have an answer; missing answers fail the round. - Judge(ctx context.Context, state interface{}, questions map[string]Question) (map[string]Answer, error) -} - -// Calibrated is implemented by providers whose answers were measured on a -// labelled set (see cmd/judgeeval): New then uses their thresholds. -type Calibrated interface { - // Calibration returns the Yes probability that counts as "yes" and the - // confidence at which a picked version is adopted. - Calibration() (threshold, versionConfidence float64) -} - -// QuestionType is the answer shape a question asks for. -type QuestionType string - -const ( - TypeBinary QuestionType = "binary" // Answer.Yes: probability of "yes"; independent per question - TypeChoice QuestionType = "choice" // Answer.Choice: one of Options, with Confidence: a rerank over options - TypeScore QuestionType = "score" // Answer.Score: position on Levels, from 0 to 1 -) - -// Question is provider-neutral: plain instructions plus the options or levels. -type Question struct { - Type QuestionType `json:"type"` - Instructions string `json:"instructions"` - // Options: for TypeChoice, option -> description ("" for none); for - // TypeBinary, optional descriptions of "true" and "false". - Options map[string]string `json:"options,omitempty"` - Levels []string `json:"levels,omitempty"` // TypeScore: 2..10 ordered level descriptions -} - -func Binary(instructions string) Question { - return Question{Type: TypeBinary, Instructions: instructions} -} - -// BinaryWith describes what "yes" and "no" mean, which sharpens the answer. -func BinaryWith(instructions, yes, no string) Question { - return Question{Type: TypeBinary, Instructions: instructions, Options: map[string]string{"true": yes, "false": no}} -} - -func Choice(instructions string, options map[string]string) Question { - return Question{Type: TypeChoice, Instructions: instructions, Options: options} -} - -func Score(instructions string, levels []string) Question { - return Question{Type: TypeScore, Instructions: instructions, Levels: levels} -} - -// Answer is one answer; only the fields of the question's type are set. -type Answer struct { - Yes float64 `json:"yes,omitempty"` - Choice string `json:"choice,omitempty"` - Score float64 `json:"score,omitempty"` - Probabilities map[string]float64 `json:"probabilities,omitempty"` // choice: per option - Confidence float64 `json:"confidence,omitempty"` -} diff --git a/judge/real_evidence_test.go b/judge/real_evidence_test.go index 3f5e4c7..69adc41 100644 --- a/judge/real_evidence_test.go +++ b/judge/real_evidence_test.go @@ -6,6 +6,8 @@ import ( "testing" "github.com/chainreactors/fingers/common" + "github.com/chainreactors/fingers/judge/internal/evidence" + "github.com/chainreactors/utils/jev" ) // Minimal public evidence retained from the 2026-09-25 corpus, not full pages. @@ -15,11 +17,11 @@ func TestStaticApplicationDescriptionIsPreservedAsEvidence(t *testing.T) { ``, ``, } { - p, err := NewPage([]byte("HTTP/1.1 200 OK\r\n\r\nIT Tools - Handy online tools for developers" + meta)) - if err != nil || p.Description != description || !strings.Contains(p.haystack(), strings.ToLower(description)) { + p, err := evidence.NewPage([]byte("HTTP/1.1 200 OK\r\n\r\nIT Tools - Handy online tools for developers" + meta)) + if err != nil || p.Description != description { t.Fatalf("description evidence missing: %+v err=%v", p, err) } - if strings.Contains(strings.Join(p.where("it tools"), ","), "header") { + if declares(p, "it tools") { t.Fatal("description incorrectly treated as deterministic header evidence") } } @@ -60,49 +62,37 @@ func TestVersionCandidatesDoNotDiscardLateProductEvidence(t *testing.T) { t.Fatal("early numeric noise evicted late library version") } } + +// Protocol features are facts of the response head: decided by code, never +// asked, whatever a provider would answer. func TestProtocolHitRequiresResponseEvidence(t *testing.T) { for _, tc := range []struct { name, header string - yes bool want bool - }{{"http基本认证", "", true, false}, {"hsts", "", true, false}, {"http基本认证", "WWW-Authenticate: Basic realm=\"test\"\r\n", false, true}, {"hsts", "Strict-Transport-Security: max-age=31536000\r\n", false, true}} { - j := New(answerProvider(func(qs map[string]Question) map[string]Answer { - out := map[string]Answer{} - for k, q := range qs { - if q.Type == TypeBinary { - v := .01 - if tc.yes { - v = .99 - } - out[k] = Answer{Yes: v} - } else if k == "page_kind" { - out[k] = Answer{Choice: string(KindLogin)} - } else if k == "primary" { - out[k] = Answer{Choice: noneOfThem} - } else { - out[k] = Answer{Choice: string(LayerNotPresent)} - } - } - return out + }{{"http基本认证", "", false}, {"hsts", "", false}, {"http基本认证", "WWW-Authenticate: Basic realm=\"test\"\r\n", true}, {"hsts", "Strict-Transport-Security: max-age=31536000\r\n", true}} { + asked := false + j := New(answerProvider(func(qs map[string]jev.Claim) map[string]jev.Ruling { + asked = true + return nil })) hits := common.Frameworks{} hits.Add(common.NewFramework(tc.name, common.FrameFromGUESS)) all, err := j.Inspect(context.Background(), []byte("HTTP/1.1 200 OK\r\n"+tc.header+"\r\n"), hits) got := all.Accepted() - if err != nil || (len(got) > 0) != tc.want { - t.Errorf("%s evidence=%q got=%v err=%v", tc.name, tc.header, got, err) + if err != nil || asked || (len(got) > 0) != tc.want { + t.Errorf("%s evidence=%q got=%v asked=%v err=%v", tc.name, tc.header, got, asked, err) } } } func TestCalendarVersionAndGeneratorName(t *testing.T) { raw := []byte("HTTP/1.1 200 OK\r\n\r\nCustom Search") - p, err := NewPage(raw) + p, err := evidence.NewPage(raw) if err != nil { t.Fatal(err) } foundName, foundVersion := false, false - for _, name := range pageNames(p) { + for _, name := range evidence.Names(p.Generator, p.Title, p.Text, p.Headers, p.Scripts) { foundName = foundName || NormalizeName(name) == "searxng" } for _, v := range extractVersions(raw, 40) { @@ -112,3 +102,30 @@ func TestCalendarVersionAndGeneratorName(t *testing.T) { t.Fatalf("name=%t version=%t", foundName, foundVersion) } } + +func TestBasicChallengeUsesFullHeaderAndQuotedRealms(t *testing.T) { + for _, tc := range []struct { + header string + want bool + }{ + {"WWW-Authenticate: Digest realm=\"" + strings.Repeat("x", 200) + "\", Basic realm=\"private\"\r\n", true}, + {"WWW-Authenticate: Digest realm=\"x, Basic fake\"\r\n", false}, + {"WWW-Authenticate: Digest realm=\"private\"\r\nWWW-Authenticate: Basic realm=\"private\"\r\n", true}, + } { + p, err := evidence.NewPage([]byte("HTTP/1.1 401 Unauthorized\r\n" + tc.header + "\r\n")) + if err != nil || protocolPresent(p, "http基本认证") != tc.want { + t.Fatalf("header %q: %v", tc.header, err) + } + } +} + +func TestIncidentalHeaderNamesAreNotDeclarations(t *testing.T) { + raw := []byte("HTTP/1.1 302 Found\r\nLocation: https://docs.test/jenkins/2.401.3\r\nSet-Cookie: jenkins=x\r\nX-Github-Request-Id: jenkins\r\n\r\n") + p, err := evidence.NewPage(raw) + if err != nil { + t.Fatal(err) + } + if declares(p, "jenkins") || declaredVersion(declarations(raw), "jenkins") != "" { + t.Fatal("incidental URL/cookie text became a fact") + } +} diff --git a/judge/recall.go b/judge/recall.go deleted file mode 100644 index 15de17c..0000000 --- a/judge/recall.go +++ /dev/null @@ -1,74 +0,0 @@ -package judge - -import ( - "sort" - "strings" - "unicode/utf8" - - "github.com/chainreactors/fingers/judge/internal/evidence" -) - -// Retriever finds fingerprint names that literally occur in a page. Every -// candidate handed to the provider is therefore anchored to evidence found -// by code, so the provider only ranks and verifies, it never invents a product. -type Retriever struct { - names []string // display names - keys []string // lowercase match keys, same index as names -} - -func NewRetriever(names []string) *Retriever { - r := &Retriever{} - seen := map[string]bool{} - for _, n := range names { - k := strings.ToLower(strings.TrimSpace(n)) - if seen[k] || !evidence.UsableKey(k) { - continue - } - seen[k] = true - r.names = append(r.names, n) - r.keys = append(r.keys, k) - } - return r -} - -// containsWord matches ASCII keys on word boundaries ("acti" must not match -// "action"); CJK keys have no word boundaries and match as substrings. -func containsWord(haystack, key string) bool { - if utf8.RuneCountInString(key) != len(key) { - return strings.Contains(haystack, key) - } - for from := 0; ; { - i := strings.Index(haystack[from:], key) - if i < 0 { - return false - } - start, end := from+i, from+i+len(key) - if (start == 0 || !isWordByte(haystack[start-1])) && (end == len(haystack) || !isWordByte(haystack[end])) { - return true - } - from = start + 1 - } -} - -func isWordByte(b byte) bool { - return b == '_' || b >= '0' && b <= '9' || b >= 'a' && b <= 'z' || b >= 'A' && b <= 'Z' -} - -// Find returns up to max names occurring in haystack, longest match first. -func (r *Retriever) Find(haystack string, max int) []string { - var hits []int - for i, k := range r.keys { - if containsWord(haystack, k) { - hits = append(hits, i) - } - } - sort.Slice(hits, func(a, b int) bool { return len(r.keys[hits[a]]) > len(r.keys[hits[b]]) }) - var out []string - for _, i := range hits { - out = append(out, r.names[i]) - if len(out) >= max { - break - } - } - return out -} diff --git a/judge/refine.go b/judge/refine.go deleted file mode 100644 index 5dc16ab..0000000 --- a/judge/refine.go +++ /dev/null @@ -1,90 +0,0 @@ -package judge - -import ( - "context" - "sort" - - "github.com/chainreactors/fingers/common" -) - -// inspect judges a copy of frames plus the known names found in the page. -// When the Judge caches answers, the page is classified in the same request, -// so a later Classify of this response costs nothing. -func (j *Judge) inspect(ctx context.Context, p *Page, frames common.Frameworks) (common.Frameworks, error) { - working := cloneFrameworks(frames) - r := newRound(p) - verifyRound(r, working, j.known(p)) - if j.Cache != nil { - var kind Kind - var generic bool - classifyRound(r, &kind, &generic) - } - if err := r.ask(ctx, j); err != nil { - return nil, err - } - return working, nil -} - -func (j *Judge) known(p *Page) []string { - if j.Known == nil { - return nil - } - return j.Known.Find(p.haystack(), maxKnown) -} - -// byImportance orders frames for versioning: the primary application first, -// then applications, servers and the rest, each by name. -func byImportance(frames common.Frameworks) []*common.Framework { - rank := func(f *common.Framework) int { - switch { - case f.Judge == nil: - return 3 - case f.Judge.Primary: - return 0 - case f.Judge.Layer == LayerApplication || f.Judge.Layer == LayerDevice: - return 1 - case f.Judge.Layer == LayerFrontend: - return 3 - } - return 2 - } - list := frames.List() - sort.Slice(list, func(a, b int) bool { - if ra, rb := rank(list[a]), rank(list[b]); ra != rb { - return ra < rb - } - return list[a].Name < list[b].Name - }) - return list -} - -func cloneFrameworks(frames common.Frameworks) common.Frameworks { - working := make(common.Frameworks, len(frames)) - for name, f := range frames { - working[name] = cloneFramework(f) - } - return working -} - -func cloneFramework(f *common.Framework) *common.Framework { - if f == nil { - return nil - } - copy := *f - copy.Tags = append([]string(nil), f.Tags...) - if f.Froms != nil { - copy.Froms = make(map[common.From]bool, len(f.Froms)) - for from, present := range f.Froms { - copy.Froms[from] = present - } - } - if f.Attributes != nil { - attrs := *f.Attributes - copy.Attributes = &attrs - } - if f.Judge != nil { - judgement := *f.Judge - copy.Judge = &judgement - } - return © -} diff --git a/judge/round.go b/judge/round.go deleted file mode 100644 index 485b6dc..0000000 --- a/judge/round.go +++ /dev/null @@ -1,88 +0,0 @@ -package judge - -import ( - "context" - "encoding/json" - "fmt" -) - -// round is the judge primitive: questions about one page, each with a -// callback, sent as a single request. Capabilities (verifyRound, -// classifyRound, versionRound) add to a round. Callbacks run only after the -// whole request succeeded, so a failed round leaves every annotated value -// untouched. -type round struct { - page *Page - questions map[string]Question - apply map[string]func(Answer) - evidence map[string]interface{} - done []func() - judge *Judge // set by ask, for thresholds in callbacks - err error -} - -func newRound(p *Page) *round { - return &round{page: p, questions: map[string]Question{}, apply: map[string]func(Answer){}} -} - -// add asks q under key; apply receives its answer. Keys must be unique in a round. -func (r *round) add(key string, q Question, apply func(Answer)) { - if _, dup := r.questions[key]; dup && r.err == nil { - r.err = fmt.Errorf("judge: duplicate question key %q", key) - } - r.questions[key] = q - r.apply[key] = apply -} - -// then registers f to run after every answer callback, in registration -// order: for decisions that combine several answers. -func (r *round) then(f func()) { r.done = append(r.done, f) } - -// show gives the provider a named piece of evidence besides the page; the -// state becomes {"response": page, name: value}. Providers read literally: -// evidence they are asked about by name weighs more than the same text in -// option descriptions. -func (r *round) show(name string, value interface{}) { - if r.evidence == nil { - r.evidence = map[string]interface{}{} - } - r.evidence[name] = value -} - -// ask sends the round; a round without questions sends nothing. -func (r *round) ask(ctx context.Context, j *Judge) error { - if r.err != nil { - return r.err - } - if len(r.questions) == 0 { - return nil - } - var state interface{} = r.page - if len(r.evidence) > 0 { - m := map[string]interface{}{"response": r.page} - for k, v := range r.evidence { - m[k] = v - } - state = m - } - // Answers are cached per question: the page matches by signature (similar - // pages share answers) plus its title, named evidence exactly. - scope, err := json.Marshal([]interface{}{r.page.similarityScope(), r.evidence}) - if err != nil { - return err - } - answers, err := j.ask(ctx, state, r.questions, scope, r.page.signature()) - if err != nil { - return err - } - r.judge = j - for key, apply := range r.apply { - if a, ok := answers[key]; ok && apply != nil { - apply(a) - } - } - for _, f := range r.done { - f() - } - return nil -} diff --git a/judge/stack.go b/judge/stack.go deleted file mode 100644 index d2becdb..0000000 --- a/judge/stack.go +++ /dev/null @@ -1,35 +0,0 @@ -package judge - -import "github.com/chainreactors/fingers/judge/internal/evidence" - -// Layers are the roles a fingerprint plays in the software stack behind one -// response, as written to Framework.Judge.Layer. A page normally carries -// several fingerprints at different layers (CDN + web server + framework + -// application + libraries); each candidate is judged independently. -const ( - LayerCDN = "cdn_or_waf" - LayerServer = "web_server" - LayerRuntime = "language_runtime" - LayerFramework = "web_framework" - LayerApplication = "application" - LayerFrontend = "frontend_library" - LayerDevice = "os_or_device" - LayerNotPresent = "not_present" -) - -// layerCriteria is the Choice criteria for classifying one candidate's layer. -var layerCriteria = map[string]string{ - LayerCDN: "A CDN, reverse proxy cloud or web application firewall in front of the site", - LayerServer: "The HTTP server or proxy software, such as nginx, Apache httpd, IIS, Tomcat", - LayerRuntime: "A programming language or runtime, such as PHP, Java, ASP.NET, Node.js", - LayerFramework: "A backend web framework, such as Spring, Django, Laravel, ThinkPHP", - LayerApplication: "The application whose interface this is: CMS, OA, admin console, SaaS app, forum, monitoring tool, search engine, or packaged static/browser-side tools; a documentation subject is not the application serving the docs", - LayerFrontend: "A browser-side library, UI kit, analytics or tag script, such as jQuery, Bootstrap, Google Analytics", - LayerDevice: "An operating system or the firmware of a hardware device such as a router, camera, NAS or firewall", - LayerNotPresent: "Not part of the software serving this response: only mentioned in text, or absent", -} - -// NormalizeName folds the spellings different engines use for one product -// ("Apache-Tomcat", "apache tomcat", "Apache HTTP Server", "apache") into one -// key. It never merges names whose remaining letters differ. -func NormalizeName(name string) string { return evidence.NormalizeName(name) } diff --git a/judge/verify.go b/judge/verify.go deleted file mode 100644 index cc0ef1d..0000000 --- a/judge/verify.go +++ /dev/null @@ -1,203 +0,0 @@ -package judge - -import ( - "fmt" - "sort" - "strings" - - "github.com/chainreactors/fingers/common" -) - -const noneOfThem = "none_of_these" // primary option for pages of no listed product - -// protocolFeatures are HTTP features some engines report as fingerprints -// (NormalizeName keys). They must be established by this response's head. -var protocolFeatures = map[string]bool{"hsts": true, "altsvc": true, "http3": true, "http2": true, "poweredby": true, "http基本认证": true} - -const ( - maxGroups = 40 // products judged per page - maxRecall = 8 // recall names judged per page - maxKnown = 24 // known names looked up per page -) - -// product is one product claimed for the page: every engine spelling of it -// (frames), or a recall name without a rule hit. -type product struct { - name string - frames []*common.Framework - strong bool // named in a header or cookie: code accepts it, models under-trust it - feature bool // protocol features have deterministic response-head evidence -} - -// verifyRound is the engine result filter. It folds the spellings of all -// engines into products, asks the provider per product whether it is really -// in the stack (a binary question, so several products can be present) and at -// which layer (a choice), and which product is the page's application. -// recall names (fingerprint names found in the page) that no rule reported -// are judged the same way; confirmed ones are added to frames. Code decides: -// a product named in a header or cookie is never rejected. -// -// The verdict is written to Framework.Judge. Frameworks that already carry -// one are not asked again. Past maxGroups products, hits are left unjudged -// (Judge == nil), which Frameworks.Accepted keeps. -func verifyRound(r *round, frames common.Frameworks, recall []string) { - page := r.page - var list []*common.Framework - judged := map[string]bool{} // keys judged before: not asked again, not recalled again - for _, f := range frames { - if f == nil { - continue - } - if f.Judge != nil { - judged[NormalizeName(f.Name)] = true - } else { - list = append(list, f) - } - } - sort.Slice(list, func(a, b int) bool { return list[a].Name < list[b].Name }) - byKey := map[string]*product{} - var products []*product - for _, f := range list { - key := NormalizeName(f.Name) - if key == "" { - continue - } - p, ok := byKey[key] - if !ok { - if len(products) >= maxGroups { - continue - } - p = &product{name: f.Name, feature: protocolFeatures[key]} - if p.feature { - p.strong = protocolPresent(page, key) - } - byKey[key] = p - products = append(products, p) - } - p.frames = append(p.frames, f) - for _, e := range page.where(f.Name) { - if !p.feature { - p.strong = p.strong || e == "header" || e == "cookie" - } - } - } - for _, p := range products { - sort.SliceStable(p.frames, func(a, b int) bool { - qa, qb := frameworkQuality(p.frames[a]), frameworkQuality(p.frames[b]) - if qa != qb { - return qa > qb - } - return p.frames[a].Name < p.frames[b].Name - }) - p.name = p.frames[0].Name - } - recalled := 0 - for _, name := range recall { - key := NormalizeName(name) - if _, ok := byKey[key]; ok || key == "" || judged[key] { - continue - } - if recalled >= maxRecall { - break - } - p := &product{name: name} - byKey[key] = p - products = append(products, p) - recalled++ - } - if len(products) == 0 { - return - } - - type answer struct { - present float64 - answered bool - layer string - } - answers := make([]answer, len(products)) - var primary string - primaryOpts := map[string]string{noneOfThem: "The page belongs to none of the listed products, e.g. a custom-built site"} - for i, p := range products { - i, name := i, p.name - primaryOpts[name] = "" - r.add(fmt.Sprintf("is_%d", i), BinaryWith(fmt.Sprintf("Is `%s` part of the software stack that produced this HTTP response "+ - "(seen in its headers, cookies, asset paths, page structure or title), rather than only mentioned in the page text? "+ - "A packaged browser-side or static web application counts even without a login or server backend. "+ - "On documentation, README or tutorial pages, the product being documented does not count as running; distinguish it from the documentation generator.", name), - name+" served, generated or is loaded by this response", - name+" only appears in the page's text, or is not present"), - func(a Answer) { answers[i].present, answers[i].answered = a.Yes, true }) - r.add(fmt.Sprintf("layer_%d", i), Choice(fmt.Sprintf("What role does `%s` play in producing this HTTP response?", name), layerCriteria), - func(a Answer) { answers[i].layer = a.Choice }) - } - r.add("primary", Choice("Which listed product is the main application this page belongs to "+ - "(the product whose login, console or content this is), as opposed to servers, frameworks and libraries underneath it?", primaryOpts), - func(a Answer) { - if a.Choice != noneOfThem { - primary = a.Choice - } - }) - - r.then(func() { - for i, p := range products { - a := answers[i] - rejected := !p.strong && (p.feature || a.answered && (a.layer == LayerNotPresent || a.present < r.judge.Threshold)) - recalled := p.frames == nil - if recalled { // recall: added only when the provider confirms it - if !a.answered || rejected { - continue - } - f := common.NewFramework(p.name, common.FrameFromGUESS) - frames.Add(f) - p.frames = []*common.Framework{f} - } - for j, f := range p.frames { - f.Judge = &common.Judgement{Layer: a.layer, Confidence: a.present, Rejected: rejected, Duplicate: j > 0, Recalled: recalled} - } - if p.name == primary && !rejected { - p.frames[0].Judge.Primary = true - } - } - }) -} - -func protocolPresent(p *Page, key string) bool { - switch key { - case "hsts": - return p.Headers["Strict-Transport-Security"] != "" - case "altsvc": - return p.Headers["Alt-Svc"] != "" - case "poweredby": - return p.Headers["X-Powered-By"] != "" - case "http2": - return strings.HasPrefix(string(p.raw), "HTTP/2") || strings.Contains(strings.ToLower(p.Headers["Alt-Svc"]), "h2=") - case "http3": - return strings.HasPrefix(string(p.raw), "HTTP/3") || strings.Contains(strings.ToLower(p.Headers["Alt-Svc"]), "h3=") - case "http基本认证": - for _, challenge := range strings.Split(p.Headers["Www-Authenticate"], ",") { - words := strings.Fields(challenge) - if len(words) > 0 && strings.EqualFold(words[0], "Basic") { - return true - } - } - } - return false -} - -// Prefer a representative that already carries a version and useful source -// metadata, since duplicate spellings are hidden from the accepted result. -func frameworkQuality(f *common.Framework) int { - quality := len(f.Froms) - if f.Attributes != nil { - if f.Version != "" { - quality += 100 - } - if f.Vendor != "" { - quality += 10 - } - } - if f.MatchDetail != nil { - quality++ - } - return quality -} diff --git a/judge/version.go b/judge/version.go index 9d2c06d..a351323 100644 --- a/judge/version.go +++ b/judge/version.go @@ -1,6 +1,7 @@ package judge import ( + "context" "fmt" "regexp" "sort" @@ -9,6 +10,17 @@ import ( "github.com/chainreactors/fingers/common" "github.com/chainreactors/fingers/judge/internal/evidence" + "github.com/chainreactors/utils/jev" +) + +var ( + // Allows a v/V/x/X prefix ("X3.4", "V8.1SP2") and a letter suffix; rejects + // digits glued to other digits or dots so IPs and long builds stay out. + reGenMajor = regexp.MustCompile(`(?i)generator["'][^>]*content=["'][A-Za-z][^"']*?\s[vV]?(\d{1,3})(?:[\s"'(]|$)|content=["'][A-Za-z][^"']*?\s[vV]?(\d{1,3})(?:[\s(][^"']*)?["'][^>]*name=["']generator`) + reVersion = regexp.MustCompile(`(?:^|[^0-9A-Za-z.])[vVxX]?(` + evidence.VersionToken + `)(?:[^0-9A-Za-z.]|\.[A-Za-z]|$)`) + // What precedes numbers reVersion catches that are never versions: the + // viewport's "initial-scale=1.0". + reNotVersion = regexp.MustCompile(`(?i)(?:initial|minimum|maximum)-scale\s*=\s*$`) ) const ( @@ -19,49 +31,34 @@ const ( maxVersioned = 8 // products versioned per page ) -// versionRound resolves the versions of frames that have none. Code decides -// first: a version the response binds to the product by name (a header such -// as "Server: nginx/1.24.0" or "X-Jenkins: 2.401.3", a generator meta such -// as "WordPress 7.0.3") is taken as is. For the rest the provider picks -// among extracted candidates, never generating a string: the candidates go -// into the state once, as a named list (Jev picked 100% right this way -// against 78% with them in the option descriptions), and each product's -// options are its own best-ranked ones. A version bound by name to another -// product is not offered, and one string is not given to two products. -func versionRound(r *round, frames ...*common.Framework) { - var targets []*common.Framework +// versions uses one finite-option claim per product, including local declarations. +func (j *Judge) versions(ctx context.Context, p *evidence.Page, frames ...*common.Framework) error { + decls := declarations(p.Raw) + extracted := extractVersions(p.Raw, maxVersions) + claims := map[string]jev.Claim{} + targets := map[string]*common.Framework{} + var shown []versionString + seen := map[string]bool{} + count := 0 for _, f := range frames { - if f != nil && (f.Attributes == nil || f.Attributes.Version == "") && len(targets) < maxVersioned { - targets = append(targets, f) + if f == nil || protocolFeatures[NormalizeName(f.Name)] || (f.Attributes != nil && f.Version != "") { + continue } - } - if len(targets) == 0 { - return - } - decls := declarations(r.page.raw) - var asked []*common.Framework - for _, f := range targets { + if count >= maxVersioned { + break + } + count++ + claim := jev.Claim{Statement: fmt.Sprintf("`version_strings` lists version-like strings with their context. Which is the version of `%s`? A product's asset URL parameter, generator meta or embedded build info counts. Ignore third-party libraries, other products, protocols, years, timestamps and build numbers.", f.Name), Options: map[string]jev.Option{ + notStated: {Description: "The response does not state the version of " + f.Name, Outcome: jev.Refuted}, + jev.OptionInsufficient: {Description: insufficientDescription, Outcome: jev.Insufficient}, + }} if v := declaredVersion(decls, f.Name); v != "" { - setVersion(f, v) - } else { - asked = append(asked, f) + claim.Options[v] = jev.Option{Description: "The response explicitly binds this version to " + f.Name, Outcome: jev.Holds} + if err := j.applyVersion(f, claim, jev.Ruling{Option: v, Confidence: 1}); err != nil { + return err + } + continue } - } - extracted := extractVersions(r.page.raw, maxVersions) - if len(asked) == 0 || len(extracted) == 0 { - return - } - type pick struct { - f *common.Framework - value string - score int - confidence float64 - } - picks := make([]pick, len(asked)) - var shown []versionString - seen := map[string]bool{} - for i, f := range asked { - i, f := i, f key := NormalizeName(f.Name) var own []versionString for _, c := range extracted { @@ -74,66 +71,39 @@ func versionRound(r *round, frames ...*common.Framework) { if len(cands) == 0 { continue } - scores := map[string]int{} - opts := map[string]string{notStated: "The response does not show the version of " + f.Name} for _, c := range cands { - opts[c.Value] = "" - scores[c.Value] = versionScore(c, key) + claim.Options[c.Value] = jev.Option{Outcome: jev.Holds} if !seen[c.Value] { seen[c.Value] = true shown = append(shown, c) } } - r.add(fmt.Sprintf("version_%d", i), Choice(fmt.Sprintf("`version_strings` lists every version-like string found in the raw response, with the text around it. "+ - "Which one is the version of `%s`? A version in an asset URL parameter, a meta tag or embedded build info of %s counts. "+ - "Ignore versions of third-party libraries, other products mentioned in text, years, timestamps and build numbers.", f.Name, f.Name), opts), - func(a Answer) { - if a.Choice != "" && a.Choice != notStated && a.Confidence >= r.judge.VersionConfidence { - picks[i] = pick{f: f, value: a.Choice, score: scores[a.Choice], confidence: a.Confidence} - } - }) + id := "version_" + key + claims[id], targets[id] = claim, f } - if len(shown) == 0 { - return + rulings, err := j.judge(ctx, map[string]interface{}{"response": p, "version_strings": shown}, claims) + if err != nil { + return err } - r.show("version_strings", shown) - count := map[string]int{} - for _, c := range extracted { - count[c.Value] = c.count - } - r.then(func() { - // A string that occurs once belongs to one product: the one whose - // context names it best, then the more confident pick. - best := map[string]int{} - for i, p := range picks { - if p.f == nil || count[p.value] > 1 { - continue - } - if j, ok := best[p.value]; !ok || p.score > picks[j].score || p.score == picks[j].score && p.confidence > picks[j].confidence { - best[p.value] = i - } - } - for i, p := range picks { - if p.f == nil { - continue - } - if j, ok := best[p.value]; ok && j != i { - continue - } - setVersion(p.f, p.value) + for id, ruling := range rulings { + if err := j.applyVersion(targets[id], claims[id], ruling); err != nil { + return err } - }) + } + return nil } -// setVersion copies on write: Attributes may be shared with other frameworks. -func setVersion(f *common.Framework, version string) { - attrs := common.NewAttributesWithAny() - if f.Attributes != nil { - copied := *f.Attributes - attrs = &copied +func (j *Judge) applyVersion(f *common.Framework, claim jev.Claim, ruling jev.Ruling) error { + if err := validRuling(claim, ruling); err != nil { + return err + } + if claim.Resolve(ruling, j.MinConfidence) == jev.Holds { + if f.Attributes == nil { + f.Attributes = common.NewAttributesWithAny() + } + f.Version = ruling.Option } - attrs.Version = version - f.Attributes = attrs + return nil } // declaration is a version the response binds to a product name. @@ -169,11 +139,22 @@ func declarations(raw []byte) []declaration { continue } key, value := line[:colon], strings.TrimSpace(line[colon+1:]) - if m := reLeadingVersion.FindStringSubmatch(value); m != nil { + if m := reLeadingVersion.FindStringSubmatch(value); m != nil && strings.HasPrefix(strings.ToLower(key), "x-") { name := strings.TrimSuffix(strings.TrimPrefix(strings.ToLower(key), "x-"), "-version") out = append(out, declaration{name, m[1]}) } - pairs(value) + if strings.EqualFold(key, "Server") || strings.EqualFold(key, "X-Powered-By") || strings.EqualFold(key, "Product") { + pairs(value) + } + if strings.HasPrefix(strings.ToLower(key), "x-") { + product := NormalizeName(strings.TrimSuffix(strings.TrimPrefix(strings.ToLower(key), "x-"), "-version")) + for _, m := range reDeclared.FindAllStringSubmatch(value, -1) { + d := declaration{m[1], m[2]} + if d.names(product) { + out = append(out, d) + } + } + } } for _, m := range evidence.Generator.FindAllStringSubmatch(body, 4) { pairs(m[1] + m[2]) @@ -242,7 +223,11 @@ func extractVersions(raw []byte, max int) []versionString { var out []versionString for _, loc := range reVersion.FindAllStringSubmatchIndex(text, -1) { v := text[loc[2]:loc[3]] - if looksLikeIPv4(v) { + before := loc[2] - 20 + if before < 0 { + before = 0 + } + if looksLikeIPv4(v) || strings.Trim(v, "0.") == "" || reNotVersion.MatchString(text[before:loc[2]]) { continue } start, end := loc[0]-60, loc[1]+30 @@ -252,7 +237,7 @@ func extractVersions(raw []byte, max int) []versionString { if end > len(text) { end = len(text) } - ctx := clean(strings.ToValidUTF8(text[start:end], "")) + ctx := evidence.Clean(strings.ToValidUTF8(text[start:end], "")) // A value repeats across a page ("?ver=7.1" on every asset, then // the generator meta); keep the context that best states a version. if i, ok := seen[v]; ok { @@ -276,7 +261,7 @@ func extractVersions(raw []byte, max int) []versionString { continue } seen[v] = len(out) - out = append(out, versionString{Value: v, Context: clean(strings.ToValidUTF8(text[m[0]:m[1]], "")), count: 1}) + out = append(out, versionString{Value: v, Context: evidence.Clean(strings.ToValidUTF8(text[m[0]:m[1]], "")), count: 1}) } } // Select evidence after scanning: SVG numbers or early library assets must diff --git a/judge_test.go b/judge_test.go index d861692..f4311d2 100644 --- a/judge_test.go +++ b/judge_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/chainreactors/fingers/judge" + "github.com/chainreactors/utils/jev" ) func TestRefineWithJudge(t *testing.T) { @@ -13,44 +14,71 @@ func TestRefineWithJudge(t *testing.T) { if err != nil { t.Fatal(err) } + engine.EnableMatchDetail() raw := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nWelcome to nginx!") frames, _ := engine.DetectContent(raw) - j := judge.New(server{}) - j.Known = judge.NewRetriever(engine.Names()) - accepted, err := j.Refine(context.Background(), raw, frames) + j := judge.New(running{}) + accepted, err := j.Inspect(context.Background(), raw, frames) + accepted = accepted.Accepted() if err != nil { t.Fatal(err) } - kind, _, err := j.Classify(context.Background(), raw) - if err != nil || kind != judge.KindDefault { - t.Fatalf("kind %q, %v", kind, err) - } - if nginx := accepted["nginx"]; nginx == nil || nginx.Judge == nil || nginx.Judge.Layer != judge.LayerServer || frames["nginx"].Judge != nil { + if nginx := accepted["nginx"]; nginx == nil || nginx.Judge == nil || nginx.Judge.Verdict != judge.OptionDeclared || frames["nginx"].Judge != nil { t.Fatalf("accepted %v, input %v", accepted["nginx"], frames["nginx"]) } - if len(engine.Names()) == 0 { - t.Fatal("engine has no names") - } } -// server is a provider that judges every product present at the server layer. -type server struct{} +// running is a provider that judges every claim running and states no version. +type running struct{} -func (server) ID() string { return "test" } +func (running) ID() string { return "test" } -func (server) Judge(ctx context.Context, state interface{}, questions map[string]judge.Question) (map[string]judge.Answer, error) { - answers := map[string]judge.Answer{} +func (running) Judge(ctx context.Context, state interface{}, questions map[string]jev.Claim) (map[string]jev.Ruling, error) { + answers := map[string]jev.Ruling{} for k := range questions { switch { - case strings.HasPrefix(k, "layer_"): - answers[k] = judge.Answer{Choice: string(judge.LayerServer)} - case k == "page_kind": - answers[k] = judge.Answer{Choice: string(judge.KindDefault)} - case k == "primary": - answers[k] = judge.Answer{Choice: "none_of_these"} + case strings.HasPrefix(k, "presence_"): + answers[k] = jev.Ruling{Option: judge.OptionRunning, Confidence: 0.9} default: - answers[k] = judge.Answer{Yes: 0.9, Choice: "not_stated"} + answers[k] = jev.Ruling{Option: "not_stated", Confidence: 0.9} } } return answers, nil } + +// Every engine that can record what a hit matched does so once enabled, and +// quotes text the response contains; disabled, results carry no detail. +func TestEnableMatchDetail(t *testing.T) { + raw := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\nX-Powered-By: PHP/7.4.3\r\nContent-Type: text/html\r\n\r\n" + + `Blog` + + ``) + engines := []string{FingerPrintEngine, WappalyzerEngine, EHoleEngine, GobyEngine} + for _, enabled := range []bool{false, true} { + engine, err := NewEngine(engines...) + if err != nil { + t.Fatal(err) + } + if enabled { + engine.EnableMatchDetail() + } + frames, _ := engine.DetectContent(raw) + seen := map[string]bool{} + for _, f := range frames { + for from := range f.Froms { + name := from.String() + seen[name] = true + if !enabled && f.MatchDetail != nil { + t.Errorf("%s from %s: detail %+v while disabled", f.Name, name, f.MatchDetail) + } + if enabled && len(f.Froms) == 1 && (f.MatchDetail == nil || !strings.Contains(strings.ToLower(string(raw)), strings.ToLower(f.MatchDetail.MatcherValue))) { + t.Errorf("%s from %s: detail %+v is not in the response", f.Name, name, f.MatchDetail) + } + } + } + for _, name := range []string{"fingerprinthub", "wappalyzer", "goby"} { + if !seen[name] { + t.Errorf("no hit from %s", name) + } + } + } +} diff --git a/wappalyzer/fingerprints.go b/wappalyzer/fingerprints.go index a9a7f8d..51d78ee 100644 --- a/wappalyzer/fingerprints.go +++ b/wappalyzer/fingerprints.go @@ -142,6 +142,17 @@ func (v *versionRegex) MatchString(value string) (bool, string) { return true, version } +// matchedText is the text a matching pattern found in value, or key when the +// pattern only requires the key to be present. +func (v *versionRegex) matchedText(value, key string) string { + if !v.skipRegex { + if text := v.regex.FindString(value); text != "" { + return text + } + } + return key +} + // part is the part of the fingerprint to match type part int @@ -242,6 +253,7 @@ func compileFingerprint(app string, fingerprint *Fingerprint) *CompiledFingerpri // matchString matches a string for the fingerprints func (f *CompiledFingerprints) matchString(data string, part part) common.Frameworks { var matched bool + var evidence string technologies := make(common.Frameworks) for _, fingerprint := range f.Apps { var version string @@ -251,6 +263,7 @@ func (f *CompiledFingerprints) matchString(data string, part part) common.Framew for _, pattern := range fingerprint.js { if valid, versionString := pattern.MatchString(data); valid { matched = true + evidence = pattern.matchedText(data, "") version = versionString } } @@ -258,6 +271,7 @@ func (f *CompiledFingerprints) matchString(data string, part part) common.Framew for _, pattern := range fingerprint.scriptSrc { if valid, versionString := pattern.MatchString(data); valid { matched = true + evidence = pattern.matchedText(data, "") version = versionString } } @@ -265,6 +279,7 @@ func (f *CompiledFingerprints) matchString(data string, part part) common.Framew for _, pattern := range fingerprint.html { if valid, versionString := pattern.MatchString(data); valid { matched = true + evidence = pattern.matchedText(data, "") version = versionString } } @@ -278,8 +293,12 @@ func (f *CompiledFingerprints) matchString(data string, part part) common.Framew } frame := fingerprint.NewFrame(version) + if evidence != "" { + frame.MatchDetail = &common.MatchDetail{MatcherType: "word", MatcherValue: evidence} + } technologies.Add(frame) matched = false + evidence = "" } return technologies } @@ -287,6 +306,7 @@ func (f *CompiledFingerprints) matchString(data string, part part) common.Framew // matchKeyValue matches a key-value store map for the fingerprints func (f *CompiledFingerprints) matchKeyValueString(key, value string, part part) common.Frameworks { var matched bool + var evidence string var technologies = make(common.Frameworks) for _, fingerprint := range f.Apps { @@ -301,6 +321,7 @@ func (f *CompiledFingerprints) matchKeyValueString(key, value string, part part) if valid, versionString := pattern.MatchString(value); valid { matched = true + evidence = pattern.matchedText(value, key) version = versionString break } @@ -313,6 +334,7 @@ func (f *CompiledFingerprints) matchKeyValueString(key, value string, part part) if valid, versionString := pattern.MatchString(value); valid { matched = true + evidence = pattern.matchedText(value, key) version = versionString break } @@ -326,6 +348,7 @@ func (f *CompiledFingerprints) matchKeyValueString(key, value string, part part) for _, pattern := range patterns { if valid, versionString := pattern.MatchString(value); valid { matched = true + evidence = pattern.matchedText(value, key) version = versionString break } @@ -338,8 +361,12 @@ func (f *CompiledFingerprints) matchKeyValueString(key, value string, part part) continue } frame := fingerprint.NewFrame(version) + if evidence != "" { + frame.MatchDetail = &common.MatchDetail{MatcherType: "word", MatcherValue: evidence} + } technologies.Add(frame) matched = false + evidence = "" } return technologies } @@ -347,6 +374,7 @@ func (f *CompiledFingerprints) matchKeyValueString(key, value string, part part) // matchMapString matches a key-value store map for the fingerprints func (f *CompiledFingerprints) matchMapString(keyValue map[string]string, part part) common.Frameworks { var matched bool + var evidence string technologies := make(common.Frameworks) for _, fingerprint := range f.Apps { @@ -361,9 +389,11 @@ func (f *CompiledFingerprints) matchMapString(keyValue map[string]string, part p } if pattern == nil { matched = true + evidence = data } if valid, versionString := pattern.MatchString(value); valid { matched = true + evidence = pattern.matchedText(value, data) version = versionString break } @@ -377,6 +407,7 @@ func (f *CompiledFingerprints) matchMapString(keyValue map[string]string, part p if valid, versionString := pattern.MatchString(value); valid { matched = true + evidence = pattern.matchedText(value, data) version = versionString break } @@ -391,6 +422,7 @@ func (f *CompiledFingerprints) matchMapString(keyValue map[string]string, part p for _, pattern := range patterns { if valid, versionString := pattern.MatchString(value); valid { matched = true + evidence = pattern.matchedText(value, data) version = versionString break } @@ -406,8 +438,12 @@ func (f *CompiledFingerprints) matchMapString(keyValue map[string]string, part p // Append the technologies as well as implied ones frame := fingerprint.NewFrame(version) + if evidence != "" { + frame.MatchDetail = &common.MatchDetail{MatcherType: "word", MatcherValue: evidence} + } technologies.Add(frame) matched = false + evidence = "" } return technologies } diff --git a/wappalyzer/wappalyzer.go b/wappalyzer/wappalyzer.go index 31d88ef..ada4e3a 100644 --- a/wappalyzer/wappalyzer.go +++ b/wappalyzer/wappalyzer.go @@ -11,6 +11,8 @@ import ( // Wappalyze is a client for working with tech detection type Wappalyze struct { fingerprints *CompiledFingerprints + // MatchDetailEnabled records on each hit the text its pattern matched. + MatchDetailEnabled bool } // NewWappalyzeEngine creates a new tech detection instance @@ -105,7 +107,7 @@ func (engine *Wappalyze) Fingerprint(headers map[string][]string, body []byte) c // Check for stuff in the body finally uniqueFingerprints.Merge(engine.checkBody(normalizedBody)) - return uniqueFingerprints + return engine.withDetail(uniqueFingerprints) } // FingerprintWithTitle identifies technologies on a target, @@ -137,9 +139,9 @@ func (engine *Wappalyze) FingerprintWithTitle(headers map[string][]string, body bodyTech := engine.checkBody(normalizedBody) uniqueFingerprints.Merge(bodyTech) title := engine.getTitle(body) - return uniqueFingerprints, title + return engine.withDetail(uniqueFingerprints), title } - return uniqueFingerprints, "" + return engine.withDetail(uniqueFingerprints), "" } // FingerprintWithInfo identifies technologies on a target, @@ -185,3 +187,13 @@ func (engine *Wappalyze) FingerprintWithCats(headers map[string][]string, body [ return result } + +// withDetail drops the matched text from hits unless MatchDetailEnabled. +func (engine *Wappalyze) withDetail(frames common.Frameworks) common.Frameworks { + if !engine.MatchDetailEnabled { + for _, frame := range frames { + frame.MatchDetail = nil + } + } + return frames +} From 3a62b4eddb0af06e76afbabb06d334a565757bf5 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Sat, 26 Sep 2026 20:32:58 -0700 Subject: [PATCH 11/16] refactor(judge): drop the last pre-Claim vocabulary and label converter - Framework.Judge.Verdict is now Judge.Option (utils/parsers fd113e9), the same word as the Ruling it mirrors. - cmd/judgeeval: testdata/labels.json is converted once to productLabel arrays (keep/drop/version, plus the Classify-era page_kind/generic fields, are gone; notes become basis), so loadLabels no longer carries the old-format converter. The converted file loads identically. - Tests speak claims/rulings/options instead of questions/answers/ choice/verdicts/refine; the unused testClaim helper is removed. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 2 +- cmd/engine/example.go | 8 +- cmd/judgeeval/README.md | 4 +- cmd/judgeeval/main.go | 48 +----- cmd/judgeeval/replay_metrics.go | 2 +- cmd/judgeeval/replay_test.go | 23 +-- cmd/judgeeval/testdata/labels.json | 241 ++++++++++++++++++++--------- go.mod | 2 +- go.sum | 4 +- judge/README.md | 6 +- judge/gen/gen_test.go | 8 +- judge/judge_test.go | 70 ++++----- judge/maintain/ledger.go | 2 +- judge/maintain/maintain_test.go | 14 +- judge/operations_test.go | 38 ++--- judge/presence.go | 4 +- judge/real_evidence_test.go | 4 +- judge_test.go | 16 +- 18 files changed, 272 insertions(+), 224 deletions(-) diff --git a/README.md b/README.md index cc3f82d..4cb9b28 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ if err != nil { } for _, f := range accepted { if f.Judge != nil { // nil：未经判定（如超过每页 40 个产品的上限） - fmt.Println(f.Name, f.Version, f.Judge.Verdict, f.Judge.Outcome) + fmt.Println(f.Name, f.Version, f.Judge.Option, f.Judge.Outcome) } } ``` diff --git a/cmd/engine/example.go b/cmd/engine/example.go index 2ff3737..c7ffd7d 100644 --- a/cmd/engine/example.go +++ b/cmd/engine/example.go @@ -228,14 +228,14 @@ func main() { judged, err := j.Inspect(ctx, content, frames) accepted := judged.Accepted() if err != nil { - fmt.Printf("jev failed, code-established verdicts only: %v\n", err) + fmt.Printf("jev failed, code-established facts only: %v\n", err) } for _, frame := range accepted { - verdict := "unjudged" + option := "unjudged" if frame.Judge != nil { - verdict = frame.Judge.Verdict + option = frame.Judge.Option } - fmt.Printf(" %-40s %-10s %s\n", frame.Name, frame.Version, verdict) + fmt.Printf(" %-40s %-10s %s\n", frame.Name, frame.Version, option) } if err == nil { fmt.Printf("accepted: %s\n", accepted.String()) diff --git a/cmd/judgeeval/README.md b/cmd/judgeeval/README.md index 695a410..2d155eb 100644 --- a/cmd/judgeeval/README.md +++ b/cmd/judgeeval/README.md @@ -45,7 +45,7 @@ go build -o bin/judgeeval.exe ./cmd/judgeeval - `metrics.json` / `report.md`：标注范围内的 TP/FP/FN/TN、误删、自然漏报恢复、版本正确/错误/缺失/无依据填值。失败和未标注输出单列。 - `generation.json` / `fingerprints/*.yaml`：候选指纹、逐样本原生 Framework 结果及排除的样本。`training` / `holdout` 按样本 ID 索引，null 表示已评估且未命中，缺失 key 表示未评估。产品、标签、训练计划统一读取 manifest；状态与分数由报告推导，不再写入另一套逐样本 DTO。 -`Inspect` 一次完成存在性和版本判断；报告比较 Baseline 与 Judged.Accepted() 两个阶段。已有非空版本保留，错误由报告揭示。缓存按模型、endpoint、完整 state 和 Claim 精确复用，使用 `claim-report-v3` 命名空间隔离旧缓存，Ruling 保存为 `option/confidence`。metrics.json 的 schema 为 3，统计使用 `claims/options`；manifest、旧 labels 和 baseline 输入格式仍支持。 +`Inspect` 一次完成存在性和版本判断；报告比较 Baseline 与 Judged.Accepted() 两个阶段。已有非空版本保留，错误由报告揭示。缓存按模型、endpoint、完整 state 和 Claim 精确复用，使用 `claim-report-v3` 命名空间隔离旧缓存，Ruling 保存为 `option/confidence`。metrics.json 的 schema 为 3，统计使用 `claims/options`。 新增版本按产品及显式别名合并计数，原基线任一别名已有版本即不算新增。漏报簇的候选名称单独统计，不能算作已补回漏报。生成器训练样本的自动版本保存在 `training` 内的 Framework；训练标签只在导出后打分。训练结果错误或缺失得到 `failed_training`，测试有错得到 `failed_holdout`，未完成全部测试或缺少独立正反例得到 `insufficient_holdout`。报告与维护收录共用这一评估逻辑。 @@ -92,6 +92,6 @@ TYPESAFE_API_KEY=... ./judgeeval -provider jev -samples cc/samples -cache judgec - 两种模式共用一个磁盘 Provider 包装器，保存完整批次的有效 Ruling、统计调用和缓存命中。缓存写入使用临时文件再重命名；失败与非法回答不落盘。 - `rows.jsonl` 保存唯一的 Baseline/Judged 记录；`metrics.json` / `report.md` 从同一结果统计，未标注预测单列，不使用正文提及或 generator 代理真值。 -- 目录模式的 `-labels` 支持旧 keep/drop/version 对象，也支持每个样本的 productLabel 数组。旧格式只在输入边界转换；统一使用规范名和显式 aliases 打分。 +- 目录模式的 `-labels` 是样本文件名到 productLabel 数组的映射（格式同 manifest 的 `labels`），统一使用规范名和显式 aliases 打分。 - `ledger.json` 保存规则审计快照；manifest 模式的 `discovery.json` 直接保存 maintain.Cluster，以样本 ID 关联，不在每行复制漏报候选。 - 新 Provider 在 `main.go` 的 `newProvider` 注册，并实现 Claim/Ruling 合约。 diff --git a/cmd/judgeeval/main.go b/cmd/judgeeval/main.go index 496c6f1..94674ad 100644 --- a/cmd/judgeeval/main.go +++ b/cmd/judgeeval/main.go @@ -12,7 +12,6 @@ import ( "os" "path/filepath" "sort" - "strings" "sync" "sync/atomic" "time" @@ -27,13 +26,13 @@ import ( func main() { manifest := flag.String("manifest", "", "replay evidence manifest with labels and generation plans") history := flag.String("history", "", "optional baseline.jsonl from a previous replay") - offline := flag.Bool("offline", false, "manifest replay using exact cached answers only; no provider requests") + offline := flag.Bool("offline", false, "manifest replay using exact cached rulings only; no provider requests") maintain := flag.Bool("maintain", false, "audit novel products and validate a native additive fingerprint library") library := flag.String("library", "", "existing local native YAML library to load before manifest replay") samples := flag.String("samples", "testdata/samples", "directory of raw HTTP responses (*.http)") labels := flag.String("labels", "", "optional ground truth (see testdata/labels.json)") provider := flag.String("provider", "jev", "judge provider: jev") - cache := flag.String("cache", "judgecache", "directory caching answers") + cache := flag.String("cache", "judgecache", "directory caching rulings") out := flag.String("out", "judgereport", "output directory") rps := flag.Float64("rps", 15, "max provider requests per second") flag.Float64Var(&minConfidence, "min-confidence", 0, "judge MinConfidence (0 = Jev default)") @@ -201,8 +200,7 @@ func evaluatePage(ctx context.Context, engine *fingers.Engine, j *judge.Judge, p return r, err } -// loadLabels converts the old keep/drop/version input once at the boundary. -// All scoring thereafter uses productLabel and exact names or explicit aliases. +// loadLabels reads ground truth: sample file name -> product labels. func loadLabels(path string) (map[string][]productLabel, error) { out := map[string][]productLabel{} if path == "" { @@ -212,47 +210,9 @@ func loadLabels(path string) (map[string][]productLabel, error) { if err != nil { return nil, err } - var values map[string]json.RawMessage - if err := json.Unmarshal(data, &values); err != nil { + if err := json.Unmarshal(data, &out); err != nil { return nil, err } - for id, raw := range values { - if strings.HasPrefix(strings.TrimSpace(string(raw)), "[") { - var labels []productLabel - if err := json.Unmarshal(raw, &labels); err != nil { - return nil, err - } - out[id] = labels - continue - } - var old struct { - Keep []string - Drop []string - Version *struct{ Product, Value string } - } - if err := json.Unmarshal(raw, &old); err != nil { - return nil, err - } - for _, name := range old.Keep { - out[id] = append(out[id], productLabel{Product: name, Present: true}) - } - for _, name := range old.Drop { - out[id] = append(out[id], productLabel{Product: name}) - } - if old.Version != nil { - found := false - for i := range out[id] { - if judge.NormalizeName(out[id][i].Product) == judge.NormalizeName(old.Version.Product) { - out[id][i].Version = &old.Version.Value - found = true - break - } - } - if !found { - out[id] = append(out[id], productLabel{Product: old.Version.Product, Present: true, Version: &old.Version.Value}) - } - } - } return out, nil } diff --git a/cmd/judgeeval/replay_metrics.go b/cmd/judgeeval/replay_metrics.go index d1175ab..4674a96 100644 --- a/cmd/judgeeval/replay_metrics.go +++ b/cmd/judgeeval/replay_metrics.go @@ -130,7 +130,7 @@ func summarizeReplay(m *replayManifest, rows []record, gen []generationResult, c accepted := r.Judged.Accepted() for _, f := range r.Judged { if f != nil && f.Judge != nil && !f.Judge.Duplicate && f.Judge.Outcome != "" { - out.Options[f.Judge.Verdict]++ + out.Options[f.Judge.Option]++ out.Outcomes[jev.Outcome(f.Judge.Outcome)]++ } } diff --git a/cmd/judgeeval/replay_test.go b/cmd/judgeeval/replay_test.go index b9f921b..08a02fa 100644 --- a/cmd/judgeeval/replay_test.go +++ b/cmd/judgeeval/replay_test.go @@ -112,11 +112,11 @@ type replayTestProvider struct { func (p *replayTestProvider) ID() string { return "test/replay/" + p.id } func (p *replayTestProvider) Judge(_ context.Context, _ interface{}, qs map[string]jev.Claim) (map[string]jev.Ruling, error) { p.calls++ - answers := map[string]jev.Ruling{} + rulings := map[string]jev.Ruling{} for key := range qs { - answers[key] = jev.Ruling{Option: "yes", Confidence: 1} + rulings[key] = jev.Ruling{Option: "yes", Confidence: 1} } - return answers, nil + return rulings, nil } func TestReplayOfflineNeverCallsProvider(t *testing.T) { @@ -135,15 +135,15 @@ func TestReplayOfflineNeverCallsProvider(t *testing.T) { t.Fatal(err) } p.offline = true - if answers, err := p.Judge(ctx, "page", qs); err != nil || answers["product"].Option != "yes" { - t.Fatalf("cached answer: %v %v", answers, err) + if rulings, err := p.Judge(ctx, "page", qs); err != nil || rulings["product"].Option != "yes" { + t.Fatalf("cached ruling: %v %v", rulings, err) } if _, err := p.Judge(ctx, "different page", qs); err == nil { - t.Fatal("different state reused cached answer") + t.Fatal("different state reused cached ruling") } provider.id = "different-endpoint" if _, err := p.Judge(ctx, "page", qs); err == nil { - t.Fatal("different endpoint reused cached answer") + t.Fatal("different endpoint reused cached ruling") } if provider.calls != 1 || p.calls != 1 || p.hits != 1 { t.Fatalf("calls=%d requests=%d hits=%d", provider.calls, p.calls, p.hits) @@ -300,16 +300,19 @@ func TestDirectoryAndReplayShareRecordAndCache(t *testing.T) { } } -func TestLabelInputConvertsOnce(t *testing.T) { +func TestLoadLabels(t *testing.T) { path := filepath.Join(t.TempDir(), "labels.json") - data := []byte(`{"old":{"keep":["nginx"],"drop":["wordpress"],"version":{"product":"nginx","value":"1.2.3"}},"new":[{"product":"Orion","aliases":["Orion-console"],"present":true}]}`) + data := []byte(`{"a":[{"product":"Orion","aliases":["Orion-console"],"present":true,"version":"1.2.3"}]}`) if err := os.WriteFile(path, data, 0600); err != nil { t.Fatal(err) } labels, err := loadLabels(path) - if err != nil || len(labels["old"]) != 2 || labels["old"][0].Version == nil || *labels["old"][0].Version != "1.2.3" || len(labels["new"][0].Aliases) != 1 { + if err != nil || len(labels["a"]) != 1 || *labels["a"][0].Version != "1.2.3" || len(labels["a"][0].Aliases) != 1 { t.Fatalf("labels=%v error=%v", labels, err) } + if _, err := loadLabels("testdata/labels.json"); err != nil { + t.Fatal(err) + } } func TestReplayHandlesBaselineWithoutAttributes(t *testing.T) { diff --git a/cmd/judgeeval/testdata/labels.json b/cmd/judgeeval/testdata/labels.json index 940cdde..9038e45 100644 --- a/cmd/judgeeval/testdata/labels.json +++ b/cmd/judgeeval/testdata/labels.json @@ -1,78 +1,167 @@ { - "blog_mentions_jenkins.http": {"page_kind": "normal_content", "generic": false, "keep": ["cloudflare", "hugo"], "drop": ["jenkins", "thinkphp"], - "note": "文章正文提到 Jenkins/ThinkPHP"}, - "confluence_login.http": {"page_kind": "login", "generic": true, "keep": ["confluence"], "drop": ["struts2", "telerik_report_server"], - "version": {"product": "confluence", "value": "7.19.8"}}, - "corporate_cn_portal.http": {"page_kind": "normal_content", "generic": false, "keep": ["nginx", "jquery"], - "note": "新闻里提到用友，规则没误报，是对照"}, - "corporate_site.http": {"page_kind": "normal_content", "generic": false, "keep": ["nginx"]}, - "custom_employee_portal.http": {"page_kind": "login", "generic": false, "keep": ["nginx"], - "note": "单位自研登录页，不是通用产品"}, - "dir_listing.http": {"page_kind": "directory_listing", "generic": true, "keep": ["apache"], - "version": {"product": "apache", "value": "2.4.41"}}, - "discuz_forum.http": {"page_kind": "normal_content", "generic": false, "keep": ["discuz"], - "version": {"product": "discuz", "value": "3.4"}, "note": "基于 CMS 的单位内容页"}, - "docs_tomcat_howto.http": {"page_kind": "normal_content", "generic": false, "keep": ["github pages", "jekyll"], "drop": ["tomcat", "apache-web-server"], - "version": {"product": "jekyll", "value": "3.9.3"}, "note": "正文里的 Tomcat/nginx 版本号是干扰"}, - "gitlab_signin.http": {"page_kind": "login", "generic": true, "keep": ["gitlab"], "drop": ["webp_server_go"], - "version": {"product": "gitlab", "value": ""}}, - "grafana_login.http": {"page_kind": "login", "generic": true, "keep": ["grafana"], - "version": {"product": "grafana", "value": "10.2.3"}}, - "harbor_portal.http": {"page_kind": "app_console", "generic": true, "keep": ["harbor"], - "version": {"product": "harbor", "value": ""}}, - "iis_default.http": {"page_kind": "default_install", "generic": true, "keep": ["iis"], - "version": {"product": "iis", "value": "10.0"}}, - "jenkins_login.http": {"page_kind": "login", "generic": true, "keep": ["jenkins"], "drop": ["log4j2"], - "version": {"product": "jenkins", "value": "2.401.3"}}, - "job_posting_cn.http": {"page_kind": "normal_content", "generic": false, "keep": ["tengine"], "drop": ["elasticsearch", "log4j2", "gitlab", "jenkins", "thinkphp"], - "note": "招聘要求里列的技术栈"}, - "json_api.http": {"page_kind": "api_response", "generic": false, "keep": ["uvicorn"], "drop": ["postgresql"]}, - "kibana_home.http": {"page_kind": "app_console", "generic": true, "keep": ["kibana"], "drop": ["log4j2"], - "version": {"product": "kibana", "value": "8.11.1"}}, - "minio_console.http": {"page_kind": "app_console", "generic": true, "keep": ["minio"], - "version": {"product": "minio", "value": ""}}, - "nacos_console.http": {"page_kind": "app_console", "generic": true, "keep": ["nacos"], - "version": {"product": "nacos", "value": "2.2.3"}}, - "news_fortios_patch.http": {"page_kind": "normal_content", "generic": false, "keep": ["cloudflare", "ghost"], "drop": ["fortianalyzer", "vnpt-technology-igate", "飞塔防火墙", "fortinet sslvpn"], - "version": {"product": "ghost", "value": "5.71"}, "note": "新闻里的 FortiOS 版本号是干扰"}, - "nginx_404.http": {"page_kind": "error_page", "generic": true, "keep": ["nginx"], - "version": {"product": "nginx", "value": "1.22.1"}}, - "nginx_default.http": {"page_kind": "default_install", "generic": true, "keep": ["nginx"], - "version": {"product": "nginx", "value": "1.18.0"}}, - "pgadmin_login.http": {"page_kind": "login", "generic": true, "keep": ["pgadmin", "gunicorn"], "drop": ["postgresql"], - "version": {"product": "pgadmin", "value": ""}, "note": "ver=80000 是内部编号，不是版本号"}, - "phpmyadmin_login.http": {"page_kind": "login", "generic": true, "keep": ["phpmyadmin", "php"], - "version": {"product": "phpmyadmin", "value": "5.2.1"}, "note": "PHP/8.1.12、Apache/2.4.54 是干扰"}, - "portainer_login.http": {"page_kind": "app_console", "generic": true, "keep": ["portainer"], - "version": {"product": "portainer", "value": ""}}, - "rabbitmq_mgmt.http": {"page_kind": "app_console", "generic": true, "keep": ["rabbitmq"], "drop": ["log4j2", "ipeakcms"], - "version": {"product": "rabbitmq", "value": ""}, "note": "js 文件名里的是第三方库版本"}, - "rebranded_grafana.http": {"page_kind": "login", "generic": true, "miss": ["grafana"], - "version": {"product": "grafana", "value": "9.5.2"}, "note": "改过品牌，规则漏报"}, - "rebranded_jenkins.http": {"page_kind": "login", "generic": true, "miss": ["jenkins"], - "version": {"product": "jenkins", "value": ""}, "note": "改过品牌，规则漏报"}, - "rebranded_zabbix.http": {"page_kind": "login", "generic": true, "keep": ["nginx"], "miss": ["zabbix"], - "version": {"product": "zabbix", "value": ""}, "note": "改过品牌的中文页面，规则漏报"}, - "seeyon_login.http": {"page_kind": "login", "generic": true, "keep": ["致远oa"], "drop": ["log4j2"], - "version": {"product": "致远oa", "value": "8.1"}}, - "shop_page.http": {"page_kind": "normal_content", "generic": false, "keep": ["shopify", "cloudflare"], - "note": "SaaS 上的单位店铺"}, - "springboot_error.http": {"page_kind": "error_page", "generic": true, "keep": ["springboot"], "drop": ["log4j2"]}, - "status_page_mentions.http": {"page_kind": "normal_content", "generic": false, "keep": ["nginx"], "drop": ["gitlab", "jenkins", "grafana_labs-公司产品"]}, - "synology_dsm.http": {"page_kind": "login", "generic": true, "keep": ["synology diskstation"], "drop": ["windows"], - "version": {"product": "synology diskstation", "value": ""}, "note": "v= 是时间戳"}, - "tomcat_default.http": {"page_kind": "default_install", "generic": true, "keep": ["tomcat"], "drop": ["tomcat_jk_connector"], - "version": {"product": "tomcat", "value": "9.0.65"}}, - "tplink_router.http": {"page_kind": "login", "generic": true, "keep": ["tp-link无线路由器"], "drop": ["apache http server", "archer-ax21"], - "version": {"product": "tp-link无线路由器", "value": "0.9.1"}}, - "unknown_admin_login.http": {"page_kind": "login", "generic": true, "keep": ["openresty"], "drop": ["zkteco"], - "note": "长尾产品，指纹库里没有；技术支持写着厂商"}, - "weaver_ecology_login.http": {"page_kind": "login", "generic": true, "keep": ["泛微 e-cology"], - "version": {"product": "泛微 e-cology", "value": ""}}, - "wordpress_home.http": {"page_kind": "normal_content", "generic": false, "keep": ["wordpress"], "drop": ["windows", "microsoft-windows远程连接"], - "version": {"product": "wordpress", "value": "6.4.2"}}, - "wordpress_login.http": {"page_kind": "login", "generic": true, "keep": ["wordpress"], "drop": ["windows", "microsoft-windows远程连接", "telerik_report_server"], - "version": {"product": "wordpress", "value": "6.4.2"}, "note": "jquery 3.7.1 是干扰"}, - "zabbix_login.http": {"page_kind": "login", "generic": true, "keep": ["zabbix"], - "version": {"product": "zabbix", "value": ""}, "note": "文档链接里的 6.0 和年份是干扰"} + "blog_mentions_jenkins.http": [ + {"product": "cloudflare", "present": true, "basis": "文章正文提到 Jenkins/ThinkPHP"}, + {"product": "hugo", "present": true, "basis": "文章正文提到 Jenkins/ThinkPHP"}, + {"product": "jenkins", "present": false, "basis": "文章正文提到 Jenkins/ThinkPHP"}, + {"product": "thinkphp", "present": false, "basis": "文章正文提到 Jenkins/ThinkPHP"} + ], + "confluence_login.http": [ + {"product": "confluence", "present": true, "version": "7.19.8"}, + {"product": "struts2", "present": false}, + {"product": "telerik_report_server", "present": false} + ], + "corporate_cn_portal.http": [ + {"product": "nginx", "present": true, "basis": "新闻里提到用友，规则没误报，是对照"}, + {"product": "jquery", "present": true, "basis": "新闻里提到用友，规则没误报，是对照"} + ], + "corporate_site.http": [ + {"product": "nginx", "present": true} + ], + "custom_employee_portal.http": [ + {"product": "nginx", "present": true, "basis": "单位自研登录页，不是通用产品"} + ], + "dir_listing.http": [ + {"product": "apache", "present": true, "version": "2.4.41"} + ], + "discuz_forum.http": [ + {"product": "discuz", "present": true, "version": "3.4", "basis": "基于 CMS 的单位内容页"} + ], + "docs_tomcat_howto.http": [ + {"product": "github pages", "present": true, "basis": "正文里的 Tomcat/nginx 版本号是干扰"}, + {"product": "jekyll", "present": true, "version": "3.9.3", "basis": "正文里的 Tomcat/nginx 版本号是干扰"}, + {"product": "tomcat", "present": false, "basis": "正文里的 Tomcat/nginx 版本号是干扰"}, + {"product": "apache-web-server", "present": false, "basis": "正文里的 Tomcat/nginx 版本号是干扰"} + ], + "gitlab_signin.http": [ + {"product": "gitlab", "present": true, "version": ""}, + {"product": "webp_server_go", "present": false} + ], + "grafana_login.http": [ + {"product": "grafana", "present": true, "version": "10.2.3"} + ], + "harbor_portal.http": [ + {"product": "harbor", "present": true, "version": ""} + ], + "iis_default.http": [ + {"product": "iis", "present": true, "version": "10.0"} + ], + "jenkins_login.http": [ + {"product": "jenkins", "present": true, "version": "2.401.3"}, + {"product": "log4j2", "present": false} + ], + "job_posting_cn.http": [ + {"product": "tengine", "present": true, "basis": "招聘要求里列的技术栈"}, + {"product": "elasticsearch", "present": false, "basis": "招聘要求里列的技术栈"}, + {"product": "log4j2", "present": false, "basis": "招聘要求里列的技术栈"}, + {"product": "gitlab", "present": false, "basis": "招聘要求里列的技术栈"}, + {"product": "jenkins", "present": false, "basis": "招聘要求里列的技术栈"}, + {"product": "thinkphp", "present": false, "basis": "招聘要求里列的技术栈"} + ], + "json_api.http": [ + {"product": "uvicorn", "present": true}, + {"product": "postgresql", "present": false} + ], + "kibana_home.http": [ + {"product": "kibana", "present": true, "version": "8.11.1"}, + {"product": "log4j2", "present": false} + ], + "minio_console.http": [ + {"product": "minio", "present": true, "version": ""} + ], + "nacos_console.http": [ + {"product": "nacos", "present": true, "version": "2.2.3"} + ], + "news_fortios_patch.http": [ + {"product": "cloudflare", "present": true, "basis": "新闻里的 FortiOS 版本号是干扰"}, + {"product": "ghost", "present": true, "version": "5.71", "basis": "新闻里的 FortiOS 版本号是干扰"}, + {"product": "fortianalyzer", "present": false, "basis": "新闻里的 FortiOS 版本号是干扰"}, + {"product": "vnpt-technology-igate", "present": false, "basis": "新闻里的 FortiOS 版本号是干扰"}, + {"product": "飞塔防火墙", "present": false, "basis": "新闻里的 FortiOS 版本号是干扰"}, + {"product": "fortinet sslvpn", "present": false, "basis": "新闻里的 FortiOS 版本号是干扰"} + ], + "nginx_404.http": [ + {"product": "nginx", "present": true, "version": "1.22.1"} + ], + "nginx_default.http": [ + {"product": "nginx", "present": true, "version": "1.18.0"} + ], + "pgadmin_login.http": [ + {"product": "pgadmin", "present": true, "version": "", "basis": "ver=80000 是内部编号，不是版本号"}, + {"product": "gunicorn", "present": true, "basis": "ver=80000 是内部编号，不是版本号"}, + {"product": "postgresql", "present": false, "basis": "ver=80000 是内部编号，不是版本号"} + ], + "phpmyadmin_login.http": [ + {"product": "phpmyadmin", "present": true, "version": "5.2.1", "basis": "PHP/8.1.12、Apache/2.4.54 是干扰"}, + {"product": "php", "present": true, "basis": "PHP/8.1.12、Apache/2.4.54 是干扰"} + ], + "portainer_login.http": [ + {"product": "portainer", "present": true, "version": ""} + ], + "rabbitmq_mgmt.http": [ + {"product": "rabbitmq", "present": true, "version": "", "basis": "js 文件名里的是第三方库版本"}, + {"product": "log4j2", "present": false, "basis": "js 文件名里的是第三方库版本"}, + {"product": "ipeakcms", "present": false, "basis": "js 文件名里的是第三方库版本"} + ], + "rebranded_grafana.http": [ + {"product": "grafana", "present": true, "version": "9.5.2", "basis": "改过品牌，规则漏报"} + ], + "rebranded_jenkins.http": [ + {"product": "jenkins", "present": true, "version": "", "basis": "改过品牌，规则漏报"} + ], + "rebranded_zabbix.http": [ + {"product": "nginx", "present": true, "basis": "改过品牌的中文页面，规则漏报"}, + {"product": "zabbix", "present": true, "version": "", "basis": "改过品牌的中文页面，规则漏报"} + ], + "seeyon_login.http": [ + {"product": "致远oa", "present": true, "version": "8.1"}, + {"product": "log4j2", "present": false} + ], + "shop_page.http": [ + {"product": "shopify", "present": true, "basis": "SaaS 上的单位店铺"}, + {"product": "cloudflare", "present": true, "basis": "SaaS 上的单位店铺"} + ], + "springboot_error.http": [ + {"product": "springboot", "present": true}, + {"product": "log4j2", "present": false} + ], + "status_page_mentions.http": [ + {"product": "nginx", "present": true}, + {"product": "gitlab", "present": false}, + {"product": "jenkins", "present": false}, + {"product": "grafana_labs-公司产品", "present": false} + ], + "synology_dsm.http": [ + {"product": "synology diskstation", "present": true, "version": "", "basis": "v= 是时间戳"}, + {"product": "windows", "present": false, "basis": "v= 是时间戳"} + ], + "tomcat_default.http": [ + {"product": "tomcat", "present": true, "version": "9.0.65"}, + {"product": "tomcat_jk_connector", "present": false} + ], + "tplink_router.http": [ + {"product": "tp-link无线路由器", "present": true, "version": "0.9.1"}, + {"product": "apache http server", "present": false}, + {"product": "archer-ax21", "present": false} + ], + "unknown_admin_login.http": [ + {"product": "openresty", "present": true, "basis": "长尾产品，指纹库里没有；技术支持写着厂商"}, + {"product": "zkteco", "present": false, "basis": "长尾产品，指纹库里没有；技术支持写着厂商"} + ], + "weaver_ecology_login.http": [ + {"product": "泛微 e-cology", "present": true, "version": ""} + ], + "wordpress_home.http": [ + {"product": "wordpress", "present": true, "version": "6.4.2"}, + {"product": "windows", "present": false}, + {"product": "microsoft-windows远程连接", "present": false} + ], + "wordpress_login.http": [ + {"product": "wordpress", "present": true, "version": "6.4.2", "basis": "jquery 3.7.1 是干扰"}, + {"product": "windows", "present": false, "basis": "jquery 3.7.1 是干扰"}, + {"product": "microsoft-windows远程连接", "present": false, "basis": "jquery 3.7.1 是干扰"}, + {"product": "telerik_report_server", "present": false, "basis": "jquery 3.7.1 是干扰"} + ], + "zabbix_login.http": [ + {"product": "zabbix", "present": true, "version": "", "basis": "文档链接里的 6.0 和年份是干扰"} + ] } diff --git a/go.mod b/go.mod index c4b09b4..36664d4 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible github.com/chainreactors/neutron v0.1.1-0.20260714062907-716c6b167cb6 github.com/chainreactors/utils/jev v0.0.0-20260927031300-6bf027367a50 - github.com/chainreactors/utils/parsers v0.0.4-0.20260926072629-6752abe35765 + github.com/chainreactors/utils/parsers v0.0.4-0.20260927032755-fd113e94d6c0 github.com/chainreactors/words v0.0.0-20260520145736-270600e60fb4 github.com/dlclark/regexp2 v1.11.5 github.com/invopop/jsonschema v0.7.0 diff --git a/go.sum b/go.sum index 917e0d8..8db9777 100644 --- a/go.sum +++ b/go.sum @@ -81,8 +81,8 @@ github.com/chainreactors/utils v0.0.0-20260629043228-93bdd2142c9a h1:4rLTKr+VlG7 github.com/chainreactors/utils v0.0.0-20260629043228-93bdd2142c9a/go.mod h1:xwbUlFoSSxLHujyb8D48o1s2DqmEAxUNfxIy0DVUmcg= github.com/chainreactors/utils/jev v0.0.0-20260927031300-6bf027367a50 h1:XnGpBYIS4u1BUNL3KtlrrXde/vSmDon34X5AEV7WOi8= github.com/chainreactors/utils/jev v0.0.0-20260927031300-6bf027367a50/go.mod h1:9w5fvcKrpQVIHWlKuvpVARpZUd+OKsYSIii2JR1ZRB8= -github.com/chainreactors/utils/parsers v0.0.4-0.20260926072629-6752abe35765 h1:5SDaMwwQfXHdkzUvFEjK4/blZ/EifEKZGtRhwzdnZ24= -github.com/chainreactors/utils/parsers v0.0.4-0.20260926072629-6752abe35765/go.mod h1:bE/znJWt08n9QOORWsWu0ggB8GWfOg3+dfUMMITmwV4= +github.com/chainreactors/utils/parsers v0.0.4-0.20260927032755-fd113e94d6c0 h1:rfhEDw1QfbIgUAhGWRbovR3ur7Ug3V7j5WxqstwlnW0= +github.com/chainreactors/utils/parsers v0.0.4-0.20260927032755-fd113e94d6c0/go.mod h1:bE/znJWt08n9QOORWsWu0ggB8GWfOg3+dfUMMITmwV4= github.com/chainreactors/words v0.0.0-20260520145736-270600e60fb4 h1:lvnDYEkatmZFHP5i321qQXK9L4vKRfso/uUfr5tOeC8= github.com/chainreactors/words v0.0.0-20260520145736-270600e60fb4/go.mod h1:zfz367PUmyaX6oAqV9SktVqyRXKlEh0sel9Wsq9dd2c= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= diff --git a/judge/README.md b/judge/README.md index 7158527..81afd43 100644 --- a/judge/README.md +++ b/judge/README.md @@ -38,7 +38,7 @@ Presence 用 `running` 表示产品参与生成响应，`mentioned` / `unrelated | 字段 | 含义 | |---|---| -| Verdict | 原始选项；即使置信度不足也保留 | +| Option | 原始选项；即使置信度不足也保留 | | Outcome | Resolve 后的 holds / refuted / insufficient；消费方按此统计 | | Confidence / Evidence | 置信度及规则命中原文；本地确定事实置信度为 1 | | Rejected | Refuted，或 Insufficient 且 DropInsufficient 为 true | @@ -48,10 +48,10 @@ Presence 用 `running` 表示产品参与生成响应，`mentioned` / `unrelated ## 离线维护 -- `maintain.Ledger.Add(id, all)` 从既有 presence 注解汇总规则表现，不再请求 Provider。`Report` 返回深复制快照，记录 Outcomes、Options 和样本 ID；`Hits()` 与 `RefutedFraction()` 从 Outcomes 推导。Presence 选项常量统一为 `judge.OptionRunning` 等 `Option*`，原生 `Framework.Judge.Verdict` 输出字段继续保留。当前引擎聚合后的 MatchDetail 仍只保留首个匹配详情，未重设计完整规则来源。 +- `maintain.Ledger.Add(id, all)` 从既有 presence 注解汇总规则表现，不再请求 Provider。`Report` 返回深复制快照，记录 Outcomes、Options 和样本 ID；`Hits()` 与 `RefutedFraction()` 从 Outcomes 推导。Presence 选项常量统一为 `judge.OptionRunning` 等 `Option*`，原生 `Framework.Judge.Option` 输出字段继续保留。当前引擎聚合后的 MatchDetail 仍只保留首个匹配详情，未重设计完整规则来源。 - `maintain.Discover` 按资源路径、表单、Cookie/自定义头名和页面文字等结构特征聚类。每个多主机簇提出 Coverage Claim：`explained` / `custom_site` → Holds，`missing` → Refuted，另有 insufficient。Cluster 直接可序列化，仅保存成员 ID、Coverage 原始 Ruling、解析的 Outcome、已报告产品和代码抽取的候选名。候选名供人命名，不作为 Provider 选项。 - `gen.Generator` 为人指定的 Name 从正反样本生成规则。需要时调用 Version。Validate 运行规则引擎，不调用模型，不修改内置库。 -`cmd/judgeeval` 两种输入模式共用 `baseline` / `judged` 行记录。接受结果、拒绝项、版本差异和指标均从它们派生，聚类通过样本 ID 关联。生成评估同样保存原生 Framework，以 manifest 为唯一标签来源，分数和状态按需推导。报告 schema 为 3，统计使用 `claims/options`；旧 manifest、labels 和 baseline 输入仍可读取，旧缓存因命名空间隔离不会被复用。详见 [judgeeval](../cmd/judgeeval/README.md)。 +`cmd/judgeeval` 两种输入模式共用 `baseline` / `judged` 行记录。接受结果、拒绝项、版本差异和指标均从它们派生，聚类通过样本 ID 关联。生成评估同样保存原生 Framework，以 manifest 为唯一标签来源，分数和状态按需推导。报告 schema 为 3，统计使用 `claims/options`；旧缓存因命名空间隔离不会被复用。详见 [judgeeval](../cmd/judgeeval/README.md)。 Provider 收到精简响应视图、规则命中上下文和版本候选上下文；完整原始响应仍仅在本地用于解析事实与提取证据。 diff --git a/judge/gen/gen_test.go b/judge/gen/gen_test.go index 130aa53..7a8d359 100644 --- a/judge/gen/gen_test.go +++ b/judge/gen/gen_test.go @@ -13,7 +13,7 @@ import ( "github.com/chainreactors/utils/jev" ) -// names picks its product whenever it is offered as a choice. +// names picks its product whenever it is offered as an option. type names string func (names) ID() string { return "names" } @@ -21,13 +21,13 @@ func (names) ID() string { return "names" } func (n names) Judge(_ context.Context, _ interface{}, qs map[string]jev.Claim) (map[string]jev.Ruling, error) { out := map[string]jev.Ruling{} for key, q := range qs { - choice := jev.OptionInsufficient + picked := jev.OptionInsufficient for option := range q.Options { if strings.EqualFold(option, string(n)) { - choice = option + picked = option } } - out[key] = jev.Ruling{Option: choice, Confidence: 0.95} + out[key] = jev.Ruling{Option: picked, Confidence: 0.95} } return out, nil } diff --git a/judge/judge_test.go b/judge/judge_test.go index 4443a45..c549a45 100644 --- a/judge/judge_test.go +++ b/judge/judge_test.go @@ -41,8 +41,8 @@ func TestNewPage(t *testing.T) { } } -// mock is a Provider answering from a table keyed by question kind and the -// product named in backticks; it records the question keys of each call. +// mock is a Provider ruling from a table keyed by claim kind and the +// product named in backticks; it records the claim keys of each call. type mock struct { mu sync.Mutex delay time.Duration @@ -50,7 +50,7 @@ type mock struct { requests [][]string fail bool confidence float64 - verdicts map[string]string // product -> presence verdict + presence map[string]string // product -> presence option versions map[string]string // product -> version picked, if among the options coverage string lastState map[string]interface{} @@ -59,7 +59,7 @@ type mock struct { func newMock() *mock { return &mock{ confidence: 0.95, - verdicts: map[string]string{"wordpress": OptionMentioned, "apache tomcat": OptionRunning, "prototype": OptionRunning}, + presence: map[string]string{"wordpress": OptionMentioned, "apache tomcat": OptionRunning, "prototype": OptionRunning}, versions: map[string]string{"jenkins": "2.401.3", "nginx": "1.24.0"}, } } @@ -77,7 +77,7 @@ func named(q jev.Claim) string { func (m *mock) ID() string { return "mock" } -func (m *mock) Judge(ctx context.Context, state interface{}, questions map[string]jev.Claim) (map[string]jev.Ruling, error) { +func (m *mock) Judge(ctx context.Context, state interface{}, claims map[string]jev.Claim) (map[string]jev.Ruling, error) { atomic.AddInt64(&m.calls, 1) time.Sleep(m.delay) m.mu.Lock() @@ -87,35 +87,35 @@ func (m *mock) Judge(ctx context.Context, state interface{}, questions map[strin } data, _ := json.Marshal(state) _ = json.Unmarshal(data, &m.lastState) - answers := map[string]jev.Ruling{} + rulings := map[string]jev.Ruling{} var keys []string - for k, q := range questions { + for k, q := range claims { keys = append(keys, k) name := strings.ToLower(named(q)) switch { case strings.HasPrefix(k, "presence_"): - v := m.verdicts[name] + v := m.presence[name] if v == "" { v = jev.OptionInsufficient } - answers[k] = jev.Ruling{Option: v, Confidence: m.confidence} + rulings[k] = jev.Ruling{Option: v, Confidence: m.confidence} case strings.HasPrefix(k, "version_"): - choice := notStated + option := notStated if v, ok := m.versions[name]; ok { if _, offered := q.Options[v]; offered { - choice = v + option = v } } - answers[k] = jev.Ruling{Option: choice, Confidence: m.confidence} + rulings[k] = jev.Ruling{Option: option, Confidence: m.confidence} case k == "coverage": - answers[k] = jev.Ruling{Option: m.coverage, Confidence: m.confidence} + rulings[k] = jev.Ruling{Option: m.coverage, Confidence: m.confidence} default: - answers[k] = jev.Ruling{Option: jev.OptionInsufficient, Confidence: m.confidence} + rulings[k] = jev.Ruling{Option: jev.OptionInsufficient, Confidence: m.confidence} } } sort.Strings(keys) m.requests = append(m.requests, keys) - return answers, nil + return rulings, nil } func testFrames() common.Frameworks { @@ -141,15 +141,15 @@ func TestInspect(t *testing.T) { if err != nil { t.Fatal(err) } - if w := all["wordpress"].Judge; w == nil || !w.Rejected || w.Verdict != OptionMentioned || w.Outcome != "refuted" || len(w.Evidence) == 0 || !strings.Contains(w.Evidence[0], "WordPress") { + if w := all["wordpress"].Judge; w == nil || !w.Rejected || w.Option != OptionMentioned || w.Outcome != "refuted" || len(w.Evidence) == 0 || !strings.Contains(w.Evidence[0], "WordPress") { t.Errorf("text-only wordpress not rejected on its evidence: %+v", w) } for _, name := range []string{"nginx", "jenkins"} { - if v := all[name].Judge; v.Rejected || v.Verdict != OptionDeclared || v.Outcome != "holds" { + if v := all[name].Judge; v.Rejected || v.Option != OptionDeclared || v.Outcome != "holds" { t.Errorf("%s named in a header must be declared: %+v", name, v) } } - if h := all["hsts"].Judge; !h.Rejected || h.Verdict != OptionAbsent || h.Outcome != "refuted" { + if h := all["hsts"].Judge; !h.Rejected || h.Option != OptionAbsent || h.Outcome != "refuted" { t.Errorf("hsts without the header must be absent: %+v", h) } if all["apache tomcat"].Judge.Duplicate == all["apache-tomcat"].Judge.Duplicate { @@ -176,20 +176,20 @@ func TestInspect(t *testing.T) { if accepted["jenkins"].Version != "2.401.3" || accepted["nginx"].Version != "1.24.0" { t.Errorf("versions: jenkins %q nginx %q", accepted["jenkins"].Version, accepted["nginx"].Version) } - // The verdicts come from the cache; tomcat is offered no version, since + // The presence come from the cache; tomcat is offered no version, since // both strings are bound by name to other products. if len(m.requests) != 1 { - t.Errorf("cached verdicts were asked again: %v", m.requests) + t.Errorf("cached presence were asked again: %v", m.requests) } } -// DropInsufficient decides what an undecided claim does; an answer below +// DropInsufficient decides what an undecided claim does; a ruling below // MinConfidence is undecided whatever it chose. func TestInsufficient(t *testing.T) { for _, c := range []struct { on bool confidence float64 - verdict string + option string rejected bool }{ {false, 0.95, OptionRunning, false}, @@ -210,10 +210,10 @@ func TestInsufficient(t *testing.T) { tomcat = all["apache-tomcat"].Judge } outcome := "holds" - if c.verdict == jev.OptionInsufficient { + if c.option == jev.OptionInsufficient { outcome = "insufficient" } - if tomcat.Verdict != OptionRunning || tomcat.Outcome != outcome || tomcat.Rejected != c.rejected { + if tomcat.Option != OptionRunning || tomcat.Outcome != outcome || tomcat.Rejected != c.rejected { t.Errorf("%+v: tomcat %+v", c, tomcat) } } @@ -223,7 +223,7 @@ func TestInsufficient(t *testing.T) { // removed like a mere mention, and the matched text reaches the provider. func TestUnrelatedIsRejected(t *testing.T) { m := newMock() - m.verdicts["apache tomcat"] = OptionUnrelated + m.presence["apache tomcat"] = OptionUnrelated frames := testFrames() for _, f := range frames { if f.Name == "apache tomcat" || f.Name == "apache-tomcat" { @@ -238,7 +238,7 @@ func TestUnrelatedIsRejected(t *testing.T) { if all["apache-tomcat"].Judge.Duplicate { tomcat = all["apache-tomcat"].Judge } - if tomcat.Verdict != OptionUnrelated || !tomcat.Rejected || len(tomcat.Evidence) == 0 || !strings.Contains(tomcat.Evidence[0], `matched "Jenkins"`) { + if tomcat.Option != OptionUnrelated || !tomcat.Rejected || len(tomcat.Evidence) == 0 || !strings.Contains(tomcat.Evidence[0], `matched "Jenkins"`) { t.Fatalf("tomcat %+v", tomcat) } } @@ -247,7 +247,7 @@ func TestUnrelatedIsRejected(t *testing.T) { // products in one request, each from its own candidates. func TestInspectVersionsEveryProduct(t *testing.T) { m := newMock() - m.verdicts["jenkins"] = OptionRunning + m.presence["jenkins"] = OptionRunning accepted, err := New(m).Inspect(context.Background(), []byte(bodyRaw), testFrames()) accepted = accepted.Accepted() if err != nil { @@ -281,7 +281,7 @@ func TestDeclarations(t *testing.T) { func TestVersionNeedsConfidence(t *testing.T) { m := newMock() - m.verdicts["jenkins"] = OptionRunning + m.presence["jenkins"] = OptionRunning m.confidence = 0.2 accepted, err := New(m).Inspect(context.Background(), []byte(bodyRaw), testFrames()) accepted = accepted.Accepted() @@ -311,14 +311,14 @@ func TestFailedProviderKeepsFacts(t *testing.T) { } // nginx, jenkins (declared), wordpress and one tomcat (unjudged); hsts absent, one tomcat a duplicate. if len(accepted) != 4 || accepted["hsts"] != nil || accepted["wordpress"] == nil || accepted["wordpress"].Judge != nil || - accepted["jenkins"].Judge.Verdict != OptionDeclared { + accepted["jenkins"].Judge.Option != OptionDeclared { t.Fatalf("accepted on failure: %v", accepted) } } func TestCache(t *testing.T) { m := newMock() - m.verdicts["jenkins"] = OptionRunning + m.presence["jenkins"] = OptionRunning j := New(jev.Cached(m, jev.DefaultCacheSize)) for i := 0; i < 2; i++ { accepted, err := j.Inspect(context.Background(), []byte(bodyRaw), testFrames()) @@ -327,7 +327,7 @@ func TestCache(t *testing.T) { t.Fatal(err) } if accepted["jenkins"].Version != "2.401.3" { - t.Fatalf("run %d: cached answers not applied", i) + t.Fatalf("run %d: cached rulings not applied", i) } } if m.calls != 2 { @@ -335,10 +335,6 @@ func TestCache(t *testing.T) { } } -func testClaim(statement string) jev.Claim { - return jev.Claim{Statement: statement, Options: map[string]jev.Option{"yes": {Description: "", Outcome: jev.Holds}, "no": {Description: "", Outcome: jev.Refuted}, jev.OptionInsufficient: {Description: insufficientDescription, Outcome: jev.Insufficient}}} -} - func TestExtractVersionsPrefixes(t *testing.T) { raw := "HTTP/1.1 200 OK\r\n\r\nV8.1SP2 10.0.0.5 jquery.min.js?ver=3.7.1{\"success_fraction\":0.0}" var got []string @@ -398,8 +394,8 @@ func TestNormalizeName(t *testing.T) { // Versions are ordered by their resolved outcome. func TestVersionOrder(t *testing.T) { frames := common.Frameworks{} - for name, j := range map[string]*common.Judgement{"jquery": {Verdict: jev.OptionInsufficient}, "apache": {Verdict: OptionDeclared, Outcome: jev.Holds.String()}, - "gitlab": {Verdict: OptionRunning, Outcome: jev.Holds.String()}, "unjudged": nil} { + for name, j := range map[string]*common.Judgement{"jquery": {Option: jev.OptionInsufficient}, "apache": {Option: OptionDeclared, Outcome: jev.Holds.String()}, + "gitlab": {Option: OptionRunning, Outcome: jev.Holds.String()}, "unjudged": nil} { f := common.NewFramework(name, common.FrameFromFingers) f.Judge = j frames.Add(f) diff --git a/judge/maintain/ledger.go b/judge/maintain/ledger.go index fc71a4c..57fa704 100644 --- a/judge/maintain/ledger.go +++ b/judge/maintain/ledger.go @@ -77,7 +77,7 @@ func (l *Ledger) Add(page string, inspected common.Frameworks) { l.rules[key] = s } s.Outcomes[o]++ - s.Options[f.Judge.Verdict]++ + s.Options[f.Judge.Option]++ if len(s.Samples[o]) < maxSamples { s.Samples[o] = append(s.Samples[o], page) } diff --git a/judge/maintain/maintain_test.go b/judge/maintain/maintain_test.go index df69125..e140e48 100644 --- a/judge/maintain/maintain_test.go +++ b/judge/maintain/maintain_test.go @@ -11,9 +11,9 @@ import ( "github.com/chainreactors/utils/jev" ) -func judged(name string, from common.From, verdict string, outcome jev.Outcome, detail *common.MatchDetail) *common.Framework { +func judged(name string, from common.From, option string, outcome jev.Outcome, detail *common.MatchDetail) *common.Framework { f := common.NewFramework(name, from) - f.Judge = &common.Judgement{Verdict: verdict, Outcome: outcome.String()} + f.Judge = &common.Judgement{Option: option, Outcome: outcome.String()} f.MatchDetail = detail return f } @@ -25,11 +25,11 @@ func TestLedgerReportsJunkRules(t *testing.T) { fs := common.Frameworks{} fs.Add(judged("webp_server_go", common.FrameFromFingers, judge.OptionUnrelated, jev.Refuted, webp)) fs.Add(judged("nginx", common.FrameFromGoby, judge.OptionDeclared, jev.Holds, nil)) - verdict, outcome := judge.OptionMentioned, jev.Refuted + option, outcome := judge.OptionMentioned, jev.Refuted if i == 0 { - verdict, outcome = judge.OptionRunning, jev.Holds + option, outcome = judge.OptionRunning, jev.Holds } - fs.Add(judged("git", common.FrameFromEhole, verdict, outcome, nil)) + fs.Add(judged("git", common.FrameFromEhole, option, outcome, nil)) fs.Add(common.NewFramework("unjudged", common.FrameFromGoby)) l.Add(fmt.Sprintf("page%d", i), fs) } @@ -122,7 +122,7 @@ func TestLedgerReportIsDeepSnapshot(t *testing.T) { } } -func TestClusterSnapshotKeepsRawChoiceAndResolvedOutcome(t *testing.T) { +func TestClusterSnapshotKeepsRawOptionAndResolvedOutcome(t *testing.T) { raw := []byte("HTTP/1.1 200 OK\r\n\r\nOrion Console") j := judge.New(coverer(CoverageMissing)) j.MinConfidence = 1 @@ -132,7 +132,7 @@ func TestClusterSnapshotKeepsRawChoiceAndResolvedOutcome(t *testing.T) { } c := clusters[0] if c.Coverage.Option != CoverageMissing || c.Outcome != jev.Insufficient { - t.Fatalf("raw choice lost: %+v", c) + t.Fatalf("raw option lost: %+v", c) } data, err := json.Marshal(c) if err != nil { diff --git a/judge/operations_test.go b/judge/operations_test.go index bd956fc..eb19a30 100644 --- a/judge/operations_test.go +++ b/judge/operations_test.go @@ -10,26 +10,26 @@ import ( "github.com/chainreactors/utils/jev" ) -type answerProvider func(map[string]jev.Claim) map[string]jev.Ruling +type rulingProvider func(map[string]jev.Claim) map[string]jev.Ruling -func (answerProvider) ID() string { return "answer-provider" } +func (rulingProvider) ID() string { return "ruling-provider" } -func (p answerProvider) Judge(_ context.Context, _ interface{}, questions map[string]jev.Claim) (map[string]jev.Ruling, error) { - return p(questions), nil +func (p rulingProvider) Judge(_ context.Context, _ interface{}, claims map[string]jev.Claim) (map[string]jev.Ruling, error) { + return p(claims), nil } type failVersionProvider struct{ *mock } -func (p failVersionProvider) Judge(ctx context.Context, state interface{}, questions map[string]jev.Claim) (map[string]jev.Ruling, error) { - if _, ok := questions["version_jenkins"]; ok { +func (p failVersionProvider) Judge(ctx context.Context, state interface{}, claims map[string]jev.Claim) (map[string]jev.Ruling, error) { + if _, ok := claims["version_jenkins"]; ok { return nil, errors.New("version unavailable") } - return p.mock.Judge(ctx, state, questions) + return p.mock.Judge(ctx, state, claims) } func TestPublicResultsDoNotMutateHits(t *testing.T) { m := newMock() - m.verdicts["jenkins"] = OptionRunning + m.presence["jenkins"] = OptionRunning j := New(m) frames := testFrames() before, _ := json.Marshal(frames) @@ -37,14 +37,14 @@ func TestPublicResultsDoNotMutateHits(t *testing.T) { if err != nil || !inspected["wordpress"].Judge.Rejected { t.Fatalf("inspect = %v, %v", inspected, err) } - refined, err := j.Inspect(context.Background(), []byte(bodyRaw), frames) - refined = refined.Accepted() - if err != nil || refined["wordpress"] != nil || refined["jenkins"].Version != "2.401.3" { - t.Fatalf("refine = %v, %v", refined, err) + accepted, err := j.Inspect(context.Background(), []byte(bodyRaw), frames) + accepted = accepted.Accepted() + if err != nil || accepted["wordpress"] != nil || accepted["jenkins"].Version != "2.401.3" { + t.Fatalf("accepted = %v, %v", accepted, err) } // The judgement is copied, not shared with the Inspect result. - refined["nginx"].Judge.Verdict = "changed" - if inspected["nginx"].Judge.Verdict == "changed" { + accepted["nginx"].Judge.Option = "changed" + if inspected["nginx"].Judge.Option == "changed" { t.Fatal("Independent Inspect calls share a Judgement") } if after, _ := json.Marshal(frames); string(after) != string(before) { @@ -56,11 +56,11 @@ func TestPublicResultsDoNotMutateHits(t *testing.T) { } } -// A failed version round keeps the verdicts and leaves the input untouched. -func TestInspectVersionFailureKeepsVerdicts(t *testing.T) { +// A failed version round keeps the presence and leaves the input untouched. +func TestInspectVersionFailureKeepsPresence(t *testing.T) { frames := testFrames() m := newMock() - m.verdicts["jenkins"] = OptionRunning + m.presence["jenkins"] = OptionRunning accepted, err := New(failVersionProvider{m}).Inspect(context.Background(), []byte(bodyRaw), frames) accepted = accepted.Accepted() if err == nil || accepted["wordpress"] != nil || accepted["jenkins"] == nil || accepted["jenkins"].Version != "" { @@ -74,7 +74,7 @@ func TestInspectVersionFailureKeepsVerdicts(t *testing.T) { } func TestInspectInvalidBatchIsNotPartlyApplied(t *testing.T) { - j := New(answerProvider(func(cs map[string]jev.Claim) map[string]jev.Ruling { + j := New(rulingProvider(func(cs map[string]jev.Claim) map[string]jev.Ruling { out := map[string]jev.Ruling{} for id := range cs { out[id] = jev.Ruling{Option: OptionRunning, Confidence: 1} @@ -94,7 +94,7 @@ func TestInspectReevaluatesAndCopiesInputs(t *testing.T) { frames := testFrames() f := frames["wordpress"] f.MatchDetail = &common.MatchDetail{MatcherValue: "WordPress"} - f.Judge = &common.Judgement{Verdict: OptionRunning, Outcome: jev.Holds.String(), Evidence: []string{"old"}} + f.Judge = &common.Judgement{Option: OptionRunning, Outcome: jev.Holds.String(), Evidence: []string{"old"}} before, _ := json.Marshal(frames) all, err := j.Inspect(context.Background(), []byte(jenkinsRaw), frames) if err != nil || !all["wordpress"].Judge.Rejected { diff --git a/judge/presence.go b/judge/presence.go index 51b0ed0..50db765 100644 --- a/judge/presence.go +++ b/judge/presence.go @@ -10,7 +10,7 @@ import ( "github.com/chainreactors/utils/jev" ) -// Presence options, as written to Framework.Judge.Verdict: the option a +// Presence options, as written to Framework.Judge.Option: the option a // presence claim was ruled with. Declared and absent are rulings code makes // from the response head; the rest are the provider's. const ( @@ -121,7 +121,7 @@ func (j *Judge) applyPresence(frames []*common.Framework, evidence []matchExcerp excerpts = append(excerpts, e.String()) } for i, f := range frames { - f.Judge = &common.Judgement{Verdict: ruling.Option, Outcome: outcome.String(), + f.Judge = &common.Judgement{Option: ruling.Option, Outcome: outcome.String(), Evidence: append([]string(nil), excerpts...), Confidence: ruling.Confidence, Rejected: j.rejects(outcome), Duplicate: i > 0} } diff --git a/judge/real_evidence_test.go b/judge/real_evidence_test.go index 69adc41..4000edd 100644 --- a/judge/real_evidence_test.go +++ b/judge/real_evidence_test.go @@ -64,14 +64,14 @@ func TestVersionCandidatesDoNotDiscardLateProductEvidence(t *testing.T) { } // Protocol features are facts of the response head: decided by code, never -// asked, whatever a provider would answer. +// asked, whatever a provider would rule. func TestProtocolHitRequiresResponseEvidence(t *testing.T) { for _, tc := range []struct { name, header string want bool }{{"http基本认证", "", false}, {"hsts", "", false}, {"http基本认证", "WWW-Authenticate: Basic realm=\"test\"\r\n", true}, {"hsts", "Strict-Transport-Security: max-age=31536000\r\n", true}} { asked := false - j := New(answerProvider(func(qs map[string]jev.Claim) map[string]jev.Ruling { + j := New(rulingProvider(func(qs map[string]jev.Claim) map[string]jev.Ruling { asked = true return nil })) diff --git a/judge_test.go b/judge_test.go index f4311d2..23e8dcb 100644 --- a/judge_test.go +++ b/judge_test.go @@ -9,7 +9,7 @@ import ( "github.com/chainreactors/utils/jev" ) -func TestRefineWithJudge(t *testing.T) { +func TestInspectWithJudge(t *testing.T) { engine, err := NewEngine(FingersEngine, WappalyzerEngine) if err != nil { t.Fatal(err) @@ -23,7 +23,7 @@ func TestRefineWithJudge(t *testing.T) { if err != nil { t.Fatal(err) } - if nginx := accepted["nginx"]; nginx == nil || nginx.Judge == nil || nginx.Judge.Verdict != judge.OptionDeclared || frames["nginx"].Judge != nil { + if nginx := accepted["nginx"]; nginx == nil || nginx.Judge == nil || nginx.Judge.Option != judge.OptionDeclared || frames["nginx"].Judge != nil { t.Fatalf("accepted %v, input %v", accepted["nginx"], frames["nginx"]) } } @@ -33,17 +33,17 @@ type running struct{} func (running) ID() string { return "test" } -func (running) Judge(ctx context.Context, state interface{}, questions map[string]jev.Claim) (map[string]jev.Ruling, error) { - answers := map[string]jev.Ruling{} - for k := range questions { +func (running) Judge(ctx context.Context, state interface{}, claims map[string]jev.Claim) (map[string]jev.Ruling, error) { + rulings := map[string]jev.Ruling{} + for k := range claims { switch { case strings.HasPrefix(k, "presence_"): - answers[k] = jev.Ruling{Option: judge.OptionRunning, Confidence: 0.9} + rulings[k] = jev.Ruling{Option: judge.OptionRunning, Confidence: 0.9} default: - answers[k] = jev.Ruling{Option: "not_stated", Confidence: 0.9} + rulings[k] = jev.Ruling{Option: "not_stated", Confidence: 0.9} } } - return answers, nil + return rulings, nil } // Every engine that can record what a hit matched does so once enabled, and From 7cc54d46fda3206c3c66f953b5c286a7113b90e6 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Sat, 26 Sep 2026 21:15:33 -0700 Subject: [PATCH 12/16] fix(engine): make merged results reproducible The same response could produce different merged results between runs, because several places iterated maps: the enabled engines, the fingers engine's candidate rules, fingerprinthub's matched templates, and wappalyzer's apps and their header / cookie / meta patterns. When several rules hit one product, which rule's MatchDetail and Attributes survived depended on that order (and fingers' stopAtFirst returned a random finger). For the judge this changed the evidence, the claims sent to the provider and therefore the cache keys. Engines now run in AllEngines order (others by name), rules and templates in index order, and wappalyzer apps and pattern keys in sorted order, precomputed at compile time. The ledger's report breaks ties on the full rule key, and judgeeval's directory mode adds pages to it in file order, so its samples are reproducible too. Co-Authored-By: Claude Opus 5.5 (1M context) --- cmd/judgeeval/main.go | 4 ++- engine.go | 28 +++++++++++++++---- fingerprinthub/fingerprinthub.go | 15 +++++++++-- fingers/common.go | 9 +++++++ judge/maintain/ledger.go | 12 ++++++++- wappalyzer/fingerprints.go | 46 +++++++++++++++++++++++++++----- wappalyzer/wappalyzer.go | 1 + 7 files changed, 100 insertions(+), 15 deletions(-) diff --git a/cmd/judgeeval/main.go b/cmd/judgeeval/main.go index 94674ad..bb87233 100644 --- a/cmd/judgeeval/main.go +++ b/cmd/judgeeval/main.go @@ -126,7 +126,6 @@ func run(providerName, samples, labelsPath, cacheDir, out, endpoint string, rps for i := range jobs { r, err := evaluatePage(ctx, engine, j, files[i]) rows[i] = r - ledger.Add(r.ID, r.Judged) var apiErr *jev.APIError if errors.As(err, &apiErr) && apiErr.Status == 401 { once.Do(func() { fatal = errors.New("unauthorized: check " + jev.EnvAPIKey); cancel() }) @@ -150,6 +149,9 @@ dispatch: if err := writeRows(filepath.Join(out, "rows.jsonl"), rows); err != nil { return err } + for _, r := range rows { // in file order, so the ledger's samples are reproducible + ledger.Add(r.ID, r.Judged) + } m := &replayManifest{} for _, r := range rows { m.Samples = append(m.Samples, replaySample{ID: r.ID, Group: r.ID, ContentSHA256: r.SHA256, Labels: labels[r.ID]}) diff --git a/engine.go b/engine.go index 1c94bf3..36eab7f 100644 --- a/engine.go +++ b/engine.go @@ -5,6 +5,7 @@ import ( "bytes" "fmt" "net/http" + "sort" "strings" "github.com/chainreactors/fingers/alias" @@ -331,11 +332,7 @@ func (engine *Engine) WebMatch(resp *http.Response) common.Frameworks { body, header, _ := httputils.SplitHttpRaw(lower) combined := make(common.Frameworks) - for name, ok := range engine.Enabled { - if !ok { - continue - } - + for _, name := range engine.enabledNames() { // Check if engine supports web fingerprinting if !engine.Capabilities[name].SupportWeb { continue @@ -373,6 +370,27 @@ func (engine *Engine) WebMatch(resp *http.Response) common.Frameworks { return combined } +// enabledNames lists the enabled engines in AllEngines order, then any others +// by name. Merging keeps the first engine's details for a shared product, so +// a fixed order makes results reproducible. +func (engine *Engine) enabledNames() []string { + var names, extra []string + known := map[string]bool{} + for _, name := range AllEngines { + known[name] = true + if engine.Enabled[name] { + names = append(names, name) + } + } + for name, ok := range engine.Enabled { + if ok && !known[name] { + extra = append(extra, name) + } + } + sort.Strings(extra) + return append(names, extra...) +} + // ServiceMatch 专门用于Service指纹识别 func (engine *Engine) ServiceMatch(host string, portStr string, level int, sender common.ServiceSender, callback common.ServiceCallback) []*common.ServiceResult { var results []*common.ServiceResult diff --git a/fingerprinthub/fingerprinthub.go b/fingerprinthub/fingerprinthub.go index 078ae8d..b1ca4fc 100644 --- a/fingerprinthub/fingerprinthub.go +++ b/fingerprinthub/fingerprinthub.go @@ -3,6 +3,7 @@ package fingerprinthub import ( "encoding/json" "fmt" + "sort" "strings" "github.com/chainreactors/fingers/common" @@ -202,7 +203,7 @@ func (engine *FingerPrintHubEngine) WebMatch(content []byte) common.Frameworks { mr := engine.webTemplateIndex.Match(lowerHeaderStr, lowerBodyStr) if engine.CaseInsensitive && !engine.MatchDetailEnabled { - for ti := range mr.Matched { + for _, ti := range sortedIndices(mr.Matched) { frames.Add(engine.newFramework(engine.webTemplates[ti])) } } @@ -212,7 +213,8 @@ func (engine *FingerPrintHubEngine) WebMatch(content []byte) common.Frameworks { mr.NeedsCheck[ti] = true } } - for ti := range mr.NeedsCheck { + // Templates run in index order: the first to hit a product keeps its details. + for _, ti := range sortedIndices(mr.NeedsCheck) { tmpl := engine.webTemplates[ti] if len(tmpl.requests) == 0 { continue @@ -266,3 +268,12 @@ func activeServiceLen(engine *FingerPrintHubEngine) int { func yamlMarshal(v interface{}) ([]byte, error) { return yaml.Marshal(v) } + +func sortedIndices(set map[int]bool) []int { + out := make([]int, 0, len(set)) + for i := range set { + out = append(out, i) + } + sort.Ints(out) + return out +} diff --git a/fingers/common.go b/fingers/common.go index fc6c03e..bdce3af 100644 --- a/fingers/common.go +++ b/fingers/common.go @@ -2,6 +2,8 @@ package fingers import ( "bytes" + "sort" + "github.com/chainreactors/fingers/common" "github.com/chainreactors/fingers/resources" "github.com/chainreactors/utils/iutils" @@ -80,7 +82,14 @@ func (fs Fingers) ACPassiveMatch(input *Content, idx *KeywordIndex, stopAtFirst frames := make(common.Frameworks) vulns := make(common.Vulns) candidates := idx.MatchCandidates(input.Header, input.Body) + // Rule order, not map order: the first finger to hit a product keeps its + // details, and stopAtFirst returns the same finger every run. + order := make([]int, 0, len(candidates)) for fi := range candidates { + order = append(order, fi) + } + sort.Ints(order) + for _, fi := range order { finger := fs[fi] if idx.IsFastPath(fi) && !finger.EnableMatchDetail { diff --git a/judge/maintain/ledger.go b/judge/maintain/ledger.go index 57fa704..04fa198 100644 --- a/judge/maintain/ledger.go +++ b/judge/maintain/ledger.go @@ -113,7 +113,17 @@ func (l *Ledger) Report(minHits int, minRefuted float64) []*RuleStats { if ra != rb { return ra > rb } - return out[a].Name < out[b].Name + ka, kb := out[a].RuleKey, out[b].RuleKey + if ka.Name != kb.Name { + return ka.Name < kb.Name + } + if ka.Engine != kb.Engine { + return ka.Engine < kb.Engine + } + if ka.Rule != kb.Rule { + return ka.Rule < kb.Rule + } + return ka.Matcher < kb.Matcher }) return out } diff --git a/wappalyzer/fingerprints.go b/wappalyzer/fingerprints.go index 51d78ee..bd9e27c 100644 --- a/wappalyzer/fingerprints.go +++ b/wappalyzer/fingerprints.go @@ -3,6 +3,7 @@ package wappalyzer import ( "github.com/chainreactors/fingers/common" "regexp" + "sort" "strconv" "strings" ) @@ -34,6 +35,21 @@ type Fingerprint struct { type CompiledFingerprints struct { // Apps is organized as Apps map[string]*CompiledFingerprint + // ordered holds Apps by name, so apps whose names differ only in case + // ("LiteSpeed Cache", "Litespeed Cache") merge the same way every run. + ordered []*CompiledFingerprint +} + +func (f *CompiledFingerprints) order() { + names := make([]string, 0, len(f.Apps)) + for name := range f.Apps { + names = append(names, name) + } + sort.Strings(names) + f.ordered = make([]*CompiledFingerprint, len(names)) + for i, name := range names { + f.ordered[i] = f.Apps[name] + } } // CompiledFingerprint contains the compiled fingerprints from the tech json @@ -63,6 +79,9 @@ type CompiledFingerprint struct { meta map[string][]*versionRegex // cpe contains the cpe for a fingerpritn cpe string + // cookieKeys, headerKeys and metaKeys list the map keys in a fixed + // order, so the evidence recorded for a hit does not vary between runs. + cookieKeys, headerKeys, metaKeys []string } func (finger *CompiledFingerprint) NewFrame(version string) *common.Framework { @@ -247,6 +266,18 @@ func compileFingerprint(app string, fingerprint *Fingerprint) *CompiledFingerpri } compiled.meta[meta] = compiledList } + for k := range compiled.cookies { + compiled.cookieKeys = append(compiled.cookieKeys, k) + } + for k := range compiled.headers { + compiled.headerKeys = append(compiled.headerKeys, k) + } + for k := range compiled.meta { + compiled.metaKeys = append(compiled.metaKeys, k) + } + sort.Strings(compiled.cookieKeys) + sort.Strings(compiled.headerKeys) + sort.Strings(compiled.metaKeys) return compiled } @@ -255,7 +286,7 @@ func (f *CompiledFingerprints) matchString(data string, part part) common.Framew var matched bool var evidence string technologies := make(common.Frameworks) - for _, fingerprint := range f.Apps { + for _, fingerprint := range f.ordered { var version string switch part { @@ -309,7 +340,7 @@ func (f *CompiledFingerprints) matchKeyValueString(key, value string, part part) var evidence string var technologies = make(common.Frameworks) - for _, fingerprint := range f.Apps { + for _, fingerprint := range f.ordered { var version string switch part { @@ -377,12 +408,13 @@ func (f *CompiledFingerprints) matchMapString(keyValue map[string]string, part p var evidence string technologies := make(common.Frameworks) - for _, fingerprint := range f.Apps { + for _, fingerprint := range f.ordered { var version string switch part { case cookiesPart: - for data, pattern := range fingerprint.cookies { + for _, data := range fingerprint.cookieKeys { + pattern := fingerprint.cookies[data] value, ok := keyValue[data] if !ok { continue @@ -399,7 +431,8 @@ func (f *CompiledFingerprints) matchMapString(keyValue map[string]string, part p } } case headersPart: - for data, pattern := range fingerprint.headers { + for _, data := range fingerprint.headerKeys { + pattern := fingerprint.headers[data] value, ok := keyValue[data] if !ok { continue @@ -413,7 +446,8 @@ func (f *CompiledFingerprints) matchMapString(keyValue map[string]string, part p } } case metaPart: - for data, patterns := range fingerprint.meta { + for _, data := range fingerprint.metaKeys { + patterns := fingerprint.meta[data] value, ok := keyValue[data] if !ok { continue diff --git a/wappalyzer/wappalyzer.go b/wappalyzer/wappalyzer.go index ada4e3a..3da8fe1 100644 --- a/wappalyzer/wappalyzer.go +++ b/wappalyzer/wappalyzer.go @@ -58,6 +58,7 @@ func (engine *Wappalyze) loadFingerprints(data []byte) error { for app, fingerprint := range fingerprintsStruct.Apps { engine.fingerprints.Apps[app] = compileFingerprint(app, fingerprint) } + engine.fingerprints.order() return nil } From 4a87b4d01a79ab130fa3950115b6726af6dec9cb Mon Sep 17 00:00:00 2001 From: M09Ic Date: Sat, 26 Sep 2026 21:15:48 -0700 Subject: [PATCH 13/16] refactor(judge): simplify the mechanism, exclude format versions, retest Simplification, verified byte for byte with a deterministic provider over the full judgeeval pipeline (189 provider requests and all outputs unchanged): - Rulings are validated once, by jev.Judge; code-built fact rulings are valid by construction, so applyPresence / applyVersion return nothing. - Presence facts (declared / absent) use one package-level factClaim. - Inspect and Version share one ready check; declares loses its nested loop; the test-only containsWord is gone. - gen.New(j, name) takes the required name; Name() and the ProbeWith wrapper are removed. - Tests are grouped by mechanism (inspect / presence / version); the legacy-field guard in judgeeval is dropped. Behaviour change: the version candidates no longer include the HTTP version that starts each Via entry ("Via: 1.1 varnish") or the format version of an SVG / XML declaration (). On the real retest Jev had picked them as varnish's and caddy's versions. Retest on the 69-page labelled set with jev-1.13.0 (docs/jev-claim-retest-20260927.md): false hits 24 -> 0, true hits removed 0, versions 36 correct / 0 wrong with 0 unsupported fills, ledger top three git / ipeakcms / webp_server_go; without the added library Discover marks SearXNG, Wakapi and IT Tools missing, and with it raises no false alarm. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 2 +- cmd/judgeeval/replay_generation.go | 2 +- cmd/judgeeval/replay_test.go | 7 - docs/jev-claim-retest-20260927.md | 56 +++++ judge/README.md | 2 +- judge/facts.go | 24 +-- judge/gen/gen.go | 32 +-- judge/gen/gen_test.go | 34 ++- judge/inspect.go | 11 +- judge/inspect_test.go | 288 +++++++++++++++++++++++++ judge/internal/evidence/evidence.go | 6 +- judge/judge.go | 17 +- judge/judge_test.go | 318 +--------------------------- judge/operations_test.go | 125 ----------- judge/presence.go | 30 ++- judge/presence_test.go | 165 +++++++++++++++ judge/real_evidence_test.go | 131 ------------ judge/version.go | 35 +-- judge/version_test.go | 125 +++++++++++ 19 files changed, 724 insertions(+), 686 deletions(-) create mode 100644 docs/jev-claim-retest-20260927.md create mode 100644 judge/inspect_test.go delete mode 100644 judge/operations_test.go create mode 100644 judge/presence_test.go delete mode 100644 judge/real_evidence_test.go create mode 100644 judge/version_test.go diff --git a/README.md b/README.md index 4cb9b28..823714e 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ for _, f := range accepted { **数据外发**：判定时会把响应的精简视图发给 Provider，包括响应头（去掉 Date、Set-Cookie 值等无关头）、Cookie 名、标题、generator/description、脚本和样式路径、内联脚本开头、HTML 注释、表单字段名，以及正文前 1500 字。扫描授权范围内的目标前，请确认允许把这些内容发送给第三方服务。 -历史 Claim 实现于 2026-09-26 在 69 份已标注的真实响应上：误报 24 → 0，真实命中误删 0，版本正确 36、错误 0，见 [历史验收报告](docs/jev-claim-acceptance-20260926.md)。这些指标尚未用本次简化实现重新请求 Jev 验证。 +当前实现于 2026-09-27 在 69 份已标注的真实响应上复测（jev-1.13.0）：误报 24 → 0，真实命中误删 0，版本正确 36、错误 0；不加载新增规则时 Discover 找出全部 3 个漏报产品，加载后无误报警。见 [复测报告](docs/jev-claim-retest-20260927.md)。 指纹判定的完整能力见 [judge/README.md](judge/README.md)；用正反样本生成原生指纹见 `judge/gen`；设计讨论见 [#34](https://github.com/chainreactors/fingers/issues/34)。 diff --git a/cmd/judgeeval/replay_generation.go b/cmd/judgeeval/replay_generation.go index 2f9ffa8..9b1cae1 100644 --- a/cmd/judgeeval/replay_generation.go +++ b/cmd/judgeeval/replay_generation.go @@ -156,7 +156,7 @@ func generateReplay(m *replayManifest, j *judge.Judge, out string) ([]generation return nil, err } r.Excluded = excluded - g := gen.New(j).Name(p.Product) + g := gen.New(j, p.Product) for _, id := range p.Positive { s := samples[id] g.Positive(s.raw) diff --git a/cmd/judgeeval/replay_test.go b/cmd/judgeeval/replay_test.go index 08a02fa..8536c01 100644 --- a/cmd/judgeeval/replay_test.go +++ b/cmd/judgeeval/replay_test.go @@ -257,13 +257,6 @@ func TestDirectoryAndReplayShareRecordAndCache(t *testing.T) { if err := json.Unmarshal(data, &r); err != nil { t.Fatal(err) } - var fields map[string]json.RawMessage - _ = json.Unmarshal(data, &fields) - for _, old := range []string{"refined", "filled_versions", "verdicts", "outcomes", "evidence", "rejected", "duplicates", "missing_candidates"} { - if _, ok := fields[old]; ok { - t.Fatalf("legacy field %s remains", old) - } - } return r } first := readRow(directory) diff --git a/docs/jev-claim-retest-20260927.md b/docs/jev-claim-retest-20260927.md new file mode 100644 index 0000000..4b217a8 --- /dev/null +++ b/docs/jev-claim-retest-20260927.md @@ -0,0 +1,56 @@ +# judge 复测：合约下沉 utils/jev 之后（2026-09-27） + +## 结论 + +**当前代码在真实 Jev 上通过了 2026-09-26 验收的全部标准，并补上了当时遗留的两个问题。** 输入与[历史验收](jev-claim-acceptance-20260926.md)相同：69 份真实响应快照，682 个已标注的产品判断。模型为 jev-1.13.0，`MinConfidence` 为 0.3，0 份请求失败。 + +| 验收标准 | 2026-09-26 | 本次 | 结论 | +|---|---:|---:|---| +| 有标注的误报删除数（要求 ≥24） | 24 | **24** | ✅ | +| 真实命中误删 | 0 | **0** | ✅ | +| 3 个宝塔默认页上的宝塔 | 全部保留 | **全部保留**（insufficient，未删除） | ✅ | +| `varnish=1.1` | 不出现 | **不出现** | ✅ | +| 版本 正确/错误 | 36/0 | **36/0**，无依据填值 0，正确弃权 26 | ✅ | +| Ledger 前三 | git、ipeakcms、webp_server_go | **git、ipeakcms、webp_server_go** | ✅ | +| Discover，不加载 library | SearXNG、Wakapi 判 missing；IT Tools 判 insufficient | **三个都判 missing**（0.88 / 0.69 / 0.40） | ✅ 3/3 | +| Discover，加载 library | IT Tools 误报警 missing | **没有误报警**，IT Tools 判 insufficient（0.21） | ✅ | + +## 本轮修正 + +**规则引擎输出不确定。** 同一份响应跑两次，合并结果可能不同。几处都按 map 顺序遍历：引擎之间、fingers 的候选规则、fingerprinthub 命中的模板、wappalyzer 的应用及其 header / cookie / meta 模式。同一产品有多条规则命中时，保留谁的 MatchDetail 和 Attributes 取决于遍历顺序，于是证据、发给 Jev 的请求和缓存键也随之变化。现在都改为固定顺序，Ledger 的排序和样本也固定了。 + +**两类格式版本不再作为候选。** 第一次真实复测中，Jev 在 4 个 GitHub Pages 页面上把 `Via: 1.1 varnish` 的 1.1 选为 varnish 的版本（置信度 0.41–0.74），在 Caddy 默认页上选了 SVG 的 `version="1.1"`（0.48）。这些都是代码能确定的事实：Via 头每一项开头是 HTTP 协议版本，SVG / XML 声明里的 version 是格式版本。现在它们和 viewport 的 `initial-scale=1.0`、IPv4 一样，在提取候选时由代码排除，没有加任何推翻 Jev 回答的逻辑。 + +**代码精简。** 以下改动都用确定性回归验证过：用一个按请求内容哈希回答的本地 Provider，跑完 judgeeval 全链路，重构前后 189 个 Provider 请求和全部输出逐字节一致。具体改动： +- Claim 合约移到 `utils/jev`。 +- 删除重复的 Ruling 校验。 +- 代码确定的事实使用固定的 `factClaim`。 +- gen 改为 `New(j, name)`，删除 `ProbeWith`。 +- 测试按机制重排。 + +## 用量 + +| 运行 | Provider 请求 | 缓存命中 | Claim 数 | 输入 token | 耗时 | +|---|---:|---:|---:|---:|---:| +| 加载 library，冷缓存 | 101 | 4 | 340 | 218,512 | 72 秒 | +| 修正版本候选后，加载 library | 11 | 91 | 331 | 25,726 | 17 秒 | +| 修正版本候选后，不加载 library | 0 | 101 | 314 | 0 | 9 秒 | + +修正只改变了含 Via 头或 SVG 的页面的版本 Claim，其余请求全部命中缓存。 + +## 局限 + +- 样本仍是 69 页、59 个主机，`MinConfidence` 需要在更大的样本集上复核。 +- IT Tools 的 Coverage 置信度低（0.40 / 0.21），文字很少的单页应用仍是 Discover 的弱项。 +- Presence 中 insufficient 从 7 增至 12，都没有删除条目（`DropInsufficient=false`），不影响上表指标。 + +## 复现 + +``` +go run ./cmd/judgeeval \ + -manifest .judge-data/expansion-20260926/run-07-final/manifest.json \ + -library .judge-data/expansion-20260926/run-07-final/library.yaml \ + -cache .judge-data/cache-v2 -out .judge-data/claim-v2b +``` + +`TYPESAFE_API_KEY` 只通过环境变量传入。去掉 `-library` 验证 Discover（输出在 `.judge-data/claim-v2b-nolib`）。`.judge-data/` 不纳入版本控制。 diff --git a/judge/README.md b/judge/README.md index 81afd43..d94572c 100644 --- a/judge/README.md +++ b/judge/README.md @@ -50,7 +50,7 @@ Presence 用 `running` 表示产品参与生成响应，`mentioned` / `unrelated - `maintain.Ledger.Add(id, all)` 从既有 presence 注解汇总规则表现，不再请求 Provider。`Report` 返回深复制快照，记录 Outcomes、Options 和样本 ID；`Hits()` 与 `RefutedFraction()` 从 Outcomes 推导。Presence 选项常量统一为 `judge.OptionRunning` 等 `Option*`，原生 `Framework.Judge.Option` 输出字段继续保留。当前引擎聚合后的 MatchDetail 仍只保留首个匹配详情，未重设计完整规则来源。 - `maintain.Discover` 按资源路径、表单、Cookie/自定义头名和页面文字等结构特征聚类。每个多主机簇提出 Coverage Claim：`explained` / `custom_site` → Holds，`missing` → Refuted，另有 insufficient。Cluster 直接可序列化，仅保存成员 ID、Coverage 原始 Ruling、解析的 Outcome、已报告产品和代码抽取的候选名。候选名供人命名，不作为 Provider 选项。 -- `gen.Generator` 为人指定的 Name 从正反样本生成规则。需要时调用 Version。Validate 运行规则引擎，不调用模型，不修改内置库。 +- `gen.New(j, name)` 为人指定的产品名从正反样本生成规则，Probe 附加主动探测的请求与响应；正例未带版本时调用 Version。Validate 运行规则引擎，不调用模型，不修改内置库。 `cmd/judgeeval` 两种输入模式共用 `baseline` / `judged` 行记录。接受结果、拒绝项、版本差异和指标均从它们派生，聚类通过样本 ID 关联。生成评估同样保存原生 Framework，以 manifest 为唯一标签来源，分数和状态按需推导。报告 schema 为 3，统计使用 `claims/options`；旧缓存因命名空间隔离不会被复用。详见 [judgeeval](../cmd/judgeeval/README.md)。 diff --git a/judge/facts.go b/judge/facts.go index 6d9ca25..fb4c9ec 100644 --- a/judge/facts.go +++ b/judge/facts.go @@ -69,23 +69,19 @@ func declares(p *evidence.Page, name string) bool { continue } for _, value := range values { + // "Apache/2.4 (Debian)", "nginx, Varnish": a token names the product... for _, token := range strings.FieldsFunc(value, func(r rune) bool { return r == ',' || r == ';' || r == '(' || r == ')' }) { - token = strings.TrimSpace(token) if i := strings.IndexByte(token, '/'); i >= 0 { - token = strings.TrimSpace(token[:i]) + token = token[:i] } - if NormalizeName(token) == key { + if NormalizeName(strings.TrimSpace(token)) == key { return true } - // A header can list several products separated by spaces. - for _, part := range strings.Fields(value) { - if !strings.Contains(part, "/") { - continue - } - part = strings.SplitN(part, "/", 2)[0] - if NormalizeName(part) == key { - return true - } + } + // ...or one of several space-separated "name/version" products does. + for _, part := range strings.Fields(value) { + if i := strings.IndexByte(part, '/'); i > 0 && NormalizeName(part[:i]) == key { + return true } } } @@ -225,10 +221,8 @@ func excerpt(text string, start, end int) matchExcerpt { return matchExcerpt{Where: where, Text: evidence.Truncate(evidence.Clean(strings.ToValidUTF8(text[lo:hi], "")), maxExcerpt)} } -// containsWord matches ASCII keys on word boundaries ("acti" must not match +// indexWord finds ASCII keys on word boundaries ("acti" must not match // "action"); CJK keys have no word boundaries and match as substrings. -func containsWord(haystack, key string) bool { return indexWord(haystack, key) >= 0 } - func indexWord(haystack, key string) int { if key == "" { return -1 diff --git a/judge/gen/gen.go b/judge/gen/gen.go index b0584a8..8ec5e80 100644 --- a/judge/gen/gen.go +++ b/judge/gen/gen.go @@ -37,14 +37,11 @@ type generatorSample struct { probes map[string][]byte } -// New creates a fingerprint generator using j for version selection. j may -// be nil when every positive carries its version (PositiveVersion). -func New(j *judge.Judge) *Generator { return &Generator{judge: j} } - -// Name sets the product the rules detect; Generate requires it. -func (g *Generator) Name(name string) *Generator { - g.name = strings.TrimSpace(name) - return g +// New creates a generator for the product a person names, using j for +// versions the positives do not carry. j may be nil when every positive +// carries its version (PositiveVersion). +func New(j *judge.Judge, name string) *Generator { + return &Generator{judge: j, name: strings.TrimSpace(name)} } // Positive adds an HTTP response known to belong to the product. @@ -83,23 +80,6 @@ func (g *Generator) Probe(request, response []byte) *Generator { return g } -// ProbeWith sends request and records the response for the last sample. The -// caller controls network behavior through send. -func (g *Generator) ProbeWith(ctx context.Context, request []byte, send func(context.Context, []byte) ([]byte, error)) error { - if g.current == nil { - return fmt.Errorf("gen: ProbeWith requires a preceding Positive or Negative") - } - if send == nil || len(request) == 0 { - return fmt.Errorf("gen: ProbeWith requires a sender and non-empty send_data") - } - response, err := send(ctx, request) - if err != nil { - return err - } - g.Probe(request, response) - return nil -} - // Generate returns one compiled, rule-validated fingerprint. It never writes // a templates repository. At least one positive and one negative are required. func (g *Generator) Generate(ctx context.Context) (*fingerlib.Finger, error) { @@ -114,7 +94,7 @@ func (g *Generator) Generate(ctx context.Context) (*fingerlib.Finger, error) { } name := g.name if name == "" { - return nil, fmt.Errorf("gen: Name is required: a person names the product, the provider never does") + return nil, fmt.Errorf("gen: a person names the product; the provider never does") } versions := make([]string, len(g.positive)) diff --git a/judge/gen/gen_test.go b/judge/gen/gen_test.go index 7a8d359..baede7a 100644 --- a/judge/gen/gen_test.go +++ b/judge/gen/gen_test.go @@ -3,7 +3,6 @@ package gen import ( "bytes" "context" - "errors" "fmt" "strings" "testing" @@ -36,7 +35,7 @@ func TestGeneratorPassiveVersionAndValidation(t *testing.T) { positive1 := []byte("HTTP/1.1 200 OK\r\nServer: nginx/2.401.3\r\nX-Jenkins: 2.401.3\r\n\r\nJenkins") positive2 := []byte("HTTP/1.1 200 OK\r\nServer: nginx/2.402.1\r\nX-Jenkins: 2.402.1\r\n\r\nJenkins") negative := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nBlog

X-Jenkins: 9.9.9 release notes

") - g := New(nil).Name("Jenkins").PositiveVersion(positive1, "2.401.3").PositiveVersion(positive2, "2.402.1").Negative(negative) + g := New(nil, "Jenkins").PositiveVersion(positive1, "2.401.3").PositiveVersion(positive2, "2.402.1").Negative(negative) f, err := g.Generate(context.Background()) if err != nil { t.Fatal(err) @@ -62,21 +61,12 @@ func TestGeneratorPassiveVersionAndValidation(t *testing.T) { } } -func TestGeneratorActiveAndProbeWith(t *testing.T) { +func TestGeneratorActive(t *testing.T) { base := []byte("HTTP/1.1 200 OK\r\n\r\nLogin") request := []byte("/admin") positive := []byte("HTTP/1.1 200 OK\r\nX-Orion: console\r\n\r\nOrion console") negative := []byte("HTTP/1.1 404 Not Found\r\n\r\nmissing") - g := New(nil).Name("Orion").Positive(base) - if err := g.ProbeWith(context.Background(), request, func(_ context.Context, got []byte) ([]byte, error) { - if !bytes.Equal(got, request) { - return nil, errors.New("wrong request") - } - return positive, nil - }); err != nil { - t.Fatal(err) - } - g.Negative(base).Probe(request, negative) + g := New(nil, "Orion").Positive(base).Probe(request, positive).Negative(base).Probe(request, negative) f, err := g.Generate(context.Background()) if err != nil { t.Fatal(err) @@ -90,18 +80,18 @@ func TestGeneratorActiveAndProbeWith(t *testing.T) { if len(f.Rules[0].Regexps.CompliedRegexp) != 0 { t.Fatal("Validate changed compiled rules") } - if _, err := New(nil).Name("Orion").Positive(base).Probe(request, positive).Negative(base).Generate(context.Background()); err == nil { + if _, err := New(nil, "Orion").Positive(base).Probe(request, positive).Negative(base).Generate(context.Background()); err == nil { t.Fatal("generated active rule without a negative probe") } } -// A person names the product: without Name, Generate fails rather than ask +// A person names the product: without a name, Generate fails rather than ask // the provider for one. func TestGeneratorRequiresName(t *testing.T) { positive := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nOrion

Orion console

") negative := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nOther") j := judge.New(names("Orion")) - if f, err := New(j).Positive(positive).Negative(negative).Generate(context.Background()); err == nil { + if f, err := New(j, "").Positive(positive).Negative(negative).Generate(context.Background()); err == nil { t.Fatalf("generated %+v without a name", f) } } @@ -111,7 +101,7 @@ func TestGeneratorActiveVersion(t *testing.T) { request := []byte("/version") positive := []byte("HTTP/1.1 200 OK\r\nX-Orion: 3.1.2\r\n\r\nversion") negative := []byte("HTTP/1.1 200 OK\r\n\r\nnot installed") - g := New(nil).Name("Orion").PositiveVersion(base, "3.1.2").Probe(request, positive).Negative(base).Probe(request, negative) + g := New(nil, "Orion").PositiveVersion(base, "3.1.2").Probe(request, positive).Negative(base).Probe(request, negative) f, err := g.Generate(context.Background()) if err != nil { t.Fatal(err) @@ -127,7 +117,7 @@ func TestGeneratorCoversDifferentPositiveVariants(t *testing.T) { first := []byte(fmt.Sprintf(base, "X-Orion-Instance: ready\r\n")) second := []byte(fmt.Sprintf(base, "X-Orion-Service: yes\r\n")) other := []byte(fmt.Sprintf(base, "Server: nginx\r\n")) - g := New(nil).Name("Orion").Positive(first).Positive(second).Negative(other) + g := New(nil, "Orion").Positive(first).Positive(second).Negative(other) f, err := g.Generate(context.Background()) if err != nil || len(f.Rules) != 2 { t.Fatalf("variant rules = %+v, %v", f, err) @@ -139,7 +129,7 @@ func TestGeneratedVersionHandlesHyphenatedHeaderAndReleaseSuffix(t *testing.T) { return []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\nX-New-Api-Version: v" + v + "\r\n\r\nNew API") } neg := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nWelcome") - g := New(nil).Name("New API").PositiveVersion(raw("1.0.0-rc.35"), "1.0.0-rc.35").Negative(neg) + g := New(nil, "New API").PositiveVersion(raw("1.0.0-rc.35"), "1.0.0-rc.35").Negative(neg) f, err := g.Generate(context.Background()) if err != nil { t.Fatal(err) @@ -157,7 +147,7 @@ func TestGeneratedMetaRuleSurvivesTitleAndCalendarVersionChanges(t *testing.T) { return []byte("HTTP/1.1 200 OK\r\n\r\n" + title + "") } negative := []byte("HTTP/1.1 200 OK\r\n\r\nSearXNG Documentation

Install SearXNG

") - f, err := New(nil).Name("SearXNG").PositiveVersion(page("Search One", "2026.9.23+3cd69d30e"), "2026.9.23+3cd69d30e").Negative(negative).Generate(context.Background()) + f, err := New(nil, "SearXNG").PositiveVersion(page("Search One", "2026.9.23+3cd69d30e"), "2026.9.23+3cd69d30e").Negative(negative).Generate(context.Background()) if err != nil { t.Fatal(err) } @@ -175,7 +165,7 @@ func TestGeneratedFooterVersionRequiresProductDeclaration(t *testing.T) { return []byte("HTTP/1.1 200 OK\r\n\r\n" + title + "
Log in
\n v" + version + " @ mysql\n
") } negative := page("Different Application", "2.18.0-be6aeb4") - f, err := New(nil).Name("Wakapi").PositiveVersion(page("Wakapi – Coding Statistics", "2.18.0-be6aeb4"), "2.18.0-be6aeb4").Negative(negative).Generate(context.Background()) + f, err := New(nil, "Wakapi").PositiveVersion(page("Wakapi – Coding Statistics", "2.18.0-be6aeb4"), "2.18.0-be6aeb4").Negative(negative).Generate(context.Background()) if err != nil { t.Fatal(err) } @@ -192,7 +182,7 @@ func TestGeneratedApplicationDoesNotLearnSharedInfrastructure(t *testing.T) { for _, name := range []string{"IT Tools", "Docmost"} { positive := []byte("HTTP/1.1 200 OK\r\nServer: cloudflare\r\nX-Frame-Options: SAMEORIGIN\r\n\r\n" + name + "") negative := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nDifferent Application") - f, err := New(nil).Name(name).Positive(positive).Negative(negative).Generate(context.Background()) + f, err := New(nil, name).Positive(positive).Negative(negative).Generate(context.Background()) if err != nil { t.Fatal(err) } diff --git a/judge/inspect.go b/judge/inspect.go index 1746ad7..859500c 100644 --- a/judge/inspect.go +++ b/judge/inspect.go @@ -2,7 +2,6 @@ package judge import ( "context" - "fmt" "sort" "github.com/chainreactors/fingers/common" @@ -21,12 +20,9 @@ func (j *Judge) Inspect(ctx context.Context, raw []byte, hits common.Frameworks) f.Judge = nil } } - if err := ctx.Err(); err != nil { + if err := j.ready(ctx); err != nil { return working, err } - if !probability(j.MinConfidence) { - return working, fmt.Errorf("invalid minimum confidence") - } p, err := evidence.NewPage(raw) if err != nil { return working, err @@ -42,12 +38,9 @@ func (j *Judge) Version(ctx context.Context, raw []byte, f *common.Framework) (s if f == nil { return "", nil } - if err := ctx.Err(); err != nil { + if err := j.ready(ctx); err != nil { return "", err } - if !probability(j.MinConfidence) { - return "", fmt.Errorf("invalid minimum confidence") - } if f.Attributes != nil && f.Version != "" { return f.Version, nil } diff --git a/judge/inspect_test.go b/judge/inspect_test.go new file mode 100644 index 0000000..d3b4ec5 --- /dev/null +++ b/judge/inspect_test.go @@ -0,0 +1,288 @@ +package judge + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "github.com/chainreactors/fingers/common" + "github.com/chainreactors/utils/jev" +) + +func TestInspect(t *testing.T) { + m := newMock() + j := New(jev.Cached(m, jev.DefaultCacheSize)) + all, err := j.Inspect(context.Background(), []byte(jenkinsRaw), testFrames()) + if err != nil { + t.Fatal(err) + } + if w := all["wordpress"].Judge; w == nil || !w.Rejected || w.Option != OptionMentioned || w.Outcome != "refuted" || len(w.Evidence) == 0 || !strings.Contains(w.Evidence[0], "WordPress") { + t.Errorf("text-only wordpress not rejected on its evidence: %+v", w) + } + for _, name := range []string{"nginx", "jenkins"} { + if v := all[name].Judge; v.Rejected || v.Option != OptionDeclared || v.Outcome != "holds" { + t.Errorf("%s named in a header must be declared: %+v", name, v) + } + } + if h := all["hsts"].Judge; !h.Rejected || h.Option != OptionAbsent || h.Outcome != "refuted" { + t.Errorf("hsts without the header must be absent: %+v", h) + } + if all["apache tomcat"].Judge.Duplicate == all["apache-tomcat"].Judge.Duplicate { + t.Errorf("exactly one tomcat spelling must be a dup") + } + // Facts are not asked: only wordpress and tomcat are claims. + if len(m.requests) != 1 || strings.Join(m.requests[0], ",") != "presence_apachetomcat,presence_wordpress" { + t.Fatalf("requests: %v", m.requests) + } + claims, _ := json.Marshal(m.lastState["matches"]) + if !strings.Contains(string(claims), "We moved here from WordPress") { + t.Errorf("claim evidence missing from the state: %s", claims) + } + + accepted, err := j.Inspect(context.Background(), []byte(jenkinsRaw), testFrames()) + accepted = accepted.Accepted() + if err != nil { + t.Fatal(err) + } + if len(accepted) != 3 { // nginx, jenkins, tomcat + t.Errorf("accepted: %v", accepted) + } + // Both versions are bound by name in the headers: taken without asking. + if accepted["jenkins"].Version != "2.401.3" || accepted["nginx"].Version != "1.24.0" { + t.Errorf("versions: jenkins %q nginx %q", accepted["jenkins"].Version, accepted["nginx"].Version) + } + // The presence come from the cache; tomcat is offered no version, since + // both strings are bound by name to other products. + if len(m.requests) != 1 { + t.Errorf("cached presence were asked again: %v", m.requests) + } +} + +// DropInsufficient decides what an undecided claim does; a ruling below +// MinConfidence is undecided whatever it chose. +func TestInsufficient(t *testing.T) { + for _, c := range []struct { + on bool + confidence float64 + option string + rejected bool + }{ + {false, 0.95, OptionRunning, false}, + {false, 0.2, jev.OptionInsufficient, false}, + {true, 0.2, jev.OptionInsufficient, true}, + {true, 0.95, OptionRunning, false}, + } { + m := newMock() + m.confidence = c.confidence + j := New(jev.Cached(m, jev.DefaultCacheSize)) + j.DropInsufficient = c.on + all, err := j.Inspect(context.Background(), []byte(jenkinsRaw), testFrames()) + if err != nil { + t.Fatal(err) + } + tomcat := all["apache tomcat"].Judge + if all["apache-tomcat"].Judge.Duplicate { + tomcat = all["apache-tomcat"].Judge + } + outcome := "holds" + if c.option == jev.OptionInsufficient { + outcome = "insufficient" + } + if tomcat.Option != OptionRunning || tomcat.Outcome != outcome || tomcat.Rejected != c.rejected { + t.Errorf("%+v: tomcat %+v", c, tomcat) + } + } +} + +// A failed provider leaves the input untouched, and the result still holds +// what code established: duplicates merged, header facts decided, the +// undecided hits kept unjudged. +func TestFailedProviderKeepsFacts(t *testing.T) { + m := newMock() + m.fail = true + frames := testFrames() + before, _ := json.Marshal(frames) + accepted, err := New(m).Inspect(context.Background(), []byte(jenkinsRaw), frames) + accepted = accepted.Accepted() + if err == nil { + t.Fatal("expected error") + } + if after, _ := json.Marshal(frames); string(after) != string(before) { + t.Fatalf("frames changed on failure:\n%s\n%s", before, after) + } + // nginx, jenkins (declared), wordpress and one tomcat (unjudged); hsts absent, one tomcat a duplicate. + if len(accepted) != 4 || accepted["hsts"] != nil || accepted["wordpress"] == nil || accepted["wordpress"].Judge != nil || + accepted["jenkins"].Judge.Option != OptionDeclared { + t.Fatalf("accepted on failure: %v", accepted) + } +} + +func TestRepeatedInspectIsCacheable(t *testing.T) { + m := newMock() + m.presence["jenkins"] = OptionRunning + j := New(jev.Cached(m, jev.DefaultCacheSize)) + for i := 0; i < 2; i++ { + accepted, err := j.Inspect(context.Background(), []byte(bodyRaw), testFrames()) + accepted = accepted.Accepted() + if err != nil { + t.Fatal(err) + } + if accepted["jenkins"].Version != "2.401.3" { + t.Fatalf("run %d: cached rulings not applied", i) + } + } + if m.calls != 2 { + t.Fatalf("calls=%d", m.calls) + } +} + +// Versions are ordered by their resolved outcome. +func TestVersionOrder(t *testing.T) { + frames := common.Frameworks{} + for name, j := range map[string]*common.Judgement{"jquery": {Option: jev.OptionInsufficient}, "apache": {Option: OptionDeclared, Outcome: jev.Holds.String()}, + "gitlab": {Option: OptionRunning, Outcome: jev.Holds.String()}, "unjudged": nil} { + f := common.NewFramework(name, common.FrameFromFingers) + f.Judge = j + frames.Add(f) + } + var got []string + for _, f := range versionOrder(frames) { + got = append(got, f.Name) + } + if strings.Join(got, ",") != "apache,gitlab,jquery,unjudged" { + t.Fatalf("order: %v", got) + } +} + +func TestPublicResultsDoNotMutateHits(t *testing.T) { + m := newMock() + m.presence["jenkins"] = OptionRunning + j := New(m) + frames := testFrames() + before, _ := json.Marshal(frames) + inspected, err := j.Inspect(context.Background(), []byte(bodyRaw), frames) + if err != nil || !inspected["wordpress"].Judge.Rejected { + t.Fatalf("inspect = %v, %v", inspected, err) + } + accepted, err := j.Inspect(context.Background(), []byte(bodyRaw), frames) + accepted = accepted.Accepted() + if err != nil || accepted["wordpress"] != nil || accepted["jenkins"].Version != "2.401.3" { + t.Fatalf("accepted = %v, %v", accepted, err) + } + // The judgement is copied, not shared with the Inspect result. + accepted["nginx"].Judge.Option = "changed" + if inspected["nginx"].Judge.Option == "changed" { + t.Fatal("Independent Inspect calls share a Judgement") + } + if after, _ := json.Marshal(frames); string(after) != string(before) { + t.Fatalf("hits changed:\n%s\n%s", before, after) + } + version, err := j.Version(context.Background(), []byte(bodyRaw), frames["jenkins"]) + if err != nil || version != "2.401.3" || frames["jenkins"].Version != "" { + t.Fatalf("version = %q, %v; input = %q", version, err, frames["jenkins"].Version) + } +} + +// A failed version round keeps the presence and leaves the input untouched. +func TestInspectVersionFailureKeepsPresence(t *testing.T) { + frames := testFrames() + m := newMock() + m.presence["jenkins"] = OptionRunning + accepted, err := New(failVersionProvider{m}).Inspect(context.Background(), []byte(bodyRaw), frames) + accepted = accepted.Accepted() + if err == nil || accepted["wordpress"] != nil || accepted["jenkins"] == nil || accepted["jenkins"].Version != "" { + t.Fatalf("failed version round = %v, %v", accepted, err) + } + for _, f := range frames { + if f.Judge != nil || f.Version != "" { + t.Fatalf("input changed after the version round failed: %+v", f) + } + } +} + +func TestInspectInvalidBatchIsNotPartlyApplied(t *testing.T) { + j := New(rulingProvider(func(cs map[string]jev.Claim) map[string]jev.Ruling { + out := map[string]jev.Ruling{} + for id := range cs { + out[id] = jev.Ruling{Option: OptionRunning, Confidence: 1} + } + out["presence_wordpress"] = jev.Ruling{Option: "invented", Confidence: 1} + return out + })) + all, err := j.Inspect(context.Background(), []byte(jenkinsRaw), testFrames()) + if err == nil || all["wordpress"].Judge != nil || (all["apache tomcat"].Judge != nil && all["apache tomcat"].Judge.Outcome != "") || all["nginx"].Judge.Outcome != jev.Holds.String() { + t.Fatalf("partially applied invalid batch: %v %v", all, err) + } +} + +func TestInspectReevaluatesAndCopiesInputs(t *testing.T) { + m := newMock() + j := New(m) + frames := testFrames() + f := frames["wordpress"] + f.MatchDetail = &common.MatchDetail{MatcherValue: "WordPress"} + f.Judge = &common.Judgement{Option: OptionRunning, Outcome: jev.Holds.String(), Evidence: []string{"old"}} + before, _ := json.Marshal(frames) + all, err := j.Inspect(context.Background(), []byte(jenkinsRaw), frames) + if err != nil || !all["wordpress"].Judge.Rejected { + t.Fatalf("old judgement reused: %v %v", all, err) + } + all["wordpress"].MatchDetail.MatcherValue = "changed" + all["wordpress"].Tags = append(all["wordpress"].Tags, "changed") + all["wordpress"].Froms[common.FrameFromFingers] = true + all["wordpress"].Version = "changed" + all["wordpress"].Judge.Evidence[0] = "changed" + after, _ := json.Marshal(frames) + if string(before) != string(after) { + t.Fatal("result mutated input") + } + partial, err := j.Inspect(context.Background(), []byte("not HTTP"), frames) + if err == nil || len(partial) != len(frames) || partial["wordpress"].Judge != nil { + t.Fatalf("parse failure lost baseline: %v %v", partial, err) + } + partial["wordpress"].MatchDetail.MatcherValue = "also changed" + if frames["wordpress"].MatchDetail.MatcherValue != "WordPress" { + t.Fatal("parse-error baseline shares pointers") + } + copied := cloneFramework(f) + copied.Judge.Evidence[0] = "changed" + if f.Judge.Evidence[0] != "old" { + t.Fatal("clone shares judgement evidence") + } +} + +// Versions stated only in the body are picked by the provider, all kept +// products in one request, each from its own candidates. +func TestInspectVersionsEveryProduct(t *testing.T) { + m := newMock() + m.presence["jenkins"] = OptionRunning + accepted, err := New(m).Inspect(context.Background(), []byte(bodyRaw), testFrames()) + accepted = accepted.Accepted() + if err != nil { + t.Fatal(err) + } + if accepted["jenkins"].Version != "2.401.3" || accepted["nginx"].Version != "1.24.0" { + t.Fatalf("versions: jenkins %q nginx %q", accepted["jenkins"].Version, accepted["nginx"].Version) + } + if len(m.requests) != 2 || !strings.HasPrefix(strings.Join(m.requests[1], ","), "version_apachetomcat,version_jenkins") { + t.Fatalf("requests: %v", m.requests) + } + if _, ok := m.lastState["version_strings"]; !ok { + t.Errorf("version strings not in the state: %v", m.lastState) + } +} + +func TestVersionNeedsConfidence(t *testing.T) { + m := newMock() + m.presence["jenkins"] = OptionRunning + m.confidence = 0.2 + accepted, err := New(m).Inspect(context.Background(), []byte(bodyRaw), testFrames()) + accepted = accepted.Accepted() + if err != nil { + t.Fatal(err) + } + if v := accepted["jenkins"].Version; v != "" { + t.Fatalf("low-confidence version written: %q", v) + } +} diff --git a/judge/internal/evidence/evidence.go b/judge/internal/evidence/evidence.go index 543b9db..d30f223 100644 --- a/judge/internal/evidence/evidence.go +++ b/judge/internal/evidence/evidence.go @@ -57,8 +57,8 @@ func NormalizeName(name string) string { return key } -// UsableKey drops keys too short to be meaningful evidence ("oa", "cms"). -func UsableKey(k string) bool { +// usableKey drops keys too short to be meaningful evidence ("oa", "cms"). +func usableKey(k string) bool { n := utf8.RuneCountInString(k) if n == len(k) { // ascii return n >= 4 @@ -103,7 +103,7 @@ func Names(generator, title, text string, headers map[string]string, assets []st byKey := map[string]candidate{} add := func(text string, weight int) { text = strings.Trim(strings.TrimSpace(NameVersion.ReplaceAllString(Clean(text), "")), "-:|/ ") - if text == "" || utf8.RuneCountInString(text) > 40 || !UsableKey(strings.ToLower(text)) { + if text == "" || utf8.RuneCountInString(text) > 40 || !usableKey(strings.ToLower(text)) { return } key := NormalizeName(text) diff --git a/judge/judge.go b/judge/judge.go index d08f5af..97e5d69 100644 --- a/judge/judge.go +++ b/judge/judge.go @@ -2,6 +2,7 @@ package judge import ( "context" + "fmt" "math" "github.com/chainreactors/fingers/judge/internal/evidence" @@ -28,14 +29,16 @@ func (j *Judge) judge(ctx context.Context, state interface{}, claims map[string] return jev.Judge(ctx, j.Provider, state, claims) } -// validRuling checks a single claim and ruling, including code-built ones. -func validRuling(c jev.Claim, r jev.Ruling) error { - return jev.ValidateRulings(map[string]jev.Claim{"claim": c}, map[string]jev.Ruling{"claim": r}) +// ready rejects a canceled context or an invalid configuration up front. +func (j *Judge) ready(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + if math.IsNaN(j.MinConfidence) || j.MinConfidence < 0 || j.MinConfidence > 1 { + return fmt.Errorf("judge: invalid MinConfidence %v", j.MinConfidence) + } + return nil } // insufficientDescription describes the abstention every claim offers. const insufficientDescription = "The evidence decides neither way." - -func probability(v float64) bool { - return !math.IsNaN(v) && !math.IsInf(v, 0) && v >= 0 && v <= 1 -} diff --git a/judge/judge_test.go b/judge/judge_test.go index c549a45..f64ff89 100644 --- a/judge/judge_test.go +++ b/judge/judge_test.go @@ -4,16 +4,13 @@ import ( "context" "encoding/json" "errors" - "fmt" "sort" "strings" "sync" "sync/atomic" - "testing" "time" "github.com/chainreactors/fingers/common" - "github.com/chainreactors/fingers/judge/internal/evidence" "github.com/chainreactors/utils/jev" ) @@ -22,25 +19,6 @@ const jenkinsRaw = "HTTP/1.1 200 OK\r\nServer: nginx/1.24.0\r\nX-Jenkins: 2.401. `

We moved here from WordPress.

` + `` -func TestNewPage(t *testing.T) { - s, err := evidence.NewPage([]byte(jenkinsRaw)) - if err != nil { - t.Fatal(err) - } - if s.Title != "Sign in [Jenkins]" || s.Headers["X-Jenkins"] != "2.401.3" { - t.Fatalf("title/header: %+v", s) - } - if len(s.Cookies) != 1 || len(s.Scripts) != 1 || len(s.Forms) != 2 { - t.Fatalf("cookies/scripts/forms: %+v", s) - } - if got := extractVersions([]byte(jenkinsRaw), 5); len(got) != 2 || got[0].Value != "1.24.0" || got[1].Value != "2.401.3" { - t.Fatalf("versions: %v", got) - } - if !declares(s, "jenkins") || !declares(s, "nginx") || declares(s, "wordpress") || declares(s, "jenk") { - t.Fatal("declares") - } -} - // mock is a Provider ruling from a table keyed by claim kind and the // product named in backticks; it records the claim keys of each call. type mock struct { @@ -134,297 +112,19 @@ const bodyRaw = "HTTP/1.1 200 OK\r\nServer: nginx\r\nSet-Cookie: JSESSIONID.1=x; `Sign in [Jenkins]` + `

We moved here from WordPress.

Jenkins 2.401.3, nginx 1.24.0
` -func TestInspect(t *testing.T) { - m := newMock() - j := New(jev.Cached(m, jev.DefaultCacheSize)) - all, err := j.Inspect(context.Background(), []byte(jenkinsRaw), testFrames()) - if err != nil { - t.Fatal(err) - } - if w := all["wordpress"].Judge; w == nil || !w.Rejected || w.Option != OptionMentioned || w.Outcome != "refuted" || len(w.Evidence) == 0 || !strings.Contains(w.Evidence[0], "WordPress") { - t.Errorf("text-only wordpress not rejected on its evidence: %+v", w) - } - for _, name := range []string{"nginx", "jenkins"} { - if v := all[name].Judge; v.Rejected || v.Option != OptionDeclared || v.Outcome != "holds" { - t.Errorf("%s named in a header must be declared: %+v", name, v) - } - } - if h := all["hsts"].Judge; !h.Rejected || h.Option != OptionAbsent || h.Outcome != "refuted" { - t.Errorf("hsts without the header must be absent: %+v", h) - } - if all["apache tomcat"].Judge.Duplicate == all["apache-tomcat"].Judge.Duplicate { - t.Errorf("exactly one tomcat spelling must be a dup") - } - // Facts are not asked: only wordpress and tomcat are claims. - if len(m.requests) != 1 || strings.Join(m.requests[0], ",") != "presence_apachetomcat,presence_wordpress" { - t.Fatalf("requests: %v", m.requests) - } - claims, _ := json.Marshal(m.lastState["matches"]) - if !strings.Contains(string(claims), "We moved here from WordPress") { - t.Errorf("claim evidence missing from the state: %s", claims) - } - - accepted, err := j.Inspect(context.Background(), []byte(jenkinsRaw), testFrames()) - accepted = accepted.Accepted() - if err != nil { - t.Fatal(err) - } - if len(accepted) != 3 { // nginx, jenkins, tomcat - t.Errorf("accepted: %v", accepted) - } - // Both versions are bound by name in the headers: taken without asking. - if accepted["jenkins"].Version != "2.401.3" || accepted["nginx"].Version != "1.24.0" { - t.Errorf("versions: jenkins %q nginx %q", accepted["jenkins"].Version, accepted["nginx"].Version) - } - // The presence come from the cache; tomcat is offered no version, since - // both strings are bound by name to other products. - if len(m.requests) != 1 { - t.Errorf("cached presence were asked again: %v", m.requests) - } -} - -// DropInsufficient decides what an undecided claim does; a ruling below -// MinConfidence is undecided whatever it chose. -func TestInsufficient(t *testing.T) { - for _, c := range []struct { - on bool - confidence float64 - option string - rejected bool - }{ - {false, 0.95, OptionRunning, false}, - {false, 0.2, jev.OptionInsufficient, false}, - {true, 0.2, jev.OptionInsufficient, true}, - {true, 0.95, OptionRunning, false}, - } { - m := newMock() - m.confidence = c.confidence - j := New(jev.Cached(m, jev.DefaultCacheSize)) - j.DropInsufficient = c.on - all, err := j.Inspect(context.Background(), []byte(jenkinsRaw), testFrames()) - if err != nil { - t.Fatal(err) - } - tomcat := all["apache tomcat"].Judge - if all["apache-tomcat"].Judge.Duplicate { - tomcat = all["apache-tomcat"].Judge - } - outcome := "holds" - if c.option == jev.OptionInsufficient { - outcome = "insufficient" - } - if tomcat.Option != OptionRunning || tomcat.Outcome != outcome || tomcat.Rejected != c.rejected { - t.Errorf("%+v: tomcat %+v", c, tomcat) - } - } -} - -// A rule that matched text unrelated to its product is a false positive, -// removed like a mere mention, and the matched text reaches the provider. -func TestUnrelatedIsRejected(t *testing.T) { - m := newMock() - m.presence["apache tomcat"] = OptionUnrelated - frames := testFrames() - for _, f := range frames { - if f.Name == "apache tomcat" || f.Name == "apache-tomcat" { - f.MatchDetail = &common.MatchDetail{MatcherType: "word", MatcherValue: "jenkins"} - } - } - all, err := New(m).Inspect(context.Background(), []byte(jenkinsRaw), frames) - if err != nil { - t.Fatal(err) - } - tomcat := all["apache tomcat"].Judge - if all["apache-tomcat"].Judge.Duplicate { - tomcat = all["apache-tomcat"].Judge - } - if tomcat.Option != OptionUnrelated || !tomcat.Rejected || len(tomcat.Evidence) == 0 || !strings.Contains(tomcat.Evidence[0], `matched "Jenkins"`) { - t.Fatalf("tomcat %+v", tomcat) - } -} - -// Versions stated only in the body are picked by the provider, all kept -// products in one request, each from its own candidates. -func TestInspectVersionsEveryProduct(t *testing.T) { - m := newMock() - m.presence["jenkins"] = OptionRunning - accepted, err := New(m).Inspect(context.Background(), []byte(bodyRaw), testFrames()) - accepted = accepted.Accepted() - if err != nil { - t.Fatal(err) - } - if accepted["jenkins"].Version != "2.401.3" || accepted["nginx"].Version != "1.24.0" { - t.Fatalf("versions: jenkins %q nginx %q", accepted["jenkins"].Version, accepted["nginx"].Version) - } - if len(m.requests) != 2 || !strings.HasPrefix(strings.Join(m.requests[1], ","), "version_apachetomcat,version_jenkins") { - t.Fatalf("requests: %v", m.requests) - } - if _, ok := m.lastState["version_strings"]; !ok { - t.Errorf("version strings not in the state: %v", m.lastState) - } -} - -func TestDeclarations(t *testing.T) { - raw := []byte("HTTP/1.1 200 OK\r\nServer: Apache/2.4.38 (Debian) OpenSSL/1.0.2q PHP/5.6.40\r\nX-Jenkins: 2.401.3\r\n" + - "X-Gitea-Version: 1.21.4\r\nX-Tomcat: Apache Tomcat/9.0.1\r\n\r\n") - decls := declarations(raw) - for product, want := range map[string]string{"Apache": "2.4.38", "openssl": "1.0.2q", "PHP": "5.6.40", "jenkins": "2.401.3", - "Gitea": "1.21.4", "Apache Tomcat": "9.0.1", "WordPress": "7.0.3", "nginx": ""} { - if got := declaredVersion(decls, product); got != want { - t.Errorf("%s: got %q want %q", product, got, want) - } - } - if !claimedByOther(decls, "5.6.40", NormalizeName("ThinkPHP")) || claimedByOther(decls, "5.6.40", NormalizeName("php")) { - t.Error("claims") - } -} - -func TestVersionNeedsConfidence(t *testing.T) { - m := newMock() - m.presence["jenkins"] = OptionRunning - m.confidence = 0.2 - accepted, err := New(m).Inspect(context.Background(), []byte(bodyRaw), testFrames()) - accepted = accepted.Accepted() - if err != nil { - t.Fatal(err) - } - if v := accepted["jenkins"].Version; v != "" { - t.Fatalf("low-confidence version written: %q", v) - } -} - -// A failed provider leaves the input untouched, and the result still holds -// what code established: duplicates merged, header facts decided, the -// undecided hits kept unjudged. -func TestFailedProviderKeepsFacts(t *testing.T) { - m := newMock() - m.fail = true - frames := testFrames() - before, _ := json.Marshal(frames) - accepted, err := New(m).Inspect(context.Background(), []byte(jenkinsRaw), frames) - accepted = accepted.Accepted() - if err == nil { - t.Fatal("expected error") - } - if after, _ := json.Marshal(frames); string(after) != string(before) { - t.Fatalf("frames changed on failure:\n%s\n%s", before, after) - } - // nginx, jenkins (declared), wordpress and one tomcat (unjudged); hsts absent, one tomcat a duplicate. - if len(accepted) != 4 || accepted["hsts"] != nil || accepted["wordpress"] == nil || accepted["wordpress"].Judge != nil || - accepted["jenkins"].Judge.Option != OptionDeclared { - t.Fatalf("accepted on failure: %v", accepted) - } -} - -func TestCache(t *testing.T) { - m := newMock() - m.presence["jenkins"] = OptionRunning - j := New(jev.Cached(m, jev.DefaultCacheSize)) - for i := 0; i < 2; i++ { - accepted, err := j.Inspect(context.Background(), []byte(bodyRaw), testFrames()) - accepted = accepted.Accepted() - if err != nil { - t.Fatal(err) - } - if accepted["jenkins"].Version != "2.401.3" { - t.Fatalf("run %d: cached rulings not applied", i) - } - } - if m.calls != 2 { - t.Fatalf("calls=%d", m.calls) - } -} - -func TestExtractVersionsPrefixes(t *testing.T) { - raw := "HTTP/1.1 200 OK\r\n\r\nV8.1SP2 10.0.0.5 jquery.min.js?ver=3.7.1{\"success_fraction\":0.0}" - var got []string - for _, c := range extractVersions([]byte(raw), 10) { - got = append(got, c.Value) - } - if strings.Join(got, ",") != "3.4,8.1SP2,3.7.1" { - t.Fatalf("got %v", got) - } -} +type rulingProvider func(map[string]jev.Claim) map[string]jev.Ruling -func TestWordBoundary(t *testing.T) { - if containsWord("take action now", "acti") || containsWord("x-github-request-id: 1", "git") { - t.Fatal("substring matched") - } - if !containsWord("sign in [jenkins]", "jenkins") || !containsWord("泛微oa", "泛微") { - t.Fatal("word missed") - } -} +func (rulingProvider) ID() string { return "ruling-provider" } -func TestRankVersions(t *testing.T) { - raw := "HTTP/1.1 200 OK\r\n\r\n" - for i := 0; i < 15; i++ { - raw += fmt.Sprintf(``, i, i) - } - raw += `` - cands := extractVersions([]byte(raw), 40) - got := rankVersions(cands, "wordpress", 3) - if got[0].Value != "7.0.3" || !strings.Contains(got[0].Context, "generator") { - t.Fatalf("wordpress generator not ranked first: %+v", got) - } - var drupal bool - for _, c := range cands { - drupal = drupal || c.Value == "10" - } - if !drupal { - t.Fatalf("major-only generator version missing: %+v", cands) - } +func (p rulingProvider) Judge(_ context.Context, _ interface{}, claims map[string]jev.Claim) (map[string]jev.Ruling, error) { + return p(claims), nil } -func TestNormalizeName(t *testing.T) { - same := [][]string{{"Apache HTTP Server", "apache-http", "apache-web-server", "Apache"}, {"Discuz! X", "discuz"}, {"Apache-Tomcat", "apache tomcat"}, {"泛微 OA", "泛微"}} - for _, names := range same { - for _, n := range names[1:] { - if NormalizeName(n) != NormalizeName(names[0]) { - t.Errorf("%q and %q not folded: %q %q", names[0], n, NormalizeName(names[0]), NormalizeName(n)) - } - } - } - for _, pair := range [][2]string{{"lighttpd", "lig"}, {"Apache Tomcat", "Apache"}, {"nginx", "ngin"}} { - if NormalizeName(pair[0]) == NormalizeName(pair[1]) { - t.Errorf("%q and %q folded", pair[0], pair[1]) - } - } -} - -// Versions are ordered by their resolved outcome. -func TestVersionOrder(t *testing.T) { - frames := common.Frameworks{} - for name, j := range map[string]*common.Judgement{"jquery": {Option: jev.OptionInsufficient}, "apache": {Option: OptionDeclared, Outcome: jev.Holds.String()}, - "gitlab": {Option: OptionRunning, Outcome: jev.Holds.String()}, "unjudged": nil} { - f := common.NewFramework(name, common.FrameFromFingers) - f.Judge = j - frames.Add(f) - } - var got []string - for _, f := range versionOrder(frames) { - got = append(got, f.Name) - } - if strings.Join(got, ",") != "apache,gitlab,jquery,unjudged" { - t.Fatalf("order: %v", got) - } -} +type failVersionProvider struct{ *mock } -// Rule excerpts include matches deep in the response. -func TestEvidenceQuotesTheMatcher(t *testing.T) { - raw := "HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\n没有找到站点" + strings.Repeat("

filler

", 300) + - "

您的请求在Web服务器中没有找到对应的站点！

" - p, _ := evidence.NewPage([]byte(raw)) - f := common.NewFramework("宝塔", common.FrameFromFingers) - f.MatchDetail = &common.MatchDetail{MatcherType: "regexp", MatcherValue: "没有找到对应的站点"} - frames := common.Frameworks{} - frames.Add(f) - groups := groupProducts(frames) - excerpts := evidenceFor(p.Raw, groups[0]) - if len(groups) != 1 || len(excerpts) == 0 || - !strings.Contains(excerpts[0].Text, "您的请求在Web服务器中没有找到对应的站点") || excerpts[0].Where != "body" { - t.Fatalf("evidence: %+v", groups[0]) - } - if strings.Contains(p.Text, "您的请求") { - t.Fatal("test page too short: the compact view already carries the match") +func (p failVersionProvider) Judge(ctx context.Context, state interface{}, claims map[string]jev.Claim) (map[string]jev.Ruling, error) { + if _, ok := claims["version_jenkins"]; ok { + return nil, errors.New("version unavailable") } + return p.mock.Judge(ctx, state, claims) } diff --git a/judge/operations_test.go b/judge/operations_test.go deleted file mode 100644 index eb19a30..0000000 --- a/judge/operations_test.go +++ /dev/null @@ -1,125 +0,0 @@ -package judge - -import ( - "context" - "encoding/json" - "errors" - "testing" - - "github.com/chainreactors/fingers/common" - "github.com/chainreactors/utils/jev" -) - -type rulingProvider func(map[string]jev.Claim) map[string]jev.Ruling - -func (rulingProvider) ID() string { return "ruling-provider" } - -func (p rulingProvider) Judge(_ context.Context, _ interface{}, claims map[string]jev.Claim) (map[string]jev.Ruling, error) { - return p(claims), nil -} - -type failVersionProvider struct{ *mock } - -func (p failVersionProvider) Judge(ctx context.Context, state interface{}, claims map[string]jev.Claim) (map[string]jev.Ruling, error) { - if _, ok := claims["version_jenkins"]; ok { - return nil, errors.New("version unavailable") - } - return p.mock.Judge(ctx, state, claims) -} - -func TestPublicResultsDoNotMutateHits(t *testing.T) { - m := newMock() - m.presence["jenkins"] = OptionRunning - j := New(m) - frames := testFrames() - before, _ := json.Marshal(frames) - inspected, err := j.Inspect(context.Background(), []byte(bodyRaw), frames) - if err != nil || !inspected["wordpress"].Judge.Rejected { - t.Fatalf("inspect = %v, %v", inspected, err) - } - accepted, err := j.Inspect(context.Background(), []byte(bodyRaw), frames) - accepted = accepted.Accepted() - if err != nil || accepted["wordpress"] != nil || accepted["jenkins"].Version != "2.401.3" { - t.Fatalf("accepted = %v, %v", accepted, err) - } - // The judgement is copied, not shared with the Inspect result. - accepted["nginx"].Judge.Option = "changed" - if inspected["nginx"].Judge.Option == "changed" { - t.Fatal("Independent Inspect calls share a Judgement") - } - if after, _ := json.Marshal(frames); string(after) != string(before) { - t.Fatalf("hits changed:\n%s\n%s", before, after) - } - version, err := j.Version(context.Background(), []byte(bodyRaw), frames["jenkins"]) - if err != nil || version != "2.401.3" || frames["jenkins"].Version != "" { - t.Fatalf("version = %q, %v; input = %q", version, err, frames["jenkins"].Version) - } -} - -// A failed version round keeps the presence and leaves the input untouched. -func TestInspectVersionFailureKeepsPresence(t *testing.T) { - frames := testFrames() - m := newMock() - m.presence["jenkins"] = OptionRunning - accepted, err := New(failVersionProvider{m}).Inspect(context.Background(), []byte(bodyRaw), frames) - accepted = accepted.Accepted() - if err == nil || accepted["wordpress"] != nil || accepted["jenkins"] == nil || accepted["jenkins"].Version != "" { - t.Fatalf("failed version round = %v, %v", accepted, err) - } - for _, f := range frames { - if f.Judge != nil || f.Version != "" { - t.Fatalf("input changed after the version round failed: %+v", f) - } - } -} - -func TestInspectInvalidBatchIsNotPartlyApplied(t *testing.T) { - j := New(rulingProvider(func(cs map[string]jev.Claim) map[string]jev.Ruling { - out := map[string]jev.Ruling{} - for id := range cs { - out[id] = jev.Ruling{Option: OptionRunning, Confidence: 1} - } - out["presence_wordpress"] = jev.Ruling{Option: "invented", Confidence: 1} - return out - })) - all, err := j.Inspect(context.Background(), []byte(jenkinsRaw), testFrames()) - if err == nil || all["wordpress"].Judge != nil || (all["apache tomcat"].Judge != nil && all["apache tomcat"].Judge.Outcome != "") || all["nginx"].Judge.Outcome != jev.Holds.String() { - t.Fatalf("partially applied invalid batch: %v %v", all, err) - } -} - -func TestInspectReevaluatesAndCopiesInputs(t *testing.T) { - m := newMock() - j := New(m) - frames := testFrames() - f := frames["wordpress"] - f.MatchDetail = &common.MatchDetail{MatcherValue: "WordPress"} - f.Judge = &common.Judgement{Option: OptionRunning, Outcome: jev.Holds.String(), Evidence: []string{"old"}} - before, _ := json.Marshal(frames) - all, err := j.Inspect(context.Background(), []byte(jenkinsRaw), frames) - if err != nil || !all["wordpress"].Judge.Rejected { - t.Fatalf("old judgement reused: %v %v", all, err) - } - all["wordpress"].MatchDetail.MatcherValue = "changed" - all["wordpress"].Tags = append(all["wordpress"].Tags, "changed") - all["wordpress"].Froms[common.FrameFromFingers] = true - all["wordpress"].Version = "changed" - all["wordpress"].Judge.Evidence[0] = "changed" - after, _ := json.Marshal(frames) - if string(before) != string(after) { - t.Fatal("result mutated input") - } - partial, err := j.Inspect(context.Background(), []byte("not HTTP"), frames) - if err == nil || len(partial) != len(frames) || partial["wordpress"].Judge != nil { - t.Fatalf("parse failure lost baseline: %v %v", partial, err) - } - partial["wordpress"].MatchDetail.MatcherValue = "also changed" - if frames["wordpress"].MatchDetail.MatcherValue != "WordPress" { - t.Fatal("parse-error baseline shares pointers") - } - copied := cloneFramework(f) - copied.Judge.Evidence[0] = "changed" - if f.Judge.Evidence[0] != "old" { - t.Fatal("clone shares judgement evidence") - } -} diff --git a/judge/presence.go b/judge/presence.go index 50db765..656269c 100644 --- a/judge/presence.go +++ b/judge/presence.go @@ -36,6 +36,14 @@ func (e matchExcerpt) String() string { return e.Where + ": " + e.Text } +// factClaim carries presence facts code establishes from the response head: +// explicit product declarations and protocol features. +var factClaim = jev.Claim{Statement: "The response head establishes the product or protocol feature.", Options: map[string]jev.Option{ + OptionDeclared: {Description: "Explicit product declaration or protocol feature", Outcome: jev.Holds}, + OptionAbsent: {Description: "The protocol feature is absent", Outcome: jev.Refuted}, + jev.OptionInsufficient: {Description: insufficientDescription, Outcome: jev.Insufficient}, +}} + // presenceClaim is a rule's claim that name is part of the software stack // that produced the response; state.matches[id] quotes where it matched. func presenceClaim(id, name string) jev.Claim { @@ -78,14 +86,7 @@ func (j *Judge) presence(ctx context.Context, p *evidence.Page, frames common.Fr option = OptionDeclared } if option != "" { - claim := jev.Claim{Statement: "The response declares " + name, Options: map[string]jev.Option{ - OptionDeclared: {Description: "Explicit product declaration or protocol feature", Outcome: jev.Holds}, - OptionAbsent: {Description: "The protocol feature is absent", Outcome: jev.Refuted}, - jev.OptionInsufficient: {Description: insufficientDescription, Outcome: jev.Insufficient}, - }} - if err := j.applyPresence(group, excerpts, claim, jev.Ruling{Option: option, Confidence: 1}); err != nil { - return err - } + j.applyPresence(group, excerpts, factClaim, jev.Ruling{Option: option, Confidence: 1}) continue } if len(claims) >= maxClaims { @@ -99,9 +100,7 @@ func (j *Judge) presence(ctx context.Context, p *evidence.Page, frames common.Fr return err } for id, ruling := range rulings { - if err := j.applyPresence(targets[id], matches[id], claims[id], ruling); err != nil { - return err - } + j.applyPresence(targets[id], matches[id], claims[id], ruling) } return nil } @@ -110,11 +109,9 @@ func (j *Judge) rejects(o jev.Outcome) bool { return o == jev.Refuted || (o == jev.Insufficient && j.DropInsufficient) } -// applyPresence is shared by deterministic and provider presence rulings. -func (j *Judge) applyPresence(frames []*common.Framework, evidence []matchExcerpt, claim jev.Claim, ruling jev.Ruling) error { - if err := validRuling(claim, ruling); err != nil { - return err - } +// applyPresence annotates a product's spellings with a ruling, whether code +// or the provider made it; spellings after the first are duplicates. +func (j *Judge) applyPresence(frames []*common.Framework, evidence []matchExcerpt, claim jev.Claim, ruling jev.Ruling) { outcome := claim.Resolve(ruling, j.MinConfidence) var excerpts []string for _, e := range evidence { @@ -125,5 +122,4 @@ func (j *Judge) applyPresence(frames []*common.Framework, evidence []matchExcerp Evidence: append([]string(nil), excerpts...), Confidence: ruling.Confidence, Rejected: j.rejects(outcome), Duplicate: i > 0} } - return nil } diff --git a/judge/presence_test.go b/judge/presence_test.go new file mode 100644 index 0000000..ac51f95 --- /dev/null +++ b/judge/presence_test.go @@ -0,0 +1,165 @@ +package judge + +import ( + "context" + "strings" + "testing" + + "github.com/chainreactors/fingers/common" + "github.com/chainreactors/fingers/judge/internal/evidence" + "github.com/chainreactors/utils/jev" +) + +func TestPageDeclarations(t *testing.T) { + s, err := evidence.NewPage([]byte(jenkinsRaw)) + if err != nil { + t.Fatal(err) + } + if s.Title != "Sign in [Jenkins]" || s.Headers["X-Jenkins"] != "2.401.3" { + t.Fatalf("title/header: %+v", s) + } + if len(s.Cookies) != 1 || len(s.Scripts) != 1 || len(s.Forms) != 2 { + t.Fatalf("cookies/scripts/forms: %+v", s) + } + if got := extractVersions([]byte(jenkinsRaw), 5); len(got) != 2 || got[0].Value != "1.24.0" || got[1].Value != "2.401.3" { + t.Fatalf("versions: %v", got) + } + if !declares(s, "jenkins") || !declares(s, "nginx") || declares(s, "wordpress") || declares(s, "jenk") { + t.Fatal("declares") + } +} + +// A rule that matched text unrelated to its product is a false positive, +// removed like a mere mention, and the matched text reaches the provider. +func TestUnrelatedIsRejected(t *testing.T) { + m := newMock() + m.presence["apache tomcat"] = OptionUnrelated + frames := testFrames() + for _, f := range frames { + if f.Name == "apache tomcat" || f.Name == "apache-tomcat" { + f.MatchDetail = &common.MatchDetail{MatcherType: "word", MatcherValue: "jenkins"} + } + } + all, err := New(m).Inspect(context.Background(), []byte(jenkinsRaw), frames) + if err != nil { + t.Fatal(err) + } + tomcat := all["apache tomcat"].Judge + if all["apache-tomcat"].Judge.Duplicate { + tomcat = all["apache-tomcat"].Judge + } + if tomcat.Option != OptionUnrelated || !tomcat.Rejected || len(tomcat.Evidence) == 0 || !strings.Contains(tomcat.Evidence[0], `matched "Jenkins"`) { + t.Fatalf("tomcat %+v", tomcat) + } +} + +func TestWordBoundary(t *testing.T) { + if indexWord("take action now", "acti") >= 0 || indexWord("x-github-request-id: 1", "git") >= 0 { + t.Fatal("substring matched") + } + if indexWord("sign in [jenkins]", "jenkins") < 0 || indexWord("泛微oa", "泛微") < 0 { + t.Fatal("word missed") + } +} + +func TestNormalizeName(t *testing.T) { + same := [][]string{{"Apache HTTP Server", "apache-http", "apache-web-server", "Apache"}, {"Discuz! X", "discuz"}, {"Apache-Tomcat", "apache tomcat"}, {"泛微 OA", "泛微"}} + for _, names := range same { + for _, n := range names[1:] { + if NormalizeName(n) != NormalizeName(names[0]) { + t.Errorf("%q and %q not folded: %q %q", names[0], n, NormalizeName(names[0]), NormalizeName(n)) + } + } + } + for _, pair := range [][2]string{{"lighttpd", "lig"}, {"Apache Tomcat", "Apache"}, {"nginx", "ngin"}} { + if NormalizeName(pair[0]) == NormalizeName(pair[1]) { + t.Errorf("%q and %q folded", pair[0], pair[1]) + } + } +} + +// Rule excerpts include matches deep in the response. +func TestEvidenceQuotesTheMatcher(t *testing.T) { + raw := "HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\n没有找到站点" + strings.Repeat("

filler

", 300) + + "

您的请求在Web服务器中没有找到对应的站点！

" + p, _ := evidence.NewPage([]byte(raw)) + f := common.NewFramework("宝塔", common.FrameFromFingers) + f.MatchDetail = &common.MatchDetail{MatcherType: "regexp", MatcherValue: "没有找到对应的站点"} + frames := common.Frameworks{} + frames.Add(f) + groups := groupProducts(frames) + excerpts := evidenceFor(p.Raw, groups[0]) + if len(groups) != 1 || len(excerpts) == 0 || + !strings.Contains(excerpts[0].Text, "您的请求在Web服务器中没有找到对应的站点") || excerpts[0].Where != "body" { + t.Fatalf("evidence: %+v", groups[0]) + } + if strings.Contains(p.Text, "您的请求") { + t.Fatal("test page too short: the compact view already carries the match") + } +} + +// Minimal public evidence retained from the 2026-09-25 corpus, not full pages. +func TestStaticApplicationDescriptionIsPreservedAsEvidence(t *testing.T) { + description := "IT Tools is a free and open-source collection of handy online tools for developers." + for _, meta := range []string{ + ``, + ``, + } { + p, err := evidence.NewPage([]byte("HTTP/1.1 200 OK\r\n\r\nIT Tools - Handy online tools for developers" + meta)) + if err != nil || p.Description != description { + t.Fatalf("description evidence missing: %+v err=%v", p, err) + } + if declares(p, "it tools") { + t.Fatal("description incorrectly treated as deterministic header evidence") + } + } +} + +// Protocol features are facts of the response head: decided by code, never +// asked, whatever a provider would rule. +func TestProtocolHitRequiresResponseEvidence(t *testing.T) { + for _, tc := range []struct { + name, header string + want bool + }{{"http基本认证", "", false}, {"hsts", "", false}, {"http基本认证", "WWW-Authenticate: Basic realm=\"test\"\r\n", true}, {"hsts", "Strict-Transport-Security: max-age=31536000\r\n", true}} { + asked := false + j := New(rulingProvider(func(qs map[string]jev.Claim) map[string]jev.Ruling { + asked = true + return nil + })) + hits := common.Frameworks{} + hits.Add(common.NewFramework(tc.name, common.FrameFromGUESS)) + all, err := j.Inspect(context.Background(), []byte("HTTP/1.1 200 OK\r\n"+tc.header+"\r\n"), hits) + got := all.Accepted() + if err != nil || asked || (len(got) > 0) != tc.want { + t.Errorf("%s evidence=%q got=%v asked=%v err=%v", tc.name, tc.header, got, asked, err) + } + } +} + +func TestBasicChallengeUsesFullHeaderAndQuotedRealms(t *testing.T) { + for _, tc := range []struct { + header string + want bool + }{ + {"WWW-Authenticate: Digest realm=\"" + strings.Repeat("x", 200) + "\", Basic realm=\"private\"\r\n", true}, + {"WWW-Authenticate: Digest realm=\"x, Basic fake\"\r\n", false}, + {"WWW-Authenticate: Digest realm=\"private\"\r\nWWW-Authenticate: Basic realm=\"private\"\r\n", true}, + } { + p, err := evidence.NewPage([]byte("HTTP/1.1 401 Unauthorized\r\n" + tc.header + "\r\n")) + if err != nil || protocolPresent(p, "http基本认证") != tc.want { + t.Fatalf("header %q: %v", tc.header, err) + } + } +} + +func TestIncidentalHeaderNamesAreNotDeclarations(t *testing.T) { + raw := []byte("HTTP/1.1 302 Found\r\nLocation: https://docs.test/jenkins/2.401.3\r\nSet-Cookie: jenkins=x\r\nX-Github-Request-Id: jenkins\r\n\r\n") + p, err := evidence.NewPage(raw) + if err != nil { + t.Fatal(err) + } + if declares(p, "jenkins") || declaredVersion(declarations(raw), "jenkins") != "" { + t.Fatal("incidental URL/cookie text became a fact") + } +} diff --git a/judge/real_evidence_test.go b/judge/real_evidence_test.go deleted file mode 100644 index 4000edd..0000000 --- a/judge/real_evidence_test.go +++ /dev/null @@ -1,131 +0,0 @@ -package judge - -import ( - "context" - "strings" - "testing" - - "github.com/chainreactors/fingers/common" - "github.com/chainreactors/fingers/judge/internal/evidence" - "github.com/chainreactors/utils/jev" -) - -// Minimal public evidence retained from the 2026-09-25 corpus, not full pages. -func TestStaticApplicationDescriptionIsPreservedAsEvidence(t *testing.T) { - description := "IT Tools is a free and open-source collection of handy online tools for developers." - for _, meta := range []string{ - ``, - ``, - } { - p, err := evidence.NewPage([]byte("HTTP/1.1 200 OK\r\n\r\nIT Tools - Handy online tools for developers" + meta)) - if err != nil || p.Description != description { - t.Fatalf("description evidence missing: %+v err=%v", p, err) - } - if declares(p, "it tools") { - t.Fatal("description incorrectly treated as deterministic header evidence") - } - } -} - -func TestVersionCandidatesFromRealResponseEvidence(t *testing.T) { - raw := []byte("HTTP/1.1 200 OK\r\nServer: Apache/2.4.38 OpenSSL/1.0.2q PHP/5.6.40\r\nX-New-Api-Version: v1.0.0-rc.35\r\n\r\n") - got := map[string]bool{} - for _, c := range extractVersions(raw, 40) { - got[c.Value] = true - } - for _, want := range []string{"2.4.38", "1.0.2q", "5.6.40", "1.0.0-rc.35", "3.6.3"} { - if !got[want] { - t.Errorf("missing %s in %v", want, got) - } - } - if got["1.0.2"] { - t.Fatal("truncated OpenSSL version") - } -} -func TestVersionCandidatesDoNotDiscardLateProductEvidence(t *testing.T) { - var b strings.Builder - b.WriteString("HTTP/1.1 200 OK\r\n\r\n") - for i := 0; i < 80; i++ { - b.WriteString(" " + string(rune('a'+i%26)) + " ") - b.WriteString(strings.Repeat("1", i/10+1) + ".25 ") - } - // Add distinct numbers to exceed the shortlist. - for _, n := range []string{"10.11", "11.12", "12.13", "13.14", "14.15", "15.16", "16.17", "17.18", "18.19", "19.20"} { - b.WriteString(n + " ") - } - b.WriteString("") - found := false - for _, c := range extractVersions([]byte(b.String()), 5) { - found = found || c.Value == "3.7.1" - } - if !found { - t.Fatal("early numeric noise evicted late library version") - } -} - -// Protocol features are facts of the response head: decided by code, never -// asked, whatever a provider would rule. -func TestProtocolHitRequiresResponseEvidence(t *testing.T) { - for _, tc := range []struct { - name, header string - want bool - }{{"http基本认证", "", false}, {"hsts", "", false}, {"http基本认证", "WWW-Authenticate: Basic realm=\"test\"\r\n", true}, {"hsts", "Strict-Transport-Security: max-age=31536000\r\n", true}} { - asked := false - j := New(rulingProvider(func(qs map[string]jev.Claim) map[string]jev.Ruling { - asked = true - return nil - })) - hits := common.Frameworks{} - hits.Add(common.NewFramework(tc.name, common.FrameFromGUESS)) - all, err := j.Inspect(context.Background(), []byte("HTTP/1.1 200 OK\r\n"+tc.header+"\r\n"), hits) - got := all.Accepted() - if err != nil || asked || (len(got) > 0) != tc.want { - t.Errorf("%s evidence=%q got=%v asked=%v err=%v", tc.name, tc.header, got, asked, err) - } - } -} - -func TestCalendarVersionAndGeneratorName(t *testing.T) { - raw := []byte("HTTP/1.1 200 OK\r\n\r\nCustom Search") - p, err := evidence.NewPage(raw) - if err != nil { - t.Fatal(err) - } - foundName, foundVersion := false, false - for _, name := range evidence.Names(p.Generator, p.Title, p.Text, p.Headers, p.Scripts) { - foundName = foundName || NormalizeName(name) == "searxng" - } - for _, v := range extractVersions(raw, 40) { - foundVersion = foundVersion || v.Value == "2026.9.23+3cd69d30e" - } - if !foundName || !foundVersion { - t.Fatalf("name=%t version=%t", foundName, foundVersion) - } -} - -func TestBasicChallengeUsesFullHeaderAndQuotedRealms(t *testing.T) { - for _, tc := range []struct { - header string - want bool - }{ - {"WWW-Authenticate: Digest realm=\"" + strings.Repeat("x", 200) + "\", Basic realm=\"private\"\r\n", true}, - {"WWW-Authenticate: Digest realm=\"x, Basic fake\"\r\n", false}, - {"WWW-Authenticate: Digest realm=\"private\"\r\nWWW-Authenticate: Basic realm=\"private\"\r\n", true}, - } { - p, err := evidence.NewPage([]byte("HTTP/1.1 401 Unauthorized\r\n" + tc.header + "\r\n")) - if err != nil || protocolPresent(p, "http基本认证") != tc.want { - t.Fatalf("header %q: %v", tc.header, err) - } - } -} - -func TestIncidentalHeaderNamesAreNotDeclarations(t *testing.T) { - raw := []byte("HTTP/1.1 302 Found\r\nLocation: https://docs.test/jenkins/2.401.3\r\nSet-Cookie: jenkins=x\r\nX-Github-Request-Id: jenkins\r\n\r\n") - p, err := evidence.NewPage(raw) - if err != nil { - t.Fatal(err) - } - if declares(p, "jenkins") || declaredVersion(declarations(raw), "jenkins") != "" { - t.Fatal("incidental URL/cookie text became a fact") - } -} diff --git a/judge/version.go b/judge/version.go index a351323..9c4852d 100644 --- a/judge/version.go +++ b/judge/version.go @@ -21,8 +21,26 @@ var ( // What precedes numbers reVersion catches that are never versions: the // viewport's "initial-scale=1.0". reNotVersion = regexp.MustCompile(`(?i)(?:initial|minimum|maximum)-scale\s*=\s*$`) + // A Via header entry starts with the HTTP version it was received with + // ("Via: 1.1 varnish", "1.0 fred, 1.1 p.example.net"), never a product's. + reViaProtocol = regexp.MustCompile(`(?i)^via:\s*(?:[^,\r\n]*,\s*)*(?:HTTP/)?$`) + // The format version of an SVG or XML declaration: . + reMarkupVersion = regexp.MustCompile(`(?i)^<(?:svg|\?xml)\b[^>]*\bversion\s*=\s*["']?$`) ) +// formatVersion reports whether the number at i in text is a protocol or +// markup format version rather than a product version. +func formatVersion(text string, i int) bool { + line := text[strings.LastIndexByte(text[:i], '\n')+1 : i] + if reViaProtocol.MatchString(line) { + return true + } + if tag := strings.LastIndexByte(text[:i], '<'); tag >= 0 && !strings.Contains(text[tag:i], ">") { + return reMarkupVersion.MatchString(text[tag:i]) + } + return false +} + const ( notStated = "not_stated" @@ -54,9 +72,7 @@ func (j *Judge) versions(ctx context.Context, p *evidence.Page, frames ...*commo }} if v := declaredVersion(decls, f.Name); v != "" { claim.Options[v] = jev.Option{Description: "The response explicitly binds this version to " + f.Name, Outcome: jev.Holds} - if err := j.applyVersion(f, claim, jev.Ruling{Option: v, Confidence: 1}); err != nil { - return err - } + j.applyVersion(f, claim, jev.Ruling{Option: v, Confidence: 1}) continue } key := NormalizeName(f.Name) @@ -86,24 +102,19 @@ func (j *Judge) versions(ctx context.Context, p *evidence.Page, frames ...*commo return err } for id, ruling := range rulings { - if err := j.applyVersion(targets[id], claims[id], ruling); err != nil { - return err - } + j.applyVersion(targets[id], claims[id], ruling) } return nil } -func (j *Judge) applyVersion(f *common.Framework, claim jev.Claim, ruling jev.Ruling) error { - if err := validRuling(claim, ruling); err != nil { - return err - } +// applyVersion writes the picked version when the ruling holds. +func (j *Judge) applyVersion(f *common.Framework, claim jev.Claim, ruling jev.Ruling) { if claim.Resolve(ruling, j.MinConfidence) == jev.Holds { if f.Attributes == nil { f.Attributes = common.NewAttributesWithAny() } f.Version = ruling.Option } - return nil } // declaration is a version the response binds to a product name. @@ -227,7 +238,7 @@ func extractVersions(raw []byte, max int) []versionString { if before < 0 { before = 0 } - if looksLikeIPv4(v) || strings.Trim(v, "0.") == "" || reNotVersion.MatchString(text[before:loc[2]]) { + if looksLikeIPv4(v) || strings.Trim(v, "0.") == "" || reNotVersion.MatchString(text[before:loc[2]]) || formatVersion(text, loc[2]) { continue } start, end := loc[0]-60, loc[1]+30 diff --git a/judge/version_test.go b/judge/version_test.go new file mode 100644 index 0000000..83bf775 --- /dev/null +++ b/judge/version_test.go @@ -0,0 +1,125 @@ +package judge + +import ( + "fmt" + "strings" + "testing" + + "github.com/chainreactors/fingers/judge/internal/evidence" +) + +func TestDeclarations(t *testing.T) { + raw := []byte("HTTP/1.1 200 OK\r\nServer: Apache/2.4.38 (Debian) OpenSSL/1.0.2q PHP/5.6.40\r\nX-Jenkins: 2.401.3\r\n" + + "X-Gitea-Version: 1.21.4\r\nX-Tomcat: Apache Tomcat/9.0.1\r\n\r\n") + decls := declarations(raw) + for product, want := range map[string]string{"Apache": "2.4.38", "openssl": "1.0.2q", "PHP": "5.6.40", "jenkins": "2.401.3", + "Gitea": "1.21.4", "Apache Tomcat": "9.0.1", "WordPress": "7.0.3", "nginx": ""} { + if got := declaredVersion(decls, product); got != want { + t.Errorf("%s: got %q want %q", product, got, want) + } + } + if !claimedByOther(decls, "5.6.40", NormalizeName("ThinkPHP")) || claimedByOther(decls, "5.6.40", NormalizeName("php")) { + t.Error("claims") + } +} + +func TestExtractVersionsPrefixes(t *testing.T) { + raw := "HTTP/1.1 200 OK\r\n\r\nV8.1SP2 10.0.0.5 jquery.min.js?ver=3.7.1{\"success_fraction\":0.0}" + var got []string + for _, c := range extractVersions([]byte(raw), 10) { + got = append(got, c.Value) + } + if strings.Join(got, ",") != "3.4,8.1SP2,3.7.1" { + t.Fatalf("got %v", got) + } +} + +func TestRankVersions(t *testing.T) { + raw := "HTTP/1.1 200 OK\r\n\r\n" + for i := 0; i < 15; i++ { + raw += fmt.Sprintf(``, i, i) + } + raw += `` + cands := extractVersions([]byte(raw), 40) + got := rankVersions(cands, "wordpress", 3) + if got[0].Value != "7.0.3" || !strings.Contains(got[0].Context, "generator") { + t.Fatalf("wordpress generator not ranked first: %+v", got) + } + var drupal bool + for _, c := range cands { + drupal = drupal || c.Value == "10" + } + if !drupal { + t.Fatalf("major-only generator version missing: %+v", cands) + } +} + +func TestVersionCandidatesFromRealResponseEvidence(t *testing.T) { + raw := []byte("HTTP/1.1 200 OK\r\nServer: Apache/2.4.38 OpenSSL/1.0.2q PHP/5.6.40\r\nX-New-Api-Version: v1.0.0-rc.35\r\n\r\n") + got := map[string]bool{} + for _, c := range extractVersions(raw, 40) { + got[c.Value] = true + } + for _, want := range []string{"2.4.38", "1.0.2q", "5.6.40", "1.0.0-rc.35", "3.6.3"} { + if !got[want] { + t.Errorf("missing %s in %v", want, got) + } + } + if got["1.0.2"] { + t.Fatal("truncated OpenSSL version") + } +} + +func TestVersionCandidatesDoNotDiscardLateProductEvidence(t *testing.T) { + var b strings.Builder + b.WriteString("HTTP/1.1 200 OK\r\n\r\n") + for i := 0; i < 80; i++ { + b.WriteString(" " + string(rune('a'+i%26)) + " ") + b.WriteString(strings.Repeat("1", i/10+1) + ".25 ") + } + // Add distinct numbers to exceed the shortlist. + for _, n := range []string{"10.11", "11.12", "12.13", "13.14", "14.15", "15.16", "16.17", "17.18", "18.19", "19.20"} { + b.WriteString(n + " ") + } + b.WriteString("") + found := false + for _, c := range extractVersions([]byte(b.String()), 5) { + found = found || c.Value == "3.7.1" + } + if !found { + t.Fatal("early numeric noise evicted late library version") + } +} + +func TestCalendarVersionAndGeneratorName(t *testing.T) { + raw := []byte("HTTP/1.1 200 OK\r\n\r\nCustom Search") + p, err := evidence.NewPage(raw) + if err != nil { + t.Fatal(err) + } + foundName, foundVersion := false, false + for _, name := range evidence.Names(p.Generator, p.Title, p.Text, p.Headers, p.Scripts) { + foundName = foundName || NormalizeName(name) == "searxng" + } + for _, v := range extractVersions(raw, 40) { + foundVersion = foundVersion || v.Value == "2026.9.23+3cd69d30e" + } + if !foundName || !foundVersion { + t.Fatalf("name=%t version=%t", foundName, foundVersion) + } +} + +// Protocol and markup format versions are facts code excludes: the provider +// is never offered them as a product's version. +func TestFormatVersionsAreNotCandidates(t *testing.T) { + raw := "HTTP/1.1 200 OK\r\nServer: GitHub.com\r\nVia: 1.1 varnish, HTTP/1.0 proxy\r\n\r\n" + + `` + + `

Varnish 7.4.2

` + var got []string + for _, c := range extractVersions([]byte(raw), 10) { + got = append(got, c.Value) + } + if strings.Join(got, ",") != "2.4.1,7.4.2" { + t.Fatalf("got %v", got) + } +} From 6750a5330f6b93bbc30cdfa0d71a20d404c5e7f9 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Sat, 26 Sep 2026 22:40:30 -0700 Subject: [PATCH 14/16] feat(engine): one key reviews every web match with Jev engine.EnableJudge(apiKey) is the only thing a user configures: every web match (WebMatch, Match, DetectContent, DetectResponse, WebMatchWithEngines) then reviews its rule results and returns those the evidence supports - false positives and duplicate spellings dropped, versions filled. An empty key reads TYPESAFE_API_KEY; EnableJudge turns on match detail and wraps the client in the exact cache. If a request fails the rule results come back unchanged. Engine.Judge stays open for another provider or policy; with it unset matching is unchanged. The example CLI's --judge flag becomes --jev-key and simply calls EnableJudge, so its output is the normal output, reviewed. README leads with the one-line setup. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 37 +++++++++++------------- cmd/engine/example.go | 50 +++++---------------------------- engine.go | 41 +++++++++++++++++++++++++-- judge/README.md | 2 ++ judge_test.go | 65 ++++++++++++++++++++++++++++++++++--------- 5 files changed, 115 insertions(+), 80 deletions(-) diff --git a/README.md b/README.md index 823714e..97580b6 100644 --- a/README.md +++ b/README.md @@ -90,37 +90,32 @@ func TestFavicon(t *testing.T) { 更多用法请见: https://chainreactors.github.io/wiki/libs/fingers/sdk/ -## 判定层 judge +## 用 Jev 提升识别效果 -规则引擎召回能力强，但会有误报、多个引擎的同名重复，也会漏掉规则库里没有的产品，版本号更是难以拿准。`judge` 包在规则结果之上加了一层审查：**它只审查规则给出的结论，不做指纹识别，模型从不说出产品名。** 它不是新的引擎，`WebMatch` 和 `DetectContent` 的行为保持不变。Claim 合约、缓存和 [TypeSafe Jev](https://docs.typesafe.ai/api.md) 客户端在 [`github.com/chainreactors/utils/jev`](https://github.com/chainreactors/utils/tree/master/jev)，fingers 只保留指纹相关的断言。 - -判定层只有一个机制，即 Claim：代码提出断言并附上证据，Provider 选出证据支持的选项，选项的 Outcome（holds / refuted / insufficient）驱动固定动作。去误报（Presence）、找漏报（Coverage）、确认版本（Version）都是 Claim；与指纹无关的自定义 Claim 直接用 `jev.Judge`。 +规则引擎召回能力强，但会有误报、多个引擎的同名重复，版本号也难以拿准。配置一个 [TypeSafe Jev](https://docs.typesafe.ai/api.md) key，引擎就会审查每一次 Web 匹配的规则结果，只返回证据支持的结果：去掉误报和重复，补全版本。调用方式不变。 ```golang -engine.EnableMatchDetail() // 让 Claim 能引用规则命中的原文 -c, _ := jev.NewClient("") // 读取 TYPESAFE_API_KEY -j := judge.New(jev.Cached(c, jev.DefaultCacheSize)) // 一个扫描任务共用一个 Judge - -hits, _ := engine.DetectContent(raw) // 同步，纯规则 -all, err := j.Inspect(ctx, raw, hits) // 一次完成 presence 与 version，保留全部注解 -accepted := all.Accepted() // 上报视图：排除拒绝和重复条目 -if err != nil { - accepted = hits // 判定失败时退回纯规则结果；hits 从不被修改 -} -for _, f := range accepted { - if f.Judge != nil { // nil：未经判定（如超过每页 40 个产品的上限） - fmt.Println(f.Name, f.Version, f.Judge.Option, f.Judge.Outcome) - } -} +engine, _ := fingers.NewEngine() +engine.EnableJudge("") // 空字符串读取 TYPESAFE_API_KEY +frames, _ := engine.DetectContent(raw) // WebMatch、Match、DetectResponse 同样生效 ``` -判定结论是 `Framework.Judge` 字段（选项、Outcome、置信度、证据、是否剔除、是否重复），随现有输出一起序列化；下游不 import judge 也能用 `frames.Accepted()` 读取。被剔除的条目和原因已经包含在 `all` 中，无须再次请求 Provider。漏报在离线维护中处理：`judge/maintain.Discover` 把多个主机上的同一页面聚成簇，逐簇提出 Coverage Claim，判为 missing 的簇连同候选名交给人命名，再用 `judge/gen` 生成规则。 +命令行：`engine -u https://example.com --jev-key `。 + +- **只审查，不识别**：Jev 只判断规则给出的结论是否成立，从不说出产品名，因此不会凭空多出结果。 +- **失败时退回规则结果**：请求失败或 key 无效时，返回的就是纯规则结果。 +- **每条结果都带依据**：保留下来的结果带有 `Framework.Judge`（选项、Outcome、置信度、证据），随现有输出一起序列化。 +- **代价**：每个有命中的页面约 1～2 个请求，相同页面命中缓存。 + +漏报不在扫描时处理，因为 Jev 不说出产品名。维护者用 `judge/maintain` 离线发现规则漏掉的产品，交给人命名后用 `judge/gen` 生成规则，更新到指纹库后所有用户受益。 + +需要自定义 Provider、阈值或查看被剔除条目及原因时，直接使用 `judge` 包，或给 `engine.Judge` 赋值，见 [judge/README.md](judge/README.md)。判定机制只有一个 Claim：代码提出断言并附上证据，Provider 选出证据支持的选项，选项的 Outcome（holds / refuted / insufficient）驱动固定动作。Claim 合约、缓存和 Jev 客户端在 [`github.com/chainreactors/utils/jev`](https://github.com/chainreactors/utils/tree/master/jev)。 **数据外发**：判定时会把响应的精简视图发给 Provider，包括响应头（去掉 Date、Set-Cookie 值等无关头）、Cookie 名、标题、generator/description、脚本和样式路径、内联脚本开头、HTML 注释、表单字段名，以及正文前 1500 字。扫描授权范围内的目标前，请确认允许把这些内容发送给第三方服务。 当前实现于 2026-09-27 在 69 份已标注的真实响应上复测（jev-1.13.0）：误报 24 → 0，真实命中误删 0，版本正确 36、错误 0；不加载新增规则时 Discover 找出全部 3 个漏报产品，加载后无误报警。见 [复测报告](docs/jev-claim-retest-20260927.md)。 -指纹判定的完整能力见 [judge/README.md](judge/README.md)；用正反样本生成原生指纹见 `judge/gen`；设计讨论见 [#34](https://github.com/chainreactors/fingers/issues/34)。 +设计讨论见 [#34](https://github.com/chainreactors/fingers/issues/34)。 ## fingers 引擎 diff --git a/cmd/engine/example.go b/cmd/engine/example.go index c7ffd7d..c75e780 100644 --- a/cmd/engine/example.go +++ b/cmd/engine/example.go @@ -1,8 +1,6 @@ package main import ( - "bytes" - "context" "crypto/tls" "encoding/json" "fmt" @@ -14,11 +12,9 @@ import ( "github.com/chainreactors/fingers" "github.com/chainreactors/fingers/common" - "github.com/chainreactors/fingers/judge" "github.com/chainreactors/fingers/resources" "github.com/chainreactors/utils/encode" "github.com/chainreactors/utils/httputils" - "github.com/chainreactors/utils/jev" "github.com/jessevdk/go-flags" "gopkg.in/yaml.v3" ) @@ -39,8 +35,8 @@ var opts struct { // 是否只检测favicon FaviconOnly bool `short:"f" long:"favicon" description:"Only detect favicon"` - // 在规则结果之上运行判定层, 目前支持的 provider: jev (需要 TYPESAFE_API_KEY) - Judge string `long:"judge" description:"Judge the rule result with a provider: jev (needs TYPESAFE_API_KEY)"` + // 配置 Jev key 后, 引擎自动去除误报与重复并补全版本 + JevKey string `long:"jev-key" description:"Jev API key: review rule results to drop false positives and fill versions"` // 资源文件覆盖 GobyFile string `long:"goby" description:"Override goby.json.gz with custom file"` @@ -181,8 +177,11 @@ func main() { fmt.Printf("Failed to create engine: %v\n", err) os.Exit(1) } - if opts.Judge != "" && engine.Fingers() != nil { - engine.EnableMatchDetail() // the judge quotes what each rule matched + if opts.JevKey != "" { + if err := engine.EnableJudge(opts.JevKey); err != nil { + fmt.Printf("Failed to enable Jev: %v\n", err) + os.Exit(1) + } } if opts.Verbose { @@ -206,42 +205,7 @@ func main() { frames.Add(frame) } } else { - var content []byte - if opts.Judge != "" { // Inspect needs the raw response; put the body back for Match - content = httputils.ReadRaw(resp) - body, _, _ := httputils.SplitHttpRaw(content) - resp.Body = ioutil.NopCloser(bytes.NewReader(body)) - } frames = engine.Match(resp) - if opts.Judge != "" { - if opts.Judge != "jev" { - fmt.Printf("unknown judge provider %q\n", opts.Judge) - os.Exit(1) - } - c, err := jev.NewClient("") - if err != nil { - fmt.Println(err) - os.Exit(1) - } - j := judge.New(jev.Cached(c, jev.DefaultCacheSize)) - ctx := context.Background() - judged, err := j.Inspect(ctx, content, frames) - accepted := judged.Accepted() - if err != nil { - fmt.Printf("jev failed, code-established facts only: %v\n", err) - } - for _, frame := range accepted { - option := "unjudged" - if frame.Judge != nil { - option = frame.Judge.Option - } - fmt.Printf(" %-40s %-10s %s\n", frame.Name, frame.Version, option) - } - if err == nil { - fmt.Printf("accepted: %s\n", accepted.String()) - return - } - } } // 输出结果 diff --git a/engine.go b/engine.go index 36eab7f..dfa48c2 100644 --- a/engine.go +++ b/engine.go @@ -3,6 +3,7 @@ package fingers import ( "bufio" "bytes" + "context" "fmt" "net/http" "sort" @@ -15,11 +16,14 @@ import ( "github.com/chainreactors/fingers/fingerprinthub" "github.com/chainreactors/fingers/fingers" "github.com/chainreactors/fingers/goby" + "github.com/chainreactors/fingers/judge" gonmap "github.com/chainreactors/fingers/nmap" "github.com/chainreactors/fingers/resources" wappalyzer "github.com/chainreactors/fingers/wappalyzer" xrayengine "github.com/chainreactors/fingers/xray" + "github.com/chainreactors/logs" "github.com/chainreactors/utils/httputils" + "github.com/chainreactors/utils/jev" "github.com/pkg/errors" ) @@ -88,7 +92,10 @@ type Engine struct { *alias.Aliases Enabled map[string]bool Capabilities map[string]common.EngineCapability // 新增：记录各引擎能力 - + // Judge, when set, reviews every web match: false positives and duplicate + // spellings are dropped and versions filled. EnableJudge sets it up with + // Jev; assign your own for another provider or policy. + Judge *judge.Judge } func (engine *Engine) String() string { @@ -207,6 +214,34 @@ func (engine *Engine) Disable(name string) { engine.Enabled[name] = false } +// EnableJudge makes every web match (WebMatch, Match, DetectContent, +// DetectResponse, WebMatchWithEngines) review its rule results with Jev and +// return only those the evidence supports: false positives and duplicate +// spellings are dropped, versions filled. An empty apiKey reads +// TYPESAFE_API_KEY. It sends a compact view of each matched response to +// the Jev API; if a request fails, the rule results are returned as they are. +func (engine *Engine) EnableJudge(apiKey string) error { + c, err := jev.NewClient(apiKey) + if err != nil { + return err + } + engine.EnableMatchDetail() + engine.Judge = judge.New(jev.Cached(c, jev.DefaultCacheSize)) + return nil +} + +// review applies Judge to the rule results of one response. +func (engine *Engine) review(content []byte, frames common.Frameworks) common.Frameworks { + if engine.Judge == nil || len(frames) == 0 { + return frames + } + all, err := engine.Judge.Inspect(context.Background(), content, frames) + if err != nil { + logs.Log.Debugf("judge: %v; unjudged hits are kept", err) + } + return all.Accepted() +} + // EnableMatchDetail makes every engine that can record what a hit matched // (fingers, fingerprinthub, wappalyzer, ehole, goby) do so in // Framework.MatchDetail. Off by default: it costs fingers and fingerprinthub @@ -367,7 +402,7 @@ func (engine *Engine) WebMatch(resp *http.Response) common.Frameworks { combined = engine.MergeFrameworks(combined, fs) } - return combined + return engine.review(content, combined) } // enabledNames lists the enabled engines in AllEngines order, then any others @@ -416,7 +451,7 @@ func (engine *Engine) WebMatchWithEngines(content []byte, engines ...string) com combined = engine.MergeFrameworks(combined, fs) } } - return combined + return engine.review(content, combined) } // MatchWithEngines (deprecated, use WebMatchWithEngines instead) diff --git a/judge/README.md b/judge/README.md index d94572c..ff56f2c 100644 --- a/judge/README.md +++ b/judge/README.md @@ -10,6 +10,8 @@ Claim 合约（Claim / Option / Outcome / Ruling / Provider）、校验、缓存 ## 在线入口 +普通用户只需 `engine.EnableJudge(key)`：引擎的 Web 匹配入口自动调用 `Inspect` 并返回 `Accepted()`，失败时返回规则结果。需要全部注解（含被剔除条目）或自定义策略时，直接使用本包： + ```go engine, _ := fingers.NewEngine() engine.EnableMatchDetail() diff --git a/judge_test.go b/judge_test.go index 23e8dcb..b6ddc96 100644 --- a/judge_test.go +++ b/judge_test.go @@ -7,38 +7,69 @@ import ( "github.com/chainreactors/fingers/judge" "github.com/chainreactors/utils/jev" + "github.com/pkg/errors" ) -func TestInspectWithJudge(t *testing.T) { +// With a Judge, the engine's own entry points return reviewed results: +// declared products are kept, a product the page only mentions is dropped, +// and a failing provider leaves the rule results in place. +func TestEngineJudge(t *testing.T) { engine, err := NewEngine(FingersEngine, WappalyzerEngine) if err != nil { t.Fatal(err) } + raw := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\nContent-Type: text/html\r\n\r\n" + + `Welcome to nginx!

Migrating from WordPress? wp-login.php

`) + rules, _ := engine.DetectContent(raw) + if rules["nginx"] == nil || len(rules) < 2 { + t.Fatalf("rule results %v", rules) + } engine.EnableMatchDetail() - raw := []byte("HTTP/1.1 200 OK\r\nServer: nginx\r\n\r\nWelcome to nginx!") - frames, _ := engine.DetectContent(raw) - j := judge.New(running{}) - accepted, err := j.Inspect(context.Background(), raw, frames) - accepted = accepted.Accepted() + engine.Judge = judge.New(verdict(judge.OptionMentioned)) + reviewed, _ := engine.DetectContent(raw) + if nginx := reviewed["nginx"]; nginx == nil || nginx.Judge == nil || nginx.Judge.Option != judge.OptionDeclared { + t.Fatalf("declared nginx: %v", reviewed["nginx"]) + } + for name, f := range reviewed { + if f.Judge != nil && f.Judge.Rejected { + t.Errorf("rejected %s returned", name) + } + } + if len(reviewed) >= len(rules) { + t.Fatalf("nothing dropped: %d reviewed of %d", len(reviewed), len(rules)) + } + engine.Judge = judge.New(failing{}) + if fallback, _ := engine.DetectContent(raw); len(fallback) != len(rules) { + t.Fatalf("failed provider changed results: %d of %d", len(fallback), len(rules)) + } +} + +func TestEnableJudge(t *testing.T) { + engine, err := NewEngine(FingersEngine) if err != nil { t.Fatal(err) } - if nginx := accepted["nginx"]; nginx == nil || nginx.Judge == nil || nginx.Judge.Option != judge.OptionDeclared || frames["nginx"].Judge != nil { - t.Fatalf("accepted %v, input %v", accepted["nginx"], frames["nginx"]) + t.Setenv(jev.EnvAPIKey, "") + if err := engine.EnableJudge(""); err == nil || engine.Judge != nil { + t.Fatal("enabled without a key") + } + if err := engine.EnableJudge("local-test"); err != nil || engine.Judge == nil || !engine.Fingers().MatchDetailEnabled { + t.Fatalf("EnableJudge: %v", err) } } -// running is a provider that judges every claim running and states no version. -type running struct{} +// verdict is a provider that rules every presence claim with one option and +// states no version. +type verdict string -func (running) ID() string { return "test" } +func (verdict) ID() string { return "test" } -func (running) Judge(ctx context.Context, state interface{}, claims map[string]jev.Claim) (map[string]jev.Ruling, error) { +func (v verdict) Judge(ctx context.Context, state interface{}, claims map[string]jev.Claim) (map[string]jev.Ruling, error) { rulings := map[string]jev.Ruling{} for k := range claims { switch { case strings.HasPrefix(k, "presence_"): - rulings[k] = jev.Ruling{Option: judge.OptionRunning, Confidence: 0.9} + rulings[k] = jev.Ruling{Option: string(v), Confidence: 0.9} default: rulings[k] = jev.Ruling{Option: "not_stated", Confidence: 0.9} } @@ -46,6 +77,14 @@ func (running) Judge(ctx context.Context, state interface{}, claims map[string]j return rulings, nil } +type failing struct{} + +func (failing) ID() string { return "failing" } + +func (failing) Judge(context.Context, interface{}, map[string]jev.Claim) (map[string]jev.Ruling, error) { + return nil, errors.New("unavailable") +} + // Every engine that can record what a hit matched does so once enabled, and // quotes text the response contains; disabled, results carry no detail. func TestEnableMatchDetail(t *testing.T) { From 8982c67af60acc87914140e2f5ab0eea8385a519 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Sat, 26 Sep 2026 23:25:16 -0700 Subject: [PATCH 15/16] docs(jev): mark historical reports, add product and design notes - The 2026-09-25/26 reports and the 1070-site report say up front that they describe removed interfaces (Refine, SuggestNames, Classify) and were not re-measured with the current implementation. - Product authenticity review of the three added fingerprints: SearXNG, Wakapi and IT Tools are real open-source projects; the IT Tools rule identifies the product family and cannot tell the original from forks. fingerprints/README.md states that scope. - Archive of the System One ("habit") design discussion behind Jev. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/jev-harness-design-20260926.md | 323 ++++++++++++++++++++ docs/jev-maintenance-validation-20260926.md | 4 + docs/jev-product-authenticity-20260926.md | 83 +++++ docs/jev-real-validation-20260925.md | 2 + docs/jev-report.md | 2 + fingerprints/README.md | 4 +- 6 files changed, 417 insertions(+), 1 deletion(-) create mode 100644 docs/jev-harness-design-20260926.md create mode 100644 docs/jev-product-authenticity-20260926.md diff --git a/docs/jev-harness-design-20260926.md b/docs/jev-harness-design-20260926.md new file mode 100644 index 0000000..0aa4bad --- /dev/null +++ b/docs/jev-harness-design-20260926.md @@ -0,0 +1,323 @@ +# Habit：一个 System One 系统 + +> 本文设计的是一个**独立系统**，不是 harness 的一个功能。它可以单独使用，也可以作为 +> harness 的底层 System One。Jev 是它的一种底层实现，不是它的抽象。 +> +> 指纹侧的断言在 [judge](../judge/README.md)，与本文的关系见 §9。 + +## 结论 + +传统 harness 是一个**没有 System One 的 System 2**：所有判断要么由 L2 做，要么根本没在做——被正则、阈值和字符串相等顶替。 + +System One 不是给 harness 加的一层，是**被编译的判定**：L2 在工作中反复做的那些判断，重复到形状已知，于是可以脱离 L2 运行。 + +它和记忆系统是对偶的： + +| | 记忆 | 习惯 | +|---|---|---| +| 做什么 | 把东西**放进**上下文 | 让东西**不必进**上下文 | +| 底层 | 存储 + 检索 | **两个模型**：一个写题，一个答题 | +| 对 L2 | 有感——它读到了召回的内容 | **无感**——判定没出现在它面前 | +| 方向 | 加法 | 减法 | + +所以叫 `habit`：记忆存"我知道什么"，习惯存"我不必再想什么"。 + +**三个概念、两个端口、一条规则、零个自由参数。** + +--- + +## 1. 不可约前提 + +| | 前提 | 性质 | +|---|---|---| +| P1 | 推理是序列到序列的函数；一轮里模型知道的一切都在序列里 | 机制 | +| P2 | 输出自回归（串行、最贵、无缓存）；输入按前缀匹配 | 机制 | +| P3 | thinking 是生成出来的 token，按 output 计价，且默认留在前缀里 | 机制 + 当前 API 行为 | +| P4 | 存在一种便宜的类型化判断，但它的概率**没有在你的分布上校准** | **产品事实，不是机制** | + +由 P4 的性质得一条铁律： + +> **任何依赖 P4 数值的设计，必须带一个测量，不能带一个常数。** + +### L1 / L2 按能力定义 + +| | 能做什么 | 代价 | +|---|---|---| +| **L2** | 产生**新的**符号序列 | 慢、贵、输出留在前缀里 | +| **L1** | 从**事先给定的**有限集合里选一个 | 快、约等于免费、不产生 token | + +只有这一条差别。四个推论全是强制的： + +1. L1 不能给自己出题 ⟹ **题只能由 L2 写** +2. 同一道判定两边都能答 ⟹ **一致率是免费的** +3. L1 不产生 output token ⟹ 短路省掉的正好是 L2 的输出 +4. L1 不能解释 ⟹ **审计靠记录，不靠理由** + +推论 1 是这套能自转的原因：**编译器就是 L2 自己。** + +## 2. 推导 + +**第一步（P1 + P3）。** L2 生成的 token 分两类：**新工作**，和**重新做已经做过的判断**。后者是纯开销。 + +**第二步（P2 + P3）。** 重复判断付两次钱：生成时按 output 付（最贵的一档，无缓存），之后留在前缀里，每一轮再付一次。 + +**第三步。** 重复，所以形状已知；形状已知，所以可类型化。 + +> **重复性本身就是可编译性的判据。** 不需要第二个筛子。 + +反面同样成立：**新工作天然不可类型化，因为你事先不知道它的形状。** 所以"会不会把 L2 的推理掏空"这个担心自动消解——能被编译的恰好只有开销。 + +**第四步：短路的正确形态是旁路，不是注入。** + +把 L1 的答案**告诉** L2（Brief、注入陈述），L2 是有感的——它知道有人给了它一个结论，而且会盲信这条陈述胜过自己读出来的东西。一条错的陈述就是毒。 + +正确的形态是**塑形入口**： + +``` +世界 ──→ [L1 旁路：塑形] ──→ 上下文 ──→ L2 +``` + +人不会先看到原始光子、再收到一张纸条说"System 1 认为这是一张脸"。**你就是看到一张脸。** 视觉系统的计算对你完全无感。 + +这一步的后果： + +| | 注入形态 | 旁路形态 | +|---|---|---| +| L2 | 有感 | **无感** | +| 失败模式 | 一条假陈述，L2 盲信 | 一次糟糕的检索，漏了东西 | +| 缓存 | 要设计注入信道 | 发生在入上下文**之前**，永远是追加 | +| 弃权 | 必须显式写进注入内容 | **不塑形 = 原样通过**，零代码 | + +判据： + +> **一条判定如果必须讲给 L2 听，它就不是旁路。** + +"目标满足了吗""我试过这个吗"是**任务自省**，只能讲给 L2 听——那属于记忆或提示，不属于习惯。真正的旁路关于**材料**：哪几段相关、这两个观测是不是同一个、这条命令碰什么、这一步要多深的 effort。 + +**第五步（P4）。** 概率没校准，就要测。对照物是现成的：**L2 自己声明过的答案**。免费、在你的分布上、每次运行都产出。 + +> **替换和测量是同一个操作。** + +## 3. 三个概念 + +```go +// Claim：L2 在工作中声明的一次判定——问什么、判成了什么。 +// 绑在具体场景上，不能独立运行。 +type Claim struct { + Statement string + Chose string +} + +// Reflex：从反复出现的 Claim 里抽出来的问题，给定了有限选项， +// 验证过可以脱离 L2 运行。 +type Reflex struct { + Statement string + Options map[string]Option // 必须含"判不了" +} + +// Compile：实例 → 类。由 L2 执行，因为 L1 只会选不会写。 +Compile([]Claim) → Reflex +``` + +Claim 是**实例**，Reflex 是**类**。分成两个类型而不是一个类型加状态标志，是因为这样**未验证的东西在类型上就跑不了**：`Claim` 没有 Options，L1 根本接不了它。约定变成结构。 + +**Provider 不在这张表里。** 它是底层设施（§8），不是抽象。换掉 Jev 换成 Laya 或 Kev，这三个概念一个都不动。 + +## 4. 两个端口 + +调用方只认识这两个，不需要知道 Claim、Reflex、Compile 的存在。 + +```go +// 写：注册 L2 刚做过的判定 +Register(ctx, []Claim) error + +// 读：这次观测触发了哪些反射 +Sense(ctx, state any) (map[string]string, error) // 题 → 选中的选项 +``` + +`Sense` 返回**决定**，不执行动作——动作是领域相关的（留哪几段、effort 调多少、丢哪个重复），属于调用方。系统本身完全不认识领域，这是它能独立存在的原因。 + +## 5. 两种用法 + +### 5.1 独立使用（像记忆系统一样） + +手写 Reflex，直接 `Sense`。不需要 `Register`，不需要 `Compile`。 + +用途：任何**重复出现的有限选择判断**——日志归类、告警分级、内容路由。 + +代价要说清：跳过 Compile 就跳过了 §6 那两个解出来的数。晋升和退役由你负责，而 P4 的铁律不会因为你手写就失效——阈值仍然必须是测出来的，不能是选出来的。 + +### 5.2 作为 harness 的 System One + +L2 边跑边注册，短路**自行形成**： + +``` +L2 工作中声明 ──→ Register ──→ Claim 日志（只增） + │ + Compile（离线、周期性、L2 执行） + │ ① 聚出反复出现的 + │ ② 抽成带有限选项的问题 + │ ③ 用日志里 L2 自己的答案验证 + ▼ + Reflex 集合 ◄─── 持续抽样；保持率掉了就移出 + │ +观测 ──→ Sense ──→ 塑形 ──→ 上下文 ──→ L2（无感） +``` + +**"之后不再需要 Claim"的准确机制**：不是禁止 L2 声明，是**那个判定不再出现在它面前**。L2 原来会声明"我在判这 48K 里哪几段有用"；编译之后它收到的就是 1.2K——那个判定对它来说不存在了。不是被压制，是 moot。 + +**不需要看见 thinking。** 声明是显式输出，不是内部推理。thinking display 默认 `omitted`、开了也只给摘要、raw CoT 任何模型都不外露——这条限制整个绕开了。 + +## 6. 自转：两个解出来的数 + +一旦出现"一致率到 0.95 就晋升"这种人挑的数，就不自转了。 + +但 L2 是随机函数（P1）——**它和自己也不完全一致**。同一情境重放两次，结论会变。这个自洽率免费可测，而且是唯一诚实的标尺： + +> **晋升**：L1 在历史声明上与 L2 的一致率 ≥ L2 与自己的重放一致率 +> **退役**：持续抽样，上式不再成立 + +你没有理由要求替代品比被替代品更稳。参照物始终是日志里 L2 自己写下的 `Chose`。 + +**采样从不停。** 晋升只是把双跑率从 1 降到 ε，不是切换模式。因为一直在测，测量不会过期，所以不需要时间戳、不需要"训练完成"标志。 + +**没有置信度阈值。** 弃权已经是一个选项（每条 Reflex 构造上必含"判不了"），而阈值那套建在一个**在 0.3–0.8 中段最不可靠**的数上（§10）。两套机制做同一件事，砍掉弱的那套。 + +### 自愈 + +Reflex 移出 → 塑形不再发生 → 原始观测流过去 → 判定重新出现在 L2 面前 → L2 又开始声明它 → 攒够了重新 Compile。 + +**没有恢复逻辑。** 系统的降级路径就是它的正常路径。 + +## 7. 一条规则 + +> **未答即原样。** 没有 Reflex、或弃权——`Sense` 返回空，调用方什么都不做，行为和没有这个系统时逐字节相同。 + +在旁路形态下这不是一条要维护的约定，是**物理事实**：塑形发生在观测入上下文之前，不塑形就是原样通过。因此不需要升级机制、不需要唤醒判据、不需要兜底逻辑。**失败恒等于不启用。** + +## 8. 底层设施 + +以下全部是设施，不是抽象。换掉它们，§3 的三个概念一个都不动。 + +**Provider。** L1 的实现。`ID()` 标识端点和模型——**它一变，所有 Reflex 的验证作废**，回到 Claim 日志重来。候选：Jev（托管）、Laya（421M，Apache-2.0，本机，含多语种变体）、Kev（实现同一 `/v1/systemone` 契约，SDK 可直接指过去）。 + +**只增。** 缓存按前缀匹配，中段编辑使其后全部失效；编辑过的历史现在还可能直接 400（preserved thinking 的 history-editing check，2026-08-31 之后创建的账号）。旁路形态天然满足：塑形在入上下文之前。 + +**Claude Code 的既有实践可直接引用**，不必自己论证：`` 追加而非改写历史；plan mode 用常驻工具进出；MCP schema 用 `defer_loading` 存根；官方对缓存中断按可用性级别告警。DeepSeek Harness 的 append-only 会话日志 + Trajectory 视图 + fork/replay，正好是 Claim 日志需要的仪器。 + +**模型不能换，effort 可以。** 缓存按模型隔离，无逃生口；effort 有——`{"role":"system","content":[],"output_config":{"effort":...}}` 不触发 messages 缓存失效（beta `mid-conversation-output-config-2026-07-01`，Opus 5 / Fable 5.1）。所以"这一步该想多深"是一条合法的 Reflex。 + +**经济学。** thinking 按 output 计价且留在前缀（P3），所以短路省两样：生成时的 output token（最贵一档、无缓存）+ 此后每轮的前缀增长。参考量级：Opus 5 $5/$25 每 MTok，Sonnet 5 $2/$10；缓存读约 0.1×，写 1.25×（5 分钟）/ 2×（1 小时）；Jev $0.042/M，输出免费。 + +## 9. 边界：旁路 ≠ 新权威 + +§1–§8 整套只对一种东西成立，必须和另一种分开： + +| | 旁路 | 新权威 | +|---|---|---| +| 做什么 | 替换掉**已经在发生**的判定 | 做一件**原本没人在做**的判断 | +| 有对照物吗 | 有——L2 的声明 | 没有 | +| 怎么定阈值 | 噪声底判据，免费 | 真标注，要花钱 | +| 失败时 | 退回原状 | 产生一个原本不存在的后果 | + +**`judge` 的 `presence` 是第二种。** 规则引擎命中之后，原本没有任何东西在判"这个产品是不是真的在跑"。两个后果： + +1. `judge.New` 默认的 `MinConfidence = jev.DefaultMinConfidence`（0.3）没有依据，**也不可能通过一致率取得依据**——没有"原来的做法"可比。而 0.3 恰好落在 Jev confidence 最不可靠的 0.3–0.8 中段下沿。 +2. `presence.go` 里 `Refuted → Rejected` 删掉了规则引擎的命中，**弃权或出错不退回原样，而是丢召回**——直接违反 §7。指纹场景里召回损失是静默的。 + +两条路都行：改成退回（`Rejected` 降级为"标记待审"），或者承认它是新权威、用真标注给它有依据的阈值。不能一边按 System One 设计、一边行使权威。 + +真标注已经在仓库里：`.judge-data/product-authenticity/` 的 17 份带标签部署响应，SHA256 已固定。 + +## 10. 已知未解决与风险 + +1. **Projection —— 唯一会随规模真正崩掉的一处。** 每条 Reflex 喂哪段 state：64K 总 / 32K 单问上限、每页 40 条 Claim、8 个产品版本，随运行时变化，**且没有明确归属者**。试过把它收进抽象（让 Reflex 自带 `Needs`），砍掉了——今天手写截断能跑。但它是最弱的工程点。 +2. **抓不到系统性偏差。** 噪声底判据只保证"不比 L2 更差"，不保证对。L1 和 L2 可以一致地错，而一致的错看起来像确证。 +3. **标定不可移植。** 独立测评：公开 benchmark ECE 0.024–0.032，自建分布 0.107（噪声底 0.024）；一篇 pentest harness 论文在自有评估集上测得 Jev ECE 0.246。confidence **两端可靠、0.3–0.8 中段最不可靠**。这是 §6 砍掉阈值的经验依据。 +4. **无内省。** L1 不给理由，可审计性只能靠记录（题、证据摘要、选项、动作），不能靠解释。 +5. **CJK 精度。** 本仓库目标大量是中文 web 资产，需专门留中文样本。 +6. **早期访问。** 无论文、无开放权重、payload 形状可能变。生产必须钉版本，不用 `jev-latest`。 +7. **唯一的端到端对照数据**来自一篇 pentest harness 论文（单次运行，作者自陈不具统计效力）：墙钟 32m12s → 26m53s，findings 16 → 18，**但场景命中 10/13 → 9/13**。方向上支持省墙钟，不支持完成率不降。 + +## 11. 术语 + +**启用**：`habit`（系统）、`Claim`（L2 声明的一次判定）、`Reflex`（抽出来的类）、`Compile`（实例→类）、`Register` / `Sense`（两个端口）。 + +**退役**：`Verdict`、`Is` / `Pick` / `IsMany` / `IsAll`、`Brief`（含 A1/A2 双写）、`Groove`、`Startle`、`Precedent`、`Impression`、`Floor`、`MeasuredAt`。 + +**从未启用**：Sieve、Assay、Attest、Ledger、Rung、SiftAct、Understudy、Harvest、CoDE。 + +对不等于现在就该有名字；命名纪律也需要停止条件。 + +--- + +## 附录 A：Jev API 参考 + +来源 、。复核于 2026-09-27——`api.md` 当前页面只给端点、鉴权、题型格式与返回字段，未给价格、限速、上下文上限和语言支持。 + +| 项 | 值 | 复核 | +|---|---|---| +| 端点 | `POST https://api.typesafe.ai/v1/systemone`；`GET /v1/models` | ✅ | +| 鉴权 | `Authorization: Bearer `，`TYPESAFE_API_KEY` | ✅ | +| 请求 | `state`（文本 / JSON 对象 / 文本数组）+ `model` + `questions` | ✅ | +| 响应 | `model` + `answers` + `usage{input_tokens, output_tokens}` | ✅ | +| 错误 | 401 鉴权、422 校验、429 限速、529 过载；429/529 指数退避 | ✅ | +| 价格 | $0.042/M 输入，输出免费 | ⚠️ 未复核 | +| 限速 | 250,000 tokens/s；1,200 req/min | ⚠️ 未复核 | +| 上下文 | 64K 总；state + 最长单问 ≤ 32K | ⚠️ 未复核 | +| 延迟 | 70–500 ms；问题并行 | ✅ 与实测一致（p50 236–276 ms，15 问一次调用） | +| 语言 | 英文为主，CJK 准确度较低 | ❌ 当前页面无此内容，按未证实处理 | + +| 类型 | criteria | 返回 | +|---|---|---| +| `noul` | 可选，`true`/`false` 各自的描述 | `noul`，0–1 概率，**不是布尔**；**无 `confidence`** | +| `choice` | 选项→描述的 map，**最多 255 项** | `choice` + `probabilities` + `confidence` | +| `score` | **2–10 个**有序等级描述的数组 | 加权 `score` + `legend` + `probabilities` + `confidence` | + +**本仓库只用 `choice`**（客户端只解码 `answer.Choice`，缺 `confidence` 直接报错）。这是个有意的选择：因为只用 choice，"判不了"才能是一个**真正的选项**而不是特殊返回值。 + +**已知坑**：`instructions` 与 `criteria` 自相矛盾会明显拉低质量。Compile 阶段的不一致样本最容易暴露这类规约错误。 + +**与本仓库现状的差异**： + +- Jev 客户端与合约已迁出本仓库，现为独立模块 `github.com/chainreactors/utils/jev`；`judge/jev/` 与 `judge/page.go` 均已不存在。 +- `cmd/judgeeval` 默认 `-rps 15`（上限 100），相对 1,200 req/min 偏保守。 +- 分页预算散落在 `judge/presence.go`（40 条）与 `judge/version.go`（`maxVersions` 40、`maxVersioned` 8），**没有对 64K/32K 做整体校验**——即 §10 风险 1。 +- `utils/jev` 现在把旁路侧和权威侧混在一个包里（`Ruling`、`Resolve(ruling, minConfidence)`、`Outcome` 的三个真值属于权威侧）。这是 §9 那条分界在代码里的位置。 + +## 附录 B：修订记录 + +### 2026-09-27 v4 —— System One 成为独立系统 + +**轴变了。** v3 沿着"一次调用的数据流"组织（Claim → Ruling → Resolve → Outcome）；本版沿着"一个判定从每次都做变成不必再做"组织。前者是实现细节，后者才是这套东西存在的理由。 + +**减法**： + +| v3 | 去向 | +|---|---| +| Brief（尾部全量 / 增量入历史）、`role:"system"` 注入信道、`clear_at` | **全部退役。** 旁路形态下没有要注入的陈述（§2 第四步） | +| 「弃权必须显式出现在注入里」 | 退役。不塑形 = 原样通过，物理事实 | +| `Groove` / `Startle` / `Precedent` / `Impression` | 退役。分别塌进 Reflex、弃权、Reflex、一个单字段返回值 | +| `Floor` / `MinConfidence` / `MeasuredAt` | 退役。弃权已是选项，阈值那套建在最不可靠的信号上；持续采样使时间戳多余 | +| `Provider` 作为核心概念 | 降级为设施（§8） | +| 「五态生命周期」「四个筛子」「影子模式」 | 分别塌成两个类型、一个判据、一个采样率 | + +**加法**： + +1. **L1 / L2 按能力定义**（§1）——一条差别推出四个推论，取代 v3 的散列前提。 +2. **旁路形态**（§2 第四步）——本版最大的结构改动。判定不讲给 L2 听，而是塑形它感知到的东西。连带消灭了 v3 一整套注入设计。 +3. **Claim / Reflex 分离**（§3）——实例与类。`Claim` 这个名字被释放给 L2 的声明；v3 叫 `Claim` 的东西其实一直是 `Reflex`。净效果是**不再需要挖 thinking**（raw CoT 不外露那条硬依赖消失）。 +4. **两个端口**（§4）——`Register` / `Sense`。系统不认识领域，因而可以独立存在。 +5. **两种用法**（§5）——独立使用 / 作为 harness 的 System One。 +6. **噪声底判据**（§6）——晋升和退役都从 L2 的自洽率解出，零自由参数。 +7. **记忆 / 习惯对偶**（结论）——定位这个系统在 harness 里的位置。 + +概念总账：v3 是 4 个概念 + 1 个参数 + 1 个配置结构；v4 是 **3 个概念 + 2 个端口 + 1 条规则 + 0 个自由参数**，Provider 降为设施。 + +### 2026-09-27 v3 —— 先减法后加法 + +砍掉四方案同心结构、`Verdict`/`Is`/`Pick` 系列、Brief 双写、五条固定判定清单、升级/唤醒机制；加入 P4 铁律、「重复⟹可类型化」、「替换与测量同一」、`Groove`、「未答即原样」、「短路≠新权威」。v4 保留了 P4 铁律、§9 的分界和「未答即原样」，其余重构。 + +### 2026-09-26 v2 —— KV 缓存被指出后重写 + +骨架由「一个约束」改为「两个约束」；新增只增约束对设计空间的裁剪；经济学重写（v1 按全价计算上下文节省，夸大约 10 倍）；指标分列。v4 保留只增约束（已升级为 API 硬错误），其余结构已重构。 diff --git a/docs/jev-maintenance-validation-20260926.md b/docs/jev-maintenance-validation-20260926.md index fd4b795..889141f 100644 --- a/docs/jev-maintenance-validation-20260926.md +++ b/docs/jev-maintenance-validation-20260926.md @@ -1,9 +1,13 @@ # Jev 新产品指纹维护验证：20260926 批次 +> 历史评估记录：文中的 Refine、SuggestNames、Classify 等机制已移除，数据保留当时含义。当前实现见 [Claim 合约](../judge/README.md) 和 [judgeeval 报告格式](../cmd/judgeeval/README.md)，本文指标未由当前实现重新验证。 + ## 结论 本次扩大真实响应范围，核验了八种原目录没有的产品，生成八份候选；其中 **SearXNG、Wakapi、IT Tools 三种通过独立样本测试及原生库加载检查**，输出可直接加载的增量库。加载该库后再运行 Jev 清洗，也保留了全部标注真命中；重复维护识别为已有产品，不重复入库。 +补充[产品真实性复核](jev-product-authenticity-20260926.md)：三个都是已有真实开源项目，“新”指本次库内新增。**IT Tools 的标题被原版和衍生版共用，因此本文对它的正例及 FP/FN 仅代表产品家族识别，不代表已区分 CorentinTh 原版与其他分支。** + 这验证了有证据标注和评审的辅助维护流程。`IsUnknownProduct` 对 IT Tools 的三份真实界面仍返回 false；`SuggestNames` 均能提出正确名称。因此当前自动发现入口有漏检，不能依赖单个 unknown 布尔值决定所有新产品是否进入维护流程。 ## 交付物 diff --git a/docs/jev-product-authenticity-20260926.md b/docs/jev-product-authenticity-20260926.md new file mode 100644 index 0000000..c2aa44b --- /dev/null +++ b/docs/jev-product-authenticity-20260926.md @@ -0,0 +1,83 @@ +# 三个新增指纹的产品真实性复核 + +## 结论 + +**SearXNG、Wakapi、IT Tools 都是真实存在、有公开源码及实际部署的项目。** 本批“新产品”指当前已审计指纹库中的新增识别对象，不代表软件近期发布，也不代表首次在全球被识别。 + +| 产品 | 官方仓库 | 仓库创建日期（UTC） | 本批指纹的准确语义 | +|---|---|---|---| +| SearXNG | `searxng/searxng` | 2021-04-12 | SearXNG 产品识别；从 generator 声明提取版本 | +| Wakapi | `muety/wakapi` | 2019-05-21 | Wakapi 产品识别；结合标题与资源/页脚证据提取版本 | +| IT Tools | `CorentinTh/it-tools` | 2020-04-05 | **IT Tools 产品家族识别**；无法区分原版与衍生版，版本留空 | + +日期来自 GitHub 仓库 API 的 `created_at`，不是首次发布日。API 和源码复核时间为 2026-09-25 17:02–17:06 UTC，即北京时间 2026-09-26 01:02–01:06。部署证据复用上一轮已归档的真实 HTTP 响应。 + +## SearXNG：身份和生成标记相互印证 + +- 官方仓库：[searxng/searxng](https://github.com/searxng/searxng)。官方 README 将其定义为元搜索引擎。 +- 官方模板 `searx/templates/simple/base.html` 第 8 行生成 `name="generator"`、`content="searxng/{{ searxng_version }}"`；第 72 行也显示产品名及版本。 +- 官方 `searx/infopage/en/about.md` 第 49 行说明 SearXNG 源自 Searx 的分支。应保留 SearXNG 独立产品身份，不能把名字包含 Searx 的页面全部认作 SearXNG。 +- 本批 10 份 SearXNG 正例均包含对应 generator，包括定制标题的实例。规则依赖产品声明，不要求站点标题恰好叫 SearXNG。 + +固定源码证据：[模板 blob](https://api.github.com/repos/searxng/searxng/git/blobs/abad3f141daaecf0465211c463319beaac30ce39)、[项目关系说明 blob](https://api.github.com/repos/searxng/searxng/git/blobs/486b2617595ee91d50853163af983a4329ee09aa)。 + +**判定：真实产品，本次库内新增身份成立。** generator 是应用自行声明的证据，不等同于主机安装审计或不可伪造的版本证明。 + +## Wakapi：独立应用，兼容 WakaTime + +- 官方仓库：[muety/wakapi](https://github.com/muety/wakapi)。官方 README 第 14 行说明它是兼容 WakaTime 的自托管编程统计后端。 +- 官方 `views/head.tpl.html` 第 2 行使用 `Wakapi – Coding Statistics` 标题，与本批 4 份正例一致。 +- 模板第 18 行使用 `assets/css/app.dist.css?v={{ getCacheBuster }}`；`views/footer.tpl.html` 第 3 行通过 `getVersion` 输出版本。缓存参数须结合具体响应验证，不能仅因出现 `?v=` 就认定它是产品版本。 +- 本批样本包括项目主页及三个独立测试站点；版本提取结果见[原验证报告](jev-maintenance-validation-20260926.md)。 +- 内置 Wappalyzer 的 `Wakav Performance Monitoring` 是另一项监控产品，不能因前缀相似认作 Wakapi。兼容 WakaTime 也不等于 Wakapi 是 WakaTime 的别名。 + +固定源码证据：[README blob](https://api.github.com/repos/muety/wakapi/git/blobs/ce5f15f2198397dd5b41a6a1eb1be08ccab0d626)、[页面头模板 blob](https://api.github.com/repos/muety/wakapi/git/blobs/12f24762324fece08a9ccad1ba9265556c62aeaa)、[页脚模板 blob](https://api.github.com/repos/muety/wakapi/git/blobs/3d50040e195b1792925994163c6b7c0139ebe022)。 + +**判定：真实产品，本次库内新增身份成立。** + +## IT Tools：真实项目，当前指纹仅识别产品家族 + +- 原项目：[CorentinTh/it-tools](https://github.com/CorentinTh/it-tools)，开发者在线工具集合。官方 `index.html` 第 7 行即当前规则使用的完整标题。 +- 衍生项目：[sharevb/it-tools](https://github.com/sharevb/it-tools)。GitHub API 返回 `fork: true`，`parent.full_name` 为 `CorentinTh/it-tools`，仓库创建于 2023-11-30。这不是本批额外发现的第四个独立产品。 +- 衍生版 `index.html` 第 26 行保留相同标题，第 37 行的 `og:url` 指向 `https://sharevb-it-tools.vercel.app/`。 + +| 本批样本 | 直接观察 | 能够支持的结论 | +|---|---|---| +| `it-tools.tech` | 完整产品标题；`og:url` 指向项目主页 | 原项目官方站点样本 | +| `tools.code.pro.vn` | 同一标题；`og:url` 改为当前站点 | IT Tools 家族，具体分支未确定 | +| `tools.quitw.org` | 同一标题；`og:url` 指向 sharevb 主页 | 携带 sharevb 衍生版元数据，不能仅凭该信息确定精确代码分支或版本 | + +固定源码证据：[原版 index.html blob](https://api.github.com/repos/CorentinTh/it-tools/git/blobs/e8b8a60ea6f6864ea062874c4d6e84cbea608de6)、[衍生版 index.html blob](https://api.github.com/repos/sharevb/it-tools/git/blobs/7e0246142e9818a8745d758aed6bd9429c221546)。分支关系来源：[仓库 API](https://api.github.com/repos/sharevb/it-tools)。 + +**判定：真实项目，本次库内新增家族级识别成立。当前规则只匹配标题，不能宣称三个站点都是 CorentinTh 原版，也不能据此补具体版本。** 原报告的 IT Tools 正例和 FP/FN 应按家族级标签理解；若目标改为原版或指定分支，需要重新标注和验证，原指标不能直接沿用。 + +## “库内新增”的核查范围 + +复核 `run-07-final/catalog-before.json` 的五个内置 HTTP 目录、别名及运行时 fingers 目录，再对以下文件进行全文检查： + +- `resources/fingers_http.json.gz` +- `resources/goby.json.gz` +- `resources/ehole.json.gz` +- `resources/wappalyzer.json.gz` +- `resources/fingerprinthub_web.json.gz` +- `resources/aliases.yaml` + +解压后忽略大小写，检索 `searx|waka|it[\s_-]*tools|corentinth|sharevb|handy[\s_-]*online`。只有 Wappalyzer 中同一个 `Wakav Performance Monitoring` 条目的五处文本命中；其余为零。未发现三个产品的同名、显式别名或上述身份标记。 + +结合此前原生检测的新增前漏报、加载后的命中，以及重复维护时三项均成为 `already_catalogued`，支持“本次审计范围中的新增指纹”。不覆盖外部所有指纹库；关键词检查也不能排除不含这些标记的规则存在语义重合。 + +## 证据与变更 + +完整证据位于本地 `.judge-data/product-authenticity/`： + +| 文件 | 内容 | +|---|---| +| `repositories.json` 与四份仓库原始 JSON | 官方来源、创建时间、分支关系、采集时间、SHA256 | +| `source-tree-index.json` 与四份 tree JSON | 源文件定位、Git tree SHA、完整性标志 | +| `source-evidence.json` 与 `source/` | 官方源码、Git blob SHA、SHA256、证据行号 | +| `deployment-evidence.json` | 17 份正例的响应路径、URL、SHA256、身份标记 | +| `library-fulltext-audit.json` | 六个本地资源的 SHA256、检索命中及上下文 | + +17 份部署响应的 SHA256 均与原 manifest 一致。其中包含训练样本，不是新增盲测样本数量。本次复核补充官方来源和身份范围，没有增加识别准确率或覆盖率声明。 + +本次仅新增证据和文档说明；原指纹 YAML、历史标签及评估结果保持原样。YAML 的 SHA256 仍为 `522a85806592e10bdffa87ccb70be90e22098fdeba85cc52452235132520065a`。 diff --git a/docs/jev-real-validation-20260925.md b/docs/jev-real-validation-20260925.md index 91fb8ec..b00d0b9 100644 --- a/docs/jev-real-validation-20260925.md +++ b/docs/jev-real-validation-20260925.md @@ -1,5 +1,7 @@ # Jev 真实响应验证：2026-09-25 +> 历史评估记录：文中的 Refine、SuggestNames 等接口已移除，数据保留当时含义。当前实现见 [Claim 合约](../judge/README.md) 和 [judgeeval 报告格式](../cmd/judgeeval/README.md)，本文指标未由当前实现重新验证。 + ## 结论 已经用新采集的真实 HTTP 响应跑通保存结果清洗、逐产品版本补全、名称建议、候选指纹生成、原生 YAML 导出加载和离线回放。发现的问题已修复并重新评估。 diff --git a/docs/jev-report.md b/docs/jev-report.md index c135d21..8772fdf 100644 --- a/docs/jev-report.md +++ b/docs/jev-report.md @@ -1,5 +1,7 @@ # Jev 判定层效果报告 +> 历史评估记录：保留当时的接口、报告字段和测量结果。当前实现以 [Claim 合约](../judge/README.md) 和 [judgeeval 报告格式](../cmd/judgeeval/README.md) 为准；本文指标未由当前实现重新验证。 + 在 1070 个真实网站上，把**传统规则指纹**与**规则 + Jev 判定**逐页对比。 ## 核心对比 diff --git a/fingerprints/README.md b/fingerprints/README.md index ae11268..9f58202 100644 --- a/fingerprints/README.md +++ b/fingerprints/README.md @@ -18,6 +18,8 @@ if err := engine.Compile(); err != nil { hits, err := engine.DetectContent(rawHTTPResponse) ``` -生成规则维持原生 `Finger` / `Frameworks` 语义。SearXNG 和 Wakapi 包含版本提取；IT Tools 在当前真实样本没有明确版本证据时只识别产品。 +生成规则维持原生 `Finger` / `Frameworks` 语义。SearXNG 和 Wakapi 包含版本提取；IT Tools 当前按完整标题识别产品家族，覆盖原版与保留该标题的衍生版，不能区分具体分支，版本留空。 + +三个项目均经官方仓库和源码复核；“新增”仅指本次已审计的指纹库范围，不是近期发布的新软件。身份依据和分支范围见[产品真实性复核](../docs/jev-product-authenticity-20260926.md)。 扩展测试报告见 [Jev 指纹库维护验证](../docs/jev-maintenance-validation-20260926.md)，运行数据和完整响应保存在本地 `.judge-data/expansion-20260926/`。这些指纹的结论仅适用于报告中列出的版本、样本和标注范围。 From e423e9e2c766a898bc518af80a7d74654a9dc638 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Sat, 26 Sep 2026 23:29:35 -0700 Subject: [PATCH 16/16] build(re2): tidy for the utils/jev dependency re2 replaces fingers with the parent directory, so its go.mod lists the parent's dependencies: add utils/jev and move utils/parsers to the merged Judgement. Co-Authored-By: Claude Opus 5.5 (1M context) --- re2/go.mod | 3 ++- re2/go.sum | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/re2/go.mod b/re2/go.mod index db18a83..24921c4 100644 --- a/re2/go.mod +++ b/re2/go.mod @@ -12,7 +12,8 @@ require ( github.com/chainreactors/logs v0.0.0-20260508055944-c678762ed15c // indirect github.com/chainreactors/neutron v0.1.1-0.20260714062907-716c6b167cb6 // indirect github.com/chainreactors/utils v0.0.0-20260629043228-93bdd2142c9a // indirect - github.com/chainreactors/utils/parsers v0.0.4-0.20260925164932-39c659c84c64 // indirect + github.com/chainreactors/utils/jev v0.0.0-20260927031300-6bf027367a50 // indirect + github.com/chainreactors/utils/parsers v0.0.4-0.20260927032755-fd113e94d6c0 // indirect github.com/chainreactors/words v0.0.0-20260520145736-270600e60fb4 // indirect github.com/dlclark/regexp2 v1.11.5 // indirect github.com/facebookincubator/nvdtools v0.1.5 // indirect diff --git a/re2/go.sum b/re2/go.sum index 4274373..2baeaaf 100644 --- a/re2/go.sum +++ b/re2/go.sum @@ -81,8 +81,10 @@ github.com/chainreactors/neutron v0.1.1-0.20260714062907-716c6b167cb6 h1:apEDnJe github.com/chainreactors/neutron v0.1.1-0.20260714062907-716c6b167cb6/go.mod h1:zok/CDxut71iw8NQTHKPvy0f+1G5639zr9GgDsXGnOs= github.com/chainreactors/utils v0.0.0-20260629043228-93bdd2142c9a h1:4rLTKr+VlG7yGbw2ym7nhNvFglvZ4acdphxKEozF7CI= github.com/chainreactors/utils v0.0.0-20260629043228-93bdd2142c9a/go.mod h1:xwbUlFoSSxLHujyb8D48o1s2DqmEAxUNfxIy0DVUmcg= -github.com/chainreactors/utils/parsers v0.0.4-0.20260925164932-39c659c84c64 h1:MgEJ0vr7png0bKb5nYL1pK5lTUMDBRbigYt74K+UwYw= -github.com/chainreactors/utils/parsers v0.0.4-0.20260925164932-39c659c84c64/go.mod h1:bE/znJWt08n9QOORWsWu0ggB8GWfOg3+dfUMMITmwV4= +github.com/chainreactors/utils/jev v0.0.0-20260927031300-6bf027367a50 h1:XnGpBYIS4u1BUNL3KtlrrXde/vSmDon34X5AEV7WOi8= +github.com/chainreactors/utils/jev v0.0.0-20260927031300-6bf027367a50/go.mod h1:9w5fvcKrpQVIHWlKuvpVARpZUd+OKsYSIii2JR1ZRB8= +github.com/chainreactors/utils/parsers v0.0.4-0.20260927032755-fd113e94d6c0 h1:rfhEDw1QfbIgUAhGWRbovR3ur7Ug3V7j5WxqstwlnW0= +github.com/chainreactors/utils/parsers v0.0.4-0.20260927032755-fd113e94d6c0/go.mod h1:bE/znJWt08n9QOORWsWu0ggB8GWfOg3+dfUMMITmwV4= github.com/chainreactors/words v0.0.0-20260520145736-270600e60fb4 h1:lvnDYEkatmZFHP5i321qQXK9L4vKRfso/uUfr5tOeC8= github.com/chainreactors/words v0.0.0-20260520145736-270600e60fb4/go.mod h1:zfz367PUmyaX6oAqV9SktVqyRXKlEh0sel9Wsq9dd2c= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=