From 93e3c06e0485edf7b2dcabba4c4ab3ff243d4c1c Mon Sep 17 00:00:00 2001 From: Zakk Date: Tue, 29 Sep 2026 03:13:30 +1000 Subject: [PATCH 1/5] fix(api): make runtime settings sections optional and recorder demand engine-defined Only observed_at and source are required; log.level and the replay ring are independent, so a level-only engine is valid. Fixed 64-record minimums leave the schema and engines advertise min_* bounds. Flow capability ceilings are the engine's limits, flows.recording gains on_demand, and honk's attachment and demand timing moves to the honk notes. --- api/discovery.yaml | 22 ++++- api/settings.yaml | 74 +++++++++------- source/openapi.yaml | 107 ++++++++++++++---------- source/v0.1.0/en/docs/flows.md | 11 ++- source/v0.1.0/en/docs/honk-mapping.md | 28 +++++++ source/v0.1.0/en/docs/runtime-status.md | 58 +++++++------ tools/check-contract.test.mjs | 73 +++++++++++++++- 7 files changed, 266 insertions(+), 107 deletions(-) diff --git a/api/discovery.yaml b/api/discovery.yaml index 5306c7a..a45684d 100644 --- a/api/discovery.yaml +++ b/api/discovery.yaml @@ -227,8 +227,9 @@ paths: max_bulk_close: 1000 flows: available: true - recording: on + recording: on_demand scopes: [ userspace_tcp, userspace_udp ] + min_flows: 64 max_flows: 10000 max_steps_per_flow: 256 retention_seconds: 300 @@ -256,6 +257,7 @@ paths: available: true levels: [ trace, debug, info, warn, error ] retention_seconds: 60 + min_buffered_records: 64 max_buffered_records: 4096 dns_query: available: true @@ -275,6 +277,7 @@ paths: entry_kinds: [ positive, negative ] dns_log: available: true + min_records: 64 max_records: 2048 max_page_size: 500 dns_rules: @@ -809,21 +812,28 @@ schemas: type: boolean recording: type: string - enum: [ off, on, sampled ] + enum: [ off, on, sampled, on_demand ] + description: How the engine records flows right now. off records none; on records every flow in scopes; sampled records a subset; on_demand records only while clients read or follow flows, which is recording.flows.mode auto in GET /runtime/settings. scopes: type: array uniqueItems: true items: $ref: ./openapi.yaml#/components/schemas/FlowScope + min_flows: + $ref: ./openapi.yaml#/components/schemas/SafeUInt + minimum: 1 + description: Smallest flows.max_flows a runtime-settings PATCH may set. Absent means 1. max_flows: type: integer minimum: 1 + description: The largest flow capacity the engine supports, which is the most a runtime-settings PATCH may set, not the current value. The current value is flows.max_flows in GET /runtime/settings. max_steps_per_flow: type: integer minimum: 1 retention_seconds: type: integer minimum: 0 + description: The longest terminal-flow retention the engine supports, which is the most a runtime-settings PATCH may set, not the current value. The current value is flows.retention_seconds in GET /runtime/settings. snapshot_ttl_seconds: type: integer minimum: 1 @@ -932,6 +942,10 @@ schemas: type: integer minimum: 1 description: Maximum age of a replayable record, in seconds. A resume cursor older than this returns 409 event_cursor_expired even when the ring has room. + min_buffered_records: + $ref: ./openapi.yaml#/components/schemas/SafeUInt + minimum: 1 + description: Smallest log.buffered_records a runtime-settings PATCH may set. Absent means 1. max_buffered_records: $ref: ./openapi.yaml#/components/schemas/SafeUInt minimum: 1 @@ -993,6 +1007,10 @@ schemas: properties: available: type: boolean + min_records: + $ref: ./openapi.yaml#/components/schemas/SafeUInt + minimum: 1 + description: Smallest dns_log.max_records a runtime-settings PATCH may set. Absent means 1. max_records: $ref: ./openapi.yaml#/components/schemas/SafeUInt minimum: 1 diff --git a/api/settings.yaml b/api/settings.yaml index 3f34cda..a45626f 100644 --- a/api/settings.yaml +++ b/api/settings.yaml @@ -5,8 +5,12 @@ paths: summary: Read the runtime-adjustable settings description: | Requires resources.runtime_settings.available. Returns the current - runtime-adjustable settings. Numeric values cannot exceed their corresponding - capability ceilings. source is config when values come from the activated + runtime-adjustable settings. Only observed_at and source are always present: + a value appears when resources.runtime_settings.fields lists it, and + recording.flows, recording.logs and recording.dns_log appear when + record_flows, record_logs and record_dns_log are listed. An engine may + report other values it cannot change. Numeric values stay inside their + capability bounds. source is config when values come from the activated configuration and runtime after a runtime override. geodata appears when resources.geodata.configurable_sources is true and carries its own source. Its URLs are returned as written, with only listener-secret values masked, @@ -73,6 +77,16 @@ paths: Content-Type: application/json Cache-Control: no-store X-Content-Type-Options: nosniff + level_only: + summary: An engine whose only runtime setting is the log level + value: + observed_at: 2026-08-15T10:00:00Z + source: config + log: { level: info } + x-headers: + Content-Type: application/json + Cache-Control: no-store + X-Content-Type-Options: nosniff "400": $ref: ./openapi.yaml#/components/responses/BadRequest "401": @@ -90,7 +104,7 @@ paths: Requires control and resources.runtime_settings.available. The body is a merge: an absent field keeps its value. Every value is checked before anything changes, and a rejected patch changes nothing. A field the schema - does not define, a ring below 64 records, or a value above its ceiling + does not define, or a value outside its schema range or advertised bounds, returns 400 invalid_request. A field not listed in resources.runtime_settings.fields, a log level not in resources.logs.levels, or pinning a recorder whose allowed is false returns @@ -129,6 +143,10 @@ paths: value: log: { level: debug } dns_log: { max_records: 1024 } + level_only: + summary: Change only the log level + value: + log: { level: info } geodata_sources: summary: Use a mirror first value: @@ -209,7 +227,7 @@ paths: Cache-Control: no-store X-Content-Type-Options: nosniff "400": - description: A field the schema does not define, or a value outside its range or advertised ceiling + description: A field the schema does not define, or a value outside its schema range or advertised bounds headers: Cache-Control: $ref: ./openapi.yaml#/components/headers/NoStore @@ -294,7 +312,7 @@ schemas: type: string enum: [ log.level, log.buffered_records, dns_log.max_records, flows.max_flows, flows.retention_seconds, record_flows, record_logs, record_dns_log, geodata ] RecorderMode: - description: true pins a permitted recorder on, false forces it off, auto (the startup default) follows flow demand for record_flows and client attachment for the other recorders. + description: true pins a permitted recorder on, false forces it off, and auto (the startup default) lets the engine record on demand, while clients read or follow what the recorder captures. What counts as demand, and how long it lasts, is engine-defined. oneOf: - type: boolean - type: string @@ -314,7 +332,8 @@ schemas: description: The recorder is capturing right now. RuntimeSettings: type: object - required: [ observed_at, source, log, dns_log, flows ] + required: [ observed_at, source ] + description: Every section is optional. A value appears when resources.runtime_settings.fields lists it; an engine may also report a value it cannot change. properties: observed_at: $ref: ./openapi.yaml#/components/schemas/Timestamp @@ -324,50 +343,43 @@ schemas: description: config while every value comes from the activated configuration; runtime once any PATCH overrode one. geodata has its own source and does not affect this one. log: type: object - required: [ level, buffered_records ] + minProperties: 1 properties: level: $ref: ./openapi.yaml#/components/schemas/LogLevel description: Minimum severity the engine emits. A lower stream level cannot recover records the engine did not emit. buffered_records: $ref: ./openapi.yaml#/components/schemas/SafeUInt - minimum: 64 - description: Log replay ring capacity, at most logs.max_buffered_records. + minimum: 1 + description: Log replay ring capacity, within logs.min_buffered_records and logs.max_buffered_records. Independent of level. dns_log: type: object required: [ max_records ] properties: max_records: $ref: ./openapi.yaml#/components/schemas/SafeUInt - minimum: 64 - description: DNS log ring capacity, at most dns_log.max_records. + minimum: 1 + description: DNS log ring capacity, within dns_log.min_records and dns_log.max_records. flows: type: object - required: [ max_flows, retention_seconds ] + minProperties: 1 properties: max_flows: $ref: ./openapi.yaml#/components/schemas/SafeUInt - minimum: 64 - description: Retained flows, at most flows.max_flows. + minimum: 1 + description: Retained flows, within flows.min_flows and flows.max_flows. retention_seconds: $ref: ./openapi.yaml#/components/schemas/SafeUInt minimum: 1 description: Maximum age of a retained terminal flow, in seconds, bounded by resources.flows.retention_seconds. Capacity pressure may evict it earlier. recording: type: object - required: [ flows, logs, dns_log, events, grace_remaining_seconds ] + minProperties: 1 description: | - Read-only recorder state. A client is attached while an admitted GET SSE stream on - /events or /logs is open, or for 60 seconds after the last stream closed or a successful - GET on /flows, /flows/{flow_id} or /dns/log; other requests, including settings reads, do not - renew attachment. Automatic log and DNS-log recorders follow attachment. The automatic - flow recorder follows flow demand instead, so that an open panel does not record full - flow traces for every connection: an admitted GET /events stream whose kinds include - flow.updated or flow.gap, or that sets a nonblank flow_id with a flow kind in its - effective kinds, holds demand while open and for 60 seconds after the last one closed, - and a successful GET on /flows or /flows/{flow_id} renews it for 60 seconds. Event streams - without kinds, /logs streams and /dns/log reads do not create flow demand. Recording - starts on attachment or demand, so the first history a client reads may be empty. + Read-only recorder state. Each recorder appears when its record_* field is listed in + resources.runtime_settings.fields. In auto mode a recorder captures on demand, so the + first history a client reads may be empty. Engine-specific demand rules are + documented with the engine; see the honk notes. properties: flows: $ref: ./openapi.yaml#/components/schemas/RecorderState @@ -381,10 +393,10 @@ schemas: properties: active: type: boolean - description: Event capture runs while a client is attached or any permitted recorder is pinned on. + description: Event capture is running. grace_remaining_seconds: $ref: ./openapi.yaml#/components/schemas/SafeUInt - description: Seconds left before the automatic log and DNS-log recorders stop, 0 while a stream is open or nothing is attached. The flow-demand grace is not reported. + description: Seconds left before automatic recorders stop after the last client left, 0 while a client is attached or nothing is recording. Absent when the engine keeps no such grace. geodata: $ref: ./openapi.yaml#/components/schemas/GeoDataSettings description: Geodata download sources and automatic updates. Present when resources.geodata.configurable_sources is true. URLs are returned as written, with only listener-secret values masked, to every admitted caller. @@ -408,7 +420,7 @@ schemas: $ref: ./openapi.yaml#/components/schemas/LogLevel buffered_records: $ref: ./openapi.yaml#/components/schemas/SafeUInt - minimum: 64 + minimum: 1 dns_log: type: object additionalProperties: false @@ -416,7 +428,7 @@ schemas: properties: max_records: $ref: ./openapi.yaml#/components/schemas/SafeUInt - minimum: 64 + minimum: 1 flows: type: object additionalProperties: false @@ -424,7 +436,7 @@ schemas: properties: max_flows: $ref: ./openapi.yaml#/components/schemas/SafeUInt - minimum: 64 + minimum: 1 retention_seconds: $ref: ./openapi.yaml#/components/schemas/SafeUInt minimum: 1 diff --git a/source/openapi.yaml b/source/openapi.yaml index a345000..f1bf1eb 100644 --- a/source/openapi.yaml +++ b/source/openapi.yaml @@ -255,10 +255,11 @@ paths: max_bulk_close: 1000 flows: available: true - recording: 'on' + recording: on_demand scopes: - userspace_tcp - userspace_udp + min_flows: 64 max_flows: 10000 max_steps_per_flow: 256 retention_seconds: 300 @@ -298,6 +299,7 @@ paths: - warn - error retention_seconds: 60 + min_buffered_records: 64 max_buffered_records: 4096 dns_query: available: true @@ -322,6 +324,7 @@ paths: - negative dns_log: available: true + min_records: 64 max_records: 2048 max_page_size: 500 dns_rules: @@ -5220,8 +5223,12 @@ paths: summary: Read the runtime-adjustable settings description: | Requires resources.runtime_settings.available. Returns the current - runtime-adjustable settings. Numeric values cannot exceed their corresponding - capability ceilings. source is config when values come from the activated + runtime-adjustable settings. Only observed_at and source are always present: + a value appears when resources.runtime_settings.fields lists it, and + recording.flows, recording.logs and recording.dns_log appear when + record_flows, record_logs and record_dns_log are listed. An engine may + report other values it cannot change. Numeric values stay inside their + capability bounds. source is config when values come from the activated configuration and runtime after a runtime override. geodata appears when resources.geodata.configurable_sources is true and carries its own source. Its URLs are returned as written, with only listener-secret values masked, @@ -5308,6 +5315,17 @@ paths: Content-Type: application/json Cache-Control: no-store X-Content-Type-Options: nosniff + level_only: + summary: An engine whose only runtime setting is the log level + value: + observed_at: '2026-08-15T10:00:00Z' + source: config + log: + level: info + x-headers: + Content-Type: application/json + Cache-Control: no-store + X-Content-Type-Options: nosniff '400': $ref: '#/components/responses/BadRequest' '401': @@ -5325,7 +5343,7 @@ paths: Requires control and resources.runtime_settings.available. The body is a merge: an absent field keeps its value. Every value is checked before anything changes, and a rejected patch changes nothing. A field the schema - does not define, a ring below 64 records, or a value above its ceiling + does not define, or a value outside its schema range or advertised bounds, returns 400 invalid_request. A field not listed in resources.runtime_settings.fields, a log level not in resources.logs.levels, or pinning a recorder whose allowed is false returns @@ -5366,6 +5384,11 @@ paths: level: debug dns_log: max_records: 1024 + level_only: + summary: Change only the log level + value: + log: + level: info geodata_sources: summary: Use a mirror first value: @@ -5464,7 +5487,7 @@ paths: Cache-Control: no-store X-Content-Type-Options: nosniff '400': - description: A field the schema does not define, or a value outside its range or advertised ceiling + description: A field the schema does not define, or a value outside its schema range or advertised bounds headers: Cache-Control: $ref: '#/components/headers/NoStore' @@ -7837,20 +7860,28 @@ components: - 'off' - 'on' - sampled + - on_demand + description: How the engine records flows right now. off records none; on records every flow in scopes; sampled records a subset; on_demand records only while clients read or follow flows, which is recording.flows.mode auto in GET /runtime/settings. scopes: type: array uniqueItems: true items: $ref: '#/components/schemas/FlowScope' + min_flows: + $ref: '#/components/schemas/SafeUInt' + minimum: 1 + description: Smallest flows.max_flows a runtime-settings PATCH may set. Absent means 1. max_flows: type: integer minimum: 1 + description: The largest flow capacity the engine supports, which is the most a runtime-settings PATCH may set, not the current value. The current value is flows.max_flows in GET /runtime/settings. max_steps_per_flow: type: integer minimum: 1 retention_seconds: type: integer minimum: 0 + description: The longest terminal-flow retention the engine supports, which is the most a runtime-settings PATCH may set, not the current value. The current value is flows.retention_seconds in GET /runtime/settings. snapshot_ttl_seconds: type: integer minimum: 1 @@ -7980,6 +8011,10 @@ components: type: integer minimum: 1 description: Maximum age of a replayable record, in seconds. A resume cursor older than this returns 409 event_cursor_expired even when the ring has room. + min_buffered_records: + $ref: '#/components/schemas/SafeUInt' + minimum: 1 + description: Smallest log.buffered_records a runtime-settings PATCH may set. Absent means 1. max_buffered_records: $ref: '#/components/schemas/SafeUInt' minimum: 1 @@ -8055,6 +8090,10 @@ components: properties: available: type: boolean + min_records: + $ref: '#/components/schemas/SafeUInt' + minimum: 1 + description: Smallest dns_log.max_records a runtime-settings PATCH may set. Absent means 1. max_records: $ref: '#/components/schemas/SafeUInt' minimum: 1 @@ -13090,9 +13129,7 @@ components: required: - observed_at - source - - log - - dns_log - - flows + description: Every section is optional. A value appears when resources.runtime_settings.fields lists it; an engine may also report a value it cannot change. properties: observed_at: $ref: '#/components/schemas/Timestamp' @@ -13104,17 +13141,15 @@ components: description: config while every value comes from the activated configuration; runtime once any PATCH overrode one. geodata has its own source and does not affect this one. log: type: object - required: - - level - - buffered_records + minProperties: 1 properties: level: $ref: '#/components/schemas/LogLevel' description: Minimum severity the engine emits. A lower stream level cannot recover records the engine did not emit. buffered_records: $ref: '#/components/schemas/SafeUInt' - minimum: 64 - description: Log replay ring capacity, at most logs.max_buffered_records. + minimum: 1 + description: Log replay ring capacity, within logs.min_buffered_records and logs.max_buffered_records. Independent of level. dns_log: type: object required: @@ -13122,42 +13157,28 @@ components: properties: max_records: $ref: '#/components/schemas/SafeUInt' - minimum: 64 - description: DNS log ring capacity, at most dns_log.max_records. + minimum: 1 + description: DNS log ring capacity, within dns_log.min_records and dns_log.max_records. flows: type: object - required: - - max_flows - - retention_seconds + minProperties: 1 properties: max_flows: $ref: '#/components/schemas/SafeUInt' - minimum: 64 - description: Retained flows, at most flows.max_flows. + minimum: 1 + description: Retained flows, within flows.min_flows and flows.max_flows. retention_seconds: $ref: '#/components/schemas/SafeUInt' minimum: 1 description: Maximum age of a retained terminal flow, in seconds, bounded by resources.flows.retention_seconds. Capacity pressure may evict it earlier. recording: type: object - required: - - flows - - logs - - dns_log - - events - - grace_remaining_seconds + minProperties: 1 description: | - Read-only recorder state. A client is attached while an admitted GET SSE stream on - /events or /logs is open, or for 60 seconds after the last stream closed or a successful - GET on /flows, /flows/{flow_id} or /dns/log; other requests, including settings reads, do not - renew attachment. Automatic log and DNS-log recorders follow attachment. The automatic - flow recorder follows flow demand instead, so that an open panel does not record full - flow traces for every connection: an admitted GET /events stream whose kinds include - flow.updated or flow.gap, or that sets a nonblank flow_id with a flow kind in its - effective kinds, holds demand while open and for 60 seconds after the last one closed, - and a successful GET on /flows or /flows/{flow_id} renews it for 60 seconds. Event streams - without kinds, /logs streams and /dns/log reads do not create flow demand. Recording - starts on attachment or demand, so the first history a client reads may be empty. + Read-only recorder state. Each recorder appears when its record_* field is listed in + resources.runtime_settings.fields. In auto mode a recorder captures on demand, so the + first history a client reads may be empty. Engine-specific demand rules are + documented with the engine; see the honk notes. properties: flows: $ref: '#/components/schemas/RecorderState' @@ -13172,10 +13193,10 @@ components: properties: active: type: boolean - description: Event capture runs while a client is attached or any permitted recorder is pinned on. + description: Event capture is running. grace_remaining_seconds: $ref: '#/components/schemas/SafeUInt' - description: Seconds left before the automatic log and DNS-log recorders stop, 0 while a stream is open or nothing is attached. The flow-demand grace is not reported. + description: Seconds left before automatic recorders stop after the last client left, 0 while a client is attached or nothing is recording. Absent when the engine keeps no such grace. geodata: $ref: '#/components/schemas/GeoDataSettings' description: Geodata download sources and automatic updates. Present when resources.geodata.configurable_sources is true. URLs are returned as written, with only listener-secret values masked, to every admitted caller. @@ -13199,7 +13220,7 @@ components: $ref: '#/components/schemas/LogLevel' buffered_records: $ref: '#/components/schemas/SafeUInt' - minimum: 64 + minimum: 1 dns_log: type: object additionalProperties: false @@ -13207,7 +13228,7 @@ components: properties: max_records: $ref: '#/components/schemas/SafeUInt' - minimum: 64 + minimum: 1 flows: type: object additionalProperties: false @@ -13215,14 +13236,14 @@ components: properties: max_flows: $ref: '#/components/schemas/SafeUInt' - minimum: 64 + minimum: 1 retention_seconds: $ref: '#/components/schemas/SafeUInt' minimum: 1 geodata: $ref: '#/components/schemas/GeoDataSettingsPatch' RecorderMode: - description: true pins a permitted recorder on, false forces it off, auto (the startup default) follows flow demand for record_flows and client attachment for the other recorders. + description: true pins a permitted recorder on, false forces it off, and auto (the startup default) lets the engine record on demand, while clients read or follow what the recorder captures. What counts as demand, and how long it lasts, is engine-defined. oneOf: - type: boolean - type: string diff --git a/source/v0.1.0/en/docs/flows.md b/source/v0.1.0/en/docs/flows.md index a28fd10..81bfbe7 100644 --- a/source/v0.1.0/en/docs/flows.md +++ b/source/v0.1.0/en/docs/flows.md @@ -324,9 +324,14 @@ Unobserved kernel-direct/blocked flows cannot be hidden behind a full userspace list. Kernel bypasses (multicast, own traffic, local services, closed admission) must be declared even where no connection exists. -Capabilities advertise `recording` (`off`, `on`, `sampled`), `scopes`, -`max_flows`, `max_steps_per_flow`, `retention_seconds`, `snapshot_ttl_seconds`, -and `max_page_size`. Retention is a **maximum age after termination**, not a +Capabilities advertise `recording` (`off`, `on`, `sampled`, `on_demand`), +`scopes`, `max_flows`, `max_steps_per_flow`, `retention_seconds`, +`snapshot_ttl_seconds`, and `max_page_size`. `on_demand` records only while +clients read or follow flows (see +[runtime settings](runtime-status.html#GET-api-v1-runtime-settings)). +`max_flows` and `retention_seconds` are the engine's limits, the most a +runtime-settings PATCH may set; the current values are in +`GET /runtime/settings`. Retention is a **maximum age after termination**, not a durable guarantee under the bounded memory limit. Eviction, recording toggles, and losses produce `flow.gap` events; per-flow loss also marks the retained record partial. Known expired IDs return `410 flow_expired` while a bounded diff --git a/source/v0.1.0/en/docs/honk-mapping.md b/source/v0.1.0/en/docs/honk-mapping.md index 839ae48..a816e89 100644 --- a/source/v0.1.0/en/docs/honk-mapping.md +++ b/source/v0.1.0/en/docs/honk-mapping.md @@ -218,3 +218,31 @@ another way. the first, at process start or the latest `auto_update` change, plus the wait and a random delay of up to 60 minutes. After a failed attempt the wait is a backoff that starts at one hour, doubles on each further failure, and never exceeds the interval. + +## Runtime settings in honk + +This section is not part of the contract. It records how honk implements +[runtime settings](runtime-status.html#GET-api-v1-runtime-settings) and the +recorders on the current `feat/native-api` branch. + +- A PATCH accepts `log.buffered_records` and `dns_log.max_records` from 64 to + 512, `flows.max_flows` from 64 to 1024, and `flows.retention_seconds` from 1 + to 300. The startup values are 512, 512, 1024 and 300. +- honk lists `log.level` and `log.buffered_records` only when the log recorder + is permitted, and reports every settings section whether or not it is listed. +- A client is attached while an admitted GET SSE stream on `/events` or + `/logs` is open, and for 60 seconds after the last one closed or after a + successful GET on `/flows`, `/flows/{flow_id}` or `/dns/log`. Settings reads, + HEAD and rejected requests do not renew attachment. In `auto` mode the log + and DNS-log recorders capture while a client is attached. +- The automatic flow recorder follows flow demand instead, so that an open + panel does not record full traces for every connection. An admitted GET + `/events` stream creates demand when its `kinds` include `flow.updated` or + `flow.gap`, or when it sets a nonblank `flow_id` and its effective kinds + include a flow kind. Demand lasts while such a stream is open, and for 60 + seconds after the last one closed or after a successful GET on `/flows` or + `/flows/{flow_id}`. Event streams without `kinds`, `/logs` streams and + `/dns/log` reads do not create demand, and attachment does not extend it. +- `grace_remaining_seconds` counts the attachment grace only, not the + flow-demand grace. Event capture runs while a client is attached or any + permitted recorder is pinned on. diff --git a/source/v0.1.0/en/docs/runtime-status.md b/source/v0.1.0/en/docs/runtime-status.md index af2bfa9..0398010 100644 --- a/source/v0.1.0/en/docs/runtime-status.md +++ b/source/v0.1.0/en/docs/runtime-status.md @@ -198,51 +198,55 @@ first open or reconnect rather than treating invalidations as samples. Requires `observe` and `resources.runtime_settings.available`. Reports the values the running engine uses for what a panel may tune without a reload: -the log level and replay ring, the DNS log ring, and flow retention. +the log level and replay ring, the DNS log ring, flow retention and the +recorders. {% api_example getRuntimeSettings 200 current %} -| Field | Ceiling | Meaning | -|-------|---------|---------| +Only `observed_at` and `source` are always present. Every other section is +optional: a value appears when `resources.runtime_settings.fields` lists it, +and an engine may also report a value it cannot change. Inside `log`, +`level` and `buffered_records` are independent, so an engine whose only +setting is the log level reports `log.level` alone: + +{% api_example getRuntimeSettings 200 level_only %} + +| Field | Bounds | Meaning | +|-------|--------|---------| | log.level | `logs.levels` | Minimum severity the engine emits. | -| log.buffered_records | `logs.max_buffered_records` | Log replay ring capacity, at least 64. | -| dns_log.max_records | `dns_log.max_records` | DNS log ring capacity, at least 64. | -| flows.max_flows | `flows.max_flows` | Retained flows, at least 64. | -| flows.retention_seconds | `flows.retention_seconds` | Maximum age after termination; capacity pressure may evict a flow sooner. | +| log.buffered_records | `logs.min_buffered_records` to `logs.max_buffered_records` | Log replay ring capacity. | +| dns_log.max_records | `dns_log.min_records` to `dns_log.max_records` | DNS log ring capacity. | +| flows.max_flows | `flows.min_flows` to `flows.max_flows` | Retained flows. | +| flows.retention_seconds | 1 to `flows.retention_seconds` | Maximum age after termination; capacity pressure may evict a flow sooner. | | source | | `config` while every value comes from the activated configuration, `runtime` once a PATCH overrode one. | | geodata | | Geodata download URLs, download route, automatic updates and checksum verification, with their own read-only `source` for the URLs; URLs are returned as written, with only listener-secret values masked. Present when `resources.geodata.configurable_sources` is true. See [Geodata](geodata.html#Configure-the-sources). | -| recording | | Read-only recorder state: `flows`, `logs` and `dns_log` each report `allowed`, `mode` (`auto`, `on`, `off`) and `active`; `events.active` reports event capture; `grace_remaining_seconds` counts down after the last attached client left and does not report the flow-demand grace. | - -A client is attached while an admitted GET SSE stream on `/events` or `/logs` -is open, or for 60 seconds after the last stream closed or a successful GET on -`/flows`, `/flows/{flow_id}` or `/dns/log`. Settings reads, HEAD and rejected -requests do not renew attachment. In `auto` mode the log and DNS-log recorders -capture only while a client is attached. - -In `auto` mode the flow recorder follows flow demand instead, so that an open -panel does not record full flow traces for every connection. An admitted GET -`/events` stream creates flow demand when its `kinds` include `flow.updated` or -`flow.gap`, or when it sets a nonblank `flow_id` and its effective kinds include -a flow kind. Demand lasts while such a stream is open, and for 60 seconds after -the last one closed or after a successful GET on `/flows` or `/flows/{flow_id}`. -Event streams without `kinds`, `/logs` streams and `/dns/log` reads do not -create demand, and general attachment does not extend the flow grace. Recording -starts on attachment or demand, so the first history a panel reads may be empty. +| recording | | Read-only recorder state. `flows`, `logs` and `dns_log` each report `allowed`, `mode` (`auto`, `on`, `off`) and `active`, and appear when `record_flows`, `record_logs` and `record_dns_log` are listed in `fields`. `events.active` reports event capture. `grace_remaining_seconds`, when present, counts down before automatic recorders stop after the last client left. | + +The capability bounds are the engine's own limits, not the current values. +An absent minimum means 1. + +In `auto` mode a recorder captures on demand: while clients read or follow +what it records. What counts as demand and how long it lasts after the last +client left is engine-defined; `resources.flows.recording` reports +`on_demand` while the flow recorder is in `auto` mode. Recording starts on +demand, so the first history a panel reads may be empty. honk's rules are in +the [honk notes](honk-mapping.html#Runtime-settings-in-honk). ## PATCH /api/v1/runtime/settings Requires `control`. Only the fields listed in `resources.runtime_settings.fields` may appear; the body merges, an absent field keeps its value. `record_flows`, `record_logs` and `record_dns_log` take `true` (keep the recorder on without -clients), `false` (force it off) or `"auto"` (the startup default: flows follow -flow demand, logs and DNS logs follow attachment); pinning a recorder the configuration forbids rejects the whole patch. +clients), `false` (force it off) or `"auto"` (the startup default: record on +demand); pinning a recorder the configuration forbids rejects the whole patch. +`{"log": {"level": "info"}}` is a complete request. {% api_request patchRuntimeSettings debug %} {% api_example patchRuntimeSettings 200 changed %} Every value is checked before anything changes, and a rejected patch changes -nothing. A ring below 64 records or a value above its ceiling returns +nothing. A value outside its schema range or advertised bounds returns `400 invalid_request`; a field not in `resources.runtime_settings.fields`, an unadvertised level, or pinning a recorder whose `allowed` is false returns `422 unsupported_value` (see [errors](errors.html#Choosing-the-status)). Shrinking a ring drops its diff --git a/tools/check-contract.test.mjs b/tools/check-contract.test.mjs index 167146a..250a1fc 100644 --- a/tools/check-contract.test.mjs +++ b/tools/check-contract.test.mjs @@ -237,13 +237,84 @@ test("runtime settings stay inside their capability ceilings and advertised fiel assertInvalid(validateExample(contract, patch), "levels come from the enum"); patch.body = {source: "runtime"}; assertInvalid(validateExample(contract, patch), "source is read-only"); + patch.body = {dns_log: {max_records: 0}}; + assertInvalid(validateExample(contract, patch), "rings hold at least one record"); patch.body = {dns_log: {max_records: 63}}; - assertInvalid(validateExample(contract, patch), "rings keep at least 64 records"); + assertValid(validateExample(contract, patch), "engines advertise their own minimum"); const rejected = example("patchRuntimeSettings:400:above_ceiling"); assert.equal(rejected.body.error.code, "invalid_request"); assertValid(validateExample(contract, rejected)); }); +test("runtime settings sections are optional and the log level stands alone", () => { + const levelOnly = example("patchRuntimeSettings:request:level_only"); + assert.deepEqual(levelOnly.body, {log: {level: "info"}}); + assertValid(validateExample(contract, levelOnly)); + const read = example("getRuntimeSettings:200:level_only"); + assert.deepEqual(Object.keys(read.body).sort(), ["log", "observed_at", "source"]); + assertValid(validateExample(contract, read)); + const current = example("getRuntimeSettings:200:current"); + for (const section of ["log", "dns_log", "flows", "geodata"]) delete current.body[section]; + assertValid(validateExample(contract, current), "only observed_at and source are required"); + for (const required of ["observed_at", "source"]) { + const missing = example("getRuntimeSettings:200:current"); + delete missing.body[required]; + assertInvalid(validateExample(contract, missing), `${required} was optional`); + } + for (const [section, value] of [["log", {}], ["dns_log", {}], ["flows", {}], ["recording", {}]]) { + const empty = example("getRuntimeSettings:200:current"); + empty.body[section] = value; + assertInvalid(validateExample(contract, empty), `an empty ${section} passed`); + } + const ring = example("getRuntimeSettings:200:current"); + ring.body.log = {buffered_records: 512}; + assertValid(validateExample(contract, ring), "the replay ring does not need the level"); + ring.body.flows = {retention_seconds: 60}; + assertValid(validateExample(contract, ring), "flow fields are independent"); +}); + +test("recorder state is allowed, mode and active, and flow recording may be on demand", () => { + const settings = example("getRuntimeSettings:200:current"); + const recorder = {allowed: true, mode: "auto", active: false}; + settings.body.recording = {flows: recorder}; + assertValid(validateExample(contract, settings), "a single recorder is enough"); + settings.body.recording = {flows: recorder, logs: recorder, dns_log: recorder, events: {active: true}, grace_remaining_seconds: 42}; + assertValid(validateExample(contract, settings)); + for (const field of ["allowed", "mode", "active"]) { + const partial = {...recorder}; + delete partial[field]; + settings.body.recording = {flows: partial}; + assertInvalid(validateExample(contract, settings), `recorder ${field} was optional`); + } + settings.body.recording = {flows: {...recorder, mode: "on_demand"}}; + assertInvalid(validateExample(contract, settings), "on_demand is a recording value, not a mode"); + const capabilities = example("getCapabilities:200:available"); + const flows = capabilities.body.resources.flows; + assert.equal(flows.recording, "on_demand"); + for (const recording of ["off", "on", "sampled", "on_demand"]) { + flows.recording = recording; + assertValid(validateExample(contract, capabilities)); + } + flows.recording = "auto"; + assertInvalid(validateExample(contract, capabilities), "recording comes from the enum"); +}); + +test("runtime setting bounds are advertised, with an absent minimum meaning 1", () => { + const capabilities = example("getCapabilities:200:available"); + const resources = capabilities.body.resources; + const current = example("getRuntimeSettings:200:current").body; + assert.ok(current.log.buffered_records >= resources.logs.min_buffered_records); + assert.ok(current.dns_log.max_records >= resources.dns_log.min_records); + assert.ok(current.flows.max_flows >= resources.flows.min_flows); + for (const [resource, bound] of [["logs", "min_buffered_records"], ["dns_log", "min_records"], ["flows", "min_flows"]]) { + const response = example("getCapabilities:200:available"); + delete response.body.resources[resource][bound]; + assertValid(validateExample(contract, response), `${resource}.${bound} is optional`); + response.body.resources[resource][bound] = 0; + assertInvalid(validateExample(contract, response), `${resource}.${bound} accepted 0`); + } +}); + test("geodata sources are patched through runtime settings and reported with their status", () => { const resources = example("getCapabilities:200:available").body.resources; assert.equal(resources.geodata.configurable_sources, true); From 0f5369f04d8783631f0ecf5b4690991a49ea42cf Mon Sep 17 00:00:00 2001 From: Zakk Date: Tue, 29 Sep 2026 03:16:18 +1000 Subject: [PATCH 2/5] fix(api): drop the DNS cache cap, advertise log filters and type eBPF attachments entry_capacity may be null for an unbounded cache and loses the 100,000 cap. Log target is nullable and resources.logs.filters lists the supported GET /logs filters. eBPF attachments carry kind interface or cgroup, so cgroup hooks such as dae's are reportable. --- api/discovery.yaml | 12 +++++- api/dns.yaml | 6 ++- api/logs.yaml | 6 +-- api/runtime.yaml | 29 +++++++++++++- source/openapi.yaml | 64 ++++++++++++++++++++++++++---- source/v0.1.0/en/docs/datapath.md | 1 + source/v0.1.0/en/docs/dns-cache.md | 4 +- source/v0.1.0/en/docs/logs.md | 15 ++++--- tools/check-contract.test.mjs | 45 ++++++++++++++++++++- 9 files changed, 159 insertions(+), 23 deletions(-) diff --git a/api/discovery.yaml b/api/discovery.yaml index a45684d..fa0e3a4 100644 --- a/api/discovery.yaml +++ b/api/discovery.yaml @@ -256,6 +256,7 @@ paths: logs: available: true levels: [ trace, debug, info, warn, error ] + filters: [ level, target ] retention_seconds: 60 min_buffered_records: 64 max_buffered_records: 4096 @@ -928,7 +929,7 @@ schemas: available: const: true then: - required: [ levels, retention_seconds, max_buffered_records ] + required: [ levels, filters, retention_seconds, max_buffered_records ] properties: available: type: boolean @@ -938,6 +939,15 @@ schemas: uniqueItems: true items: $ref: ./openapi.yaml#/components/schemas/LogLevel + filters: + type: array + uniqueItems: true + description: The GET /logs query filters this engine applies. level is always listed; a filter not listed returns 422 unsupported_value. + items: + type: string + enum: [ level, target ] + contains: + const: level retention_seconds: type: integer minimum: 1 diff --git a/api/dns.yaml b/api/dns.yaml index 80e6505..323a19e 100644 --- a/api/dns.yaml +++ b/api/dns.yaml @@ -796,8 +796,10 @@ schemas: entries: $ref: ./openapi.yaml#/components/schemas/UInt64 entry_capacity: - $ref: ./openapi.yaml#/components/schemas/UInt64 - description: Effective entry limit after the engine applies its bounds, at most 100,000. + oneOf: + - $ref: ./openapi.yaml#/components/schemas/UInt64 + - type: "null" + description: Effective entry limit after the engine applies its bounds, or null when the cache has no entry limit or the engine cannot report it. DnsLogRecord: type: object required: [ id, observed_at, src, question, status, cached, upstream, route, elapsed_ms, answers ] diff --git a/api/logs.yaml b/api/logs.yaml index fc7f446..01936ff 100644 --- a/api/logs.yaml +++ b/api/logs.yaml @@ -35,7 +35,7 @@ paths: example: info - name: target in: query - description: Case-sensitive literal module prefix; omit for all targets. + description: Case-sensitive literal prefix of LogRecord.target; omit for all targets. Requires target in resources.logs.filters, otherwise 422 unsupported_value. Records whose target is null never match. schema: type: string minLength: 1 @@ -102,9 +102,9 @@ schemas: level: $ref: ./openapi.yaml#/components/schemas/LogLevel target: - type: string + type: [ string, "null" ] minLength: 1 - description: Engine module name, not a network destination. + description: The engine component that emitted the record, such as a module name; not a network destination. null when the engine does not report one. message: type: string minLength: 1 diff --git a/api/runtime.yaml b/api/runtime.yaml index eb89eea..f971537 100644 --- a/api/runtime.yaml +++ b/api/runtime.yaml @@ -458,9 +458,14 @@ paths: epoch: "3" attachments: - name: wan_ingress + kind: interface interface: eth0 direction: ingress state: attached + - name: connect4 + kind: cgroup + cgroup: / + state: attached maps: state: ready conn_state: @@ -874,20 +879,42 @@ schemas: $ref: ./openapi.yaml#/components/schemas/EbpfMaps EbpfAttachment: type: object - required: [ name, interface, direction, state ] + required: [ name, kind, state ] + description: One program attachment. An interface attachment names the interface and direction; a cgroup attachment names the cgroup instead and has neither. properties: name: type: string minLength: 1 + description: Program or hook name. + kind: + type: string + enum: [ interface, cgroup ] interface: type: string minLength: 1 direction: type: string enum: [ ingress, egress ] + cgroup: + type: string + minLength: 1 + description: cgroup v2 path relative to the cgroup2 mount; / is the root cgroup. state: type: string enum: [ attached, detached, error, unknown ] + if: + properties: + kind: + const: interface + then: + required: [ interface, direction ] + properties: + cgroup: false + else: + required: [ cgroup ] + properties: + interface: false + direction: false EbpfMaps: type: object required: [ state, conn_state ] diff --git a/source/openapi.yaml b/source/openapi.yaml index f1bf1eb..33707d6 100644 --- a/source/openapi.yaml +++ b/source/openapi.yaml @@ -298,6 +298,9 @@ paths: - info - warn - error + filters: + - level + - target retention_seconds: 60 min_buffered_records: 64 max_buffered_records: 4096 @@ -2085,9 +2088,14 @@ paths: epoch: '3' attachments: - name: wan_ingress + kind: interface interface: eth0 direction: ingress state: attached + - name: connect4 + kind: cgroup + cgroup: / + state: attached maps: state: ready conn_state: @@ -5164,7 +5172,7 @@ paths: example: info - name: target in: query - description: Case-sensitive literal module prefix; omit for all targets. + description: Case-sensitive literal prefix of LogRecord.target; omit for all targets. Requires target in resources.logs.filters, otherwise 422 unsupported_value. Records whose target is null never match. schema: type: string minLength: 1 @@ -7996,6 +8004,7 @@ components: then: required: - levels + - filters - retention_seconds - max_buffered_records properties: @@ -8007,6 +8016,17 @@ components: uniqueItems: true items: $ref: '#/components/schemas/LogLevel' + filters: + type: array + uniqueItems: true + description: The GET /logs query filters this engine applies. level is always listed; a filter not listed returns 422 unsupported_value. + items: + type: string + enum: + - level + - target + contains: + const: level retention_seconds: type: integer minimum: 1 @@ -9167,13 +9187,19 @@ components: type: object required: - name - - interface - - direction + - kind - state + description: One program attachment. An interface attachment names the interface and direction; a cgroup attachment names the cgroup instead and has neither. properties: name: type: string minLength: 1 + description: Program or hook name. + kind: + type: string + enum: + - interface + - cgroup interface: type: string minLength: 1 @@ -9182,6 +9208,10 @@ components: enum: - ingress - egress + cgroup: + type: string + minLength: 1 + description: cgroup v2 path relative to the cgroup2 mount; / is the root cgroup. state: type: string enum: @@ -9189,6 +9219,22 @@ components: - detached - error - unknown + if: + properties: + kind: + const: interface + then: + required: + - interface + - direction + properties: + cgroup: false + else: + required: + - cgroup + properties: + interface: false + direction: false EbpfMaps: type: object required: @@ -12647,8 +12693,10 @@ components: entries: $ref: '#/components/schemas/UInt64' entry_capacity: - $ref: '#/components/schemas/UInt64' - description: Effective entry limit after the engine applies its bounds, at most 100,000. + oneOf: + - $ref: '#/components/schemas/UInt64' + - type: 'null' + description: Effective entry limit after the engine applies its bounds, or null when the cache has no entry limit or the engine cannot report it. DnsLogRecord: type: object required: @@ -13099,9 +13147,11 @@ components: level: $ref: '#/components/schemas/LogLevel' target: - type: string + type: + - string + - 'null' minLength: 1 - description: Engine module name, not a network destination. + description: The engine component that emitted the record, such as a module name; not a network destination. null when the engine does not report one. message: type: string minLength: 1 diff --git a/source/v0.1.0/en/docs/datapath.md b/source/v0.1.0/en/docs/datapath.md index 799968b..558dae0 100644 --- a/source/v0.1.0/en/docs/datapath.md +++ b/source/v0.1.0/en/docs/datapath.md @@ -38,6 +38,7 @@ usable. `programs: loaded` alone does not mean that traffic is being handled. | ebpf.routing.generation_id | string or null | Generation currently published to eBPF. | | ebpf.routing.epoch | string or null | Engine routing epoch when exposed. | | ebpf.attachments | array | Engine-visible hook attachments. It may be empty when details are unavailable. | +| ebpf.attachments[].kind | string | `interface` (with `interface` and `direction`) or `cgroup` (with `cgroup`, the cgroup v2 path relative to the mount). | | ebpf.maps.state | string | `ready`, `partial`, `error`, or `unknown`. | | ebpf.maps.conn_state | object or null | Conntrack occupancy when the backend exposes it. | | ebpf.health | string | `healthy`, `degraded`, `failed`, or `unknown`. | diff --git a/source/v0.1.0/en/docs/dns-cache.md b/source/v0.1.0/en/docs/dns-cache.md index 4300ae9..ae725d2 100644 --- a/source/v0.1.0/en/docs/dns-cache.md +++ b/source/v0.1.0/en/docs/dns-cache.md @@ -70,12 +70,12 @@ cache class they claim to expose. `usage` describes the whole runtime cache at snapshot time and ignores the filters. Every page of one snapshot repeats it. Both fields are UInt64 -decimal strings: +decimal strings, and `entry_capacity` may be null: | Field | Description | |-------|-------------| | entries | Entries currently retained, including expired entries not yet evicted | -| entry_capacity | Effective entry limit after the engine applies its bounds, at most 100,000 | +| entry_capacity | Effective entry limit after the engine applies its bounds, or null when the cache has no entry limit or the engine cannot report it | The entry count is the cache's only limit; the size of an entry is not bounded. The engine evicts when `entries` reaches `entry_capacity`, so a client showing diff --git a/source/v0.1.0/en/docs/logs.md b/source/v0.1.0/en/docs/logs.md index 9e9a6df..9a0309e 100644 --- a/source/v0.1.0/en/docs/logs.md +++ b/source/v0.1.0/en/docs/logs.md @@ -14,13 +14,16 @@ title: Logs | Parameter | Default | Meaning | |-----------|---------|---------| | level | all advertised levels | Minimum severity: `trace`, `debug`, `info`, `warn`, `error`, in ascending order. | -| target | absent | Case-sensitive literal module prefix. | +| target | absent | Case-sensitive literal prefix of a record's `target`. Requires `target` in `filters`. | | Last-Event-ID | absent | Header containing the last processed opaque cursor. | A level outside the five above returns `400 invalid_request`; one of them that -`levels` does not advertise returns `422 unsupported_value` (see [errors](errors.html#Choosing-the-status)). Capabilities -advertise `levels`, `retention_seconds` and `max_buffered_records`; clients -must not infer them from the engine version. +`levels` does not advertise returns `422 unsupported_value` (see [errors](errors.html#Choosing-the-status)). +`filters` lists the query filters the engine applies; `level` is always +listed, and `target` on an engine that does not list it returns +`422 unsupported_value`. Capabilities advertise `levels`, `filters`, +`retention_seconds` and `max_buffered_records`; clients must not infer them +from the engine version. ## Response @@ -32,7 +35,9 @@ line break. `stream.ready` is the first frame on every connection, including a resume. Its data uses the [events](events.html) payload: `instance_id`, `observed_at`. Each `event: log` frame has an `id` and JSON data containing `ts`, `level`, -`target` (module), `message`, and `fields` (object or null). Heartbeat comments +`target` (the emitting component, or null when the engine does not report +one), `message`, and `fields` (object or null). A `target` filter never +matches a record whose `target` is null. Heartbeat comments arrive at most 15 seconds apart while idle and do not advance the cursor. Sanitize messages and structured fields before buffering. No secrets, diff --git a/tools/check-contract.test.mjs b/tools/check-contract.test.mjs index 250a1fc..80ed1db 100644 --- a/tools/check-contract.test.mjs +++ b/tools/check-contract.test.mjs @@ -1413,7 +1413,7 @@ test("observability resources expose discovery, permissions and examples for eve test("observability capabilities require usable bounds only when available", () => { for (const [resource, fields] of [ - ["logs", ["levels", "retention_seconds", "max_buffered_records"]], + ["logs", ["levels", "filters", "retention_seconds", "max_buffered_records"]], ["providers", ["can_refresh", "can_manage", "max_page_size"]], ["rules", ["max_rules"]], ["geodata", ["can_update", "assets"]], @@ -1470,7 +1470,8 @@ test("log payloads and filters preserve typed records and the shared cursor erro const schema = { $ref: bindings.log }; record.fields = null; assertValid(contract.validate(schema, record)); - for (const [field, invalid] of [["ts", "yesterday"], ["level", "fatal"], ["fields", []]]) { + assertValid(contract.validate(schema, { ...record, target: null }), "an engine without targets reports null"); + for (const [field, invalid] of [["ts", "yesterday"], ["level", "fatal"], ["fields", []], ["target", ""]]) { const changed = { ...record, [field]: invalid }; assertInvalid(contract.validate(schema, changed)); } @@ -1880,3 +1881,43 @@ test("only operations with replay semantics take Idempotency-Key", () => { assert.equal(replays(spec.paths["/api/v1/operations/reload"].post), true); assert.equal(replays(spec.paths["/api/v1/groups/{group_id}"].patch), true); }); + +test("log filters are advertised and always include level", () => { + const capabilities = example("getCapabilities:200:available"); + const logs = capabilities.body.resources.logs; + assert.deepEqual(logs.filters, ["level", "target"]); + logs.filters = ["level"]; + assertValid(validateExample(contract, capabilities), "an engine without targets lists level alone"); + for (const invalid of [["target"], [], ["level", "module"], ["level", "level"]]) { + logs.filters = invalid; + assertInvalid(validateExample(contract, capabilities), `filters ${JSON.stringify(invalid)} passed`); + } +}); + +test("DNS cache capacity may be unbounded and is not capped by the contract", () => { + const usage = { $ref: "#/components/schemas/DnsCacheUsage" }; + assertValid(contract.validate(usage, { entries: "5", entry_capacity: null }), "null capacity"); + assertValid(contract.validate(usage, { entries: "5", entry_capacity: "1000000" }), "no fixed cap"); + assertInvalid(contract.validate(usage, { entries: "5" }), "capacity may be null but not absent"); +}); + +test("eBPF attachments are interface or cgroup attachments", () => { + const response = example("getDatapath:200:active"); + const attachments = response.body.ebpf.attachments; + assert.deepEqual(attachments.map((attachment) => attachment.kind), ["interface", "cgroup"]); + assertValid(validateExample(contract, response)); + const schema = { $ref: "#/components/schemas/EbpfAttachment" }; + const [iface, cgroup] = attachments; + for (const [label, value] of [ + ["kind is required", (({ kind, ...rest }) => rest)(iface)], + ["an interface attachment needs its direction", (({ direction, ...rest }) => rest)(iface)], + ["an interface attachment needs its interface", (({ interface: _, ...rest }) => rest)(iface)], + ["an interface attachment has no cgroup", { ...iface, cgroup: "/" }], + ["a cgroup attachment needs its path", (({ cgroup: _, ...rest }) => rest)(cgroup)], + ["a cgroup attachment has no interface", { ...cgroup, interface: "eth0" }], + ["a cgroup attachment has no direction", { ...cgroup, direction: "egress" }], + ["kind comes from the enum", { ...cgroup, kind: "xdp" }], + ]) { + assertInvalid(contract.validate(schema, value), label); + } +}); From 32ebfd7019a54fad10fc994c4fa19a20fd1535ca Mon Sep 17 00:00:00 2001 From: Zakk Date: Tue, 29 Sep 2026 03:17:45 +1000 Subject: [PATCH 3/5] fix(api): open datapath step actions to engine-defined values action keeps pass, redirect, hold and drop as the core set and accepts other lowercase engine values, which clients show as they are. honk's NFQUEUE actions, UDP decision token and Clash-mode override move to the honk notes. --- api/flow-steps.yaml | 6 ++++-- source/openapi.yaml | 13 ++++--------- source/v0.1.0/en/docs/flows.md | 10 ++++++---- source/v0.1.0/en/docs/honk-mapping.md | 13 +++++++++++++ tools/check-contract.test.mjs | 12 ++++++++++++ 5 files changed, 39 insertions(+), 15 deletions(-) diff --git a/api/flow-steps.yaml b/api/flow-steps.yaml index a412cb3..0764124 100644 --- a/api/flow-steps.yaml +++ b/api/flow-steps.yaml @@ -211,7 +211,9 @@ schemas: enum: [ kernel, userspace ] action: type: string - enum: [ pass, redirect, hold, arm_direct, activate_direct, activate_proxy, drop ] + pattern: ^[a-z][a-z0-9_]*$ + maxLength: 64 + description: pass lets the packet continue without the proxy, redirect hands it to the userspace proxy, hold keeps it until a pending decision completes, and drop discards it. Any other value is an engine-defined step documented with that engine; clients show an unknown value as it is. reason: type: string minLength: 1 @@ -372,7 +374,7 @@ schemas: mode_override: type: string enum: [ none, direct, global, unknown ] - description: Clash-mode override observed for this outbound attempt, separate from the configured dial mode. This field does not expose a native runtime-mode setting. none means no override was applied; unknown means the recorder could not determine it. + description: Engine mode override observed for this outbound attempt, such as honk's Clash mode, separate from the configured dial mode. This field does not expose a native runtime-mode setting. none means no override was applied; unknown means the recorder could not determine it. selection_path: type: array items: diff --git a/source/openapi.yaml b/source/openapi.yaml index 33707d6..de4216b 100644 --- a/source/openapi.yaml +++ b/source/openapi.yaml @@ -11637,14 +11637,9 @@ components: - userspace action: type: string - enum: - - pass - - redirect - - hold - - arm_direct - - activate_direct - - activate_proxy - - drop + pattern: ^[a-z][a-z0-9_]*$ + maxLength: 64 + description: pass lets the packet continue without the proxy, redirect hands it to the userspace proxy, hold keeps it until a pending decision completes, and drop discards it. Any other value is an engine-defined step documented with that engine; clients show an unknown value as it is. reason: type: string minLength: 1 @@ -11935,7 +11930,7 @@ components: - direct - global - unknown - description: Clash-mode override observed for this outbound attempt, separate from the configured dial mode. This field does not expose a native runtime-mode setting. none means no override was applied; unknown means the recorder could not determine it. + description: Engine mode override observed for this outbound attempt, such as honk's Clash mode, separate from the configured dial mode. This field does not expose a native runtime-mode setting. none means no override was applied; unknown means the recorder could not determine it. selection_path: type: array items: diff --git a/source/v0.1.0/en/docs/flows.md b/source/v0.1.0/en/docs/flows.md index 81bfbe7..1779f22 100644 --- a/source/v0.1.0/en/docs/flows.md +++ b/source/v0.1.0/en/docs/flows.md @@ -21,7 +21,7 @@ resolve DNS, probe nodes, or dial anything. - `id` is an opaque, instance-scoped flow ID, allocated at the first decision hook, before sniffing, DNS, or dialing can fail. Together with `instance_id` it is never reused. Neither a five-tuple, PID, socket cookie, outbound index, - nor honk's UDP decision token alone is an API identity. + nor an engine-internal decision token alone is an API identity. - Kernel and userspace observations join only through an incarnation-safe handoff. If correlation cannot be proved, return separate partial records; never join by IP, name, five-tuple, or a nearby timestamp alone. @@ -33,7 +33,7 @@ resolve DNS, probe nodes, or dial anything. relabel old steps with the current generation, group name, or selected leaf. `rule_id` is meaningful only with that generation and rule chain. - A new engine process gets a new `instance_id`. API IDs are not BPF map ABI; - do not change the persisted UDP token allocator to implement them. + implementing them must not change identifiers the datapath persists. ## GET /api/v1/flows @@ -177,8 +177,10 @@ configured UDP DNS carried over TCP through a proxy). A kernel `drop` used to complete proxy handoff is not a policy `block`. Likewise direct activation ends userspace setup, not the native connection. -Record NFQUEUE hold/arm/verdict/publication order in `datapath` steps without -exposing mutable verdict tokens. Static port-53 interception and early +Record the order of datapath decisions in `datapath` steps without exposing +mutable verdict tokens. A step's `action` is `pass`, `redirect`, `hold`, +`drop`, or an engine-defined value; clients show an unknown value as it is. +honk's NFQUEUE steps are in the [honk notes](honk-mapping.html#Flow-steps-in-honk). Static port-53 interception and early bypasses are enforcement reasons, not invented configured rule matches. `server_addr` is the physical proxy server/socket peer if observed; `dial_ip` diff --git a/source/v0.1.0/en/docs/honk-mapping.md b/source/v0.1.0/en/docs/honk-mapping.md index a816e89..e227397 100644 --- a/source/v0.1.0/en/docs/honk-mapping.md +++ b/source/v0.1.0/en/docs/honk-mapping.md @@ -246,3 +246,16 @@ recorders on the current `feat/native-api` branch. - `grace_remaining_seconds` counts the attachment grace only, not the flow-demand grace. Event capture runs while a client is attached or any permitted recorder is pinned on. + +## Flow steps in honk + +This section is not part of the contract. It records honk-specific detail +behind the [flow record](flows.html) rules. + +- honk hands UDP decisions between the kernel and userspace through NFQUEUE. + Its `datapath` steps record the hold, arm, verdict and publication order, + and may use the engine-defined actions `arm_direct`, `activate_direct` and + `activate_proxy` alongside the core values. +- honk's UDP decision token is not a flow ID, and the persisted UDP token + allocator is not changed to produce flow IDs. +- `mode_override` reports honk's Clash-mode override (`direct` or `global`). diff --git a/tools/check-contract.test.mjs b/tools/check-contract.test.mjs index 80ed1db..ce3b36f 100644 --- a/tools/check-contract.test.mjs +++ b/tools/check-contract.test.mjs @@ -1921,3 +1921,15 @@ test("eBPF attachments are interface or cgroup attachments", () => { assertInvalid(contract.validate(schema, value), label); } }); + +test("datapath step actions have a core set and accept engine-defined values", () => { + const schema = { $ref: "#/components/schemas/DatapathStepData" }; + const data = { plane: "kernel", action: "pass", reason: "bypass", error: null }; + for (const action of ["pass", "redirect", "hold", "drop", "arm_direct", "sk_assign"]) { + assertValid(contract.validate(schema, { ...data, action }), action); + } + for (const action of ["", "Redirect", "x y", "1drop", "a".repeat(65)]) { + assertInvalid(contract.validate(schema, { ...data, action }), `action ${JSON.stringify(action)} passed`); + } + assertInvalid(contract.validate(schema, (({ action, ...rest }) => rest)(data)), "action is required"); +}); From 8545bb127285417f43d1fa350a82bdae648aa568 Mon Sep 17 00:00:00 2001 From: Zakk Date: Tue, 29 Sep 2026 03:39:40 +1000 Subject: [PATCH 4/5] fix(api): settle recorder policy, grace scope and partial eBPF attachments - attachments gain an `other` kind with a free `hook`; the list may be partial - `flows.recording` reports policy (off/on/on_demand), not current capture - `grace_remaining_seconds` is the engine's attachment grace, optional - RecorderMode (PATCH) and RecorderState.mode (GET string) documented apart - DNS eviction and fill ratio only with a positive capacity - flow capability retention is at least 1 second - GET /logs declares 422; enumerations list filters and min_* fields - flow-steps drops the honk Clash-mode example; honk notes list the actions honk emits today --- api/discovery.yaml | 6 +- api/flow-steps.yaml | 2 +- api/logs.yaml | 23 ++++++ api/runtime.yaml | 56 ++++++++++---- api/settings.yaml | 5 +- source/openapi.yaml | 98 +++++++++++++++++++------ source/v0.1.0/en/docs/capabilities.md | 8 +- source/v0.1.0/en/docs/datapath.md | 4 +- source/v0.1.0/en/docs/dns-cache.md | 9 ++- source/v0.1.0/en/docs/flows.md | 4 +- source/v0.1.0/en/docs/honk-mapping.md | 7 +- source/v0.1.0/en/docs/logs.md | 3 +- source/v0.1.0/en/docs/runtime-status.md | 22 ++++-- tools/check-contract.test.mjs | 15 +++- 14 files changed, 192 insertions(+), 70 deletions(-) diff --git a/api/discovery.yaml b/api/discovery.yaml index fa0e3a4..37bff88 100644 --- a/api/discovery.yaml +++ b/api/discovery.yaml @@ -814,7 +814,7 @@ schemas: recording: type: string enum: [ off, on, sampled, on_demand ] - description: How the engine records flows right now. off records none; on records every flow in scopes; sampled records a subset; on_demand records only while clients read or follow flows, which is recording.flows.mode auto in GET /runtime/settings. + description: The engine's flow recording policy, not whether the recorder is capturing now. off records none, because the configuration does not permit the flow recorder or its mode is off; on records every flow in scopes (mode on); sampled records a subset; on_demand is mode auto, recording while clients create demand, whether or not any client does now. recording.flows.active in GET /runtime/settings reports whether the recorder is capturing. scopes: type: array uniqueItems: true @@ -833,8 +833,8 @@ schemas: minimum: 1 retention_seconds: type: integer - minimum: 0 - description: The longest terminal-flow retention the engine supports, which is the most a runtime-settings PATCH may set, not the current value. The current value is flows.retention_seconds in GET /runtime/settings. + minimum: 1 + description: The longest terminal-flow retention the engine supports, which is the most a runtime-settings PATCH may set, not the current value. The smallest a PATCH may set is always 1. The current value is flows.retention_seconds in GET /runtime/settings. snapshot_ttl_seconds: type: integer minimum: 1 diff --git a/api/flow-steps.yaml b/api/flow-steps.yaml index 0764124..26eee14 100644 --- a/api/flow-steps.yaml +++ b/api/flow-steps.yaml @@ -374,7 +374,7 @@ schemas: mode_override: type: string enum: [ none, direct, global, unknown ] - description: Engine mode override observed for this outbound attempt, such as honk's Clash mode, separate from the configured dial mode. This field does not expose a native runtime-mode setting. none means no override was applied; unknown means the recorder could not determine it. + description: Engine mode override observed for this outbound attempt, separate from the configured dial mode. This field does not expose a native runtime-mode setting. none means no override was applied; unknown means the recorder could not determine it. selection_path: type: array items: diff --git a/api/logs.yaml b/api/logs.yaml index 01936ff..ab3112e 100644 --- a/api/logs.yaml +++ b/api/logs.yaml @@ -85,6 +85,29 @@ paths: $ref: ./openapi.yaml#/components/responses/EventCursorExpired "413": $ref: ./openapi.yaml#/components/responses/TooLarge + "422": + description: level is a LogLevel member not in resources.logs.levels, or target is set while resources.logs.filters omits target (unsupported_value). + headers: + Cache-Control: + $ref: ./openapi.yaml#/components/headers/NoStore + X-Content-Type-Options: + $ref: ./openapi.yaml#/components/headers/NoSniff + content: + application/json: + schema: + $ref: ./openapi.yaml#/components/schemas/ErrorResponse + examples: + unadvertised_filter: + value: + error: + code: unsupported_value + message: target is not an advertised log filter. + details: null + request_id: request-5 + x-headers: + Content-Type: application/json + Cache-Control: no-store + X-Content-Type-Options: nosniff "429": $ref: ./openapi.yaml#/components/responses/RateLimited "503": diff --git a/api/runtime.yaml b/api/runtime.yaml index f971537..d25aeeb 100644 --- a/api/runtime.yaml +++ b/api/runtime.yaml @@ -466,6 +466,10 @@ paths: kind: cgroup cgroup: / state: attached + - name: sk_msg_verdict + kind: other + hook: sk_msg verdict on the socket map + state: attached maps: state: ready conn_state: @@ -873,6 +877,7 @@ schemas: properties: attachments: type: array + description: The attachments the engine checks. The list may be partial; an engine need not report every program it attached. items: $ref: ./openapi.yaml#/components/schemas/EbpfAttachment maps: @@ -880,7 +885,7 @@ schemas: EbpfAttachment: type: object required: [ name, kind, state ] - description: One program attachment. An interface attachment names the interface and direction; a cgroup attachment names the cgroup instead and has neither. + description: One program attachment. An interface attachment names the interface and direction, a cgroup attachment names the cgroup, and any other attachment, such as a sockmap verdict or a tracing program, describes its hook in `hook`. properties: name: type: string @@ -888,7 +893,7 @@ schemas: description: Program or hook name. kind: type: string - enum: [ interface, cgroup ] + enum: [ interface, cgroup, other ] interface: type: string minLength: 1 @@ -899,22 +904,43 @@ schemas: type: string minLength: 1 description: cgroup v2 path relative to the cgroup2 mount; / is the root cgroup. + hook: + type: string + minLength: 1 + description: Where an `other` attachment is attached, in the engine's words, such as the sockmap a verdict program serves or the kernel function a tracing program hooks. state: type: string enum: [ attached, detached, error, unknown ] - if: - properties: - kind: - const: interface - then: - required: [ interface, direction ] - properties: - cgroup: false - else: - required: [ cgroup ] - properties: - interface: false - direction: false + allOf: + - if: + properties: + kind: + const: interface + then: + required: [ interface, direction ] + properties: + cgroup: false + hook: false + - if: + properties: + kind: + const: cgroup + then: + required: [ cgroup ] + properties: + interface: false + direction: false + hook: false + - if: + properties: + kind: + const: other + then: + required: [ hook ] + properties: + interface: false + direction: false + cgroup: false EbpfMaps: type: object required: [ state, conn_state ] diff --git a/api/settings.yaml b/api/settings.yaml index a45626f..c5bc76b 100644 --- a/api/settings.yaml +++ b/api/settings.yaml @@ -312,7 +312,7 @@ schemas: type: string enum: [ log.level, log.buffered_records, dns_log.max_records, flows.max_flows, flows.retention_seconds, record_flows, record_logs, record_dns_log, geodata ] RecorderMode: - description: true pins a permitted recorder on, false forces it off, and auto (the startup default) lets the engine record on demand, while clients read or follow what the recorder captures. What counts as demand, and how long it lasts, is engine-defined. + description: The value a PATCH sets for a recorder. true pins a permitted recorder on, false forces it off, and "auto" (the startup default) lets the engine record on demand. What counts as demand, and how long it lasts, is engine-defined. GET reports the result as the string RecorderState.mode, not as this value. oneOf: - type: boolean - type: string @@ -327,6 +327,7 @@ schemas: mode: type: string enum: [ auto, "on", "off" ] + description: The recorder's mode as reported by GET, always a string. "on" follows a PATCH of true, "off" a PATCH of false, and auto a PATCH of "auto" or no PATCH at all. active: type: boolean description: The recorder is capturing right now. @@ -396,7 +397,7 @@ schemas: description: Event capture is running. grace_remaining_seconds: $ref: ./openapi.yaml#/components/schemas/SafeUInt - description: Seconds left before automatic recorders stop after the last client left, 0 while a client is attached or nothing is recording. Absent when the engine keeps no such grace. + description: Seconds left in the engine's attachment grace after the last client left, 0 while a client is attached or once the grace has run out. Which recorders follow this grace is engine-defined, and a recorder may keep its own demand timer that this value does not count. Absent when the engine keeps no attachment grace. geodata: $ref: ./openapi.yaml#/components/schemas/GeoDataSettings description: Geodata download sources and automatic updates. Present when resources.geodata.configurable_sources is true. URLs are returned as written, with only listener-secret values masked, to every admitted caller. diff --git a/source/openapi.yaml b/source/openapi.yaml index de4216b..6cf6631 100644 --- a/source/openapi.yaml +++ b/source/openapi.yaml @@ -2096,6 +2096,10 @@ paths: kind: cgroup cgroup: / state: attached + - name: sk_msg_verdict + kind: other + hook: sk_msg verdict on the socket map + state: attached maps: state: ready conn_state: @@ -5221,6 +5225,29 @@ paths: $ref: '#/components/responses/EventCursorExpired' '413': $ref: '#/components/responses/TooLarge' + '422': + description: level is a LogLevel member not in resources.logs.levels, or target is set while resources.logs.filters omits target (unsupported_value). + headers: + Cache-Control: + $ref: '#/components/headers/NoStore' + X-Content-Type-Options: + $ref: '#/components/headers/NoSniff' + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + examples: + unadvertised_filter: + value: + error: + code: unsupported_value + message: target is not an advertised log filter. + details: null + request_id: request-5 + x-headers: + Content-Type: application/json + Cache-Control: no-store + X-Content-Type-Options: nosniff '429': $ref: '#/components/responses/RateLimited' '503': @@ -7869,7 +7896,7 @@ components: - 'on' - sampled - on_demand - description: How the engine records flows right now. off records none; on records every flow in scopes; sampled records a subset; on_demand records only while clients read or follow flows, which is recording.flows.mode auto in GET /runtime/settings. + description: The engine's flow recording policy, not whether the recorder is capturing now. off records none, because the configuration does not permit the flow recorder or its mode is off; on records every flow in scopes (mode on); sampled records a subset; on_demand is mode auto, recording while clients create demand, whether or not any client does now. recording.flows.active in GET /runtime/settings reports whether the recorder is capturing. scopes: type: array uniqueItems: true @@ -7888,8 +7915,8 @@ components: minimum: 1 retention_seconds: type: integer - minimum: 0 - description: The longest terminal-flow retention the engine supports, which is the most a runtime-settings PATCH may set, not the current value. The current value is flows.retention_seconds in GET /runtime/settings. + minimum: 1 + description: The longest terminal-flow retention the engine supports, which is the most a runtime-settings PATCH may set, not the current value. The smallest a PATCH may set is always 1. The current value is flows.retention_seconds in GET /runtime/settings. snapshot_ttl_seconds: type: integer minimum: 1 @@ -9179,6 +9206,7 @@ components: properties: attachments: type: array + description: The attachments the engine checks. The list may be partial; an engine need not report every program it attached. items: $ref: '#/components/schemas/EbpfAttachment' maps: @@ -9189,7 +9217,7 @@ components: - name - kind - state - description: One program attachment. An interface attachment names the interface and direction; a cgroup attachment names the cgroup instead and has neither. + description: One program attachment. An interface attachment names the interface and direction, a cgroup attachment names the cgroup, and any other attachment, such as a sockmap verdict or a tracing program, describes its hook in `hook`. properties: name: type: string @@ -9200,6 +9228,7 @@ components: enum: - interface - cgroup + - other interface: type: string minLength: 1 @@ -9212,6 +9241,10 @@ components: type: string minLength: 1 description: cgroup v2 path relative to the cgroup2 mount; / is the root cgroup. + hook: + type: string + minLength: 1 + description: Where an `other` attachment is attached, in the engine's words, such as the sockmap a verdict program serves or the kernel function a tracing program hooks. state: type: string enum: @@ -9219,22 +9252,40 @@ components: - detached - error - unknown - if: - properties: - kind: - const: interface - then: - required: - - interface - - direction - properties: - cgroup: false - else: - required: - - cgroup - properties: - interface: false - direction: false + allOf: + - if: + properties: + kind: + const: interface + then: + required: + - interface + - direction + properties: + cgroup: false + hook: false + - if: + properties: + kind: + const: cgroup + then: + required: + - cgroup + properties: + interface: false + direction: false + hook: false + - if: + properties: + kind: + const: other + then: + required: + - hook + properties: + interface: false + direction: false + cgroup: false EbpfMaps: type: object required: @@ -11930,7 +11981,7 @@ components: - direct - global - unknown - description: Engine mode override observed for this outbound attempt, such as honk's Clash mode, separate from the configured dial mode. This field does not expose a native runtime-mode setting. none means no override was applied; unknown means the recorder could not determine it. + description: Engine mode override observed for this outbound attempt, separate from the configured dial mode. This field does not expose a native runtime-mode setting. none means no override was applied; unknown means the recorder could not determine it. selection_path: type: array items: @@ -13241,7 +13292,7 @@ components: description: Event capture is running. grace_remaining_seconds: $ref: '#/components/schemas/SafeUInt' - description: Seconds left before automatic recorders stop after the last client left, 0 while a client is attached or nothing is recording. Absent when the engine keeps no such grace. + description: Seconds left in the engine's attachment grace after the last client left, 0 while a client is attached or once the grace has run out. Which recorders follow this grace is engine-defined, and a recorder may keep its own demand timer that this value does not count. Absent when the engine keeps no attachment grace. geodata: $ref: '#/components/schemas/GeoDataSettings' description: Geodata download sources and automatic updates. Present when resources.geodata.configurable_sources is true. URLs are returned as written, with only listener-secret values masked, to every admitted caller. @@ -13288,7 +13339,7 @@ components: geodata: $ref: '#/components/schemas/GeoDataSettingsPatch' RecorderMode: - description: true pins a permitted recorder on, false forces it off, and auto (the startup default) lets the engine record on demand, while clients read or follow what the recorder captures. What counts as demand, and how long it lasts, is engine-defined. + description: The value a PATCH sets for a recorder. true pins a permitted recorder on, false forces it off, and "auto" (the startup default) lets the engine record on demand. What counts as demand, and how long it lasts, is engine-defined. GET reports the result as the string RecorderState.mode, not as this value. oneOf: - type: boolean - type: string @@ -13310,6 +13361,7 @@ components: - auto - 'on' - 'off' + description: The recorder's mode as reported by GET, always a string. "on" follows a PATCH of true, "off" a PATCH of false, and auto a PATCH of "auto" or no PATCH at all. active: type: boolean description: The recorder is capturing right now. diff --git a/source/v0.1.0/en/docs/capabilities.md b/source/v0.1.0/en/docs/capabilities.md index 27b1800..54b834d 100644 --- a/source/v0.1.0/en/docs/capabilities.md +++ b/source/v0.1.0/en/docs/capabilities.md @@ -54,8 +54,8 @@ entire selected set, including non-closable entries. See [Closing connections](connections.html#Closing) for permissions, ownership, filters, and errors. -`logs` advertises supported `levels`, `retention_seconds` and -`max_buffered_records`. Its bounded SSE feed carries sanitized log records, +`logs` advertises supported `levels`, the query `filters`, `retention_seconds`, +`max_buffered_records` and the optional `min_buffered_records`. Its bounded SSE feed carries sanitized log records, separately from invalidation events. `providers` advertises `can_refresh`, `can_manage` and `max_page_size` (1–1000); refresh requires `control` and the operation resource, and @@ -78,7 +78,9 @@ are positive safe integers. See [Logs](logs.html), [Providers](providers.html), when true, `fields` lists which settings the PATCH accepts on this backend. `dns_log.available` declares the ring of recent client resolutions; when -true, `max_records` and `max_page_size` are required positive safe integers. +true, `max_records` and `max_page_size` are required positive safe integers, +and the optional `min_records` is the smallest ring a runtime-settings PATCH +may set. `dns_rules.available` declares `GET /api/v1/dns/rules`; when true, `max_rules` bounds each of its two lists, including the fallback entry. See diff --git a/source/v0.1.0/en/docs/datapath.md b/source/v0.1.0/en/docs/datapath.md index 558dae0..26125c9 100644 --- a/source/v0.1.0/en/docs/datapath.md +++ b/source/v0.1.0/en/docs/datapath.md @@ -37,8 +37,8 @@ usable. `programs: loaded` alone does not mean that traffic is being handled. | ebpf.routing.state | string | `published`, `not_published`, `error`, or `unknown`. | | ebpf.routing.generation_id | string or null | Generation currently published to eBPF. | | ebpf.routing.epoch | string or null | Engine routing epoch when exposed. | -| ebpf.attachments | array | Engine-visible hook attachments. It may be empty when details are unavailable. | -| ebpf.attachments[].kind | string | `interface` (with `interface` and `direction`) or `cgroup` (with `cgroup`, the cgroup v2 path relative to the mount). | +| ebpf.attachments | array | The attachments the engine checks. The list may be partial, and it may be empty when details are unavailable. | +| ebpf.attachments[].kind | string | `interface` (with `interface` and `direction`), `cgroup` (with `cgroup`, the cgroup v2 path relative to the mount), or `other` (with `hook`, the engine's description of the attach point, such as a sockmap verdict or a tracing hook). | | ebpf.maps.state | string | `ready`, `partial`, `error`, or `unknown`. | | ebpf.maps.conn_state | object or null | Conntrack occupancy when the backend exposes it. | | ebpf.health | string | `healthy`, `degraded`, `failed`, or `unknown`. | diff --git a/source/v0.1.0/en/docs/dns-cache.md b/source/v0.1.0/en/docs/dns-cache.md index ae725d2..8cd0b88 100644 --- a/source/v0.1.0/en/docs/dns-cache.md +++ b/source/v0.1.0/en/docs/dns-cache.md @@ -69,8 +69,8 @@ runtime in-memory cache are included. Implementations must not silently omit a cache class they claim to expose. `usage` describes the whole runtime cache at snapshot time and ignores the -filters. Every page of one snapshot repeats it. Both fields are UInt64 -decimal strings, and `entry_capacity` may be null: +filters. Every page of one snapshot repeats it. `usage.entries` is a UInt64 +decimal string; `usage.entry_capacity` is a UInt64 decimal string or null: | Field | Description | |-------|-------------| @@ -78,8 +78,9 @@ decimal strings, and `entry_capacity` may be null: | entry_capacity | Effective entry limit after the engine applies its bounds, or null when the cache has no entry limit or the engine cannot report it | The entry count is the cache's only limit; the size of an entry is not bounded. -The engine evicts when `entries` reaches `entry_capacity`, so a client showing -how full the cache is should use that ratio. `entries` may exceed +When `entry_capacity` is positive, the engine evicts at that limit; clients can +use `entries / entry_capacity` to show fullness. A null or zero capacity does +not support that ratio. `entries` may exceed `total`, which counts only entries that match the filters and the listing's expiry rule. When `usage` is absent, the client has no capacity information and must not infer one from `total`. diff --git a/source/v0.1.0/en/docs/flows.md b/source/v0.1.0/en/docs/flows.md index 1779f22..9f16b7a 100644 --- a/source/v0.1.0/en/docs/flows.md +++ b/source/v0.1.0/en/docs/flows.md @@ -327,8 +327,8 @@ userspace list. Kernel bypasses (multicast, own traffic, local services, closed admission) must be declared even where no connection exists. Capabilities advertise `recording` (`off`, `on`, `sampled`, `on_demand`), -`scopes`, `max_flows`, `max_steps_per_flow`, `retention_seconds`, -`snapshot_ttl_seconds`, and `max_page_size`. `on_demand` records only while +`scopes`, the optional `min_flows`, `max_flows`, `max_steps_per_flow`, +`retention_seconds`, `snapshot_ttl_seconds`, and `max_page_size`. `on_demand` records only while clients read or follow flows (see [runtime settings](runtime-status.html#GET-api-v1-runtime-settings)). `max_flows` and `retention_seconds` are the engine's limits, the most a diff --git a/source/v0.1.0/en/docs/honk-mapping.md b/source/v0.1.0/en/docs/honk-mapping.md index e227397..ec9138a 100644 --- a/source/v0.1.0/en/docs/honk-mapping.md +++ b/source/v0.1.0/en/docs/honk-mapping.md @@ -253,9 +253,10 @@ This section is not part of the contract. It records honk-specific detail behind the [flow record](flows.html) rules. - honk hands UDP decisions between the kernel and userspace through NFQUEUE. - Its `datapath` steps record the hold, arm, verdict and publication order, - and may use the engine-defined actions `arm_direct`, `activate_direct` and - `activate_proxy` alongside the core values. + Its `datapath` steps currently use the core action `drop` and the + engine-defined actions `activate_direct` and `activate_proxy` + (`control/connection/udp.rs`). They record the NFQUEUE activation, not each + hold, arm, verdict and publication step. - honk's UDP decision token is not a flow ID, and the persisted UDP token allocator is not changed to produce flow IDs. - `mode_override` reports honk's Clash-mode override (`direct` or `global`). diff --git a/source/v0.1.0/en/docs/logs.md b/source/v0.1.0/en/docs/logs.md index 9a0309e..8df36d1 100644 --- a/source/v0.1.0/en/docs/logs.md +++ b/source/v0.1.0/en/docs/logs.md @@ -22,7 +22,8 @@ A level outside the five above returns `400 invalid_request`; one of them that `filters` lists the query filters the engine applies; `level` is always listed, and `target` on an engine that does not list it returns `422 unsupported_value`. Capabilities advertise `levels`, `filters`, -`retention_seconds` and `max_buffered_records`; clients must not infer them +`retention_seconds`, `max_buffered_records` and the optional +`min_buffered_records`; clients must not infer them from the engine version. ## Response diff --git a/source/v0.1.0/en/docs/runtime-status.md b/source/v0.1.0/en/docs/runtime-status.md index 0398010..0aa1957 100644 --- a/source/v0.1.0/en/docs/runtime-status.md +++ b/source/v0.1.0/en/docs/runtime-status.md @@ -220,17 +220,22 @@ setting is the log level reports `log.level` alone: | flows.retention_seconds | 1 to `flows.retention_seconds` | Maximum age after termination; capacity pressure may evict a flow sooner. | | source | | `config` while every value comes from the activated configuration, `runtime` once a PATCH overrode one. | | geodata | | Geodata download URLs, download route, automatic updates and checksum verification, with their own read-only `source` for the URLs; URLs are returned as written, with only listener-secret values masked. Present when `resources.geodata.configurable_sources` is true. See [Geodata](geodata.html#Configure-the-sources). | -| recording | | Read-only recorder state. `flows`, `logs` and `dns_log` each report `allowed`, `mode` (`auto`, `on`, `off`) and `active`, and appear when `record_flows`, `record_logs` and `record_dns_log` are listed in `fields`. `events.active` reports event capture. `grace_remaining_seconds`, when present, counts down before automatic recorders stop after the last client left. | +| recording | | Read-only recorder state. `flows`, `logs` and `dns_log` each report `allowed`, `mode` (`auto`, `on`, `off`) and `active`, and appear when `record_flows`, `record_logs` and `record_dns_log` are listed in `fields`. `events.active` reports event capture. `grace_remaining_seconds`, when present, counts the engine's attachment grace after the last client left; which recorders follow it is engine-defined, and an engine without such a grace omits it. | The capability bounds are the engine's own limits, not the current values. An absent minimum means 1. -In `auto` mode a recorder captures on demand: while clients read or follow -what it records. What counts as demand and how long it lasts after the last -client left is engine-defined; `resources.flows.recording` reports -`on_demand` while the flow recorder is in `auto` mode. Recording starts on -demand, so the first history a panel reads may be empty. honk's rules are in -the [honk notes](honk-mapping.html#Runtime-settings-in-honk). +In `auto` mode, a recorder captures according to client demand. The requests +and streams that create demand, and any grace period after clients leave, are +engine-defined. Recording starts on demand, so the first history a panel reads +may be empty. honk's rules are in the +[honk notes](honk-mapping.html#Runtime-settings-in-honk). + +`resources.flows.recording` reports the flow recorder's policy, and +`recording.flows.active` reports whether it is capturing now. The policy is +`off` when the configuration does not permit the recorder or its mode is `off`, +`on` when its mode is `on`, and `on_demand` when its mode is `auto`, whether or +not a client creates demand at the moment. ## PATCH /api/v1/runtime/settings @@ -238,7 +243,8 @@ Requires `control`. Only the fields listed in `resources.runtime_settings.fields may appear; the body merges, an absent field keeps its value. `record_flows`, `record_logs` and `record_dns_log` take `true` (keep the recorder on without clients), `false` (force it off) or `"auto"` (the startup default: record on -demand); pinning a recorder the configuration forbids rejects the whole patch. +demand). GET reports the result in `recording.*.mode` as the string `on`, `off` +or `auto`, never as a boolean. Pinning a recorder the configuration forbids rejects the whole patch. `{"log": {"level": "info"}}` is a complete request. {% api_request patchRuntimeSettings debug %} diff --git a/tools/check-contract.test.mjs b/tools/check-contract.test.mjs index ce3b36f..cef89f0 100644 --- a/tools/check-contract.test.mjs +++ b/tools/check-contract.test.mjs @@ -297,6 +297,9 @@ test("recorder state is allowed, mode and active, and flow recording may be on d } flows.recording = "auto"; assertInvalid(validateExample(contract, capabilities), "recording comes from the enum"); + flows.recording = "on_demand"; + flows.retention_seconds = 0; + assertInvalid(validateExample(contract, capabilities), "flow retention is at least 1 second"); }); test("runtime setting bounds are advertised, with an absent minimum meaning 1", () => { @@ -1484,6 +1487,7 @@ test("log payloads and filters preserve typed records and the shared cursor erro const expired = example("streamLogs:409:event_cursor_expired"); assert.deepEqual(expired.body, example("streamEvents:409:event_cursor_expired").body); assert.equal(expired.mediaType, "application/json"); + assert.ok(spec.paths["/api/v1/logs"].get.responses["422"], "an unadvertised level or filter is declared"); }); test("native EventSource receives log readiness, record IDs and heartbeat framing", { timeout: 3_000 }, async () => { @@ -1901,13 +1905,13 @@ test("DNS cache capacity may be unbounded and is not capped by the contract", () assertInvalid(contract.validate(usage, { entries: "5" }), "capacity may be null but not absent"); }); -test("eBPF attachments are interface or cgroup attachments", () => { +test("eBPF attachments are interface, cgroup or other attachments", () => { const response = example("getDatapath:200:active"); const attachments = response.body.ebpf.attachments; - assert.deepEqual(attachments.map((attachment) => attachment.kind), ["interface", "cgroup"]); + assert.deepEqual(attachments.map((attachment) => attachment.kind), ["interface", "cgroup", "other"]); assertValid(validateExample(contract, response)); const schema = { $ref: "#/components/schemas/EbpfAttachment" }; - const [iface, cgroup] = attachments; + const [iface, cgroup, other] = attachments; for (const [label, value] of [ ["kind is required", (({ kind, ...rest }) => rest)(iface)], ["an interface attachment needs its direction", (({ direction, ...rest }) => rest)(iface)], @@ -1917,6 +1921,11 @@ test("eBPF attachments are interface or cgroup attachments", () => { ["a cgroup attachment has no interface", { ...cgroup, interface: "eth0" }], ["a cgroup attachment has no direction", { ...cgroup, direction: "egress" }], ["kind comes from the enum", { ...cgroup, kind: "xdp" }], + ["an interface attachment has no hook", { ...iface, hook: "tc" }], + ["a cgroup attachment has no hook", { ...cgroup, hook: "connect4" }], + ["an other attachment needs its hook", (({ hook, ...rest }) => rest)(other)], + ["an other attachment has no interface", { ...other, interface: "eth0" }], + ["an other attachment has no cgroup", { ...other, cgroup: "/" }], ]) { assertInvalid(contract.validate(schema, value), label); } From e30a67bbd433cf9fdcd4679189209f4dd2cffcf8 Mon Sep 17 00:00:00 2001 From: Zakk Date: Tue, 29 Sep 2026 03:40:10 +1000 Subject: [PATCH 5/5] docs(runtime): tighten settings, flow and log wording; record honk's current capabilities --- source/v0.1.0/en/docs/flows.md | 11 +++++------ source/v0.1.0/en/docs/honk-mapping.md | 12 ++++++++++-- source/v0.1.0/en/docs/logs.md | 2 +- source/v0.1.0/en/docs/runtime-status.md | 10 +++++----- 4 files changed, 21 insertions(+), 14 deletions(-) diff --git a/source/v0.1.0/en/docs/flows.md b/source/v0.1.0/en/docs/flows.md index 9f16b7a..600c13b 100644 --- a/source/v0.1.0/en/docs/flows.md +++ b/source/v0.1.0/en/docs/flows.md @@ -328,12 +328,11 @@ admission) must be declared even where no connection exists. Capabilities advertise `recording` (`off`, `on`, `sampled`, `on_demand`), `scopes`, the optional `min_flows`, `max_flows`, `max_steps_per_flow`, -`retention_seconds`, `snapshot_ttl_seconds`, and `max_page_size`. `on_demand` records only while -clients read or follow flows (see -[runtime settings](runtime-status.html#GET-api-v1-runtime-settings)). -`max_flows` and `retention_seconds` are the engine's limits, the most a -runtime-settings PATCH may set; the current values are in -`GET /runtime/settings`. Retention is a **maximum age after termination**, not a +`retention_seconds`, `snapshot_ttl_seconds`, and `max_page_size`. +`on_demand` follows the +[runtime settings](runtime-status.html#GET-api-v1-runtime-settings) demand +rules. `max_flows` and `retention_seconds` are supported maxima, not active +values; that endpoint reports the active values. Retention is a **maximum age after termination**, not a durable guarantee under the bounded memory limit. Eviction, recording toggles, and losses produce `flow.gap` events; per-flow loss also marks the retained record partial. Known expired IDs return `410 flow_expired` while a bounded diff --git a/source/v0.1.0/en/docs/honk-mapping.md b/source/v0.1.0/en/docs/honk-mapping.md index ec9138a..1057d10 100644 --- a/source/v0.1.0/en/docs/honk-mapping.md +++ b/source/v0.1.0/en/docs/honk-mapping.md @@ -228,8 +228,10 @@ recorders on the current `feat/native-api` branch. - A PATCH accepts `log.buffered_records` and `dns_log.max_records` from 64 to 512, `flows.max_flows` from 64 to 1024, and `flows.retention_seconds` from 1 to 300. The startup values are 512, 512, 1024 and 300. -- honk lists `log.level` and `log.buffered_records` only when the log recorder - is permitted, and reports every settings section whether or not it is listed. +- `resources.runtime_settings.fields` lists `log.level` and + `log.buffered_records` only when the log recorder is permitted. GET + `/runtime/settings` reports every settings section, including values absent + from that list. - A client is attached while an admitted GET SSE stream on `/events` or `/logs` is open, and for 60 seconds after the last one closed or after a successful GET on `/flows`, `/flows/{flow_id}` or `/dns/log`. Settings reads, @@ -243,6 +245,12 @@ recorders on the current `feat/native-api` branch. seconds after the last one closed or after a successful GET on `/flows` or `/flows/{flow_id}`. Event streams without `kinds`, `/logs` streams and `/dns/log` reads do not create demand, and attachment does not extend it. +- honk currently reports `resources.flows.recording` as `on` or `off` from + whether the flow recorder is capturing, advertises the current `max_flows` + and `retention_seconds` rather than its maxima, and omits `min_flows`, + `logs.min_buffered_records`, `dns_log.min_records`, `logs.filters` and the + eBPF attachment `kind`. A planned honk change reports all of them as the + contract requires, with `on_demand` in `auto` mode. - `grace_remaining_seconds` counts the attachment grace only, not the flow-demand grace. Event capture runs while a client is attached or any permitted recorder is pinned on. diff --git a/source/v0.1.0/en/docs/logs.md b/source/v0.1.0/en/docs/logs.md index 8df36d1..3099331 100644 --- a/source/v0.1.0/en/docs/logs.md +++ b/source/v0.1.0/en/docs/logs.md @@ -14,7 +14,7 @@ title: Logs | Parameter | Default | Meaning | |-----------|---------|---------| | level | all advertised levels | Minimum severity: `trace`, `debug`, `info`, `warn`, `error`, in ascending order. | -| target | absent | Case-sensitive literal prefix of a record's `target`. Requires `target` in `filters`. | +| target | absent | Case-sensitive literal prefix of a record's `target`. | | Last-Event-ID | absent | Header containing the last processed opaque cursor. | A level outside the five above returns `400 invalid_request`; one of them that diff --git a/source/v0.1.0/en/docs/runtime-status.md b/source/v0.1.0/en/docs/runtime-status.md index 0aa1957..3175564 100644 --- a/source/v0.1.0/en/docs/runtime-status.md +++ b/source/v0.1.0/en/docs/runtime-status.md @@ -203,11 +203,11 @@ recorders. {% api_example getRuntimeSettings 200 current %} -Only `observed_at` and `source` are always present. Every other section is -optional: a value appears when `resources.runtime_settings.fields` lists it, -and an engine may also report a value it cannot change. Inside `log`, -`level` and `buffered_records` are independent, so an engine whose only -setting is the log level reports `log.level` alone: +Only `observed_at` and `source` are always present. Values listed in +`resources.runtime_settings.fields` appear in GET; the engine may also report +read-only values. Within `log`, `level` and `buffered_records` are +independent, so an engine whose only setting is the log level reports +`log.level` alone: {% api_example getRuntimeSettings 200 level_only %}