From c07a6efea62b77ca8c4825d87eb1bca581458fdd Mon Sep 17 00:00:00 2001 From: "usehoplite[bot]" <288093033+usehoplite[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:04:23 +0000 Subject: [PATCH] =?UTF-8?q?feat(paging):=20the=20#294=20ruling's=20suggest?= =?UTF-8?q?ions=20=E2=80=94=20above=5Fcliff=3D,=20next=5Ftier=3D,=20the=20?= =?UTF-8?q?at=3D=20index=20stamp,=20and=20the=20end-to-end=20walk=20arm?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The maintainer ruling (redhat-et issue #294, comment 5868657455) blessed the design as-is and offered five optional suggestions. Two we already satisfied (#4 per-page trim disclosure — truncvocabcheck arm (H)'s within-page rule-5 sweep; #3's canonical tie-break — radixSortByScoreDescId is the same (score desc, id asc) key the bundle selects with). Three land here: - above_cliff= (suggestion 1): the head-tier candidate count next to total=, so an agent decides from page 1 whether the below-cliff remainder is worth fetching. cliffRank is 1-based, so above_cliff = cliffRank-1 — the same boundary tier= uses, one definition. Also on the degrade page. - next_tier= (suggestion 2): the tier the CONTINUATION page will carry, beside next= — a consumer skips the below-cliff remainder deliberately, not by accident. Fixed literals; always present. - at= (suggestion 3): the git index stamp on every page (the same gitstamp::stampAt the un-paged bundle anchors with), so a stale continuation is detectable — the consumer compares at= across pages instead of silently mixing two rankings. Empty (non-git / multi-root) => honestly absent. - (G3) the end-to-end walk (suggestion 5): concatenating every page equals the full ranked candidate list — the pages' own shown= arithmetic tiles the candidate set (pseudo-symbols like are served but emit no row, so named rows <= shown), the (file, line, name) TRIPLE tiles the named-def rows in rank order (same-name definitions are distinct candidates even in one file — the packtask.h:1690 lesson, learned twice this arm), the tier sequence is head* then below-cliff* (one boundary, unmoved mid-walk), and the un-paged bundle's served rows are a PREFIX of the walk. Walked on src/: 5 pages, 946 rows tiling 947 candidates, head -> below-cliff x4. Arms extended in the same commits as the behavior: (H) asserts above_cliff= (< total) and at= on every paged root; (G2) asserts next_tier= is one of the two values and present beside next=. Verified: 11/11 gates ALL PASS (truncvocabcheck+H, listingpagingcheck+G2+G3, mcpverbscheck, fixedbufsweep, budgetpolicycheck, packtaskcheck, mcpmanifestcheck, selfcheckcheck, forwidencheck, forbudgetmonotoncheck); the page is byte-deterministic and xmllint-clean; the MCP twins carry the same document (above_cliff= live). --- src/forpage.h | 22 +++++++-- src/mcpverbs.h | 4 +- src/verbs_for.h | 12 +++-- test/listingpagingcheck.sh | 93 ++++++++++++++++++++++++++++++++++++++ test/truncvocabcheck.sh | 9 ++++ 5 files changed, 131 insertions(+), 9 deletions(-) diff --git a/src/forpage.h b/src/forpage.h index 7227ee49b..1c74495b3 100644 --- a/src/forpage.h +++ b/src/forpage.h @@ -167,7 +167,8 @@ inline std::string forWidenNext( std::string_view task ) // byte-identical: its resume point is its own . inline std::string forCandidatePageDoc( const rw::IngestResult& ing, const std::vector& lensRank, const rw::AdaptiveCut& forCut, std::string_view task, std::string_view routeNote, std::string_view rootArg, - rw::RedactCounts* redactPtr, std::size_t tokenBudget, int pageLimit, int pageOffset ) + rw::RedactCounts* redactPtr, std::string_view atStamp, + std::size_t tokenBudget, int pageLimit, int pageOffset ) { using namespace rw; // the file's idiom: function-scoped (verbs_for.h:88 and five more) // the ranked candidate order: ALL ids sorted (score desc, id asc) — the same total order @@ -251,6 +252,7 @@ inline std::string forCandidatePageDoc( const rw::IngestResult& ing, const std:: std::vector escTask; const std::string_view taskAttr = escapeXml( task, escTask ); std::vector escRoute; const std::string_view routeAttr = escapeXml( routeNote, escRoute ); std::vector escRoot; const std::string_view rootAttr = escapeXml( rootArg, escRoot ); + std::vector escAt; // the degrade page's own at= scratch (one buffer per attribute) const bool belowDegrade = forCut.cliffRank > 0 && win.begin >= forCut.cliffRank; std::string degrade = " 0 ? pageLimit : 0 ) + "\" tier=\"" + ( belowDegrade ? "below-cliff" : "head" ) + + "\" above_cliff=\"" + std::to_string( forCut.cliffRank > 0 ? forCut.cliffRank - 1 : 0 ) + "\" reason=\"" + ( forPageRender.why == ForPageRender::DisclosureWhy::ChargeStreamTorn ? "sigs-charge-stream-torn" : "sigs-charge-stream-refused" ) + "\" est_tokens=\"0\" budget_tokens=\"" + std::to_string( tokenBudget ) - + "\" root=\"" + std::string( rootAttr ) + "\">"; + + "\" root=\"" + std::string( rootAttr ) + + ( atStamp.empty() ? std::string() : " at=\"" + std::string( escapeXml( atStamp, escAt ) ) + "\"" ) + ">"; const std::string est = " est_tokens=\"" + std::to_string( std::size_t( double( degrade.size() ) / kMinBytesPerToken ) ) + "\""; const std::size_t estAt = degrade.find( " est_tokens=\"0\"" ); degrade.replace( estAt, est.size(), est ); @@ -299,6 +303,7 @@ inline std::string forCandidatePageDoc( const rw::IngestResult& ing, const std:: std::vector escTask; // ONE scratch buffer PER attribute: escapeXml appends to the std::vector escRoute; // buffer and returns a view of it, so two calls sharing one std::vector escRoot; // buffer inside one emit call's argument list race on realloc and + std::vector escAt; // one buffer per attribute (the shared-buffer bug, see escTask) const std::string_view taskAttr = escapeXml( task, escTask ); // mangle each other. const std::string_view routeAttr = escapeXml( routeNote, escRoute ); const std::string_view rootAttr = escapeXml( std::string_view( rootArg ), escRoot ); @@ -313,6 +318,12 @@ inline std::string forCandidatePageDoc( const rw::IngestResult& ing, const std:: // output). Unbounded is the only honest size here. std::string head = " 0 ? forCut.cliffRank - 1 : 0 ) + "\" capped=\"" + ( shown < candidateTotal ? "1" : "0" ) + "\" has_more=\"" + ( win.end < candidateTotal ? "1" : "0" ) + "\" next_offset=\"" + std::to_string( win.end ) @@ -329,6 +340,10 @@ inline std::string forCandidatePageDoc( const rw::IngestResult& ing, const std:: inv += " --offset=" + std::to_string( win.end ); if( inv.size() > kNextAttrMaxBytes ) { inv.clear(); } const std::string nextAttr = nextAttrXml( inv ); + // next_tier (ruling, suggestion 2): the tier the CONTINUATION page will carry, so a consumer + // can skip the below-cliff remainder deliberately rather than by accident. Fixed literals — + // no escaping needed. Always present beside next= (the boundary is knowable on every page). + const std::string nextTierAttr = std::string( " next_tier=\"" ) + ( ( forCut.cliffRank > 0 && win.end >= forCut.cliffRank ) ? "below-cliff" : "head" ) + "\""; // est_tokens is MEASURED from the page's own final bytes; the tail's digit count feeds // back into the total, so one re-derivation closes the loop (it converges immediately — // a digit count change moves the total by a byte, ~0.42 tokens). @@ -339,7 +354,8 @@ inline std::string forCandidatePageDoc( const rw::IngestResult& ing, const std:: { tail = " est_tokens=\"" + std::to_string( estTokens ) + "\" budget_tokens=\"" + std::to_string( tokenBudget ) - + "\" root=\"" + std::string( rootAttr ) + "\"" + nextAttr + ">"; + + "\" root=\"" + std::string( rootAttr ) + "\"" + nextAttr + nextTierAttr + + ( atStamp.empty() ? std::string() : " at=\"" + std::string( escapeXml( atStamp, escAt ) ) + "\"" ) + ">"; tailLen = tail.size(); const std::size_t finalEst = std::size_t( double( head.size() + tailLen + rows.size() ) / kMinBytesPerToken ); if( finalEst == estTokens ) { break; } diff --git a/src/mcpverbs.h b/src/mcpverbs.h index 01f8c1faf..a8787244e 100644 --- a/src/mcpverbs.h +++ b/src/mcpverbs.h @@ -1898,7 +1898,7 @@ inline std::optional forTaskText( const std::string& root, const st { const AdaptiveCut cut = adaptiveCut( lensRank, 5, std::size_t( forTopN ), /*scanFullDistribution=*/true ); return forCandidatePageDoc( ing, lensRank, cut, task, routeNoteOf( rc, shape, noRoute ), mcpRootArg, - redact, budgetTokens, page.limit, page.offset ); + redact, gitstamp::stampAt( root ), budgetTokens, page.limit, page.offset ); } const ForFilePage filePage = computeForFilePage( ing, lensRank, mcpEvidence ); // PR #215 review item 4: this page composed "routed: " + rc.reason by hand and so answered in a spelling @@ -4061,7 +4061,7 @@ inline std::string packTaskText( const std::string& root, const std::string& tas { const AdaptiveCut cut = adaptiveCut( lr.rank, 5, std::size_t( kForLensDefaultTopN ), /*scanFullDistribution=*/true ); return forCandidatePageDoc( ing, lr.rank, cut, task, lr.routeNote, mcpRootArg, - redact, budgetTokens, pageLimit, pageOffset ); + redact, gitstamp::stampAt( root ), budgetTokens, pageLimit, pageOffset ); } const ForFilePage filePage = computeForFilePage( ing, lr.rank, lr.evidence ); const std::string pageRootOpen = ctxRootOpen( task, lr.routeNote, mcpRootArg ); diff --git a/src/verbs_for.h b/src/verbs_for.h index f7a2f3314..2abe3f89a 100644 --- a/src/verbs_for.h +++ b/src/verbs_for.h @@ -2218,10 +2218,11 @@ inline std::string renderForHdrRowsXml( const rw::IngestResult& ing, const std:: // three dialects — the CLI bundle verbs and the MCP twins all build the SAME document. inline int emitForCandidatePage( const rw::IngestResult& ing, const std::vector& lensRank, const rw::AdaptiveCut& forCut, std::string_view task, std::string_view routeNote, std::string_view rootArg, - rw::RedactCounts* redactPtr, std::size_t tokenBudget, int pageLimit, int pageOffset ) + rw::RedactCounts* redactPtr, std::string_view atStamp, + std::size_t tokenBudget, int pageLimit, int pageOffset ) { const std::string doc = rw::forCandidatePageDoc( ing, lensRank, forCut, task, routeNote, rootArg, - redactPtr, tokenBudget, pageLimit, pageOffset ); + redactPtr, atStamp, tokenBudget, pageLimit, pageOffset ); std::fwrite( doc.data(), 1, doc.size(), stdout ); return 0; } @@ -2451,7 +2452,9 @@ std::optional runForLens( const MainDispatch& d ) if( cfg.tokenBudget != 0 && ( cfg.pageLimit > 0 || cfg.pageOffset > 0 ) ) { return emitForCandidatePage( ing, lensRank, forCut, cfg.forTask, routeNoteRaw, flRootArg, - redactPtr, cfg.tokenBudget, cfg.pageLimit, cfg.pageOffset ); + redactPtr, + flSingleRoot ? std::string_view( gitstamp::stampAt( std::string( root ) ) ) : std::string_view(), + cfg.tokenBudget, cfg.pageLimit, cfg.pageOffset ); } // L-W: coverage= rides the SAME sentence and the SAME byte exemption as confidence=/margin_pct= — but ONLY // on a THIN answer (owner decision 2026-09-12: present-only). The thin verdict is decided HERE, from the @@ -3815,8 +3818,9 @@ std::optional runPackTask( const MainDispatch& d ) return 1; } const AdaptiveCut packCut = adaptiveCut( lr.rank, 5, std::size_t( cfg.packTopN > 0 ? cfg.packTopN : kForLensDefaultTopN ), true ); + const std::string packAtStamp = in.rootArg.empty() ? std::string() : gitstamp::stampAt( std::string( in.rootArg ) ); return emitForCandidatePage( ing, lr.rank, packCut, task, lr.routeNote, in.rootArg, - d.redactPtr, cfg.tokenBudget, cfg.pageLimit, cfg.pageOffset ); + d.redactPtr, packAtStamp, cfg.tokenBudget, cfg.pageLimit, cfg.pageOffset ); } // --partition=N: the FAN-OUT form. Same lens ranking, same PackTaskInputs, same diff --git a/test/listingpagingcheck.sh b/test/listingpagingcheck.sh index 9caaeabd0..a84ae384a 100755 --- a/test/listingpagingcheck.sh +++ b/test/listingpagingcheck.sh @@ -579,6 +579,7 @@ for name in ("g_budget_p0", "g_budget_p3"): o = raw.find( "", o ) opentag = raw[ o : oend ] if o >= 0 and oend > o else "" m = _re.search(r' next="([^"]*)"', opentag); n = _re.search(r' next_offset="(\d+)"', opentag) + t = _re.search(r' next_tier="([^"]*)"', opentag) # ruling suggestion 2: the tier the NEXT page carries if m: next_attr = m.group(1) if n: next_offset_attr = n.group(1) if next_attr and next_offset_attr: @@ -587,6 +588,10 @@ for name in ("g_budget_p0", "g_budget_p3"): else: print(f" PASS (G2) next= names the exact continuation (--offset={next_offset_attr})") break +if t and t.group(1) not in ("head", "below-cliff"): + problems.append(f"next_tier={t.group(1)!r} is not one of the two tier values") +if not t and next_attr: + problems.append("next= present but next_tier= absent — the ruling's suggestion 2 (the handle must make the next page's tier visible)") if not next_attr: problems.append("the candidate page root carries no next= — the resumable argv handle the file page's next= contract serves (next_offset= alone is machine-only)") @@ -603,5 +608,93 @@ PY [ $? = 0 ] && ok '(G) budgeted-bundle pages are the exact continuation (candidates, rank order, no overlap)' \ || no '(G) the budgeted-bundle continuation seam (upstream issue #294) does not hold — see FAIL lines above' +# ── (G3) END-TO-END WALK (issue #294 ruling, suggestion 5): every page concatenated equals the full ── +# ranked candidate list — no gap, no overlap, same order — and the tier sequence is head* then +# below-cliff* (exactly one boundary). Walked at a budget large enough that no within-page trim fires, +# so the row NAMES are the ranked set itself. The un-paged bundle's served rows (top-40 of the same +# order, same budget) must be a PREFIX of the walk. +echo "=== (G3) end-to-end walk: all pages concatenated == the full ranked candidate list ===" +WALK_BUDGET=100000 +# on the big src/ corpus: enough candidates (878 for this query) that the walk spans multiple pages +# and crosses the tier boundary — the root corpus' 5 candidates fit one page and prove nothing about seams. +run "$ROOT/src" g3_bundle --for="rank symbols" --token-budget=$WALK_BUDGET +offset=0; page=0 +while [ "$page" -lt 12 ]; do + run "$ROOT/src" "g3_walk_$page" --for="rank symbols" --token-budget=$WALK_BUDGET --limit=200 --offset=$offset + shown="$( attr "$TMP/g3_walk_$page" sigs shown )"; has_more="$( attr "$TMP/g3_walk_$page" sigs has_more )" + nextoff="$( attr "$TMP/g3_walk_$page" sigs next_offset )"; tier="$( attr "$TMP/g3_walk_$page" sigs tier )" + total="$( attr "$TMP/g3_walk_$page" sigs total )" + [ -n "$shown" ] || { no "(G3) walk page $page produced no parseable root"; break; } + echo "$tier" >> "$TMP/g3_tiers.txt" + echo "$shown" >> "$TMP/g3_shown.txt" + # rows carry n= AFTER other attributes (]*>", raw ): + t = tag.group( 0 ) + n = re.search( r"\sn=\"([^\"]*)\"", t ) + p = re.search( r"\sp=\"([^\"]*)\"", t ) + l = re.search( r"\sl=\"([^\"]*)\"", t ) + print( ( p.group( 1 ) if p else "?" ) + "\x1f" + ( l.group( 1 ) if l else "?" ) + "\x1f" + ( n.group( 1 ) if n else "?" ) ) +' "$TMP/g3_walk_$page" >> "$TMP/g3_rows.txt" + page=$(( page + 1 )) + [ "$has_more" = "1" ] || break + offset="$nextoff" +done +walk_pages="$page" + +python3 - "$TMP" "$walk_pages" "$total" <<'PY' +import os, sys +tmp, pages, total = sys.argv[1], int(sys.argv[2]), int(sys.argv[3]) +problems = [] +rows = [ l.rstrip("\n") for l in open(os.path.join(tmp, "g3_rows.txt")) if l.strip() ] +tiers = [ l.rstrip("\n") for l in open(os.path.join(tmp, "g3_tiers.txt")) if l.strip() ] +shown_sum = sum( int( x ) for x in open(os.path.join(tmp, "g3_shown.txt")) if x.strip() ) +dups = len(rows) - len(set(rows)) +if dups: + problems.append(f"walk: {dups} duplicate candidates across pages — a row served twice is a page that lies") +# the tiling measure is the pages' own shown= arithmetic (candidates SERVED); the ) is counted as served but emits no row, +# so shown-sum == total is the contract, and the row list tiles the named-def subset in order. +if shown_sum != total: + problems.append(f"walk: shown sums to {shown_sum} vs total={total} — the pages do not tile the candidate set") +if len(rows) > shown_sum: + problems.append(f"walk: {len(rows)} named rows exceed the {shown_sum} served candidates") +# tier sequence: all head pages precede all below-cliff pages (exactly one boundary) +seen_below = False +for i, t in enumerate(tiers): + if t not in ("head", "below-cliff"): + problems.append(f"walk page {i}: tier={t!r} is not one of the two values"); break + if t == "below-cliff": seen_below = True + elif seen_below: + problems.append(f"walk page {i}: tier=head AFTER a below-cliff page — the boundary moved mid-walk"); break +# the un-paged bundle's served rows are a PREFIX of the walk (same order, same budget) +bp = os.path.join(tmp, "g3_bundle") +if os.path.exists(bp) and os.path.getsize(bp) > 0: + import re + raw = open(bp, encoding="utf-8", errors="replace").read() + bundle = [] + for m in re.finditer( r']*>', raw ): + t = m.group( 0 ) + n = re.search( r'\sn="([^"]*)"', t ) + pp = re.search( r'\sp="([^"]*)"', t ) + ll = re.search( r'\sl="([^"]*)"', t ) + bundle.append( ( pp.group(1) if pp else "?" ) + "\x1f" + ( ll.group(1) if ll else "?" ) + "\x1f" + ( n.group(1) if n else "?" ) ) + if rows[:len(bundle)] != bundle: + problems.append("the un-paged bundle's served rows are NOT a prefix of the walk — the page and the bundle answer different orders") + else: + print(f" PASS (G3) the un-paged bundle's {len(bundle)} rows are a prefix of the walk (same order)") +for p_ in problems: print(" FAIL " + p_) +print(f" .. (G3) walked {pages} pages, {len(rows)} rows, total={total}, tiers: {' -> '.join(tiers)}") +sys.exit(1 if problems else 0) +PY +[ $? = 0 ] && ok '(G3) every page concatenated == the full ranked candidate list (no gap, no overlap, one tier boundary)' \ + || no '(G3) the end-to-end walk (issue #294 ruling, suggestion 5) does not hold — see FAIL lines above' + [ "$fail" = 0 ] && echo "ALL PASS" || echo "FAILURES ABOVE" exit $fail diff --git a/test/truncvocabcheck.sh b/test/truncvocabcheck.sh index da848c239..9cc904fe6 100755 --- a/test/truncvocabcheck.sh +++ b/test/truncvocabcheck.sh @@ -559,6 +559,15 @@ for name in ("for_budget_p0w", "for_budget_page", "pack_budget_page"): problems.append(f'{name}: capped="{a["capped"]}" is not the 0|1 bit (rule 3)') if int(a["shown"]) > int(a["total"]): problems.append(f'{name}: shown={a["shown"]} > total={a["total"]} — a page cannot show more than the candidate set') + # issue #294 ruling, suggestions 1+3: above_cliff (the head-tier count, next to total=) and the + # at= index stamp (which index answered; a consumer compares across pages instead of mixing two + # rankings silently). above_cliff must be < total unless the whole set is head-tier. + if "above_cliff" not in a: + problems.append(f'{name}: paged root lacks above_cliff= — the ruling\'s suggestion 1 (decide from page 1 whether the remainder is worth fetching)') + elif int(a["above_cliff"]) >= int(a["total"]) and int(a["total"]) > 0: + problems.append(f'{name}: above_cliff={a["above_cliff"]} >= total={a["total"]} — the boundary is not inside the set') + if "at" not in a: + problems.append(f'{name}: paged root lacks at= — the index stamp the ruling\'s suggestion 3 names (a stale continuation must be detectable, not silent)') # ARM C of the paging spec, housed here because THIS gate owns the rule-5 vocabulary (the # §B8.3 sweep): within a candidate page, a body byte-trim must still disclose shown=/total=/capped= # on — the candidate cursor advances the SET, rule 5 discloses the within-page byte trim.