diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 948117d..6a66cf4 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "agentlinkops", - "version": "0.6.3", + "version": "0.6.4", "description": "Connect an agent to AgentLinkOps and run backlink work with it: the connect skill (views, the three discovery verbs, evidence rules), local ledger and campaign skills, a portable CRM, the source-cited content pipeline, user-managed disavow lists, HTML reports and monitoring-event sync.", "author": { "name": "AgentLinkOps" diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index 78910ee..2b9584d 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "agentlinkops", - "version": "0.6.3", + "version": "0.6.4", "description": "Connect an agent to AgentLinkOps and run backlink work with it: the connect skill (views, the three discovery verbs, evidence rules), local ledger and campaign skills, a portable CRM, the source-cited content pipeline, user-managed disavow lists, HTML reports and monitoring-event sync.", "author": { "name": "AgentLinkOps" diff --git a/CHANGELOG.md b/CHANGELOG.md index 36f6ce8..6b83622 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,12 @@ All notable changes to the `agentlinkops` package are recorded here. The version follows the plugin manifests in `.claude-plugin/` and `.codex-plugin/`. +## 0.6.4 (2026-09-20) + +- Import and export immutable local IndexNow submission receipts, with hashes and replay checks. These commands send no submissions. +- Join each submission attempt to later retained index observations for the exact URL and project. JSON and CSV preserve source, date, reason and unknown results. +- Submission responses remain received, validation pending, failed or unknown. Later Google observations do not verify indexing by an IndexNow participant or establish that submission caused indexing. + ## 0.6.3 (2026-09-20) - Agent installation preserves customer edits and supports previewed removal and interrupted-install recovery. diff --git a/cli/client.js b/cli/client.js index 10d0005..a32fe03 100644 --- a/cli/client.js +++ b/cli/client.js @@ -2,7 +2,7 @@ import {restResponseContract} from '../shared/response-contract.js'; // A small HTTP client for the cloud. Scoped keys only; no human session ever reaches here. // DP-0036-T13: the CLI names itself so the usage counters can tell it from other REST callers. // The version follows the plugin manifests; bump it with them. -export const CLI_VERSION = '0.6.3'; +export const CLI_VERSION = '0.6.4'; export const CLI_USER_AGENT = `agentlinkops-cli/${CLI_VERSION}`; export class CloudError extends Error { constructor(code, status, details = null) { diff --git a/cli/config.js b/cli/config.js index 3085ad4..bd5f713 100644 --- a/cli/config.js +++ b/cli/config.js @@ -20,6 +20,7 @@ export const DEFAULTS = Object.freeze({ receipts: 'receipts.jsonl', observations: 'observations.jsonl', indexObservations: 'index-observations.jsonl', + indexSubmissions: 'index-submissions.jsonl', events: 'events.jsonl', candidates: 'candidates.jsonl', state: 'state.json', @@ -99,7 +100,7 @@ export async function loadConfig({ cwd = process.cwd(), root = null } = {}) { defaults: { ...DEFAULTS, ...file.defaults }, paths: { ledger: at('ledger'), observations: at('observations'), events: at('events'), - candidates: at('candidates'), state: at('state'), receipts: at('receipts'), indexObservations: at('indexObservations'), + candidates: at('candidates'), state: at('state'), receipts: at('receipts'), indexObservations: at('indexObservations'), indexSubmissions: at('indexSubmissions'), }, }; } diff --git a/cli/index-observations.js b/cli/index-observations.js index 6e2592b..b65cde8 100644 --- a/cli/index-observations.js +++ b/cli/index-observations.js @@ -1,3 +1,4 @@ +import {boundedSubmissionText,readSubmissions,saveSubmissions,reconcileSubmissions,submissionCsv} from './index-submissions.js'; import {readFile,writeFile,open,rename,unlink,mkdir,stat} from 'node:fs/promises'; import {dirname,resolve} from 'node:path'; import {createHash} from 'node:crypto'; @@ -9,7 +10,7 @@ import {indexEvidenceReference,describeEvidenceReference} from '../src/evidence- import {indexReceiptResponse} from '../shared/index-observation-contract.js'; const sha=value=>createHash('sha256').update(value).digest('hex'); const fail=message=>{throw new ConfigError(message);}; -export const INDEX_CLI_HELP='agentlinkops index pull --watch-id ID [--limit 50] [--before ID] | import RECEIPTS.json | export [--out FILE] | csv URLS.csv [--out FILE] | check-csv URLS.csv --connection-id ID --request-id ID --out FILE'; +export const INDEX_CLI_HELP='agentlinkops index pull --watch-id ID [--limit 50] [--before ID] | import RECEIPTS.json | export [--out FILE] | csv URLS.csv [--out FILE] | check-csv URLS.csv --connection-id ID --request-id ID --out FILE | submissions-import FILE | submissions-export --project-id ID [--out FILE] | reconcile-submissions --project-id ID [--format json|csv] [--out FILE]'; function validate(row){ if(row?.v!==1||! /^[\w-]{1,128}$/.test(row.project_id??'')||! /^[\w-]{1,128}$/.test(row.watch_id??''))fail('Invalid local index receipt scope'); const receipt=indexReceiptResponse.parse(row.receipt); @@ -61,6 +62,19 @@ function csvInput(text){ } export async function indexMain(args,{cwd=process.cwd(),env=process.env,out=console.log,fetchImpl=globalThis.fetch}={}){ const command=args._[1],config=await loadConfig({cwd}),path=config.paths.indexObservations; + if(['submissions-import','submissions-export','reconcile-submissions'].includes(command)){ + const allowed=new Set(['_','tag','out','project-id','format']); + if(args.tag?.length||Object.keys(args).some(key=>!allowed.has(key))||args._.length!==(command==='submissions-import'?3:2))fail(INDEX_CLI_HELP); + if(command==='submissions-import'){ + if(args.out||args['project-id']||args.format)fail(INDEX_CLI_HELP); + out(JSON.stringify(await saveSubmissions(config.paths.indexSubmissions,JSON.parse(await boundedSubmissionText(resolve(cwd,args._[2]))))));return 0; + } + if(args.format&&!['json','csv'].includes(args.format))fail('Submission format must be json or csv'); + if(command==='submissions-export'&&args.format)fail(INDEX_CLI_HELP); + const report=reconcileSubmissions(await readSubmissions(config.paths.indexSubmissions),command==='reconcile-submissions'?await readIndexReceipts(path):[],args['project-id']); + const text=command==='submissions-export'?JSON.stringify(report.submissions.map(({outcome,observations,verification,...row})=>row),null,2):args.format==='csv'?submissionCsv(report):JSON.stringify(report,null,2); + if(args.out){if(typeof args.out!=='string')fail('--out requires a file');await atomicOutput(resolve(cwd,args.out),text);}else out(text);return 0; + } const allowed=new Set(['_','tag','out','watch-id','limit','before','connection-id','request-id']);if(args.tag?.length||Object.keys(args).some(key=>!allowed.has(key)))fail(INDEX_CLI_HELP); if(command==='import'){ if(!args._[2])fail(INDEX_CLI_HELP);const parsed=JSON.parse(await boundedText(resolve(cwd,args._[2])));if(!Array.isArray(parsed)||parsed.length>100)fail('Import requires an array of at most 100 scoped index receipts');out(JSON.stringify(await saveIndexReceipts(path,parsed)));return 0; diff --git a/cli/index-submissions.js b/cli/index-submissions.js new file mode 100644 index 0000000..6f35bc1 --- /dev/null +++ b/cli/index-submissions.js @@ -0,0 +1,50 @@ +import {readFile,writeFile,open,rename,unlink,mkdir,stat} from 'node:fs/promises'; +import {dirname} from 'node:path'; +import {createHash} from 'node:crypto'; +import {ConfigError} from './config.js'; +import {indexEvidenceReference} from '../src/evidence-reference.js'; +const fail=message=>{throw new ConfigError(message);}; +const digest=value=>createHash('sha256').update(JSON.stringify(value)).digest('hex'); +const id=value=>typeof value==='string'&&/^[\w-]{1,128}$/.test(value); +const fields=['v','id','project_id','url','channel','endpoint','submitted_at','http_status','evidence_sha256']; +export const SUBMISSION_NOTE='Imported submission records describe reported HTTP responses. Hashes check local integrity, not provider authenticity. Later Google observations do not verify indexing by an IndexNow participant or show that submission caused indexing.'; +export function validateSubmission(row){ + if(!row||Object.keys(row).some(k=>!fields.includes(k))||fields.some(k=>!Object.hasOwn(row,k))||row.v!==1||!id(row.id)||!id(row.project_id)||row.channel!=='indexnow'||row.endpoint!=='https://api.indexnow.org/indexnow')fail('Invalid IndexNow submission receipt'); + let url;try{url=new URL(row.url);}catch{fail('Invalid submission URL');} + if(typeof row.url!=='string'||url.href!==row.url||!['https:','http:'].includes(url.protocol)||url.username||url.password||url.hash)fail('Submission URL must be exact HTTP(S) without credentials or fragment'); + if(typeof row.submitted_at!=='string'||!Number.isFinite(Date.parse(row.submitted_at))||new Date(row.submitted_at).toISOString()!==row.submitted_at)fail('Submission date must be an ISO UTC timestamp'); + if(row.http_status!==null&&(!Number.isInteger(row.http_status)||row.http_status<100||row.http_status>599))fail('Invalid submission HTTP status'); + const body=Object.fromEntries(fields.filter(k=>k!=='evidence_sha256').map(k=>[k,row[k]])); + if(row.evidence_sha256!==digest(body))fail('Submission receipt hash mismatch'); + return {...body,evidence_sha256:row.evidence_sha256}; +} +export async function boundedSubmissionText(path){const info=await stat(path);if(info.size>32*1024*1024)fail('Submission input exceeds 32 MiB');return readFile(path,'utf8');} +export async function readSubmissions(path){try{return (await boundedSubmissionText(path)).split('\n').filter(line=>line.trim()).map(line=>validateSubmission(JSON.parse(line)));}catch(error){if(error.code==='ENOENT')return [];throw error;}} +export async function saveSubmissions(path,rows){ + if(!Array.isArray(rows)||rows.length>100)fail('Import requires at most 100 submission receipts'); + const checked=rows.map(validateSubmission);await mkdir(dirname(path),{recursive:true});let lock; + try{lock=await open(`${path}.lock`,'wx');}catch(error){if(error.code==='EEXIST')fail('Submission receipt writer already locked');throw error;} + const tmp=`${path}.${crypto.randomUUID()}.tmp`; + try{ + const current=await readSubmissions(path),byId=new Map(current.map(r=>[`${r.project_id}:${r.id}`,r])); + for(const row of checked){const key=`${row.project_id}:${row.id}`,old=byId.get(key);if(old&&JSON.stringify(old)!==JSON.stringify(row))fail('Submission receipt identity conflict');byId.set(key,row);} + const text=[...byId.values()].sort((a,b)=>a.project_id.localeCompare(b.project_id)||a.submitted_at.localeCompare(b.submitted_at)||a.id.localeCompare(b.id)).map(r=>JSON.stringify(r)).join('\n')+'\n'; + if(Buffer.byteLength(text)>32*1024*1024)fail('Submission receipts exceed 32 MiB; archive before importing more'); + await writeFile(tmp,text,{flag:'wx'});await rename(tmp,path);return {imported:checked.length,total:byId.size}; + }finally{await unlink(tmp).catch(()=>{});await lock.close();await unlink(`${path}.lock`);} +} +export function reconcileSubmissions(submissions,observations,projectId){ + if(!id(projectId))fail('A valid --project-id is required'); + return {v:1,project_id:projectId,note:SUBMISSION_NOTE,submissions:submissions.filter(r=>r.project_id===projectId).map(validateSubmission).map(row=>{ + const subsequent=observations.filter(r=>r.project_id===projectId&&r.receipt.observation.url===row.url&&Date.parse(r.receipt.observation.checked_at)>Date.parse(row.submitted_at)).map(r=>{ + const o=r.receipt.observation;return {watch_id:r.watch_id,observation_id:o.id,checked_at:o.checked_at,backend:o.backend,source_key:o.source_key,tier:o.tier,reason:o.reason,confidence:o.confidence,evidence_sha256:o.evidenceSha256,evidence_available:r.receipt.evidenceAvailable,evidence_reference:indexEvidenceReference({receiptId:o.id}),relationship:'observation_after_submission',participant_indexing_verified:false,causation_established:false}; + }).sort((a,b)=>a.checked_at.localeCompare(b.checked_at)||a.observation_id.localeCompare(b.observation_id)); + return {...row,outcome:row.http_status===200?'received':row.http_status===202?'key_validation_pending':row.http_status===null?'unknown':'failed',observations:subsequent,verification:'not_established'}; + })}; +} +const cell=value=>{const raw=String(value??''),safe=/^[\s]*[=+\-@]|^[\t\r]/.test(raw)?`'${raw}`:raw;return `"${safe.replaceAll('"','""')}"`;}; +export function submissionCsv(report){ + const rows=[['Project','Submission ID','URL','Submitted At','HTTP Status','Submission Outcome','Submission Hash','Observation ID','Watch ID','Backend','Source','Check Date','Verdict','Reason','Confidence','Evidence Available','Observation Hash','Evidence Reference','Participant Indexing Verified','Causation Established','Reading Note']]; + for(const s of report.submissions)for(const o of s.observations.length?s.observations:[null])rows.push([report.project_id,s.id,s.url,s.submitted_at,s.http_status,s.outcome,s.evidence_sha256,o?.observation_id,o?.watch_id,o?.backend,o?.source_key,o?.checked_at,o?.tier??'unknown',o?.reason??'no_later_observation',o?.confidence??'unknown',o?.evidence_available??false,o?.evidence_sha256,o?.evidence_reference,false,false,report.note]); + return rows.map(row=>row.map(cell).join(',')).join('\r\n')+'\r\n'; +} diff --git a/cli/main.js b/cli/main.js index a61fa23..b36cd16 100644 --- a/cli/main.js +++ b/cli/main.js @@ -56,6 +56,8 @@ const USAGE = `agentlinkops — a backlink ledger that lives in your repository agentlinkops admission list|add|update|remove|evaluate|reevaluate|receipt --project ID agentlinkops disavow list|history|propose|import|export|approve|reject|delete agentlinkops index import FILE | export [--out FILE] | csv URLS.csv [--out FILE] + agentlinkops index submissions-import FILE | submissions-export --project-id ID [--out FILE] + agentlinkops index reconcile-submissions --project-id ID [--format json|csv] [--out FILE] agentlinkops index check-csv URLS.csv --connection-id ID --request-id ID --out FILE agentlinkops import FILE --target DOMAIN [--from SUPPLIER] [--map source=COL,target=COL] [--exact-url] [--no-subdomains] [--generated-at ISO] [--json] diff --git a/package-lock.json b/package-lock.json index 1a2027c..6c1c12d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "agentlinkops", - "version": "0.6.3", + "version": "0.6.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "agentlinkops", - "version": "0.6.3", + "version": "0.6.4", "license": "Apache-2.0", "dependencies": { "@modelcontextprotocol/server": "2.0.0", diff --git a/package.json b/package.json index fb98398..6b53234 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "agentlinkops", - "version": "0.6.3", + "version": "0.6.4", "description": "A backlink and AI citation ledger that lives in your repository: local link and citation checks with dated evidence, supplier-export imports and optional sync to the hosted AgentLinkOps service.", "author": "AgentLinkOps (https://agentlinkops.com)", "license": "Apache-2.0", diff --git a/test/index-submissions-cli.test.js b/test/index-submissions-cli.test.js new file mode 100644 index 0000000..8581c29 --- /dev/null +++ b/test/index-submissions-cli.test.js @@ -0,0 +1,44 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {mkdtemp,mkdir,writeFile,readFile,rm} from 'node:fs/promises'; +import {tmpdir} from 'node:os'; +import {join} from 'node:path'; +import {createHash} from 'node:crypto'; +import {execFileSync} from 'node:child_process'; +import {validateSubmission,saveSubmissions,readSubmissions,reconcileSubmissions,submissionCsv} from '../cli/index-submissions.js'; +import {saveIndexReceipts} from '../cli/index-observations.js'; +import {gscIndexObservation} from '../src/index-observation.js'; +const url='https://example.com/article',at='2026-09-20T00:00:00.000Z'; +const hash=value=>createHash('sha256').update(JSON.stringify(value)).digest('hex'); +function submission(changes={}){const body={v:1,id:'attempt1',project_id:'project1',url,channel:'indexnow',endpoint:'https://api.indexnow.org/indexnow',submitted_at:at,http_status:200,...changes};return {...body,evidence_sha256:hash(body)};} +function observation({project='project1',href=url,date='2026-09-20T01:00:00.000Z',status=429,id='a'}={}){ + const raw={error:{code:status}},o=gscIndexObservation({observationId:`observation${id}`,inspectionUrl:href,checkedAt:date,siteUrl:'sc-domain:example.com',grantedProperties:['sc-domain:example.com'],payload:raw});delete o.raw_response; + return {v:1,project_id:project,watch_id:'watch1',receipt:{observation:{...o,id:id.repeat(64),observationId:`observation${id}`,checkedAt:date,recordedAt:date,expiresAt:'2026-10-20T00:00:00.000Z',evidenceSha256:hash(raw),eventId:null,ignored:null},rawResponse:raw,evidenceAvailable:true}}; +} +async function setup(t){const cwd=await mkdtemp(join(tmpdir(),'index-submissions-'));t.after(()=>rm(cwd,{recursive:true,force:true}));await mkdir(join(cwd,'.agentlinkops'));await writeFile(join(cwd,'.agentlinkops','links.jsonl'),'');return {cwd,path:join(cwd,'.agentlinkops','index-submissions.jsonl')};} +test('immutable attempts preserve status distinctions and reject hash, field and identity corruption',async t=>{ + const {path}=await setup(t),first=submission(),pending=submission({id:'attempt2',http_status:202});await saveSubmissions(path,[first,pending]);await saveSubmissions(path,[first]);assert.equal((await readSubmissions(path)).length,2); + const before=await readFile(path,'utf8');await assert.rejects(saveSubmissions(path,[submission({http_status:202})]),/identity conflict/);assert.equal(await readFile(path,'utf8'),before); + assert.throws(()=>validateSubmission({...first,http_status:500}),/hash mismatch/);assert.throws(()=>validateSubmission({...first,key:'secret'}),/Invalid/); + for(const edit of [{url:`${url}#x`},{url:'https://user:pass@example.com/article'},{submitted_at:'2026-09-20'},{http_status:'200'},{endpoint:'https://other.example/'},{url:'HTTPS://example.com/article'}])assert.throws(()=>validateSubmission(submission(edit))); + const report=reconcileSubmissions([first,pending,submission({id:'failure',http_status:403}),submission({id:'timeout',http_status:null})],[],'project1');assert.deepEqual(report.submissions.map(s=>s.outcome),['received','key_validation_pending','failed','unknown']);assert.ok(report.submissions.every(s=>s.verification==='not_established')); +}); +test('chronological join keeps exact project and URL boundaries, unknowns and all attempts',()=>{ + const rows=[observation({date:at,id:'b'}),observation({project:'project2',id:'c'}),observation({href:`${url}/`,id:'d'}),observation({date:'2026-09-19T23:59:59.000Z',id:'e'}),observation({date:'2026-09-20T02:00:00.000Z',id:'f'}),observation()]; + const report=reconcileSubmissions([submission(),submission({id:'attempt2',http_status:202}),submission({project_id:'project2'})],rows,'project1');assert.equal(report.submissions.length,2); + for(const s of report.submissions){assert.deepEqual(s.observations.map(o=>o.observation_id),['a'.repeat(64),'f'.repeat(64)]);assert.ok(s.observations.every(o=>o.tier==='unknown'&&!o.participant_indexing_verified&&!o.causation_established));assert.equal(s.observations[0].evidence_sha256,hash({error:{code:429}}));} + assert.throws(()=>reconcileSubmissions([],[],undefined),/project-id/); + const indexed=observation();indexed.receipt.observation.tier='indexed';const joined=reconcileSubmissions([submission()],[indexed],'project1');assert.equal(joined.submissions[0].observations[0].participant_indexing_verified,false);assert.equal(joined.submissions[0].verification,'not_established'); + indexed.receipt.observation.reason=' =HYPERLINK("bad")';assert.match(submissionCsv(reconcileSubmissions([submission()],[indexed],'project1')),/"' =HYPERLINK/); +}); +test('real CLI process imports, exports and reconciles offline with retained observation provenance',async t=>{ + const {cwd,path}=await setup(t),entry=new URL('../cli/agentlinkops.mjs',import.meta.url).pathname; + const run=(...args)=>execFileSync(process.execPath,[entry,'index',...args],{cwd,encoding:'utf8',env:{PATH:process.env.PATH,HOME:process.env.HOME}}); + await writeFile(join(cwd,'submissions.json'),JSON.stringify([submission(),submission({id:'attempt2',http_status:202})])); + assert.equal(JSON.parse(run('submissions-import','submissions.json')).total,2);run('submissions-import','submissions.json');assert.equal((await readSubmissions(path)).length,2); + await saveIndexReceipts(join(cwd,'.agentlinkops','index-observations.jsonl'),[observation()]); + run('submissions-export','--project-id','project1','--out','nested/export.json');assert.deepEqual(JSON.parse(await readFile(join(cwd,'nested/export.json'),'utf8')),[submission(),submission({id:'attempt2',http_status:202})]); + const report=JSON.parse(run('reconcile-submissions','--project-id','project1'));assert.equal(report.submissions[0].observations.length,1);assert.match(report.note,/do not verify/); + run('reconcile-submissions','--project-id','project1','--format','csv','--out','nested/report.csv');assert.match(await readFile(join(cwd,'nested/report.csv'),'utf8'),/key_validation_pending/); + assert.deepEqual(JSON.parse(run('reconcile-submissions','--project-id','other')).submissions,[]); +});