From b64712a9906fe150cb52aabda22f2dec45d6c647 Mon Sep 17 00:00:00 2001 From: kzoeps Date: Mon, 28 Sep 2026 10:54:19 +0600 Subject: [PATCH 1/8] hypercerts-api: add actor-follow query handlers --- .../app.certified.graph.getFollow.json | 70 +++++++ ...pp.certified.graph.listActorFollowers.json | 58 ++++++ ...pp.certified.graph.listActorFollowing.json | 58 ++++++ hypercerts-api/lua/endpoints/getFollow.lua | 67 ++++++ .../lua/endpoints/listActorFollowers.lua | 195 ++++++++++++++++++ .../lua/endpoints/listActorFollowing.lua | 195 ++++++++++++++++++ hypercerts-api/lua/shared/actorFollow.lua | 46 +++++ hypercerts-api/lua/shared/actorFollowList.lua | 144 +++++++++++++ .../lua/shared/actorFollowLookup.lua | 16 ++ hypercerts-api/lua/src/getFollow.lua | 3 + hypercerts-api/lua/src/listActorFollowers.lua | 3 + hypercerts-api/lua/src/listActorFollowing.lua | 3 + hypercerts-api/manifest.json | 47 ++++- .../modules/actor-follow/manifest.json | 89 ++++++++ hypercerts-api/tooling/build-lua.js | 16 +- 15 files changed, 998 insertions(+), 12 deletions(-) create mode 100644 hypercerts-api/lexicons/app.certified.graph.getFollow.json create mode 100644 hypercerts-api/lexicons/app.certified.graph.listActorFollowers.json create mode 100644 hypercerts-api/lexicons/app.certified.graph.listActorFollowing.json create mode 100644 hypercerts-api/lua/endpoints/getFollow.lua create mode 100644 hypercerts-api/lua/endpoints/listActorFollowers.lua create mode 100644 hypercerts-api/lua/endpoints/listActorFollowing.lua create mode 100644 hypercerts-api/lua/shared/actorFollow.lua create mode 100644 hypercerts-api/lua/shared/actorFollowList.lua create mode 100644 hypercerts-api/lua/shared/actorFollowLookup.lua create mode 100644 hypercerts-api/lua/src/getFollow.lua create mode 100644 hypercerts-api/lua/src/listActorFollowers.lua create mode 100644 hypercerts-api/lua/src/listActorFollowing.lua create mode 100644 hypercerts-api/modules/actor-follow/manifest.json diff --git a/hypercerts-api/lexicons/app.certified.graph.getFollow.json b/hypercerts-api/lexicons/app.certified.graph.getFollow.json new file mode 100644 index 00000000..1c2abdc2 --- /dev/null +++ b/hypercerts-api/lexicons/app.certified.graph.getFollow.json @@ -0,0 +1,70 @@ +{ + "lexicon": 1, + "id": "app.certified.graph.getFollow", + "defs": { + "main": { + "type": "query", + "description": "Returns the earliest indexed follow record for one actor-to-actor relationship, or null when no matching relationship is indexed. Authentication is not required and the response is not personalized.", + "parameters": { + "type": "params", + "required": ["actor", "subject"], + "properties": { + "actor": { + "type": "string", + "format": "did", + "description": "DID of the actor who published the follow record. Required." + }, + "subject": { + "type": "string", + "format": "did", + "description": "DID of the actor named as the follow record's subject. Required." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { "type": "ref", "ref": "#output" } + }, + "errors": [ + { "name": "InvalidRequest", "description": "A required DID is invalid, repeated, or accompanied by an unknown parameter." } + ] + }, + "followRecordView": { + "type": "object", + "description": "Raw indexed follow record view, including the DID that published the relationship.", + "required": ["uri", "cid", "indexedAt", "did", "record"], + "properties": { + "uri": { "type": "string", "format": "at-uri" }, + "cid": { "type": "string", "format": "cid" }, + "indexedAt": { "type": "string", "format": "datetime" }, + "did": { "type": "string", "format": "did" }, + "record": { "type": "ref", "ref": "app.certified.graph.follow" } + } + }, + "actorFollowView": { + "type": "object", + "description": "The displayed actor and the raw follow record representing its relationship.", + "required": ["did", "profile", "organization", "follow"], + "nullable": ["profile", "organization"], + "properties": { + "did": { "type": "string", "format": "did" }, + "profile": { "type": "ref", "ref": "org.hypercerts.api.defs#profileView" }, + "organization": { "type": "ref", "ref": "org.hypercerts.api.defs#organizationView" }, + "follow": { "type": "ref", "ref": "#followRecordView" } + } + }, + "output": { + "type": "object", + "description": "The raw representative follow record, or explicit null when the relationship is not indexed.", + "required": ["follow"], + "nullable": ["follow"], + "properties": { + "follow": { + "type": "ref", + "ref": "#followRecordView", + "description": "The earliest-created indexed record for the actor and subject pair, with URI and publisher metadata." + } + } + } + } +} diff --git a/hypercerts-api/lexicons/app.certified.graph.listActorFollowers.json b/hypercerts-api/lexicons/app.certified.graph.listActorFollowers.json new file mode 100644 index 00000000..ceb1b024 --- /dev/null +++ b/hypercerts-api/lexicons/app.certified.graph.listActorFollowers.json @@ -0,0 +1,58 @@ +{ + "lexicon": 1, + "id": "app.certified.graph.listActorFollowers", + "defs": { + "main": { + "type": "query", + "description": "Lists actors whose indexed follow records name the requested actor as subject. Authentication is not required and the response is not personalized. Duplicate relationships are collapsed before pagination.", + "parameters": { + "type": "params", + "required": ["actor"], + "properties": { + "actor": { + "type": "string", + "format": "did", + "description": "DID of the actor whose incoming follows are listed. Required." + }, + "sortDirection": { + "type": "string", + "enum": ["asc", "desc"], + "description": "Sort by each representative follow's createdAt and URI in this direction; the default is desc." + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 100, + "description": "Maximum relationships returned; the handler default is 25." + }, + "cursor": { + "type": "string", + "description": "Opaque cursor from the previous page, bound to sortDirection. Keep other query parameters unchanged between pages." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { "type": "ref", "ref": "#output" } + }, + "errors": [ + { "name": "InvalidRequest", "description": "A required DID, sort direction, limit, cursor, or query parameter is invalid." } + ] + }, + "output": { + "type": "object", + "required": ["followers"], + "properties": { + "followers": { + "type": "array", + "items": { "type": "ref", "ref": "app.certified.graph.getFollow#actorFollowView" }, + "description": "Incoming relationships, with each displayed DID identifying the actor who published the follow." + }, + "cursor": { + "type": "string", + "description": "Opaque cursor for the next page; omitted when there are no more relationships." + } + } + } + } +} diff --git a/hypercerts-api/lexicons/app.certified.graph.listActorFollowing.json b/hypercerts-api/lexicons/app.certified.graph.listActorFollowing.json new file mode 100644 index 00000000..95c1a407 --- /dev/null +++ b/hypercerts-api/lexicons/app.certified.graph.listActorFollowing.json @@ -0,0 +1,58 @@ +{ + "lexicon": 1, + "id": "app.certified.graph.listActorFollowing", + "defs": { + "main": { + "type": "query", + "description": "Lists actors named as subjects by the requested actor's indexed follow records. Authentication is not required and the response is not personalized. Duplicate relationships are collapsed before pagination.", + "parameters": { + "type": "params", + "required": ["actor"], + "properties": { + "actor": { + "type": "string", + "format": "did", + "description": "DID of the actor whose outgoing follows are listed. Required." + }, + "sortDirection": { + "type": "string", + "enum": ["asc", "desc"], + "description": "Sort by each representative follow's createdAt and URI in this direction; the default is desc." + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 100, + "description": "Maximum relationships returned; the handler default is 25." + }, + "cursor": { + "type": "string", + "description": "Opaque cursor from the previous page, bound to sortDirection. Keep other query parameters unchanged between pages." + } + } + }, + "output": { + "encoding": "application/json", + "schema": { "type": "ref", "ref": "#output" } + }, + "errors": [ + { "name": "InvalidRequest", "description": "A required DID, sort direction, limit, cursor, or query parameter is invalid." } + ] + }, + "output": { + "type": "object", + "required": ["following"], + "properties": { + "following": { + "type": "array", + "items": { "type": "ref", "ref": "app.certified.graph.getFollow#actorFollowView" }, + "description": "Outgoing relationships, with each displayed DID identifying the actor named as the follow subject." + }, + "cursor": { + "type": "string", + "description": "Opaque cursor for the next page; omitted when there are no more relationships." + } + } + } + } +} diff --git a/hypercerts-api/lua/endpoints/getFollow.lua b/hypercerts-api/lua/endpoints/getFollow.lua new file mode 100644 index 00000000..948aa07f --- /dev/null +++ b/hypercerts-api/lua/endpoints/getFollow.lua @@ -0,0 +1,67 @@ +local FOLLOW = "app.certified.graph.follow" +local NULL = json.decode("null") + +local function invalid(message) + error("InvalidRequest: " .. message, 0) +end + +local function keys_only(values, allowed) + for key in pairs(values) do + if not allowed[key] then invalid("unknown query parameter") end + end +end + +local function scalar(values, key) + local value = values[key] + if value == nil then return nil end + if type(value) ~= "string" and type(value) ~= "number" then + invalid(key .. " must occur once") + end + return tostring(value) +end + +local function valid_did(value) + if #value > 2048 then return false end + local method, specific = value:match("^did:([a-z]+):(.+)$") + if not method or not specific or specific:sub(-1) == ":" or specific:sub(-1) == "%" + or value:find("[^%w%.:_%%%-]") then return false end + return true +end + +local function query(sql, values) + if db.backend() ~= "postgres" then error("ActorFollowQueryFailed: actor-follow queries require PostgreSQL", 0) end + local ok, result = pcall(db.raw, sql, values) + if not ok then error("ActorFollowQueryFailed: actor-follow lookup failed", 0) end + return result +end + +local function row_view(row) + return { + uri = row.uri, + cid = row.cid, + indexedAt = row.indexed_at, + did = row.did, + record = json.decode(row.record), + } +end + +local function query_follow(actor, subject) + local rows = query( + "SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record FROM happyview_records WHERE collection = $1 AND did = $2 AND record::jsonb->>'subject' = $3 ORDER BY (record::jsonb->>'createdAt')::timestamptz ASC, uri ASC LIMIT 1", + { FOLLOW, actor, subject }) + if #rows == 0 then return NULL end + return row_view(rows[1]) +end + +local function get_follow() + keys_only(params, { actor = true, subject = true }) + local actor = scalar(params, "actor") + local subject = scalar(params, "subject") + if not actor or not valid_did(actor) then invalid("actor must be a valid DID") end + if not subject or not valid_did(subject) then invalid("subject must be a valid DID") end + return { follow = query_follow(actor, subject) } +end + +function handle() + return get_follow() +end diff --git a/hypercerts-api/lua/endpoints/listActorFollowers.lua b/hypercerts-api/lua/endpoints/listActorFollowers.lua new file mode 100644 index 00000000..98169e69 --- /dev/null +++ b/hypercerts-api/lua/endpoints/listActorFollowers.lua @@ -0,0 +1,195 @@ +local FOLLOW = "app.certified.graph.follow" +local NULL = json.decode("null") + +local function invalid(message) + error("InvalidRequest: " .. message, 0) +end + +local function keys_only(values, allowed) + for key in pairs(values) do + if not allowed[key] then invalid("unknown query parameter") end + end +end + +local function scalar(values, key) + local value = values[key] + if value == nil then return nil end + if type(value) ~= "string" and type(value) ~= "number" then + invalid(key .. " must occur once") + end + return tostring(value) +end + +local function valid_did(value) + if #value > 2048 then return false end + local method, specific = value:match("^did:([a-z]+):(.+)$") + if not method or not specific or specific:sub(-1) == ":" or specific:sub(-1) == "%" + or value:find("[^%w%.:_%%%-]") then return false end + return true +end + +local function query(sql, values) + if db.backend() ~= "postgres" then error("ActorFollowQueryFailed: actor-follow queries require PostgreSQL", 0) end + local ok, result = pcall(db.raw, sql, values) + if not ok then error("ActorFollowQueryFailed: actor-follow lookup failed", 0) end + return result +end + +local function row_view(row) + return { + uri = row.uri, + cid = row.cid, + indexedAt = row.indexed_at, + did = row.did, + record = json.decode(row.record), + } +end + +local PROFILE = "app.certified.actor.profile" +local ORGANIZATION = "app.certified.actor.organization" + +local function valid_record_key(value) + return #value >= 1 and #value <= 512 and value ~= "." and value ~= ".." + and not value:find("[^%w_~%.:%-]") +end + +local function valid_follow_uri(value) + if type(value) ~= "string" or value:find("[?#]") then return false end + local authority, collection, rkey = value:match("^at://([^/]+)/([^/]+)/([^/]+)$") + return authority ~= nil and valid_did(authority) and collection == FOLLOW and valid_record_key(rkey) +end + +local function valid_datetime(value) + local year, month, day, hour, minute, second, suffix = value:match( + "^(%d%d%d%d)%-(%d%d)%-(%d%d)T(%d%d):(%d%d):(%d%d)(.*)$") + if not year then return false end + year, month, day = tonumber(year), tonumber(month), tonumber(day) + hour, minute, second = tonumber(hour), tonumber(minute), tonumber(second) + if month < 1 or month > 12 or hour > 23 or minute > 59 or second > 59 then return false end + local leap = year % 4 == 0 and (year % 100 ~= 0 or year % 400 == 0) + local month_days = { 31, leap and 29 or 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 } + if day < 1 or day > month_days[month] then return false end + local fraction, zone = suffix:match("^(%.%d+)(Z)$") + if not fraction then fraction, zone = suffix:match("^(%.%d+)([+-]%d%d:%d%d)$") end + if not fraction then zone = suffix:match("^(Z)$") end + if not zone then zone = suffix:match("^([+-]%d%d:%d%d)$") end + if not zone or zone == "-00:00" then return false end + if zone ~= "Z" then + local zh, zm = zone:match("^[+-](%d%d):(%d%d)$") + if not zh or tonumber(zh) > 23 or tonumber(zm) > 59 then return false end + end + return true +end + +local function hydrate(views) + if #views == 0 then return end + local dids, seen = {}, {} + for _, view in ipairs(views) do + if not seen[view.did] then + seen[view.did] = true + dids[#dids + 1] = view.did + end + end + local profiles, organizations = {}, {} + local function load(collection, target) + local values, marks = { collection }, {} + for _, did in ipairs(dids) do + values[#values + 1] = did + marks[#marks + 1] = "$" .. #values + end + local rows = query("SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record FROM happyview_records WHERE collection = $1 AND rkey = 'self' AND did IN (" .. table.concat(marks, ",") .. ")", values) + for _, row in ipairs(rows) do target[row.did] = row end + end + load(PROFILE, profiles) + load(ORGANIZATION, organizations) + for _, view in ipairs(views) do + local profile, organization = profiles[view.did], organizations[view.did] + view.profile = profile and row_view(profile) or NULL + view.organization = organization and row_view(organization) or NULL + end +end + +local function cursor_encode(value) + local encoded = json.encode(value) + return (encoded:gsub(".", function(char) return string.format("%02x", string.byte(char)) end)) +end + +local function cursor_decode(token, direction) + if not token then return nil end + if #token % 2 ~= 0 or token:find("[^0-9a-f]") then invalid("cursor is malformed") end + local decoded = token:gsub("..", function(pair) return string.char(tonumber(pair, 16)) end) + local ok, value = pcall(json.decode, decoded) + if not ok or type(value) ~= "table" or value.v ~= 1 or value.d ~= direction + or type(value.t) ~= "string" or type(value.u) ~= "string" then + invalid("cursor is malformed or belongs to another sortDirection") + end + for key in pairs(value) do + if key ~= "v" and key ~= "d" and key ~= "t" and key ~= "u" then invalid("cursor is malformed") end + end + if not valid_follow_uri(value.u) or not valid_datetime(value.t) then invalid("cursor is malformed") end + return value +end + +local function query_actor_follows(mode, actor, limit, cursor, direction) + local filters, values = { "collection = $1" }, { FOLLOW } + values[#values + 1] = actor + if mode == "followers" then + filters[#filters + 1] = "record::jsonb->>'subject' = $2" + else + filters[#filters + 1] = "did = $2" + end + + local cursor_filter = "" + if cursor then + values[#values + 1] = cursor.t + local timestamp = "$" .. #values .. "::timestamptz" + values[#values + 1] = cursor.u + local uri = "$" .. #values + local operator = direction == "asc" and ">" or "<" + cursor_filter = " WHERE (sort_at, uri) " .. operator .. " (" .. timestamp .. ", " .. uri .. ")" + end + + values[#values + 1] = limit + 1 + local ordering = direction == "asc" and "ASC" or "DESC" + local displayed_did = mode == "followers" and "did" or "subject_did" + local sql = "WITH ranked_follows AS (SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record, record::jsonb->>'subject' AS subject_did, (record::jsonb->>'createdAt')::timestamptz AS sort_at, ROW_NUMBER() OVER (PARTITION BY did, record::jsonb->>'subject' ORDER BY (record::jsonb->>'createdAt')::timestamptz ASC, uri ASC) AS relationship_rank FROM happyview_records WHERE " .. table.concat(filters, " AND ") .. "), representatives AS (SELECT uri, did, cid, indexed_at, record, sort_at, " .. displayed_did .. " AS actor_did FROM ranked_follows WHERE relationship_rank = 1) SELECT uri, did, cid, indexed_at, record, actor_did, to_char(sort_at AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') AS sort_timestamp FROM representatives" .. cursor_filter .. " ORDER BY sort_at " .. ordering .. ", uri " .. ordering .. " LIMIT $" .. #values + local rows = query(sql, values) + local more = #rows > limit + if more then rows[#rows] = nil end + + local views = {} + for _, row in ipairs(rows) do + views[#views + 1] = { did = row.actor_did, follow = row_view(row) } + end + hydrate(views) + + local next_cursor + if more then + local last = rows[#rows] + next_cursor = cursor_encode({ v = 1, d = direction, t = last.sort_timestamp, u = last.uri }) + end + return views, next_cursor +end + +local function list_actor_follows(mode) + keys_only(params, { actor = true, sortDirection = true, limit = true, cursor = true }) + local actor = scalar(params, "actor") + if not actor or not valid_did(actor) then invalid("actor must be a valid DID") end + + local limit_value = scalar(params, "limit") + if limit_value and not limit_value:match("^%d+$") then invalid("limit must be an integer from 1 through 100") end + local limit = limit_value and tonumber(limit_value) or 25 + if not limit or limit % 1 ~= 0 or limit < 1 or limit > 100 then invalid("limit must be an integer from 1 through 100") end + + local direction = scalar(params, "sortDirection") or "desc" + if direction ~= "asc" and direction ~= "desc" then invalid("sortDirection must be 'asc' or 'desc'") end + local cursor = cursor_decode(scalar(params, "cursor"), direction) + local views, next_cursor = query_actor_follows(mode, actor, limit, cursor, direction) + local response = { [mode] = toarray(views) } + if next_cursor then response.cursor = next_cursor end + return response +end + +function handle() + return list_actor_follows("followers") +end diff --git a/hypercerts-api/lua/endpoints/listActorFollowing.lua b/hypercerts-api/lua/endpoints/listActorFollowing.lua new file mode 100644 index 00000000..b0cb8759 --- /dev/null +++ b/hypercerts-api/lua/endpoints/listActorFollowing.lua @@ -0,0 +1,195 @@ +local FOLLOW = "app.certified.graph.follow" +local NULL = json.decode("null") + +local function invalid(message) + error("InvalidRequest: " .. message, 0) +end + +local function keys_only(values, allowed) + for key in pairs(values) do + if not allowed[key] then invalid("unknown query parameter") end + end +end + +local function scalar(values, key) + local value = values[key] + if value == nil then return nil end + if type(value) ~= "string" and type(value) ~= "number" then + invalid(key .. " must occur once") + end + return tostring(value) +end + +local function valid_did(value) + if #value > 2048 then return false end + local method, specific = value:match("^did:([a-z]+):(.+)$") + if not method or not specific or specific:sub(-1) == ":" or specific:sub(-1) == "%" + or value:find("[^%w%.:_%%%-]") then return false end + return true +end + +local function query(sql, values) + if db.backend() ~= "postgres" then error("ActorFollowQueryFailed: actor-follow queries require PostgreSQL", 0) end + local ok, result = pcall(db.raw, sql, values) + if not ok then error("ActorFollowQueryFailed: actor-follow lookup failed", 0) end + return result +end + +local function row_view(row) + return { + uri = row.uri, + cid = row.cid, + indexedAt = row.indexed_at, + did = row.did, + record = json.decode(row.record), + } +end + +local PROFILE = "app.certified.actor.profile" +local ORGANIZATION = "app.certified.actor.organization" + +local function valid_record_key(value) + return #value >= 1 and #value <= 512 and value ~= "." and value ~= ".." + and not value:find("[^%w_~%.:%-]") +end + +local function valid_follow_uri(value) + if type(value) ~= "string" or value:find("[?#]") then return false end + local authority, collection, rkey = value:match("^at://([^/]+)/([^/]+)/([^/]+)$") + return authority ~= nil and valid_did(authority) and collection == FOLLOW and valid_record_key(rkey) +end + +local function valid_datetime(value) + local year, month, day, hour, minute, second, suffix = value:match( + "^(%d%d%d%d)%-(%d%d)%-(%d%d)T(%d%d):(%d%d):(%d%d)(.*)$") + if not year then return false end + year, month, day = tonumber(year), tonumber(month), tonumber(day) + hour, minute, second = tonumber(hour), tonumber(minute), tonumber(second) + if month < 1 or month > 12 or hour > 23 or minute > 59 or second > 59 then return false end + local leap = year % 4 == 0 and (year % 100 ~= 0 or year % 400 == 0) + local month_days = { 31, leap and 29 or 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 } + if day < 1 or day > month_days[month] then return false end + local fraction, zone = suffix:match("^(%.%d+)(Z)$") + if not fraction then fraction, zone = suffix:match("^(%.%d+)([+-]%d%d:%d%d)$") end + if not fraction then zone = suffix:match("^(Z)$") end + if not zone then zone = suffix:match("^([+-]%d%d:%d%d)$") end + if not zone or zone == "-00:00" then return false end + if zone ~= "Z" then + local zh, zm = zone:match("^[+-](%d%d):(%d%d)$") + if not zh or tonumber(zh) > 23 or tonumber(zm) > 59 then return false end + end + return true +end + +local function hydrate(views) + if #views == 0 then return end + local dids, seen = {}, {} + for _, view in ipairs(views) do + if not seen[view.did] then + seen[view.did] = true + dids[#dids + 1] = view.did + end + end + local profiles, organizations = {}, {} + local function load(collection, target) + local values, marks = { collection }, {} + for _, did in ipairs(dids) do + values[#values + 1] = did + marks[#marks + 1] = "$" .. #values + end + local rows = query("SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record FROM happyview_records WHERE collection = $1 AND rkey = 'self' AND did IN (" .. table.concat(marks, ",") .. ")", values) + for _, row in ipairs(rows) do target[row.did] = row end + end + load(PROFILE, profiles) + load(ORGANIZATION, organizations) + for _, view in ipairs(views) do + local profile, organization = profiles[view.did], organizations[view.did] + view.profile = profile and row_view(profile) or NULL + view.organization = organization and row_view(organization) or NULL + end +end + +local function cursor_encode(value) + local encoded = json.encode(value) + return (encoded:gsub(".", function(char) return string.format("%02x", string.byte(char)) end)) +end + +local function cursor_decode(token, direction) + if not token then return nil end + if #token % 2 ~= 0 or token:find("[^0-9a-f]") then invalid("cursor is malformed") end + local decoded = token:gsub("..", function(pair) return string.char(tonumber(pair, 16)) end) + local ok, value = pcall(json.decode, decoded) + if not ok or type(value) ~= "table" or value.v ~= 1 or value.d ~= direction + or type(value.t) ~= "string" or type(value.u) ~= "string" then + invalid("cursor is malformed or belongs to another sortDirection") + end + for key in pairs(value) do + if key ~= "v" and key ~= "d" and key ~= "t" and key ~= "u" then invalid("cursor is malformed") end + end + if not valid_follow_uri(value.u) or not valid_datetime(value.t) then invalid("cursor is malformed") end + return value +end + +local function query_actor_follows(mode, actor, limit, cursor, direction) + local filters, values = { "collection = $1" }, { FOLLOW } + values[#values + 1] = actor + if mode == "followers" then + filters[#filters + 1] = "record::jsonb->>'subject' = $2" + else + filters[#filters + 1] = "did = $2" + end + + local cursor_filter = "" + if cursor then + values[#values + 1] = cursor.t + local timestamp = "$" .. #values .. "::timestamptz" + values[#values + 1] = cursor.u + local uri = "$" .. #values + local operator = direction == "asc" and ">" or "<" + cursor_filter = " WHERE (sort_at, uri) " .. operator .. " (" .. timestamp .. ", " .. uri .. ")" + end + + values[#values + 1] = limit + 1 + local ordering = direction == "asc" and "ASC" or "DESC" + local displayed_did = mode == "followers" and "did" or "subject_did" + local sql = "WITH ranked_follows AS (SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record, record::jsonb->>'subject' AS subject_did, (record::jsonb->>'createdAt')::timestamptz AS sort_at, ROW_NUMBER() OVER (PARTITION BY did, record::jsonb->>'subject' ORDER BY (record::jsonb->>'createdAt')::timestamptz ASC, uri ASC) AS relationship_rank FROM happyview_records WHERE " .. table.concat(filters, " AND ") .. "), representatives AS (SELECT uri, did, cid, indexed_at, record, sort_at, " .. displayed_did .. " AS actor_did FROM ranked_follows WHERE relationship_rank = 1) SELECT uri, did, cid, indexed_at, record, actor_did, to_char(sort_at AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') AS sort_timestamp FROM representatives" .. cursor_filter .. " ORDER BY sort_at " .. ordering .. ", uri " .. ordering .. " LIMIT $" .. #values + local rows = query(sql, values) + local more = #rows > limit + if more then rows[#rows] = nil end + + local views = {} + for _, row in ipairs(rows) do + views[#views + 1] = { did = row.actor_did, follow = row_view(row) } + end + hydrate(views) + + local next_cursor + if more then + local last = rows[#rows] + next_cursor = cursor_encode({ v = 1, d = direction, t = last.sort_timestamp, u = last.uri }) + end + return views, next_cursor +end + +local function list_actor_follows(mode) + keys_only(params, { actor = true, sortDirection = true, limit = true, cursor = true }) + local actor = scalar(params, "actor") + if not actor or not valid_did(actor) then invalid("actor must be a valid DID") end + + local limit_value = scalar(params, "limit") + if limit_value and not limit_value:match("^%d+$") then invalid("limit must be an integer from 1 through 100") end + local limit = limit_value and tonumber(limit_value) or 25 + if not limit or limit % 1 ~= 0 or limit < 1 or limit > 100 then invalid("limit must be an integer from 1 through 100") end + + local direction = scalar(params, "sortDirection") or "desc" + if direction ~= "asc" and direction ~= "desc" then invalid("sortDirection must be 'asc' or 'desc'") end + local cursor = cursor_decode(scalar(params, "cursor"), direction) + local views, next_cursor = query_actor_follows(mode, actor, limit, cursor, direction) + local response = { [mode] = toarray(views) } + if next_cursor then response.cursor = next_cursor end + return response +end + +function handle() + return list_actor_follows("following") +end diff --git a/hypercerts-api/lua/shared/actorFollow.lua b/hypercerts-api/lua/shared/actorFollow.lua new file mode 100644 index 00000000..5fd594b6 --- /dev/null +++ b/hypercerts-api/lua/shared/actorFollow.lua @@ -0,0 +1,46 @@ +local FOLLOW = "app.certified.graph.follow" +local NULL = json.decode("null") + +local function invalid(message) + error("InvalidRequest: " .. message, 0) +end + +local function keys_only(values, allowed) + for key in pairs(values) do + if not allowed[key] then invalid("unknown query parameter") end + end +end + +local function scalar(values, key) + local value = values[key] + if value == nil then return nil end + if type(value) ~= "string" and type(value) ~= "number" then + invalid(key .. " must occur once") + end + return tostring(value) +end + +local function valid_did(value) + if #value > 2048 then return false end + local method, specific = value:match("^did:([a-z]+):(.+)$") + if not method or not specific or specific:sub(-1) == ":" or specific:sub(-1) == "%" + or value:find("[^%w%.:_%%%-]") then return false end + return true +end + +local function query(sql, values) + if db.backend() ~= "postgres" then error("ActorFollowQueryFailed: actor-follow queries require PostgreSQL", 0) end + local ok, result = pcall(db.raw, sql, values) + if not ok then error("ActorFollowQueryFailed: actor-follow lookup failed", 0) end + return result +end + +local function row_view(row) + return { + uri = row.uri, + cid = row.cid, + indexedAt = row.indexed_at, + did = row.did, + record = json.decode(row.record), + } +end diff --git a/hypercerts-api/lua/shared/actorFollowList.lua b/hypercerts-api/lua/shared/actorFollowList.lua new file mode 100644 index 00000000..83aa45ef --- /dev/null +++ b/hypercerts-api/lua/shared/actorFollowList.lua @@ -0,0 +1,144 @@ +local PROFILE = "app.certified.actor.profile" +local ORGANIZATION = "app.certified.actor.organization" + +local function valid_record_key(value) + return #value >= 1 and #value <= 512 and value ~= "." and value ~= ".." + and not value:find("[^%w_~%.:%-]") +end + +local function valid_follow_uri(value) + if type(value) ~= "string" or value:find("[?#]") then return false end + local authority, collection, rkey = value:match("^at://([^/]+)/([^/]+)/([^/]+)$") + return authority ~= nil and valid_did(authority) and collection == FOLLOW and valid_record_key(rkey) +end + +local function valid_datetime(value) + local year, month, day, hour, minute, second, suffix = value:match( + "^(%d%d%d%d)%-(%d%d)%-(%d%d)T(%d%d):(%d%d):(%d%d)(.*)$") + if not year then return false end + year, month, day = tonumber(year), tonumber(month), tonumber(day) + hour, minute, second = tonumber(hour), tonumber(minute), tonumber(second) + if month < 1 or month > 12 or hour > 23 or minute > 59 or second > 59 then return false end + local leap = year % 4 == 0 and (year % 100 ~= 0 or year % 400 == 0) + local month_days = { 31, leap and 29 or 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 } + if day < 1 or day > month_days[month] then return false end + local fraction, zone = suffix:match("^(%.%d+)(Z)$") + if not fraction then fraction, zone = suffix:match("^(%.%d+)([+-]%d%d:%d%d)$") end + if not fraction then zone = suffix:match("^(Z)$") end + if not zone then zone = suffix:match("^([+-]%d%d:%d%d)$") end + if not zone or zone == "-00:00" then return false end + if zone ~= "Z" then + local zh, zm = zone:match("^[+-](%d%d):(%d%d)$") + if not zh or tonumber(zh) > 23 or tonumber(zm) > 59 then return false end + end + return true +end + +local function hydrate(views) + if #views == 0 then return end + local dids, seen = {}, {} + for _, view in ipairs(views) do + if not seen[view.did] then + seen[view.did] = true + dids[#dids + 1] = view.did + end + end + local profiles, organizations = {}, {} + local function load(collection, target) + local values, marks = { collection }, {} + for _, did in ipairs(dids) do + values[#values + 1] = did + marks[#marks + 1] = "$" .. #values + end + local rows = query("SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record FROM happyview_records WHERE collection = $1 AND rkey = 'self' AND did IN (" .. table.concat(marks, ",") .. ")", values) + for _, row in ipairs(rows) do target[row.did] = row end + end + load(PROFILE, profiles) + load(ORGANIZATION, organizations) + for _, view in ipairs(views) do + local profile, organization = profiles[view.did], organizations[view.did] + view.profile = profile and row_view(profile) or NULL + view.organization = organization and row_view(organization) or NULL + end +end + +local function cursor_encode(value) + local encoded = json.encode(value) + return (encoded:gsub(".", function(char) return string.format("%02x", string.byte(char)) end)) +end + +local function cursor_decode(token, direction) + if not token then return nil end + if #token % 2 ~= 0 or token:find("[^0-9a-f]") then invalid("cursor is malformed") end + local decoded = token:gsub("..", function(pair) return string.char(tonumber(pair, 16)) end) + local ok, value = pcall(json.decode, decoded) + if not ok or type(value) ~= "table" or value.v ~= 1 or value.d ~= direction + or type(value.t) ~= "string" or type(value.u) ~= "string" then + invalid("cursor is malformed or belongs to another sortDirection") + end + for key in pairs(value) do + if key ~= "v" and key ~= "d" and key ~= "t" and key ~= "u" then invalid("cursor is malformed") end + end + if not valid_follow_uri(value.u) or not valid_datetime(value.t) then invalid("cursor is malformed") end + return value +end + +local function query_actor_follows(mode, actor, limit, cursor, direction) + local filters, values = { "collection = $1" }, { FOLLOW } + values[#values + 1] = actor + if mode == "followers" then + filters[#filters + 1] = "record::jsonb->>'subject' = $2" + else + filters[#filters + 1] = "did = $2" + end + + local cursor_filter = "" + if cursor then + values[#values + 1] = cursor.t + local timestamp = "$" .. #values .. "::timestamptz" + values[#values + 1] = cursor.u + local uri = "$" .. #values + local operator = direction == "asc" and ">" or "<" + cursor_filter = " WHERE (sort_at, uri) " .. operator .. " (" .. timestamp .. ", " .. uri .. ")" + end + + values[#values + 1] = limit + 1 + local ordering = direction == "asc" and "ASC" or "DESC" + local displayed_did = mode == "followers" and "did" or "subject_did" + local sql = "WITH ranked_follows AS (SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record, record::jsonb->>'subject' AS subject_did, (record::jsonb->>'createdAt')::timestamptz AS sort_at, ROW_NUMBER() OVER (PARTITION BY did, record::jsonb->>'subject' ORDER BY (record::jsonb->>'createdAt')::timestamptz ASC, uri ASC) AS relationship_rank FROM happyview_records WHERE " .. table.concat(filters, " AND ") .. "), representatives AS (SELECT uri, did, cid, indexed_at, record, sort_at, " .. displayed_did .. " AS actor_did FROM ranked_follows WHERE relationship_rank = 1) SELECT uri, did, cid, indexed_at, record, actor_did, to_char(sort_at AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') AS sort_timestamp FROM representatives" .. cursor_filter .. " ORDER BY sort_at " .. ordering .. ", uri " .. ordering .. " LIMIT $" .. #values + local rows = query(sql, values) + local more = #rows > limit + if more then rows[#rows] = nil end + + local views = {} + for _, row in ipairs(rows) do + views[#views + 1] = { did = row.actor_did, follow = row_view(row) } + end + hydrate(views) + + local next_cursor + if more then + local last = rows[#rows] + next_cursor = cursor_encode({ v = 1, d = direction, t = last.sort_timestamp, u = last.uri }) + end + return views, next_cursor +end + +local function list_actor_follows(mode) + keys_only(params, { actor = true, sortDirection = true, limit = true, cursor = true }) + local actor = scalar(params, "actor") + if not actor or not valid_did(actor) then invalid("actor must be a valid DID") end + + local limit_value = scalar(params, "limit") + if limit_value and not limit_value:match("^%d+$") then invalid("limit must be an integer from 1 through 100") end + local limit = limit_value and tonumber(limit_value) or 25 + if not limit or limit % 1 ~= 0 or limit < 1 or limit > 100 then invalid("limit must be an integer from 1 through 100") end + + local direction = scalar(params, "sortDirection") or "desc" + if direction ~= "asc" and direction ~= "desc" then invalid("sortDirection must be 'asc' or 'desc'") end + local cursor = cursor_decode(scalar(params, "cursor"), direction) + local views, next_cursor = query_actor_follows(mode, actor, limit, cursor, direction) + local response = { [mode] = toarray(views) } + if next_cursor then response.cursor = next_cursor end + return response +end diff --git a/hypercerts-api/lua/shared/actorFollowLookup.lua b/hypercerts-api/lua/shared/actorFollowLookup.lua new file mode 100644 index 00000000..ad3147e4 --- /dev/null +++ b/hypercerts-api/lua/shared/actorFollowLookup.lua @@ -0,0 +1,16 @@ +local function query_follow(actor, subject) + local rows = query( + "SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record FROM happyview_records WHERE collection = $1 AND did = $2 AND record::jsonb->>'subject' = $3 ORDER BY (record::jsonb->>'createdAt')::timestamptz ASC, uri ASC LIMIT 1", + { FOLLOW, actor, subject }) + if #rows == 0 then return NULL end + return row_view(rows[1]) +end + +local function get_follow() + keys_only(params, { actor = true, subject = true }) + local actor = scalar(params, "actor") + local subject = scalar(params, "subject") + if not actor or not valid_did(actor) then invalid("actor must be a valid DID") end + if not subject or not valid_did(subject) then invalid("subject must be a valid DID") end + return { follow = query_follow(actor, subject) } +end diff --git a/hypercerts-api/lua/src/getFollow.lua b/hypercerts-api/lua/src/getFollow.lua new file mode 100644 index 00000000..4806e844 --- /dev/null +++ b/hypercerts-api/lua/src/getFollow.lua @@ -0,0 +1,3 @@ +function handle() + return get_follow() +end diff --git a/hypercerts-api/lua/src/listActorFollowers.lua b/hypercerts-api/lua/src/listActorFollowers.lua new file mode 100644 index 00000000..9d81c739 --- /dev/null +++ b/hypercerts-api/lua/src/listActorFollowers.lua @@ -0,0 +1,3 @@ +function handle() + return list_actor_follows("followers") +end diff --git a/hypercerts-api/lua/src/listActorFollowing.lua b/hypercerts-api/lua/src/listActorFollowing.lua new file mode 100644 index 00000000..dd383e03 --- /dev/null +++ b/hypercerts-api/lua/src/listActorFollowing.lua @@ -0,0 +1,3 @@ +function handle() + return list_actor_follows("following") +end diff --git a/hypercerts-api/manifest.json b/hypercerts-api/manifest.json index 3fa86f90..30c5981a 100644 --- a/hypercerts-api/manifest.json +++ b/hypercerts-api/manifest.json @@ -1,22 +1,36 @@ { - "name": "hypercerts-location-api", + "name": "hypercerts-location-actor-follow-api", "collection": "app.certified.location", "targetHappyViewRevision": "a2f347e605d7cd9c11669b320e43c10dbfe982e3", - "runtimeCompatibility": "pending-runtime-validation", - "status": "handlers-implemented-runtime-validation-pending", + "runtimeCompatibility": "compatible-at-target-revision", + "status": "handlers-implemented-runtime-validation-passed-at-target-revision", "handlerStatus": { "getLocation": "implemented", - "listLocations": "implemented" + "listLocations": "implemented", + "getFollow": "implemented", + "listActorFollowers": "implemented", + "listActorFollowing": "implemented" }, "authentication": { "decision": "public", "unresolved": false, - "note": "Both endpoints expose public record reads and require no caller authentication." + "note": "The location and actor-follow query endpoints expose public record reads and require no caller authentication." }, "luaBuild": { "command": "pnpm build:lua", - "sharedSource": "lua/shared/location.lua", - "endpointSources": ["lua/src/getLocation.lua", "lua/src/listLocations.lua"], + "sharedSources": [ + "lua/shared/location.lua", + "lua/shared/actorFollow.lua", + "lua/shared/actorFollowLookup.lua", + "lua/shared/actorFollowList.lua" + ], + "endpointSources": [ + "lua/src/getLocation.lua", + "lua/src/listLocations.lua", + "lua/src/getFollow.lua", + "lua/src/listActorFollowers.lua", + "lua/src/listActorFollowing.lua" + ], "installedHandlers": "lua/endpoints/" }, "validationLexicons": [ @@ -28,6 +42,22 @@ "id": "org.hypercerts.api.defs", "path": "lexicons/org.hypercerts.api.defs.json" }, + { + "id": "app.certified.graph.follow", + "packagePath": "lexicons/app/certified/graph/follow.json" + }, + { + "id": "app.certified.graph.getFollow", + "path": "lexicons/app.certified.graph.getFollow.json" + }, + { + "id": "app.certified.graph.listActorFollowers", + "path": "lexicons/app.certified.graph.listActorFollowers.json" + }, + { + "id": "app.certified.graph.listActorFollowing", + "path": "lexicons/app.certified.graph.listActorFollowing.json" + }, { "id": "app.certified.location.getLocation", "path": "lexicons/app.certified.location.getLocation.json" @@ -162,6 +192,7 @@ } ], "modules": [ - "modules/location/manifest.json" + "modules/location/manifest.json", + "modules/actor-follow/manifest.json" ] } diff --git a/hypercerts-api/modules/actor-follow/manifest.json b/hypercerts-api/modules/actor-follow/manifest.json new file mode 100644 index 00000000..259f9feb --- /dev/null +++ b/hypercerts-api/modules/actor-follow/manifest.json @@ -0,0 +1,89 @@ +{ + "assets": [ + { + "kind": "lexicon", + "id": "app.certified.graph.follow", + "packagePath": "lexicons/app/certified/graph/follow.json", + "config": { + "backfill": true + }, + "dependsOn": ["app.certified.signature.defs"], + "registrationGroup": "actor-follow-record-schema-sources" + }, + { + "kind": "lexicon", + "id": "app.certified.graph.getFollow", + "path": "../../lexicons/app.certified.graph.getFollow.json", + "config": { + "backfill": false, + "target_collection": "app.certified.graph.follow" + }, + "dependsOn": ["app.certified.graph.follow", "org.hypercerts.api.defs"] + }, + { + "kind": "lexicon", + "id": "app.certified.graph.listActorFollowers", + "path": "../../lexicons/app.certified.graph.listActorFollowers.json", + "config": { + "backfill": false, + "target_collection": "app.certified.graph.follow" + }, + "dependsOn": ["app.certified.graph.getFollow"] + }, + { + "kind": "lexicon", + "id": "app.certified.graph.listActorFollowing", + "path": "../../lexicons/app.certified.graph.listActorFollowing.json", + "config": { + "backfill": false, + "target_collection": "app.certified.graph.follow" + }, + "dependsOn": ["app.certified.graph.getFollow"] + }, + { + "kind": "script", + "id": "xrpc.query:app.certified.graph.getFollow", + "path": "../../lua/endpoints/getFollow.lua", + "sourcePath": "../../lua/src/getFollow.lua", + "sharedSourcePaths": [ + "../../lua/shared/actorFollow.lua", + "../../lua/shared/actorFollowLookup.lua" + ], + "config": { + "script_type": "lua", + "description": "Actor follow API handler" + }, + "dependsOn": ["app.certified.graph.getFollow"] + }, + { + "kind": "script", + "id": "xrpc.query:app.certified.graph.listActorFollowers", + "path": "../../lua/endpoints/listActorFollowers.lua", + "sourcePath": "../../lua/src/listActorFollowers.lua", + "sharedSourcePaths": [ + "../../lua/shared/actorFollow.lua", + "../../lua/shared/actorFollowList.lua" + ], + "config": { + "script_type": "lua", + "description": "Actor follow API handler" + }, + "dependsOn": ["app.certified.graph.listActorFollowers"] + }, + { + "kind": "script", + "id": "xrpc.query:app.certified.graph.listActorFollowing", + "path": "../../lua/endpoints/listActorFollowing.lua", + "sourcePath": "../../lua/src/listActorFollowing.lua", + "sharedSourcePaths": [ + "../../lua/shared/actorFollow.lua", + "../../lua/shared/actorFollowList.lua" + ], + "config": { + "script_type": "lua", + "description": "Actor follow API handler" + }, + "dependsOn": ["app.certified.graph.listActorFollowing"] + } + ] +} diff --git a/hypercerts-api/tooling/build-lua.js b/hypercerts-api/tooling/build-lua.js index 53b3a6eb..8f90ed50 100644 --- a/hypercerts-api/tooling/build-lua.js +++ b/hypercerts-api/tooling/build-lua.js @@ -3,8 +3,16 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; const root = fileURLToPath(new URL('../', import.meta.url)); -const shared = await readFile(path.join(root, 'lua/shared/location.lua'), 'utf8'); -for (const name of ['getLocation', 'listLocations']) { - const endpoint = await readFile(path.join(root, `lua/src/${name}.lua`), 'utf8'); - await writeFile(path.join(root, `lua/endpoints/${name}.lua`), `${shared.trimEnd()}\n\n${endpoint}`); +const bundles = [ + { shared: ['location'], endpoints: ['getLocation', 'listLocations'] }, + { shared: ['actorFollow', 'actorFollowLookup'], endpoints: ['getFollow'] }, + { shared: ['actorFollow', 'actorFollowList'], endpoints: ['listActorFollowers', 'listActorFollowing'] }, +]; + +for (const { shared, endpoints } of bundles) { + const sharedSources = await Promise.all(shared.map((name) => readFile(path.join(root, `lua/shared/${name}.lua`), 'utf8'))); + for (const name of endpoints) { + const endpoint = await readFile(path.join(root, `lua/src/${name}.lua`), 'utf8'); + await writeFile(path.join(root, `lua/endpoints/${name}.lua`), `${[...sharedSources, endpoint].map((source) => source.trimEnd()).join('\n\n')}\n`); + } } From c64508d44098ca04d8fc229183601480c35bd5fa Mon Sep 17 00:00:00 2001 From: kzoeps Date: Mon, 28 Sep 2026 10:54:34 +0600 Subject: [PATCH 2/8] hypercerts-api: add actor-follow fixtures and tests --- .../contracts/actor-follow.contract.test.js | 150 ++++++++ .../tests/fixtures/actor-follows.js | 86 +++++ .../tests/fixtures/fixtures.test.js | 42 ++- hypercerts-api/tooling/actor-follow.test.js | 323 ++++++++++++++++++ hypercerts-api/tooling/seed.js | 6 +- hypercerts-api/tooling/seed.test.js | 12 +- 6 files changed, 614 insertions(+), 5 deletions(-) create mode 100644 hypercerts-api/tests/contracts/actor-follow.contract.test.js create mode 100644 hypercerts-api/tests/fixtures/actor-follows.js create mode 100644 hypercerts-api/tooling/actor-follow.test.js diff --git a/hypercerts-api/tests/contracts/actor-follow.contract.test.js b/hypercerts-api/tests/contracts/actor-follow.contract.test.js new file mode 100644 index 00000000..977dd5d1 --- /dev/null +++ b/hypercerts-api/tests/contracts/actor-follow.contract.test.js @@ -0,0 +1,150 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { + actorFollowDids, + actorFollowOrganizationRecords, + actorFollowProfileRecords, + actorFollowRecords, +} from '../fixtures/actor-follows.js'; +import { contractUrl, requireContractTarget } from './helpers.js'; + +const baseUrl = requireContractTarget(); +const followCollection = 'app.certified.graph.follow'; +const followByRkey = new Map(actorFollowRecords.map((record) => [record.rkey, record])); + +function row(rkey) { + const record = followByRkey.get(rkey); + assert.ok(record, `missing actor-follow fixture ${rkey}`); + return record; +} + +function recordView(record) { + return { + uri: record.uri, + cid: record.cid, + indexedAt: record.indexedAt, + did: record.did, + record: record.record, + }; +} + +function actorView(did, follow, profile, organization) { + return { + did, + profile: profile ? recordView(profile) : null, + organization: organization ? recordView(organization) : null, + follow: recordView(follow), + }; +} + +function sidecar(records, did) { + return records.find((record) => record.did === did); +} + +async function get(nsid, params) { + const response = await fetch(contractUrl(baseUrl, nsid, params)); + const body = await response.json(); + assert.equal(response.status, 200, `${nsid}: ${JSON.stringify(body)}`); + return body; +} + +async function collectPages(nsid, outputKey, actor, direction, pageCount) { + const items = []; + let cursor; + for (let pageIndex = 0; pageIndex < pageCount; pageIndex += 1) { + const page = await get(nsid, { actor, limit: 1, sortDirection: direction, cursor }); + assert.equal(page[outputKey].length, 1, `${nsid} ${direction} page ${pageIndex + 1} should contain one relationship`); + items.push(page[outputKey][0]); + const hasNextPage = pageIndex < pageCount - 1; + if (hasNextPage) { + assert.ok(page.cursor, `${nsid} ${direction} page ${pageIndex + 1} should return a cursor`); + cursor = page.cursor; + } else { + assert.equal(page.cursor, undefined, `${nsid} ${direction} final page should not return a cursor`); + } + } + return items; +} + +test('getFollow returns the earliest URI-tiebroken record unchanged and null for a missing pair', async () => { + const { publisher, primarySubject, otherPublisher, secondSubject } = actorFollowDids; + const result = await get('app.certified.graph.getFollow', { actor: publisher, subject: primarySubject }); + const representative = row('3jzfcijpj2z2b'); + assert.deepEqual(result, { follow: recordView(representative) }); + assert.deepEqual(result.follow.record.via, representative.record.via); + assert.equal(result.follow.did, publisher); + + const missing = await get('app.certified.graph.getFollow', { actor: otherPublisher, subject: secondSubject }); + assert.deepEqual(missing, { follow: null }); +}); + +test('follower and following queries preserve direction, publisher metadata, and nullable hydration', async () => { + const { publisher, otherPublisher, primarySubject, secondSubject, thirdSubject } = actorFollowDids; + const incoming = await get('app.certified.graph.listActorFollowers', { + actor: primarySubject, sortDirection: 'asc', limit: 100, + }); + assert.deepEqual(incoming.followers, [ + actorView( + publisher, + row('3jzfcijpj2z2b'), + sidecar(actorFollowProfileRecords, publisher), + sidecar(actorFollowOrganizationRecords, publisher), + ), + actorView(otherPublisher, row('3jzfcijpj2z2g'), undefined, undefined), + ]); + assert.equal(incoming.cursor, undefined); + + const outgoing = await get('app.certified.graph.listActorFollowing', { + actor: publisher, sortDirection: 'asc', limit: 100, + }); + assert.deepEqual(outgoing.following, [ + actorView( + primarySubject, + row('3jzfcijpj2z2b'), + sidecar(actorFollowProfileRecords, primarySubject), + sidecar(actorFollowOrganizationRecords, primarySubject), + ), + actorView(secondSubject, row('3jzfcijpj2z2d'), sidecar(actorFollowProfileRecords, secondSubject), undefined), + actorView(thirdSubject, row('3jzfcijpj2z2f'), undefined, undefined), + ]); + assert.equal(outgoing.cursor, undefined); + assert.equal(outgoing.following[0].follow.did, publisher, 'nested record metadata identifies its publisher'); + assert.equal(outgoing.following[0].follow.record.$type, followCollection); +}); + +test('follower and following pages deduplicate before pagination and keep stable tuple order', async () => { + const { publisher, otherPublisher, primarySubject } = actorFollowDids; + const cases = [ + { + nsid: 'app.certified.graph.listActorFollowers', + outputKey: 'followers', + actor: primarySubject, + count: 2, + asc: [row('3jzfcijpj2z2b'), row('3jzfcijpj2z2g')], + desc: [row('3jzfcijpj2z2g'), row('3jzfcijpj2z2b')], + ascDids: [publisher, otherPublisher], + descDids: [otherPublisher, publisher], + }, + { + nsid: 'app.certified.graph.listActorFollowing', + outputKey: 'following', + actor: publisher, + count: 3, + asc: [row('3jzfcijpj2z2b'), row('3jzfcijpj2z2d'), row('3jzfcijpj2z2f')], + desc: [row('3jzfcijpj2z2f'), row('3jzfcijpj2z2d'), row('3jzfcijpj2z2b')], + ascDids: [actorFollowDids.primarySubject, actorFollowDids.secondSubject, actorFollowDids.thirdSubject], + descDids: [actorFollowDids.thirdSubject, actorFollowDids.secondSubject, actorFollowDids.primarySubject], + }, + ]; + + for (const query of cases) { + for (const direction of ['asc', 'desc']) { + const items = await collectPages(query.nsid, query.outputKey, query.actor, direction, query.count); + const expected = query[direction]; + assert.deepEqual(items.map(({ follow }) => follow.uri), expected.map(({ uri }) => uri)); + assert.deepEqual(items.map(({ did }) => did), query[`${direction}Dids`]); + assert.equal(new Set(items.map(({ follow }) => follow.uri)).size, query.count, `${query.nsid} ${direction} pages repeat or omit a relationship`); + assert.equal(items[0].follow.record.$type, followCollection); + } + } +}); diff --git a/hypercerts-api/tests/fixtures/actor-follows.js b/hypercerts-api/tests/fixtures/actor-follows.js new file mode 100644 index 00000000..f63c717e --- /dev/null +++ b/hypercerts-api/tests/fixtures/actor-follows.js @@ -0,0 +1,86 @@ +import { encode } from '@atcute/cbor'; +import * as CID from '@atcute/cid'; + +const followCollection = 'app.certified.graph.follow'; +const profileCollection = 'app.certified.actor.profile'; +const organizationCollection = 'app.certified.actor.organization'; +const indexedAt = '2025-01-02T03:04:05.000Z'; + +export const actorFollowDids = { + publisher: 'did:plc:aaaaaaaaaaaaaaaaaaaaaaaa', + otherPublisher: 'did:plc:bbbbbbbbbbbbbbbbbbbbbbbb', + primarySubject: 'did:plc:cccccccccccccccccccccccc', + secondSubject: 'did:plc:dddddddddddddddddddddddd', + thirdSubject: 'did:plc:eeeeeeeeeeeeeeeeeeeeeeee', + curator: 'did:plc:ffffffffffffffffffffffff', +}; + +async function row(collection, rkey, fields, did) { + const record = { $type: collection, ...fields }; + const uri = `at://${did}/${collection}/${rkey}`; + const cid = CID.toString(await CID.create(0x71, encode(record))); + return { uri, did, collection, rkey, cid, indexedAt, record }; +} + +export const actorFollowRecords = await Promise.all([ + row(followCollection, '3jzfcijpj2z2a', { + subject: actorFollowDids.primarySubject, + createdAt: '2025-01-04T00:00:00.000Z', + }, actorFollowDids.publisher), + row(followCollection, '3jzfcijpj2z2b', { + subject: actorFollowDids.primarySubject, + createdAt: '2025-01-01T00:00:00.000Z', + via: { + uri: `at://${actorFollowDids.curator}/app.certified.graph.list/3jzfcijpj2z2a`, + cid: 'bafkreigh2akiscaildcqabsyg3dfr6chu3fgpregiymsck7e7aqa4s52zy', + }, + }, actorFollowDids.publisher), + row(followCollection, '3jzfcijpj2z2c', { + subject: actorFollowDids.primarySubject, + createdAt: '2025-01-01T00:00:00.000Z', + }, actorFollowDids.publisher), + row(followCollection, '3jzfcijpj2z2d', { + subject: actorFollowDids.secondSubject, + createdAt: '2025-01-02T00:00:00.000Z', + }, actorFollowDids.publisher), + row(followCollection, '3jzfcijpj2z2e', { + subject: actorFollowDids.secondSubject, + createdAt: '2025-01-02T00:00:00.000Z', + }, actorFollowDids.publisher), + row(followCollection, '3jzfcijpj2z2f', { + subject: actorFollowDids.thirdSubject, + createdAt: '2025-01-02T00:00:00.000Z', + }, actorFollowDids.publisher), + row(followCollection, '3jzfcijpj2z2g', { + subject: actorFollowDids.primarySubject, + createdAt: '2025-01-02T00:00:00.000Z', + }, actorFollowDids.otherPublisher), +]); + +export const actorFollowProfileRecords = await Promise.all([ + row(profileCollection, 'self', { + displayName: 'Follow fixture publisher', + createdAt: indexedAt, + }, actorFollowDids.publisher), + row(profileCollection, 'self', { + displayName: 'Follow fixture primary subject', + createdAt: indexedAt, + }, actorFollowDids.primarySubject), + row(profileCollection, 'self', { + displayName: 'Follow fixture second subject', + createdAt: indexedAt, + }, actorFollowDids.secondSubject), +]); + +export const actorFollowOrganizationRecords = await Promise.all([ + row(organizationCollection, 'self', { + organizationType: ['nonprofit'], + visibility: 'public', + createdAt: indexedAt, + }, actorFollowDids.publisher), + row(organizationCollection, 'self', { + organizationType: ['community'], + visibility: 'public', + createdAt: indexedAt, + }, actorFollowDids.primarySubject), +]); diff --git a/hypercerts-api/tests/fixtures/fixtures.test.js b/hypercerts-api/tests/fixtures/fixtures.test.js index 8581a494..99c6c652 100644 --- a/hypercerts-api/tests/fixtures/fixtures.test.js +++ b/hypercerts-api/tests/fixtures/fixtures.test.js @@ -1,16 +1,26 @@ import test from 'node:test'; import assert from 'node:assert/strict'; +import { encode } from '@atcute/cbor'; import * as CID from '@atcute/cid'; import { isValidDid, isValidTid } from '@atproto/syntax'; import { locationRecords, profileRecords, organizationRecords, seedSql } from './records.js'; +import { + actorFollowDids, + actorFollowOrganizationRecords, + actorFollowProfileRecords, + actorFollowRecords, +} from './actor-follows.js'; test('fixtures have consistent full AT-URIs, valid DID/TID/CID identifiers, types, and fixed timestamps', () => { - const records = [...locationRecords, ...profileRecords, ...organizationRecords]; + const records = [ + ...locationRecords, ...profileRecords, ...organizationRecords, + ...actorFollowRecords, ...actorFollowProfileRecords, ...actorFollowOrganizationRecords, + ]; for (const row of records) { assert.equal(row.uri, `at://${row.did}/${row.collection}/${row.rkey}`); assert.equal(row.record.$type, row.collection); assert.equal(isValidDid(row.did), true); - if (row.collection === 'app.certified.location') assert.equal(isValidTid(row.rkey), true); + if (row.collection === 'app.certified.location' || row.collection === 'app.certified.graph.follow') assert.equal(isValidTid(row.rkey), true); const cid = CID.fromString(row.cid); assert.equal(cid.version, 1); assert.equal(cid.codec, 0x71); @@ -20,6 +30,34 @@ test('fixtures have consistent full AT-URIs, valid DID/TID/CID identifiers, type } }); +test('actor-follow fixtures isolate publishers and cover date precedence, URI ties, and sparse sidecars', async () => { + const publishers = new Set([actorFollowDids.publisher, actorFollowDids.otherPublisher]); + assert.equal(publishers.size, 2); + assert.equal(new Set(actorFollowRecords.map(({ uri }) => uri)).size, 7); + assert.deepEqual( + actorFollowRecords.filter(({ did, record }) => did === actorFollowDids.publisher && record.subject === actorFollowDids.primarySubject) + .map(({ rkey }) => rkey).sort(), + ['3jzfcijpj2z2a', '3jzfcijpj2z2b', '3jzfcijpj2z2c'], + ); + const primaryPair = new Map(actorFollowRecords + .filter(({ did, record }) => did === actorFollowDids.publisher && record.subject === actorFollowDids.primarySubject) + .map((record) => [record.rkey, record])); + assert.equal(primaryPair.get('3jzfcijpj2z2a').record.createdAt, '2025-01-04T00:00:00.000Z'); + assert.equal(primaryPair.get('3jzfcijpj2z2b').record.createdAt, '2025-01-01T00:00:00.000Z'); + assert.equal(primaryPair.get('3jzfcijpj2z2c').record.createdAt, '2025-01-01T00:00:00.000Z'); + assert.ok(primaryPair.get('3jzfcijpj2z2b').uri < primaryPair.get('3jzfcijpj2z2c').uri); + assert.deepEqual(Object.keys(primaryPair.get('3jzfcijpj2z2b').record.via).sort(), ['cid', 'uri']); + assert.deepEqual(actorFollowProfileRecords.map(({ did }) => did).sort(), [ + actorFollowDids.primarySubject, actorFollowDids.publisher, actorFollowDids.secondSubject, + ].sort()); + assert.deepEqual(actorFollowOrganizationRecords.map(({ did }) => did).sort(), [ + actorFollowDids.primarySubject, actorFollowDids.publisher, + ].sort()); + for (const row of [...actorFollowRecords, ...actorFollowProfileRecords, ...actorFollowOrganizationRecords]) { + assert.equal(CID.toString(await CID.create(0x71, encode(row.record))), row.cid); + } +}); + test('location fixtures cover the smallBlob union variant with a shaped blob reference', () => { const fixture = locationRecords.find(({ record }) => record.location?.$type === 'org.hypercerts.defs#smallBlob'); assert.ok(fixture, 'expected a location fixture using the smallBlob union variant'); diff --git a/hypercerts-api/tooling/actor-follow.test.js b/hypercerts-api/tooling/actor-follow.test.js new file mode 100644 index 00000000..d0273aac --- /dev/null +++ b/hypercerts-api/tooling/actor-follow.test.js @@ -0,0 +1,323 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { existsSync } from 'node:fs'; +import { readFile } from 'node:fs/promises'; +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { loadAssets } from './installer.js'; +import { validatePackageLexicons } from './validate-lexicons.js'; +import { + actorFollowOrganizationRecords, + actorFollowProfileRecords, + actorFollowRecords, +} from '../tests/fixtures/actor-follows.js'; + +const root = fileURLToPath(new URL('../', import.meta.url)); +const actor = 'did:plc:aaaaaaaaaaaaaaaaaaaaaaaa'; +const follower = 'did:plc:bbbbbbbbbbbbbbbbbbbbbbbb'; +const subject = 'did:plc:cccccccccccccccccccccccc'; +const followCollection = 'app.certified.graph.follow'; +const profileCollection = 'app.certified.actor.profile'; +const organizationCollection = 'app.certified.actor.organization'; +const indexedAt = '2025-01-02T03:04:05.000Z'; + +function followRow(id, did, actorDid, createdAt, { subjectDid = actor, via = false } = {}) { + return { + uri: `at://${did}/${followCollection}/${id}`, + did, + cid: `bafy${id}`, + indexed_at: indexedAt, + record: `follow-${id}`, + actor_did: actorDid, + sort_timestamp: createdAt.replace('Z', '.000000Z'), + follow_record: { + $type: followCollection, + subject: subjectDid, + createdAt, + ...(via ? { via: { uri: 'at://did:plc:list/app.certified.list/one', cid: 'bafylink' } } : {}), + }, + }; +} + +function sidecarRow(collection, did, record) { + return { + uri: `at://${did}/${collection}/self`, did, cid: `bafy-${collection.endsWith('profile') ? 'profile' : 'org'}-${did.slice(-4)}`, + indexed_at: indexedAt, record: `sidecar-${collection}-${did}`, sidecar_record: record, + }; +} + +function lua(value) { + if (typeof value === 'string') return JSON.stringify(value); + if (typeof value === 'number' || typeof value === 'boolean') return String(value); + if (Array.isArray(value)) return `{${value.map(lua).join(',')}}`; + if (value && typeof value === 'object') { + return `{${Object.entries(value).map(([key, item]) => `[${JSON.stringify(key)}]=${lua(item)}`).join(',')}}`; + } + throw new TypeError(`Cannot encode ${typeof value} as a Lua fixture`); +} + +function cursor({ direction = 'desc', timestamp = '2025-01-02T03:04:05.000000Z', uri = `at://${follower}/${followCollection}/cursor` } = {}) { + return Buffer.from(JSON.stringify({ v: 1, d: direction, t: timestamp, u: uri }), 'utf8').toString('hex'); +} + +function runLua({ endpoint, params, rows = [], profiles = [], organizations = [], backend = 'postgres', queryFailure = false, expectError, expectedCalls = 0, assertions }) { + const endpointPath = `lua/endpoints/${endpoint}.lua`; + assert.ok(existsSync(`${root}/${endpointPath}`), `${endpointPath} must be generated before exercising the handler`); + const records = Object.fromEntries([ + ...rows.map(({ record, follow_record }) => [record, follow_record]), + ...profiles.map(({ record, sidecar_record }) => [record, sidecar_record]), + ...organizations.map(({ record, sidecar_record }) => [record, sidecar_record]), + ]); + const luaSource = ` +local NULL_VALUE = {} +local RECORDS = ${lua(records)} +local rows = ${lua(rows)} +local profiles = ${lua(profiles)} +local organizations = ${lua(organizations)} +local calls = {} +local function decode_cursor(value) + local version = tonumber(value:match('"v":(%d+)')) + local direction = value:match('"d":"([^"]*)"') + local timestamp = value:match('"t":"([^"]*)"') + local uri = value:match('"u":"([^"]*)"') + if not version or not direction or not timestamp or not uri then error('invalid cursor JSON') end + return { v = version, d = direction, t = timestamp, u = uri } +end +json = { + decode = function(value) + if value == 'null' then return NULL_VALUE end + if RECORDS[value] then return RECORDS[value] end + if value:sub(1, 1) == '{' then return decode_cursor(value) end + local decoded = value:gsub('..', function(pair) return string.char(tonumber(pair, 16)) end) + return decode_cursor(decoded) + end, + encode = function(value) + return string.format('{"v":%d,"d":"%s","t":"%s","u":"%s"}', value.v, value.d, value.t, value.u) + end, +} +toarray = function(value) return value end +params = ${lua(params)} +db = { + backend = function() return '${backend}' end, + raw = function(sql, values) + calls[#calls + 1] = { sql = sql, values = values } + if ${queryFailure ? 'true' : 'false'} then error('fixture database failure') end + if values[1] == '${followCollection}' then return rows end + if values[1] == '${profileCollection}' then return profiles end + if values[1] == '${organizationCollection}' then return organizations end + error('unexpected collection ' .. tostring(values[1])) + end, +} +dofile('${endpointPath}') +local ok, result = pcall(handle) +${expectError + ? `assert(not ok, 'expected handler to reject the request')\nassert(tostring(result):find('${expectError}', 1, true), tostring(result))\nassert(#calls == ${expectedCalls}, 'unexpected PostgreSQL query count')` + : `assert(ok, tostring(result))\n${assertions}`} +`; + const result = spawnSync('lua5.4', ['-e', luaSource], { cwd: root, encoding: 'utf8' }); + assert.equal(result.status, 0, `${result.stderr}${result.stdout}`); + return result.stdout; +} + +test('actor-follow module installs the follow record schema, three queries, and their handlers', async () => { + const manifest = JSON.parse(await readFile(`${root}/manifest.json`, 'utf8')); + assert.ok(manifest.modules.includes('modules/actor-follow/manifest.json')); + assert.ok(manifest.validationLexicons.some(({ id }) => id === followCollection)); + + const { assets } = await loadAssets(`${root}/manifest.json`); + const expected = [ + followCollection, + 'app.certified.graph.getFollow', + 'app.certified.graph.listActorFollowers', + 'app.certified.graph.listActorFollowing', + 'xrpc.query:app.certified.graph.getFollow', + 'xrpc.query:app.certified.graph.listActorFollowers', + 'xrpc.query:app.certified.graph.listActorFollowing', + ]; + for (const id of expected) assert.ok(assets.some((asset) => asset.id === id), `missing installed asset ${id}`); + assert.equal(assets.find((asset) => asset.id === followCollection).config.backfill, true); + assert.deepEqual( + assets.filter(({ kind, id }) => kind === 'script' && id.startsWith('xrpc.query:app.certified.graph.')).map(({ id }) => id).sort(), + expected.filter((id) => id.startsWith('xrpc.query:app.certified.graph.')).sort(), + ); +}); + +test('actor-follow query Lexicons validate the endpoint inputs, nullable lookup, and shared actor views', async () => { + const { lexicons, documents } = await validatePackageLexicons(); + const byId = new Map(documents.map((document) => [document.id, document])); + const follow = byId.get(followCollection); + const getFollow = byId.get('app.certified.graph.getFollow'); + const followers = byId.get('app.certified.graph.listActorFollowers'); + const following = byId.get('app.certified.graph.listActorFollowing'); + assert.ok(follow && getFollow && followers && following, 'the follow record and all query Lexicons belong to the validation closure'); + assert.equal(lexicons.getDefOrThrow(followCollection).type, 'record'); + assert.deepEqual(Object.keys(getFollow.defs.main.parameters.properties).sort(), ['actor', 'subject']); + assert.deepEqual(getFollow.defs.output.required, ['follow']); + assert.deepEqual(getFollow.defs.output.nullable, ['follow']); + assert.deepEqual(Object.keys(followers.defs.main.parameters.properties).sort(), ['actor', 'cursor', 'limit', 'sortDirection']); + assert.deepEqual(Object.keys(following.defs.main.parameters.properties).sort(), ['actor', 'cursor', 'limit', 'sortDirection']); + for (const query of [followers, following]) { + assert.equal(query.defs.main.parameters.properties.limit.minimum, 1); + assert.equal(query.defs.main.parameters.properties.limit.maximum, 100); + assert.equal(query.defs.main.output.encoding, 'application/json'); + } + const sharedDefinitions = byId.get('org.hypercerts.api.defs'); + const followView = lexicons.getDefOrThrow('app.certified.graph.getFollow#followRecordView'); + const actorFollowView = lexicons.getDefOrThrow('app.certified.graph.getFollow#actorFollowView'); + assert.deepEqual(followView.required, ['uri', 'cid', 'indexedAt', 'did', 'record']); + assert.equal(followView.properties.record.ref, `lex:${followCollection}`); + assert.deepEqual(actorFollowView.required, ['did', 'profile', 'organization', 'follow']); + assert.deepEqual(actorFollowView.nullable, ['profile', 'organization']); + assert.equal(actorFollowView.properties.profile.ref, 'lex:org.hypercerts.api.defs#profileView'); + assert.equal(actorFollowView.properties.organization.ref, 'lex:org.hypercerts.api.defs#organizationView'); + assert.equal(actorFollowView.properties.follow.ref, 'lex:app.certified.graph.getFollow#followRecordView'); + assert.equal(sharedDefinitions.defs.actorFollowView, undefined); + assert.equal(followers.defs.output.properties.followers.items.ref, 'lex:app.certified.graph.getFollow#actorFollowView'); + assert.equal(following.defs.output.properties.following.items.ref, 'lex:app.certified.graph.getFollow#actorFollowView'); + const { jsonToLex, lexToJson } = await import('@atproto/lexicon'); + const record = { $type: followCollection, subject, createdAt: '2025-01-01T00:00:00.000Z' }; + const decoded = jsonToLex(record); + lexicons.assertValidRecord(followCollection, decoded); + assert.deepEqual(lexToJson(decoded), record); +}); + +test('actor-follow fixture records validate against their pinned Lexicons', async () => { + const { lexicons } = await validatePackageLexicons(); + const { jsonToLex, lexToJson } = await import('@atproto/lexicon'); + for (const record of [...actorFollowRecords, ...actorFollowProfileRecords, ...actorFollowOrganizationRecords]) { + const decoded = jsonToLex(record.record); + lexicons.assertValidRecord(record.collection, decoded); + assert.deepEqual(lexToJson(decoded), record.record); + } +}); + +test('actor-follow views do not mutate the already-installed shared actor definitions', async () => { + const { documents } = await validatePackageLexicons(); + const shared = documents.find(({ id }) => id === 'org.hypercerts.api.defs'); + const getFollow = documents.find(({ id }) => id === 'app.certified.graph.getFollow'); + assert.equal(shared.defs.followRecordView, undefined); + assert.equal(shared.defs.actorFollowView, undefined); + assert.ok(getFollow.defs.followRecordView && getFollow.defs.actorFollowView); +}); + +test('getFollow selects one raw representative, preserves via, and returns null for no relationship', () => { + const earliest = followRow('early', actor, actor, '2025-01-01T00:00:00Z', { subjectDid: subject, via: true }); + const output = runLua({ + endpoint: 'getFollow', params: { actor, subject }, rows: [earliest], + assertions: ` +assert(result.follow.uri == '${earliest.uri}') +assert(result.follow.cid == '${earliest.cid}' and result.follow.indexedAt == '${indexedAt}') +assert(result.follow.did == '${actor}' and result.follow.record.createdAt == '2025-01-01T00:00:00Z') +assert(result.follow.record.via.uri == 'at://did:plc:list/app.certified.list/one') +assert(#calls == 1) +assert(calls[1].values[1] == '${followCollection}' and calls[1].values[2] == '${actor}' and calls[1].values[3] == '${subject}') +assert(calls[1].sql:find("ORDER BY (record::jsonb->>'createdAt')::timestamptz ASC, uri ASC LIMIT 1", 1, true)) +assert(result.follow.via == nil, 'via stays inside the unchanged raw record') +`, + }); + assert.equal(output, ''); + + runLua({ endpoint: 'getFollow', params: { actor, subject }, rows: [], assertions: 'assert(result.follow == NULL_VALUE)\nassert(#calls == 1)' }); +}); + +test('listActorFollowers deduplicates before cursor pagination, sorts newest first, and hydrates only the returned page', () => { + const rows = [ + followRow('follower-a', follower, follower, '2025-01-05T00:00:00Z', { via: true }), + followRow('follower-b', subject, subject, '2025-01-04T00:00:00Z'), + followRow('next-page', 'did:plc:dddddddddddddddddddddddd', 'did:plc:dddddddddddddddddddddddd', '2025-01-03T00:00:00Z'), + ]; + const profile = sidecarRow(profileCollection, follower, { $type: profileCollection, displayName: 'Follower profile' }); + const organization = sidecarRow(organizationCollection, follower, { $type: organizationCollection, visibility: 'public' }); + runLua({ + endpoint: 'listActorFollowers', params: { + actor, limit: '2', sortDirection: 'desc', + cursor: cursor({ direction: 'desc', timestamp: '2025-01-06T00:00:00.000000Z' }), + }, rows, profiles: [profile], organizations: [organization], + assertions: ` +assert(#result.followers == 2 and result.followers[1].did == '${follower}' and result.followers[2].did == '${subject}') +assert(result.followers[1].follow.did == '${follower}' and result.followers[1].follow.record.via.uri ~= nil) +assert(result.followers[1].profile.uri == '${profile.uri}' and result.followers[1].profile.did == '${follower}') +assert(result.followers[1].profile.cid == '${profile.cid}' and result.followers[1].profile.indexedAt == '${indexedAt}') +assert(result.followers[1].profile.record.displayName == 'Follower profile') +assert(result.followers[1].organization.uri == '${organization.uri}') +assert(result.followers[1].organization.cid == '${organization.cid}' and result.followers[1].organization.indexedAt == '${indexedAt}') +assert(result.followers[1].organization.record.visibility == 'public') +assert(result.followers[2].profile == NULL_VALUE and result.followers[2].organization == NULL_VALUE) +assert(result.cursor ~= nil) +local sql = calls[1].sql +assert(sql:find("record::jsonb->>'subject' = $2", 1, true)) +assert(not sql:find('did = $2', 1, true), 'followers are selected by subject, not publisher') +assert(sql:find('ROW_NUMBER() OVER', 1, true)) +assert(sql:find("PARTITION BY did, record::jsonb->>'subject'", 1, true)) +assert(sql:find("ORDER BY (record::jsonb->>'createdAt')::timestamptz ASC, uri ASC", 1, true)) +local representativeFilter = assert(sql:find('relationship_rank = 1', 1, true)) +local cursorFilter = assert(sql:find('(sort_at, uri) <', 1, true)) +assert(representativeFilter < cursorFilter, 'collapse duplicate relationships before applying the page cursor') +assert(sql:find('ORDER BY sort_at DESC, uri DESC', 1, true)) +assert(#calls == 3) +assert(calls[1].values[5] == 3, 'fetch one lookahead row after the two-item page') +assert(calls[2].values[1] == '${profileCollection}' and calls[2].values[2] == '${follower}' and calls[2].values[3] == '${subject}') +assert(calls[2].values[4] == nil, 'do not hydrate the lookahead row') +assert(calls[3].values[1] == '${organizationCollection}' and calls[3].values[2] == '${follower}' and calls[3].values[3] == '${subject}') +assert(calls[3].values[4] == nil, 'do not hydrate the lookahead row') +local tokenJson = result.cursor:gsub('..', function(pair) return string.char(tonumber(pair, 16)) end) +local token = decode_cursor(tokenJson) +assert(token.v == 1 and token.d == 'desc' and token.t == '2025-01-04T00:00:00.000000Z') +assert(token.u == 'at://${subject}/${followCollection}/follower-b') +`, + }); +}); + +test('listActorFollowing exposes the followed DID while keeping the publisher DID in the raw record view', () => { + const row = followRow('outgoing', actor, subject, '2025-01-06T00:00:00Z', { subjectDid: subject }); + const profile = sidecarRow(profileCollection, subject, { $type: profileCollection, displayName: 'Followed profile' }); + runLua({ + endpoint: 'listActorFollowing', params: { actor, sortDirection: 'asc', limit: '1' }, rows: [row], profiles: [profile], + assertions: ` +assert(#result.following == 1 and result.following[1].did == '${subject}') +assert(result.following[1].follow.did == '${actor}' and result.following[1].follow.uri == '${row.uri}') +assert(result.following[1].profile.uri == '${profile.uri}' and result.following[1].organization == NULL_VALUE) +assert(result.cursor == nil) +assert(calls[1].values[1] == '${followCollection}' and calls[1].values[2] == '${actor}') +assert(calls[1].sql:find('did = $2', 1, true) and calls[1].sql:find('subject_did AS actor_did', 1, true)) +assert(calls[1].sql:find("ORDER BY sort_at ASC, uri ASC", 1, true)) +`, + }); +}); + +test('actor-follow listing defaults to 25, allows 100, and returns empty arrays without cursors', () => { + runLua({ + endpoint: 'listActorFollowers', params: { actor }, rows: [], + assertions: ` +assert(#result.followers == 0 and result.cursor == nil) +assert(calls[1].values[3] == 26) +assert(calls[1].sql:find('ORDER BY sort_at DESC, uri DESC', 1, true)) +`, + }); + runLua({ + endpoint: 'listActorFollowing', params: { actor, limit: '100' }, rows: [], + assertions: 'assert(#result.following == 0 and calls[1].values[3] == 101)', + }); +}); + +test('actor-follow queries reject invalid DIDs, limits, scalar repetition, unknown keys, and cursors before querying', () => { + for (const params of [ + { actor: 'alice.example' }, + { actor, limit: '0' }, + { actor, limit: '101' }, + { actor, limit: { repeated: true } }, + { actor, sortDirection: 'sideways' }, + { actor, extra: 'not-supported' }, + { actor, cursor: 'not-hex' }, + { actor, cursor: cursor({ timestamp: '2025-02-30T00:00:00Z' }) }, + { actor, cursor: cursor({ direction: 'asc' }), sortDirection: 'desc' }, + ]) { + runLua({ endpoint: 'listActorFollowers', params, expectError: 'InvalidRequest:' }); + } + runLua({ endpoint: 'getFollow', params: { actor, subject: 'alice.example' }, expectError: 'InvalidRequest:' }); +}); + +test('actor-follow queries surface the beta PostgreSQL-only and operational failure errors', () => { + runLua({ endpoint: 'listActorFollowers', params: { actor }, backend: 'sqlite', expectError: 'ActorFollowQueryFailed:', expectedCalls: 0 }); + runLua({ endpoint: 'listActorFollowers', params: { actor }, queryFailure: true, expectError: 'ActorFollowQueryFailed:', expectedCalls: 1 }); +}); diff --git a/hypercerts-api/tooling/seed.js b/hypercerts-api/tooling/seed.js index 21518022..167f8fc7 100644 --- a/hypercerts-api/tooling/seed.js +++ b/hypercerts-api/tooling/seed.js @@ -3,6 +3,7 @@ import { accessSync, constants, statSync } from 'node:fs'; import path from 'node:path'; import { pathToFileURL } from 'node:url'; import { seedSql, locationRecords, profileRecords, organizationRecords } from '../tests/fixtures/records.js'; +import { actorFollowRecords, actorFollowProfileRecords, actorFollowOrganizationRecords } from '../tests/fixtures/actor-follows.js'; import { badDateSeedSql } from '../tests/fixtures/bad-dates.js'; import { badDateLocations } from '../tests/fixtures/bad-location-dates.js'; @@ -33,7 +34,10 @@ function sqlInput(statements, types = 'text, text, text, text, jsonb, text, text ].join('\n'); } -export function buildSeedInput(env = process.env, rows = [...locationRecords, ...profileRecords, ...organizationRecords]) { +export function buildSeedInput(env = process.env, rows = [ + ...locationRecords, ...profileRecords, ...organizationRecords, + ...actorFollowRecords, ...actorFollowProfileRecords, ...actorFollowOrganizationRecords, +]) { disposableTarget(env); return sqlInput(seedSql(rows, { disposableTestTarget: true })); } diff --git a/hypercerts-api/tooling/seed.test.js b/hypercerts-api/tooling/seed.test.js index b433d6a3..1ce5c6fb 100644 --- a/hypercerts-api/tooling/seed.test.js +++ b/hypercerts-api/tooling/seed.test.js @@ -5,6 +5,7 @@ import { chmod, copyFile, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:f import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { locationRecords, profileRecords, organizationRecords } from '../tests/fixtures/records.js'; +import { actorFollowRecords, actorFollowProfileRecords, actorFollowOrganizationRecords } from '../tests/fixtures/actor-follows.js'; import { badDateLocations } from '../tests/fixtures/bad-location-dates.js'; import { buildBadDateSeedInput, buildSeedInput, psqlTargetArgs } from './seed.js'; @@ -17,6 +18,7 @@ async function withCopiedSeedModule(run) { await mkdir(path.join(root, 'tests/fixtures'), { recursive: true }); await copyFile(new URL('./seed.js', import.meta.url), path.join(root, 'tooling/seed.js')); await copyFile(new URL('../tests/fixtures/records.js', import.meta.url), path.join(root, 'tests/fixtures/records.js')); + await copyFile(new URL('../tests/fixtures/actor-follows.js', import.meta.url), path.join(root, 'tests/fixtures/actor-follows.js')); await copyFile(new URL('../tests/fixtures/bad-dates.js', import.meta.url), path.join(root, 'tests/fixtures/bad-dates.js')); await copyFile(new URL('../tests/fixtures/bad-location-dates.js', import.meta.url), path.join(root, 'tests/fixtures/bad-location-dates.js')); await writeFile(path.join(root, 'package.json'), '{"type":"module"}\n'); @@ -167,7 +169,10 @@ test('seed CLI rejects an absolute path that is not executable', async () => { }); test('bad-date location rows stay isolated from routine fixtures', () => { - const normal = [...locationRecords, ...profileRecords, ...organizationRecords]; + const normal = [ + ...locationRecords, ...profileRecords, ...organizationRecords, + ...actorFollowRecords, ...actorFollowProfileRecords, ...actorFollowOrganizationRecords, + ]; const normalDids = new Set(normal.map(({ did }) => did)); const normalUris = new Set(normal.map(({ uri }) => uri)); assert.equal(new Set(badDateLocations.map(({ uri }) => uri)).size, 14); @@ -210,7 +215,10 @@ test('bad-date CLI is opt-in, passes eight-parameter SQL to selected executable, assert.equal(normal.status, 0, normal.stderr); const normalSql = await readFile(capture, 'utf8'); const executions = normalSql.split('\n').filter((line) => line.startsWith('EXECUTE happyview_fixture(')); - const expectedRows = [...locationRecords, ...profileRecords, ...organizationRecords]; + const expectedRows = [ + ...locationRecords, ...profileRecords, ...organizationRecords, + ...actorFollowRecords, ...actorFollowProfileRecords, ...actorFollowOrganizationRecords, + ]; assert.equal(executions.length, expectedRows.length); for (const row of expectedRows) { const matching = executions.filter((line) => line.startsWith(`EXECUTE happyview_fixture(E'${row.uri}', `)); From 2eccd048019274f7cd40cd351ca6b0da686085c2 Mon Sep 17 00:00:00 2001 From: kzoeps Date: Mon, 28 Sep 2026 10:54:47 +0600 Subject: [PATCH 3/8] hypercerts-api: integrate actor-follow package checks --- hypercerts-api/README.md | 10 ++- hypercerts-api/package.json | 2 +- hypercerts-api/tooling/installer.test.js | 86 ++++++++----------- hypercerts-api/tooling/lexicons.test.js | 15 ++++ hypercerts-api/tooling/lua.test.js | 104 ++++++++++++++--------- 5 files changed, 121 insertions(+), 96 deletions(-) diff --git a/hypercerts-api/README.md b/hypercerts-api/README.md index e5f86e99..facc9c78 100644 --- a/hypercerts-api/README.md +++ b/hypercerts-api/README.md @@ -2,7 +2,9 @@ This directory contains the tooling for building and testing a Hypercerts XRPC API on HappyView. It provides a reusable installer for HappyView admin assets (Lexicons and Lua scripts), pinned Hypercerts Lexicon dependencies, and test fixtures for checking record behavior. The installer lets API modules declare their assets together, install missing assets in dependency order, and refuse to overwrite assets that differ from what is already installed. -**Current status:** The root `manifest.json` includes the location API bundle. Build its Lua handlers and run the offline checks before installing it on an approved HappyView target. +**Current status:** The root `manifest.json` includes the location and actor-follow query slices. Build their Lua handlers and run the offline checks before installing the bundle on an approved HappyView target. + +The actor-follow slice exposes `app.certified.graph.getFollow`, `app.certified.graph.listActorFollowers`, and `app.certified.graph.listActorFollowing`, backed by the pinned `app.certified.graph.follow` record Lexicon. ## Get started @@ -14,7 +16,7 @@ pnpm build:lua pnpm check ``` -These checks run offline; they do not require a running HappyView instance or database. `build:lua` builds the standalone location handlers in `lua/endpoints/`. `pnpm check` runs JavaScript and Lua lint, a strict `checkJs` typecheck of the installer, then the offline unit tests; run `pnpm run typecheck` to run just the typecheck. TypeScript is a development-time checker only: the installer remains JavaScript and runs directly with Node, with no transpilation. The bundle combines local API Lexicons and scripts with schemas from the pinned `@hypercerts-org/lexicon` package. HTTP contract tests require an installed bundle and a separately approved disposable target. +These checks run offline; they do not require a running HappyView instance or database. `build:lua` builds the standalone location and actor-follow handlers in `lua/endpoints/`. `pnpm check` runs JavaScript and Lua lint, a strict `checkJs` typecheck of the installer, then the offline unit tests; run `pnpm run typecheck` to run just the typecheck. TypeScript is a development-time checker only: the installer remains JavaScript and runs directly with Node, with no transpilation. The bundle combines local API Lexicons and scripts with schemas from the pinned `@hypercerts-org/lexicon` package. HTTP contract tests require an installed bundle and a separately approved disposable target. ESLint is installed with the package dependencies. It uses ESLint's recommended checks plus strict equality, no implicit coercion, no shadowed names, no reassigned parameters, no `var`, and `const` where possible. Console output is allowed only in CLI tooling. When a branch contains Lua scripts, Luacheck checks the generated handlers in `lua/endpoints/`; it rejects unknown globals and unused arguments, allowing only HappyView's `db`, `json`, `params`, `toarray`, and `handle`. It targets Lua 5.4 and skips line-length checks for long SQL expressions. Install Lua 5.4 and LuaRocks, then run `luarocks --lua-version=5.4 --local install luacheck 1.2.0` to enable that check locally. The lint runner also finds the default `~/.luarocks/bin` install if it is not on `PATH`. Branches without Lua files skip Luacheck. If Lua sources exist but generated handlers are missing, run `pnpm build:lua` first. @@ -92,11 +94,11 @@ The root manifest's `validationLexicons` is the complete local schema closure us The bootstrap command above should target only an explicitly approved HappyView instance. It sends `Authorization: Bearer `; session-cookie authentication is not supported. -The token must have `lexicons:read` and `lexicons:create` for lexicon assets. If the bundle includes scripts, it also needs `scripts:read` and `scripts:manage`. If a manifest requests backfill for a new record lexicon, `backfill:create` is additionally needed to start that job; without it, the lexicon is still uploaded but no backfill starts. Remote targets must use HTTPS; HTTP is allowed only on `localhost`, `127.0.0.1`, or `::1`. URL credentials and HTTP redirects are rejected. Resolve installed-asset conflicts manually. The location handlers and fixtures require PostgreSQL. +The token must have `lexicons:read` and `lexicons:create` for lexicon assets. If the bundle includes scripts, it also needs `scripts:read` and `scripts:manage`. If a manifest requests backfill for a new record lexicon, `backfill:create` is additionally needed to start that job; without it, the lexicon is still uploaded but no backfill starts. Remote targets must use HTTPS; HTTP is allowed only on `localhost`, `127.0.0.1`, or `::1`. URL credentials and HTTP redirects are rejected. Resolve installed-asset conflicts manually. The location and actor-follow handlers, and the fixtures, require PostgreSQL. ## Test fixtures (disposable databases only) -The fixture tools seed records directly into PostgreSQL, bypassing HappyView ingestion. Use them **only with a separately approved, disposable loopback test database**, never persistent data. The `seed:test` and `seed:bad-dates` scripts require `HAPPYVIEW_DISPOSABLE_TEST_TARGET=YES`, a loopback `PGHOST`, a `PGDATABASE` name containing a `test` marker, and an absolute `PSQL_PATH` to a trusted `psql` executable. Standard `PGPORT` and `PGUSER` can select the test instance and user. The normal `seed:test` command seeds location, profile, and organization examples by default; `seed:bad-dates` seeds only location examples by default. Both tools also accept supplied record rows through `buildSeedInput` and `buildBadDateSeedInput` in `tooling/seed.js`. +The fixture tools seed records directly into PostgreSQL, bypassing HappyView ingestion. Use them **only with a separately approved, disposable loopback test database**, never persistent data. The `seed:test` and `seed:bad-dates` scripts require `HAPPYVIEW_DISPOSABLE_TEST_TARGET=YES`, a loopback `PGHOST`, a `PGDATABASE` name containing a `test` marker, and an absolute `PSQL_PATH` to a trusted `psql` executable. Standard `PGPORT` and `PGUSER` can select the test instance and user. The normal `seed:test` command seeds location, profile, organization, and actor-follow examples by default; `seed:bad-dates` seeds only location examples and remains separate. `pnpm test:contracts` runs the location and actor-follow HTTP contract suites; `pnpm test:bad-dates` remains a separate opt-in suite. Both seed tools also accept supplied record rows through `buildSeedInput` and `buildBadDateSeedInput` in `tooling/seed.js`. Fixture CIDs are computed from stored record JSON with `@atcute/cbor` and `@atcute/cid`. If you seeded an older version of the blob-backed fixture, reseed the **disposable test database** before comparing CIDs: the earlier `jsonToLex` conversion produced a different CID. diff --git a/hypercerts-api/package.json b/hypercerts-api/package.json index f4f7a8e1..2af9c78c 100644 --- a/hypercerts-api/package.json +++ b/hypercerts-api/package.json @@ -17,7 +17,7 @@ "check": "pnpm run lint && pnpm run typecheck && pnpm run test:unit", "build:lua": "node tooling/build-lua.js", "test:unit": "node --test tooling/*.test.js tests/fixtures/*.test.js tests/contracts/helpers.test.js tests/contracts/location-filters.test.js", - "test:contracts": "node --test tests/contracts/location.contract.test.js", + "test:contracts": "node --test tests/contracts/location.contract.test.js tests/contracts/actor-follow.contract.test.js", "test:bad-dates": "node --test tests/contracts/location-bad-dates.contract.test.js", "install:api": "node tooling/installer.js", "seed:test": "node tooling/seed.js", diff --git a/hypercerts-api/tooling/installer.test.js b/hypercerts-api/tooling/installer.test.js index 8d2988cc..5a29c433 100644 --- a/hypercerts-api/tooling/installer.test.js +++ b/hypercerts-api/tooling/installer.test.js @@ -1,62 +1,46 @@ import test from 'node:test'; import assert from 'node:assert/strict'; -import { readFileSync, readdirSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; import { loadAssets, orderAssets } from './installer.js'; +import { validatePackageLexicons } from './validate-lexicons.js'; -test('installer deploys only location records, required definitions, API Lexicons, and handlers', async () => { - const manifest = JSON.parse(readFileSync(new URL('../manifest.json', import.meta.url), 'utf8')); - const { assets } = await loadAssets(fileURLToPath(new URL('../manifest.json', import.meta.url))); - const installed = new Map(assets.filter(({ kind }) => kind === 'lexicon').map((entry) => [entry.id, entry])); - const expectedAssetIds = [ - 'app.certified.actor.organization', - 'app.certified.actor.profile', - 'app.certified.location', - 'app.certified.signature.defs', - 'org.hypercerts.defs', - 'org.hypercerts.api.defs', - 'app.certified.location.getLocation', - 'app.certified.location.listLocations', - 'xrpc.query:app.certified.location.getLocation', - 'xrpc.query:app.certified.location.listLocations', - ]; - assert.deepEqual(assets.map(({ id }) => id).sort(), expectedAssetIds.sort()); +const manifestPath = fileURLToPath(new URL('../manifest.json', import.meta.url)); - const validationSources = new Map(manifest.validationLexicons.map((source) => [source.id, source])); - for (const asset of installed.values()) { - const isRecordLexicon = asset.lexicon_json.defs?.main?.type === 'record'; - assert.equal(asset.config.backfill, isRecordLexicon, `${asset.id} backfill configuration`); - assert.equal(asset.lexicon_json.id, asset.id); - if (asset.packagePath) assert.equal(asset.packagePath, validationSources.get(asset.id)?.packagePath); +async function loadBundle() { + const { assets } = await loadAssets(manifestPath); + return assets; +} + +test('installer gives every declared query a handler that depends on its Lexicon', async () => { + const assets = await loadBundle(); + const assetsById = new Map(assets.map((asset) => [asset.id, asset])); + const orderedIds = orderAssets(assets).map(({ id }) => id); + const position = (id) => orderedIds.indexOf(id); + const queries = assets.filter(({ kind, lexicon_json }) => kind === 'lexicon' && lexicon_json.defs?.main?.type === 'query'); + + assert.ok(queries.length > 0, 'bundle must declare at least one query Lexicon'); + for (const query of queries) { + const handlerId = `xrpc.query:${query.id}`; + const handler = assetsById.get(handlerId); + assert.ok(handler, `missing handler ${handlerId} for query ${query.id}`); + assert.equal(handler.kind, 'script', `${handlerId} must be a script`); + assert.ok(handler.dependsOn?.includes(query.id), `${handlerId} must depend on ${query.id}`); + assert.ok(position(query.id) < position(handlerId), `${query.id} must install before ${handlerId}`); } - const ordered = orderAssets(assets).map(({ id }) => id); - const position = (id) => ordered.indexOf(id); - const sharedViews = installed.get('org.hypercerts.api.defs'); - const locationRecord = installed.get('app.certified.location'); - const getLocation = installed.get('app.certified.location.getLocation'); - const listLocations = installed.get('app.certified.location.listLocations'); - assert.ok(sharedViews, 'missing shared API view definitions'); - assert.ok(locationRecord, 'missing pinned location record schema'); - assert.ok(getLocation, 'missing getLocation Lexicon'); - assert.ok(listLocations, 'missing listLocations Lexicon'); - assert.deepEqual(sharedViews.dependsOn, ['app.certified.actor.organization', 'app.certified.actor.profile']); - assert.deepEqual(getLocation.dependsOn, ['app.certified.location', 'org.hypercerts.api.defs']); - assert.deepEqual(listLocations.dependsOn, ['app.certified.location.getLocation']); - assert.equal(locationRecord.packagePath, 'lexicons/app/certified/location.json'); - assert.ok(position('app.certified.actor.organization') < position('org.hypercerts.api.defs')); - assert.ok(position('app.certified.actor.profile') < position('org.hypercerts.api.defs')); - assert.ok(position('app.certified.location') < position('app.certified.location.getLocation')); - assert.ok(position('org.hypercerts.api.defs') < position('app.certified.location.getLocation')); - assert.ok(position('app.certified.location.getLocation') < position('app.certified.location.listLocations')); - assert.ok(position('app.certified.location.getLocation') < position('xrpc.query:app.certified.location.getLocation')); - assert.ok(position('app.certified.location.listLocations') < position('xrpc.query:app.certified.location.listLocations')); }); -test('the local location API schemas remain the only checked-in schemas', () => { - const files = readdirSync(new URL('../lexicons/', import.meta.url)).filter((file) => file.endsWith('.json')); - assert.deepEqual(files.sort(), [ - 'app.certified.location.getLocation.json', - 'app.certified.location.listLocations.json', - 'org.hypercerts.api.defs.json', +test('declared query schemas exist in the validation Lexicon set', async () => { + const [assets, { lexicons, documents }] = await Promise.all([ + loadBundle(), + validatePackageLexicons(), ]); + const validatedDocuments = new Map(documents.map((document) => [document.id, document])); + const queries = assets.filter(({ kind, lexicon_json }) => kind === 'lexicon' && lexicon_json.defs?.main?.type === 'query'); + + for (const query of queries) { + const document = validatedDocuments.get(query.id); + assert.ok(document, `query schema ${query.id} is missing from validationLexicons`); + assert.equal(document.id, query.id); + assert.ok(lexicons.getDefOrThrow(query.id), `${query.id} must resolve in the validated Lexicon set`); + } }); diff --git a/hypercerts-api/tooling/lexicons.test.js b/hypercerts-api/tooling/lexicons.test.js index 8c20151c..4439b8b6 100644 --- a/hypercerts-api/tooling/lexicons.test.js +++ b/hypercerts-api/tooling/lexicons.test.js @@ -19,6 +19,7 @@ test('the full validation Lexicon closure resolves locally while only selected p assert.deepEqual(deployedPackageAssets.map(({ id }) => id).sort(), [ 'app.certified.actor.organization', 'app.certified.actor.profile', + 'app.certified.graph.follow', 'app.certified.location', 'app.certified.signature.defs', 'org.hypercerts.defs', @@ -64,6 +65,20 @@ test('location query result refs resolve to shared actor views and getLocation-o assert.equal(lexicons.getDefOrThrow('app.certified.location.getLocation#locationView').type, 'object'); }); +test('follow query Lexicons declare all required DID parameters', async () => { + const { documents } = await validatePackageLexicons(); + const byId = new Map(documents.map((document) => [document.id, document])); + assert.deepEqual({ + getFollow: byId.get('app.certified.graph.getFollow').defs.main.parameters.required, + listActorFollowers: byId.get('app.certified.graph.listActorFollowers').defs.main.parameters.required, + listActorFollowing: byId.get('app.certified.graph.listActorFollowing').defs.main.parameters.required, + }, { + getFollow: ['actor', 'subject'], + listActorFollowers: ['actor'], + listActorFollowing: ['actor'], + }); +}); + test('installed ATProto validator accepts package language, transitive refs, and real fixture records', async () => { const { lexicons, isValidDid, isValidTid } = await validatePackageLexicons(); const { jsonToLex, lexToJson } = await import('@atproto/lexicon'); diff --git a/hypercerts-api/tooling/lua.test.js b/hypercerts-api/tooling/lua.test.js index a5b6b7a0..de77e3b1 100644 --- a/hypercerts-api/tooling/lua.test.js +++ b/hypercerts-api/tooling/lua.test.js @@ -1,51 +1,75 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import { readFile } from 'node:fs/promises'; +import path from 'node:path'; import { fileURLToPath } from 'node:url'; -import { spawnSync } from 'node:child_process'; import { loadAssets } from './installer.js'; const root = fileURLToPath(new URL('../', import.meta.url)); -const read = (relative) => readFile(new URL(`../${relative}`, import.meta.url), 'utf8'); - -test('checked-in Lua bundles reproduce from shared and endpoint sources', async () => { - const [shared, getSource, listSource, getBeforeBuild, listBeforeBuild] = await Promise.all([ - read('lua/shared/location.lua'), read('lua/src/getLocation.lua'), read('lua/src/listLocations.lua'), - readFile(new URL('../lua/endpoints/getLocation.lua', import.meta.url)), - readFile(new URL('../lua/endpoints/listLocations.lua', import.meta.url)), - ]); - const expectedGet = Buffer.from(`${shared.trimEnd()}\n\n${getSource}`); - const expectedList = Buffer.from(`${shared.trimEnd()}\n\n${listSource}`); - assert.deepEqual(getBeforeBuild, expectedGet, 'getLocation bundle is stale; run pnpm build:lua'); - assert.deepEqual(listBeforeBuild, expectedList, 'listLocations bundle is stale; run pnpm build:lua'); - - const result = spawnSync(process.execPath, ['tooling/build-lua.js'], { cwd: root, encoding: 'utf8' }); - assert.equal(result.status, 0, result.stderr); - const [getBuilt, listBuilt] = await Promise.all([ - read('lua/endpoints/getLocation.lua'), read('lua/endpoints/listLocations.lua'), - ]); - assert.equal(getBuilt, `${shared.trimEnd()}\n\n${getSource}`); - assert.equal(listBuilt, `${shared.trimEnd()}\n\n${listSource}`); - assert.match(getBuilt, /local function get_location\(\)/); - assert.doesNotMatch(getBuilt, /local function list_locations\(\)/); - assert.match(listBuilt, /local function list_locations\(\)/); - assert.doesNotMatch(listBuilt, /local function get_location\(\)/); - assert.doesNotMatch(getBuilt, /\brequire\s*\(/); - assert.doesNotMatch(listBuilt, /\brequire\s*\(/); +const read = (relative) => readFile(path.resolve(root, relative), 'utf8'); +const manifestPath = path.join(root, 'manifest.json'); + +async function declaredLuaHandlers() { + const bundle = JSON.parse(await read('manifest.json')); + const { assets } = await loadAssets(manifestPath); + const loadedById = new Map(assets.map((asset) => [asset.id, asset])); + const handlers = []; + + for (const modulePath of bundle.modules) { + const moduleFile = path.resolve(root, modulePath); + const moduleManifest = JSON.parse(await readFile(moduleFile, 'utf8')); + for (const declaration of moduleManifest.assets) { + if (declaration.kind !== 'script' || declaration.config.script_type !== 'lua' || !declaration.id.startsWith('xrpc.query:')) continue; + handlers.push({ + declaration, + moduleDirectory: path.dirname(moduleFile), + loaded: loadedById.get(declaration.id), + }); + } + } + + return { bundle, handlers }; +} + +function sourcePaths({ declaration, moduleDirectory }) { + const shared = declaration.sharedSourcePaths ?? (declaration.sharedSourcePath ? [declaration.sharedSourcePath] : []); + assert.ok(declaration.sourcePath, `${declaration.id} must declare its endpoint source`); + return { + endpoint: path.resolve(moduleDirectory, declaration.sourcePath), + shared: shared.map((source) => path.resolve(moduleDirectory, source)), + }; +} + +test('checked-in Lua bundles reproduce from declared handler sources', async () => { + const { handlers } = await declaredLuaHandlers(); + assert.ok(handlers.length > 0, 'bundle must declare at least one Lua query handler'); + for (const handler of handlers) { + const sources = sourcePaths(handler); + const contents = await Promise.all([...sources.shared, sources.endpoint].map((file) => readFile(file, 'utf8'))); + const bundle = `${contents.map((source) => source.trimEnd()).join('\n\n')}\n`; + const built = await readFile(path.resolve(handler.moduleDirectory, handler.declaration.path), 'utf8'); + assert.equal(built, bundle, `${handler.declaration.id} bundle is stale; run pnpm build:lua`); + assert.match(built, /function handle\(\)/, `${handler.declaration.id} must define handle()`); + assert.doesNotMatch(built, /\brequire\s*\(/, `${handler.declaration.id} must be standalone`); + } }); -test('manifest installs only built standalone handlers and records their source inputs', async () => { - const manifest = JSON.parse(await read('manifest.json')); - const { assets } = await loadAssets(fileURLToPath(new URL('../manifest.json', import.meta.url))); - assert.equal(manifest.handlerStatus.getLocation, 'implemented'); - assert.equal(manifest.handlerStatus.listLocations, 'implemented'); - assert.equal(manifest.authentication.unresolved, false); - for (const name of ['getLocation', 'listLocations']) { - const asset = assets.find(({ id }) => id === `xrpc.query:app.certified.location.${name}`); - assert.equal(asset.kind, 'script'); - assert.equal(asset.path, `../../lua/endpoints/${name}.lua`); - assert.equal(asset.sourcePath, `../../lua/src/${name}.lua`); - assert.equal(asset.sharedSourcePath, '../../lua/shared/location.lua'); - assert.match(asset.body, /function handle\(\)/); +test('manifest installs declared Lua query handlers from their built bundles and sources', async () => { + const { bundle, handlers } = await declaredLuaHandlers(); + assert.equal(bundle.authentication.unresolved, false); + assert.ok(handlers.length > 0, 'bundle must declare at least one Lua query handler'); + + for (const handler of handlers) { + const { declaration, loaded } = handler; + const name = declaration.id.slice('xrpc.query:'.length).split('.').at(-1); + assert.ok(loaded, `missing loaded handler ${declaration.id}`); + assert.equal(bundle.handlerStatus[name], 'implemented', `${name} must be marked implemented`); + assert.equal(loaded.kind, 'script'); + assert.equal(loaded.path, declaration.path); + assert.equal(loaded.sourcePath, declaration.sourcePath); + const declaredShared = declaration.sharedSourcePaths ?? (declaration.sharedSourcePath ? [declaration.sharedSourcePath] : []); + const loadedShared = loaded.sharedSourcePaths ?? (loaded.sharedSourcePath ? [loaded.sharedSourcePath] : []); + assert.deepEqual(loadedShared, declaredShared); + assert.match(loaded.body, /function handle\(\)/); } }); From 60174b9799e644f138e1e8073adafead2e9d7cfa Mon Sep 17 00:00:00 2001 From: kzoeps Date: Mon, 28 Sep 2026 11:24:00 +0600 Subject: [PATCH 4/8] hypercerts-api: tighten checks and refine actor-follow descriptions --- .github/workflows/hypercerts-api.yml | 19 ++++- hypercerts-api/.luacheckrc | 7 +- hypercerts-api/README.md | 4 +- hypercerts-api/eslint.config.js | 3 + .../app.certified.graph.getFollow.json | 2 +- ...pp.certified.graph.listActorFollowers.json | 2 +- ...pp.certified.graph.listActorFollowing.json | 2 +- hypercerts-api/package.json | 5 +- hypercerts-api/tooling/build-lua.js | 39 ++++++--- hypercerts-api/tooling/build-lua.test.js | 79 +++++++++++++++++++ hypercerts-api/tooling/installer.js | 16 ++-- .../tooling/lint-security-rules.test.js | 40 ++++++++++ hypercerts-api/tooling/lua-bundles.js | 47 +++++++++++ 13 files changed, 235 insertions(+), 30 deletions(-) create mode 100644 hypercerts-api/tooling/build-lua.test.js create mode 100644 hypercerts-api/tooling/lint-security-rules.test.js create mode 100644 hypercerts-api/tooling/lua-bundles.js diff --git a/.github/workflows/hypercerts-api.yml b/.github/workflows/hypercerts-api.yml index 6b89cc44..550b2532 100644 --- a/.github/workflows/hypercerts-api.yml +++ b/.github/workflows/hypercerts-api.yml @@ -27,6 +27,12 @@ jobs: with: luaVersion: "5.4" + - name: Add Lua 5.4 executable alias + run: | + lua_bin="$(command -v lua)" + ln -s "$lua_bin" "${lua_bin%/*}/lua5.4" + lua5.4 -e 'assert(_VERSION == "Lua 5.4", "expected Lua 5.4")' + - uses: leafo/gh-actions-luarocks@35d062def313a7699a0d513e996bcf4352f05389 # v6.1.0 - name: Install Luacheck @@ -45,5 +51,14 @@ jobs: - name: Install Node dependencies run: pnpm install --frozen-lockfile - - name: Lint and test - run: pnpm check + - name: Check generated Lua bundles + run: pnpm run check:generated + + - name: Lint + run: pnpm run lint + + - name: Typecheck + run: pnpm run typecheck + + - name: Unit tests + run: pnpm run test:unit diff --git a/hypercerts-api/.luacheckrc b/hypercerts-api/.luacheckrc index b6358c8a..aa1f3dda 100644 --- a/hypercerts-api/.luacheckrc +++ b/hypercerts-api/.luacheckrc @@ -1,5 +1,10 @@ std = "lua54" -globals = { "db", "json", "params", "toarray", "handle" } +globals = { "handle" } +read_globals = { "db", "json", "params", "toarray" } +not_globals = { + "io", "debug", "package", "require", "dofile", "loadfile", "load", "collectgarbage", + "os.execute", "os.exit", "os.getenv", "os.remove", "os.rename", "os.setlocale", "os.tmpname", +} unused_args = true -- Generated SQL expressions can be long; keep Luacheck focused on code issues. max_line_length = false diff --git a/hypercerts-api/README.md b/hypercerts-api/README.md index facc9c78..e7fbc8ae 100644 --- a/hypercerts-api/README.md +++ b/hypercerts-api/README.md @@ -16,9 +16,9 @@ pnpm build:lua pnpm check ``` -These checks run offline; they do not require a running HappyView instance or database. `build:lua` builds the standalone location and actor-follow handlers in `lua/endpoints/`. `pnpm check` runs JavaScript and Lua lint, a strict `checkJs` typecheck of the installer, then the offline unit tests; run `pnpm run typecheck` to run just the typecheck. TypeScript is a development-time checker only: the installer remains JavaScript and runs directly with Node, with no transpilation. The bundle combines local API Lexicons and scripts with schemas from the pinned `@hypercerts-org/lexicon` package. HTTP contract tests require an installed bundle and a separately approved disposable target. +These checks run offline; they do not require a running HappyView instance or database. `build:lua` builds the standalone location and actor-follow handlers in `lua/endpoints/`. `pnpm check` verifies the generated Lua bundles are current, runs JavaScript and Lua lint, a strict `checkJs` typecheck of the installer, then the offline unit tests; run `pnpm run typecheck` to run just the typecheck. TypeScript is a development-time checker only: the installer remains JavaScript and runs directly with Node, with no transpilation. Run `pnpm check:generated` to check bundle freshness on its own; it does not modify files. If it fails, run `pnpm build:lua` and review the generated changes. The bundle combines local API Lexicons and scripts with schemas from the pinned `@hypercerts-org/lexicon` package. HTTP contract tests require an installed bundle and a separately approved disposable target. -ESLint is installed with the package dependencies. It uses ESLint's recommended checks plus strict equality, no implicit coercion, no shadowed names, no reassigned parameters, no `var`, and `const` where possible. Console output is allowed only in CLI tooling. When a branch contains Lua scripts, Luacheck checks the generated handlers in `lua/endpoints/`; it rejects unknown globals and unused arguments, allowing only HappyView's `db`, `json`, `params`, `toarray`, and `handle`. It targets Lua 5.4 and skips line-length checks for long SQL expressions. Install Lua 5.4 and LuaRocks, then run `luarocks --lua-version=5.4 --local install luacheck 1.2.0` to enable that check locally. The lint runner also finds the default `~/.luarocks/bin` install if it is not on `PATH`. Branches without Lua files skip Luacheck. If Lua sources exist but generated handlers are missing, run `pnpm build:lua` first. +ESLint is installed with the package dependencies. It uses ESLint's recommended checks plus strict equality, no implicit coercion, no shadowed names, no reassigned parameters, no `var`, and `const` where possible. It also rejects direct `eval`, implied evaluation (such as string-based timers), and `Function` constructors. Console output is allowed only in CLI tooling, and unused ESLint disable directives fail lint. When a branch contains Lua scripts, Luacheck checks the generated handlers in `lua/endpoints/`; it rejects unknown globals, sandbox-removed Lua APIs, and unused arguments. It rejects `io`, `debug`, `package`, `require`, `dofile`, `loadfile`, `load`, `collectgarbage`, and unsafe `os` functions; only `os.time`, `os.date`, `os.difftime`, and `os.clock` are allowed from `os`. Only the HappyView `handle` global is writable; `db`, `json`, `params`, and `toarray` are read-only. It targets Lua 5.4 and skips line-length checks for long SQL expressions. Install Lua 5.4 and LuaRocks, then run `luarocks --lua-version=5.4 --local install luacheck 1.2.0` to enable that check locally. The lint runner also finds the default `~/.luarocks/bin` install if it is not on `PATH`. Branches without Lua files skip Luacheck. If Lua sources exist but generated handlers are missing, run `pnpm build:lua` first. ## Bootstrap a running HappyView instance diff --git a/hypercerts-api/eslint.config.js b/hypercerts-api/eslint.config.js index 2a289d22..5bb91131 100644 --- a/hypercerts-api/eslint.config.js +++ b/hypercerts-api/eslint.config.js @@ -15,6 +15,9 @@ export default [ rules: { ...js.configs.recommended.rules, eqeqeq: ["error", "always", { null: "ignore" }], + "no-eval": "error", + "no-implied-eval": "error", + "no-new-func": "error", "no-console": "error", "no-implicit-coercion": "error", "no-param-reassign": ["error", { props: true }], diff --git a/hypercerts-api/lexicons/app.certified.graph.getFollow.json b/hypercerts-api/lexicons/app.certified.graph.getFollow.json index 1c2abdc2..6fc3298c 100644 --- a/hypercerts-api/lexicons/app.certified.graph.getFollow.json +++ b/hypercerts-api/lexicons/app.certified.graph.getFollow.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "Returns the earliest indexed follow record for one actor-to-actor relationship, or null when no matching relationship is indexed. Authentication is not required and the response is not personalized.", + "description": "Returns the earliest indexed follow record for one actor-to-actor relationship, or null when no matching relationship is indexed. Authentication is not required.", "parameters": { "type": "params", "required": ["actor", "subject"], diff --git a/hypercerts-api/lexicons/app.certified.graph.listActorFollowers.json b/hypercerts-api/lexicons/app.certified.graph.listActorFollowers.json index ceb1b024..77276d6c 100644 --- a/hypercerts-api/lexicons/app.certified.graph.listActorFollowers.json +++ b/hypercerts-api/lexicons/app.certified.graph.listActorFollowers.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "Lists actors whose indexed follow records name the requested actor as subject. Authentication is not required and the response is not personalized. Duplicate relationships are collapsed before pagination.", + "description": "Lists actors whose indexed follow records name the requested actor as subject. Authentication is not required. Duplicate relationships are collapsed before pagination.", "parameters": { "type": "params", "required": ["actor"], diff --git a/hypercerts-api/lexicons/app.certified.graph.listActorFollowing.json b/hypercerts-api/lexicons/app.certified.graph.listActorFollowing.json index 95c1a407..f8dba0d5 100644 --- a/hypercerts-api/lexicons/app.certified.graph.listActorFollowing.json +++ b/hypercerts-api/lexicons/app.certified.graph.listActorFollowing.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "Lists actors named as subjects by the requested actor's indexed follow records. Authentication is not required and the response is not personalized. Duplicate relationships are collapsed before pagination.", + "description": "Lists actors named as subjects by the requested actor's indexed follow records. Authentication is not required. Duplicate relationships are collapsed before pagination.", "parameters": { "type": "params", "required": ["actor"], diff --git a/hypercerts-api/package.json b/hypercerts-api/package.json index 2af9c78c..cf365c67 100644 --- a/hypercerts-api/package.json +++ b/hypercerts-api/package.json @@ -10,11 +10,12 @@ "@hypercerts-org/lexicon": "1.4.0" }, "scripts": { - "lint:js": "eslint --max-warnings=0 .", + "lint:js": "eslint --max-warnings=0 --report-unused-disable-directives .", "lint:lua": "node tooling/lint-lua.js", "lint": "pnpm run lint:js && pnpm run lint:lua", "typecheck": "tsc --project tsconfig.json", - "check": "pnpm run lint && pnpm run typecheck && pnpm run test:unit", + "check:generated": "node tooling/build-lua.js --check", + "check": "pnpm run check:generated && pnpm run lint && pnpm run typecheck && pnpm run test:unit", "build:lua": "node tooling/build-lua.js", "test:unit": "node --test tooling/*.test.js tests/fixtures/*.test.js tests/contracts/helpers.test.js tests/contracts/location-filters.test.js", "test:contracts": "node --test tests/contracts/location.contract.test.js tests/contracts/actor-follow.contract.test.js", diff --git a/hypercerts-api/tooling/build-lua.js b/hypercerts-api/tooling/build-lua.js index 8f90ed50..18e1e8c4 100644 --- a/hypercerts-api/tooling/build-lua.js +++ b/hypercerts-api/tooling/build-lua.js @@ -1,18 +1,33 @@ -import { readFile, writeFile } from 'node:fs/promises'; import path from 'node:path'; -import { fileURLToPath } from 'node:url'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { buildLuaBundles, checkLuaBundles } from './lua-bundles.js'; const root = fileURLToPath(new URL('../', import.meta.url)); -const bundles = [ - { shared: ['location'], endpoints: ['getLocation', 'listLocations'] }, - { shared: ['actorFollow', 'actorFollowLookup'], endpoints: ['getFollow'] }, - { shared: ['actorFollow', 'actorFollowList'], endpoints: ['listActorFollowers', 'listActorFollowing'] }, -]; -for (const { shared, endpoints } of bundles) { - const sharedSources = await Promise.all(shared.map((name) => readFile(path.join(root, `lua/shared/${name}.lua`), 'utf8'))); - for (const name of endpoints) { - const endpoint = await readFile(path.join(root, `lua/src/${name}.lua`), 'utf8'); - await writeFile(path.join(root, `lua/endpoints/${name}.lua`), `${[...sharedSources, endpoint].map((source) => source.trimEnd()).join('\n\n')}\n`); +async function main() { + const [option, ...extraOptions] = process.argv.slice(2); + if (extraOptions.length > 0 || (option && option !== '--check')) { + throw new Error('Usage: node tooling/build-lua.js [--check]'); + } + + if (option === '--check') { + const stale = await checkLuaBundles(root); + if (stale.length > 0) { + const staleFiles = stale.map((file) => ` - ${file}`).join('\n'); + throw new Error(`Generated Lua bundles are stale:\n${staleFiles}\nRun \`pnpm build:lua\` to regenerate and commit them.`); + } + process.stdout.write('Generated Lua bundles are up to date.\n'); + return; + } + + await buildLuaBundles(root); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) { + try { + await main(); + } catch (error) { + process.stderr.write(`${error.message}\n`); + process.exitCode = 1; } } diff --git a/hypercerts-api/tooling/build-lua.test.js b/hypercerts-api/tooling/build-lua.test.js new file mode 100644 index 00000000..1bfe8fd8 --- /dev/null +++ b/hypercerts-api/tooling/build-lua.test.js @@ -0,0 +1,79 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { buildLuaBundles, checkLuaBundles } from './lua-bundles.js'; + +const sources = [ + ['lua/shared/location.lua', 'local function query() return "location" end\n'], + ['lua/shared/actorFollow.lua', 'local function query() return "follow" end\n'], + ['lua/shared/actorFollowLookup.lua', 'local function lookup() return true end\n'], + ['lua/shared/actorFollowList.lua', 'local function list() return true end\n'], + ['lua/src/getLocation.lua', 'function handle() return query() end\n'], + ['lua/src/listLocations.lua', 'function handle() return query() end\n'], + ['lua/src/getFollow.lua', 'function handle() return query() end\n'], + ['lua/src/listActorFollowers.lua', 'function handle() return list() end\n'], + ['lua/src/listActorFollowing.lua', 'function handle() return list() end\n'], +]; + +async function withLuaRoot(run) { + const root = await mkdtemp(path.join(os.tmpdir(), 'hypercerts-api-build-lua-')); + try { + for (const [relativePath, content] of sources) { + const file = path.join(root, relativePath); + await mkdir(path.dirname(file), { recursive: true }); + await writeFile(file, content); + } + await mkdir(path.join(root, 'lua/endpoints')); + await run(root); + } finally { + await rm(root, { recursive: true, force: true }); + } +} + +test('builds a handler bundle from its shared and endpoint sources', async () => { + await withLuaRoot(async (root) => { + await buildLuaBundles(root); + assert.equal( + await readFile(path.join(root, 'lua/endpoints/getLocation.lua'), 'utf8'), + 'local function query() return "location" end\n\nfunction handle() return query() end\n', + ); + }); +}); + +test('checks generated bundles without rewriting stale outputs', async () => { + await withLuaRoot(async (root) => { + const expectedStale = [ + 'lua/endpoints/getLocation.lua', + 'lua/endpoints/listLocations.lua', + 'lua/endpoints/getFollow.lua', + 'lua/endpoints/listActorFollowers.lua', + 'lua/endpoints/listActorFollowing.lua', + ]; + for (const relativePath of expectedStale) { + await writeFile(path.join(root, relativePath), 'stale bundle\n'); + } + + assert.deepEqual(await checkLuaBundles(root), expectedStale); + for (const relativePath of expectedStale) { + assert.equal(await readFile(path.join(root, relativePath), 'utf8'), 'stale bundle\n'); + } + }); +}); + +test('reports missing generated bundles and accepts fresh bundles', async () => { + await withLuaRoot(async (root) => { + const expected = [ + 'lua/endpoints/getLocation.lua', + 'lua/endpoints/listLocations.lua', + 'lua/endpoints/getFollow.lua', + 'lua/endpoints/listActorFollowers.lua', + 'lua/endpoints/listActorFollowing.lua', + ]; + assert.deepEqual(await checkLuaBundles(root), expected); + + await buildLuaBundles(root); + assert.deepEqual(await checkLuaBundles(root), []); + }); +}); diff --git a/hypercerts-api/tooling/installer.js b/hypercerts-api/tooling/installer.js index d6164a1e..5747b4fa 100644 --- a/hypercerts-api/tooling/installer.js +++ b/hypercerts-api/tooling/installer.js @@ -351,7 +351,7 @@ function validateAssetEntry(entry, modulePath, assetIndex, owners) { return /** @type {ValidatedManifestAsset} */ (candidate); } -/* eslint-disable no-param-reassign -- the caller passes a private copy to populate with its loaded source. */ +/* eslint-disable no-param-reassign -- loadAssetSource passes a private copy to populate with its loaded source. */ /** @param {LexiconManifestAsset & { lexicon_json?: unknown }} asset @param {string} root */ async function loadLexiconAssetSource(asset, root) { asset.lexicon_json = await readLexiconSource(asset, root); @@ -382,18 +382,19 @@ function assetSourcePath(asset) { /** @param {ValidatedManifestAsset & { lexicon_json?: unknown; body?: string }} asset @param {string} modulePath @param {string} root @returns {Promise} */ async function loadAssetSource(asset, modulePath, root) { + const loadedAsset = { ...asset }; try { - if (asset.kind === 'lexicon') { - await loadLexiconAssetSource(asset, root); + if (loadedAsset.kind === 'lexicon') { + await loadLexiconAssetSource(loadedAsset, root); } else { - await loadScriptAssetSource(asset, root); + await loadScriptAssetSource(loadedAsset, root); } + return /** @type {LoadedAsset} */ (loadedAsset); } catch (cause) { // Node filesystem and package-source operations throw Error instances; preserve their original message interpolation. const detail = /** @type {Error} */ (cause).message; - throw new Error(`Asset ${asset.id} in module ${modulePath}: source ${assetSourcePath(asset)} is missing, invalid or empty (${detail}); fix the declaration or source before installing`, { cause }); + throw new Error(`Asset ${loadedAsset.id} in module ${modulePath}: source ${assetSourcePath(loadedAsset)} is missing, invalid or empty (${detail}); fix the declaration or source before installing`, { cause }); } - return /** @type {LoadedAsset} */ (asset); } /** @param {string} modulePath @param {string} file @param {Map} owners @returns {Promise} */ @@ -402,8 +403,7 @@ async function loadModuleAssets(modulePath, file, owners) { /** @type {LoadedAsset[]} */ const assets = []; for (const entry of module.assets) { - const asset = { ...entry }; - assets.push(await loadAssetSource(asset, modulePath, path.dirname(file))); + assets.push(await loadAssetSource(entry, modulePath, path.dirname(file))); } return assets; } diff --git a/hypercerts-api/tooling/lint-security-rules.test.js b/hypercerts-api/tooling/lint-security-rules.test.js new file mode 100644 index 00000000..18b05081 --- /dev/null +++ b/hypercerts-api/tooling/lint-security-rules.test.js @@ -0,0 +1,40 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import { fileURLToPath } from 'node:url'; + +const packageRoot = fileURLToPath(new URL('../', import.meta.url)); + +function lintLua(file) { + const args = ['--config', '.luacheckrc', '--no-color', file]; + let result = spawnSync('luacheck', args, { cwd: packageRoot, encoding: 'utf8' }); + if (result.error?.code === 'ENOENT') { + result = spawnSync(path.join(os.homedir(), '.luarocks', 'bin', 'luacheck'), args, { + cwd: packageRoot, + encoding: 'utf8', + }); + } + assert.ifError(result.error); + return result; +} + +test('Luacheck distinguishes HappyView sandbox APIs from Lua 5.4 globals', async () => { + const directory = await mkdtemp(path.join(os.tmpdir(), 'hypercerts-lint-')); + const file = path.join(directory, 'handler.lua'); + try { + await writeFile(file, 'function handle() return io.open("file"), os.execute("true") end\n'); + const rejected = lintLua(file); + assert.notEqual(rejected.status, 0, 'sandbox-removed APIs should fail lint'); + assert.match(rejected.stdout, /undefined variable 'io'/); + assert.match(rejected.stdout, /undefined field 'execute'/); + + await writeFile(file, 'function handle() return db.backend(), json.encode(params), toarray({}), os.time() end\n'); + const accepted = lintLua(file); + assert.equal(accepted.status, 0, `${accepted.stdout}\n${accepted.stderr}`); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); diff --git a/hypercerts-api/tooling/lua-bundles.js b/hypercerts-api/tooling/lua-bundles.js new file mode 100644 index 00000000..8f6f5db0 --- /dev/null +++ b/hypercerts-api/tooling/lua-bundles.js @@ -0,0 +1,47 @@ +import { readFile, writeFile } from 'node:fs/promises'; +import path from 'node:path'; + +const bundles = [ + { shared: ['location'], endpoints: ['getLocation', 'listLocations'] }, + { shared: ['actorFollow', 'actorFollowLookup'], endpoints: ['getFollow'] }, + { shared: ['actorFollow', 'actorFollowList'], endpoints: ['listActorFollowers', 'listActorFollowing'] }, +]; + +async function renderBundles(root) { + const outputs = []; + for (const { shared, endpoints } of bundles) { + const sharedSources = await Promise.all(shared.map((name) => readFile(path.join(root, `lua/shared/${name}.lua`), 'utf8'))); + for (const name of endpoints) { + const endpoint = await readFile(path.join(root, `lua/src/${name}.lua`), 'utf8'); + outputs.push({ + path: `lua/endpoints/${name}.lua`, + content: `${[...sharedSources, endpoint].map((source) => source.trimEnd()).join('\n\n')}\n`, + }); + } + } + return outputs; +} + +export async function buildLuaBundles(root) { + for (const output of await renderBundles(root)) { + await writeFile(path.join(root, output.path), output.content); + } +} + +export async function checkLuaBundles(root) { + const stale = []; + for (const output of await renderBundles(root)) { + let current; + try { + current = await readFile(path.join(root, output.path), 'utf8'); + } catch (error) { + if (error.code === 'ENOENT') { + stale.push(output.path); + continue; + } + throw error; + } + if (current !== output.content) stale.push(output.path); + } + return stale; +} From 271584deb6635fd173c265fdc2a790ffab08c983 Mon Sep 17 00:00:00 2001 From: kzoeps Date: Mon, 28 Sep 2026 15:33:00 +0600 Subject: [PATCH 5/8] hypercerts-api: share Lua query and actor view helpers --- hypercerts-api/lua/endpoints/getFollow.lua | 27 +-- hypercerts-api/lua/endpoints/getLocation.lua | 85 ++++++---- .../lua/endpoints/listActorFollowers.lua | 121 ++++++++------ .../lua/endpoints/listActorFollowing.lua | 121 ++++++++------ .../lua/endpoints/listLocations.lua | 155 +++++++++++------- hypercerts-api/lua/shared/actorFollow.lua | 38 ----- hypercerts-api/lua/shared/actorFollowList.lua | 81 +-------- .../lua/shared/actorFollowLookup.lua | 2 +- hypercerts-api/lua/shared/actorView.lua | 29 ++++ hypercerts-api/lua/shared/listQuery.lua | 40 +++++ hypercerts-api/lua/shared/location.lua | 74 +-------- hypercerts-api/lua/shared/query.lua | 26 +++ .../lua/shared/recordIdentifier.lua | 11 ++ hypercerts-api/lua/shared/recordView.lua | 11 ++ hypercerts-api/lua/src/getLocation.lua | 4 +- hypercerts-api/lua/src/listLocations.lua | 41 +---- .../modules/actor-follow/manifest.json | 12 ++ hypercerts-api/modules/location/manifest.json | 17 +- hypercerts-api/tooling/actor-follow.test.js | 3 + hypercerts-api/tooling/build-lua.test.js | 13 +- hypercerts-api/tooling/lua-bundles.js | 7 +- 21 files changed, 476 insertions(+), 442 deletions(-) create mode 100644 hypercerts-api/lua/shared/actorView.lua create mode 100644 hypercerts-api/lua/shared/listQuery.lua create mode 100644 hypercerts-api/lua/shared/query.lua create mode 100644 hypercerts-api/lua/shared/recordIdentifier.lua create mode 100644 hypercerts-api/lua/shared/recordView.lua diff --git a/hypercerts-api/lua/endpoints/getFollow.lua b/hypercerts-api/lua/endpoints/getFollow.lua index 948aa07f..735e2329 100644 --- a/hypercerts-api/lua/endpoints/getFollow.lua +++ b/hypercerts-api/lua/endpoints/getFollow.lua @@ -1,6 +1,3 @@ -local FOLLOW = "app.certified.graph.follow" -local NULL = json.decode("null") - local function invalid(message) error("InvalidRequest: " .. message, 0) end @@ -11,8 +8,8 @@ local function keys_only(values, allowed) end end -local function scalar(values, key) - local value = values[key] +local function scalar(params, key) + local value = params[key] if value == nil then return nil end if type(value) ~= "string" and type(value) ~= "number" then invalid(key .. " must occur once") @@ -28,14 +25,9 @@ local function valid_did(value) return true end -local function query(sql, values) - if db.backend() ~= "postgres" then error("ActorFollowQueryFailed: actor-follow queries require PostgreSQL", 0) end - local ok, result = pcall(db.raw, sql, values) - if not ok then error("ActorFollowQueryFailed: actor-follow lookup failed", 0) end - return result -end +local NULL = json.decode("null") -local function row_view(row) +local function record_view(row) return { uri = row.uri, cid = row.cid, @@ -45,12 +37,21 @@ local function row_view(row) } end +local FOLLOW = "app.certified.graph.follow" + +local function query(sql, values) + if db.backend() ~= "postgres" then error("ActorFollowQueryFailed: actor-follow queries require PostgreSQL", 0) end + local ok, result = pcall(db.raw, sql, values) + if not ok then error("ActorFollowQueryFailed: actor-follow lookup failed", 0) end + return result +end + local function query_follow(actor, subject) local rows = query( "SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record FROM happyview_records WHERE collection = $1 AND did = $2 AND record::jsonb->>'subject' = $3 ORDER BY (record::jsonb->>'createdAt')::timestamptz ASC, uri ASC LIMIT 1", { FOLLOW, actor, subject }) if #rows == 0 then return NULL end - return row_view(rows[1]) + return record_view(rows[1]) end local function get_follow() diff --git a/hypercerts-api/lua/endpoints/getLocation.lua b/hypercerts-api/lua/endpoints/getLocation.lua index d98d939d..19d5f64d 100644 --- a/hypercerts-api/lua/endpoints/getLocation.lua +++ b/hypercerts-api/lua/endpoints/getLocation.lua @@ -1,8 +1,3 @@ -local COLLECTION = "app.certified.location" -local PROFILE = "app.certified.actor.profile" -local ORGANIZATION = "app.certified.actor.organization" -local NULL = json.decode("null") - local function invalid(message) error("InvalidRequest: " .. message, 0) end @@ -35,57 +30,83 @@ local function valid_record_key(value) and not value:find("[^%w_~%.:%-]") end -local function valid_uri(value) - if value:find("[?#]") then return false end +local function valid_record_uri(value) + if type(value) ~= "string" or value:find("[?#]") then return false end local authority, collection, rkey = value:match("^at://([^/]+)/([^/]+)/([^/]+)$") - return authority ~= nil and valid_did(authority) and collection == COLLECTION and valid_record_key(rkey) + if not authority or not valid_did(authority) or not valid_record_key(rkey) then return false end + return true, collection end -local function query(sql, values) - local ok, result = pcall(db.raw, sql, values) - if not ok then error("LocationQueryFailed: location lookup failed", 0) end - return result -end +local NULL = json.decode("null") -local function row_view(row) - local record = json.decode(row.record) +local function record_view(row) return { - uri = row.uri, cid = row.cid, indexedAt = row.indexed_at, did = row.did, - record = record, + uri = row.uri, + cid = row.cid, + indexedAt = row.indexed_at, + did = row.did, + record = json.decode(row.record), } end -local function hydrate(views) - if #views == 0 then return end +local PROFILE = "app.certified.actor.profile" +local ORGANIZATION = "app.certified.actor.organization" + +local function hydrate_actor_views(actors, run_query) + if #actors == 0 then return end local dids, seen = {}, {} - for _, view in ipairs(views) do - if not seen[view.did] then seen[view.did] = true; dids[#dids + 1] = view.did end + for _, actor in ipairs(actors) do + if not seen[actor.did] then + seen[actor.did] = true + dids[#dids + 1] = actor.did + end end local profiles, organizations = {}, {} local function load(collection, target) - if #dids == 0 then return end - local params, marks = {}, {} - params[1] = collection - for _, did in ipairs(dids) do params[#params + 1] = did; marks[#marks + 1] = "$" .. #params end - local rows = query("SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record FROM happyview_records WHERE collection = $1 AND rkey = 'self' AND did IN (" .. table.concat(marks, ",") .. ")", params) + local params, marks = { collection }, {} + for _, did in ipairs(dids) do + params[#params + 1] = did + marks[#marks + 1] = "$" .. #params + end + local rows = run_query("SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record FROM happyview_records WHERE collection = $1 AND rkey = 'self' AND did IN (" .. table.concat(marks, ",") .. ")", params) for _, row in ipairs(rows) do target[row.did] = row end end load(PROFILE, profiles) load(ORGANIZATION, organizations) + for _, actor in ipairs(actors) do + actor.profile = profiles[actor.did] and record_view(profiles[actor.did]) or NULL + actor.organization = organizations[actor.did] and record_view(organizations[actor.did]) or NULL + end +end + +local COLLECTION = "app.certified.location" + +local function valid_location_uri(value) + local valid, collection = valid_record_uri(value) + return valid and collection == COLLECTION +end + +local function query(sql, values) + local ok, result = pcall(db.raw, sql, values) + if not ok then error("LocationQueryFailed: location lookup failed", 0) end + return result +end + +local function hydrate(views) + local authors = {} for _, view in ipairs(views) do - local profile, organization = profiles[view.did], organizations[view.did] - local author = { did = view.did, profile = NULL, organization = NULL } - if profile then author.profile = row_view(profile) end - if organization then author.organization = row_view(organization) end + local author = { did = view.did } view.author = author + authors[#authors + 1] = author end + hydrate_actor_views(authors, query) end local function query_location(uri) if db.backend() ~= "postgres" then error("LocationQueryFailed: location API requires PostgreSQL", 0) end local rows = query("SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record FROM happyview_records WHERE collection = $1 AND uri = $2 LIMIT 1", { COLLECTION, uri }) local views = {} - for _, row in ipairs(rows) do views[#views + 1] = row_view(row) end + for _, row in ipairs(rows) do views[#views + 1] = record_view(row) end hydrate(views) return views end @@ -93,7 +114,7 @@ end local function get_location() keys_only(params, { uri = true }) local uri = scalar(params, "uri") - if not uri or not valid_uri(uri) then invalid("uri must be a full app.certified.location AT-URI with a DID authority") end + if not uri or not valid_location_uri(uri) then invalid("uri must be a full app.certified.location AT-URI with a DID authority") end local views = query_location(uri) if #views == 0 then error("RecordNotFound: location record is not indexed", 0) end return { location = views[1] } diff --git a/hypercerts-api/lua/endpoints/listActorFollowers.lua b/hypercerts-api/lua/endpoints/listActorFollowers.lua index 98169e69..f6ceb241 100644 --- a/hypercerts-api/lua/endpoints/listActorFollowers.lua +++ b/hypercerts-api/lua/endpoints/listActorFollowers.lua @@ -1,6 +1,3 @@ -local FOLLOW = "app.certified.graph.follow" -local NULL = json.decode("null") - local function invalid(message) error("InvalidRequest: " .. message, 0) end @@ -11,8 +8,8 @@ local function keys_only(values, allowed) end end -local function scalar(values, key) - local value = values[key] +local function scalar(params, key) + local value = params[key] if value == nil then return nil end if type(value) ~= "string" and type(value) ~= "number" then invalid(key .. " must occur once") @@ -28,35 +25,16 @@ local function valid_did(value) return true end -local function query(sql, values) - if db.backend() ~= "postgres" then error("ActorFollowQueryFailed: actor-follow queries require PostgreSQL", 0) end - local ok, result = pcall(db.raw, sql, values) - if not ok then error("ActorFollowQueryFailed: actor-follow lookup failed", 0) end - return result -end - -local function row_view(row) - return { - uri = row.uri, - cid = row.cid, - indexedAt = row.indexed_at, - did = row.did, - record = json.decode(row.record), - } -end - -local PROFILE = "app.certified.actor.profile" -local ORGANIZATION = "app.certified.actor.organization" - local function valid_record_key(value) return #value >= 1 and #value <= 512 and value ~= "." and value ~= ".." and not value:find("[^%w_~%.:%-]") end -local function valid_follow_uri(value) +local function valid_record_uri(value) if type(value) ~= "string" or value:find("[?#]") then return false end local authority, collection, rkey = value:match("^at://([^/]+)/([^/]+)/([^/]+)$") - return authority ~= nil and valid_did(authority) and collection == FOLLOW and valid_record_key(rkey) + if not authority or not valid_did(authority) or not valid_record_key(rkey) then return false end + return true, collection end local function valid_datetime(value) @@ -81,37 +59,79 @@ local function valid_datetime(value) return true end -local function hydrate(views) - if #views == 0 then return end +local function parse_list_limit(params) + local limit_value = scalar(params, "limit") + if limit_value and not limit_value:match("^%d+$") then invalid("limit must be an integer from 1 through 100") end + local limit = limit_value and tonumber(limit_value) or 25 + if not limit or limit % 1 ~= 0 or limit < 1 or limit > 100 then invalid("limit must be an integer from 1 through 100") end + return limit +end + +local function parse_sort_direction(params) + local direction = scalar(params, "sortDirection") or "desc" + if direction ~= "asc" and direction ~= "desc" then invalid("sortDirection must be 'asc' or 'desc'") end + return direction +end + +local function cursor_encode(value) + local encoded = json.encode(value) + return (encoded:gsub(".", function(char) return string.format("%02x", string.byte(char)) end)) +end + +local NULL = json.decode("null") + +local function record_view(row) + return { + uri = row.uri, + cid = row.cid, + indexedAt = row.indexed_at, + did = row.did, + record = json.decode(row.record), + } +end + +local PROFILE = "app.certified.actor.profile" +local ORGANIZATION = "app.certified.actor.organization" + +local function hydrate_actor_views(actors, run_query) + if #actors == 0 then return end local dids, seen = {}, {} - for _, view in ipairs(views) do - if not seen[view.did] then - seen[view.did] = true - dids[#dids + 1] = view.did + for _, actor in ipairs(actors) do + if not seen[actor.did] then + seen[actor.did] = true + dids[#dids + 1] = actor.did end end local profiles, organizations = {}, {} local function load(collection, target) - local values, marks = { collection }, {} + local params, marks = { collection }, {} for _, did in ipairs(dids) do - values[#values + 1] = did - marks[#marks + 1] = "$" .. #values + params[#params + 1] = did + marks[#marks + 1] = "$" .. #params end - local rows = query("SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record FROM happyview_records WHERE collection = $1 AND rkey = 'self' AND did IN (" .. table.concat(marks, ",") .. ")", values) + local rows = run_query("SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record FROM happyview_records WHERE collection = $1 AND rkey = 'self' AND did IN (" .. table.concat(marks, ",") .. ")", params) for _, row in ipairs(rows) do target[row.did] = row end end load(PROFILE, profiles) load(ORGANIZATION, organizations) - for _, view in ipairs(views) do - local profile, organization = profiles[view.did], organizations[view.did] - view.profile = profile and row_view(profile) or NULL - view.organization = organization and row_view(organization) or NULL + for _, actor in ipairs(actors) do + actor.profile = profiles[actor.did] and record_view(profiles[actor.did]) or NULL + actor.organization = organizations[actor.did] and record_view(organizations[actor.did]) or NULL end end -local function cursor_encode(value) - local encoded = json.encode(value) - return (encoded:gsub(".", function(char) return string.format("%02x", string.byte(char)) end)) +local FOLLOW = "app.certified.graph.follow" + +local function query(sql, values) + if db.backend() ~= "postgres" then error("ActorFollowQueryFailed: actor-follow queries require PostgreSQL", 0) end + local ok, result = pcall(db.raw, sql, values) + if not ok then error("ActorFollowQueryFailed: actor-follow lookup failed", 0) end + return result +end + +local function valid_follow_uri(value) + local valid, collection = valid_record_uri(value) + return valid and collection == FOLLOW end local function cursor_decode(token, direction) @@ -159,9 +179,9 @@ local function query_actor_follows(mode, actor, limit, cursor, direction) local views = {} for _, row in ipairs(rows) do - views[#views + 1] = { did = row.actor_did, follow = row_view(row) } + views[#views + 1] = { did = row.actor_did, follow = record_view(row) } end - hydrate(views) + hydrate_actor_views(views, query) local next_cursor if more then @@ -176,13 +196,8 @@ local function list_actor_follows(mode) local actor = scalar(params, "actor") if not actor or not valid_did(actor) then invalid("actor must be a valid DID") end - local limit_value = scalar(params, "limit") - if limit_value and not limit_value:match("^%d+$") then invalid("limit must be an integer from 1 through 100") end - local limit = limit_value and tonumber(limit_value) or 25 - if not limit or limit % 1 ~= 0 or limit < 1 or limit > 100 then invalid("limit must be an integer from 1 through 100") end - - local direction = scalar(params, "sortDirection") or "desc" - if direction ~= "asc" and direction ~= "desc" then invalid("sortDirection must be 'asc' or 'desc'") end + local limit = parse_list_limit(params) + local direction = parse_sort_direction(params) local cursor = cursor_decode(scalar(params, "cursor"), direction) local views, next_cursor = query_actor_follows(mode, actor, limit, cursor, direction) local response = { [mode] = toarray(views) } diff --git a/hypercerts-api/lua/endpoints/listActorFollowing.lua b/hypercerts-api/lua/endpoints/listActorFollowing.lua index b0cb8759..69799254 100644 --- a/hypercerts-api/lua/endpoints/listActorFollowing.lua +++ b/hypercerts-api/lua/endpoints/listActorFollowing.lua @@ -1,6 +1,3 @@ -local FOLLOW = "app.certified.graph.follow" -local NULL = json.decode("null") - local function invalid(message) error("InvalidRequest: " .. message, 0) end @@ -11,8 +8,8 @@ local function keys_only(values, allowed) end end -local function scalar(values, key) - local value = values[key] +local function scalar(params, key) + local value = params[key] if value == nil then return nil end if type(value) ~= "string" and type(value) ~= "number" then invalid(key .. " must occur once") @@ -28,35 +25,16 @@ local function valid_did(value) return true end -local function query(sql, values) - if db.backend() ~= "postgres" then error("ActorFollowQueryFailed: actor-follow queries require PostgreSQL", 0) end - local ok, result = pcall(db.raw, sql, values) - if not ok then error("ActorFollowQueryFailed: actor-follow lookup failed", 0) end - return result -end - -local function row_view(row) - return { - uri = row.uri, - cid = row.cid, - indexedAt = row.indexed_at, - did = row.did, - record = json.decode(row.record), - } -end - -local PROFILE = "app.certified.actor.profile" -local ORGANIZATION = "app.certified.actor.organization" - local function valid_record_key(value) return #value >= 1 and #value <= 512 and value ~= "." and value ~= ".." and not value:find("[^%w_~%.:%-]") end -local function valid_follow_uri(value) +local function valid_record_uri(value) if type(value) ~= "string" or value:find("[?#]") then return false end local authority, collection, rkey = value:match("^at://([^/]+)/([^/]+)/([^/]+)$") - return authority ~= nil and valid_did(authority) and collection == FOLLOW and valid_record_key(rkey) + if not authority or not valid_did(authority) or not valid_record_key(rkey) then return false end + return true, collection end local function valid_datetime(value) @@ -81,37 +59,79 @@ local function valid_datetime(value) return true end -local function hydrate(views) - if #views == 0 then return end +local function parse_list_limit(params) + local limit_value = scalar(params, "limit") + if limit_value and not limit_value:match("^%d+$") then invalid("limit must be an integer from 1 through 100") end + local limit = limit_value and tonumber(limit_value) or 25 + if not limit or limit % 1 ~= 0 or limit < 1 or limit > 100 then invalid("limit must be an integer from 1 through 100") end + return limit +end + +local function parse_sort_direction(params) + local direction = scalar(params, "sortDirection") or "desc" + if direction ~= "asc" and direction ~= "desc" then invalid("sortDirection must be 'asc' or 'desc'") end + return direction +end + +local function cursor_encode(value) + local encoded = json.encode(value) + return (encoded:gsub(".", function(char) return string.format("%02x", string.byte(char)) end)) +end + +local NULL = json.decode("null") + +local function record_view(row) + return { + uri = row.uri, + cid = row.cid, + indexedAt = row.indexed_at, + did = row.did, + record = json.decode(row.record), + } +end + +local PROFILE = "app.certified.actor.profile" +local ORGANIZATION = "app.certified.actor.organization" + +local function hydrate_actor_views(actors, run_query) + if #actors == 0 then return end local dids, seen = {}, {} - for _, view in ipairs(views) do - if not seen[view.did] then - seen[view.did] = true - dids[#dids + 1] = view.did + for _, actor in ipairs(actors) do + if not seen[actor.did] then + seen[actor.did] = true + dids[#dids + 1] = actor.did end end local profiles, organizations = {}, {} local function load(collection, target) - local values, marks = { collection }, {} + local params, marks = { collection }, {} for _, did in ipairs(dids) do - values[#values + 1] = did - marks[#marks + 1] = "$" .. #values + params[#params + 1] = did + marks[#marks + 1] = "$" .. #params end - local rows = query("SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record FROM happyview_records WHERE collection = $1 AND rkey = 'self' AND did IN (" .. table.concat(marks, ",") .. ")", values) + local rows = run_query("SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record FROM happyview_records WHERE collection = $1 AND rkey = 'self' AND did IN (" .. table.concat(marks, ",") .. ")", params) for _, row in ipairs(rows) do target[row.did] = row end end load(PROFILE, profiles) load(ORGANIZATION, organizations) - for _, view in ipairs(views) do - local profile, organization = profiles[view.did], organizations[view.did] - view.profile = profile and row_view(profile) or NULL - view.organization = organization and row_view(organization) or NULL + for _, actor in ipairs(actors) do + actor.profile = profiles[actor.did] and record_view(profiles[actor.did]) or NULL + actor.organization = organizations[actor.did] and record_view(organizations[actor.did]) or NULL end end -local function cursor_encode(value) - local encoded = json.encode(value) - return (encoded:gsub(".", function(char) return string.format("%02x", string.byte(char)) end)) +local FOLLOW = "app.certified.graph.follow" + +local function query(sql, values) + if db.backend() ~= "postgres" then error("ActorFollowQueryFailed: actor-follow queries require PostgreSQL", 0) end + local ok, result = pcall(db.raw, sql, values) + if not ok then error("ActorFollowQueryFailed: actor-follow lookup failed", 0) end + return result +end + +local function valid_follow_uri(value) + local valid, collection = valid_record_uri(value) + return valid and collection == FOLLOW end local function cursor_decode(token, direction) @@ -159,9 +179,9 @@ local function query_actor_follows(mode, actor, limit, cursor, direction) local views = {} for _, row in ipairs(rows) do - views[#views + 1] = { did = row.actor_did, follow = row_view(row) } + views[#views + 1] = { did = row.actor_did, follow = record_view(row) } end - hydrate(views) + hydrate_actor_views(views, query) local next_cursor if more then @@ -176,13 +196,8 @@ local function list_actor_follows(mode) local actor = scalar(params, "actor") if not actor or not valid_did(actor) then invalid("actor must be a valid DID") end - local limit_value = scalar(params, "limit") - if limit_value and not limit_value:match("^%d+$") then invalid("limit must be an integer from 1 through 100") end - local limit = limit_value and tonumber(limit_value) or 25 - if not limit or limit % 1 ~= 0 or limit < 1 or limit > 100 then invalid("limit must be an integer from 1 through 100") end - - local direction = scalar(params, "sortDirection") or "desc" - if direction ~= "asc" and direction ~= "desc" then invalid("sortDirection must be 'asc' or 'desc'") end + local limit = parse_list_limit(params) + local direction = parse_sort_direction(params) local cursor = cursor_decode(scalar(params, "cursor"), direction) local views, next_cursor = query_actor_follows(mode, actor, limit, cursor, direction) local response = { [mode] = toarray(views) } diff --git a/hypercerts-api/lua/endpoints/listLocations.lua b/hypercerts-api/lua/endpoints/listLocations.lua index 04d769f2..875fcc47 100644 --- a/hypercerts-api/lua/endpoints/listLocations.lua +++ b/hypercerts-api/lua/endpoints/listLocations.lua @@ -1,8 +1,3 @@ -local COLLECTION = "app.certified.location" -local PROFILE = "app.certified.actor.profile" -local ORGANIZATION = "app.certified.actor.organization" -local NULL = json.decode("null") - local function invalid(message) error("InvalidRequest: " .. message, 0) end @@ -35,50 +30,11 @@ local function valid_record_key(value) and not value:find("[^%w_~%.:%-]") end -local function valid_uri(value) - if value:find("[?#]") then return false end +local function valid_record_uri(value) + if type(value) ~= "string" or value:find("[?#]") then return false end local authority, collection, rkey = value:match("^at://([^/]+)/([^/]+)/([^/]+)$") - return authority ~= nil and valid_did(authority) and collection == COLLECTION and valid_record_key(rkey) -end - -local function query(sql, values) - local ok, result = pcall(db.raw, sql, values) - if not ok then error("LocationQueryFailed: location lookup failed", 0) end - return result -end - -local function row_view(row) - local record = json.decode(row.record) - return { - uri = row.uri, cid = row.cid, indexedAt = row.indexed_at, did = row.did, - record = record, - } -end - -local function hydrate(views) - if #views == 0 then return end - local dids, seen = {}, {} - for _, view in ipairs(views) do - if not seen[view.did] then seen[view.did] = true; dids[#dids + 1] = view.did end - end - local profiles, organizations = {}, {} - local function load(collection, target) - if #dids == 0 then return end - local params, marks = {}, {} - params[1] = collection - for _, did in ipairs(dids) do params[#params + 1] = did; marks[#marks + 1] = "$" .. #params end - local rows = query("SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record FROM happyview_records WHERE collection = $1 AND rkey = 'self' AND did IN (" .. table.concat(marks, ",") .. ")", params) - for _, row in ipairs(rows) do target[row.did] = row end - end - load(PROFILE, profiles) - load(ORGANIZATION, organizations) - for _, view in ipairs(views) do - local profile, organization = profiles[view.did], organizations[view.did] - local author = { did = view.did, profile = NULL, organization = NULL } - if profile then author.profile = row_view(profile) end - if organization then author.organization = row_view(organization) end - view.author = author - end + if not authority or not valid_did(authority) or not valid_record_key(rkey) then return false end + return true, collection end local function valid_datetime(value) @@ -103,6 +59,90 @@ local function valid_datetime(value) return true end +local function parse_list_limit(params) + local limit_value = scalar(params, "limit") + if limit_value and not limit_value:match("^%d+$") then invalid("limit must be an integer from 1 through 100") end + local limit = limit_value and tonumber(limit_value) or 25 + if not limit or limit % 1 ~= 0 or limit < 1 or limit > 100 then invalid("limit must be an integer from 1 through 100") end + return limit +end + +local function parse_sort_direction(params) + local direction = scalar(params, "sortDirection") or "desc" + if direction ~= "asc" and direction ~= "desc" then invalid("sortDirection must be 'asc' or 'desc'") end + return direction +end + +local function cursor_encode(value) + local encoded = json.encode(value) + return (encoded:gsub(".", function(char) return string.format("%02x", string.byte(char)) end)) +end + +local NULL = json.decode("null") + +local function record_view(row) + return { + uri = row.uri, + cid = row.cid, + indexedAt = row.indexed_at, + did = row.did, + record = json.decode(row.record), + } +end + +local PROFILE = "app.certified.actor.profile" +local ORGANIZATION = "app.certified.actor.organization" + +local function hydrate_actor_views(actors, run_query) + if #actors == 0 then return end + local dids, seen = {}, {} + for _, actor in ipairs(actors) do + if not seen[actor.did] then + seen[actor.did] = true + dids[#dids + 1] = actor.did + end + end + local profiles, organizations = {}, {} + local function load(collection, target) + local params, marks = { collection }, {} + for _, did in ipairs(dids) do + params[#params + 1] = did + marks[#marks + 1] = "$" .. #params + end + local rows = run_query("SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record FROM happyview_records WHERE collection = $1 AND rkey = 'self' AND did IN (" .. table.concat(marks, ",") .. ")", params) + for _, row in ipairs(rows) do target[row.did] = row end + end + load(PROFILE, profiles) + load(ORGANIZATION, organizations) + for _, actor in ipairs(actors) do + actor.profile = profiles[actor.did] and record_view(profiles[actor.did]) or NULL + actor.organization = organizations[actor.did] and record_view(organizations[actor.did]) or NULL + end +end + +local COLLECTION = "app.certified.location" + +local function valid_location_uri(value) + local valid, collection = valid_record_uri(value) + return valid and collection == COLLECTION +end + +local function query(sql, values) + local ok, result = pcall(db.raw, sql, values) + if not ok then error("LocationQueryFailed: location lookup failed", 0) end + return result +end + +local function hydrate(views) + local authors = {} + for _, view in ipairs(views) do + local author = { did = view.did } + view.author = author + authors[#authors + 1] = author + end + hydrate_actor_views(authors, query) +end + local function array(params, key, validate, description, max_bytes) local value = params[key] if value == nil then return nil end @@ -135,11 +175,6 @@ local function add_in(where, params, column, values) where[#where + 1] = column .. " IN (" .. table.concat(placeholders, ",") .. ")" end -local function cursor_encode(value) - local encoded = json.encode(value) - return (encoded:gsub(".", function(char) return string.format("%02x", string.byte(char)) end)) -end - local function cursor_decode(token, direction) if not token then return nil end if #token % 2 ~= 0 or token:find("[^0-9a-f]") then invalid("cursor is malformed") end @@ -152,7 +187,7 @@ local function cursor_decode(token, direction) for key in pairs(value) do if key ~= "v" and key ~= "d" and key ~= "t" and key ~= "u" then invalid("cursor is malformed") end end - if not valid_uri(value.u) or not valid_datetime(value.t) then invalid("cursor is malformed") end + if not valid_location_uri(value.u) or not valid_datetime(value.t) then invalid("cursor is malformed") end return value end @@ -181,7 +216,7 @@ local function query_locations(filters, limit, cursor, direction) local more = #rows > limit if more then rows[#rows] = nil end local views = {} - for _, row in ipairs(rows) do views[#views + 1] = row_view(row) end + for _, row in ipairs(rows) do views[#views + 1] = record_view(row) end hydrate(views) local next_cursor if more then @@ -194,14 +229,10 @@ end local function list_locations() keys_only(params, { authors = true, uris = true, locationTypes = true, limit = true, cursor = true, sortDirection = true }) local authors = array(params, "authors", valid_did, "valid DIDs") - local uris = array(params, "uris", valid_uri, "full app.certified.location AT-URIs with DID authorities") + local uris = array(params, "uris", valid_location_uri, "full app.certified.location AT-URIs with DID authorities") local types = array(params, "locationTypes", nil, nil, 20) - local limit_value = scalar(params, "limit") - if limit_value and not limit_value:match("^%d+$") then invalid("limit must be an integer from 1 through 100") end - local limit = limit_value and tonumber(limit_value) or 25 - if not limit or limit % 1 ~= 0 or limit < 1 or limit > 100 then invalid("limit must be an integer from 1 through 100") end - local direction = scalar(params, "sortDirection") or "desc" - if direction ~= "asc" and direction ~= "desc" then invalid("sortDirection must be 'asc' or 'desc'") end + local limit = parse_list_limit(params) + local direction = parse_sort_direction(params) local cursor = cursor_decode(scalar(params, "cursor"), direction) local views, next_cursor = query_locations({ authors = authors, uris = uris, locationTypes = types }, limit, cursor, direction) local response = { locations = toarray(views) } diff --git a/hypercerts-api/lua/shared/actorFollow.lua b/hypercerts-api/lua/shared/actorFollow.lua index 5fd594b6..ad552b53 100644 --- a/hypercerts-api/lua/shared/actorFollow.lua +++ b/hypercerts-api/lua/shared/actorFollow.lua @@ -1,32 +1,4 @@ local FOLLOW = "app.certified.graph.follow" -local NULL = json.decode("null") - -local function invalid(message) - error("InvalidRequest: " .. message, 0) -end - -local function keys_only(values, allowed) - for key in pairs(values) do - if not allowed[key] then invalid("unknown query parameter") end - end -end - -local function scalar(values, key) - local value = values[key] - if value == nil then return nil end - if type(value) ~= "string" and type(value) ~= "number" then - invalid(key .. " must occur once") - end - return tostring(value) -end - -local function valid_did(value) - if #value > 2048 then return false end - local method, specific = value:match("^did:([a-z]+):(.+)$") - if not method or not specific or specific:sub(-1) == ":" or specific:sub(-1) == "%" - or value:find("[^%w%.:_%%%-]") then return false end - return true -end local function query(sql, values) if db.backend() ~= "postgres" then error("ActorFollowQueryFailed: actor-follow queries require PostgreSQL", 0) end @@ -34,13 +6,3 @@ local function query(sql, values) if not ok then error("ActorFollowQueryFailed: actor-follow lookup failed", 0) end return result end - -local function row_view(row) - return { - uri = row.uri, - cid = row.cid, - indexedAt = row.indexed_at, - did = row.did, - record = json.decode(row.record), - } -end diff --git a/hypercerts-api/lua/shared/actorFollowList.lua b/hypercerts-api/lua/shared/actorFollowList.lua index 83aa45ef..8d0a9556 100644 --- a/hypercerts-api/lua/shared/actorFollowList.lua +++ b/hypercerts-api/lua/shared/actorFollowList.lua @@ -1,70 +1,6 @@ -local PROFILE = "app.certified.actor.profile" -local ORGANIZATION = "app.certified.actor.organization" - -local function valid_record_key(value) - return #value >= 1 and #value <= 512 and value ~= "." and value ~= ".." - and not value:find("[^%w_~%.:%-]") -end - local function valid_follow_uri(value) - if type(value) ~= "string" or value:find("[?#]") then return false end - local authority, collection, rkey = value:match("^at://([^/]+)/([^/]+)/([^/]+)$") - return authority ~= nil and valid_did(authority) and collection == FOLLOW and valid_record_key(rkey) -end - -local function valid_datetime(value) - local year, month, day, hour, minute, second, suffix = value:match( - "^(%d%d%d%d)%-(%d%d)%-(%d%d)T(%d%d):(%d%d):(%d%d)(.*)$") - if not year then return false end - year, month, day = tonumber(year), tonumber(month), tonumber(day) - hour, minute, second = tonumber(hour), tonumber(minute), tonumber(second) - if month < 1 or month > 12 or hour > 23 or minute > 59 or second > 59 then return false end - local leap = year % 4 == 0 and (year % 100 ~= 0 or year % 400 == 0) - local month_days = { 31, leap and 29 or 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 } - if day < 1 or day > month_days[month] then return false end - local fraction, zone = suffix:match("^(%.%d+)(Z)$") - if not fraction then fraction, zone = suffix:match("^(%.%d+)([+-]%d%d:%d%d)$") end - if not fraction then zone = suffix:match("^(Z)$") end - if not zone then zone = suffix:match("^([+-]%d%d:%d%d)$") end - if not zone or zone == "-00:00" then return false end - if zone ~= "Z" then - local zh, zm = zone:match("^[+-](%d%d):(%d%d)$") - if not zh or tonumber(zh) > 23 or tonumber(zm) > 59 then return false end - end - return true -end - -local function hydrate(views) - if #views == 0 then return end - local dids, seen = {}, {} - for _, view in ipairs(views) do - if not seen[view.did] then - seen[view.did] = true - dids[#dids + 1] = view.did - end - end - local profiles, organizations = {}, {} - local function load(collection, target) - local values, marks = { collection }, {} - for _, did in ipairs(dids) do - values[#values + 1] = did - marks[#marks + 1] = "$" .. #values - end - local rows = query("SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record FROM happyview_records WHERE collection = $1 AND rkey = 'self' AND did IN (" .. table.concat(marks, ",") .. ")", values) - for _, row in ipairs(rows) do target[row.did] = row end - end - load(PROFILE, profiles) - load(ORGANIZATION, organizations) - for _, view in ipairs(views) do - local profile, organization = profiles[view.did], organizations[view.did] - view.profile = profile and row_view(profile) or NULL - view.organization = organization and row_view(organization) or NULL - end -end - -local function cursor_encode(value) - local encoded = json.encode(value) - return (encoded:gsub(".", function(char) return string.format("%02x", string.byte(char)) end)) + local valid, collection = valid_record_uri(value) + return valid and collection == FOLLOW end local function cursor_decode(token, direction) @@ -112,9 +48,9 @@ local function query_actor_follows(mode, actor, limit, cursor, direction) local views = {} for _, row in ipairs(rows) do - views[#views + 1] = { did = row.actor_did, follow = row_view(row) } + views[#views + 1] = { did = row.actor_did, follow = record_view(row) } end - hydrate(views) + hydrate_actor_views(views, query) local next_cursor if more then @@ -129,13 +65,8 @@ local function list_actor_follows(mode) local actor = scalar(params, "actor") if not actor or not valid_did(actor) then invalid("actor must be a valid DID") end - local limit_value = scalar(params, "limit") - if limit_value and not limit_value:match("^%d+$") then invalid("limit must be an integer from 1 through 100") end - local limit = limit_value and tonumber(limit_value) or 25 - if not limit or limit % 1 ~= 0 or limit < 1 or limit > 100 then invalid("limit must be an integer from 1 through 100") end - - local direction = scalar(params, "sortDirection") or "desc" - if direction ~= "asc" and direction ~= "desc" then invalid("sortDirection must be 'asc' or 'desc'") end + local limit = parse_list_limit(params) + local direction = parse_sort_direction(params) local cursor = cursor_decode(scalar(params, "cursor"), direction) local views, next_cursor = query_actor_follows(mode, actor, limit, cursor, direction) local response = { [mode] = toarray(views) } diff --git a/hypercerts-api/lua/shared/actorFollowLookup.lua b/hypercerts-api/lua/shared/actorFollowLookup.lua index ad3147e4..729d902b 100644 --- a/hypercerts-api/lua/shared/actorFollowLookup.lua +++ b/hypercerts-api/lua/shared/actorFollowLookup.lua @@ -3,7 +3,7 @@ local function query_follow(actor, subject) "SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record FROM happyview_records WHERE collection = $1 AND did = $2 AND record::jsonb->>'subject' = $3 ORDER BY (record::jsonb->>'createdAt')::timestamptz ASC, uri ASC LIMIT 1", { FOLLOW, actor, subject }) if #rows == 0 then return NULL end - return row_view(rows[1]) + return record_view(rows[1]) end local function get_follow() diff --git a/hypercerts-api/lua/shared/actorView.lua b/hypercerts-api/lua/shared/actorView.lua new file mode 100644 index 00000000..11187d16 --- /dev/null +++ b/hypercerts-api/lua/shared/actorView.lua @@ -0,0 +1,29 @@ +local PROFILE = "app.certified.actor.profile" +local ORGANIZATION = "app.certified.actor.organization" + +local function hydrate_actor_views(actors, run_query) + if #actors == 0 then return end + local dids, seen = {}, {} + for _, actor in ipairs(actors) do + if not seen[actor.did] then + seen[actor.did] = true + dids[#dids + 1] = actor.did + end + end + local profiles, organizations = {}, {} + local function load(collection, target) + local params, marks = { collection }, {} + for _, did in ipairs(dids) do + params[#params + 1] = did + marks[#marks + 1] = "$" .. #params + end + local rows = run_query("SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record FROM happyview_records WHERE collection = $1 AND rkey = 'self' AND did IN (" .. table.concat(marks, ",") .. ")", params) + for _, row in ipairs(rows) do target[row.did] = row end + end + load(PROFILE, profiles) + load(ORGANIZATION, organizations) + for _, actor in ipairs(actors) do + actor.profile = profiles[actor.did] and record_view(profiles[actor.did]) or NULL + actor.organization = organizations[actor.did] and record_view(organizations[actor.did]) or NULL + end +end diff --git a/hypercerts-api/lua/shared/listQuery.lua b/hypercerts-api/lua/shared/listQuery.lua new file mode 100644 index 00000000..9dede23f --- /dev/null +++ b/hypercerts-api/lua/shared/listQuery.lua @@ -0,0 +1,40 @@ +local function valid_datetime(value) + local year, month, day, hour, minute, second, suffix = value:match( + "^(%d%d%d%d)%-(%d%d)%-(%d%d)T(%d%d):(%d%d):(%d%d)(.*)$") + if not year then return false end + year, month, day = tonumber(year), tonumber(month), tonumber(day) + hour, minute, second = tonumber(hour), tonumber(minute), tonumber(second) + if month < 1 or month > 12 or hour > 23 or minute > 59 or second > 59 then return false end + local leap = year % 4 == 0 and (year % 100 ~= 0 or year % 400 == 0) + local month_days = { 31, leap and 29 or 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 } + if day < 1 or day > month_days[month] then return false end + local fraction, zone = suffix:match("^(%.%d+)(Z)$") + if not fraction then fraction, zone = suffix:match("^(%.%d+)([+-]%d%d:%d%d)$") end + if not fraction then zone = suffix:match("^(Z)$") end + if not zone then zone = suffix:match("^([+-]%d%d:%d%d)$") end + if not zone or zone == "-00:00" then return false end + if zone ~= "Z" then + local zh, zm = zone:match("^[+-](%d%d):(%d%d)$") + if not zh or tonumber(zh) > 23 or tonumber(zm) > 59 then return false end + end + return true +end + +local function parse_list_limit(params) + local limit_value = scalar(params, "limit") + if limit_value and not limit_value:match("^%d+$") then invalid("limit must be an integer from 1 through 100") end + local limit = limit_value and tonumber(limit_value) or 25 + if not limit or limit % 1 ~= 0 or limit < 1 or limit > 100 then invalid("limit must be an integer from 1 through 100") end + return limit +end + +local function parse_sort_direction(params) + local direction = scalar(params, "sortDirection") or "desc" + if direction ~= "asc" and direction ~= "desc" then invalid("sortDirection must be 'asc' or 'desc'") end + return direction +end + +local function cursor_encode(value) + local encoded = json.encode(value) + return (encoded:gsub(".", function(char) return string.format("%02x", string.byte(char)) end)) +end diff --git a/hypercerts-api/lua/shared/location.lua b/hypercerts-api/lua/shared/location.lua index 8c8037c5..a5a778c1 100644 --- a/hypercerts-api/lua/shared/location.lua +++ b/hypercerts-api/lua/shared/location.lua @@ -1,44 +1,8 @@ local COLLECTION = "app.certified.location" -local PROFILE = "app.certified.actor.profile" -local ORGANIZATION = "app.certified.actor.organization" -local NULL = json.decode("null") -local function invalid(message) - error("InvalidRequest: " .. message, 0) -end - -local function keys_only(values, allowed) - for key in pairs(values) do - if not allowed[key] then invalid("unknown query parameter") end - end -end - -local function scalar(params, key) - local value = params[key] - if value == nil then return nil end - if type(value) ~= "string" and type(value) ~= "number" then - invalid(key .. " must occur once") - end - return tostring(value) -end - -local function valid_did(value) - if #value > 2048 then return false end - local method, specific = value:match("^did:([a-z]+):(.+)$") - if not method or not specific or specific:sub(-1) == ":" or specific:sub(-1) == "%" - or value:find("[^%w%.:_%%%-]") then return false end - return true -end - -local function valid_record_key(value) - return #value >= 1 and #value <= 512 and value ~= "." and value ~= ".." - and not value:find("[^%w_~%.:%-]") -end - -local function valid_uri(value) - if value:find("[?#]") then return false end - local authority, collection, rkey = value:match("^at://([^/]+)/([^/]+)/([^/]+)$") - return authority ~= nil and valid_did(authority) and collection == COLLECTION and valid_record_key(rkey) +local function valid_location_uri(value) + local valid, collection = valid_record_uri(value) + return valid and collection == COLLECTION end local function query(sql, values) @@ -47,36 +11,12 @@ local function query(sql, values) return result end -local function row_view(row) - local record = json.decode(row.record) - return { - uri = row.uri, cid = row.cid, indexedAt = row.indexed_at, did = row.did, - record = record, - } -end - local function hydrate(views) - if #views == 0 then return end - local dids, seen = {}, {} - for _, view in ipairs(views) do - if not seen[view.did] then seen[view.did] = true; dids[#dids + 1] = view.did end - end - local profiles, organizations = {}, {} - local function load(collection, target) - if #dids == 0 then return end - local params, marks = {}, {} - params[1] = collection - for _, did in ipairs(dids) do params[#params + 1] = did; marks[#marks + 1] = "$" .. #params end - local rows = query("SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record FROM happyview_records WHERE collection = $1 AND rkey = 'self' AND did IN (" .. table.concat(marks, ",") .. ")", params) - for _, row in ipairs(rows) do target[row.did] = row end - end - load(PROFILE, profiles) - load(ORGANIZATION, organizations) + local authors = {} for _, view in ipairs(views) do - local profile, organization = profiles[view.did], organizations[view.did] - local author = { did = view.did, profile = NULL, organization = NULL } - if profile then author.profile = row_view(profile) end - if organization then author.organization = row_view(organization) end + local author = { did = view.did } view.author = author + authors[#authors + 1] = author end + hydrate_actor_views(authors, query) end diff --git a/hypercerts-api/lua/shared/query.lua b/hypercerts-api/lua/shared/query.lua new file mode 100644 index 00000000..057159d6 --- /dev/null +++ b/hypercerts-api/lua/shared/query.lua @@ -0,0 +1,26 @@ +local function invalid(message) + error("InvalidRequest: " .. message, 0) +end + +local function keys_only(values, allowed) + for key in pairs(values) do + if not allowed[key] then invalid("unknown query parameter") end + end +end + +local function scalar(params, key) + local value = params[key] + if value == nil then return nil end + if type(value) ~= "string" and type(value) ~= "number" then + invalid(key .. " must occur once") + end + return tostring(value) +end + +local function valid_did(value) + if #value > 2048 then return false end + local method, specific = value:match("^did:([a-z]+):(.+)$") + if not method or not specific or specific:sub(-1) == ":" or specific:sub(-1) == "%" + or value:find("[^%w%.:_%%%-]") then return false end + return true +end diff --git a/hypercerts-api/lua/shared/recordIdentifier.lua b/hypercerts-api/lua/shared/recordIdentifier.lua new file mode 100644 index 00000000..2c015333 --- /dev/null +++ b/hypercerts-api/lua/shared/recordIdentifier.lua @@ -0,0 +1,11 @@ +local function valid_record_key(value) + return #value >= 1 and #value <= 512 and value ~= "." and value ~= ".." + and not value:find("[^%w_~%.:%-]") +end + +local function valid_record_uri(value) + if type(value) ~= "string" or value:find("[?#]") then return false end + local authority, collection, rkey = value:match("^at://([^/]+)/([^/]+)/([^/]+)$") + if not authority or not valid_did(authority) or not valid_record_key(rkey) then return false end + return true, collection +end diff --git a/hypercerts-api/lua/shared/recordView.lua b/hypercerts-api/lua/shared/recordView.lua new file mode 100644 index 00000000..00a2b5ac --- /dev/null +++ b/hypercerts-api/lua/shared/recordView.lua @@ -0,0 +1,11 @@ +local NULL = json.decode("null") + +local function record_view(row) + return { + uri = row.uri, + cid = row.cid, + indexedAt = row.indexed_at, + did = row.did, + record = json.decode(row.record), + } +end diff --git a/hypercerts-api/lua/src/getLocation.lua b/hypercerts-api/lua/src/getLocation.lua index 578bb781..3a27b95c 100644 --- a/hypercerts-api/lua/src/getLocation.lua +++ b/hypercerts-api/lua/src/getLocation.lua @@ -2,7 +2,7 @@ local function query_location(uri) if db.backend() ~= "postgres" then error("LocationQueryFailed: location API requires PostgreSQL", 0) end local rows = query("SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record FROM happyview_records WHERE collection = $1 AND uri = $2 LIMIT 1", { COLLECTION, uri }) local views = {} - for _, row in ipairs(rows) do views[#views + 1] = row_view(row) end + for _, row in ipairs(rows) do views[#views + 1] = record_view(row) end hydrate(views) return views end @@ -10,7 +10,7 @@ end local function get_location() keys_only(params, { uri = true }) local uri = scalar(params, "uri") - if not uri or not valid_uri(uri) then invalid("uri must be a full app.certified.location AT-URI with a DID authority") end + if not uri or not valid_location_uri(uri) then invalid("uri must be a full app.certified.location AT-URI with a DID authority") end local views = query_location(uri) if #views == 0 then error("RecordNotFound: location record is not indexed", 0) end return { location = views[1] } diff --git a/hypercerts-api/lua/src/listLocations.lua b/hypercerts-api/lua/src/listLocations.lua index 45b178cd..f3eaeb07 100644 --- a/hypercerts-api/lua/src/listLocations.lua +++ b/hypercerts-api/lua/src/listLocations.lua @@ -1,25 +1,3 @@ -local function valid_datetime(value) - local year, month, day, hour, minute, second, suffix = value:match( - "^(%d%d%d%d)%-(%d%d)%-(%d%d)T(%d%d):(%d%d):(%d%d)(.*)$") - if not year then return false end - year, month, day = tonumber(year), tonumber(month), tonumber(day) - hour, minute, second = tonumber(hour), tonumber(minute), tonumber(second) - if month < 1 or month > 12 or hour > 23 or minute > 59 or second > 59 then return false end - local leap = year % 4 == 0 and (year % 100 ~= 0 or year % 400 == 0) - local month_days = { 31, leap and 29 or 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 } - if day < 1 or day > month_days[month] then return false end - local fraction, zone = suffix:match("^(%.%d+)(Z)$") - if not fraction then fraction, zone = suffix:match("^(%.%d+)([+-]%d%d:%d%d)$") end - if not fraction then zone = suffix:match("^(Z)$") end - if not zone then zone = suffix:match("^([+-]%d%d:%d%d)$") end - if not zone or zone == "-00:00" then return false end - if zone ~= "Z" then - local zh, zm = zone:match("^[+-](%d%d):(%d%d)$") - if not zh or tonumber(zh) > 23 or tonumber(zm) > 59 then return false end - end - return true -end - local function array(params, key, validate, description, max_bytes) local value = params[key] if value == nil then return nil end @@ -52,11 +30,6 @@ local function add_in(where, params, column, values) where[#where + 1] = column .. " IN (" .. table.concat(placeholders, ",") .. ")" end -local function cursor_encode(value) - local encoded = json.encode(value) - return (encoded:gsub(".", function(char) return string.format("%02x", string.byte(char)) end)) -end - local function cursor_decode(token, direction) if not token then return nil end if #token % 2 ~= 0 or token:find("[^0-9a-f]") then invalid("cursor is malformed") end @@ -69,7 +42,7 @@ local function cursor_decode(token, direction) for key in pairs(value) do if key ~= "v" and key ~= "d" and key ~= "t" and key ~= "u" then invalid("cursor is malformed") end end - if not valid_uri(value.u) or not valid_datetime(value.t) then invalid("cursor is malformed") end + if not valid_location_uri(value.u) or not valid_datetime(value.t) then invalid("cursor is malformed") end return value end @@ -98,7 +71,7 @@ local function query_locations(filters, limit, cursor, direction) local more = #rows > limit if more then rows[#rows] = nil end local views = {} - for _, row in ipairs(rows) do views[#views + 1] = row_view(row) end + for _, row in ipairs(rows) do views[#views + 1] = record_view(row) end hydrate(views) local next_cursor if more then @@ -111,14 +84,10 @@ end local function list_locations() keys_only(params, { authors = true, uris = true, locationTypes = true, limit = true, cursor = true, sortDirection = true }) local authors = array(params, "authors", valid_did, "valid DIDs") - local uris = array(params, "uris", valid_uri, "full app.certified.location AT-URIs with DID authorities") + local uris = array(params, "uris", valid_location_uri, "full app.certified.location AT-URIs with DID authorities") local types = array(params, "locationTypes", nil, nil, 20) - local limit_value = scalar(params, "limit") - if limit_value and not limit_value:match("^%d+$") then invalid("limit must be an integer from 1 through 100") end - local limit = limit_value and tonumber(limit_value) or 25 - if not limit or limit % 1 ~= 0 or limit < 1 or limit > 100 then invalid("limit must be an integer from 1 through 100") end - local direction = scalar(params, "sortDirection") or "desc" - if direction ~= "asc" and direction ~= "desc" then invalid("sortDirection must be 'asc' or 'desc'") end + local limit = parse_list_limit(params) + local direction = parse_sort_direction(params) local cursor = cursor_decode(scalar(params, "cursor"), direction) local views, next_cursor = query_locations({ authors = authors, uris = uris, locationTypes = types }, limit, cursor, direction) local response = { locations = toarray(views) } diff --git a/hypercerts-api/modules/actor-follow/manifest.json b/hypercerts-api/modules/actor-follow/manifest.json index 259f9feb..59ad7dbf 100644 --- a/hypercerts-api/modules/actor-follow/manifest.json +++ b/hypercerts-api/modules/actor-follow/manifest.json @@ -46,6 +46,8 @@ "path": "../../lua/endpoints/getFollow.lua", "sourcePath": "../../lua/src/getFollow.lua", "sharedSourcePaths": [ + "../../lua/shared/query.lua", + "../../lua/shared/recordView.lua", "../../lua/shared/actorFollow.lua", "../../lua/shared/actorFollowLookup.lua" ], @@ -61,6 +63,11 @@ "path": "../../lua/endpoints/listActorFollowers.lua", "sourcePath": "../../lua/src/listActorFollowers.lua", "sharedSourcePaths": [ + "../../lua/shared/query.lua", + "../../lua/shared/recordIdentifier.lua", + "../../lua/shared/listQuery.lua", + "../../lua/shared/recordView.lua", + "../../lua/shared/actorView.lua", "../../lua/shared/actorFollow.lua", "../../lua/shared/actorFollowList.lua" ], @@ -76,6 +83,11 @@ "path": "../../lua/endpoints/listActorFollowing.lua", "sourcePath": "../../lua/src/listActorFollowing.lua", "sharedSourcePaths": [ + "../../lua/shared/query.lua", + "../../lua/shared/recordIdentifier.lua", + "../../lua/shared/listQuery.lua", + "../../lua/shared/recordView.lua", + "../../lua/shared/actorView.lua", "../../lua/shared/actorFollow.lua", "../../lua/shared/actorFollowList.lua" ], diff --git a/hypercerts-api/modules/location/manifest.json b/hypercerts-api/modules/location/manifest.json index d5f3fd9e..d6c220bf 100644 --- a/hypercerts-api/modules/location/manifest.json +++ b/hypercerts-api/modules/location/manifest.json @@ -92,7 +92,13 @@ "id": "xrpc.query:app.certified.location.getLocation", "path": "../../lua/endpoints/getLocation.lua", "sourcePath": "../../lua/src/getLocation.lua", - "sharedSourcePath": "../../lua/shared/location.lua", + "sharedSourcePaths": [ + "../../lua/shared/query.lua", + "../../lua/shared/recordIdentifier.lua", + "../../lua/shared/recordView.lua", + "../../lua/shared/actorView.lua", + "../../lua/shared/location.lua" + ], "config": { "script_type": "lua", "description": "Location API handler" @@ -106,7 +112,14 @@ "id": "xrpc.query:app.certified.location.listLocations", "path": "../../lua/endpoints/listLocations.lua", "sourcePath": "../../lua/src/listLocations.lua", - "sharedSourcePath": "../../lua/shared/location.lua", + "sharedSourcePaths": [ + "../../lua/shared/query.lua", + "../../lua/shared/recordIdentifier.lua", + "../../lua/shared/listQuery.lua", + "../../lua/shared/recordView.lua", + "../../lua/shared/actorView.lua", + "../../lua/shared/location.lua" + ], "config": { "script_type": "lua", "description": "Location API handler" diff --git a/hypercerts-api/tooling/actor-follow.test.js b/hypercerts-api/tooling/actor-follow.test.js index d0273aac..3d2af1f8 100644 --- a/hypercerts-api/tooling/actor-follow.test.js +++ b/hypercerts-api/tooling/actor-follow.test.js @@ -310,6 +310,9 @@ test('actor-follow queries reject invalid DIDs, limits, scalar repetition, unkno { actor, extra: 'not-supported' }, { actor, cursor: 'not-hex' }, { actor, cursor: cursor({ timestamp: '2025-02-30T00:00:00Z' }) }, + { actor, cursor: cursor({ uri: `at://${follower}/app.certified.graph.other/key` }) }, + { actor, cursor: cursor({ uri: `at://${follower}/${followCollection}/.` }) }, + { actor, cursor: cursor({ uri: `at://${follower}/${followCollection}/${'a'.repeat(513)}` }) }, { actor, cursor: cursor({ direction: 'asc' }), sortDirection: 'desc' }, ]) { runLua({ endpoint: 'listActorFollowers', params, expectError: 'InvalidRequest:' }); diff --git a/hypercerts-api/tooling/build-lua.test.js b/hypercerts-api/tooling/build-lua.test.js index 1bfe8fd8..49dff267 100644 --- a/hypercerts-api/tooling/build-lua.test.js +++ b/hypercerts-api/tooling/build-lua.test.js @@ -6,6 +6,11 @@ import path from 'node:path'; import { buildLuaBundles, checkLuaBundles } from './lua-bundles.js'; const sources = [ + ['lua/shared/query.lua', 'local function query_common() return "query" end\n'], + ['lua/shared/recordIdentifier.lua', 'local function record_identifier_common() return "identifier" end\n'], + ['lua/shared/listQuery.lua', 'local function list_query_common() return "list query" end\n'], + ['lua/shared/recordView.lua', 'local function record_view_common() return "record" end\n'], + ['lua/shared/actorView.lua', 'local function actor_view_common() return "actor" end\n'], ['lua/shared/location.lua', 'local function query() return "location" end\n'], ['lua/shared/actorFollow.lua', 'local function query() return "follow" end\n'], ['lua/shared/actorFollowLookup.lua', 'local function lookup() return true end\n'], @@ -32,13 +37,11 @@ async function withLuaRoot(run) { } } -test('builds a handler bundle from its shared and endpoint sources', async () => { +test('builds a handler bundle with shared source before its endpoint', async () => { await withLuaRoot(async (root) => { await buildLuaBundles(root); - assert.equal( - await readFile(path.join(root, 'lua/endpoints/getLocation.lua'), 'utf8'), - 'local function query() return "location" end\n\nfunction handle() return query() end\n', - ); + const built = await readFile(path.join(root, 'lua/endpoints/getLocation.lua'), 'utf8'); + assert.ok(built.endsWith('local function query() return "location" end\n\nfunction handle() return query() end\n')); }); }); diff --git a/hypercerts-api/tooling/lua-bundles.js b/hypercerts-api/tooling/lua-bundles.js index 8f6f5db0..fd493090 100644 --- a/hypercerts-api/tooling/lua-bundles.js +++ b/hypercerts-api/tooling/lua-bundles.js @@ -2,9 +2,10 @@ import { readFile, writeFile } from 'node:fs/promises'; import path from 'node:path'; const bundles = [ - { shared: ['location'], endpoints: ['getLocation', 'listLocations'] }, - { shared: ['actorFollow', 'actorFollowLookup'], endpoints: ['getFollow'] }, - { shared: ['actorFollow', 'actorFollowList'], endpoints: ['listActorFollowers', 'listActorFollowing'] }, + { shared: ['query', 'recordIdentifier', 'recordView', 'actorView', 'location'], endpoints: ['getLocation'] }, + { shared: ['query', 'recordIdentifier', 'listQuery', 'recordView', 'actorView', 'location'], endpoints: ['listLocations'] }, + { shared: ['query', 'recordView', 'actorFollow', 'actorFollowLookup'], endpoints: ['getFollow'] }, + { shared: ['query', 'recordIdentifier', 'listQuery', 'recordView', 'actorView', 'actorFollow', 'actorFollowList'], endpoints: ['listActorFollowers', 'listActorFollowing'] }, ]; async function renderBundles(root) { From 5927cc53571b04658f51eb8024c71c3deeaefcdf Mon Sep 17 00:00:00 2001 From: kzoeps Date: Mon, 28 Sep 2026 18:00:41 +0600 Subject: [PATCH 6/8] hypercerts-api: include actor-follow totals in listing responses --- hypercerts-api/README.md | 2 +- ...pp.certified.graph.listActorFollowers.json | 7 ++- ...pp.certified.graph.listActorFollowing.json | 7 ++- .../lua/endpoints/listActorFollowers.lua | 14 +++-- .../lua/endpoints/listActorFollowing.lua | 14 +++-- hypercerts-api/lua/shared/actorFollowList.lua | 14 +++-- .../contracts/actor-follow.contract.test.js | 3 ++ hypercerts-api/tooling/actor-follow.test.js | 54 +++++++++++++++++-- 8 files changed, 97 insertions(+), 18 deletions(-) diff --git a/hypercerts-api/README.md b/hypercerts-api/README.md index e7fbc8ae..ca3b7028 100644 --- a/hypercerts-api/README.md +++ b/hypercerts-api/README.md @@ -4,7 +4,7 @@ This directory contains the tooling for building and testing a Hypercerts XRPC A **Current status:** The root `manifest.json` includes the location and actor-follow query slices. Build their Lua handlers and run the offline checks before installing the bundle on an approved HappyView target. -The actor-follow slice exposes `app.certified.graph.getFollow`, `app.certified.graph.listActorFollowers`, and `app.certified.graph.listActorFollowing`, backed by the pinned `app.certified.graph.follow` record Lexicon. +The actor-follow slice exposes `app.certified.graph.getFollow`, `app.certified.graph.listActorFollowers`, and `app.certified.graph.listActorFollowing`, backed by the pinned `app.certified.graph.follow` record Lexicon. Both listing queries return a required `totalCount` for the complete deduplicated relationship set, independent of cursor and limit. ## Get started diff --git a/hypercerts-api/lexicons/app.certified.graph.listActorFollowers.json b/hypercerts-api/lexicons/app.certified.graph.listActorFollowers.json index 77276d6c..16516cf3 100644 --- a/hypercerts-api/lexicons/app.certified.graph.listActorFollowers.json +++ b/hypercerts-api/lexicons/app.certified.graph.listActorFollowers.json @@ -41,13 +41,18 @@ }, "output": { "type": "object", - "required": ["followers"], + "required": ["followers", "totalCount"], "properties": { "followers": { "type": "array", "items": { "type": "ref", "ref": "app.certified.graph.getFollow#actorFollowView" }, "description": "Incoming relationships, with each displayed DID identifying the actor who published the follow." }, + "totalCount": { + "type": "integer", + "minimum": 0, + "description": "Number of matching relationships after duplicate relationships are collapsed, before cursor and limit are applied." + }, "cursor": { "type": "string", "description": "Opaque cursor for the next page; omitted when there are no more relationships." diff --git a/hypercerts-api/lexicons/app.certified.graph.listActorFollowing.json b/hypercerts-api/lexicons/app.certified.graph.listActorFollowing.json index f8dba0d5..a4cc6e83 100644 --- a/hypercerts-api/lexicons/app.certified.graph.listActorFollowing.json +++ b/hypercerts-api/lexicons/app.certified.graph.listActorFollowing.json @@ -41,13 +41,18 @@ }, "output": { "type": "object", - "required": ["following"], + "required": ["following", "totalCount"], "properties": { "following": { "type": "array", "items": { "type": "ref", "ref": "app.certified.graph.getFollow#actorFollowView" }, "description": "Outgoing relationships, with each displayed DID identifying the actor named as the follow subject." }, + "totalCount": { + "type": "integer", + "minimum": 0, + "description": "Number of matching relationships after duplicate relationships are collapsed, before cursor and limit are applied." + }, "cursor": { "type": "string", "description": "Opaque cursor for the next page; omitted when there are no more relationships." diff --git a/hypercerts-api/lua/endpoints/listActorFollowers.lua b/hypercerts-api/lua/endpoints/listActorFollowers.lua index f6ceb241..f4df1d6b 100644 --- a/hypercerts-api/lua/endpoints/listActorFollowers.lua +++ b/hypercerts-api/lua/endpoints/listActorFollowers.lua @@ -172,8 +172,14 @@ local function query_actor_follows(mode, actor, limit, cursor, direction) values[#values + 1] = limit + 1 local ordering = direction == "asc" and "ASC" or "DESC" local displayed_did = mode == "followers" and "did" or "subject_did" - local sql = "WITH ranked_follows AS (SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record, record::jsonb->>'subject' AS subject_did, (record::jsonb->>'createdAt')::timestamptz AS sort_at, ROW_NUMBER() OVER (PARTITION BY did, record::jsonb->>'subject' ORDER BY (record::jsonb->>'createdAt')::timestamptz ASC, uri ASC) AS relationship_rank FROM happyview_records WHERE " .. table.concat(filters, " AND ") .. "), representatives AS (SELECT uri, did, cid, indexed_at, record, sort_at, " .. displayed_did .. " AS actor_did FROM ranked_follows WHERE relationship_rank = 1) SELECT uri, did, cid, indexed_at, record, actor_did, to_char(sort_at AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') AS sort_timestamp FROM representatives" .. cursor_filter .. " ORDER BY sort_at " .. ordering .. ", uri " .. ordering .. " LIMIT $" .. #values + local sql = "WITH ranked_follows AS (SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record, record::jsonb->>'subject' AS subject_did, (record::jsonb->>'createdAt')::timestamptz AS sort_at, ROW_NUMBER() OVER (PARTITION BY did, record::jsonb->>'subject' ORDER BY (record::jsonb->>'createdAt')::timestamptz ASC, uri ASC) AS relationship_rank FROM happyview_records WHERE " .. table.concat(filters, " AND ") .. "), representatives AS (SELECT uri, did, cid, indexed_at, record, sort_at, " .. displayed_did .. " AS actor_did FROM ranked_follows WHERE relationship_rank = 1), " .. + "total AS (SELECT COUNT(*) AS total_count FROM representatives), " .. + "page AS (SELECT uri, did, cid, indexed_at, record, sort_at, actor_did FROM representatives" .. cursor_filter .. " ORDER BY sort_at " .. ordering .. ", uri " .. ordering .. " LIMIT $" .. #values .. ") " .. + "SELECT page.uri, page.did, page.cid, page.indexed_at, page.record, page.actor_did, total.total_count, to_char(page.sort_at AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') AS sort_timestamp FROM total LEFT JOIN page ON TRUE ORDER BY page.sort_at " .. ordering .. ", page.uri " .. ordering local rows = query(sql, values) + local total_count = rows[1] and tonumber(rows[1].total_count) + if not total_count or total_count < 0 or total_count % 1 ~= 0 then error("ActorFollowQueryFailed: actor-follow count unavailable", 0) end + if rows[1].uri == nil then rows = {} end local more = #rows > limit if more then rows[#rows] = nil end @@ -188,7 +194,7 @@ local function query_actor_follows(mode, actor, limit, cursor, direction) local last = rows[#rows] next_cursor = cursor_encode({ v = 1, d = direction, t = last.sort_timestamp, u = last.uri }) end - return views, next_cursor + return views, next_cursor, total_count end local function list_actor_follows(mode) @@ -199,8 +205,8 @@ local function list_actor_follows(mode) local limit = parse_list_limit(params) local direction = parse_sort_direction(params) local cursor = cursor_decode(scalar(params, "cursor"), direction) - local views, next_cursor = query_actor_follows(mode, actor, limit, cursor, direction) - local response = { [mode] = toarray(views) } + local views, next_cursor, total_count = query_actor_follows(mode, actor, limit, cursor, direction) + local response = { [mode] = toarray(views), totalCount = total_count } if next_cursor then response.cursor = next_cursor end return response end diff --git a/hypercerts-api/lua/endpoints/listActorFollowing.lua b/hypercerts-api/lua/endpoints/listActorFollowing.lua index 69799254..4300f2a0 100644 --- a/hypercerts-api/lua/endpoints/listActorFollowing.lua +++ b/hypercerts-api/lua/endpoints/listActorFollowing.lua @@ -172,8 +172,14 @@ local function query_actor_follows(mode, actor, limit, cursor, direction) values[#values + 1] = limit + 1 local ordering = direction == "asc" and "ASC" or "DESC" local displayed_did = mode == "followers" and "did" or "subject_did" - local sql = "WITH ranked_follows AS (SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record, record::jsonb->>'subject' AS subject_did, (record::jsonb->>'createdAt')::timestamptz AS sort_at, ROW_NUMBER() OVER (PARTITION BY did, record::jsonb->>'subject' ORDER BY (record::jsonb->>'createdAt')::timestamptz ASC, uri ASC) AS relationship_rank FROM happyview_records WHERE " .. table.concat(filters, " AND ") .. "), representatives AS (SELECT uri, did, cid, indexed_at, record, sort_at, " .. displayed_did .. " AS actor_did FROM ranked_follows WHERE relationship_rank = 1) SELECT uri, did, cid, indexed_at, record, actor_did, to_char(sort_at AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') AS sort_timestamp FROM representatives" .. cursor_filter .. " ORDER BY sort_at " .. ordering .. ", uri " .. ordering .. " LIMIT $" .. #values + local sql = "WITH ranked_follows AS (SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record, record::jsonb->>'subject' AS subject_did, (record::jsonb->>'createdAt')::timestamptz AS sort_at, ROW_NUMBER() OVER (PARTITION BY did, record::jsonb->>'subject' ORDER BY (record::jsonb->>'createdAt')::timestamptz ASC, uri ASC) AS relationship_rank FROM happyview_records WHERE " .. table.concat(filters, " AND ") .. "), representatives AS (SELECT uri, did, cid, indexed_at, record, sort_at, " .. displayed_did .. " AS actor_did FROM ranked_follows WHERE relationship_rank = 1), " .. + "total AS (SELECT COUNT(*) AS total_count FROM representatives), " .. + "page AS (SELECT uri, did, cid, indexed_at, record, sort_at, actor_did FROM representatives" .. cursor_filter .. " ORDER BY sort_at " .. ordering .. ", uri " .. ordering .. " LIMIT $" .. #values .. ") " .. + "SELECT page.uri, page.did, page.cid, page.indexed_at, page.record, page.actor_did, total.total_count, to_char(page.sort_at AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') AS sort_timestamp FROM total LEFT JOIN page ON TRUE ORDER BY page.sort_at " .. ordering .. ", page.uri " .. ordering local rows = query(sql, values) + local total_count = rows[1] and tonumber(rows[1].total_count) + if not total_count or total_count < 0 or total_count % 1 ~= 0 then error("ActorFollowQueryFailed: actor-follow count unavailable", 0) end + if rows[1].uri == nil then rows = {} end local more = #rows > limit if more then rows[#rows] = nil end @@ -188,7 +194,7 @@ local function query_actor_follows(mode, actor, limit, cursor, direction) local last = rows[#rows] next_cursor = cursor_encode({ v = 1, d = direction, t = last.sort_timestamp, u = last.uri }) end - return views, next_cursor + return views, next_cursor, total_count end local function list_actor_follows(mode) @@ -199,8 +205,8 @@ local function list_actor_follows(mode) local limit = parse_list_limit(params) local direction = parse_sort_direction(params) local cursor = cursor_decode(scalar(params, "cursor"), direction) - local views, next_cursor = query_actor_follows(mode, actor, limit, cursor, direction) - local response = { [mode] = toarray(views) } + local views, next_cursor, total_count = query_actor_follows(mode, actor, limit, cursor, direction) + local response = { [mode] = toarray(views), totalCount = total_count } if next_cursor then response.cursor = next_cursor end return response end diff --git a/hypercerts-api/lua/shared/actorFollowList.lua b/hypercerts-api/lua/shared/actorFollowList.lua index 8d0a9556..141eb352 100644 --- a/hypercerts-api/lua/shared/actorFollowList.lua +++ b/hypercerts-api/lua/shared/actorFollowList.lua @@ -41,8 +41,14 @@ local function query_actor_follows(mode, actor, limit, cursor, direction) values[#values + 1] = limit + 1 local ordering = direction == "asc" and "ASC" or "DESC" local displayed_did = mode == "followers" and "did" or "subject_did" - local sql = "WITH ranked_follows AS (SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record, record::jsonb->>'subject' AS subject_did, (record::jsonb->>'createdAt')::timestamptz AS sort_at, ROW_NUMBER() OVER (PARTITION BY did, record::jsonb->>'subject' ORDER BY (record::jsonb->>'createdAt')::timestamptz ASC, uri ASC) AS relationship_rank FROM happyview_records WHERE " .. table.concat(filters, " AND ") .. "), representatives AS (SELECT uri, did, cid, indexed_at, record, sort_at, " .. displayed_did .. " AS actor_did FROM ranked_follows WHERE relationship_rank = 1) SELECT uri, did, cid, indexed_at, record, actor_did, to_char(sort_at AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') AS sort_timestamp FROM representatives" .. cursor_filter .. " ORDER BY sort_at " .. ordering .. ", uri " .. ordering .. " LIMIT $" .. #values + local sql = "WITH ranked_follows AS (SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record, record::jsonb->>'subject' AS subject_did, (record::jsonb->>'createdAt')::timestamptz AS sort_at, ROW_NUMBER() OVER (PARTITION BY did, record::jsonb->>'subject' ORDER BY (record::jsonb->>'createdAt')::timestamptz ASC, uri ASC) AS relationship_rank FROM happyview_records WHERE " .. table.concat(filters, " AND ") .. "), representatives AS (SELECT uri, did, cid, indexed_at, record, sort_at, " .. displayed_did .. " AS actor_did FROM ranked_follows WHERE relationship_rank = 1), " .. + "total AS (SELECT COUNT(*) AS total_count FROM representatives), " .. + "page AS (SELECT uri, did, cid, indexed_at, record, sort_at, actor_did FROM representatives" .. cursor_filter .. " ORDER BY sort_at " .. ordering .. ", uri " .. ordering .. " LIMIT $" .. #values .. ") " .. + "SELECT page.uri, page.did, page.cid, page.indexed_at, page.record, page.actor_did, total.total_count, to_char(page.sort_at AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') AS sort_timestamp FROM total LEFT JOIN page ON TRUE ORDER BY page.sort_at " .. ordering .. ", page.uri " .. ordering local rows = query(sql, values) + local total_count = rows[1] and tonumber(rows[1].total_count) + if not total_count or total_count < 0 or total_count % 1 ~= 0 then error("ActorFollowQueryFailed: actor-follow count unavailable", 0) end + if rows[1].uri == nil then rows = {} end local more = #rows > limit if more then rows[#rows] = nil end @@ -57,7 +63,7 @@ local function query_actor_follows(mode, actor, limit, cursor, direction) local last = rows[#rows] next_cursor = cursor_encode({ v = 1, d = direction, t = last.sort_timestamp, u = last.uri }) end - return views, next_cursor + return views, next_cursor, total_count end local function list_actor_follows(mode) @@ -68,8 +74,8 @@ local function list_actor_follows(mode) local limit = parse_list_limit(params) local direction = parse_sort_direction(params) local cursor = cursor_decode(scalar(params, "cursor"), direction) - local views, next_cursor = query_actor_follows(mode, actor, limit, cursor, direction) - local response = { [mode] = toarray(views) } + local views, next_cursor, total_count = query_actor_follows(mode, actor, limit, cursor, direction) + local response = { [mode] = toarray(views), totalCount = total_count } if next_cursor then response.cursor = next_cursor end return response end diff --git a/hypercerts-api/tests/contracts/actor-follow.contract.test.js b/hypercerts-api/tests/contracts/actor-follow.contract.test.js index 977dd5d1..e399d029 100644 --- a/hypercerts-api/tests/contracts/actor-follow.contract.test.js +++ b/hypercerts-api/tests/contracts/actor-follow.contract.test.js @@ -53,6 +53,7 @@ async function collectPages(nsid, outputKey, actor, direction, pageCount) { let cursor; for (let pageIndex = 0; pageIndex < pageCount; pageIndex += 1) { const page = await get(nsid, { actor, limit: 1, sortDirection: direction, cursor }); + assert.equal(page.totalCount, pageCount, `${nsid} ${direction} totalCount should be independent of pagination`); assert.equal(page[outputKey].length, 1, `${nsid} ${direction} page ${pageIndex + 1} should contain one relationship`); items.push(page[outputKey][0]); const hasNextPage = pageIndex < pageCount - 1; @@ -83,6 +84,7 @@ test('follower and following queries preserve direction, publisher metadata, and const incoming = await get('app.certified.graph.listActorFollowers', { actor: primarySubject, sortDirection: 'asc', limit: 100, }); + assert.equal(incoming.totalCount, 2); assert.deepEqual(incoming.followers, [ actorView( publisher, @@ -107,6 +109,7 @@ test('follower and following queries preserve direction, publisher metadata, and actorView(secondSubject, row('3jzfcijpj2z2d'), sidecar(actorFollowProfileRecords, secondSubject), undefined), actorView(thirdSubject, row('3jzfcijpj2z2f'), undefined, undefined), ]); + assert.equal(outgoing.totalCount, 3); assert.equal(outgoing.cursor, undefined); assert.equal(outgoing.following[0].follow.did, publisher, 'nested record metadata identifies its publisher'); assert.equal(outgoing.following[0].follow.record.$type, followCollection); diff --git a/hypercerts-api/tooling/actor-follow.test.js b/hypercerts-api/tooling/actor-follow.test.js index 3d2af1f8..a38a24c7 100644 --- a/hypercerts-api/tooling/actor-follow.test.js +++ b/hypercerts-api/tooling/actor-follow.test.js @@ -60,7 +60,7 @@ function cursor({ direction = 'desc', timestamp = '2025-01-02T03:04:05.000000Z', return Buffer.from(JSON.stringify({ v: 1, d: direction, t: timestamp, u: uri }), 'utf8').toString('hex'); } -function runLua({ endpoint, params, rows = [], profiles = [], organizations = [], backend = 'postgres', queryFailure = false, expectError, expectedCalls = 0, assertions }) { +function runLua({ endpoint, params, rows = [], profiles = [], organizations = [], totalCount = rows.length, backend = 'postgres', queryFailure = false, expectError, expectedCalls = 0, assertions }) { const endpointPath = `lua/endpoints/${endpoint}.lua`; assert.ok(existsSync(`${root}/${endpointPath}`), `${endpointPath} must be generated before exercising the handler`); const records = Object.fromEntries([ @@ -74,6 +74,7 @@ local RECORDS = ${lua(records)} local rows = ${lua(rows)} local profiles = ${lua(profiles)} local organizations = ${lua(organizations)} +local totalCount = ${totalCount} local calls = {} local function decode_cursor(value) local version = tonumber(value:match('"v":(%d+)')) @@ -102,7 +103,18 @@ db = { raw = function(sql, values) calls[#calls + 1] = { sql = sql, values = values } if ${queryFailure ? 'true' : 'false'} then error('fixture database failure') end - if values[1] == '${followCollection}' then return rows end + if values[1] == '${followCollection}' then + if not sql:find('total_count', 1, true) then return rows end + if #rows == 0 then return { { total_count = totalCount } } end + local countedRows = {} + for index, row in ipairs(rows) do + local countedRow = {} + for key, value in pairs(row) do countedRow[key] = value end + countedRow.total_count = totalCount + countedRows[index] = countedRow + end + return countedRows + end if values[1] == '${profileCollection}' then return profiles end if values[1] == '${organizationCollection}' then return organizations end error('unexpected collection ' .. tostring(values[1])) @@ -156,10 +168,13 @@ test('actor-follow query Lexicons validate the endpoint inputs, nullable lookup, assert.deepEqual(getFollow.defs.output.nullable, ['follow']); assert.deepEqual(Object.keys(followers.defs.main.parameters.properties).sort(), ['actor', 'cursor', 'limit', 'sortDirection']); assert.deepEqual(Object.keys(following.defs.main.parameters.properties).sort(), ['actor', 'cursor', 'limit', 'sortDirection']); - for (const query of [followers, following]) { + for (const [query, outputKey] of [[followers, 'followers'], [following, 'following']]) { assert.equal(query.defs.main.parameters.properties.limit.minimum, 1); assert.equal(query.defs.main.parameters.properties.limit.maximum, 100); assert.equal(query.defs.main.output.encoding, 'application/json'); + assert.deepEqual(query.defs.output.required, [outputKey, 'totalCount']); + assert.equal(query.defs.output.properties.totalCount.type, 'integer'); + assert.equal(query.defs.output.properties.totalCount.minimum, 0); } const sharedDefinitions = byId.get('org.hypercerts.api.defs'); const followView = lexicons.getDefOrThrow('app.certified.graph.getFollow#followRecordView'); @@ -285,6 +300,39 @@ assert(calls[1].sql:find("ORDER BY sort_at ASC, uri ASC", 1, true)) }); }); +test('actor-follow totalCount covers all collapsed relationships before pagination, including empty pages', () => { + const cases = [ + { endpoint: 'listActorFollowers', outputKey: 'followers', row: followRow('counted-follower', follower, follower, '2025-01-05T00:00:00Z') }, + { endpoint: 'listActorFollowing', outputKey: 'following', row: followRow('counted-following', actor, subject, '2025-01-05T00:00:00Z', { subjectDid: subject }) }, + ]; + + for (const { endpoint, outputKey, row } of cases) { + const assertions = ` +assert(result.totalCount == 4 and type(result.totalCount) == 'number') +assert(#result.${outputKey} == 1) +local sql = calls[1].sql +local countCte = assert(sql:find('total AS (SELECT COUNT(*) AS total_count FROM representatives)', 1, true)) +local pageCte = assert(sql:find('page AS (SELECT', 1, true)) +local cursorFilter = assert(sql:find('(sort_at, uri) <', 1, true)) +assert(countCte < pageCte and pageCte < cursorFilter, 'count the full representative set before applying cursor pagination') +assert(sql:find('LEFT JOIN page', 1, true), 'retain the count when the page has no rows') +`; + runLua({ + endpoint, + params: { actor, limit: '1', sortDirection: 'desc', cursor: cursor({ direction: 'desc', timestamp: '2025-01-06T00:00:00.000000Z' }) }, + rows: [row], totalCount: 4, + assertions, + }); + + runLua({ + endpoint, + params: { actor, limit: '1', sortDirection: 'desc', cursor: cursor({ direction: 'desc', timestamp: '2024-12-31T00:00:00.000000Z' }) }, + rows: [], totalCount: 4, + assertions: `assert(#result.${outputKey} == 0 and result.cursor == nil)\nassert(result.totalCount == 4 and type(result.totalCount) == 'number')\nassert(calls[1].sql:find('(sort_at, uri) <', 1, true), 'apply the beyond-end cursor to the page')\n`, + }); + } +}); + test('actor-follow listing defaults to 25, allows 100, and returns empty arrays without cursors', () => { runLua({ endpoint: 'listActorFollowers', params: { actor }, rows: [], From 3bf8156a22f15051c030a05e9a17132a516e2a53 Mon Sep 17 00:00:00 2001 From: kzoeps Date: Mon, 28 Sep 2026 20:59:18 +0600 Subject: [PATCH 7/8] hypercerts-api: guard malformed actor-follow timestamps --- hypercerts-api/README.md | 2 +- hypercerts-api/lua/endpoints/getFollow.lua | 9 ++++++++- .../lua/endpoints/listActorFollowers.lua | 9 ++++++++- .../lua/endpoints/listActorFollowing.lua | 9 ++++++++- hypercerts-api/lua/shared/actorFollow.lua | 7 +++++++ hypercerts-api/lua/shared/actorFollowList.lua | 2 +- .../lua/shared/actorFollowLookup.lua | 2 +- hypercerts-api/manifest.json | 5 +++++ hypercerts-api/tooling/actor-follow.test.js | 19 +++++++++++++++++-- 9 files changed, 56 insertions(+), 8 deletions(-) diff --git a/hypercerts-api/README.md b/hypercerts-api/README.md index ca3b7028..404bcd04 100644 --- a/hypercerts-api/README.md +++ b/hypercerts-api/README.md @@ -4,7 +4,7 @@ This directory contains the tooling for building and testing a Hypercerts XRPC A **Current status:** The root `manifest.json` includes the location and actor-follow query slices. Build their Lua handlers and run the offline checks before installing the bundle on an approved HappyView target. -The actor-follow slice exposes `app.certified.graph.getFollow`, `app.certified.graph.listActorFollowers`, and `app.certified.graph.listActorFollowing`, backed by the pinned `app.certified.graph.follow` record Lexicon. Both listing queries return a required `totalCount` for the complete deduplicated relationship set, independent of cursor and limit. +The actor-follow slice exposes `app.certified.graph.getFollow`, `app.certified.graph.listActorFollowers`, and `app.certified.graph.listActorFollowing`, backed by the pinned `app.certified.graph.follow` record Lexicon. Both listing queries return a required `totalCount` for the complete deduplicated relationship set, independent of cursor and limit. Lookup and listings sort follows by `createdAt`; if it is missing or malformed, they use `indexed_at` (or the database insertion time when `indexed_at` is absent) so those follows remain visible and counted. ## Get started diff --git a/hypercerts-api/lua/endpoints/getFollow.lua b/hypercerts-api/lua/endpoints/getFollow.lua index 735e2329..84931d61 100644 --- a/hypercerts-api/lua/endpoints/getFollow.lua +++ b/hypercerts-api/lua/endpoints/getFollow.lua @@ -39,6 +39,13 @@ end local FOLLOW = "app.certified.graph.follow" +-- Guard the cast inside CASE; a WHERE filter cannot protect it from planner reordering. +local function follow_sort_key() + local created = "record::jsonb->>'createdAt'" + local timestamp_pattern = "^[0-9]{4}-[0-9]{2}-[0-9]{2}T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]([.][0-9]+)?(Z|[+-]([01][0-9]|2[0-3]):[0-5][0-9])$" + return "CASE WHEN jsonb_typeof(record::jsonb->'createdAt') = 'string' AND " .. created .. " ~ '" .. timestamp_pattern .. "' AND " .. created .. " !~ '-00:00$' AND pg_input_is_valid(" .. created .. ", 'timestamptz') THEN (" .. created .. ")::timestamptz ELSE COALESCE(indexed_at::timestamptz, created_at::timestamptz) END" +end + local function query(sql, values) if db.backend() ~= "postgres" then error("ActorFollowQueryFailed: actor-follow queries require PostgreSQL", 0) end local ok, result = pcall(db.raw, sql, values) @@ -48,7 +55,7 @@ end local function query_follow(actor, subject) local rows = query( - "SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record FROM happyview_records WHERE collection = $1 AND did = $2 AND record::jsonb->>'subject' = $3 ORDER BY (record::jsonb->>'createdAt')::timestamptz ASC, uri ASC LIMIT 1", + "SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record FROM happyview_records CROSS JOIN LATERAL (SELECT " .. follow_sort_key() .. " AS sort_at) sorted WHERE collection = $1 AND did = $2 AND record::jsonb->>'subject' = $3 ORDER BY sorted.sort_at ASC, uri ASC LIMIT 1", { FOLLOW, actor, subject }) if #rows == 0 then return NULL end return record_view(rows[1]) diff --git a/hypercerts-api/lua/endpoints/listActorFollowers.lua b/hypercerts-api/lua/endpoints/listActorFollowers.lua index f4df1d6b..93c746d2 100644 --- a/hypercerts-api/lua/endpoints/listActorFollowers.lua +++ b/hypercerts-api/lua/endpoints/listActorFollowers.lua @@ -122,6 +122,13 @@ end local FOLLOW = "app.certified.graph.follow" +-- Guard the cast inside CASE; a WHERE filter cannot protect it from planner reordering. +local function follow_sort_key() + local created = "record::jsonb->>'createdAt'" + local timestamp_pattern = "^[0-9]{4}-[0-9]{2}-[0-9]{2}T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]([.][0-9]+)?(Z|[+-]([01][0-9]|2[0-3]):[0-5][0-9])$" + return "CASE WHEN jsonb_typeof(record::jsonb->'createdAt') = 'string' AND " .. created .. " ~ '" .. timestamp_pattern .. "' AND " .. created .. " !~ '-00:00$' AND pg_input_is_valid(" .. created .. ", 'timestamptz') THEN (" .. created .. ")::timestamptz ELSE COALESCE(indexed_at::timestamptz, created_at::timestamptz) END" +end + local function query(sql, values) if db.backend() ~= "postgres" then error("ActorFollowQueryFailed: actor-follow queries require PostgreSQL", 0) end local ok, result = pcall(db.raw, sql, values) @@ -172,7 +179,7 @@ local function query_actor_follows(mode, actor, limit, cursor, direction) values[#values + 1] = limit + 1 local ordering = direction == "asc" and "ASC" or "DESC" local displayed_did = mode == "followers" and "did" or "subject_did" - local sql = "WITH ranked_follows AS (SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record, record::jsonb->>'subject' AS subject_did, (record::jsonb->>'createdAt')::timestamptz AS sort_at, ROW_NUMBER() OVER (PARTITION BY did, record::jsonb->>'subject' ORDER BY (record::jsonb->>'createdAt')::timestamptz ASC, uri ASC) AS relationship_rank FROM happyview_records WHERE " .. table.concat(filters, " AND ") .. "), representatives AS (SELECT uri, did, cid, indexed_at, record, sort_at, " .. displayed_did .. " AS actor_did FROM ranked_follows WHERE relationship_rank = 1), " .. + local sql = "WITH ranked_follows AS (SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record, record::jsonb->>'subject' AS subject_did, sorted.sort_at, ROW_NUMBER() OVER (PARTITION BY did, record::jsonb->>'subject' ORDER BY sorted.sort_at ASC, uri ASC) AS relationship_rank FROM happyview_records CROSS JOIN LATERAL (SELECT " .. follow_sort_key() .. " AS sort_at) sorted WHERE " .. table.concat(filters, " AND ") .. "), representatives AS (SELECT uri, did, cid, indexed_at, record, sort_at, " .. displayed_did .. " AS actor_did FROM ranked_follows WHERE relationship_rank = 1), " .. "total AS (SELECT COUNT(*) AS total_count FROM representatives), " .. "page AS (SELECT uri, did, cid, indexed_at, record, sort_at, actor_did FROM representatives" .. cursor_filter .. " ORDER BY sort_at " .. ordering .. ", uri " .. ordering .. " LIMIT $" .. #values .. ") " .. "SELECT page.uri, page.did, page.cid, page.indexed_at, page.record, page.actor_did, total.total_count, to_char(page.sort_at AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') AS sort_timestamp FROM total LEFT JOIN page ON TRUE ORDER BY page.sort_at " .. ordering .. ", page.uri " .. ordering diff --git a/hypercerts-api/lua/endpoints/listActorFollowing.lua b/hypercerts-api/lua/endpoints/listActorFollowing.lua index 4300f2a0..cc3fc84d 100644 --- a/hypercerts-api/lua/endpoints/listActorFollowing.lua +++ b/hypercerts-api/lua/endpoints/listActorFollowing.lua @@ -122,6 +122,13 @@ end local FOLLOW = "app.certified.graph.follow" +-- Guard the cast inside CASE; a WHERE filter cannot protect it from planner reordering. +local function follow_sort_key() + local created = "record::jsonb->>'createdAt'" + local timestamp_pattern = "^[0-9]{4}-[0-9]{2}-[0-9]{2}T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]([.][0-9]+)?(Z|[+-]([01][0-9]|2[0-3]):[0-5][0-9])$" + return "CASE WHEN jsonb_typeof(record::jsonb->'createdAt') = 'string' AND " .. created .. " ~ '" .. timestamp_pattern .. "' AND " .. created .. " !~ '-00:00$' AND pg_input_is_valid(" .. created .. ", 'timestamptz') THEN (" .. created .. ")::timestamptz ELSE COALESCE(indexed_at::timestamptz, created_at::timestamptz) END" +end + local function query(sql, values) if db.backend() ~= "postgres" then error("ActorFollowQueryFailed: actor-follow queries require PostgreSQL", 0) end local ok, result = pcall(db.raw, sql, values) @@ -172,7 +179,7 @@ local function query_actor_follows(mode, actor, limit, cursor, direction) values[#values + 1] = limit + 1 local ordering = direction == "asc" and "ASC" or "DESC" local displayed_did = mode == "followers" and "did" or "subject_did" - local sql = "WITH ranked_follows AS (SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record, record::jsonb->>'subject' AS subject_did, (record::jsonb->>'createdAt')::timestamptz AS sort_at, ROW_NUMBER() OVER (PARTITION BY did, record::jsonb->>'subject' ORDER BY (record::jsonb->>'createdAt')::timestamptz ASC, uri ASC) AS relationship_rank FROM happyview_records WHERE " .. table.concat(filters, " AND ") .. "), representatives AS (SELECT uri, did, cid, indexed_at, record, sort_at, " .. displayed_did .. " AS actor_did FROM ranked_follows WHERE relationship_rank = 1), " .. + local sql = "WITH ranked_follows AS (SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record, record::jsonb->>'subject' AS subject_did, sorted.sort_at, ROW_NUMBER() OVER (PARTITION BY did, record::jsonb->>'subject' ORDER BY sorted.sort_at ASC, uri ASC) AS relationship_rank FROM happyview_records CROSS JOIN LATERAL (SELECT " .. follow_sort_key() .. " AS sort_at) sorted WHERE " .. table.concat(filters, " AND ") .. "), representatives AS (SELECT uri, did, cid, indexed_at, record, sort_at, " .. displayed_did .. " AS actor_did FROM ranked_follows WHERE relationship_rank = 1), " .. "total AS (SELECT COUNT(*) AS total_count FROM representatives), " .. "page AS (SELECT uri, did, cid, indexed_at, record, sort_at, actor_did FROM representatives" .. cursor_filter .. " ORDER BY sort_at " .. ordering .. ", uri " .. ordering .. " LIMIT $" .. #values .. ") " .. "SELECT page.uri, page.did, page.cid, page.indexed_at, page.record, page.actor_did, total.total_count, to_char(page.sort_at AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') AS sort_timestamp FROM total LEFT JOIN page ON TRUE ORDER BY page.sort_at " .. ordering .. ", page.uri " .. ordering diff --git a/hypercerts-api/lua/shared/actorFollow.lua b/hypercerts-api/lua/shared/actorFollow.lua index ad552b53..2119779e 100644 --- a/hypercerts-api/lua/shared/actorFollow.lua +++ b/hypercerts-api/lua/shared/actorFollow.lua @@ -1,5 +1,12 @@ local FOLLOW = "app.certified.graph.follow" +-- Guard the cast inside CASE; a WHERE filter cannot protect it from planner reordering. +local function follow_sort_key() + local created = "record::jsonb->>'createdAt'" + local timestamp_pattern = "^[0-9]{4}-[0-9]{2}-[0-9]{2}T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]([.][0-9]+)?(Z|[+-]([01][0-9]|2[0-3]):[0-5][0-9])$" + return "CASE WHEN jsonb_typeof(record::jsonb->'createdAt') = 'string' AND " .. created .. " ~ '" .. timestamp_pattern .. "' AND " .. created .. " !~ '-00:00$' AND pg_input_is_valid(" .. created .. ", 'timestamptz') THEN (" .. created .. ")::timestamptz ELSE COALESCE(indexed_at::timestamptz, created_at::timestamptz) END" +end + local function query(sql, values) if db.backend() ~= "postgres" then error("ActorFollowQueryFailed: actor-follow queries require PostgreSQL", 0) end local ok, result = pcall(db.raw, sql, values) diff --git a/hypercerts-api/lua/shared/actorFollowList.lua b/hypercerts-api/lua/shared/actorFollowList.lua index 141eb352..1e1c716c 100644 --- a/hypercerts-api/lua/shared/actorFollowList.lua +++ b/hypercerts-api/lua/shared/actorFollowList.lua @@ -41,7 +41,7 @@ local function query_actor_follows(mode, actor, limit, cursor, direction) values[#values + 1] = limit + 1 local ordering = direction == "asc" and "ASC" or "DESC" local displayed_did = mode == "followers" and "did" or "subject_did" - local sql = "WITH ranked_follows AS (SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record, record::jsonb->>'subject' AS subject_did, (record::jsonb->>'createdAt')::timestamptz AS sort_at, ROW_NUMBER() OVER (PARTITION BY did, record::jsonb->>'subject' ORDER BY (record::jsonb->>'createdAt')::timestamptz ASC, uri ASC) AS relationship_rank FROM happyview_records WHERE " .. table.concat(filters, " AND ") .. "), representatives AS (SELECT uri, did, cid, indexed_at, record, sort_at, " .. displayed_did .. " AS actor_did FROM ranked_follows WHERE relationship_rank = 1), " .. + local sql = "WITH ranked_follows AS (SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record, record::jsonb->>'subject' AS subject_did, sorted.sort_at, ROW_NUMBER() OVER (PARTITION BY did, record::jsonb->>'subject' ORDER BY sorted.sort_at ASC, uri ASC) AS relationship_rank FROM happyview_records CROSS JOIN LATERAL (SELECT " .. follow_sort_key() .. " AS sort_at) sorted WHERE " .. table.concat(filters, " AND ") .. "), representatives AS (SELECT uri, did, cid, indexed_at, record, sort_at, " .. displayed_did .. " AS actor_did FROM ranked_follows WHERE relationship_rank = 1), " .. "total AS (SELECT COUNT(*) AS total_count FROM representatives), " .. "page AS (SELECT uri, did, cid, indexed_at, record, sort_at, actor_did FROM representatives" .. cursor_filter .. " ORDER BY sort_at " .. ordering .. ", uri " .. ordering .. " LIMIT $" .. #values .. ") " .. "SELECT page.uri, page.did, page.cid, page.indexed_at, page.record, page.actor_did, total.total_count, to_char(page.sort_at AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') AS sort_timestamp FROM total LEFT JOIN page ON TRUE ORDER BY page.sort_at " .. ordering .. ", page.uri " .. ordering diff --git a/hypercerts-api/lua/shared/actorFollowLookup.lua b/hypercerts-api/lua/shared/actorFollowLookup.lua index 729d902b..24c2bd90 100644 --- a/hypercerts-api/lua/shared/actorFollowLookup.lua +++ b/hypercerts-api/lua/shared/actorFollowLookup.lua @@ -1,6 +1,6 @@ local function query_follow(actor, subject) local rows = query( - "SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record FROM happyview_records WHERE collection = $1 AND did = $2 AND record::jsonb->>'subject' = $3 ORDER BY (record::jsonb->>'createdAt')::timestamptz ASC, uri ASC LIMIT 1", + "SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record FROM happyview_records CROSS JOIN LATERAL (SELECT " .. follow_sort_key() .. " AS sort_at) sorted WHERE collection = $1 AND did = $2 AND record::jsonb->>'subject' = $3 ORDER BY sorted.sort_at ASC, uri ASC LIMIT 1", { FOLLOW, actor, subject }) if #rows == 0 then return NULL end return record_view(rows[1]) diff --git a/hypercerts-api/manifest.json b/hypercerts-api/manifest.json index 30c5981a..d3e36ad7 100644 --- a/hypercerts-api/manifest.json +++ b/hypercerts-api/manifest.json @@ -19,6 +19,11 @@ "luaBuild": { "command": "pnpm build:lua", "sharedSources": [ + "lua/shared/query.lua", + "lua/shared/recordIdentifier.lua", + "lua/shared/listQuery.lua", + "lua/shared/recordView.lua", + "lua/shared/actorView.lua", "lua/shared/location.lua", "lua/shared/actorFollow.lua", "lua/shared/actorFollowLookup.lua", diff --git a/hypercerts-api/tooling/actor-follow.test.js b/hypercerts-api/tooling/actor-follow.test.js index a38a24c7..d4c14bb2 100644 --- a/hypercerts-api/tooling/actor-follow.test.js +++ b/hypercerts-api/tooling/actor-follow.test.js @@ -226,7 +226,7 @@ assert(result.follow.did == '${actor}' and result.follow.record.createdAt == '20 assert(result.follow.record.via.uri == 'at://did:plc:list/app.certified.list/one') assert(#calls == 1) assert(calls[1].values[1] == '${followCollection}' and calls[1].values[2] == '${actor}' and calls[1].values[3] == '${subject}') -assert(calls[1].sql:find("ORDER BY (record::jsonb->>'createdAt')::timestamptz ASC, uri ASC LIMIT 1", 1, true)) +assert(calls[1].sql:find('ORDER BY sorted.sort_at ASC, uri ASC LIMIT 1', 1, true)) assert(result.follow.via == nil, 'via stays inside the unchanged raw record') `, }); @@ -235,6 +235,21 @@ assert(result.follow.via == nil, 'via stays inside the unchanged raw record') runLua({ endpoint: 'getFollow', params: { actor, subject }, rows: [], assertions: 'assert(result.follow == NULL_VALUE)\nassert(#calls == 1)' }); }); +test('actor-follow lookup and listings guard malformed createdAt and fall back to indexed_at for sorting', () => { + for (const endpoint of ['getFollow', 'listActorFollowers', 'listActorFollowing']) { + runLua({ + endpoint, + params: endpoint === 'getFollow' ? { actor, subject } : { actor }, + assertions: ` +assert(calls[1].sql:find("jsonb_typeof(record::jsonb->'createdAt') = 'string'", 1, true)) +assert(calls[1].sql:find('pg_input_is_valid(', 1, true)) +assert(calls[1].sql:find('ELSE COALESCE(indexed_at::timestamptz, created_at::timestamptz) END', 1, true)) +assert(calls[1].sql:find('ORDER BY sorted.sort_at ASC, uri ASC', 1, true)) +`, + }); + } +}); + test('listActorFollowers deduplicates before cursor pagination, sorts newest first, and hydrates only the returned page', () => { const rows = [ followRow('follower-a', follower, follower, '2025-01-05T00:00:00Z', { via: true }), @@ -264,7 +279,7 @@ assert(sql:find("record::jsonb->>'subject' = $2", 1, true)) assert(not sql:find('did = $2', 1, true), 'followers are selected by subject, not publisher') assert(sql:find('ROW_NUMBER() OVER', 1, true)) assert(sql:find("PARTITION BY did, record::jsonb->>'subject'", 1, true)) -assert(sql:find("ORDER BY (record::jsonb->>'createdAt')::timestamptz ASC, uri ASC", 1, true)) +assert(sql:find('ORDER BY sorted.sort_at ASC, uri ASC', 1, true)) local representativeFilter = assert(sql:find('relationship_rank = 1', 1, true)) local cursorFilter = assert(sql:find('(sort_at, uri) <', 1, true)) assert(representativeFilter < cursorFilter, 'collapse duplicate relationships before applying the page cursor') From ab77f58fd6bedae49dba2ddd39ecb0552bb66d94 Mon Sep 17 00:00:00 2001 From: kzoeps Date: Mon, 28 Sep 2026 21:46:53 +0600 Subject: [PATCH 8/8] hypercerts-api: split actor-follow list query into focused helpers Signed-off-by: kzoeps --- .../lua/endpoints/listActorFollowers.lua | 23 +++++++++++++------ .../lua/endpoints/listActorFollowing.lua | 23 +++++++++++++------ hypercerts-api/lua/shared/actorFollowList.lua | 23 +++++++++++++------ 3 files changed, 48 insertions(+), 21 deletions(-) diff --git a/hypercerts-api/lua/endpoints/listActorFollowers.lua b/hypercerts-api/lua/endpoints/listActorFollowers.lua index 93c746d2..e99d5d01 100644 --- a/hypercerts-api/lua/endpoints/listActorFollowers.lua +++ b/hypercerts-api/lua/endpoints/listActorFollowers.lua @@ -157,7 +157,7 @@ local function cursor_decode(token, direction) return value end -local function query_actor_follows(mode, actor, limit, cursor, direction) +local function build_actor_follow_query(mode, actor, limit, cursor, direction) local filters, values = { "collection = $1" }, { FOLLOW } values[#values + 1] = actor if mode == "followers" then @@ -183,19 +183,28 @@ local function query_actor_follows(mode, actor, limit, cursor, direction) "total AS (SELECT COUNT(*) AS total_count FROM representatives), " .. "page AS (SELECT uri, did, cid, indexed_at, record, sort_at, actor_did FROM representatives" .. cursor_filter .. " ORDER BY sort_at " .. ordering .. ", uri " .. ordering .. " LIMIT $" .. #values .. ") " .. "SELECT page.uri, page.did, page.cid, page.indexed_at, page.record, page.actor_did, total.total_count, to_char(page.sort_at AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') AS sort_timestamp FROM total LEFT JOIN page ON TRUE ORDER BY page.sort_at " .. ordering .. ", page.uri " .. ordering - local rows = query(sql, values) - local total_count = rows[1] and tonumber(rows[1].total_count) - if not total_count or total_count < 0 or total_count % 1 ~= 0 then error("ActorFollowQueryFailed: actor-follow count unavailable", 0) end - if rows[1].uri == nil then rows = {} end - local more = #rows > limit - if more then rows[#rows] = nil end + return sql, values +end +local function build_actor_follow_views(rows) local views = {} for _, row in ipairs(rows) do views[#views + 1] = { did = row.actor_did, follow = record_view(row) } end hydrate_actor_views(views, query) + return views +end + +local function query_actor_follows(mode, actor, limit, cursor, direction) + local sql, values = build_actor_follow_query(mode, actor, limit, cursor, direction) + local rows = query(sql, values) + local total_count = rows[1] and tonumber(rows[1].total_count) + if not total_count or total_count < 0 or total_count % 1 ~= 0 then error("ActorFollowQueryFailed: actor-follow count unavailable", 0) end + if rows[1].uri == nil then rows = {} end + local more = #rows > limit + if more then rows[#rows] = nil end + local views = build_actor_follow_views(rows) local next_cursor if more then local last = rows[#rows] diff --git a/hypercerts-api/lua/endpoints/listActorFollowing.lua b/hypercerts-api/lua/endpoints/listActorFollowing.lua index cc3fc84d..ea266a63 100644 --- a/hypercerts-api/lua/endpoints/listActorFollowing.lua +++ b/hypercerts-api/lua/endpoints/listActorFollowing.lua @@ -157,7 +157,7 @@ local function cursor_decode(token, direction) return value end -local function query_actor_follows(mode, actor, limit, cursor, direction) +local function build_actor_follow_query(mode, actor, limit, cursor, direction) local filters, values = { "collection = $1" }, { FOLLOW } values[#values + 1] = actor if mode == "followers" then @@ -183,19 +183,28 @@ local function query_actor_follows(mode, actor, limit, cursor, direction) "total AS (SELECT COUNT(*) AS total_count FROM representatives), " .. "page AS (SELECT uri, did, cid, indexed_at, record, sort_at, actor_did FROM representatives" .. cursor_filter .. " ORDER BY sort_at " .. ordering .. ", uri " .. ordering .. " LIMIT $" .. #values .. ") " .. "SELECT page.uri, page.did, page.cid, page.indexed_at, page.record, page.actor_did, total.total_count, to_char(page.sort_at AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') AS sort_timestamp FROM total LEFT JOIN page ON TRUE ORDER BY page.sort_at " .. ordering .. ", page.uri " .. ordering - local rows = query(sql, values) - local total_count = rows[1] and tonumber(rows[1].total_count) - if not total_count or total_count < 0 or total_count % 1 ~= 0 then error("ActorFollowQueryFailed: actor-follow count unavailable", 0) end - if rows[1].uri == nil then rows = {} end - local more = #rows > limit - if more then rows[#rows] = nil end + return sql, values +end +local function build_actor_follow_views(rows) local views = {} for _, row in ipairs(rows) do views[#views + 1] = { did = row.actor_did, follow = record_view(row) } end hydrate_actor_views(views, query) + return views +end + +local function query_actor_follows(mode, actor, limit, cursor, direction) + local sql, values = build_actor_follow_query(mode, actor, limit, cursor, direction) + local rows = query(sql, values) + local total_count = rows[1] and tonumber(rows[1].total_count) + if not total_count or total_count < 0 or total_count % 1 ~= 0 then error("ActorFollowQueryFailed: actor-follow count unavailable", 0) end + if rows[1].uri == nil then rows = {} end + local more = #rows > limit + if more then rows[#rows] = nil end + local views = build_actor_follow_views(rows) local next_cursor if more then local last = rows[#rows] diff --git a/hypercerts-api/lua/shared/actorFollowList.lua b/hypercerts-api/lua/shared/actorFollowList.lua index 1e1c716c..ff8106a0 100644 --- a/hypercerts-api/lua/shared/actorFollowList.lua +++ b/hypercerts-api/lua/shared/actorFollowList.lua @@ -19,7 +19,7 @@ local function cursor_decode(token, direction) return value end -local function query_actor_follows(mode, actor, limit, cursor, direction) +local function build_actor_follow_query(mode, actor, limit, cursor, direction) local filters, values = { "collection = $1" }, { FOLLOW } values[#values + 1] = actor if mode == "followers" then @@ -45,19 +45,28 @@ local function query_actor_follows(mode, actor, limit, cursor, direction) "total AS (SELECT COUNT(*) AS total_count FROM representatives), " .. "page AS (SELECT uri, did, cid, indexed_at, record, sort_at, actor_did FROM representatives" .. cursor_filter .. " ORDER BY sort_at " .. ordering .. ", uri " .. ordering .. " LIMIT $" .. #values .. ") " .. "SELECT page.uri, page.did, page.cid, page.indexed_at, page.record, page.actor_did, total.total_count, to_char(page.sort_at AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') AS sort_timestamp FROM total LEFT JOIN page ON TRUE ORDER BY page.sort_at " .. ordering .. ", page.uri " .. ordering - local rows = query(sql, values) - local total_count = rows[1] and tonumber(rows[1].total_count) - if not total_count or total_count < 0 or total_count % 1 ~= 0 then error("ActorFollowQueryFailed: actor-follow count unavailable", 0) end - if rows[1].uri == nil then rows = {} end - local more = #rows > limit - if more then rows[#rows] = nil end + return sql, values +end +local function build_actor_follow_views(rows) local views = {} for _, row in ipairs(rows) do views[#views + 1] = { did = row.actor_did, follow = record_view(row) } end hydrate_actor_views(views, query) + return views +end + +local function query_actor_follows(mode, actor, limit, cursor, direction) + local sql, values = build_actor_follow_query(mode, actor, limit, cursor, direction) + local rows = query(sql, values) + local total_count = rows[1] and tonumber(rows[1].total_count) + if not total_count or total_count < 0 or total_count % 1 ~= 0 then error("ActorFollowQueryFailed: actor-follow count unavailable", 0) end + if rows[1].uri == nil then rows = {} end + local more = #rows > limit + if more then rows[#rows] = nil end + local views = build_actor_follow_views(rows) local next_cursor if more then local last = rows[#rows]