Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions .changeset/api-cursor-pagination.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
---
'lens': minor
---

Add cursor-and-limit pagination to every list route — `GET /pools`,
`GET /pairs`, `GET /candles/:assetA/:assetB` and `GET /prices/history` — matching
the convention `GET /screener` already used. Each accepts `?cursor=` and
`?limit=` (absent limit → the route default, out-of-range values clamped, a
non-numeric limit is a 400) and now returns `nextCursor` + `hasMore` beside its
own key (`pools`, `pairs`, `candles`, `points`), so an unpaged caller still gets
a sensible first page. `GET /prices/history?format=csv` carries the same
envelope in the `X-Next-Cursor` / `X-Has-More` response headers, keeping the CSV
body the byte-stable `ts,price,volume` contract.

Pagination is keyset-based, not offset-based: the cursor is an opaque base64
`[sortValue, id]` and the predicate is `(sort, id) > (value, id)` (or `<` for
descending), so a row inserted while a client is paging can never shift a page
boundary and cause a duplicate or a skipped row. The shared helpers live in
`src/pagination.ts`; the per-route default/maximum page sizes are 50/200
(`/pools`, `/pairs`), 500/5000 (`/candles`), 1000/10000 (`/prices/history`) and
20/100 (`/screener`), documented in `openapi.yaml` and exercised by a
mid-page-insert regression test.
10 changes: 10 additions & 0 deletions clients/typescript/src/schema.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -948,6 +948,16 @@ export interface paths {
export type webhooks = Record<string, never>;
export interface components {
schemas: {
/**
* @description Stable machine-readable error code. Branch on this, never on `error`, which is human-readable.
* @enum {string}
*/
ErrorCode: "missing_parameter" | "invalid_parameter" | "bad_network" | "bad_interval" | "bad_window" | "unsupported" | "unknown_pair" | "unwatched_pair" | "pair_already_watched" | "unauthorized" | "not_found" | "upstream_unavailable" | "internal_error" | "rate_limited" | "payment_required" | "payment_invalid" | "payment_verification_failed" | "quota_exceeded" | "admin_not_configured" | "admin_unauthorized" | "missing" | "malformed" | "unknown" | "revoked" | "expired" | "insufficient_scope" | "discovery_database_unavailable" | "VALIDATION_ERROR" | "UNSUPPORTED_PARAMETER" | "INVALID_NETWORK" | "STALE_DATA" | "INVALID_PRICE" | "INTERNAL_ERROR" | "DB_POOL_EXHAUSTED";
Error: {
/** @description Human-readable error message. Do not branch on it. */
error: string;
code: components["schemas"]["ErrorCode"];
};
LendingRateEntry: {
id: string;
/** @example blend */
Expand Down
56 changes: 56 additions & 0 deletions openapi.json
Original file line number Diff line number Diff line change
Expand Up @@ -4287,6 +4287,62 @@
}
},
"schemas": {
"ErrorCode": {
"type": "string",
"description": "Stable machine-readable error code. Branch on this, never on `error`, which is human-readable.",
"enum": [
"missing_parameter",
"invalid_parameter",
"bad_network",
"bad_interval",
"bad_window",
"unsupported",
"unknown_pair",
"unwatched_pair",
"pair_already_watched",
"unauthorized",
"not_found",
"upstream_unavailable",
"internal_error",
"rate_limited",
"payment_required",
"payment_invalid",
"payment_verification_failed",
"quota_exceeded",
"admin_not_configured",
"admin_unauthorized",
"missing",
"malformed",
"unknown",
"revoked",
"expired",
"insufficient_scope",
"discovery_database_unavailable",
"VALIDATION_ERROR",
"UNSUPPORTED_PARAMETER",
"INVALID_NETWORK",
"STALE_DATA",
"INVALID_PRICE",
"INTERNAL_ERROR",
"DB_POOL_EXHAUSTED"
]
},
"Error": {
"type": "object",
"required": [
"error",
"code"
],
"properties": {
"error": {
"type": "string",
"description": "Human-readable error message. Do not branch on it."
},
"code": {
"$ref": "#/components/schemas/ErrorCode"
}
}
},
"LendingRateEntry": {
"type": "object",
"required": [
Expand Down
47 changes: 47 additions & 0 deletions openapi.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3009,6 +3009,53 @@ components:
example: 'public, max-age=300'

schemas:
ErrorCode:
type: string
description: Stable machine-readable error code. Branch on this, never on `error`, which is human-readable.
enum:
- missing_parameter
- invalid_parameter
- bad_network
- bad_interval
- bad_window
- unsupported
- unknown_pair
- unwatched_pair
- pair_already_watched
- unauthorized
- not_found
- upstream_unavailable
- internal_error
- rate_limited
- payment_required
- payment_invalid
- payment_verification_failed
- quota_exceeded
- admin_not_configured
- admin_unauthorized
- missing
- malformed
- unknown
- revoked
- expired
- insufficient_scope
- discovery_database_unavailable
- VALIDATION_ERROR
- UNSUPPORTED_PARAMETER
- INVALID_NETWORK
- STALE_DATA
- INVALID_PRICE
- INTERNAL_ERROR
- DB_POOL_EXHAUSTED
Error:
type: object
required: [error, code]
properties:
error:
type: string
description: Human-readable error message. Do not branch on it.
code:
$ref: '#/components/schemas/ErrorCode'
LendingRateEntry:
type: object
required: [id, venue, source, network, poolId, asset, supplyApy, borrowApy, timestamp]
Expand Down
2 changes: 1 addition & 1 deletion src/__tests__/benchmark.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ describe('GET /benchmark/:asset', () => {
})

expect(res.statusCode).toBe(404)
expect(res.json()).toEqual({ error: 'Pair USDC/USD not watched' })
expect(res.json()).toEqual({ error: 'Pair USDC/USD not watched', code: 'unwatched_pair' })
})

it('calculates benchmark deviation correctly when asset is assetB in the pair (e.g. USD/USDC)', async () => {
Expand Down
67 changes: 67 additions & 0 deletions src/__tests__/errorCodes.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
import { describe, expect, it } from 'vitest'
import { readFileSync } from 'node:fs'
import { fileURLToPath } from 'node:url'
import { readdirSync } from 'node:fs'
import YAML from 'js-yaml'
import { ERROR_CODES, ERROR_CODE_VALUES } from '../errors'

function readSpec(): any {
const specPath = fileURLToPath(new URL('../../openapi.yaml', import.meta.url))
const spec = YAML.load(readFileSync(specPath, 'utf8'))
return spec
}

function collectCodesFromFile(filePath: string): string[] {
const content = readFileSync(filePath, 'utf8')
const codes: string[] = []
const re1 = /apiError\(\s*['"]([a-z0-9_]+)['"]/g
let m: RegExpExecArray | null
while ((m = re1.exec(content))) {
codes.push(m[1])
}
const re2 = /\bcode:\s*['"]([a-z][a-z0-9_]*)['"]/g
while ((m = re2.exec(content))) {
codes.push(m[1])
}
return codes
}

function getModuleFiles(): string[] {
const modules: string[] = []
const routesDir = fileURLToPath(new URL('../routes', import.meta.url))
for (const f of readdirSync(routesDir)) {
if (f.endsWith('.ts')) modules.push(fileURLToPath(new URL(`../routes/${f}`, import.meta.url)))
}
const apiFiles = ['rest.ts', 'history.ts', 'admin.ts', 'auth.ts', 'usage.ts'].map(f =>
fileURLToPath(new URL(`../api/${f}`, import.meta.url)),
)
modules.push(...apiFiles)
modules.push(fileURLToPath(new URL('../middleware/network.ts', import.meta.url)))
modules.push(fileURLToPath(new URL('../middleware/x402.ts', import.meta.url)))
modules.push(fileURLToPath(new URL('../app.ts', import.meta.url)))
return modules
}

describe('error codes', () => {
it('documents the error code enum in openapi and keeps it in sync with ERROR_CODES', () => {
const spec = readSpec()
const enumVals = spec.components?.schemas?.ErrorCode?.enum as string[] | undefined
expect(enumVals).toBeDefined()
const expected = Object.keys(ERROR_CODES).sort()
const actual = [...(enumVals || [])].sort()
expect(actual).toEqual(expected)
})

it('no route/error-emitting module emits a code outside the enumerated set', () => {
const allowed = new Set(Object.keys(ERROR_CODES))
const modules = getModuleFiles()
const all = modules.flatMap(collectCodesFromFile)
const bad = all.filter(c => !allowed.has(c))
expect(bad, `Codes outside set: ${JSON.stringify(bad)}`).toEqual([])
})

it('error code set is non-empty and matches values', () => {
expect(ERROR_CODE_VALUES.length).toBeGreaterThan(0)
expect(Object.keys(ERROR_CODES).length).toBe(ERROR_CODE_VALUES.length)
})
})
2 changes: 1 addition & 1 deletion src/__tests__/movers.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -279,7 +279,7 @@ describe('GET /movers', () => {
const res = await app.inject({ method: 'GET', url: '/movers' })

expect(res.statusCode).toBe(500)
expect(res.json()).toEqual({ error: 'Internal server error' })
expect(res.json()).toEqual({ error: 'Internal server error', code: 'internal_error' })
expect(res.body).not.toMatch(/secret/)

await app.close()
Expand Down
11 changes: 6 additions & 5 deletions src/api/admin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { formatValidationError } from '../validation'
import { hashApiKey, isAdminAuthorized, checkAdminAuth, invalidateApiKey } from './auth'
import { copyUsageHistory } from '../x402/metering'

import { apiError } from '../errors'
/** Generates a new opaque API key with a recognizable `lens_` prefix. */
export function generateApiKey(): string {
return `lens_${randomBytes(24).toString('hex')}`
Expand Down Expand Up @@ -136,7 +137,7 @@ export async function registerAdminRoutes(app: FastifyInstance) {
}
const requestedLimit = Number(req.query.limit ?? 25)
if (!Number.isInteger(requestedLimit) || requestedLimit < 1) {
return reply.status(400).send({ error: 'limit must be a positive integer' })
return reply.status(400).send(apiError('invalid_parameter', 'limit must be a positive integer'))
}
const failures = await prisma.webhookDeliveryFailure.findMany({
orderBy: { createdAt: 'desc' },
Expand Down Expand Up @@ -224,7 +225,7 @@ export async function registerAdminRoutes(app: FastifyInstance) {
const revoked = req.query.revoked ?? 'false'
if (limit === null || limit < 1 || offset === null || !['true', 'false'].includes(revoked)) {
return reply.status(400).send({
error: 'Invalid pagination or revoked filter. Use limit (1-100), offset (0 or greater), and revoked=true|false.',
...apiError('invalid_parameter', 'Invalid pagination or revoked filter. Use limit (1-100), offset (0 or greater), and revoked=true|false.'),
})
}

Expand Down Expand Up @@ -299,7 +300,7 @@ export async function registerAdminRoutes(app: FastifyInstance) {
const { id } = req.params
const existing = await prisma.apiKey.findUnique({ where: { id } })
if (!existing) {
return reply.status(404).send({ error: 'API key not found' })
return reply.status(404).send(apiError('not_found', 'API key not found'))
}
if (existing.revokedAt) {
return reply.status(200).send({ id, revoked: true, alreadyRevoked: true })
Expand Down Expand Up @@ -334,10 +335,10 @@ export async function registerAdminRoutes(app: FastifyInstance) {
const { id } = req.params
const oldKey = await prisma.apiKey.findUnique({ where: { id } })
if (!oldKey) {
return reply.status(404).send({ error: 'API key not found' })
return reply.status(404).send(apiError('not_found', 'API key not found'))
}
if (oldKey.revokedAt) {
return reply.status(400).send({ error: 'Cannot rotate a revoked API key' })
return reply.status(400).send(apiError('invalid_parameter', 'Cannot rotate a revoked API key'))
}

const defaultOverlap = parseInt(process.env.KEY_ROTATION_OVERLAP_HOURS ?? '24', 10) || 24
Expand Down
3 changes: 2 additions & 1 deletion src/api/auth.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { prisma } from '../db'
import { config } from '../config'
import { api_key_auth_failures_total, admin_auth_failures_total, api_key_cache_lookups_total } from '../metrics'

import { apiError } from '../errors'
const log = createLogger('auth')

type ApiKeyAuthFailureReason = 'missing' | 'malformed' | 'unknown' | 'revoked' | 'expired' | 'insufficient_scope'
Expand Down Expand Up @@ -518,7 +519,7 @@ async function apiKeyAuthPlugin(app: FastifyInstance) {
reply.header('Retry-After', retryAfterSec)
return reply.status(429).send({
statusCode: 429,
error: 'Too Many Requests',
...apiError('rate_limited', 'Too Many Requests'),
message: `Daily rate limit of ${keyContext.ratePerDay} requests exceeded for this API key. Quota resets at UTC midnight.`,
retryAfter: retryAfterSec,
})
Expand Down
21 changes: 11 additions & 10 deletions src/api/history.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import '../middleware/network' // declares req.network on the FastifyRequest typ
import { findOverlappingGaps } from '../backfill/gaps'
import { formatPrice, formatVolume } from '../pricing/precision'

import { apiError } from '../errors'
/** Supported aggregation intervals → bucket width in seconds. */
export const HISTORY_INTERVAL_SECONDS: Record<string, number> = {
'1m': 60,
Expand Down Expand Up @@ -158,39 +159,39 @@ export async function registerHistoryRoutes(app: FastifyInstance) {
}>('/prices/history', { config: { scope: 'read:history' } }, async (req, reply) => {
const { pair } = req.query
if (!pair) {
return reply.status(400).send({ error: 'pair query parameter is required' })
return reply.status(400).send(apiError('missing_parameter', 'pair query parameter is required'))
}

// Validated before the DB round-trip so an unknown format never queries.
const format = req.query.format ?? 'json'
if (format !== 'json' && format !== 'csv') {
return reply.status(400).send({ error: 'format must be one of: json, csv' })
return reply.status(400).send(apiError('invalid_parameter', 'format must be one of: json, csv'))
}

const interval = req.query.interval ?? '1m'
const intervalSecs = HISTORY_INTERVAL_SECONDS[interval]
if (!intervalSecs) {
return reply.status(400).send({
error: `interval must be one of: ${Object.keys(HISTORY_INTERVAL_SECONDS).join(', ')}`,
...apiError('bad_interval', `interval must be one of: ${Object.keys(HISTORY_INTERVAL_SECONDS).join(', ')}`),
})
}

const parsedLimit = parseLimit(req.query.limit, HISTORY_DEFAULT_LIMIT, MAX_HISTORY_POINTS)
if ('error' in parsedLimit) {
return reply.status(400).send({ error: parsedLimit.error })
return reply.status(400).send(apiError('invalid_parameter', parsedLimit.error))
}
const { limit } = parsedLimit

let afterTs: Date | undefined
if (req.query.cursor) {
const decoded = decodeCursor(req.query.cursor)
if (decoded === null) {
return reply.status(400).send({ error: 'Invalid cursor format' })
return reply.status(400).send(apiError('invalid_parameter', 'Invalid cursor format'))
}
if (decoded !== 'skip') {
afterTs = new Date(String(decoded.value))
if (isNaN(afterTs.getTime())) {
return reply.status(400).send({ error: 'Invalid cursor format' })
return reply.status(400).send(apiError('invalid_parameter', 'Invalid cursor format'))
}
}
}
Expand All @@ -199,20 +200,20 @@ export async function registerHistoryRoutes(app: FastifyInstance) {
const to = req.query.to ? new Date(req.query.to) : new Date()

if (isNaN(from.getTime())) {
return reply.status(400).send({ error: 'from must be a valid ISO 8601 date' })
return reply.status(400).send(apiError('invalid_parameter', 'from must be a valid ISO 8601 date'))
}
if (isNaN(to.getTime())) {
return reply.status(400).send({ error: 'to must be a valid ISO 8601 date' })
return reply.status(400).send(apiError('invalid_parameter', 'to must be a valid ISO 8601 date'))
}
if (from.getTime() > to.getTime()) {
return reply.status(400).send({ error: 'from must be before to' })
return reply.status(400).send(apiError('invalid_parameter', 'from must be before to'))
}

// Guard against unbounded scans: cap the number of buckets the window can produce.
const spanSecs = (to.getTime() - from.getTime()) / 1000
if (spanSecs / intervalSecs > MAX_HISTORY_POINTS) {
return reply.status(400).send({
error: `requested window is too large for interval=${interval} (max ${MAX_HISTORY_POINTS} points); narrow the range or use a coarser interval`,
...apiError('invalid_parameter', `requested window is too large for interval=${interval} (max ${MAX_HISTORY_POINTS} points); narrow the range or use a coarser interval`),
})
}

Expand Down
Loading
Loading