diff --git a/scripts/builders/packages.env b/scripts/builders/packages.env index d178e59ec3..7c3dc10b60 100644 --- a/scripts/builders/packages.env +++ b/scripts/builders/packages.env @@ -1,4 +1,4 @@ DOCKERFILE="./services/docker/Dockerfile.packages" CONTEXT="../" REPO="sjc.ocir.io/axbydjxa5zuh/packages" -SERVICES="github-repos-enricher bq-dataset-ingest npm-worker pypi-worker maven-worker osv-worker dockerhub-sync cargo-worker go-worker nuget-worker security-contacts-worker" +SERVICES="github-repos-enricher bq-dataset-ingest npm-worker pypi-worker maven-worker osv-worker dockerhub-sync cargo-worker go-worker nuget-worker security-contacts-worker rubygems-worker" diff --git a/scripts/services/rubygems-worker.yaml b/scripts/services/rubygems-worker.yaml new file mode 100644 index 0000000000..5f7b1ed878 --- /dev/null +++ b/scripts/services/rubygems-worker.yaml @@ -0,0 +1,65 @@ +version: '3.1' + +x-env-args: &env-args + DOCKER_BUILDKIT: 1 + NODE_ENV: docker + SERVICE: rubygems-worker + SHELL: /bin/sh + SUPPRESS_NO_CONFIG_WARNING: 'true' + CROWD_TEMPORAL_TASKQUEUE: rubygems-worker + +services: + rubygems-worker: + build: + context: ../../ + dockerfile: ./scripts/services/docker/Dockerfile.packages + command: 'pnpm run start:rubygems-worker' + working_dir: /usr/crowd/app/services/apps/packages_worker + env_file: + - ../../backend/.env.dist.local + - ../../backend/.env.dist.composed + - ../../backend/.env.override.local + - ../../backend/.env.override.composed + environment: + <<: *env-args + restart: always + networks: + - crowd-bridge + + rubygems-worker-dev: + build: + context: ../../ + dockerfile: ./scripts/services/docker/Dockerfile.packages + command: 'pnpm run dev:rubygems-worker' + working_dir: /usr/crowd/app/services/apps/packages_worker + env_file: + - ../../backend/.env.dist.local + - ../../backend/.env.dist.composed + - ../../backend/.env.override.local + - ../../backend/.env.override.composed + environment: + <<: *env-args + hostname: rubygems-worker + networks: + - crowd-bridge + volumes: + - ../../services/libs/audit-logs/src:/usr/crowd/app/services/libs/audit-logs/src + - ../../services/libs/common/src:/usr/crowd/app/services/libs/common/src + - ../../services/libs/common_services/src:/usr/crowd/app/services/libs/common_services/src + - ../../services/libs/data-access-layer/src:/usr/crowd/app/services/libs/data-access-layer/src + - ../../services/libs/database/src:/usr/crowd/app/services/libs/database/src + - ../../services/libs/integrations/src:/usr/crowd/app/services/libs/integrations/src + - ../../services/libs/logging/src:/usr/crowd/app/services/libs/logging/src + - ../../services/libs/nango/src:/usr/crowd/app/services/libs/nango/src + - ../../services/libs/opensearch/src:/usr/crowd/app/services/libs/opensearch/src + - ../../services/libs/queue/src:/usr/crowd/app/services/libs/queue/src + - ../../services/libs/redis/src:/usr/crowd/app/services/libs/redis/src + - ../../services/libs/snowflake/src:/usr/crowd/app/services/libs/snowflake/src + - ../../services/libs/telemetry/src:/usr/crowd/app/services/libs/telemetry/src + - ../../services/libs/temporal/src:/usr/crowd/app/services/libs/temporal/src + - ../../services/libs/types/src:/usr/crowd/app/services/libs/types/src + - ../../services/apps/packages_worker/src:/usr/crowd/app/services/apps/packages_worker/src + +networks: + crowd-bridge: + external: true diff --git a/services/apps/packages_worker/package.json b/services/apps/packages_worker/package.json index d500ab3301..6d6936478f 100644 --- a/services/apps/packages_worker/package.json +++ b/services/apps/packages_worker/package.json @@ -49,6 +49,9 @@ "start:nuget-worker": "CROWD_TEMPORAL_TASKQUEUE=nuget-worker SERVICE=nuget-worker tsx src/bin/nuget-worker.ts", "dev:nuget-worker": "CROWD_TEMPORAL_TASKQUEUE=nuget-worker SERVICE=nuget-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9242 src/bin/nuget-worker.ts", "dev:nuget-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=nuget-worker SERVICE=nuget-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9242 src/bin/nuget-worker.ts", + "start:rubygems-worker": "CROWD_TEMPORAL_TASKQUEUE=rubygems-worker SERVICE=rubygems-worker tsx src/bin/rubygems-worker.ts", + "dev:rubygems-worker": "CROWD_TEMPORAL_TASKQUEUE=rubygems-worker SERVICE=rubygems-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9244 src/bin/rubygems-worker.ts", + "dev:rubygems-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=rubygems-worker SERVICE=rubygems-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9244 src/bin/rubygems-worker.ts", "backfill:maven": "SERVICE=maven tsx src/bin/maven-backfill.ts", "backfill:maven:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=maven LOG_LEVEL=info tsx src/bin/maven-backfill.ts", "import:maven-maintainers": "SERVICE=maven tsx src/maven/scripts/importMaintainersFromCsv.ts", diff --git a/services/apps/packages_worker/src/activities.ts b/services/apps/packages_worker/src/activities.ts index 5b035b52cb..2985d91f79 100644 --- a/services/apps/packages_worker/src/activities.ts +++ b/services/apps/packages_worker/src/activities.ts @@ -32,6 +32,7 @@ export { } from './pypi/activities' export { getCriticalPypiCount } from './pypi/downloads/getCriticalPypiCount' export { processNuGetBatch } from './nuget/activities' +export { processRubyGemsCoreBatch, processRubyGemsCriticalBatch } from './rubygems/activities' export { processSecurityContactsBatch, ingestSecurityContactsForPurlActivity, diff --git a/services/apps/packages_worker/src/bin/rubygems-worker.ts b/services/apps/packages_worker/src/bin/rubygems-worker.ts new file mode 100644 index 0000000000..cd5415b53c --- /dev/null +++ b/services/apps/packages_worker/src/bin/rubygems-worker.ts @@ -0,0 +1,9 @@ +import { scheduleRubyGemsCriticalIngestion, scheduleRubyGemsIngestion } from '../rubygems/schedule' +import { svc } from '../service' + +setImmediate(async () => { + await svc.init() + await scheduleRubyGemsIngestion() + await scheduleRubyGemsCriticalIngestion() + await svc.start() +}) diff --git a/services/apps/packages_worker/src/config.ts b/services/apps/packages_worker/src/config.ts index cc5307f161..6b587381cf 100644 --- a/services/apps/packages_worker/src/config.ts +++ b/services/apps/packages_worker/src/config.ts @@ -86,6 +86,22 @@ export function getNuGetConfig() { } } +export function getRubyGemsConfig() { + return { + batchSize: parseInt(process.env.RUBYGEMS_FETCHER_BATCH_SIZE ?? '1000', 10), + concurrency: parseInt(process.env.RUBYGEMS_FETCHER_CONCURRENCY ?? '8', 10), + groupDelayMs: parseInt(process.env.RUBYGEMS_FETCHER_GROUP_DELAY_MS ?? '1000', 10), + } +} + +export function getRubyGemsCriticalConfig() { + return { + batchSize: parseInt(process.env.RUBYGEMS_CRITICAL_FETCHER_BATCH_SIZE ?? '5000', 10), + concurrency: parseInt(process.env.RUBYGEMS_CRITICAL_FETCHER_CONCURRENCY ?? '4', 10), + groupDelayMs: parseInt(process.env.RUBYGEMS_CRITICAL_FETCHER_GROUP_DELAY_MS ?? '1000', 10), + } +} + export function getDockerhubConfig() { return { hubBaseUrl: requireEnv('DOCKERHUB_API_BASE_URL'), diff --git a/services/apps/packages_worker/src/rubygems/activities.ts b/services/apps/packages_worker/src/rubygems/activities.ts new file mode 100644 index 0000000000..4dd4ba1f26 --- /dev/null +++ b/services/apps/packages_worker/src/rubygems/activities.ts @@ -0,0 +1,27 @@ +import { getServiceChildLogger } from '@crowd/logging' + +import { getRubyGemsConfig, getRubyGemsCriticalConfig } from '../config' +import { getPackagesDb } from '../db' + +import { processBatch as processCoreBatch } from './runRubyGemsCoreLoop' +import { processBatch as processCriticalBatch } from './runRubyGemsCriticalLoop' +import { BatchResult } from './types' + +const log = getServiceChildLogger('rubygems-activity') + +export async function processRubyGemsCoreBatch(): Promise { + const config = getRubyGemsConfig() + const qx = await getPackagesDb() + const today = new Date().toISOString().split('T')[0] + const result = await processCoreBatch(qx, config, today) + log.info({ ...result }, 'RubyGems core batch complete') + return result +} + +export async function processRubyGemsCriticalBatch(): Promise { + const config = getRubyGemsCriticalConfig() + const qx = await getPackagesDb() + const result = await processCriticalBatch(qx, config) + log.info({ ...result }, 'RubyGems critical batch complete') + return result +} diff --git a/services/apps/packages_worker/src/rubygems/client.ts b/services/apps/packages_worker/src/rubygems/client.ts new file mode 100644 index 0000000000..bb613a4267 --- /dev/null +++ b/services/apps/packages_worker/src/rubygems/client.ts @@ -0,0 +1,61 @@ +import axios from 'axios' + +import { + RubyGemsFetchError, + RubyGemsFetchResult, + RubyGemsGemResponse, + RubyGemsOwner, + RubyGemsVersionItem, +} from './types' + +function classifyError(err: unknown): RubyGemsFetchError | null { + if (!axios.isAxiosError(err)) return null + const status = err.response?.status + if (status === 404) return { kind: 'NOT_FOUND', status, message: err.message } + if (status === 429) return { kind: 'RATE_LIMIT', status, message: err.message } + return null +} + +export async function fetchGem(name: string): Promise> { + try { + const resp = await axios.get( + `https://rubygems.org/api/v1/gems/${encodeURIComponent(name)}.json`, + { timeout: 15000 }, + ) + return resp.data + } catch (err) { + const classified = classifyError(err) + if (classified) return classified + throw err + } +} + +export async function fetchVersions( + name: string, +): Promise> { + try { + const resp = await axios.get( + `https://rubygems.org/api/v1/versions/${encodeURIComponent(name)}.json`, + { timeout: 15000 }, + ) + return resp.data + } catch (err) { + const classified = classifyError(err) + if (classified) return classified + throw err + } +} + +export async function fetchOwners(name: string): Promise> { + try { + const resp = await axios.get( + `https://rubygems.org/api/v1/gems/${encodeURIComponent(name)}/owners.json`, + { timeout: 15000 }, + ) + return resp.data + } catch (err) { + const classified = classifyError(err) + if (classified) return classified + throw err + } +} diff --git a/services/apps/packages_worker/src/rubygems/normalize.ts b/services/apps/packages_worker/src/rubygems/normalize.ts new file mode 100644 index 0000000000..a7997cc91d --- /dev/null +++ b/services/apps/packages_worker/src/rubygems/normalize.ts @@ -0,0 +1,63 @@ +import { + NormalizedRubyGemsOwner, + NormalizedRubyGemsPackage, + NormalizedRubyGemsVersion, + RubyGemsGemResponse, + RubyGemsOwner, + RubyGemsVersionItem, +} from './types' + +function nonEmpty(value: string | null | undefined): string | null { + if (!value) return null + const trimmed = value.trim() + return trimmed === '' ? null : trimmed +} + +export function normalizeRubyGemsPackage(doc: RubyGemsGemResponse): NormalizedRubyGemsPackage { + const licenses = doc.licenses && doc.licenses.length > 0 ? doc.licenses : null + return { + description: nonEmpty(doc.info), + homepage: nonEmpty(doc.homepage_uri), + declaredRepositoryUrl: nonEmpty(doc.source_code_uri), + licenses, + licensesRaw: licenses ? licenses.join(', ') : null, + latestVersion: nonEmpty(doc.version), + totalDownloads: doc.downloads ?? 0, + } +} + +function parseCreatedAt(value: string | undefined): Date | null { + if (!value) return null + const date = new Date(value) + return isNaN(date.getTime()) ? null : date +} + +export function normalizeRubyGemsVersions( + items: RubyGemsVersionItem[], +): NormalizedRubyGemsVersion[] { + return items.map((item) => ({ + number: item.number, + publishedAt: parseCreatedAt(item.created_at), + isPrerelease: item.prerelease ?? false, + licenses: item.licenses && item.licenses.length > 0 ? item.licenses : null, + })) +} + +export function pickLatestRubyGemsVersion( + versions: NormalizedRubyGemsVersion[], +): NormalizedRubyGemsVersion | null { + if (versions.length === 0) return null + const sorted = [...versions].sort((a, b) => { + const at = a.publishedAt ? a.publishedAt.getTime() : 0 + const bt = b.publishedAt ? b.publishedAt.getTime() : 0 + return bt - at + }) + const stable = sorted.find((v) => !v.isPrerelease) + return stable ?? sorted[0] +} + +export function normalizeRubyGemsOwners(owners: RubyGemsOwner[]): NormalizedRubyGemsOwner[] { + return owners + .filter((o): o is RubyGemsOwner & { handle: string } => !!o.handle && o.handle.trim() !== '') + .map((o) => ({ username: o.handle, email: nonEmpty(o.email) })) +} diff --git a/services/apps/packages_worker/src/rubygems/runRubyGemsCoreLoop.ts b/services/apps/packages_worker/src/rubygems/runRubyGemsCoreLoop.ts new file mode 100644 index 0000000000..9176b8faa7 --- /dev/null +++ b/services/apps/packages_worker/src/rubygems/runRubyGemsCoreLoop.ts @@ -0,0 +1,182 @@ +import { + QueryExecutor, + RubyGemsPackageToSync, + listRubyGemsPackagesToSync, + logAuditFieldChange, + recordDownloadSnapshot, + upsertPackage, +} from '@crowd/data-access-layer' +import { getServiceChildLogger } from '@crowd/logging' + +import { fetchGem } from './client' +import { normalizeRubyGemsPackage } from './normalize' +import { BatchResult, isRubyGemsFetchError } from './types' + +const log = getServiceChildLogger('rubygems-core') + +async function withDeadlockRetry(fn: () => Promise, maxAttempts = 4): Promise { + for (let attempt = 1; ; attempt++) { + try { + return await fn() + } catch (err) { + const code = (err as { code?: string }).code + const isDeadlock = + code === '40P01' || /deadlock detected/i.test(String((err as Error)?.message)) + if (isDeadlock && attempt < maxAttempts) { + await new Promise((r) => setTimeout(r, 50 * attempt + Math.random() * 100)) + log.debug({ attempt }, 'Deadlock detected — retrying transaction') + continue + } + throw err + } + } +} + +function rubyGemsRegistryUrl(name: string): string { + return `https://rubygems.org/gems/${name}` +} + +async function markPackageError(qx: QueryExecutor, pkg: RubyGemsPackageToSync): Promise { + await upsertPackage(qx, { + purl: pkg.purl, + ecosystem: 'rubygems', + namespace: null, + name: pkg.name, + description: null, + homepage: null, + declaredRepositoryUrl: null, + licenses: null, + licensesRaw: null, + latestVersion: pkg.latestVersion, + ingestionSource: 'rubygems_error', + }) +} + +type PackageStatus = 'processed' | 'skipped' | 'error' | 'unchanged' + +export type RubyGemsCoreConfig = { batchSize: number; concurrency: number; groupDelayMs: number } + +async function processPackage( + qx: QueryExecutor, + pkg: RubyGemsPackageToSync, + today: string, +): Promise { + const gemResult = await fetchGem(pkg.name) + + if (isRubyGemsFetchError(gemResult)) { + if (gemResult.kind === 'NOT_FOUND') { + await upsertPackage(qx, { + purl: pkg.purl, + ecosystem: 'rubygems', + namespace: null, + name: pkg.name, + description: null, + homepage: null, + declaredRepositoryUrl: null, + licenses: null, + licensesRaw: null, + latestVersion: pkg.latestVersion, + registryUrl: rubyGemsRegistryUrl(pkg.name), + ingestionSource: 'rubygems_not_found', + }) + log.warn( + { purl: pkg.purl }, + 'Package not found on RubyGems registry — writing minimal record', + ) + return 'skipped' + } + if (gemResult.kind === 'RATE_LIMIT') { + log.warn({ purl: pkg.purl }, 'Rate limited by RubyGems registry — will retry next pass') + await markPackageError(qx, pkg) + return 'error' + } + throw new Error(`Transient error fetching ${pkg.purl}: ${gemResult.message}`) + } + + const normalized = normalizeRubyGemsPackage(gemResult) + + await withDeadlockRetry(() => + qx.tx(async (t) => { + const changed = new Set() + + const { id: packageDbId, changedFields: pkgChanged } = await upsertPackage(t, { + purl: pkg.purl, + ecosystem: 'rubygems', + namespace: null, + name: pkg.name, + description: normalized.description, + homepage: normalized.homepage, + declaredRepositoryUrl: normalized.declaredRepositoryUrl, + licenses: normalized.licenses, + licensesRaw: normalized.licensesRaw, + latestVersion: normalized.latestVersion, + registryUrl: rubyGemsRegistryUrl(pkg.name), + ingestionSource: 'rubygems-registry', + }) + pkgChanged.forEach((f) => changed.add(f)) + + if (normalized.totalDownloads > 0) { + const dlChanged = await recordDownloadSnapshot(t, { + packageId: packageDbId, + purl: pkg.purl, + totalDownloads: normalized.totalDownloads, + today, + }) + dlChanged.forEach((f) => changed.add(f)) + } + + await logAuditFieldChange(t, 'rubygems', pkg.purl, Array.from(changed)) + + log.debug({ purl: pkg.purl, totalDownloads: normalized.totalDownloads }, 'ok') + }), + ) + + return 'processed' +} + +export async function processBatch( + qx: QueryExecutor, + config: RubyGemsCoreConfig, + today: string, +): Promise { + const packages = await listRubyGemsPackagesToSync(qx, { limit: config.batchSize }) + + if (packages.length === 0) return { processed: 0, skipped: 0, error: 0, unchanged: 0 } + + log.info({ count: packages.length }, 'Batch started') + + const counts = { processed: 0, skipped: 0, error: 0, unchanged: 0 } + + for (let batchStart = 0; batchStart < packages.length; batchStart += config.concurrency) { + const group = packages.slice(batchStart, batchStart + config.concurrency) + + if (config.groupDelayMs > 0 && batchStart > 0) { + await new Promise((r) => setTimeout(r, config.groupDelayMs)) + } + + await Promise.all( + group.map(async (pkg) => { + try { + const status = await processPackage(qx, pkg, today) + counts[status]++ + } catch (err) { + const message = err instanceof Error ? err.message : String(err) + log.error({ purl: pkg.purl, error: message }, 'Unexpected error processing package') + try { + await markPackageError(qx, pkg) + } catch { + // best-effort — don't let a failed mark crash the batch + } + counts.error++ + } + }), + ) + + const done = batchStart + group.length + if (done % 1000 === 0 || done === packages.length) { + log.info({ done, total: packages.length, ...counts }, 'Progress') + } + } + + return counts +} diff --git a/services/apps/packages_worker/src/rubygems/runRubyGemsCriticalLoop.ts b/services/apps/packages_worker/src/rubygems/runRubyGemsCriticalLoop.ts new file mode 100644 index 0000000000..f148a55656 --- /dev/null +++ b/services/apps/packages_worker/src/rubygems/runRubyGemsCriticalLoop.ts @@ -0,0 +1,188 @@ +import { + IDbVersionUpsert, + QueryExecutor, + RubyGemsCriticalPackageToSync, + listRubyGemsCriticalPackagesToSync, + logAuditFieldChange, + replacePackageMaintainers, + upsertMaintainer, + upsertPackage, + upsertVersionsBatch, +} from '@crowd/data-access-layer' +import { getServiceChildLogger } from '@crowd/logging' + +import { fetchOwners, fetchVersions } from './client' +import { + normalizeRubyGemsOwners, + normalizeRubyGemsVersions, + pickLatestRubyGemsVersion, +} from './normalize' +import { BatchResult, isRubyGemsFetchError } from './types' + +const log = getServiceChildLogger('rubygems-critical') + +export type RubyGemsCriticalConfig = { + batchSize: number + concurrency: number + groupDelayMs: number +} + +async function withDeadlockRetry(fn: () => Promise, maxAttempts = 4): Promise { + for (let attempt = 1; ; attempt++) { + try { + return await fn() + } catch (err) { + const code = (err as { code?: string }).code + const isDeadlock = + code === '40P01' || /deadlock detected/i.test(String((err as Error)?.message)) + if (isDeadlock && attempt < maxAttempts) { + await new Promise((r) => setTimeout(r, 50 * attempt + Math.random() * 100)) + log.debug({ attempt }, 'Deadlock detected — retrying transaction') + continue + } + throw err + } + } +} + +type PackageStatus = 'processed' | 'skipped' | 'error' | 'unchanged' + +async function processPackage( + qx: QueryExecutor, + pkg: RubyGemsCriticalPackageToSync, +): Promise { + const [versionsResult, ownersResult] = await Promise.all([ + fetchVersions(pkg.name), + fetchOwners(pkg.name), + ]) + + if (isRubyGemsFetchError(versionsResult)) { + if (versionsResult.kind === 'NOT_FOUND') { + log.warn({ purl: pkg.purl }, 'Package not found on RubyGems versions endpoint — skipping') + return 'skipped' + } + if (versionsResult.kind === 'RATE_LIMIT') { + log.warn({ purl: pkg.purl }, 'Rate limited fetching RubyGems versions — will retry next pass') + return 'error' + } + throw new Error(`Transient error fetching versions for ${pkg.purl}: ${versionsResult.message}`) + } + + const normalizedVersions = normalizeRubyGemsVersions(versionsResult) + const latest = pickLatestRubyGemsVersion(normalizedVersions) + const ownersFetchFailed = isRubyGemsFetchError(ownersResult) + const owners = ownersFetchFailed ? [] : normalizeRubyGemsOwners(ownersResult) + + await withDeadlockRetry(() => + qx.tx(async (t) => { + const changed = new Set() + + const { id: packageDbId, changedFields: pkgChanged } = await upsertPackage(t, { + purl: pkg.purl, + ecosystem: 'rubygems', + namespace: null, + name: pkg.name, + description: null, + homepage: null, + declaredRepositoryUrl: null, + licenses: null, + licensesRaw: null, + latestVersion: latest?.number ?? null, + versionsCount: normalizedVersions.length > 0 ? normalizedVersions.length : null, + latestReleaseAt: latest?.publishedAt ?? null, + ingestionSource: 'rubygems-registry', + }) + pkgChanged.forEach((f) => changed.add(f)) + + if (normalizedVersions.length > 0) { + const versionRows: IDbVersionUpsert[] = normalizedVersions.map((v) => ({ + packageId: packageDbId, + ecosystem: 'rubygems', + namespace: null, + name: pkg.name, + number: v.number, + isLatest: v.number === latest?.number, + isPrerelease: v.isPrerelease, + license: v.licenses && v.licenses.length > 0 ? v.licenses[0] : null, + })) + const verChanged = await upsertVersionsBatch(t, versionRows) + verChanged.forEach((f) => changed.add(f)) + } + + const maintainerLinks: Array<{ maintainerId: number; role: 'maintainer' }> = [] + for (const owner of owners) { + const { id: maintainerId, changedFields: mChanged } = await upsertMaintainer(t, { + ecosystem: 'rubygems', + username: owner.username, + displayName: owner.username, + url: null, + email: owner.email, + }) + mChanged.forEach((f) => changed.add(f)) + maintainerLinks.push({ maintainerId, role: 'maintainer' }) + } + + if (!ownersFetchFailed) { + const pmChanged = await replacePackageMaintainers(t, packageDbId, maintainerLinks) + pmChanged.forEach((f) => changed.add(f)) + } + + await logAuditFieldChange(t, 'rubygems', pkg.purl, Array.from(changed)) + + log.debug( + { + purl: pkg.purl, + versions: normalizedVersions.length, + maintainers: maintainerLinks.length, + }, + 'ok', + ) + }), + ) + + return 'processed' +} + +export async function processBatch( + qx: QueryExecutor, + config: RubyGemsCriticalConfig, +): Promise { + const packages = await listRubyGemsCriticalPackagesToSync(qx, { limit: config.batchSize }) + + if (packages.length === 0) return { processed: 0, skipped: 0, error: 0, unchanged: 0 } + + log.info({ count: packages.length }, 'Critical batch started') + + const counts = { processed: 0, skipped: 0, error: 0, unchanged: 0 } + + for (let batchStart = 0; batchStart < packages.length; batchStart += config.concurrency) { + const group = packages.slice(batchStart, batchStart + config.concurrency) + + if (config.groupDelayMs > 0 && batchStart > 0) { + await new Promise((r) => setTimeout(r, config.groupDelayMs)) + } + + await Promise.all( + group.map(async (pkg) => { + try { + const status = await processPackage(qx, pkg) + counts[status]++ + } catch (err) { + const message = err instanceof Error ? err.message : String(err) + log.error( + { purl: pkg.purl, error: message }, + 'Unexpected error processing critical package', + ) + counts.error++ + } + }), + ) + + const done = batchStart + group.length + if (done % 1000 === 0 || done === packages.length) { + log.info({ done, total: packages.length, ...counts }, 'Critical progress') + } + } + + return counts +} diff --git a/services/apps/packages_worker/src/rubygems/schedule.ts b/services/apps/packages_worker/src/rubygems/schedule.ts new file mode 100644 index 0000000000..1d06224a3c --- /dev/null +++ b/services/apps/packages_worker/src/rubygems/schedule.ts @@ -0,0 +1,78 @@ +import { ScheduleAlreadyRunning, ScheduleOverlapPolicy } from '@temporalio/client' + +import { svc } from '../service' +import { ingestRubyGemsCriticalDetails, ingestRubyGemsPackages } from '../workflows' + +export async function scheduleRubyGemsIngestion(): Promise { + const { temporal } = svc + if (!temporal) throw new Error('Temporal client not initialized') + + try { + await temporal.schedule.create({ + scheduleId: 'rubygems-registry-ingest', + spec: { + cronExpressions: ['0 7 * * *'], + }, + policies: { + overlap: ScheduleOverlapPolicy.SKIP, + catchupWindow: '1 hour', + }, + action: { + type: 'startWorkflow', + workflowType: ingestRubyGemsPackages, + workflowId: 'rubygems-daily-core', + taskQueue: 'rubygems-worker', + workflowRunTimeout: '24 hours', + retry: { + initialInterval: '30 seconds', + backoffCoefficient: 2, + maximumAttempts: 5, + }, + args: [], + }, + }) + } catch (err) { + if (err instanceof ScheduleAlreadyRunning) { + svc.log.info('Schedule rubygems-registry-ingest already exists, skipping creation.') + } else { + throw err + } + } +} + +export async function scheduleRubyGemsCriticalIngestion(): Promise { + const { temporal } = svc + if (!temporal) throw new Error('Temporal client not initialized') + + try { + await temporal.schedule.create({ + scheduleId: 'rubygems-critical-ingest', + spec: { + cronExpressions: ['0 9 * * *'], + }, + policies: { + overlap: ScheduleOverlapPolicy.SKIP, + catchupWindow: '1 hour', + }, + action: { + type: 'startWorkflow', + workflowType: ingestRubyGemsCriticalDetails, + workflowId: 'rubygems-daily-critical', + taskQueue: 'rubygems-worker', + workflowRunTimeout: '24 hours', + retry: { + initialInterval: '30 seconds', + backoffCoefficient: 2, + maximumAttempts: 5, + }, + args: [], + }, + }) + } catch (err) { + if (err instanceof ScheduleAlreadyRunning) { + svc.log.info('Schedule rubygems-critical-ingest already exists, skipping creation.') + } else { + throw err + } + } +} diff --git a/services/apps/packages_worker/src/rubygems/types.ts b/services/apps/packages_worker/src/rubygems/types.ts new file mode 100644 index 0000000000..f1948ce1d1 --- /dev/null +++ b/services/apps/packages_worker/src/rubygems/types.ts @@ -0,0 +1,62 @@ +export interface BatchResult { + processed: number + skipped: number + error: number + unchanged: number +} + +export interface RubyGemsFetchError { + kind: 'NOT_FOUND' | 'RATE_LIMIT' + status?: number + message: string +} + +export type RubyGemsFetchResult = T | RubyGemsFetchError + +export function isRubyGemsFetchError(r: RubyGemsFetchResult): r is RubyGemsFetchError { + return typeof r === 'object' && r !== null && 'kind' in (r as object) +} + +export interface RubyGemsGemResponse { + name: string + version: string + info?: string | null + homepage_uri?: string | null + source_code_uri?: string | null + licenses?: string[] | null + downloads?: number +} + +export interface RubyGemsVersionItem { + number: string + created_at: string + prerelease?: boolean + licenses?: string[] | null +} + +export interface RubyGemsOwner { + handle?: string | null + email?: string | null +} + +export interface NormalizedRubyGemsPackage { + description: string | null + homepage: string | null + declaredRepositoryUrl: string | null + licenses: string[] | null + licensesRaw: string | null + latestVersion: string | null + totalDownloads: number +} + +export interface NormalizedRubyGemsVersion { + number: string + publishedAt: Date | null + isPrerelease: boolean + licenses: string[] | null +} + +export interface NormalizedRubyGemsOwner { + username: string + email: string | null +} diff --git a/services/apps/packages_worker/src/rubygems/workflows.ts b/services/apps/packages_worker/src/rubygems/workflows.ts new file mode 100644 index 0000000000..80fa5e5b21 --- /dev/null +++ b/services/apps/packages_worker/src/rubygems/workflows.ts @@ -0,0 +1,26 @@ +import { continueAsNew, log, proxyActivities } from '@temporalio/workflow' + +import type * as activities from './activities' + +const acts = proxyActivities({ + startToCloseTimeout: '30 minutes', + retry: { initialInterval: '30 seconds', backoffCoefficient: 2, maximumAttempts: 5 }, +}) + +export async function ingestRubyGemsPackages(): Promise { + const result = await acts.processRubyGemsCoreBatch() + if (result.processed + result.skipped + result.error + result.unchanged === 0) { + log.info('RubyGems core ingestion complete — no more work, exiting.', { ...result }) + return + } + await continueAsNew() +} + +export async function ingestRubyGemsCriticalDetails(): Promise { + const result = await acts.processRubyGemsCriticalBatch() + if (result.processed + result.skipped + result.error + result.unchanged === 0) { + log.info('RubyGems critical ingestion complete — no more work, exiting.', { ...result }) + return + } + await continueAsNew() +} diff --git a/services/apps/packages_worker/src/workflows/index.ts b/services/apps/packages_worker/src/workflows/index.ts index 9b762282b4..a14aee8ed0 100644 --- a/services/apps/packages_worker/src/workflows/index.ts +++ b/services/apps/packages_worker/src/workflows/index.ts @@ -25,6 +25,7 @@ export { ingestPypiDownloadsDaily, } from '../pypi/downloads/ingestPypiDownloads' export { ingestNuGetPackages } from '../nuget/workflows' +export { ingestRubyGemsCriticalDetails, ingestRubyGemsPackages } from '../rubygems/workflows' export { ingestSecurityContacts, ingestSecurityContactsForPurlWorkflow, diff --git a/services/libs/data-access-layer/src/index.ts b/services/libs/data-access-layer/src/index.ts index 2bcbdacc47..9ede0d95cd 100644 --- a/services/libs/data-access-layer/src/index.ts +++ b/services/libs/data-access-layer/src/index.ts @@ -21,6 +21,9 @@ export * from './osspckgs/maintainers' export * from './osspckgs/packages' export * from './osspckgs/repos' export * from './osspckgs/stewardships' +export * from './osspckgs/types' export * from './osspckgs/versions' export * from './osspckgs/nuget' +export * from './osspckgs/downloads' +export * from './osspckgs/rubygems' export * from './osspckgs/api' diff --git a/services/libs/data-access-layer/src/osspckgs/downloads.ts b/services/libs/data-access-layer/src/osspckgs/downloads.ts new file mode 100644 index 0000000000..14340b919c --- /dev/null +++ b/services/libs/data-access-layer/src/osspckgs/downloads.ts @@ -0,0 +1,58 @@ +import { insertDailyDownloads } from '../packages/downloadsDaily' +import { upsertLast30dDownload } from '../packages/downloadsLast30d' +import { QueryExecutor } from '../queryExecutor' + +export async function recordDownloadSnapshot( + qx: QueryExecutor, + params: { + packageId: number + purl: string + totalDownloads: number + today: string // YYYY-MM-DD + }, +): Promise { + const { packageId, purl, totalDownloads, today } = params + const changed: string[] = [] + + const prevRow = await qx.selectOne( + `SELECT total_downloads FROM packages WHERE id = $(packageId)`, + { packageId }, + ) + const prev: number | null = + prevRow?.total_downloads != null ? Number(prevRow.total_downloads) : null + + if (prev !== null && totalDownloads > prev) { + const delta = totalDownloads - prev + const dailyChanged = await insertDailyDownloads(qx, String(packageId), [ + { day: today, downloads: delta }, + ]) + dailyChanged.forEach((f) => changed.push(f)) + } + + const updated = await qx.result( + `UPDATE packages SET total_downloads = $(totalDownloads) + WHERE id = $(packageId) + AND (total_downloads IS NULL OR total_downloads < $(totalDownloads))`, + { totalDownloads, packageId }, + ) + if (updated > 0) changed.push('packages.total_downloads') + + const sumRow = await qx.selectOne( + `SELECT COALESCE(SUM(count), 0)::bigint AS total + FROM downloads_daily + WHERE package_id = $(packageId)::bigint + AND date > $(today)::date - INTERVAL '30 days' + AND date <= $(today)::date`, + { packageId, today }, + ) + const monthlySum = Number(sumRow.total) + + const thirtyDaysAgo = new Date(today) + thirtyDaysAgo.setDate(thirtyDaysAgo.getDate() - 29) + const startDate = thirtyDaysAgo.toISOString().split('T')[0] + + const monthlyChanged = await upsertLast30dDownload(qx, purl, startDate, today, monthlySum, true) + monthlyChanged.forEach((f) => changed.push(f)) + + return changed +} diff --git a/services/libs/data-access-layer/src/osspckgs/rubygems.ts b/services/libs/data-access-layer/src/osspckgs/rubygems.ts new file mode 100644 index 0000000000..f29921ae80 --- /dev/null +++ b/services/libs/data-access-layer/src/osspckgs/rubygems.ts @@ -0,0 +1,68 @@ +import { QueryExecutor } from '../queryExecutor' + +export type RubyGemsPackageToSync = { + id: number + purl: string + name: string + latestVersion: string | null +} + +const RUBYGEMS_WORKER_OUTCOMES = ['rubygems-registry', 'rubygems_not_found', 'rubygems_error'] + +export async function listRubyGemsPackagesToSync( + qx: QueryExecutor, + options: { limit: number }, +): Promise { + const { limit } = options + return qx.select( + ` + SELECT + p.id, + p.purl, + p.name, + p.latest_version AS "latestVersion" + FROM packages p + WHERE + p.ecosystem = 'rubygems' + AND ( + p.ingestion_source IS NULL + OR p.ingestion_source <> ALL($(workerOutcomes)::text[]) + OR p.last_synced_at < NOW() - INTERVAL '1 day' + ) + ORDER BY + p.dependent_count DESC NULLS LAST, + p.id ASC + LIMIT $(limit) + `, + { limit, workerOutcomes: RUBYGEMS_WORKER_OUTCOMES }, + ) +} + +export type RubyGemsCriticalPackageToSync = { + id: number + purl: string + name: string +} + +export async function listRubyGemsCriticalPackagesToSync( + qx: QueryExecutor, + options: { limit: number }, +): Promise { + const { limit } = options + return qx.select( + ` + SELECT p.id, p.purl, p.name + FROM packages p + LEFT JOIN LATERAL ( + SELECT MAX(v.last_synced_at) AS last_synced + FROM versions v WHERE v.package_id = p.id + ) v ON true + WHERE p.ecosystem = 'rubygems' + AND p.is_critical + AND (v.last_synced IS NULL OR v.last_synced < NOW() - INTERVAL '1 day') + ORDER BY p.dependent_count DESC NULLS LAST, p.id ASC + LIMIT $(limit) + `, + { limit }, + ) +}