fix: 2026-08 production audit

.env.example

@@ -27,6 +27,7 @@ VALKEY_ADDR=localhost:6379
 # GH_MEMBERSHIP_CACHE_TTL=300         # seconds
 # JWT_TTL_SECONDS=900                 # deploy-session JWT TTL
 # VALKEY_PASSWORD=                    # empty for unauthenticated dev
+# VALKEY_CONNECT_RETRY_WINDOW=5s      # boot-time retry window for the initial dial; 0 disables retry
 # REGISTRY_AUTHZ_TEAM=staff           # GitHub team allowed to mutate the sites registry
 # ALIAS_PRODUCTION_KEY_FORMAT=<site>/production
 # ALIAS_PREVIEW_KEY_FORMAT=<site>/preview

.github/workflows/test.yml

@@ -28,6 +28,9 @@ jobs:
           go-version-file: go.mod
           cache-dependency-path: go.sum
 
+      - name: go mod tidy check
+        run: go mod tidy -diff
+
       - name: go vet
         run: go vet ./...
 

README.md

@@ -2,7 +2,7 @@
 
 Static-apps deploy proxy for the freeCodeCamp Universe platform. Public hostname: `uploads.freecode.camp`.
 
-Staff devs and CI run `universe deploy`; the artifact lands on R2 behind a Caddy `r2_alias` upstream. Zero R2 tokens reach staff hands or CI secrets — Artemis is the sole holder of the admin S3 token. Identity is GitHub team membership.
+Staff developers and CI run `universe static deploy`. The CLI uploads the build artifact to artemis, artemis writes it to R2, and a Caddy `r2_alias` upstream serves it. Staff and CI hold no R2 tokens — artemis is the only holder of the admin S3 token. Caller identity comes from GitHub team membership.
 
 ## Quick start
 
@@ -15,10 +15,12 @@ just                   # list every recipe
 
 ## Docs
 
+- **[`docs/ORIENTATION.md`](docs/ORIENTATION.md)** — the read sequence for a new contributor.
+- **[`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md)** — what the service does and how it is built, written from the source code.
 - **[`docs/README.md`](docs/README.md)** — API contract, configuration, observability, R2 layout, sites registry, integration testing, curl examples.
 - **[`docs/RELEASING.md`](docs/RELEASING.md)** — versioning rule, release-please flow, image build, downstream deploy pin.
 
-The CLI ↔ artemis contract and per-site authorization model are specified in ADR-016 (Universe platform repo).
+ADR-016 (Universe platform repo) specifies the CLI ↔ artemis contract and the per-site authorization model.
 
 ## License
 

cmd/artemis/gcworkflows.go

@@ -8,6 +8,7 @@ import (
 	"errors"
 	"fmt"
 	"log/slog"
+	mrand "math/rand/v2"
 	"time"
 
 	"github.com/freeCodeCamp/artemis/internal/gc"
@@ -71,13 +72,39 @@ const (
 	cronTombstonePurge         = "0 3 * * *"
 	cronReconcile              = "0 4 * * *"
 	relayInterval              = 5 * time.Second
-
-	reconcilePublishFloor   = 4 * time.Second
-	reconcilePublishPerSite = 150 * time.Millisecond
 )
 
-func reconcilePublishDeadline(n int) time.Duration {
-	return reconcilePublishFloor + time.Duration(n)*reconcilePublishPerSite
+func publishReconcileEvents(ctx context.Context, publisher worker.Publisher, sites []string, perPublish time.Duration) (int, error) {
+	shuffled := append([]string(nil), sites...)
+	mrand.Shuffle(len(shuffled), func(i, j int) { shuffled[i], shuffled[j] = shuffled[j], shuffled[i] })
+	var firstErr error
+	published := 0
+	for _, site := range shuffled {
+		if ctx.Err() != nil {
+			if firstErr == nil {
+				firstErr = ctx.Err()
+			}
+			break
+		}
+		payload, err := json.Marshal(map[string]string{"site": site})
+		if err != nil {
+			if firstErr == nil {
+				firstErr = err
+			}
+			continue
+		}
+		pctx, cancel := context.WithTimeout(ctx, perPublish)
+		err = publisher.Publish(pctx, topicSiteReconcile, payload)
+		cancel()
+		if err != nil {
+			if firstErr == nil {
+				firstErr = err
+			}
+			continue
+		}
+		published++
+	}
+	return published, firstErr
 }
 
 func runRelayLoop(ctx context.Context, relay *worker.Relay, interval time.Duration) {
@@ -118,28 +145,12 @@ func gcWorkflowDefs(gcw *gcWiring, dryRun bool, publisher worker.Publisher, reco
 			Cron: []string{cronReconcile},
 			Handler: withCheckIn(workflowReconcileScheduler, cronReconcile, observeWorkflow(workflowReconcileScheduler, func(ctx context.Context, _ map[string]any) error {
 				sites := reconcileSites()
-				pctx, cancel := context.WithTimeout(ctx, reconcilePublishDeadline(len(sites)))
-				defer cancel()
-				var firstErr error
-				for _, site := range sites {
-					if pctx.Err() != nil {
-						if firstErr == nil {
-							firstErr = pctx.Err()
-						}
-						break
-					}
-					payload, err := json.Marshal(map[string]string{"site": site})
-					if err != nil {
-						if firstErr == nil {
-							firstErr = err
-						}
-						continue
-					}
-					if err := publisher.Publish(pctx, topicSiteReconcile, payload); err != nil {
-						if firstErr == nil {
-							firstErr = err
-						}
-					}
+				published, firstErr := publishReconcileEvents(ctx, publisher, sites, worker.DefaultPublishTimeout)
+				if published < len(sites) {
+					slog.ErrorContext(ctx, "reconcile.schedule.incomplete",
+						"sites", len(sites),
+						"published", published,
+						"skipped", len(sites)-published)
 				}
 				if firstErr != nil {
 					captureBackground("reconcile.schedule", firstErr)

cmd/artemis/gcworkflows_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"log/slog"
 	"sync"
 	"testing"
@@ -17,6 +18,8 @@ import (
 	"github.com/jackc/pgx/v5/pgconn"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"sort"
+	"strings"
 )
 
 type fakeReaper struct{}
@@ -214,6 +217,67 @@ func TestReconcileScheduler_BoundsPublishDeadline(t *testing.T) {
 	assert.LessOrEqual(t, d, 30*time.Second, "publish deadline is bounded, not open-ended")
 }
 
+type slowPublisher struct {
+	stallOn string
+	mu      sync.Mutex
+	sites   []string
+}
+
+func (p *slowPublisher) Publish(ctx context.Context, _ string, payload []byte) error {
+	var m map[string]string
+	if err := json.Unmarshal(payload, &m); err != nil {
+		return err
+	}
+	if p.stallOn != "" && m["site"] == p.stallOn {
+		<-ctx.Done()
+		return ctx.Err()
+	}
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.sites = append(p.sites, m["site"])
+	return nil
+}
+
+func (p *slowPublisher) published() []string {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	return append([]string(nil), p.sites...)
+}
+
+func reconcileSiteNames(n int) []string {
+	out := make([]string, n)
+	for i := range out {
+		out[i] = fmt.Sprintf("site-%03d", i)
+	}
+	return out
+}
+
+func TestPublishReconcileEvents_PublishesEverySite(t *testing.T) {
+	sites := reconcileSiteNames(50)
+	pub := &slowPublisher{}
+
+	published, err := publishReconcileEvents(context.Background(), pub, sites, 100*time.Millisecond)
+
+	require.NoError(t, err)
+	assert.Equal(t, len(sites), published,
+		"every registered site must get a reconcile event; a per-run budget must never truncate the list")
+	assert.ElementsMatch(t, sites, pub.published())
+}
+
+func TestPublishReconcileEvents_StalledSiteDoesNotDropTheRest(t *testing.T) {
+	sites := reconcileSiteNames(20)
+	pub := &slowPublisher{stallOn: sites[5]}
+
+	published, err := publishReconcileEvents(context.Background(), pub, sites, 20*time.Millisecond)
+
+	require.Error(t, err, "a stalled publish is still reported to the caller")
+	assert.Equal(t, len(sites)-1, published,
+		"one stalled site must not stop the sites after it")
+	assert.NotContains(t, pub.published(), sites[5])
+	assert.Contains(t, pub.published(), sites[19],
+		"the last site in the list must still be reached")
+}
+
 func TestGCWorkflowDefs(t *testing.T) {
 	gcw := &gcWiring{SiteGC: &gc.SiteGC{}, Purge: &gc.TombstonePurge{}, Reconciler: &gc.Reconciler{}}
 	defs := gcWorkflowDefs(gcw, true, &capturingPublisher{}, noSites)
@@ -259,8 +323,10 @@ func TestReconcileScheduler_PublishesPerSite(t *testing.T) {
 
 	require.Len(t, pub.topics, 2, "one site.reconcile event published per registered site")
 	assert.Equal(t, []string{topicSiteReconcile, topicSiteReconcile}, pub.topics)
-	assert.Contains(t, string(pub.payloads[0]), `"site":"www"`)
-	assert.Contains(t, string(pub.payloads[1]), `"site":"learn"`)
+	payloads := []string{string(pub.payloads[0]), string(pub.payloads[1])}
+	sort.Strings(payloads)
+	assert.Equal(t, []string{`{"site":"learn"}`, `{"site":"www"}`}, payloads,
+		"every site gets one event; publish order is shuffled by design")
 }
 
 type exhaustingPublisher struct {
@@ -461,3 +527,48 @@ func TestRegisterGCWorkflows(t *testing.T) {
 	require.NoError(t, registerGCWorkflows(rt, gcw, false, &capturingPublisher{}, noSites))
 	assert.Len(t, rt.Registered(), 4)
 }
+
+type truncatingPublisher struct {
+	mu     sync.Mutex
+	sites  []string
+	after  int
+	cancel context.CancelFunc
+}
+
+func (p *truncatingPublisher) Publish(_ context.Context, _ string, payload []byte) error {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	var m map[string]string
+	if err := json.Unmarshal(payload, &m); err != nil {
+		return err
+	}
+	p.sites = append(p.sites, m["site"])
+	if len(p.sites) >= p.after {
+		p.cancel()
+	}
+	return nil
+}
+
+func TestReconcileScheduler_TruncatedRunsCoverDisjointSuffixes(t *testing.T) {
+	sites := reconcileSiteNames(40)
+	const keep = 8
+	const runs = 6
+
+	publishedSets := make([]string, 0, runs)
+	for range runs {
+		ctx, cancel := context.WithCancel(context.Background())
+		pub := &truncatingPublisher{after: keep, cancel: cancel}
+		_, _ = publishReconcileEvents(ctx, pub, sites, time.Second)
+		cancel()
+		got := append([]string(nil), pub.sites...)
+		sort.Strings(got)
+		publishedSets = append(publishedSets, strings.Join(got, ","))
+	}
+
+	distinct := map[string]bool{}
+	for _, s := range publishedSets {
+		distinct[s] = true
+	}
+	require.Greater(t, len(distinct), 1,
+		"a truncated run must not cover the identical site prefix every time; a fixed order starves the same tail every night")
+}

cmd/artemis/main_test.go

@@ -21,7 +21,7 @@ func TestBootMigrations(t *testing.T) {
 
 	testcontainers.SkipIfProviderIsNotHealthy(t)
 
-	container, err := postgres.Run(ctx, "postgres:16-alpine",
+	container, err := postgres.Run(ctx, testPostgresImage,
 		postgres.WithDatabase("artemis_test"),
 		postgres.WithUsername("artemis"),
 		postgres.WithPassword("artemis"),
@@ -46,3 +46,5 @@ func TestBootMigrations(t *testing.T) {
 		require.Truef(t, exists, "table %q must exist after boot migrations", table)
 	}
 }
+
+const testPostgresImage = "postgres:16-alpine"

docker-compose.yml

@@ -2,7 +2,7 @@ name: artemis-local
 
 services:
   postgres:
-    image: postgres:17-alpine
+    image: postgres:16-alpine
     environment:
       POSTGRES_USER: artemis
       POSTGRES_PASSWORD: artemis