Add a data pipeline page for curators

src/App.jsx

@@ -19,6 +19,7 @@ const Review = lazy(() => import('./pages/Review'))
 const InfoPage = lazy(() => import('./pages/InfoPage'))
 const Register = lazy(() => import('./pages/Register'))
 const AdminRegistrations = lazy(() => import('./pages/AdminRegistrations'))
+const DataPipeline = lazy(() => import('./pages/DataPipeline'))
 const NotFound = lazy(() => import('./pages/NotFound'))
 
 function App() {
@@ -32,6 +33,7 @@ function App() {
           <Route path="admin/users/new" element={<RequireAuth><AdminCreateUser /></RequireAuth>} />
           <Route path="review" element={<RequireAuth><Review /></RequireAuth>} />
           <Route path="admin/registrations" element={<RequireAuth><AdminRegistrations /></RequireAuth>} />
+          <Route path="admin/pipeline" element={<RequireAuth><DataPipeline /></RequireAuth>} />
           <Route path="register" element={<Register />} />
           <Route path="reports" element={<Reports />} />
           {/* Same paths as the legacy site, so existing links keep working. */}

src/index.css

@@ -742,3 +742,8 @@ p {
 .registration-row { display: flex; flex-wrap: wrap; align-items: flex-start; justify-content: space-between; gap: 1rem; padding: 1rem 1.25rem; }
 .registration-row:hover { transform: none; }
 .registration-actions { display: flex; flex-wrap: wrap; align-items: center; gap: 0.5rem; }
+
+/* Data pipeline page (pages/DataPipeline.jsx) */
+.pipeline-runs { list-style: none; margin: 0; padding: 0; display: grid; gap: 0.6rem; font-size: 0.9rem; }
+.pipeline-worklist { display: grid; gap: 0.75rem; margin-top: 1rem; }
+.pipeline-page .btn-sm { padding: 0.25rem 0.75rem; font-size: 0.8rem; }

src/pages/DataPipeline.jsx

@@ -0,0 +1,213 @@
+/**
+ * DataPipeline.jsx — curators' read-only view of the data pipeline
+ * (validate-submission permission).
+ *
+ * What each source last loaded and what the app holds from it, projects where
+ * an app edit and a source change disagree, and the data-quality issues the
+ * pipeline found, each opening a worklist of the projects it affects. Loading
+ * data is done with the pipeline's command line (nccrd-build/PIPELINE.md).
+ */
+import { useEffect, useState } from 'react';
+import { Link } from 'react-router-dom';
+import { ArrowLeft } from 'lucide-react';
+import { getPipelineStatus, getPipelineIssueProjects } from '../services/api';
+import { useCurrentUser } from '../context/CurrentUserContext';
+import { sourceLabel } from '../utils/dataSources';
+
+const PAGE_SIZE = 50;
+
+const FIELDS = {
+    'geo_location.province': 'Province',
+    'geo_location.coordinates': 'Coordinates',
+    estimated_budget_cost: 'Budget range',
+    implementation_status: 'Implementation status',
+    intervention_measurement: 'Type of measure',
+    funding_type: 'Funding type',
+};
+const fieldLabel = (field) => FIELDS[field] || field.replace(/[._]/g, ' ').replace(/^\w/, (c) => c.toUpperCase());
+
+const formatDate = (iso) => (iso ? new Date(iso).toLocaleString('en-ZA', { day: 'numeric', month: 'short', year: 'numeric', hour: '2-digit', minute: '2-digit' }) : '');
+
+/** "update 309, excluded 877" from a run's summary (merge actions, or a clean-up's counts). */
+const runSummary = (summary) => {
+    const counts = summary?.actions || summary || {};
+    return Object.entries(counts).filter(([, n]) => typeof n === 'number').map(([k, n]) => `${k.replace(/_/g, ' ')} ${n.toLocaleString()}`).join(', ');
+};
+
+const Worklist = ({ issue }) => {
+    const [projects, setProjects] = useState([]);
+    const [total, setTotal] = useState(null);
+    const [error, setError] = useState(null);
+    const [loading, setLoading] = useState(false);
+
+    const load = (offset) => {
+        setLoading(true);
+        getPipelineIssueProjects({ data_source: issue.data_source, field: issue.field, issue: issue.issue, limit: PAGE_SIZE, offset })
+            .then((data) => {
+                setProjects((prev) => (offset ? [...prev, ...data.projects] : data.projects));
+                setTotal(data.total);
+                setLoading(false);
+            })
+            .catch((err) => { setError(err.message || 'Could not load the projects.'); setLoading(false); });
+    };
+    useEffect(() => { load(0); }, [issue.data_source, issue.field, issue.issue]); // eslint-disable-line react-hooks/exhaustive-deps
+
+    if (error) return <p className="report-error" role="alert">{error}</p>;
+    return (
+        <div className="pipeline-worklist">
+            {total === null ? <p className="report-empty">Loading…</p> : (
+                <ul className="review-list">
+                    {projects.map((p) => (
+                        <li key={p.id}>
+                            <Link to={`/submission/${p.id}`} className="glass-panel review-row">
+                                <span className="review-row-main">
+                                    <strong>{p.title || 'Untitled project'}</strong>
+                                    <span className="review-row-meta">
+                                        {[p.implementation_organization, p.public ? 'public' : 'not public', p.value && `found: ${p.value}`].filter(Boolean).join(' · ')}
+                                    </span>
+                                </span>
+                            </Link>
+                        </li>
+                    ))}
+                </ul>
+            )}
+            {total !== null && projects.length < total && (
+                <button type="button" className="btn btn-outline" disabled={loading} onClick={() => load(projects.length)}>
+                    Show more ({(total - projects.length).toLocaleString()} remaining)
+                </button>
+            )}
+        </div>
+    );
+};
+
+const DataPipeline = () => {
+    const { hasPermission, loading: authLoading } = useCurrentUser();
+    const [status, setStatus] = useState(null);
+    const [error, setError] = useState(null);
+    const [open, setOpen] = useState(null);
+    const isCurator = hasPermission('validate-submission');
+
+    useEffect(() => {
+        if (!isCurator) return;
+        getPipelineStatus().then(setStatus).catch((err) => setError(err.message || 'Could not load the data pipeline status.'));
+    }, [isCurator]);
+
+    if (authLoading) return null;
+    if (!isCurator) return (
+        <div className="container" style={{ paddingBlock: '2rem' }}>
+            <div className="glass-panel" style={{ padding: '2rem', borderLeft: '4px solid #ef4444' }}>
+                <h3 style={{ color: '#b91c1c' }}>Curators only</h3>
+                <p>You need the reviewer permission (validate-submission) to see the data pipeline.</p>
+            </div>
+        </div>
+    );
+
+    const issueKey = (i) => `${i.data_source}|${i.field}|${i.issue}`;
+
+    return (
+        <div className="container pipeline-page" style={{ paddingBlock: '2rem 4rem' }}>
+            <Link to="/review" className="report-back"><ArrowLeft size={16} /> Back to the review queue</Link>
+            <header>
+                <h1 style={{ fontSize: '2rem', margin: '0 0 0.5rem' }}>Data pipeline</h1>
+                <p className="report-lede">
+                    What the data pipeline has loaded into the app from the legacy NCCRD and the provincial registers, projects where an
+                    edit in the app and a change in the source disagree, and the data-quality issues it found. Open an issue for the list of
+                    projects to fix.
+                </p>
+            </header>
+
+            {error && <div className="glass-panel report-error" role="alert">{error}</div>}
+            {!status && !error && <p className="report-empty">Loading…</p>}
+            {status && !status.available && (
+                <div className="glass-panel" style={{ padding: '2rem' }}>
+                    <p className="report-empty">The data pipeline hasn’t been run on this database yet.</p>
+                </div>
+            )}
+            {status?.available && (
+                <div className="report-grid">
+                    <section className="glass-panel report-panel is-wide" aria-labelledby="pipeline-sources">
+                        <h2 className="report-panel-title" id="pipeline-sources">Sources</h2>
+                        <p className="report-panel-subtitle">
+                            {status.loads.map((l) => `${l.source}: ${l.file_name}, loaded ${formatDate(l.loaded_at)}`).join(' · ')}
+                        </p>
+                        <div className="chart-scroll">
+                            <table className="report-table">
+                                <thead>
+                                    <tr><th>Source</th><th>Rows in the source</th><th>Kept out</th><th>In the app</th><th>Public</th></tr>
+                                </thead>
+                                <tbody>
+                                    {status.sources.map((s) => (
+                                        <tr key={s.data_source}>
+                                            <td>{sourceLabel(s.data_source)}</td>
+                                            <td>{s.in_source.toLocaleString()}</td>
+                                            <td>{s.excluded.toLocaleString()}</td>
+                                            <td>{s.in_app.toLocaleString()}</td>
+                                            <td>{s.public.toLocaleString()}</td>
+                                        </tr>
+                                    ))}
+                                </tbody>
+                            </table>
+                        </div>
+                    </section>
+
+                    <section className="glass-panel report-panel" aria-labelledby="pipeline-runs">
+                        <h2 className="report-panel-title" id="pipeline-runs">Recent runs</h2>
+                        {status.runs.length === 0 ? <p className="report-empty">None yet.</p> : (
+                            <ul className="pipeline-runs">
+                                {status.runs.map((r) => (
+                                    <li key={`${r.kind}-${r.ran_at}`}><strong>{r.kind}</strong> · {formatDate(r.ran_at)}<br /><span className="review-row-meta">{runSummary(r.summary)}</span></li>
+                                ))}
+                            </ul>
+                        )}
+                    </section>
+
+                    <section className="glass-panel report-panel" aria-labelledby="pipeline-conflicts">
+                        <h2 className="report-panel-title" id="pipeline-conflicts">Conflicts</h2>
+                        <p className="report-panel-subtitle">Changed in the app and in the source. The app’s version is kept.</p>
+                        {status.conflicts.length === 0 ? <p className="report-empty">None.</p> : (
+                            <ul className="pipeline-runs">
+                                {status.conflicts.map((c) => (
+                                    <li key={c.id}>
+                                        <Link to={`/submission/${c.id}`}>{c.title || 'Untitled project'}</Link>
+                                        <br /><span className="review-row-meta">{[c.reason, (c.fields || []).map(fieldLabel).join(', ')].filter(Boolean).join(' · ')}</span>
+                                    </li>
+                                ))}
+                            </ul>
+                        )}
+                    </section>
+
+                    <section className="glass-panel report-panel is-wide" aria-labelledby="pipeline-issues">
+                        <h2 className="report-panel-title" id="pipeline-issues">Data quality</h2>
+                        <p className="report-panel-subtitle">Issues found while loading, counted over projects in the app.</p>
+                        {status.issues.length === 0 ? <p className="report-empty">No issues.</p> : (
+                            <div className="chart-scroll">
+                                <table className="report-table">
+                                    <thead><tr><th>Source</th><th>Field</th><th>Issue</th><th>Projects</th><th /></tr></thead>
+                                    <tbody>
+                                        {status.issues.map((i) => (
+                                            <tr key={issueKey(i)}>
+                                                <td>{sourceLabel(i.data_source)}</td>
+                                                <td>{fieldLabel(i.field)}</td>
+                                                <td>{i.issue}</td>
+                                                <td>{i.projects.toLocaleString()}</td>
+                                                <td>
+                                                    <button type="button" className="btn btn-outline btn-sm" aria-expanded={open === issueKey(i)}
+                                                        onClick={() => setOpen(open === issueKey(i) ? null : issueKey(i))}>
+                                                        {open === issueKey(i) ? 'Hide' : 'Show projects'}
+                                                    </button>
+                                                </td>
+                                            </tr>
+                                        ))}
+                                    </tbody>
+                                </table>
+                            </div>
+                        )}
+                        {open && <Worklist key={open} issue={status.issues.find((i) => issueKey(i) === open)} />}
+                    </section>
+                </div>
+            )}
+        </div>
+    );
+};
+
+export default DataPipeline;

src/pages/DataPipeline.test.jsx

@@ -0,0 +1,57 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { render, screen, fireEvent } from '@testing-library/react';
+import { MemoryRouter } from 'react-router-dom';
+import { getPipelineStatus, getPipelineIssueProjects } from '../services/api';
+import DataPipeline from './DataPipeline';
+
+const auth = { curator: true };
+vi.mock('../context/CurrentUserContext', () => ({
+    useCurrentUser: () => ({ hasPermission: (p) => auth.curator && p === 'validate-submission', loading: false }),
+}));
+vi.mock('../services/api', () => ({ getPipelineStatus: vi.fn(), getPipelineIssueProjects: vi.fn() }));
+
+const status = {
+    available: true,
+    loads: [{ source: 'legacy', file_name: 'nccrd_stable.bak', loaded_at: '2026-10-06T09:03:00Z' }],
+    sources: [{ data_source: 'sqlserver_legacy', in_source: 2540, excluded: 865, in_app: 1435, public: 1051 }],
+    runs: [{ kind: 'gold', ran_at: '2026-10-06T09:05:00Z', summary: { actions: { update: 43, insert: 125 } } }],
+    conflicts: [{ id: 'c1', title: 'Solar clinic', reason: 'changed in the app since the last sync', fields: ['title'] }],
+    issues: [{ data_source: 'sqlserver_legacy', field: 'geo_location.province', issue: 'no province', projects: 380 }],
+};
+
+describe('DataPipeline', () => {
+    beforeEach(() => {
+        auth.curator = true;
+        getPipelineStatus.mockReset().mockResolvedValue(status);
+        getPipelineIssueProjects.mockReset().mockResolvedValue({
+            total: 380, projects: [{ id: 'p1', title: 'Coastal plan', implementation_organization: 'City', public: true, value: null }],
+        });
+    });
+
+    it('shows sources, runs, conflicts and issues, and opens a worklist', async () => {
+        render(<MemoryRouter><DataPipeline /></MemoryRouter>);
+        expect(await screen.findAllByText('Legacy NCCRD')).toHaveLength(2); // sources and issues tables
+        expect(screen.getByText('2,540')).toBeInTheDocument();
+        expect(screen.getByText(/update 43, insert 125/)).toBeInTheDocument();
+        expect(screen.getByRole('link', { name: 'Solar clinic' })).toHaveAttribute('href', '/submission/c1');
+        expect(screen.getByText('Province')).toBeInTheDocument();
+
+        fireEvent.click(screen.getByRole('button', { name: 'Show projects' }));
+        expect(await screen.findByRole('link', { name: /Coastal plan/ })).toHaveAttribute('href', '/submission/p1');
+        expect(getPipelineIssueProjects).toHaveBeenCalledWith(expect.objectContaining({ field: 'geo_location.province', issue: 'no province', offset: 0 }));
+        expect(screen.getByRole('button', { name: /Show more \(379 remaining\)/ })).toBeInTheDocument();
+    });
+
+    it('says so when the pipeline has never run', async () => {
+        getPipelineStatus.mockResolvedValue({ available: false });
+        render(<MemoryRouter><DataPipeline /></MemoryRouter>);
+        expect(await screen.findByText(/hasn’t been run on this database yet/)).toBeInTheDocument();
+    });
+
+    it('turns away non-curators', () => {
+        auth.curator = false;
+        render(<MemoryRouter><DataPipeline /></MemoryRouter>);
+        expect(screen.getByText('Curators only')).toBeInTheDocument();
+        expect(getPipelineStatus).not.toHaveBeenCalled();
+    });
+});

src/pages/Review.jsx

@@ -13,17 +13,11 @@ import { useCurrentUser } from '../context/CurrentUserContext';
 import { REVIEW_STATES } from '../utils/reviewStatus';
 import StatusBadge from '../components/StatusBadge';
 import InterventionBadge from '../components/InterventionBadge';
+import { sourceLabel } from '../utils/dataSources';
 
 const TABS = ['awaiting_review', 'not_accepted', 'draft', 'published'];
 const PAGE_SIZE = 50;
 
-const SOURCES = {
-    sqlserver_legacy: 'Legacy NCCRD',
-    gauteng_register_2024: 'Gauteng register 2024',
-    wc_project_database_2020: 'Western Cape database 2020',
-    react_app: 'Submission form',
-};
-
 const formatDate = (iso) => (iso ? new Date(iso).toLocaleDateString('en-ZA', { day: 'numeric', month: 'short', year: 'numeric' }) : '');
 
 const Review = () => {
@@ -78,6 +72,7 @@ const Review = () => {
                 <h1 style={{ fontSize: '2rem', margin: '0 0 0.5rem' }}>Review submissions</h1>
                 <p className="report-lede">
                     Open a project to read it in full, then accept it to publish it, or mark it not accepted with a reason for the submitter.
+                    {' '}<Link to="/admin/pipeline">Data pipeline: loaded sources and data-quality worklists →</Link>
                 </p>
             </header>
 
@@ -116,7 +111,7 @@ const Review = () => {
                                     </span>
                                     <strong>{s.title || 'Untitled project'}</strong>
                                     <span className="review-row-meta">
-                                        {[s.implementation_organization, SOURCES[s.data_source] || s.data_source, s.createdate && `added ${formatDate(s.createdate)}`]
+                                        {[s.implementation_organization, sourceLabel(s.data_source), s.createdate && `added ${formatDate(s.createdate)}`]
                                             .filter(Boolean).join(' · ')}
                                     </span>
                                 </span>

src/services/api.js

@@ -121,6 +121,19 @@ export const reviewSubmission = async (id, decision, comments) => {
 };
 
 /** Submissions per review state, for the review queue. GET /submission/review/counts. */
+export const getPipelineStatus = async () => {
+    const res = await fetch(`${API_BASE_URL}/pipeline/status`, { headers: { ..._authHeaders() } });
+    if (!res.ok) await _throwOnErrorWithAuthCheck(res);
+    return res.json();
+};
+
+/** Projects behind one data-quality issue: { data_source, field, issue, limit, offset }. */
+export const getPipelineIssueProjects = async (params) => {
+    const res = await fetch(`${API_BASE_URL}/pipeline/issues?${new URLSearchParams(params)}`, { headers: { ..._authHeaders() } });
+    if (!res.ok) await _throwOnErrorWithAuthCheck(res);
+    return res.json();
+};
+
 export const getReviewCounts = async () => {
     const res = await fetch(`${API_BASE_URL}/submission/review/counts`, { headers: { ..._authHeaders() } });
     if (!res.ok) await _throwOnErrorWithAuthCheck(res);

src/utils/dataSources.js

@@ -0,0 +1,10 @@
+// Readable names for nccrd.submission.data_source, shared by the review
+// queue and the data pipeline page.
+export const DATA_SOURCES = {
+    sqlserver_legacy: 'Legacy NCCRD',
+    gauteng_register_2024: 'Gauteng register 2024',
+    wc_project_database_2020: 'Western Cape database 2020',
+    react_app: 'Submission form',
+};
+
+export const sourceLabel = (dataSource) => DATA_SOURCES[dataSource] || dataSource;