diff --git a/src/App.jsx b/src/App.jsx index f1f319f..93945cd 100644 --- a/src/App.jsx +++ b/src/App.jsx @@ -19,6 +19,7 @@ const Review = lazy(() => import('./pages/Review')) const InfoPage = lazy(() => import('./pages/InfoPage')) const Register = lazy(() => import('./pages/Register')) const AdminRegistrations = lazy(() => import('./pages/AdminRegistrations')) +const DataPipeline = lazy(() => import('./pages/DataPipeline')) const NotFound = lazy(() => import('./pages/NotFound')) function App() { @@ -32,6 +33,7 @@ function App() { } /> } /> } /> + } /> } /> } /> {/* Same paths as the legacy site, so existing links keep working. */} diff --git a/src/index.css b/src/index.css index 1a67d7b..192b435 100644 --- a/src/index.css +++ b/src/index.css @@ -742,3 +742,8 @@ p { .registration-row { display: flex; flex-wrap: wrap; align-items: flex-start; justify-content: space-between; gap: 1rem; padding: 1rem 1.25rem; } .registration-row:hover { transform: none; } .registration-actions { display: flex; flex-wrap: wrap; align-items: center; gap: 0.5rem; } + +/* Data pipeline page (pages/DataPipeline.jsx) */ +.pipeline-runs { list-style: none; margin: 0; padding: 0; display: grid; gap: 0.6rem; font-size: 0.9rem; } +.pipeline-worklist { display: grid; gap: 0.75rem; margin-top: 1rem; } +.pipeline-page .btn-sm { padding: 0.25rem 0.75rem; font-size: 0.8rem; } diff --git a/src/pages/DataPipeline.jsx b/src/pages/DataPipeline.jsx new file mode 100644 index 0000000..2720643 --- /dev/null +++ b/src/pages/DataPipeline.jsx @@ -0,0 +1,213 @@ +/** + * DataPipeline.jsx — curators' read-only view of the data pipeline + * (validate-submission permission). + * + * What each source last loaded and what the app holds from it, projects where + * an app edit and a source change disagree, and the data-quality issues the + * pipeline found, each opening a worklist of the projects it affects. Loading + * data is done with the pipeline's command line (nccrd-build/PIPELINE.md). + */ +import { useEffect, useState } from 'react'; +import { Link } from 'react-router-dom'; +import { ArrowLeft } from 'lucide-react'; +import { getPipelineStatus, getPipelineIssueProjects } from '../services/api'; +import { useCurrentUser } from '../context/CurrentUserContext'; +import { sourceLabel } from '../utils/dataSources'; + +const PAGE_SIZE = 50; + +const FIELDS = { + 'geo_location.province': 'Province', + 'geo_location.coordinates': 'Coordinates', + estimated_budget_cost: 'Budget range', + implementation_status: 'Implementation status', + intervention_measurement: 'Type of measure', + funding_type: 'Funding type', +}; +const fieldLabel = (field) => FIELDS[field] || field.replace(/[._]/g, ' ').replace(/^\w/, (c) => c.toUpperCase()); + +const formatDate = (iso) => (iso ? new Date(iso).toLocaleString('en-ZA', { day: 'numeric', month: 'short', year: 'numeric', hour: '2-digit', minute: '2-digit' }) : ''); + +/** "update 309, excluded 877" from a run's summary (merge actions, or a clean-up's counts). */ +const runSummary = (summary) => { + const counts = summary?.actions || summary || {}; + return Object.entries(counts).filter(([, n]) => typeof n === 'number').map(([k, n]) => `${k.replace(/_/g, ' ')} ${n.toLocaleString()}`).join(', '); +}; + +const Worklist = ({ issue }) => { + const [projects, setProjects] = useState([]); + const [total, setTotal] = useState(null); + const [error, setError] = useState(null); + const [loading, setLoading] = useState(false); + + const load = (offset) => { + setLoading(true); + getPipelineIssueProjects({ data_source: issue.data_source, field: issue.field, issue: issue.issue, limit: PAGE_SIZE, offset }) + .then((data) => { + setProjects((prev) => (offset ? [...prev, ...data.projects] : data.projects)); + setTotal(data.total); + setLoading(false); + }) + .catch((err) => { setError(err.message || 'Could not load the projects.'); setLoading(false); }); + }; + useEffect(() => { load(0); }, [issue.data_source, issue.field, issue.issue]); // eslint-disable-line react-hooks/exhaustive-deps + + if (error) return

{error}

; + return ( +
+ {total === null ?

Loading…

: ( +
    + {projects.map((p) => ( +
  • + + + {p.title || 'Untitled project'} + + {[p.implementation_organization, p.public ? 'public' : 'not public', p.value && `found: ${p.value}`].filter(Boolean).join(' · ')} + + + +
    • + ))} +
+ )} + {total !== null && projects.length < total && ( + + )} +
+ ); +}; + +const DataPipeline = () => { + const { hasPermission, loading: authLoading } = useCurrentUser(); + const [status, setStatus] = useState(null); + const [error, setError] = useState(null); + const [open, setOpen] = useState(null); + const isCurator = hasPermission('validate-submission'); + + useEffect(() => { + if (!isCurator) return; + getPipelineStatus().then(setStatus).catch((err) => setError(err.message || 'Could not load the data pipeline status.')); + }, [isCurator]); + + if (authLoading) return null; + if (!isCurator) return ( +
+
+

Curators only

+

You need the reviewer permission (validate-submission) to see the data pipeline.

+
+
+ ); + + const issueKey = (i) => `${i.data_source}|${i.field}|${i.issue}`; + + return ( +
+ Back to the review queue +
+

Data pipeline

+

+ What the data pipeline has loaded into the app from the legacy NCCRD and the provincial registers, projects where an + edit in the app and a change in the source disagree, and the data-quality issues it found. Open an issue for the list of + projects to fix. +

+
+ + {error &&
{error}
} + {!status && !error &&

Loading…

} + {status && !status.available && ( +
+

The data pipeline hasn’t been run on this database yet.

+
+ )} + {status?.available && ( +
+
+

Sources

+

+ {status.loads.map((l) => `${l.source}: ${l.file_name}, loaded ${formatDate(l.loaded_at)}`).join(' · ')} +

+
+ + + + + + {status.sources.map((s) => ( + + + + + + + + ))} + +
SourceRows in the sourceKept outIn the appPublic
{sourceLabel(s.data_source)}{s.in_source.toLocaleString()}{s.excluded.toLocaleString()}{s.in_app.toLocaleString()}{s.public.toLocaleString()}
+
+
+ +
+

Recent runs

+ {status.runs.length === 0 ?

None yet.

: ( +
    + {status.runs.map((r) => ( +
  • {r.kind} · {formatDate(r.ran_at)}
    {runSummary(r.summary)}
    • + ))} +
+ )} +
+ +
+

Conflicts

+

Changed in the app and in the source. The app’s version is kept.

+ {status.conflicts.length === 0 ?

None.

: ( +
    + {status.conflicts.map((c) => ( +
  • + {c.title || 'Untitled project'} +
    {[c.reason, (c.fields || []).map(fieldLabel).join(', ')].filter(Boolean).join(' · ')} +
    • + ))} +
+ )} +
+ +
+

Data quality

+

Issues found while loading, counted over projects in the app.

+ {status.issues.length === 0 ?

No issues.

: ( +
+ + + + {status.issues.map((i) => ( + + + + + + + + ))} + +
SourceFieldIssueProjects
{sourceLabel(i.data_source)}{fieldLabel(i.field)}{i.issue}{i.projects.toLocaleString()} + +
+
+ )} + {open && issueKey(i) === open)} />} +
+
+ )} +
+ ); +}; + +export default DataPipeline; diff --git a/src/pages/DataPipeline.test.jsx b/src/pages/DataPipeline.test.jsx new file mode 100644 index 0000000..e43d783 --- /dev/null +++ b/src/pages/DataPipeline.test.jsx @@ -0,0 +1,57 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { render, screen, fireEvent } from '@testing-library/react'; +import { MemoryRouter } from 'react-router-dom'; +import { getPipelineStatus, getPipelineIssueProjects } from '../services/api'; +import DataPipeline from './DataPipeline'; + +const auth = { curator: true }; +vi.mock('../context/CurrentUserContext', () => ({ + useCurrentUser: () => ({ hasPermission: (p) => auth.curator && p === 'validate-submission', loading: false }), +})); +vi.mock('../services/api', () => ({ getPipelineStatus: vi.fn(), getPipelineIssueProjects: vi.fn() })); + +const status = { + available: true, + loads: [{ source: 'legacy', file_name: 'nccrd_stable.bak', loaded_at: '2026-10-06T09:03:00Z' }], + sources: [{ data_source: 'sqlserver_legacy', in_source: 2540, excluded: 865, in_app: 1435, public: 1051 }], + runs: [{ kind: 'gold', ran_at: '2026-10-06T09:05:00Z', summary: { actions: { update: 43, insert: 125 } } }], + conflicts: [{ id: 'c1', title: 'Solar clinic', reason: 'changed in the app since the last sync', fields: ['title'] }], + issues: [{ data_source: 'sqlserver_legacy', field: 'geo_location.province', issue: 'no province', projects: 380 }], +}; + +describe('DataPipeline', () => { + beforeEach(() => { + auth.curator = true; + getPipelineStatus.mockReset().mockResolvedValue(status); + getPipelineIssueProjects.mockReset().mockResolvedValue({ + total: 380, projects: [{ id: 'p1', title: 'Coastal plan', implementation_organization: 'City', public: true, value: null }], + }); + }); + + it('shows sources, runs, conflicts and issues, and opens a worklist', async () => { + render(); + expect(await screen.findAllByText('Legacy NCCRD')).toHaveLength(2); // sources and issues tables + expect(screen.getByText('2,540')).toBeInTheDocument(); + expect(screen.getByText(/update 43, insert 125/)).toBeInTheDocument(); + expect(screen.getByRole('link', { name: 'Solar clinic' })).toHaveAttribute('href', '/submission/c1'); + expect(screen.getByText('Province')).toBeInTheDocument(); + + fireEvent.click(screen.getByRole('button', { name: 'Show projects' })); + expect(await screen.findByRole('link', { name: /Coastal plan/ })).toHaveAttribute('href', '/submission/p1'); + expect(getPipelineIssueProjects).toHaveBeenCalledWith(expect.objectContaining({ field: 'geo_location.province', issue: 'no province', offset: 0 })); + expect(screen.getByRole('button', { name: /Show more \(379 remaining\)/ })).toBeInTheDocument(); + }); + + it('says so when the pipeline has never run', async () => { + getPipelineStatus.mockResolvedValue({ available: false }); + render(); + expect(await screen.findByText(/hasn’t been run on this database yet/)).toBeInTheDocument(); + }); + + it('turns away non-curators', () => { + auth.curator = false; + render(); + expect(screen.getByText('Curators only')).toBeInTheDocument(); + expect(getPipelineStatus).not.toHaveBeenCalled(); + }); +}); diff --git a/src/pages/Review.jsx b/src/pages/Review.jsx index 27e3a99..556cf6c 100644 --- a/src/pages/Review.jsx +++ b/src/pages/Review.jsx @@ -13,17 +13,11 @@ import { useCurrentUser } from '../context/CurrentUserContext'; import { REVIEW_STATES } from '../utils/reviewStatus'; import StatusBadge from '../components/StatusBadge'; import InterventionBadge from '../components/InterventionBadge'; +import { sourceLabel } from '../utils/dataSources'; const TABS = ['awaiting_review', 'not_accepted', 'draft', 'published']; const PAGE_SIZE = 50; -const SOURCES = { - sqlserver_legacy: 'Legacy NCCRD', - gauteng_register_2024: 'Gauteng register 2024', - wc_project_database_2020: 'Western Cape database 2020', - react_app: 'Submission form', -}; - const formatDate = (iso) => (iso ? new Date(iso).toLocaleDateString('en-ZA', { day: 'numeric', month: 'short', year: 'numeric' }) : ''); const Review = () => { @@ -78,6 +72,7 @@ const Review = () => {

Review submissions

Open a project to read it in full, then accept it to publish it, or mark it not accepted with a reason for the submitter. + {' '}Data pipeline: loaded sources and data-quality worklists →

@@ -116,7 +111,7 @@ const Review = () => { {s.title || 'Untitled project'} - {[s.implementation_organization, SOURCES[s.data_source] || s.data_source, s.createdate && `added ${formatDate(s.createdate)}`] + {[s.implementation_organization, sourceLabel(s.data_source), s.createdate && `added ${formatDate(s.createdate)}`] .filter(Boolean).join(' · ')} diff --git a/src/services/api.js b/src/services/api.js index 2f7ebef..c6a3c13 100644 --- a/src/services/api.js +++ b/src/services/api.js @@ -121,6 +121,19 @@ export const reviewSubmission = async (id, decision, comments) => { }; /** Submissions per review state, for the review queue. GET /submission/review/counts. */ +export const getPipelineStatus = async () => { + const res = await fetch(`${API_BASE_URL}/pipeline/status`, { headers: { ..._authHeaders() } }); + if (!res.ok) await _throwOnErrorWithAuthCheck(res); + return res.json(); +}; + +/** Projects behind one data-quality issue: { data_source, field, issue, limit, offset }. */ +export const getPipelineIssueProjects = async (params) => { + const res = await fetch(`${API_BASE_URL}/pipeline/issues?${new URLSearchParams(params)}`, { headers: { ..._authHeaders() } }); + if (!res.ok) await _throwOnErrorWithAuthCheck(res); + return res.json(); +}; + export const getReviewCounts = async () => { const res = await fetch(`${API_BASE_URL}/submission/review/counts`, { headers: { ..._authHeaders() } }); if (!res.ok) await _throwOnErrorWithAuthCheck(res); diff --git a/src/utils/dataSources.js b/src/utils/dataSources.js new file mode 100644 index 0000000..775ad8d --- /dev/null +++ b/src/utils/dataSources.js @@ -0,0 +1,10 @@ +// Readable names for nccrd.submission.data_source, shared by the review +// queue and the data pipeline page. +export const DATA_SOURCES = { + sqlserver_legacy: 'Legacy NCCRD', + gauteng_register_2024: 'Gauteng register 2024', + wc_project_database_2020: 'Western Cape database 2020', + react_app: 'Submission form', +}; + +export const sourceLabel = (dataSource) => DATA_SOURCES[dataSource] || dataSource;