diff --git a/docs/pages/users/how_to/import_export_dhis2_mappings.en.md b/docs/pages/users/how_to/import_export_dhis2_mappings.en.md
new file mode 100644
index 0000000000..374f74c0a1
--- /dev/null
+++ b/docs/pages/users/how_to/import_export_dhis2_mappings.en.md
@@ -0,0 +1,84 @@
+# Import and export DHIS2 mappings
+
+To export submissions to DHIS2, each question of a form is mapped to a DHIS2 data element (see the
+"DHIS2 mappings" section of the [user guide](../reference/user_guide.md)). This page explains how to
+reuse those mappings instead of redoing them by hand.
+
+## Which mapping type to choose
+
+The mapping type depends on how many submissions the form gets, and on what they must become in
+DHIS2:
+
+| Mapping type | Use it when | Each submission becomes, in DHIS2 |
+|---|---|---|
+| **Aggregate** | there is **one submission per org unit and period** (monthly report, quantity form...). The form needs a period type. | a data value set of a **dataset**, for the org unit and period of the submission; the dataset is marked as complete. A second submission for the same org unit and period overwrites the values of the first one. |
+| **Event** | there are **several submissions per org unit and period**, with no person to follow (consultations, visits, surveys...) | an event of an **event program without registration**. Its date is the creation date or the start of the period of the submission (**Event date source**). |
+| **Event Tracker** | the **same person (or object) is followed over time** | a tracked entity of a **tracker program** (found by its identifier, or created), with its attributes, an enrollment and an event per program stage. A repeat group can create related tracked entities, linked by a relationship. |
+
+The **Derived** type does not export to DHIS2: it computes a statistics form from another form, see
+[Deduce a form from another form](deduce_form_from_another_form.md).
+
+## A mapping per form version
+
+A mapping is attached to one **version** of a form, not to the form itself: questions can be added,
+removed or renamed from one version to the next.
+
+When you upload a new version of a form, IASO copies the mappings of the previous version:
+
+- mappings of questions still present in the new version are kept;
+- mappings of questions that no longer exist are dropped;
+- new questions are not mapped yet: map them, or mark them as never mapped.
+
+## Export the mappings of a version
+
+On the mapping page of a form version, click **Export mappings**. A `.json` file is downloaded with
+the question mappings and the DHIS2 dataset (aggregate mappings) or program (event and tracker
+mappings) they target. Questions marked as never mapped are not exported.
+
+Use it to keep a copy before a big change, or to copy the mappings to another server or account.
+
+## Import mappings
+
+On the mapping page of the form version to complete, click **Import mappings**. Nothing is changed
+until you confirm the last step.
+
+### 1. Pick the mappings to reuse
+
+Choose either:
+
+- another **version of the same form** that uses the same DHIS2 data source, or
+- a **mapping export** (`.json`), for example made on another account or another server.
+
+A file is refused when it is not a mapping export, or when it was made for another mapping type
+(e.g. a tracker export into an aggregate mapping). When it was made for another DHIS2 dataset or
+program, a warning asks you to check that its data elements belong to this one.
+
+### 2. Review the differences
+
+The incoming mappings are compared with the current ones, question by question, in four tabs:
+
+| Tab | What it contains | Default decision |
+|---|---|---|
+| **Conflicts** | the question is already mapped (or marked as never mapped) to something else | **Keep** the current mapping |
+| **To add** | the question is not mapped yet in this version | **Add** the incoming mapping |
+| **Identical** | same DHIS2 target on both sides, nothing to resolve | no change |
+| **Dropped** | cannot be imported: the question does not exist in this version, or the mapping is not valid for this mapping type | not imported |
+
+For each conflict, choose **Keep** or **Overwrite**; for each addition, **Add** or **Skip**. The
+buttons above the table decide for all the rows of the tab at once (**Keep current everywhere**,
+**Overwrite everywhere**, **Add all**, **Skip all**).
+
+### 3. Apply
+
+Click **Apply** (the button shows the number of changes). A message sums up how many mappings were
+added, overwritten, kept and not imported.
+
+!!! warning
+ The DHIS2 ids of the imported mappings are not checked against DHIS2. If a data element does not
+ exist in the target dataset or program, the error only shows up when exporting submissions.
+
+## Go back to a previous mapping
+
+Every change of a mapping version, including an import, is recorded. On the mapping page, the
+**Link to changes log** shows the history, with the mappings before and after each change: use it
+to find a mapping that was overwritten by mistake.
diff --git a/docs/pages/users/how_to/import_export_dhis2_mappings.fr.md b/docs/pages/users/how_to/import_export_dhis2_mappings.fr.md
new file mode 100644
index 0000000000..69a1974587
--- /dev/null
+++ b/docs/pages/users/how_to/import_export_dhis2_mappings.fr.md
@@ -0,0 +1,90 @@
+# Importer et exporter des mappings DHIS2
+
+Pour exporter des soumissions vers DHIS2, chaque question d'un formulaire est mappée vers un data
+element DHIS2 (voir la section « Correspondances avec DHIS2 » du [guide utilisateur](../reference/user_guide.md)).
+Cette page explique comment réutiliser ces mappings au lieu de les refaire à la main.
+
+## Quel type de mapping choisir
+
+Le type de mapping dépend du nombre de soumissions du formulaire, et de ce qu'elles doivent devenir
+dans DHIS2 :
+
+| Type de mapping | À utiliser quand | Chaque soumission devient, dans DHIS2 |
+|---|---|---|
+| **Aggregate** | il y a **une soumission par unité d'organisation et par période** (rapport mensuel, formulaire de quantités...). Le formulaire doit avoir un type de période. | un ensemble de valeurs d'un **dataset**, pour l'unité d'organisation et la période de la soumission ; le dataset est marqué comme complet. Une deuxième soumission pour la même unité d'organisation et la même période écrase les valeurs de la première. |
+| **Event** | il y a **plusieurs soumissions par unité d'organisation et par période**, sans personne à suivre (consultations, visites, enquêtes...) | un évènement d'un **programme d'évènements sans enregistrement** (event program without registration). Sa date est la date de création ou le début de la période de la soumission (**Source pour la date d'évènement**). |
+| **Event Tracker** | la **même personne (ou le même objet) est suivie dans le temps** | une entité suivie (tracked entity) d'un **programme tracker** (retrouvée par son identifiant, ou créée), avec ses attributs, une inscription (enrollment) et un évènement par étape du programme. Un groupe répété peut créer des entités suivies liées, reliées par une relation. |
+
+Le type **Derived** n'exporte pas vers DHIS2 : il calcule un formulaire de statistiques à partir
+d'un autre formulaire, voir [Déduire un formulaire à partir d'un autre formulaire](deduce_form_from_another_form.md).
+
+## Un mapping par version de formulaire
+
+Un mapping est attaché à une **version** d'un formulaire, pas au formulaire lui-même : des questions
+peuvent être ajoutées, supprimées ou renommées d'une version à l'autre.
+
+Quand vous chargez une nouvelle version d'un formulaire, IASO copie les mappings de la version
+précédente :
+
+- les mappings des questions encore présentes dans la nouvelle version sont gardés ;
+- les mappings des questions qui n'existent plus sont supprimés ;
+- les nouvelles questions ne sont pas encore mappées : mappez-les, ou marquez-les comme jamais mappées.
+
+## Exporter les mappings d'une version
+
+Sur la page de mapping d'une version de formulaire, cliquez sur **Exporter les mappings**. Un
+fichier `.json` est téléchargé avec les mappings des questions et le dataset DHIS2 (mappings
+agrégés) ou le programme (mappings event et tracker) qu'ils visent. Les questions marquées comme
+jamais mappées ne sont pas exportées.
+
+Utilisez-le pour garder une copie avant un gros changement, ou pour copier les mappings vers un
+autre serveur ou un autre compte.
+
+## Importer des mappings
+
+Sur la page de mapping de la version de formulaire à compléter, cliquez sur **Importer des
+mappings**. Rien n'est modifié avant la confirmation de la dernière étape.
+
+### 1. Choisir les mappings à réutiliser
+
+Choisissez :
+
+- une autre **version du même formulaire** qui utilise la même source de données DHIS2, ou
+- un **export de mapping** (`.json`), fait par exemple sur un autre compte ou un autre serveur.
+
+Un fichier est refusé s'il n'est pas un export de mapping, ou s'il a été fait pour un autre type de
+mapping (par exemple un export tracker dans un mapping agrégé). S'il a été fait pour un autre
+dataset ou programme DHIS2, un avertissement vous demande de vérifier que ses data elements font
+partie de celui-ci.
+
+### 2. Vérifier les différences
+
+Les mappings importés sont comparés aux mappings actuels, question par question, dans quatre
+onglets :
+
+| Onglet | Contenu | Décision par défaut |
+|---|---|---|
+| **Conflits** | la question est déjà mappée (ou marquée comme jamais mappée) vers autre chose | **Garder** le mapping actuel |
+| **À ajouter** | la question n'est pas encore mappée dans cette version | **Ajouter** le mapping importé |
+| **Identiques** | même cible DHIS2 des deux côtés, rien à résoudre | aucun changement |
+| **Ignorés** | ne peuvent pas être importés : la question n'existe pas dans cette version, ou le mapping n'est pas valide pour ce type de mapping | non importé |
+
+Pour chaque conflit, choisissez **Garder** ou **Écraser** ; pour chaque ajout, **Ajouter** ou
+**Ignorer**. Les boutons au-dessus du tableau décident pour toutes les lignes de l'onglet à la fois
+(**Garder l'actuel partout**, **Écraser partout**, **Tout ajouter**, **Tout ignorer**).
+
+### 3. Appliquer
+
+Cliquez sur **Appliquer** (le bouton indique le nombre de changements). Un message résume combien
+de mappings ont été ajoutés, écrasés, gardés et non importés.
+
+!!! warning "Attention"
+ Les identifiants DHIS2 des mappings importés ne sont pas vérifiés auprès de DHIS2. Si un data
+ element n'existe pas dans le dataset ou le programme visé, l'erreur n'apparaît qu'à l'export des
+ soumissions.
+
+## Revenir à un mapping précédent
+
+Chaque modification d'une version de mapping, import compris, est enregistrée. Sur la page de
+mapping, le **Lien vers le Journal des modifications** affiche l'historique, avec les mappings
+avant et après chaque modification : utilisez-le pour retrouver un mapping écrasé par erreur.
diff --git a/docs/pages/users/reference/user_guide.en.md b/docs/pages/users/reference/user_guide.en.md
index ed653f3092..371aa0544f 100644
--- a/docs/pages/users/reference/user_guide.en.md
+++ b/docs/pages/users/reference/user_guide.en.md
@@ -309,13 +309,13 @@ In the Form view you can see details of:
- Versions
-- Type of the form :
+- Type of the mapping (see [which mapping type to choose](../how_to/import_export_dhis2_mappings.md#which-mapping-type-to-choose)):
- - Aggregate : fix
+ - Aggregate : one submission per org unit and period, exported to a DHIS2 dataset
- - Event : series of singular events
+ - Event : several submissions per org unit and period, exported to an event program without registration
- - Event Tracker : continuous
+ - Event Tracker : the same person followed over time, exported to a tracker program
- Number of questionnaire to be mapped
@@ -342,6 +342,9 @@ box by using the name, code or ID and then confirm.
Once confirmed, the question will turn to green and be counted.
+To reuse the mappings of another form version or of an export file, see
+[Import and export DHIS2 mappings](../how_to/import_export_dhis2_mappings.md).
+
;
+ setDecision: (questionKey: string, decision: Decision) => void;
+ setBulkDecision: (kind: DiffKind, decision: Decision) => void;
+ plan: ImportPlan;
+};
+
+const BUCKET_MESSAGES: Record = {
+ [DiffKind.CONFLICT]: MESSAGES.bucketConflict,
+ [DiffKind.ADD]: MESSAGES.bucketAdd,
+ [DiffKind.IDENTICAL]: MESSAGES.bucketIdentical,
+ [DiffKind.DROPPED]: MESSAGES.bucketDropped,
+};
+
+type Choice = { decision: Decision; message: IntlMessage };
+
+const CHOICES: Partial> = {
+ [DiffKind.CONFLICT]: [
+ { decision: Decision.KEEP, message: MESSAGES.keep },
+ { decision: Decision.OVERWRITE, message: MESSAGES.overwrite },
+ ],
+ [DiffKind.ADD]: [
+ { decision: Decision.SKIP, message: MESSAGES.skip },
+ { decision: Decision.APPLY, message: MESSAGES.add },
+ ],
+};
+
+const BULK: Record = {
+ [DiffKind.CONFLICT]: {
+ message: MESSAGES.resolveAllConflicts,
+ actions: [
+ { decision: Decision.KEEP, message: MESSAGES.keepEverywhere },
+ {
+ decision: Decision.OVERWRITE,
+ message: MESSAGES.overwriteEverywhere,
+ },
+ ],
+ },
+ [DiffKind.ADD]: {
+ message: MESSAGES.allAdditions,
+ actions: [
+ { decision: Decision.APPLY, message: MESSAGES.addAll },
+ { decision: Decision.SKIP, message: MESSAGES.skipAll },
+ ],
+ },
+ [DiffKind.IDENTICAL]: { message: MESSAGES.identicalHint, actions: [] },
+ [DiffKind.DROPPED]: { message: MESSAGES.droppedHint, actions: [] },
+};
+
+const styles: SxStyles = {
+ tabs: { borderBottom: 1, borderColor: 'divider', mt: 1 },
+ bulkRow: { display: 'flex', alignItems: 'center', gap: 1, minHeight: 44 },
+ table: { maxHeight: '40vh' },
+ // fixed layout: long question names or mappings must not widen their column
+ tableLayout: { tableLayout: 'fixed' },
+ questionColumn: { width: 240 },
+ questionCell: { overflow: 'hidden' },
+ mappingCell: { overflowWrap: 'anywhere' },
+ decisionColumn: { width: 210 },
+ current: { color: 'text.primary', overflowWrap: 'anywhere' },
+ noCurrent: { color: 'text.disabled', overflowWrap: 'anywhere' },
+ choices: { display: 'flex', gap: 0.75, justifyContent: 'flex-end' },
+ summary: { mt: 1.5 },
+};
+
+export const CompareStep: FunctionComponent = ({
+ sourceTitle,
+ versionId,
+ mappingType,
+ rows,
+ bucket,
+ setBucket,
+ decisions,
+ setDecision,
+ setBulkDecision,
+ plan,
+}) => {
+ const { formatMessage } = useSafeIntl();
+ const counts = useMemo(() => {
+ const result = Object.fromEntries(
+ DIFF_KINDS.map(kind => [kind, 0]),
+ ) as Record;
+ rows.forEach(row => {
+ result[row.kind] += 1;
+ });
+ return result;
+ }, [rows]);
+ // empty buckets are hidden: only offer the tabs with something to show
+ const visibleKinds = DIFF_KINDS.filter(kind => counts[kind] > 0);
+ const bucketRows = useMemo(
+ () => rows.filter(row => row.kind === bucket),
+ [rows, bucket],
+ );
+ const bulk = BULK[bucket];
+
+ const getCurrentLabel = (row: DiffRow): string => {
+ if (isNeverMapped(row.current)) {
+ return formatMessage(MESSAGES.markedNeverMapped);
+ }
+ if (row.kind !== DiffKind.DROPPED) {
+ return getMappingLabel(row.current) || textPlaceholder;
+ }
+ return row.invalid
+ ? formatMessage(MESSAGES.importInvalidMapping, {
+ type: mappingType,
+ })
+ : formatMessage(MESSAGES.questionAbsent, { versionId });
+ };
+
+ return (
+
+
+ {formatMessage(MESSAGES.compareLabel, {
+ source: sourceTitle,
+ versionId,
+ })}
+
+ setBucket(value)}
+ indicatorColor="primary"
+ textColor="primary"
+ sx={styles.tabs}
+ >
+ {visibleKinds.map(kind => (
+
+
+
+ {formatMessage(bulk.message)}
+
+ {bulk.actions.map(action => (
+ setBulkDecision(bucket, action.decision)}
+ >
+ {formatMessage(action.message)}
+
+ ))}
+
+
+
+
+
+
+ {formatMessage(MESSAGES.question)}
+
+
+ {formatMessage(
+ bucket === DiffKind.DROPPED
+ ? MESSAGES.question
+ : MESSAGES.currentMapping,
+ )}
+
+
+ {formatMessage(MESSAGES.incomingMapping)}
+
+
+ {formatMessage(MESSAGES.decision)}
+
+
+
+
+ {bucketRows.map(row => {
+ const decision = decisions[row.questionKey];
+ const choices = CHOICES[row.kind];
+ return (
+
+
+ {/* dropped questions have no label: the name alone is the title */}
+
+ {row.questionLabel ??
+ row.questionKey}
+
+ {row.questionLabel &&
+ row.questionLabel !==
+ row.questionKey && (
+
+ {row.questionKey}
+
+ )}
+
+
+ {getCurrentLabel(row)}
+
+
+ {getMappingLabel(row.incoming)}
+
+
+ {choices ? (
+
+ {choices.map(choice => (
+
+ setDecision(
+ row.questionKey,
+ choice.decision,
+ )
+ }
+ >
+ {formatMessage(
+ choice.message,
+ )}
+
+ ))}
+
+ ) : (
+
+ {formatMessage(
+ row.kind ===
+ DiffKind.IDENTICAL
+ ? MESSAGES.noChange
+ : MESSAGES.notImportable,
+ )}
+
+ )}
+
+
+ );
+ })}
+
+
+
+
+ {formatMessage(MESSAGES.importSummary, {
+ added: plan.added,
+ overwritten: plan.overwritten,
+ ignored: plan.dropped + plan.skipped,
+ })}
+
+
+ );
+};
diff --git a/hat/assets/js/apps/Iaso/domains/mappings/components/ImportMappings/ImportMappingsDialog.test.tsx b/hat/assets/js/apps/Iaso/domains/mappings/components/ImportMappings/ImportMappingsDialog.test.tsx
new file mode 100644
index 0000000000..320a5593dc
--- /dev/null
+++ b/hat/assets/js/apps/Iaso/domains/mappings/components/ImportMappings/ImportMappingsDialog.test.tsx
@@ -0,0 +1,203 @@
+import React from 'react';
+import { fireEvent, screen, waitFor } from '@testing-library/react';
+import { describe, expect, it, vi } from 'vitest';
+import { renderWithThemeAndIntlProvider } from '../../../../../../tests/helpers';
+import { getMappableQuestions } from '../../importMappings';
+import { ImportMappingsDialog } from './ImportMappingsDialog';
+
+const de = (id: string) => ({
+ id,
+ name: `DE ${id}`,
+ valueType: 'NUMBER',
+ categoryOptionCombo: 'coc',
+ categoryOptionComboName: 'default',
+});
+
+const form = { id: 5, name: 'Form' };
+const mapping = {
+ mapping_type: 'AGGREGATE',
+ data_source: { id: 3, name: 'dhis2' },
+};
+
+const mappingVersion = {
+ id: 1,
+ form_version: { id: 10, version_id: '2021111101', form },
+ mapping,
+ question_mappings: { q1: de('a'), q2: de('b') },
+ derivate_settings: { data_set_id: 'ds1' },
+};
+
+const sources = [
+ mappingVersion,
+ {
+ id: 2,
+ form_version: { id: 9, version_id: '2020100801', form },
+ mapping,
+ question_mappings: {
+ q1: de('a'),
+ q2: de('c'),
+ q3: de('d'),
+ gone: de('e'),
+ },
+ derivate_settings: { data_set_id: 'ds1' },
+ updated_at: 1600000000,
+ },
+ {
+ // other data source: never offered
+ id: 3,
+ form_version: { id: 8, version_id: '2019', form },
+ mapping: { ...mapping, data_source: { id: 4, name: 'other' } },
+ question_mappings: { q3: de('z') },
+ derivate_settings: { data_set_id: 'ds1' },
+ },
+];
+
+const { useGetMappingImportSources } = vi.hoisted(() => ({
+ useGetMappingImportSources: vi.fn(),
+}));
+vi.mock('../../hooks/requests/useGetMappingImportSources', () => ({
+ useGetMappingImportSources,
+}));
+useGetMappingImportSources.mockImplementation(() => ({
+ data: sources,
+ isLoading: false,
+}));
+
+const questions = getMappableQuestions({
+ name: 'survey',
+ type: 'survey',
+ children: ['q1', 'q2', 'q3'].map(name => ({
+ name,
+ type: 'integer',
+ label: `Label ${name}`,
+ })),
+});
+
+describe('ImportMappingsDialog', () => {
+ it('compares a previous version and applies the decisions', async () => {
+ const onApply = vi.fn().mockResolvedValue(undefined);
+ const closeDialog = vi.fn();
+ renderWithThemeAndIntlProvider(
+ ;
+ onApply: (plan: ImportPlan) => Promise;
+};
+
+const FILE_SOURCE_ID = 'file';
+
+const IMPORT_ERROR_MESSAGES: Record = {
+ [MappingImportErrorReason.INVALID_JSON]: MESSAGES.importInvalidJson,
+ [MappingImportErrorReason.INVALID_FORMAT]: MESSAGES.importInvalidFormat,
+ [MappingImportErrorReason.MAPPING_TYPE_MISMATCH]:
+ MESSAGES.importMappingTypeMismatch,
+ [MappingImportErrorReason.NO_VALID_MAPPING]: MESSAGES.importNoValidMapping,
+};
+
+const OTHER_TARGET_MESSAGES = {
+ dataset: MESSAGES.importOtherDataset,
+ program: MESSAGES.importOtherProgram,
+};
+
+const styles: SxStyles = {
+ content: { position: 'relative', minHeight: 200 },
+};
+
+export const ImportMappingsDialog: FunctionComponent = ({
+ open,
+ closeDialog,
+ mappingVersion,
+ questions,
+ onApply,
+}) => {
+ const { formatMessage } = useSafeIntl();
+ const [step, setStep] = useState<1 | 2>(1);
+ const [selectedId, setSelectedId] = useState();
+ const [fileSource, setFileSource] = useState();
+ const [fileError, setFileError] = useState();
+ const [fileWarning, setFileWarning] = useState();
+ const [bucket, setBucket] = useState(DiffKind.CONFLICT);
+ const [decisions, setDecisions] = useState<
+ Record
+ >({});
+ const [isApplying, setIsApplying] = useState(false);
+
+ const mappingType = mappingVersion.mapping.mapping_type;
+ const { sources: versionSources, isLoading } = useImportSources(
+ mappingVersion,
+ questions,
+ open,
+ );
+ const sources = fileSource
+ ? [fileSource, ...versionSources]
+ : versionSources;
+ const selectedSource = sources.find(s => s.id === selectedId) ?? sources[0];
+
+ const rows = useMemo(
+ () =>
+ selectedSource && step === 2
+ ? computeMappingsDiff(
+ mappingVersion.question_mappings,
+ selectedSource.questionMappings,
+ questions,
+ mappingType,
+ )
+ : [],
+ [
+ selectedSource,
+ step,
+ mappingVersion.question_mappings,
+ questions,
+ mappingType,
+ ],
+ );
+ const plan = useMemo(
+ () => buildImportPlan(rows, decisions),
+ [rows, decisions],
+ );
+ const changesCount = Object.keys(plan.changes).length;
+
+ const reset = () => {
+ setStep(1);
+ setSelectedId(undefined);
+ setFileSource(undefined);
+ setFileError(undefined);
+ setFileWarning(undefined);
+ setDecisions({});
+ };
+ const handleClose = () => {
+ reset();
+ closeDialog();
+ };
+
+ const onFileChosen = async (file: File) => {
+ setFileError(undefined);
+ setFileWarning(undefined);
+ try {
+ const content = parseMappingExport(await file.text(), mappingType);
+ const otherTarget = getOtherTarget(content, mappingVersion);
+ if (otherTarget) {
+ setFileWarning(
+ formatMessage(OTHER_TARGET_MESSAGES[otherTarget.kind], {
+ name: otherTarget.name,
+ }),
+ );
+ }
+ const questionMappings = getImportableMappings(
+ content.question_mappings,
+ );
+ setFileSource({
+ id: FILE_SOURCE_ID,
+ title: file.name,
+ meta: content.form
+ ? formatMessage(MESSAGES.importFileMeta, {
+ formName: content.form.name,
+ versionId: content.form_version?.version_id,
+ })
+ : '',
+ mappingsCount: Object.keys(questionMappings).length,
+ matchingCount: countMatchingMappings(
+ questionMappings,
+ questions,
+ mappingType,
+ ),
+ questionMappings,
+ });
+ setSelectedId(FILE_SOURCE_ID);
+ } catch (e) {
+ if (e instanceof MappingImportError) {
+ setFileError(
+ formatMessage(IMPORT_ERROR_MESSAGES[e.reason], {
+ type: mappingType,
+ }),
+ );
+ } else {
+ throw e;
+ }
+ }
+ };
+
+ const goToCompare = () => {
+ const diff = computeMappingsDiff(
+ mappingVersion.question_mappings,
+ selectedSource.questionMappings,
+ questions,
+ mappingType,
+ );
+ setDecisions(
+ Object.fromEntries(
+ diff.map(row => [row.questionKey, getDefaultDecision(row)]),
+ ),
+ );
+ // empty tabs are hidden: open on the first one with rows
+ const firstBucket = DIFF_KINDS.find(kind =>
+ diff.some(row => row.kind === kind),
+ );
+ setBucket(firstBucket ?? DiffKind.CONFLICT);
+ setStep(2);
+ };
+
+ const apply = async () => {
+ setIsApplying(true);
+ try {
+ await onApply(plan);
+ handleClose();
+ } finally {
+ setIsApplying(false);
+ }
+ };
+
+ const setDecision = (questionKey: string, decision: Decision) =>
+ setDecisions(current => ({ ...current, [questionKey]: decision }));
+ const setBulkDecision = (kind: DiffKind, decision: Decision) =>
+ setDecisions(current => {
+ const next = { ...current };
+ rows.filter(row => row.kind === kind).forEach(row => {
+ next[row.questionKey] = decision;
+ });
+ return next;
+ });
+
+ return (
+
+ {formatMessage(MESSAGES.importMappings)}
+
+ {step === 1 && (
+
+
+ setStep(1)}
+ color="primary"
+ data-test="cancel-button"
+ >
+ {formatMessage(
+ step === 1 ? MESSAGES.cancel : MESSAGES.back,
+ )}
+
+ {step === 1 ? (
+
+ {formatMessage(MESSAGES.compare)}
+
+ ) : (
+
+ {formatMessage(MESSAGES.applyChanges, {
+ count: changesCount,
+ })}
+
+ )}
+
+
+ );
+};
diff --git a/hat/assets/js/apps/Iaso/domains/mappings/components/ImportMappings/MappingImportActions.tsx b/hat/assets/js/apps/Iaso/domains/mappings/components/ImportMappings/MappingImportActions.tsx
new file mode 100644
index 0000000000..151b9cb220
--- /dev/null
+++ b/hat/assets/js/apps/Iaso/domains/mappings/components/ImportMappings/MappingImportActions.tsx
@@ -0,0 +1,94 @@
+import React, { FunctionComponent, useState } from 'react';
+import DownloadIcon from '@mui/icons-material/Download';
+import InputIcon from '@mui/icons-material/Input';
+import { Box, Button } from '@mui/material';
+import { useSafeIntl } from 'bluesquare-components';
+import { openSnackBar } from '../../../../components/snackBars/EventDispatcher';
+import { succesfullSnackBar } from '../../../../constants/snackBars';
+import { SxStyles } from '../../../../types/general';
+import { useBulkUpdateQuestionMappings } from '../../hooks/requests/useBulkUpdateQuestionMappings';
+import { buildMappingExport, getExportFileName } from '../../importMappings';
+import MESSAGES from '../../messages';
+import { ImportPlan, MappingVersionRow } from '../../types';
+import { ImportMappingsDialog } from './ImportMappingsDialog';
+
+type Props = {
+ mappingVersion: MappingVersionRow;
+ questions: Record;
+};
+
+const styles: SxStyles = {
+ actions: { display: 'flex', justifyContent: 'flex-end', gap: 1 },
+};
+
+const downloadJson = (content: unknown, fileName: string) => {
+ const blob = new Blob([JSON.stringify(content, null, 2)], {
+ type: 'application/json',
+ });
+ const url = URL.createObjectURL(blob);
+ const link = document.createElement('a');
+ link.href = url;
+ link.download = fileName;
+ link.click();
+ URL.revokeObjectURL(url);
+};
+
+export const MappingImportActions: FunctionComponent = ({
+ mappingVersion,
+ questions,
+}) => {
+ const { formatMessage } = useSafeIntl();
+ const [open, setOpen] = useState(false);
+ const { mutateAsync: bulkUpdate } = useBulkUpdateQuestionMappings();
+
+ const onApply = async (plan: ImportPlan) => {
+ await bulkUpdate({
+ mappingVersionId: mappingVersion.id,
+ questionMappings: plan.changes,
+ });
+ openSnackBar(
+ succesfullSnackBar(
+ undefined,
+ formatMessage(MESSAGES.importDone, {
+ added: plan.added,
+ overwritten: plan.overwritten,
+ kept: plan.kept,
+ ignored: plan.dropped + plan.skipped,
+ }),
+ ),
+ );
+ };
+
+ return (
+
+ setOpen(false)}
+ mappingVersion={mappingVersion}
+ questions={questions}
+ onApply={onApply}
+ />
+
+ );
+};
diff --git a/hat/assets/js/apps/Iaso/domains/mappings/components/ImportMappings/SourceCard.tsx b/hat/assets/js/apps/Iaso/domains/mappings/components/ImportMappings/SourceCard.tsx
new file mode 100644
index 0000000000..49a81e0390
--- /dev/null
+++ b/hat/assets/js/apps/Iaso/domains/mappings/components/ImportMappings/SourceCard.tsx
@@ -0,0 +1,90 @@
+import React, { FunctionComponent, KeyboardEvent } from 'react';
+import RadioButtonCheckedIcon from '@mui/icons-material/RadioButtonChecked';
+import RadioButtonUncheckedIcon from '@mui/icons-material/RadioButtonUnchecked';
+import { Box, Chip, Typography } from '@mui/material';
+import { useSafeIntl } from 'bluesquare-components';
+import { SxStyles } from '../../../../types/general';
+import MESSAGES from '../../messages';
+import { ImportSource } from '../../types';
+
+type Props = {
+ source: ImportSource;
+ selected: boolean;
+ onSelect: () => void;
+};
+
+const card = {
+ display: 'flex',
+ alignItems: 'center',
+ gap: 1.5,
+ p: 1.5,
+ mb: 1,
+ cursor: 'pointer',
+ border: 1,
+ borderRadius: 1,
+ borderColor: 'divider',
+ backgroundColor: 'transparent',
+};
+
+const styles: SxStyles = {
+ card,
+ selectedCard: {
+ ...card,
+ borderColor: 'primary.main',
+ backgroundColor: 'action.hover',
+ },
+ text: { flex: '1 1 auto', minWidth: 0 },
+};
+
+export const SourceCard: FunctionComponent = ({
+ source,
+ selected,
+ onSelect,
+}) => {
+ const { formatMessage } = useSafeIntl();
+ const RadioIcon = selected
+ ? RadioButtonCheckedIcon
+ : RadioButtonUncheckedIcon;
+ const isGoodMatch =
+ source.mappingsCount > 0 &&
+ source.matchingCount / source.mappingsCount >= 0.5;
+ const handleKeyDown = (event: KeyboardEvent) => {
+ if (event.key === 'Enter' || event.key === ' ') {
+ event.preventDefault();
+ onSelect();
+ }
+ };
+ return (
+
+
+ {source.title}
+
+ {source.meta}
+
+
+
+ {formatMessage(MESSAGES.importMappingsCount, {
+ count: source.mappingsCount,
+ })}
+
+
+ );
+};
diff --git a/hat/assets/js/apps/Iaso/domains/mappings/components/ImportMappings/SourceStep.tsx b/hat/assets/js/apps/Iaso/domains/mappings/components/ImportMappings/SourceStep.tsx
new file mode 100644
index 0000000000..0840fc6259
--- /dev/null
+++ b/hat/assets/js/apps/Iaso/domains/mappings/components/ImportMappings/SourceStep.tsx
@@ -0,0 +1,104 @@
+import React, { ChangeEvent, FunctionComponent, useRef } from 'react';
+import UploadFileIcon from '@mui/icons-material/UploadFile';
+import { Alert, Box, Button, Divider, Typography } from '@mui/material';
+import { LoadingSpinner, useSafeIntl } from 'bluesquare-components';
+import { SxStyles } from '../../../../types/general';
+import MESSAGES from '../../messages';
+import { ImportSource } from '../../types';
+import { SourceCard } from './SourceCard';
+
+type Props = {
+ sources: ImportSource[];
+ selectedId?: string;
+ onSelect: (id: string) => void;
+ onFileChosen: (file: File) => void;
+ fileError?: string;
+ // the file targets another DHIS2 dataset or program
+ fileWarning?: string;
+ isLoading: boolean;
+};
+
+const styles: SxStyles = {
+ divider: { mt: 2 },
+ fileRow: { display: 'flex', alignItems: 'center', gap: 1.5, pt: 2 },
+ fileText: { flex: '1 1 auto' },
+ fileAlert: { mt: 1.5 },
+};
+
+export const SourceStep: FunctionComponent = ({
+ sources,
+ selectedId,
+ onSelect,
+ onFileChosen,
+ fileError,
+ fileWarning,
+ isLoading,
+}) => {
+ const { formatMessage } = useSafeIntl();
+ const fileInput = useRef(null);
+ const onFileChange = (event: ChangeEvent) => {
+ const file = event.target.files?.[0];
+ if (file) {
+ onFileChosen(file);
+ }
+ // allow choosing the same file again
+ event.target.value = '';
+ };
+ return (
+
+
+ {formatMessage(MESSAGES.importPickSource)}
+
+ {isLoading &&
+ {formatMessage(MESSAGES.importNoSource)}
+
+ )}
+ {sources.map(source => (
+ onSelect(source.id)}
+ />
+ ))}
+
+
+ {formatMessage(MESSAGES.importFromFile)}
+
+
+ {fileError && (
+
+ {fileError}
+
+ )}
+ {fileWarning && (
+
+ {fileWarning}
+
+ )}
+
+ );
+};
diff --git a/hat/assets/js/apps/Iaso/domains/mappings/details.tsx b/hat/assets/js/apps/Iaso/domains/mappings/details.tsx
index 17fcf00605..6af7961402 100644
--- a/hat/assets/js/apps/Iaso/domains/mappings/details.tsx
+++ b/hat/assets/js/apps/Iaso/domains/mappings/details.tsx
@@ -1,9 +1,10 @@
-import React, { useState } from 'react';
-import { Grid, Box } from '@mui/material';
-import { makeStyles } from '@mui/styles';
+import React, { useMemo, useState } from 'react';
+import { History } from '@mui/icons-material';
+import { Grid, Box, Typography, useTheme } from '@mui/material';
import {
commonStyles,
+ ExternalLink,
LoadingSpinner,
useRedirectToReplace,
useSafeIntl,
@@ -12,8 +13,10 @@ import {
import TopBar from '../../components/nav/TopBarComponent';
import { baseUrls } from '../../constants/urls';
import { useParamsObject } from '../../routing/hooks/useParamsObject';
+import { SxStyles } from '../../types/general';
import DerivedQuestionMappingForm from './components/DerivedQuestionMappingForm';
import GeneraMappingInfo from './components/GeneraMappingInfo';
+import { MappingImportActions } from './components/ImportMappings/MappingImportActions';
import QuestionInfos from './components/QuestionInfos';
import QuestionMappingForm from './components/QuestionMappingForm';
import RecursiveTreeView from './components/RecursiveTreeView';
@@ -23,19 +26,25 @@ import {
useApplyUpdate,
useGetMappingVersionDetail,
} from './hooks';
+import { getMappableQuestions } from './importMappings';
import MESSAGES from './messages';
const baseUrl = baseUrls.mappingDetail;
-const useStyles = makeStyles(theme => ({
- ...commonStyles(theme),
- icon: {
- width: 30,
- height: 'auto',
- display: 'block',
- cursor: 'pointer',
+const styles: SxStyles = {
+ actions: {
+ display: 'flex',
+ justifyContent: 'flex-end',
+ alignItems: 'center',
+ gap: 2,
+ mb: 2,
},
-}));
+ linkWithIcon: {
+ display: 'flex',
+ alignItems: 'center',
+ gap: '0.5em',
+ },
+};
const iasoFieldOptions = formatMessage => [
{ value: undefined, label: formatMessage(MESSAGES.useValueFromForm) },
@@ -55,7 +64,7 @@ const fieldTypeOptions = formatMessage => [
const MappingDetails = () => {
const redirectToReplace = useRedirectToReplace();
- const classes = useStyles();
+ const theme = useTheme();
const { formatMessage } = useSafeIntl();
const params = useParamsObject(baseUrl);
const currentMappingVersionQuery = useGetMappingVersionDetail(
@@ -73,6 +82,11 @@ const MappingDetails = () => {
? Descriptor.indexQuestions(currentFormVersion.descriptor)
: {};
+ const mappableQuestions = useMemo(
+ () => getMappableQuestions(currentFormVersion?.descriptor),
+ [currentFormVersion],
+ );
+
const [currentQuestion, setCurrentQuestion] = useState(
indexedQuestions[params.questionName],
);
@@ -137,7 +151,7 @@ const MappingDetails = () => {
currentMappingVersion.mapping.mapping_type !== 'DERIVED';
return (
-
+
{
{fetching &&
+
+
+
+
+
+
+ {currentFormVersion && isDataElementMappable && (
+
{currentFormVersion && currentMappingVersion && (
@@ -227,7 +257,7 @@ const MappingDetails = () => {
)}
-
+
);
};
diff --git a/hat/assets/js/apps/Iaso/domains/mappings/hooks/requests/useBulkUpdateQuestionMappings.ts b/hat/assets/js/apps/Iaso/domains/mappings/hooks/requests/useBulkUpdateQuestionMappings.ts
new file mode 100644
index 0000000000..833689edef
--- /dev/null
+++ b/hat/assets/js/apps/Iaso/domains/mappings/hooks/requests/useBulkUpdateQuestionMappings.ts
@@ -0,0 +1,18 @@
+import { UseMutationResult } from 'react-query';
+import { patchRequest } from '../../../../libs/Api';
+import { useSnackMutation } from '../../../../libs/apiHooks';
+import { BulkUpdateVariables, MappingVersionRow } from '../../types';
+
+export const useBulkUpdateQuestionMappings = (): UseMutationResult<
+ MappingVersionRow,
+ unknown,
+ BulkUpdateVariables
+> =>
+ useSnackMutation({
+ mutationFn: ({ mappingVersionId, questionMappings }) =>
+ patchRequest(`/api/mappingversions/${mappingVersionId}/`, {
+ question_mappings: questionMappings,
+ }),
+ invalidateQueryKey: ['mappingversions'],
+ showSuccessSnackBar: false,
+ });
diff --git a/hat/assets/js/apps/Iaso/domains/mappings/hooks/requests/useGetMappingImportSources.ts b/hat/assets/js/apps/Iaso/domains/mappings/hooks/requests/useGetMappingImportSources.ts
new file mode 100644
index 0000000000..57fe92e610
--- /dev/null
+++ b/hat/assets/js/apps/Iaso/domains/mappings/hooks/requests/useGetMappingImportSources.ts
@@ -0,0 +1,22 @@
+import { UseQueryResult } from 'react-query';
+import { getRequest } from '../../../../libs/Api';
+import { useSnackQuery } from '../../../../libs/apiHooks';
+import { MappingVersionRow } from '../../types';
+
+const SOURCE_FIELDS =
+ 'id,form_version,mapping,question_mappings,derivate_settings,updated_at';
+
+/** The mapping versions of a form, for one mapping type. */
+export const useGetMappingImportSources = (
+ formId: number,
+ mappingType: string,
+ enabled: boolean,
+): UseQueryResult =>
+ useSnackQuery({
+ queryKey: ['mappingversions', 'importSources', formId, mappingType],
+ queryFn: () =>
+ getRequest(
+ `/api/mappingversions/?formId=${formId}&mappingTypes=${mappingType}&fields=${SOURCE_FIELDS}`,
+ ).then(r => r.mapping_versions),
+ options: { enabled: Boolean(formId && mappingType) && enabled },
+ });
diff --git a/hat/assets/js/apps/Iaso/domains/mappings/hooks/useImportSources.ts b/hat/assets/js/apps/Iaso/domains/mappings/hooks/useImportSources.ts
new file mode 100644
index 0000000000..daf87d8248
--- /dev/null
+++ b/hat/assets/js/apps/Iaso/domains/mappings/hooks/useImportSources.ts
@@ -0,0 +1,65 @@
+import { useMemo } from 'react';
+import { useSafeIntl } from 'bluesquare-components';
+import { DateTimeCell } from '../../../components/Cells/DateTimeCell';
+import {
+ countMatchingMappings,
+ getImportableMappings,
+} from '../importMappings';
+import MESSAGES from '../messages';
+import { ImportSource, MappingVersionRow } from '../types';
+import { useGetMappingImportSources } from './requests/useGetMappingImportSources';
+
+const sortByVersionDesc = (a: MappingVersionRow, b: MappingVersionRow) =>
+ `${b.form_version.version_id}`.localeCompare(
+ `${a.form_version.version_id}`,
+ );
+
+/**
+ * Other versions of the same form, with the same mapping type and data
+ * source, the wizard can import from. Mappings of another form go through an
+ * export / import of the JSON file.
+ */
+export const useImportSources = (
+ mappingVersion: MappingVersionRow,
+ questions: Record,
+ enabled: boolean,
+): { sources: ImportSource[]; isLoading: boolean } => {
+ const { formatMessage } = useSafeIntl();
+ const { data, isLoading } = useGetMappingImportSources(
+ mappingVersion.form_version.form.id,
+ mappingVersion.mapping.mapping_type,
+ enabled,
+ );
+ const sources = useMemo(() => {
+ const dataSourceId = mappingVersion.mapping.data_source.id;
+ return (data ?? [])
+ .filter(
+ candidate =>
+ candidate.id !== mappingVersion.id &&
+ candidate.mapping.data_source.id === dataSourceId,
+ )
+ .sort(sortByVersionDesc)
+ .map(candidate => {
+ const questionMappings = getImportableMappings(
+ candidate.question_mappings,
+ );
+ return {
+ id: `${candidate.id}`,
+ title: formatMessage(MESSAGES.importVersionTitle, {
+ versionId: candidate.form_version.version_id,
+ }),
+ meta: formatMessage(MESSAGES.importVersionMeta, {
+ date: DateTimeCell({ value: candidate.updated_at }),
+ }),
+ mappingsCount: Object.keys(questionMappings).length,
+ matchingCount: countMatchingMappings(
+ questionMappings,
+ questions,
+ mappingVersion.mapping.mapping_type,
+ ),
+ questionMappings,
+ };
+ });
+ }, [data, mappingVersion, questions, formatMessage]);
+ return { sources, isLoading };
+};
diff --git a/hat/assets/js/apps/Iaso/domains/mappings/importMappings.test.ts b/hat/assets/js/apps/Iaso/domains/mappings/importMappings.test.ts
new file mode 100644
index 0000000000..1c23be4f36
--- /dev/null
+++ b/hat/assets/js/apps/Iaso/domains/mappings/importMappings.test.ts
@@ -0,0 +1,531 @@
+import { describe, expect, it } from 'vitest';
+import {
+ buildImportPlan,
+ buildMappingExport,
+ computeMappingsDiff,
+ getDefaultDecision,
+ getExportFileName,
+ getMappableQuestions,
+ getMappingLabel,
+ getOtherTarget,
+ isValidForMappingType,
+ MappingImportError,
+ parseMappingExport,
+} from './importMappings';
+import { DiffKind, MappingImportErrorReason } from './types';
+
+const descriptor = {
+ name: 'survey',
+ type: 'survey',
+ children: [
+ {
+ name: 'grp1',
+ type: 'group',
+ label: 'Group 1',
+ children: [
+ { name: 'q1', type: 'integer', label: 'Question 1' },
+ { name: 'q2', type: 'integer', label: 'Question 2' },
+ ],
+ },
+ { name: 'q3', type: 'integer', label: 'Question 3' },
+ { name: 'q4', type: 'integer', label: 'Question 4' },
+ { name: 'q5', type: 'integer', label: 'Question 5' },
+ ],
+};
+
+const de = (id: string, extra = {}) => ({
+ id,
+ name: `DE ${id}`,
+ valueType: 'NUMBER',
+ categoryOptionCombo: 'coc1',
+ categoryOptionComboName: 'default',
+ ...extra,
+});
+
+describe('getMappableQuestions', () => {
+ it('keeps repeat groups, select all that apply and their choices', () => {
+ const questions = getMappableQuestions({
+ name: 'survey',
+ type: 'survey',
+ children: [
+ {
+ name: 'household',
+ type: 'repeat',
+ children: [{ name: 'age', type: 'integer' }],
+ },
+ {
+ name: 'symptoms',
+ type: 'select all that apply',
+ children: [
+ { name: 'fever', label: 'Fever' },
+ { name: 'cough', label: 'Cough' },
+ ],
+ },
+ {
+ name: 'sex',
+ type: 'select one',
+ children: [{ name: 'male' }],
+ },
+ ],
+ });
+ expect(Object.keys(questions).sort()).toEqual([
+ 'age',
+ 'household',
+ 'sex',
+ 'symptoms',
+ 'symptoms__cough',
+ 'symptoms__fever',
+ ]);
+ });
+
+ it('skips groups', () => {
+ expect(Object.keys(getMappableQuestions(descriptor)).sort()).toEqual([
+ 'q1',
+ 'q2',
+ 'q3',
+ 'q4',
+ 'q5',
+ ]);
+ });
+});
+
+describe('getMappingLabel', () => {
+ it('shows the combo when not default', () => {
+ expect(getMappingLabel(de('a'))).toBe('DE a');
+ expect(
+ getMappingLabel(de('a', { categoryOptionComboName: 'Male' })),
+ ).toBe('DE a (Male)');
+ });
+ it('joins event tracker mappings', () => {
+ expect(
+ getMappingLabel([
+ { dataElement: { id: 'x', name: 'X' } },
+ { trackedEntityAttribute: { id: 'y', name: 'Y' } },
+ ]),
+ ).toBe('X, Y');
+ });
+});
+
+describe('mapping shapes', () => {
+ const multiple = (fever: string) => ({
+ type: 'multiple',
+ values: { fever: de(fever), cough: de('cough') },
+ });
+ const tracker = (stage: string) => [
+ {
+ dataElement: { id: 'de1', name: 'Age' },
+ programStage: stage,
+ parent: 'household',
+ },
+ ];
+ const repeat = (relationship: string) => [
+ {
+ type: 'repeat',
+ program_id: 'p1',
+ program_name: 'Program',
+ tracked_entity_type: 'tet',
+ tracked_entity_identifier: 'uid',
+ relationship_type: relationship,
+ relationship_type_name: `Rel ${relationship}`,
+ },
+ ];
+ const questions = {
+ symptoms: { name: 'symptoms', type: 'select all that apply' },
+ age: { name: 'age', type: 'integer' },
+ household: { name: 'household', type: 'repeat' },
+ empty: { name: 'empty', type: 'integer' },
+ };
+
+ const kindsOf = (rows: { questionKey: string; kind: string }[]) =>
+ Object.fromEntries(rows.map(r => [r.questionKey, r.kind]));
+
+ it('compares each shape on its DHIS2 target', () => {
+ expect(
+ kindsOf(
+ computeMappingsDiff(
+ { symptoms: multiple('a'), empty: [] },
+ { symptoms: multiple('b'), empty: de('x') },
+ questions,
+ 'AGGREGATE',
+ ),
+ ),
+ ).toEqual({
+ // a "multiple" mapping has no id but is a real mapping
+ symptoms: DiffKind.CONFLICT,
+ // an empty list is not a mapping
+ empty: DiffKind.ADD,
+ });
+ expect(
+ kindsOf(
+ computeMappingsDiff(
+ { age: tracker('s1'), household: repeat('r1') },
+ {
+ age: [
+ { ...tracker('s1')[0], dataElement: { id: 'de1' } },
+ ],
+ household: repeat('r2'),
+ },
+ questions,
+ 'EVENT_TRACKER',
+ ),
+ ),
+ ).toEqual({ age: DiffKind.IDENTICAL, household: DiffKind.CONFLICT });
+ expect(
+ computeMappingsDiff(
+ { age: tracker('s1') },
+ { age: tracker('s2') },
+ questions,
+ 'EVENT_TRACKER',
+ )[0].kind,
+ ).toBe(DiffKind.CONFLICT);
+ });
+
+ it.each([
+ ['a data element', de('a'), 'AGGREGATE', true],
+ ['a data element without valueType', { id: 'a' }, 'AGGREGATE', false],
+ ['a select all that apply', multiple('a'), 'EVENT', true],
+ [
+ 'a select all that apply choice without id',
+ { type: 'multiple', values: { fever: { name: 'Fever' } } },
+ 'AGGREGATE',
+ false,
+ ],
+ ['a tracker list', tracker('s1'), 'AGGREGATE', false],
+ ['a tracker list', tracker('s1'), 'EVENT_TRACKER', true],
+ ['a repeat group', repeat('r1'), 'EVENT_TRACKER', true],
+ [
+ 'a tracked entity attribute',
+ [{ trackedEntityAttribute: { id: 'tea' } }],
+ 'EVENT_TRACKER',
+ true,
+ ],
+ ['a data element', de('a'), 'EVENT_TRACKER', false],
+ ['an empty list', [], 'EVENT_TRACKER', false],
+ ['a list of junk', [{ foo: 1 }], 'EVENT_TRACKER', false],
+ [
+ 'a never mapped marker',
+ { type: 'neverMapped' },
+ 'EVENT_TRACKER',
+ true,
+ ],
+ ])('%s in a %s mapping is valid: %s', (_, mapping, mappingType, valid) => {
+ expect(isValidForMappingType(mapping as any, mappingType)).toBe(valid);
+ });
+
+ it('drops mappings that do not fit the mapping type', () => {
+ const rows = computeMappingsDiff(
+ {},
+ { symptoms: tracker('s1'), age: de('a') },
+ questions,
+ 'AGGREGATE',
+ );
+ expect(kindsOf(rows)).toEqual({
+ symptoms: DiffKind.DROPPED,
+ age: DiffKind.ADD,
+ });
+ expect(rows.find(r => r.questionKey === 'symptoms')?.invalid).toBe(
+ true,
+ );
+ });
+
+ it('labels each shape', () => {
+ expect(getMappingLabel(multiple('a'))).toBe(
+ 'fever: DE a, cough: DE cough',
+ );
+ expect(getMappingLabel(repeat('r1'))).toBe('Program (Rel r1)');
+ expect(getMappingLabel(tracker('s1'))).toBe('Age');
+ expect(
+ getMappingLabel(
+ de('a', {
+ displayName: 'DE a - Male',
+ categoryOptionComboName: 'Male',
+ }),
+ ),
+ ).toBe('DE a - Male');
+ });
+});
+
+describe('computeMappingsDiff', () => {
+ const current = {
+ q1: de('a'),
+ q2: de('b'),
+ q4: { type: 'neverMapped' },
+ };
+ const incoming = {
+ q1: de('a', { name: 'renamed' }),
+ q2: de('c'),
+ q3: de('d'),
+ q4: de('e'),
+ q5: { type: 'neverMapped' },
+ gone: de('f'),
+ };
+ const rows = computeMappingsDiff(
+ current,
+ incoming,
+ getMappableQuestions(descriptor),
+ 'AGGREGATE',
+ );
+ const kinds = Object.fromEntries(rows.map(r => [r.questionKey, r.kind]));
+
+ it('classifies each incoming mapping', () => {
+ expect(kinds).toEqual({
+ q1: DiffKind.IDENTICAL,
+ q2: DiffKind.CONFLICT,
+ q3: DiffKind.ADD,
+ // overwriting a never mapped marker is a decision to take
+ q4: DiffKind.CONFLICT,
+ gone: DiffKind.DROPPED,
+ });
+ });
+
+ it('only imports choice keys of select all that apply questions', () => {
+ const choiceQuestions = getMappableQuestions({
+ name: 'survey',
+ type: 'survey',
+ children: [
+ {
+ name: 'symptoms',
+ type: 'select all that apply',
+ children: [{ name: 'fever' }],
+ },
+ {
+ name: 'sex',
+ type: 'select one',
+ children: [{ name: 'male' }],
+ },
+ ],
+ });
+ const choiceRows = computeMappingsDiff(
+ {},
+ { symptoms__fever: de('a'), sex__male: de('b') },
+ choiceQuestions,
+ 'AGGREGATE',
+ );
+ expect(
+ Object.fromEntries(choiceRows.map(r => [r.questionKey, r.kind])),
+ ).toEqual({
+ symptoms__fever: DiffKind.ADD,
+ sex__male: DiffKind.DROPPED,
+ });
+ });
+
+ it('builds the patch payload', () => {
+ const decisions = Object.fromEntries(
+ rows.map(r => [r.questionKey, getDefaultDecision(r, true)]),
+ );
+ const plan = buildImportPlan(rows, decisions);
+ expect(plan.changes).toEqual({
+ q2: de('c'),
+ q3: de('d'),
+ q4: de('e'),
+ });
+ expect(plan).toMatchObject({
+ added: 1,
+ overwritten: 2,
+ kept: 0,
+ skipped: 0,
+ dropped: 1,
+ });
+ });
+
+ it('keeps current mappings by default', () => {
+ const decisions = Object.fromEntries(
+ rows.map(r => [r.questionKey, getDefaultDecision(r)]),
+ );
+ const plan = buildImportPlan(rows, decisions);
+ expect(Object.keys(plan.changes)).toEqual(['q3']);
+ expect(plan.kept).toBe(2);
+ });
+});
+
+describe('export / import', () => {
+ const mappingVersion = {
+ id: 1,
+ form_version: {
+ id: 10,
+ version_id: '2021111101',
+ form: { id: 5, name: "Cadre de l'ECZS" },
+ },
+ mapping: {
+ mapping_type: 'AGGREGATE',
+ data_source: { id: 3, name: 'dhis2' },
+ },
+ question_mappings: { q1: de('a'), q2: { type: 'neverMapped' } },
+ derivate_settings: { data_set_id: 'ds1', data_set_name: 'DS 1' },
+ };
+
+ it('exports without never mapped markers', () => {
+ const exported = buildMappingExport(
+ mappingVersion,
+ new Date('2026-01-01T00:00:00Z'),
+ );
+ expect(exported.question_mappings).toEqual({ q1: de('a') });
+ expect(exported.dataset).toEqual({ id: 'ds1', name: 'DS 1' });
+ expect(exported.program).toBeUndefined();
+ expect(getExportFileName(mappingVersion)).toBe(
+ 'mapping-cadre-de-l-eczs-2021111101-aggregate.json',
+ );
+ });
+
+ it('round trips', () => {
+ const text = JSON.stringify(buildMappingExport(mappingVersion));
+ expect(parseMappingExport(text, 'AGGREGATE').question_mappings).toEqual(
+ { q1: de('a') },
+ );
+ });
+
+ it.each([
+ [
+ 'AGGREGATE',
+ {
+ q1: de('a'),
+ // select all that apply as a whole, and one boolean data element per choice
+ symptoms: {
+ type: 'multiple',
+ values: { fever: de('b'), cough: de('c') },
+ },
+ symptoms__fever: de('d', { valueType: 'BOOLEAN' }),
+ q2: { type: 'neverMapped' },
+ },
+ ],
+ [
+ 'EVENT_TRACKER',
+ {
+ household: [
+ {
+ type: 'repeat',
+ program_id: 'p1',
+ tracked_entity_type: 'tet1',
+ tracked_entity_identifier: 'uid',
+ relationship_type: 'rel1',
+ },
+ ],
+ // inside the repeat group
+ age: [
+ {
+ dataElement: { id: 'de1', name: 'Age' },
+ programStage: 'stage1',
+ parent: 'household',
+ },
+ ],
+ q1: [
+ {
+ trackedEntityAttribute: { id: 'tea1', name: 'Name' },
+ iaso_field: 'instance.uuid',
+ },
+ ],
+ },
+ ],
+ ])(
+ 're-importing its own %s export shows every mapping as identical',
+ (mappingType, questionMappings) => {
+ const formDescriptor = {
+ name: 'survey',
+ type: 'survey',
+ children: [
+ {
+ name: 'grp1',
+ type: 'group',
+ children: [{ name: 'q1', type: 'integer' }],
+ },
+ { name: 'q2', type: 'integer' },
+ {
+ name: 'symptoms',
+ type: 'select all that apply',
+ children: [{ name: 'fever' }, { name: 'cough' }],
+ },
+ {
+ name: 'household',
+ type: 'repeat',
+ children: [{ name: 'age', type: 'integer' }],
+ },
+ ],
+ };
+ const version = {
+ ...mappingVersion,
+ mapping: {
+ ...mappingVersion.mapping,
+ mapping_type: mappingType,
+ },
+ question_mappings: questionMappings,
+ };
+
+ const text = JSON.stringify(buildMappingExport(version));
+ const rows = computeMappingsDiff(
+ questionMappings,
+ parseMappingExport(text, mappingType).question_mappings,
+ getMappableQuestions(formDescriptor),
+ mappingType,
+ );
+
+ const kinds = Object.fromEntries(
+ rows.map(r => [r.questionKey, r.kind]),
+ );
+ // never mapped markers are not exported
+ const expected = Object.fromEntries(
+ Object.keys(questionMappings)
+ .filter(key => key !== 'q2')
+ .map(key => [key, DiffKind.IDENTICAL]),
+ );
+ expect(kinds).toEqual(expected);
+ },
+ );
+
+ it('accepts a bare question_mappings dictionary', () => {
+ expect(
+ parseMappingExport(JSON.stringify({ q1: de('a') }), 'AGGREGATE')
+ .question_mappings,
+ ).toEqual({ q1: de('a') });
+ });
+
+ it.each([
+ ['not json', MappingImportErrorReason.INVALID_JSON],
+ ['[]', MappingImportErrorReason.INVALID_FORMAT],
+ ['{"q1": 3}', MappingImportErrorReason.INVALID_FORMAT],
+ ['{"q1": [3]}', MappingImportErrorReason.INVALID_FORMAT],
+ [
+ '{"mapping_type": "EVENT", "question_mappings": {}}',
+ MappingImportErrorReason.MAPPING_TYPE_MISMATCH,
+ ],
+ // any JSON object whose values are objects, e.g. a tsconfig.json
+ [
+ '{"compilerOptions": {"strict": true}}',
+ MappingImportErrorReason.NO_VALID_MAPPING,
+ ],
+ [
+ '{"q1": [{"dataElement": {"id": "x"}}]}',
+ MappingImportErrorReason.NO_VALID_MAPPING,
+ ],
+ [
+ '{"question_mappings": {}}',
+ MappingImportErrorReason.NO_VALID_MAPPING,
+ ],
+ ])('rejects %s', (text, reason) => {
+ // the error message is the reason
+ expect(() => parseMappingExport(text, 'AGGREGATE')).toThrow(
+ new MappingImportError(reason),
+ );
+ });
+});
+
+describe('getOtherTarget', () => {
+ const aggregate = { derivate_settings: { data_set_id: 'ds1' } };
+ const tracker = { derivate_settings: { program_id: 'p1' } };
+
+ it('flags an export made for another dataset or program', () => {
+ expect(
+ getOtherTarget({ dataset: { id: 'ds2', name: 'DS 2' } }, aggregate),
+ ).toEqual({ kind: 'dataset', name: 'DS 2' });
+ expect(getOtherTarget({ program: { id: 'p2' } }, tracker)).toEqual({
+ kind: 'program',
+ name: 'p2',
+ });
+ });
+
+ it('accepts the same target, or an export without one', () => {
+ expect(getOtherTarget({ dataset: { id: 'ds1' } }, aggregate)).toBe(
+ undefined,
+ );
+ expect(getOtherTarget({}, aggregate)).toBe(undefined);
+ });
+});
diff --git a/hat/assets/js/apps/Iaso/domains/mappings/importMappings.ts b/hat/assets/js/apps/Iaso/domains/mappings/importMappings.ts
new file mode 100644
index 0000000000..1ea3cbb4da
--- /dev/null
+++ b/hat/assets/js/apps/Iaso/domains/mappings/importMappings.ts
@@ -0,0 +1,443 @@
+import { isEqual } from 'lodash';
+import Descriptor from './descriptor';
+import { isNeverMapped } from './question_mappings';
+import {
+ Decision,
+ DiffKind,
+ DiffRow,
+ ImportPlan,
+ MappingExport,
+ MappingImportErrorReason,
+ QuestionMapping,
+ QuestionMappings,
+} from './types';
+
+// in the order of the wizard tabs
+export const DIFF_KINDS: DiffKind[] = [
+ DiffKind.CONFLICT,
+ DiffKind.ADD,
+ DiffKind.IDENTICAL,
+ DiffKind.DROPPED,
+];
+
+export const EXPORT_FORMAT = 'iaso-mapping-export';
+export const EXPORT_FORMAT_VERSION = 1;
+
+/*
+ * question_mappings values come in several shapes:
+ * - AGGREGATE / EVENT data element: { id, valueType, categoryOptionCombo, ... }
+ * - select all that apply as a whole: { type: 'multiple', values: { choice: { id, ... } } }
+ * or one boolean data element per choice, keyed `question__choice`
+ * - EVENT_TRACKER: a list [{ dataElement | trackedEntityAttribute, programStage, iaso_field, parent }]
+ * - EVENT_TRACKER repeat group: [{ type: 'repeat', program_id, relationship_type, tracked_entity_identifier, ... }]
+ * - never mapped marker: { type: 'neverMapped' }
+ */
+
+// Same rule as the backend copy_mappings_from_previous_version: everything
+// but survey and plain groups can carry a mapping (repeat groups and select
+// all that apply questions included).
+const NOT_MAPPABLE_TYPES = ['survey', 'group'];
+
+/**
+ * Questions of a form version that can be mapped, indexed by the key used in
+ * question_mappings.
+ */
+export const getMappableQuestions = (
+ descriptor: Record | undefined,
+): Record => {
+ const result: Record = {};
+ Object.values(Descriptor.indexQuestions(descriptor)).forEach(
+ (question: any) => {
+ if (!NOT_MAPPABLE_TYPES.includes(question.type)) {
+ result[Descriptor.getKey(question)] = question;
+ }
+ },
+ );
+ return result;
+};
+
+/** A mapping pointing to something in DHIS2 (not a never mapped marker). */
+export const hasTarget = (mapping?: QuestionMapping): boolean => {
+ if (!mapping || isNeverMapped(mapping)) {
+ return false;
+ }
+ return Array.isArray(mapping) ? mapping.length > 0 : true;
+};
+
+/** Mappings that carry a target, i.e. no "never mapped" markers. */
+export const getImportableMappings = (
+ questionMappings: QuestionMappings | undefined,
+): QuestionMappings => {
+ const result: QuestionMappings = {};
+ Object.entries(questionMappings ?? {}).forEach(([key, mapping]) => {
+ if (hasTarget(mapping)) {
+ result[key] = mapping;
+ }
+ });
+ return result;
+};
+
+const mappingItemLabel = (mapping: Record): string => {
+ if (mapping.type === 'multiple') {
+ return Object.entries(mapping.values ?? {})
+ .map(
+ ([choice, target]: [string, any]) =>
+ `${choice}: ${mappingItemLabel(target)}`,
+ )
+ .join(', ');
+ }
+ if (mapping.type === 'repeat') {
+ const program = mapping.program_name ?? mapping.program_id;
+ return mapping.relationship_type_name
+ ? `${program} (${mapping.relationship_type_name})`
+ : program;
+ }
+ const name =
+ mapping.displayName ??
+ mapping.name ??
+ mapping.dataElement?.name ??
+ mapping.trackedEntityAttribute?.name ??
+ mapping.id ??
+ '';
+ const comboName = mapping.categoryOptionComboName;
+ return comboName && comboName !== 'default' && !name.includes(comboName)
+ ? `${name} (${comboName})`
+ : name;
+};
+
+export const getMappingLabel = (mapping?: QuestionMapping): string => {
+ if (!mapping) {
+ return '';
+ }
+ if (Array.isArray(mapping)) {
+ return mapping.map(mappingItemLabel).join(', ');
+ }
+ return mappingItemLabel(mapping);
+};
+
+// Fields identifying the DHIS2 target of a mapping, ignoring the cached
+// metadata (names, option sets, datasets, ...).
+const mappingTargetKey = (
+ mapping: Record,
+): Record => ({
+ type: mapping.type,
+ id: mapping.id,
+ categoryOptionCombo: mapping.categoryOptionCombo,
+ dataElement: mapping.dataElement?.id,
+ trackedEntityAttribute: mapping.trackedEntityAttribute?.id,
+ programStage: mapping.programStage,
+ iaso_field: mapping.iaso_field,
+ parent: mapping.parent,
+ program_id: mapping.program_id,
+ tracked_entity_type: mapping.tracked_entity_type,
+ tracked_entity_identifier: mapping.tracked_entity_identifier,
+ relationship_type: mapping.relationship_type,
+ values: mapping.values
+ ? Object.fromEntries(
+ Object.entries(mapping.values).map(
+ ([choice, target]: [string, any]) => [
+ choice,
+ mappingTargetKey(target),
+ ],
+ ),
+ )
+ : undefined,
+});
+
+/**
+ * Two mappings are identical when they point to the same DHIS2 target, even
+ * if the cached metadata (names, datasets, ...) differs.
+ */
+export const isSameMapping = (a: QuestionMapping, b: QuestionMapping) => {
+ if (Array.isArray(a) !== Array.isArray(b)) {
+ return false;
+ }
+ if (Array.isArray(a) && Array.isArray(b)) {
+ return isEqual(a.map(mappingTargetKey), b.map(mappingTargetKey));
+ }
+ return isEqual(
+ mappingTargetKey(a as Record),
+ mappingTargetKey(b as Record),
+ );
+};
+
+const isObject = (value: unknown): value is Record =>
+ Boolean(value) && typeof value === 'object' && !Array.isArray(value);
+
+const isTrackerItem = (item: unknown): boolean =>
+ isObject(item) &&
+ Boolean(
+ item.dataElement?.id ||
+ item.trackedEntityAttribute?.id ||
+ (item.type === 'repeat' && item.program_id),
+ );
+
+/**
+ * Whether a question mapping has the shape the exporter of the mapping type
+ * reads. Same rules as get_question_mapping_shape_error on the backend.
+ */
+export const isValidForMappingType = (
+ mapping: QuestionMapping,
+ mappingType: string,
+): boolean => {
+ if (isNeverMapped(mapping)) {
+ return true;
+ }
+ if (mappingType === 'EVENT_TRACKER') {
+ return (
+ Array.isArray(mapping) &&
+ mapping.length > 0 &&
+ mapping.every(isTrackerItem)
+ );
+ }
+ if (Array.isArray(mapping) || !isObject(mapping)) {
+ return false;
+ }
+ const { type, values, id, valueType } = mapping;
+ if (type === 'multiple') {
+ return (
+ isObject(values) &&
+ Object.values(values).every(
+ value => isObject(value) && Boolean(value.id),
+ )
+ );
+ }
+ return Boolean(id && valueType);
+};
+
+export const computeMappingsDiff = (
+ currentMappings: QuestionMappings,
+ incomingMappings: QuestionMappings,
+ questions: Record,
+ mappingType: string,
+): DiffRow[] =>
+ Object.entries(getImportableMappings(incomingMappings)).map(
+ ([questionKey, incoming]) => {
+ const question = questions[questionKey];
+ const current = currentMappings[questionKey];
+ const row: DiffRow = {
+ kind: DiffKind.ADD,
+ questionKey,
+ questionLabel: question
+ ? Descriptor.getHumanLabel(question)
+ : undefined,
+ // a never mapped marker is a decision too: overwriting it is a conflict
+ current:
+ hasTarget(current) || isNeverMapped(current)
+ ? current
+ : undefined,
+ incoming,
+ };
+ if (!question) {
+ row.kind = DiffKind.DROPPED;
+ } else if (!isValidForMappingType(incoming, mappingType)) {
+ row.kind = DiffKind.DROPPED;
+ row.invalid = true;
+ } else if (row.current) {
+ row.kind = isSameMapping(row.current, incoming)
+ ? DiffKind.IDENTICAL
+ : DiffKind.CONFLICT;
+ }
+ return row;
+ },
+ );
+
+/** Mappings of a question of this version, valid for the mapping type. */
+export const countMatchingMappings = (
+ questionMappings: QuestionMappings,
+ questions: Record,
+ mappingType: string,
+): number =>
+ Object.entries(questionMappings).filter(
+ ([key, mapping]) =>
+ questions[key] && isValidForMappingType(mapping, mappingType),
+ ).length;
+
+export const countValidMappings = (
+ questionMappings: QuestionMappings,
+ mappingType: string,
+): number =>
+ Object.values(questionMappings).filter(mapping =>
+ isValidForMappingType(mapping, mappingType),
+ ).length;
+
+export const getDefaultDecision = (
+ row: DiffRow,
+ overwriteConflicts = false,
+): Decision | undefined => {
+ if (row.kind === DiffKind.CONFLICT) {
+ return overwriteConflicts ? Decision.OVERWRITE : Decision.KEEP;
+ }
+ if (row.kind === DiffKind.ADD) {
+ return Decision.APPLY;
+ }
+ return undefined;
+};
+
+export const willApply = (decision?: Decision) =>
+ decision === Decision.APPLY || decision === Decision.OVERWRITE;
+
+export const buildImportPlan = (
+ rows: DiffRow[],
+ decisions: Record,
+): ImportPlan => {
+ const plan: ImportPlan = {
+ changes: {},
+ added: 0,
+ overwritten: 0,
+ kept: 0,
+ skipped: 0,
+ dropped: 0,
+ };
+ rows.forEach(row => {
+ const decision = decisions[row.questionKey];
+ if (row.kind === DiffKind.DROPPED) {
+ plan.dropped += 1;
+ } else if (
+ row.kind === DiffKind.CONFLICT &&
+ decision === Decision.OVERWRITE
+ ) {
+ plan.changes[row.questionKey] = row.incoming;
+ plan.overwritten += 1;
+ } else if (row.kind === DiffKind.CONFLICT) {
+ plan.kept += 1;
+ } else if (row.kind === DiffKind.ADD && decision === Decision.APPLY) {
+ plan.changes[row.questionKey] = row.incoming;
+ plan.added += 1;
+ } else if (row.kind === DiffKind.ADD) {
+ plan.skipped += 1;
+ }
+ });
+ return plan;
+};
+
+export const buildMappingExport = (
+ mappingVersion: Record,
+ now: Date = new Date(),
+): MappingExport => {
+ const { mapping, form_version: formVersion } = mappingVersion;
+ const settings = mappingVersion.derivate_settings ?? {};
+ const result: MappingExport = {
+ format: EXPORT_FORMAT,
+ format_version: EXPORT_FORMAT_VERSION,
+ exported_at: now.toISOString(),
+ mapping_type: mapping.mapping_type,
+ data_source: mapping.data_source,
+ form: { id: formVersion.form.id, name: formVersion.form.name },
+ form_version: {
+ id: formVersion.id,
+ version_id: formVersion.version_id,
+ },
+ question_mappings: getImportableMappings(
+ mappingVersion.question_mappings,
+ ),
+ };
+ if (settings.data_set_id) {
+ result.dataset = {
+ id: settings.data_set_id,
+ name: settings.data_set_name,
+ };
+ }
+ if (settings.program_id) {
+ result.program = {
+ id: settings.program_id,
+ name: settings.program_name,
+ };
+ }
+ return result;
+};
+
+export const getExportFileName = (mappingVersion: Record) => {
+ const { form_version: formVersion, mapping } = mappingVersion;
+ const slug =
+ `${formVersion.form.name}-${formVersion.version_id}-${mapping.mapping_type}`
+ .toLowerCase()
+ .replace(/[^a-z0-9]+/g, '-')
+ .replace(/^-|-$/g, '');
+ return `mapping-${slug}.json`;
+};
+
+const isValidMapping = (mapping: unknown): boolean =>
+ Array.isArray(mapping) ? mapping.every(isObject) : isObject(mapping);
+
+export class MappingImportError extends Error {
+ constructor(public reason: MappingImportErrorReason) {
+ super(reason);
+ }
+}
+
+/**
+ * Accepts an export produced by buildMappingExport, or a bare
+ * question_mappings dictionary.
+ */
+export const parseMappingExport = (
+ text: string,
+ expectedMappingType: string,
+): Partial & { question_mappings: QuestionMappings } => {
+ let parsed;
+ try {
+ parsed = JSON.parse(text);
+ } catch {
+ throw new MappingImportError(MappingImportErrorReason.INVALID_JSON);
+ }
+ if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
+ throw new MappingImportError(MappingImportErrorReason.INVALID_FORMAT);
+ }
+ const content =
+ 'question_mappings' in parsed ? parsed : { question_mappings: parsed };
+ const { question_mappings: questionMappings } = content;
+ if (
+ !questionMappings ||
+ typeof questionMappings !== 'object' ||
+ Array.isArray(questionMappings) ||
+ Object.values(questionMappings).some(m => !isValidMapping(m))
+ ) {
+ throw new MappingImportError(MappingImportErrorReason.INVALID_FORMAT);
+ }
+ if (content.mapping_type && content.mapping_type !== expectedMappingType) {
+ throw new MappingImportError(
+ MappingImportErrorReason.MAPPING_TYPE_MISMATCH,
+ );
+ }
+ if (
+ countValidMappings(
+ getImportableMappings(questionMappings),
+ expectedMappingType,
+ ) === 0
+ ) {
+ throw new MappingImportError(MappingImportErrorReason.NO_VALID_MAPPING);
+ }
+ return content;
+};
+
+/**
+ * The DHIS2 dataset or program an export was made for, when it is not the
+ * one of the mapping version it is imported in.
+ */
+export const getOtherTarget = (
+ content: Partial,
+ mappingVersion: Record,
+): { kind: 'dataset' | 'program'; name: string } | undefined => {
+ const settings = mappingVersion.derivate_settings ?? {};
+ if (
+ content.dataset?.id &&
+ settings.data_set_id &&
+ content.dataset.id !== settings.data_set_id
+ ) {
+ return {
+ kind: 'dataset',
+ name: content.dataset.name ?? content.dataset.id,
+ };
+ }
+ if (
+ content.program?.id &&
+ settings.program_id &&
+ content.program.id !== settings.program_id
+ ) {
+ return {
+ kind: 'program',
+ name: content.program.name ?? content.program.id,
+ };
+ }
+ return undefined;
+};
diff --git a/hat/assets/js/apps/Iaso/domains/mappings/messages.js b/hat/assets/js/apps/Iaso/domains/mappings/messages.js
index 7f188f4187..9794b9c7f3 100644
--- a/hat/assets/js/apps/Iaso/domains/mappings/messages.js
+++ b/hat/assets/js/apps/Iaso/domains/mappings/messages.js
@@ -233,6 +233,204 @@ const MESSAGES = defineMessages({
id: 'iaso.mappings.startTypingFormVersion',
defaultMessage: "Start typing to search by form's name or version id",
},
+ importMappings: {
+ id: 'iaso.mappings.import.title',
+ defaultMessage: 'Import mappings',
+ },
+ exportMappings: {
+ id: 'iaso.mappings.export',
+ defaultMessage: 'Export mappings',
+ },
+ importPickSource: {
+ id: 'iaso.mappings.import.pickSource',
+ defaultMessage:
+ 'Pick the mappings to reuse. Nothing is applied until you confirm.',
+ },
+ importVersionTitle: {
+ id: 'iaso.mappings.import.versionTitle',
+ defaultMessage: 'Version {versionId}',
+ },
+ importVersionMeta: {
+ id: 'iaso.mappings.import.versionMeta',
+ defaultMessage: 'Last updated {date}',
+ },
+ importFileMeta: {
+ id: 'iaso.mappings.import.fileMeta',
+ defaultMessage: 'Exported from {formName} - {versionId}',
+ },
+ importMappingsCount: {
+ id: 'iaso.mappings.import.mappingsCount',
+ defaultMessage: '{count} mappings',
+ },
+ importMatchCount: {
+ id: 'iaso.mappings.import.matchCount',
+ defaultMessage: '{matching} / {total} match this version',
+ },
+ importNoSource: {
+ id: 'iaso.mappings.import.noSource',
+ defaultMessage:
+ 'No other mapping version uses this data source with the same DHIS2 dataset or program.',
+ },
+ importFromFile: {
+ id: 'iaso.mappings.import.fromFile',
+ defaultMessage:
+ 'Or import a mapping export (.json), for example from another account.',
+ },
+ chooseFile: {
+ id: 'iaso.mappings.import.chooseFile',
+ defaultMessage: 'Choose a file',
+ },
+ importInvalidJson: {
+ id: 'iaso.mappings.import.invalidJson',
+ defaultMessage: 'This file is not valid JSON.',
+ },
+ importInvalidFormat: {
+ id: 'iaso.mappings.import.invalidFormat',
+ defaultMessage: 'This file is not a mapping export.',
+ },
+ importMappingTypeMismatch: {
+ id: 'iaso.mappings.import.mappingTypeMismatch',
+ defaultMessage: 'This export was made for another mapping type.',
+ },
+ compare: {
+ id: 'iaso.label.compare',
+ defaultMessage: 'Compare',
+ },
+ back: {
+ id: 'iaso.label.back',
+ defaultMessage: 'Back',
+ },
+ applyChanges: {
+ id: 'iaso.mappings.import.applyChanges',
+ defaultMessage: 'Apply {count} changes',
+ },
+ compareLabel: {
+ id: 'iaso.mappings.import.compareLabel',
+ defaultMessage:
+ 'Comparing {source} with version {versionId}. Nothing is applied until you confirm.',
+ },
+ bucketConflict: {
+ id: 'iaso.mappings.import.bucket.conflict',
+ defaultMessage: 'Conflicts ({count})',
+ },
+ bucketAdd: {
+ id: 'iaso.mappings.import.bucket.add',
+ defaultMessage: 'To add ({count})',
+ },
+ bucketIdentical: {
+ id: 'iaso.mappings.import.bucket.identical',
+ defaultMessage: 'Identical ({count})',
+ },
+ bucketDropped: {
+ id: 'iaso.mappings.import.bucket.dropped',
+ defaultMessage: 'Dropped ({count})',
+ },
+ resolveAllConflicts: {
+ id: 'iaso.mappings.import.resolveAllConflicts',
+ defaultMessage: 'Resolve all conflicts:',
+ },
+ keepEverywhere: {
+ id: 'iaso.mappings.import.keepEverywhere',
+ defaultMessage: 'Keep current everywhere',
+ },
+ overwriteEverywhere: {
+ id: 'iaso.mappings.import.overwriteEverywhere',
+ defaultMessage: 'Overwrite everywhere',
+ },
+ allAdditions: {
+ id: 'iaso.mappings.import.allAdditions',
+ defaultMessage: 'All additions:',
+ },
+ addAll: {
+ id: 'iaso.mappings.import.addAll',
+ defaultMessage: 'Add all',
+ },
+ skipAll: {
+ id: 'iaso.mappings.import.skipAll',
+ defaultMessage: 'Skip all',
+ },
+ identicalHint: {
+ id: 'iaso.mappings.import.identicalHint',
+ defaultMessage: 'Same DHIS2 target on both sides. Nothing to resolve.',
+ },
+ droppedHint: {
+ id: 'iaso.mappings.import.droppedHint',
+ defaultMessage:
+ 'These mappings cannot be imported: the question does not exist in this version, or the mapping is not valid for this mapping type.',
+ },
+ question: {
+ id: 'iaso.label.question',
+ defaultMessage: 'Question',
+ },
+ incomingMapping: {
+ id: 'iaso.mappings.import.incomingMapping',
+ defaultMessage: 'Incoming mapping',
+ },
+ decision: {
+ id: 'iaso.mappings.import.decision',
+ defaultMessage: 'Decision',
+ },
+ keep: {
+ id: 'iaso.mappings.import.keep',
+ defaultMessage: 'Keep',
+ },
+ overwrite: {
+ id: 'iaso.mappings.import.overwrite',
+ defaultMessage: 'Overwrite',
+ },
+ skip: {
+ id: 'iaso.mappings.import.skip',
+ defaultMessage: 'Skip',
+ },
+ noChange: {
+ id: 'iaso.mappings.import.noChange',
+ defaultMessage: 'No change',
+ },
+ notImportable: {
+ id: 'iaso.mappings.import.notImportable',
+ defaultMessage: 'Not importable',
+ },
+ questionAbsent: {
+ id: 'iaso.mappings.import.questionAbsent',
+ defaultMessage: 'Question absent from version {versionId}',
+ },
+ markedNeverMapped: {
+ id: 'iaso.mappings.import.markedNeverMapped',
+ defaultMessage: 'Marked as never mapped',
+ },
+ importSummary: {
+ id: 'iaso.mappings.import.summary',
+ defaultMessage:
+ '{added} to add, {overwritten} to overwrite, {ignored} not imported',
+ },
+ importDone: {
+ id: 'iaso.mappings.import.done',
+ defaultMessage:
+ '{added} mappings added, {overwritten} overwritten, {kept} kept, {ignored} not imported',
+ },
+ importInvalidMapping: {
+ id: 'iaso.mappings.import.invalidMapping',
+ defaultMessage: 'Not valid for a {type} mapping',
+ },
+ importNoValidMapping: {
+ id: 'iaso.mappings.import.noValidMapping',
+ defaultMessage:
+ 'This file contains no mapping valid for a {type} mapping.',
+ },
+ importOtherDataset: {
+ id: 'iaso.mappings.import.otherDataset',
+ defaultMessage:
+ 'This export was made for another DHIS2 dataset ({name}). Check that its data elements belong to this one.',
+ },
+ importOtherProgram: {
+ id: 'iaso.mappings.import.otherProgram',
+ defaultMessage:
+ 'This export was made for another DHIS2 program ({name}). Check that its data elements belong to this one.',
+ },
+ changeLog: {
+ id: 'iaso.mappings.changeLog',
+ defaultMessage: 'Link to changes log',
+ },
});
export default MESSAGES;
diff --git a/hat/assets/js/apps/Iaso/domains/mappings/types.ts b/hat/assets/js/apps/Iaso/domains/mappings/types.ts
new file mode 100644
index 0000000000..89130216ea
--- /dev/null
+++ b/hat/assets/js/apps/Iaso/domains/mappings/types.ts
@@ -0,0 +1,77 @@
+// mapping versions come from the untyped /api/mappingversions/ serializer
+export type MappingVersionRow = Record;
+
+// See importMappings.ts for the shapes a question mapping can take
+export type QuestionMapping = Record | Record[];
+export type QuestionMappings = Record;
+
+export enum DiffKind {
+ CONFLICT = 'conflict',
+ ADD = 'add',
+ IDENTICAL = 'identical',
+ DROPPED = 'dropped',
+}
+
+export enum Decision {
+ KEEP = 'keep',
+ OVERWRITE = 'overwrite',
+ APPLY = 'apply',
+ SKIP = 'skip',
+}
+
+// why an uploaded file can't be imported
+export enum MappingImportErrorReason {
+ INVALID_JSON = 'invalidJson',
+ INVALID_FORMAT = 'invalidFormat',
+ MAPPING_TYPE_MISMATCH = 'mappingTypeMismatch',
+ NO_VALID_MAPPING = 'noValidMapping',
+}
+
+export type DiffRow = {
+ kind: DiffKind;
+ questionKey: string;
+ questionLabel?: string;
+ // a mapping or a never mapped marker
+ current?: QuestionMapping;
+ incoming: QuestionMapping;
+ // dropped because its shape does not fit the mapping type
+ invalid?: boolean;
+};
+
+export type ImportPlan = {
+ // payload for PATCH question_mappings
+ changes: QuestionMappings;
+ added: number;
+ overwritten: number;
+ kept: number;
+ skipped: number;
+ dropped: number;
+};
+
+export type MappingExport = {
+ format: string;
+ format_version: number;
+ exported_at: string;
+ mapping_type: string;
+ data_source: { id: number; name: string };
+ form: { id: number; name: string };
+ form_version: { id: number; version_id: string };
+ dataset?: { id?: string; name?: string };
+ program?: { id?: string; name?: string };
+ question_mappings: QuestionMappings;
+};
+
+// a mapping version, or an uploaded export, the wizard can import from
+export type ImportSource = {
+ id: string;
+ title: string;
+ meta: string;
+ mappingsCount: number;
+ matchingCount: number;
+ questionMappings: QuestionMappings;
+};
+
+export type BulkUpdateVariables = {
+ mappingVersionId: number;
+ questionMappings: Record;
+};
diff --git a/hat/audit/models.py b/hat/audit/models.py
index 56f6436d73..943807c157 100644
--- a/hat/audit/models.py
+++ b/hat/audit/models.py
@@ -27,6 +27,7 @@
GROUP_SET_API = "group_set_api"
INSTANCE_API = "instance_api"
INSTANCE_API_BULK = "instance_api_bulk"
+MAPPING_VERSION_API = "mapping_version_api"
ORG_UNIT_API = "org_unit_api"
ORG_UNIT_API_BULK = "org_unit_api_bulk"
ORG_UNIT_CHANGE_REQUEST = "org_unit_change_request"
diff --git a/iaso/api/logs.py b/iaso/api/logs.py
index 9766609773..c2baf1c6e4 100644
--- a/iaso/api/logs.py
+++ b/iaso/api/logs.py
@@ -13,11 +13,12 @@
from hat.api.authentication import CsrfExemptSessionAuthentication
from hat.audit.models import Modification
-from iaso.models import Form, Instance, OrgUnit
+from iaso.models import Form, Instance, MappingVersion, OrgUnit
from iaso.models.base import Profile
from iaso.models.org_unit import OrgUnitChangeRequest
from iaso.models.payments import Payment, PaymentLot
from iaso.permissions.core_permissions import (
+ CORE_MAPPINGS_PERMISSION,
CORE_PAYMENTS_PERMISSION,
CORE_SUBMISSIONS_PERMISSION,
CORE_SUBMISSIONS_UPDATE_PERMISSION,
@@ -53,6 +54,9 @@ def has_access_to(user: User, obj: Union[OrgUnit, Instance, models.Model]):
if isinstance(obj, Profile):
profiles = Profile.objects.filter(account=user.iaso_profile.account)
return profiles.filter(id=obj.id).exists() and user.has_perm(CORE_USERS_ADMIN_PERMISSION.full_name())
+ if isinstance(obj, MappingVersion):
+ mapping_versions = MappingVersion.objects.filter_for_user(user)
+ return mapping_versions.filter(id=obj.id).exists() and user.has_perm(CORE_MAPPINGS_PERMISSION.full_name())
# Now checking models that are part of plugins
if is_polio_plugin_active():
@@ -78,7 +82,7 @@ class LogsViewSet(viewsets.ViewSet):
- new_value
- field_diffs
- contentType parameter can be one of: iaso.orgunit, iaso.form, iaso.instance, polio.campaign
+ contentType parameter can be one of: iaso.orgunit, iaso.form, iaso.instance, iaso.mappingversion, polio.campaign
list:
Returns the list of modifications
diff --git a/iaso/api/mapping_versions.py b/iaso/api/mapping_versions.py
index 0ad2c2d29d..36a8ef18e2 100644
--- a/iaso/api/mapping_versions.py
+++ b/iaso/api/mapping_versions.py
@@ -3,19 +3,112 @@
from django.core.exceptions import ObjectDoesNotExist
from django.db.models import Q
from django_filters.rest_framework import DjangoFilterBackend
-from drf_spectacular.utils import extend_schema
+from drf_spectacular.types import OpenApiTypes
+from drf_spectacular.utils import extend_schema, extend_schema_field
from rest_framework import permissions, serializers
import iaso.models as m
from dynamic_fields.filter_backends import DynamicFieldsFilterBackendBackwardCompatible
from dynamic_fields.serializer import DynamicFieldsModelSerializerBackwardCompatible
-from iaso.models import FormVersion, MappingVersion
+from hat.audit.models import MAPPING_VERSION_API, log_modification, serialize_instance
+from iaso.models import MappingVersion
from iaso.permissions.core_permissions import CORE_MAPPINGS_PERMISSION
from .common import HasPermission, ModelViewSet, TimestampField
+def get_question_mapping_shape_error(mapping_type, data_element):
+ """The shape a question mapping must have for the exporters of its mapping type, or None if it is valid.
+
+ The id and valueType of plain data element mappings are checked separately."""
+ if isinstance(data_element, dict) and data_element.get("type") == MappingVersion.QUESTION_MAPPING_NEVER_MAPPED:
+ return None
+
+ if mapping_type == m.EVENT_TRACKER:
+ if not isinstance(data_element, list) or not data_element:
+ return "should be a list for EVENT_TRACKER mappings"
+ for item in data_element:
+ if not isinstance(item, dict):
+ return "should only contain objects"
+ is_data_element = isinstance(item.get("dataElement"), dict) and item["dataElement"].get("id")
+ is_attribute = isinstance(item.get("trackedEntityAttribute"), dict) and item["trackedEntityAttribute"].get(
+ "id"
+ )
+ is_repeat_group = item.get("type") == "repeat" and item.get("program_id")
+ if not (is_data_element or is_attribute or is_repeat_group):
+ return "should map a data element, a tracked entity attribute or a repeat group"
+ return None
+
+ if not isinstance(data_element, dict):
+ return f"should not be a list for {mapping_type} mappings"
+ if data_element.get("type") == MappingVersion.QUESTION_MAPPING_MULTIPLE:
+ values = data_element.get("values")
+ if not isinstance(values, dict) or not all(isinstance(v, dict) and v.get("id") for v in values.values()):
+ return "should map each choice to a data element id"
+ return None
+
+
+def is_unmap(data_element):
+ return isinstance(data_element, dict) and data_element.get("action") == "unmap"
+
+
+def get_question_mapping_error(question_name, data_element, mapping_type, mappable_questions):
+ """Why a question mapping of a PATCH can't be saved, or None if it is valid."""
+ # unmapping stays allowed, to clean up mappings of questions removed from the form
+ if is_unmap(data_element):
+ return None
+
+ if mappable_questions and question_name not in mappable_questions:
+ return "question does not exist in this form version"
+
+ if data_element is None:
+ return None
+
+ shape_error = get_question_mapping_shape_error(mapping_type, data_element)
+ if shape_error:
+ return shape_error
+
+ if (
+ isinstance(data_element, dict)
+ and data_element
+ and data_element.get("type")
+ not in (MappingVersion.QUESTION_MAPPING_MULTIPLE, MappingVersion.QUESTION_MAPPING_NEVER_MAPPED)
+ ):
+ if data_element.get("id") is None:
+ return "should have a least an data element id"
+ if data_element.get("valueType") is None:
+ return "should have a valueType"
+ return None
+
+
+class MappingVersionFormSerializer(serializers.Serializer):
+ id = serializers.IntegerField()
+ name = serializers.CharField()
+ periodType = serializers.CharField(allow_null=True)
+
+
+class MappingVersionFormVersionSerializer(serializers.Serializer):
+ id = serializers.IntegerField()
+ form = MappingVersionFormSerializer()
+ version_id = serializers.CharField()
+
+
+class MappingVersionDataSourceSerializer(serializers.Serializer):
+ id = serializers.IntegerField()
+ name = serializers.CharField()
+
+
+class MappingVersionMappingSerializer(serializers.Serializer):
+ mapping_type = serializers.CharField()
+ data_source = MappingVersionDataSourceSerializer()
+
+
+class MappingVersionDatasetSerializer(serializers.Serializer):
+ id = serializers.CharField(allow_null=True)
+ name = serializers.CharField(allow_null=True)
+
+
class MappingVersionSerializer(DynamicFieldsModelSerializerBackwardCompatible):
class Meta:
model = MappingVersion
@@ -52,23 +145,29 @@ class Meta:
total_questions = serializers.SerializerMethodField()
derivate_settings = serializers.SerializerMethodField()
+ @extend_schema_field(OpenApiTypes.OBJECT)
def get_derivate_settings(self, mapping_version):
return mapping_version.json
+ @extend_schema_field(OpenApiTypes.INT)
def get_mapped_questions(self, mapping_version):
return len(mapping_version.json.get("question_mappings", {}))
+ @extend_schema_field(OpenApiTypes.INT)
def get_total_questions(self, mapping_version):
questions_by_name = mapping_version.form_version.questions_by_name()
return len(questions_by_name)
+ @extend_schema_field(OpenApiTypes.OBJECT)
def get_question_mappings(self, mapping_version):
return mapping_version.json.get("question_mappings", {})
+ @extend_schema_field(MappingVersionMappingSerializer)
def get_mapping(self, mapping_version):
m = mapping_version.mapping
return {"mapping_type": m.mapping_type, "data_source": {"id": m.data_source.id, "name": m.data_source.name}}
+ @extend_schema_field(MappingVersionFormVersionSerializer)
def get_form_version(self, mapping_version):
v = mapping_version.form_version
return {
@@ -77,6 +176,7 @@ def get_form_version(self, mapping_version):
"version_id": v.version_id,
}
+ @extend_schema_field(MappingVersionDatasetSerializer)
def get_dataset(self, mapping_version):
return {
"id": mapping_version.json.get("data_set_id", None),
@@ -141,34 +241,37 @@ def create(self, validated_data: typing.MutableMapping):
return m.MappingVersion.objects.create(mapping=mapping, form_version=form_version, json=validated_data["json"])
def update(self, instance, validated_data):
+ # serialized before any change: question mappings are edited in place in instance.json
+ past_value = serialize_instance(instance)
+
# partial update only question mappings
if "question_mappings" in validated_data:
- for question_name, data_element in validated_data["question_mappings"].items():
- path = "question_mappings." + question_name
-
- if type(data_element) is list:
- instance.json["question_mappings"][question_name] = data_element
+ question_mappings = validated_data["question_mappings"]
+ # empty when the form version has no descriptor: nothing to check against
+ mappable_questions = instance.form_version.mappable_questions_by_name()
+
+ # validate every question mapping first, to report all the errors of an import at once
+ errors = {}
+ for question_name, data_element in question_mappings.items():
+ error = get_question_mapping_error(
+ question_name, data_element, instance.mapping.mapping_type, mappable_questions
+ )
+ if error:
+ errors["question_mappings." + question_name] = error
+ if errors:
+ raise serializers.ValidationError(errors)
+
+ for question_name, data_element in question_mappings.items():
+ if is_unmap(data_element):
+ instance.json["question_mappings"].pop(question_name, None)
else:
- if data_element and data_element.get("action") == "unmap":
- instance.json["question_mappings"].pop(question_name, None)
- continue
-
- if data_element and data_element.get("type") not in (
- MappingVersion.QUESTION_MAPPING_MULTIPLE,
- MappingVersion.QUESTION_MAPPING_NEVER_MAPPED,
- ):
- if data_element.get("id") is None:
- raise serializers.ValidationError({path: "should have a least an data element id"})
-
- if data_element.get("valueType") is None:
- raise serializers.ValidationError({path: "should have a valueType"})
-
- instance.json["question_mappings"][question_name] = data_element
+ instance.json["question_mappings"][question_name] = data_element
if "event_date_source" in validated_data:
instance.json["event_date_source"] = validated_data["event_date_source"]
instance.save()
+ log_modification(past_value, instance, source=MAPPING_VERSION_API, user=self.context["request"].user)
return instance
@@ -201,9 +304,9 @@ def get_queryset(self):
"order", "form_version__form__name,form_version__version_id,mapping__mapping_type"
).split(",")
- profile = self.request.user.iaso_profile
- queryset = MappingVersion.objects.filter(
- form_version_id__in=FormVersion.objects.filter(form__projects__account=profile.account)
+ # the serializer reads form_version.form and mapping.data_source of each row
+ queryset = MappingVersion.objects.filter_for_user(self.request.user).select_related(
+ "form_version__form", "mapping__data_source"
)
search_term = self.request.GET.get("search") or self.request.GET.get("search")
diff --git a/iaso/dhis2/form_mapping.py b/iaso/dhis2/form_mapping.py
index e11dae1477..0e0b554880 100644
--- a/iaso/dhis2/form_mapping.py
+++ b/iaso/dhis2/form_mapping.py
@@ -42,11 +42,13 @@ def copy_mappings_from_previous_version(form_version, previous_form_version):
mapping_version.form_version = form_version
questions_by_name = form_version.questions_by_name()
+ # unlike questions_by_name, also has the questions inside repeat groups
+ mappable_questions = form_version.mappable_questions_by_name()
if "question_mappings" in mapping_version.json:
filtered_question_mappings = {}
for k, v in mapping_version.json["question_mappings"].items():
- if k in questions_by_name:
+ if k in mappable_questions:
# preserve all question mappings except the questions that don't exist in the newer version
filtered_question_mappings[k] = v
elif "__" in k:
diff --git a/iaso/management/commands/seed_test_data.py b/iaso/management/commands/seed_test_data.py
index 8c247f08df..8687be6ffe 100644
--- a/iaso/management/commands/seed_test_data.py
+++ b/iaso/management/commands/seed_test_data.py
@@ -1,10 +1,12 @@
import csv
import json
+import os
from io import BytesIO
-from random import randint, random, sample
+from random import choice, randint, random, sample
from uuid import uuid4
+import openpyxl
import requests
from dhis2 import Api
@@ -239,7 +241,7 @@ def handle(self, *args, **options):
credentials,
user=user,
mapping_type="AGGREGATE",
- mapping_file="./testdata/seed-data-command-form-mapping.json",
+ mapping_file="./testdata/seed-data-command-quality-form-mapping.json",
xls_file="testdata/seed-data-command-form-i18n.xlsx",
)
project.save()
@@ -516,6 +518,45 @@ def seed_geom_code_demo_change_requests(self, datasource, source_version):
run_demo_sync_and_report(datasource, source_version, demo_version, swapped_pair, self.user, self.stdout)
+ @staticmethod
+ def xls_file_for_form(xls_file, form):
+ """The xlsform with settings matching the seeded form, so that it can be downloaded and uploaded back as
+ a new version: the odk form_id of the form (it contains the dhis2 version) and no version, generated on
+ upload."""
+ workbook = openpyxl.load_workbook(xls_file)
+ settings = workbook["settings"] if "settings" in workbook.sheetnames else workbook.create_sheet("settings")
+ headers = [cell.value for cell in settings[1]]
+ while headers and headers[-1] is None:
+ headers.pop()
+
+ def set_setting(names, value):
+ # pyxform also accepts the old id_string alias for form_id
+ column = next((i + 1 for i, header in enumerate(headers) if header in names), None)
+ if column is None:
+ if value is None:
+ return
+ headers.append(names[0])
+ column = len(headers)
+ settings.cell(row=1, column=column, value=names[0])
+ # not cell(..., value=value): openpyxl ignores value=None there
+ settings.cell(row=2, column=column).value = value
+
+ set_setting(("form_id", "id_string"), form.form_id)
+ set_setting(("version",), None)
+
+ buffer = BytesIO()
+ workbook.save(buffer)
+ size = buffer.tell()
+ buffer.seek(0)
+ return InMemoryUploadedFile(
+ buffer,
+ field_name="xls_file",
+ name=os.path.basename(xls_file),
+ content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+ size=size,
+ charset=None,
+ )
+
def seed_form(
self,
form,
@@ -534,7 +575,7 @@ def seed_form(
# TODO: use better fixture
open(xls_xml_file)
)
- form_version.xls_file = UploadedFile(open(xls_file, "rb+"))
+ form_version.xls_file = self.xls_file_for_form(xls_file, form)
form_version.save()
@@ -611,6 +652,8 @@ def seed_instances(
out = OrgUnitType.objects.filter(org_units__version=source_version).distinct()
form.org_unit_types.set(out)
+ questions_by_name = mapping_version.form_version.questions_by_name() if mapping_version else {}
+
for org_unit in source_version.orgunit_set.all():
instances = []
for period in periods:
@@ -645,7 +688,9 @@ def seed_instances(
if mapping_version and "question_mappings" in mapping_version.json:
# quality or quantity
for key in mapping_version.json["question_mappings"]:
- test_data[key] = randint(1, 10)
+ # a choice of the select questions (0/1 for quality), a number for the others
+ choices = questions_by_name.get(key, {}).get("children")
+ test_data[key] = str(choice(choices)["name"]) if choices else randint(1, 10)
instance.json = test_data
instance.form = form
diff --git a/iaso/models/base.py b/iaso/models/base.py
index 1510d542ec..dd552f1637 100644
--- a/iaso/models/base.py
+++ b/iaso/models/base.py
@@ -6,7 +6,7 @@
from django.contrib.auth.models import AnonymousUser, User
from django.contrib.postgres.aggregates import ArrayAgg
from django.db import models
-from django.db.models import Q
+from django.db.models import Exists, OuterRef, Q
from django.utils.functional import cached_property
from django.utils.translation import gettext_lazy as _
from phonenumber_field.modelfields import PhoneNumberField
@@ -371,6 +371,14 @@ def as_dict(self):
}
+class MappingVersionQuerySet(models.QuerySet):
+ def filter_for_user(self, user: User):
+ # Exists() rather than a join on form__projects to avoid duplicate rows (form <-> project is m2m)
+ return self.filter(
+ Exists(Project.objects.filter(account=user.iaso_profile.account, forms=OuterRef("form_version__form")))
+ )
+
+
class MappingVersion(models.Model):
QUESTION_MAPPING_NEVER_MAPPED = "neverMapped"
QUESTION_MAPPING_MULTIPLE = "multiple"
@@ -391,6 +399,8 @@ class MappingVersion(models.Model):
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
+ objects = MappingVersionQuerySet.as_manager()
+
class Meta:
unique_together = [["form_version", "name"]]
diff --git a/iaso/models/forms.py b/iaso/models/forms.py
index 533b44f23e..f1b5cccd59 100644
--- a/iaso/models/forms.py
+++ b/iaso/models/forms.py
@@ -377,6 +377,9 @@ def get_or_save_form_descriptor(self): # TODO: remove me - should be populated
def questions_by_name(self):
return parsing.to_questions_by_name(self.get_or_save_form_descriptor())
+ def mappable_questions_by_name(self):
+ return parsing.to_mappable_questions_by_name(self.get_or_save_form_descriptor())
+
def repeat_groups(self):
questions = self.questions_by_name()
repeats = []
diff --git a/iaso/odk/__init__.py b/iaso/odk/__init__.py
index 0eb724738b..2bf1f4716d 100644
--- a/iaso/odk/__init__.py
+++ b/iaso/odk/__init__.py
@@ -1,4 +1,4 @@
-from .parsing import ParsingError, Survey, parse_xls_form, to_questions_by_name
+from .parsing import ParsingError, Survey, parse_xls_form, to_mappable_questions_by_name, to_questions_by_name
from .validator import validate_xls_form
@@ -6,6 +6,7 @@
"ParsingError",
"Survey",
"parse_xls_form",
+ "to_mappable_questions_by_name",
"to_questions_by_name",
"validate_xls_form",
]
diff --git a/iaso/odk/parsing.py b/iaso/odk/parsing.py
index ad86ad6972..1a2af4a9cc 100644
--- a/iaso/odk/parsing.py
+++ b/iaso/odk/parsing.py
@@ -95,6 +95,40 @@ def to_questions_by_name(form_descriptor):
return questions_by_name
+NOT_MAPPABLE_TYPES = ("survey", "group")
+SELECT_MULTIPLE_TYPE = "select all that apply"
+
+
+def visit_mappable(node, mappable_questions):
+ node_type = node.get("type")
+ node_name = node.get("name")
+ if node_type not in NOT_MAPPABLE_TYPES and node_name is not None:
+ mappable_questions[node_name] = node
+ if node_type == SELECT_MULTIPLE_TYPE:
+ # one boolean data element per choice
+ for choice in node.get("children", []):
+ mappable_questions[f"{node_name}__{choice['name']}"] = choice
+ return
+
+ if node_type in (*NOT_MAPPABLE_TYPES, "repeat"):
+ for child in node.get("children", []):
+ visit_mappable(child, mappable_questions)
+
+
+def to_mappable_questions_by_name(form_descriptor):
+ """Keys a question_mappings entry can use: like to_questions_by_name, plus the questions inside repeat
+ groups (EVENT_TRACKER mappings) and the `question__choice` keys of the select all that apply questions
+ (one boolean data element per choice).
+
+ Kept apart from to_questions_by_name on purpose: its callers (repeat_groups, the form possible fields,
+ the DHIS2 exporter, ...) rely on repeats being a single entry and on choices not being questions."""
+ mappable_questions = {}
+ if not form_descriptor:
+ return mappable_questions
+ visit_mappable(form_descriptor, mappable_questions)
+ return mappable_questions
+
+
def visit_by_path(node, questions_by_name, current_path):
parent = node.get("type", None) is not None and (node["type"] == "survey" or node["type"] == "group")
diff --git a/iaso/tests/api/test_logs.py b/iaso/tests/api/test_logs.py
index 98128d1be8..1e5c17b5f8 100644
--- a/iaso/tests/api/test_logs.py
+++ b/iaso/tests/api/test_logs.py
@@ -1,10 +1,16 @@
from copy import deepcopy
+from unittest import mock
+from django.core.files import File
from rest_framework import status
from hat.audit.models import log_modification
from iaso import models as m
-from iaso.permissions.core_permissions import CORE_FORMS_PERMISSION, CORE_SUBMISSIONS_PERMISSION
+from iaso.permissions.core_permissions import (
+ CORE_FORMS_PERMISSION,
+ CORE_MAPPINGS_PERMISSION,
+ CORE_SUBMISSIONS_PERMISSION,
+)
from iaso.test import APITestCase
@@ -195,3 +201,42 @@ def test_perm_instance(self):
self.client.force_authenticate(user_superuser)
response = self.client.get(f"/api/logs/{modification.id}/")
self.assertJSONResponse(response, status.HTTP_200_OK)
+
+ def test_perm_mapping_version(self):
+ """To read mapping version history the user needs the mappings perm, within its account"""
+ form_file_mock = mock.MagicMock(spec=File)
+ form_file_mock.name = "test.xml"
+ form_version = self.reference_form.form_versions.create(file=form_file_mock, version_id="2020022401")
+ data_source = m.DataSource.objects.create(name="DHIS2")
+ mapping = m.Mapping.objects.create(form=self.reference_form, data_source=data_source, mapping_type=m.AGGREGATE)
+ mapping_version = m.MappingVersion.objects.create(
+ mapping=mapping, form_version=form_version, json={"question_mappings": {}}
+ )
+ modification = log_modification(None, mapping_version, user=self.jane, source="myunittest")
+
+ user_with_mappings_perm = self.create_user_with_profile(
+ username="bob", account=self.ghi, permissions=[CORE_MAPPINGS_PERMISSION]
+ )
+ user_no_mappings_perm = self.create_user_with_profile(username="bob2", account=self.ghi, permissions=[])
+ other_account = m.Account.objects.create(name="other")
+ user_other_account = self.create_user_with_profile(
+ username="jim", account=other_account, permissions=[CORE_MAPPINGS_PERMISSION]
+ )
+ list_url = f"/api/logs/?contentType=iaso.mappingversion&objectId={mapping_version.id}&fields=field_diffs"
+
+ self.client.force_authenticate(user_with_mappings_perm)
+ response = self.client.get(f"/api/logs/{modification.id}/")
+ self.assertJSONResponse(response, status.HTTP_200_OK)
+ response = self.client.get(list_url)
+ r = self.assertJSONResponse(response, status.HTTP_200_OK)
+ self.assertEqual([log["id"] for log in r["list"]], [modification.id])
+
+ for user in (user_no_mappings_perm, user_other_account):
+ with self.subTest(user=user.username):
+ self.client.force_authenticate(user)
+ response = self.client.get(f"/api/logs/{modification.id}/")
+ r = self.assertJSONResponse(response, status.HTTP_401_UNAUTHORIZED)
+ self.assertEqual(r, {"error": "Unauthorized"})
+ response = self.client.get(list_url)
+ r = self.assertJSONResponse(response, status.HTTP_403_FORBIDDEN)
+ self.assertEqual(r, {"error": "Forbidden"})
diff --git a/iaso/tests/api/test_mappingversions.py b/iaso/tests/api/test_mappingversions.py
index 8b9df57521..cc5683bf12 100644
--- a/iaso/tests/api/test_mappingversions.py
+++ b/iaso/tests/api/test_mappingversions.py
@@ -3,16 +3,19 @@
from unittest import mock
from django.core.files import File
-from django.test import override_settings
+from django.test import SimpleTestCase, override_settings
from rest_framework import status
+from hat.audit.models import MAPPING_VERSION_API, Modification
from iaso import models as m
+from iaso.api.mapping_versions import get_question_mapping_shape_error
from iaso.permissions.core_permissions import CORE_MAPPINGS_PERMISSION
-from iaso.test import APITestCase
+from iaso.test import APITestCase, SwaggerTestCaseMixin
+from iaso.tests.utils.query_profiler import QueryProfiler
@override_settings(MEDIA_ROOT=tempfile.mkdtemp())
-class FormsVersionAPITestCase(APITestCase):
+class FormsVersionAPITestCase(SwaggerTestCaseMixin, APITestCase):
@classmethod
def setUpTestData(cls):
cls.maxDiff = None
@@ -263,13 +266,13 @@ def test_mappingversions_list_filters_mapping_types(self):
resp = self.client.get("/api/mappingversions/")
- self.assertEqual(len(resp.json()["mapping_versions"]), 1)
+ self.assertMappingVersionsCount(resp, 1)
resp = self.client.get("/api/mappingversions/?mappingTypes=EVENT")
- self.assertEqual(len(resp.json()["mapping_versions"]), 0)
+ self.assertMappingVersionsCount(resp, 0)
resp = self.client.get("/api/mappingversions/?mappingTypes=AGGREGATE")
- self.assertEqual(len(resp.json()["mapping_versions"]), 1)
+ self.assertMappingVersionsCount(resp, 1)
def test_mappingversions_list_filters_form_id(self):
self.client.force_authenticate(self.yoda)
@@ -279,13 +282,13 @@ def test_mappingversions_list_filters_form_id(self):
resp = self.client.get("/api/mappingversions/")
- self.assertEqual(len(resp.json()["mapping_versions"]), 1)
+ self.assertMappingVersionsCount(resp, 1)
resp = self.client.get(f"/api/mappingversions/?formId={self.form_1.id}")
- self.assertEqual(len(resp.json()["mapping_versions"]), 0)
+ self.assertMappingVersionsCount(resp, 0)
resp = self.client.get(f"/api/mappingversions/?formId={self.form_2.id}")
- self.assertEqual(len(resp.json()["mapping_versions"]), 1)
+ self.assertMappingVersionsCount(resp, 1)
def test_mappingversions_list_filters_org_unit_type_ids(self):
self.client.force_authenticate(self.yoda)
@@ -297,16 +300,16 @@ def test_mappingversions_list_filters_org_unit_type_ids(self):
resp = self.client.get("/api/mappingversions/")
- self.assertEqual(len(resp.json()["mapping_versions"]), 1)
+ self.assertMappingVersionsCount(resp, 1)
resp = self.client.get(f"/api/mappingversions/?orgUnitTypeIds={self.sith_council.id}")
- self.assertEqual(len(resp.json()["mapping_versions"]), 1)
+ self.assertMappingVersionsCount(resp, 1)
resp = self.client.get(f"/api/mappingversions/?orgUnitTypeIds={other_org_unit_type.id}")
- self.assertEqual(len(resp.json()["mapping_versions"]), 0)
+ self.assertMappingVersionsCount(resp, 0)
resp = self.client.get(f"/api/mappingversions/?orgUnitTypeIds={other_org_unit_type.id},{self.sith_council.id}")
- self.assertEqual(len(resp.json()["mapping_versions"]), 1)
+ self.assertMappingVersionsCount(resp, 1)
def test_mappingversions_list_filters_project_ids(self):
self.client.force_authenticate(self.yoda)
@@ -320,16 +323,428 @@ def test_mappingversions_list_filters_project_ids(self):
resp = self.client.get("/api/mappingversions/")
- self.assertEqual(len(resp.json()["mapping_versions"]), 1)
+ self.assertMappingVersionsCount(resp, 1)
resp = self.client.get(f"/api/mappingversions/?projectsIds={self.project.id}")
- self.assertEqual(len(resp.json()["mapping_versions"]), 1)
+ self.assertMappingVersionsCount(resp, 1)
resp = self.client.get(f"/api/mappingversions/?projectsIds={other_project.id}")
- self.assertEqual(len(resp.json()["mapping_versions"]), 0)
+ self.assertMappingVersionsCount(resp, 0)
resp = self.client.get(f"/api/mappingversions/?projectsIds={other_project.id},{self.project.id}")
- self.assertEqual(len(resp.json()["mapping_versions"]), 1)
+ self.assertMappingVersionsCount(resp, 1)
+
+ def test_mappingversions_not_visible_from_other_account(self):
+ self.client.force_authenticate(self.yoda)
+ form_version = self.create_form_version()
+ mapping_version = self.create_mapping_version(form_version, self.sw_source)
+
+ self.client.force_authenticate(self.batman)
+ resp = self.client.get("/api/mappingversions/")
+ self.assertMappingVersionsCount(resp, 0)
+
+ resp = self.client.get(f"/api/mappingversions/{mapping_version['id']}/")
+ self.assertJSONResponse(resp, status.HTTP_404_NOT_FOUND)
+
+ self.assertEqual(m.MappingVersion.objects.filter_for_user(self.batman).count(), 0)
+
+ def test_mappingversions_no_duplicates_when_form_in_several_projects(self):
+ """form <-> project is m2m: a form shared by 2 projects of the account must not duplicate rows"""
+ self.client.force_authenticate(self.yoda)
+ form_version = self.create_form_version()
+ self.create_mapping_version(form_version, self.sw_source)
+
+ other_project = m.Project.objects.create(
+ name="Cloning facility", app_id="stars.empire.cloning", account=self.star_wars
+ )
+ other_project.forms.add(form_version.form)
+
+ resp = self.client.get("/api/mappingversions/")
+ self.assertMappingVersionsCount(resp, 1)
+
+ self.assertEqual(m.MappingVersion.objects.filter_for_user(self.yoda).count(), 1)
+
+ def test_mappingversions_bulk_patch(self):
+ """PATCH /mappingversions/: several question mappings at once, as the import wizard does, then
+ restoring and unmapping some of them"""
+
+ self.client.force_authenticate(self.yoda)
+ form_version = self.create_form_version()
+ mapping_version_id = self.create_mapping_version(form_version, self.sw_source)["id"]
+
+ original = {
+ "question_1": {"id": "de1", "valueType": "NUMBER", "categoryOptionCombo": "coc1"},
+ "question_2": {"id": "de2", "valueType": "NUMBER", "categoryOptionCombo": "coc1"},
+ "question_6": {"type": "neverMapped"},
+ }
+ response = self.patch_question_mappings(mapping_version_id, original)
+ self.assertJSONResponse(response, status.HTTP_200_OK)
+
+ imported = {
+ # overwrite
+ "question_2": {"id": "de2bis", "valueType": "INTEGER", "categoryOptionCombo": "coc2"},
+ # overwrite a never mapped marker
+ "question_6": {"id": "de6", "valueType": "NUMBER"},
+ # select all that apply, no data element id at the top level
+ "question_3": {"type": "multiple", "values": {"a": {"id": "de3a", "valueType": "BOOLEAN"}}},
+ }
+ response = self.patch_question_mappings(mapping_version_id, imported)
+ self.assertJSONResponse(response, status.HTTP_200_OK)
+ self.assertEqual(self.get_question_mappings(mapping_version_id), {**original, **imported})
+
+ restore = {
+ "question_2": original["question_2"],
+ "question_6": {"type": "neverMapped"},
+ "question_3": {"action": "unmap"},
+ }
+ response = self.patch_question_mappings(mapping_version_id, restore)
+ self.assertJSONResponse(response, status.HTTP_200_OK)
+ self.assertEqual(self.get_question_mappings(mapping_version_id), original)
+
+ def test_mappingversions_bulk_patch_event_tracker(self):
+ """PATCH /mappingversions/: event tracker mappings are lists, repeat groups included"""
+
+ self.client.force_authenticate(self.yoda)
+ form_version = self.create_form_version()
+ mapping_version_id = self.create_tracker_mapping_version(form_version, self.sw_source)
+ original = {"question_1": [{"trackedEntityAttribute": {"id": "tea1"}, "iaso_field": "instance.uuid"}]}
+ self.patch_question_mappings(mapping_version_id, original)
+
+ imported = {
+ # event tracker question inside a repeat group
+ "question_4": [{"dataElement": {"id": "de4"}, "programStage": "stage1", "parent": "question_5"}],
+ # event tracker repeat group
+ "question_5": [
+ {
+ "type": "repeat",
+ "program_id": "program1",
+ "tracked_entity_type": "tet1",
+ "tracked_entity_identifier": "uid",
+ "relationship_type": "rel1",
+ }
+ ],
+ }
+ response = self.patch_question_mappings(mapping_version_id, imported)
+ self.assertJSONResponse(response, status.HTTP_200_OK)
+ self.assertEqual(self.get_question_mappings(mapping_version_id), {**original, **imported})
+
+ response = self.patch_question_mappings(
+ mapping_version_id, {"question_4": {"action": "unmap"}, "question_5": {"action": "unmap"}}
+ )
+ self.assertJSONResponse(response, status.HTTP_200_OK)
+ self.assertEqual(self.get_question_mappings(mapping_version_id), original)
+
+ def test_mappingversions_patch_checks_shape_for_mapping_type(self):
+ """PATCH /mappingversions/: a question mapping must have the shape the exporter of its type reads"""
+
+ self.client.force_authenticate(self.yoda)
+ form_version = self.create_form_version()
+ aggregate_id = self.create_mapping_version(form_version, self.sw_source)["id"]
+ tracker_id = self.create_tracker_mapping_version(form_version, self.sw_source)
+ data_element = {"id": "de1", "valueType": "NUMBER"}
+
+ cases = [
+ (aggregate_id, [{"dataElement": {"id": "de1"}}], "should not be a list for AGGREGATE mappings"),
+ (
+ aggregate_id,
+ {"type": "multiple", "values": {"a": {"name": "no id"}}},
+ "should map each choice to a data element id",
+ ),
+ (aggregate_id, {"type": "multiple"}, "should map each choice to a data element id"),
+ (tracker_id, data_element, "should be a list for EVENT_TRACKER mappings"),
+ (tracker_id, [], "should be a list for EVENT_TRACKER mappings"),
+ (tracker_id, ["de1"], "should only contain objects"),
+ (tracker_id, [{"foo": 1}], "should map a data element, a tracked entity attribute or a repeat group"),
+ (
+ tracker_id,
+ [{"type": "repeat"}],
+ "should map a data element, a tracked entity attribute or a repeat group",
+ ),
+ ]
+ for mapping_version_id, question_mapping, error in cases:
+ with self.subTest(question_mapping=question_mapping):
+ response = self.patch_question_mappings(mapping_version_id, {"question_1": question_mapping})
+ result = self.assertJSONResponse(response, status.HTTP_400_BAD_REQUEST)
+ self.assertEqual(result, {"question_mappings.question_1": error})
+
+ # never mapped markers and unmapping are valid for every mapping type
+ for mapping_version_id in (aggregate_id, tracker_id):
+ response = self.patch_question_mappings(mapping_version_id, {"question_1": {"type": "neverMapped"}})
+ self.assertJSONResponse(response, status.HTTP_200_OK)
+ response = self.patch_question_mappings(mapping_version_id, {"question_1": {"action": "unmap"}})
+ self.assertJSONResponse(response, status.HTTP_200_OK)
+ self.assertEqual(self.get_question_mappings(mapping_version_id), {})
+
+ def test_mappingversions_bulk_patch_invalid_entry_saves_nothing(self):
+ """PATCH /mappingversions/: one invalid question mapping rejects the whole import"""
+
+ self.client.force_authenticate(self.yoda)
+ form_version = self.create_form_version()
+ mapping_version_id = self.create_mapping_version(form_version, self.sw_source)["id"]
+ original = {"question_1": {"id": "de1", "valueType": "NUMBER"}}
+ self.patch_question_mappings(mapping_version_id, original)
+
+ response = self.patch_question_mappings(
+ mapping_version_id,
+ {
+ "question_1": {"action": "unmap"},
+ "question_2": {"id": "de2", "valueType": "NUMBER"},
+ "question_3": {"id": "de3"},
+ },
+ )
+
+ result = self.assertJSONResponse(response, status.HTTP_400_BAD_REQUEST)
+ self.assertEqual(result, {"question_mappings.question_3": "should have a valueType"})
+ self.assertEqual(self.get_question_mappings(mapping_version_id), original)
+ self.assertEqual(Modification.objects.filter(object_id=mapping_version_id).count(), 1)
+
+ def test_mappingversions_bulk_patch_reports_all_errors(self):
+ """PATCH /mappingversions/: every invalid question mapping of an import is reported, not only the first"""
+
+ self.client.force_authenticate(self.yoda)
+ form_version = self.create_form_version()
+ mapping_version_id = self.create_mapping_version(form_version, self.sw_source)["id"]
+
+ response = self.patch_question_mappings(
+ mapping_version_id,
+ {
+ "question_1": {"valueType": "NUMBER"},
+ "question_2": {"id": "de2", "valueType": "NUMBER"},
+ "question_3": {"id": "de3"},
+ "question_4": [{"dataElement": {"id": "de4"}}],
+ },
+ )
+
+ result = self.assertJSONResponse(response, status.HTTP_400_BAD_REQUEST)
+ self.assertEqual(
+ result,
+ {
+ "question_mappings.question_1": "should have a least an data element id",
+ "question_mappings.question_3": "should have a valueType",
+ "question_mappings.question_4": "should not be a list for AGGREGATE mappings",
+ },
+ )
+ self.assertEqual(self.get_question_mappings(mapping_version_id), {})
+
+ def test_mappingversions_patch_logs_modification(self):
+ """PATCH /mappingversions/: each patch is audited with the question mappings before and after"""
+
+ self.client.force_authenticate(self.yoda)
+ form_version = self.create_form_version()
+ mapping_version_id = self.create_mapping_version(form_version, self.sw_source)["id"]
+ original = {"question_1": {"id": "de1", "valueType": "NUMBER"}}
+ self.patch_question_mappings(mapping_version_id, original)
+
+ imported = {
+ "question_1": {"id": "de1bis", "valueType": "INTEGER"},
+ "question_2": {"type": "neverMapped"},
+ }
+ response = self.patch_question_mappings(mapping_version_id, imported)
+ self.assertJSONResponse(response, status.HTTP_200_OK)
+
+ modifications = Modification.objects.filter(object_id=mapping_version_id).order_by("id")
+ self.assertEqual(modifications.count(), 2)
+ modification = modifications.last()
+ self.assertEqual(modification.source, MAPPING_VERSION_API)
+ self.assertEqual(modification.user, self.yoda)
+ self.assertEqual(modification.content_object, m.MappingVersion.objects.get(id=mapping_version_id))
+ self.assertEqual(modification.past_value[0]["fields"]["json"]["question_mappings"], original)
+ self.assertEqual(modification.new_value[0]["fields"]["json"]["question_mappings"], imported)
+
+ def test_mappingversions_list_import_sources_fields(self):
+ """GET /mappingversions/ with the fields the import wizard needs to list its sources"""
+
+ self.client.force_authenticate(self.yoda)
+ form_version = self.create_form_version()
+ mapping_version_id = self.create_mapping_version(form_version, self.sw_source)["id"]
+ question_mappings = {"question_1": {"id": "de1", "valueType": "NUMBER"}}
+ self.patch_question_mappings(mapping_version_id, question_mappings)
+
+ response = self.client.get(
+ "/api/mappingversions/",
+ {
+ "mappingTypes": "AGGREGATE",
+ "fields": "id,form_version,mapping,question_mappings,derivate_settings,updated_at",
+ },
+ )
+
+ mapping_versions = self.assertJSONResponse(response, status.HTTP_200_OK)["mapping_versions"]
+ self.assertEqual(len(mapping_versions), 1)
+ mapping_version = mapping_versions[0]
+ self.assertEqual(
+ set(mapping_version.keys()),
+ {"id", "form_version", "mapping", "question_mappings", "derivate_settings", "updated_at"},
+ )
+ self.assertEqual(mapping_version["question_mappings"], question_mappings)
+ self.assertEqual(mapping_version["derivate_settings"]["data_set_id"], "ERTFDG")
+ self.assertEqual(mapping_version["mapping"]["data_source"]["id"], self.sw_source.id)
+ self.assertEqual(mapping_version["form_version"]["form"]["id"], form_version.form_id)
+
+ def test_mappingversions_list_query_count(self):
+ """GET /mappingversions/: no query per row, for the mapping list and the import wizard sources"""
+
+ mapping, _ = m.Mapping.objects.get_or_create(
+ form=self.form_1, data_source=self.sw_source, mapping_type=m.AGGREGATE
+ )
+ descriptor = {"name": "data", "type": "survey", "children": [{"name": "weight", "type": "decimal"}]}
+ for version_id in ("1", "2", "3"):
+ form_version = m.FormVersion.objects.create(
+ form=self.form_1, version_id=version_id, form_descriptor=descriptor
+ )
+ m.MappingVersion.objects.create(
+ mapping=mapping,
+ form_version=form_version,
+ name="aggregate",
+ json={"question_mappings": {"weight": {"id": "de1", "valueType": "NUMBER"}}, "data_set_id": "ds1"},
+ )
+ self.client.force_authenticate(self.yoda)
+
+ urls = {
+ "mapping list": f"/api/mappingversions/?formId={self.form_1.id}",
+ "import sources": (
+ f"/api/mappingversions/?formId={self.form_1.id}&mappingTypes=AGGREGATE"
+ "&fields=id,form_version,mapping,question_mappings,derivate_settings,updated_at"
+ ),
+ }
+ for label, url in urls.items():
+ with self.subTest(label):
+ with QueryProfiler(
+ trace_tables=["iaso_formversion", "iaso_form", "iaso_mapping", "iaso_datasource"]
+ ) as profiler:
+ response = self.client.get(url)
+ self.assertMappingVersionsCount(response, 3)
+
+ # a single query for the 3 rows: form version, form, mapping and data source are joined (the
+ # profiler only counts the FROM table), iaso_project is the EXISTS of filter_for_user in that
+ # same query. auth_permission is the permission check, not related to the rows.
+ with profiler.report_on_failure(f"mappingversions_{label.replace(' ', '_')}.md", title=label):
+ profiler.assertLessEqualQueryCount(
+ {
+ "iaso_mappingversion": 1,
+ "iaso_project": 1,
+ "iaso_formversion": 0,
+ "iaso_form": 0,
+ "iaso_mapping": 0,
+ "iaso_datasource": 0,
+ },
+ exclude=["auth_permission"],
+ )
+
+ def test_mappingversions_responses_compliant_to_swagger(self):
+ """POST, PATCH, GET and list /mappingversions/ responses match the OpenAPI schema"""
+
+ self.client.force_authenticate(self.yoda)
+ form_version = self.create_form_version()
+ # the schema lists every field, the responses only the default ones unless asked for all
+ response = self.client.post(
+ "/api/mappingversions/?fields=:all",
+ data={
+ "form_version": {"id": form_version.id},
+ "mapping": {"type": "AGGREGATE", "datasource": {"id": self.sw_source.id}},
+ "dataset": {"id": "ERTFDG", "name": "My dataset name"},
+ },
+ format="json",
+ )
+ created = self.assertJSONResponse(response, status.HTTP_201_CREATED)
+ self.assertResponseCompliantToSwagger(created, "MappingVersion")
+
+ question_mappings = {
+ "question_1": {"id": "de1", "valueType": "NUMBER"},
+ "question_3": {"type": "multiple", "values": {"a": {"id": "de3a", "valueType": "BOOLEAN"}}},
+ "question_6": {"type": "neverMapped"},
+ }
+ response = self.client.patch(
+ f"/api/mappingversions/{created['id']}/?fields=:all",
+ data={"question_mappings": question_mappings},
+ format="json",
+ )
+ patched = self.assertJSONResponse(response, status.HTTP_200_OK)
+ self.assertResponseCompliantToSwagger(patched, "MappingVersion")
+
+ response = self.client.get(f"/api/mappingversions/{created['id']}/?fields=:all")
+ retrieved = self.assertJSONResponse(response, status.HTTP_200_OK)
+ self.assertResponseCompliantToSwagger(retrieved, "MappingVersion")
+ self.assertEqual(retrieved["question_mappings"], question_mappings)
+
+ response = self.client.get("/api/mappingversions/?fields=:all")
+ listed = self.assertJSONResponse(response, status.HTTP_200_OK)["mapping_versions"]
+ self.assertResponseCompliantToSwagger(listed, "MappingVersion", as_array=True)
+ self.assertEqual([mapping_version["id"] for mapping_version in listed], [created["id"]])
+
+ def test_mappingversions_patch_checks_questions_exist(self):
+ """PATCH /mappingversions/: question mappings must target a question of the form version"""
+
+ self.client.force_authenticate(self.yoda)
+ form_version = self.create_form_version()
+ form_version.form_descriptor = {
+ "name": "data",
+ "type": "survey",
+ "children": [
+ {"name": "weight", "type": "decimal"},
+ {"name": "household", "type": "repeat", "children": [{"name": "age", "type": "integer"}]},
+ {"name": "symptoms", "type": "select all that apply", "children": [{"name": "fever"}]},
+ {"name": "sex", "type": "select one", "children": [{"name": "male"}]},
+ ],
+ }
+ form_version.save()
+ mapping_version_id = self.create_mapping_version(form_version, self.sw_source)["id"]
+ data_element = {"id": "de1", "valueType": "NUMBER"}
+
+ response = self.patch_question_mappings(mapping_version_id, {"unknown": data_element})
+ result = self.assertJSONResponse(response, status.HTTP_400_BAD_REQUEST)
+ self.assertEqual(result, {"question_mappings.unknown": "question does not exist in this form version"})
+
+ response = self.patch_question_mappings(mapping_version_id, {"unknown": {"type": "neverMapped"}})
+ self.assertJSONResponse(response, status.HTTP_400_BAD_REQUEST)
+
+ # choice keys only exist for select all that apply questions (one boolean data element per choice),
+ # even though copy_mappings_from_previous_version still copies the select one ones
+ response = self.patch_question_mappings(mapping_version_id, {"sex__male": data_element})
+ result = self.assertJSONResponse(response, status.HTTP_400_BAD_REQUEST)
+ self.assertEqual(result, {"question_mappings.sex__male": "question does not exist in this form version"})
+
+ valid = {
+ "weight": data_element,
+ "symptoms": {"type": "multiple", "values": {"fever": data_element}},
+ "symptoms__fever": data_element,
+ }
+ response = self.patch_question_mappings(mapping_version_id, valid)
+ self.assertJSONResponse(response, status.HTTP_200_OK)
+ self.assertEqual(self.get_question_mappings(mapping_version_id), valid)
+
+ tracker_id = self.create_tracker_mapping_version(form_version, self.sw_source)
+ valid_tracker = {
+ "household": [{"type": "repeat", "program_id": "p1"}],
+ # inside the repeat group
+ "age": [{"dataElement": {"id": "de2"}, "programStage": "s1", "parent": "household"}],
+ }
+ response = self.patch_question_mappings(tracker_id, valid_tracker)
+ self.assertJSONResponse(response, status.HTTP_200_OK)
+ self.assertEqual(self.get_question_mappings(tracker_id), valid_tracker)
+
+ # a mapping left over for a question removed from the form can still be unmapped
+ mapping_version = m.MappingVersion.objects.get(id=mapping_version_id)
+ mapping_version.json["question_mappings"]["removed_question"] = data_element
+ mapping_version.save()
+ response = self.patch_question_mappings(mapping_version_id, {"removed_question": {"action": "unmap"}})
+ self.assertJSONResponse(response, status.HTTP_200_OK)
+ self.assertEqual(self.get_question_mappings(mapping_version_id), valid)
+
+ def assertMappingVersionsCount(self, response, expected_count):
+ result = self.assertJSONResponse(response, status.HTTP_200_OK)
+ self.assertEqual(len(result["mapping_versions"]), expected_count)
+
+ def patch_question_mappings(self, mapping_version_id, question_mappings):
+ return self.client.patch(
+ f"/api/mappingversions/{mapping_version_id}/",
+ data={"question_mappings": question_mappings},
+ format="json",
+ headers={"accept": "application/json"},
+ )
+
+ def get_question_mappings(self, mapping_version_id):
+ response = self.client.get(f"/api/mappingversions/{mapping_version_id}/?fields=:all")
+ return self.assertJSONResponse(response, status.HTTP_200_OK)["question_mappings"]
def create_form_version(self):
with open("iaso/tests/fixtures/odk_form_valid_sample1_2020022401.xlsx", "rb") as xls_file:
@@ -342,6 +757,18 @@ def create_form_version(self):
return m.FormVersion.objects.all()[0]
+ def create_tracker_mapping_version(self, form_version, source):
+ # created directly: the API names every mapping version "", and (form_version, name) is unique
+ mapping, _ = m.Mapping.objects.get_or_create(
+ form=form_version.form, data_source=source, mapping_type=m.EVENT_TRACKER
+ )
+ return m.MappingVersion.objects.create(
+ mapping=mapping,
+ form_version=form_version,
+ name="tracker",
+ json={"question_mappings": {}, "program_id": "PRGRM"},
+ ).id
+
def create_mapping_version(self, form_version, source):
resp = self.client.post(
"/api/mappingversions/",
@@ -358,3 +785,57 @@ def create_mapping_version(self, form_version, source):
format="json",
headers={"accept": "application/json"},
).json()
+
+
+class QuestionMappingShapeErrorTestCase(SimpleTestCase):
+ """get_question_mapping_shape_error, without going through the API"""
+
+ def test_valid_shapes(self):
+ cases = [
+ (m.AGGREGATE, {"id": "de1", "valueType": "NUMBER"}),
+ (m.EVENT, {"id": "de1", "valueType": "NUMBER"}),
+ (m.AGGREGATE, {"type": "multiple", "values": {"a": {"id": "de1"}, "b": {"id": "de2"}}}),
+ (m.EVENT_TRACKER, [{"dataElement": {"id": "de1"}, "programStage": "stage1"}]),
+ (m.EVENT_TRACKER, [{"trackedEntityAttribute": {"id": "tea1"}}]),
+ (m.EVENT_TRACKER, [{"type": "repeat", "program_id": "program1"}]),
+ # never mapped markers are valid for every mapping type
+ (m.AGGREGATE, {"type": "neverMapped"}),
+ (m.EVENT_TRACKER, {"type": "neverMapped"}),
+ ]
+ for mapping_type, data_element in cases:
+ with self.subTest(mapping_type=mapping_type, data_element=data_element):
+ self.assertIsNone(get_question_mapping_shape_error(mapping_type, data_element))
+
+ def test_invalid_shapes(self):
+ cases = [
+ (m.AGGREGATE, [{"dataElement": {"id": "de1"}}], "should not be a list for AGGREGATE mappings"),
+ (m.EVENT, [{"dataElement": {"id": "de1"}}], "should not be a list for EVENT mappings"),
+ (m.AGGREGATE, {"type": "multiple"}, "should map each choice to a data element id"),
+ (m.AGGREGATE, {"type": "multiple", "values": []}, "should map each choice to a data element id"),
+ (
+ m.AGGREGATE,
+ {"type": "multiple", "values": {"a": {"id": "de1"}, "b": {"name": "no id"}}},
+ "should map each choice to a data element id",
+ ),
+ (m.EVENT_TRACKER, {"id": "de1", "valueType": "NUMBER"}, "should be a list for EVENT_TRACKER mappings"),
+ (m.EVENT_TRACKER, [], "should be a list for EVENT_TRACKER mappings"),
+ (m.EVENT_TRACKER, ["de1"], "should only contain objects"),
+ (
+ m.EVENT_TRACKER,
+ [{"dataElement": {"name": "no id"}}],
+ "should map a data element, a tracked entity attribute or a repeat group",
+ ),
+ (
+ m.EVENT_TRACKER,
+ [{"type": "repeat"}],
+ "should map a data element, a tracked entity attribute or a repeat group",
+ ),
+ (
+ m.EVENT_TRACKER,
+ [{"dataElement": {"id": "de1"}}, {"foo": 1}],
+ "should map a data element, a tracked entity attribute or a repeat group",
+ ),
+ ]
+ for mapping_type, data_element, error in cases:
+ with self.subTest(mapping_type=mapping_type, data_element=data_element):
+ self.assertEqual(get_question_mapping_shape_error(mapping_type, data_element), error)
diff --git a/iaso/tests/dhis2/tests_form_mapping.py b/iaso/tests/dhis2/tests_form_mapping.py
new file mode 100644
index 0000000000..a5a4fe2eb7
--- /dev/null
+++ b/iaso/tests/dhis2/tests_form_mapping.py
@@ -0,0 +1,77 @@
+from iaso import models as m
+from iaso.dhis2.form_mapping import copy_mappings_from_previous_version
+from iaso.test import TestCase
+
+
+DESCRIPTOR = {
+ "name": "data",
+ "type": "survey",
+ "children": [
+ {"name": "weight", "type": "decimal"},
+ {
+ "name": "household",
+ "type": "repeat",
+ "children": [{"name": "age", "type": "integer"}],
+ },
+ {
+ "name": "symptoms",
+ "type": "select all that apply",
+ "children": [{"name": "fever"}, {"name": "cough"}],
+ },
+ {"name": "sex", "type": "select one", "children": [{"name": "male"}]},
+ ],
+}
+
+
+class CopyMappingsFromPreviousVersionTestCase(TestCase):
+ @classmethod
+ def setUpTestData(cls):
+ account = m.Account.objects.create(name="account")
+ cls.data_source = m.DataSource.objects.create(name="dhis2")
+ cls.form = m.Form.objects.create(name="form")
+ project = m.Project.objects.create(name="project", app_id="app", account=account)
+ project.forms.add(cls.form)
+
+ def test_keeps_mappings_of_questions_still_in_the_new_version(self):
+ previous = m.FormVersion.objects.create(form=self.form, version_id="1", form_descriptor=DESCRIPTOR)
+ new_descriptor = {
+ **DESCRIPTOR,
+ # "weight" and the "cough" choice were removed
+ "children": [
+ DESCRIPTOR["children"][1],
+ {**DESCRIPTOR["children"][2], "children": [{"name": "fever"}]},
+ DESCRIPTOR["children"][3],
+ ],
+ }
+ new = m.FormVersion.objects.create(form=self.form, version_id="2", form_descriptor=new_descriptor)
+ mapping = m.Mapping.objects.create(form=self.form, data_source=self.data_source, mapping_type=m.EVENT_TRACKER)
+ repeat_mapping = [{"type": "repeat", "program_id": "p1", "relationship_type": "r1"}]
+ age_mapping = [{"dataElement": {"id": "de_age"}, "programStage": "s1", "parent": "household"}]
+ m.MappingVersion.objects.create(
+ mapping=mapping,
+ form_version=previous,
+ json={
+ "question_mappings": {
+ "weight": {"id": "de_weight", "valueType": "NUMBER"},
+ "household": repeat_mapping,
+ "age": age_mapping,
+ "symptoms__fever": {"id": "de_fever", "valueType": "BOOLEAN"},
+ "symptoms__cough": {"id": "de_cough", "valueType": "BOOLEAN"},
+ "sex__male": {"id": "de_male", "valueType": "BOOLEAN"},
+ }
+ },
+ )
+
+ copy_mappings_from_previous_version(new, previous)
+
+ self.assertEqual(
+ new.mapping_versions.get().json["question_mappings"],
+ {
+ "household": repeat_mapping,
+ # inside the repeat group
+ "age": age_mapping,
+ "symptoms__fever": {"id": "de_fever", "valueType": "BOOLEAN"},
+ # choice keys are kept for any question with choices, as before
+ "sex__male": {"id": "de_male", "valueType": "BOOLEAN"},
+ },
+ )
diff --git a/iaso/tests/models/test_mapping_version.py b/iaso/tests/models/test_mapping_version.py
new file mode 100644
index 0000000000..9b2103359e
--- /dev/null
+++ b/iaso/tests/models/test_mapping_version.py
@@ -0,0 +1,52 @@
+from iaso import models as m
+from iaso.test import TestCase
+
+
+class MappingVersionQuerySetTestCase(TestCase):
+ @classmethod
+ def setUpTestData(cls):
+ cls.account = m.Account.objects.create(name="Belgium")
+ cls.other_account = m.Account.objects.create(name="France")
+ cls.project_1 = m.Project.objects.create(
+ name="Belgium health 1", app_id="belgium.health.1", account=cls.account
+ )
+ cls.project_2 = m.Project.objects.create(
+ name="Belgium health 2", app_id="belgium.health.2", account=cls.account
+ )
+ cls.other_project = m.Project.objects.create(
+ name="France health", app_id="france.health", account=cls.other_account
+ )
+ cls.user = cls.create_user_with_profile(username="belgium_user", account=cls.account)
+ cls.data_source = m.DataSource.objects.create(name="DHIS2")
+
+ def create_mapping_version(self, form_name):
+ form = m.Form.objects.create(name=form_name)
+ form_version = m.FormVersion.objects.create(form=form, version_id="1")
+ mapping = m.Mapping.objects.create(form=form, data_source=self.data_source, mapping_type=m.AGGREGATE)
+ mapping_version = m.MappingVersion.objects.create(
+ mapping=mapping, form_version=form_version, name="aggregate", json={"question_mappings": {}}
+ )
+ return form, mapping_version
+
+ def test_filter_for_user_no_duplicates_when_form_in_several_projects(self):
+ form, mapping_version = self.create_mapping_version("Shared form")
+ form.projects.set([self.project_1, self.project_2])
+
+ result = m.MappingVersion.objects.filter_for_user(self.user)
+
+ self.assertQuerySetEqual(result, [mapping_version])
+
+ def test_filter_for_user_excludes_forms_of_other_accounts(self):
+ form, _ = self.create_mapping_version("Other account form")
+ form.projects.set([self.other_project])
+
+ result = m.MappingVersion.objects.filter_for_user(self.user)
+
+ self.assertQuerySetEqual(result, [])
+
+ def test_filter_for_user_excludes_forms_without_project(self):
+ self.create_mapping_version("Orphan form")
+
+ result = m.MappingVersion.objects.filter_for_user(self.user)
+
+ self.assertQuerySetEqual(result, [])
diff --git a/iaso/tests/odk/test_parsing.py b/iaso/tests/odk/test_parsing.py
index c378c3e747..ad2dda9cd1 100644
--- a/iaso/tests/odk/test_parsing.py
+++ b/iaso/tests/odk/test_parsing.py
@@ -5,7 +5,7 @@
from django.test import SimpleTestCase
from django.utils.dateparse import parse_datetime
-from iaso.odk import ParsingError, Survey, parse_xls_form, to_questions_by_name
+from iaso.odk import ParsingError, Survey, parse_xls_form, to_mappable_questions_by_name, to_questions_by_name
class ParsingTestCase(SimpleTestCase):
@@ -174,3 +174,51 @@ def test_to_questions_by_name_with_node_without_children(self):
{"name": "parent", "type": "survey", "children": [{"name": "group_without_children", "type": "group"}]}
)
self.assertEqual({}, flattened)
+
+ def test_to_mappable_questions_by_name(self):
+ descriptor = {
+ "name": "data",
+ "type": "survey",
+ "choices": {"yes_no": [{"name": "yes"}, {"name": "no"}]},
+ "children": [
+ {"name": "weight", "type": "decimal"},
+ {"name": "grp", "type": "group", "children": [{"name": "in_group", "type": "text"}]},
+ {"name": "household", "type": "repeat", "children": [{"name": "age", "type": "integer"}]},
+ {"name": "symptoms", "type": "select all that apply", "children": [{"name": "fever"}]},
+ # pyxform 2.x: the choices list is referenced and still inlined as children
+ {
+ "name": "answers",
+ "type": "select all that apply",
+ "itemset": "yes_no",
+ "list_name": "yes_no",
+ "children": [{"name": "yes"}, {"name": "no"}],
+ },
+ {"name": "sex", "type": "select one", "children": [{"name": "male"}]},
+ {"name": "meta", "type": "group", "children": [{"name": "instanceID", "type": "calculate"}]},
+ ],
+ }
+
+ mappable = to_mappable_questions_by_name(descriptor)
+
+ self.assertEqual(
+ sorted(mappable.keys()),
+ [
+ "age",
+ "answers",
+ "answers__no",
+ "answers__yes",
+ "household",
+ "in_group",
+ "instanceID",
+ "sex",
+ "symptoms",
+ "symptoms__fever",
+ "weight",
+ ],
+ )
+ # to_questions_by_name is unchanged: no repeat children nor choices
+ self.assertNotIn("age", to_questions_by_name(descriptor))
+
+ def test_to_mappable_questions_by_name_with_empty_descriptor(self):
+ self.assertEqual(to_mappable_questions_by_name({}), {})
+ self.assertEqual(to_mappable_questions_by_name(None), {})
diff --git a/mkdocs.yml b/mkdocs.yml
index dafb9c2a61..b86db73fda 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -119,6 +119,7 @@ nav: # nav configures the side bar menu - this is the ONLY place the nav tree is
- IASO and DHIS2:
- ./pages/users/how_to/setup_dhis2_login_in_iaso.md
- ./pages/users/how_to/deduce_form_from_another_form.md
+ - ./pages/users/how_to/import_export_dhis2_mappings.md
- FAQ: ./pages/users/FAQ/faq.md
- Developers:
- References:
diff --git a/testdata/seed-data-command-form-i18n.xlsx b/testdata/seed-data-command-form-i18n.xlsx
index 8f87aae4c3..d2bef1dd71 100644
Binary files a/testdata/seed-data-command-form-i18n.xlsx and b/testdata/seed-data-command-form-i18n.xlsx differ
diff --git a/testdata/seed-data-command-form.xlsx b/testdata/seed-data-command-form.xlsx
index 6ac890ba7e..dff451c6ab 100644
Binary files a/testdata/seed-data-command-form.xlsx and b/testdata/seed-data-command-form.xlsx differ
diff --git a/testdata/seed-data-command-quality-form-mapping.json b/testdata/seed-data-command-quality-form-mapping.json
new file mode 100644
index 0000000000..c04961512b
--- /dev/null
+++ b/testdata/seed-data-command-quality-form-mapping.json
@@ -0,0 +1,60 @@
+{
+ "data_set_id": "EKWVBc5C0ms",
+ "data_set_name": "MNCH Quarterly Report",
+ "question_mappings": {
+ "PBF_PCA_qlt_8_1": {
+ "id": "ijIsjbvXFJL",
+ "name": "MNCH Breech/shoulder delivery",
+ "valueType": "INTEGER",
+ "categoryOptionCombo": "HllvX50cXC0"
+ },
+ "PBF_PCA_qlt_8_2": {
+ "id": "WQSdEl9xUMA",
+ "name": "MNCH ANC Attendance",
+ "valueType": "INTEGER",
+ "categoryOptionCombo": "HllvX50cXC0"
+ },
+ "PBF_PCA_qlt_8_3_1": {
+ "id": "g9Tso8plX88",
+ "name": "MNCH vacuum/forceps delivery",
+ "valueType": "INTEGER",
+ "categoryOptionCombo": "HllvX50cXC0"
+ },
+ "PBF_PCA_qlt_8_4": {
+ "id": "QhKQD2wHowC",
+ "name": "MNCH Cases of anaemia during ANC",
+ "valueType": "INTEGER",
+ "categoryOptionCombo": "HllvX50cXC0"
+ },
+ "PBF_PCA_qlt_8_5": {
+ "id": "DosHq4vnsYv",
+ "name": "MNCH Caesarean section deliveries",
+ "valueType": "INTEGER",
+ "categoryOptionCombo": "HllvX50cXC0"
+ },
+ "PBF_PCA_qlt_8_6_1": {
+ "id": "Q8lOv2CAUjf",
+ "name": "MNCH Destructive operations delivery",
+ "valueType": "INTEGER",
+ "categoryOptionCombo": "HllvX50cXC0"
+ },
+ "PBF_PCA_qlt_8_7_1": {
+ "id": "SqvCDfGBFzM",
+ "name": "MNCH Normal deliveries",
+ "valueType": "INTEGER",
+ "categoryOptionCombo": "HllvX50cXC0"
+ },
+ "PBF_PCA_qlt_8_8": {
+ "id": "aLF5wmCZrKF",
+ "name": "MNCH Heamoglobin checks performed during ANC",
+ "valueType": "INTEGER",
+ "categoryOptionCombo": "HllvX50cXC0"
+ },
+ "PBF_PCA_qlt_8_10": {
+ "id": "iYCGteGcr6Q",
+ "name": "MNCH ANC registrants",
+ "valueType": "INTEGER",
+ "categoryOptionCombo": "HllvX50cXC0"
+ }
+ }
+}
diff --git a/docs/pages/users/reference/user_guide.fr.md b/docs/pages/users/reference/user_guide.fr.md
index 982e14bc82..55b42b5f08 100644
--- a/docs/pages/users/reference/user_guide.fr.md
+++ b/docs/pages/users/reference/user_guide.fr.md
@@ -270,10 +270,10 @@ Dans la vue du formulaire, vous pouvez consulter les détails suivants :
- Actions
- Nom des formulaires disponibles pour les correspondances
- Versions
-- Type du formulaire :
- - Aggregate : données fixes
- - Event : série d'événements singuliers
- - Event Tracker : continu
+- Type du mapping (voir [quel type de mapping choisir](../how_to/import_export_dhis2_mappings.md#quel-type-de-mapping-choisir)) :
+ - Aggregate : une soumission par unité d'organisation et par période, exportée vers un dataset DHIS2
+ - Event : plusieurs soumissions par unité d'organisation et par période, exportées vers un programme d'évènements sans enregistrement
+ - Event Tracker : la même personne suivie dans le temps, exportée vers un programme tracker
- Nombre de questionnaires à mapper
- Nombre total de questionnaires
- Couverture des correspondances
@@ -289,6 +289,9 @@ Si la question doit être mappée, recherchez l'élément correspondant dans le
Une fois confirmée, la question deviendra verte et sera comptabilisée.
+Pour réutiliser les mappings d'une autre version du formulaire ou d'un fichier d'export, voir
+[Importer et exporter des mappings DHIS2](../how_to/import_export_dhis2_mappings.md).
+
### Complétude
Cette fonctionnalité est destinée aux cas où des périodes ont été définies pour les formulaires de collecte de données.
diff --git a/hat/assets/js/apps/Iaso/domains/app/translations/en.json b/hat/assets/js/apps/Iaso/domains/app/translations/en.json
index 25530c4d83..b0202b33fc 100644
--- a/hat/assets/js/apps/Iaso/domains/app/translations/en.json
+++ b/hat/assets/js/apps/Iaso/domains/app/translations/en.json
@@ -1118,11 +1118,55 @@
"iaso.map.shape.editLocationDisabled": "Location edition is disabled on this account",
"iaso.map.title.markerClustering": "Clustering",
"iaso.mapping.mappingType": "Mapping type",
+ "iaso.mappings.changeLog": "Link to changes log",
"iaso.mappings.changeMapping": "Change the mapping to existing one :",
"iaso.mappings.confirm": "Confirm",
"iaso.mappings.coverage": "Coverage",
"iaso.mappings.create": "Create Mapping",
"iaso.mappings.currentMapping": "Current mapping",
+ "iaso.mappings.export": "Export mappings",
+ "iaso.mappings.import.addAll": "Add all",
+ "iaso.mappings.import.allAdditions": "All additions:",
+ "iaso.mappings.import.applyChanges": "Apply {count} changes",
+ "iaso.mappings.import.bucket.add": "To add ({count})",
+ "iaso.mappings.import.bucket.conflict": "Conflicts ({count})",
+ "iaso.mappings.import.bucket.dropped": "Dropped ({count})",
+ "iaso.mappings.import.bucket.identical": "Identical ({count})",
+ "iaso.mappings.import.chooseFile": "Choose a file",
+ "iaso.mappings.import.compareLabel": "Comparing {source} with version {versionId}. Nothing is applied until you confirm.",
+ "iaso.mappings.import.decision": "Decision",
+ "iaso.mappings.import.done": "{added} mappings added, {overwritten} overwritten, {kept} kept, {ignored} not imported",
+ "iaso.mappings.import.droppedHint": "These mappings cannot be imported: the question does not exist in this version, or the mapping is not valid for this mapping type.",
+ "iaso.mappings.import.fileMeta": "Exported from {formName} - {versionId}",
+ "iaso.mappings.import.fromFile": "Or import a mapping export (.json), for example from another account.",
+ "iaso.mappings.import.identicalHint": "Same DHIS2 target on both sides. Nothing to resolve.",
+ "iaso.mappings.import.incomingMapping": "Incoming mapping",
+ "iaso.mappings.import.invalidFormat": "This file is not a mapping export.",
+ "iaso.mappings.import.invalidJson": "This file is not valid JSON.",
+ "iaso.mappings.import.invalidMapping": "Not valid for a {type} mapping",
+ "iaso.mappings.import.keep": "Keep",
+ "iaso.mappings.import.keepEverywhere": "Keep current everywhere",
+ "iaso.mappings.import.mappingsCount": "{count} mappings",
+ "iaso.mappings.import.mappingTypeMismatch": "This export was made for another mapping type.",
+ "iaso.mappings.import.markedNeverMapped": "Marked as never mapped",
+ "iaso.mappings.import.matchCount": "{matching} / {total} match this version",
+ "iaso.mappings.import.noChange": "No change",
+ "iaso.mappings.import.noSource": "No other mapping version uses this data source with the same DHIS2 dataset or program.",
+ "iaso.mappings.import.notImportable": "Not importable",
+ "iaso.mappings.import.noValidMapping": "This file contains no mapping valid for a {type} mapping.",
+ "iaso.mappings.import.otherDataset": "This export was made for another DHIS2 dataset ({name}). Check that its data elements belong to this one.",
+ "iaso.mappings.import.otherProgram": "This export was made for another DHIS2 program ({name}). Check that its data elements belong to this one.",
+ "iaso.mappings.import.overwrite": "Overwrite",
+ "iaso.mappings.import.overwriteEverywhere": "Overwrite everywhere",
+ "iaso.mappings.import.pickSource": "Pick the mappings to reuse. Nothing is applied until you confirm.",
+ "iaso.mappings.import.questionAbsent": "Question absent from version {versionId}",
+ "iaso.mappings.import.resolveAllConflicts": "Resolve all conflicts:",
+ "iaso.mappings.import.skip": "Skip",
+ "iaso.mappings.import.skipAll": "Skip all",
+ "iaso.mappings.import.summary": "{added} to add, {overwritten} to overwrite, {ignored} not imported",
+ "iaso.mappings.import.title": "Import mappings",
+ "iaso.mappings.import.versionMeta": "Last updated {date}",
+ "iaso.mappings.import.versionTitle": "Version {versionId}",
"iaso.mappings.label.atLeastAMapping": "at least a mapping",
"iaso.mappings.label.dataset": "Dataset",
"iaso.mappings.label.duplicateMappingAlert": "Duplicate mapping ! Will be used in both {duplicates}",
diff --git a/hat/assets/js/apps/Iaso/domains/app/translations/es.json b/hat/assets/js/apps/Iaso/domains/app/translations/es.json
index 8cf471c60c..b219a16490 100644
--- a/hat/assets/js/apps/Iaso/domains/app/translations/es.json
+++ b/hat/assets/js/apps/Iaso/domains/app/translations/es.json
@@ -916,11 +916,55 @@
"iaso.map.shape.editLocationDisabled": "La edición de ubicación está deshabilitada en esta cuenta",
"iaso.map.title.markerClustering": "Agrupación",
"iaso.mapping.mappingType": "Tipo de mapeo",
+ "iaso.mappings.changeLog": "Enlace al registro de cambios",
"iaso.mappings.changeMapping": "Cambiar el mapeo a uno existente:",
"iaso.mappings.confirm": "Confirmar",
"iaso.mappings.coverage": "Cobertura",
"iaso.mappings.create": "Crear Mapeo",
"iaso.mappings.currentMapping": "Mapeo actual",
+ "iaso.mappings.export": "Exportar mapeos",
+ "iaso.mappings.import.addAll": "Añadir todo",
+ "iaso.mappings.import.allAdditions": "Todas las adiciones:",
+ "iaso.mappings.import.applyChanges": "Aplicar {count} cambios",
+ "iaso.mappings.import.bucket.add": "Para añadir ({count})",
+ "iaso.mappings.import.bucket.conflict": "Conflictos ({count})",
+ "iaso.mappings.import.bucket.dropped": "Descartados ({count})",
+ "iaso.mappings.import.bucket.identical": "Idénticos ({count})",
+ "iaso.mappings.import.chooseFile": "Elegir un archivo",
+ "iaso.mappings.import.compareLabel": "Comparando {source} con la versión {versionId}. No se aplica nada hasta que confirme.",
+ "iaso.mappings.import.decision": "Decisión",
+ "iaso.mappings.import.done": "{added} mapeos añadidos, {overwritten} sobrescritos, {kept} mantenidos, {ignored} no importados",
+ "iaso.mappings.import.droppedHint": "Estos mapeos no se pueden importar: la pregunta no existe en esta versión, o el mapeo no es válido para este tipo de mapeo.",
+ "iaso.mappings.import.fileMeta": "Exportado desde {formName} - {versionId}",
+ "iaso.mappings.import.fromFile": "O importe una exportación de mapeo (.json), por ejemplo desde otra cuenta.",
+ "iaso.mappings.import.identicalHint": "Mismo destino DHIS2 en ambos lados. Nada que resolver.",
+ "iaso.mappings.import.incomingMapping": "Mapeo entrante",
+ "iaso.mappings.import.invalidFormat": "Este archivo no es una exportación de mapeo.",
+ "iaso.mappings.import.invalidJson": "Este archivo no es un JSON válido.",
+ "iaso.mappings.import.invalidMapping": "No válido para un mapeo {type}",
+ "iaso.mappings.import.keep": "Mantener",
+ "iaso.mappings.import.keepEverywhere": "Mantener el actual en todas partes",
+ "iaso.mappings.import.mappingsCount": "{count} mapeos",
+ "iaso.mappings.import.mappingTypeMismatch": "Esta exportación se hizo para otro tipo de mapeo.",
+ "iaso.mappings.import.markedNeverMapped": "Marcada como nunca mapeada",
+ "iaso.mappings.import.matchCount": "{matching} / {total} coinciden con esta versión",
+ "iaso.mappings.import.noChange": "Sin cambios",
+ "iaso.mappings.import.noSource": "Ninguna otra versión de mapeo usa esta fuente de datos con el mismo dataset o programa DHIS2.",
+ "iaso.mappings.import.notImportable": "No importable",
+ "iaso.mappings.import.noValidMapping": "Este archivo no contiene ningún mapeo válido para un mapeo {type}.",
+ "iaso.mappings.import.otherDataset": "Esta exportación se hizo para otro dataset DHIS2 ({name}). Verifique que sus data elements pertenecen a este.",
+ "iaso.mappings.import.otherProgram": "Esta exportación se hizo para otro programa DHIS2 ({name}). Verifique que sus data elements pertenecen a este.",
+ "iaso.mappings.import.overwrite": "Sobrescribir",
+ "iaso.mappings.import.overwriteEverywhere": "Sobrescribir en todas partes",
+ "iaso.mappings.import.pickSource": "Elija los mapeos a reutilizar. No se aplica nada hasta que confirme.",
+ "iaso.mappings.import.questionAbsent": "Pregunta ausente de la versión {versionId}",
+ "iaso.mappings.import.resolveAllConflicts": "Resolver todos los conflictos:",
+ "iaso.mappings.import.skip": "Omitir",
+ "iaso.mappings.import.skipAll": "Omitir todo",
+ "iaso.mappings.import.summary": "{added} para añadir, {overwritten} para sobrescribir, {ignored} no importados",
+ "iaso.mappings.import.title": "Importar mapeos",
+ "iaso.mappings.import.versionMeta": "Actualizado el {date}",
+ "iaso.mappings.import.versionTitle": "Versión {versionId}",
"iaso.mappings.label.atLeastAMapping": "al menos un mapeo",
"iaso.mappings.label.dataset": "Conjunto de datos",
"iaso.mappings.label.duplicateMappingAlert": "¡Mapeo duplicado! Se usará en ambos {duplicates}",
diff --git a/hat/assets/js/apps/Iaso/domains/app/translations/fr.json b/hat/assets/js/apps/Iaso/domains/app/translations/fr.json
index d8fda0a58f..aeeebabb8a 100644
--- a/hat/assets/js/apps/Iaso/domains/app/translations/fr.json
+++ b/hat/assets/js/apps/Iaso/domains/app/translations/fr.json
@@ -1119,11 +1119,55 @@
"iaso.map.shape.editLocationDisabled": "L'édition de la localisation est désactivée sur ce compte",
"iaso.map.title.markerClustering": "Regroupement",
"iaso.mapping.mappingType": "Type de correspondance",
+ "iaso.mappings.changeLog": "Lien vers le Journal des modifications",
"iaso.mappings.changeMapping": "Changer pour une correspondance existante :",
"iaso.mappings.confirm": "Confirmer",
"iaso.mappings.coverage": "Couverture",
"iaso.mappings.create": "Ajouter une correspondance",
"iaso.mappings.currentMapping": "Correspondance actuelle",
+ "iaso.mappings.export": "Exporter les mappings",
+ "iaso.mappings.import.addAll": "Tout ajouter",
+ "iaso.mappings.import.allAdditions": "Tous les ajouts :",
+ "iaso.mappings.import.applyChanges": "Appliquer {count} changements",
+ "iaso.mappings.import.bucket.add": "À ajouter ({count})",
+ "iaso.mappings.import.bucket.conflict": "Conflits ({count})",
+ "iaso.mappings.import.bucket.dropped": "Ignorés ({count})",
+ "iaso.mappings.import.bucket.identical": "Identiques ({count})",
+ "iaso.mappings.import.chooseFile": "Choisir un fichier",
+ "iaso.mappings.import.compareLabel": "Comparaison de {source} avec la version {versionId}. Rien n'est appliqué avant votre confirmation.",
+ "iaso.mappings.import.decision": "Décision",
+ "iaso.mappings.import.done": "{added} mappings ajoutés, {overwritten} écrasés, {kept} gardés, {ignored} non importés",
+ "iaso.mappings.import.droppedHint": "Ces mappings ne peuvent pas être importés : la question n'existe pas dans cette version, ou le mapping n'est pas valide pour ce type de mapping.",
+ "iaso.mappings.import.fileMeta": "Exporté depuis {formName} - {versionId}",
+ "iaso.mappings.import.fromFile": "Ou importez un export de mapping (.json), par exemple depuis un autre compte.",
+ "iaso.mappings.import.identicalHint": "Même cible DHIS2 des deux côtés. Rien à résoudre.",
+ "iaso.mappings.import.incomingMapping": "Mapping importé",
+ "iaso.mappings.import.invalidFormat": "Ce fichier n'est pas un export de mapping.",
+ "iaso.mappings.import.invalidJson": "Ce fichier n'est pas un JSON valide.",
+ "iaso.mappings.import.invalidMapping": "Non valide pour un mapping {type}",
+ "iaso.mappings.import.keep": "Garder",
+ "iaso.mappings.import.keepEverywhere": "Garder l'actuel partout",
+ "iaso.mappings.import.mappingsCount": "{count} mappings",
+ "iaso.mappings.import.mappingTypeMismatch": "Cet export concerne un autre type de mapping.",
+ "iaso.mappings.import.markedNeverMapped": "Marquée comme jamais mappée",
+ "iaso.mappings.import.matchCount": "{matching} / {total} correspondent à cette version",
+ "iaso.mappings.import.noChange": "Aucun changement",
+ "iaso.mappings.import.noSource": "Aucune autre version de mapping n'utilise cette source avec le même dataset ou programme DHIS2.",
+ "iaso.mappings.import.notImportable": "Non importable",
+ "iaso.mappings.import.noValidMapping": "Ce fichier ne contient aucun mapping valide pour un mapping {type}.",
+ "iaso.mappings.import.otherDataset": "Cet export a été fait pour un autre dataset DHIS2 ({name}). Vérifiez que ses data elements font partie de celui-ci.",
+ "iaso.mappings.import.otherProgram": "Cet export a été fait pour un autre programme DHIS2 ({name}). Vérifiez que ses data elements font partie de celui-ci.",
+ "iaso.mappings.import.overwrite": "Écraser",
+ "iaso.mappings.import.overwriteEverywhere": "Écraser partout",
+ "iaso.mappings.import.pickSource": "Choisissez les mappings à réutiliser. Rien n'est appliqué avant votre confirmation.",
+ "iaso.mappings.import.questionAbsent": "Question absente de la version {versionId}",
+ "iaso.mappings.import.resolveAllConflicts": "Résoudre tous les conflits :",
+ "iaso.mappings.import.skip": "Ignorer",
+ "iaso.mappings.import.skipAll": "Tout ignorer",
+ "iaso.mappings.import.summary": "{added} à ajouter, {overwritten} à écraser, {ignored} non importés",
+ "iaso.mappings.import.title": "Importer des mappings",
+ "iaso.mappings.import.versionMeta": "Mis à jour le {date}",
+ "iaso.mappings.import.versionTitle": "Version {versionId}",
"iaso.mappings.label.atLeastAMapping": "au moins une correspondance",
"iaso.mappings.label.dataset": "Dataset",
"iaso.mappings.label.duplicateMappingAlert": "Doublon ! Sera utilisé dans {duplicates}",
diff --git a/hat/assets/js/apps/Iaso/domains/mappings/components/ImportMappings/CompareStep.tsx b/hat/assets/js/apps/Iaso/domains/mappings/components/ImportMappings/CompareStep.tsx
new file mode 100644
index 0000000000..a6a39b9aea
--- /dev/null
+++ b/hat/assets/js/apps/Iaso/domains/mappings/components/ImportMappings/CompareStep.tsx
@@ -0,0 +1,306 @@
+import React, { FunctionComponent, useMemo } from 'react';
+import {
+ Box,
+ Button,
+ Tab,
+ Table,
+ TableBody,
+ TableCell,
+ TableContainer,
+ TableHead,
+ TableRow,
+ Tabs,
+ Typography,
+} from '@mui/material';
+import { IntlMessage, useSafeIntl } from 'bluesquare-components';
+import { textPlaceholder } from '../../../../constants/uiConstants';
+import { SxStyles } from '../../../../types/general';
+import { DIFF_KINDS, getMappingLabel } from '../../importMappings';
+import MESSAGES from '../../messages';
+import { isNeverMapped } from '../../question_mappings';
+import { Decision, DiffKind, DiffRow, ImportPlan } from '../../types';
+
+type Props = {
+ sourceTitle: string;
+ versionId: string;
+ mappingType: string;
+ rows: DiffRow[];
+ bucket: DiffKind;
+ setBucket: (bucket: DiffKind) => void;
+ decisions: Record