diff --git a/.gitignore b/.gitignore index 70dbaae..a933294 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,6 @@ relay.json # Raw (unsanitized) supplier payloads — never commit /sabre/ +/amadeus_response.xml +/sabre_response.xml +/*_response.xml diff --git a/openspec/changes/close-remark-free-text-pii-gap/.openspec.yaml b/openspec/changes/close-remark-free-text-pii-gap/.openspec.yaml new file mode 100644 index 0000000..5081c98 --- /dev/null +++ b/openspec/changes/close-remark-free-text-pii-gap/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-12 diff --git a/openspec/changes/close-remark-free-text-pii-gap/proposal.md b/openspec/changes/close-remark-free-text-pii-gap/proposal.md new file mode 100644 index 0000000..77a6169 --- /dev/null +++ b/openspec/changes/close-remark-free-text-pii-gap/proposal.md @@ -0,0 +1,71 @@ +## Why + +A customer reported PII surviving anonymization, "mainly in remarks". Two real captured responses +(Amadeus `PNR_Reply`, Sabre `GetReservationRS`) were replayed through the shipped baked ruleset with +the real engine. The passenger's own identity redacts correctly — structured name, phone, email, +DOCS/APIS entries, frequent-flyer number and card all scrubbed. Two specific classes still leak: + +- **PII that exists only in free text.** The phase-2 reference pass can only scrub values that a + phase-1 field rule already collected, so any value with no structured counterpart is invisible to + it. This is exactly the third-party population: emergency contact (`EMER-`, `EMER-`), + authority-to-charge (`TPREF-AUTH-AUTHORITY TO CHARGE-`), travel arranger (`TRAVEL ARRANGER/…`, + `*84-…`, the `CTC` phone node, `ReceivedFrom/Name`, `FutureTicketing/Comment`) and the Amadeus + orderer (`ACEORB-…`, `ACECRM-ORDERER-…`). The observed proof is a single node reading + `EMER-YAARA `: the passenger surname beside it *was* scrubbed by the reference rule, while + the emergency contact's given name survived because nothing had collected it. +- **Sabre history mirrors for one-way types.** Every coded passenger remark is re-emitted three times + — `RemarkLine/Text`, `HistoryAssociationElement`, and `AssociationChild`/`AssociationParent/Content` + (with the remark code turned into a `P!`/`V!`/`E!` prefix). `ReferenceRule` accepts only + `EncryptAction`, so date-of-birth, passport number, nationality and card data — whose policy is + one-way replacement — cannot be propagated to those mirrors by the reference pass at all. Confirmed + leaks: `BIRTHDATE-07AUG70`, `PSPT-IL …`, `PSPT/CO-IL/NR-/IS-…/EX-…/CI-IL`, + `CTZN-IL`, and a masked PAN `*CA5XXXXXXXXXXX8630/1127` appearing 20 times. + +Two further gaps are structural rather than free-text: no rule covers the Amadeus `FP` (form of +payment) or `OS` (other service information) `dataElementsIndiv` at all, so a card number sits in +cleartext in `PAX CCDCXXXXXXXXXX7298/0629`; and person-linked pseudonymous identifiers (employee id, +traveller profile id, traveller-attached loyalty/contract numbers) are unclassified today and survive +in both channels. + +## What Changes + +- **Scope decision: traveller-adjacent third parties are data subjects.** Emergency contacts, + authority-to-charge holders, travel arrangers and the Amadeus orderer SHALL be redacted like the + passenger. **Agency-agent personal names and sign-in codes remain operational** and SHALL be + preserved — this keeps the existing "agent sign-in codes are not PII" stance and keeps remark + audit trails legible for the customer. +- **Scope decision: person-linked pseudonymous identifiers are PII.** Employee ids, traveller profile + ids, and loyalty/contract numbers attached to a traveller SHALL be encrypted (reversible — clients + legitimately correlate them). Organisation-wide commercial codes (corporate account ids such as + `OIN CHEVRON`/`CMP …`, tour codes, discount codes) are NOT personal data and SHALL be preserved, + since redacting them breaks fare reissue and policy display. +- **Identity-document data found in free text gets format-preserving sentinels** (`01JAN00`, + `00000000`, `31DEC99`, `ZZ`, `M`) rather than `REDACTED`, extending the existing typed-field + sentinel policy from structured nodes to free-text spans so downstream text parsers keep working. +- **Rules only reach values via literal markers.** Every new `extract_patterns` entry anchors on an + operational marker observed in a payload (`EMER-`, `BIRTHDATE-`, `PSPT-`, `PSPT/CO-`, `CTZN-`, + `AUTHORITY TO CHARGE-`, `TRAVEL ARRANGER/`, `ACEORB-`, `ACECRM-EMPLOYEE ID-`, `CYTRIC PROFILE REF:`, + `PREF HTL ID/`, `DHSNBR-PAX-`, `ISS TKT FR`). No generic shape matching over free text: an + unanchored date or digit-run pattern would redact fares, flight dates and city pairs. +- **One-way types name their mirrors explicitly.** Rules for date-of-birth, gender, passport, + nationality and card fragments in Sabre remarks SHALL also target `HistoryAssociationElement` and + `AssociationChild`/`AssociationParent/Content`, because the reference pass cannot carry a + non-encrypt action. +- **New Amadeus paths.** The `FP` and `OS` `dataElementsIndiv` free-text nodes gain coverage. + +No engine change, no schema change, no new `PiiType`. The whole change is baked-ruleset authoring plus +contract tests over sanitized fixtures. + +## Capabilities + +### New Capabilities + + +### Modified Capabilities +- `pii-rules`: defines which non-passenger persons and which pseudonymous identifiers are in scope, + and requires free-text identity-document spans to use format-preserving sentinels. +- `amadeus-pii-baseline`: gains orderer-name, remark date-of-birth/gender, employee-id, profile-ref + and loyalty-pref rules, plus first-time coverage of the `FP` and `OS` free-text elements. +- `sabre-pii-baseline`: gains third-party name/phone rules, remark identity-document rules that also + cover the two history mirrors, card-fragment rules for history nodes, and employee/traveller-profile + id rules. diff --git a/openspec/changes/close-remark-free-text-pii-gap/specs/amadeus-pii-baseline/spec.md b/openspec/changes/close-remark-free-text-pii-gap/specs/amadeus-pii-baseline/spec.md new file mode 100644 index 0000000..babd5a3 --- /dev/null +++ b/openspec/changes/close-remark-free-text-pii-gap/specs/amadeus-pii-baseline/spec.md @@ -0,0 +1,42 @@ +## ADDED Requirements + +### Requirement: Amadeus form-of-payment and other-service free text are covered + +The Amadeus baseline SHALL cover the `FP` (form of payment) and `OS` (other service information) +`dataElementsIndiv` free-text nodes, not only the `AP` and `RM` elements it targeted previously. All +four use the same `otherDataFreetext/longFreetext` carrier; `FP` holds a card number and expiry, and +`OS` holds traveller-attached contract numbers. + +#### Scenario: Card number in the FP element is redacted +- **WHEN** a `PNR_Reply` carries a form-of-payment line such as `PAX CC/` +- **THEN** the card number and the expiry are redacted and the operational prefix is preserved — + supplier-side masking is not trusted + +#### Scenario: Traveller contract number in the OS element is encrypted +- **WHEN** an `OS` line carries a traveller-attached contract or loyalty number (for example + `CP/`) +- **THEN** the number is encrypted, while organisation-level corporate account codes in sibling `OS` + lines (for example `CMP …`, `OIN …`, `NCA …`) are preserved + +### Requirement: Amadeus RM remark rules cover both remark mirrors + +Amadeus remark extraction patterns SHALL NOT require a leading `*`, so that both renderings of every +remark match. Each Amadeus RM/RIR element is emitted twice for the same remark: once under +`miscellaneousRemarks/remarks/freetext` with the leading `*` inside the text, and once under +`extendedRemark/structuredRemark/freetext` with that `*` hoisted into the sibling `category` element. + +#### Scenario: Orderer name is redacted in both mirrors +- **WHEN** a `PNR_Reply` carries an orderer name as `*ACEORB-` and `*ACECRM-ORDERER-`, each + duplicated into the `structuredRemark` mirror without the leading `*` +- **THEN** every occurrence is encrypted and the `ACEORB-` / `ACECRM-ORDERER-` markers are preserved + +#### Scenario: Date of birth and gender in a remark are redacted +- **WHEN** a remark carries `DOB-/GENDER-` in both mirrors +- **THEN** both spans are replaced with format-preserving sentinels in both mirrors + +#### Scenario: Person-linked identifiers in remarks are encrypted +- **WHEN** remarks carry an employee id (`ACECRM-EMPLOYEE ID-`), a booking-tool traveller + profile reference (`CYTRIC PROFILE REF:`), or a traveller hotel-loyalty preference id + (`PREF HTL ID/SUP-/ID-`) +- **THEN** each identifier is encrypted while its marker text and any template remark that carries the + marker with no value are preserved diff --git a/openspec/changes/close-remark-free-text-pii-gap/specs/pii-rules/spec.md b/openspec/changes/close-remark-free-text-pii-gap/specs/pii-rules/spec.md new file mode 100644 index 0000000..678e837 --- /dev/null +++ b/openspec/changes/close-remark-free-text-pii-gap/specs/pii-rules/spec.md @@ -0,0 +1,71 @@ +## ADDED Requirements + +### Requirement: Traveller-adjacent third parties are data subjects; agency staff are operational + +The baked ruleset SHALL treat a natural person recorded in a booking *because of their relationship to +the traveller* as a data subject, redacted on the same terms as the passenger. This covers at least +emergency contacts, authority-to-charge / payment-authorising persons, travel arrangers and bookers, +and the Amadeus orderer. + +Personal names and identifiers belonging to **agency staff acting in their professional capacity** — +full agent names in audit remarks, agent sign-in codes, duty codes, pseudo-city codes — SHALL NOT be +redacted. Redacting them would destroy the audit legibility the remark trail exists to provide, and +extends the existing "agent sign-in codes are not PII" classification consistently to the name form. + +#### Scenario: Emergency contact is redacted +- **WHEN** a response carries an emergency-contact name or phone number in free text with no + structured counterpart anywhere in the document +- **THEN** the value is redacted and the operational marker around it (for example `EMER-`) is + preserved + +#### Scenario: Travel arranger is redacted across every rendering +- **WHEN** the same travel-arranger name appears in a contact phone node, a remark line, an invoice + remark field, and a ticketing comment +- **THEN** every occurrence is redacted, and occurrences reached by the reference pass carry the same + token as the occurrence the field rule collected + +#### Scenario: Agent name and sine are preserved +- **WHEN** a remark records the acting agent, either by full name (for example + `ADVISED BY - `) or by short sign-in code (for example `12AUG2026--…`) +- **THEN** the value is preserved verbatim in the anonymized response + +### Requirement: Person-linked pseudonymous identifiers are PII; organisation codes are not + +An identifier that resolves to one natural person SHALL be treated as PII and encrypted (reversible, +because clients legitimately correlate travellers across bookings by these keys). This covers at least +employer employee ids, traveller profile ids in every namespace that mirrors them, and +loyalty/contract numbers attached to an individual traveller. + +An identifier that resolves only to an organisation — corporate account ids, corporate contract and +deal codes, tour codes, negotiated-fare discount codes — SHALL NOT be redacted; it is commercial data, +and removing it breaks fare reissue and corporate policy display. + +#### Scenario: Employee id is encrypted wherever it appears +- **WHEN** an employee id appears both as a structured attribute and inside remark free text +- **THEN** both occurrences are replaced by the same reversible token + +#### Scenario: Traveller profile id is encrypted in every mirror namespace +- **WHEN** a traveller profile id is emitted in more than one namespace for the same reservation +- **THEN** every occurrence is encrypted, while agency-level and corporate-level profile ids for the + same reservation are preserved + +#### Scenario: Corporate account code is preserved +- **WHEN** a response carries an organisation-level corporate account, contract, tour or discount code +- **THEN** the value is preserved verbatim + +### Requirement: Free-text identity-document spans use format-preserving sentinels + +A rule redacting identity-document data embedded in free text SHALL replace the span with a sentinel +that preserves the original shape in the format used at that location, rather than a `REDACTED` +literal. This applies to date of birth, gender, document number, document dates and nationality, and +extends the existing typed-field sentinel policy to free text so that consumer-side text parsing of +the surrounding operational line keeps working. + +#### Scenario: Date of birth in a remark keeps its date shape +- **WHEN** a remark line carries a date of birth in a `DDMMMYY` free-text span +- **THEN** the span is replaced with a `DDMMMYY`-shaped sentinel, and the surrounding marker text is + preserved + +#### Scenario: Passport number in a remark keeps its length class +- **WHEN** a remark line carries a passport number in a free-text span +- **THEN** the span is replaced with a document-number-shaped sentinel, not a `REDACTED` literal diff --git a/openspec/changes/close-remark-free-text-pii-gap/specs/sabre-pii-baseline/spec.md b/openspec/changes/close-remark-free-text-pii-gap/specs/sabre-pii-baseline/spec.md new file mode 100644 index 0000000..f0dd133 --- /dev/null +++ b/openspec/changes/close-remark-free-text-pii-gap/specs/sabre-pii-baseline/spec.md @@ -0,0 +1,94 @@ +## ADDED Requirements + +### Requirement: Reservation remark PII is redacted in every history mirror + +A field rule redacting remark PII with a non-encrypt action SHALL target Sabre's history mirror paths +explicitly, in addition to the remark path. Sabre re-emits each coded passenger remark up to three +times for one reservation: as `RemarkLine/Text`, as `HistoryAssociationElement`, and as +`AssociationChild`/`AssociationParent` `Content`, where the remark's `@code` becomes a `!` text +prefix. Reference rules accept only an encrypt action, so a value whose policy is one-way replacement +can never be propagated to those mirrors by the reference pass. + +#### Scenario: Remark date of birth is redacted in all three renderings +- **WHEN** a `GetReservationRS` carries `BIRTHDATE-` in a passenger remark, and the same line + is mirrored as `P!BIRTHDATE-` in both history renderings +- **THEN** all three occurrences carry the sentinel, and no plaintext date of birth remains in the + response + +#### Scenario: Remark passport, document dates and nationality are redacted in all three renderings +- **WHEN** a passenger remark carries `PSPT- `, or + `PSPT/CO-/NR-/IS-/EX-/CI-`, or `CTZN-`, mirrored into + history +- **THEN** every document number, document date and country span is replaced by its sentinel in all + renderings, while the operational markers (`PSPT-`, `/NR-`, `CTZN-`) are preserved + +#### Scenario: Masked card fragment in history is redacted +- **WHEN** a form-of-payment value appears in history nodes as a supplier-masked card fragment with an + expiry (for example `*CA/` or the `¥`-delimited remark rendering) +- **THEN** the card fragment and the expiry are both redacted in every rendering — supplier masking is + not trusted, since it still reveals the leading digit and last four + +### Requirement: Non-passenger persons in reservation free text are redacted + +Each non-passenger person recorded in a `GetReservationRS` SHALL be reached by a field rule anchored on +its operational marker, so that the value also enters the collector and the reference pass scrubs its +history mirrors. These persons — emergency contact, authority-to-charge holder, travel arranger — have +no structured name node anywhere in the document, so nothing else can collect them. + +#### Scenario: Emergency contact name and phone +- **WHEN** a passenger remark carries `EMER-` and `EMER-` +- **THEN** the name is encrypted, the phone is redacted, and `EMER-OTHER` and other non-PII `EMER-` + values are left untouched + +#### Scenario: Authority-to-charge holder +- **WHEN** a passenger remark carries `TPREF-AUTH-AUTHORITY TO CHARGE-` +- **THEN** the name is encrypted and the preference marker is preserved + +#### Scenario: Travel arranger reached through the contact phone node +- **WHEN** the arranger's name is embedded in a `CTC` contact phone value (for example + `M--`), which a separate rule masks as a whole node +- **THEN** the arranger name rule runs before the whole-node phone rule, so the name is collected and + encrypted rather than destroyed unread, and the collected value scrubs the arranger's other + occurrences in remarks and the ticketing comment + +#### Scenario: Received-from field mixes arranger and agent +- **WHEN** `ReceivedFrom/Name` carries an arranger name, a phone number, and an agency agent name in + one value +- **THEN** the arranger name and the phone are redacted and the agent name is preserved + +### Requirement: A rewritten span must never leave a token abutting adjacent text + +A rule rewriting a span inside free text SHALL NOT leave an `ENC_` token immediately adjacent to +surrounding alphanumeric or `-`/`_` text. `ENC_` payloads are base64url, so `-` and `_` read as token +characters: the greedy token scan on the request path re-consumes the adjacent text into the token, the +value never de-anonymizes, and the relay would forward a Wenrix token to the channel — breaking +transparency. Where an operational suffix would abut, the rule SHALL either fold that suffix into the +encrypted span (keeping the round-trip exact) or use a one-way sentinel containing no token characters. + +#### Scenario: Phone followed by an agent name +- **WHEN** `ReceivedFrom/Name` holds a phone number followed directly by `-` +- **THEN** the number is replaced with a one-way sentinel rather than a token, so nothing abuts the + suffix and the response still de-anonymizes cleanly on the way back upstream + +#### Scenario: Name followed by an operational qualifier +- **WHEN** a remark name is followed directly by a `-` qualifier (for example + `TRAVEL ARRANGER/-B`) +- **THEN** the qualifier is folded into the encrypted span, so the marker stays visible and + de-anonymizing the response restores the original line byte for byte + +### Requirement: Person-linked identifiers in a reservation are encrypted + +Employee ids and traveller profile ids carried in a `GetReservationRS` SHALL be encrypted in every +location and namespace that mirrors them, while profile ids belonging to the agency, the corporate +account, or any other non-traveller entity SHALL be preserved. + +#### Scenario: Employee id in attribute and remark +- **WHEN** an employee id appears as `Passenger/@referenceNumber` and inside a `PROFILE -` + remark +- **THEN** both are encrypted with the same token + +#### Scenario: Traveller profile id in both namespaces +- **WHEN** the traveller's profile id is emitted under the passenger's structured profile node and + again in the open-reservation-element mirror namespace +- **THEN** both are encrypted, and the agency, corporate and other non-traveller profile ids for the + same reservation are preserved diff --git a/openspec/changes/close-remark-free-text-pii-gap/tasks.md b/openspec/changes/close-remark-free-text-pii-gap/tasks.md new file mode 100644 index 0000000..c79ab1a --- /dev/null +++ b/openspec/changes/close-remark-free-text-pii-gap/tasks.md @@ -0,0 +1,64 @@ +## 1. Sanitized evidence + +- [x] 1.1 Add `tests/fixtures/amadeus/pnr_retrieve_third_party_remarks_response.xml` — trimmed + `PNR_Reply` keeping both RM remark mirrors, the `FP` and `OS` `dataElementsIndiv`, and the structured + traveller name so the reference pass has a bucket. +- [x] 1.2 Add `tests/fixtures/sabre/get_reservation_third_party_remarks_response.xml` — trimmed + `GetReservationRS` keeping the passenger coded-remark block, reservation `HS`/`INVOICE`/`CODED` + lines, `ReceivedFrom`, `FutureTicketing`, the `CTC` phone nodes, both profile namespaces, and three + history transactions so `HistoryAssociationElement` and both `Association*/Content` are exercised. + Preserve the `¤` and `¥` delimiters verbatim. +- [x] 1.3 Delete the raw captured payloads and add them to `.gitignore` (live passenger data and a live + Sabre `BinarySecurityToken`). + +## 2. Failing tests first (TDD) + +- [x] 2.1 `TestThirdPartyRemarks` in `test_pii_amadeus.py`: orderer name in both mirrors, remark + date-of-birth/gender sentinels, employee id, profile ref, loyalty pref, `FP` card + expiry, `OS` + contract number; round-trip; template remark and organisation codes preserved. +- [x] 2.2 `TestThirdPartyRemarks` in `test_pii_sabre.py`: emergency contact name/phone, + authority-to-charge, arranger via `CTC` phone / remark / invoice field / `ReceivedFrom` / + `FutureTicketing`, remark passport/document-dates/nationality/date-of-birth/gender sentinels, history + card fragment; agent name and sine preserved. +- [x] 2.3 Mirror test: the same sentinel appears in `RemarkLine/Text`, `HistoryAssociationElement`, and + both `Association*/Content` — the regression a reference-rule-only fix would miss. +- [x] 2.4 Token-identity test: the arranger name collected from the `CTC` phone node yields the same + token in the remark, invoice field and ticketing comment. +- [x] 2.5 Rule-ordering regression test: the `CTC` arranger name is tokenized, not swallowed by the + whole-node phone rule — fails if the rule is moved after it. +- [x] 2.6 Operational-survival negatives: agent name and sine, record locators, ticket numbers, fares, + policy blocks, corporate account/tour/discount codes, and the corporate URL are byte-present. +- [x] 2.7 Relay-level case per channel in `tests/integration/test_pii_extended_operations_relay.py`. + +## 3. Rules + +- [x] 3.1 Amadeus: orderer name, remark date-of-birth, remark gender, employee id, profile ref, loyalty + pref on the existing RM triple path; patterns anchored without the leading `*`. +- [x] 3.2 Amadeus: new `FP` card + expiry rules and new `OS` contract-number rule. +- [x] 3.3 Sabre: third-party name and phone rules on the remark path. +- [x] 3.4 Sabre: `ctc_arranger_name` inserted **before** `sabre.res.phone`; `received_from_arranger`; + `future_ticketing_name`. +- [x] 3.5 Sabre: remark date-of-birth, gender, passport, document dates, nationality and card-fragment + rules, each unioning the remark path with `HistoryAssociationElement` and both `Association*/Content`. +- [x] 3.6 Sabre: employee-id (attribute + remark) and traveller-profile-id rules. +- [x] 3.7 Add `//s19:Comment` to `sabre.res.remarks_reference` so collected third-party names reach the + ticketing comment. +- [x] 3.8 Bump `rules_version`. + +## 4. Regression sweep + +- [x] 4.1 Re-derive the `counts` assertions in `test_pii_amadeus.py`, `test_pii_sabre.py`, + `test_pii_amadeus_coverage_gaps.py` and `test_pii_sabre_coverage_gaps.py` that shift because the new + marker patterns fire on existing fixtures. Derive each number from the engine. +- [x] 4.2 Confirm no new rule fires on an operational-only fixture (`queue_access_response.xml` stays + uncovered; locators and ticket numbers unchanged everywhere). + +## 5. Close out + +- [x] 5.1 Replay both raw payloads through the engine: no plaintext third-party PII, identity-document + data or person-linked identifier survives; agent names, fares and organisation codes do survive. +- [x] 5.2 Tests green (692 passed), `just types` clean, coverage 95.8% (gate 85%). NOTE: `just lint` + and `just fmt-check` are red, but identically so on pristine `master` — 40 `ruff check` violations + and 4 unformatted files, none in code this change touches. Pre-existing, tracked separately; this + change adds no new violations. +- [ ] 5.3 Conventional commit; sync specs and archive the change. diff --git a/src/channel_relay/pii/rules_fallback.json b/src/channel_relay/pii/rules_fallback.json index 60a96a5..c915d5b 100644 --- a/src/channel_relay/pii/rules_fallback.json +++ b/src/channel_relay/pii/rules_fallback.json @@ -1,6 +1,6 @@ { "schema_version": "1.0", - "rules_version": "amadeus+sabre+travelfusion+travelport+farelogix-expanded-2026-08-05", + "rules_version": "amadeus+sabre+travelfusion+travelport+farelogix-remarks-2026-08-12", "rules": [ { "id": "amadeus.pnr.surname", @@ -293,6 +293,154 @@ } ] }, + { + "id": "amadeus.pnr.remark_orderer_name", + "channel": "amadeus", + "operation": "^PNR_Reply", + "path": "//a:dataElementsIndiv[a:elementManagementData/a:segmentName='RM']/a:otherDataFreetext/a:longFreetext | //a:dataElementsIndiv/a:miscellaneousRemarks/a:remarks/a:freetext | //a:dataElementsIndiv/a:extendedRemark/a:structuredRemark/a:freetext", + "namespaces": { + "a": "http://xml.amadeus.com/PNRACC_17_1_1A" + }, + "pii_type": "person", + "method": "encrypt", + "extract_patterns": [ + { + "pattern": "\\bACE(?:ORB|CRM-ORDERER)-([A-Z][A-Z ]{2,60}?)\\s*$" + } + ] + }, + { + "id": "amadeus.pnr.remark_dob", + "channel": "amadeus", + "operation": "^PNR_Reply", + "path": "//a:dataElementsIndiv[a:elementManagementData/a:segmentName='RM']/a:otherDataFreetext/a:longFreetext | //a:dataElementsIndiv/a:miscellaneousRemarks/a:remarks/a:freetext | //a:dataElementsIndiv/a:extendedRemark/a:structuredRemark/a:freetext", + "namespaces": { + "a": "http://xml.amadeus.com/PNRACC_17_1_1A" + }, + "pii_type": "dob", + "method": "replace", + "replacement": "01JAN00", + "extract_patterns": [ + { + "pattern": "\\bDOB-(\\d{2}[A-Z]{3}\\d{2})\\b" + } + ] + }, + { + "id": "amadeus.pnr.remark_gender", + "channel": "amadeus", + "operation": "^PNR_Reply", + "path": "//a:dataElementsIndiv[a:elementManagementData/a:segmentName='RM']/a:otherDataFreetext/a:longFreetext | //a:dataElementsIndiv/a:miscellaneousRemarks/a:remarks/a:freetext | //a:dataElementsIndiv/a:extendedRemark/a:structuredRemark/a:freetext", + "namespaces": { + "a": "http://xml.amadeus.com/PNRACC_17_1_1A" + }, + "pii_type": "gender", + "method": "replace", + "replacement": "M", + "extract_patterns": [ + { + "pattern": "\\bGENDER-([MF])\\b" + } + ] + }, + { + "id": "amadeus.pnr.remark_employee_id", + "channel": "amadeus", + "operation": "^PNR_Reply", + "path": "//a:dataElementsIndiv[a:elementManagementData/a:segmentName='RM']/a:otherDataFreetext/a:longFreetext | //a:dataElementsIndiv/a:miscellaneousRemarks/a:remarks/a:freetext | //a:dataElementsIndiv/a:extendedRemark/a:structuredRemark/a:freetext", + "namespaces": { + "a": "http://xml.amadeus.com/PNRACC_17_1_1A" + }, + "pii_type": "person", + "method": "encrypt", + "extract_patterns": [ + { + "pattern": "\\bACECRM-EMPLOYEE ID-(\\d{3,})" + } + ] + }, + { + "id": "amadeus.pnr.remark_profile_ref", + "channel": "amadeus", + "operation": "^PNR_Reply", + "path": "//a:dataElementsIndiv[a:elementManagementData/a:segmentName='RM']/a:otherDataFreetext/a:longFreetext | //a:dataElementsIndiv/a:miscellaneousRemarks/a:remarks/a:freetext | //a:dataElementsIndiv/a:extendedRemark/a:structuredRemark/a:freetext", + "namespaces": { + "a": "http://xml.amadeus.com/PNRACC_17_1_1A" + }, + "pii_type": "person", + "method": "encrypt", + "extract_patterns": [ + { + "pattern": "\\bCYTRIC PROFILE REF:\\s*(\\d{3,})" + } + ] + }, + { + "id": "amadeus.pnr.remark_loyalty_pref", + "channel": "amadeus", + "operation": "^PNR_Reply", + "path": "//a:dataElementsIndiv[a:elementManagementData/a:segmentName='RM']/a:otherDataFreetext/a:longFreetext | //a:dataElementsIndiv/a:miscellaneousRemarks/a:remarks/a:freetext | //a:dataElementsIndiv/a:extendedRemark/a:structuredRemark/a:freetext", + "namespaces": { + "a": "http://xml.amadeus.com/PNRACC_17_1_1A" + }, + "pii_type": "frequent_flyer", + "method": "encrypt", + "extract_patterns": [ + { + "pattern": "\\bPREF HTL ID/SUP-[A-Z]{2}/ID-(\\d{3,})" + } + ] + }, + { + "id": "amadeus.pnr.fp_card_expiry", + "channel": "amadeus", + "operation": "^PNR_Reply", + "path": "//a:dataElementsIndiv[a:elementManagementData/a:segmentName='FP']/a:otherDataFreetext/a:longFreetext", + "namespaces": { + "a": "http://xml.amadeus.com/PNRACC_17_1_1A" + }, + "pii_type": "payment", + "method": "replace", + "replacement": "0000", + "extract_patterns": [ + { + "pattern": "X{4,}\\d{2,}/(\\d{4})\\s*$" + } + ] + }, + { + "id": "amadeus.pnr.fp_card", + "channel": "amadeus", + "operation": "^PNR_Reply", + "path": "//a:dataElementsIndiv[a:elementManagementData/a:segmentName='FP']/a:otherDataFreetext/a:longFreetext", + "namespaces": { + "a": "http://xml.amadeus.com/PNRACC_17_1_1A" + }, + "pii_type": "payment", + "method": "replace", + "replacement": "REDACTED", + "extract_patterns": [ + { + "pattern": "\\bCC[A-Z]{2}(X{4,}\\d{2,})" + } + ] + }, + { + "id": "amadeus.pnr.osi_contract_id", + "channel": "amadeus", + "operation": "^PNR_Reply", + "path": "//a:dataElementsIndiv[a:elementManagementData/a:segmentName='OS']/a:otherDataFreetext/a:longFreetext", + "namespaces": { + "a": "http://xml.amadeus.com/PNRACC_17_1_1A" + }, + "pii_type": "frequent_flyer", + "method": "encrypt", + "extract_patterns": [ + { + "pattern": "\\bCP/([A-Z0-9]{6,})" + } + ] + }, { "id": "amadeus.pnr.remarks_reference", "channel": "amadeus", @@ -733,6 +881,22 @@ "pii_type": "email", "method": "encrypt" }, + { + "id": "sabre.res.ctc_arranger_name", + "channel": "sabre", + "operation": "^(GetReservationRS|UpdateReservationRS)$", + "path": "//s19:PhoneNumber[s19:CityCode='CTC']/s19:Number", + "namespaces": { + "s19": "http://webservices.sabre.com/pnrbuilder/v1_19" + }, + "pii_type": "person", + "method": "encrypt", + "extract_patterns": [ + { + "pattern": "^[ME]-([A-Z][A-Z ]+?)-\\d" + } + ] + }, { "id": "sabre.res.phone", "channel": "sabre", @@ -1083,6 +1247,22 @@ } ] }, + { + "id": "sabre.res.received_from_arranger", + "channel": "sabre", + "operation": "^(GetReservationRS|UpdateReservationRS)$", + "path": "//s19:ReceivedFrom/s19:Name", + "namespaces": { + "s19": "http://webservices.sabre.com/pnrbuilder/v1_19" + }, + "pii_type": "person", + "method": "encrypt", + "extract_patterns": [ + { + "pattern": "^([A-Z]{2,} [A-Z]{2,}) \\d" + } + ] + }, { "id": "sabre.res.received_from_phone", "channel": "sabre", @@ -1092,19 +1272,260 @@ "s19": "http://webservices.sabre.com/pnrbuilder/v1_19" }, "pii_type": "phone", - "method": "encrypt", + "method": "replace", + "replacement": "REDACTED", "extract_patterns": [ { "pattern": "(\\d{7,15})" } ] }, + { + "id": "sabre.res.remark_third_party_name", + "channel": "sabre", + "operation": "^(GetReservationRS|UpdateReservationRS)$", + "path": "//s19:RemarkLine/s19:Text", + "namespaces": { + "s19": "http://webservices.sabre.com/pnrbuilder/v1_19" + }, + "pii_type": "person", + "method": "encrypt", + "extract_patterns": [ + { + "pattern": "\\bEMER-([A-Z]{2,}(?: [A-Z]{2,})+)" + }, + { + "pattern": "AUTHORITY TO CHARGE-([A-Z][A-Z ]{2,40}?)\\s*$" + }, + { + "pattern": "\\bTRAVEL ARRANGER/([A-Z][A-Z ]{2,40}?(?:-[A-Z])?)\\s*$" + }, + { + "pattern": "\\*84-([A-Z][A-Z ]{2,40}?)\\s*$" + } + ] + }, + { + "id": "sabre.res.remark_third_party_phone", + "channel": "sabre", + "operation": "^(GetReservationRS|UpdateReservationRS)$", + "path": "//s19:RemarkLine/s19:Text | //s19:HistoryAssociationElement | //s19:AssociationChild/s19:Content | //s19:AssociationParent/s19:Content", + "namespaces": { + "s19": "http://webservices.sabre.com/pnrbuilder/v1_19" + }, + "pii_type": "phone", + "method": "replace", + "replacement": "REDACTED", + "extract_patterns": [ + { + "pattern": "\\bEMER-(\\d{7,15})\\b" + } + ] + }, + { + "id": "sabre.res.future_ticketing_name", + "channel": "sabre", + "operation": "^(GetReservationRS|UpdateReservationRS)$", + "path": "//s19:FutureTicketing/s19:Comment", + "namespaces": { + "s19": "http://webservices.sabre.com/pnrbuilder/v1_19" + }, + "pii_type": "person", + "method": "encrypt", + "extract_patterns": [ + { + "pattern": "\\bISS TKT FR ([A-Z]{3,})" + } + ] + }, + { + "id": "sabre.res.remark_dob", + "channel": "sabre", + "operation": "^(GetReservationRS|UpdateReservationRS)$", + "path": "//s19:RemarkLine/s19:Text | //s19:HistoryAssociationElement | //s19:AssociationChild/s19:Content | //s19:AssociationParent/s19:Content", + "namespaces": { + "s19": "http://webservices.sabre.com/pnrbuilder/v1_19" + }, + "pii_type": "dob", + "method": "replace", + "replacement": "01JAN00", + "extract_patterns": [ + { + "pattern": "\\bBIRTHDATE-(\\d{2}[A-Z]{3}\\d{2})\\b" + }, + { + "pattern": "\\bDHSNBR-PAX-[\\d.]+,(\\d{2}[A-Z]{3}\\d{2})" + } + ] + }, + { + "id": "sabre.res.remark_gender", + "channel": "sabre", + "operation": "^(GetReservationRS|UpdateReservationRS)$", + "path": "//s19:RemarkLine/s19:Text | //s19:HistoryAssociationElement | //s19:AssociationChild/s19:Content | //s19:AssociationParent/s19:Content", + "namespaces": { + "s19": "http://webservices.sabre.com/pnrbuilder/v1_19" + }, + "pii_type": "gender", + "method": "replace", + "replacement": "M", + "extract_patterns": [ + { + "pattern": "\\bGENDER-([MF])\\b" + }, + { + "pattern": "\\bDHSNBR-PAX-[\\d.]+,\\d{2}[A-Z]{3}\\d{2},([MF])" + } + ] + }, + { + "id": "sabre.res.remark_passport", + "channel": "sabre", + "operation": "^(GetReservationRS|UpdateReservationRS)$", + "path": "//s19:RemarkLine/s19:Text | //s19:HistoryAssociationElement | //s19:AssociationChild/s19:Content | //s19:AssociationParent/s19:Content", + "namespaces": { + "s19": "http://webservices.sabre.com/pnrbuilder/v1_19" + }, + "pii_type": "passport_id", + "method": "replace", + "replacement": "00000000", + "extract_patterns": [ + { + "pattern": "\\bPSPT-[A-Z]{2} ([A-Z0-9]{5,15})" + }, + { + "pattern": "\\bPSPT/CO-[A-Z]{2}/NR-([A-Z0-9]{5,15})" + } + ] + }, + { + "id": "sabre.res.remark_doc_dates", + "channel": "sabre", + "operation": "^(GetReservationRS|UpdateReservationRS)$", + "path": "//s19:RemarkLine/s19:Text | //s19:HistoryAssociationElement | //s19:AssociationChild/s19:Content | //s19:AssociationParent/s19:Content", + "namespaces": { + "s19": "http://webservices.sabre.com/pnrbuilder/v1_19" + }, + "pii_type": "passport_id", + "method": "replace", + "replacement": "31DEC99", + "extract_patterns": [ + { + "pattern": "\\bPSPT-[A-Z]{2} [A-Z0-9]{5,15} (\\d{2}[A-Z]{3}\\d{2})" + }, + { + "pattern": "/IS-(\\d{2}[A-Z]{3}\\d{2})" + }, + { + "pattern": "/EX-(\\d{2}[A-Z]{3}\\d{2})" + } + ] + }, + { + "id": "sabre.res.remark_nationality", + "channel": "sabre", + "operation": "^(GetReservationRS|UpdateReservationRS)$", + "path": "//s19:RemarkLine/s19:Text | //s19:HistoryAssociationElement | //s19:AssociationChild/s19:Content | //s19:AssociationParent/s19:Content", + "namespaces": { + "s19": "http://webservices.sabre.com/pnrbuilder/v1_19" + }, + "pii_type": "nationality", + "method": "replace", + "replacement": "ZZ", + "extract_patterns": [ + { + "pattern": "\\bCTZN-([A-Z]{2})\\b" + }, + { + "pattern": "\\bPSPT-([A-Z]{2}) " + }, + { + "pattern": "\\bPSPT/CO-([A-Z]{2})/" + }, + { + "pattern": "/CI-([A-Z]{2})\\b" + } + ] + }, + { + "id": "sabre.res.history_card_expiry", + "channel": "sabre", + "operation": "^(GetReservationRS|UpdateReservationRS)$", + "path": "//s19:RemarkLine/s19:Text | //s19:HistoryAssociationElement | //s19:AssociationChild/s19:Content | //s19:AssociationParent/s19:Content", + "namespaces": { + "s19": "http://webservices.sabre.com/pnrbuilder/v1_19" + }, + "pii_type": "payment", + "method": "replace", + "replacement": "0000", + "extract_patterns": [ + { + "pattern": "X{4,}\\d{2,}[/¥](\\d{4})" + } + ] + }, + { + "id": "sabre.res.history_card_fragment", + "channel": "sabre", + "operation": "^(GetReservationRS|UpdateReservationRS)$", + "path": "//s19:RemarkLine/s19:Text | //s19:HistoryAssociationElement | //s19:AssociationChild/s19:Content | //s19:AssociationParent/s19:Content", + "namespaces": { + "s19": "http://webservices.sabre.com/pnrbuilder/v1_19" + }, + "pii_type": "payment", + "method": "replace", + "replacement": "REDACTED", + "extract_patterns": [ + { + "pattern": "(\\d{0,6}X{4,}\\d{2,})" + } + ] + }, + { + "id": "sabre.res.employee_id_attribute", + "channel": "sabre", + "operation": "^(GetReservationRS|UpdateReservationRS)$", + "path": "//s19:Passenger/@referenceNumber", + "namespaces": { + "s19": "http://webservices.sabre.com/pnrbuilder/v1_19" + }, + "pii_type": "person", + "method": "encrypt" + }, + { + "id": "sabre.res.employee_id_remark", + "channel": "sabre", + "operation": "^(GetReservationRS|UpdateReservationRS)$", + "path": "//s19:RemarkLine/s19:Text", + "namespaces": { + "s19": "http://webservices.sabre.com/pnrbuilder/v1_19" + }, + "pii_type": "person", + "method": "encrypt", + "extract_patterns": [ + { + "pattern": "\\bPROFILE [A-Z]+-(\\d{5,})" + } + ] + }, + { + "id": "sabre.res.traveller_profile_id", + "channel": "sabre", + "operation": "^(GetReservationRS|UpdateReservationRS)$", + "path": "//s19:Passenger/s19:Profiles/s19:Profile[s19:ProfileType='TVL']/s19:ProfileID | //o14:Profile[o14:Type='TRAVELER']/o14:ID", + "namespaces": { + "s19": "http://webservices.sabre.com/pnrbuilder/v1_19", + "o14": "http://services.sabre.com/res/or/v1_14" + }, + "pii_type": "person", + "method": "encrypt" + }, { "id": "sabre.res.remarks_reference", "channel": "sabre", "operation": "^(GetReservationRS|UpdateReservationRS)$", "rule_type": "reference", - "path": "//s19:Text | //s19:Content | //s19:HistoryAssociationElement | //s19:AccountingLineText | //s19:GenericSpecialRequest/s19:FreeText | //s19:GenericSpecialRequest/s19:FullText | //s19:GenericSpecialRequests/s19:FreeText | //s19:GenericSpecialRequests/s19:FullText | //s19:ETicketNumber | //s19:OriginalTicketDetails", + "path": "//s19:Text | //s19:Content | //s19:HistoryAssociationElement | //s19:AccountingLineText | //s19:Comment | //s19:GenericSpecialRequest/s19:FreeText | //s19:GenericSpecialRequest/s19:FullText | //s19:GenericSpecialRequests/s19:FreeText | //s19:GenericSpecialRequests/s19:FullText | //s19:ETicketNumber | //s19:OriginalTicketDetails", "namespaces": { "s19": "http://webservices.sabre.com/pnrbuilder/v1_19" }, diff --git a/tests/fixtures/amadeus/pnr_retrieve_third_party_remarks_response.xml b/tests/fixtures/amadeus/pnr_retrieve_third_party_remarks_response.xml new file mode 100644 index 0000000..e9bc2f3 --- /dev/null +++ b/tests/fixtures/amadeus/pnr_retrieve_third_party_remarks_response.xml @@ -0,0 +1,325 @@ + + + + http://www.w3.org/2005/08/addressing/anonymous + http://webservices.amadeus.com/PNRRET_17_1_1A + + + + + + + 1A + TQ7XF4 + + + + + + + PT + 1 + + NM + 1 + + + + + NIEMI + 1 + + + LAURA MS + ADT + + + + + + + + + + OT + 45 + + RM + 91 + + + + RM + *ACEORB-GREENE ALICE + + + + + RM + * + ACEORB-GREENE ALICE + + + + + + + OT + 46 + + RM + 92 + + + + RM + *ACECRM-ORDERER-GREENE ALICE + + + + + RM + * + ACECRM-ORDERER-GREENE ALICE + + + + + + + + OT + 47 + + RM + 93 + + + + RM + P + DOB-02FEB90/GENDER-F + + + + + RM + P + DOB-02FEB90/GENDER-F + + + + + + + + OT + 48 + + RM + 94 + + + + RM + *ACECRM-EMPLOYEE ID-37000001 + + + + + RM + * + ACECRM-EMPLOYEE ID-37000001 + + + + + + + OT + 49 + + RM + 95 + + + + RM + CYTRIC PROFILE REF:15000000001 + + + + + RM + CYTRIC PROFILE REF:15000000001 + + + + + + + OT + 50 + + RM + 96 + + + + RM + H + PREF HTL ID/SUP-CY/ID-770000001 + + + + + RM + H + PREF HTL ID/SUP-CY/ID-770000001 + + + + + + + + OT + 51 + + RM + 97 + + + + RM + I + EMPLOYEE ID MANDATORY, RM*ACECRM-EMPLOYEE ID- + + + + + + + + OT + 81 + + OS + 29 + + + + 3 + 28 + YY + + CP/LH100000000001 + + + + + + OT + 82 + + OS + 30 + + + + 3 + 28 + YY + + CMP OPK000FI + + + + + + OT + 83 + + OS + 31 + + + + 3 + 28 + YY + + OIN FI00000 + + + + + + OT + 84 + + OS + 32 + + + + 3 + 28 + YY + + NCA TSTCOOP + + + + + + + OT + 155 + + FA + 123 + + + + 3 + P06 + + PAX 105-2400000001/ETAY/12AUG26/HELMK0000/19000000 + + + + + + OT + 156 + + FP + 124 + + + + 3 + 16 + + PAX CCDCXXXXXXXXXX1111/0630 + + + + PT + 1 + + + + + + + diff --git a/tests/fixtures/sabre/get_reservation_third_party_remarks_response.xml b/tests/fixtures/sabre/get_reservation_third_party_remarks_response.xml new file mode 100644 index 0000000..b1912c2 --- /dev/null +++ b/tests/fixtures/sabre/get_reservation_third_party_remarks_response.xml @@ -0,0 +1,326 @@ + + + + + + + YPQGBZ + Z99 + + + + + CTC + M-DANA COHEN-972500000002 + + + CTC + E-DANACOHEN//EXAMPLECORP.COM + + + + + DANA COHEN 972500000002-MARK TAYLOR + + + + + 118000001 + AGY + + + 430000001 + CRP + + + + + + + TRAVEL ARRANGER/DANA COHEN-B + + + + + + + + D/PASSPORT 6 MONTH VALIDITY ADVISED BY - MARK TAYLOR - 12AUG + + + + + + + 12AUG2026-MTAYLR-ADVISE TRAVELER-ISRAEL IS CONTROLLED. + + + F/LOWEST FARE ACCEPTED - 3100.00-MTAYLR-08/12/26 + + + + + + + + DHSNBR-PAX-1.1,02FEB90,F,TESTER/ANNA, + + + + + + + PSPT/CO-IL/NR-39000001/IS-03DEC22/EX-04DEC32/CI-IL + + + + + + + PROFILE ILTESTCORP-12300001 + + + + + + + + *84-DANA COHEN + + + + + + + + OIN TESTCORP + + + ZT/CD-XZ00S00 + + + CONT- TOTAL FARE - USD 7100.00 + + + + + + + *CA5XXXXXXXXXXX1111¥11/27-XN + + + + + + + + TESTER + ANNA MS + + + 712300001 + TVL + + + + + + + XTRAF-//ANNA TESTER + + + + + + + BIRTHDATE-02FEB90 + + + + + + + GENDER-F + + + + + + + PSPT-IL 39000001 04DEC32 TESTER ANNA + + + + + + + TPREF-AUTH-AUTHORITY TO CHARGE-SAM BARNES + + + + + + + EMER-MAYA COHEN + + + + + + + EMER-972500000001 + + + + + + + + EMER-OTHER + + + + + + + CTZN-IL + + + + + + + DRIVELIC----TESTER/ANNA + + + + + + + + + TAW + T00F + 12AUG + 236 + AWAITING ADVS TO ISS TKT FR DANA + + + + + + + + Z99 + T00F + + + 215 + ANA + P!BIRTHDATE-02FEB90 + + TESTER/ANNA MS + + + Name + TESTER/ANNA MS + + + Remark + P!BIRTHDATE-02FEB90 + + + + 217 + ANA + V!PSPT-IL 39000001 04DEC32 TESTER ANNA + + Remark + V!PSPT-IL 39000001 04DEC32 TESTER ANNA + + + + 226 + ANA + E!EMER-MAYA COHEN + + Remark + E!EMER-MAYA COHEN + + + + 235 + ANA + E!EMER-972500000001 + + Remark + E!EMER-972500000001 + + + + 223 + ANA + F!TPREF-AUTH-AUTHORITY TO CHARGE-SAM BARNES + + Remark + F!TPREF-AUTH-AUTHORITY TO CHARGE-SAM BARNES + + + + 228 + ANA + V!CTZN-IL + + Remark + V!CTZN-IL + + + + + 309 + ANA + *CA5XXXXXXXXXXX1111/1127-XN + + FOP + *CA5XXXXXXXXXXX1111/1127 + + + FOP + *CA5XXXXXXXXXXX1111/1127-XN + + + + + + + + 118000001 + TAGENCY + AGY + T00F + + + + + 712300001 + TRAVELER + TVL + T00F + + + + + + + diff --git a/tests/integration/test_pii_extended_operations_relay.py b/tests/integration/test_pii_extended_operations_relay.py index 23af5c6..6238612 100644 --- a/tests/integration/test_pii_extended_operations_relay.py +++ b/tests/integration/test_pii_extended_operations_relay.py @@ -70,3 +70,56 @@ def test_sabre_update_reservation_redacted_end_to_end( assert value not in response.content assert b"ENC_" in response.content assert b"PNR004" in response.content + + +def test_amadeus_third_party_remarks_redacted_end_to_end( + relay_client_factory: RelayClientFactory, monkeypatch: pytest.MonkeyPatch +) -> None: + client = _client( + relay_client_factory, + monkeypatch, + channel_type=ChannelType.AMADEUS, + fixture="pnr_retrieve_third_party_remarks_response.xml", + ) + with client: + response = client.post("/channel/amadeus/op", content=b"", headers={"content-type": "text/xml"}) + assert response.status_code == 200 + # Orderer name, remark date of birth, person-linked identifiers, and the FP card. + for value in (b"GREENE", b"ALICE", b"02FEB90", b"37000001", b"770000001", b"XXXXXXXXXX1111"): + assert value not in response.content + assert b"ENC_" in response.content + # Organisation-level codes and the ticket number stay verbatim through the relay. + assert b"CMP OPK000FI" in response.content + assert b"PAX 105-2400000001/ETAY" in response.content + + +def test_sabre_third_party_remarks_redacted_end_to_end( + relay_client_factory: RelayClientFactory, monkeypatch: pytest.MonkeyPatch +) -> None: + client = _client( + relay_client_factory, + monkeypatch, + channel_type=ChannelType.SABRE, + fixture="get_reservation_third_party_remarks_response.xml", + ) + with client: + response = client.post("/channel/sabre/op", content=b"", headers={"content-type": "text/xml"}) + assert response.status_code == 200 + # Third parties, identity documents in remarks, the history card fragment, and the + # person-linked identifiers. + for value in ( + b"DANA COHEN", + b"MAYA", + b"SAM BARNES", + b"02FEB90", + b"39000001", + b"5XXXXXXXXXXX1111", + b"12300001", + b"712300001", + ): + assert value not in response.content + assert b"ENC_" in response.content + # Agency-agent identity and commercial data are operational and survive the relay. + assert b"MARK TAYLOR" in response.content + assert b"OIN TESTCORP" in response.content + assert b"YPQGBZ" in response.content diff --git a/tests/unit/test_pii_amadeus.py b/tests/unit/test_pii_amadeus.py index 986c77e..c3b21af 100644 --- a/tests/unit/test_pii_amadeus.py +++ b/tests/unit/test_pii_amadeus.py @@ -287,6 +287,117 @@ def test_remark_tokens_round_trip( assert b"NAME-PETER/HUGO/JOHNSON" in restored +THIRD_PARTY_FIXTURE = FIXTURES_DIR / "amadeus" / "pnr_retrieve_third_party_remarks_response.xml" + + +@pytest.fixture(name="third_party_body") +def third_party_body_fixture() -> bytes: + return THIRD_PARTY_FIXTURE.read_bytes() + + +class TestThirdPartyRemarks: + """A customer-reported gap: PII carried only by RM remark free text, plus the FP and OS + elements that had no rule at all. + + Every RM remark is emitted twice — once under ``miscellaneousRemarks/remarks`` with the + leading ``*`` inside the text, once under ``extendedRemark/structuredRemark`` with that ``*`` + hoisted into ``category``. Patterns must therefore not require the ``*``, and each assertion + below covers both mirrors. + """ + + def test_orderer_name_encrypted_in_both_mirrors( + self, third_party_body: bytes, baked_ruleset: RuleSet, pii_keyring: Keyring, xml_texts: XmlTexts + ) -> None: + redacted, _ = redact_response_body( + third_party_body, channel="amadeus", ruleset=baked_ruleset, keyring=pii_keyring + ) + # The orderer is a third party: no structured name node carries them, so the reference + # pass cannot see the value and a field rule has to reach it. + assert b"GREENE" not in redacted + assert b"ALICE" not in redacted + # Both markers survive, in both mirrors (2 x ACEORB + 2 x ACECRM-ORDERER). + assert redacted.count(b"ACEORB-") == 2 + assert redacted.count(b"ACECRM-ORDERER-") == 2 + orderer_texts = [t for t in xml_texts(redacted, "freetext") if "ACEORB-" in t] + assert len(orderer_texts) == 2 + for text in orderer_texts: + token = text.split("ACEORB-")[1] + assert TOKEN_RE.fullmatch(token) + assert decrypt(token, pii_keyring) == "GREENE ALICE" + + def test_remark_dob_and_gender_use_format_preserving_sentinels( + self, third_party_body: bytes, baked_ruleset: RuleSet, pii_keyring: Keyring, xml_texts: XmlTexts + ) -> None: + redacted, _ = redact_response_body( + third_party_body, channel="amadeus", ruleset=baked_ruleset, keyring=pii_keyring + ) + assert b"02FEB90" not in redacted + # Identity-document spans in free text keep their shape (a DDMMMYY-shaped sentinel and the + # existing gender sentinel), so a consumer parsing the remark line still succeeds. + dob_texts = [t for t in xml_texts(redacted, "freetext") if t.startswith("DOB-")] + assert dob_texts == ["DOB-01JAN00/GENDER-M", "DOB-01JAN00/GENDER-M"] + + def test_person_linked_identifiers_encrypted( + self, third_party_body: bytes, baked_ruleset: RuleSet, pii_keyring: Keyring, xml_texts: XmlTexts + ) -> None: + redacted, _ = redact_response_body( + third_party_body, channel="amadeus", ruleset=baked_ruleset, keyring=pii_keyring + ) + for gone in (b"37000001", b"15000000001", b"770000001", b"LH100000000001"): + assert gone not in redacted + texts = xml_texts(redacted, "freetext") + employee = next(t for t in texts if t.startswith("*ACECRM-EMPLOYEE ID-")) + token = employee.removeprefix("*ACECRM-EMPLOYEE ID-") + assert TOKEN_RE.fullmatch(token) + assert decrypt(token, pii_keyring) == "37000001" + # The template remark carries the same marker with no value; it must be untouched, + # trailing hyphen included. + assert b"EMPLOYEE ID MANDATORY, RM*ACECRM-EMPLOYEE ID-<" in redacted + + def test_form_of_payment_element_card_redacted( + self, third_party_body: bytes, baked_ruleset: RuleSet, pii_keyring: Keyring, xml_texts: XmlTexts + ) -> None: + redacted, _ = redact_response_body( + third_party_body, channel="amadeus", ruleset=baked_ruleset, keyring=pii_keyring + ) + # The FP element had no rule at all. Supplier-side masking is not trusted: the surviving + # digits still reveal the last four. + assert b"XXXXXXXXXX1111" not in redacted + assert b"0630" not in redacted + fp = next(t for t in xml_texts(redacted, "longFreetext") if t.startswith("PAX CCDC")) + assert fp == "PAX CCDCREDACTED/0000" + + def test_organisation_codes_and_ticket_number_preserved( + self, third_party_body: bytes, baked_ruleset: RuleSet, pii_keyring: Keyring + ) -> None: + redacted, _ = redact_response_body( + third_party_body, channel="amadeus", ruleset=baked_ruleset, keyring=pii_keyring + ) + # Organisation-level corporate account codes are commercial data, not personal data: + # redacting them breaks fare reissue. Ticket number and locator are operational (§7). + for kept in ( + b"CMP OPK000FI", + b"OIN FI00000", + b"NCA TSTCOOP", + b"PAX 105-2400000001/ETAY/12AUG26/HELMK0000/19000000", + b"TQ7XF4", + ): + assert kept in redacted + + def test_third_party_tokens_round_trip( + self, third_party_body: bytes, baked_ruleset: RuleSet, pii_keyring: Keyring + ) -> None: + redacted, _ = redact_response_body( + third_party_body, channel="amadeus", ruleset=baked_ruleset, keyring=pii_keyring + ) + restored, _ = deanonymize_request_body(redacted, keyring=pii_keyring) + assert b"*ACEORB-GREENE ALICE" in restored + assert b"ACECRM-ORDERER-GREENE ALICE" in restored + assert b"*ACECRM-EMPLOYEE ID-37000001" in restored + assert b"CYTRIC PROFILE REF:15000000001" in restored + assert b"CP/LH100000000001" in restored + + def test_ruleset_version_covers_amadeus(baked_ruleset: RuleSet) -> None: assert any(rule.channel == "amadeus" for rule in baked_ruleset.rules) assert "amadeus" in baked_ruleset.rules_version diff --git a/tests/unit/test_pii_sabre.py b/tests/unit/test_pii_sabre.py index 8d53030..81a9561 100644 --- a/tests/unit/test_pii_sabre.py +++ b/tests/unit/test_pii_sabre.py @@ -781,6 +781,177 @@ def test_trip_search_given_name_gone_from_remarks_and_history( assert b"SEBDWC" in redacted +class TestThirdPartyAndHistoryMirrors: + """A customer-reported gap with two distinct causes. + + 1. Non-passenger persons — emergency contact, authority-to-charge holder, travel arranger — + have no structured name node anywhere in the document. The phase-2 reference pass can only + scrub values a phase-1 field rule already collected, so nothing could see them. Observed in + the live payload as ``EMER- ``: the passenger surname beside the + emergency contact's name *was* scrubbed, the contact's own given name was not. + 2. Sabre re-emits every coded passenger remark three times — ``RemarkLine/Text``, + ``HistoryAssociationElement``, and ``AssociationChild``/``AssociationParent`` ``Content``, + with ``@code`` turned into a ``P!``/``V!``/``E!`` prefix. Reference rules accept only an + encrypt action, so date of birth, passport, nationality and card data (all one-way) can + never be propagated into those mirrors by the reference pass; their field rules have to name + the mirror paths. + """ + + FIXTURE = "get_reservation_third_party_remarks_response.xml" + + def test_emergency_contact_name_and_phone_redacted(self, baked_ruleset: RuleSet, pii_keyring: Keyring) -> None: + redacted, _ = _redact(_fixture(self.FIXTURE), baked_ruleset, pii_keyring) + assert b"MAYA" not in redacted + assert b"972500000001" not in redacted + # The marker survives, and EMER-OTHER carries no PII so it is left alone. + assert b"EMER-" in redacted + assert b"EMER-OTHER" in redacted + + def test_authority_to_charge_holder_redacted(self, baked_ruleset: RuleSet, pii_keyring: Keyring) -> None: + redacted, _ = _redact(_fixture(self.FIXTURE), baked_ruleset, pii_keyring) + assert b"SAM BARNES" not in redacted + assert b"TPREF-AUTH-AUTHORITY TO CHARGE-" in redacted + + def test_arranger_redacted_everywhere_and_shares_one_token( + self, baked_ruleset: RuleSet, pii_keyring: Keyring, xml_texts: XmlTexts + ) -> None: + """The arranger's name is introduced by the ``CTC`` contact phone node. Collecting it there + in phase 1 is what lets the phase-2 reference pass reach its other renderings, so those + carry the same token.""" + redacted, _ = _redact(_fixture(self.FIXTURE), baked_ruleset, pii_keyring) + assert b"DANA COHEN" not in redacted + assert b"DANACOHEN" not in redacted + # The invoice UDID field and received-from both resolve to the ciphertext of the value the + # CTC node contributed to the collector. + udid = next(t for t in xml_texts(redacted, "Text") if t.startswith("*84-")) + token = udid.removeprefix("*84-") + assert TOKEN_RE.fullmatch(token) + assert decrypt(token, pii_keyring) == "DANA COHEN" + assert token in next(t for t in xml_texts(redacted, "Name") if "MARK TAYLOR" in t) + # The TRAVEL ARRANGER line ends in an operational "-B" qualifier. That is folded INTO the + # encrypted span deliberately: "-" is a base64url character, so a token placed immediately + # before "-B" is re-consumed by the greedy token scan on the way back upstream and never + # de-anonymizes — which would send a Wenrix token to the channel. Folding the suffix in + # keeps the marker visible, keeps the round-trip exact, and costs only a distinct token. + arranger = next(t for t in xml_texts(redacted, "Text") if t.startswith("TRAVEL ARRANGER/")) + arranger_token = arranger.removeprefix("TRAVEL ARRANGER/") + assert TOKEN_RE.fullmatch(arranger_token) + assert decrypt(arranger_token, pii_keyring) == "DANA COHEN-B" + + def test_arranger_name_survives_the_whole_node_phone_rule( + self, baked_ruleset: RuleSet, pii_keyring: Keyring, xml_texts: XmlTexts + ) -> None: + """Rules mutate the tree in list order, and two existing rules overwrite the very spans the + arranger rules read: ``sabre.res.phone`` replaces the whole ``CTC`` node with a sentinel, + and ``sabre.res.received_from_phone`` tokenizes the digits that + ``sabre.res.received_from_arranger`` keys on to tell an arranger from a lone agent name. + Both arranger rules must be ordered *before* their respective phone rule; otherwise the name + is destroyed or unmatchable, never enters the collector, and every other rendering leaks. + This test fails if either rule is moved after its phone rule.""" + redacted, _ = _redact(_fixture(self.FIXTURE), baked_ruleset, pii_keyring) + # The CTC node itself is masked whole (no ENC_ token abutting the phone digits) ... + ctc = list(xml_texts(redacted, "Number")) + assert ctc and all(_MASKED_RE.fullmatch(t) for t in ctc) + # ... yet the name it carried was collected, so the reference pass reached the remarks. + assert b"DANA" not in redacted + # ReceivedFrom: the arranger name is a reversible token, the phone is a one-way sentinel, + # and the agent name is untouched. + received_from = next(t for t in xml_texts(redacted, "Name") if "MARK TAYLOR" in t) + arranger, phone = received_from.removesuffix("-MARK TAYLOR").split(" ") + assert TOKEN_RE.fullmatch(arranger) + assert decrypt(arranger, pii_keyring) == "DANA COHEN" + assert _MASKED_RE.fullmatch(phone) + + def test_ticketing_comment_name_redacted(self, baked_ruleset: RuleSet, pii_keyring: Keyring) -> None: + # A bare given name in a free-text comment: the collected value is the full name, so the + # reference pass cannot match it and the comment needs its own marker-anchored rule. + redacted, _ = _redact(_fixture(self.FIXTURE), baked_ruleset, pii_keyring) + assert b"AWAITING ADVS TO ISS TKT FR " in redacted + comment = redacted.split(b"AWAITING ADVS TO ISS TKT FR ")[1].split(b"<")[0] + assert TOKEN_RE.fullmatch(comment.decode()) + assert decrypt(comment.decode(), pii_keyring) == "DANA" + + def test_identity_documents_redacted_in_all_three_renderings( + self, baked_ruleset: RuleSet, pii_keyring: Keyring + ) -> None: + """The regression a reference-rule-only fix would miss.""" + redacted, _ = _redact(_fixture(self.FIXTURE), baked_ruleset, pii_keyring) + for gone in (b"02FEB90", b"39000001", b"03DEC22", b"04DEC32", b"CTZN-IL", b"GENDER-F"): + assert gone not in redacted, gone + # Each of the three renderings carries the sentinel: remark, HistoryAssociationElement, + # AssociationChild/Content. + assert redacted.count(b"BIRTHDATE-01JAN00") == 3 + assert redacted.count(b"CTZN-ZZ") == 3 + assert redacted.count(b"PSPT-ZZ 00000000 31DEC99") == 3 + assert b"PSPT/CO-ZZ/NR-00000000/IS-31DEC99/EX-31DEC99/CI-ZZ" in redacted + assert b"GENDER-M" in redacted + assert b"DHSNBR-PAX-1.1,01JAN00,M," in redacted + + def test_history_card_fragment_and_expiry_redacted(self, baked_ruleset: RuleSet, pii_keyring: Keyring) -> None: + # The same card renders with "¥" in the remark and "/" in history. Only the remark form was + # covered before, and the history form appeared many times over. + redacted, _ = _redact(_fixture(self.FIXTURE), baked_ruleset, pii_keyring) + assert b"5XXXXXXXXXXX1111" not in redacted + assert b"1127" not in redacted + assert b"11/27" not in redacted + assert b"*CAREDACTED/0000-XN" in redacted + + def test_person_linked_identifiers_encrypted_and_org_profiles_preserved( + self, baked_ruleset: RuleSet, pii_keyring: Keyring, xml_texts: XmlTexts + ) -> None: + redacted, _ = _redact(_fixture(self.FIXTURE), baked_ruleset, pii_keyring) + assert b"12300001" not in redacted + assert b"712300001" not in redacted + # Employee id: same token in the attribute and in the remark that echoes it. + employee_tokens = _attrs(redacted, "referenceNumber") + assert len(employee_tokens) == 1 + assert decrypt(employee_tokens[0], pii_keyring) == "12300001" + profile_remark = next(t for t in xml_texts(redacted, "Text") if t.startswith("PROFILE ILTESTCORP-")) + assert profile_remark == f"PROFILE ILTESTCORP-{employee_tokens[0]}" + # Traveller profile id encrypted in both namespaces; agency and corporate ids preserved. + assert _ns_texts(redacted, "//s19:Passenger/s19:Profiles/s19:Profile/s19:ProfileID") != ["712300001"] + assert b"118000001" in redacted + assert b"430000001" in redacted + + def test_traveller_profile_id_also_caught_in_the_older_fixture( + self, baked_ruleset: RuleSet, pii_keyring: Keyring + ) -> None: + """The traveller-profile rule reaches back into evidence captured before this change: this + fixture carried a plaintext ``TVL`` profile id all along. Locked here because the suite that + owns that fixture asserts values rather than counts, so the new catch would otherwise be + invisible.""" + redacted, counts = _redact(_fixture("get_reservation_remark_name_response.xml"), baked_ruleset, pii_keyring) + assert b"120402457" not in redacted + assert counts["person"] == 21 + + def test_agent_identity_and_commercial_data_preserved(self, baked_ruleset: RuleSet, pii_keyring: Keyring) -> None: + """Agency staff acting professionally are not redacted: the remark trail exists to be + auditable, and sign-in codes are classified operational. Fares and organisation-level + codes must survive too.""" + redacted, _ = _redact(_fixture(self.FIXTURE), baked_ruleset, pii_keyring) + for kept in ( + b"MARK TAYLOR", + b"MTAYLR", + b"Z99", + b"YPQGBZ", + b"CONT- TOTAL FARE - USD 7100.00", + b"3100.00", + b"OIN TESTCORP", + b"ZT/CD-XZ00S00", + b"ADVISE TRAVELER-ISRAEL IS CONTROLLED.", + ): + assert kept in redacted, kept + + def test_third_party_tokens_round_trip(self, baked_ruleset: RuleSet, pii_keyring: Keyring) -> None: + redacted, _ = _redact(_fixture(self.FIXTURE), baked_ruleset, pii_keyring) + restored, _ = deanonymize_request_body(redacted, keyring=pii_keyring) + assert b"TRAVEL ARRANGER/DANA COHEN-B" in restored + assert b"*84-DANA COHEN" in restored + assert b"EMER-MAYA COHEN" in restored + assert b"TPREF-AUTH-AUTHORITY TO CHARGE-SAM BARNES" in restored + assert b"PROFILE ILTESTCORP-12300001" in restored + + def test_ruleset_version_covers_sabre(baked_ruleset: RuleSet) -> None: assert any(rule.channel == "sabre" for rule in baked_ruleset.rules) assert "sabre" in baked_ruleset.rules_version