Construct legacy Dataset IDs from selected roots without changing coverage

.claude/agents/scripts/d4d_builder.py

@@ -7,8 +7,18 @@
 """
 
 from datetime import datetime
+from pathlib import Path
+import sys
 from typing import Any, Dict, List, Optional, Union
 
+# Like the parser wrapper, support direct execution outside the checkout.
+_source = Path(__file__).resolve().parents[3] / 'src'
+if (_source / 'data_sheets_schema/legacy_doi.py').is_file() and str(_source) not in sys.path:
+    sys.path.insert(0, str(_source))
+
+from data_sheets_schema.legacy_doi import doi_value, source_value
+from data_sheets_schema.legacy_root_identity import root_identity_route, resolve_root_identity
+
 
 class D4DBuilder:
     """Build D4D YAML structure from mapped RO-Crate data."""
@@ -34,7 +44,9 @@ def build_dataset(self, rocrate_parser) -> Dict[str, Any]:
             Dict with D4D Dataset structure
         """
         # Use the same production gate as the packaged builder (#4588).
-        rocrate_parser.require_root_dataset()
+        root = rocrate_parser.require_root_dataset()
+        marked_id = root_identity_route(self.mapping)
+        identity = resolve_root_identity(root) if marked_id else None
         self.d4d_data = {}
 
         # Get all covered D4D fields
@@ -45,6 +57,11 @@ def build_dataset(self, rocrate_parser) -> Dict[str, Any]:
         # Map each covered field
         mapped_count = 0
         for d4d_field in covered_fields:
+            if marked_id and d4d_field == 'id':
+                if identity['id'] is not None:
+                    self.d4d_data['id'] = identity['id']
+                    mapped_count += 1
+                continue
             rocrate_property = self.mapping.get_rocrate_property(d4d_field)
             if not rocrate_property:
                 continue
@@ -54,10 +71,13 @@ def build_dataset(self, rocrate_parser) -> Dict[str, Any]:
 
             # Try to extract value from RO-Crate
             value = None
-            for rc_prop in rocrate_props:
-                value = rocrate_parser.get_property(rc_prop)
-                if value is not None:
-                    break
+            if d4d_field == 'doi':
+                value = source_value(rocrate_parser, rocrate_props)
+            else:
+                for rc_prop in rocrate_props:
+                    value = rocrate_parser.get_property(rc_prop)
+                    if value is not None:
+                        break
 
             if value is not None:
                 # Apply transformations based on field type
@@ -85,6 +105,10 @@ def apply_field_transformation(self, field_name: str, value: Any) -> Any:
         if not mapping_info:
             return value
 
+        # Dataset.doi requires the bare form, independently of stale TSV types.
+        if field_name == 'doi':
+            return doi_value(value)
+
         field_type = mapping_info.get('Type', '').lower()
 
         # Date transformations

.claude/agents/scripts/informativeness_scorer.py

@@ -9,6 +9,14 @@
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
 
+import sys
+
+_source = Path(__file__).resolve().parents[3] / 'src'
+if (_source / 'data_sheets_schema/legacy_root_identity.py').is_file() and str(_source) not in sys.path:
+    sys.path.insert(0, str(_source))
+
+from data_sheets_schema.legacy_root_identity import scoring_fields
+
 
 class InformativenessScorer:
     """Score and rank RO-Crate sources by D4D informativeness."""
@@ -51,7 +59,7 @@ def score_rocrate(
         scores = {}
 
         # 1. D4D Coverage - how many fields can this RO-Crate populate?
-        covered_fields = mapping_loader.get_covered_fields()
+        covered_fields = scoring_fields(mapping_loader)
         populated_count = 0
 
         for d4d_field in covered_fields:
@@ -144,7 +152,7 @@ def _count_unique_fields(
         other_parsers: List
     ) -> int:
         """Count fields unique to this RO-Crate vs others."""
-        covered_fields = mapping_loader.get_covered_fields()
+        covered_fields = scoring_fields(mapping_loader)
         unique_count = 0
 
         for d4d_field in covered_fields:

.claude/agents/scripts/mapping_loader.py

@@ -10,6 +10,14 @@
 from pathlib import Path
 from typing import Dict, List, Optional, Set
 
+import sys
+
+_source = Path(__file__).resolve().parents[3] / 'src'
+if (_source / 'data_sheets_schema/legacy_root_identity.py').is_file() and str(_source) not in sys.path:
+    sys.path.insert(0, str(_source))
+
+from data_sheets_schema.legacy_root_identity import MARKER, validate_rows
+
 
 class MappingLoader:
     """Load and manage D4D to RO-Crate field mappings from TSV file."""
@@ -36,9 +44,10 @@ def __init__(self, tsv_path: str):
     def _load_mappings(self):
         """Load and parse the TSV mapping file."""
         with open(self.tsv_path, 'r', encoding='utf-8') as f:
-            reader = csv.DictReader(f, delimiter='\t')
+            rows = list(csv.DictReader(f, delimiter='\t'))
+            validate_rows(rows)
 
-            for row in reader:
+            for row in rows:
                 # Skip header rows and empty rows
                 if not row.get('D4D Property') or row['D4D Property'].startswith('D4D:'):
                     continue
@@ -61,7 +70,10 @@ def _load_mappings(self):
                         for rc_prop in rocrate_field.split(','):
                             rc_prop = rc_prop.strip()
                             if rc_prop:
-                                self.rocrate_to_d4d[rc_prop] = d4d_field
+                                if row.get('Func', '').strip() == MARKER:
+                                    self.rocrate_to_d4d.setdefault(rc_prop, d4d_field)
+                                else:
+                                    self.rocrate_to_d4d[rc_prop] = d4d_field
 
                         # Track direct mappings (1:1 relationships)
                         direct = row.get('Direct mapping? Yes =1; No = 0', '0').strip()

.claude/agents/scripts/rocrate_merger.py

@@ -13,6 +13,10 @@
 from field_prioritizer import FieldPrioritizer, MergeStrategy
 from d4d_builder import D4DBuilder
 
+from data_sheets_schema.legacy_root_identity import (
+    root_identity_route, merge_identity_evidence, identity_report_lines,
+)
+
 
 class ROCrateMerger:
     """Merge multiple RO-Crate sources into single D4D dataset."""
@@ -25,6 +29,7 @@ def __init__(self, mapping_loader):
             mapping_loader: MappingLoader instance with field mappings
         """
         self.mapping = mapping_loader
+        self.root_identity_sources = None
         self.primary_index = 0
         self.primary_name = ""
         self.prioritizer = FieldPrioritizer()
@@ -66,6 +71,24 @@ def merge_rocrates(
         for parser in rocrate_parsers:
             parser.require_root_dataset()
 
+        marked_id = root_identity_route(self.mapping)
+        identity_sources = None
+        if marked_id:
+            if source_names is None:
+                source_names = [Path(parser.rocrate_path).name.replace(
+                    '-ro-crate-metadata.json', '') for parser in rocrate_parsers]
+            if len(source_names) != len(rocrate_parsers):
+                raise ValueError('source_names must match the selected RO-Crates')
+            identity_sources = merge_identity_evidence(
+                rocrate_parsers, source_names, primary_index)
+        if marked_id or self.root_identity_sources is not None:
+            # A new marked merge (or returning to a custom mapping) must not
+            # inherit an earlier Dataset.id when its primary has no identity.
+            self.merged_data = {}
+            self.provenance = {}
+            self.merge_stats = {key: 0 for key in self.merge_stats}
+        self.root_identity_sources = identity_sources
+
         self.merge_stats['total_sources'] = len(rocrate_parsers)
 
         # Get source names
@@ -114,12 +137,16 @@ def merge_rocrates(
             ]
 
             # Merge this field
-            merged_value, sources = self.merge_field(
-                field_name,
-                primary_value,
-                secondary_values,
-                primary_name
-            )
+            if marked_id and field_name == 'id':
+                merged_value = primary_value
+                sources = [primary_name] if primary_value is not None else []
+            else:
+                merged_value, sources = self.merge_field(
+                    field_name,
+                    primary_value,
+                    secondary_values,
+                    primary_name
+                )
 
             if merged_value is not None:
                 self.merged_data[field_name] = merged_value
@@ -247,6 +274,8 @@ def generate_merge_report(
             report.append(f"   - D4D fields contributed: {contributed_fields}")
             report.append("")
 
+        report.extend(identity_report_lines(self.root_identity_sources))
+
         # Merge statistics
         report.append("MERGE STATISTICS")
         report.append("-"*80)

.claude/agents/scripts/rocrate_to_d4d.py

@@ -43,6 +43,7 @@
 from rocrate_merger import ROCrateMerger
 from informativeness_scorer import InformativenessScorer
 from data_sheets_schema.legacy_publication import prepare_dataset, publish, diagnostic
+from data_sheets_schema.legacy_root_identity import root_identity_route, scoring_fields
 
 
 def render_transformation_report(
@@ -58,15 +59,20 @@ def render_transformation_report(
     f.write("="*80 + "\n\n")
 
     # Transformation summary
-    covered_fields = mapping_loader.get_covered_fields()
+    covered_fields = scoring_fields(mapping_loader)
     dataset = d4d_builder.get_dataset()
+    identity_construction = root_identity_route(mapping_loader)
+    populated_count = len(dataset) - int(identity_construction and 'id' in dataset)
+    percentage = populated_count / len(covered_fields) * 100 if covered_fields else 0
 
     f.write("TRANSFORMATION SUMMARY\n")
     f.write("-"*80 + "\n")
     f.write(f"Total D4D fields in mapping: {len(covered_fields)}\n")
-    f.write(f"Fields populated from RO-Crate: {len(dataset)}\n")
-    f.write(f"Coverage: {len(dataset)}/{len(covered_fields)} ")
-    f.write(f"({len(dataset)/len(covered_fields)*100:.1f}%)\n\n")
+    f.write(f"Fields populated from RO-Crate: {populated_count}\n")
+    f.write(f"Coverage: {populated_count}/{len(covered_fields)} ")
+    f.write(f"({percentage:.1f}%)\n\n")
+    if identity_construction:
+        f.write("Required Dataset.id construction is excluded from coverage.\n\n")
 
     # Unmapped RO-Crate properties
     mapped_props = mapping_loader.get_all_mapped_rocrate_properties()
@@ -456,8 +462,14 @@ def main():
 
     print(f"Output: {output_path}")
     print(f"\nFields populated: {len(dataset)}")
-    print(f"Coverage: {len(dataset)}/{len(mapping.get_covered_fields())} mapped fields")
-    print(f"Percentage: {len(dataset)/len(mapping.get_covered_fields())*100:.1f}%")
+    covered_fields = scoring_fields(mapping)
+    identity_construction = root_identity_route(mapping)
+    populated_count = len(dataset) - int(identity_construction and 'id' in dataset)
+    percentage = populated_count / len(covered_fields) * 100 if covered_fields else 0
+    print(f"Coverage: {populated_count}/{len(covered_fields)} mapped fields")
+    print(f"Percentage: {percentage:.1f}%")
+    if identity_construction:
+        print("Required Dataset.id construction is excluded from coverage.")
 
     return 0
 

data/ro-crate_mapping/d4d_rocrate_mapping_v2_semantic.tsv

@@ -82,3 +82,4 @@ RO-Crate: Fairscape Release RO-Crate	acquisition_methods			(slot exists but no d
 	version_access			Will older versions of the dataset continue to be supported/hosted/maintained? If so, how? If not, how will obsolescence be communicated to dataset consumers?	version			1	0	0		relatedMatch	http://www.w3.org/2004/02/skos/core#relatedMatch	minimal	version_access		"""All versions available"""	"""All versions available"""
 	vulnerable_populations			Information about protections for vulnerable populations in human subjects research.	atRiskPopulations			1	1	0		exactMatch	http://www.w3.org/2004/02/skos/core#exactMatch	none	vulnerable_populations		"""Children excluded"""	"""Children excluded"""
 	was_derived_from	str		(slot exists but no description)	generatedBy			1	1	0		exactMatch	http://www.w3.org/2004/02/skos/core#exactMatch	none	was_derived_from	xsd:string constraint	"""10.5281/zenodo.111"""	"""10.5281/zenodo.111"""
+D4D: Dataset	id	URI		Required identity from the selected crate root.	identifier,@id	root_identifier_v1	Scalar identifier, else root @id; preserve the complete DOI suffix; no minted ID or list selection. Construction only, excluded from informativeness scoring. Semantic fidelity unassessed.	1	0	0							doi:10.1234/Example/	https://doi.org/10.1234/Example/

notes/legacy_doi_2026-10-08/README.md

@@ -0,0 +1,101 @@
+# Legacy DOI construction
+
+This is the bounded repair in
+[#4670](https://github.com/bridge2ai/data-sheets-schema/issues/4670).
+
+The packaged and hidden TSV builders now give the `doi` target its own
+construction rule. Previously a bare source DOI such as `10.1234/MixedCase`
+became `https://doi.org/10.1234/MixedCase`, contradicting the Dataset schema's
+bare-DOI requirement. Generic URI targets keep their existing behavior.
+
+Recognized scalar `doi:`, HTTP(S) `doi.org`, and HTTP(S) `dx.doi.org` forms use
+the existing `scope.bare_doi` parser only to recognize an eligible repair.
+Prefix matching is case insensitive; the entire suffix, including its case and
+one or more trailing slashes, is taken from the source string. After the existing
+outer-whitespace trim, only the known representation prefix is removed. The
+parser's slash-stripped return value is never used as the constructed DOI.
+Strings already starting with `10.` stay exactly as supplied, including
+schema-valid bare forms outside the repair parser's narrower grammar. The
+mandatory final Dataset validator still decides whether a retained value is valid.
+
+Malformed strings, objects, booleans, numbers and lists stay intact in drafts.
+A singleton, repeated, mixed or conflicting list never contributes just its
+first or recognizable DOI. No source assertion is dropped to obtain a valid
+scalar, and returned mutable DOI values do not alias the parser's source data.
+Missing/null source properties retain the builder's existing missing-value rule.
+
+When an explicitly selected TSV maps several properties to `doi`, every
+nonmissing root assertion is checked. Exact repeated scalar strings agree;
+different strings or competing nonscalar values refuse construction. This
+includes different spellings of the same DOI: the legacy builder has no
+sidecar that retains the discarded alternative, so the caller must explicitly
+disambiguate the selected mapping. Member properties cannot fill a missing root
+value or participate in its conflict decision. Other targets retain their
+existing property precedence.
+
+The current default TSV is unchanged and still omits required `Dataset.id`.
+This repair does not make that producer generally schema valid, reroute it,
+mint IDs, reinterpret person roles, convert rounded sizes to exact byte counts,
+or rewrite historical outputs. Parent issues
+[#4594](https://github.com/bridge2ai/data-sheets-schema/issues/4594) and
+[#2915](https://github.com/bridge2ai/data-sheets-schema/issues/2915) remain open.
+Neither source coverage nor semantic equivalence follows from this correction;
+row retirement is not a coverage gain, and intact text does not establish
+`exactMatch`/`none`.
+
+## Pending suffix-preservation correction
+
+The second review round found [#4671](https://github.com/bridge2ai/data-sheets-schema/issues/4671)
+before publication: the first implementation used the recognizer's return value,
+which silently removed trailing slashes from prefixed DOI suffixes. The earlier
+suite covered this ending only for bare inputs. The correction now removes only
+the known prefix and adds both-builder controls for all accepted prefix forms,
+mixed case, internal slashes and one/multiple trailing slashes, through actual
+Dataset validation and file publication. Unrecognized forms remain intact for
+refusal. Production code and tests changed; independent review and a fresh
+focused run are pending. The 318-pass evidence below belongs to the earlier
+implementation and does not validate this correction.
+
+## Earlier validation results
+
+The serialized eight-module run passed **318 tests**, with zero failures,
+errors or skips, at commit
+`f85d132ebdb3b26c1d80d6414ec39220c226334d`, tree
+`5371e1734c2f7aeed14b1c0668c3154dbc3136e2`. The coordinator's terminal reported
+43.51 seconds and 14 dependency deprecation warnings. JUnit records 43.473
+seconds; its XML does not encode the warning count or attest the tested commit.
+
+| Module | Passed |
+| --- | ---: |
+| `tests/test_rocrate/test_legacy_doi.py` | 117 |
+| `tests/test_fairscape_integration/test_d4d_builder.py` | 37 |
+| `tests/test_rocrate/test_legacy_publication.py` | 51 |
+| `tests/test_rocrate/test_legacy_publication_adversarial.py` | 39 |
+| `tests/test_rocrate/test_legacy_result_contract.py` | 19 |
+| `tests/test_rocrate/test_legacy_envelope_adversarial.py` | 28 |
+| `tests/test_cli/test_rocrate_transform_mapping.py` | 17 |
+| `tests/test_cli/test_rocrate_transform_mapping_adversarial.py` | 10 |
+
+The earlier 117 new cases exercised both real builders and the current closed Dataset
+validator, packaged/hidden publishers, the real API batch path with an explicit
+valid-ID mapping, and direct hidden-script use from another working directory.
+They covered the then-tested accepted forms and preservation/refusal boundaries,
+including a later invalid batch DOI preventing every destination replacement.
+Existing publication, result-contract and wrapper controls also passed.
+Initial independent source review found no issues before that execution; the
+later review found the missing prefixed-suffix case documented above.
+
+[validation.json](validation.json) records the command with symbolic paths,
+the eight module counts, and 26 selected production/test/resource/dependency
+pins for that earlier commit. Each selected working file then matched the tested
+Git blob; the current correction intentionally changes the DOI helper and tests.
+These pins cover the named files, not a complete installed environment.
+The retained external JUnit artifact `legacy-doi-tests-01.xml` is 51,571 bytes,
+SHA-256 `30c50542813c258f877d4631d55341ab69cbe564b0afb2de163465a4d6ce3905`.
+The earlier artifact and its source pins are retained without relabeling them
+as evidence for the pending correction.
+
+No real-project replay, provider call, scientific rating or new historical label
+was produced. All five retained-input legacy replays and the remaining default
+construction defects remain obligations of #4594; #2915 stays open. The successful
+software controls establish this DOI slice, not overall legacy mapping repair.