Feature/nctcog integration

.dockerignore

@@ -0,0 +1,11 @@
+.env*
+**/.env*
+.aws/
+**/.aws/
+*.pem
+*.key
+.venv/
+cdk.out/
+**/__pycache__/
+infrastructure/
+infrastructure-as-code/

.github/workflows/ecr-image.yml

@@ -0,0 +1,40 @@
+name: Push ECR image
+
+on:
+  push:
+    branches: [main]
+    paths-ignore:
+      - 'infrastructure/**'
+      - 'infrastructure-as-code/**'
+      - '.github/workflows/infrastructure.yml'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  push:
+    if: github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ vars.ECR_PUSH_ROLE_ARN }}
+          aws-region: ${{ vars.AWS_REGION }}
+      - name: Verify ECR repository
+        run: aws ecr describe-repositories --repository-names flows2fim --query 'repositories[0].repositoryName' --output text > /dev/null
+      - uses: aws-actions/amazon-ecr-login@v2
+        id: login
+      - uses: docker/setup-buildx-action@v3
+      - uses: docker/build-push-action@v6
+        with:
+          context: .
+          target: prod
+          push: true
+          platforms: linux/amd64,linux/arm64
+          tags: ${{ steps.login.outputs.registry }}/flows2fim:sha-${{ github.sha }}

.github/workflows/infrastructure.yml

@@ -0,0 +1,65 @@
+name: Deploy ECR infrastructure
+
+on:
+  pull_request:
+    branches: [master]
+    paths:
+      - 'infrastructure/**'
+      - '.github/workflows/infrastructure.yml'
+  push:
+    branches: [master]
+    paths:
+      - 'infrastructure/**'
+      - '.github/workflows/infrastructure.yml'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  validate:
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: infrastructure
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v6
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+      - run: npm install --global aws-cdk@2.1143.0
+      - run: uv sync --locked 
+      - run: uv run --locked  ruff check app.py
+      - run: uv run --locked  pyright
+      - run: uv run --locked  cdk synth
+
+  deploy:
+    if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request'
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: infrastructure
+    permissions:
+      contents: read
+      id-token: write
+    concurrency:
+      group: flows2fim-registry
+      cancel-in-progress: false
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v6
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+      - run: npm install --global aws-cdk@2.1143.0
+      - run: uv sync --locked
+      - uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ vars.CDK_DEPLOY_ROLE_ARN }}
+          aws-region: ${{ vars.AWS_REGION }}
+      - run: uv run --locked cdk synth
+      - run: uv run --locked cdk deploy --require-approval never --role-arn "$CFN_EXECUTION_ROLE_ARN"
+        env:
+          CFN_EXECUTION_ROLE_ARN: ${{ vars.CFN_EXECUTION_ROLE_ARN }}

.gitignore

@@ -25,6 +25,14 @@ builds/*
 # Go workspace file
 go.work
 
+# Python CDK tooling and synthesized templates
+.venv/
+__pycache__/
+*.egg-info/
+.ruff_cache/
+.pyright/
+cdk.out/
+
 # Data files
 *.csv
 *.vrt

CHANGELOG.md

@@ -7,6 +7,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- `controls` and `validate` now fail with an actionable message, including SQL to backfill the column, when the `scenarios` table has no `fim_path` column
+- `controls` output file now has a `fim_path` column, appended after `map_exists`
+
+### Changed
+
+- **BREAKING** - FIM file paths are now read from a `fim_path` column in the `scenarios` table (relative to the FIM library root) instead of being derived from the `<reach_id>/z_<stage>/f_<flow>.tif` folder layout. `controls` requires the column, and `fim` requires it in the controls file and resolves relative paths against `-lib`. Absolute and GDAL VSI paths are used as-is
+- `fim` no longer requires the `z_<stage>/f_<flow>.tif` library layout. `domain.tif` is still read from `<lib>/<reach_id>/` when `-with_domain` is set
+- **BREAKING** - `validate` now matches library `.tif` files against `scenarios.fim_path` instead of parsing reach, flow and stage out of the folder layout, so any file layout is accepted. `<reach_id>/domain.tif` files are ignored. Scenarios with `map_exists = 1` and an empty `fim_path` are reported as missing FIMs
+- **BREAKING** - `validate` output columns: `-o_fims` gains a trailing `fim_path` column, and `-o_scenarios` now has a single `fim_path` column, since unmatched files are no longer parsed into scenario fields
+
+### Removed
+
+- **BREAKING** - `validate` flag `-o_unexpected_fims` and its check. `fim_path` is only recorded for `map_exists = 1` scenarios, so a raster for a mapless scenario cannot be matched to it; such files are reported in `-o_scenarios` instead
+
 ## [0.5.0] - 2026-08-04
 
 **Note:** Compatible with outputs from Ripple1D Pipeline version 0.11.0 (the release containing the `scenarios` rename) to present. Not compatible with earlier outputs, which carry a `rating_curves` table instead, see the Changed section below.

cmd/controls/controls.go

@@ -30,6 +30,7 @@ Database file must have a table 'scenarios' and contain following columns
         ds_wse REAL
         boundary_condition TEXT CHECK(boundary_condition IN ('nd','kwse'))
         map_exists BOOL CHECK(map_exists IN (0, 1))
+        fim_path TEXT (FIM file path relative to the FIM library root, NULL when map_exists = 0)
         UNIQUE(reach_id, us_flow, ds_wse, boundary_condition)
 
 Database file must have a table 'network' and contain following columns
@@ -57,13 +58,15 @@ type ScenarioRecord struct {
 	ControlReachStage float32
 	BoundaryCondition string
 	MapExists         bool
+	FimPath           string
 }
 
 type ResultRecord struct {
 	ReachID              int
 	Flow                 int
 	ControlReachStageStr string
 	MapExists            bool
+	FimPath              string
 }
 
 func ReadFlows(filePath string) (map[int]float32, error) {
@@ -180,7 +183,7 @@ func FetchUpstreamReaches(db *sql.DB, controlReachID int) ([]int, error) {
 
 func FetchNormalDepthFlowStage(db *sql.DB, reachID int, flow float32) (ScenarioRecord, error) {
 	row := db.QueryRow(`
-		SELECT us_flow, us_wse, ds_wse, map_exists
+		SELECT us_flow, us_wse, ds_wse, map_exists, COALESCE(fim_path, '')
 		FROM scenarios
 		WHERE reach_id = ?
 		AND boundary_condition = 'nd'
@@ -190,7 +193,7 @@ func FetchNormalDepthFlowStage(db *sql.DB, reachID int, flow float32) (ScenarioR
 	)
 
 	var scenario ScenarioRecord
-	if err := row.Scan(&scenario.Flow, &scenario.Stage, &scenario.ControlReachStage, &scenario.MapExists); err != nil {
+	if err := row.Scan(&scenario.Flow, &scenario.Stage, &scenario.ControlReachStage, &scenario.MapExists, &scenario.FimPath); err != nil {
 		// Check if the error is because of no rows
 		if err == sql.ErrNoRows {
 			// No rows found, not an error in this context
@@ -206,14 +209,14 @@ func FetchNormalDepthFlowStage(db *sql.DB, reachID int, flow float32) (ScenarioR
 
 func FetchNearestFlowStage(db *sql.DB, reachID int, flow, controlStage float32) (ScenarioRecord, error) {
 	row := db.QueryRow(`
-	SELECT us_flow, us_wse, ds_wse, boundary_condition, map_exists
+	SELECT us_flow, us_wse, ds_wse, boundary_condition, map_exists, COALESCE(fim_path, '')
 	FROM scenarios
 	WHERE reach_id = ?
 	ORDER BY ABS(us_flow - ? ), ABS(ds_wse - ?)
 	LIMIT 1;
 	`, reachID, flow, controlStage)
 	var scenario ScenarioRecord
-	if err := row.Scan(&scenario.Flow, &scenario.Stage, &scenario.ControlReachStage, &scenario.BoundaryCondition, &scenario.MapExists); err != nil {
+	if err := row.Scan(&scenario.Flow, &scenario.Stage, &scenario.ControlReachStage, &scenario.BoundaryCondition, &scenario.MapExists, &scenario.FimPath); err != nil {
 		// Check if the error is because of no rows
 		if err == sql.ErrNoRows {
 			// No rows found, not an error in this context
@@ -288,7 +291,7 @@ func TraverseUpstream(db *sql.DB, flows map[int]float32, startReaches []ControlD
 			}
 		}
 
-		result := ResultRecord{ReachID: scenario.ReachID, Flow: scenario.Flow, MapExists: scenario.MapExists}
+		result := ResultRecord{ReachID: scenario.ReachID, Flow: scenario.Flow, MapExists: scenario.MapExists, FimPath: scenario.FimPath}
 		if scenario.BoundaryCondition == "nd" {
 			result.ControlReachStageStr = "nd"
 		} else {
@@ -302,8 +305,8 @@ func TraverseUpstream(db *sql.DB, flows map[int]float32, startReaches []ControlD
 	return results, nil
 }
 
-// WriteCSV writes the control table. The map_exists column is appended last so
-// that positional readers of the first three columns are unaffected.
+// WriteCSV writes the control table. The map_exists and fim_path columns are
+// appended last so that positional readers of the first three columns are unaffected.
 func WriteCSV(data []ResultRecord, filePath string) error {
 	file, err := os.Create(filePath)
 	if err != nil {
@@ -314,16 +317,16 @@ func WriteCSV(data []ResultRecord, filePath string) error {
 	writer := csv.NewWriter(file)
 	defer writer.Flush()
 
-	if err := writer.Write([]string{"reach_id", "flow", "control_stage", "map_exists"}); err != nil {
+	if err := writer.Write([]string{"reach_id", "flow", "control_stage", "map_exists", "fim_path"}); err != nil {
 		return err
 	}
 
 	for _, d := range data {
-		mapExists := "0"
+		mapExists, fimPath := "0", ""
 		if d.MapExists {
-			mapExists = "1"
+			mapExists, fimPath = "1", d.FimPath
 		}
-		record := []string{strconv.Itoa(d.ReachID), fmt.Sprint(d.Flow), d.ControlReachStageStr, mapExists}
+		record := []string{strconv.Itoa(d.ReachID), fmt.Sprint(d.Flow), d.ControlReachStageStr, mapExists, fimPath}
 		if err := writer.Write(record); err != nil {
 			return err
 		}
@@ -422,6 +425,9 @@ func Run(args []string) (err error) {
 	if err := utils.CheckScenariosTable(db); err != nil {
 		return err
 	}
+	if err := utils.CheckFimPathColumn(db); err != nil {
+		return err
+	}
 
 	results, err := TraverseUpstream(db, flows, startReaches)
 	if err != nil {