fix(firestore-bigquery-export): normalize NONE/omit partitioning params for change-tracker 2.x (#2698)

* fix(firestore-bigquery-export): implement buildPartitioningConfig function and update tests

* fix(tests): improve readability and structure of partitioning tests

* chore(firebase-bigquery-export): release version 0.3.0 with breaking changes and improved partitioning validation

Author: andyma-star

firestore-bigquery-export/CHANGELOG.md

@@ -1,3 +1,9 @@
+## Version 0.3.0
+
+breaking change: reject invalid partitioning configuration combinations at startup with explicit error messages
+
+fix: normalize `NONE` / `omit` partitioning sentinels before mapping to change-tracker 2.x partitioning strategy
+
 ## Version 0.2.11
 
 chore: bump firestore-bigquery-change-tracker dependency to v2 in functions package

firestore-bigquery-export/extension.yaml

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 name: firestore-bigquery-export
-version: 0.2.11
+version: 0.3.0
 specVersion: v1beta
 
 displayName: Stream Firestore to BigQuery

firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning.test.ts

@@ -15,8 +15,12 @@ let dataset: Dataset;
 let table: Table;
 let randomID: string;
 let datasetId: string;
+const describeIfBigQueryIntegration =
+  process.env.RUN_BIGQUERY_INTEGRATION_TESTS === "true"
+    ? describe
+    : describe.skip;
 
-describe("processing partitions on a new table", () => {
+describeIfBigQueryIntegration("processing partitions on a new table", () => {
   beforeAll(async () => {
     jest.spyOn(logger, "debug").mockImplementation(() => {});
     jest.spyOn(logger, "info").mockImplementation(() => {});
@@ -576,7 +580,7 @@ describe("processing partitions on a new table", () => {
   });
 });
 
-describe("updateTableMetadata", () => {
+describeIfBigQueryIntegration("updateTableMetadata", () => {
   let testTable: Table;
   let testDataset: Dataset;
 
@@ -728,103 +732,106 @@ describe("updateTableMetadata", () => {
   });
 });
 
-describe("getPartitionValue with DELETE operations", () => {
-  let testTable: Table;
-  let testDataset: Dataset;
-
-  beforeAll(async () => {
-    const randomID = (Math.random() + 1).toString(36).substring(7);
-    const testDatasetId = `bq_delete_${randomID}`;
-    [testDataset] = await bq.createDataset(testDatasetId, {
-      location: "europe-west2",
-    });
-    [testTable] = await testDataset.createTable(`bq_delete_${randomID}`, {});
-  });
-
-  afterAll(async () => {
-    await deleteTable({
-      datasetId: testDataset.id,
+describeIfBigQueryIntegration(
+  "getPartitionValue with DELETE operations",
+  () => {
+    let testTable: Table;
+    let testDataset: Dataset;
+
+    beforeAll(async () => {
+      const randomID = (Math.random() + 1).toString(36).substring(7);
+      const testDatasetId = `bq_delete_${randomID}`;
+      [testDataset] = await bq.createDataset(testDatasetId, {
+        location: "europe-west2",
+      });
+      [testTable] = await testDataset.createTable(`bq_delete_${randomID}`, {});
     });
-  });
 
-  test("uses oldData for DELETE operations", () => {
-    const partitioningConfig = new PartitioningConfig({
-      granularity: "DAY",
-      bigqueryColumnName: "end_date",
-      bigqueryColumnType: "TIMESTAMP",
-      firestoreFieldName: "endDate",
+    afterAll(async () => {
+      await deleteTable({
+        datasetId: testDataset.id,
+      });
     });
 
-    const oldDate = admin.firestore.Timestamp.fromDate(
-      new Date("2024-01-15T10:00:00Z")
-    );
+    test("uses oldData for DELETE operations", () => {
+      const partitioningConfig = new PartitioningConfig({
+        granularity: "DAY",
+        bigqueryColumnName: "end_date",
+        bigqueryColumnType: "TIMESTAMP",
+        firestoreFieldName: "endDate",
+      });
 
-    const event: FirestoreDocumentChangeEvent = {
-      timestamp: "",
-      operation: ChangeType.DELETE,
-      documentName: "test/doc",
-      eventId: "event1",
-      documentId: "doc",
-      data: null,
-      oldData: { endDate: oldDate },
-    };
+      const oldDate = admin.firestore.Timestamp.fromDate(
+        new Date("2024-01-15T10:00:00Z")
+      );
 
-    const partitioning = new Partitioning(partitioningConfig, testTable);
-    const value = partitioning.getPartitionValue(event);
+      const event: FirestoreDocumentChangeEvent = {
+        timestamp: "",
+        operation: ChangeType.DELETE,
+        documentName: "test/doc",
+        eventId: "event1",
+        documentId: "doc",
+        data: null,
+        oldData: { endDate: oldDate },
+      };
 
-    expect(value.end_date).toBeDefined();
-  });
+      const partitioning = new Partitioning(partitioningConfig, testTable);
+      const value = partitioning.getPartitionValue(event);
 
-  test("returns empty object for DELETE when oldData is null", () => {
-    const partitioningConfig = new PartitioningConfig({
-      granularity: "DAY",
-      bigqueryColumnName: "end_date",
-      bigqueryColumnType: "TIMESTAMP",
-      firestoreFieldName: "endDate",
+      expect(value.end_date).toBeDefined();
     });
 
-    const event: FirestoreDocumentChangeEvent = {
-      timestamp: "",
-      operation: ChangeType.DELETE,
-      documentName: "test/doc",
-      eventId: "event1",
-      documentId: "doc",
-      data: null,
-      oldData: null,
-    };
+    test("returns empty object for DELETE when oldData is null", () => {
+      const partitioningConfig = new PartitioningConfig({
+        granularity: "DAY",
+        bigqueryColumnName: "end_date",
+        bigqueryColumnType: "TIMESTAMP",
+        firestoreFieldName: "endDate",
+      });
 
-    const partitioning = new Partitioning(partitioningConfig, testTable);
-    const value = partitioning.getPartitionValue(event);
+      const event: FirestoreDocumentChangeEvent = {
+        timestamp: "",
+        operation: ChangeType.DELETE,
+        documentName: "test/doc",
+        eventId: "event1",
+        documentId: "doc",
+        data: null,
+        oldData: null,
+      };
 
-    expect(value).toEqual({});
-  });
+      const partitioning = new Partitioning(partitioningConfig, testTable);
+      const value = partitioning.getPartitionValue(event);
 
-  test("returns empty object for DELETE when oldData lacks the field", () => {
-    const partitioningConfig = new PartitioningConfig({
-      granularity: "DAY",
-      bigqueryColumnName: "end_date",
-      bigqueryColumnType: "TIMESTAMP",
-      firestoreFieldName: "endDate",
+      expect(value).toEqual({});
     });
 
-    const event: FirestoreDocumentChangeEvent = {
-      timestamp: "",
-      operation: ChangeType.DELETE,
-      documentName: "test/doc",
-      eventId: "event1",
-      documentId: "doc",
-      data: null,
-      oldData: { otherField: "value" },
-    };
+    test("returns empty object for DELETE when oldData lacks the field", () => {
+      const partitioningConfig = new PartitioningConfig({
+        granularity: "DAY",
+        bigqueryColumnName: "end_date",
+        bigqueryColumnType: "TIMESTAMP",
+        firestoreFieldName: "endDate",
+      });
 
-    const partitioning = new Partitioning(partitioningConfig, testTable);
-    const value = partitioning.getPartitionValue(event);
+      const event: FirestoreDocumentChangeEvent = {
+        timestamp: "",
+        operation: ChangeType.DELETE,
+        documentName: "test/doc",
+        eventId: "event1",
+        documentId: "doc",
+        data: null,
+        oldData: { otherField: "value" },
+      };
 
-    expect(value).toEqual({});
-  });
-});
+      const partitioning = new Partitioning(partitioningConfig, testTable);
+      const value = partitioning.getPartitionValue(event);
+
+      expect(value).toEqual({});
+    });
+  }
+);
 
-describe("isValidPartitionForExistingTable", () => {
+describeIfBigQueryIntegration("isValidPartitionForExistingTable", () => {
   let testTable: Table;
   let testDataset: Dataset;
   let partitionedTable: Table;
@@ -886,7 +893,7 @@ describe("isValidPartitionForExistingTable", () => {
   });
 });
 
-describe("addPartitioningToSchema with DATE type", () => {
+describeIfBigQueryIntegration("addPartitioningToSchema with DATE type", () => {
   let testTable: Table;
   let testDataset: Dataset;
 

firestore-bigquery-export/functions/__tests__/__snapshots__/config.test.ts.snap

@@ -26,6 +26,9 @@ Object {
   "maxDispatchesPerSecond": 10,
   "maxEnqueueAttempts": 3,
   "maxStaleness": undefined,
+  "partitioning": Object {
+    "granularity": "NONE",
+  },
   "projectId": undefined,
   "refreshIntervalMinutes": undefined,
   "tableId": "my_table",

firestore-bigquery-export/functions/__tests__/config.test.ts

@@ -5,7 +5,7 @@ import { resolve as pathResolve } from "path";
 import * as yaml from "js-yaml";
 import mockedEnv from "mocked-env";
 
-import { clustering } from "../src/config";
+import { buildPartitioningConfig, clustering } from "../src/config";
 
 let restoreEnv;
 let extensionYaml;
@@ -150,4 +150,105 @@ describe("extension config", () => {
       });
     });
   });
+
+  describe("buildPartitioningConfig", () => {
+    test("uses ingestion-time partitioning when optional partition fields are empty", () => {
+      expect(
+        buildPartitioningConfig({
+          timePartitioning: "HOUR",
+          timePartitioningField: undefined,
+          timePartitioningFieldType: undefined,
+          timePartitioningFirestoreField: undefined,
+        })
+      ).toEqual({
+        granularity: "HOUR",
+      });
+    });
+
+    test("normalizes NONE/omit sentinels to ingestion-time partitioning", () => {
+      expect(
+        buildPartitioningConfig({
+          timePartitioning: "DAY",
+          timePartitioningField: "NONE",
+          timePartitioningFieldType: "omit",
+          timePartitioningFirestoreField: "NONE",
+        })
+      ).toEqual({
+        granularity: "DAY",
+      });
+    });
+
+    test("returns NONE strategy when table partitioning is disabled and optional fields are empty", () => {
+      expect(
+        buildPartitioningConfig({
+          timePartitioning: null,
+          timePartitioningField: undefined,
+          timePartitioningFieldType: undefined,
+          timePartitioningFirestoreField: undefined,
+        })
+      ).toEqual({
+        granularity: "NONE",
+      });
+    });
+
+    test("returns custom Firestore field strategy only when all required fields exist", () => {
+      expect(
+        buildPartitioningConfig({
+          timePartitioning: "HOUR",
+          timePartitioningField: "partition_column",
+          timePartitioningFieldType: "TIMESTAMP",
+          timePartitioningFirestoreField: "time",
+        })
+      ).toEqual({
+        granularity: "HOUR",
+        bigqueryColumnName: "partition_column",
+        bigqueryColumnType: "TIMESTAMP",
+        firestoreFieldName: "time",
+      });
+    });
+
+    test("supports partitioning by the built-in timestamp field", () => {
+      expect(
+        buildPartitioningConfig({
+          timePartitioning: "MONTH",
+          timePartitioningField: "timestamp",
+          timePartitioningFieldType: "DATETIME",
+          timePartitioningFirestoreField: undefined,
+        })
+      ).toEqual({
+        granularity: "MONTH",
+        bigqueryColumnName: "timestamp",
+        bigqueryColumnType: "DATETIME",
+      });
+    });
+
+    test("throws a useful error for invalid partial custom field configs", () => {
+      expect(() =>
+        buildPartitioningConfig({
+          timePartitioning: "HOUR",
+          timePartitioningField: "partition_column",
+          timePartitioningFieldType: "omit",
+          timePartitioningFirestoreField: undefined,
+        })
+      ).toThrow(/Invalid partitioning configuration/);
+
+      expect(() =>
+        buildPartitioningConfig({
+          timePartitioning: "HOUR",
+          timePartitioningField: undefined,
+          timePartitioningFieldType: "TIMESTAMP",
+          timePartitioningFirestoreField: "time",
+        })
+      ).toThrow(/Valid combinations are/);
+
+      expect(() =>
+        buildPartitioningConfig({
+          timePartitioning: null,
+          timePartitioningField: "created_at",
+          timePartitioningFieldType: "TIMESTAMP",
+          timePartitioningFirestoreField: "createdAt",
+        })
+      ).toThrow(/TABLE_PARTITIONING is NONE/);
+    });
+  });
 });