Skip to content

Commit 025ed29

Browse files
cabljaccabljac
authored
feat: update change tracker to 2.0.2 with fixed types (#2696)
* refactor(firestore-bigquery-change-tracker): improve partitioning types and clean up exports - Replace flat config with PartitioningStrategy discriminated union for type-safe partitioning - Rename Config to ChangeTrackerConfig for clarity - Export PartitioningStrategy, PartitioningFieldType, TimePartitioningGranularity - Remove unused dependencies (generate-schema, inquirer, lodash) - Bump version to 2.0.1 * chore(firestore-bigquery-change-tracker): bump to 2.0.2 * fix(firestore-bigquery-change-tracker): address PR feedback - Use explicit { granularity: "NONE" } default instead of empty object - Add deprecated Config alias for backwards compatibility * test(firestore-bigquery-change-tracker): re-add tests for malformed partitioning configs * fix(firestore-bigquery-change-tracker): throw on invalid partitioning strategy instead of silent fallback * revert(firestore-bigquery-change-tracker): keep NONE fallback for invalid partitioning configs
1 parent 5dd7e25 commit 025ed29

16 files changed

Lines changed: 110 additions & 150 deletions

File tree

‎firestore-bigquery-export/firestore-bigquery-change-tracker/package-lock.json‎

Lines changed: 2 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎firestore-bigquery-export/firestore-bigquery-change-tracker/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
"url": "git+https://github.com/firebase/extensions.git",
66
"directory": "firestore-bigquery-export/firestore-bigquery-change-tracker"
77
},
8-
"version": "2.0.1",
8+
"version": "2.0.2",
99
"description": "Core change-tracker library for Cloud Firestore Collection BigQuery Exports",
1010
"main": "./lib/index.js",
1111
"scripts": {

‎firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/failedTransaction.test.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import * as admin from "firebase-admin";
2-
import { Config } from "../../bigquery/types";
2+
import { ChangeTrackerConfig } from "../../bigquery/types";
33

44
import handleFailedTransactions from "../../bigquery/handleFailedTransactions";
55

@@ -15,7 +15,7 @@ describe("handleFailedTransactions", () => {
1515
const collectionName = "testing";
1616
const doc = db.collection("testing").doc("600");
1717

18-
const config: Config = {
18+
const config: ChangeTrackerConfig = {
1919
backupTableId: collectionName,
2020
datasetId: "",
2121
tableId: "",

‎firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/materializedViews/initializeLatestView.test.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { initializeLatestView } from "../../../bigquery/initializeLatestView";
22
import { initializeLatestMaterializedView } from "../../../bigquery/initializeLatestMaterializedView";
3-
import { Config } from "../../../bigquery/types";
3+
import { ChangeTrackerConfig } from "../../../bigquery/types";
44

55
jest.mock("../../../bigquery/initializeLatestMaterializedView");
66

@@ -12,7 +12,7 @@ describe("initializeLatestView", () => {
1212
create: jest.fn(),
1313
};
1414

15-
const mockConfig: Config = {
15+
const mockConfig: ChangeTrackerConfig = {
1616
datasetId: "test_dataset",
1717
tableId: "test_raw_table",
1818
datasetLocation: "US",

‎firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning.test.ts‎

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import * as admin from "firebase-admin";
33
import { BigQuery, Dataset, Table } from "@google-cloud/bigquery";
44
import { ChangeType, FirestoreDocumentChangeEvent } from "../..";
55

6-
import { Config } from "../../bigquery/types";
6+
import { ChangeTrackerConfig } from "../../bigquery/types";
77
import { Partitioning } from "../../bigquery/partitioning";
88
import { PartitioningConfig } from "../../bigquery/partitioning/config";
99
import { deleteTable } from "../fixtures/clearTables";
@@ -50,7 +50,7 @@ describe("processing partitions on a new table", () => {
5050
});
5151
describe("addPartitioningToSchema", () => {
5252
test("adds a custom TIMESTAMP to a schema", async () => {
53-
const config: Config = {
53+
const config: ChangeTrackerConfig = {
5454
datasetId: "dataset",
5555
tableId: "table",
5656
datasetLocation: "US",
@@ -84,7 +84,7 @@ describe("processing partitions on a new table", () => {
8484
});
8585

8686
test("adds a custom DATETIME to a schema", async () => {
87-
const config: Config = {
87+
const config: ChangeTrackerConfig = {
8888
datasetId: "dataset",
8989
tableId: "table",
9090
datasetLocation: "US",
@@ -118,7 +118,7 @@ describe("processing partitions on a new table", () => {
118118
});
119119

120120
test("does not add an invalid time partition type to a schema", async () => {
121-
const config: Config = {
121+
const config: ChangeTrackerConfig = {
122122
datasetId: "dataset",
123123
tableId: "table",
124124
datasetLocation: "US",
@@ -147,7 +147,7 @@ describe("processing partitions on a new table", () => {
147147
});
148148

149149
test("does not add partitioning without a valid timePartitioning value ", async () => {
150-
const config: Config = {
150+
const config: ChangeTrackerConfig = {
151151
datasetId: "",
152152
tableId: "",
153153
datasetLocation: "",
@@ -173,7 +173,7 @@ describe("processing partitions on a new table", () => {
173173
});
174174

175175
test("does not add partitioning without a timePartitioningFirestoreField", async () => {
176-
const config: Config = {
176+
const config: ChangeTrackerConfig = {
177177
datasetId: "",
178178
tableId: "",
179179
datasetLocation: "",
@@ -201,7 +201,7 @@ describe("processing partitions on a new table", () => {
201201

202202
describe("getPartitionValue", () => {
203203
test("returns a value when timePartitioningField and timePartitioningFirestoreField string value has been defined", async () => {
204-
const config: Config = {
204+
const config: ChangeTrackerConfig = {
205205
datasetId: "",
206206
tableId: "",
207207
datasetLocation: "",
@@ -235,7 +235,7 @@ describe("processing partitions on a new table", () => {
235235
});
236236

237237
test("returns a value when timePartitioningField and timePartitioningFirestoreField string value has been defined, with a timestamp-like value", async () => {
238-
const config: Config = {
238+
const config: ChangeTrackerConfig = {
239239
datasetId: "",
240240
tableId: "",
241241
datasetLocation: "",
@@ -273,7 +273,7 @@ describe("processing partitions on a new table", () => {
273273
});
274274

275275
test("returns an empty object when _seconds or _nanoseconds is not a number", async () => {
276-
const config: Config = {
276+
const config: ChangeTrackerConfig = {
277277
datasetId: "",
278278
tableId: "",
279279
datasetLocation: "",
@@ -310,7 +310,7 @@ describe("processing partitions on a new table", () => {
310310
});
311311

312312
test("returns a value when timePartitioningField and timePartitioningFirestoreField string value has been defined, and is timestamp-like", async () => {
313-
const config: Config = {
313+
const config: ChangeTrackerConfig = {
314314
datasetId: "",
315315
tableId: "",
316316
datasetLocation: "",
@@ -347,7 +347,7 @@ describe("processing partitions on a new table", () => {
347347
});
348348

349349
test("returns an empty object if timePartitioningFirestoreField has not been provided", async () => {
350-
const config: Config = {
350+
const config: ChangeTrackerConfig = {
351351
datasetId: "",
352352
tableId: "",
353353
datasetLocation: "",
@@ -380,7 +380,7 @@ describe("processing partitions on a new table", () => {
380380
expect(value).toEqual({});
381381
});
382382
test("returns an empty object if timePartitioningFirestoreField has not been provided", async () => {
383-
const config: Config = {
383+
const config: ChangeTrackerConfig = {
384384
datasetId: "",
385385
tableId: "",
386386
datasetLocation: "",
@@ -414,7 +414,7 @@ describe("processing partitions on a new table", () => {
414414
});
415415

416416
test("returns an empty object if timePartitioningFirestoreField timePartitioningField", async () => {
417-
const config: Config = {
417+
const config: ChangeTrackerConfig = {
418418
datasetId: "",
419419
tableId: "",
420420
datasetLocation: "",
@@ -448,7 +448,7 @@ describe("processing partitions on a new table", () => {
448448
});
449449

450450
test("returns an empty object if no event data has been provided", async () => {
451-
const config: Config = {
451+
const config: ChangeTrackerConfig = {
452452
datasetId: "",
453453
tableId: "",
454454
datasetLocation: "",
@@ -480,7 +480,7 @@ describe("processing partitions on a new table", () => {
480480
});
481481

482482
test("returns an empty object if a non string or Timestamp value is synced from Firestore", async () => {
483-
const config: Config = {
483+
const config: ChangeTrackerConfig = {
484484
datasetId: "",
485485
tableId: "",
486486
datasetLocation: "",
@@ -516,7 +516,7 @@ describe("processing partitions on a new table", () => {
516516

517517
describe("isTablePartitioned", () => {
518518
test("partition return false if table is not provided", async () => {
519-
const config: Config = {
519+
const config: ChangeTrackerConfig = {
520520
datasetId: "",
521521
tableId: "",
522522
datasetLocation: "",
@@ -541,7 +541,7 @@ describe("processing partitions on a new table", () => {
541541
});
542542

543543
test("partition return false if table is not provided", async () => {
544-
const config: Config = {
544+
const config: ChangeTrackerConfig = {
545545
datasetId: "",
546546
tableId: "",
547547
datasetLocation: "",
@@ -596,7 +596,7 @@ describe("updateTableMetadata", () => {
596596
});
597597

598598
test("updates the table metadata with the timestamp field", async () => {
599-
const config: Config = {
599+
const config: ChangeTrackerConfig = {
600600
datasetId: "",
601601
tableId: "",
602602
datasetLocation: "",
@@ -623,7 +623,7 @@ describe("updateTableMetadata", () => {
623623
});
624624
});
625625
test("Should not update if there is a custom option with the timestamp option", async () => {
626-
const config: Config = {
626+
const config: ChangeTrackerConfig = {
627627
datasetId: "",
628628
tableId: "",
629629
datasetLocation: "",

‎firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/config.test.ts‎

Lines changed: 11 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -87,65 +87,30 @@ describe("PartitioningConfig", () => {
8787
});
8888
});
8989

90-
describe("static factory methods", () => {
91-
test("none() creates a NONE config", () => {
92-
const config = PartitioningConfig.none();
93-
expect(config.getType()).toBe(PartitioningType.NONE);
94-
expect(config.getGranularity()).toBe("NONE");
95-
});
96-
97-
test("ingestionTime() creates an INGESTION_TIME config", () => {
98-
const config = PartitioningConfig.ingestionTime("MONTH");
99-
expect(config.getType()).toBe(PartitioningType.INGESTION_TIME);
100-
expect(config.getGranularity()).toBe("MONTH");
101-
});
102-
103-
test("firestoreTimestamp() creates a FIRESTORE_TIMESTAMP config", () => {
104-
const config = PartitioningConfig.firestoreTimestamp("DAY");
105-
expect(config.getType()).toBe(PartitioningType.FIRESTORE_TIMESTAMP);
106-
expect(config.getBigQueryColumnName()).toBe("timestamp");
107-
expect(config.getBigQueryColumnType()).toBeUndefined();
108-
});
109-
110-
test("firestoreTimestamp() with columnType creates correct config", () => {
111-
const config = PartitioningConfig.firestoreTimestamp("DAY", "DATE");
112-
expect(config.getType()).toBe(PartitioningType.FIRESTORE_TIMESTAMP);
113-
expect(config.getBigQueryColumnType()).toBe("DATE");
114-
});
115-
116-
test("firestoreField() creates a FIRESTORE_FIELD config", () => {
117-
const config = PartitioningConfig.firestoreField(
118-
"YEAR",
119-
"event_date",
120-
"DATE",
121-
"eventDate"
122-
);
123-
expect(config.getType()).toBe(PartitioningType.FIRESTORE_FIELD);
124-
expect(config.getGranularity()).toBe("YEAR");
125-
expect(config.getBigQueryColumnName()).toBe("event_date");
126-
expect(config.getBigQueryColumnType()).toBe("DATE");
127-
expect(config.getFirestoreFieldName()).toBe("eventDate");
128-
});
129-
});
130-
13190
describe("getter methods", () => {
132-
test("getStrategy() returns the original config", () => {
133-
const originalConfig = {
91+
test("getStrategy() returns the original strategy", () => {
92+
const strategy = {
13493
granularity: "DAY" as const,
13594
bigqueryColumnName: "test",
13695
bigqueryColumnType: "TIMESTAMP" as const,
13796
firestoreFieldName: "test",
13897
};
139-
const config = new PartitioningConfig(originalConfig);
140-
expect(config.getStrategy()).toEqual(originalConfig);
98+
const config = new PartitioningConfig(strategy);
99+
expect(config.getStrategy()).toEqual(strategy);
141100
});
142101

143102
test("all boolean methods return false for non-matching types", () => {
144-
const config = PartitioningConfig.ingestionTime("DAY");
103+
const config = new PartitioningConfig({ granularity: "DAY" });
145104
expect(config.isNoPartitioning()).toBe(false);
146105
expect(config.isFirestoreTimestampPartitioning()).toBe(false);
147106
expect(config.isFirestoreFieldPartitioning()).toBe(false);
148107
expect(config.isIngestionTimePartitioning()).toBe(true);
149108
});
109+
110+
test("constructor defaults to no partitioning when undefined", () => {
111+
const config = new PartitioningConfig();
112+
expect(config.getType()).toBe(PartitioningType.NONE);
113+
expect(config.getStrategy()).toEqual({ granularity: "NONE" });
114+
});
150115
});
151116
});

‎firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/fixtures/changeTracker.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import {
55
FirestoreDocumentChangeEvent,
66
} from "../..";
77
import { LogLevel } from "../../logger";
8-
import { Config } from "../../bigquery/types";
8+
import { ChangeTrackerConfig } from "../../bigquery/types";
99

1010
export const changeTracker = ({
1111
datasetId = "",
@@ -24,7 +24,7 @@ export const changeTracker = ({
2424
maxStaleness = undefined,
2525
refreshIntervalMinutes = undefined,
2626
logLevel = LogLevel.DEBUG,
27-
}: Partial<Config>): FirestoreBigQueryEventHistoryTracker => {
27+
}: Partial<ChangeTrackerConfig>): FirestoreBigQueryEventHistoryTracker => {
2828
return new FirestoreBigQueryEventHistoryTracker({
2929
datasetId,
3030
tableId,

‎firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/checkUpdates.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,11 @@ import { Table, TableMetadata } from "@google-cloud/bigquery/build/src/table";
22
import { Partitioning } from "./partitioning";
33
import { PartitioningConfig } from "./partitioning/config";
44

5-
import { Config } from ".";
5+
import { ChangeTrackerConfig } from ".";
66

77
interface TableRequiresUpdateOptions {
88
table: Table;
9-
config: Config;
9+
config: ChangeTrackerConfig;
1010
documentIdColExists: boolean;
1111
pathParamsColExists: boolean;
1212
oldDataColExists: boolean;
@@ -49,7 +49,7 @@ export async function tableRequiresUpdate({
4949

5050
interface ViewRequiresUpdateOptions {
5151
metadata?: TableMetadata;
52-
config: Config;
52+
config: ChangeTrackerConfig;
5353
documentIdColExists: boolean;
5454
pathParamsColExists: boolean;
5555
oldDataColExists: boolean;

‎firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/clustering.ts‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { Config } from ".";
1+
import { ChangeTrackerConfig } from ".";
22

33
import * as logs from "../logs";
44
import * as bigquery from "@google-cloud/bigquery";
@@ -25,11 +25,15 @@ interface InvalidFieldType {
2525
}
2626

2727
export class Clustering {
28-
public config: Config;
28+
public config: ChangeTrackerConfig;
2929
public table: bigquery.Table;
3030
public schema: object;
3131

32-
constructor(config: Config, table?: bigquery.Table, schema?: object) {
32+
constructor(
33+
config: ChangeTrackerConfig,
34+
table?: bigquery.Table,
35+
schema?: object
36+
) {
3337
this.config = config;
3438
this.table = table;
3539
this.schema = schema;

‎firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/handleFailedTransactions.ts‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,17 @@
11
import * as admin from "firebase-admin";
22
import { initializeApp } from "firebase-admin/app";
33
import { getFirestore } from "firebase-admin/firestore";
4-
import { Config } from ".";
4+
import { ChangeTrackerConfig } from ".";
55

66
if (!admin.apps.length) {
77
initializeApp();
88
}
99

10-
export default async (rows: any[], config: Config, e: Error): Promise<void> => {
10+
export default async (
11+
rows: any[],
12+
config: ChangeTrackerConfig,
13+
e: Error
14+
): Promise<void> => {
1115
const db = getFirestore(config.firestoreInstanceId!);
1216
db.settings({
1317
ignoreUndefinedProperties: true,

0 commit comments

Comments
 (0)