Skip to content

Commit 45be403

Browse files
tyler-dunkeltyler-dunkel
committed
Merge branch 'feature/CG-433-add-gcp-bigquery' into 'main'
CG-699/CG-433: "Feature/ add gcp bigquery" Closes CG-433 and CG-699 See merge request auto-cloud/cloudgraph/provider/cloudgraph-provider-gcp!22
2 parents 3737a45 + ee20fcc commit 45be403

47 files changed

Lines changed: 1876 additions & 45 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎README.md‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,9 +67,15 @@ CloudGraph GCP Provider will ask you what regions you would like to crawl and wi
6767
| alertPolicy | project |
6868
| apiKeys | project |
6969
| assets | project |
70+
| bigQueryDataset | project |
71+
| bigQueryConnection | project |
72+
| bigQueryDataTransfer | bigQueryDataTransferRun, project |
73+
| bigQueryDataTransferRun | project |
74+
| bigQueryReservation | project |
75+
| bigQueryReservationCapacityCommitment | project |
7076
| cloudFunction | project, vpc |
7177
| computeProject | project |
72-
| kmsCryptoKeys | iamPolicy, kmsKeyRing, project |
78+
| kmsCryptoKeys | iamPolicy, kmsKeyRing, project |
7379
| dnsManagedZone | project |
7480
| dnsPolicy | project, network |
7581
| firewall | network, project |

‎package.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,12 @@
4040
"@google-cloud/kms": "^2.10.0",
4141
"@google-cloud/logging": "^9.6.4",
4242
"@google-cloud/monitoring": "^2.3.5",
43+
"@google-cloud/bigquery": "^5.9.3",
44+
"@google-cloud/bigquery-connection": "^1.5.1",
45+
"@google-cloud/bigquery-data-transfer": "^2.3.0",
46+
"@google-cloud/bigquery-reservation": "^1.4.0",
47+
"@google-cloud/kms": "^2.10.0",
48+
"@google-cloud/logging": "^9.6.4",
4349
"@google-cloud/resource-manager": "^3.0.0",
4450
"@google-cloud/secret-manager": "^3.10.1",
4551
"@google-cloud/storage": "^5.16.1",

‎src/config/constants.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import services from '../enums/services'
33

44
export const GLOBAL_REGION = 'global'
55
export const DEFAULT_REGION = regionMap.usEast1
6+
export const MULTI_REGIONS = ['EU', 'US']
67
export const DEFAULT_RESOURCES = Object.values(services).join(',')
78
export const ENV_VAR_CREDS_LOG = 'Using ENV variable credentials'
89
export const MESSAGE_INTERVAL = 45000

‎src/enums/regions.ts‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,7 @@ export const regionMap = {
2323
asiaSoutheast1: 'asia-southeast1',
2424
asiaSoutheast2: 'asia-southeast2',
2525
// Europe
26-
europeCentral2: 'europe-central2',
27-
europeWest: 'europe-west',
26+
europeCentral2: 'europe-central2',
2827
europeWest2: 'europe-west2',
2928
europeWest3: 'europe-west3',
3029
europeWest6: 'europe-west6',
@@ -55,8 +54,7 @@ const {
5554
asiaSoutheast1,
5655
asiaSoutheast2,
5756
// Europe
58-
europeCentral2,
59-
europeWest,
57+
europeCentral2,
6058
europeWest2,
6159
europeWest3,
6260
europeWest6,
@@ -87,8 +85,7 @@ export default [
8785
asiaSoutheast1,
8886
asiaSoutheast2,
8987
// Europe
90-
europeCentral2,
91-
europeWest,
88+
europeCentral2,
9289
europeWest2,
9390
europeWest3,
9491
europeWest6,

‎src/enums/relations.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,5 +12,6 @@ export default {
1212
[services.project]: [services.iamPolicy],
1313
[services.folder]: [services.iamPolicy],
1414
[services.kmsKeyRing]: [services.kmsCryptoKeys],
15-
[services.kmsCryptoKeys]: [services.iamPolicy]
15+
[services.kmsCryptoKeys]: [services.iamPolicy],
16+
[services.bigQueryDataTransfer]: [services.bigQueryDataTransferRun],
1617
}

‎src/enums/schemasMap.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,12 @@ import services from './services'
66
export default {
77
[services.firewall]: 'gcpFirewall',
88
[services.project]: 'gcpProject',
9+
[services.bigQueryDataset]: 'gcpBigQueryDataset',
10+
[services.bigQueryConnection]: 'gcpBigQueryConnection',
11+
[services.bigQueryReservation]: 'gcpBigQueryReservation',
12+
[services.bigQueryReservationCapacityCommitment]: 'gcpBigQueryReservationCapacityCommitment',
13+
[services.bigQueryDataTransfer]: 'gcpBigQueryDataTransfer',
14+
[services.bigQueryDataTransferRun]: 'gcpBigQueryDataTransferRun',
915
[services.kmsKeyRing]: 'gcpKmsKeyRing',
1016
[services.kmsCryptoKeys]: 'gcpKmsCryptoKey',
1117
[services.dnsManagedZone]: 'gcpDnsManagedZone',

‎src/enums/serviceMap.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,11 @@
11
import services from './services'
22
import GcpProject from '../services/project'
3+
import GcpBigQuery from '../services/bigQuery'
4+
import GcpBigQueryConnection from '../services/bigQueryConnection'
5+
import GcpBigQueryReservation from '../services/bigQueryReservation'
6+
import GcpBigQueryReservationCapacityCommitment from '../services/bigQueryReservationCapacityCommitment'
7+
import GcpBigQueryDataTransfer from '../services/bigQueryDataTransfer'
8+
import GcpBigQueryDataTransferRun from '../services/bigQueryDataTransferRun'
39
import GcpKmsKeyRing from '../services/kms'
410
import GcpKmsCryptoKey from '../services/kmsCryptoKey'
511
import GcpDnsManagedZone from '../services/dnsManagedZone'
@@ -37,6 +43,12 @@ import GcpComputeProject from '../services/computeProject'
3743
export default {
3844
[services.firewall]: GcpFirewall,
3945
[services.project]: GcpProject,
46+
[services.bigQueryDataset]: GcpBigQuery,
47+
[services.bigQueryConnection]: GcpBigQueryConnection,
48+
[services.bigQueryReservation]: GcpBigQueryReservation,
49+
[services.bigQueryReservationCapacityCommitment]: GcpBigQueryReservationCapacityCommitment,
50+
[services.bigQueryDataTransfer]: GcpBigQueryDataTransfer,
51+
[services.bigQueryDataTransferRun]: GcpBigQueryDataTransferRun,
4052
[services.kmsKeyRing]: GcpKmsKeyRing,
4153
[services.kmsCryptoKeys]: GcpKmsCryptoKey,
4254
[services.dnsManagedZone]: GcpDnsManagedZone,

‎src/enums/services.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,12 @@ export default {
3232
// metastore: 'metastore',
3333
// pubsub: 'pubsub',
3434
// bigtable: 'bigtable',
35+
bigQueryDataset: 'bigQueryDataset',
36+
bigQueryConnection: 'bigQueryConnection',
37+
bigQueryReservation: 'bigQueryReservation',
38+
bigQueryReservationCapacityCommitment: 'bigQueryReservationCapacityCommitment',
39+
bigQueryDataTransfer: 'bigQueryDataTransfer',
40+
bigQueryDataTransferRun: 'bigQueryDataTransferRun',
3541
// databaseMigration: 'database-migration',
3642
// datastore: 'datastore',
3743
// firestore: 'firestore',

‎src/services/base/schema.graphql‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,6 @@ interface gcpBaseResource
88
projectId: String! @search(by: [hash, regexp])
99
region: String @search(by: [hash, regexp])
1010
name: String @search(by: [hash, regexp])
11+
kind: String @search(by: [hash, regexp])
12+
labels: [gcpRawLabel]
1113
}

‎src/services/bigQuery/data.ts‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
import { BigQuery } from '@google-cloud/bigquery'
2+
import CloudGraph from '@cloudgraph/sdk'
3+
import groupBy from 'lodash/groupBy'
4+
import gcpLoggerText from '../../properties/logger'
5+
import { GcpServiceInput } from '../../types'
6+
import { generateGcpErrorLog, initTestEndpoint } from '../../utils'
7+
import { RawGcpBigQueryDataset } from './types'
8+
import { MULTI_REGIONS } from '../../config/constants'
9+
10+
const lt = { ...gcpLoggerText }
11+
const { logger } = CloudGraph
12+
const serviceName = 'BigQuery Dataset'
13+
const apiEndpoint = initTestEndpoint(serviceName)
14+
15+
export default async ({
16+
regions,
17+
config,
18+
}: GcpServiceInput): Promise<{
19+
[region: string]: RawGcpBigQueryDataset[]
20+
}> => {
21+
const bigQueryClient = new BigQuery({ ...config, apiEndpoint })
22+
const datasetsResult: RawGcpBigQueryDataset[] = []
23+
const { projectId } = config
24+
const allRegions = regions.split(',').concat(MULTI_REGIONS)
25+
try {
26+
const dataSetIter = bigQueryClient.getDatasetsStream()
27+
for await (const dataSetResponse of dataSetIter) {
28+
if (allRegions.includes(dataSetResponse.location)) {
29+
const dsMetaData = dataSetResponse.metadata
30+
const result = {
31+
...dsMetaData,
32+
region: dataSetResponse.location,
33+
Labels: dataSetResponse.labels,
34+
tables: [],
35+
projectId,
36+
}
37+
try {
38+
const tableIter = dataSetResponse.getTablesStream()
39+
for await (const tableResponse of tableIter) {
40+
result.tables.push(tableResponse.metadata)
41+
}
42+
datasetsResult.push(result)
43+
} catch (error) {
44+
generateGcpErrorLog(serviceName, 'bigQuery:getTablesStream', error)
45+
}
46+
}
47+
}
48+
} catch (error) {
49+
generateGcpErrorLog(serviceName, 'bigQuery:getDatasetsStream', error)
50+
}
51+
52+
logger.debug(lt.foundResources(serviceName, datasetsResult.length))
53+
return groupBy(datasetsResult, 'region')
54+
}

0 commit comments

Comments
 (0)