Merge branch 'feature/CG-668-firestore' into 'main'

Resolve CG-668 "Feature/ firestore"

Closes CG-668

See merge request auto-cloud/cloudgraph/provider/cloudgraph-provider-gcp!48

Author: tyler-dunkel

README.md

@@ -86,6 +86,7 @@ CloudGraph GCP Provider will ask you what regions you would like to crawl and wi
 | kmsCryptoKeys                         | iamPolicy, kmsKeyRing, project                                                                      |
 | dnsManagedZone                        | project                                                                                             |
 | dnsPolicy                             | project, network                                                                                    |
+| firestores                            | project                                                                                             |
 | firewall                              | network, project                                                                                    |
 | folder                                | iamPolicy, organization, project                                                                    |
 | kmsKeyRing                            | kmsCryptoKeys, project                                                                              |

package.json

@@ -40,7 +40,9 @@
     "@google-cloud/bigquery-reservation": "^1.4.0",
     "@google-cloud/compute": "^3.0.0",
     "@google-cloud/dataproc": "^3.2.0",
+    "@google-cloud/datastore": "^6.6.2",
     "@google-cloud/dns": "^2.2.3",
+    "@google-cloud/firestore": "^5.0.2",
     "@google-cloud/functions": "^1.2.0",
     "@google-cloud/kms": "^2.10.0",
     "@google-cloud/logging": "^9.6.4",

src/enums/schemasMap.ts

@@ -47,5 +47,6 @@ export default {
   [services.dataprocAutoscalingPolicy]: 'gcpDataprocAutoscalingPolicy',
   [services.dataprocJob]: 'gcpDataprocJob',
   [services.dataprocWorkflowTemplate]: 'gcpDataprocWorkflowTemplate',
+  [services.firestoreDatabase]: 'gcpFirestoreDatabase',
   tag: 'gcpTag',
 }

src/enums/serviceMap.ts

@@ -43,6 +43,8 @@ import GcpDataprocCluster from '../services/dataprocCluster'
 import GcpDataprocAutoscalingPolicy from '../services/dataprocAutoscalingPolicy'
 import GcpDataprocJob from '../services/dataprocJob'
 import GcpDataprocWorkflowTemplate from '../services/dataprocWorkflowTemplate'
+import GcpFirestoreDatabase from '../services/firestore'
+
 /**
  * serviceMap is an object that contains all currently supported services
  * serviceMap is used by the serviceFactory to produce instances of service classes
@@ -91,5 +93,6 @@ export default {
   [services.dataprocAutoscalingPolicy]: GcpDataprocAutoscalingPolicy,
   [services.dataprocJob]: GcpDataprocJob,
   [services.dataprocWorkflowTemplate]: GcpDataprocWorkflowTemplate,
+  [services.firestoreDatabase]: GcpFirestoreDatabase,
   tag: GcpTag,
 }

src/enums/services.ts

@@ -46,7 +46,7 @@ export default {
   bigQueryDataTransferRun: 'bigQueryDataTransferRun',
   // databaseMigration: 'database-migration',
   // datastore: 'datastore',
-  // firestore: 'firestore',
+  firestoreDatabase: 'firestoreDatabases',
   cloudRouter: 'cloudRouters',
   firewall: 'firewall',
   // spanner: 'spanner',

src/services/firestore/data.ts

@@ -0,0 +1,67 @@
+import CloudGraph from '@cloudgraph/sdk'
+import groupBy from 'lodash/groupBy'
+import { Datastore } from '@google-cloud/datastore'
+import gcpLoggerText from '../../properties/logger'
+import { GcpServiceInput } from '../../types'
+import { initTestEndpoint, generateGcpErrorLog } from '../../utils'
+import { GLOBAL_REGION } from '../../config/constants'
+import { listData } from '../../utils/fetchUtils'
+
+const lt = { ...gcpLoggerText }
+const { logger } = CloudGraph
+const serviceName = 'Firestore Database'
+const apiEndpoint = initTestEndpoint(serviceName)
+
+export interface RawGcpFirestoreDatabase {
+  id: string
+  projectId: string
+  region: string
+  name: string
+  createTime: string
+  updateTime: string
+  locationId: string
+  type: string
+  concurrencyMode: string
+  appEngineIntegrationMode: string
+  keyPrefix: string
+}
+
+export default async ({
+  config,
+}: GcpServiceInput): Promise<{
+  [region: string]: RawGcpFirestoreDatabase[]
+}> =>
+  new Promise(async resolve => {
+    const firestoreDatabaseList: RawGcpFirestoreDatabase[] = []
+    const { projectId } = config
+
+    try {
+      const service = new Datastore({ ...config, apiEndpoint })
+      const data = await listData({
+        accessToken: await service.auth.getAccessToken(),
+        apiUri: `https://firestore.googleapis.com/v1/projects/${projectId}/databases`,
+        dataFieldName: 'databases',
+      })
+      
+      for (const {name, createTime, updateTime, locationId, type, concurrencyMode, appEngineIntegrationMode, keyPrefix} of data) {
+        firestoreDatabaseList.push({
+          id: name,
+          projectId,
+          region: GLOBAL_REGION,
+          name,
+          createTime,
+          updateTime,
+          locationId,
+          type,
+          concurrencyMode,
+          appEngineIntegrationMode,
+          keyPrefix,
+        })
+      }
+    } catch (error) {
+      generateGcpErrorLog(serviceName, 'firestore:listDatabase', error)
+    }
+    
+    logger.debug(lt.foundResources(serviceName, firestoreDatabaseList.length))
+    resolve(groupBy(firestoreDatabaseList, 'region'))
+  })

src/services/firestore/format.ts

@@ -0,0 +1,37 @@
+import { GcpFirestoreDatabase } from '../../types/generated'
+import { RawGcpFirestoreDatabase } from './data'
+
+export default ({
+  service,
+}: {
+  service: RawGcpFirestoreDatabase
+  region: string
+}): GcpFirestoreDatabase => {
+  const {
+    id,
+    projectId,
+    region,
+    name,
+    createTime,
+    updateTime,
+    locationId,
+    type,
+    concurrencyMode,
+    appEngineIntegrationMode,
+    keyPrefix,
+  } = service
+
+  return {
+    id,
+    projectId,
+    region,
+    name,
+    createTime,
+    updateTime,
+    locationId,
+    type,
+    concurrencyMode,
+    appEngineIntegrationMode,
+    keyPrefix,
+  }
+}

src/services/firestore/index.ts

@@ -0,0 +1,13 @@
+import {Service} from '@cloudgraph/sdk'
+import BaseService from '../base'
+import format from './format'
+import getData from './data'
+import mutation from './mutation'
+
+export default class GcpFirestoreDatabase extends BaseService implements Service {
+  format = format.bind(this)
+
+  getData = getData.bind(this)
+
+  mutation = mutation
+}

src/services/firestore/mutation.ts

@@ -0,0 +1,5 @@
+export default `mutation($input: [AddgcpFirestoreDatabaseInput!]!) {
+  addgcpFirestoreDatabase(input: $input, upsert: true) {
+    numUids
+  }
+}`;

src/services/firestore/schema.graphql

@@ -0,0 +1,16 @@
+type gcpFirestoreDatabase implements gcpBaseResource
+  @generate(
+    query: { get: true, query: true, aggregate: true }
+    mutation: { add: true, delete: false }
+    subscription: false
+  )
+  @key(fields: "id") {
+    createTime: String @search(by: [hash, regexp])
+    updateTime: String @search(by: [hash, regexp])
+    locationId: String @search(by: [hash, regexp])
+    type: String @search(by: [hash, regexp])
+    concurrencyMode: String @search(by: [hash, regexp])
+    appEngineIntegrationMode: String @search(by: [hash, regexp])
+    keyPrefix: String @search(by: [hash, regexp])
+    project: [gcpProject] @hasInverse(field: firestoreDatabases)
+}