Merge branch 'main' into feature/CG-664-ai-platform-notebook

Author: james-zhou-inspire11

CHANGELOG.md

@@ -1,3 +1,11 @@
+# [0.32.0](https://gitlab.com/auto-cloud/cloudgraph/provider/cloudgraph-provider-gcp/compare/0.31.0...0.32.0) (2022-02-21)
+
+
+### Features
+
+* **apigateway:** add api gateway ([0f3a5a9](https://gitlab.com/auto-cloud/cloudgraph/provider/cloudgraph-provider-gcp/commit/0f3a5a907adfd1c434bd8b168a2f1c634d52f469))
+* **apigateway:** update README ([05ff0a5](https://gitlab.com/auto-cloud/cloudgraph/provider/cloudgraph-provider-gcp/commit/05ff0a55f4dbc9e980892a8e224a19ed4ae2ce01))
+
 # [0.31.0](https://gitlab.com/auto-cloud/cloudgraph/provider/cloudgraph-provider-gcp/compare/0.30.0...0.31.0) (2022-02-18)
 
 

README.md

@@ -66,6 +66,9 @@ CloudGraph GCP Provider will ask you what regions you would like to crawl and wi
 | ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ |
 | aiPlatformNotebooks                   | project, kmsCryptoKeys, network, subnet                                                                                  |
 | alertPolicy                           | project                                                                                                                  |
+| apiGatewayGateways                    | project, apiGatewayApis, apiGatewayApiConfigs                                                                            |
+| apiGatewayApis                        | project, apiGatewayGateways                                                                                              |
+| apiGatewayApiConfigs                  | project, apiGatewayGateways                                                                                              |
 | apiKeys                               | project                                                                                                                  |
 | assets                                | project                                                                                                                  |
 | bigQueryDataset                       | project                                                                                                                  |

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cloudgraph/cg-provider-gcp",
-  "version": "0.31.0",
+  "version": "0.32.0",
   "description": "CloudGraph provider plugin for GCP used to fetch GCP cloud data.",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
@@ -33,6 +33,7 @@
   },
   "dependencies": {
     "@cloudgraph/sdk": "^0.11.0",
+    "@google-cloud/api-gateway": "^1.2.1",
     "@google-cloud/asset": "^3.22.0",
     "@google-cloud/bigquery": "^5.10.0",
     "@google-cloud/bigquery-connection": "^1.5.1",

src/enums/relations.ts

@@ -3,7 +3,7 @@ import services from './services'
 /**
  * Set relations between services to data sharing
  * The key of the object represents the parent or base service,
- * it might contain a array of dependant or childs that must be executed after the parent
+ * it might contain an array of dependant or children that must be executed after the parent
  */
 export default {
   [services.storageBucket]: [services.iamPolicy],
@@ -14,4 +14,5 @@ export default {
   [services.kmsKeyRing]: [services.kmsCryptoKeys],
   [services.kmsCryptoKeys]: [services.iamPolicy],
   [services.bigQueryDataTransfer]: [services.bigQueryDataTransferRun],
+  [services.apiGatewayApi]: [services.apiGatewayApiConfig],
 }

src/enums/schemasMap.ts

@@ -48,6 +48,9 @@ export default {
   [services.dataprocJob]: 'gcpDataprocJob',
   [services.dataprocWorkflowTemplate]: 'gcpDataprocWorkflowTemplate',
   [services.aiPlatformNotebook]: 'gcpAiPlatformNotebook',
+  [services.apiGatewayGateway]: 'gcpApiGatewayGateway',
+  [services.apiGatewayApi]: 'gcpApiGatewayApi',
+  [services.apiGatewayApiConfig]: 'gcpApiGatewayApiConfig',
   [services.firestoreDatabase]: 'gcpFirestoreDatabase',
   tag: 'gcpTag',
 }

src/enums/serviceMap.ts

@@ -43,6 +43,9 @@ import GcpDataprocAutoscalingPolicy from '../services/dataprocAutoscalingPolicy'
 import GcpDataprocJob from '../services/dataprocJob'
 import GcpDataprocWorkflowTemplate from '../services/dataprocWorkflowTemplate'
 import GcpAiPlatformNotebook from '../services/aiPlatformNotebook'
+import GcpApiGatewayGateway from '../services/apiGatewayGateway'
+import GcpApiGatewayApi from '../services/apiGatewayApi'
+import GcpApiGatewayApiConfig from '../services/apiGatewayApiConfig'
 import GcpFirestoreDatabase from '../services/firestore'
 import GcpLabel from '../services/label'
 import GcpTag from '../services/tag'
@@ -96,6 +99,9 @@ export default {
   [services.dataprocJob]: GcpDataprocJob,
   [services.dataprocWorkflowTemplate]: GcpDataprocWorkflowTemplate,
   [services.aiPlatformNotebook]: GcpAiPlatformNotebook,
+  [services.apiGatewayGateway]: GcpApiGatewayGateway,
+  [services.apiGatewayApi]: GcpApiGatewayApi,
+  [services.apiGatewayApiConfig]: GcpApiGatewayApiConfig,
   [services.firestoreDatabase]: GcpFirestoreDatabase,
   label: GcpLabel,
   tag: GcpTag,

src/enums/services.ts

@@ -8,7 +8,9 @@ export default {
   // ml: 'ml',
   // mlEngine: 'ml-engine',
   // notebooks: 'notebooks',
-  // apiGateway: 'api-gateway',
+  apiGatewayGateway: 'apiGatewayGateways',
+  apiGatewayApi: 'apiGatewayApis',
+  apiGatewayApiConfig: 'apiGatewayApiConfigs',
   // apigee: 'apigee',
   // endpoints: 'endpoints',
   // recommender: 'recommender',

src/services/apiGatewayApi/connections.ts

@@ -0,0 +1,56 @@
+import { ServiceConnection } from '@cloudgraph/sdk'
+import { RawGcpApiGatewayApi } from './data'
+
+import services from '../../enums/services'
+import { RawGcpApiGatewayGateway } from '../apiGatewayGateway/data'
+
+export default ({
+  service,
+  data,
+}: {
+  service: RawGcpApiGatewayApi
+  data: { name: string; data: { [property: string]: any[] } }[]
+  region: string
+}): {
+  [property: string]: ServiceConnection[]
+} => {
+  const { id } = service
+  const connections: ServiceConnection[] = []
+
+  /**
+   * Find Api Gateway Gateways
+   */
+  const gateways: {
+    name: string
+    data: { [property: string]: any[] }
+  } = data.find(({ name }) => name === services.apiGatewayGateway)
+
+  if (gateways?.data) {
+    for (const region of Object.keys(gateways.data)) {
+      const filteredGateways = gateways.data[region].filter(
+        ({ apiConfig }: RawGcpApiGatewayGateway) => {
+          const idSessions = apiConfig.split('/')
+          idSessions[1] = service.projectId
+          const apiIdWithNumber =  idSessions.join('/')
+          return apiIdWithNumber.startsWith(`${id}/`)
+        }
+      )
+  
+      if (filteredGateways) {
+        for (const filteredGateway of filteredGateways) {
+          connections.push({
+            id: filteredGateway.id,
+            resourceType: services.apiGatewayGateway,
+            relation: 'child',
+            field: 'apiGatewayGateways',
+          })
+        }
+      }
+    }
+  }
+
+  const result = {
+    [service.id]: connections,
+  }
+  return result
+}

src/services/apiGatewayApi/data.ts

@@ -0,0 +1,61 @@
+import groupBy from 'lodash/groupBy'
+import { ApiGatewayServiceClient } from '@google-cloud/api-gateway'
+import CloudGraph from '@cloudgraph/sdk'
+import { google } from '@google-cloud/api-gateway/build/protos/protos'
+import gcpLoggerText from '../../properties/logger'
+import { GcpServiceInput, LabelMap } from '../../types'
+import { generateGcpErrorLog, initTestEndpoint } from '../../utils'
+import { GLOBAL_REGION } from '../../config/constants'
+
+const lt = { ...gcpLoggerText }
+const { logger } = CloudGraph
+const serviceName = 'API Gateway Api'
+const apiEndpoint = initTestEndpoint(serviceName)
+
+export interface RawGcpApiGatewayApi extends 
+Omit<google.cloud.apigateway.v1.IApi, 'labels'> {
+  id: string,
+  region: string,
+  projectId: string,
+  Labels: LabelMap,
+}
+
+export const listApiGatewayApiData = async (
+  apiGatewayClient: ApiGatewayServiceClient,
+  projectId: string,
+  apiData: RawGcpApiGatewayApi[],
+): Promise<void> => {
+  try {
+    const apiIter = apiGatewayClient.listApisAsync({
+      parent: `projects/${projectId}/locations/${GLOBAL_REGION}`,
+    })
+    for await (const {name, labels, ...api} of apiIter) {
+      apiData.push({
+        id: name,
+        projectId,
+        region: GLOBAL_REGION,
+        Labels: labels,
+        ...api,
+      })
+    }
+  } catch (error) {
+    generateGcpErrorLog(serviceName, 'ApiGateway:listApisAsync', error)
+  }
+}
+
+export default async ({
+  config,
+}: GcpServiceInput): Promise<{
+  [region: string]: RawGcpApiGatewayApi[]
+}> => {
+  const { projectId } = config
+
+  const apiData: RawGcpApiGatewayApi[] = []
+  const apiGatewayClient = new ApiGatewayServiceClient({ ...config, apiEndpoint })
+
+  await listApiGatewayApiData(apiGatewayClient, projectId, apiData)
+
+  logger.debug(lt.foundResources(serviceName, apiData.length))
+
+  return groupBy(apiData, 'region')
+}

src/services/apiGatewayApi/format.ts

@@ -0,0 +1,37 @@
+import { google } from '@google-cloud/api-gateway/build/protos/protos'
+import { RawGcpApiGatewayApi } from './data'
+import { enumKeyToString, formatLabelsFromMap } from '../../utils/format'
+import { GcpApiGatewayApi } from '../../types/generated'
+import { toISOString } from '../../utils/dateutils'
+
+export default ({
+  service,
+  account,
+  region,
+}: {
+  service: RawGcpApiGatewayApi
+  account: string
+  region: string
+}): GcpApiGatewayApi => {
+  const {
+    id,
+    Labels,
+    createTime,
+    updateTime,
+    displayName,
+    managedService,
+    state,
+  } = service
+
+  return {
+    id,
+    name: displayName,
+    projectId: account,
+    region,
+    labels: formatLabelsFromMap(Labels),
+    createTime: toISOString(createTime?.seconds?.toString()) || '',
+    updateTime: toISOString(updateTime?.seconds?.toString()) || '',
+    managedService,
+    state: enumKeyToString(google.cloud.apigateway.v1.Api.State, state),
+  }
+}