Merge branch 'feature/CG-665-add-api-gateway' into 'main'

feat(apigateway): CG-665 add api gateway

Closes CG-665

See merge request auto-cloud/cloudgraph/provider/cloudgraph-provider-gcp!46

Author: tyler-dunkel

README.md

@@ -65,6 +65,9 @@ CloudGraph GCP Provider will ask you what regions you would like to crawl and wi
 | Service                               | Relations                                                                                           |
 | ------------------------------------- | --------------------------------------------------------------------------------------------------- |
 | alertPolicy                           | project                                                                                             |
+| apiGatewayGateways                    | project, apiGatewayApis, apiGatewayApiConfigs                                                       |
+| apiGatewayApis                        | project, apiGatewayGateways                                                                         |
+| apiGatewayApiConfigs                  | project, apiGatewayGateways                                                                         |
 | apiKeys                               | project                                                                                             |
 | assets                                | project                                                                                             |
 | bigQueryDataset                       | project                                                                                             |

package.json

@@ -33,6 +33,7 @@
   },
   "dependencies": {
     "@cloudgraph/sdk": "^0.11.0",
+    "@google-cloud/api-gateway": "^1.2.1",
     "@google-cloud/asset": "^3.22.0",
     "@google-cloud/bigquery": "^5.10.0",
     "@google-cloud/bigquery-connection": "^1.5.1",

src/enums/relations.ts

@@ -3,7 +3,7 @@ import services from './services'
 /**
  * Set relations between services to data sharing
  * The key of the object represents the parent or base service,
- * it might contain a array of dependant or childs that must be executed after the parent
+ * it might contain an array of dependant or children that must be executed after the parent
  */
 export default {
   [services.storageBucket]: [services.iamPolicy],
@@ -14,4 +14,5 @@ export default {
   [services.kmsKeyRing]: [services.kmsCryptoKeys],
   [services.kmsCryptoKeys]: [services.iamPolicy],
   [services.bigQueryDataTransfer]: [services.bigQueryDataTransferRun],
+  [services.apiGatewayApi]: [services.apiGatewayApiConfig],
 }

src/enums/schemasMap.ts

@@ -47,6 +47,9 @@ export default {
   [services.dataprocAutoscalingPolicy]: 'gcpDataprocAutoscalingPolicy',
   [services.dataprocJob]: 'gcpDataprocJob',
   [services.dataprocWorkflowTemplate]: 'gcpDataprocWorkflowTemplate',
+  [services.apiGatewayGateway]: 'gcpApiGatewayGateway',
+  [services.apiGatewayApi]: 'gcpApiGatewayApi',
+  [services.apiGatewayApiConfig]: 'gcpApiGatewayApiConfig',
   [services.firestoreDatabase]: 'gcpFirestoreDatabase',
   tag: 'gcpTag',
 }

src/enums/serviceMap.ts

@@ -42,6 +42,9 @@ import GcpDataprocCluster from '../services/dataprocCluster'
 import GcpDataprocAutoscalingPolicy from '../services/dataprocAutoscalingPolicy'
 import GcpDataprocJob from '../services/dataprocJob'
 import GcpDataprocWorkflowTemplate from '../services/dataprocWorkflowTemplate'
+import GcpApiGatewayGateway from '../services/apiGatewayGateway'
+import GcpApiGatewayApi from '../services/apiGatewayApi'
+import GcpApiGatewayApiConfig from '../services/apiGatewayApiConfig'
 import GcpFirestoreDatabase from '../services/firestore'
 import GcpLabel from '../services/label'
 import GcpTag from '../services/tag'
@@ -94,6 +97,9 @@ export default {
   [services.dataprocAutoscalingPolicy]: GcpDataprocAutoscalingPolicy,
   [services.dataprocJob]: GcpDataprocJob,
   [services.dataprocWorkflowTemplate]: GcpDataprocWorkflowTemplate,
+  [services.apiGatewayGateway]: GcpApiGatewayGateway,
+  [services.apiGatewayApi]: GcpApiGatewayApi,
+  [services.apiGatewayApiConfig]: GcpApiGatewayApiConfig,
   [services.firestoreDatabase]: GcpFirestoreDatabase,
   label: GcpLabel,
   tag: GcpTag,

src/enums/services.ts

@@ -7,7 +7,9 @@ export default {
   // ml: 'ml',
   // mlEngine: 'ml-engine',
   // notebooks: 'notebooks',
-  // apiGateway: 'api-gateway',
+  apiGatewayGateway: 'apiGatewayGateways',
+  apiGatewayApi: 'apiGatewayApis',
+  apiGatewayApiConfig: 'apiGatewayApiConfigs',
   // apigee: 'apigee',
   // endpoints: 'endpoints',
   // recommender: 'recommender',

src/services/apiGatewayApi/connections.ts

@@ -0,0 +1,56 @@
+import { ServiceConnection } from '@cloudgraph/sdk'
+import { RawGcpApiGatewayApi } from './data'
+
+import services from '../../enums/services'
+import { RawGcpApiGatewayGateway } from '../apiGatewayGateway/data'
+
+export default ({
+  service,
+  data,
+}: {
+  service: RawGcpApiGatewayApi
+  data: { name: string; data: { [property: string]: any[] } }[]
+  region: string
+}): {
+  [property: string]: ServiceConnection[]
+} => {
+  const { id } = service
+  const connections: ServiceConnection[] = []
+
+  /**
+   * Find Api Gateway Gateways
+   */
+  const gateways: {
+    name: string
+    data: { [property: string]: any[] }
+  } = data.find(({ name }) => name === services.apiGatewayGateway)
+
+  if (gateways?.data) {
+    for (const region of Object.keys(gateways.data)) {
+      const filteredGateways = gateways.data[region].filter(
+        ({ apiConfig }: RawGcpApiGatewayGateway) => {
+          const idSessions = apiConfig.split('/')
+          idSessions[1] = service.projectId
+          const apiIdWithNumber =  idSessions.join('/')
+          return apiIdWithNumber.startsWith(`${id}/`)
+        }
+      )
+  
+      if (filteredGateways) {
+        for (const filteredGateway of filteredGateways) {
+          connections.push({
+            id: filteredGateway.id,
+            resourceType: services.apiGatewayGateway,
+            relation: 'child',
+            field: 'apiGatewayGateways',
+          })
+        }
+      }
+    }
+  }
+
+  const result = {
+    [service.id]: connections,
+  }
+  return result
+}

src/services/apiGatewayApi/data.ts

@@ -0,0 +1,61 @@
+import groupBy from 'lodash/groupBy'
+import { ApiGatewayServiceClient } from '@google-cloud/api-gateway'
+import CloudGraph from '@cloudgraph/sdk'
+import { google } from '@google-cloud/api-gateway/build/protos/protos'
+import gcpLoggerText from '../../properties/logger'
+import { GcpServiceInput, LabelMap } from '../../types'
+import { generateGcpErrorLog, initTestEndpoint } from '../../utils'
+import { GLOBAL_REGION } from '../../config/constants'
+
+const lt = { ...gcpLoggerText }
+const { logger } = CloudGraph
+const serviceName = 'API Gateway Api'
+const apiEndpoint = initTestEndpoint(serviceName)
+
+export interface RawGcpApiGatewayApi extends 
+Omit<google.cloud.apigateway.v1.IApi, 'labels'> {
+  id: string,
+  region: string,
+  projectId: string,
+  Labels: LabelMap,
+}
+
+export const listApiGatewayApiData = async (
+  apiGatewayClient: ApiGatewayServiceClient,
+  projectId: string,
+  apiData: RawGcpApiGatewayApi[],
+): Promise<void> => {
+  try {
+    const apiIter = apiGatewayClient.listApisAsync({
+      parent: `projects/${projectId}/locations/${GLOBAL_REGION}`,
+    })
+    for await (const {name, labels, ...api} of apiIter) {
+      apiData.push({
+        id: name,
+        projectId,
+        region: GLOBAL_REGION,
+        Labels: labels,
+        ...api,
+      })
+    }
+  } catch (error) {
+    generateGcpErrorLog(serviceName, 'ApiGateway:listApisAsync', error)
+  }
+}
+
+export default async ({
+  config,
+}: GcpServiceInput): Promise<{
+  [region: string]: RawGcpApiGatewayApi[]
+}> => {
+  const { projectId } = config
+
+  const apiData: RawGcpApiGatewayApi[] = []
+  const apiGatewayClient = new ApiGatewayServiceClient({ ...config, apiEndpoint })
+
+  await listApiGatewayApiData(apiGatewayClient, projectId, apiData)
+
+  logger.debug(lt.foundResources(serviceName, apiData.length))
+
+  return groupBy(apiData, 'region')
+}

src/services/apiGatewayApi/format.ts

@@ -0,0 +1,37 @@
+import { google } from '@google-cloud/api-gateway/build/protos/protos'
+import { RawGcpApiGatewayApi } from './data'
+import { enumKeyToString, formatLabelsFromMap } from '../../utils/format'
+import { GcpApiGatewayApi } from '../../types/generated'
+import { toISOString } from '../../utils/dateutils'
+
+export default ({
+  service,
+  account,
+  region,
+}: {
+  service: RawGcpApiGatewayApi
+  account: string
+  region: string
+}): GcpApiGatewayApi => {
+  const {
+    id,
+    Labels,
+    createTime,
+    updateTime,
+    displayName,
+    managedService,
+    state,
+  } = service
+
+  return {
+    id,
+    name: displayName,
+    projectId: account,
+    region,
+    labels: formatLabelsFromMap(Labels),
+    createTime: toISOString(createTime?.seconds?.toString()) || '',
+    updateTime: toISOString(updateTime?.seconds?.toString()) || '',
+    managedService,
+    state: enumKeyToString(google.cloud.apigateway.v1.Api.State, state),
+  }
+}

src/services/apiGatewayApi/index.ts

@@ -0,0 +1,16 @@
+import { Service } from '@cloudgraph/sdk'
+import BaseService from '../base'
+import format from './format'
+import getData from './data'
+import getConnections from './connections'
+import mutation from './mutation'
+
+export default class GcpApiGatewayApi extends BaseService implements Service {
+  format = format.bind(this)
+
+  getData = getData.bind(this)
+
+  getConnections = getConnections.bind(this)
+
+  mutation = mutation;
+}