Merge branch 'feature/CG-530' into 'main'

CG-530: feat(label/tag): add labels/tags as services

Closes CG-530

See merge request auto-cloud/cloudgraph/provider/cloudgraph-provider-gcp!47

Author: tyler-dunkel

src/enums/serviceMap.ts

@@ -15,7 +15,6 @@ import GcpCdnUrlMap from '../services/cdnUrlMap'
 import GcpDnsManagedZone from '../services/dnsManagedZone'
 import GcpVpc from '../services/vpc'
 import GcpIam from '../services/iam'
-import GcpTag from '../services/tag'
 import GcpLogBucket from '../services/logBucket'
 import GcpLogSink from '../services/logSink'
 import GcpLogView from '../services/logView'
@@ -44,6 +43,8 @@ import GcpDataprocAutoscalingPolicy from '../services/dataprocAutoscalingPolicy'
 import GcpDataprocJob from '../services/dataprocJob'
 import GcpDataprocWorkflowTemplate from '../services/dataprocWorkflowTemplate'
 import GcpFirestoreDatabase from '../services/firestore'
+import GcpLabel from '../services/label'
+import GcpTag from '../services/tag'
 
 /**
  * serviceMap is an object that contains all currently supported services
@@ -94,5 +95,6 @@ export default {
   [services.dataprocJob]: GcpDataprocJob,
   [services.dataprocWorkflowTemplate]: GcpDataprocWorkflowTemplate,
   [services.firestoreDatabase]: GcpFirestoreDatabase,
+  label: GcpLabel,
   tag: GcpTag,
 }

src/enums/services.ts

@@ -67,7 +67,7 @@ export default {
   // recaptcha: 'recaptcha',
   // resourceManager: 'resource-manager',
   // resourceSettings: 'resource-settings',
-  secretManager: 'secretManager',
+  secretManager: 'secrets',
   // iot: 'iot',
   // cloudShell: 'cloud-shell',
   // debug: 'debug',

src/services/alertPolicy/format.ts

@@ -35,7 +35,7 @@ export default ({
     region,
     displayName,
     documentation,
-    userLabels: formatLabelsFromMap(userLabels),
+    labels: formatLabelsFromMap(userLabels),
     conditions: conditions?.map(condition => ({
       id: cuid(),
       name: condition?.name,

src/services/alertPolicy/schema.graphql

@@ -1,7 +1,7 @@
 type gcpAlertPolicy implements gcpBaseResource @key(fields: "id") {
   displayName: String @search(by: [hash, regexp])
   documentation: gcpAlertPolicyDocumentation
-  userLabels: [gcpRawLabel]
+  labels: [gcpRawLabel]
   conditions: [gcpAlertPolicyCondition]
   combiner: String @search(by: [hash, regexp])
   enabled: gcpBoolValue

src/services/index.ts

@@ -279,7 +279,8 @@ export default class Provider extends CloudGraph.Client {
     }
     const resourceNames: string[] = sortResourcesDependencies(relations, [
       ...new Set<string>(configuredResources.split(',')),
-    ])
+    ]).concat(['label', 'tag'])
+
     const { projectId } = account
     const config = { ...account }
 
@@ -351,8 +352,6 @@ export default class Provider extends CloudGraph.Client {
     // data so we can pass along accountId
     // TODO: find a better way to handle this
     let mergedRawData: rawDataInterface[] = []
-    const tagRegion = GLOBAL_REGION
-    const tags = { name: 'tag', data: { [tagRegion]: [] } }
     const crawledAccounts = []
     for (const account of configuredAccounts) {
       const { projectId } = account
@@ -370,43 +369,6 @@ export default class Provider extends CloudGraph.Client {
       }
     }
 
-    // Handle global tag entities
-    try {
-      for (const { data: entityData } of rawData) {
-        for (const region of Object.keys(entityData)) {
-          const dataAtRegion = entityData[region]
-          dataAtRegion.forEach(singleEntity => {
-            if (!isEmpty(singleEntity.Tags)) {
-              for (const [key, value] of Object.entries(singleEntity.Tags)) {
-                if (
-                  !tags.data[tagRegion].find(
-                    ({ id }) => id === `${key}:${value}`
-                  )
-                ) {
-                  tags.data[tagRegion].push({
-                    id: `${key}:${value}`,
-                    key,
-                    value,
-                  })
-                }
-              }
-            }
-          })
-        }
-      }
-      const existingTagsIdx = rawData.findIndex(({ name }) => {
-        return name === 'tag'
-      })
-      if (existingTagsIdx > -1) {
-        rawData[existingTagsIdx] = tags
-      } else {
-        rawData.push(tags)
-      }
-    } catch (error: any) {
-      this.logger.error('There was an error aggregating tags')
-      this.logger.debug(error)
-    }
-
     /**
      * Loop through the sdk data to format entities and build connections
      * 1. Format data with provider service format function

src/services/label/connections.ts

@@ -0,0 +1,52 @@
+import { ServiceConnection } from '@cloudgraph/sdk'
+import { rawDataInterface } from '../../types'
+import { GLOBAL_REGION } from '../../config/constants'
+import services from '../../enums/services'
+
+export default ({
+  service,
+  data,
+  region,
+}: {
+  service: any
+  data: Array<{ name: string; data: { [property: string]: any[] } }>
+  region: string
+}): {
+  [property: string]: ServiceConnection[]
+} => {
+  const { id } = service
+  const connections: ServiceConnection[] = []
+  const connectTo = Object.values(services)
+
+  for (const serviceName of connectTo) {
+    const instances: {
+      name: string
+      data: { [property: string]: any[] }
+    } = data.find(({ name }) => name === serviceName)
+
+    const regions = [region, GLOBAL_REGION]
+    for (const region of regions) {
+      if (instances?.data?.[region]) {
+        const filtered = instances.data[region].filter(
+          ({ labels }: rawDataInterface) => !!labels
+        )
+
+        for (const instance of filtered) {
+          if (instance) {
+            connections.push({
+              id: instance.id,
+              resourceType: serviceName,
+              relation: 'child',
+              field: serviceName,
+            })
+          }
+        }
+      }
+    }
+  }
+  
+  const labelResult = {
+    [id]: connections,
+  }
+  return labelResult
+}

src/services/label/data.ts

@@ -0,0 +1,60 @@
+import groupBy from 'lodash/groupBy'
+import isEmpty from 'lodash/isEmpty'
+import CloudGraph from '@cloudgraph/sdk'
+import gcpLoggerText from '../../properties/logger'
+import { generateGcpErrorLog } from '../../utils'
+import { GcpServiceInput } from '../../types'
+import { GcpLabel } from '../../types/generated'
+
+const lt = { ...gcpLoggerText }
+const { logger } = CloudGraph
+const serviceName = 'Label'
+
+export interface RawGcpLabel extends GcpLabel {
+  id: string
+  projectId: string
+  region: string
+}
+
+export default async ({
+  config,
+  rawData,
+}: GcpServiceInput): Promise<{
+  [region: string]: RawGcpLabel[]
+}> =>
+  new Promise(async resolve => {
+    const labelList: RawGcpLabel[] = []
+    const { projectId } = config
+
+    try {
+      for (const { data: entityData } of rawData) {
+        for (const region of Object.keys(entityData)) {
+          const dataAtRegion = entityData[region]
+          dataAtRegion.forEach(singleEntity => {
+            if (!isEmpty(singleEntity.labels)) {
+              for (const [key, value] of Object.entries(singleEntity.labels)) {
+                if (
+                  !labelList.find(
+                    ({ id }) => id === `${singleEntity.id}:${key}:${value}`
+                  )
+                ) {
+                  labelList.push({
+                    id: `${singleEntity.id}:${key}:${value}`,
+                    projectId,
+                    key,
+                    value: String(value),
+                    region,
+                  })
+                }
+              }
+            }
+          })
+        }
+      }
+    } catch (error: any) {
+      generateGcpErrorLog(serviceName, '', error)
+    }
+
+    logger.debug(lt.foundResources(serviceName, labelList.length))
+    resolve(groupBy(labelList, 'region'))
+  })

src/services/label/index.ts

@@ -0,0 +1,15 @@
+import { Service } from '@cloudgraph/sdk';
+import BaseService from '../base';
+import getConnections from './connections';
+import getData from './data'
+import mutation from './mutation';
+
+export default class GcpLabel extends BaseService implements Service {
+  format = ({service}: {service: any}): any => service
+
+  getConnections = getConnections.bind(this)
+
+  getData = getData.bind(this)
+
+  mutation = mutation;
+}

src/services/label/mutation.ts

@@ -0,0 +1,5 @@
+export default `mutation($input: [AddgcpLabelInput!]!) {
+  addgcpLabel(input: $input, upsert: true) {
+    numUids
+  }
+}`;

src/services/label/schema.graphql

@@ -0,0 +1,24 @@
+type gcpLabel implements gcpBaseResource @key(fields: "id") {
+  id: String! @id @search(by: [hash])
+  key: String! @search(by: [hash, regexp])
+  value: String! @search(by: [hash, regexp])
+  alertPolicy: [gcpAlertPolicy]
+  cloudFunction: [gcpCloudFunction]
+  dnsManagedZone: [gcpDnsManagedZone]
+  kmsCryptoKey: [gcpKmsCryptoKey]
+  project: [gcpProject]
+  secrets: [gcpSecret]
+  storageBucket: [gcpStorageBucket]
+  vmInstance: [gcpVmInstance]
+}
+
+type gcpRawLabel
+  @generate(
+    query: { get: false, query: false, aggregate: false }
+    mutation: { add: false, delete: false }
+    subscription: false
+  ) {
+  id: String! @id
+  key: String
+  value: String
+}