Merge branch 'feature/cg-670-gcp-dataproc' into 'main'

Feature/cg-670-gcp-dataproc

See merge request auto-cloud/cloudgraph/provider/cloudgraph-provider-gcp!41

Author: tyler-dunkel

README.md

@@ -76,6 +76,10 @@ CloudGraph GCP Provider will ask you what regions you would like to crawl and wi
 | cloudFunction                         | project, vpc                                                                     |
 | cloudRouter                           | project                                                                          |
 | computeProject                        | project                                                                          |
+| dataprocClusters                      | project, dataprocJobs, dataprocWorkflowTemplate                                  |
+| dataprocAutoscalingPolicies           | project                                                                          |
+| dataprocJobs                          | project, dataprocClusters                                                        |
+| dataprocWorkflowTemplate              | project, dataprocClusters                                                        |
 | kmsCryptoKeys                         | iamPolicy, kmsKeyRing, project                                                   |
 | dnsManagedZone                        | project                                                                          |
 | dnsPolicy                             | project, network                                                                 |

package.json

@@ -35,6 +35,7 @@
     "@cloudgraph/sdk": "^0.11.0",
     "@google-cloud/asset": "^3.22.0",
     "@google-cloud/compute": "^3.0.0",
+    "@google-cloud/dataproc": "^3.2.0",
     "@google-cloud/dns": "^2.2.3",
     "@google-cloud/functions": "^1.2.0",
     "@google-cloud/kms": "^2.10.0",

src/enums/schemasMap.ts

@@ -40,5 +40,9 @@ export default {
   [services.serviceAccount]: 'gcpServiceAccount',
   [services.apiKey]: 'gcpApiKey',
   [services.computeProject]: 'gcpComputeProject',
+  [services.dataprocCluster]: 'gcpDataprocCluster',
+  [services.dataprocAutoscalingPolicy]: 'gcpDataprocAutoscalingPolicy',
+  [services.dataprocJob]: 'gcpDataprocJob',
+  [services.dataprocWorkflowTemplate]: 'gcpDataprocWorkflowTemplate',
   tag: 'gcpTag',
 }

src/enums/serviceMap.ts

@@ -36,7 +36,10 @@ import GcpSslPolicies from '../services/sslPolicy'
 import GcpServiceAccount from '../services/serviceAccount'
 import GcpApiKey from '../services/apiKey'
 import GcpComputeProject from '../services/computeProject'
-
+import GcpDataprocCluster from '../services/dataprocCluster'
+import GcpDataprocAutoscalingPolicy from '../services/dataprocAutoscalingPolicy'
+import GcpDataprocJob from '../services/dataprocJob'
+import GcpDataprocWorkflowTemplate from '../services/dataprocWorkflowTemplate'
 /**
  * serviceMap is an object that contains all currently supported services
  * serviceMap is used by the serviceFactory to produce instances of service classes
@@ -78,5 +81,9 @@ export default {
   [services.serviceAccount]: GcpServiceAccount,
   [services.apiKey]: GcpApiKey,
   [services.computeProject]: GcpComputeProject,
+  [services.dataprocCluster]: GcpDataprocCluster,
+  [services.dataprocAutoscalingPolicy]: GcpDataprocAutoscalingPolicy,
+  [services.dataprocJob]: GcpDataprocJob,
+  [services.dataprocWorkflowTemplate]: GcpDataprocWorkflowTemplate,
   tag: GcpTag,
 }

src/enums/services.ts

@@ -28,7 +28,10 @@ export default {
   // composer: 'composer',
   // dataCatalog: 'data-catalog',
   // dataflow: 'dataflow',
-  // dataproc: 'dataproc',
+  dataprocCluster: 'dataprocClusters',
+  dataprocAutoscalingPolicy: 'dataprocAutoscalingPolicies',
+  dataprocJob: 'dataprocJobs',
+  dataprocWorkflowTemplate: 'dataprocWorkflowTemplates',
   // metastore: 'metastore',
   // pubsub: 'pubsub',
   // bigtable: 'bigtable',

src/services/base/schema.graphql

@@ -11,3 +11,15 @@ interface gcpBaseResource
     kind: String @search(by: [hash, regexp])
     labels: [gcpRawLabel]
   }
+
+type gcpKeyValue
+  @generate(
+    query: { get: true, query: true, aggregate: true }
+    mutation: { add: true, delete: false }
+    subscription: false
+  ) 
+  @key(fields: "id") {
+    id: String! @id @search(by: [hash])
+    key: String! @search(by: [hash, regexp])
+    value: String @search(by: [hash, regexp])
+  }

src/services/dataprocAutoscalingPolicy/data.ts

@@ -0,0 +1,62 @@
+import { AutoscalingPolicyServiceClient } from '@google-cloud/dataproc'
+import CloudGraph from '@cloudgraph/sdk'
+import groupBy from 'lodash/groupBy'
+import { google } from '@google-cloud/dataproc/build/protos/protos'
+import gcpLoggerText from '../../properties/logger'
+import { GcpServiceInput } from '../../types'
+import { generateGcpErrorLog } from '../../utils'
+
+const lt = { ...gcpLoggerText }
+const { logger } = CloudGraph
+const serviceName = 'Dataproc Autoscaling Policy'
+
+export interface RawGcpDataprocAutoscalingPolicy extends Omit<google.cloud.dataproc.v1.IAutoscalingPolicy, 'id'|'labels'> {
+  id: string
+  region: string
+  projectId: string
+  Labels: { [key: string]: string }
+}
+
+export default async ({
+  regions,
+  config,
+}: GcpServiceInput): Promise<{
+  [region: string]: RawGcpDataprocAutoscalingPolicy[]
+}> => {
+    const autoscalingPolicyList: RawGcpDataprocAutoscalingPolicy[] = []
+    const { projectId } = config
+
+    for (const region of regions.split(',')) {
+      /**
+       * Get all the Dataproc Autoscaling Policies
+       */
+        
+      try {
+        const dataprocClient = new AutoscalingPolicyServiceClient({ 
+          ...config,
+          apiEndpoint: `${region}-dataproc.googleapis.com`,
+          projectId,
+        })
+        const locationName = dataprocClient.locationPath(projectId, region)
+        const iterable =  dataprocClient.listAutoscalingPoliciesAsync({
+          parent: locationName,
+        })
+        for await (const { id, labels, ...response } of iterable) {
+          if (response) {
+            autoscalingPolicyList.push({
+              ...response,
+              id,
+              projectId,
+              region,
+              Labels: labels,
+            })
+          }
+        }
+      } catch (error) {
+        generateGcpErrorLog(serviceName, 'dataprocAutoscalingPolicy:listAutoscalingPoliciesAsync', error)
+      }
+    }
+
+    logger.debug(lt.foundResources(serviceName, autoscalingPolicyList.length))
+    return groupBy(autoscalingPolicyList, 'region')
+  }

src/services/dataprocAutoscalingPolicy/format.ts

@@ -0,0 +1,43 @@
+import {
+  GcpDataprocAutoscalingPolicy
+} from '../../types/generated'
+import { RawGcpDataprocAutoscalingPolicy } from './data'
+import { formatLabelsFromMap } from '../../utils/format'
+
+export default ({
+  service,
+  region,
+}: {
+  service: RawGcpDataprocAutoscalingPolicy
+  region: string
+}): GcpDataprocAutoscalingPolicy => {
+  const {
+    id,
+    projectId,
+    name,
+    basicAlgorithm = {},
+    workerConfig = {},
+    secondaryWorkerConfig = {},
+    Labels = {},
+  } = service
+
+  return {
+    id,
+    projectId,
+    region,
+    name,
+    basicAlgorithmYarnConfigGracefulDecommissionTimeout: basicAlgorithm?.yarnConfig?.gracefulDecommissionTimeout?.seconds?.toString() || '',
+    basicAlgorithmYarnConfigScaleUpFactor: basicAlgorithm?.yarnConfig?.scaleUpFactor || 0.0,
+    basicAlgorithmYarnConfigScaleDownFactor: basicAlgorithm?.yarnConfig?.scaleDownFactor || 0.0,
+    basicAlgorithmYarnConfigScaleUpMinWorkerFraction: basicAlgorithm?.yarnConfig?.scaleUpMinWorkerFraction || 0.0,
+    basicAlgorithmYarnConfigScaleDownMinWorkerFraction: basicAlgorithm?.yarnConfig?.scaleDownMinWorkerFraction || 0.0,
+    basicAlgorithmCooldownPeriod: basicAlgorithm?.cooldownPeriod?.seconds?.toString() || '',
+    workerConfigMinInstances: workerConfig?.minInstances || 0,
+    workerConfigMaxInstances: workerConfig?.maxInstances || 0,
+    workerConfigWeight: workerConfig?.weight || 0,
+    secondaryWorkerConfigMinInstances: secondaryWorkerConfig?.minInstances || 0,
+    secondaryWorkerConfigMaxInstances: secondaryWorkerConfig?.maxInstances || 0,
+    secondaryWorkerConfigWeight: secondaryWorkerConfig?.weight || 0,
+    labels: formatLabelsFromMap(Labels),
+  }
+}

src/services/dataprocAutoscalingPolicy/index.ts

@@ -0,0 +1,13 @@
+import {Service} from '@cloudgraph/sdk'
+import BaseService from '../base'
+import format from './format'
+import getData from './data'
+import mutation from './mutation'
+
+export default class GcpDataprocAutoscalingPolicy extends BaseService implements Service {
+  format = format.bind(this)
+
+  getData = getData.bind(this)
+
+  mutation = mutation
+}

src/services/dataprocAutoscalingPolicy/mutation.ts

@@ -0,0 +1,5 @@
+export default `mutation($input: [AddgcpDataprocAutoscalingPolicyInput!]!) {
+  addgcpDataprocAutoscalingPolicy(input: $input, upsert: true) {
+    numUids
+  }
+}`;