feat(dataproc): couple updates/fixes to the cluster

Author: james-zhou-inspire11

src/enums/services.ts

@@ -28,7 +28,7 @@ export default {
   // composer: 'composer',
   // dataCatalog: 'data-catalog',
   // dataflow: 'dataflow',
-  dataprocCluster: 'dataprocCluster',
+  dataprocCluster: 'dataprocClusters',
   // metastore: 'metastore',
   // pubsub: 'pubsub',
   // bigtable: 'bigtable',

src/services/dataprocCluster/data.ts

@@ -5,12 +5,11 @@ import groupBy from 'lodash/groupBy'
 import { google } from '@google-cloud/dataproc/build/protos/protos'
 import gcpLoggerText from '../../properties/logger'
 import { GcpServiceInput } from '../../types'
-import { initTestEndpoint, generateGcpErrorLog } from '../../utils'
+import { generateGcpErrorLog } from '../../utils'
 
 const lt = { ...gcpLoggerText }
 const { logger } = CloudGraph
 const serviceName = 'Dataproc Cluster'
-const apiEndpoint = initTestEndpoint(serviceName)
 
 export interface RawGcpDataprocCluster extends Omit<google.cloud.dataproc.v1.ICluster, 'projectId' | 'labels'> {
   id: string
@@ -34,7 +33,11 @@ export default async ({
        */
 
       try {
-        const dataprocClient = new ClusterControllerClient({ ...config, apiEndpoint })
+        const dataprocClient = new ClusterControllerClient({ 
+          ...config,
+          apiEndpoint: `${region}-dataproc.googleapis.com`,
+          projectId,
+        })
 
         const iterable =  dataprocClient.listClustersAsync({ projectId, region })
         for await (const { labels, ...response } of iterable) {

src/services/dataprocCluster/format.ts

@@ -146,9 +146,9 @@ const formatClusterConfigLifecycle = ({
   idleStartTime,
 }: google.cloud.dataproc.v1.ILifecycleConfig): GcpDataprocClusterConfigLifecycleConfig => {
   return {
-    idleDeleteTtl: idleDeleteTtl.seconds?.toString() || '',
+    idleDeleteTtl: idleDeleteTtl?.seconds?.toString() || '',
     autoDeleteTime: toISOString(autoDeleteTime?.seconds?.toString()),
-    autoDeleteTtl: autoDeleteTtl.seconds?.toString() || '',
+    autoDeleteTtl: autoDeleteTtl?.seconds?.toString() || '',
     idleStartTime: toISOString(idleStartTime?.seconds?.toString()),
   }
 }
@@ -192,19 +192,19 @@ const formatClusterConfig = ({
   return {
     configBucket,
     tempBucket,
-    gceClusterConfig: formatClusterConfigGceCluster(gceClusterConfig),
-    masterConfig: formatClusterConfigInstanceGroup(masterConfig),
-    workerConfig: formatClusterConfigInstanceGroup(workerConfig),
-    secondaryWorkerConfig: formatClusterConfigInstanceGroup(secondaryWorkerConfig),
-    softwareConfig: formatClusterConfigSoftware(softwareConfig),
+    gceClusterConfig: formatClusterConfigGceCluster(gceClusterConfig || {}),
+    masterConfig: formatClusterConfigInstanceGroup(masterConfig || {}),
+    workerConfig: formatClusterConfigInstanceGroup(workerConfig || {}),
+    secondaryWorkerConfig: formatClusterConfigInstanceGroup(secondaryWorkerConfig || {}),
+    softwareConfig: formatClusterConfigSoftware(softwareConfig || {}),
     initializationActions: initializationActions?.map(formatClusterConfigNodeInitializationAction),
     encryptionConfigGcePdKmsKeyName: encryptionConfig?.gcePdKmsKeyName || '',
     autoscalingConfigPolicyUri: autoscalingConfig?.policyUri || '',
-    securityConfig: formatClusterConfigSecurity(securityConfig),
-    lifecycleConfig: formatClusterConfigLifecycle(lifecycleConfig),
-    endpointConfig: formatClusterConfigEndpoint(endpointConfig),
+    securityConfig: formatClusterConfigSecurity(securityConfig || {}),
+    lifecycleConfig: formatClusterConfigLifecycle(lifecycleConfig || {}),
+    endpointConfig: formatClusterConfigEndpoint(endpointConfig || {}),
     metastoreMetastoreServiceConfig: metastoreConfig?.dataprocMetastoreService || '',
-    gkeClusterConfig: formatClusterConfigGke(gkeClusterConfig),
+    gkeClusterConfig: formatClusterConfigGke(gkeClusterConfig || {}),
   }
 }
 

src/services/dataprocCluster/schema.graphql

@@ -197,5 +197,5 @@ type gcpDataprocCluster implements gcpBaseResource
     clusterUuid: String @search(by: [hash, regexp])
     hdfsMetrics: [gcpDataprocClusterMetric]
     yarnMetrics: [gcpDataprocClusterMetric]
-    project: [gcpProject] @hasInverse(field: dataprocCluster)
+    project: [gcpProject] @hasInverse(field: dataprocClusters)
   }

src/services/project/schema.graphql

@@ -46,4 +46,5 @@ type gcpProject @key(fields: "id") {
   serviceAccounts: [gcpServiceAccount] @hasInverse(field: project)
   kmsCryptoKeys: [gcpKmsCryptoKey] @hasInverse(field: project)
   dataprocCluster: [gcpDataprocCluster] @hasInverse(field: project)
+  dataprocClusters: [gcpDataprocCluster] @hasInverse(field: project)
 }

src/types/generated.ts

@@ -1446,6 +1446,7 @@ export type GcpProject = {
   computeProject?: Maybe<Array<Maybe<GcpComputeProject>>>;
   createTime?: Maybe<Scalars['String']>;
   dataprocCluster?: Maybe<Array<Maybe<GcpDataprocCluster>>>;
+  dataprocClusters?: Maybe<Array<Maybe<GcpDataprocCluster>>>;
   deleteTime?: Maybe<Scalars['String']>;
   displayName?: Maybe<Scalars['String']>;
   dnsManagedZone?: Maybe<Array<Maybe<GcpDnsManagedZone>>>;