Merge pull request #9 from cloudgraphdev/feature/CG-1259

feat(service): Added GCP Billing service

Author: tyler-dunkel

src/enums/schemasMap.ts

@@ -12,6 +12,7 @@ export default {
   [services.bigQueryReservationCapacityCommitment]: 'gcpBigQueryReservationCapacityCommitment',
   [services.bigQueryDataTransfer]: 'gcpBigQueryDataTransfer',
   [services.bigQueryDataTransferRun]: 'gcpBigQueryDataTransferRun',
+  [services.billing]: 'gcpBilling',
   [services.kmsKeyRing]: 'gcpKmsKeyRing',
   [services.kmsCryptoKeys]: 'gcpKmsCryptoKey',
   [services.cdnBackendBucket]: 'gcpCdnBackendBucket',

src/enums/serviceMap.ts

@@ -49,6 +49,7 @@ import GcpApiGatewayApiConfig from '../services/apiGatewayApiConfig'
 import GcpFirestoreDatabase from '../services/firestore'
 import GcpLabel from '../services/label'
 import GcpTag from '../services/tag'
+import GcpBilling from '../services/billing'
 
 /**
  * serviceMap is an object that contains all currently supported services
@@ -63,6 +64,7 @@ export default {
   [services.bigQueryReservationCapacityCommitment]: GcpBigQueryReservationCapacityCommitment,
   [services.bigQueryDataTransfer]: GcpBigQueryDataTransfer,
   [services.bigQueryDataTransferRun]: GcpBigQueryDataTransferRun,
+  [services.billing]: GcpBilling,
   [services.kmsKeyRing]: GcpKmsKeyRing,
   [services.kmsCryptoKeys]: GcpKmsCryptoKey,
   [services.cloudRouter]: GcpCloudRouter,

src/enums/services.ts

@@ -16,7 +16,7 @@ export default {
   // recommender: 'recommender',
   // services: 'services',
   // anthos: 'anthos',
-  // billing: 'billing',
+  billing: 'billing',
   // artifacts: 'artifacts',
   // builds: 'builds',
   // scheduler: 'scheduler',

src/properties/logger.ts

@@ -1,29 +1,48 @@
 export default {
   platform: 'Platform: GCP',
-  fetchingResourceData: (resource: string): string => `Fetching ${resource} data...`,
+  fetchingResourceData: (resource: string): string =>
+    `Fetching ${resource} data...`,
   doneFetchingResourceData: (resource: string, num: number): string =>
     `✅ Done fetching ${resource} data in ${num}s ✅`,
-  foundResources: (resource: string, num: number): string => `Found ${num} ${resource}`,
+  foundResources: (resource: string, num: number): string =>
+    `Found ${num} ${resource}`,
   lookingForResourcesAddToRegion: (region: string, resource: string): string =>
     `Looking for ${resource} to add to Region ${region}...`,
-  addingServiceToRegion: (region: string, resource: string, num: number): string =>
-    `Adding ${num} ${resource} to Region ${region}`,
+  addingServiceToRegion: (
+    region: string,
+    resource: string,
+    num: number
+  ): string => `Adding ${num} ${resource} to Region ${region}`,
   lookingForResourcesAddToProject: (resource: string): string =>
     `Looking for ${resource} to add to the Project...`,
   addingServiceToProject: (resource: string, num: number): string =>
     `Adding ${num} ${resource} to the Project`,
   /**
-  * VPC
-  */
+   * VPC
+   */
   foundVpcs: (num: number): string => `Found ${num} VPCs`,
   /**
-  * IAM
-  */
+   * IAM
+   */
   foundPolicies: (num: number): string => `Found ${num} IAM Policies`,
   /**
-  * Logging
-  */
+   * Logging
+   */
   foundLogBuckets: (num: number): string => `Found ${num} Log Buckets`,
   foundLogSinks: (num: number): string => `Found ${num} Log Sinks`,
-  foundLogViews: (num: number): string => `Found ${num} Log Views`
+  foundLogViews: (num: number): string => `Found ${num} Log Views`,
+  /**
+   * Billing
+   */
+  fetchingAggregateFinOpsData:
+    'Fetching aggregate FinOps data for this GCP account via the Azure SDK...',
+  unableToFindFinOpsAggregateData:
+    '❌ Unable to get billing data for this GCP account, billing data was missing. ❌',
+  queryingAggregateFinOpsDataForRegion: (
+    region: string,
+    type: string
+  ): string =>
+    `Querying aggregate FinOps data for the ${region} region using the ${type} method...`,
+  doneFetchingAggregateFinOpsData: (num: number): string =>
+    `🕒 Done fetching aggregate FinOps data in ${num} 🕘`,
 }

src/services/billing/data.ts

@@ -0,0 +1,306 @@
+import CloudGraph from '@cloudgraph/sdk'
+import { BigQuery, RowMetadata } from '@google-cloud/bigquery'
+import * as _ from 'lodash'
+import { GLOBAL_REGION } from '../../config/constants'
+import gcpLoggerText from '../../properties/logger'
+import { GcpServiceInput } from '../../types'
+import { generateGcpErrorLog, initTestEndpoint } from '../../utils'
+import {
+  createDiffSecs,
+  getCurrentDayOfMonth,
+  getDaysAgo,
+  getFirstDayOfMonth,
+} from '../../utils/dateutils'
+import {
+  formatAmmountAndUnit,
+  getCurrency,
+  getTotalCost,
+  RawGcpTotalCost,
+  totalCostQuery,
+  totalCostGroupByServiceQuery,
+} from './utils'
+
+const lt = { ...gcpLoggerText }
+const { logger } = CloudGraph
+const serviceName = 'Billing'
+const apiEndpoint = initTestEndpoint(serviceName)
+
+export interface costInterface {
+  cost?: number
+  currency?: string
+  formattedCost?: string
+}
+
+export interface RawGcpBilling {
+  totalCostLast30Days: costInterface
+  totalCostMonthToDate: costInterface
+  monthToDateDailyAverage: { [key: string]: costInterface }
+  last30DaysDailyAverage: { [key: string]: costInterface }
+  monthToDate: { [key: string]: costInterface }
+  last30Days: { [key: string]: costInterface }
+  individualData: { [key: string]: costInterface }
+}
+
+export const listBillingData = async (
+  client: BigQuery,
+  projectId: string,
+  billings: RawGcpTotalCost[],
+  billingAccountId: string,
+  dataset: string,
+  groupBy?: boolean,
+  startDate?: string,
+  endDate?: string
+): Promise<void> =>
+  new Promise<void>(async resolve => {
+    try {
+      const billingTable = `${projectId}.${dataset}.gcp_billing_export_resource_v1_${billingAccountId}`
+
+      let sqlQuery = ''
+      if (groupBy) {
+        sqlQuery = totalCostGroupByServiceQuery(billingTable, startDate, endDate)
+      } else {
+        sqlQuery = totalCostQuery(billingTable, startDate, endDate)
+      }
+
+      const options = {
+        query: sqlQuery,
+      }
+
+      // Run the query
+      const [rows] = await client.query(options)
+
+      if (!_.isEmpty(rows)) {
+        billings.push(...rows)
+      }
+    } catch (error) {
+      generateGcpErrorLog(serviceName, 'bigQuery:getTotalCost', error)
+    }
+
+    resolve()
+  })
+
+export default async ({
+  config,
+}: GcpServiceInput): Promise<{
+  [region: string]: RawGcpBilling[]
+}> => {
+  const startDate = new Date()
+  const region = GLOBAL_REGION
+  const results: RawGcpBilling = {
+    totalCostLast30Days: {},
+    totalCostMonthToDate: {},
+    monthToDateDailyAverage: {},
+    last30DaysDailyAverage: {},
+    monthToDate: {},
+    last30Days: {},
+    individualData: {},
+  }
+  const resultPromises = []
+  const {
+    projectId,
+    billing: { billingAccountId, bigQueryDataset },
+  } = config
+
+  try {
+    const client = new BigQuery({ ...config, apiEndpoint })
+
+    const listAggregateFinOpsData = async ({
+      resolve,
+      type,
+      groupBy = true,
+      individualData = false,
+      timePeriod: TimePeriod,
+    }: {
+      resolve: () => void
+      type: string
+      groupBy?: boolean
+      individualData?: boolean
+      timePeriod: { Start: string; End: string }
+    }): Promise<void> => {
+      logger.debug(lt.queryingAggregateFinOpsDataForRegion(region, type))
+      const billingData: RowMetadata[] = []
+
+      await listBillingData(
+        client,
+        projectId,
+        billingData,
+        billingAccountId,
+        bigQueryDataset,
+        groupBy,
+        TimePeriod.Start,
+        TimePeriod.End
+      )
+
+      if (_.isEmpty(billingData)) {
+        logger.debug(lt.unableToFindFinOpsAggregateData)
+        return resolve()
+      }
+
+      if (groupBy || individualData) {
+        const services = _.groupBy(billingData, u => u.service)
+        Object.keys(services).map(name => {
+          const serviceUsages = services[name]
+          const currency = getCurrency(serviceUsages)
+          const cost = getTotalCost(serviceUsages)
+          const costData = {
+            cost,
+            currency,
+            formattedCost: formatAmmountAndUnit({
+              Amount: cost,
+              Unit: currency,
+            }),
+          }
+          if (individualData) {
+            results.individualData[name] = costData
+          } else {
+            results[type][name] = costData
+          }
+        })
+      } else {
+        const currency = getCurrency(billingData)
+        const cost = getTotalCost(billingData)
+        results[type] = {
+          cost,
+          currency,
+          formattedCost: formatAmmountAndUnit({
+            Amount: cost,
+            Unit: currency,
+          }),
+        }
+      }
+
+      resolve()
+    }
+
+    /**
+     * Now we make 4 queries to the api in order to get aggregate pricing data sliced in various ways
+     */
+
+    const today = new Date().toLocaleDateString('en-ca')
+    const startOfMonth = getFirstDayOfMonth()
+
+    const commonArgs = {
+      timePeriod: {
+        Start: getDaysAgo(60), // TODO: change to 30 !!!
+        End: today,
+      },
+    }
+
+    /**
+     * Breakdown by service types and spend for last 30 days
+     */
+    const last30DaysData = new Promise<void>(resolve =>
+      listAggregateFinOpsData({
+        ...commonArgs,
+        resolve,
+        type: 'last30Days',
+      })
+    )
+    resultPromises.push(last30DaysData)
+
+    /**
+     * Breakdown by service types and spend since the beginning of the month
+     */
+    if (!(today === startOfMonth)) {
+      const monthToDateData = new Promise<void>(resolve =>
+        listAggregateFinOpsData({
+          resolve,
+          type: 'monthToDate',
+          timePeriod: {
+            Start: startOfMonth,
+            End: today,
+          },
+        })
+      )
+      resultPromises.push(monthToDateData)
+    }
+
+    /**
+     * The single total cost of everything in the last 30 days
+     */
+    const totalCostLast30Days = new Promise<void>(resolve =>
+      listAggregateFinOpsData({
+        ...commonArgs,
+        resolve,
+        type: 'totalCostLast30Days',
+        groupBy: false,
+      })
+    )
+    resultPromises.push(totalCostLast30Days)
+
+    /**
+     * The single total cost of everything in the current month
+     */
+    if (!(today === startOfMonth)) {
+      const totalCostMonthToDate = new Promise<void>(resolve =>
+        listAggregateFinOpsData({
+          resolve,
+          type: 'totalCostMonthToDate',
+          groupBy: false,
+          timePeriod: {
+            Start: startOfMonth,
+            End: today,
+          },
+        })
+      )
+      resultPromises.push(totalCostMonthToDate)
+    }
+
+    const individualDataPromise = new Promise<void>(resolve =>
+      listAggregateFinOpsData({
+        resolve,
+        type: 'individualData',
+        individualData: true,
+        timePeriod: {
+          Start: getDaysAgo(1), // i.e. get the daily cost
+          End: today,
+        },
+      })
+    )
+    resultPromises.push(individualDataPromise)
+
+    await Promise.all(resultPromises)
+
+    /**
+     * Create Daily Averages
+     */
+
+    const createDailyAverage = ({
+      days,
+      resultMonthlyData,
+      resultAverageData,
+    }): void[] =>
+      Object.keys(resultMonthlyData).map(service => {
+        const { cost: aggregateCost, currency } = resultMonthlyData[service]
+        const cost = parseFloat((aggregateCost / days).toFixed(10))
+        results[resultAverageData][service] = {
+          cost,
+          currency,
+          formattedCost: formatAmmountAndUnit({ Amount: cost, Unit: currency }),
+        }
+      })
+
+    if (!_.isEmpty(results.monthToDate)) {
+      createDailyAverage({
+        days: parseInt(getCurrentDayOfMonth(), 10),
+        resultMonthlyData: results.monthToDate,
+        resultAverageData: 'monthToDateDailyAverage',
+      })
+    }
+    if (!_.isEmpty(results.last30Days)) {
+      createDailyAverage({
+        days: 30,
+        resultMonthlyData: results.last30Days,
+        resultAverageData: 'last30DaysDailyAverage',
+      })
+    }
+
+    logger.debug(lt.doneFetchingAggregateFinOpsData(createDiffSecs(startDate)))
+    return { [region]: [results] }
+  } catch (e) {
+    logger.error(e)
+  }
+
+  logger.debug(lt.doneFetchingAggregateFinOpsData(createDiffSecs(startDate)))
+  return { [region]: [results] }
+}