diff --git a/handwritten/storage/conformance-test/conformanceCommon.ts b/handwritten/storage/conformance-test/conformanceCommon.ts index a206ea064fe8..ddec27bddfa3 100644 --- a/handwritten/storage/conformance-test/conformanceCommon.ts +++ b/handwritten/storage/conformance-test/conformanceCommon.ts @@ -14,12 +14,11 @@ * limitations under the License. */ import * as jsonToNodeApiMapping from './test-data/retryInvocationMap.json'; -import * as libraryMethods from './libraryMethods'; +import * as libraryMethods from './libraryMethods.js'; import {Bucket, File, HmacKey, Notification, Storage} from '../src/'; import * as crypto from 'crypto'; import * as assert from 'assert'; import {DecorateRequestOptions} from '../src/nodejs-common'; -import fetch from 'node-fetch'; interface RetryCase { instructions: String[]; diff --git a/handwritten/storage/package.json b/handwritten/storage/package.json index ab9d8dd36101..eed089d8dc0d 100644 --- a/handwritten/storage/package.json +++ b/handwritten/storage/package.json @@ -99,9 +99,8 @@ "@types/mocha": "^9.1.1", "@types/mockery": "^1.4.29", "@types/node": "^24.0.0", - "@types/node-fetch": "^2.1.3", "@types/proxyquire": "^1.3.28", - "@types/request": "^2.48.4", + "@types/request": "^2.48.12", "@types/sinon": "^17.0.0", "@types/tmp": "0.2.6", "@types/yargs": "^17.0.35", @@ -115,7 +114,6 @@ "mocha": "^11.1.0", "mockery": "^2.1.0", "nock": "~13.5.0", - "node-fetch": "^2.6.7", "pack-n-play": "^5.0.1", "proxyquire": "^2.1.3", "sinon": "^18.0.0", diff --git a/handwritten/storage/src/bucket.ts b/handwritten/storage/src/bucket.ts index c12622f1b457..23aefac9e3fe 100644 --- a/handwritten/storage/src/bucket.ts +++ b/handwritten/storage/src/bucket.ts @@ -308,6 +308,47 @@ export interface EncryptionEnforcementConfig { restrictionMode?: 'NotRestricted' | 'FullyRestricted'; readonly effectiveTime?: string; } + +/** + * Configuration for a bucket's IP Filter. + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('my-bucket'); + * + * const metadata = { + * ipFilter: { + * mode: 'Enabled', + * publicNetworkSource: { + * allowedIpCidrRanges: ['192.168.1.1/32'] + * } + * } + * }; + * + * bucket.setMetadata(metadata, (err, apiResponse) => { + * if (err) { + * console.error(err); + * } else { + * console.log('IP filter updated successfully.'); + * } + * }); + * ``` + */ +export interface IpFilter { + mode?: 'Enabled' | 'Disabled'; + publicNetworkSource?: { + allowedIpCidrRanges?: string[]; + }; + vpcNetworkSources?: { + network?: string; + allowedIpCidrRanges?: string[]; + }[]; + allowAllServiceAgentAccess?: boolean; + allowCrossOrgVpcs?: boolean; +} + export interface BucketMetadata extends BaseMetadata { acl?: AclMetadata[] | null; autoclass?: { @@ -341,6 +382,7 @@ export interface BucketMetadata extends BaseMetadata { lockedTime?: string; }; }; + ipFilter?: IpFilter | null; labels?: { [key: string]: string | null; }; diff --git a/handwritten/storage/src/index.ts b/handwritten/storage/src/index.ts index 78285e225105..f5450e978b7d 100644 --- a/handwritten/storage/src/index.ts +++ b/handwritten/storage/src/index.ts @@ -146,6 +146,7 @@ export { GetNotificationsCallback, GetNotificationsOptions, GetNotificationsResponse, + IpFilter, Labels, LifecycleAction, LifecycleCondition, diff --git a/handwritten/storage/src/nodejs-common/index.ts b/handwritten/storage/src/nodejs-common/index.ts index 02c9310c5da3..6cdaa371024e 100644 --- a/handwritten/storage/src/nodejs-common/index.ts +++ b/handwritten/storage/src/nodejs-common/index.ts @@ -45,7 +45,11 @@ export { AbortableDuplex, ApiError, BodyResponseCallback, + DecorateHeadersOptions, + DecorateHeadersResult, DecorateRequestOptions, + decorateHeaders, + Headers, ResponseBody, util, } from './util.js'; diff --git a/handwritten/storage/src/nodejs-common/service.ts b/handwritten/storage/src/nodejs-common/service.ts index 4853142638f0..7cbc3a478645 100644 --- a/handwritten/storage/src/nodejs-common/service.ts +++ b/handwritten/storage/src/nodejs-common/service.ts @@ -20,7 +20,6 @@ import { GoogleAuthOptions, } from 'google-auth-library'; import type {Request} from 'teeny-request'; -import * as crypto from 'crypto'; import {Interceptor} from './service-object.js'; import { @@ -29,13 +28,9 @@ import { GCCL_GCS_CMD_KEY, MakeAuthenticatedRequest, PackageJson, + decorateHeaders, util, } from './util.js'; -import { - getRuntimeTrackingString, - getUserAgentString, - getModuleFormat, -} from '../util.js'; export const DEFAULT_PROJECT_ID_TOKEN = '{{projectId}}'; @@ -271,23 +266,12 @@ export class Service { delete reqOpts.interceptors_; - const pkg = this.packageJson; - let userAgent = getUserAgentString(); - if (this.providedUserAgent) { - userAgent = `${this.providedUserAgent} ${userAgent}`; - } - reqOpts.headers = { - ...reqOpts.headers, - 'User-Agent': userAgent, - 'x-goog-api-client': `${getRuntimeTrackingString()} gccl/${ - pkg.version - }-${getModuleFormat()} gccl-invocation-id/${crypto.randomUUID()}`, - }; - - if (reqOpts[GCCL_GCS_CMD_KEY]) { - reqOpts.headers['x-goog-api-client'] += - ` gccl-gcs-cmd/${reqOpts[GCCL_GCS_CMD_KEY]}`; - } + const {headers} = decorateHeaders(reqOpts.headers, { + packageJson: this.packageJson, + providedUserAgent: this.providedUserAgent, + gcclGcsCmd: reqOpts[GCCL_GCS_CMD_KEY], + }); + reqOpts.headers = headers; if (reqOpts.shouldReturnStream) { return this.makeAuthenticatedRequest(reqOpts) as {} as Request; diff --git a/handwritten/storage/src/nodejs-common/util.ts b/handwritten/storage/src/nodejs-common/util.ts index 08e19b477162..9e9908820193 100644 --- a/handwritten/storage/src/nodejs-common/util.ts +++ b/handwritten/storage/src/nodejs-common/util.ts @@ -89,6 +89,9 @@ const MAX_RETRY_DEFAULT = 3; // eslint-disable-next-line @typescript-eslint/no-explicit-any export type ResponseBody = any; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export type Headers = {[header: string]: any}; + // Directly copy over Duplexify interfaces export interface DuplexifyOptions extends DuplexOptions { autoDestroy?: boolean; @@ -1052,20 +1055,88 @@ export class Util { : [optionsOrCallback as T, cb as C]; } + decorateHeaders( + headers?: CoreOptions['headers'], + options?: DecorateHeadersOptions + ) { + return decorateHeaders(headers, options); + } + _getDefaultHeaders(gcclGcsCmd?: string) { - const headers = { - 'User-Agent': getUserAgentString(), - 'x-goog-api-client': `${getRuntimeTrackingString()} gccl/${ - packageJson.version - }-${getModuleFormat()} gccl-invocation-id/${crypto.randomUUID()}`, - }; + return decorateHeaders(undefined, {gcclGcsCmd}).headers; + } +} - if (gcclGcsCmd) { - headers['x-goog-api-client'] += ` gccl-gcs-cmd/${gcclGcsCmd}`; - } +export interface DecorateHeadersOptions { + idempotencyToken?: string; + packageJson?: PackageJson; + providedUserAgent?: string; + gcclGcsCmd?: string; +} + +export interface DecorateHeadersResult { + headers: Headers; + idempotencyToken: string; +} - return headers; +/** + * Decorates and sanitizes headers for GCS requests: + * - Checks for user-provided `x-goog-gcs-idempotency-token` case-insensitively. + * - If a valid non-empty string user token is provided, uses it as the idempotency token and preserves the header. + * - If not provided or invalid, removes any invalid header key and sets `x-goog-gcs-idempotency-token` to either the provided fallback token or a generated UUID. + * - Adds `User-Agent` and `x-goog-api-client` (with tracking string, package version, gccl-invocation-id, and optional gccl-gcs-cmd). + * + * @param headers Existing headers object (optional). + * @param options Decoration options (idempotencyToken, packageJson, providedUserAgent, gcclGcsCmd). + * @returns An object containing the decorated headers and the effective idempotency token. + */ +export function decorateHeaders( + headers?: CoreOptions['headers'], + options?: DecorateHeadersOptions +): DecorateHeadersResult { + const sanitizedHeaders: Headers = {...headers}; + const userTokenKey = Object.keys(sanitizedHeaders).find( + key => key.toLowerCase() === 'x-goog-gcs-idempotency-token' + ); + const userTokenValue = userTokenKey + ? sanitizedHeaders[userTokenKey] + : undefined; + const hasValidUserToken = + typeof userTokenValue === 'string' && userTokenValue.trim() !== ''; + + const idempotencyToken = hasValidUserToken + ? (userTokenValue as string) + : options?.idempotencyToken || crypto.randomUUID(); + + let userAgent = getUserAgentString(); + if (options?.providedUserAgent) { + userAgent = `${options.providedUserAgent} ${userAgent}`; + } + + const pkg = options?.packageJson || packageJson; + let googAPIClient = `${getRuntimeTrackingString()} gccl/${ + pkg.version + }-${getModuleFormat()} gccl-invocation-id/${idempotencyToken}`; + + const gcclGcsCmd = options?.gcclGcsCmd; + if (gcclGcsCmd) { + googAPIClient += ` gccl-gcs-cmd/${gcclGcsCmd}`; } + + sanitizedHeaders['User-Agent'] = userAgent; + sanitizedHeaders['x-goog-api-client'] = googAPIClient; + + if (!hasValidUserToken) { + if (userTokenKey) { + delete sanitizedHeaders[userTokenKey]; + } + sanitizedHeaders['x-goog-gcs-idempotency-token'] = idempotencyToken; + } + + return { + headers: sanitizedHeaders, + idempotencyToken, + }; } /** diff --git a/handwritten/storage/src/resumable-upload.ts b/handwritten/storage/src/resumable-upload.ts index 39e12291f35c..49a1af237b8d 100644 --- a/handwritten/storage/src/resumable-upload.ts +++ b/handwritten/storage/src/resumable-upload.ts @@ -30,21 +30,12 @@ import {Readable, Writable, WritableOptions} from 'stream'; import AsyncRetry from 'async-retry'; import {RetryOptions, PreconditionOptions} from './storage.js'; import * as crypto from 'crypto'; -import { - getRuntimeTrackingString, - getModuleFormat, - getUserAgentString, -} from './util.js'; -import {GCCL_GCS_CMD_KEY} from './nodejs-common/util.js'; +import {GCCL_GCS_CMD_KEY, decorateHeaders} from './nodejs-common/util.js'; import {FileExceptionMessages, FileMetadata, RequestError} from './file.js'; -// eslint-disable-next-line @typescript-eslint/ban-ts-comment -// @ts-ignore -import {getPackageJSON} from './package-json-helper.cjs'; import {HashStreamValidator} from './hash-stream-validator.js'; const NOT_FOUND_STATUS_CODE = 404; const RESUMABLE_INCOMPLETE_STATUS_CODE = 308; -const packageJson = getPackageJSON(); export const PROTOCOL_REGEX = /^(\w*):\/\//; @@ -338,7 +329,11 @@ export class Upload extends Writable { timeOfFirstRequest: number; isPartialUpload: boolean; - private currentInvocationId = { + private currentInvocationId: { + checkUploadStatus: string; + chunk: string; + uri: string; + } = { checkUploadStatus: crypto.randomUUID(), chunk: crypto.randomUUID(), uri: crypto.randomUUID(), @@ -821,14 +816,22 @@ export class Upload extends Writable { delete metadata.contentType; } - let googAPIClient = `${getRuntimeTrackingString()} gccl/${ - packageJson.version - }-${getModuleFormat()} gccl-invocation-id/${this.currentInvocationId.uri}`; - - if (this.#gcclGcsCmd) { - googAPIClient += ` gccl-gcs-cmd/${this.#gcclGcsCmd}`; + if (this.origin) { + headers.Origin = this.origin; } + const {headers: reqHeaders, idempotencyToken} = decorateHeaders( + { + ...this.customRequestOptions?.headers, + ...headers, + }, + { + idempotencyToken: this.currentInvocationId.uri, + gcclGcsCmd: this.#gcclGcsCmd, + } + ); + this.currentInvocationId.uri = idempotencyToken; + // Check if headers already exist before creating new ones const reqOpts: GaxiosOptions = { method: 'POST', @@ -841,11 +844,7 @@ export class Upload extends Writable { this.params ), data: metadata, - headers: { - 'User-Agent': getUserAgentString(), - 'x-goog-api-client': googAPIClient, - ...headers, - }, + headers: reqHeaders, }; if (metadata.contentLength) { @@ -1010,20 +1009,14 @@ export class Upload extends Writable { }, }); - let googAPIClient = `${getRuntimeTrackingString()} gccl/${ - packageJson.version - }-${getModuleFormat()} gccl-invocation-id/${ - this.currentInvocationId.chunk - }`; - - if (this.#gcclGcsCmd) { - googAPIClient += ` gccl-gcs-cmd/${this.#gcclGcsCmd}`; - } - - const headers: GaxiosOptions['headers'] = { - 'User-Agent': getUserAgentString(), - 'x-goog-api-client': googAPIClient, - }; + const {headers, idempotencyToken} = decorateHeaders( + this.customRequestOptions?.headers, + { + idempotencyToken: this.currentInvocationId.chunk, + gcclGcsCmd: this.#gcclGcsCmd, + } + ); + this.currentInvocationId.chunk = idempotencyToken; // If using multiple chunk upload, set appropriate header if (multiChunkMode) { @@ -1224,25 +1217,24 @@ export class Upload extends Writable { async checkUploadStatus( config: CheckUploadStatusConfig = {} ): Promise> { - let googAPIClient = `${getRuntimeTrackingString()} gccl/${ - packageJson.version - }-${getModuleFormat()} gccl-invocation-id/${ - this.currentInvocationId.checkUploadStatus - }`; - - if (this.#gcclGcsCmd) { - googAPIClient += ` gccl-gcs-cmd/${this.#gcclGcsCmd}`; - } + const localHeaders: Record = { + ...this.customRequestOptions?.headers, + 'Content-Length': 0, + 'Content-Range': 'bytes */*', + }; + const {headers: reqHeaders, idempotencyToken} = decorateHeaders( + localHeaders, + { + idempotencyToken: this.currentInvocationId.checkUploadStatus, + gcclGcsCmd: this.#gcclGcsCmd, + } + ); + this.currentInvocationId.checkUploadStatus = idempotencyToken; const opts: GaxiosOptions = { method: 'PUT', url: this.uri, - headers: { - 'Content-Length': 0, - 'Content-Range': 'bytes */*', - 'User-Agent': getUserAgentString(), - 'x-goog-api-client': googAPIClient, - }, + headers: reqHeaders, }; try { @@ -1321,7 +1313,7 @@ export class Upload extends Writable { ); }; - const combinedReqOpts = { + const combinedReqOpts: GaxiosOptions = { ...this.customRequestOptions, ...reqOpts, headers: { @@ -1330,6 +1322,19 @@ export class Upload extends Writable { }, }; + if (combinedReqOpts.headers) { + const headers = combinedReqOpts.headers as Record; + const userTokenKey = Object.keys(headers).find( + key => key.toLowerCase() === 'x-goog-gcs-idempotency-token' + ); + const userTokenValue = userTokenKey ? headers[userTokenKey] : undefined; + const hasValidUserToken = + typeof userTokenValue === 'string' && userTokenValue.trim() !== ''; + if (!hasValidUserToken && userTokenKey) { + delete headers[userTokenKey]; + } + } + const res = await this.authClient.request<{error?: object}>( combinedReqOpts ); @@ -1351,7 +1356,7 @@ export class Upload extends Writable { reqOpts.signal = controller.signal; reqOpts.validateStatus = () => true; - const combinedReqOpts = { + const combinedReqOpts: GaxiosOptions = { ...this.customRequestOptions, ...reqOpts, headers: { @@ -1359,6 +1364,20 @@ export class Upload extends Writable { ...reqOpts.headers, }, }; + + if (combinedReqOpts.headers) { + const headers = combinedReqOpts.headers as Record; + const userTokenKey = Object.keys(headers).find( + key => key.toLowerCase() === 'x-goog-gcs-idempotency-token' + ); + const userTokenValue = userTokenKey ? headers[userTokenKey] : undefined; + const hasValidUserToken = + typeof userTokenValue === 'string' && userTokenValue.trim() !== ''; + if (!hasValidUserToken && userTokenKey) { + delete headers[userTokenKey]; + } + } + const res = await this.authClient.request(combinedReqOpts); const successfulRequest = this.onResponse(res); this.removeListener('error', errorCallback); diff --git a/handwritten/storage/system-test/storage.ts b/handwritten/storage/system-test/storage.ts index d9351f703732..c674dd42d1e5 100644 --- a/handwritten/storage/system-test/storage.ts +++ b/handwritten/storage/system-test/storage.ts @@ -16,8 +16,6 @@ import assert from 'assert'; import {after, afterEach, before, beforeEach, describe, it} from 'mocha'; import * as crypto from 'crypto'; import * as fs from 'fs'; -import fetch from 'node-fetch'; -import FormData from 'form-data'; import pLimit from 'p-limit'; import {promisify} from 'util'; import * as path from 'path'; @@ -44,6 +42,7 @@ interface ErrorCallbackFunction { } import {PubSub, Subscription, Topic} from '@google-cloud/pubsub'; import {getDirName} from '../src/util.js'; +import {BucketMetadata} from '../src/bucket.js'; class HTTPError extends Error { code: number; @@ -1294,6 +1293,101 @@ describe('storage', function () { }); describe('bucket metadata', () => { + describe('ipFilter', () => { + let ipFilterBucket: Bucket; + + before(async () => { + ipFilterBucket = storage.bucket(generateName()); + await ipFilterBucket.create(); + }); + + after(async () => { + await ipFilterBucket.delete().catch(() => { }); + }); + + + it('should create a bucket with ipFilter', async () => { + const metadata: BucketMetadata = { + ipFilter: { + mode: 'Disabled', + publicNetworkSource: { + allowedIpCidrRanges: ['0.0.0.0/0'], + }, + allowAllServiceAgentAccess: true, + }, + }; + const bucketToCreate = storage.bucket(generateName()); + const [bucket, apiResponse] = await bucketToCreate.create(metadata); + + assert.strictEqual(apiResponse.ipFilter?.mode, metadata.ipFilter?.mode); + assert.deepStrictEqual(apiResponse.ipFilter?.publicNetworkSource?.allowedIpCidrRanges, metadata.ipFilter?.publicNetworkSource?.allowedIpCidrRanges); + assert.strictEqual(apiResponse.ipFilter?.allowAllServiceAgentAccess, metadata.ipFilter?.allowAllServiceAgentAccess); + + await bucket.delete().catch(() => {}); + }); + + it('should set ipFilter', async () => { + const metadata: BucketMetadata = { + ipFilter: { + mode: 'Disabled', + publicNetworkSource: { + allowedIpCidrRanges: ['0.0.0.0/0', '::/0'], + }, + allowAllServiceAgentAccess: false, + }, + }; + const [meta] = await ipFilterBucket.setMetadata(metadata); + assert.deepStrictEqual(meta.ipFilter, metadata.ipFilter); + }); + + it('should get ipFilter', async () => { + const [meta] = await ipFilterBucket.getMetadata(); + assert.strictEqual(meta.ipFilter?.mode, 'Disabled'); + assert.deepStrictEqual( + meta.ipFilter?.publicNetworkSource?.allowedIpCidrRanges, + ['0.0.0.0/0', '::/0'] + ); + }); + + it('should update ipFilter', async () => { + const metadata: BucketMetadata = { + ipFilter: { + mode: 'Disabled', + publicNetworkSource: { + allowedIpCidrRanges: ['203.0.113.0/24'], + }, + allowAllServiceAgentAccess: false, + }, + }; + const [meta] = await ipFilterBucket.setMetadata(metadata); + assert.deepStrictEqual(meta.ipFilter, metadata.ipFilter); + }); + + it('should clear allowedIpCidrRanges', async () => { + const [getMeta] = await ipFilterBucket.getMetadata(); + assert.strictEqual(getMeta.ipFilter?.mode, 'Disabled'); + assert.deepStrictEqual( + getMeta.ipFilter?.publicNetworkSource?.allowedIpCidrRanges, + ['203.0.113.0/24'] + ); + + const metadata: BucketMetadata = { + ipFilter: { + mode: 'Disabled', + publicNetworkSource: { + // The API omits the field when the array is cleared. + allowedIpCidrRanges: [], + }, + allowAllServiceAgentAccess: false, + }, + }; + const [meta] = await ipFilterBucket.setMetadata(metadata); + assert.strictEqual(meta.ipFilter?.mode, 'Disabled'); + assert.strictEqual(meta.ipFilter?.publicNetworkSource?.allowedIpCidrRanges, undefined); + assert.strictEqual(meta.ipFilter?.allowAllServiceAgentAccess, false); + }); + }); + it('should allow setting metadata on a bucket', async () => { const metadata = { website: { @@ -4608,7 +4702,15 @@ describe('storage', function () { setTimeout(resolve, RETENTION_DURATION_SECONDS * 1000), ); return Promise.all( - buckets.map(bucket => limit(() => deleteBucketAsync(bucket))), + buckets.map(bucket => + limit(() => + deleteBucketAsync(bucket).catch((err: ApiError) => { + if (err.code !== 404) { + throw err; + } + }) + ) + ) ); } diff --git a/handwritten/storage/test/bucket.ts b/handwritten/storage/test/bucket.ts index c995d243c592..531db6415888 100644 --- a/handwritten/storage/test/bucket.ts +++ b/handwritten/storage/test/bucket.ts @@ -3714,5 +3714,185 @@ describe('Bucket', () => { bucket.setMetadata(clearMetadata, assert.ifError); }); }); + + describe('ipFilter', () => { + it('should pass ipFilter to create', done => { + const metadata = { + ipFilter: { + mode: 'Disabled', + publicNetworkSource: { + allowedIpCidrRanges: ['192.168.0.0/16'], + }, + allowAllServiceAgentAccess: true, + }, + }; + + const storageMock = Object.assign({}, bucket.storage, { + createBucket: ( + name: string, + options: unknown, + callback: Function + ) => { + assert.strictEqual(name, bucket.name); + assert.deepStrictEqual(options, metadata); + callback(null, bucket, metadata); + }, + }); + + const testBucket = new Bucket(storageMock, bucket.name); + testBucket.create(metadata, (err: Error | null) => { + assert.ifError(err); + done(); + }); + }); + + it('should enable ipFilter', done => { + const metadata = { + ipFilter: { + mode: 'Enabled', + publicNetworkSource: { + allowedIpCidrRanges: ['192.168.1.1/32'], + }, + }, + }; + + bucket.parent.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + assert.strictEqual(reqOpts.method, 'PATCH'); + assert.deepStrictEqual(reqOpts.json.ipFilter, metadata.ipFilter); + callback(null, metadata); + }; + + bucket.setMetadata(metadata, (err: Error | null) => { + assert.ifError(err); + done(); + }); + }); + + it('should update ipFilter', done => { + const metadata = { + ipFilter: { + mode: 'Enabled', + vpcNetworkSources: [ + { + network: 'projects/my-project/global/networks/my-vpc', + allowedIpCidrRanges: ['10.0.0.0/8'], + }, + ], + }, + }; + + bucket.parent.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + assert.strictEqual(reqOpts.method, 'PATCH'); + assert.deepStrictEqual(reqOpts.json.ipFilter, metadata.ipFilter); + callback(null, metadata); + }; + + bucket.setMetadata(metadata, (err: Error | null) => { + assert.ifError(err); + done(); + }); + }); + + it('should get ipFilter', async () => { + const ipFilter = { + mode: 'Enabled', + publicNetworkSource: { + allowedIpCidrRanges: ['192.168.1.1/32'], + }, + vpcNetworkSources: [ + { + network: 'projects/my-project/global/networks/my-vpc', + allowedIpCidrRanges: ['10.0.0.0/8'], + }, + ], + allowAllServiceAgentAccess: true, + allowCrossOrgVpcs: true, + }; + + bucket.parent.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(null, {ipFilter}); + }; + + const [metadata] = (await bucket.getMetadata()) as [BucketMetadata]; + assert.deepStrictEqual(metadata.ipFilter, ipFilter); + }); + + it('should clear allowedIpCidrRanges', done => { + const initialIpFilter = { + mode: 'Disabled', + publicNetworkSource: { + allowedIpCidrRanges: ['203.0.113.0/24'], + }, + }; + + const updatedIpFilter = { + mode: 'Disabled', + publicNetworkSource: { + allowedIpCidrRanges: undefined, + }, + allowAllServiceAgentAccess: false, + }; + + bucket.parent.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + if (reqOpts.method === 'PATCH') { + assert.deepStrictEqual( + reqOpts.json.ipFilter?.publicNetworkSource?.allowedIpCidrRanges, + [] + ); + callback(null, {ipFilter: updatedIpFilter}); + } else { + callback(null, {ipFilter: initialIpFilter}); + } + }; + + bucket.getMetadata((err: Error | null, getMeta?: BucketMetadata) => { + assert.ifError(err); + assert.strictEqual(getMeta?.ipFilter?.mode, 'Disabled'); + assert.deepStrictEqual( + getMeta?.ipFilter?.publicNetworkSource?.allowedIpCidrRanges, + ['203.0.113.0/24'] + ); + + const metadataUpdate = { + ipFilter: { + mode: 'Disabled', + publicNetworkSource: { + allowedIpCidrRanges: [], + }, + allowAllServiceAgentAccess: false, + }, + }; + + bucket.setMetadata( + metadataUpdate, + (err: Error | null, meta?: BucketMetadata) => { + assert.ifError(err); + assert.strictEqual(meta?.ipFilter?.mode, 'Disabled'); + assert.strictEqual( + meta?.ipFilter?.publicNetworkSource?.allowedIpCidrRanges, + undefined + ); + assert.strictEqual( + meta?.ipFilter?.allowAllServiceAgentAccess, + false + ); + done(); + } + ); + }); + }); + }); }); }); diff --git a/handwritten/storage/test/headers.ts b/handwritten/storage/test/headers.ts index 9ccc685814bb..b786ae54d4e0 100644 --- a/handwritten/storage/test/headers.ts +++ b/handwritten/storage/test/headers.ts @@ -65,11 +65,16 @@ describe('headers', () => { } catch (err) { if (err !== error) throw err; } - assert.ok( - /^gl-node\/(?[^W]+) gccl\/(?[^W]+) gccl-invocation-id\/(?[^W]+)$/.test( - requests[0].headers['x-goog-api-client'] - ) - ); + const apiClientHeader = requests[0].headers['x-goog-api-client']; + const match = + /^gl-node\/(?\S+) gccl\/(?\S+) gccl-invocation-id\/(?\S+)$/.exec( + apiClientHeader + ); + assert.ok(match); + const invocationId = match.groups!.gcclInvocationId; + const idempotencyToken = + requests[0].headers['x-goog-gcs-idempotency-token']; + assert.strictEqual(idempotencyToken, invocationId); }); it('populates x-goog-api-client header (deno)', async () => { @@ -87,10 +92,37 @@ describe('headers', () => { } catch (err) { if (err !== error) throw err; } - assert.ok( - /^gl-deno\/0.00.0 gccl\/(?[^W]+) gccl-invocation-id\/(?[^W]+)$/.test( - requests[1].headers['x-goog-api-client'] - ) - ); + const apiClientHeader = requests[1].headers['x-goog-api-client']; + const match = + /^gl-deno\/0.00.0 gccl\/(?\S+) gccl-invocation-id\/(?\S+)$/.exec( + apiClientHeader + ); + assert.ok(match); + const invocationId = match.groups!.gcclInvocationId; + const idempotencyToken = + requests[1].headers['x-goog-gcs-idempotency-token']; + assert.strictEqual(idempotencyToken, invocationId); + }); + + it('generates unique tokens for different requests', async () => { + const storage = new Storage(); + const bucket = storage.bucket('foo-bucket'); + try { + await bucket.create(); + } catch (err) { + if (err !== error) throw err; + } + try { + await bucket.create(); + } catch (err) { + if (err !== error) throw err; + } + const token1 = + requests[requests.length - 2].headers['x-goog-gcs-idempotency-token']; + const token2 = + requests[requests.length - 1].headers['x-goog-gcs-idempotency-token']; + assert.ok(token1); + assert.ok(token2); + assert.notStrictEqual(token1, token2); }); }); diff --git a/handwritten/storage/test/index.ts b/handwritten/storage/test/index.ts index 7d095bd11601..f615cbeb1ffa 100644 --- a/handwritten/storage/test/index.ts +++ b/handwritten/storage/test/index.ts @@ -1256,6 +1256,53 @@ describe('Storage', () => { } ); }); + + it('should list buckets with ipFilter summary', done => { + const bucketsResponse = [ + { + id: 'bucket-with-filter', + name: 'bucket-with-filter', + ipFilter: { + mode: 'Enabled', + allowCrossOrgVpcs: true, + allowAllServiceAgentAccess: true, + }, + }, + { + id: 'bucket-without-filter', + name: 'bucket-without-filter', + location: 'US', + }, + ]; + storage.request = ( + reqOpts: DecorateRequestOptions, + callback: Function + ) => { + callback(null, {items: bucketsResponse}); + }; + + storage.getBuckets((err: Error | null, buckets: Bucket[]) => { + if (err) return done(err); + + const filteredBucket = buckets.find( + (b: Bucket) => b.name === 'bucket-with-filter' + )!; + const normalBucket = buckets.find( + (b: Bucket) => b.name === 'bucket-without-filter' + )!; + + assert.ok(filteredBucket.metadata.ipFilter); + assert.strictEqual(filteredBucket.metadata.ipFilter.mode, 'Enabled'); + assert.strictEqual( + filteredBucket.metadata.ipFilter.allowCrossOrgVpcs, + true + ); + + assert.strictEqual(normalBucket.metadata.ipFilter, undefined); + + done(); + }); + }); }); describe('getHmacKeys', () => { diff --git a/handwritten/storage/test/nodejs-common/service.ts b/handwritten/storage/test/nodejs-common/service.ts index 502c4e5419f9..e7aaa8c58d5a 100644 --- a/handwritten/storage/test/nodejs-common/service.ts +++ b/handwritten/storage/test/nodejs-common/service.ts @@ -478,7 +478,7 @@ describe('Service', () => { const r = new RegExp( `^gl-node/${process.versions.node} gccl/${ pkg.version - }-${getModuleFormat()} gccl-invocation-id/(?[^W]+)$` + }-${getModuleFormat()} gccl-invocation-id/(?\\S+)$` ); assert.ok(r.test(reqOpts.headers!['x-goog-api-client'])); done(); @@ -487,6 +487,91 @@ describe('Service', () => { service.request_(reqOpts, assert.ifError); }); + it('should add the x-goog-gcs-idempotency-token header matching the gccl-invocation-id', done => { + service.makeAuthenticatedRequest = (reqOpts: DecorateRequestOptions) => { + const pkg = service.packageJson; + const r = new RegExp( + `^gl-node/${process.versions.node} gccl/${ + pkg.version + }-${getModuleFormat()} gccl-invocation-id/(?\\S+)$` + ); + const match = r.exec(reqOpts.headers!['x-goog-api-client']); + assert.ok(match); + const invocationId = match.groups!.gcclInvocationId; + const idempotencyToken = + reqOpts.headers!['x-goog-gcs-idempotency-token']; + assert.strictEqual(idempotencyToken, invocationId); + done(); + }; + + service.request_(reqOpts, assert.ifError); + }); + + it('should respect user-provided x-goog-gcs-idempotency-token case-insensitively and align it with gccl-invocation-id', done => { + const customToken = 'Custom-Token-With-W-123'; + const customReqOpts = { + ...reqOpts, + headers: { + 'X-Goog-Gcs-Idempotency-Token': customToken, + }, + }; + + service.makeAuthenticatedRequest = (reqOpts: DecorateRequestOptions) => { + const pkg = service.packageJson; + const r = new RegExp( + `^gl-node/${process.versions.node} gccl/${ + pkg.version + }-${getModuleFormat()} gccl-invocation-id/(?\\S+)$` + ); + const match = r.exec(reqOpts.headers!['x-goog-api-client']); + assert.ok(match); + const invocationId = match.groups!.gcclInvocationId; + assert.strictEqual(invocationId, customToken); + + // Verify there is no duplicate x-goog-gcs-idempotency-token header + assert.strictEqual( + reqOpts.headers!['x-goog-gcs-idempotency-token'], + undefined + ); + assert.strictEqual( + reqOpts.headers!['X-Goog-Gcs-Idempotency-Token'], + customToken + ); + done(); + }; + + service.request_(customReqOpts, assert.ifError); + }); + + it('should ignore invalid user-provided idempotency tokens and fallback to generating a UUID', done => { + const customReqOpts = { + ...reqOpts, + headers: { + 'X-Goog-Gcs-Idempotency-Token': undefined as unknown as string, + }, + }; + + service.makeAuthenticatedRequest = (reqOpts: DecorateRequestOptions) => { + const pkg = service.packageJson; + const r = new RegExp( + `^gl-node/${process.versions.node} gccl/${ + pkg.version + }-${getModuleFormat()} gccl-invocation-id/(?\\S+)$` + ); + const match = r.exec(reqOpts.headers!['x-goog-api-client']); + assert.ok(match); + const invocationId = match.groups!.gcclInvocationId; + + // Verify a fallback token was generated and matches the invocation ID + const idempotencyToken = + reqOpts.headers!['x-goog-gcs-idempotency-token']; + assert.strictEqual(idempotencyToken, invocationId); + done(); + }; + + service.request_(customReqOpts, assert.ifError); + }); + it('should add the `gccl-gcs-cmd` to the api-client header when provided', done => { const expected = 'example.expected/value'; service.makeAuthenticatedRequest = (reqOpts: DecorateRequestOptions) => { @@ -494,7 +579,7 @@ describe('Service', () => { const r = new RegExp( `^gl-node/${process.versions.node} gccl/${ pkg.version - }-${getModuleFormat()} gccl-invocation-id/(?[^W]+) gccl-gcs-cmd/${expected}$` + }-${getModuleFormat()} gccl-invocation-id/(?\\S+) gccl-gcs-cmd/${expected}$` ); assert.ok(r.test(reqOpts.headers!['x-goog-api-client'])); done(); diff --git a/handwritten/storage/test/nodejs-common/util.ts b/handwritten/storage/test/nodejs-common/util.ts index 7c554377047e..a85ef9b1c69f 100644 --- a/handwritten/storage/test/nodejs-common/util.ts +++ b/handwritten/storage/test/nodejs-common/util.ts @@ -42,6 +42,7 @@ import {teenyRequest} from 'teeny-request'; import { Abortable, ApiError, + decorateHeaders, DecorateRequestOptions, Duplexify, GCCL_GCS_CMD_KEY, @@ -53,6 +54,7 @@ import { Util, } from '../../src/nodejs-common/util.js'; import {DEFAULT_PROJECT_ID_TOKEN} from '../../src/nodejs-common/service.js'; +import {getModuleFormat} from '../../src/util.js'; import duplexify from 'duplexify'; nock.disableNetConnect(); @@ -82,10 +84,15 @@ function fakeRequest() { } fakeRequest.defaults = (defaults: CoreOptions) => { - assert.ok( - /^gl-node\/(?[^W]+) gccl\/(?[^W]+) gccl-invocation-id\/(?[^W]+)$/.test( - defaults.headers!['x-goog-api-client'] - ) + const match = + /^gl-node\/(?\S+) gccl\/(?\S+) gccl-invocation-id\/(?\S+)$/.exec( + defaults.headers!['x-goog-api-client'] as string + ); + assert.ok(match); + const invocationId = match.groups!.gcclInvocationId; + assert.strictEqual( + defaults.headers!['x-goog-gcs-idempotency-token'], + invocationId ); return fakeRequest; }; @@ -1953,4 +1960,173 @@ describe('common/util', () => { assert.strictEqual(cb, callback); }); }); + + describe('decorateHeaders', () => { + const X_GOOG_API_HEADER_REGEX = + /^gl-node\/(?\S+) gccl\/(?\S+) gccl-invocation-id\/(?\S+)$/; + + it('should return default headers when no headers are provided', () => { + const result = decorateHeaders(); + assert(result.headers); + assert.ok(result.idempotencyToken); + assert.strictEqual( + result.headers['x-goog-gcs-idempotency-token'], + result.idempotencyToken + ); + assert.ok(result.headers['User-Agent']); + const match = X_GOOG_API_HEADER_REGEX.exec( + result.headers['x-goog-api-client'] + ); + assert.ok(match); + assert.strictEqual( + match.groups!.gcclInvocationId, + result.idempotencyToken + ); + }); + + it('should preserve custom headers passed in', () => { + const result = decorateHeaders({ + 'X-Custom-Header': 'custom-value', + }); + assert.strictEqual(result.headers['X-Custom-Header'], 'custom-value'); + }); + + it('should not mutate the input headers object', () => { + const inputHeaders = { + 'X-Goog-Gcs-Idempotency-Token': '', + 'X-Keep-Header': 'stay', + }; + const result = decorateHeaders(inputHeaders); + assert.strictEqual(inputHeaders['X-Goog-Gcs-Idempotency-Token'], ''); + assert.strictEqual(inputHeaders['X-Keep-Header'], 'stay'); + assert.strictEqual( + result.headers['X-Goog-Gcs-Idempotency-Token'], + undefined + ); + assert.strictEqual( + result.headers['x-goog-gcs-idempotency-token'], + result.idempotencyToken + ); + }); + + it('should respect user-provided x-goog-gcs-idempotency-token case-insensitively and align it with gccl-invocation-id', () => { + const customToken = 'Custom-Token-With-W-456'; + const result = decorateHeaders({ + 'X-Goog-Gcs-Idempotency-Token': customToken, + }); + assert.strictEqual(result.idempotencyToken, customToken); + assert.strictEqual( + result.headers['X-Goog-Gcs-Idempotency-Token'], + customToken + ); + assert.strictEqual( + result.headers['x-goog-gcs-idempotency-token'], + undefined + ); + const match = X_GOOG_API_HEADER_REGEX.exec( + result.headers['x-goog-api-client'] + ); + assert.ok(match); + assert.strictEqual(match.groups!.gcclInvocationId, customToken); + }); + + it('should ignore invalid user-provided idempotency tokens and fallback to generating a UUID', () => { + const result = decorateHeaders({ + 'X-Goog-Gcs-Idempotency-Token': '', + }); + assert.ok(result.idempotencyToken); + assert.strictEqual( + result.headers['X-Goog-Gcs-Idempotency-Token'], + undefined + ); + assert.strictEqual( + result.headers['x-goog-gcs-idempotency-token'], + result.idempotencyToken + ); + const match = X_GOOG_API_HEADER_REGEX.exec( + result.headers['x-goog-api-client'] + ); + assert.ok(match); + assert.strictEqual( + match.groups!.gcclInvocationId, + result.idempotencyToken + ); + }); + + it('should ignore whitespace-only user-provided idempotency tokens and fallback to generating a UUID', () => { + const result = decorateHeaders({ + 'X-Goog-Gcs-Idempotency-Token': ' ', + }); + assert.ok(result.idempotencyToken); + assert.strictEqual( + result.headers['X-Goog-Gcs-Idempotency-Token'], + undefined + ); + assert.strictEqual( + result.headers['x-goog-gcs-idempotency-token'], + result.idempotencyToken + ); + const match = X_GOOG_API_HEADER_REGEX.exec( + result.headers['x-goog-api-client'] + ); + assert.ok(match); + assert.strictEqual( + match.groups!.gcclInvocationId, + result.idempotencyToken + ); + }); + + it('should use provided fallback idempotencyToken when user token is not provided or invalid', () => { + const fallback = 'my-fallback-uuid-123'; + const result = decorateHeaders( + { + 'X-Goog-Gcs-Idempotency-Token': '', + }, + {idempotencyToken: fallback} + ); + assert.strictEqual(result.idempotencyToken, fallback); + assert.strictEqual( + result.headers['X-Goog-Gcs-Idempotency-Token'], + undefined + ); + assert.strictEqual( + result.headers['x-goog-gcs-idempotency-token'], + fallback + ); + const match = X_GOOG_API_HEADER_REGEX.exec( + result.headers['x-goog-api-client'] + ); + assert.ok(match); + assert.strictEqual(match.groups!.gcclInvocationId, fallback); + }); + + it('should append providedUserAgent if provided in options', () => { + const result = decorateHeaders(undefined, { + providedUserAgent: 'custom-agent/1.0.0', + }); + assert.ok(result.headers['User-Agent'].startsWith('custom-agent/1.0.0 ')); + }); + + it('should append gcclGcsCmd if provided in options', () => { + const result = decorateHeaders(undefined, { + gcclGcsCmd: 'Storage.createBucket', + }); + assert.ok( + result.headers['x-goog-api-client'].endsWith( + ' gccl-gcs-cmd/Storage.createBucket' + ) + ); + }); + + it('should use custom packageJson if provided in options', () => { + const result = decorateHeaders(undefined, { + packageJson: {name: 'custom-pkg', version: '7.7.7'}, + }); + assert.ok( + result.headers['x-goog-api-client'].includes( + `gccl/7.7.7-${getModuleFormat()}` + ) + ); + }); + }); }); diff --git a/handwritten/storage/test/resumable-upload.ts b/handwritten/storage/test/resumable-upload.ts index 178fafecaa9d..e5cb5e875f8d 100644 --- a/handwritten/storage/test/resumable-upload.ts +++ b/handwritten/storage/test/resumable-upload.ts @@ -56,8 +56,8 @@ const RESUMABLE_INCOMPLETE_STATUS_CODE = 308; const CHUNK_SIZE_MULTIPLE = 2 ** 18; const queryPath = '/?userProject=user-project-id'; const X_GOOG_API_HEADER_REGEX = - /^gl-node\/(?[^W]+) gccl\/(?[^W]+) gccl-invocation-id\/(?[^W]+) gccl-gcs-cmd\/(?[^W]+)$/; -const USER_AGENT_REGEX = /^gcloud-node-storage\/(?[^W]+)$/; + /^gl-node\/(?\S+) gccl\/(?\S+) gccl-invocation-id\/(?\S+) gccl-gcs-cmd\/(?\S+)$/; +const USER_AGENT_REGEX = /^gcloud-node-storage\/(?\S+)$/; const CORRECT_CLIENT_CRC32C = 'Q2hlY2tzdW0h'; const INCORRECT_SERVER_CRC32C = 'Q2hlY2tzdVUa'; const CORRECT_CLIENT_MD5 = 'CorrectMD5Hash'; @@ -878,12 +878,140 @@ describe('resumable-upload', () => { delete metadataNoHeaders.contentLength; delete metadataNoHeaders.contentType; assert.deepStrictEqual(reqOpts.data, metadataNoHeaders); + assert(reqOpts.headers); + const apiClientHeader = reqOpts.headers['x-goog-api-client']; + const match = X_GOOG_API_HEADER_REGEX.exec(apiClientHeader as string); + assert.ok(match); + const invocationId = match.groups!.gcclInvocationId; + const idempotencyToken = + reqOpts.headers['x-goog-gcs-idempotency-token']; + assert.strictEqual(idempotencyToken, invocationId); done(); return {headers: {location: '/foo'}}; }; up.createURI(); }); + it('should respect user-provided x-goog-gcs-idempotency-token case-insensitively and align it with gccl-invocation-id in createURI', async () => { + const customToken = 'my-custom-resumable-token'; + up.customRequestOptions = { + headers: { + 'X-Goog-Gcs-Idempotency-Token': customToken, + }, + }; + + up.authClient.request = async (combinedReqOpts: GaxiosOptions) => { + assert(combinedReqOpts.headers); + const apiClientHeader = combinedReqOpts.headers['x-goog-api-client']; + const match = X_GOOG_API_HEADER_REGEX.exec(apiClientHeader as string); + assert.ok(match); + const invocationId = match.groups!.gcclInvocationId; + assert.strictEqual(invocationId, customToken); + + // Verify there is no duplicate x-goog-gcs-idempotency-token header + assert.strictEqual( + combinedReqOpts.headers['x-goog-gcs-idempotency-token'], + undefined + ); + assert.strictEqual( + combinedReqOpts.headers['X-Goog-Gcs-Idempotency-Token'], + customToken + ); + return {headers: {location: '/foo'}}; + }; + + await up.createURI(); + assert.strictEqual( + up.customRequestOptions.headers!['X-Goog-Gcs-Idempotency-Token'], + customToken + ); + }); + + it('should ignore invalid user-provided idempotency tokens and fallback to generating a UUID in createURI', async () => { + up.customRequestOptions = { + headers: { + 'X-Goog-Gcs-Idempotency-Token': '', // invalid empty string + }, + }; + + up.authClient.request = async (combinedReqOpts: GaxiosOptions) => { + assert(combinedReqOpts.headers); + const apiClientHeader = combinedReqOpts.headers['x-goog-api-client']; + const match = X_GOOG_API_HEADER_REGEX.exec(apiClientHeader as string); + assert.ok(match); + const invocationId = match.groups!.gcclInvocationId; + + // Verify a fallback token was generated and matches the invocation ID + const idempotencyToken = + combinedReqOpts.headers['x-goog-gcs-idempotency-token']; + assert.strictEqual(idempotencyToken, invocationId); + return {headers: {location: '/foo'}}; + }; + + await up.createURI(); + assert.strictEqual( + up.customRequestOptions.headers!['X-Goog-Gcs-Idempotency-Token'], + '' + ); + }); + + it('should ignore whitespace-only user-provided idempotency tokens and fallback to generating a UUID in createURI', async () => { + up.customRequestOptions = { + headers: { + 'X-Goog-Gcs-Idempotency-Token': ' ', // whitespace only + }, + }; + + up.authClient.request = async (combinedReqOpts: GaxiosOptions) => { + assert(combinedReqOpts.headers); + const apiClientHeader = combinedReqOpts.headers['x-goog-api-client']; + const match = X_GOOG_API_HEADER_REGEX.exec(apiClientHeader as string); + assert.ok(match); + const invocationId = match.groups!.gcclInvocationId; + + // Verify a fallback token was generated and matches the invocation ID + const idempotencyToken = + combinedReqOpts.headers['x-goog-gcs-idempotency-token']; + assert.strictEqual(idempotencyToken, invocationId); + return {headers: {location: '/foo'}}; + }; + + await up.createURI(); + assert.strictEqual( + up.customRequestOptions.headers!['X-Goog-Gcs-Idempotency-Token'], + ' ' + ); + }); + + it('should reuse the same x-goog-gcs-idempotency-token on retry of createURI', async () => { + let invocationCount = 0; + let token1 = ''; + let token2 = ''; + + up.makeRequest = async (reqOpts: GaxiosOptions) => { + invocationCount++; + assert(reqOpts.headers); + if (invocationCount === 1) { + token1 = reqOpts.headers['x-goog-gcs-idempotency-token'] as string; + const error = new GaxiosError( + 'Retriable error', + {} as GaxiosOptions, + {status: 500} as GaxiosResponse + ); + throw error; + } else if (invocationCount === 2) { + token2 = reqOpts.headers['x-goog-gcs-idempotency-token'] as string; + return {headers: {location: '/foo'}}; + } + return {headers: {location: '/foo'}}; + }; + + await up.createURI(); + assert.strictEqual(invocationCount, 2); + assert.ok(token1); + assert.strictEqual(token1, token2); + }); + it('should pass through the KMS key name', done => { const kmsKeyName = 'kms-key-name'; const up = upload({ @@ -1294,6 +1422,218 @@ describe('resumable-upload', () => { assert.equal(data.byteLength, CONTENT_LENGTH - NUM_BYTES_WRITTEN); }); }); + + describe('idempotency and header decoration', () => { + it('should attach x-goog-gcs-idempotency-token matching gccl-invocation-id in sendChunk', async () => { + let capturedHeaders: GaxiosOptions['headers']; + up.makeRequestStream = async (requestOptions: GaxiosOptions) => { + capturedHeaders = requestOptions.headers; + return null; + }; + up.writeBuffers = [Buffer.alloc(16)]; + await up.startUploading(); + + assert(capturedHeaders); + const apiClientHeader = capturedHeaders[ + 'x-goog-api-client' + ] as string; + const match = X_GOOG_API_HEADER_REGEX.exec(apiClientHeader); + assert.ok(match); + const invocationId = match.groups!.gcclInvocationId; + assert.strictEqual( + capturedHeaders['x-goog-gcs-idempotency-token'], + invocationId + ); + }); + + it('should respect user-provided x-goog-gcs-idempotency-token case-insensitively in sendChunk', async () => { + const customToken = 'my-custom-chunk-token'; + up.customRequestOptions = { + headers: { + 'X-Goog-Gcs-Idempotency-Token': customToken, + }, + }; + let capturedHeaders: GaxiosOptions['headers']; + up.makeRequestStream = async (requestOptions: GaxiosOptions) => { + capturedHeaders = requestOptions.headers; + return null; + }; + up.writeBuffers = [Buffer.alloc(16)]; + await up.startUploading(); + + assert(capturedHeaders); + const apiClientHeader = capturedHeaders[ + 'x-goog-api-client' + ] as string; + const match = X_GOOG_API_HEADER_REGEX.exec(apiClientHeader); + assert.ok(match); + const invocationId = match.groups!.gcclInvocationId; + assert.strictEqual(invocationId, customToken); + assert.strictEqual( + capturedHeaders['X-Goog-Gcs-Idempotency-Token'], + customToken + ); + assert.strictEqual( + capturedHeaders['x-goog-gcs-idempotency-token'], + undefined + ); + assert.strictEqual( + up.customRequestOptions.headers!['X-Goog-Gcs-Idempotency-Token'], + customToken + ); + }); + + it('should ignore invalid user-provided idempotency tokens and fallback to generating a UUID in sendChunk', async () => { + up.customRequestOptions = { + headers: { + 'X-Goog-Gcs-Idempotency-Token': '', + }, + }; + let capturedHeaders: GaxiosOptions['headers']; + up.makeRequestStream = async (requestOptions: GaxiosOptions) => { + capturedHeaders = requestOptions.headers; + return null; + }; + up.writeBuffers = [Buffer.alloc(16)]; + await up.startUploading(); + + assert(capturedHeaders); + const apiClientHeader = capturedHeaders[ + 'x-goog-api-client' + ] as string; + const match = X_GOOG_API_HEADER_REGEX.exec(apiClientHeader); + assert.ok(match); + const invocationId = match.groups!.gcclInvocationId; + assert.strictEqual( + capturedHeaders['x-goog-gcs-idempotency-token'], + invocationId + ); + assert.strictEqual( + up.customRequestOptions.headers!['X-Goog-Gcs-Idempotency-Token'], + '' + ); + }); + + it('should ignore whitespace-only user-provided idempotency tokens and fallback to generating a UUID in sendChunk', async () => { + up.customRequestOptions = { + headers: { + 'X-Goog-Gcs-Idempotency-Token': ' ', + }, + }; + let capturedHeaders: GaxiosOptions['headers']; + up.makeRequestStream = async (requestOptions: GaxiosOptions) => { + capturedHeaders = requestOptions.headers; + return null; + }; + up.writeBuffers = [Buffer.alloc(16)]; + await up.startUploading(); + + assert(capturedHeaders); + const apiClientHeader = capturedHeaders[ + 'x-goog-api-client' + ] as string; + const match = X_GOOG_API_HEADER_REGEX.exec(apiClientHeader); + assert.ok(match); + const invocationId = match.groups!.gcclInvocationId; + assert.strictEqual( + capturedHeaders['x-goog-gcs-idempotency-token'], + invocationId + ); + assert.strictEqual( + up.customRequestOptions.headers!['X-Goog-Gcs-Idempotency-Token'], + ' ' + ); + }); + + it('should reuse the same idempotency token on retry of the same chunk', async () => { + up.writeBuffers = [Buffer.alloc(16)]; + up.retryOptions.retryableErrorFn = () => true; + + let invocationCount = 0; + let token1 = ''; + let token2 = ''; + + up.makeRequestStream = async (requestOptions: GaxiosOptions) => { + invocationCount++; + const token = requestOptions.headers![ + 'x-goog-gcs-idempotency-token' + ] as string; + if (invocationCount === 1) { + token1 = token; + const err = new Error('Retriable error') as ApiError; + err.code = 500; + throw err; + } + token2 = token; + return null; + }; + + const retryPromise = new Promise(resolve => { + up.attemptDelayedRetry = async () => { + await up.startUploading(); + resolve(); + }; + }); + + await up.startUploading(); + await retryPromise; + + assert.strictEqual(invocationCount, 2); + assert.ok(token1); + assert.strictEqual(token1, token2); + }); + + it('should generate distinct idempotency tokens for Chunk 1 and Chunk 2 in multi-chunk uploads', done => { + up.chunkSize = 256; + up.contentLength = 512; + up.writeBuffers = [Buffer.alloc(256), Buffer.alloc(256)]; + up.upstreamEnded = false; + + const chunkTokens: string[] = []; + const chunkInvocationIds: string[] = []; + + up.makeRequestStream = async (requestOptions: GaxiosOptions) => { + const headers = requestOptions.headers!; + const token = headers['x-goog-gcs-idempotency-token'] as string; + const apiClientHeader = headers['x-goog-api-client'] as string; + const match = X_GOOG_API_HEADER_REGEX.exec(apiClientHeader); + assert.ok(match); + chunkTokens.push(token); + chunkInvocationIds.push(match.groups!.gcclInvocationId); + + if (chunkTokens.length === 1) { + return { + status: 308, + headers: {range: 'bytes 0-255'}, + data: {}, + } as unknown as GaxiosResponse; + } + return { + status: 200, + headers: {}, + data: {size: 512}, + } as unknown as GaxiosResponse; + }; + + up.on('metadata', () => { + try { + assert.strictEqual(chunkTokens.length, 2); + assert.strictEqual(chunkTokens[0], chunkInvocationIds[0]); + assert.strictEqual(chunkTokens[1], chunkInvocationIds[1]); + assert.notStrictEqual(chunkTokens[0], chunkTokens[1]); + assert.notStrictEqual( + chunkInvocationIds[0], + chunkInvocationIds[1] + ); + done(); + } catch (err) { + done(err); + } + }); + + up.startUploading(); + }); + }); }); describe('X-Goog-Hash header injection', () => { @@ -1825,6 +2165,153 @@ describe('resumable-upload', () => { await destroyCalled; }); + describe('#checkUploadStatus', () => { + beforeEach(() => { + up.uri = 'http://example.com/upload'; + }); + + it('should attach x-goog-gcs-idempotency-token matching gccl-invocation-id in checkUploadStatus', async () => { + let capturedHeaders: GaxiosOptions['headers']; + up.makeRequest = async (reqOpts: GaxiosOptions) => { + capturedHeaders = reqOpts.headers; + return {status: 308, headers: {range: 'bytes 0-10'}}; + }; + + await up.checkUploadStatus(); + + assert(capturedHeaders); + const apiClientHeader = capturedHeaders['x-goog-api-client'] as string; + const match = X_GOOG_API_HEADER_REGEX.exec(apiClientHeader); + assert.ok(match); + const invocationId = match.groups!.gcclInvocationId; + assert.strictEqual( + capturedHeaders['x-goog-gcs-idempotency-token'], + invocationId + ); + }); + + it('should respect user-provided x-goog-gcs-idempotency-token case-insensitively in checkUploadStatus', async () => { + const customToken = 'my-custom-status-token'; + up.customRequestOptions = { + headers: { + 'X-Goog-Gcs-Idempotency-Token': customToken, + }, + }; + let capturedHeaders: GaxiosOptions['headers']; + up.makeRequest = async (reqOpts: GaxiosOptions) => { + capturedHeaders = reqOpts.headers; + return {status: 308, headers: {range: 'bytes 0-10'}}; + }; + + await up.checkUploadStatus(); + + assert(capturedHeaders); + const apiClientHeader = capturedHeaders['x-goog-api-client'] as string; + const match = X_GOOG_API_HEADER_REGEX.exec(apiClientHeader); + assert.ok(match); + const invocationId = match.groups!.gcclInvocationId; + assert.strictEqual(invocationId, customToken); + assert.strictEqual( + capturedHeaders['X-Goog-Gcs-Idempotency-Token'], + customToken + ); + assert.strictEqual( + capturedHeaders['x-goog-gcs-idempotency-token'], + undefined + ); + assert.strictEqual( + up.customRequestOptions.headers!['X-Goog-Gcs-Idempotency-Token'], + customToken + ); + }); + + it('should ignore invalid user-provided idempotency tokens and fallback to generating a UUID in checkUploadStatus', async () => { + up.customRequestOptions = { + headers: { + 'X-Goog-Gcs-Idempotency-Token': '', + }, + }; + let capturedHeaders: GaxiosOptions['headers']; + up.makeRequest = async (reqOpts: GaxiosOptions) => { + capturedHeaders = reqOpts.headers; + return {status: 308, headers: {range: 'bytes 0-10'}}; + }; + + await up.checkUploadStatus(); + + assert(capturedHeaders); + const apiClientHeader = capturedHeaders['x-goog-api-client'] as string; + const match = X_GOOG_API_HEADER_REGEX.exec(apiClientHeader); + assert.ok(match); + const invocationId = match.groups!.gcclInvocationId; + assert.strictEqual( + capturedHeaders['x-goog-gcs-idempotency-token'], + invocationId + ); + assert.strictEqual( + up.customRequestOptions.headers!['X-Goog-Gcs-Idempotency-Token'], + '' + ); + }); + + it('should ignore whitespace-only user-provided idempotency tokens and fallback to generating a UUID in checkUploadStatus', async () => { + up.customRequestOptions = { + headers: { + 'X-Goog-Gcs-Idempotency-Token': ' ', + }, + }; + let capturedHeaders: GaxiosOptions['headers']; + up.makeRequest = async (reqOpts: GaxiosOptions) => { + capturedHeaders = reqOpts.headers; + return {status: 308, headers: {range: 'bytes 0-10'}}; + }; + + await up.checkUploadStatus(); + + assert(capturedHeaders); + const apiClientHeader = capturedHeaders['x-goog-api-client'] as string; + const match = X_GOOG_API_HEADER_REGEX.exec(apiClientHeader); + assert.ok(match); + const invocationId = match.groups!.gcclInvocationId; + assert.strictEqual( + capturedHeaders['x-goog-gcs-idempotency-token'], + invocationId + ); + assert.strictEqual( + up.customRequestOptions.headers!['X-Goog-Gcs-Idempotency-Token'], + ' ' + ); + }); + + it('should reuse the same x-goog-gcs-idempotency-token on retry of checkUploadStatus', async () => { + let invocationCount = 0; + let token1 = ''; + let token2 = ''; + + up.getRetryDelay = () => 1; + up.retryOptions.retryableErrorFn = () => true; + + up.makeRequest = async (reqOpts: GaxiosOptions) => { + invocationCount++; + const token = reqOpts.headers![ + 'x-goog-gcs-idempotency-token' + ] as string; + if (invocationCount === 1) { + token1 = token; + throw new Error('Transient error'); + } + token2 = token; + return {status: 308, headers: {range: 'bytes 0-10'}}; + }; + + await up.checkUploadStatus(); + + assert.strictEqual(invocationCount, 2); + assert.ok(token1); + assert.strictEqual(token1, token2); + }); + }); + describe('#getAndSetOffset', () => { const RANGE = 123456; const RESP = {status: 308, headers: {range: `range-${RANGE}`}};