Skip to content

Commit 08bca57

Browse files
max-ostapenkomax-ostapenko
committed
reservation setter
1 parent e62f22c commit 08bca57

9 files changed

Lines changed: 154 additions & 55 deletions

File tree

‎.github/linters/eslint.config.mjs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@ export default [
2121
operate: 'readonly',
2222
ctx: 'readonly',
2323
constants: 'readonly',
24-
reports: 'readonly'
24+
reports: 'readonly',
25+
reservations: 'readonly'
2526
}
2627
},
2728
rules: {

‎definitions/output/crawl/pages.js‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
assert('corrupted_technology_values')
33
.tags(['crawl_complete'])
44
.query(ctx => `
5+
${reservations.reservation_setter(ctx)}
6+
57
SELECT
68
/*
79
date,
@@ -82,7 +84,7 @@ publish('pages', {
8284
tags: ['crawl_complete'],
8385
dependOnDependencyAssertions: true
8486
}).preOps(ctx => `
85-
SET @@RESERVATION='${constants.reservation_id}';
87+
${reservations.reservation_setter(ctx)}
8688
8789
DELETE FROM ${ctx.self()}
8890
WHERE date = '${constants.currentMonth}' AND

‎definitions/output/crawl/parsed_css.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ publish('parsed_css', {
1919
},
2020
tags: ['crawl_complete']
2121
}).preOps(ctx => `
22-
SET @@RESERVATION='${constants.reservation_id}';
22+
${reservations.reservation_setter(ctx)}
2323
2424
DELETE FROM ${ctx.self()}
2525
WHERE date = '${constants.currentMonth}'

‎definitions/output/crawl/requests.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ publish('requests', {
3838
},
3939
tags: ['crawl_complete']
4040
}).preOps(ctx => `
41-
SET @@RESERVATION='${constants.reservation_id}';
41+
${reservations.reservation_setter(ctx)}
4242
4343
FOR client_var IN (SELECT * FROM UNNEST(['desktop', 'mobile']) AS value) DO
4444
FOR is_root_page_var IN (SELECT * FROM UNNEST([TRUE, FALSE]) AS value) DO

‎definitions/output/f1/pages_latest.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@ publish('pages_latest', {
77
clusterBy: ['client', 'is_root_page', 'rank', 'page']
88
},
99
tags: ['crawl_complete']
10-
}).preOps(`
11-
SET @@RESERVATION='${constants.reservation_id}';
10+
}).preOps(ctx => `
11+
${reservations.reservation_setter(ctx)}
1212
`).query(ctx => `
1313
SELECT
1414
date,

‎definitions/output/f1/requests_latest.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@ publish('requests_latest', {
77
clusterBy: ['client', 'is_root_page', 'rank', 'type']
88
},
99
tags: ['crawl_complete']
10-
}).preOps(`
11-
SET @@RESERVATION='${constants.reservation_id}';
10+
}).preOps(ctx => `
11+
${reservations.reservation_setter(ctx)}
1212
`).query(ctx => `
1313
SELECT
1414
date,

‎includes/constants.js‎

Lines changed: 1 addition & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -22,49 +22,6 @@ const [
2222
: ['', '']
2323
const bucket = 'httparchive'
2424
const storagePath = 'reports/'
25-
const reservation_id = 'projects/httparchive/locations/US/reservations/pipeline'
26-
27-
class DataformTemplateBuilder {
28-
/**
29-
* Create a Dataform SQL template that can be dynamically interpolated
30-
* @param {function} templateFn - A function that returns the SQL template string
31-
* @returns {function} A function that can be called with a context to resolve the template
32-
*/
33-
static create (templateFn) {
34-
return (ctx, params) => {
35-
// Custom replacer function to handle nested variables
36-
const resolveVariable = (path, scope) => {
37-
// Split the path into parts (handles nested objects like 'constants.devRankFilter')
38-
const parts = path.split('.')
39-
40-
// Traverse the provided scope (ctx or global) to find the value
41-
let value = scope
42-
for (const part of parts) {
43-
if (value === undefined || value === null) break
44-
value = value[part]
45-
}
46-
47-
// Convert value to appropriate string representation
48-
if (value === undefined || value === null) return ''
49-
if (typeof value === 'string') return `'${value}'`
50-
if (typeof value === 'number') return value.toString()
51-
if (typeof value === 'boolean') return value.toString()
52-
if (typeof value === 'function') return value.toString()
53-
54-
// For objects or arrays, use JSON.stringify
55-
return JSON.stringify(value)
56-
}
57-
58-
// Generate the template with the provided context and global context
59-
return templateFn(ctx, params).replace(/\${(.*?)}/g, (match, p1) => {
60-
const [scope, path] = p1.includes(':') ? p1.split(':') : ['params', p1.trim()]
61-
return scope === 'ctx'
62-
? resolveVariable(path.trim(), ctx)
63-
: resolveVariable(path.trim(), params)
64-
})
65-
}
66-
}
67-
}
6825

6926
module.exports = {
7027
today,
@@ -76,8 +33,6 @@ module.exports = {
7633
environment,
7734
devTABLESAMPLE,
7835
devRankFilter,
79-
DataformTemplateBuilder,
8036
bucket,
81-
storagePath,
82-
reservation_id
37+
storagePath
8338
}

‎includes/reports.js‎

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,44 @@
1-
const { DataformTemplateBuilder } = require('./constants')
1+
class DataformTemplateBuilder {
2+
/**
3+
* Create a Dataform SQL template that can be dynamically interpolated
4+
* @param {function} templateFn - A function that returns the SQL template string
5+
* @returns {function} A function that can be called with a context to resolve the template
6+
*/
7+
static create (templateFn) {
8+
return (ctx, params) => {
9+
// Custom replacer function to handle nested variables
10+
const resolveVariable = (path, scope) => {
11+
// Split the path into parts (handles nested objects like 'constants.devRankFilter')
12+
const parts = path.split('.')
13+
14+
// Traverse the provided scope (ctx or global) to find the value
15+
let value = scope
16+
for (const part of parts) {
17+
if (value === undefined || value === null) break
18+
value = value[part]
19+
}
20+
21+
// Convert value to appropriate string representation
22+
if (value === undefined || value === null) return ''
23+
if (typeof value === 'string') return `'${value}'`
24+
if (typeof value === 'number') return value.toString()
25+
if (typeof value === 'boolean') return value.toString()
26+
if (typeof value === 'function') return value.toString()
27+
28+
// For objects or arrays, use JSON.stringify
29+
return JSON.stringify(value)
30+
}
31+
32+
// Generate the template with the provided context and global context
33+
return templateFn(ctx, params).replace(/\${(.*?)}/g, (match, p1) => {
34+
const [scope, path] = p1.includes(':') ? p1.split(':') : ['params', p1.trim()]
35+
return scope === 'ctx'
36+
? resolveVariable(path.trim(), ctx)
37+
: resolveVariable(path.trim(), params)
38+
})
39+
}
40+
}
41+
}
242

343
const config = {
444
_metrics: {

‎includes/reservations.js‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
// Reservation references
2+
const RESERVATIONS = {
3+
HIGH_SLOTS: 'projects/httparchive/locations/US/reservations/pipeline',
4+
LOW_SLOTS: null,
5+
ON_DEMAND: 'none',
6+
DEFAULT: null
7+
}
8+
9+
// Configuration for actions (JSON format for dynamic injection)
10+
const RESERVATION_CONFIG = {
11+
'highSlots': [
12+
'httparchive.crawl.pages',
13+
'httparchive.crawl.requests',
14+
'httparchive.crawl.parsed_css',
15+
'httparchive.f1.pages_latest',
16+
'httparchive.f1.requests_latest'
17+
],
18+
'lowSlots': [],
19+
'onDemand': [
20+
'httparchive.dataform_assertions.corrupted_technology_values'
21+
]
22+
}
23+
24+
// Convert arrays to Sets for O(1) lookup performance
25+
const RESERVATION_SETS = {
26+
highSlots: new Set(RESERVATION_CONFIG.highSlots),
27+
lowSlots: new Set(RESERVATION_CONFIG.lowSlots),
28+
onDemand: new Set(RESERVATION_CONFIG.onDemand)
29+
}
30+
31+
/**
32+
* Determines the appropriate reservation for a given action name
33+
* @param {string} actionName - The fully qualified table name (with or without backticks)
34+
* @returns {string|null} The reservation identifier or null if no reservation assignment needed
35+
*/
36+
function getReservation(actionName) {
37+
if (!actionName || typeof actionName !== 'string') {
38+
return RESERVATIONS.DEFAULT
39+
}
40+
41+
// Strip backticks if present and normalize
42+
const normalizedName = actionName.replace(/`/g, '').trim()
43+
44+
if (RESERVATION_SETS.highSlots.has(normalizedName)) {
45+
return RESERVATIONS.HIGH_SLOTS
46+
} else if (RESERVATION_SETS.lowSlots.has(normalizedName)) {
47+
return RESERVATIONS.LOW_SLOTS
48+
} else if (RESERVATION_SETS.onDemand.has(normalizedName)) {
49+
return RESERVATIONS.ON_DEMAND
50+
} else {
51+
return RESERVATIONS.DEFAULT
52+
}
53+
}
54+
55+
/**
56+
* Extracts action name from Dataform context object
57+
* @param {Object} ctx - Dataform context object
58+
* @returns {string|null} The extracted action name or null if not found
59+
*/
60+
function extractActionName(ctx) {
61+
if (!ctx) {
62+
return null
63+
}
64+
65+
// Try primary method: ctx.self()
66+
if (typeof ctx.self === 'function') {
67+
const selfName = ctx.self()
68+
if (selfName) {
69+
return selfName
70+
}
71+
}
72+
73+
// Fallback: construct from proto target
74+
if (ctx?.operation?.proto?.target) {
75+
const operationTarget = ctx?.operation?.proto?.target
76+
const parts = []
77+
78+
if (operationTarget.database) parts.push(operationTarget.database)
79+
if (operationTarget.schema) parts.push(operationTarget.schema)
80+
if (operationTarget.name) parts.push(operationTarget.name)
81+
82+
return parts.length > 0 ? parts.join('.') : null
83+
}
84+
85+
return null
86+
}
87+
88+
/**
89+
* Generates the reservation SQL statement for a given Dataform context
90+
* @param {Object} ctx - Dataform context object with self() method and/or proto.target
91+
* @returns {string} The SQL statement to set reservation or empty string
92+
*/
93+
function reservation_setter(ctx) {
94+
const actionName = extractActionName(ctx)
95+
const reservation = getReservation(actionName)
96+
return reservation ? `SET @@RESERVATION='${reservation}';` : ''
97+
}
98+
99+
module.exports = {
100+
reservation_setter
101+
}

0 commit comments

Comments
 (0)