From 505efa0bd911204d0a72f75ec6c70efed309566f Mon Sep 17 00:00:00 2001 From: banteg <4562643+banteg@users.noreply.github.com> Date: Sat, 11 Jul 2026 23:27:53 +0400 Subject: [PATCH 1/2] feat(solidity): recover historical storage layouts --- packages/compilers-types/src/SolidityTypes.ts | 42 +- packages/compilers/src/index.ts | 1 + .../src/lib/solidityStorageLayout.ts | 974 ++++++++++++++++++ .../test/solidityStorageLayout.spec.ts | 631 ++++++++++++ .../src/Compilation/CompilationTypes.ts | 7 + .../src/Compilation/SolidityCompilation.ts | 45 +- .../Compilation/SolidityCompilation.spec.ts | 193 +++- packages/lib-sourcify/test/utils.ts | 31 + .../private/stateless/customReplaceMethods.ts | 2 + .../stateless/private.stateless.handlers.ts | 3 +- .../stateless/private.stateless.paths.yaml | 10 + .../stateless/solidityStorageLayoutReplace.ts | 166 +++ .../services/compiler/local/SolcLocal.ts | 21 +- .../solidity-storage-layout.stateless.spec.ts | 207 ++++ .../unit/solidityStorageLayoutReplace.spec.ts | 202 ++++ 15 files changed, 2512 insertions(+), 23 deletions(-) create mode 100644 packages/compilers/src/lib/solidityStorageLayout.ts create mode 100644 packages/compilers/test/solidityStorageLayout.spec.ts create mode 100644 services/server/src/server/apiv1/verification/private/stateless/solidityStorageLayoutReplace.ts create mode 100644 services/server/test/integration/apiv1/verification-handlers/solidity-storage-layout.stateless.spec.ts create mode 100644 services/server/test/unit/solidityStorageLayoutReplace.spec.ts diff --git a/packages/compilers-types/src/SolidityTypes.ts b/packages/compilers-types/src/SolidityTypes.ts index b615d6c0d0..94deeb9600 100644 --- a/packages/compilers-types/src/SolidityTypes.ts +++ b/packages/compilers-types/src/SolidityTypes.ts @@ -133,32 +133,36 @@ interface SolidityOutputEvmDeployedBytecode extends SolidityOutputEvmBytecode { export interface SolidityOutputSource { // In older solidity versions, solcjs returns the id as a string id: number | string; - ast: any; - legacyAST: any; + ast?: any; + legacyAST?: any; } export interface SolidityOutputSources { [globalName: string]: SolidityOutputSource; } +export interface StorageLayoutItem { + astId: number; + contract: string; + label: string; + offset: number; + slot: string; + type: string; +} + +export interface StorageLayoutType { + encoding: string; + label: string; + numberOfBytes: string; + base?: string; + key?: string; + value?: string; + members?: StorageLayoutItem[]; +} + export interface StorageLayout { - storage: Array<{ - astId: number; - contract: string; - label: string; - offset: number; - slot: string; - type: string; - }>; - types: { - [index: string]: { - encoding: string; - label: string; - numberOfBytes: string; - key?: string; - value?: string; - }; - }; + storage: StorageLayoutItem[]; + types: Record | null; } export type TransientStorageLayout = StorageLayout; diff --git a/packages/compilers/src/index.ts b/packages/compilers/src/index.ts index c1ad311269..41f72fb2e5 100644 --- a/packages/compilers/src/index.ts +++ b/packages/compilers/src/index.ts @@ -5,5 +5,6 @@ export const setCompilersLoggerLevel = setLevel; export type ICompilersLogger = ILogger; export * from './lib/solidityCompiler'; +export * from './lib/solidityStorageLayout'; export * from './lib/vyperCompiler'; export * from './lib/feCompiler'; diff --git a/packages/compilers/src/lib/solidityStorageLayout.ts b/packages/compilers/src/lib/solidityStorageLayout.ts new file mode 100644 index 0000000000..3fc0cb64d1 --- /dev/null +++ b/packages/compilers/src/lib/solidityStorageLayout.ts @@ -0,0 +1,974 @@ +import semver from 'semver'; +import type { + SolidityJsonInput, + SolidityOutput, + StorageLayout, + StorageLayoutItem, + StorageLayoutType, +} from '@ethereum-sourcify/compilers-types'; + +const MINIMUM_VERSION = '0.4.0'; +const NATIVE_STORAGE_LAYOUT_VERSION = '0.5.13'; +const FIXED_POINT_WIDTH_CHANGE_VERSION = '0.4.14-nightly.2017.7.20'; +const WORD_BYTES = toBigInt(32); +const ZERO = toBigInt(0); +const ONE = toBigInt(1); + +type JsonObject = Record; + +interface CompilationTarget { + path: string; + name: string; +} + +interface StoragePosition { + slot: bigint; + offset: number; +} + +interface HistoricalType { + key: string; + label: string; + kind: + | 'value' + | 'bytes' + | 'dynamic_array' + | 'fixed_array' + | 'mapping' + | 'struct'; + astId: number; + storageBytes: bigint; + base?: HistoricalType; + keyType?: HistoricalType; + valueType?: HistoricalType; + length?: bigint; + definition?: JsonObject; + contextContractId?: number; +} + +interface TypeDefinition { + node: JsonObject; + kind: 'ContractDefinition' | 'EnumDefinition' | 'StructDefinition'; + name: string; + contractId?: number; +} + +function normalizeCompilerVersion(version: string): string { + return version.trim().replace(/^v/, '').replace('-ci.', '-nightly.'); +} + +/** + * Returns true for the standalone historical reconstruction range. Persisting + * layouts before 0.4.7 requires additional source-identity safeguards. + */ +export function supportsHistoricalSolidityStorageLayout( + version: string, +): boolean { + const normalized = semver.valid(normalizeCompilerVersion(version)); + return Boolean( + normalized && + semver.gte(normalized, `${MINIMUM_VERSION}-0`) && + semver.lt(normalized, NATIVE_STORAGE_LAYOUT_VERSION), + ); +} + +/** + * Reconstructs solc's storageLayout artifact from the analyzed historical AST. + * + * The implementation mirrors the storage allocation routines present in solc + * 0.4.x and 0.5.x. It intentionally throws on unresolved AST constructs so a + * caller can fail closed instead of persisting a partial layout. + */ +export function generateHistoricalSolidityStorageLayout( + version: string, + input: SolidityJsonInput, + output: SolidityOutput, + target: CompilationTarget, +): StorageLayout | undefined { + if (!supportsHistoricalSolidityStorageLayout(version)) return undefined; + + return new HistoricalStorageLayoutBuilder( + normalizeCompilerVersion(version), + input, + output, + target, + ).generate(); +} + +class HistoricalStorageLayoutBuilder { + private readonly nodePaths = new Map(); + private readonly nodeContracts = new Map(); + private readonly definitions: TypeDefinition[] = []; + private readonly definitionsById = new Map(); + private readonly contractsById = new Map(); + private readonly types = new Map(); + private readonly generatedTypes: Record = {}; + private readonly fullyQualifiedName: string; + + constructor( + private readonly version: string, + private readonly input: SolidityJsonInput, + private readonly output: SolidityOutput, + private readonly target: CompilationTarget, + ) { + this.fullyQualifiedName = `${target.path}:${target.name}`; + this.indexAsts(); + } + + generate(): StorageLayout { + const contract = this.findTargetContract(); + const linearizedBaseContracts = this.linearizedBaseContracts(contract); + const variables: JsonObject[] = []; + + for (const contractId of [...linearizedBaseContracts].reverse()) { + const base = this.contractsById.get(contractId); + if (!base) { + throw new Error( + `Cannot resolve linearized base contract AST id ${contractId}`, + ); + } + for (const variable of directChildren(base).filter( + (node) => nodeKind(node) === 'VariableDeclaration', + )) { + if (!this.isConstant(variable)) variables.push(variable); + } + } + + const variableTypes = variables.map((variable) => + this.typeFromVariable(variable), + ); + const { positions } = this.computeOffsets(variableTypes); + const storage = variables.map((variable, index) => { + const type = variableTypes[index]; + const position = positions[index]; + this.generateType(type); + return this.storageItem(variable, type, position); + }); + + return { + storage, + types: storage.length === 0 ? null : this.generatedTypes, + }; + } + + private indexAsts() { + if (!this.output.sources) { + throw new Error('Historical compiler output does not contain sources'); + } + + for (const [path, source] of Object.entries(this.output.sources)) { + const ast = source.ast ?? source.legacyAST; + if (!ast) continue; + this.visit(ast, path, undefined); + } + } + + private visit(node: unknown, path: string, contractId?: number) { + if (!isObject(node)) return; + + const kind = nodeKind(node); + let currentContractId = contractId; + if (kind === 'ContractDefinition') { + currentContractId = requiredNodeId(node); + this.contractsById.set(currentContractId, node); + } + + if (typeof node.id === 'number' || typeof node.id === 'string') { + this.nodePaths.set(node, path); + if (currentContractId !== undefined) { + this.nodeContracts.set(node, currentContractId); + } + } + + if ( + kind === 'ContractDefinition' || + kind === 'EnumDefinition' || + kind === 'StructDefinition' + ) { + const definition: TypeDefinition = { + node, + kind, + name: requiredNodeName(node), + ...(kind !== 'ContractDefinition' && currentContractId !== undefined + ? { contractId: currentContractId } + : {}), + }; + this.definitions.push(definition); + this.definitionsById.set(requiredNodeId(node), definition); + } + + for (const child of astChildren(node)) { + this.visit(child, path, currentContractId); + } + } + + private findTargetContract(): JsonObject { + const candidates = [...this.contractsById.values()].filter( + (contract) => requiredNodeName(contract) === this.target.name, + ); + const exact = candidates.find( + (contract) => this.nodePaths.get(contract) === this.target.path, + ); + if (exact) return exact; + throw new Error( + `Cannot resolve compilation target ${this.fullyQualifiedName}`, + ); + } + + private linearizedBaseContracts(contract: JsonObject): number[] { + const value = + contract.linearizedBaseContracts ?? + contract.attributes?.linearizedBaseContracts; + if (!Array.isArray(value)) { + throw new Error( + `Contract ${requiredNodeName(contract)} has no linearized base contracts`, + ); + } + return value.map(requiredInteger); + } + + private isConstant(variable: JsonObject): boolean { + const constant = variable.constant ?? variable.attributes?.constant; + if (typeof constant === 'boolean') return constant; + + const path = this.nodePaths.get(variable); + const src = parseSrc(variable.src); + const source = path ? this.input.sources[path]?.content : undefined; + if (!source || !src) { + throw new Error( + `Cannot determine whether ${requiredNodeName(variable)} is constant`, + ); + } + const declaration = sliceUtf8Bytes(source, src.start, src.length); + return /\bconstant\b/.test(stripCommentsAndStrings(declaration)); + } + + private typeFromVariable(variable: JsonObject): HistoricalType { + const typeName = + variable.typeName ?? + directChildren(variable).find((child) => isTypeName(child)); + if (!typeName) { + throw new Error( + `Variable ${requiredNodeName(variable)} has no type-name AST`, + ); + } + const contextContractId = this.nodeContracts.get(variable); + return this.typeFromNode( + typeName, + contextContractId, + resolvedTypeString(variable), + ); + } + + private typeFromNode( + node: JsonObject, + contextContractId?: number, + resolvedType?: string, + ): HistoricalType { + const kind = nodeKind(node); + + if (kind === 'ElementaryTypeName') { + return this.elementaryType( + node, + resolvedTypeString(node) ?? resolvedType, + ); + } + if (kind === 'ArrayTypeName') { + const children = directChildren(node); + const baseNode = + node.baseType ?? children.find((child) => isTypeName(child)); + if (!baseNode) throw new Error('Array type has no base type'); + const arraySemanticType = resolvedTypeString(node) ?? resolvedType; + const semanticArray = arraySemanticType + ? peelArrayDimension(arraySemanticType) + : undefined; + const base = this.typeFromNode( + baseNode, + contextContractId, + semanticArray?.base, + ); + const lengthNode = + node.length ?? + children.find((child) => child !== baseNode && isObject(child)); + if (lengthNode === null || lengthNode === undefined) { + return this.intern({ + key: `t_array(${base.key})dyn_storage`, + label: `${base.label}[]`, + kind: 'dynamic_array', + astId: requiredNodeId(node), + storageBytes: WORD_BYTES, + base, + contextContractId, + }); + } + const length = constantInteger(lengthNode, semanticArray?.length); + return this.intern({ + key: `t_array(${base.key})${length.toString()}_storage`, + label: `${base.label}[${length.toString()}]`, + kind: 'fixed_array', + astId: requiredNodeId(node), + storageBytes: WORD_BYTES, + base, + length, + contextContractId, + }); + } + if (kind === 'Mapping') { + const children = directChildren(node).filter(isTypeName); + const keyNode = node.keyType ?? children[0]; + const valueNode = node.valueType ?? children[1]; + if (!keyNode || !valueNode) throw new Error('Mapping type is incomplete'); + const semanticMapping = splitMappingType( + resolvedTypeString(node) ?? resolvedType, + ); + const keyType = this.typeFromNode( + keyNode, + contextContractId, + semanticMapping?.key, + ); + const valueType = this.typeFromNode( + valueNode, + contextContractId, + semanticMapping?.value, + ); + return this.intern({ + key: `t_mapping(${keyType.key},${valueType.key})`, + label: `mapping(${keyType.label} => ${valueType.label})`, + kind: 'mapping', + astId: requiredNodeId(node), + storageBytes: WORD_BYTES, + keyType, + valueType, + contextContractId, + }); + } + if (kind === 'UserDefinedTypeName') { + return this.userDefinedType( + node, + contextContractId, + resolvedTypeString(node) ?? resolvedType, + ); + } + if (kind === 'FunctionTypeName') { + const visibility = node.visibility ?? node.attributes?.visibility; + const external = visibility === 'external'; + const semanticType = resolvedTypeString(node) ?? resolvedType; + return this.intern({ + key: this.functionTypeIdentifier(node, external), + label: semanticType ?? `function ${external ? 'external' : 'internal'}`, + kind: 'value', + astId: requiredNodeId(node), + storageBytes: toBigInt(external ? 24 : 8), + contextContractId, + }); + } + + throw new Error(`Unsupported historical Solidity type AST ${kind}`); + } + + private functionTypeIdentifier(node: JsonObject, external: boolean): string { + const parameterLists = directChildren(node).filter( + (child) => nodeKind(child) === 'ParameterList', + ); + const parameters = node.parameterTypes ?? parameterLists[0]; + const returns = node.returnParameterTypes ?? parameterLists[1]; + if (!parameters || !returns) { + throw new Error('Legacy function type has incomplete parameter lists'); + } + + const parameterTypes = parameterListVariables(parameters).map((parameter) => + this.functionParameterTypeIdentifier(parameter), + ); + const returnTypes = parameterListVariables(returns).map((parameter) => + this.functionParameterTypeIdentifier(parameter), + ); + const rawMutability = + node.stateMutability ?? node.attributes?.stateMutability; + const mutability = + rawMutability === 'pure' || + rawMutability === 'view' || + rawMutability === 'payable' + ? rawMutability + : node.payable === true || node.attributes?.payable === true + ? 'payable' + : node.constant === true || + node.attributes?.constant === true || + node.isDeclaredConst === true || + node.attributes?.isDeclaredConst === true + ? 'view' + : 'nonpayable'; + + return `t_function_${external ? 'external' : 'internal'}_${mutability}(${parameterTypes.join(',')})returns(${returnTypes.join(',')})`; + } + + private functionParameterTypeIdentifier(parameter: JsonObject): string { + const type = this.typeFromVariable(parameter); + const semanticType = resolvedTypeString(parameter); + const astLocation = + parameter.storageLocation ?? parameter.attributes?.storageLocation; + const location = + astLocation === 'storage' || + astLocation === 'memory' || + astLocation === 'calldata' + ? astLocation + : semanticType?.match(/\b(storage|memory|calldata)\b/)?.[1]; + if (location && type.kind !== 'mapping') { + return this.typeIdentifierAtLocation(type, location); + } + return type.key; + } + + private typeIdentifierAtLocation( + type: HistoricalType, + location: string, + ): string { + if (type.kind === 'dynamic_array' || type.kind === 'fixed_array') { + if (!type.base) throw new Error(`Array ${type.label} has no base type`); + const base = this.typeIdentifierAtLocation(type.base, location); + const length = + type.kind === 'dynamic_array' + ? 'dyn' + : requiredBigInt(type.length, `Array ${type.label} has no length`); + return `t_array(${base})${length}_${location}_ptr`; + } + if (type.kind === 'bytes' || type.kind === 'struct') { + if (!type.key.endsWith('_storage')) { + throw new Error(`Reference type ${type.label} has no storage suffix`); + } + return `${type.key.slice(0, -'_storage'.length)}_${location}_ptr`; + } + return type.key; + } + + private elementaryType( + node: JsonObject, + resolvedType?: string, + ): HistoricalType { + let name = String( + (node.nodeType ? node.name : node.attributes?.name) ?? resolvedType ?? '', + ); + if ( + resolvedType && + (name === 'var' || name === 'fixed' || name === 'ufixed') + ) { + name = resolvedType; + } + name = name.replace(/\s+(storage|memory|calldata)(\s+(ref|pointer))?$/, ''); + name = name.replace(/\s+payable$/, ''); + + if (name === 'uint') name = 'uint256'; + if (name === 'int') name = 'int256'; + if (name === 'byte') name = 'bytes1'; + if (name === 'address') { + const payable = /\baddress payable\b/.test(resolvedType ?? ''); + return this.valueType( + payable ? 't_address_payable' : 't_address', + payable ? 'address payable' : 'address', + 20, + node, + ); + } + if (name === 'bool') return this.valueType('t_bool', 'bool', 1, node); + if (name === 'bytes' || name === 'string') { + return this.intern({ + key: `t_${name}_storage`, + label: name, + kind: 'bytes', + astId: requiredNodeId(node), + storageBytes: WORD_BYTES, + }); + } + + const integer = /^(u?int)(\d+)$/.exec(name); + if (integer) { + return this.valueType(`t_${name}`, name, Number(integer[2]) / 8, node); + } + const fixedBytes = /^bytes(\d+)$/.exec(name); + if (fixedBytes) { + return this.valueType(`t_${name}`, name, Number(fixedBytes[1]), node); + } + if (name === 'fixed' || name === 'ufixed') { + const decimals = semver.lt(this.version, FIXED_POINT_WIDTH_CHANGE_VERSION) + ? 128 + : semver.lt(this.version, '0.4.22') + ? 19 + : 18; + name = `${name}128x${decimals}`; + } + const fixedPoint = /^(u?fixed)(\d+)(?:x(\d+))?$/.exec(name); + if (fixedPoint) { + const bits = semver.lt(this.version, FIXED_POINT_WIDTH_CHANGE_VERSION) + ? Number(fixedPoint[2]) + Number(fixedPoint[3] ?? 0) + : Number(fixedPoint[2]); + return this.valueType(`t_${name}`, name, bits / 8, node); + } + + throw new Error(`Unsupported elementary storage type ${name}`); + } + + private valueType( + key: string, + label: string, + bytes: number, + node: JsonObject, + ): HistoricalType { + if (!Number.isInteger(bytes) || bytes < 1 || bytes > 32) { + throw new Error(`Invalid storage width ${bytes} for ${label}`); + } + return this.intern({ + key, + label, + kind: 'value', + astId: requiredNodeId(node), + storageBytes: toBigInt(bytes), + }); + } + + private userDefinedType( + node: JsonObject, + contextContractId?: number, + resolvedType?: string, + ): HistoricalType { + const referencedDeclaration = + node.referencedDeclaration ?? node.attributes?.referencedDeclaration; + let definition = + referencedDeclaration !== undefined + ? this.definitionsById.get(requiredInteger(referencedDeclaration)) + : undefined; + if (!definition) { + definition = this.resolveLegacyDefinition( + requiredNodeName(node), + contextContractId, + resolvedType, + ); + } + if (!definition) { + throw new Error( + `Cannot resolve user-defined type ${requiredNodeName(node)}`, + ); + } + + const id = requiredNodeId(definition.node); + if (definition.kind === 'ContractDefinition') { + const contractKind = + definition.node.contractKind === 'library' || + definition.node.attributes?.contractKind === 'library' || + definition.node.attributes?.isLibrary === true + ? 'library' + : 'contract'; + return this.intern({ + key: `t_contract(${definition.name})${id}`, + label: `${contractKind} ${definition.name}`, + kind: 'value', + astId: id, + storageBytes: toBigInt(20), + definition: definition.node, + contextContractId, + }); + } + + const owner = + definition.contractId !== undefined + ? this.contractsById.get(definition.contractId) + : undefined; + const canonicalName = owner + ? `${requiredNodeName(owner)}.${definition.name}` + : definition.name; + if (definition.kind === 'EnumDefinition') { + const memberCount = directChildren(definition.node).filter( + (child) => nodeKind(child) === 'EnumValue', + ).length; + const bytes = bytesRequired(Math.max(0, memberCount - 1)); + return this.intern({ + key: `t_enum(${definition.name})${id}`, + label: `enum ${canonicalName}`, + kind: 'value', + astId: id, + storageBytes: toBigInt(Math.max(1, bytes)), + definition: definition.node, + contextContractId: definition.contractId, + }); + } + + return this.intern({ + key: `t_struct(${definition.name})${id}_storage`, + label: `struct ${canonicalName}`, + kind: 'struct', + astId: id, + storageBytes: WORD_BYTES, + definition: definition.node, + contextContractId: definition.contractId, + }); + } + + private resolveLegacyDefinition( + rawName: string, + contextContractId?: number, + resolvedType?: string, + ): TypeDefinition | undefined { + const hint = resolvedType?.trim().split(/\s+/)[0]; + const expectedKind = + hint === 'struct' + ? 'StructDefinition' + : hint === 'enum' + ? 'EnumDefinition' + : hint === 'contract' || hint === 'library' + ? 'ContractDefinition' + : undefined; + const qualified = (resolvedType ?? rawName) + .replace(/^(struct|enum|contract|library)\s+/, '') + .replace(/\s+(storage|memory|calldata).*$/, ''); + const parts = qualified.split('.'); + const name = parts[parts.length - 1]; + let candidates = this.definitions.filter( + (definition) => + definition.name === name && + (!expectedKind || definition.kind === expectedKind), + ); + + const ownerName = parts.length > 1 ? parts[parts.length - 2] : undefined; + if (ownerName) { + candidates = candidates.filter((definition) => { + if (definition.contractId === undefined) return false; + const owner = this.contractsById.get(definition.contractId); + return owner && requiredNodeName(owner) === ownerName; + }); + } else if (contextContractId !== undefined) { + const context = this.contractsById.get(contextContractId); + const scope = context ? this.linearizedBaseContracts(context) : []; + for (const contractId of scope) { + const scoped = candidates.find( + (definition) => definition.contractId === contractId, + ); + if (scoped) return scoped; + } + } + + return candidates.length === 1 ? candidates[0] : undefined; + } + + private storageSize( + type: HistoricalType, + visiting = new Set(), + ): bigint { + if ( + type.kind === 'value' || + type.kind === 'bytes' || + type.kind === 'dynamic_array' || + type.kind === 'mapping' + ) { + return ONE; + } + if (type.kind === 'fixed_array') { + if (!type.base || type.length === undefined) { + throw new Error(`Incomplete fixed-array type ${type.key}`); + } + const baseBytes = type.base.storageBytes; + let size: bigint; + if (baseBytes < WORD_BYTES) { + const itemsPerSlot = WORD_BYTES / baseBytes; + size = ceilDiv(type.length, itemsPerSlot); + } else { + size = type.length * this.storageSize(type.base, visiting); + } + return size > ZERO ? size : ONE; + } + if (!type.definition) throw new Error(`Incomplete struct type ${type.key}`); + if (visiting.has(type.key)) { + throw new Error(`Illegal direct recursive struct ${type.label}`); + } + const nextVisiting = new Set(visiting).add(type.key); + const members = this.structMembers(type); + return this.computeOffsets( + members.map((member) => member.type), + nextVisiting, + ).size; + } + + private computeOffsets( + types: HistoricalType[], + visiting = new Set(), + ): { positions: StoragePosition[]; size: bigint } { + let slot = ZERO; + let byteOffset = ZERO; + const positions: StoragePosition[] = []; + + for (const type of types) { + if (byteOffset + type.storageBytes > WORD_BYTES) { + slot += ONE; + byteOffset = ZERO; + } + positions.push({ slot, offset: Number(byteOffset) }); + const size = this.storageSize(type, visiting); + if (size === ONE && byteOffset + type.storageBytes <= WORD_BYTES) { + byteOffset += type.storageBytes; + } else { + slot += size; + byteOffset = ZERO; + } + } + if (byteOffset > ZERO) slot += ONE; + return { positions, size: slot }; + } + + private structMembers(type: HistoricalType) { + if (!type.definition) throw new Error(`Struct ${type.label} has no AST`); + return directChildren(type.definition) + .filter((node) => nodeKind(node) === 'VariableDeclaration') + .map((node) => ({ node, type: this.typeFromVariable(node) })); + } + + private generateType(type: HistoricalType) { + if (this.generatedTypes[type.key]) return; + + const typeInfo: StorageLayoutType = { + encoding: + type.kind === 'mapping' + ? 'mapping' + : type.kind === 'dynamic_array' + ? 'dynamic_array' + : type.kind === 'bytes' + ? 'bytes' + : 'inplace', + label: type.label, + numberOfBytes: (type.storageBytes * this.storageSize(type)).toString(), + }; + this.generatedTypes[type.key] = typeInfo; + + if (type.kind === 'fixed_array' || type.kind === 'dynamic_array') { + if (!type.base) throw new Error(`Array ${type.label} has no base type`); + typeInfo.base = type.base.key; + this.generateType(type.base); + } else if (type.kind === 'mapping') { + if (!type.keyType || !type.valueType) { + throw new Error(`Mapping ${type.label} is incomplete`); + } + typeInfo.key = type.keyType.key; + typeInfo.value = type.valueType.key; + this.generateType(type.keyType); + this.generateType(type.valueType); + } else if (type.kind === 'struct') { + const members = this.structMembers(type); + const { positions } = this.computeOffsets( + members.map((member) => member.type), + ); + typeInfo.members = members.map((member, index) => { + this.generateType(member.type); + return this.storageItem(member.node, member.type, positions[index]); + }); + } + } + + private storageItem( + node: JsonObject, + type: HistoricalType, + position: StoragePosition, + ): StorageLayoutItem { + return { + astId: requiredNodeId(node), + contract: this.fullyQualifiedName, + label: requiredNodeName(node), + offset: position.offset, + slot: position.slot.toString(), + type: type.key, + }; + } + + private intern(type: HistoricalType): HistoricalType { + const existing = this.types.get(type.key); + if (existing) return existing; + this.types.set(type.key, type); + return type; + } +} + +function nodeKind(node: JsonObject): string { + return String(node.nodeType ?? node.name ?? ''); +} + +function requiredNodeName(node: JsonObject): string { + const name = node.nodeType ? node.name : node.attributes?.name; + if (typeof name !== 'string' || name.length === 0) { + throw new Error(`AST ${nodeKind(node)} node has no name`); + } + return name; +} + +function requiredNodeId(node: JsonObject): number { + return requiredInteger(node.id); +} + +function requiredInteger(value: unknown): number { + const number = typeof value === 'string' ? Number(value) : value; + if (typeof number !== 'number' || !Number.isInteger(number)) { + throw new Error(`Expected an integer, got ${String(value)}`); + } + return number; +} + +function requiredBigInt(value: bigint | undefined, message: string): bigint { + if (value === undefined) throw new Error(message); + return value; +} + +function directChildren(node: JsonObject): JsonObject[] { + if (Array.isArray(node.nodes)) return node.nodes.filter(isObject); + if (Array.isArray(node.members)) return node.members.filter(isObject); + if (Array.isArray(node.parameters)) return node.parameters.filter(isObject); + if (Array.isArray(node.children)) return node.children.filter(isObject); + return []; +} + +function parameterListVariables(node: JsonObject): JsonObject[] { + return directChildren(node).filter( + (child) => nodeKind(child) === 'VariableDeclaration', + ); +} + +function astChildren(node: JsonObject): JsonObject[] { + const children: JsonObject[] = []; + for (const [key, value] of Object.entries(node)) { + if (key === 'attributes' || key === 'typeDescriptions') continue; + if (Array.isArray(value)) children.push(...value.filter(isAstNode)); + else if (isAstNode(value)) children.push(value); + } + return children; +} + +function isAstNode(value: unknown): value is JsonObject { + return ( + isObject(value) && + (typeof value.nodeType === 'string' || + (typeof value.name === 'string' && value.id !== undefined)) + ); +} + +function isObject(value: unknown): value is JsonObject { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function isTypeName(node: JsonObject): boolean { + return [ + 'ArrayTypeName', + 'ElementaryTypeName', + 'FunctionTypeName', + 'Mapping', + 'UserDefinedTypeName', + ].includes(nodeKind(node)); +} + +function resolvedTypeString(node: JsonObject): string | undefined { + const value = node.typeDescriptions?.typeString ?? node.attributes?.type; + return typeof value === 'string' ? value : undefined; +} + +function constantInteger(node: JsonObject, fallback?: string): bigint { + const candidates = [ + node.value, + node.attributes?.value, + node.typeDescriptions?.typeString, + node.attributes?.type, + fallback, + ]; + for (const candidate of candidates) { + if (typeof candidate !== 'string' && typeof candidate !== 'number') { + continue; + } + const match = String(candidate).match(/^(?:int_const\s+)?(\d+)$/); + if (match) return toBigInt(match[1]); + } + throw new Error('Cannot resolve fixed-array length from historical AST'); +} + +function peelArrayDimension( + type: string, +): { base: string; length: string } | undefined { + const normalized = type + .trim() + .replace(/\s+(storage|memory|calldata)\s+(ref|pointer)$/, ''); + const match = + /^(.*)\[(\d*)\](?:\s+(storage|memory|calldata)\s+(ref|pointer))?$/.exec( + normalized, + ); + if (!match) return undefined; + return { base: match[1].trim(), length: match[2] }; +} + +function splitMappingType( + type: string | undefined, +): { key: string; value: string } | undefined { + if (!type) return undefined; + const normalized = type + .trim() + .replace(/\s+(storage|memory|calldata)(\s+(ref|pointer))?$/, ''); + if (!normalized.startsWith('mapping(') || !normalized.endsWith(')')) { + return undefined; + } + const contents = normalized.slice('mapping('.length, -1); + let depth = 0; + for (let index = 0; index < contents.length - 1; index += 1) { + const character = contents[index]; + if (character === '(' || character === '[') depth += 1; + else if (character === ')' || character === ']') depth -= 1; + else if (depth === 0 && character === '=' && contents[index + 1] === '>') { + return { + key: contents.slice(0, index).trim(), + value: contents.slice(index + 2).trim(), + }; + } + } + return undefined; +} + +function bytesRequired(value: number): number { + let remaining = value; + let bytes = 0; + do { + bytes += 1; + remaining = Math.floor(remaining / 256); + } while (remaining > 0); + return bytes; +} + +function ceilDiv(value: bigint, divisor: bigint): bigint { + return (value + divisor - ONE) / divisor; +} + +function toBigInt(value: string | number): bigint { + const bigint = ( + globalThis as unknown as { + BigInt: (input: string | number) => bigint; + } + ).BigInt; + return bigint(value); +} + +function parseSrc(src: unknown): { start: number; length: number } | undefined { + if (typeof src !== 'string') return undefined; + const [start, length] = src.split(':').map(Number); + if ( + !Number.isSafeInteger(start) || + !Number.isSafeInteger(length) || + start < 0 || + length < 0 + ) { + return undefined; + } + return { start, length }; +} + +function sliceUtf8Bytes(source: string, start: number, length: number): string { + const bytes = Buffer.from(source, 'utf8'); + if (start + length > bytes.length) { + throw new Error('AST source range exceeds source contents'); + } + return bytes.subarray(start, start + length).toString('utf8'); +} + +function stripCommentsAndStrings(source: string): string { + return source + .replace(/\/\*[\s\S]*?\*\//g, ' ') + .replace(/\/\/[^\n\r]*/g, ' ') + .replace(/"(?:\\.|[^"\\])*"/g, ' ') + .replace(/'(?:\\.|[^'\\])*'/g, ' '); +} diff --git a/packages/compilers/test/solidityStorageLayout.spec.ts b/packages/compilers/test/solidityStorageLayout.spec.ts new file mode 100644 index 0000000000..430e485042 --- /dev/null +++ b/packages/compilers/test/solidityStorageLayout.spec.ts @@ -0,0 +1,631 @@ +import { expect } from 'chai'; +import solc from 'solc'; +import type { + SolidityJsonInput, + SolidityOutput, +} from '@ethereum-sourcify/compilers-types'; +import { + generateHistoricalSolidityStorageLayout, + supportsHistoricalSolidityStorageLayout, +} from '../src/lib/solidityStorageLayout'; + +describe('historical Solidity storage layouts', () => { + it('supports only the pre-native 0.4.x and 0.5.x range', () => { + expect(supportsHistoricalSolidityStorageLayout('0.3.6')).to.equal(false); + expect(supportsHistoricalSolidityStorageLayout('0.4.0')).to.equal(true); + expect( + supportsHistoricalSolidityStorageLayout( + '0.5.13-nightly.2019.10.15+commit.abc12345', + ), + ).to.equal(true); + expect(supportsHistoricalSolidityStorageLayout('0.5.13')).to.equal(false); + }); + + it('matches native layout for packing, inheritance, arrays, mappings and structs', () => { + const source = ` + pragma solidity >=0.5.0; + contract Other {} + contract $Other {} + contract A_$_B {} + contract Base { + enum E { A, B, C } + struct Inner { uint8 x; uint16 y; } + struct A_storage { uint256 x; } + struct Outer { + Inner inner; + uint8[5] packed; + mapping(address => uint256) map; + } + uint8 c1; + uint256 constant SKIP = 3; + E e; + bytes3 fixedBytes; + address addr; + Other other; + uint256 constant N = 5; + uint8[N] packed; + uint16[2][3] nested; + bytes data; + string text; + mapping(address => Outer) map; + Outer outer; + function(uint256, bytes32) external returns (bool) callback; + function($Other) external callbackWithDollarType; + function(A_$_B) external callbackWithEscapedDollarType; + function(uint256[][2] calldata) external callbackWithArray; + function(mapping(address => uint256) storage) internal callbackWithMapping; + function(A_storage memory) internal callbackWithStorageInTypeName; + function() external payable callbackPayable; + function(uint8) internal pure returns (bytes4) internalCallback; + } + contract Sibling { uint128 z; } + contract Child is Sibling, Base { bool tail; } + `; + const input: SolidityJsonInput = { + language: 'Solidity', + sources: { 'Probe.sol': { content: source } }, + settings: { + outputSelection: { + '*': { + '': ['ast'], + '*': ['storageLayout'], + }, + }, + }, + }; + const output = JSON.parse( + solc.compile(JSON.stringify(input)), + ) as SolidityOutput; + expect( + output.errors?.filter((error) => error.severity === 'error'), + ).to.deep.equal([]); + + const reconstructed = generateHistoricalSolidityStorageLayout( + '0.5.12+commit.7709ece9', + input, + output, + { path: 'Probe.sol', name: 'Child' }, + ); + const native = output.contracts['Probe.sol'].Child.storageLayout; + + expect(reconstructed).to.deep.equal(native); + }); + + it('normalizes legacy ASTs and excludes constants from their source span', () => { + const source = + '// 😀😀😀😀😀😀😀😀\n' + + 'contract Base { uint8 a; uint256 constant SKIP = 1; uint16 b; } ' + + 'contract Child is Base { bool c; }'; + const variable = ( + id: number, + name: string, + type: string, + ): Record => { + const declarationStart = source.indexOf( + name === 'SKIP' ? 'uint256 constant SKIP' : `${type} ${name}`, + ); + const declarationEnd = source.indexOf(';', declarationStart) + 1; + const byteStart = Buffer.byteLength( + source.slice(0, declarationStart), + 'utf8', + ); + const byteLength = Buffer.byteLength( + source.slice(declarationStart, declarationEnd), + 'utf8', + ); + return { + id, + name: 'VariableDeclaration', + attributes: { name, type }, + src: `${byteStart}:${byteLength}:0`, + children: [ + { + id: id + 1000, + name: 'ElementaryTypeName', + attributes: { name: type }, + }, + ], + }; + }; + const input: SolidityJsonInput = { + language: 'Solidity', + sources: { 'Legacy.sol': { content: source } }, + settings: {}, + }; + const output = { + contracts: {}, + sources: { + 'Legacy.sol': { + id: 0, + legacyAST: { + id: 500, + name: 'SourceUnit', + children: [ + { + id: 100, + name: 'ContractDefinition', + attributes: { + name: 'Base', + linearizedBaseContracts: [100], + }, + children: [ + variable(1, 'a', 'uint8'), + variable(2, 'SKIP', 'uint256'), + variable(3, 'b', 'uint16'), + ], + }, + { + id: 200, + name: 'ContractDefinition', + attributes: { + name: 'Child', + linearizedBaseContracts: [200, 100], + }, + children: [variable(4, 'c', 'bool')], + }, + ], + }, + }, + }, + } as unknown as SolidityOutput; + + const layout = generateHistoricalSolidityStorageLayout( + '0.4.0+commit.acd334c9', + input, + output, + { path: 'Legacy.sol', name: 'Child' }, + ); + + expect(layout?.storage).to.deep.equal([ + { + astId: 1, + contract: 'Legacy.sol:Child', + label: 'a', + offset: 0, + slot: '0', + type: 't_uint8', + }, + { + astId: 3, + contract: 'Legacy.sol:Child', + label: 'b', + offset: 1, + slot: '0', + type: 't_uint16', + }, + { + astId: 4, + contract: 'Legacy.sol:Child', + label: 'c', + offset: 3, + slot: '0', + type: 't_bool', + }, + ]); + }); + + it('resolves library types from the legacy AST dialect', () => { + const input: SolidityJsonInput = { + language: 'Solidity', + sources: { + 'Library.sol': { + content: 'library L {} contract C { L value; }', + }, + }, + settings: {}, + }; + const output = { + contracts: {}, + sources: { + 'Library.sol': { + id: 0, + legacyAST: { + id: 500, + name: 'SourceUnit', + children: [ + { + id: 100, + name: 'ContractDefinition', + attributes: { + isLibrary: true, + name: 'L', + linearizedBaseContracts: [100], + }, + children: [], + }, + { + id: 200, + name: 'ContractDefinition', + attributes: { + name: 'C', + linearizedBaseContracts: [200], + }, + children: [ + { + id: 1, + name: 'VariableDeclaration', + attributes: { + constant: false, + name: 'value', + type: 'library L', + }, + children: [ + { + id: 2, + name: 'UserDefinedTypeName', + attributes: { name: 'L' }, + }, + ], + }, + ], + }, + ], + }, + }, + }, + } as unknown as SolidityOutput; + + const layout = generateHistoricalSolidityStorageLayout( + '0.4.11+commit.68ef5810', + input, + output, + { path: 'Library.sol', name: 'C' }, + ); + + expect(layout?.storage[0]).to.include({ + offset: 0, + slot: '0', + type: 't_contract(L)100', + }); + expect(layout?.types?.['t_contract(L)100']).to.deep.equal({ + encoding: 'inplace', + label: 'library L', + numberOfBytes: '20', + }); + }); + + it('builds canonical function identifiers from legacy parameter lists', () => { + const input: SolidityJsonInput = { + language: 'Solidity', + sources: { + 'Functions.sol': { + content: + 'contract C { function(uint256, bytes32) external returns (bool) cb; ' + + 'function(uint8) internal constant returns (bytes4) icb; }', + }, + }, + settings: {}, + }; + const output = legacyContractOutput('Functions.sol', 'C', [ + legacyFunctionVariable( + 1, + 'cb', + 'external', + false, + ['uint256', 'bytes32'], + ['bool'], + ), + legacyFunctionVariable( + 20, + 'icb', + 'internal', + true, + ['uint8'], + ['bytes4'], + ), + ]); + + const layout = generateHistoricalSolidityStorageLayout( + '0.4.11+commit.68ef5810', + input, + output, + { path: 'Functions.sol', name: 'C' }, + ); + + expect( + layout?.storage.map(({ offset, slot, type }) => ({ + offset, + slot, + type, + })), + ).to.deep.equal([ + { + offset: 0, + slot: '0', + type: 't_function_external_nonpayable(t_uint256,t_bytes32)returns(t_bool)', + }, + { + offset: 24, + slot: '0', + type: 't_function_internal_view(t_uint8)returns(t_bytes4)', + }, + ]); + }); + + it('normalizes the compact AST isDeclaredConst function flag', () => { + const parameter = (id: number, type: string) => ({ + id, + nodeType: 'VariableDeclaration', + name: '', + typeDescriptions: { typeString: type }, + typeName: { + id: id + 100, + name: type, + nodeType: 'ElementaryTypeName', + }, + }); + const input: SolidityJsonInput = { + language: 'Solidity', + sources: { + 'Compact.sol': { + content: + 'contract C { function(uint8) internal constant returns (bytes4) cb; }', + }, + }, + settings: {}, + }; + const output = { + contracts: {}, + sources: { + 'Compact.sol': { + id: 0, + ast: { + id: 500, + nodeType: 'SourceUnit', + nodes: [ + { + id: 100, + name: 'C', + nodeType: 'ContractDefinition', + linearizedBaseContracts: [100], + nodes: [ + { + constant: false, + id: 1, + name: 'cb', + nodeType: 'VariableDeclaration', + typeDescriptions: { + typeString: 'function (uint8) constant returns (bytes4)', + }, + typeName: { + id: 2, + isDeclaredConst: true, + nodeType: 'FunctionTypeName', + parameterTypes: { + id: 3, + nodeType: 'ParameterList', + parameters: [parameter(4, 'uint8')], + }, + returnParameterTypes: { + id: 5, + nodeType: 'ParameterList', + parameters: [parameter(6, 'bytes4')], + }, + typeDescriptions: { + typeString: + 'function (uint8) constant returns (bytes4)', + }, + visibility: 'internal', + }, + }, + ], + }, + ], + }, + }, + }, + } as unknown as SolidityOutput; + + const layout = generateHistoricalSolidityStorageLayout( + '0.4.12+commit.194ff033', + input, + output, + { path: 'Compact.sol', name: 'C' }, + ); + + expect(layout?.storage[0].type).to.equal( + 't_function_internal_view(t_uint8)returns(t_bytes4)', + ); + }); + + it('uses the historical fixed-point width rule before 0.4.14', () => { + const input: SolidityJsonInput = { + language: 'Solidity', + sources: { + 'Fixed.sol': { content: 'contract C { fixed128x128 a; uint8 b; }' }, + }, + settings: {}, + }; + const output = legacyContractOutput('Fixed.sol', 'C', [ + legacyElementaryVariable(1, 'a', 'fixed128x128'), + legacyElementaryVariable(2, 'b', 'uint8'), + ]); + const oldLayout = generateHistoricalSolidityStorageLayout( + '0.4.13+commit.0fb4cb1a', + input, + output, + { path: 'Fixed.sol', name: 'C' }, + ); + + expect( + oldLayout?.storage.map(({ slot, offset }) => ({ slot, offset })), + ).to.deep.equal([ + { slot: '0', offset: 0 }, + { slot: '1', offset: 0 }, + ]); + }); + + it('switches fixed-point storage widths at the historical nightly cutoff', () => { + const input: SolidityJsonInput = { + language: 'Solidity', + sources: { + 'Fixed.sol': { content: 'contract C { fixed128x16 a; uint8 b; }' }, + }, + settings: {}, + }; + const output = legacyContractOutput('Fixed.sol', 'C', [ + legacyElementaryVariable(1, 'a', 'fixed128x16'), + legacyElementaryVariable(2, 'b', 'uint8'), + ]); + const offsets = (version: string) => + generateHistoricalSolidityStorageLayout(version, input, output, { + path: 'Fixed.sol', + name: 'C', + })?.storage.map(({ offset }) => offset); + + expect(offsets('0.4.14-nightly.2017.7.19+commit.aaaa')).to.deep.equal([ + 0, 18, + ]); + expect(offsets('0.4.14-nightly.2017.7.20+commit.bbbb')).to.deep.equal([ + 0, 16, + ]); + expect(offsets('0.4.14-ci.2017.7.19+commit.cccc')).to.deep.equal([0, 18]); + expect(offsets('0.4.14-ci.2017.7.20+commit.dddd')).to.deep.equal([0, 16]); + }); + + it('returns the native null type table for an empty contract', () => { + const input: SolidityJsonInput = { + language: 'Solidity', + sources: { 'Empty.sol': { content: 'contract Empty {}' } }, + settings: {}, + }; + const output = legacyContractOutput('Empty.sol', 'Empty', []); + + expect( + generateHistoricalSolidityStorageLayout( + '0.4.11+commit.68ef5810', + input, + output, + { path: 'Empty.sol', name: 'Empty' }, + ), + ).to.deep.equal({ storage: [], types: null }); + }); + + it('fails closed when a user-defined type cannot be resolved', () => { + const input: SolidityJsonInput = { + language: 'Solidity', + sources: { 'Broken.sol': { content: 'contract C { Missing value; }' } }, + settings: {}, + }; + const output = legacyContractOutput('Broken.sol', 'C', [ + { + id: 1, + name: 'VariableDeclaration', + attributes: { + constant: false, + name: 'value', + type: 'struct Missing storage ref', + }, + children: [ + { + id: 2, + name: 'UserDefinedTypeName', + attributes: { name: 'Missing' }, + }, + ], + }, + ]); + + expect(() => + generateHistoricalSolidityStorageLayout( + '0.4.11+commit.68ef5810', + input, + output, + { path: 'Broken.sol', name: 'C' }, + ), + ).to.throw('Cannot resolve user-defined type Missing'); + }); +}); + +function legacyElementaryVariable(id: number, name: string, type: string) { + return { + id, + name: 'VariableDeclaration', + attributes: { constant: false, name, type }, + children: [ + { + id: id + 1000, + name: 'ElementaryTypeName', + attributes: { name: type }, + }, + ], + }; +} + +function legacyFunctionVariable( + id: number, + name: string, + visibility: 'external' | 'internal', + constant: boolean, + parameters: string[], + returns: string[], +) { + let nextId = id + 2; + const parameterList = (types: string[]) => ({ + id: nextId++, + name: 'ParameterList', + children: types.map((type) => { + const typeId = nextId++; + return { + id: nextId++, + name: 'VariableDeclaration', + attributes: { constant: false, name: '', type }, + children: [ + { + id: typeId, + name: 'ElementaryTypeName', + attributes: { name: type }, + }, + ], + }; + }), + }); + const parameterTypes = parameterList(parameters); + const returnParameterTypes = parameterList(returns); + const mutability = constant ? ' constant' : ''; + const returnLabel = returns.length ? ` returns (${returns.join(',')})` : ''; + return { + id, + name: 'VariableDeclaration', + attributes: { + constant: false, + name, + type: `function (${parameters.join(',')}) ${visibility}${mutability}${returnLabel}`, + }, + children: [ + { + id: id + 1, + name: 'FunctionTypeName', + attributes: { constant, payable: false, visibility }, + children: [parameterTypes, returnParameterTypes], + }, + ], + }; +} + +function legacyContractOutput( + path: string, + name: string, + variables: Record[], +): SolidityOutput { + return { + contracts: {}, + sources: { + [path]: { + id: 0, + legacyAST: { + id: 500, + name: 'SourceUnit', + children: [ + { + id: 100, + name: 'ContractDefinition', + attributes: { name, linearizedBaseContracts: [100] }, + children: variables, + }, + ], + }, + }, + }, + } as unknown as SolidityOutput; +} diff --git a/packages/lib-sourcify/src/Compilation/CompilationTypes.ts b/packages/lib-sourcify/src/Compilation/CompilationTypes.ts index 6f5d559469..a1a1bc86b1 100644 --- a/packages/lib-sourcify/src/Compilation/CompilationTypes.ts +++ b/packages/lib-sourcify/src/Compilation/CompilationTypes.ts @@ -1,6 +1,7 @@ import type { SolidityJsonInput, SolidityOutput, + StorageLayout, VyperJsonInput, VyperOutput, FeJsonInput, @@ -64,6 +65,12 @@ export interface ISolidityCompiler { solcJsonInput: SolidityJsonInput, forceEmscripten?: boolean, ): Promise; + extractStorageLayout?( + version: string, + solcJsonInput: SolidityJsonInput, + compilerOutput: SolidityOutput, + compilationTarget: CompilationTarget, + ): Promise; } export interface IVyperCompiler { diff --git a/packages/lib-sourcify/src/Compilation/SolidityCompilation.ts b/packages/lib-sourcify/src/Compilation/SolidityCompilation.ts index 5f75651ac9..6bea71342a 100644 --- a/packages/lib-sourcify/src/Compilation/SolidityCompilation.ts +++ b/packages/lib-sourcify/src/Compilation/SolidityCompilation.ts @@ -42,6 +42,15 @@ export const DEFAULT_OUTPUT_SELECTION = { }, } as const; +export function supportsHistoricalSolidityStorageLayoutExtraction( + version: string, +): boolean { + // Before 0.4.7, bytecode did not bind the source through compiler metadata. + // Distinct source layouts can therefore deduplicate to one compiled_contracts + // row, which cannot safely carry a source-specific storageLayout artifact. + return semver.gte(version, '0.4.7') && semver.lt(version, '0.5.13'); +} + /** * Abstraction of a solidity compilation */ @@ -75,7 +84,17 @@ export class SolidityCompilation extends AbstractCompilation { } initSolidityJsonInput() { - this.jsonInput.settings.outputSelection = DEFAULT_OUTPUT_SELECTION; + const outputSelection = structuredClone( + DEFAULT_OUTPUT_SELECTION, + ) as unknown as Record>; + if ( + supportsHistoricalSolidityStorageLayoutExtraction(this.compilerVersion) + ) { + outputSelection['*'][''] = [ + semver.gte(this.compilerVersion, '0.4.12') ? 'ast' : 'legacyAST', + ]; + } + this.jsonInput.settings.outputSelection = outputSelection; } /** Generates an edited contract with a space at the end of each source file to create a different source file hash and consequently a different metadata hash. @@ -279,6 +298,30 @@ export class SolidityCompilation extends AbstractCompilation { public async compile(forceEmscripten = false) { const contract = await this.compileAndReturnCompilationTarget(forceEmscripten); + if ( + contract.storageLayout === undefined && + this.compilerOutput && + this.compiler.extractStorageLayout && + supportsHistoricalSolidityStorageLayoutExtraction(this.compilerVersion) + ) { + try { + const storageLayout = await this.compiler.extractStorageLayout( + this.compilerVersion, + this.jsonInput, + this.compilerOutput, + this.compilationTarget, + ); + if (storageLayout !== undefined) { + contract.storageLayout = storageLayout; + } + } catch (error) { + logWarn('Cannot recover historical Solidity storage layout', { + error, + compilerVersion: this.compilerVersion, + compilationTarget: this.compilationTarget, + }); + } + } if (contract.metadata) { this._metadata = JSON.parse(contract.metadata.trim()); } else { diff --git a/packages/lib-sourcify/test/Compilation/SolidityCompilation.spec.ts b/packages/lib-sourcify/test/Compilation/SolidityCompilation.spec.ts index 5a324b70e4..91aac7ba3c 100644 --- a/packages/lib-sourcify/test/Compilation/SolidityCompilation.spec.ts +++ b/packages/lib-sourcify/test/Compilation/SolidityCompilation.spec.ts @@ -2,15 +2,21 @@ import { describe, it } from 'mocha'; import { expect, use } from 'chai'; import path from 'path'; import fs from 'fs'; -import { SolidityCompilation } from '../../src/Compilation/SolidityCompilation'; +import { + SolidityCompilation, + supportsHistoricalSolidityStorageLayoutExtraction, +} from '../../src/Compilation/SolidityCompilation'; import { solc } from '../utils'; import { CompilationError, type CompilationTarget, + type ISolidityCompiler, } from '../../src/Compilation/CompilationTypes'; import type { SolidityJsonInput, + SolidityOutput, SolidityOutputContract, + StorageLayout, Metadata, } from '@ethereum-sourcify/compilers-types'; import chaiAsPromised from 'chai-as-promised'; @@ -36,6 +42,29 @@ function getCompilationTargetFromMetadata( } describe('SolidityCompilation', () => { + it('recovers historical layouts only when bytecode binds the source metadata', () => { + expect( + supportsHistoricalSolidityStorageLayoutExtraction( + '0.4.6+commit.2dabbdf0', + ), + ).to.equal(false); + expect( + supportsHistoricalSolidityStorageLayoutExtraction( + '0.4.7+commit.822622cf', + ), + ).to.equal(true); + expect( + supportsHistoricalSolidityStorageLayoutExtraction( + '0.5.12+commit.7709ece9', + ), + ).to.equal(true); + expect( + supportsHistoricalSolidityStorageLayoutExtraction( + '0.5.13+commit.5b0b510c', + ), + ).to.equal(false); + }); + it('should compile a simple contract', async () => { const contractPath = path.join(__dirname, '..', 'sources', 'Storage'); const metadata = JSON.parse( @@ -96,6 +125,7 @@ describe('SolidityCompilation', () => { ); await compilation.compile(); + await compilation.generateCborAuxdataPositions(); expect(compilation.runtimeBytecodeCborAuxdata).to.deep.equal({ '1': { @@ -149,6 +179,7 @@ describe('SolidityCompilation', () => { ); await compilation.compile(); + await compilation.generateCborAuxdataPositions(); expect(compilation.runtimeBytecodeCborAuxdata).to.deep.equal({ @@ -453,6 +484,19 @@ describe('SolidityCompilation', () => { ); await compilation.compile(); + + const storageLayout = ( + compilation.contractCompilerOutput as SolidityOutputContract + ).storageLayout; + const owner = storageLayout?.storage.find(({ label }) => label === 'owner'); + expect(owner).to.include({ + contract: 'Multidrop.sol:Multidrop', + label: 'owner', + offset: 0, + slot: '0', + type: 't_address', + }); + await compilation.generateCborAuxdataPositions(); // For Solidity 0.4.11, auxdata should be extracted from bytecode directly @@ -486,6 +530,127 @@ describe('SolidityCompilation', () => { } }); + it('should attach a recovered historical storage layout', async () => { + const recoveredLayout: StorageLayout = { + storage: [ + { + astId: 1, + contract: 'Legacy.sol:Legacy', + label: 'value', + offset: 0, + slot: '0', + type: 't_uint256', + }, + ], + types: { + t_uint256: { + encoding: 'inplace', + label: 'uint256', + numberOfBytes: '32', + }, + }, + }; + let extractionCalls = 0; + const compiler: ISolidityCompiler = { + async compile() { + return emptySolidityOutput(); + }, + async extractStorageLayout() { + extractionCalls += 1; + return recoveredLayout; + }, + }; + const compilation = new SolidityCompilation( + compiler, + '0.5.12+commit.7709ece9', + basicSolidityInput(), + { path: 'Legacy.sol', name: 'Legacy' }, + ); + + expect( + compilation.jsonInput.settings.outputSelection?.['*']?.[''], + ).to.deep.equal(['ast']); + await compilation.compile(); + + expect(extractionCalls).to.equal(1); + expect( + (compilation.contractCompilerOutput as SolidityOutputContract) + .storageLayout, + ).to.deep.equal(recoveredLayout); + }); + + it('should preserve native storage layouts without invoking recovery', async () => { + const nativeLayout: StorageLayout = { storage: [], types: null }; + let extractionCalls = 0; + const compiler: ISolidityCompiler = { + async compile() { + return emptySolidityOutput(nativeLayout); + }, + async extractStorageLayout() { + extractionCalls += 1; + throw new Error('must not run'); + }, + }; + const compilation = new SolidityCompilation( + compiler, + '0.5.12+commit.7709ece9', + basicSolidityInput(), + { path: 'Legacy.sol', name: 'Legacy' }, + ); + + await compilation.compile(); + + expect(extractionCalls).to.equal(0); + expect( + (compilation.contractCompilerOutput as SolidityOutputContract) + .storageLayout, + ).to.equal(nativeLayout); + }); + + it('should leave the layout absent when historical recovery fails', async () => { + const compiler: ISolidityCompiler = { + async compile() { + return emptySolidityOutput(); + }, + async extractStorageLayout() { + throw new Error('unresolved historical type'); + }, + }; + const compilation = new SolidityCompilation( + compiler, + '0.4.11+commit.68ef5810', + basicSolidityInput(), + { path: 'Legacy.sol', name: 'Legacy' }, + ); + + expect( + compilation.jsonInput.settings.outputSelection?.['*']?.[''], + ).to.deep.equal(['legacyAST']); + await compilation.compile(); + + expect( + (compilation.contractCompilerOutput as SolidityOutputContract) + .storageLayout, + ).to.equal(undefined); + }); + + it('should not request historical ASTs for native layout versions', () => { + const compilation = new SolidityCompilation( + { + async compile() { + return emptySolidityOutput(); + }, + }, + '0.5.13+commit.5b0b510c', + basicSolidityInput(), + { path: 'Legacy.sol', name: 'Legacy' }, + ); + + expect( + compilation.jsonInput.settings.outputSelection?.['*']?.[''], + ).to.equal(undefined); + }); + it('should output transientStorageLayout for contracts with transient storage variables', async () => { const source = `// SPDX-License-Identifier: MIT pragma solidity ^0.8.28; @@ -583,3 +748,29 @@ contract TransientStorage { expect(compilation.creationBytecodeCborAuxdata).to.deep.equal({}); }); }); + +function basicSolidityInput(): SolidityJsonInput { + return { + language: 'Solidity', + sources: { 'Legacy.sol': { content: 'contract Legacy { uint value; }' } }, + settings: {}, + }; +} + +function emptySolidityOutput(storageLayout?: StorageLayout): SolidityOutput { + return { + contracts: { + 'Legacy.sol': { + Legacy: { + abi: [], + ...(storageLayout ? { storageLayout } : {}), + evm: { + bytecode: { object: '' }, + deployedBytecode: { object: '' }, + }, + }, + }, + }, + sources: { 'Legacy.sol': { id: 0 } }, + }; +} diff --git a/packages/lib-sourcify/test/utils.ts b/packages/lib-sourcify/test/utils.ts index 88948dfbdb..feeb9014b9 100644 --- a/packages/lib-sourcify/test/utils.ts +++ b/packages/lib-sourcify/test/utils.ts @@ -3,6 +3,7 @@ import { expect } from 'chai'; import type { Signer } from 'ethers'; import { ContractFactory, type JsonRpcSigner } from 'ethers'; import { + generateHistoricalSolidityStorageLayout, useSolidityCompiler, useVyperCompiler, useFeCompiler, @@ -20,10 +21,12 @@ import type { import type { Verification } from '../src/Verification/Verification'; import type { CompiledContractCborAuxdata, + CompilationTarget, ISolidityCompiler, IVyperCompiler, IFeCompiler, } from '../src/Compilation/CompilationTypes'; +import type { StorageLayout } from '@ethereum-sourcify/compilers-types'; import fs from 'fs'; import { type PathContent, @@ -81,6 +84,20 @@ class Solc implements ISolidityCompiler { forceEmscripten, ); } + + async extractStorageLayout( + version: string, + solcJsonInput: SolidityJsonInput, + compilerOutput: SolidityOutput, + compilationTarget: CompilationTarget, + ): Promise { + return generateHistoricalSolidityStorageLayout( + version, + solcJsonInput, + compilerOutput, + compilationTarget, + ); + } } export const solc = new Solc(); @@ -259,6 +276,20 @@ export class TestSolidityCompiler implements ISolidityCompiler { forceEmscripten, ); } + + async extractStorageLayout( + version: string, + solcJsonInput: SolidityJsonInput, + compilerOutput: SolidityOutput, + compilationTarget: CompilationTarget, + ): Promise { + return generateHistoricalSolidityStorageLayout( + version, + solcJsonInput, + compilerOutput, + compilationTarget, + ); + } } export function assertCborTransformations(transformations?: any[]) { diff --git a/services/server/src/server/apiv1/verification/private/stateless/customReplaceMethods.ts b/services/server/src/server/apiv1/verification/private/stateless/customReplaceMethods.ts index 944f95044f..d5fb0cd04c 100644 --- a/services/server/src/server/apiv1/verification/private/stateless/customReplaceMethods.ts +++ b/services/server/src/server/apiv1/verification/private/stateless/customReplaceMethods.ts @@ -9,6 +9,7 @@ import { import type { SourcifyDatabaseService } from "../../../../services/storageServices/SourcifyDatabaseService"; import { BadRequestError } from "../../../../../common/errors"; import logger from "../../../../../common/logger"; +import { replaceSolidityStorageLayout } from "./solidityStorageLayoutReplace"; /** * Result of a custom replace method: @@ -276,4 +277,5 @@ export const REPLACE_METHODS: Record = { "replace-creation-information": replaceCreationInformation, "replace-metadata": replaceMetadata, "replace-vyper-immutable-references": replaceVyperImmutableReferences, + "replace-solidity-storage-layout": replaceSolidityStorageLayout, }; diff --git a/services/server/src/server/apiv1/verification/private/stateless/private.stateless.handlers.ts b/services/server/src/server/apiv1/verification/private/stateless/private.stateless.handlers.ts index f598c80849..0c1306fc40 100644 --- a/services/server/src/server/apiv1/verification/private/stateless/private.stateless.handlers.ts +++ b/services/server/src/server/apiv1/verification/private/stateless/private.stateless.handlers.ts @@ -324,7 +324,7 @@ export async function replaceContract( try { rpcFailedFetchingCreationBytecode = verification.onchainCreationBytecode === undefined; - } catch (error) { + } catch { // verification.onchainCreationBytecode throws if not available rpcFailedFetchingCreationBytecode = true; } @@ -339,6 +339,7 @@ export async function replaceContract( rpcFailedFetchingCreationBytecode, }); } catch (error: any) { + if (typeof error?.statusCode === "number") throw error; throw new InternalServerError(error.message); } } diff --git a/services/server/src/server/apiv1/verification/private/stateless/private.stateless.paths.yaml b/services/server/src/server/apiv1/verification/private/stateless/private.stateless.paths.yaml index 8cdead8908..1d6b3f280d 100644 --- a/services/server/src/server/apiv1/verification/private/stateless/private.stateless.paths.yaml +++ b/services/server/src/server/apiv1/verification/private/stateless/private.stateless.paths.yaml @@ -159,6 +159,7 @@ paths: - address - forceCompilation - forceRPCRequest + - customReplaceMethod properties: chainId: oneOf: @@ -212,6 +213,15 @@ paths: whose `immutableReferences` were never persisted; use with `forceCompilation: true`. Only Vyper is supported (throws otherwise); contracts without immutables are left untouched. + - "replace-solidity-storage-layout": Reconstructs the storage layout of + a verified Solidity 0.4.7 through 0.5.12 contract from its historical + compiler AST and backfills only + `compiled_contracts.compilation_artifacts.storageLayout`. Use with + `forceCompilation: true`. The fresh compilation must match the deployment + and exactly match the stored version, target, settings, additional input, + and source set. Existing non-null layouts are never overwritten. + Versions before 0.4.7 are excluded because their bytecode does not bind + source metadata, so distinct layouts can share one compiled-contract row. example: "replace-creation-information" responses: "200": diff --git a/services/server/src/server/apiv1/verification/private/stateless/solidityStorageLayoutReplace.ts b/services/server/src/server/apiv1/verification/private/stateless/solidityStorageLayoutReplace.ts new file mode 100644 index 0000000000..11f06c50c8 --- /dev/null +++ b/services/server/src/server/apiv1/verification/private/stateless/solidityStorageLayoutReplace.ts @@ -0,0 +1,166 @@ +import semver from "semver"; +import type { + StorageLayout, + VerificationExport, +} from "@ethereum-sourcify/lib-sourcify"; +import { + bytesFromString, + prepareCompilerSettingsFromVerification, +} from "../../../../services/utils/database-util"; +import type { SourcifyDatabaseService } from "../../../../services/storageServices/SourcifyDatabaseService"; +import { BadRequestError } from "../../../../../common/errors"; +import logger from "../../../../../common/logger"; +import type { CustomReplaceMethod } from "./customReplaceMethods"; + +// 0.4.7 is the first release whose bytecode metadata binds the source. Older +// compilations with different layouts can otherwise deduplicate to one shared +// compiled_contracts row because that table is keyed by bytecode hashes. +const MINIMUM_VERSION = "0.4.7"; +const NATIVE_STORAGE_LAYOUT_VERSION = "0.5.13"; + +/** Backfills only a missing historical Solidity storageLayout artifact. */ +export const replaceSolidityStorageLayout: CustomReplaceMethod = async ( + sourcifyDatabaseService, + verification, +) => { + if (verification.compilation.language !== "Solidity") { + throw new BadRequestError( + `replace-solidity-storage-layout only supports Solidity contracts, got ${verification.compilation.language}`, + ); + } + + const normalizedVersion = semver.valid( + verification.compilation.compilerVersion.trim().replace(/^v/, ""), + ); + if ( + !normalizedVersion || + semver.lt(normalizedVersion, MINIMUM_VERSION) || + !semver.lt(normalizedVersion, NATIVE_STORAGE_LAYOUT_VERSION) + ) { + throw new BadRequestError( + "replace-solidity-storage-layout only supports Solidity versions from 0.4.7 through 0.5.12", + ); + } + + const { runtimeMatch, creationMatch } = verification.status; + if ( + ![runtimeMatch, creationMatch].some( + (match) => match === "perfect" || match === "partial", + ) + ) { + throw new BadRequestError( + "Cannot backfill a Solidity storage layout from a compilation that did not match the deployment", + ); + } + + const storageLayout = + verification.compilation.contractCompilerOutput.storageLayout; + if (storageLayout === undefined) { + const reason = "Historical Solidity storage layout could not be recovered"; + logger.info(reason, { + chainId: verification.chainId, + address: verification.address, + compilerVersion: verification.compilation.compilerVersion, + }); + return { reason, replaced: false }; + } + if (!isSolidityStorageLayout(storageLayout)) { + throw new BadRequestError( + "Recovered Solidity storage layout has an invalid structure", + ); + } + + const compilationId = await getSingleCompilationId( + sourcifyDatabaseService, + verification, + ); + await writeMissingStorageLayout( + sourcifyDatabaseService, + verification, + compilationId, + storageLayout, + ); +}; + +function isSolidityStorageLayout(value: unknown): value is StorageLayout { + if (typeof value !== "object" || value === null) return false; + const layout = value as Partial; + return ( + Array.isArray(layout.storage) && + (layout.types === null || + (typeof layout.types === "object" && !Array.isArray(layout.types))) + ); +} + +async function getSingleCompilationId( + sourcifyDatabaseService: SourcifyDatabaseService, + verification: VerificationExport, +): Promise { + const result = await sourcifyDatabaseService.database.pool.query( + `SELECT vc.compilation_id + FROM verified_contracts vc + JOIN contract_deployments cd ON cd.id = vc.deployment_id + INNER JOIN sourcify_matches sm ON sm.verified_contract_id = vc.id + WHERE cd.chain_id = $1 AND cd.address = $2`, + [verification.chainId.toString(), bytesFromString(verification.address)], + ); + if (result.rows.length === 0) { + throw new Error( + `No existing verified contract found for address ${verification.address} on chain ${verification.chainId}`, + ); + } + if (result.rows.length > 1) { + throw new Error( + `Multiple verified contracts found for address ${verification.address} on chain ${verification.chainId}; cannot safely backfill storageLayout`, + ); + } + return result.rows[0].compilation_id; +} + +async function writeMissingStorageLayout( + sourcifyDatabaseService: SourcifyDatabaseService, + verification: VerificationExport, + compilationId: string, + storageLayout: StorageLayout, +) { + const { path, name } = verification.compilation.compilationTarget; + const settings = prepareCompilerSettingsFromVerification(verification); + const additionalInput = verification.compilation.additionalInput ?? null; + const result = await sourcifyDatabaseService.database.pool.query( + `UPDATE compiled_contracts cc + SET compilation_artifacts = jsonb_set( + COALESCE(cc.compilation_artifacts, '{}'::jsonb), + '{storageLayout}', $7::jsonb, true) + WHERE cc.id = $1 + AND cc.language = 'solidity' + AND cc.version = $2 + AND cc.fully_qualified_name = $3 + AND cc.compiler_settings = $4::jsonb + AND cc.additional_input IS NOT DISTINCT FROM $5::jsonb + AND ( + SELECT jsonb_object_agg(ccs.path, sources.content) + FROM compiled_contracts_sources ccs + JOIN sources ON sources.source_hash = ccs.source_hash + WHERE ccs.compilation_id = cc.id + ) = $6::jsonb + AND ( + cc.compilation_artifacts->'storageLayout' IS NULL + OR cc.compilation_artifacts->'storageLayout' = 'null'::jsonb + ) + RETURNING cc.id`, + [ + compilationId, + verification.compilation.compilerVersion, + `${path}:${name}`, + JSON.stringify(settings), + additionalInput === null ? null : JSON.stringify(additionalInput), + JSON.stringify(verification.compilation.sources), + JSON.stringify(storageLayout), + ], + ); + if (result.rows.length !== 1) { + throw new BadRequestError( + "Fresh Solidity compilation identity does not match the stored compilation, or storageLayout is already populated; refusing to replace storageLayout", + ); + } +} diff --git a/services/server/src/server/services/compiler/local/SolcLocal.ts b/services/server/src/server/services/compiler/local/SolcLocal.ts index 0308272649..cbfa8952ec 100644 --- a/services/server/src/server/services/compiler/local/SolcLocal.ts +++ b/services/server/src/server/services/compiler/local/SolcLocal.ts @@ -1,9 +1,14 @@ import type { + CompilationTarget, + StorageLayout, SolidityOutput, ISolidityCompiler, SolidityJsonInput, } from "@ethereum-sourcify/lib-sourcify"; -import { useSolidityCompiler } from "@ethereum-sourcify/compilers"; +import { + generateHistoricalSolidityStorageLayout, + useSolidityCompiler, +} from "@ethereum-sourcify/compilers"; export class SolcLocal implements ISolidityCompiler { constructor( @@ -24,4 +29,18 @@ export class SolcLocal implements ISolidityCompiler { forceEmscripten, ); } + + async extractStorageLayout( + version: string, + solcJsonInput: SolidityJsonInput, + compilerOutput: SolidityOutput, + compilationTarget: CompilationTarget, + ): Promise { + return generateHistoricalSolidityStorageLayout( + version, + solcJsonInput, + compilerOutput, + compilationTarget, + ); + } } diff --git a/services/server/test/integration/apiv1/verification-handlers/solidity-storage-layout.stateless.spec.ts b/services/server/test/integration/apiv1/verification-handlers/solidity-storage-layout.stateless.spec.ts new file mode 100644 index 0000000000..469f17bb54 --- /dev/null +++ b/services/server/test/integration/apiv1/verification-handlers/solidity-storage-layout.stateless.spec.ts @@ -0,0 +1,207 @@ +import chai from "chai"; +import chaiHttp from "chai-http"; +import { StatusCodes } from "http-status-codes"; +import { + SourcifyChain, + type ISolidityCompiler, + type SourcifyChainMap, +} from "@ethereum-sourcify/lib-sourcify"; +import type { SolidityJsonInput } from "@ethereum-sourcify/compilers-types"; +import { LocalChainFixture } from "../../../helpers/LocalChainFixture"; +import { ServerFixture } from "../../../helpers/ServerFixture"; +import { deployFromAbiAndBytecodeForCreatorTxHash } from "../../../helpers/helpers"; +import { assertVerification } from "../../../helpers/assertions"; + +chai.use(chaiHttp); + +describe("/private/replace-solidity-storage-layout", function () { + const chainId = 31339; + const port = 8555; + const chainFixture = new LocalChainFixture({ + chainId: chainId.toString(), + port, + }); + const serverFixture = new ServerFixture({ + chains: { + [chainId]: new SourcifyChain({ + name: "Historical storage layout test chain", + chainId, + supported: true, + rpcs: [ + { + rpc: `http://localhost:${port}`, + urlWithoutApiKey: `http://localhost:${port}`, + }, + ], + }), + } as SourcifyChainMap, + }); + + it("should backfill and expose a historical Solidity storage layout", async () => { + const compilerVersion = "0.5.12+commit.7709ece9"; + const contractPath = "HistoricalLayout.sol"; + const contractName = "HistoricalLayout"; + const source = ` + pragma solidity 0.5.12; + + contract HistoricalLayout { + struct Record { + uint8 flag; + uint16 count; + } + + uint128 first; + uint8 second; + uint16[3] packed; + mapping(address => Record) records; + Record latest; + } + `; + const jsonInput: SolidityJsonInput = { + language: "Solidity", + sources: { [contractPath]: { content: source } }, + settings: { + optimizer: { enabled: false, runs: 200 }, + outputSelection: { + "*": { + "*": ["abi", "evm.bytecode.object"], + }, + }, + }, + }; + const solc = serverFixture.server.app.get("solc") as ISolidityCompiler; + const compilerOutput = await solc.compile(compilerVersion, jsonInput); + const contractOutput = compilerOutput.contracts[contractPath][contractName]; + const { contractAddress, txHash } = + await deployFromAbiAndBytecodeForCreatorTxHash( + chainFixture.localSigner, + contractOutput.abi, + `0x${contractOutput.evm.bytecode.object}`, + ); + + const verificationResponse = await chai + .request(serverFixture.server.app) + .post("/verify/solc-json") + .attach("files", Buffer.from(JSON.stringify(jsonInput)), "solc.json") + .field("address", contractAddress) + .field("chain", chainFixture.chainId) + .field("creatorTxHash", txHash) + .field("compilerVersion", compilerVersion) + .field("contractName", contractName); + await assertVerification( + serverFixture, + null, + verificationResponse, + null, + contractAddress, + chainFixture.chainId, + "perfect", + ); + + const addressBuffer = Buffer.from(contractAddress.substring(2), "hex"); + const storedResult = await serverFixture.sourcifyDatabase.query( + `SELECT cc.id, cc.compilation_artifacts + FROM verified_contracts vc + JOIN contract_deployments cd ON cd.id = vc.deployment_id + JOIN compiled_contracts cc ON cc.id = vc.compilation_id + WHERE cd.chain_id = $1 AND cd.address = $2`, + [chainFixture.chainId, addressBuffer], + ); + chai.expect(storedResult.rows).to.have.length(1); + const compilationId = storedResult.rows[0].id; + const originalArtifacts = storedResult.rows[0].compilation_artifacts; + chai + .expect( + originalArtifacts.storageLayout.storage.map( + ({ label, offset, slot }: Record) => ({ + label, + offset, + slot, + }), + ), + ) + .to.deep.equal([ + { label: "first", offset: 0, slot: "0" }, + { label: "second", offset: 16, slot: "0" }, + { label: "packed", offset: 0, slot: "1" }, + { label: "records", offset: 0, slot: "2" }, + { label: "latest", offset: 0, slot: "3" }, + ]); + + await serverFixture.sourcifyDatabase.query( + `UPDATE compiled_contracts + SET compilation_artifacts = jsonb_set( + compilation_artifacts, '{storageLayout}', 'null'::jsonb) + WHERE id = $1`, + [compilationId], + ); + + const replaceBody = { + address: contractAddress, + chainId: chainFixture.chainId, + forceCompilation: true, + forceRPCRequest: false, + customReplaceMethod: "replace-solidity-storage-layout", + jsonInput, + compilerVersion, + compilationTarget: `${contractPath}:${contractName}`, + }; + const mismatchedSourceResponse = await chai + .request(serverFixture.server.app) + .post("/private/replace-contract") + .set("authorization", "Bearer sourcify-test-token") + .send({ + ...replaceBody, + jsonInput: { + ...jsonInput, + sources: { + [contractPath]: { content: `${source}\n` }, + }, + }, + }); + chai + .expect(mismatchedSourceResponse.status) + .to.equal(StatusCodes.BAD_REQUEST); + chai + .expect(mismatchedSourceResponse.body.message) + .to.contain("compilation identity does not match"); + + const replaceResponse = await chai + .request(serverFixture.server.app) + .post("/private/replace-contract") + .set("authorization", "Bearer sourcify-test-token") + .send(replaceBody); + chai.expect(replaceResponse.status).to.equal(StatusCodes.OK); + chai.expect(replaceResponse.body.replaced).to.be.true; + + const restoredResult = await serverFixture.sourcifyDatabase.query( + "SELECT compilation_artifacts FROM compiled_contracts WHERE id = $1", + [compilationId], + ); + chai + .expect(restoredResult.rows[0].compilation_artifacts) + .to.deep.equal(originalArtifacts); + + const lookupResponse = await chai + .request(serverFixture.server.app) + .get( + `/v2/contract/${chainFixture.chainId}/${contractAddress}?fields=storageLayout`, + ); + chai.expect(lookupResponse.status).to.equal(StatusCodes.OK); + chai + .expect(lookupResponse.body.storageLayout) + .to.deep.equal(originalArtifacts.storageLayout); + + const repeatedReplaceResponse = await chai + .request(serverFixture.server.app) + .post("/private/replace-contract") + .set("authorization", "Bearer sourcify-test-token") + .send(replaceBody); + chai + .expect(repeatedReplaceResponse.status) + .to.equal(StatusCodes.BAD_REQUEST); + chai + .expect(repeatedReplaceResponse.body.message) + .to.contain("storageLayout is already populated"); + }); +}); diff --git a/services/server/test/unit/solidityStorageLayoutReplace.spec.ts b/services/server/test/unit/solidityStorageLayoutReplace.spec.ts new file mode 100644 index 0000000000..8efe0e8d06 --- /dev/null +++ b/services/server/test/unit/solidityStorageLayoutReplace.spec.ts @@ -0,0 +1,202 @@ +import { expect } from "chai"; +import sinon from "sinon"; +import { replaceSolidityStorageLayout } from "../../src/server/apiv1/verification/private/stateless/solidityStorageLayoutReplace"; + +const storageLayout = { + storage: [ + { + astId: 1, + contract: "Fixture.sol:Fixture", + label: "owner", + offset: 0, + slot: "0", + type: "t_address", + }, + ], + types: { + t_address: { + encoding: "inplace", + label: "address", + numberOfBytes: "20", + }, + }, +}; + +function verification(options?: { + language?: string; + version?: string; + runtimeMatch?: "perfect" | "partial" | null; + layout?: unknown; +}) { + return { + address: "0x0000000000000000000000000000000000000001", + chainId: 1, + status: { + runtimeMatch: + options && "runtimeMatch" in options ? options.runtimeMatch : "perfect", + creationMatch: null, + }, + compilation: { + language: options?.language ?? "Solidity", + compilerVersion: options?.version ?? "0.5.12+commit.7709ece9", + compilationTarget: { path: "Fixture.sol", name: "Fixture" }, + sources: { "Fixture.sol": "contract Fixture { address owner; }\n" }, + jsonInput: { + settings: { + evmVersion: "petersburg", + outputSelection: { "*": { "*": ["storageLayout"] } }, + }, + }, + contractCompilerOutput: { + storageLayout: + options && "layout" in options ? options.layout : storageLayout, + }, + }, + } as any; +} + +describe("replaceSolidityStorageLayout", () => { + it("rejects non-Solidity compilations", async () => { + const query = sinon.stub(); + + await expectFailure( + replaceSolidityStorageLayout( + { database: { pool: { query } } } as any, + verification({ language: "Vyper" }), + ), + "only supports Solidity contracts", + ); + expect(query.called).to.equal(false); + }); + + for (const version of [ + "0.3.6+commit.3fc68da5", + "0.4.6+commit.2dabbdf0", + "0.5.13+commit.5b0b510c", + ]) { + it(`rejects compiler version ${version}`, async () => { + const query = sinon.stub(); + + await expectFailure( + replaceSolidityStorageLayout( + { database: { pool: { query } } } as any, + verification({ version }), + ), + "versions from 0.4.7 through 0.5.12", + ); + expect(query.called).to.equal(false); + }); + } + + it("rejects a layout from a compilation that did not match", async () => { + const query = sinon.stub(); + + await expectFailure( + replaceSolidityStorageLayout( + { database: { pool: { query } } } as any, + verification({ runtimeMatch: null }), + ), + "did not match the deployment", + ); + expect(query.called).to.equal(false); + }); + + it("leaves the row untouched when recovery returned no layout", async () => { + const query = sinon.stub(); + + const result = await replaceSolidityStorageLayout( + { database: { pool: { query } } } as any, + verification({ layout: undefined }), + ); + + expect(result).to.deep.equal({ + reason: "Historical Solidity storage layout could not be recovered", + replaced: false, + }); + expect(query.called).to.equal(false); + }); + + it("rejects a malformed recovered layout", async () => { + const query = sinon.stub(); + + await expectFailure( + replaceSolidityStorageLayout( + { database: { pool: { query } } } as any, + verification({ layout: {} }), + ), + "invalid structure", + ); + expect(query.called).to.equal(false); + }); + + it("writes only a missing layout for an exact compilation identity", async () => { + const query = sinon.stub(); + query.onFirstCall().resolves({ rows: [{ compilation_id: "7" }] }); + query.onSecondCall().resolves({ rows: [{ id: "7" }] }); + + await replaceSolidityStorageLayout( + { database: { pool: { query } } } as any, + verification(), + ); + + expect(query.callCount).to.equal(2); + const updateSql = query.secondCall.args[0] as string; + expect(updateSql).to.contain("compilation_artifacts = jsonb_set"); + expect(updateSql).to.contain("cc.language = 'solidity'"); + expect(updateSql).to.contain("compiler_settings = $4::jsonb"); + expect(updateSql).to.contain( + "cc.compilation_artifacts->'storageLayout' IS NULL", + ); + expect(updateSql).to.contain( + "cc.compilation_artifacts->'storageLayout' = 'null'::jsonb", + ); + expect(query.secondCall.args[1]).to.deep.equal([ + "7", + "0.5.12+commit.7709ece9", + "Fixture.sol:Fixture", + '{"evmVersion":"petersburg"}', + null, + '{"Fixture.sol":"contract Fixture { address owner; }\\n"}', + JSON.stringify(storageLayout), + ]); + }); + + it("accepts the empty native layout shape", async () => { + const query = sinon.stub(); + query.onFirstCall().resolves({ rows: [{ compilation_id: "7" }] }); + query.onSecondCall().resolves({ rows: [{ id: "7" }] }); + + await replaceSolidityStorageLayout( + { database: { pool: { query } } } as any, + verification({ layout: { storage: [], types: null } }), + ); + + expect(query.secondCall.args[1][6]).to.equal('{"storage":[],"types":null}'); + }); + + it("refuses identity mismatches and populated layouts", async () => { + const query = sinon.stub(); + query.onFirstCall().resolves({ rows: [{ compilation_id: "7" }] }); + query.onSecondCall().resolves({ rows: [] }); + + await expectFailure( + replaceSolidityStorageLayout( + { database: { pool: { query } } } as any, + verification(), + ), + "already populated", + ); + }); +}); + +async function expectFailure( + promise: Promise, + expectedMessage: string, +) { + try { + await promise; + expect.fail("expected operation to fail"); + } catch (error) { + expect((error as Error).message).to.contain(expectedMessage); + } +} From 2e8bf295e4f8a597ad065045e191d7d912d43cba Mon Sep 17 00:00:00 2001 From: banteg <4562643+banteg@users.noreply.github.com> Date: Sun, 12 Jul 2026 00:17:59 +0400 Subject: [PATCH 2/2] test(server): update historical layout fixtures --- .../testdata/match_sol_0_4_7.json | 20 ++++++++++++++++++- .../testdata/match_sol_0_4_9.json | 20 ++++++++++++++++++- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/services/server/test/integration/verification-cases/testdata/match_sol_0_4_7.json b/services/server/test/integration/verification-cases/testdata/match_sol_0_4_7.json index 34d6630657..d92227b0ce 100644 --- a/services/server/test/integration/verification-cases/testdata/match_sol_0_4_7.json +++ b/services/server/test/integration/verification-cases/testdata/match_sol_0_4_7.json @@ -57,7 +57,25 @@ ], "devdoc": null, "userdoc": null, - "storageLayout": null, + "storageLayout": { + "storage": [ + { + "astId": 5610124, + "contract": "contracts/SimpleStorage.sol:SimpleStorage", + "label": "storedData", + "offset": 0, + "slot": "0", + "type": "t_uint256" + } + ], + "types": { + "t_uint256": { + "encoding": "inplace", + "label": "uint256", + "numberOfBytes": "32" + } + } + }, "transientStorageLayout": null, "sources": { "contracts/SimpleStorage.sol": { diff --git a/services/server/test/integration/verification-cases/testdata/match_sol_0_4_9.json b/services/server/test/integration/verification-cases/testdata/match_sol_0_4_9.json index fdb3a8088b..f65e9f308c 100644 --- a/services/server/test/integration/verification-cases/testdata/match_sol_0_4_9.json +++ b/services/server/test/integration/verification-cases/testdata/match_sol_0_4_9.json @@ -57,7 +57,25 @@ ], "devdoc": null, "userdoc": null, - "storageLayout": null, + "storageLayout": { + "storage": [ + { + "astId": 3, + "contract": "contracts/SimpleStorage.sol:SimpleStorage", + "label": "storedData", + "offset": 0, + "slot": "0", + "type": "t_uint256" + } + ], + "types": { + "t_uint256": { + "encoding": "inplace", + "label": "uint256", + "numberOfBytes": "32" + } + } + }, "transientStorageLayout": null, "sources": { "contracts/SimpleStorage.sol": {