From 6ab912b3e440a37aa2550baa3128c3e96586a6af Mon Sep 17 00:00:00 2001 From: nischit Date: Fri, 29 Aug 2025 21:41:50 +0545 Subject: [PATCH 1/4] null table schema update --- ...04_clickhouse_create_insert_null_table.sql | 34 ++-- ...7_clickhouse_create_token_transfers_mv.sql | 158 ------------------ 2 files changed, 22 insertions(+), 170 deletions(-) delete mode 100644 internal/tools/clickhouse/0007_clickhouse_create_token_transfers_mv.sql diff --git a/internal/tools/clickhouse/0004_clickhouse_create_insert_null_table.sql b/internal/tools/clickhouse/0004_clickhouse_create_insert_null_table.sql index 8597fcd..8ac3f55 100644 --- a/internal/tools/clickhouse/0004_clickhouse_create_insert_null_table.sql +++ b/internal/tools/clickhouse/0004_clickhouse_create_insert_null_table.sql @@ -1,6 +1,6 @@ -CREATE TABLE IF NOT EXISTS insert_null_block_data ( +CREATE TABLE default.insert_null_block_data +( chain_id UInt256, - block Tuple( block_number UInt256, block_timestamp DateTime, @@ -22,8 +22,7 @@ CREATE TABLE IF NOT EXISTS insert_null_block_data ( gas_limit UInt256, gas_used UInt256, withdrawals_root FixedString(66), - base_fee_per_gas Nullable(UInt64) - ), + base_fee_per_gas Nullable(UInt64)), transactions Array(Tuple( hash FixedString(66), nonce UInt64, @@ -55,8 +54,7 @@ CREATE TABLE IF NOT EXISTS insert_null_block_data ( blob_gas_used Nullable(UInt64), blob_gas_price Nullable(UInt256), logs_bloom Nullable(String), - status Nullable(UInt64) - )), + status Nullable(UInt64))), logs Array(Tuple( block_number UInt256, block_hash FixedString(66), @@ -69,8 +67,7 @@ CREATE TABLE IF NOT EXISTS insert_null_block_data ( topic_0 String, topic_1 String, topic_2 String, - topic_3 String - )), + topic_3 String)), traces Array(Tuple( block_number UInt256, block_hash FixedString(66), @@ -91,9 +88,22 @@ CREATE TABLE IF NOT EXISTS insert_null_block_data ( value UInt256, author Nullable(FixedString(42)), reward_type LowCardinality(Nullable(String)), - refund_address Nullable(FixedString(42)) - )), - + refund_address Nullable(FixedString(42)))), + token_transfers Array(Tuple( + chain_id UInt256, + token_type LowCardinality(String), + token_address FixedString(42), + token_id UInt256, + from_address FixedString(42), + to_address FixedString(42), + block_number UInt256, + block_timestamp DateTime, + transaction_hash FixedString(66), + transaction_index UInt64, + amount UInt256, + log_index UInt64, + batch_index Nullable(UInt16))), insert_timestamp DateTime DEFAULT now(), is_deleted UInt8 DEFAULT 0 -) ENGINE = Null; +) +ENGINE = Null \ No newline at end of file diff --git a/internal/tools/clickhouse/0007_clickhouse_create_token_transfers_mv.sql b/internal/tools/clickhouse/0007_clickhouse_create_token_transfers_mv.sql deleted file mode 100644 index 30d01a5..0000000 --- a/internal/tools/clickhouse/0007_clickhouse_create_token_transfers_mv.sql +++ /dev/null @@ -1,158 +0,0 @@ --- ERC20 -CREATE MATERIALIZED VIEW IF NOT EXISTS token_transfers_erc20_mv -TO token_transfers -AS -SELECT - chain_id, - address AS token_address, - 'erc20' AS token_type, - 0 AS token_id, - concat('0x', substring(topic_1, 27, 40)) AS from_address, - concat('0x', substring(topic_2, 27, 40)) AS to_address, - block_number, - block_timestamp, - transaction_hash, - transaction_index, - reinterpretAsUInt256(reverse(unhex(substring(data, 3, 64)))) AS amount, - log_index, - CAST(NULL AS Nullable(UInt16)) AS batch_index, - insert_timestamp, - is_deleted -FROM logs -WHERE topic_0 = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' -- Transfer - AND length(topic_1) = 66 AND startsWith(topic_1, '0x') - AND length(topic_2) = 66 AND startsWith(topic_2, '0x') - AND topic_3 = '' - AND length(data) = 66; - --- ERC721 -CREATE MATERIALIZED VIEW IF NOT EXISTS token_transfers_erc721_mv -TO token_transfers -AS -SELECT - chain_id, - address AS token_address, - 'erc721' AS token_type, - reinterpretAsUInt256(reverse(unhex(substring(topic_3, 3, 64)))) AS token_id, - concat('0x', substring(topic_1, 27, 40)) AS from_address, - concat('0x', substring(topic_2, 27, 40)) AS to_address, - block_number, - block_timestamp, - transaction_hash, - transaction_index, - toUInt8(1) AS amount, - log_index, - CAST(NULL AS Nullable(UInt16)) AS batch_index, - insert_timestamp, - is_deleted -FROM logs -WHERE topic_0 = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' - AND length(topic_1) = 66 AND startsWith(topic_1, '0x') - AND length(topic_2) = 66 AND startsWith(topic_2, '0x') - AND length(topic_3) = 66 AND startsWith(topic_3, '0x') - AND length(data) = 2; - --- ERC1155 (single) -CREATE MATERIALIZED VIEW IF NOT EXISTS token_transfers_erc1155_single_mv -TO token_transfers -AS -SELECT - chain_id, - address AS token_address, - 'erc1155' AS token_type, - reinterpretAsUInt256(reverse(unhex(substring(data, 3, 64)))) AS token_id, - concat('0x', substring(topic_2, 27, 40)) AS from_address, - concat('0x', substring(topic_3, 27, 40)) AS to_address, - block_number, - block_timestamp, - transaction_hash, - transaction_index, - reinterpretAsUInt256(reverse(unhex(substring(data, 67, 64)))) AS amount, - log_index, - toNullable(toUInt16(0)) AS batch_index, - insert_timestamp, - is_deleted -FROM logs -WHERE topic_0 = '0xc3d58168c5ae7397731d063d5bbf3d657854427343f4c083240f7aacaa2d0f62' -- TransferSingle - AND length(topic_2) = 66 AND length(topic_3) = 66 - AND length(data) = (2 + 2*64); - --- ERC1155 (batch) -CREATE MATERIALIZED VIEW IF NOT EXISTS token_transfers_erc1155_batch_mv -TO token_transfers -AS -SELECT - chain_id, - address AS token_address, - 'erc1155' AS token_type, - reinterpretAsUInt256(reverse(substring(bin, (ids_base + ((i - 1) * 32)) + 1, 32))) AS token_id, - concat('0x', substring(topic_2, 27, 40)) AS from_address, - concat('0x', substring(topic_3, 27, 40)) AS to_address, - block_number, - block_timestamp, - transaction_hash, - transaction_index, - reinterpretAsUInt256(reverse(substring(bin, (am_base + ((i - 1) * 32)) + 1, 32))) AS amount, - log_index, - toNullable(toUInt16(i - 1)) AS batch_index, - insert_timestamp, - is_deleted -FROM ( - SELECT - chain_id, - address, - topic_2, - topic_3, - block_number, - block_timestamp, - transaction_hash, - transaction_index, - log_index, - is_deleted, - insert_timestamp, - unhex(substring(data, 3)) AS bin, - length(unhex(substring(data, 3))) AS bin_len, - toUInt32(reinterpretAsUInt256(reverse(substring(unhex(substring(data, 3)), 1, 32)))) AS ids_off, - toUInt32(reinterpretAsUInt256(reverse(substring(unhex(substring(data, 3)), 33, 32)))) AS am_off, - toUInt32(reinterpretAsUInt256(reverse(substring(unhex(substring(data, 3)), ids_off + 1, 32)))) AS ids_len, - toUInt32(reinterpretAsUInt256(reverse(substring(unhex(substring(data, 3)), am_off + 1, 32)))) AS am_len, - ids_off + 32 AS ids_base, - am_off + 32 AS am_base -FROM default.logs -WHERE (topic_0 = '0x4a39dc06d4c0dbc64b70af90fd698a233a518aa5d07e595d983b8c0526c8f7fb') - AND (length(topic_2) = 66) - AND (length(topic_3) = 66) - AND (ids_len = am_len) - AND (ids_len > 0) - AND ((ids_off + 32) <= bin_len) - AND ((am_off + 32) <= bin_len) - AND ((ids_base + (ids_len * 32)) <= bin_len) - AND ((am_base + (am_len * 32)) <= bin_len) -) ARRAY JOIN range(1, ids_len + 1) AS i; - - --- ERC6909 -CREATE MATERIALIZED VIEW IF NOT EXISTS token_transfers_erc6909_mv -TO token_transfers -AS -SELECT - chain_id, - address AS token_address, - 'erc6909' AS token_type, - reinterpretAsUInt256(reverse(unhex(substring(topic_3, 3, 64)))) AS token_id, - concat('0x', substring(topic_1, 27, 40)) AS from_address, - concat('0x', substring(topic_2, 27, 40)) AS to_address, - block_number, - block_timestamp, - transaction_hash, - transaction_index, - reinterpretAsUInt256(reverse(unhex(substring(data, 67, 64)))) AS amount, - log_index, - CAST(NULL AS Nullable(UInt16)) AS batch_index, - insert_timestamp, - is_deleted -FROM logs -WHERE topic_0 = '0x1b3d7edb2e9c0b0e7c525b20aaaef0f5940d2ed71663c7d39266ecafac728859' - AND length(topic_1) = 66 - AND length(topic_2) = 66 - AND length(data) == 2 + 128; \ No newline at end of file From c0ebbc1b47230bb22ded2e298e5ad9291c9c57b7 Mon Sep 17 00:00:00 2001 From: nischit Date: Fri, 29 Aug 2025 23:04:32 +0545 Subject: [PATCH 2/4] insert token transfers mv --- ...house_create_insert_token_transfers_mv.sql | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 internal/tools/clickhouse/0004_clickhouse_create_insert_token_transfers_mv.sql diff --git a/internal/tools/clickhouse/0004_clickhouse_create_insert_token_transfers_mv.sql b/internal/tools/clickhouse/0004_clickhouse_create_insert_token_transfers_mv.sql new file mode 100644 index 0000000..7ed7faf --- /dev/null +++ b/internal/tools/clickhouse/0004_clickhouse_create_insert_token_transfers_mv.sql @@ -0,0 +1,36 @@ +CREATE MATERIALIZED VIEW default.insert_token_transfers_mv TO default.token_transfers +( + chain_id UInt256, + token_type LowCardinality(String), + token_address FixedString(42), + token_id UInt256, + from_address FixedString(42), + to_address FixedString(42), + block_number UInt256, + block_timestamp DateTime, + transaction_hash FixedString(66), + transaction_index UInt64, + amount UInt256, + log_index UInt64, + batch_index Nullable(UInt16), + insert_timestamp DateTime, + is_deleted UInt8 +) +AS SELECT + chain_id, + tt.2 AS token_type, + tt.3 AS token_address, + tt.4 AS token_id, + tt.5 AS from_address, + tt.6 AS to_address, + tt.7 AS block_number, + tt.8 AS block_timestamp, + tt.9 AS transaction_hash, + tt.10 AS transaction_index, + tt.11 AS amount, + tt.12 AS log_index, + tt.13 AS batch_index, + insert_timestamp, + is_deleted +FROM default.insert_null_block_data +ARRAY JOIN token_transfers AS tt \ No newline at end of file From dc75352db74aaed5d2114a722968de2238ac79d7 Mon Sep 17 00:00:00 2001 From: nischit Date: Fri, 29 Aug 2025 23:05:14 +0545 Subject: [PATCH 3/4] minor change --- ...v.sql => 0014_clickhouse_create_insert_token_transfers_mv.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename internal/tools/clickhouse/{0004_clickhouse_create_insert_token_transfers_mv.sql => 0014_clickhouse_create_insert_token_transfers_mv.sql} (100%) diff --git a/internal/tools/clickhouse/0004_clickhouse_create_insert_token_transfers_mv.sql b/internal/tools/clickhouse/0014_clickhouse_create_insert_token_transfers_mv.sql similarity index 100% rename from internal/tools/clickhouse/0004_clickhouse_create_insert_token_transfers_mv.sql rename to internal/tools/clickhouse/0014_clickhouse_create_insert_token_transfers_mv.sql From eea1630bf95a99e5f60519b241c8b4f04ff9f39e Mon Sep 17 00:00:00 2001 From: Jake Loo <2171134+jakeloo@users.noreply.github.com> Date: Fri, 29 Aug 2025 20:37:07 +0000 Subject: [PATCH 4/4] nit schema update --- ...04_clickhouse_create_insert_null_table.sql | 3 +- ...ckhouse_create_address_transactions_mv.sql | 2 +- ...clickhouse_create_address_transfers_mv.sql | 2 +- ...house_create_insert_token_transfers_mv.sql | 46 ++++++------------- 4 files changed, 18 insertions(+), 35 deletions(-) diff --git a/internal/tools/clickhouse/0004_clickhouse_create_insert_null_table.sql b/internal/tools/clickhouse/0004_clickhouse_create_insert_null_table.sql index 8ac3f55..6388831 100644 --- a/internal/tools/clickhouse/0004_clickhouse_create_insert_null_table.sql +++ b/internal/tools/clickhouse/0004_clickhouse_create_insert_null_table.sql @@ -1,4 +1,4 @@ -CREATE TABLE default.insert_null_block_data +CREATE TABLE IF NOT EXISTS insert_null_block_data ( chain_id UInt256, block Tuple( @@ -90,7 +90,6 @@ CREATE TABLE default.insert_null_block_data reward_type LowCardinality(Nullable(String)), refund_address Nullable(FixedString(42)))), token_transfers Array(Tuple( - chain_id UInt256, token_type LowCardinality(String), token_address FixedString(42), token_id UInt256, diff --git a/internal/tools/clickhouse/0011_clickhouse_create_address_transactions_mv.sql b/internal/tools/clickhouse/0011_clickhouse_create_address_transactions_mv.sql index c5b5ac7..db16ab5 100644 --- a/internal/tools/clickhouse/0011_clickhouse_create_address_transactions_mv.sql +++ b/internal/tools/clickhouse/0011_clickhouse_create_address_transactions_mv.sql @@ -1,4 +1,4 @@ -CREATE MATERIALIZED VIEW address_transactions_mv +CREATE MATERIALIZED VIEW IF NOT EXISTS address_transactions_mv TO address_transactions AS SELECT diff --git a/internal/tools/clickhouse/0013_clickhouse_create_address_transfers_mv.sql b/internal/tools/clickhouse/0013_clickhouse_create_address_transfers_mv.sql index 0a7d2cc..d3bdfcb 100644 --- a/internal/tools/clickhouse/0013_clickhouse_create_address_transfers_mv.sql +++ b/internal/tools/clickhouse/0013_clickhouse_create_address_transfers_mv.sql @@ -1,4 +1,4 @@ -CREATE MATERIALIZED VIEW address_transfers_mv +CREATE MATERIALIZED VIEW IF NOT EXISTS address_transfers_mv TO address_transfers AS SELECT diff --git a/internal/tools/clickhouse/0014_clickhouse_create_insert_token_transfers_mv.sql b/internal/tools/clickhouse/0014_clickhouse_create_insert_token_transfers_mv.sql index 7ed7faf..1db6386 100644 --- a/internal/tools/clickhouse/0014_clickhouse_create_insert_token_transfers_mv.sql +++ b/internal/tools/clickhouse/0014_clickhouse_create_insert_token_transfers_mv.sql @@ -1,36 +1,20 @@ -CREATE MATERIALIZED VIEW default.insert_token_transfers_mv TO default.token_transfers -( - chain_id UInt256, - token_type LowCardinality(String), - token_address FixedString(42), - token_id UInt256, - from_address FixedString(42), - to_address FixedString(42), - block_number UInt256, - block_timestamp DateTime, - transaction_hash FixedString(66), - transaction_index UInt64, - amount UInt256, - log_index UInt64, - batch_index Nullable(UInt16), - insert_timestamp DateTime, - is_deleted UInt8 -) +CREATE MATERIALIZED VIEW IF NOT EXISTS insert_token_transfers_mv +TO token_transfers AS SELECT chain_id, - tt.2 AS token_type, - tt.3 AS token_address, - tt.4 AS token_id, - tt.5 AS from_address, - tt.6 AS to_address, - tt.7 AS block_number, - tt.8 AS block_timestamp, - tt.9 AS transaction_hash, - tt.10 AS transaction_index, - tt.11 AS amount, - tt.12 AS log_index, - tt.13 AS batch_index, + tt.1 AS token_type, + tt.2 AS token_address, + tt.3 AS token_id, + tt.4 AS from_address, + tt.5 AS to_address, + tt.6 AS block_number, + tt.7 AS block_timestamp, + tt.8 AS transaction_hash, + tt.9 AS transaction_index, + tt.10 AS amount, + tt.11 AS log_index, + tt.12 AS batch_index, insert_timestamp, is_deleted -FROM default.insert_null_block_data +FROM insert_null_block_data ARRAY JOIN token_transfers AS tt \ No newline at end of file