Skip to content

Commit e6b2825

Browse files
committed
starrocks: add some test cases for partition
Signed-off-by: jaogoy <jaogoy@gmail.com>
1 parent 90b1fad commit e6b2825

3 files changed

Lines changed: 222 additions & 8 deletions

File tree

tests/core/engine_adapter/integration/test_integration_starrocks.py

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1434,6 +1434,163 @@ def test_e2e_partition_list(self, starrocks_adapter: StarRocksEngineAdapter):
14341434
finally:
14351435
starrocks_adapter.drop_schema(db_name, ignore_if_not_exists=True)
14361436

1437+
# ========================================
1438+
# Case 6B: Expression Partitioning (table + MV)
1439+
# Covers: expression partitioning with/without functions, table vs MV
1440+
# ========================================
1441+
1442+
@pytest.mark.parametrize(
1443+
"partition_expr",
1444+
[
1445+
"(event_date, region)", # plain columns
1446+
"date_trunc('day', event_date)", # one function expression
1447+
"(from_unixtime(ts), region)", # multiple expressions
1448+
],
1449+
)
1450+
def test_e2e_partition_expression_for_table(
1451+
self,
1452+
starrocks_adapter: StarRocksEngineAdapter,
1453+
partition_expr: str,
1454+
):
1455+
"""Expression partitioning for regular tables (outer paren only when no functions)."""
1456+
db_name = "sr_e2e_part_expr_tbl_db"
1457+
table_name = f"{db_name}.sr_part_expr_table"
1458+
1459+
model_sql = f"""
1460+
MODEL (
1461+
name test.partition_expr_table,
1462+
kind FULL,
1463+
dialect starrocks,
1464+
columns (
1465+
id BIGINT,
1466+
ts BIGINT,
1467+
event_date DATE,
1468+
region VARCHAR(50)
1469+
),
1470+
partitioned_by {partition_expr},
1471+
);
1472+
SELECT *
1473+
"""
1474+
1475+
try:
1476+
starrocks_adapter.create_schema(db_name, ignore_if_exists=True)
1477+
params = self._parse_model_and_get_all_params(model_sql)
1478+
starrocks_adapter.create_table(table_name, **params)
1479+
1480+
ddl = fetchone_or_fail(starrocks_adapter, f"SHOW CREATE TABLE {table_name}")[1]
1481+
logger.info(f"Case 6B DDL:\n{ddl}")
1482+
ddl_upper = ddl.upper()
1483+
assert "PARTITION BY" in ddl_upper
1484+
1485+
before, after = ddl_upper.split("PARTITION BY", 1)
1486+
after = after.lstrip()
1487+
1488+
# Column/function presence
1489+
if "DATE" in partition_expr.upper():
1490+
assert "EVENT_DATE" in after
1491+
else:
1492+
assert "REGION" in after
1493+
if "FROM_UNIXTIME" in partition_expr.upper():
1494+
assert "FROM_UNIXTIME" in after or \
1495+
("FROM_UNIXTIME" in before and "__GENERATED_PARTITION_COLUMN" in after)
1496+
if "DATE_TRUNC" in partition_expr.upper():
1497+
assert "DATE_TRUNC" in after
1498+
finally:
1499+
starrocks_adapter.drop_schema(db_name, ignore_if_not_exists=True)
1500+
1501+
@pytest.mark.parametrize(
1502+
"partition_clause,has_func",
1503+
[
1504+
("(event_date, region)", False),
1505+
("(date_trunc('day', event_date), region)", True),
1506+
],
1507+
)
1508+
def test_e2e_partition_expression_for_mv(
1509+
self,
1510+
starrocks_adapter: StarRocksEngineAdapter,
1511+
partition_clause: str,
1512+
has_func: bool,
1513+
):
1514+
"""Expression partitioning for MVs should always keep outer parentheses."""
1515+
db_name = "sr_e2e_part_expr_mv_db"
1516+
src_table = f"{db_name}.sr_part_expr_src"
1517+
mv_table = f"{db_name}.sr_part_expr_mv"
1518+
1519+
try:
1520+
starrocks_adapter.create_schema(db_name, ignore_if_exists=True)
1521+
1522+
# Source table + data
1523+
starrocks_adapter.create_table(
1524+
src_table,
1525+
target_columns_to_types={
1526+
"id": exp.DataType.build("BIGINT"),
1527+
"ts": exp.DataType.build("BIGINT"),
1528+
"event_date": exp.DataType.build("DATE"),
1529+
"region": exp.DataType.build("VARCHAR(50)"),
1530+
},
1531+
primary_key=("id", "event_date", "region"),
1532+
table_properties={
1533+
"partitioned_by": partition_clause,
1534+
},
1535+
)
1536+
starrocks_adapter.execute(
1537+
f"""
1538+
INSERT INTO {src_table} (id, ts, event_date, region)
1539+
VALUES (1, 1700000000, '2024-01-01', 'us')
1540+
"""
1541+
)
1542+
1543+
model_sql = f"""
1544+
MODEL (
1545+
name test.partition_expr_mv,
1546+
kind VIEW (
1547+
materialized true
1548+
),
1549+
dialect starrocks,
1550+
columns (
1551+
id BIGINT,
1552+
ts BIGINT,
1553+
event_date DATE,
1554+
region VARCHAR(50)
1555+
),
1556+
partitioned_by {partition_clause},
1557+
physical_properties (
1558+
distributed_by = 'HASH(id) BUCKETS 2',
1559+
refresh_moment = 'IMMEDIATE',
1560+
refresh_scheme = 'ASYNC'
1561+
)
1562+
);
1563+
SELECT id, ts, event_date, region FROM {src_table};
1564+
"""
1565+
1566+
model = _load_sql_model(model_sql)
1567+
query = model.render_query()
1568+
assert query is not None
1569+
materialized_properties = _materialized_properties_from_model(model)
1570+
1571+
starrocks_adapter.create_view(
1572+
mv_table,
1573+
query,
1574+
replace=True,
1575+
materialized=True,
1576+
target_columns_to_types=model.columns_to_types,
1577+
materialized_properties=materialized_properties,
1578+
view_properties=model.physical_properties,
1579+
)
1580+
1581+
ddl = fetchone_or_fail(
1582+
starrocks_adapter, f"SHOW CREATE MATERIALIZED VIEW {mv_table}"
1583+
)[1]
1584+
logger.info(f"Case 6B DDL:\n{ddl}")
1585+
ddl_upper = ddl.upper()
1586+
assert "PARTITION BY" in ddl_upper
1587+
after = ddl_upper.split("PARTITION BY", 1)[1].lstrip()
1588+
assert after.startswith("("), f"MV partition should keep parentheses, got: {after[:50]}"
1589+
assert "REGION" in after
1590+
assert ("DATE_TRUNC" in after) == has_func
1591+
finally:
1592+
starrocks_adapter.drop_schema(db_name, ignore_if_not_exists=True)
1593+
14371594
# ========================================
14381595
# Case 7: Other Key Types (test_design.md Case 7)
14391596
# Covers: duplicate_key, unique_key, aggregate_key

tests/core/engine_adapter/test_starrocks.py

Lines changed: 64 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -809,9 +809,11 @@ class TestPartitionPropertyBuilding:
809809
"partition_expr,expected_clause",
810810
[
811811
# Expression partitioning - single column
812-
("'dt'", "PARTITION BY (dt)"),
812+
("'dt'", "PARTITION BY (`dt`)"),
813813
# Expression partitioning - multi-column
814-
("(year, month)", "PARTITION BY (year, month)"),
814+
("(year, month)", "PARTITION BY (`year`, `month`)"),
815+
# Expression partitioning - multi-column with func
816+
("(date_trunc('day', dt), region)", "PARTITION BY DATE_TRUNC('DAY', `dt`), `region`"),
815817
# RANGE partitioning
816818
("RANGE (dt)", "PARTITION BY RANGE (`dt`) ()"),
817819
# LIST partitioning
@@ -852,6 +854,61 @@ def test_partitioned_by_forms(
852854
sql = to_sql_calls(adapter)[0]
853855
assert expected_clause in sql
854856

857+
@pytest.mark.parametrize(
858+
"partition_expr,expected_clause",
859+
[
860+
("(year, month)", "PARTITION BY (`year`, `month`)"),
861+
(
862+
"(date_trunc('day', dt), region)",
863+
"PARTITION BY (DATE_TRUNC('DAY', `dt`), `region`)",
864+
),
865+
(
866+
"(from_unixtime(dt))",
867+
"PARTITION BY (FROM_UNIXTIME(`dt`))",
868+
),
869+
],
870+
)
871+
def test_partitioned_by_forms_for_mv(
872+
self,
873+
make_mocked_engine_adapter: t.Callable[..., StarRocksEngineAdapter],
874+
partition_expr: str,
875+
expected_clause: str,
876+
):
877+
"""MV partition_by should keep outer parentheses when rendering partition tuples."""
878+
model_sql = f"""
879+
MODEL (
880+
name test_schema.test_mv_partition,
881+
kind VIEW (
882+
materialized true,
883+
),
884+
dialect starrocks,
885+
columns (dt DATE, region STRING, year INT, month INT),
886+
physical_properties (
887+
partition_by = {partition_expr}
888+
)
889+
);
890+
SELECT dt, region, year, month FROM src;
891+
"""
892+
893+
model = _load_sql_model(model_sql)
894+
materialized_properties = (
895+
{"partitioned_by": model.partitioned_by} if model.partitioned_by else None
896+
)
897+
898+
adapter = make_mocked_engine_adapter(StarRocksEngineAdapter)
899+
adapter.create_view(
900+
model.name,
901+
model.render_query(),
902+
materialized=True,
903+
replace=False,
904+
target_columns_to_types=_columns(model),
905+
materialized_properties=materialized_properties,
906+
view_properties=model.physical_properties,
907+
)
908+
909+
sql = to_sql_calls(adapter)[0]
910+
assert expected_clause in sql
911+
855912
def test_partition_by_alias(
856913
self, make_mocked_engine_adapter: t.Callable[..., StarRocksEngineAdapter]
857914
):
@@ -881,7 +938,7 @@ def test_partition_by_alias(
881938
)
882939

883940
sql = to_sql_calls(adapter)[0]
884-
assert "PARTITION BY (year, month)" in sql
941+
assert "PARTITION BY (year, month)" in sql or "PARTITION BY (`year`, `month`)" in sql
885942

886943
def test_partitioned_by_as_model_parameter(
887944
self, make_mocked_engine_adapter: t.Callable[..., StarRocksEngineAdapter]
@@ -909,7 +966,7 @@ def test_partitioned_by_as_model_parameter(
909966
)
910967

911968
sql = to_sql_calls(adapter)[0]
912-
assert "PARTITION BY (year, month)" in sql
969+
assert "PARTITION BY (year, month)" in sql or "PARTITION BY (`year`, `month`)" in sql
913970

914971
def test_partitions_value_forms(
915972
self, make_mocked_engine_adapter: t.Callable[..., StarRocksEngineAdapter]
@@ -1354,10 +1411,10 @@ def _build_mv_model(self, property_sql: str) -> SqlModel:
13541411
model_sql = f"""
13551412
MODEL (
13561413
name test_schema.test_mv_refresh_model,
1357-
kind VIEW,
1414+
kind VIEW (materialized true),
13581415
dialect starrocks,
13591416
columns (a INT),
1360-
virtual_properties (
1417+
physical_properties (
13611418
{property_sql}
13621419
)
13631420
);
@@ -1377,7 +1434,7 @@ def _create_simple_mv(
13771434
replace=False,
13781435
materialized=True,
13791436
target_columns_to_types=_columns(model),
1380-
view_properties=model.virtual_properties,
1437+
view_properties=model.physical_properties,
13811438
)
13821439
# replace=False → only CREATE statement is emitted
13831440
return to_sql_calls(adapter)[-1]

tests/core/test_connection_config.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1971,7 +1971,7 @@ def test_starrocks(make_config):
19711971
assert config.database == "testdb"
19721972
assert config.DIALECT == "starrocks"
19731973
assert config.DISPLAY_NAME == "StarRocks"
1974-
assert config.DISPLAY_ORDER == 19
1974+
assert config.DISPLAY_ORDER == 18
19751975
assert config.is_recommended_for_state_sync is False
19761976

19771977
# Test with minimal configuration (using default port)

0 commit comments

Comments
 (0)