Skip to content

Commit d46927e

Browse files
themisvaltinosthemisvaltinos
committed
refactor; clean up code
1 parent d49e987 commit d46927e

3 files changed

Lines changed: 71 additions & 112 deletions

File tree

‎sqlmesh/core/console.py‎

Lines changed: 36 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -2150,42 +2150,44 @@ def show_table_diff(
21502150
"""
21512151
Display the table diff between all mismatched tables.
21522152
"""
2153-
mismatched_tables = []
2154-
fully_matched = []
2155-
for table_diff in table_diffs:
2156-
if (
2157-
table_diff.row_diff(
2158-
temp_schema=temp_schema, skip_grain_check=skip_grain_check
2159-
).full_match_pct
2160-
== 100
2161-
):
2162-
fully_matched.append(table_diff)
2163-
else:
2164-
mismatched_tables.append(table_diff)
2153+
if len(table_diffs) > 1:
2154+
mismatched_tables = []
2155+
fully_matched = []
2156+
for table_diff in table_diffs:
2157+
if (
2158+
table_diff.row_diff(
2159+
temp_schema=temp_schema, skip_grain_check=skip_grain_check
2160+
).full_match_pct
2161+
== 100
2162+
):
2163+
fully_matched.append(table_diff)
2164+
else:
2165+
mismatched_tables.append(table_diff)
2166+
table_diffs = mismatched_tables if mismatched_tables else []
2167+
if fully_matched:
2168+
m_tree = Tree("\n[b]Identical Tables")
2169+
for m in fully_matched:
2170+
m_tree.add(
2171+
f"[{self.TABLE_DIFF_SOURCE_BLUE}]{m.source}[/{self.TABLE_DIFF_SOURCE_BLUE}] - [{self.TABLE_DIFF_TARGET_GREEN}]{m.target}[/{self.TABLE_DIFF_TARGET_GREEN}]"
2172+
)
2173+
self._print(m_tree)
21652174

2166-
if fully_matched:
2167-
m_tree = Tree("\n[b]Identical Tables")
2168-
for m in fully_matched:
2169-
m_tree.add(
2170-
f"[{self.TABLE_DIFF_SOURCE_BLUE}]{m.source}[/{self.TABLE_DIFF_SOURCE_BLUE}] - [{self.TABLE_DIFF_TARGET_GREEN}]{m.target}[/{self.TABLE_DIFF_TARGET_GREEN}]"
2171-
)
2172-
self._print(m_tree)
2175+
if mismatched_tables:
2176+
m_tree = Tree("\n[b]Mismatched Tables")
2177+
for m in mismatched_tables:
2178+
m_tree.add(
2179+
f"[{self.TABLE_DIFF_SOURCE_BLUE}]{m.source}[/{self.TABLE_DIFF_SOURCE_BLUE}] - [{self.TABLE_DIFF_TARGET_GREEN}]{m.target}[/{self.TABLE_DIFF_TARGET_GREEN}]"
2180+
)
2181+
self._print(m_tree)
21732182

2174-
if mismatched_tables:
2175-
m_tree = Tree("\n[b]Mismatched Tables")
2176-
for m in mismatched_tables:
2177-
m_tree.add(
2178-
f"[{self.TABLE_DIFF_SOURCE_BLUE}]{m.source}[/{self.TABLE_DIFF_SOURCE_BLUE}] - [{self.TABLE_DIFF_TARGET_GREEN}]{m.target}[/{self.TABLE_DIFF_TARGET_GREEN}]"
2179-
)
2180-
self._print(m_tree)
2181-
for table_diff in mismatched_tables:
2182-
self.show_table_diff_summary(table_diff)
2183-
self.show_schema_diff(table_diff.schema_diff())
2184-
self.show_row_diff(
2185-
table_diff.row_diff(temp_schema=temp_schema, skip_grain_check=skip_grain_check),
2186-
show_sample=show_sample,
2187-
skip_grain_check=skip_grain_check,
2188-
)
2183+
for table_diff in table_diffs:
2184+
self.show_table_diff_summary(table_diff)
2185+
self.show_schema_diff(table_diff.schema_diff())
2186+
self.show_row_diff(
2187+
table_diff.row_diff(temp_schema=temp_schema, skip_grain_check=skip_grain_check),
2188+
show_sample=show_sample,
2189+
skip_grain_check=skip_grain_check,
2190+
)
21892191

21902192
def print_environments(self, environments_summary: t.List[EnvironmentSummary]) -> None:
21912193
"""Prints all environment names along with expiry datetime."""
@@ -2776,53 +2778,6 @@ def show_model_difference_summary(
27762778
context_diff, modified_snapshots, environment_naming_info, default_catalog, no_diff
27772779
)
27782780

2779-
def show_table_diff(
2780-
self,
2781-
table_diffs: t.List[TableDiff],
2782-
show_sample: bool = True,
2783-
skip_grain_check: bool = False,
2784-
temp_schema: t.Optional[str] = None,
2785-
) -> None:
2786-
"""
2787-
Display the table diff between all mismatched tables.
2788-
"""
2789-
mismatched_tables = []
2790-
fully_matched = []
2791-
for table_diff in table_diffs:
2792-
if (
2793-
table_diff.row_diff(
2794-
temp_schema=temp_schema, skip_grain_check=skip_grain_check
2795-
).full_match_pct
2796-
== 100
2797-
):
2798-
fully_matched.append(table_diff)
2799-
else:
2800-
mismatched_tables.append(table_diff)
2801-
2802-
if fully_matched:
2803-
m_tree = Tree("\n[b]Identical Tables")
2804-
for m in fully_matched:
2805-
m_tree.add(
2806-
f"[{self.TABLE_DIFF_SOURCE_BLUE}]{m.source}[/{self.TABLE_DIFF_SOURCE_BLUE}] - [{self.TABLE_DIFF_TARGET_GREEN}]{m.target}[/{self.TABLE_DIFF_TARGET_GREEN}]"
2807-
)
2808-
self._print(m_tree)
2809-
2810-
if mismatched_tables:
2811-
m_tree = Tree("\n[b]Mismatched Tables")
2812-
for m in mismatched_tables:
2813-
m_tree.add(
2814-
f"[{self.TABLE_DIFF_SOURCE_BLUE}]{m.source}[/{self.TABLE_DIFF_SOURCE_BLUE}] - [{self.TABLE_DIFF_TARGET_GREEN}]{m.target}[/{self.TABLE_DIFF_TARGET_GREEN}]"
2815-
)
2816-
self._print(m_tree)
2817-
for table_diff in mismatched_tables:
2818-
self.show_table_diff_summary(table_diff)
2819-
self.show_schema_diff(table_diff.schema_diff())
2820-
self.show_row_diff(
2821-
table_diff.row_diff(temp_schema=temp_schema, skip_grain_check=skip_grain_check),
2822-
show_sample=show_sample,
2823-
skip_grain_check=skip_grain_check,
2824-
)
2825-
28262781
def _print_models_with_threshold(
28272782
self,
28282783
environment_naming_info: EnvironmentNamingInfo,

‎sqlmesh/core/context.py‎

Lines changed: 16 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1613,6 +1613,7 @@ def table_diff(
16131613
if not target_env:
16141614
raise SQLMeshError(f"Could not find environment '{target}'")
16151615

1616+
modified_snapshots: t.Set[ModelOrSnapshot] = { model_or_snapshot } if model_or_snapshot else set()
16161617
if select_model:
16171618
models_to_diff = self._new_selector().expand_model_selections(select_model)
16181619
target_snapshots = {
@@ -1629,34 +1630,21 @@ def table_diff(
16291630
current_snapshot.snapshot_id.name
16301631
for _, (current_snapshot, _) in context_diff.modified_snapshots.items()
16311632
}
1632-
tasks_num = min(len(modified_snapshots), self.concurrent_tasks)
1633-
table_diffs = concurrent_apply_to_values(
1634-
list(modified_snapshots),
1635-
lambda s: self._model_diff(
1636-
source_env=source_env,
1637-
target_env=target_env,
1638-
model_or_snapshot=s,
1639-
limit=limit,
1640-
decimals=decimals,
1641-
on=on,
1642-
skip_columns=skip_columns,
1643-
where=where,
1644-
),
1645-
tasks_num=tasks_num,
1646-
)
1647-
elif model_or_snapshot:
1648-
table_diffs = [
1649-
self._model_diff(
1650-
source_env=source_env,
1651-
target_env=target_env,
1652-
model_or_snapshot=model_or_snapshot,
1653-
limit=limit,
1654-
decimals=decimals,
1655-
on=on,
1656-
skip_columns=skip_columns,
1657-
where=where,
1658-
)
1659-
]
1633+
tasks_num = min(len(modified_snapshots), self.concurrent_tasks)
1634+
table_diffs = concurrent_apply_to_values(
1635+
list(modified_snapshots),
1636+
lambda s: self._model_diff(
1637+
source_env=source_env,
1638+
target_env=target_env,
1639+
model_or_snapshot=s,
1640+
limit=limit,
1641+
decimals=decimals,
1642+
on=on,
1643+
skip_columns=skip_columns,
1644+
where=where,
1645+
),
1646+
tasks_num=tasks_num,
1647+
)
16601648
else:
16611649
table_diffs = [
16621650
self._table_diff(

‎tests/integrations/jupyter/test_magics.py‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -641,11 +641,27 @@ def test_table_diff(notebook, loaded_sushi_context, convert_all_html_output_to_t
641641

642642
assert not output.stdout
643643
assert not output.stderr
644-
assert len(output.outputs) == 1
644+
assert len(output.outputs) == 5
645645

646646
assert convert_all_html_output_to_text(output) == [
647-
"""Identical Tables
648-
└── memory.sushi__dev.top_waiters - memory.sushi.top_waiters""",
647+
"""Table Diff
648+
├── Model:
649+
│ └── sushi.top_waiters
650+
├── Environment:
651+
│ ├── Source: dev
652+
│ └── Target: prod
653+
├── Tables:
654+
│ ├── Source: memory.sushi__dev.top_waiters
655+
│ └── Target: memory.sushi.top_waiters
656+
└── Join On:
657+
└── waiter_id""",
658+
"""Schema Diff Between 'DEV' and 'PROD' environments for model 'sushi.top_waiters':
659+
└── Schemas match""",
660+
"""Row Counts:
661+
└── FULL MATCH: 8 rows (100.0%)""",
662+
"""COMMON ROWS column comparison stats:""",
663+
"""pct_match
664+
revenue 100.0""",
649665
]
650666

651667

0 commit comments

Comments
 (0)