Skip to content

Commit ecdb4d4

Browse files
KLongeKLonge
committed
refactor(sql): update SQL query and improve test context
- Change GROUP BY clause in `full_model.sql` to use `GROUP BY ALL`. - Update docstrings in `conftest.py` to indicate debugging context for fixtures. - Uncomment and utilize `initialize_test_source` in `model_change_test_context`. - Replace `model_change_test_context` with `sample_sqlmesh_test_context` in tests for consistency. - Remove commented-out code in `test_model_code_change.py` to enhance clarity and focus on current implementation.
1 parent dbe64ee commit ecdb4d4

File tree

3 files changed

+77
-166
lines changed

3 files changed

+77
-166
lines changed

‎sample/sqlmesh_project/models/marts/full_model.sql

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,4 +15,4 @@ SELECT
1515
COUNT(DISTINCT id) AS num_orders,
1616
FROM
1717
sqlmesh_example.intermediate_model_1
18-
GROUP BY item_id
18+
GROUP BY ALL

‎tests/conftest.py

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -384,7 +384,7 @@ def sample_sqlmesh_test_context(
384384

385385
@pytest.fixture
386386
def permanent_sqlmesh_project() -> str:
387-
"""Returns the path to the permanent sample SQLMesh project.
387+
"""FOR DEBUGGING ONLY: Returns the path to the permanent sample SQLMesh project.
388388
389389
This fixture provides access to the sample project without copying to a temp directory,
390390
which is useful for debugging and investigating issues with file handling.
@@ -408,9 +408,9 @@ def permanent_sqlmesh_project() -> str:
408408
shutil.copytree(source_dir, project_dir)
409409

410410
# Clean up any existing db file
411-
# db_path = os.path.join(project_dir, "db.db")
412-
# if os.path.exists(db_path):
413-
# os.remove(db_path)
411+
db_path = os.path.join(project_dir, "db.db")
412+
if os.path.exists(db_path):
413+
os.remove(db_path)
414414

415415
return project_dir
416416

@@ -419,7 +419,7 @@ def permanent_sqlmesh_project() -> str:
419419
def model_change_test_context(
420420
permanent_sqlmesh_project: str,
421421
) -> t.Generator[SQLMeshTestContext, None, None]:
422-
"""Creates a SQLMesh test context specifically for testing model code changes.
422+
"""FOR DEBUGGING ONLY: Creates a SQLMesh test context specifically for testing model code changes.
423423
424424
This fixture provides a context that allows modifying SQL model files and ensures
425425
they are properly restored after the test completes. It uses a permanent project
@@ -448,7 +448,7 @@ def model_change_test_context(
448448
context_config=context_config,
449449
project_path=permanent_sqlmesh_project,
450450
)
451-
# test_context.initialize_test_source()
451+
test_context.initialize_test_source()
452452

453453
yield test_context
454454

‎tests/context/plan_and_run/test_model_code_change.py

Lines changed: 70 additions & 159 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,9 @@
2626
"only_skip_backfill",
2727
],
2828
)
29-
@pytest.mark.skip(reason="Work in progress test")
29+
# @pytest.mark.skip(reason="Work in progress test")
3030
def test_given_model_chain_when_running_with_different_flags_then_behaves_as_expected(
31-
model_change_test_context: SQLMeshTestContext,
31+
sample_sqlmesh_test_context: SQLMeshTestContext,
3232
no_auto_upstream: bool,
3333
skip_backfill: bool,
3434
expected_changes: dict[str, str],
@@ -55,175 +55,86 @@ def test_given_model_chain_when_running_with_different_flags_then_behaves_as_exp
5555
">=" means final count should be greater than or equal to initial
5656
"""
5757
# Initial run to set up all models
58-
# model_change_test_context.plan_and_run(
59-
# environment="dev",
60-
# start="2023-02-01",
61-
# end="2023-02-03",
62-
# plan_options=PlanOptions(
63-
# execution_time="2023-02-03",
64-
# ),
65-
# run_options=RunOptions(
66-
# execution_time="2023-02-03",
67-
# ),
68-
# )
69-
70-
# # Get initial counts for the model chain
71-
# initial_counts = {
72-
# "staging_1": model_change_test_context.query(
73-
# "SELECT COUNT(*) FROM sqlmesh_example__dev.staging_model_1"
74-
# )[0][0],
75-
# "staging_2": model_change_test_context.query(
76-
# "SELECT COUNT(*) FROM sqlmesh_example__dev.staging_model_2"
77-
# )[0][0],
78-
# "intermediate": model_change_test_context.query(
79-
# "SELECT COUNT(*) FROM sqlmesh_example__dev.intermediate_model_1"
80-
# )[0][0],
81-
# "full": model_change_test_context.query(
82-
# "SELECT COUNT(*) FROM sqlmesh_example__dev.full_model"
83-
# )[0][0],
84-
# }
85-
86-
# print(f"initial_counts: {initial_counts}")
87-
# print(
88-
# f"intermediate_model_1 first run: {
89-
# model_change_test_context.query(
90-
# 'SELECT * FROM sqlmesh_example__dev.intermediate_model_1',
91-
# return_df=True,
92-
# )
93-
# }"
94-
# )
95-
96-
# # Modify staging_model_1 to include more data
97-
# model_change_test_context.modify_model_file(
98-
# "intermediate_model_1.sql",
99-
# """
100-
# MODEL (
101-
# name sqlmesh_example.intermediate_model_1,
102-
# kind INCREMENTAL_BY_TIME_RANGE (
103-
# time_column event_date
104-
# ),
105-
# start '2020-01-01',
106-
# cron '@daily',
107-
# grain (id, event_date)
108-
# );
109-
110-
# SELECT
111-
# main.id,
112-
# main.item_id,
113-
# main.event_date,
114-
# CONCAT(sub.item_name, ' - modified18') as item_name
115-
# FROM sqlmesh_example.staging_model_1 AS main
116-
# INNER JOIN sqlmesh_example.staging_model_2 as sub
117-
# ON main.id = sub.id
118-
# WHERE
119-
# event_date BETWEEN @start_date AND @end_date
120-
121-
# """,
122-
# )
123-
124-
# raise Exception("Stop here")
58+
sample_sqlmesh_test_context.plan_and_run(
59+
environment="dev",
60+
)
61+
62+
63+
print(
64+
f"intermediate_model_1 first run: {
65+
sample_sqlmesh_test_context.query(
66+
'SELECT * FROM sqlmesh_example__dev.intermediate_model_1',
67+
return_df=True,
68+
)
69+
}"
70+
)
71+
print(
72+
f"full_model first run: {
73+
sample_sqlmesh_test_context.query(
74+
'SELECT * FROM sqlmesh_example__dev.full_model',
75+
return_df=True,
76+
)
77+
}"
78+
)
79+
80+
# # Modify intermediate_model_1 sql to cause breaking change
81+
sample_sqlmesh_test_context.modify_model_file(
82+
"intermediate_model_1.sql",
83+
"""
84+
MODEL (
85+
name sqlmesh_example.intermediate_model_1,
86+
kind INCREMENTAL_BY_TIME_RANGE (
87+
time_column event_date
88+
),
89+
start '2020-01-01',
90+
cron '@daily',
91+
grain (id, event_date)
92+
);
93+
94+
SELECT
95+
main.id,
96+
main.item_id,
97+
main.event_date,
98+
CONCAT(sub.item_name, ' - modified1') as item_name
99+
FROM sqlmesh_example.staging_model_1 AS main
100+
INNER JOIN sqlmesh_example.staging_model_2 as sub
101+
ON main.id = sub.id
102+
WHERE
103+
event_date BETWEEN @start_date AND @end_date
104+
105+
""",
106+
)
107+
125108

126109
# Run with specified flags
127-
model_change_test_context.plan_and_run(
110+
sample_sqlmesh_test_context.plan_and_run(
128111
environment="dev",
129112
plan_options=PlanOptions(
130113
skip_backfill=skip_backfill,
131114
enable_preview=True,
132115
),
133-
select_models=["sqlmesh_example.intermediate_model_1"],
134116
)
135117

136-
# Get final counts and debug info
137-
final_counts = {
138-
"seed_1": model_change_test_context.query(
139-
"SELECT COUNT(*) FROM sqlmesh_example__dev.seed_model_1"
140-
)[0][0],
141-
"staging_1": model_change_test_context.query(
142-
"SELECT COUNT(*) FROM sqlmesh_example__dev.staging_model_1"
143-
)[0][0],
144-
"staging_2": model_change_test_context.query(
145-
"SELECT COUNT(*) FROM sqlmesh_example__dev.staging_model_2"
146-
)[0][0],
147-
"intermediate": model_change_test_context.query(
148-
"SELECT COUNT(*) FROM sqlmesh_example__dev.intermediate_model_1"
149-
)[0][0],
150-
"full": model_change_test_context.query(
151-
"SELECT COUNT(*) FROM sqlmesh_example__dev.full_model"
152-
)[0][0],
153-
}
154-
print(f"first_model_change_counts: {final_counts}")
155-
# print(
156-
# f"intermediate_model_1 after first model change to upstream model: {
157-
# model_change_test_context.query(
158-
# 'SELECT * FROM sqlmesh_example__dev.intermediate_model_1',
159-
# return_df=True,
160-
# )
161-
# }"
162-
# )
163-
164-
# # Modify staging_model_1 to include more data
165-
# model_change_test_context.modify_model_file(
166-
# "staging_model_2.sql",
167-
# """
168-
# MODEL (
169-
# name sqlmesh_example.staging_model_2,
170-
# grain id
171-
# );
172-
173-
# SELECT
174-
# id,
175-
# CONCAT(item_name, ' - modified again') as item_name
176-
# FROM
177-
# sqlmesh_example.seed_model_2
178-
# """,
179-
# )
180-
181-
# # Run with specified flags
182-
# model_change_test_context.plan_and_run(
183-
# environment="dev",
184-
# start="2023-02-01",
185-
# end="2023-02-03",
186-
# execution_time="2023-02-03",
187-
# plan_options=PlanOptions(
188-
# select_models=[
189-
# "sqlmesh_example.staging_model_2",
190-
# ],
191-
# skip_backfill=skip_backfill,
192-
# enable_preview=True,
193-
# ),
194-
# # run_options=RunOptions(
195-
# # select_models=[
196-
# # "sqlmesh_example.staging_model_1",
197-
# # ],
198-
# # no_auto_upstream=no_auto_upstream,
199-
# # ),
200-
# )
201-
202-
# print(
203-
# f"intermediate_model_1 after second model change to upstream model: {
204-
# model_change_test_context.query(
205-
# 'SELECT * FROM sqlmesh_example__dev.intermediate_model_1',
206-
# return_df=True,
207-
# )
208-
# }"
209-
# )
118+
print(
119+
f"intermediate_model_1 after first model change to upstream model: {
120+
sample_sqlmesh_test_context.query(
121+
'SELECT * FROM sqlmesh_example__dev.intermediate_model_1',
122+
return_df=True,
123+
)
124+
}"
125+
)
126+
127+
print(
128+
f"full_model after first model change to upstream model: {
129+
sample_sqlmesh_test_context.query(
130+
'SELECT * FROM sqlmesh_example__dev.full_model',
131+
return_df=True,
132+
)
133+
}"
134+
)
210135

211136
raise Exception("Stop here")
212137

213-
# # Verify counts match expectations
214-
# for model, expected_change in expected_changes.items():
215-
# if expected_change == "==":
216-
# assert final_counts[model] == initial_counts[model], (
217-
# f"{model} count should remain unchanged when "
218-
# f"no_auto_upstream={no_auto_upstream} and skip_backfill={skip_backfill}"
219-
# )
220-
# elif expected_change == ">=":
221-
# assert final_counts[model] >= initial_counts[model], (
222-
# f"{model} count should increase when "
223-
# f"no_auto_upstream={no_auto_upstream} and skip_backfill={skip_backfill}"
224-
# )
225-
# else:
226-
# raise ValueError(f"Invalid expected change: {expected_change}")
227138

228139

229140
if __name__ == "__main__":

0 commit comments

Comments
 (0)