Skip to content
This repository was archived by the owner on Jan 7, 2025. It is now read-only.

Commit af4f113

Browse files
skyzhskyzh
authored
Merge branch 'main' into skyzh/optimizer-structured-trace
2 parents e05b193 + d1e27a5 commit af4f113

File tree

6 files changed

+142
-90
lines changed

6 files changed

+142
-90
lines changed

‎Cargo.lock‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎docs/book.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ authors = ["Alex Chi"]
33
language = "en"
44
multilingual = false
55
src = "src"
6-
title = "Introduction to optd"
6+
title = "the optd book"
77

88
[output.html]
99
additional-css = ["custom.css"]

‎docs/src/SUMMARY.md‎

Lines changed: 69 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,73 @@
11
# Summary
22

3-
# The Core Framework
3+
# optd book
44

5-
- [Optimizer](./optimizer.md)
6-
- [Plan Representation](./plan_repr.md)
7-
- [Rule Engine](./rule_engine.md)
8-
- [Cost Model](./cost_model.md)
9-
- [Properties](./properties.md)
5+
[Intro to optd]()
6+
- [The Core]()
7+
- [Plan Representation]()
8+
- [Memo Table and Logical Equivalence]()
9+
- [Cascades Framework]()
10+
- [Basic Cascades Tasks]()
11+
- [Cycle Avoidance]()
12+
- [Upper Bound Pruning]()
13+
- [Multi-Stage Optimization]()
14+
- [Rule IR and Matcher]()
15+
- [Cost and Statistics]()
16+
- [Logical Properties]()
17+
- [Physical Properties and Enforcers]()
18+
- [Memo Table: Subgoals and Winners]()
19+
- [Cascades Tasks: Required Physical Properties]()
20+
- [Exploration Budget]()
21+
- [Heuristics Optimizer]()
22+
- [Integration with Datafusion]()
23+
- [Datafusion Plan Representation]()
24+
- [Datafusion Bridge]()
25+
- [Rule Engine and Rules]()
26+
- [Basic Cost Model]()
27+
- [Logical and Physical Properties]()
28+
- [Optimization Passes]()
29+
- [Miscellaneous]()
30+
- [Explain]()
31+
- [Research]()
32+
- [Partial Exploration and Re-Optimization]()
33+
- [Advanced Cost Model]()
34+
- [The Hyper Subquery Unnesting Ruleset]()
35+
- [Testing and Benchmark]()
36+
- [sqlplannertest]()
37+
- [sqllogictest]()
38+
- [perfbench]()
39+
- [Debugging and Tracing]()
40+
- [optd-core Tracing]()
41+
- [Memo Table Visualization]()
42+
- [Optimizer Dump]()
43+
- [Contribution Guide]()
44+
- [Install Tools]()
45+
- [Contribution Workflow]()
46+
- [Add a Datafusion Rule]()
47+
- [What's Next]()
48+
- [Ideas]()
49+
- [RFCs]()
50+
---
1051

11-
# Integration
12-
13-
- [Apache Arrow Datafusion](./datafusion.md)
14-
15-
# Adaptive Optimization
16-
17-
- [Re-optimization](./reoptimization.md)
18-
- [Partial Exploration](./partial_exploration.md)
19-
20-
# Demo
21-
22-
- [Three Join Demo](./demo_three_join.md)
23-
- [TPC-H Q8 Demo](./demo_tpch_q8.md)
24-
25-
# Performance Benchmarking
26-
- [Cost Model Cardinality Benchmarking](./cost_model_benchmarking.md)
27-
28-
# Functional Testing
29-
30-
- [SQLPlannerTest](./sqlplannertest.md)
31-
- [Datafusion CLI](./datafusion_cli.md)
32-
33-
# Miscellaneous
34-
35-
- [Miscellaneous](./miscellaneous.md)
52+
# DEPRECATED
53+
- [old optd book]()
54+
- [Core Framework]()
55+
- [Optimizer](./optimizer.md)
56+
- [Plan Representation](./plan_repr.md)
57+
- [Rule Engine](./rule_engine.md)
58+
- [Cost Model](./cost_model.md)
59+
- [Properties](./properties.md)
60+
- [Integration]()
61+
- [Apache Arrow Datafusion](./datafusion.md)
62+
- [Adaptive Optimization]()
63+
- [Re-optimization](./reoptimization.md)
64+
- [Partial Exploration](./partial_exploration.md)
65+
- [Demo]()
66+
- [Three Join Demo](./demo_three_join.md)
67+
- [TPC-H Q8 Demo](./demo_tpch_q8.md)
68+
- [Performance Benchmarking]()
69+
- [Cost Model Cardinality Benchmarking](./cost_model_benchmarking.md)
70+
- [Functional Testing]()
71+
- [SQLPlannerTest](./sqlplannertest.md)
72+
- [Datafusion CLI](./datafusion_cli.md)
73+
- [Miscellaneous](./miscellaneous.md)

‎optd-core/src/cascades/memo.rs‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -410,14 +410,29 @@ impl<T: NodeType> NaiveMemo<T> {
410410
trace!(event = "merge_group", merge_into = %merge_into, merge_from = %merge_from);
411411
let group_merge_from = self.groups.remove(&merge_from).unwrap();
412412
let group_merge_into = self.groups.get_mut(&merge_into).unwrap();
413-
// TODO: update winner, cost and properties
413+
414+
// Merge expressions
414415
for from_expr in group_merge_from.group_exprs {
415416
let ret = self.expr_id_to_group_id.insert(from_expr, merge_into);
416417
assert!(ret.is_some());
417418
group_merge_into.group_exprs.insert(from_expr);
418419
}
419420
self.merged_group_mapping.insert(merge_from, merge_into);
420421

422+
// Merge winner
423+
if let Some(winner) = group_merge_from.info.winner.as_full_winner() {
424+
match &group_merge_into.info.winner {
425+
Winner::Impossible | Winner::Unknown => {
426+
group_merge_into.info.winner = Winner::Full(winner.clone());
427+
}
428+
Winner::Full(winner_into) => {
429+
if winner.total_weighted_cost < winner_into.total_weighted_cost {
430+
group_merge_into.info.winner = Winner::Full(winner.clone());
431+
}
432+
}
433+
}
434+
}
435+
421436
// Update all indexes and other data structures
422437
// 1. update merged group mapping -- could be optimized with union find
423438
for (_, mapped_to) in self.merged_group_mapping.iter_mut() {

‎optd-sqlplannertest/tests/joins/join_enumerate.planner.sql‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -132,11 +132,12 @@ PhysicalHashJoin { join_type: Inner, left_keys: [ #1 ], right_keys: [ #0 ] }
132132
(Join (Join t3 t1) t2)
133133
(Join (Join t3 t2) t1)
134134
135-
PhysicalHashJoin { join_type: Inner, left_keys: [ #1 ], right_keys: [ #0 ] }
136-
├── PhysicalHashJoin { join_type: Inner, left_keys: [ #0 ], right_keys: [ #0 ] }
137-
│ ├── PhysicalScan { table: t1 }
138-
│ └── PhysicalScan { table: t2 }
139-
└── PhysicalScan { table: t3 }
135+
PhysicalProjection { exprs: [ #4, #5, #0, #1, #2, #3 ] }
136+
└── PhysicalHashJoin { join_type: Inner, left_keys: [ #0 ], right_keys: [ #2 ] }
137+
├── PhysicalScan { table: t2 }
138+
└── PhysicalHashJoin { join_type: Inner, left_keys: [ #0 ], right_keys: [ #1 ] }
139+
├── PhysicalScan { table: t3 }
140+
└── PhysicalScan { table: t1 }
140141
0 0 0 200 0 300
141142
1 1 1 201 1 301
142143
2 2 2 202 2 302

‎optd-sqlplannertest/tests/tpch/q17.planner.sql‎

Lines changed: 49 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -75,57 +75,55 @@ PhysicalProjection
7575
│ └── [ #5 ]
7676
├── groups: []
7777
└── PhysicalProjection { exprs: [ #0, #1, #2, #3, #4, #5, #6, #7, #8, #9, #10, #11, #12, #13, #14, #15, #16, #17, #18, #19, #20, #21, #22, #23, #24, #26 ] }
78-
└── PhysicalProjection { exprs: [ #11, #12, #13, #14, #15, #16, #17, #18, #19, #20, #21, #22, #23, #24, #25, #26, #2, #3, #4, #5, #6, #7, #8, #9, #10, #0, #1 ] }
79-
└── PhysicalNestedLoopJoin
80-
├── join_type: Inner
81-
├── cond:And
82-
│ ├── Eq
83-
│ │ ├── #2
84-
│ │ └── #12
85-
│ └── Lt
86-
│ ├── Cast { cast_to: Decimal128(30, 15), child: #15 }
87-
│ └── #1
88-
├── PhysicalProjection { exprs: [ #9, #10, #0, #1, #2, #3, #4, #5, #6, #7, #8 ] }
89-
│ └── PhysicalHashJoin { join_type: Inner, left_keys: [ #0 ], right_keys: [ #0 ] }
90-
│ ├── PhysicalFilter
91-
│ │ ├── cond:And
92-
│ │ │ ├── Eq
93-
│ │ │ │ ├── #3
94-
│ │ │ │ └── "Brand#13"
95-
│ │ │ └── Eq
96-
│ │ │ ├── #6
97-
│ │ │ └── "JUMBO PKG"
98-
│ │ └── PhysicalScan { table: part }
99-
│ └── PhysicalProjection
100-
│ ├── exprs:
101-
│ │ ┌── #0
102-
│ │ └── Cast
103-
│ │ ├── cast_to: Decimal128(30, 15)
104-
│ │ ├── child:Mul
105-
│ │ │ ├── 0.2(float)
106-
│ │ │ └── Cast { cast_to: Float64, child: #1 }
78+
└── PhysicalNestedLoopJoin
79+
├── join_type: Inner
80+
├── cond:And
81+
│ ├── Eq
82+
│ │ ├── #16
83+
│ │ └── #1
84+
│ └── Lt
85+
│ ├── Cast { cast_to: Decimal128(30, 15), child: #4 }
86+
│ └── #26
87+
├── PhysicalScan { table: lineitem }
88+
└── PhysicalHashJoin { join_type: Inner, left_keys: [ #0 ], right_keys: [ #0 ] }
89+
├── PhysicalFilter
90+
│ ├── cond:And
91+
│ │ ├── Eq
92+
│ │ │ ├── #3
93+
│ │ │ └── "Brand#13"
94+
│ │ └── Eq
95+
│ │ ├── #6
96+
│ │ └── "JUMBO PKG"
97+
│ └── PhysicalScan { table: part }
98+
└── PhysicalProjection
99+
├── exprs:
100+
│ ┌── #0
101+
│ └── Cast
102+
│ ├── cast_to: Decimal128(30, 15)
103+
│ ├── child:Mul
104+
│ │ ├── 0.2(float)
105+
│ │ └── Cast { cast_to: Float64, child: #1 }
107106
108-
│ └── PhysicalProjection { exprs: [ #0, #2 ] }
109-
│ └── PhysicalNestedLoopJoin
110-
│ ├── join_type: LeftOuter
111-
│ ├── cond:And
112-
│ │ └── Eq
113-
│ │ ├── #0
114-
│ │ └── #1
115-
│ ├── PhysicalAgg { aggrs: [], groups: [ #16 ] }
116-
│ │ └── PhysicalNestedLoopJoin { join_type: Inner, cond: true }
117-
│ │ ├── PhysicalScan { table: lineitem }
118-
│ │ └── PhysicalScan { table: part }
119-
│ └── PhysicalAgg
120-
│ ├── aggrs:Agg(Avg)
121-
│ │ └── [ #5 ]
122-
│ ├── groups: [ #0 ]
123-
│ └── PhysicalHashJoin { join_type: Inner, left_keys: [ #0 ], right_keys: [ #1 ] }
124-
│ ├── PhysicalAgg { aggrs: [], groups: [ #16 ] }
125-
│ │ └── PhysicalNestedLoopJoin { join_type: Inner, cond: true }
126-
│ │ ├── PhysicalScan { table: lineitem }
127-
│ │ └── PhysicalScan { table: part }
128-
│ └── PhysicalScan { table: lineitem }
129-
└── PhysicalScan { table: lineitem }
107+
└── PhysicalProjection { exprs: [ #0, #2 ] }
108+
└── PhysicalNestedLoopJoin
109+
├── join_type: LeftOuter
110+
├── cond:And
111+
│ └── Eq
112+
│ ├── #0
113+
│ └── #1
114+
├── PhysicalAgg { aggrs: [], groups: [ #16 ] }
115+
│ └── PhysicalNestedLoopJoin { join_type: Inner, cond: true }
116+
│ ├── PhysicalScan { table: lineitem }
117+
│ └── PhysicalScan { table: part }
118+
└── PhysicalAgg
119+
├── aggrs:Agg(Avg)
120+
│ └── [ #5 ]
121+
├── groups: [ #0 ]
122+
└── PhysicalHashJoin { join_type: Inner, left_keys: [ #0 ], right_keys: [ #1 ] }
123+
├── PhysicalAgg { aggrs: [], groups: [ #16 ] }
124+
│ └── PhysicalNestedLoopJoin { join_type: Inner, cond: true }
125+
│ ├── PhysicalScan { table: lineitem }
126+
│ └── PhysicalScan { table: part }
127+
└── PhysicalScan { table: lineitem }
130128
*/
131129

0 commit comments

Comments
 (0)