Skip to content
This repository has been archived by the owner on Jan 7, 2025. It is now read-only.

Commit

Permalink
fix(cost): use 1000 as default table row cnt (#208)
Browse files Browse the repository at this point in the history 
Helps us generate better plans across all regression tests, otherwise
row_cnt=1 does not make much sense. This also fixes NLJ within subquery
unnesting test case.

Call simplify expr when applying join pushdown rules, which in the
future should be done automatically in the core.

Signed-off-by: Alex Chi <[email protected]>
  • Loading branch information
@skyzh
skyzh authored Nov 1, 2024
1 parent bdf638a commit 4c8f4b4
Show file tree
Hide file tree
Showing 13 changed files with 256 additions and 324 deletions.
6 changes: 4 additions & 2 deletions optd-datafusion-repr/src/cost/adaptive_cost.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ use optd_core::{
rel_node::{RelNode, Value},
};

use super::base_cost::DEFAULT_TABLE_ROW_CNT;

pub type RuntimeAdaptionStorage = Arc<Mutex<RuntimeAdaptionStorageInner>>;

#[derive(Default, Debug)]
Expand Down Expand Up @@ -52,10 +54,10 @@ impl CostModel<OptRelNodeTyp> for AdaptiveCostModel {
let runtime_row_cnt = (*runtime_row_cnt).max(1) as f64;
return OptCostModel::cost(runtime_row_cnt, 0.0, runtime_row_cnt);
} else {
return OptCostModel::cost(1.0, 0.0, 1.0);
return OptCostModel::cost(DEFAULT_TABLE_ROW_CNT as f64, 0.0, 1.0);
}
} else {
return OptCostModel::cost(1.0, 0.0, 1.0);
return OptCostModel::cost(DEFAULT_TABLE_ROW_CNT as f64, 0.0, 1.0);
}
}
let (mut row_cnt, compute_cost, io_cost) = OptCostModel::cost_tuple(
Expand Down
4 changes: 3 additions & 1 deletion optd-datafusion-repr/src/cost/base_cost.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ pub const ROW_COUNT: usize = 1;
pub const COMPUTE_COST: usize = 2;
pub const IO_COST: usize = 3;

pub(crate) const DEFAULT_TABLE_ROW_CNT: usize = 1000;

impl OptCostModel {
pub fn row_cnt(Cost(cost): &Cost) -> f64 {
cost[ROW_COUNT]
Expand Down Expand Up @@ -104,7 +106,7 @@ impl CostModel<OptRelNodeTyp> for OptCostModel {
.table_stat
.get(table_name.as_ref())
.copied()
.unwrap_or(1) as f64;
.unwrap_or(DEFAULT_TABLE_ROW_CNT) as f64;
Self::cost(row_cnt, 0.0, row_cnt)
}
OptRelNodeTyp::PhysicalLimit => {
Expand Down
2 changes: 1 addition & 1 deletion optd-datafusion-repr/src/rules/filter.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ define_rule!(
// - Replaces the Or operator with True if any operand is True
// - Replaces the And operator with False if any operand is False
// - Removes Duplicates
fn simplify_log_expr(log_expr: OptRelNodeRef, changed: &mut bool) -> OptRelNodeRef {
pub(crate) fn simplify_log_expr(log_expr: OptRelNodeRef, changed: &mut bool) -> OptRelNodeRef {
let log_expr = LogOpExpr::from_rel_node(log_expr).unwrap();
let op = log_expr.op_type();
// we need a new children vec to output deterministic order
Expand Down
79 changes: 0 additions & 79 deletions optd-datafusion-repr/src/rules/filter_join.rs
View file

This file was deleted.

7 changes: 6 additions & 1 deletion optd-datafusion-repr/src/rules/filter_pushdown.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ use crate::plan_nodes::{
};
use crate::properties::schema::SchemaPropertyBuilder;

use super::filter::simplify_log_expr;
use super::macros::define_rule;

/// Emits a LogOpExpr AND if the list has more than one element
Expand All @@ -36,7 +37,11 @@ fn and_expr_list_to_expr(exprs: Vec<Expr>) -> Expr {
fn merge_conds(first: Expr, second: Expr) -> Expr {
let new_expr_list = ExprList::new(vec![first, second]);
// Flatten nested logical expressions if possible
LogOpExpr::new_flattened_nested_logical(LogOpType::And, new_expr_list).into_expr()
let flattened =
LogOpExpr::new_flattened_nested_logical(LogOpType::And, new_expr_list).into_expr();
let mut changed = false;
// TODO: such simplifications should be invoked from optd-core, instead of ad-hoc
Expr::from_rel_node(simplify_log_expr(flattened.into_rel_node(), &mut changed)).unwrap()
}

#[derive(Debug, Clone, Copy)]
Expand Down
8 changes: 4 additions & 4 deletions optd-sqlplannertest/tests/basic/filter.planner.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,17 +89,17 @@ LogicalProjection { exprs: [ #0, #1, #2, #3 ] }
└── LogicalJoin { join_type: Cross, cond: true }
├── LogicalScan { table: t1 }
└── LogicalScan { table: t2 }
PhysicalNestedLoopJoin
├── join_type: Inner
PhysicalFilter
├── cond:Or
│ ├── Eq
│ │ ├── #0
│ │ └── #2
│ └── Eq
│ ├── #0
│ └── #3
├── PhysicalScan { table: t1 }
└── PhysicalScan { table: t2 }
└── PhysicalNestedLoopJoin { join_type: Cross, cond: true }
├── PhysicalScan { table: t1 }
└── PhysicalScan { table: t2 }
0 0 0 200
1 1 1 201
2 2 2 202
Expand Down
6 changes: 3 additions & 3 deletions optd-sqlplannertest/tests/basic/verbose.planner.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ PhysicalScan { table: t1 }
select * from t1;

/*
PhysicalScan { table: t1, cost: weighted=1.00,row_cnt=1.00,compute=0.00,io=1.00 }
PhysicalScan { table: t1, cost: weighted=1.00,row_cnt=1000.00,compute=0.00,io=1.00 }
*/

-- Test verbose explain with aggregation
Expand All @@ -28,7 +28,7 @@ PhysicalAgg
├── aggrs:Agg(Count)
│ └── [ 1(u8) ]
├── groups: []
├── cost: weighted=21.12,row_cnt=1.00,compute=20.12,io=1.00
└── PhysicalScan { table: t1, cost: weighted=1.00,row_cnt=1.00,compute=0.00,io=1.00 }
├── cost: weighted=10071.06,row_cnt=1000.00,compute=10070.06,io=1.00
└── PhysicalScan { table: t1, cost: weighted=1.00,row_cnt=1000.00,compute=0.00,io=1.00 }
*/

20 changes: 10 additions & 10 deletions optd-sqlplannertest/tests/pushdowns/fliter_transpose.planner.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,11 +47,11 @@ LogicalProjection { exprs: [ #0, #1, #3 ] }
└── LogicalJoin { join_type: Cross, cond: true }
├── LogicalScan { table: t1 }
└── LogicalScan { table: t2 }
PhysicalFilter
├── cond:Eq
│ ├── #0
│ └── #2
└── PhysicalProjection { exprs: [ #0, #1, #3 ] }
PhysicalProjection { exprs: [ #0, #1, #3 ] }
└── PhysicalFilter
├── cond:Eq
│ ├── #0
│ └── #3
└── PhysicalNestedLoopJoin { join_type: Cross, cond: true }
├── PhysicalScan { table: t1 }
└── PhysicalScan { table: t2 }
Expand All @@ -72,11 +72,11 @@ LogicalProjection { exprs: [ #0, #1, #2 ] }
└── LogicalJoin { join_type: Cross, cond: true }
├── LogicalScan { table: t1 }
└── LogicalScan { table: t2 }
PhysicalFilter
├── cond:Eq
│ ├── #0
│ └── #2
└── PhysicalProjection { exprs: [ #0, #1, #3 ] }
PhysicalProjection { exprs: [ #0, #1, #3 ] }
└── PhysicalFilter
├── cond:Eq
│ ├── #0
│ └── #3
└── PhysicalNestedLoopJoin { join_type: Cross, cond: true }
├── PhysicalScan { table: t1 }
└── PhysicalScan { table: t2 }
Expand Down
33 changes: 15 additions & 18 deletions optd-sqlplannertest/tests/subqueries/subquery_unnesting.planner.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,24 +53,21 @@ LogicalProjection { exprs: [ #0, #1 ] }
├── LogicalAgg { exprs: [], groups: [ #0 ] }
│ └── LogicalScan { table: t1 }
└── LogicalScan { table: t2 }
PhysicalProjection { exprs: [ #0, #1 ] }
PhysicalProjection { exprs: [ #2, #3 ] }
└── PhysicalHashJoin { join_type: Inner, left_keys: [ #0 ], right_keys: [ #0 ] }
├── PhysicalScan { table: t1 }
└── PhysicalFilter
├── cond:Gt
│ ├── #1
│ └── 100(i64)
└── PhysicalAgg
├── aggrs:Agg(Sum)
│ └── [ Cast { cast_to: Int64, expr: #2 } ]
├── groups: [ #1 ]
└── PhysicalFilter
├── cond:Eq
│ ├── #1
│ └── #0
└── PhysicalNestedLoopJoin { join_type: Inner, cond: true }
├── PhysicalAgg { aggrs: [], groups: [ #0 ] }
│ └── PhysicalScan { table: t1 }
└── PhysicalScan { table: t2 }
├── PhysicalAgg
│ ├── aggrs:Agg(Sum)
│ │ └── [ Cast { cast_to: Int64, expr: #2 } ]
│ ├── groups: [ #1 ]
│ └── PhysicalProjection { exprs: [ #2, #0, #1 ] }
│ └── PhysicalHashJoin { join_type: Inner, left_keys: [ #0 ], right_keys: [ #0 ] }
│ ├── PhysicalFilter
│ │ ├── cond:Gt
│ │ │ ├── #0
│ │ │ └── 100(i64)
│ │ └── PhysicalScan { table: t2 }
│ └── PhysicalAgg { aggrs: [], groups: [ #0 ] }
│ └── PhysicalScan { table: t1 }
└── PhysicalScan { table: t1 }
*/

75 changes: 38 additions & 37 deletions optd-sqlplannertest/tests/tpch/tpch-01-05.planner.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -362,21 +362,21 @@ PhysicalLimit { skip: 0(u64), fetch: 100(u64) }
│ │ └── #1
│ └── SortOrder { order: Asc }
│ └── #3
└── PhysicalProjection { exprs: [ #5, #1, #22, #7, #9, #2, #4, #6 ] }
└── PhysicalHashJoin { join_type: Inner, left_keys: [ #7, #19 ], right_keys: [ #1, #0 ] }
└── PhysicalProjection { exprs: [ #19, #15, #22, #0, #2, #16, #18, #20 ] }
└── PhysicalHashJoin { join_type: Inner, left_keys: [ #0, #12 ], right_keys: [ #1, #0 ] }
├── PhysicalHashJoin { join_type: Inner, left_keys: [ #23 ], right_keys: [ #0 ] }
│ ├── PhysicalHashJoin { join_type: Inner, left_keys: [ #3 ], right_keys: [ #0 ] }
│ │ ├── PhysicalHashJoin { join_type: Inner, left_keys: [ #0 ], right_keys: [ #10 ] }
│ │ │ ├── PhysicalScan { table: supplier }
│ │ │ └── PhysicalHashJoin { join_type: Inner, left_keys: [ #0 ], right_keys: [ #0 ] }
│ │ │ ├── PhysicalFilter
│ │ │ │ ├── cond:And
│ │ │ │ │ ├── Eq
│ │ │ │ │ │ ├── #5
│ │ │ │ │ │ └── 4(i32)
│ │ │ │ │ └── Like { expr: #4, pattern: "%TIN", negated: false, case_insensitive: false }
│ │ │ │ └── PhysicalScan { table: part }
│ │ │ └── PhysicalScan { table: partsupp }
│ ├── PhysicalHashJoin { join_type: Inner, left_keys: [ #17 ], right_keys: [ #0 ] }
│ │ ├── PhysicalHashJoin { join_type: Inner, left_keys: [ #10 ], right_keys: [ #0 ] }
│ │ │ ├── PhysicalHashJoin { join_type: Inner, left_keys: [ #0 ], right_keys: [ #0 ] }
│ │ │ │ ├── PhysicalFilter
│ │ │ │ │ ├── cond:And
│ │ │ │ │ │ ├── Eq
│ │ │ │ │ │ │ ├── #5
│ │ │ │ │ │ │ └── 4(i32)
│ │ │ │ │ │ └── Like { expr: #4, pattern: "%TIN", negated: false, case_insensitive: false }
│ │ │ │ │ └── PhysicalScan { table: part }
│ │ │ │ └── PhysicalScan { table: partsupp }
│ │ │ └── PhysicalScan { table: supplier }
│ │ └── PhysicalProjection { exprs: [ #0, #1, #2 ] }
│ │ └── PhysicalScan { table: nation }
│ └── PhysicalProjection { exprs: [ #0 ] }
Expand Down Expand Up @@ -609,28 +609,29 @@ PhysicalSort
│ ├── Cast { cast_to: Decimal128(20, 0), expr: 1(i64) }
│ └── #23
├── groups: [ #41 ]
└── PhysicalHashJoin { join_type: Inner, left_keys: [ #19, #3 ], right_keys: [ #0, #3 ] }
├── PhysicalHashJoin { join_type: Inner, left_keys: [ #0 ], right_keys: [ #1 ] }
│ ├── PhysicalScan { table: customer }
│ └── PhysicalHashJoin { join_type: Inner, left_keys: [ #0 ], right_keys: [ #0 ] }
│ ├── PhysicalFilter
│ │ ├── cond:And
│ │ │ ├── Geq
│ │ │ │ ├── #4
│ │ │ │ └── Cast { cast_to: Date32, expr: "2023-01-01" }
│ │ │ └── Lt
│ │ │ ├── #4
│ │ │ └── Cast { cast_to: Date32, expr: "2024-01-01" }
│ │ └── PhysicalScan { table: orders }
│ └── PhysicalScan { table: lineitem }
└── PhysicalHashJoin { join_type: Inner, left_keys: [ #9 ], right_keys: [ #0 ] }
├── PhysicalHashJoin { join_type: Inner, left_keys: [ #3 ], right_keys: [ #0 ] }
│ ├── PhysicalScan { table: supplier }
│ └── PhysicalScan { table: nation }
└── PhysicalFilter
├── cond:Eq
│ ├── #1
│ └── "Asia"
└── PhysicalScan { table: region }
└── PhysicalHashJoin { join_type: Inner, left_keys: [ #42 ], right_keys: [ #0 ] }
├── PhysicalHashJoin { join_type: Inner, left_keys: [ #36 ], right_keys: [ #0 ] }
│ ├── PhysicalHashJoin { join_type: Inner, left_keys: [ #19, #3 ], right_keys: [ #0, #3 ] }
│ │ ├── PhysicalProjection { exprs: [ #25, #26, #27, #28, #29, #30, #31, #32, #0, #1, #2, #3, #4, #5, #6, #7, #8, #9, #10, #11, #12, #13, #14, #15, #16, #17, #18, #19, #20, #21, #22, #23, #24 ] }
│ │ │ └── PhysicalHashJoin { join_type: Inner, left_keys: [ #1 ], right_keys: [ #0 ] }
│ │ │ ├── PhysicalHashJoin { join_type: Inner, left_keys: [ #0 ], right_keys: [ #0 ] }
│ │ │ │ ├── PhysicalFilter
│ │ │ │ │ ├── cond:And
│ │ │ │ │ │ ├── Geq
│ │ │ │ │ │ │ ├── #4
│ │ │ │ │ │ │ └── Cast { cast_to: Date32, expr: "2023-01-01" }
│ │ │ │ │ │ └── Lt
│ │ │ │ │ │ ├── #4
│ │ │ │ │ │ └── Cast { cast_to: Date32, expr: "2024-01-01" }
│ │ │ │ │ └── PhysicalScan { table: orders }
│ │ │ │ └── PhysicalScan { table: lineitem }
│ │ │ └── PhysicalScan { table: customer }
│ │ └── PhysicalScan { table: supplier }
│ └── PhysicalScan { table: nation }
└── PhysicalFilter
├── cond:Eq
│ ├── #1
│ └── "Asia"
└── PhysicalScan { table: region }
*/

Loading

0 comments on commit 4c8f4b4

Please sign in to comment.