Skip to content

wip: opt: attempt at building routine parameters as placeholders #158407

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 4 commits into
base: master
Choose a base branch
from
Draft

wip: opt: attempt at building routine parameters as placeholders #158407

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
bcaeb50
sql: add optimizer_build_routine_params_as_placeholders
mgartner Nov 25, 2025
9e2e17c
opt: refactor parameter column synthesis
mgartner Nov 26, 2025
860d0e9
opt: implement optimizer_build_routine_params_as_placeholders
mgartner Nov 26, 2025
67339dc
wip
mgartner Nov 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions pkg/sql/logictest/testdata/logic_test/information_schema
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4186,6 +4186,7 @@ on_update_rehome_row_enabled on
opt_split_scan_limit 2048
optimizer on
optimizer_always_use_histograms on
optimizer_build_routine_params_as_placeholders off
optimizer_check_input_min_row_count 1
optimizer_clamp_inequality_selectivity on
optimizer_clamp_low_histogram_selectivity on
Expand Down
3 changes: 3 additions & 0 deletions pkg/sql/logictest/testdata/logic_test/pg_catalog
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3138,6 +3138,7 @@ null_ordered_last off
on_update_rehome_row_enabled on NULL NULL NULL string
opt_split_scan_limit 2048 NULL NULL NULL string
optimizer_always_use_histograms on NULL NULL NULL string
optimizer_build_routine_params_as_placeholders off NULL NULL NULL string
optimizer_check_input_min_row_count 1 NULL NULL NULL string
optimizer_clamp_inequality_selectivity on NULL NULL NULL string
optimizer_clamp_low_histogram_selectivity on NULL NULL NULL string
Expand Down Expand Up @@ -3387,6 +3388,7 @@ null_ordered_last off
on_update_rehome_row_enabled on NULL user NULL on on
opt_split_scan_limit 2048 NULL user NULL 2048 2048
optimizer_always_use_histograms on NULL user NULL on on
optimizer_build_routine_params_as_placeholders off NULL user NULL off off
optimizer_check_input_min_row_count 1 NULL user NULL 1 1
optimizer_clamp_inequality_selectivity on NULL user NULL on on
optimizer_clamp_low_histogram_selectivity on NULL user NULL on on
Expand Down Expand Up @@ -3627,6 +3629,7 @@ on_update_rehome_row_enabled NULL NULL
opt_split_scan_limit NULL NULL NULL NULL NULL
optimizer NULL NULL NULL NULL NULL
optimizer_always_use_histograms NULL NULL NULL NULL NULL
optimizer_build_routine_params_as_placeholders NULL NULL NULL NULL NULL
optimizer_check_input_min_row_count NULL NULL NULL NULL NULL
optimizer_clamp_inequality_selectivity NULL NULL NULL NULL NULL
optimizer_clamp_low_histogram_selectivity NULL NULL NULL NULL NULL
Expand Down
1 change: 1 addition & 0 deletions pkg/sql/logictest/testdata/logic_test/show_source
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,7 @@ null_ordered_last off
on_update_rehome_row_enabled on
opt_split_scan_limit 2048
optimizer_always_use_histograms on
optimizer_build_routine_params_as_placeholders off
optimizer_check_input_min_row_count 1
optimizer_clamp_inequality_selectivity on
optimizer_clamp_low_histogram_selectivity on
Expand Down
5 changes: 5 additions & 0 deletions pkg/sql/opt/exec/execbuilder/scalar.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1228,6 +1228,11 @@ func (b *Builder) buildRoutinePlanGenerator(
if ord, ok := argOrd(t.Col); ok {
return f.ConstructConstVal(args[ord], t.Typ)
}
case *memo.PlaceholderExpr:
ord := int(t.Value.(*tree.Placeholder).Idx)
if ord < len(args) {
return f.ConstructConstVal(args[ord], t.Typ)
}

case *memo.WithScanExpr:
// Allow referring to "outer" With expressions, if
Expand Down
2 changes: 2 additions & 0 deletions pkg/sql/opt/memo/expr.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -731,6 +731,8 @@ type UDFDefinition struct {
// i-th column in the list corresponds to the i-th parameter of the function.
// During execution of the UDF, these columns are replaced with the arguments
// of the function invocation.
// TODO(mgartner): This can be replaced with the number of params if we
// always build parameter columns as placeholders instead of variables.
Params opt.ColList

// Body contains a relational expression for each statement in the function
Expand Down
4 changes: 1 addition & 3 deletions pkg/sql/opt/memo/expr_format.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1070,9 +1070,7 @@ func (f *ExprFmtCtx) formatScalarWithLabel(
// Ensure that the definition of the UDF is not printed out again if it
// is recursively called.
f.seenUDFs[def] = struct{}{}
if len(def.Params) > 0 {
f.formatColList(tp, "params:", def.Params, opt.ColSet{} /* notNullCols */)
}
f.formatColList(tp, "params:", def.Params, opt.ColSet{} /* notNullCols */)
n := tp.Child("body")
for i := range def.Body {
stmtNode := n
Expand Down
3 changes: 3 additions & 0 deletions pkg/sql/opt/memo/memo.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,7 @@ type Memo struct {
clampLowHistogramSelectivity bool
clampInequalitySelectivity bool
useMaxFrequencySelectivity bool
buildRoutineParamsAsPlaceholders bool

// txnIsoLevel is the isolation level under which the plan was created. This
// affects the planning of some locking operations, so it must be included in
Expand Down Expand Up @@ -330,6 +331,7 @@ func (m *Memo) Init(ctx context.Context, evalCtx *eval.Context) {
clampLowHistogramSelectivity: evalCtx.SessionData().OptimizerClampLowHistogramSelectivity,
clampInequalitySelectivity: evalCtx.SessionData().OptimizerClampInequalitySelectivity,
useMaxFrequencySelectivity: evalCtx.SessionData().OptimizerUseMaxFrequencySelectivity,
buildRoutineParamsAsPlaceholders: evalCtx.SessionData().OptimizerBuildRoutineParamsAsPlaceholders,
txnIsoLevel: evalCtx.TxnIsoLevel,
}
m.metadata.Init()
Expand Down Expand Up @@ -509,6 +511,7 @@ func (m *Memo) IsStale(
m.clampLowHistogramSelectivity != evalCtx.SessionData().OptimizerClampLowHistogramSelectivity ||
m.clampInequalitySelectivity != evalCtx.SessionData().OptimizerClampInequalitySelectivity ||
m.useMaxFrequencySelectivity != evalCtx.SessionData().OptimizerUseMaxFrequencySelectivity ||
m.buildRoutineParamsAsPlaceholders != evalCtx.SessionData().OptimizerBuildRoutineParamsAsPlaceholders ||
m.txnIsoLevel != evalCtx.TxnIsoLevel {
return true, nil
}
Expand Down
5 changes: 5 additions & 0 deletions pkg/sql/opt/memo/memo_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -611,6 +611,11 @@ func TestMemoIsStale(t *testing.T) {
evalCtx.SessionData().OptimizerClampInequalitySelectivity = false
notStale()

evalCtx.SessionData().OptimizerBuildRoutineParamsAsPlaceholders = true
stale()
evalCtx.SessionData().OptimizerBuildRoutineParamsAsPlaceholders = false
notStale()

// User no longer has access to view.
catalog.View(tree.NewTableNameWithSchema("t", catconstants.PublicSchemaName, "abcview")).Revoked = true
_, err = o.Memo().IsStale(ctx, &evalCtx, catalog)
Expand Down
5 changes: 4 additions & 1 deletion pkg/sql/opt/norm/factory.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -393,7 +393,10 @@ func (f *Factory) AssignPlaceholders(from *memo.Memo) (err error) {
newDef := *t.Def
newDef.Body = make([]memo.RelExpr, len(t.Def.Body))
for i := range t.Def.Body {
newDef.Body[i] = f.CopyAndReplaceDefault(t.Def.Body[i], replaceFn).(memo.RelExpr)
// Do not replace placeholders in the body of the UDF - those
// placeholders are references to routine paramaters not
// prepared statement parameters.
newDef.Body[i] = f.CopyWithoutAssigningPlaceholders(t.Def.Body[i]).(memo.RelExpr)
}
return f.ConstructUDFCall(newArgs, &memo.UDFCallPrivate{Def: &newDef})
case *memo.RecursiveCTEExpr:
Expand Down
21 changes: 17 additions & 4 deletions pkg/sql/opt/norm/inline_funcs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"github.com/cockroachdb/cockroach/pkg/sql/catalog/colinfo"
"github.com/cockroachdb/cockroach/pkg/sql/opt"
"github.com/cockroachdb/cockroach/pkg/sql/opt/memo"
"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
"github.com/cockroachdb/cockroach/pkg/sql/sem/volatility"
"github.com/cockroachdb/cockroach/pkg/util/intsets"
"github.com/cockroachdb/errors"
Expand Down Expand Up @@ -458,26 +459,38 @@ func (c *CustomFuncs) IsInlinableUDF(args memo.ScalarListExpr, udfp *memo.UDFCal
func (c *CustomFuncs) ConvertUDFToSubquery(
args memo.ScalarListExpr, udfp *memo.UDFCallPrivate,
) opt.ScalarExpr {
// argForParam returns the argument that can be substituted for the given
// argForParamCol returns the argument that can be substituted for the given
// column, if the column is a parameter of the UDF. It returns ok=false if
// the column is not a UDF parameter.
argForParam := func(col opt.ColumnID) (e opt.Expr, ok bool) {
argForParamCol := func(col opt.ColumnID) (e opt.Expr, ok bool) {
for i := range udfp.Def.Params {
if udfp.Def.Params[i] == col {
return args[i], true
}
}
return nil, false
}
// argForParam returns the argument that can be substituted for the given
// placeholder.
argForParam := func(idx tree.PlaceholderIdx) opt.Expr {
if int(idx) > len(args) {
panic(errors.AssertionFailedf("routine parameter out of range"))
}
return args[int(idx)]
}

// replace substitutes variables that are UDF parameters with the
// corresponding argument from the invocation of the UDF.
var replace ReplaceFunc
replace = func(nd opt.Expr) opt.Expr {
if t, ok := nd.(*memo.VariableExpr); ok {
if arg, ok := argForParam(t.Col); ok {
switch t := nd.(type) {
case *memo.VariableExpr:
if arg, ok := argForParamCol(t.Col); ok {
return arg
}
case *memo.PlaceholderExpr:
p := t.Value.(*tree.Placeholder)
return argForParam(p.Idx)
}
return c.f.Replace(nd, replace)
}
Expand Down
189 changes: 189 additions & 0 deletions pkg/sql/opt/norm/testdata/rules/decorrelate
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -669,6 +669,7 @@ project
# --------------------------------------------------
# TryDecorrelateSelect
# --------------------------------------------------

norm expect=TryDecorrelateSelect
SELECT * FROM a WHERE EXISTS(SELECT * FROM (VALUES (k), (i)) WHERE column1=k)
----
Expand Down Expand Up @@ -8775,3 +8776,191 @@ semi-join-apply
│ ├── x:8 = u:12 [outer=(8,12), constraints=(/8: (/NULL - ]; /12: (/NULL - ]), fd=(8)==(12), (12)==(8)]
│ └── i:2 = 100 [outer=(2), constraints=(/2: [/100 - /100]; tight), fd=()-->(2)]
└── filters (true)


# --------------------------------------------------
# Routine-related decorrelation regression tests
# --------------------------------------------------

# Regression tests for #157840.
exec-ddl
CREATE PROCEDURE p157840a(abs ab[]) LANGUAGE SQL AS $$
WITH vals AS (
SELECT (v).a, (v).b
FROM ROWS FROM (unnest(abs)) AS v
)
SELECT * FROM xy JOIN vals ON x = a
$$
----

# Join filters with parameter columns are not pull up above the join.
norm format=show-scalars
CALL p157840a(ARRAY[
ROW(1, 'one')::ab,
ROW(2, 'two')::ab,
ROW(3, 'three')::ab
])
----
call
├── cardinality: [0 - 0]
├── volatile
└── procedure: p157840a
├── args
│ └── array:
│ ├── cast: RECORD
│ │ └── tuple
│ │ ├── const: 1
│ │ └── const: 'one'
│ ├── cast: RECORD
│ │ └── tuple
│ │ ├── const: 2
│ │ └── const: 'two'
│ └── cast: RECORD
│ └── tuple
│ ├── const: 3
│ └── const: 'three'
├── params: abs:1
└── body
├── inner-join (hash)
│ ├── columns: x:5!null y:6 a:9!null b:10
│ ├── multiplicity: left-rows(zero-or-more), right-rows(zero-or-one)
│ ├── immutable, has-placeholder
│ ├── fd: (5)-->(6), (5)==(9), (9)==(5)
│ ├── scan xy
│ │ ├── columns: x:5!null y:6
│ │ ├── key: (5)
│ │ └── fd: (5)-->(6)
│ ├── project
│ │ ├── columns: a:9 b:10
│ │ ├── immutable, has-placeholder
│ │ ├── project-set
│ │ │ ├── columns: unnest:2
│ │ │ ├── immutable, has-placeholder
│ │ │ ├── values
│ │ │ │ ├── cardinality: [1 - 1]
│ │ │ │ ├── key: ()
│ │ │ │ └── tuple
│ │ │ └── zip
│ │ │ └── function: unnest [immutable]
│ │ │ └── placeholder: $1
│ │ └── projections
│ │ ├── column-access: 0 [as=a:9, outer=(2)]
│ │ │ └── variable: unnest:2
│ │ └── column-access: 1 [as=b:10, outer=(2)]
│ │ └── variable: unnest:2
│ └── filters
│ └── eq [outer=(5,9), constraints=(/5: (/NULL - ]; /9: (/NULL - ]), fd=(5)==(9), (9)==(5)]
│ ├── variable: x:5
│ └── variable: a:9
└── values
├── columns: column1:11
├── cardinality: [1 - 1]
├── key: ()
├── fd: ()-->(11)
└── tuple
└── null

exec-ddl
CREATE PROCEDURE p157840b(abs ab[]) LANGUAGE SQL AS $$
WITH vals AS (
SELECT (v).a, (v).b
FROM ROWS FROM (unnest(abs)) AS v
)
SELECT * FROM xy
JOIN vals ON true
JOIN LATERAL (SELECT u, u/1.1 AS div FROM uv WHERE x=5 AND y=b) ON true
$$
----

# The x=5 filter can be pushed above the xy scan and the y=b filter remains in
# the lowest join filters possible.
norm format=show-scalars
CALL p157840b(ARRAY[
ROW(1, 'one')::ab,
ROW(2, 'two')::ab,
ROW(3, 'three')::ab
])
----
call
├── cardinality: [0 - 0]
├── volatile
└── procedure: p157840b
├── args
│ └── array:
│ ├── cast: RECORD
│ │ └── tuple
│ │ ├── const: 1
│ │ └── const: 'one'
│ ├── cast: RECORD
│ │ └── tuple
│ │ ├── const: 2
│ │ └── const: 'two'
│ └── cast: RECORD
│ └── tuple
│ ├── const: 3
│ └── const: 'three'
├── params: abs:1
└── body
├── project
│ ├── columns: div:15!null x:5!null y:6!null a:9 b:10!null u:11!null
│ ├── immutable, has-placeholder
│ ├── fd: ()-->(5,6,10), (11)-->(15), (6)==(10), (10)==(6)
│ ├── inner-join (cross)
│ │ ├── columns: x:5!null y:6!null a:9 b:10!null u:11!null
│ │ ├── immutable, has-placeholder
│ │ ├── fd: ()-->(5,6,10), (6)==(10), (10)==(6)
│ │ ├── inner-join (hash)
│ │ │ ├── columns: x:5!null y:6!null a:9 b:10!null
│ │ │ ├── multiplicity: left-rows(zero-or-more), right-rows(zero-or-one)
│ │ │ ├── immutable, has-placeholder
│ │ │ ├── fd: ()-->(5,6,10), (6)==(10), (10)==(6)
│ │ │ ├── select
│ │ │ │ ├── columns: x:5!null y:6
│ │ │ │ ├── cardinality: [0 - 1]
│ │ │ │ ├── key: ()
│ │ │ │ ├── fd: ()-->(5,6)
│ │ │ │ ├── scan xy
│ │ │ │ │ ├── columns: x:5!null y:6
│ │ │ │ │ ├── key: (5)
│ │ │ │ │ └── fd: (5)-->(6)
│ │ │ │ └── filters
│ │ │ │ └── eq [outer=(5), constraints=(/5: [/5 - /5]; tight), fd=()-->(5)]
│ │ │ │ ├── variable: x:5
│ │ │ │ └── const: 5
│ │ │ ├── project
│ │ │ │ ├── columns: a:9 b:10
│ │ │ │ ├── immutable, has-placeholder
│ │ │ │ ├── project-set
│ │ │ │ │ ├── columns: unnest:2
│ │ │ │ │ ├── immutable, has-placeholder
│ │ │ │ │ ├── values
│ │ │ │ │ │ ├── cardinality: [1 - 1]
│ │ │ │ │ │ ├── key: ()
│ │ │ │ │ │ └── tuple
│ │ │ │ │ └── zip
│ │ │ │ │ └── function: unnest [immutable]
│ │ │ │ │ └── placeholder: $1
│ │ │ │ └── projections
│ │ │ │ ├── column-access: 0 [as=a:9, outer=(2)]
│ │ │ │ │ └── variable: unnest:2
│ │ │ │ └── column-access: 1 [as=b:10, outer=(2)]
│ │ │ │ └── variable: unnest:2
│ │ │ └── filters
│ │ │ └── eq [outer=(6,10), constraints=(/6: (/NULL - ]; /10: (/NULL - ]), fd=(6)==(10), (10)==(6)]
│ │ │ ├── variable: y:6
│ │ │ └── variable: b:10
│ │ ├── scan uv
│ │ │ ├── columns: u:11!null
│ │ │ └── key: (11)
│ │ └── filters (true)
│ └── projections
│ └── div [as=div:15, outer=(11)]
│ ├── variable: u:11
│ └── const: 1.1
└── values
├── columns: column1:16
├── cardinality: [1 - 1]
├── key: ()
├── fd: ()-->(16)
└── tuple
└── null
Loading
Loading