From 54f1fdc5e101f5117d10dd4d3afbc954d3406001 Mon Sep 17 00:00:00 2001 From: Michal Charemza Date: Sat, 18 Jan 2025 08:12:12 +0000 Subject: [PATCH] Optimize slow query that uses a high amount of temporary disk space to find relations Resolves https://github.com/dbt-labs/dbt-postgres/issues/189 The macro postgres_get_relations in relations.sql was extremely slow and used an extremely high amount of temporary disk space on a system with high numbers of schemas, tables, and dependencies between database objects (rows in pg_depend). Slow to the point of not completing in 50 minutes and using more than 160GB disk space (at which point PostgreSQL ran out of disk space and aborted the query). The solution here optimises the query and so it runs in just under 1 second on my system. It does this by being heavily inspired by the definition of information_schema.view_table_usage, and specifically: - Stripping out CTEs that can be optimisation blockers, often by causing CTEs to be materialised to disk (especially in older PostgreSQL, but I suspect in recent too in some cases). - Removing unnecessary filtering on relkind: going via pg_rewrite (or rather, the equivalent row on pg_depend) is equivalent to that - Avoiding sequential scans on any table by structuring joins/where clause to leverage indexes, especially on pg_depend - Removing unnecessary filtering out system catalog tables from dependents (they are excluded by the remaining filters on referenced tables). - Not having `select distinct ... from pg_dependent` in the innards of the query, and instead having a top level `select distinct` - on my system this saved over 45 seconds. - Excluding self-relations that depend on themselves by using oid rather than using the names of tables and schemas. I suspect this is also more robust because oids I think _can_ be repeated between system tables, and so when querying pg_depend filtering on classid and refclassid is required (and I think also means indexes are better leveraged). Comparing calls to `explain` it reduces the largest "rows" value from 5,284,141,410,595,979 (over five quadrillion) to 219 and the actual run time from never completing within 50 minutes (because it used all of the 160GB available) to completing in ~500ms. It also has some style/naming changes: - Using a `distinct` on the top level rather than a group by for clarity (performance seemed the same in my case). - Flips the definition of "referenced" and "dependent" in the query to match both the definitions in pg_depend, and the code at https://github.com/dbt-labs/dbt-postgres/blob/05f0337d6b05c9c68617e41c0b5bca9c2a733783/dbt/adapters/postgres/impl.py#L113 - Re-orders the join to I think a slightly clearer order that "flows" from views -> the linking table (pg_depend) to the tables referenced in the views. - Lowers the abstraction/indirection levels in naming/aliases, using names closer to the PostgreSQL catalog tables - this made it easier to write and understand, and so I suspect easier to make changes in future (I found I had to keep in mind the PostgreSQL definitions more than the output of the query when making changes). --- .../unreleased/Fixes-20250118-084103.yaml | 6 ++ dbt/include/postgres/macros/relations.sql | 93 +++++++------------ 2 files changed, 38 insertions(+), 61 deletions(-) create mode 100644 .changes/unreleased/Fixes-20250118-084103.yaml diff --git a/.changes/unreleased/Fixes-20250118-084103.yaml b/.changes/unreleased/Fixes-20250118-084103.yaml new file mode 100644 index 00000000..66dcd3ec --- /dev/null +++ b/.changes/unreleased/Fixes-20250118-084103.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Optimize slow query that uses a high amount of temporary disk space to find relations +time: 2025-01-18T08:41:03.022013Z +custom: + Author: michalc + Issue: "189" diff --git a/dbt/include/postgres/macros/relations.sql b/dbt/include/postgres/macros/relations.sql index dd50cf00..dc0a8a8a 100644 --- a/dbt/include/postgres/macros/relations.sql +++ b/dbt/include/postgres/macros/relations.sql @@ -7,68 +7,39 @@ #} {%- call statement('relations', fetch_result=True) -%} - with relation as ( - select - pg_rewrite.ev_class as class, - pg_rewrite.oid as id - from pg_rewrite - ), - class as ( - select - oid as id, - relname as name, - relnamespace as schema, - relkind as kind - from pg_class - ), - dependency as ( - select distinct - pg_depend.objid as id, - pg_depend.refobjid as ref - from pg_depend - ), - schema as ( - select - pg_namespace.oid as id, - pg_namespace.nspname as name - from pg_namespace - where nspname != 'information_schema' and nspname not like 'pg\_%' - ), - referenced as ( - select - relation.id AS id, - referenced_class.name , - referenced_class.schema , - referenced_class.kind - from relation - join class as referenced_class on relation.class=referenced_class.id - where referenced_class.kind in ('r', 'v', 'm') - ), - relationships as ( - select - referenced.name as referenced_name, - referenced.schema as referenced_schema_id, - dependent_class.name as dependent_name, - dependent_class.schema as dependent_schema_id, - referenced.kind as kind - from referenced - join dependency on referenced.id=dependency.id - join class as dependent_class on dependency.ref=dependent_class.id - where - (referenced.name != dependent_class.name or - referenced.schema != dependent_class.schema) - ) + select distinct + dependent_namespace.nspname as dependent_schema, + dependent_class.relname as dependent_name, + referenced_namespace.nspname as referenced_schema, + referenced_class.relname as referenced_name - select - referenced_schema.name as referenced_schema, - relationships.referenced_name as referenced_name, - dependent_schema.name as dependent_schema, - relationships.dependent_name as dependent_name - from relationships - join schema as dependent_schema on relationships.dependent_schema_id=dependent_schema.id - join schema as referenced_schema on relationships.referenced_schema_id=referenced_schema.id - group by referenced_schema, referenced_name, dependent_schema, dependent_name - order by referenced_schema, referenced_name, dependent_schema, dependent_name; + -- Query for views: views are entries in pg_class with an entry in pg_rewrite, but we avoid + -- a seq scan on pg_rewrite by leveraging the fact there is an "internal" row in pg_depend for + -- the view... + from pg_class as dependent_class + join pg_namespace as dependent_namespace on dependent_namespace.oid = dependent_class.relnamespace + join pg_depend as dependent_depend on dependent_depend.refobjid = dependent_class.oid + and dependent_depend.classid = 'pg_rewrite'::regclass + and dependent_depend.refclassid = 'pg_class'::regclass + and dependent_depend.deptype = 'i' + + -- ... and via pg_depend (that has a row per column, hence the need for "distinct" above, and + -- making sure to exclude the internal row to avoid a view appearing to depend on itself)... + join pg_depend as joining_depend on joining_depend.objid = dependent_depend.objid + and joining_depend.classid = 'pg_rewrite'::regclass + and joining_depend.refclassid = 'pg_class'::regclass + and joining_depend.refobjid != dependent_depend.refobjid + + -- ... we can find the tables they query from in pg_class, but excluding system tables. Note we + -- don't need need to exclude _dependent_ system tables, because they only query from other + -- system tables, and so are automatically excluded by excluding _referenced_ system tables + join pg_class as referenced_class on referenced_class.oid = joining_depend.refobjid + join pg_namespace as referenced_namespace on referenced_namespace.oid = referenced_class.relnamespace + and referenced_namespace.nspname != 'information_schema' + and referenced_namespace.nspname not like 'pg\_%' + + order by + dependent_schema, dependent_name, referenced_schema, referenced_name; {%- endcall -%}