Snowflake Web: Optimise partition pruning (Close #140)

snowplow · Oct 13, 2022 · b49931e · b49931e
1 parent 7b2c528
commit b49931e
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 4 deletions.
diff --git a/web/v1/snowflake/sql-runner/sql/standard/01-base/01-main/03-sessions-to-process.sql b/web/v1/snowflake/sql-runner/sql/standard/01-base/01-main/03-sessions-to-process.sql
@@ -15,6 +15,10 @@
 */
 
 
+-- Use variable to set scan limits
+
+SET (LOWER_LIMIT, UPPER_LIMIT) = (SELECT lower_limit, upper_limit FROM {{.scratch_schema}}.base_new_events_limits{{.entropy}});
+
 -- Get sessionids for new events
 CREATE OR REPLACE TABLE {{.scratch_schema}}.base_sessions_to_process{{.entropy}}
 AS (
@@ -29,8 +33,8 @@ AS (
     ON a.event_id = b.event_id
 
   WHERE b.event_id IS NULL
-    AND a.collector_tstamp >= (SELECT lower_limit FROM {{.scratch_schema}}.base_new_events_limits{{.entropy}})
-    AND a.collector_tstamp <= (SELECT upper_limit FROM {{.scratch_schema}}.base_new_events_limits{{.entropy}})
+    AND a.collector_tstamp >= $LOWER_LIMIT
+    AND a.collector_tstamp <= $UPPER_LIMIT
     AND a.domain_sessionid IS NOT NULL
     AND TIMESTAMPDIFF(DAY, a.dvce_created_tstamp, a.dvce_sent_tstamp) <= {{or .days_late_allowed 3}}
 

diff --git a/web/v1/snowflake/sql-runner/sql/standard/01-base/01-main/06-events-this-run.sql b/web/v1/snowflake/sql-runner/sql/standard/01-base/01-main/06-events-this-run.sql
@@ -18,6 +18,7 @@
 CREATE OR REPLACE PROCEDURE {{.scratch_schema}}.create_events_this_run()
   RETURNS VARCHAR
   LANGUAGE JAVASCRIPT
+  EXECUTE AS CALLER
   AS
   $$
 
@@ -37,16 +38,20 @@ CREATE OR REPLACE PROCEDURE {{.scratch_schema}}.create_events_this_run()
       new_col = new_col + ',';
   }
 
+  var sql_stmt2 =`SET (LOWER_LIMIT, UPPER_LIMIT) = (SELECT lower_limit, upper_limit FROM {{.scratch_schema}}.base_run_limits{{.entropy}});`
+  snowflake.createStatement({sqlText: sql_stmt2}).execute();
+
   var fin_query=`
+
     CREATE OR REPLACE TABLE {{.scratch_schema}}.events_this_run{{.entropy}}
     AS
       SELECT
         ` + new_col + ` ` + result + `
       FROM {{.input_schema}}.events AS a
       INNER JOIN {{.scratch_schema}}.base_sessions_to_include{{.entropy}} AS b
         ON a.domain_sessionid = b.session_id
-      WHERE a.collector_tstamp >= (SELECT lower_limit FROM {{.scratch_schema}}.base_run_limits{{.entropy}})
-        AND a.collector_tstamp <= (SELECT upper_limit FROM {{.scratch_schema}}.base_run_limits{{.entropy}});`;
+      WHERE a.collector_tstamp >= $LOWER_LIMIT
+        AND a.collector_tstamp <= $UPPER_LIMIT;`;
 
   snowflake.createStatement({sqlText: fin_query}).execute();