docs(python): Add Polars & LLMs page to the user guide

.github/workflows/benchmark-remote.yml

@@ -74,6 +74,7 @@ jobs:
         working-directory: py-polars
         run: |
           "$HOME/py-polars-cache/add-data.py" "$PWD/polars" < ./benchmark-results
+          pip install seaborn
           "$HOME/py-polars-cache/create-plots.py"
           touch "$HOME/py-polars-cache/upload-probe"
           "$HOME/py-polars-cache/cache-build.sh" "$PWD/polars"

crates/polars-core/src/frame/mod.rs

@@ -368,8 +368,13 @@ impl DataFrame {
     /// static EMPTY: DataFrame = DataFrame::empty();
     /// ```
     pub const fn empty() -> Self {
+        Self::empty_with_height(0)
+    }
+
+    /// Creates an empty `DataFrame` with a specific `height`.
+    pub const fn empty_with_height(height: usize) -> Self {
         DataFrame {
-            height: 0,
+            height,
             columns: vec![],
             cached_schema: OnceLock::new(),
         }

crates/polars-error/src/lib.rs

@@ -338,6 +338,14 @@ macro_rules! polars_err {
     (opq = $op:ident, $lhs:expr, $rhs:expr) => {
         $crate::polars_err!(op = stringify!($op), $lhs, $rhs)
     };
+    (bigidx, ctx = $ctx:expr, size = $size:expr) => {
+        polars_err!(ComputeError: "\
+{} produces {} rows which is more than maximum allowed pow(2, 32) rows; \
+consider compiling with bigidx feature (polars-u64-idx package on python)",
+            $ctx,
+            $size,
+        )
+    };
     (append) => {
         polars_err!(SchemaMismatch: "cannot append series, data types don't match")
     };

crates/polars-expr/src/hash_keys.rs

@@ -143,6 +143,13 @@ impl HashKeys {
             Self::Single(s) => Self::Single(s.gather(idxs)),
         }
     }
+
+    pub fn sketch_cardinality(&self, sketch: &mut CardinalitySketch) {
+        match self {
+            HashKeys::RowEncoded(s) => s.sketch_cardinality(sketch),
+            HashKeys::Single(s) => s.sketch_cardinality(sketch),
+        }
+    }
 }
 
 #[derive(Clone, Debug)]
@@ -231,6 +238,20 @@ impl RowEncodedKeys {
             keys,
         }
     }
+
+    pub fn sketch_cardinality(&self, sketch: &mut CardinalitySketch) {
+        if let Some(validity) = self.keys.validity() {
+            for (h, is_v) in self.hashes.values_iter().zip(validity) {
+                if is_v {
+                    sketch.insert(*h);
+                }
+            }
+        } else {
+            for h in self.hashes.values_iter() {
+                sketch.insert(*h);
+            }
+        }
+    }
 }
 
 /// Single keys. Does not pre-hash for boolean & integer types, only for strings
@@ -284,4 +305,8 @@ impl SingleKeys {
             keys: self.keys.take_slice_unchecked(idxs),
         }
     }
+
+    pub fn sketch_cardinality(&self, _sketch: &mut CardinalitySketch) {
+        todo!()
+    }
 }

crates/polars-io/src/csv/read/parser.rs

@@ -6,6 +6,7 @@ use polars_core::prelude::*;
 use polars_core::{config, POOL};
 use polars_error::feature_gated;
 use polars_utils::index::Bounded;
+use polars_utils::select::select_unpredictable;
 use rayon::prelude::*;
 
 use super::buffer::Buffer;
@@ -607,6 +608,13 @@ pub struct CountLines {
     quoting: bool,
 }
 
+#[derive(Copy, Clone, Debug)]
+pub struct LineStats {
+    newline_count: usize,
+    last_newline_offset: usize,
+    end_inside_string: bool,
+}
+
 impl CountLines {
     pub fn new(quote_char: Option<u8>, eol_char: u8) -> Self {
         let quoting = quote_char.is_some();
@@ -626,6 +634,98 @@ impl CountLines {
         }
     }
 
+    /// Analyzes a chunk of CSV data.
+    ///
+    /// Returns (newline_count, last_newline_offset, end_inside_string) twice,
+    /// the first is assuming the start of the chunk is *not* inside a string,
+    /// the second assuming the start is inside a string.
+    pub fn analyze_chunk(&self, bytes: &[u8]) -> [LineStats; 2] {
+        let mut scan_offset = 0;
+        let mut states = [
+            LineStats {
+                newline_count: 0,
+                last_newline_offset: 0,
+                end_inside_string: false,
+            },
+            LineStats {
+                newline_count: 0,
+                last_newline_offset: 0,
+                end_inside_string: false,
+            },
+        ];
+
+        // false if even number of quotes seen so far, true otherwise.
+        #[allow(unused_assignments)]
+        let mut global_quote_parity = false;
+
+        #[cfg(feature = "simd")]
+        {
+            // 0 if even number of quotes seen so far, u64::MAX otherwise.
+            let mut global_quote_parity_mask = 0;
+            while scan_offset + 64 <= bytes.len() {
+                let block: [u8; 64] = unsafe {
+                    bytes
+                        .get_unchecked(scan_offset..scan_offset + 64)
+                        .try_into()
+                        .unwrap_unchecked()
+                };
+                let simd_bytes = SimdVec::from(block);
+                let eol_mask = simd_bytes.simd_eq(self.simd_eol_char).to_bitmask();
+                if self.quoting {
+                    let quote_mask = simd_bytes.simd_eq(self.simd_quote_char).to_bitmask();
+                    let quote_parity =
+                        prefix_xorsum_inclusive(quote_mask) ^ global_quote_parity_mask;
+                    global_quote_parity_mask = ((quote_parity as i64) >> 63) as u64;
+
+                    let start_outside_string_eol_mask = eol_mask & !quote_parity;
+                    states[0].newline_count += start_outside_string_eol_mask.count_ones() as usize;
+                    states[0].last_newline_offset = select_unpredictable(
+                        start_outside_string_eol_mask != 0,
+                        (scan_offset + 63)
+                            .wrapping_sub(start_outside_string_eol_mask.leading_zeros() as usize),
+                        states[0].last_newline_offset,
+                    );
+
+                    let start_inside_string_eol_mask = eol_mask & quote_parity;
+                    states[1].newline_count += start_inside_string_eol_mask.count_ones() as usize;
+                    states[1].last_newline_offset = select_unpredictable(
+                        start_inside_string_eol_mask != 0,
+                        (scan_offset + 63)
+                            .wrapping_sub(start_inside_string_eol_mask.leading_zeros() as usize),
+                        states[1].last_newline_offset,
+                    );
+                } else {
+                    states[0].newline_count += eol_mask.count_ones() as usize;
+                    states[0].last_newline_offset = select_unpredictable(
+                        eol_mask != 0,
+                        (scan_offset + 63).wrapping_sub(eol_mask.leading_zeros() as usize),
+                        states[0].last_newline_offset,
+                    );
+                }
+
+                scan_offset += 64;
+            }
+
+            global_quote_parity = global_quote_parity_mask > 0;
+        }
+
+        while scan_offset < bytes.len() {
+            let c = unsafe { *bytes.get_unchecked(scan_offset) };
+            global_quote_parity ^= (c == self.quote_char) & self.quoting;
+
+            let state = &mut states[global_quote_parity as usize];
+            state.newline_count += (c == self.eol_char) as usize;
+            state.last_newline_offset =
+                select_unpredictable(c == self.eol_char, scan_offset, state.last_newline_offset);
+
+            scan_offset += 1;
+        }
+
+        states[0].end_inside_string = global_quote_parity;
+        states[1].end_inside_string = !global_quote_parity;
+        states
+    }
+
     pub fn find_next(&self, bytes: &[u8], chunk_size: &mut usize) -> (usize, usize) {
         loop {
             let b = unsafe { bytes.get_unchecked(..(*chunk_size).min(bytes.len())) };

crates/polars-plan/src/plans/ir/format.rs

@@ -3,6 +3,7 @@ use std::fmt::{self, Display, Formatter};
 use polars_core::datatypes::AnyValue;
 use polars_core::schema::Schema;
 use polars_io::RowIndex;
+use polars_utils::format_list_truncated;
 use recursive::recursive;
 
 use self::ir::dot::ScanSourcesDisplay;
@@ -272,16 +273,20 @@ impl<'a> IRDisplay<'a> {
                 ..
             } => {
                 let total_columns = schema.len();
-                let n_columns = if let Some(columns) = output_schema {
-                    columns.len().to_string()
+                let (n_columns, projected) = if let Some(schema) = output_schema {
+                    (
+                        format!("{}", schema.len()),
+                        format_list_truncated!(schema.iter_names(), 4, '"'),
+                    )
                 } else {
-                    "*".to_string()
+                    ("*".to_string(), "".to_string())
                 };
                 write!(
                     f,
-                    "{:indent$}DF {:?}; PROJECT {}/{} COLUMNS",
+                    "{:indent$}DF {}; PROJECT{} {}/{} COLUMNS",
                     "",
-                    schema.iter_names().take(4).collect::<Vec<_>>(),
+                    format_list_truncated!(schema.iter_names(), 4, '"'),
+                    projected,
                     n_columns,
                     total_columns,
                 )

crates/polars-plan/src/plans/ir/tree_format.rs

@@ -1,6 +1,7 @@
 use std::fmt;
 
 use polars_core::error::*;
+use polars_utils::{format_list_container_truncated, format_list_truncated};
 #[cfg(feature = "regex")]
 use regex::Regex;
 
@@ -190,22 +191,33 @@ impl<'a> TreeFmtNode<'a> {
                         schema,
                         output_schema,
                         ..
-                    } => ND(
-                        wh(
-                            h,
-                            &format!(
-                                "DF {:?}\nPROJECT {}/{} COLUMNS",
-                                schema.iter_names().take(4).collect::<Vec<_>>(),
-                                if let Some(columns) = output_schema {
-                                    format!("{}", columns.len())
-                                } else {
-                                    "*".to_string()
-                                },
-                                schema.len()
+                    } => {
+                        let (n_columns, projected) = if let Some(schema) = output_schema {
+                            (
+                                format!("{}", schema.len()),
+                                format!(
+                                    ": {};",
+                                    format_list_truncated!(schema.iter_names(), 4, '"')
+                                ),
+                            )
+                        } else {
+                            ("*".to_string(), "".to_string())
+                        };
+                        ND(
+                            wh(
+                                h,
+                                &format!(
+                                    "DF {}\nPROJECT{} {}/{} COLUMNS",
+                                    format_list_truncated!(schema.iter_names(), 4, '"'),
+                                    projected,
+                                    n_columns,
+                                    schema.len()
+                                ),
                             ),
-                        ),
-                        vec![],
-                    ),
+                            vec![],
+                        )
+                    },
+
                     Union { inputs, .. } => ND(
                         wh(
                             h,

crates/polars-plan/src/plans/optimizer/mod.rs

@@ -71,6 +71,7 @@ pub fn optimize(
     #[allow(dead_code)]
     let verbose = verbose();
 
+    #[cfg(feature = "python")]
     if opt_flags.streaming() {
         polars_warn!(
             Deprecation,

crates/polars-plan/src/plans/optimizer/projection_pushdown/functions/mod.rs

@@ -42,14 +42,9 @@ pub(super) fn process_functions(
             Ok(lp)
         },
         Explode { columns, .. } => {
-            columns.iter().for_each(|name| {
-                add_str_to_accumulated(
-                    name.clone(),
-                    &mut ctx.acc_projections,
-                    &mut ctx.projected_names,
-                    expr_arena,
-                )
-            });
+            columns
+                .iter()
+                .for_each(|name| add_str_to_accumulated(name.clone(), &mut ctx, expr_arena));
             proj_pd.pushdown_and_assign(input, ctx, lp_arena, expr_arena)?;
             Ok(IRBuilder::new(input, expr_arena, lp_arena)
                 .explode(columns.clone())

crates/polars-plan/src/plans/optimizer/projection_pushdown/functions/unpivot.rs

@@ -14,36 +14,25 @@ pub(super) fn process_unpivot(
         // restart projection pushdown
         proj_pd.no_pushdown_restart_opt(lp, ctx, lp_arena, expr_arena)
     } else {
-        let (mut acc_projections, mut local_projections, mut projected_names) =
-            split_acc_projections(
-                ctx.acc_projections,
-                lp_arena.get(input).schema(lp_arena).as_ref(),
-                expr_arena,
-                false,
-            );
+        let (acc_projections, mut local_projections, projected_names) = split_acc_projections(
+            ctx.acc_projections,
+            lp_arena.get(input).schema(lp_arena).as_ref(),
+            expr_arena,
+            false,
+        );
 
         if !local_projections.is_empty() {
             local_projections.extend_from_slice(&acc_projections);
         }
+        let mut ctx = ProjectionContext::new(acc_projections, projected_names, ctx.inner);
 
         // make sure that the requested columns are projected
-        args.index.iter().for_each(|name| {
-            add_str_to_accumulated(
-                name.clone(),
-                &mut acc_projections,
-                &mut projected_names,
-                expr_arena,
-            )
-        });
-        args.on.iter().for_each(|name| {
-            add_str_to_accumulated(
-                name.clone(),
-                &mut acc_projections,
-                &mut projected_names,
-                expr_arena,
-            )
-        });
-        let ctx = ProjectionContext::new(acc_projections, projected_names, ctx.inner);
+        args.index
+            .iter()
+            .for_each(|name| add_str_to_accumulated(name.clone(), &mut ctx, expr_arena));
+        args.on
+            .iter()
+            .for_each(|name| add_str_to_accumulated(name.clone(), &mut ctx, expr_arena));
 
         proj_pd.pushdown_and_assign(input, ctx, lp_arena, expr_arena)?;