Skip to content

fix: http handler cut block to pages using memory size after gc. #19071

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
youngsofun wants to merge 4 commits into
base: main
Choose a base branch
from
Open

fix: http handler cut block to pages using memory size after gc. #19071

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
12b8a85
Datablock add fn memory_size_with_options
youngsofun Dec 5, 2025
255457f
fix: http handler cut block to pages using memory size after gc.
youngsofun Dec 5, 2025
cb0f3fe
Column::memory_size add arg gc.
youngsofun Dec 5, 2025
0d6aa5f
Column::memory_size add arg gc.
youngsofun Dec 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions src/common/column/src/binview/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -314,8 +314,12 @@ impl<T: ViewType + ?Sized> BinaryViewColumnGeneric<T> {
.get_or_init(|| self.views.iter().map(|v| v.length as usize).sum::<usize>())
}

pub fn memory_size(&self) -> usize {
self.total_buffer_len() + self.len() * 16
pub fn memory_size(&self, gc: bool) -> usize {
if gc {
self.total_bytes_len() + self.len() * 16
} else {
self.total_buffer_len() + self.len() * 16
}
}

fn total_unshared_buffer_len(&self) -> usize {
Expand Down
4 changes: 2 additions & 2 deletions src/common/column/tests/it/binview/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,15 +151,15 @@ fn test_slice() {
];

let array: Utf8ViewColumn = data.into_iter().collect();
assert_eq!(array.memory_size(), 156);
assert_eq!(array.memory_size(false), 156);

let a3 = array.sliced(2, 3);
assert_eq!(a3.into_iter().collect::<Vec<_>>(), vec![
"databend",
"yyyyyyyyyyyyyyyyyyyyy",
"zzzzzzzzzzzzzzzzzzzzz",
]);
assert_eq!(a3.memory_size(), 108);
assert_eq!(a3.memory_size(false), 108);
}

#[test]
Expand Down
9 changes: 8 additions & 1 deletion src/query/expression/src/block.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,14 @@ impl BlockEntry {
pub fn memory_size(&self) -> usize {
match self {
BlockEntry::Const(scalar, _, _) => scalar.as_ref().memory_size(),
BlockEntry::Column(column) => column.memory_size(),
BlockEntry::Column(column) => column.memory_size(false),
}
}

pub fn memory_size_with_options(&self, gc: bool) -> usize {
match self {
BlockEntry::Const(scalar, _, _) => scalar.as_ref().memory_size(),
BlockEntry::Column(column) => column.memory_size(gc),
}
}

Expand Down
3 changes: 2 additions & 1 deletion src/query/expression/src/types.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -607,7 +607,8 @@ pub trait AccessType: Debug + Clone + PartialEq + Sized + 'static {
std::mem::size_of::<Self::Scalar>()
}

fn column_memory_size(col: &Self::Column) -> usize {
fn column_memory_size(col: &Self::Column, gc: bool) -> usize {
Copy link
Collaborator

@forsaken628 forsaken628 Dec 6, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thinking of adding a new function?
https://docs.rs/arrow/latest/arrow/array/struct.ArrayData.html#method.get_buffer_memory_size

let _ = gc;
Self::column_len(col) * std::mem::size_of::<Self::Scalar>()
}

Expand Down
4 changes: 2 additions & 2 deletions src/query/expression/src/types/any.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,8 +83,8 @@ impl AccessType for AnyType {
scalar.memory_size()
}

fn column_memory_size(col: &Self::Column) -> usize {
col.memory_size()
fn column_memory_size(col: &Self::Column, gc: bool) -> usize {
col.memory_size(gc)
}

#[inline(always)]
Expand Down
10 changes: 5 additions & 5 deletions src/query/expression/src/types/array.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,11 +101,11 @@ impl<T: AccessType> AccessType for ArrayType<T> {
}

fn scalar_memory_size(scalar: &Self::ScalarRef<'_>) -> usize {
<T as AccessType>::column_memory_size(scalar)
<T as AccessType>::column_memory_size(scalar, false)
}

fn column_memory_size(col: &Self::Column) -> usize {
col.memory_size()
fn column_memory_size(col: &Self::Column, gc: bool) -> usize {
col.memory_size(gc)
}

fn compare(a: T::Column, b: T::Column) -> Ordering {
Expand Down Expand Up @@ -269,8 +269,8 @@ impl<T: AccessType> ArrayColumn<T> {
}
}

pub fn memory_size(&self) -> usize {
T::column_memory_size(&self.underlying_column()) + self.offsets.len() * 8
pub fn memory_size(&self, gc: bool) -> usize {
T::column_memory_size(&self.underlying_column(), gc) + self.offsets.len() * 8
}

// Note: if the array column has been sliced, the number of values may not match the offsets.
Expand Down
2 changes: 1 addition & 1 deletion src/query/expression/src/types/binary.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ impl AccessType for BinaryType {
scalar.len()
}

fn column_memory_size(col: &Self::Column) -> usize {
fn column_memory_size(col: &Self::Column, _gc: bool) -> usize {
col.data().len() + col.offsets().len() * 8
}

Expand Down
2 changes: 1 addition & 1 deletion src/query/expression/src/types/bitmap.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ impl AccessType for BitmapType {
scalar.len()
}

fn column_memory_size(col: &Self::Column) -> usize {
fn column_memory_size(col: &Self::Column, _gc: bool) -> usize {
col.memory_size()
}

Expand Down
2 changes: 1 addition & 1 deletion src/query/expression/src/types/compute_view.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ where
std::mem::size_of::<F>()
}

fn column_memory_size(col: &Self::Column) -> usize {
fn column_memory_size(col: &Self::Column, _gc: bool) -> usize {
F::column_len(col) * std::mem::size_of::<F>()
}

Expand Down
4 changes: 2 additions & 2 deletions src/query/expression/src/types/generic.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,8 +86,8 @@ impl<const INDEX: usize> AccessType for GenericType<INDEX> {
scalar.memory_size()
}

fn column_memory_size(col: &Self::Column) -> usize {
col.memory_size()
fn column_memory_size(col: &Self::Column, gc: bool) -> usize {
col.memory_size(gc)
}

#[inline(always)]
Expand Down
2 changes: 1 addition & 1 deletion src/query/expression/src/types/geography.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ impl AccessType for GeographyType {
scalar.0.len()
}

fn column_memory_size(col: &Self::Column) -> usize {
fn column_memory_size(col: &Self::Column, _gc: bool) -> usize {
col.memory_size()
}

Expand Down
2 changes: 1 addition & 1 deletion src/query/expression/src/types/geometry.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ impl AccessType for GeometryType {
scalar.len()
}

fn column_memory_size(col: &Self::Column) -> usize {
fn column_memory_size(col: &Self::Column, _gc: bool) -> usize {
col.memory_size()
}

Expand Down
12 changes: 6 additions & 6 deletions src/query/expression/src/types/map.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,8 +111,8 @@ impl<K: AccessType, V: AccessType> AccessType for KvPair<K, V> {
K::scalar_memory_size(k) + V::scalar_memory_size(v)
}

fn column_memory_size(col: &Self::Column) -> usize {
col.memory_size()
fn column_memory_size(col: &Self::Column, gc: bool) -> usize {
col.memory_size(gc)
}

#[inline(always)]
Expand Down Expand Up @@ -289,8 +289,8 @@ impl<K: AccessType, V: AccessType> KvColumn<K, V> {
}
}

pub fn memory_size(&self) -> usize {
K::column_memory_size(&self.keys) + V::column_memory_size(&self.values)
pub fn memory_size(&self, gc: bool) -> usize {
K::column_memory_size(&self.keys, gc) + V::column_memory_size(&self.values, gc)
}
}

Expand Down Expand Up @@ -501,8 +501,8 @@ impl<K: AccessType, V: AccessType> AccessType for MapType<K, V> {
MapInternal::<K, V>::scalar_memory_size(scalar)
}

fn column_memory_size(col: &Self::Column) -> usize {
MapInternal::<K, V>::column_memory_size(col)
fn column_memory_size(col: &Self::Column, gc: bool) -> usize {
MapInternal::<K, V>::column_memory_size(col, gc)
}

#[inline(always)]
Expand Down
8 changes: 4 additions & 4 deletions src/query/expression/src/types/nullable.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,8 +119,8 @@ impl<T: AccessType> AccessType for NullableType<T> {
}
}

fn column_memory_size(col: &Self::Column) -> usize {
col.memory_size()
fn column_memory_size(col: &Self::Column, gc: bool) -> usize {
col.memory_size(gc)
}

// Null default lastly
Expand Down Expand Up @@ -350,8 +350,8 @@ impl<T: AccessType> NullableColumn<T> {
}
}

pub fn memory_size(&self) -> usize {
T::column_memory_size(&self.column) + self.validity.as_slice().0.len()
pub fn memory_size(&self, gc: bool) -> usize {
T::column_memory_size(&self.column, gc) + self.validity.as_slice().0.len()
}

pub fn destructure(self) -> (T::Column, Bitmap) {
Expand Down
2 changes: 1 addition & 1 deletion src/query/expression/src/types/simple_type.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ impl<T: SimpleType> AccessType for SimpleValueType<T> {
std::mem::size_of::<Self::Scalar>()
}

fn column_memory_size(buffer: &Buffer<Self::Scalar>) -> usize {
fn column_memory_size(buffer: &Buffer<Self::Scalar>, _gc: bool) -> usize {
buffer.len() * std::mem::size_of::<Self::Scalar>()
}

Expand Down
4 changes: 2 additions & 2 deletions src/query/expression/src/types/string.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,8 +100,8 @@ impl AccessType for StringType {
scalar.len()
}

fn column_memory_size(col: &Self::Column) -> usize {
col.memory_size()
fn column_memory_size(col: &Self::Column, gc: bool) -> usize {
col.memory_size(gc)
}

#[inline(always)]
Expand Down
2 changes: 1 addition & 1 deletion src/query/expression/src/types/variant.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ impl AccessType for VariantType {
scalar.len()
}

fn column_memory_size(col: &Self::Column) -> usize {
fn column_memory_size(col: &Self::Column, _gc: bool) -> usize {
col.memory_size()
}

Expand Down
2 changes: 1 addition & 1 deletion src/query/expression/src/types/vector.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ impl AccessType for VectorType {
scalar.memory_size()
}

fn column_memory_size(col: &Self::Column) -> usize {
fn column_memory_size(col: &Self::Column, _gc: bool) -> usize {
col.memory_size()
}

Expand Down
2 changes: 1 addition & 1 deletion src/query/expression/src/types/zero_size_type.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ impl<T: ZeroSizeType> AccessType for ZeroSizeValueType<T> {
0
}

fn column_memory_size(_: &usize) -> usize {
fn column_memory_size(_: &usize, _: bool) -> usize {
std::mem::size_of::<usize>()
}

Expand Down
28 changes: 14 additions & 14 deletions src/query/expression/src/values.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -297,10 +297,10 @@ impl<T: AccessType> Value<T> {
}
}

pub fn memory_size(&self) -> usize {
pub fn memory_size(&self, gc: bool) -> usize {
match self {
Value::Scalar(scalar) => T::scalar_memory_size(&T::to_scalar_ref(scalar)),
Value::Column(c) => T::column_memory_size(c),
Value::Column(c) => T::column_memory_size(c, gc),
}
}

Expand Down Expand Up @@ -707,8 +707,8 @@ impl ScalarRef<'_> {
ScalarRef::TimestampTz(_) => 16,
ScalarRef::Date(_) => 4,
ScalarRef::Interval(_) => 16,
ScalarRef::Array(col) => col.memory_size(),
ScalarRef::Map(col) => col.memory_size(),
ScalarRef::Array(col) => col.memory_size(false),
ScalarRef::Map(col) => col.memory_size(false),
ScalarRef::Bitmap(b) => b.len(),
ScalarRef::Tuple(scalars) => scalars.iter().map(|s| s.memory_size()).sum(),
ScalarRef::Variant(buf) => buf.len(),
Expand Down Expand Up @@ -897,8 +897,8 @@ impl ScalarRef<'_> {
ScalarRef::TimestampTz(_) => n * 16,
ScalarRef::Date(_) => n * 4,
ScalarRef::Interval(_) => n * 16,
ScalarRef::Array(col) => col.memory_size() * n + (n + 1) * 8,
ScalarRef::Map(col) => col.memory_size() * n + (n + 1) * 8,
ScalarRef::Array(col) => col.memory_size(false) * n + (n + 1) * 8,
ScalarRef::Map(col) => col.memory_size(false) * n + (n + 1) * 8,
ScalarRef::Bitmap(b) => b.len() * n + (n + 1) * 8,
ScalarRef::Tuple(fields) => {
let DataType::Tuple(fields_ty) = data_type else {
Expand Down Expand Up @@ -1754,7 +1754,7 @@ impl Column {
}
}

pub fn memory_size(&self) -> usize {
pub fn memory_size(&self, gc: bool) -> usize {
match self {
Column::Null { .. } => std::mem::size_of::<usize>(),
Column::EmptyArray { .. } => std::mem::size_of::<usize>(),
Expand All @@ -1774,19 +1774,19 @@ impl Column {
Column::Decimal(DecimalColumn::Decimal256(col, _)) => col.len() * 32,
Column::Boolean(c) => c.as_slice().0.len(),
Column::Binary(col) => col.memory_size(),
Column::String(col) => col.memory_size(),
Column::String(col) => col.memory_size(gc),
Column::Timestamp(col) => col.len() * 8,
Column::TimestampTz(col) => col.len() * 16,
Column::Date(col) => col.len() * 4,
Column::Interval(col) => col.len() * 16,
Column::Array(col) => col.memory_size(),
Column::Map(col) => col.memory_size(),
Column::Array(col) => col.memory_size(gc),
Column::Map(col) => col.memory_size(gc),
Column::Bitmap(col) => col.memory_size(),
Column::Nullable(c) => c.column.memory_size() + c.validity.as_slice().0.len(),
Column::Tuple(fields) => fields.iter().map(|f| f.memory_size()).sum(),
Column::Nullable(c) => c.column.memory_size(gc) + c.validity.as_slice().0.len(),
Column::Tuple(fields) => fields.iter().map(|f| f.memory_size(gc)).sum(),
Column::Variant(col) => col.memory_size(),
Column::Geometry(col) => col.memory_size(),
Column::Geography(col) => GeographyType::column_memory_size(col),
Column::Geography(col) => GeographyType::column_memory_size(col, gc),
Column::Vector(col) => col.memory_size(),
Column::Opaque(col) => col.memory_size(),
}
Expand All @@ -1810,7 +1810,7 @@ impl Column {
Column::Decimal(DecimalColumn::Decimal128(col, _)) => col.len() * 16,
Column::Decimal(DecimalColumn::Decimal256(col, _)) => col.len() * 32,
Column::Interval(col) => col.len() * 16,
Column::Geography(col) => GeographyType::column_memory_size(col),
Column::Geography(col) => GeographyType::column_memory_size(col, false),
Column::Boolean(c) => c.len(),
// 8 * len + size of bytes
Column::Binary(col)
Expand Down
8 changes: 4 additions & 4 deletions src/query/expression/tests/it/block.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,16 +90,16 @@ fn test_block_entry_memory_size() {
assert_eq!(3, entry.memory_size());

let col = StringType::from_data((0..10).map(|x| x.to_string()).collect::<Vec<_>>());
assert_eq!(col.memory_size(), 10 * 16);
assert_eq!(col.memory_size(false), 10 * 16);

let array = ArrayColumn::<Int64Type>::new(
Buffer::from_iter(0..10i64),
Buffer::from(vec![0u64, 1, 3, 6, 10]),
);
let total_memory_size = array.memory_size(); // 10 * 8 + 5 * 8 = 80 + 40 = 120
let total_memory_size = array.memory_size(false); // 10 * 8 + 5 * 8 = 80 + 40 = 120
let expected = array
.iter()
.map(|x| Int64Type::column_memory_size(&x))
.map(|x| Int64Type::column_memory_size(&x, false))
.sum::<usize>()
+ (array.len() + 1) * 8;
assert_eq!(total_memory_size, expected);
Expand All @@ -108,6 +108,6 @@ fn test_block_entry_memory_size() {
let array3 = array.slice(2..4);
assert_eq!(
total_memory_size,
array2.memory_size() + array3.memory_size() - 8
array2.memory_size(false) + array3.memory_size(false) - 8
);
}
2 changes: 1 addition & 1 deletion src/query/expression/tests/it/kernel.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -625,7 +625,7 @@ fn assert_estimated_scalar_repeat_size(scalar: ScalarRef, num_rows: usize, ty: D
let col = builder.build();
assert_eq!(
scalar.estimated_scalar_repeat_size(num_rows, &ty),
col.memory_size()
col.memory_size(false)
);
}

Expand Down
2 changes: 1 addition & 1 deletion src/query/service/src/pipelines/processors/transforms/transform_srf.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ impl BlockingTransform for TransformSRF {
.map(|srf_result| {
let (result, rows) = srf_result.get(i).unwrap();
if *rows > 0 {
result.memory_size()
result.memory_size(false)
} else {
0
}
Expand Down
2 changes: 1 addition & 1 deletion src/query/service/src/servers/http/v1/query/sized_spsc.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,7 +199,7 @@ impl PageBuilder {
.iter()
.map(|entry| match entry {
BlockEntry::Const(scalar, _, n) => *n * scalar.as_ref().memory_size(),
BlockEntry::Column(column) => column.memory_size(),
BlockEntry::Column(column) => column.memory_size(true),
})
.sum::<usize>();

Expand Down
2 changes: 1 addition & 1 deletion src/query/storages/fuse/src/io/read/inverted_index/inverted_index_loader.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -264,7 +264,7 @@ pub(crate) async fn load_inverted_index_files<'a>(
inverted_binary,
&databend_common_expression::types::DataType::Binary,
)?;
inverted_bytes_len += column.memory_size();
inverted_bytes_len += column.memory_size(false);
let value = unsafe { column.index_unchecked(0) };
let bytes = value.as_binary().unwrap();
let file = InvertedIndexFile::create(name.clone(), bytes.to_vec());
Expand Down
Loading
Loading