Skip to content

Commit 9fae32b

Browse files
committed
chore: add some extra code comments
1 parent 871da0e commit 9fae32b

File tree

2 files changed

+92
-6
lines changed

2 files changed

+92
-6
lines changed

src/query/functions/src/scalars/hilbert.rs

Lines changed: 43 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,12 +34,20 @@ use databend_common_expression::FunctionSignature;
3434
use databend_common_expression::ScalarRef;
3535
use databend_common_expression::Value;
3636

37+
/// Registers Hilbert curve related functions with the function registry.
3738
pub fn register(registry: &mut FunctionRegistry) {
39+
// Register the hilbert_range_index function that calculates Hilbert indices for multi-dimensional data
3840
registry.register_function_factory("hilbert_range_index", |_, args_type| {
3941
let args_num = args_type.len();
42+
// The function supports 2, 3, 4, or 5 dimensions (each dimension requires 2 arguments)
4043
if ![4, 6, 8, 10].contains(&args_num) {
4144
return None;
4245
}
46+
47+
// Create the function signature with appropriate argument types
48+
// For each dimension, we need:
49+
// 1. A value (the point coordinate in that dimension)
50+
// 2. An array of boundaries (for partitioning that dimension)
4351
let sig_args_type = (0..args_num / 2)
4452
.flat_map(|idx| {
4553
[
@@ -48,6 +56,7 @@ pub fn register(registry: &mut FunctionRegistry) {
4856
]
4957
})
5058
.collect();
59+
5160
Some(Arc::new(Function {
5261
signature: FunctionSignature {
5362
name: "hilbert_range_index".to_string(),
@@ -57,33 +66,49 @@ pub fn register(registry: &mut FunctionRegistry) {
5766
eval: FunctionEval::Scalar {
5867
calc_domain: Box::new(|_, _| FunctionDomain::Full),
5968
eval: Box::new(move |args, ctx| {
69+
// Determine if we're processing scalar values or columns
6070
let input_all_scalars = args.iter().all(|arg| arg.as_scalar().is_some());
6171
let process_rows = if input_all_scalars { 1 } else { ctx.num_rows };
6272
let mut builder = BinaryType::create_builder(process_rows, &[]);
73+
6374
for index in 0..process_rows {
6475
let mut points = Vec::with_capacity(args_num / 2);
76+
77+
// Process each dimension (each dimension has a value and boundaries array)
6578
for i in (0..args_num).step_by(2) {
66-
let arg1 = &args[i];
67-
let arg2 = &args[i + 1];
68-
79+
let arg1 = &args[i]; // The value in this dimension
80+
let arg2 = &args[i + 1]; // The boundaries array for this dimension
81+
82+
// Get the value and boundaries for this row
6983
let val = unsafe { arg1.index_unchecked(index) };
7084
let arr = unsafe { arg2.index_unchecked(index) };
85+
86+
// Calculate the partition ID for this dimension (capped at 65535, i.e. 2 bytes or 16 bits)
87+
// This effectively discretizes the continuous dimension into buckets
7188
let id = arr
7289
.as_array()
7390
.map(|arr| calc_range_partition_id(val, arr).min(65535) as u16)
7491
.unwrap_or(0);
92+
93+
// Encode the partition ID as bytes
7594
let key = id.encode();
7695
points.push(key);
7796
}
97+
98+
// Convert the multi-dimensional point to a Hilbert index
99+
// This maps the n-dimensional point to a 1-dimensional value
78100
let points = points
79101
.iter()
80102
.map(|array| array.as_slice())
81103
.collect::<Vec<_>>();
82104
let slice = hilbert_index(&points, 2);
105+
106+
// Store the Hilbert index in the result
83107
builder.put_slice(&slice);
84108
builder.commit_row();
85109
}
86110

111+
// Return the appropriate result type based on input
87112
if input_all_scalars {
88113
Value::Scalar(BinaryType::upcast_scalar(BinaryType::build_scalar(builder)))
89114
} else {
@@ -127,6 +152,21 @@ pub fn register(registry: &mut FunctionRegistry) {
127152
);
128153
}
129154

155+
/// Calculates the partition ID for a value based on range boundaries.
156+
///
157+
/// # Arguments
158+
/// * `val` - The value to find the partition for
159+
/// * `arr` - The array of boundary values that define the partitions
160+
///
161+
/// # Returns
162+
/// * The partition ID as a u64 (0 to arr.len())
163+
///
164+
/// # Example
165+
/// For boundaries [10, 20, 30]:
166+
/// - Values < 10 get partition ID 0
167+
/// - Values >= 10 and < 20 get partition ID 1
168+
/// - Values >= 20 and < 30 get partition ID 2
169+
/// - Values >= 30 get partition ID 3
130170
fn calc_range_partition_id(val: ScalarRef, arr: &Column) -> u64 {
131171
let mut low = 0;
132172
let mut high = arr.len();

src/query/service/src/interpreters/interpreter_table_recluster.rs

Lines changed: 49 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -286,6 +286,13 @@ impl ReclusterTableInterpreter {
286286
Ok(false)
287287
}
288288

289+
/// Builds physical plan for Hilbert clustering.
290+
/// # Arguments
291+
/// * `tbl` - Reference to the table being reclustered
292+
/// * `push_downs` - Optional filter conditions to push down to storage
293+
/// * `hilbert_info` - Cached Hilbert mapping information (built if None)
294+
/// # Returns
295+
/// * `Result<Option<PhysicalPlan>>` - The physical plan if reclustering is needed, None otherwise
289296
async fn build_hilbert_plan(
290297
&self,
291298
tbl: &Arc<dyn Table>,
@@ -299,6 +306,7 @@ impl ReclusterTableInterpreter {
299306
.do_hilbert_clustering(tbl.clone(), self.ctx.clone(), push_downs.clone())
300307
.await?
301308
else {
309+
// No reclustering needed (e.g., table already optimally clustered)
302310
return Ok(None);
303311
};
304312

@@ -311,38 +319,49 @@ impl ReclusterTableInterpreter {
311319
let total_rows = recluster_info.removed_statistics.row_count as usize;
312320
let total_compressed = recluster_info.removed_statistics.compressed_byte_size as usize;
313321

322+
// Determine rows per block based on data size and compression ratio
314323
let rows_per_block =
315324
block_thresholds.calc_rows_per_block(total_bytes, total_rows, total_compressed);
325+
326+
// Calculate initial partition count based on data volume and block size
316327
let mut total_partitions = std::cmp::max(total_rows / rows_per_block, 1);
328+
329+
// Adjust number of partitions according to the block size thresholds
317330
if total_partitions < block_thresholds.block_per_segment
318331
&& block_thresholds.check_perfect_segment(
319-
block_thresholds.block_per_segment,
332+
block_thresholds.block_per_segment, // this effectively by-pass the total_blocks criteria
320333
total_rows,
321334
total_bytes,
322335
total_compressed,
323336
)
324337
{
325338
total_partitions = block_thresholds.block_per_segment;
326339
}
340+
327341
warn!(
328342
"Do hilbert recluster, total_bytes: {}, total_rows: {}, total_partitions: {}",
329343
total_bytes, total_rows, total_partitions
330344
);
331345

346+
// Create a subquery executor for running Hilbert mapping calculations
332347
let subquery_executor = Arc::new(ServiceQueryExecutor::new(QueryContext::create_from(
333348
self.ctx.as_ref(),
334349
)));
350+
335351
let partitions = settings.get_hilbert_num_range_ids()? as usize;
336352

353+
// Ensure Hilbert mapping information is built (if not already)
337354
self.build_hilbert_info(tbl, hilbert_info).await?;
338355
let HilbertBuildInfo {
339356
keys_bound,
340357
index_bound,
341358
query,
342359
} = hilbert_info.as_ref().unwrap();
343360

361+
// Variables will store the calculated bounds for Hilbert mapping
344362
let mut variables = VecDeque::new();
345363

364+
// Execute the `kyes_bound` plan to calculate bounds for each clustering key
346365
let keys_bounds = self
347366
.execute_hilbert_plan(
348367
&subquery_executor,
@@ -352,11 +371,15 @@ impl ReclusterTableInterpreter {
352371
tbl,
353372
)
354373
.await?;
374+
375+
// Store each clustering key's bounds in the variables collection
355376
for entry in keys_bounds.columns().iter() {
356377
let v = entry.value.index(0).unwrap().to_owned();
357378
variables.push_back(v);
358379
}
359380

381+
// Execute the `index_bound` plan to calculate the Hilbert index bounds
382+
// i.e. `range_bound(..)(hilbert_range_index(..))`
360383
let index_bounds = self
361384
.execute_hilbert_plan(
362385
&subquery_executor,
@@ -366,28 +389,38 @@ impl ReclusterTableInterpreter {
366389
tbl,
367390
)
368391
.await?;
392+
393+
// Add the Hilbert index bound to the front of variables
369394
let val = index_bounds.value_at(0, 0).unwrap().to_owned();
370395
variables.push_front(val);
371396

372-
// reset the scan progress.
397+
// Reset the scan progress to its original value
373398
self.ctx.get_scan_progress().set(&scan_progress_value);
399+
374400
let Plan::Query {
375401
s_expr,
376402
metadata,
377403
bind_context,
378404
..
379405
} = query
380406
else {
381-
unreachable!()
407+
unreachable!("Expected a Query plan, but got {:?}", query.kind());
382408
};
409+
410+
// Replace placeholders in the expression
411+
// `range_partition_id(hilbert_range_index(cluster_key, [$key_range_bound], ..), [$hilbert_index_range_bound])`
412+
// with calculated constants.
383413
let mut s_expr = replace_with_constant(s_expr, &variables, total_partitions as u16);
414+
384415
if tbl.change_tracking_enabled() {
385416
s_expr = set_update_stream_columns(&s_expr)?;
386417
}
418+
387419
metadata.write().replace_all_tables(tbl.clone());
388420
let mut builder = PhysicalPlanBuilder::new(metadata.clone(), self.ctx.clone(), false);
389421
let mut plan = Box::new(builder.build(&s_expr, bind_context.column_set()).await?);
390422

423+
// Check if the plan already has an exchange operator
391424
let mut is_exchange = false;
392425
if let PhysicalPlan::Exchange(Exchange {
393426
input,
@@ -399,16 +432,24 @@ impl ReclusterTableInterpreter {
399432
plan = input.clone();
400433
}
401434

435+
// Determine if we need distributed execution
402436
let cluster = self.ctx.get_cluster();
403437
let is_distributed = is_exchange || !cluster.is_empty();
438+
439+
// For distributed execution, add an exchange operator to distribute work
404440
if is_distributed {
441+
// Create an expression for the partition column,
442+
// i.e.`range_partition_id(hilbert_range_index({hilbert_keys_str}), [...]) AS _predicate`
405443
let expr = scalar_expr_to_remote_expr(
406444
&ScalarExpr::BoundColumnRef(BoundColumnRef {
407445
span: None,
408446
column: bind_context.columns.last().unwrap().clone(),
409447
}),
410448
plan.output_schema()?.as_ref(),
411449
)?;
450+
451+
// Add exchange operator for data distribution,
452+
// shuffling data based on the hash of range partition IDs derived from the Hilbert index.
412453
plan = Box::new(PhysicalPlan::Exchange(Exchange {
413454
plan_id: 0,
414455
input: plan,
@@ -422,13 +463,18 @@ impl ReclusterTableInterpreter {
422463
let table_meta_timestamps = self
423464
.ctx
424465
.get_table_meta_timestamps(tbl.as_ref(), Some(snapshot.clone()))?;
466+
467+
// Create the Hilbert partition physical plan,
468+
// collecting data into partitions and persist them
425469
let plan = PhysicalPlan::HilbertPartition(Box::new(HilbertPartition {
426470
plan_id: 0,
427471
input: plan,
428472
table_info: table_info.clone(),
429473
num_partitions: total_partitions,
430474
table_meta_timestamps,
431475
}));
476+
477+
// Finally, commit the newly clustered table
432478
Ok(Some(Self::add_commit_sink(
433479
plan,
434480
is_distributed,

0 commit comments

Comments
 (0)