Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Improvement](runtime-filter) refactor of RuntimeFilterSlots #47524

Closed
wants to merge 8 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions be/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -510,6 +510,7 @@ set(DORIS_LINK_LIBS
IO
Olap
Runtime
RuntimeFilter
Service
Udf
Util
Expand Down Expand Up @@ -778,6 +779,7 @@ add_subdirectory(${SRC_DIR}/http)
add_subdirectory(${SRC_DIR}/io)
add_subdirectory(${SRC_DIR}/olap)
add_subdirectory(${SRC_DIR}/runtime)
add_subdirectory(${SRC_DIR}/runtime_filter)
add_subdirectory(${SRC_DIR}/service) # this include doris_be
add_subdirectory(${SRC_DIR}/udf)
add_subdirectory(${SRC_DIR}/cloud)
Expand Down
26 changes: 10 additions & 16 deletions be/src/exec/olap_common.h
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@

#include "common/status.h"
#include "exec/olap_utils.h"
#include "olap/filter_olap_param.h"
#include "olap/olap_common.h"
#include "olap/olap_tuple.h"
#include "runtime/define_primitive_type.h"
Expand Down Expand Up @@ -186,7 +187,7 @@ class ColumnValueRange {

size_t get_fixed_value_size() const { return _fixed_values.size(); }

void to_olap_filter(std::vector<TCondition>& filters) {
void to_olap_filter(std::vector<FilterOlapParam<TCondition>>& filters) {
if (is_fixed_value_range()) {
// 1. convert to in filter condition
to_in_condition(filters, true);
Expand All @@ -202,34 +203,32 @@ class ColumnValueRange {
}

if (null_pred.condition_values.size() != 0) {
filters.push_back(std::move(null_pred));
filters.emplace_back(_column_name, null_pred, _runtime_filter_id);
return;
}

TCondition low;
if (TYPE_MIN != _low_value || FILTER_LARGER_OR_EQUAL != _low_op) {
low.__set_column_name(_column_name);
low.__set_condition_op((_low_op == FILTER_LARGER_OR_EQUAL ? ">=" : ">>"));
low.__set_marked_by_runtime_filter(_marked_runtime_filter_predicate);
low.condition_values.push_back(
cast_to_string<primitive_type, CppType>(_low_value, _scale));
}

if (low.condition_values.size() != 0) {
filters.push_back(std::move(low));
filters.emplace_back(_column_name, low, _runtime_filter_id);
}

TCondition high;
if (TYPE_MAX != _high_value || FILTER_LESS_OR_EQUAL != _high_op) {
high.__set_column_name(_column_name);
high.__set_condition_op((_high_op == FILTER_LESS_OR_EQUAL ? "<=" : "<<"));
high.__set_marked_by_runtime_filter(_marked_runtime_filter_predicate);
high.condition_values.push_back(
cast_to_string<primitive_type, CppType>(_high_value, _scale));
}

if (high.condition_values.size() != 0) {
filters.push_back(std::move(high));
filters.emplace_back(_column_name, high, _runtime_filter_id);
}
} else {
// 3. convert to is null and is not null filter condition
Expand All @@ -242,24 +241,23 @@ class ColumnValueRange {
}

if (null_pred.condition_values.size() != 0) {
filters.push_back(std::move(null_pred));
filters.emplace_back(_column_name, null_pred, _runtime_filter_id);
}
}
}

void to_in_condition(std::vector<TCondition>& filters, bool is_in = true) {
void to_in_condition(std::vector<FilterOlapParam<TCondition>>& filters, bool is_in = true) {
TCondition condition;
condition.__set_column_name(_column_name);
condition.__set_condition_op(is_in ? "*=" : "!*=");
condition.__set_marked_by_runtime_filter(_marked_runtime_filter_predicate);

for (const auto& value : _fixed_values) {
condition.condition_values.push_back(
cast_to_string<primitive_type, CppType>(value, _scale));
}

if (condition.condition_values.size() != 0) {
filters.push_back(std::move(condition));
filters.emplace_back(_column_name, condition, _runtime_filter_id);
}
}

Expand Down Expand Up @@ -296,11 +294,7 @@ class ColumnValueRange {
_contain_null = _is_nullable_col && contain_null;
}

void mark_runtime_filter_predicate(bool is_runtime_filter_predicate) {
_marked_runtime_filter_predicate = is_runtime_filter_predicate;
}

bool get_marked_by_runtime_filter() const { return _marked_runtime_filter_predicate; }
void set_runtime_filter_id(int runtime_filter_id) { _runtime_filter_id = runtime_filter_id; }

int precision() const { return _precision; }

Expand Down Expand Up @@ -362,7 +356,7 @@ class ColumnValueRange {
primitive_type == PrimitiveType::TYPE_DATETIME ||
primitive_type == PrimitiveType::TYPE_DATETIMEV2;

bool _marked_runtime_filter_predicate = false;
int _runtime_filter_id = -1;
};

class OlapScanKeys {
Expand Down
4 changes: 2 additions & 2 deletions be/src/exprs/bitmapfilter_predicate.h
Original file line number Diff line number Diff line change
Expand Up @@ -19,16 +19,16 @@

#include <algorithm>

#include "exprs/runtime_filter.h"
#include "gutil/integral_types.h"
#include "runtime/define_primitive_type.h"
#include "runtime/primitive_type.h"
#include "runtime_filter/runtime_filter_definitions.h"
#include "util/bitmap_value.h"

namespace doris {

// only used in Runtime Filter
class BitmapFilterFuncBase : public RuntimeFilterFuncBase {
class BitmapFilterFuncBase {
public:
virtual void insert_many(const std::vector<const BitmapValue*>& bitmaps) = 0;
virtual uint16_t find_fixed_len_olap_engine(const char* data, const uint8* nullmap,
Expand Down
Loading
Loading