integrate ByteSlice into Bliss, resolve conflicts

CMakeLists.txt

@@ -46,6 +46,9 @@ add_subdirectory(tests)
 # HEADER bliss
 #   Bliss lib files
 # =============================================================================
+
+option(COMPILE_COLUMNSKETCHES "compile_column_sketches" OFF)
+
 add_library(bliss OBJECT
     ${CMAKE_SOURCE_DIR}/src/bliss/util/timer.h
     ${CMAKE_SOURCE_DIR}/src/bliss/util/reader.h
@@ -55,6 +58,8 @@ add_library(bliss OBJECT
     ${CMAKE_SOURCE_DIR}/src/bliss/bench_lipp.h
     ${CMAKE_SOURCE_DIR}/src/bliss/bench_alex.h
     ${CMAKE_SOURCE_DIR}/src/bliss/bench_btree.h
+    ${CMAKE_SOURCE_DIR}/src/bliss/bench_imprints.h
+    ${CMAKE_SOURCE_DIR}/src/bliss/bench_columnsketches.h
     ${CMAKE_SOURCE_DIR}/src/bliss/bench_skiplist.h
     ${CMAKE_SOURCE_DIR}/src/bliss/bench_byteslice.h
     ${CMAKE_SOURCE_DIR}/src/bliss/bench_pgm.h
@@ -73,11 +78,18 @@ target_link_libraries(bliss PUBLIC
     lipp
     tlx
     byteslice
+    imprints
     skiplist
     pgm
     art
 )
 
+if(COMPILE_COLUMNSKETCHES)
+    target_link_libraries(bliss PUBLIC
+        columnsketches
+    )
+endif()
+
 target_include_directories(bliss PUBLIC
     ${CMAKE_SOURCE_DIR}/src
 )

external/CMakeLists.txt

@@ -72,6 +72,45 @@ endif()
 add_library(tlx INTERFACE)
 target_include_directories(tlx INTERFACE ${tlx_SOURCE_DIR}/)
 
+FetchContent_Declare(
+        imprints
+        GIT_REPOSITORY https://github.com/altramarine/imprints.git
+        GIT_TAG cmake-version
+)
+FetchContent_GetProperties(imprints)
+if (NOT imprints_POPULATED)
+    FetchContent_Populate(imprints)
+endif()
+add_library(imprints INTERFACE)
+target_include_directories(imprints INTERFACE ${imprints_SOURCE_DIR}/include)
+
+FetchContent_Declare(
+        columnsketches
+        GIT_REPOSITORY https://github.com/altramarine/ColumnSketches-cpp.git
+        GIT_TAG main
+)
+
+if(COMPILE_COLUMNSKETCHES)
+    FetchContent_GetProperties(columnsketches)
+    if (NOT columnsketches_POPULATED)
+        FetchContent_Populate(columnsketches)
+    endif()
+    file(GLOB_RECURSE columnsketches_SOURCES
+    "${columnsketches_SOURCE_DIR}/src/*.cpp"
+    )
+    add_library(columnsketches STATIC ${columnsketches_SOURCES})
+    target_include_directories(columnsketches PUBLIC ${columnsketches_SOURCE_DIR}/include)
+    target_link_libraries(columnsketches PUBLIC pthread)
+    target_compile_options(columnsketches PUBLIC 
+    "-pthread"
+    "-mavx"
+    "-mavx2"
+    "-msse"
+    "-msse2"
+    "-O3"
+    "-DCOMPILE_COLUMNSKETCHES=1"
+    )
+endif()
 
 FetchContent_Declare(
         byteslice

src/bliss/bench_byteslice.h

@@ -13,7 +13,7 @@ template <typename KEY_TYPE, typename VALUE_TYPE>
 class BlissByteSliceIndex : public BlissIndex<KEY_TYPE, VALUE_TYPE> {
    public:
     BlissByteSliceIndex(int numrows = 1000000, int numbits = 32) {
-        byteslice_ = new ByteSlice(numrows, numbits);
+        byteslice_ = new byteslice::ByteSlice(numrows, numbits);
     };
 
     ~BlissByteSliceIndex()  {
@@ -29,8 +29,8 @@ class BlissByteSliceIndex : public BlissIndex<KEY_TYPE, VALUE_TYPE> {
     }
 
 
-    bool get(VALUE_TYPE low, VALUE_TYPE high) {
-        uint32_t * result = byteslice_->query(static_cast<unsigned long long>(low), static_cast<unsigned long long>(high));
+    bool get(VALUE_TYPE start, VALUE_TYPE end) {
+        uint32_t * result = byteslice_->query(static_cast<unsigned long long>(start), static_cast<unsigned long long>(end));
         bool check_ = result != nullptr;
         if(check_) delete result;
         return check_;
@@ -48,7 +48,7 @@ class BlissByteSliceIndex : public BlissIndex<KEY_TYPE, VALUE_TYPE> {
 
     void end_routine() override {}
   private:
-    ByteSlice *byteslice_;
+    byteslice::ByteSlice *byteslice_;
 };
 
 }  // namespace bliss

src/bliss/bench_columnsketches.h

@@ -0,0 +1,56 @@
+#ifndef BLISS_BENCH_COLUMNSKETCHES
+#define BLISS_BENCH_COLUMNSKETCHES
+#ifdef COMPILE_COLUMNSKETCHES
+#include <vector>
+
+#include "bliss/bliss_index.h"
+#include "column_sketches.h"
+
+namespace bliss {
+
+template <typename KEY_TYPE, typename VALUE_TYPE>
+class BlissColumnSketchesIndex : public BlissIndex<KEY_TYPE, VALUE_TYPE> {
+   public:
+    BlissColumnSketchesIndex() {
+        columnsketches_ = new ColumnSketchesInteger();
+    };
+
+    ~BlissColumnSketchesIndex()  {
+        delete columnsketches_;
+    }
+
+    void bulkload(
+        std::vector<std::pair<KEY_TYPE, VALUE_TYPE>> values) override {
+        std::vector<int32_t> vals;
+        for(auto x: values) {
+            vals.push_back(x.second);
+        }
+        columnsketches_->create(&vals[0], vals.size(), vals.size());
+    }
+
+    bool get(VALUE_TYPE start, VALUE_TYPE end) {
+        uint32_t *res = columnsketches_->query_ge_lt(start, end);
+        bool succeed = res != nullptr;
+        if(succeed) delete res;
+        return succeed;
+    }
+
+    bool get(KEY_TYPE key) override { 
+        std::runtime_error("Column Imprints does not support get(key, value).");
+        return false; 
+    }
+
+    void put(KEY_TYPE key, VALUE_TYPE value) {
+        std::runtime_error("Column Imprints does not support put(key, value).");
+        return;
+    }
+
+    void end_routine() override {}
+  private:
+
+    ColumnSketchesInteger *columnsketches_;
+};
+
+}  // namespace bliss
+#endif
+#endif  // !BLISS_BENCH_BTREE

src/bliss/bench_imprints.h

@@ -0,0 +1,65 @@
+#ifndef BLISS_BENCH_IMPRINTS
+#define BLISS_BENCH_IMPRINTS
+
+#include <vector>
+
+#include "bliss/bliss_index.h"
+#include "column_imprints.h"
+#include <string>
+
+namespace bliss {
+
+template <typename KEY_TYPE, typename VALUE_TYPE>
+class BlissImprintsIndex : public BlissIndex<KEY_TYPE, VALUE_TYPE> {
+   public:
+    BlissImprintsIndex(int blocksize = 64, int maxbins = 64, std::string type_name = std::string("unsigned long")) : blocksize_(blocksize), maxbins_(maxbins) {
+        imprints_ = new ColumnImprints::Imprints<VALUE_TYPE>(blocksize, maxbins, type_name);
+    };
+
+    ~BlissImprintsIndex()  {
+        delete imprints_;
+    }
+
+    void bulkload(
+        std::vector<std::pair<KEY_TYPE, VALUE_TYPE>> values) override {
+        std::vector<VALUE_TYPE> vals;
+        for(auto x: values) {
+            vals.push_back(x.second);
+        }
+        imprints_->bulkload(vals);
+    }
+
+
+    bool get(VALUE_TYPE start, VALUE_TYPE end) {
+        unsigned int * res = imprints_->range_scan(start, end);
+        if(res != nullptr) {
+            delete res;
+            return true;
+        } else
+            return false;
+    }
+
+    uint32_t * get_bitmask(VALUE_TYPE start, VALUE_TYPE end) {
+        return imprints_->range_scan(start, end);
+    }
+
+    bool get(KEY_TYPE key) override { 
+        std::runtime_error("Column Imprints does not support get(key, value).");
+        return false; 
+    }
+
+    void put(KEY_TYPE key, VALUE_TYPE value) {
+        std::runtime_error("Column Imprints does not support put(key, value).");
+        return;
+    }
+
+    void end_routine() override {}
+  private:
+    ColumnImprints::Imprints<VALUE_TYPE> *imprints_;
+    int blocksize_, maxbins_;
+    std::string type_name;
+};
+
+}  // namespace bliss
+
+#endif  // !BLISS_BENCH_BTREE

src/bliss/util/execute.h

@@ -46,5 +46,4 @@ void execute_non_empty_reads(bliss::BlissIndex<key_type, value_type> &tree,
 }  // namespace executor
 }  // namespace utils
 }  // namespace bliss
-
 #endif

src/bliss_bench.cpp

@@ -12,6 +12,7 @@
 #include "bliss/bench_leveldb.h"
 #include "bliss/bench_lipp.h"
 #include "bliss/bench_byteslice.h"
+#include "bliss/bench_columnsketches.h"
 #include "bliss/bench_pgm.h"
 #include "bliss/bench_skiplist.h"
 #include "bliss/bliss_index.h"
@@ -20,6 +21,7 @@
 #include "bliss/util/execute.h"
 #include "bliss/util/reader.h"
 #include "bliss/util/timer.h"
+#include "bliss/bench_imprints.h"
 #include "include/pgm/pgm_index_dynamic.hpp"
 #include "skip_list.h"
 
@@ -177,6 +179,15 @@ int main(int argc, char *argv[]) {
         index.reset(new bliss::BlissBTreeIndex<key_type, value_type>());
     } else if (config.index == "byteslice") {
         index.reset(new bliss::BlissByteSliceIndex<key_type, value_type>());
+    } else if (config.index == "imprints") {
+        index.reset(new bliss::BlissImprintsIndex<key_type, value_type>(/* block_size */64, /* max_bins */64));
+    } else if (config.index == "columnskteches") {
+#ifdef COMPILE_COLUMNSKETCHES
+        index.reset(new bliss::BlissColumnSketchesIndex<key_type, value_type>());
+#else
+        std::runtime_error("Column Sketches is not Imported");
+        exit(0);
+#endif
     } else if (config.index == "skiplist") {
         index.reset(new bliss::BlissSkipListIndex<key_type, value_type>());
     } else if (config.index == "art") {

tests/CMakeLists.txt

@@ -17,6 +17,7 @@ target_include_directories(bliss_test_infra PUBLIC
 add_subdirectory(test_alex)
 add_subdirectory(test_lipp)
 add_subdirectory(test_btree)
+add_subdirectory(test_imprints)
 add_subdirectory(test_skiplist)
 add_subdirectory(test_art)
 add_subdirectory(test_leveldb)

tests/bliss_index_tests.h

@@ -16,6 +16,8 @@
 #include "bliss/bench_btree.h"
 #include "bliss/bench_leveldb.h"
 #include "bliss/bench_lipp.h"
+#include "bliss/bench_imprints.h"
+#include "bliss/bench_skiplist.h"
 #include "bliss/bench_pgm.h"
 #include "bliss/bench_skiplist.h"
 #include "bliss/bliss_index.h"

tests/test_imprints/CMakeLists.txt

@@ -0,0 +1,9 @@
+get_filename_component(EXEC ${CMAKE_CURRENT_SOURCE_DIR} NAME)
+file(GLOB_RECURSE CPP_TESTS "*_tests.cpp")
+add_executable(${EXEC} ${CPP_TESTS})
+target_link_libraries(${EXEC} PRIVATE 
+bliss
+bliss_test_infra 
+GTest::gtest_main)
+include(GoogleTest)
+gtest_discover_tests(${EXEC})

tests/test_imprints/imprint_tests.cpp

@@ -0,0 +1,28 @@
+#include "bliss_index_tests.h"
+
+class ImprintsTest : public BlissIndexTest {};
+
+TEST_F(ImprintsTest, TestImprint_Random) {
+    index.reset(new bliss::BlissImprintsIndex<size_t, key_type>(64, 64, std::string("unsigned long")));
+    std::vector<key_type> data;
+    GenerateData(data, num_keys, false);
+    std::vector < std::pair<size_t, key_type> > bulkload_data;
+    for (size_t i = 0; i < data.size(); i ++) {
+        bulkload_data.push_back(std::make_pair(i, data[i]));
+    }
+    index->bulkload(bulkload_data);
+    auto minimum = 0;
+    for (size_t key = 0; key < num_keys; key += std::max(1,num_keys / 20)) {
+        uint32_t *result = nullptr;
+        bliss::BlissImprintsIndex<size_t, key_type> * index_copy = (bliss::BlissImprintsIndex<size_t, key_type> *)index.get();
+        result = (index_copy)->get_bitmask(minimum, data[key]);
+        for(size_t i = 0; i < num_keys; i ++) {
+            if((result[i / 32] >> (i & 31)) & 1u) {
+                EXPECT_TRUE(data[i] <= data[key] && data[i] > minimum);
+            } else {
+                EXPECT_FALSE(data[i] <= data[key] && data[i] > minimum);
+            }
+        }
+        free(result);
+    }
+}