210 Integration of descriptor sets into neo4j based handler

CMakeLists.txt

@@ -71,6 +71,7 @@ else()
     src/ImageCommand.cc
     src/Neo4jBaseCommands.cc
     src/Neo4JHandlerCommands.cc
+    src/Neo4JDescriptorCommands.cc
     src/OpsIOCoordinator.cc
     src/PMGDIterators.cc
     src/PMGDQuery.cc

src/BackendNeo4j.cc

@@ -105,7 +105,22 @@ neo4j_result_stream_t *BackendNeo4j::run_in_tx(char *cypher_string,
 }
 int BackendNeo4j::commit_tx(neo4j_transaction_t *tx) {
   int rc = 0;
+  char buf[128];
+  const char *err_msg;
+  const char *fail_code;
   rc = neo4j_commit(tx);
+
+
+  if(rc != 0){
+      err_msg = neo4j_tx_failure_message(tx);
+      rc =  neo4j_tx_failure(tx);
+      fail_code = neo4j_tx_failure_code(tx);
+
+      printf("TX Failure: %d\n", rc);
+      printf("Fail Code: %s\n", fail_code);
+      printf("Neo4j Transaction Failure: %s\n", err_msg);
+  }
+
   neo4j_free_tx(tx);
 
   return rc;

src/BackendNeo4j.h

@@ -8,6 +8,8 @@
 #include <neo4j-client.h>
 #include <tbb/concurrent_queue.h>
 #include <transaction.h>
+// #include <connection.h>
+// #include <result_stream.h>
 
 #include <cstdint>
 #include <string>

src/DescriptorsCommand.cc

@@ -56,7 +56,6 @@ DescriptorsCommand::DescriptorsCommand(const std::string &cmd_name)
 std::string DescriptorsCommand::get_set_path(PMGDQuery &query_tx,
                                              const std::string &set_name,
                                              int &dim) {
-
   // Check cache for descriptor set, if its found set dimensions and return
   // path,
   // otherwise we go forward and query PMGD to locate the descriptor set
@@ -67,7 +66,6 @@ std::string DescriptorsCommand::get_set_path(PMGDQuery &query_tx,
     dim = _desc_set_dims[set_name];
     return mapped_path;
   }
-
   // Will issue a read-only transaction to check
   // if the Set exists
   PMGDQuery query(query_tx.get_pmgd_qh());
@@ -87,7 +85,6 @@ std::string DescriptorsCommand::get_set_path(PMGDQuery &query_tx,
   results["list"] = list_arr;
 
   bool unique = true;
-
   // Query set node
   query.add_group();
   query.QueryNode(-1, VDMS_DESC_SET_TAG, link, constraints, results, unique,
@@ -110,8 +107,10 @@ std::string DescriptorsCommand::get_set_path(PMGDQuery &query_tx,
     dim = ent[VDMS_DESC_SET_DIM_PROP].asInt();
     _desc_set_dims[set_name] = dim;
     _desc_set_locator[set_name] = set_path;
+
     return set_path;
   }
+
   return "";
 }
 
@@ -305,7 +304,7 @@ Json::Value AddDescriptorSet::construct_responses(
   else if (eng_str == "Flinng")
     _eng = VCL::Flinng;
   else if (eng_str == "FaissHNSWFlat")
-    _eng = VCL::FaissHNSWFlat;
+    _eng = VCL::FaissHNSWFlat; //WARNING
   else
     throw ExceptionCommand(DescriptorSetError, "Engine not supported");
 
@@ -326,10 +325,6 @@ Json::Value AddDescriptorSet::construct_responses(
     }
 
     desc_set.store();
-    if (output_vcl_timing) {
-      desc_set.timers.print_map_runtimes();
-    }
-    desc_set.timers.clear_all_timers();
 
     delete (param);
   } catch (VCL::Exception e) {
@@ -373,10 +368,6 @@ long AddDescriptor::insert_descriptor(const std::string &blob,
       id_first = desc_set->add((float *)blob.data(), nr_desc);
     }
 
-    if (output_vcl_timing) {
-      desc_set->timers.print_map_runtimes();
-    }
-    desc_set->timers.clear_all_timers();
   } catch (VCL::Exception e) {
     print_exception(e);
     error["info"] = "VCL Descriptors Exception";
@@ -1007,10 +998,6 @@ void FindDescriptor::populate_blobs(const std::string &set_path,
       desc_blob->resize(sizeof(float) * dim);
 
       set->get_descriptors(&id, 1, (float *)(*desc_blob).data());
-      if (output_vcl_timing) {
-        set->timers.print_map_runtimes();
-      }
-      set->timers.clear_all_timers();
     }
   }
 }

src/Neo4JCommands.h

@@ -33,12 +33,16 @@
 
 #include <jsoncpp/json/value.h>
 
+#include "DescriptorsManager.h"
 #include "queryMessage.pb.h"
+#include "tbb/concurrent_unordered_map.h"
 #include "vcl/Image.h"
 #include "vcl/VCL.h"
 
 namespace VDMS {
 
+typedef std::pair<std::vector<long>, std::vector<float>> IDDistancePair;
+
 class Neo4jCommand {
  protected:
   const std::string _cmd_name;
@@ -101,4 +105,120 @@ class Neo4jNeoFind : public Neo4jCommand {
                                   const std::string &blob);
 };
 
+class NeoDescriptorsCommand : public Neo4jCommand {
+ protected:
+  DescriptorsManager *_dm;
+  VCL::DescriptorSetEngine _eng;
+  bool output_vcl_timing;
+
+  // IDDistancePair is a pointer so that we can free its content
+  // without having to use erase methods, which are not lock free
+  // for this data structure in tbb
+  tbb::concurrent_unordered_map<long, IDDistancePair *> _cache_map;
+
+  static tbb::concurrent_unordered_map<std::string, std::string>
+      _desc_set_locator;
+  static tbb::concurrent_unordered_map<std::string, int> _desc_set_dims;
+
+  // Will return the path to the set and the dimensions
+  std::string get_set_path(const std::string &set, int &dim);
+
+  void add_vec_id_idx(const std::string &setname);
+
+  bool check_blob_size(const std::string &blob, const int dimensions,
+                       const long n_desc);
+
+ public:
+  NeoDescriptorsCommand(const std::string &cmd_name);
+
+  virtual bool need_blob(const Json::Value &cmd) { return false; }
+
+  int data_processing(std::string &tx, const Json::Value &root,
+                      const std::string &blob, int grp_id,
+                      Json::Value &error) = 0;
+
+  virtual Json::Value construct_responses(Json::Value &json_responses,
+                                          const Json::Value &json,
+                                          protobufs::queryMessage &response,
+                                          const std::string &blob) = 0;
+};
+
+class Neo4jNeoAddDescSet : public NeoDescriptorsCommand {
+  std::string _storage_sets;
+  uint64_t _flinng_num_rows;
+  uint64_t _flinng_cells_per_row;
+  uint64_t _flinng_num_hash_tables;
+  uint64_t _flinng_hashes_per_table;
+  uint64_t _flinng_sub_hash_bits;  // sub_hash_bits * hashes_per_table must be
+  // less than 32, otherwise segfault will happen
+  uint64_t _flinng_cut_off;
+
+ public:
+  Neo4jNeoAddDescSet();
+  bool need_blob(const Json::Value &cmd) { return false; };
+  int data_processing(std::string &tx, const Json::Value &root,
+                      const std::string &blob, int grp_id, Json::Value &error);
+  Json::Value construct_responses(Json::Value &json_responses,
+                                  const Json::Value &json,
+                                  protobufs::queryMessage &response,
+                                  const std::string &blob);
+};
+
+class Neo4jNeoFindDescSet : public NeoDescriptorsCommand {
+  std::string _storage_sets;
+
+ public:
+  Neo4jNeoFindDescSet();
+  bool need_blob(const Json::Value &cmd) { return false; };
+  int data_processing(std::string &tx, const Json::Value &root,
+                      const std::string &blob, int grp_id, Json::Value &error);
+  Json::Value construct_responses(Json::Value &json_responses,
+                                  const Json::Value &json,
+                                  protobufs::queryMessage &response,
+                                  const std::string &blob);
+};
+
+class Neo4jNeoAddDesc : public NeoDescriptorsCommand {
+  int add_single_descriptor(std::string &tx, const Json::Value &root,
+                            const std::string &blob, int grp_id,
+                            Json::Value &error);
+
+  int add_descriptor_batch(std::string &tx, const Json::Value &root,
+                           const std::string &blob, int grp_id,
+                           Json::Value &error);
+
+  long insert_descriptor(const std::string &blob, const std::string &set_path,
+                         int nr_desc, const std::string &label,
+                         Json::Value &error);
+
+ public:
+  Neo4jNeoAddDesc();
+  bool need_blob(const Json::Value &cmd) { return true; };
+  int data_processing(std::string &tx, const Json::Value &root,
+                      const std::string &blob, int grp_id, Json::Value &error);
+  Json::Value construct_responses(Json::Value &json_responses,
+                                  const Json::Value &json,
+                                  protobufs::queryMessage &response,
+                                  const std::string &blob);
+};
+
+class Neo4jNeoFindDesc : public NeoDescriptorsCommand {
+ private:
+  void convert_properties(Json::Value &entities, Json::Value &list,
+                          std::string set_name);
+  void populate_blobs(const std::string &set_path, std::string set_name,
+                      const Json::Value &results, Json::Value &entities,
+                      protobufs::queryMessage &query_res);
+
+ public:
+  Neo4jNeoFindDesc();
+  bool need_blob(const Json::Value &cmd);
+  int data_processing(std::string &tx, const Json::Value &root,
+                      const std::string &blob, int grp_id, Json::Value &error);
+  Json::Value construct_responses(Json::Value &json_responses,
+                                  const Json::Value &json,
+                                  protobufs::queryMessage &response,
+                                  const std::string &blob);
+};
+
 }  // namespace VDMS

src/Neo4jBaseCommands.cc

@@ -63,15 +63,12 @@ Json::Value Neo4jCommand::check_responses(Json::Value &responses) {
   bool flag_error = false;
   Json::Value ret;
   if (responses.size() == 0) {
-    printf("NO responses found! Setting Errror!\n");
     ret["status"] = Neo4jCommand::Error;
-    ret["info"] = "No responses!";
-    printf("Error Response!\n");
+    ret["info"] = "No Neo4J responses!";
     return ret;
   }
-  printf("No Error found!\n");
 
-  ret = responses[0];
+  ret = responses["metadata_res"];
 
   if (!flag_error) {
     ret["status"] = Neo4jCommand::Success;

src/QueryHandlerNeo4j.cc

@@ -48,12 +48,24 @@ using namespace VDMS;
 
 std::unordered_map<std::string, Neo4jCommand *> QueryHandlerNeo4j::_rs_cmds;
 BackendNeo4j *QueryHandlerNeo4j::neoconn_pool;
-// VCL::RemoteConnection *global_s3_connection;
+
+// Static globals for use in looking up descriptor set locations, defined in
+// DescriptorCommand.h
+tbb::concurrent_unordered_map<std::string, std::string>
+        NeoDescriptorsCommand::_desc_set_locator;
+tbb::concurrent_unordered_map<std::string, int> NeoDescriptorsCommand::_desc_set_dims;
+
 
 void QueryHandlerNeo4j::init() {
+  DescriptorsManager::init();
 
   _rs_cmds["NeoAdd"] = new Neo4jNeoAdd();
   _rs_cmds["NeoFind"] = new Neo4jNeoFind();
+  _rs_cmds["NeoAddDescriptorSet"] = new Neo4jNeoAddDescSet();
+  _rs_cmds["NeoFindDescriptorSet"] = new Neo4jNeoFindDescSet();
+  _rs_cmds["NeoAddDescriptor"] = new Neo4jNeoAddDesc();
+  _rs_cmds["NeoFindDescriptor"] = new Neo4jNeoFindDesc();
+
   // seed random time
   srand((unsigned)time(NULL));
 
@@ -62,7 +74,7 @@ void QueryHandlerNeo4j::init() {
   char *pass = getenv("NEO4J_PASS");
 
   uint_fast32_t flags = NEO4J_INSECURE;
-  int nr_conns = 16;
+  int nr_conns = 32; //TODO update to be configurable
 
   neoconn_pool = new BackendNeo4j(nr_conns, (char *)tgtdb, user, pass, flags);
 
@@ -94,7 +106,7 @@ bool QueryHandlerNeo4j::syntax_checker(const Json::Value &root,
                                        Json::Value &error) {
   valijson::ValidationResults results;
   valijson::adapters::JsonCppAdapter user_query(root);
-  std::cerr << root.toStyledString() << std::endl; // TEMPORARY
+  //std::cerr << root.toStyledString() << std::endl; // TEMPORARY
   if (!_validator.validate(*_schema, user_query, &results)) {
     std::cerr << "API validation failed for:" << std::endl;
     std::cerr << root.toStyledString() << std::endl;
@@ -170,7 +182,7 @@ void QueryHandlerNeo4j::process_query(protobufs::queryMessage &proto_query,
   int rc;
 
   Json::FastWriter fastWriter;
-  Json::Value hello_res;
+  Json::Value final_resp;
   Json::Value json_responses;
   Json::Value cmd_result;
 
@@ -186,11 +198,15 @@ void QueryHandlerNeo4j::process_query(protobufs::queryMessage &proto_query,
     Json::Value neo4j_resp;
     std::string cypher;
 
-    const Json::Value &query = root[j];
+    Json::Value &query = root[j];
     std::string cmd = query.getMemberNames()[0];
 
-    Neo4jCommand *rscmd = _rs_cmds[cmd];
 
+    if (_rs_cmds.count(cmd) == 0) {
+        std::cout<<"Command: " << cmd << "Does not exist!" << std::endl;
+    }
+
+    Neo4jCommand *rscmd = _rs_cmds[cmd];
     cypher = query[cmd]["cypher"].asString();
 
     const std::string &blob =
@@ -199,28 +215,40 @@ void QueryHandlerNeo4j::process_query(protobufs::queryMessage &proto_query,
     rc = rscmd->data_processing(cypher, query, blob, 0, cmd_result);
 
     if (rc != 0) {
-      printf("Data Processing failed, aborting transaction...\n");
       error = true;
+      proto_res.set_json(fastWriter.write(cmd_result));
       break;
     }
 
     res_stream = neoconn_pool->run_in_tx((char *)cypher.c_str(), tx);
     neo4j_resp = neoconn_pool->results_to_json(res_stream);
+    std::cout << "Neo4J Resp:" << neo4j_resp << std::endl;
+    query["cp_result"] = cmd_result;
 
-    rscmd->construct_responses(neo4j_resp, query, proto_res, blob);
+    Json::Value resp_retval = rscmd->construct_responses(neo4j_resp, query, proto_res, blob);
 
-    if (neo4j_resp.isMember("metadata_res")) {
-      hello_res["metadata_res"] = neo4j_resp["metadata_res"];
+    //THIS IS VERY CLUNKY and confusing, NEEDS TO BE REFACTORED
+    if (neo4j_resp.isMember("metadata_res") && (cmd == "NeoAdd" || cmd == "NeoFind")) {
+        resp_retval["metadata_res"] = neo4j_resp["metadata_res"];
+    } else {
+        std::cout << "Non NeoAdd/Find" << std::endl;
     }
 
-    json_responses.append(hello_res);
+    json_responses.append(resp_retval);
+
 
-    proto_res.set_json(fastWriter.write(json_responses));
   }
+    proto_res.set_json(fastWriter.write(json_responses));
   // commit neo4j transaction, needs to be updated in future to account for
   // errors on response construction
   if (error == false) {
-    neoconn_pool->commit_tx(tx);
+    rc = neoconn_pool->commit_tx(tx);
+
+    if(rc != 0){
+        printf("Warning! Transaction Error: %d\n", rc);
+        exit(1);
+    }
+
   }
 
   neoconn_pool->put_conn(conn);

src/QueryHandlerNeo4j.h

@@ -39,14 +39,15 @@ namespace VDMS {
 
 class QueryHandlerNeo4j : public QueryHandlerBase {
  protected:
-  static BackendNeo4j *neoconn_pool;
+  // static BackendNeo4j *neoconn_pool;
   static std::unordered_map<std::string, Neo4jCommand *> _rs_cmds;
   friend class QueryHandlerTester;
   bool syntax_checker(const Json::Value &root, Json::Value &error);
   int parse_commands(const protobufs::queryMessage &proto_query,
                      Json::Value &root);
 
  public:
+  static BackendNeo4j *neoconn_pool;
   static void init();
   QueryHandlerNeo4j();
   void process_query(protobufs::queryMessage &proto_query,

src/vcl/DescriptorSet.cc

@@ -98,16 +98,13 @@ DescriptorSet::DescriptorSet(const std::string &set_path, unsigned dim,
 DescriptorSet::~DescriptorSet() { delete _set; }
 
 void DescriptorSet::write_set_info() {
-  timers.add_timestamp("write_set_info");
   std::string path = _set->get_path() + "/" + INFO_FILE_NAME;
   std::ofstream info_file(path);
   info_file << _eng << std::endl;
   info_file.close();
-  timers.add_timestamp("write_set_info");
 }
 
 void DescriptorSet::read_set_info(const std::string &set_path) {
-  timers.add_timestamp("read_set_info");
   std::string path = set_path + "/" + INFO_FILE_NAME;
   std::ifstream info_file(path);
 
@@ -123,7 +120,6 @@ void DescriptorSet::read_set_info(const std::string &set_path) {
   sstr >> num;
   _eng = (DescriptorSetEngine)num;
   info_file.close();
-  timers.add_timestamp("read_set_info");
 }
 
 /*  *********************** */
@@ -139,78 +135,57 @@ long DescriptorSet::get_n_descriptors() { return _set->get_n_total(); }
 void DescriptorSet::search(DescDataArray queries, unsigned n_queries,
                            unsigned k, long *descriptors_ids,
                            float *distances) {
-  timers.add_timestamp("desc_set_search");
   _set->search(queries, n_queries, k, descriptors_ids, distances);
-  timers.add_timestamp("desc_set_search");
 }
 
 void DescriptorSet::search(DescDataArray queries, unsigned n_queries,
                            unsigned k, long *descriptors_ids) {
-  timers.add_timestamp("desc_set_search");
   _set->search(queries, n_queries, k, descriptors_ids);
-  timers.add_timestamp("desc_set_search");
 }
 
 void DescriptorSet::radius_search(DescData queries, float radius,
                                   long *descriptors_ids, float *distances) {
-  timers.add_timestamp("desc_set_radius_search");
   _set->radius_search(queries, radius, descriptors_ids, distances);
-  timers.add_timestamp("desc_set_radius_search");
 }
 
 long DescriptorSet::add(DescDataArray descriptors, unsigned n, long *labels) {
   long rc;
-  timers.add_timestamp("desc_set_add");
   rc = _set->add(descriptors, n, labels);
-  timers.add_timestamp("desc_set_add");
   return rc;
 }
 
 long DescriptorSet::add_and_store(DescDataArray descriptors, unsigned n,
                                   long *labels) {
   long rc;
-  timers.add_timestamp("desc_set_add_and_store");
   rc = _set->add_and_store(descriptors, n, labels);
-  timers.add_timestamp("desc_set_add_and_store");
   return rc;
 }
 
 void DescriptorSet::train() {
-  timers.add_timestamp("desc_set_add_and_store");
   _set->train();
-  timers.add_timestamp("desc_set_add_and_store");
 }
 
 void DescriptorSet::finalize_index() {
-  timers.add_timestamp("desc_set_finalize_idx");
   _set->finalize_index();
-  timers.add_timestamp("desc_set_finalize_idx");
 }
 
 void DescriptorSet::train(DescDataArray descriptors, unsigned n) {
-  timers.add_timestamp("desc_set_train");
   _set->train(descriptors, n);
-  timers.add_timestamp("desc_set_train");
 }
 
 bool DescriptorSet::is_trained() { return _set->is_trained(); }
 
 void DescriptorSet::classify(DescDataArray descriptors, unsigned n,
                              long *labels, unsigned quorum) {
-  timers.add_timestamp("desc_set_classify");
   _set->classify(descriptors, n, labels, quorum);
-  timers.add_timestamp("desc_set_classify");
 }
 
 void DescriptorSet::get_descriptors(long *ids, unsigned n,
                                     DescDataArray descriptors) {
-  timers.add_timestamp("desc_set_get_descs");
   _set->get_descriptors(ids, n, descriptors);
-  timers.add_timestamp("desc_set_get_descs");
 }
 
 void DescriptorSet::store() {
-  timers.add_timestamp("desc_set_store");
   _set->store();
   write_set_info();
 
@@ -234,14 +209,11 @@ void DescriptorSet::store() {
       // std::remove(filename.c_str());
     }
   }
-  timers.add_timestamp("desc_set_store");
 }
 
 void DescriptorSet::store(std::string set_path) {
-  timers.add_timestamp("desc_set_store");
   _set->store(set_path);
   write_set_info();
-  timers.add_timestamp("desc_set_store");
 }
 
 /*  *********************** */
@@ -251,61 +223,49 @@ void DescriptorSet::store(std::string set_path) {
 long DescriptorSet::add(DescDataArray descriptors, unsigned n,
                         LabelIdVector &labels) {
   long rc;
-  timers.add_timestamp("desc_set_add_vec");
   if (n != labels.size() && labels.size() != 0)
     throw VCLException(SizeMismatch, "Labels Vector of Wrong Size");
   rc = add(descriptors, n, labels.size() > 0 ? (long *)labels.data() : NULL);
-  timers.add_timestamp("desc_set_add_vec");
 
   return rc;
 }
 
 long DescriptorSet::add_and_store(DescDataArray descriptors, unsigned n,
                                   LabelIdVector &labels) {
-  timers.add_timestamp("desc_set_add_store_vec");
   long rc;
   if (n != labels.size() && labels.size() != 0)
     throw VCLException(SizeMismatch, "Labels Vector of Wrong Size");
 
   rc = add_and_store(descriptors, n,
                      labels.size() > 0 ? (long *)labels.data() : NULL);
 
-  timers.add_timestamp("desc_set_add_store_vec");
   return rc;
 }
 
 void DescriptorSet::search(DescDataArray queries, unsigned n, unsigned k,
                            DescIdVector &ids, DistanceVector &distances) {
-  timers.add_timestamp("search");
   ids.resize(n * k);
   distances.resize(n * k);
   search(queries, n, k, ids.data(), distances.data());
-  timers.add_timestamp("search");
 }
 
 void DescriptorSet::search(DescDataArray queries, unsigned n, unsigned k,
                            DescIdVector &ids) {
-  timers.add_timestamp("desc_set_search");
   ids.resize(n * k);
   search(queries, n, k, ids.data());
-  timers.add_timestamp("desc_set_search");
 }
 
 std::vector<long> DescriptorSet::classify(DescDataArray descriptors, unsigned n,
                                           unsigned quorum) {
-  timers.add_timestamp("desc_set_vec_classify");
   LabelIdVector labels;
   labels.resize(n);
   classify(descriptors, n, labels.data(), quorum);
-  timers.add_timestamp("desc_set_vec_classify");
   return labels;
 }
 
 void DescriptorSet::get_descriptors(std::vector<long> &ids,
                                     float *descriptors) {
-  timers.add_timestamp("desc_set_vec_get_desc");
   get_descriptors(ids.data(), ids.size(), descriptors);
-  timers.add_timestamp("desc_set_vec_get_desc");
 }
 
 /*  *********************** */

utils/src/api_schema/api_schema.json

@@ -67,7 +67,13 @@
       { "$ref": "#/definitions/FindBlobTop" },
 
       { "$ref": "#/definitions/NeoAddTop" },
-      { "$ref": "#/definitions/NeoFindTop" }
+      { "$ref": "#/definitions/NeoFindTop" },
+
+      { "$ref": "#/definitions/NeoAddDescriptorSetTop" },
+      { "$ref": "#/definitions/NeoFindDescriptorSetTop" },
+      { "$ref": "#/definitions/NeoAddDescriptorTop" },
+      { "$ref": "#/definitions/NeoClassifyDescriptorTop" },
+      { "$ref": "#/definitions/NeoFindDescriptorTop" }
     ]
   },
   "uniqueItems": false,
@@ -130,7 +136,7 @@
 
     "engineFormatString": {
       "type": "string",
-      "enum": ["FaissFlat", "FaissHNSWFlat", "FaissIVFFlat", "TileDBDense", "TileDBSparse", "Flinng"]
+      "enum": ["FaissFlat", "FaissIVFFlat", "TileDBDense", "TileDBSparse", "Flinng"]
     },
 
     "vidCodecString": {
@@ -452,20 +458,46 @@
       "additionalProperties": false
     },
 
+    "NeoAddDescriptorTop": {
+      "properties": {
+        "NeoAddDescriptor" : { "type": "object",
+          "$ref": "#/definitions/NeoAddDescriptor" }
+      },
+      "additionalProperties": false
+    },
+
     "AddDescriptorSetTop": {
       "properties": {
         "AddDescriptorSet" : { "type": "object",
                                "$ref": "#/definitions/AddDescriptorSet" }
       },
       "additionalProperties": false
     },
+
+    "NeoAddDescriptorSetTop": {
+      "properties": {
+        "NeoAddDescriptorSet" : { "type": "object",
+          "$ref": "#/definitions/NeoAddDescriptorSet" }
+      },
+      "additionalProperties": false
+    },
+
     "FindDescriptorSetTop": {
       "properties": {
         "FindDescriptorSet" : { "type": "object",
                                "$ref": "#/definitions/FindDescriptorSet" }
       },
       "additionalProperties": false
     },
+
+    "NeoFindDescriptorSetTop": {
+      "properties": {
+        "NeoFindDescriptorSet" : { "type": "object",
+          "$ref": "#/definitions/NeoFindDescriptorSet" }
+      },
+      "additionalProperties": false
+    },
+
     "DeleteExpiredTop": {
       "properties": {
         "DeleteExpired" : { "type": "object", "$ref": "#/definitions/DeleteExpired" }
@@ -481,13 +513,27 @@
       "additionalProperties": false
     },
 
+    "NeoClassifyDescriptorTop": {
+      "properties": {
+        "NeoClassifyDescriptor" : { "type": "object", "$ref": "#/definitions/NeoClassifyDescriptor" }
+      },
+      "additionalProperties": false
+    },
+
     "FindDescriptorTop": {
       "properties": {
         "FindDescriptor" : { "type": "object", "$ref": "#/definitions/FindDescriptor" }
       },
       "additionalProperties": false
     },
 
+    "NeoFindDescriptorTop": {
+      "properties": {
+        "NeoFindDescriptor" : { "type": "object", "$ref": "#/definitions/NeoFindDescriptor" }
+      },
+      "additionalProperties": false
+    },
+
     "AddBoundingBoxTop": {
       "properties": {
         "AddBoundingBox" : { "type": "object", "$ref": "#/definitions/AddBoundingBox" }
@@ -728,6 +774,26 @@
       "required": ["name", "dimensions"],
       "additionalProperties": false
     },
+
+    "NeoAddDescriptorSet": {
+      "properties": {
+        "name":       { "type": "string" },
+        "dimensions": { "$ref": "#/definitions/refInt" },
+        "metric":     { "$ref": "#/definitions/metricFormatString" },
+        "engine":     { "$ref": "#/definitions/engineFormatString" },
+        "link":       { "$ref": "#/definitions/blockLink" },
+        "properties": { "type": "object" },
+        "flinng_num_rows":{ "$ref": "#/definitions/refInt" },
+        "flinng_cells_per_row":{ "$ref": "#/definitions/refInt" },
+        "flinng_num_hash_tables":{ "$ref": "#/definitions/refInt" },
+        "flinng_hashes_per_table":{ "$ref": "#/definitions/refInt" },
+        "flinng_sub_hash_bits":{ "$ref": "#/definitions/refInt" },
+        "flinng_cut_off":{ "$ref": "#/definitions/refInt" }
+      },
+      "required": ["name", "dimensions"],
+      "additionalProperties": false
+    },
+
     "FindDescriptorSet": {
       "properties": {
         "_ref":        { "$ref": "#/definitions/refInt" },
@@ -743,6 +809,21 @@
       "additionalProperties": false
     },
 
+    "NeoFindDescriptorSet": {
+      "properties": {
+        "_ref":        { "$ref": "#/definitions/refInt" },
+        "results":     { "$ref": "#/definitions/blockResults" },
+        "set":       { "type": "string" },
+        "storeIndex" : { "type": "boolean" },
+        "constraints": { "type": "object" },
+        "link":       { "$ref": "#/definitions/blockLink" }
+
+      },
+      "required": ["set"],
+
+      "additionalProperties": false
+    },
+
     "AddDescriptor": {
       "properties": {
         "set":        { "type": "string" },
@@ -756,6 +837,17 @@
       "additionalProperties": false
     },
 
+    "NeoAddDescriptor": {
+      "properties": {
+        "set":        { "type": "string" },
+        "label":      { "type": "string" },
+        "properties": { "type": "object" },
+        "batch_properties": {"$ref": "#/definitions/propertyArray"}
+      },
+      "required": ["set"],
+      "additionalProperties": false
+    },
+
     "ClassifyDescriptor": {
       "properties": {
         "set":         { "type": "string" },
@@ -766,6 +858,15 @@
       "additionalProperties": false
     },
 
+    "NeoClassifyDescriptor": {
+      "properties": {
+        "set":         { "type": "string" },
+        "k_neighbors": { "$ref": "#/definitions/positiveInt" }
+      },
+      "required": ["set"],
+      "additionalProperties": false
+    },
+
     "FindDescriptor": {
       "properties": {
         "set":         { "type": "string" },
@@ -780,6 +881,19 @@
       "additionalProperties": false
     },
 
+    "NeoFindDescriptor": {
+      "properties": {
+        "set":         { "type": "string" },
+        "_ref":        { "$ref": "#/definitions/refInt" },
+        "k_neighbors": { "$ref": "#/definitions/positiveInt" },
+        "results":     { "$ref": "#/definitions/blockResults" },
+        "constraints": { "type": "object" }
+      },
+      "required": ["set"],
+      "additionalProperties": false
+    },
+
+
     "AddBoundingBox": {
       "properties": {
         "_ref":       { "$ref": "#/definitions/refInt" },