From 0f6be5f02278485e0e68d997b6b840c7df4ff65e Mon Sep 17 00:00:00 2001 From: "xj.lin" Date: Sat, 11 May 2019 12:04:47 +0800 Subject: [PATCH 01/16] 1 update conf files 2 new scripts dir 3 update Readme Former-commit-id: 0b0c4ca55fc05e459c3e2ed7e8c9c83b3752cbd3 --- cpp/CMakeLists.txt | 4 ++-- cpp/README.md | 1 + cpp/conf/server_config.yaml | 14 +++++++------- cpp/scripts/start_server.sh | 4 ++++ cpp/scripts/stop_server.sh | 16 ++++++++++++++++ 5 files changed, 30 insertions(+), 9 deletions(-) create mode 100755 cpp/scripts/start_server.sh create mode 100755 cpp/scripts/stop_server.sh diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index a8c49657c5..040e2a73fa 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -79,8 +79,8 @@ add_custom_target(Clean-All COMMAND ${CMAKE_BUILD_TOOL} clean) #install install(FILES - start_server.sh - stop_server.sh + scripts/start_server.sh + scripts/stop_server.sh DESTINATION scripts) install(FILES diff --git a/cpp/README.md b/cpp/README.md index d8cb62991b..c870304ea4 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -23,6 +23,7 @@ cmake_build/src/libvecwise_engine.a is the static library cd [sourcecode path]/cpp ./build.sh -t Debug ./build.sh -t Release + ./build.sh -g # Build GPU version #### To build unittest: diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml index d91161830b..14570588bd 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/conf/server_config.yaml @@ -5,7 +5,7 @@ server_config: server_mode: thread_pool #optional: simple, thread_pool db_config: - db_path: /tmp/vecwise + db_path: /var/vecwise db_backend_url: http://127.0.0.1 db_flush_interval: 5 #unit: second idmapper_max_open_file: 128 @@ -13,7 +13,7 @@ db_config: log_config: global: format: "%datetime | %level | %logger | %msg" - filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-global.log" + filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-global.log" enabled: true to_file: true to_standard_output: true @@ -21,22 +21,22 @@ log_config: performance_tracking: false max_log_file_size: 2097152 # throw log files away after 2mb debug: - filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-debug.log" + filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-debug.log" enabled: true warning: - filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-warning.log" + filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-warning.log" trace: - filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-trace.log" + filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-trace.log" verbose: format: "%datetime{%d/%m/%y} | %level-%vlevel | %msg" to_file: false to_standard_output: true error: enabled: false - filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-error.log" + filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-error.log" fatal: enabled: false - filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-fatal.log" + filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-fatal.log" cache_config: cpu_cache_capacity: 16 # unit: GB diff --git a/cpp/scripts/start_server.sh b/cpp/scripts/start_server.sh new file mode 100755 index 0000000000..6f5db9490c --- /dev/null +++ b/cpp/scripts/start_server.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +../bin/vecwise_server -c ../conf/server_config.yaml -l ../conf/vecwise_engine_log.conf + diff --git a/cpp/scripts/stop_server.sh b/cpp/scripts/stop_server.sh new file mode 100755 index 0000000000..89bd9398e0 --- /dev/null +++ b/cpp/scripts/stop_server.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +function kill_progress() +{ + kill -s SIGUSR2 $(pgrep $1) + + sleep 2 +} + +STATUS=$(kill_progress "vecwise_server" ) + +if [[ ${STATUS} == "false" ]];then + echo "vecwise_server closed abnormally!" +else + echo "vecwise_server closed successfully!" +fi From 527651bbc683d9852943fefd492c743e9451a9e5 Mon Sep 17 00:00:00 2001 From: "xj.lin" Date: Sat, 11 May 2019 12:10:30 +0800 Subject: [PATCH 02/16] update conf files Former-commit-id: 28cf4be27eb0fe9d74046bb609dba103d162e7b2 --- cpp/conf/vecwise_engine_log.conf | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cpp/conf/vecwise_engine_log.conf b/cpp/conf/vecwise_engine_log.conf index 4935b2fd9a..d0164b4158 100644 --- a/cpp/conf/vecwise_engine_log.conf +++ b/cpp/conf/vecwise_engine_log.conf @@ -1,6 +1,6 @@ * GLOBAL: FORMAT = "%datetime | %level | %logger | %msg" - FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-global.log" + FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-global.log" ENABLED = true TO_FILE = true TO_STANDARD_OUTPUT = true @@ -8,12 +8,12 @@ PERFORMANCE_TRACKING = false MAX_LOG_FILE_SIZE = 2097152 ## Throw log files away after 2MB * DEBUG: - FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-debug.log" + FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-debug.log" ENABLED = true * WARNING: - FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-warning.log" + FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-warning.log" * TRACE: - FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-trace.log" + FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-trace.log" * VERBOSE: FORMAT = "%datetime{%d/%M/%y} | %level-%vlevel | %msg" TO_FILE = false @@ -21,7 +21,7 @@ ## Error logs * ERROR: ENABLED = false - FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-error.log" + FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-error.log" * FATAL: ENABLED = false - FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-fatal.log" \ No newline at end of file + FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-fatal.log" \ No newline at end of file From fddf73204a3a715beeffcad32304f8223e686db5 Mon Sep 17 00:00:00 2001 From: "xj.lin" Date: Sat, 11 May 2019 13:21:19 +0800 Subject: [PATCH 03/16] update config Former-commit-id: 5004449f97f88acac31e01c3d83b7302059091b5 --- cpp/conf/server_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml index 14570588bd..790bca4ba5 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/conf/server_config.yaml @@ -1,5 +1,5 @@ server_config: - address: 127.0.0.1 + address: 0.0.0.0 port: 33001 transfer_protocol: json #optional: binary, compact, json, debug server_mode: thread_pool #optional: simple, thread_pool From c4244fdc6db402f3b0244174ae87779a0a18d50c Mon Sep 17 00:00:00 2001 From: "xj.lin" Date: Sat, 11 May 2019 16:52:16 +0800 Subject: [PATCH 04/16] fix search stack overflow Former-commit-id: 39801544686f061a63f4c3f1dec11565164a928d --- cpp/src/db/DBImpl.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index f977222535..4218ffa953 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -107,7 +107,7 @@ Status DBImpl::search(const std::string& group_id, size_t k, size_t nq, using SearchResult = std::pair, std::vector>; std::vector batchresult(nq); // allocate nq cells. - auto cluster = [&](long *nns, float *dis) -> void { + auto cluster = [&](long *nns, float *dis, const int& k) -> void { for (int i = 0; i < nq; ++i) { auto f_begin = batchresult[i].first.cbegin(); auto s_begin = batchresult[i].second.cbegin(); @@ -134,8 +134,10 @@ Status DBImpl::search(const std::string& group_id, size_t k, size_t nq, search_set_size += file_size; LOG(DEBUG) << "Search file_type " << file.file_type << " Of Size: " << file_size << " M"; - index.Search(nq, vectors, k, output_distence, output_ids); - cluster(output_ids, output_distence); // cluster to each query + + int inner_k = index.Count() < k ? index.Count() : k; + index.Search(nq, vectors, inner_k, output_distence, output_ids); + cluster(output_ids, output_distence, inner_k); // cluster to each query memset(output_distence, 0, k * nq * sizeof(float)); memset(output_ids, 0, k * nq * sizeof(long)); } @@ -161,8 +163,11 @@ Status DBImpl::search(const std::string& group_id, size_t k, size_t nq, for (auto &result_pair : batchresult) { auto &dis = result_pair.second; auto &nns = result_pair.first; + topk_cpu(dis, k, output_distence, output_ids); - for (int i = 0; i < k; ++i) { + + int inner_k = dis.size() < k ? dis.size() : k; + for (int i = 0; i < inner_k; ++i) { res.emplace_back(nns[output_ids[i]]); // mapping } results.push_back(res); // append to result list From 354f68a96feaa2a31a63f136eef47c9bf5eb3425 Mon Sep 17 00:00:00 2001 From: "xj.lin" Date: Sat, 11 May 2019 18:49:30 +0800 Subject: [PATCH 05/16] fix TopK bug Former-commit-id: 76e24617b83d49b2a4808a0cd7406edb849e767e --- cpp/src/db/DBImpl.cpp | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 4218ffa953..3df27ba063 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -147,15 +147,25 @@ Status DBImpl::search(const std::string& group_id, size_t k, size_t nq, const int &k, float *output_distence, long *output_ids) -> void { - std::map inverted_table; + std::map> inverted_table; for (int i = 0; i < input_data.size(); ++i) { - inverted_table[input_data[i]] = i; + if (inverted_table.count(input_data[i]) == 1) { + auto& ori_vec = inverted_table[input_data[i]]; + ori_vec.push_back(i); + } + else { + inverted_table[input_data[i]] = std::vector{i}; + } } int count = 0; - for (auto it = inverted_table.begin(); it != inverted_table.end() && count < k; ++it, ++count) { - output_distence[count] = it->first; - output_ids[count] = it->second; + for (auto &item : inverted_table){ + if (count == k) break; + for (auto &id : item.second){ + if (++count == k) break; + output_distence[count] = item.first; + output_ids[count] = id; + } } }; auto cluster_topk = [&]() -> void { From 56291b218d2070b615c4b61151e9fb890bf2918b Mon Sep 17 00:00:00 2001 From: groot Date: Mon, 13 May 2019 15:17:05 +0800 Subject: [PATCH 06/16] use rocksdb column family Former-commit-id: 8cbc5d91e02e6bbc0eff1e329b4c76687d2c1f74 --- cpp/conf/server_config.yaml | 16 +++--- cpp/conf/vecwise_engine_log.conf | 12 ++-- cpp/src/server/VecIdMapper.cpp | 81 +++++++++++++++++++++------ cpp/src/server/VecIdMapper.h | 33 +++++------ cpp/unittest/server/idmapper_test.cpp | 10 +++- 5 files changed, 101 insertions(+), 51 deletions(-) diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml index 790bca4ba5..d91161830b 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/conf/server_config.yaml @@ -1,11 +1,11 @@ server_config: - address: 0.0.0.0 + address: 127.0.0.1 port: 33001 transfer_protocol: json #optional: binary, compact, json, debug server_mode: thread_pool #optional: simple, thread_pool db_config: - db_path: /var/vecwise + db_path: /tmp/vecwise db_backend_url: http://127.0.0.1 db_flush_interval: 5 #unit: second idmapper_max_open_file: 128 @@ -13,7 +13,7 @@ db_config: log_config: global: format: "%datetime | %level | %logger | %msg" - filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-global.log" + filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-global.log" enabled: true to_file: true to_standard_output: true @@ -21,22 +21,22 @@ log_config: performance_tracking: false max_log_file_size: 2097152 # throw log files away after 2mb debug: - filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-debug.log" + filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-debug.log" enabled: true warning: - filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-warning.log" + filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-warning.log" trace: - filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-trace.log" + filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-trace.log" verbose: format: "%datetime{%d/%m/%y} | %level-%vlevel | %msg" to_file: false to_standard_output: true error: enabled: false - filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-error.log" + filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-error.log" fatal: enabled: false - filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-fatal.log" + filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-fatal.log" cache_config: cpu_cache_capacity: 16 # unit: GB diff --git a/cpp/conf/vecwise_engine_log.conf b/cpp/conf/vecwise_engine_log.conf index d0164b4158..4935b2fd9a 100644 --- a/cpp/conf/vecwise_engine_log.conf +++ b/cpp/conf/vecwise_engine_log.conf @@ -1,6 +1,6 @@ * GLOBAL: FORMAT = "%datetime | %level | %logger | %msg" - FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-global.log" + FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-global.log" ENABLED = true TO_FILE = true TO_STANDARD_OUTPUT = true @@ -8,12 +8,12 @@ PERFORMANCE_TRACKING = false MAX_LOG_FILE_SIZE = 2097152 ## Throw log files away after 2MB * DEBUG: - FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-debug.log" + FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-debug.log" ENABLED = true * WARNING: - FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-warning.log" + FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-warning.log" * TRACE: - FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-trace.log" + FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-trace.log" * VERBOSE: FORMAT = "%datetime{%d/%M/%y} | %level-%vlevel | %msg" TO_FILE = false @@ -21,7 +21,7 @@ ## Error logs * ERROR: ENABLED = false - FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-error.log" + FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-error.log" * FATAL: ENABLED = false - FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-fatal.log" \ No newline at end of file + FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-fatal.log" \ No newline at end of file diff --git a/cpp/src/server/VecIdMapper.cpp b/cpp/src/server/VecIdMapper.cpp index 2a50d4db2d..b0dfcc6db2 100644 --- a/cpp/src/server/VecIdMapper.cpp +++ b/cpp/src/server/VecIdMapper.cpp @@ -13,6 +13,8 @@ #include "rocksdb/slice.h" #include "rocksdb/options.h" +#include + namespace zilliz { namespace vecwise { namespace server { @@ -36,26 +38,33 @@ SimpleIdMapper::~SimpleIdMapper() { } -ServerError SimpleIdMapper::Put(const std::string& nid, const std::string& sid) { - ids_[nid] = sid; +//not thread-safe +ServerError SimpleIdMapper::Put(const std::string& nid, const std::string& sid, const std::string& group) { + ID_MAPPING& mapping = id_groups_[group]; + mapping[nid] = sid; return SERVER_SUCCESS; } -ServerError SimpleIdMapper::Put(const std::vector& nid, const std::vector& sid) { +//not thread-safe +ServerError SimpleIdMapper::Put(const std::vector& nid, const std::vector& sid, const std::string& group) { if(nid.size() != sid.size()) { return SERVER_INVALID_ARGUMENT; } + ID_MAPPING& mapping = id_groups_[group]; for(size_t i = 0; i < nid.size(); i++) { - ids_[nid[i]] = sid[i]; + mapping[nid[i]] = sid[i]; } return SERVER_SUCCESS; } -ServerError SimpleIdMapper::Get(const std::string& nid, std::string& sid) const { - auto iter = ids_.find(nid); - if(iter == ids_.end()) { +//not thread-safe +ServerError SimpleIdMapper::Get(const std::string& nid, std::string& sid, const std::string& group) const { + ID_MAPPING& mapping = id_groups_[group]; + + auto iter = mapping.find(nid); + if(iter == mapping.end()) { return SERVER_INVALID_ARGUMENT; } @@ -64,13 +73,16 @@ ServerError SimpleIdMapper::Get(const std::string& nid, std::string& sid) const return SERVER_SUCCESS; } -ServerError SimpleIdMapper::Get(const std::vector& nid, std::vector& sid) const { +//not thread-safe +ServerError SimpleIdMapper::Get(const std::vector& nid, std::vector& sid, const std::string& group) const { sid.clear(); + ID_MAPPING& mapping = id_groups_[group]; + ServerError err = SERVER_SUCCESS; for(size_t i = 0; i < nid.size(); i++) { - auto iter = ids_.find(nid[i]); - if(iter == ids_.end()) { + auto iter = mapping.find(nid[i]); + if(iter == mapping.end()) { sid.push_back(""); SERVER_LOG_ERROR << "ID mapper failed to find id: " << nid[i]; err = SERVER_INVALID_ARGUMENT; @@ -83,8 +95,10 @@ ServerError SimpleIdMapper::Get(const std::vector& nid, std::vector return err; } -ServerError SimpleIdMapper::Delete(const std::string& nid) { - ids_.erase(nid); +//not thread-safe +ServerError SimpleIdMapper::Delete(const std::string& nid, const std::string& group) { + ID_MAPPING& mapping = id_groups_[group]; + mapping.erase(nid); return SERVER_SUCCESS; } @@ -104,12 +118,43 @@ RocksIdMapper::RocksIdMapper() { options.create_if_missing = true; options.max_open_files = config.GetInt32Value(CONFIG_DB_IDMAPPER_MAX_FILE, 128); + //load column families + std::vector column_names; + rocksdb::Status s = rocksdb::DB::ListColumnFamilies(options, db_path, &column_names); + if (!s.ok()) { + SERVER_LOG_ERROR << "ID mapper failed to initialize:" << s.ToString(); + } + + std::vector column_families; + for(auto& column_name : column_names) { + rocksdb::ColumnFamilyDescriptor desc; + desc.name = column_name; + column_families.emplace_back(desc); + } + // open DB - rocksdb::Status s = rocksdb::DB::Open(options, db_path, &db_); + std::vector column_handles; + s = rocksdb::DB::Open(options, db_path, column_families, &column_handles, &db_); if(!s.ok()) { SERVER_LOG_ERROR << "ID mapper failed to initialize:" << s.ToString(); db_ = nullptr; } + + try { + rocksdb::ColumnFamilyHandle *cf; + s = db_->CreateColumnFamily(rocksdb::ColumnFamilyOptions(), "222", &cf); + if (!s.ok()) { + SERVER_LOG_ERROR << "ID mapper failed to initialize:" << s.ToString(); + } + + std::vector column_families; + s = db_->ListColumnFamilies(options, db_path, &column_families); + if (!s.ok()) { + SERVER_LOG_ERROR << "ID mapper failed to initialize:" << s.ToString(); + } + } catch(std::exception& ex) { + std::cout << ex.what() << std::endl; + } } RocksIdMapper::~RocksIdMapper() { if(db_) { @@ -118,7 +163,7 @@ RocksIdMapper::~RocksIdMapper() { } } -ServerError RocksIdMapper::Put(const std::string& nid, const std::string& sid) { +ServerError RocksIdMapper::Put(const std::string& nid, const std::string& sid, const std::string& group) { if(db_ == nullptr) { return SERVER_NULL_POINTER; } @@ -134,7 +179,7 @@ ServerError RocksIdMapper::Put(const std::string& nid, const std::string& sid) { return SERVER_SUCCESS; } -ServerError RocksIdMapper::Put(const std::vector& nid, const std::vector& sid) { +ServerError RocksIdMapper::Put(const std::vector& nid, const std::vector& sid, const std::string& group) { if(nid.size() != sid.size()) { return SERVER_INVALID_ARGUMENT; } @@ -150,7 +195,7 @@ ServerError RocksIdMapper::Put(const std::vector& nid, const std::v return err; } -ServerError RocksIdMapper::Get(const std::string& nid, std::string& sid) const { +ServerError RocksIdMapper::Get(const std::string& nid, std::string& sid, const std::string& group) const { if(db_ == nullptr) { return SERVER_NULL_POINTER; } @@ -165,7 +210,7 @@ ServerError RocksIdMapper::Get(const std::string& nid, std::string& sid) const { return SERVER_SUCCESS; } -ServerError RocksIdMapper::Get(const std::vector& nid, std::vector& sid) const { +ServerError RocksIdMapper::Get(const std::vector& nid, std::vector& sid, const std::string& group) const { sid.clear(); ServerError err = SERVER_SUCCESS; @@ -185,7 +230,7 @@ ServerError RocksIdMapper::Get(const std::vector& nid, std::vector< return err; } -ServerError RocksIdMapper::Delete(const std::string& nid) { +ServerError RocksIdMapper::Delete(const std::string& nid, const std::string& group) { if(db_ == nullptr) { return SERVER_NULL_POINTER; } diff --git a/cpp/src/server/VecIdMapper.h b/cpp/src/server/VecIdMapper.h index b376c12638..216218b51e 100644 --- a/cpp/src/server/VecIdMapper.h +++ b/cpp/src/server/VecIdMapper.h @@ -25,14 +25,14 @@ public: virtual ~IVecIdMapper(){} - virtual ServerError Put(const std::string& nid, const std::string& sid) = 0; - virtual ServerError Put(const std::vector& nid, const std::vector& sid) = 0; + virtual ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") = 0; + virtual ServerError Put(const std::vector& nid, const std::vector& sid, const std::string& group = "") = 0; - virtual ServerError Get(const std::string& nid, std::string& sid) const = 0; + virtual ServerError Get(const std::string& nid, std::string& sid, const std::string& group = "") const = 0; //NOTE: the 'sid' will be cleared at begin of the function - virtual ServerError Get(const std::vector& nid, std::vector& sid) const = 0; + virtual ServerError Get(const std::vector& nid, std::vector& sid, const std::string& group = "") const = 0; - virtual ServerError Delete(const std::string& nid) = 0; + virtual ServerError Delete(const std::string& nid, const std::string& group = "") = 0; }; class SimpleIdMapper : public IVecIdMapper{ @@ -40,16 +40,17 @@ public: SimpleIdMapper(); ~SimpleIdMapper(); - ServerError Put(const std::string& nid, const std::string& sid) override; - ServerError Put(const std::vector& nid, const std::vector& sid) override; + ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") override; + ServerError Put(const std::vector& nid, const std::vector& sid, const std::string& group = "") override; - ServerError Get(const std::string& nid, std::string& sid) const override; - ServerError Get(const std::vector& nid, std::vector& sid) const override; + ServerError Get(const std::string& nid, std::string& sid, const std::string& group = "") const override; + ServerError Get(const std::vector& nid, std::vector& sid, const std::string& group = "") const override; - ServerError Delete(const std::string& nid) override; + ServerError Delete(const std::string& nid, const std::string& group = "") override; private: - std::unordered_map ids_; + using ID_MAPPING = std::unordered_map; + mutable std::unordered_map id_groups_; }; class RocksIdMapper : public IVecIdMapper{ @@ -57,13 +58,13 @@ public: RocksIdMapper(); ~RocksIdMapper(); - ServerError Put(const std::string& nid, const std::string& sid) override; - ServerError Put(const std::vector& nid, const std::vector& sid) override; + ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") override; + ServerError Put(const std::vector& nid, const std::vector& sid, const std::string& group = "") override; - ServerError Get(const std::string& nid, std::string& sid) const override; - ServerError Get(const std::vector& nid, std::vector& sid) const override; + ServerError Get(const std::string& nid, std::string& sid, const std::string& group = "") const override; + ServerError Get(const std::vector& nid, std::vector& sid, const std::string& group = "") const override; - ServerError Delete(const std::string& nid) override; + ServerError Delete(const std::string& nid, const std::string& group = "") override; private: rocksdb::DB* db_; diff --git a/cpp/unittest/server/idmapper_test.cpp b/cpp/unittest/server/idmapper_test.cpp index 80bbb2e402..d0831c825b 100644 --- a/cpp/unittest/server/idmapper_test.cpp +++ b/cpp/unittest/server/idmapper_test.cpp @@ -44,9 +44,13 @@ TEST(IdMapperTest, IDMAPPER_TEST) { nid.clear(); sid.clear(); const int64_t count = 1000000; - for(int64_t i = 0; i < count; i++) { - nid.push_back(std::to_string(i+100000)); - sid.push_back("val_" + std::to_string(i)); + { + server::TimeRecorder rc("prepare id data"); + for (int64_t i = 0; i < count; i++) { + nid.push_back(std::to_string(i + 100000)); + sid.push_back("val_" + std::to_string(i)); + } + rc.Record("done!"); } { From a56333b55d23a14ad6d3a35848ef062fd6b01b23 Mon Sep 17 00:00:00 2001 From: groot Date: Mon, 13 May 2019 17:05:24 +0800 Subject: [PATCH 07/16] throw exception Former-commit-id: 6a8ae6f0c0b73fc9411b039143a1650e6ab7cc02 --- cpp/src/server/VecServiceHandler.cpp | 89 ++++++++--- cpp/src/server/VecServiceScheduler.h | 2 + cpp/src/server/VecServiceTask.cpp | 139 +++++++++++------- cpp/src/thrift/cpp_gen.sh | 2 +- cpp/src/thrift/gen-py/py_sample.py | 4 +- ...VectorService.thrift => megasearch.thrift} | 0 cpp/src/thrift/py_gen.sh | 2 +- cpp/src/utils/Error.h | 5 +- cpp/test_client/src/ClientTest.cpp | 17 +++ 9 files changed, 177 insertions(+), 83 deletions(-) rename cpp/src/thrift/{VectorService.thrift => megasearch.thrift} (100%) diff --git a/cpp/src/server/VecServiceHandler.cpp b/cpp/src/server/VecServiceHandler.cpp index de5942cd8b..843b37a8d5 100644 --- a/cpp/src/server/VecServiceHandler.cpp +++ b/cpp/src/server/VecServiceHandler.cpp @@ -38,6 +38,62 @@ namespace { void TimeRecord(const std::string& func_name) { } + + const std::map& ErrorMap() { + static const std::map code_map = { + {SERVER_UNEXPECTED_ERROR, zilliz::VecErrCode::ILLEGAL_ARGUMENT}, + {SERVER_NULL_POINTER, zilliz::VecErrCode::ILLEGAL_ARGUMENT}, + {SERVER_INVALID_ARGUMENT, zilliz::VecErrCode::ILLEGAL_ARGUMENT}, + {SERVER_FILE_NOT_FOUND, zilliz::VecErrCode::ILLEGAL_ARGUMENT}, + {SERVER_NOT_IMPLEMENT, zilliz::VecErrCode::ILLEGAL_ARGUMENT}, + {SERVER_BLOCKING_QUEUE_EMPTY, zilliz::VecErrCode::ILLEGAL_ARGUMENT}, + {SERVER_GROUP_NOT_EXIST, zilliz::VecErrCode::GROUP_NOT_EXISTS}, + {SERVER_INVALID_TIME_RANGE, zilliz::VecErrCode::ILLEGAL_TIME_RANGE}, + {SERVER_INVALID_VECTOR_DIMENSION, zilliz::VecErrCode::ILLEGAL_VECTOR_DIMENSION}, + }; + + return code_map; + } + + const std::map& ErrorMessage() { + static const std::map msg_map = { + {SERVER_UNEXPECTED_ERROR, "unexpected error occurs"}, + {SERVER_NULL_POINTER, "null pointer error"}, + {SERVER_INVALID_ARGUMENT, "invalid argument"}, + {SERVER_FILE_NOT_FOUND, "file not found"}, + {SERVER_NOT_IMPLEMENT, "not implemented"}, + {SERVER_BLOCKING_QUEUE_EMPTY, "queue empty"}, + {SERVER_GROUP_NOT_EXIST, "group not exist"}, + {SERVER_INVALID_TIME_RANGE, "invalid time range"}, + {SERVER_INVALID_VECTOR_DIMENSION, "invalid vector dimension"}, + }; + + return msg_map; + } + + void ExecTask(BaseTaskPtr& task_ptr) { + if(task_ptr == nullptr) { + return; + } + + VecServiceScheduler& scheduler = VecServiceScheduler::GetInstance(); + scheduler.ExecuteTask(task_ptr); + + if(!task_ptr->IsAsync()) { + task_ptr->WaitToFinish(); + ServerError err = task_ptr->ErrorCode(); + if (err != SERVER_SUCCESS) { + zilliz::VecException ex; + ex.__set_code(ErrorMap().at(err)); + std::string msg = task_ptr->ErrorMsg(); + if(msg.empty()){ + msg = ErrorMessage().at(err); + } + ex.__set_reason(msg); + throw ex; + } + } + } } void @@ -47,8 +103,7 @@ VecServiceHandler::add_group(const VecGroup &group) { << ", group.index_type = " << group.index_type; BaseTaskPtr task_ptr = AddGroupTask::Create(group.dimension, group.id); - VecServiceScheduler& scheduler = VecServiceScheduler::GetInstance(); - scheduler.ExecuteTask(task_ptr); + ExecTask(task_ptr); } void @@ -58,8 +113,7 @@ VecServiceHandler::get_group(VecGroup &_return, const std::string &group_id) { _return.id = group_id; BaseTaskPtr task_ptr = GetGroupTask::Create(group_id, _return.dimension); - VecServiceScheduler& scheduler = VecServiceScheduler::GetInstance(); - scheduler.ExecuteTask(task_ptr); + ExecTask(task_ptr); } void @@ -68,8 +122,7 @@ VecServiceHandler::del_group(const std::string &group_id) { SERVER_LOG_TRACE << "group_id = " << group_id; BaseTaskPtr task_ptr = DeleteGroupTask::Create(group_id); - VecServiceScheduler& scheduler = VecServiceScheduler::GetInstance(); - scheduler.ExecuteTask(task_ptr); + ExecTask(task_ptr); } @@ -79,8 +132,7 @@ VecServiceHandler::add_vector(std::string& _return, const std::string &group_id, SERVER_LOG_TRACE << "group_id = " << group_id << ", vector size = " << tensor.tensor.size(); BaseTaskPtr task_ptr = AddVectorTask::Create(group_id, &tensor, _return); - VecServiceScheduler& scheduler = VecServiceScheduler::GetInstance(); - scheduler.ExecuteTask(task_ptr); + ExecTask(task_ptr); } void @@ -92,8 +144,7 @@ VecServiceHandler::add_vector_batch(std::vector & _return, << tensor_list.tensor_list.size(); BaseTaskPtr task_ptr = AddBatchVectorTask::Create(group_id, &tensor_list, _return); - VecServiceScheduler& scheduler = VecServiceScheduler::GetInstance(); - scheduler.ExecuteTask(task_ptr); + ExecTask(task_ptr); } void @@ -104,8 +155,7 @@ VecServiceHandler::add_binary_vector(std::string& _return, SERVER_LOG_TRACE << "group_id = " << group_id << ", vector size = " << tensor.tensor.size()/4; BaseTaskPtr task_ptr = AddVectorTask::Create(group_id, &tensor, _return); - VecServiceScheduler& scheduler = VecServiceScheduler::GetInstance(); - scheduler.ExecuteTask(task_ptr); + ExecTask(task_ptr); } void @@ -117,8 +167,7 @@ VecServiceHandler::add_binary_vector_batch(std::vector & _return, << tensor_list.tensor_list.size(); BaseTaskPtr task_ptr = AddBatchVectorTask::Create(group_id, &tensor_list, _return); - VecServiceScheduler& scheduler = VecServiceScheduler::GetInstance(); - scheduler.ExecuteTask(task_ptr); + ExecTask(task_ptr); } void @@ -135,8 +184,7 @@ VecServiceHandler::search_vector(VecSearchResult &_return, tensor_list.tensor_list.push_back(tensor); VecSearchResultList result; BaseTaskPtr task_ptr = SearchVectorTask::Create(group_id, top_k, &tensor_list, filter, result); - VecServiceScheduler& scheduler = VecServiceScheduler::GetInstance(); - scheduler.ExecuteTask(task_ptr); + ExecTask(task_ptr); if(!result.result_list.empty()) { _return = result.result_list[0]; @@ -156,8 +204,7 @@ VecServiceHandler::search_vector_batch(VecSearchResultList &_return, << ", vector list size = " << tensor_list.tensor_list.size(); BaseTaskPtr task_ptr = SearchVectorTask::Create(group_id, top_k, &tensor_list, filter, _return); - VecServiceScheduler& scheduler = VecServiceScheduler::GetInstance(); - scheduler.ExecuteTask(task_ptr); + ExecTask(task_ptr); } void @@ -174,8 +221,7 @@ VecServiceHandler::search_binary_vector(VecSearchResult& _return, tensor_list.tensor_list.push_back(tensor); VecSearchResultList result; BaseTaskPtr task_ptr = SearchVectorTask::Create(group_id, top_k, &tensor_list, filter, result); - VecServiceScheduler& scheduler = VecServiceScheduler::GetInstance(); - scheduler.ExecuteTask(task_ptr); + ExecTask(task_ptr); if(!result.result_list.empty()) { _return = result.result_list[0]; @@ -195,8 +241,7 @@ VecServiceHandler::search_binary_vector_batch(VecSearchResultList& _return, << ", vector list size = " << tensor_list.tensor_list.size(); BaseTaskPtr task_ptr = SearchVectorTask::Create(group_id, top_k, &tensor_list, filter, _return); - VecServiceScheduler& scheduler = VecServiceScheduler::GetInstance(); - scheduler.ExecuteTask(task_ptr); + ExecTask(task_ptr); } diff --git a/cpp/src/server/VecServiceScheduler.h b/cpp/src/server/VecServiceScheduler.h index ffcecbce5b..65b234c84d 100644 --- a/cpp/src/server/VecServiceScheduler.h +++ b/cpp/src/server/VecServiceScheduler.h @@ -27,6 +27,7 @@ public: std::string TaskGroup() const { return task_group_; } ServerError ErrorCode() const { return error_code_; } + std::string ErrorMsg() const { return error_msg_; } bool IsAsync() const { return async_; } @@ -41,6 +42,7 @@ protected: bool async_; bool done_; ServerError error_code_; + std::string error_msg_; }; using BaseTaskPtr = std::shared_ptr; diff --git a/cpp/src/server/VecServiceTask.cpp b/cpp/src/server/VecServiceTask.cpp index 9eda08495a..7afdb93f5d 100644 --- a/cpp/src/server/VecServiceTask.cpp +++ b/cpp/src/server/VecServiceTask.cpp @@ -92,12 +92,16 @@ ServerError AddGroupTask::OnExecute() { group_info.group_id = group_id_; engine::Status stat = DB()->add_group(group_info); if(!stat.ok()) { - SERVER_LOG_ERROR << "Engine failed: " << stat.ToString(); - return SERVER_UNEXPECTED_ERROR; + error_code_ = SERVER_UNEXPECTED_ERROR; + error_msg_ = "Engine failed: " + stat.ToString(); + SERVER_LOG_ERROR << error_msg_; + return error_code_; } } catch (std::exception& ex) { - SERVER_LOG_ERROR << ex.what(); + error_code_ = SERVER_UNEXPECTED_ERROR; + error_msg_ = ex.what(); + SERVER_LOG_ERROR << error_msg_; return SERVER_UNEXPECTED_ERROR; } @@ -124,14 +128,18 @@ ServerError GetGroupTask::OnExecute() { group_info.group_id = group_id_; engine::Status stat = DB()->get_group(group_info); if(!stat.ok()) { - SERVER_LOG_ERROR << "Engine failed: " << stat.ToString(); - return SERVER_UNEXPECTED_ERROR; + error_code_ = SERVER_GROUP_NOT_EXIST; + error_msg_ = "Engine failed: " + stat.ToString(); + SERVER_LOG_ERROR << error_msg_; + return error_code_; } else { dimension_ = (int32_t)group_info.dimension; } } catch (std::exception& ex) { - SERVER_LOG_ERROR << ex.what(); + error_code_ = SERVER_UNEXPECTED_ERROR; + error_msg_ = ex.what(); + SERVER_LOG_ERROR << error_msg_; return SERVER_UNEXPECTED_ERROR; } @@ -150,14 +158,10 @@ BaseTaskPtr DeleteGroupTask::Create(const std::string& group_id) { } ServerError DeleteGroupTask::OnExecute() { - try { - - - } catch (std::exception& ex) { - SERVER_LOG_ERROR << ex.what(); - } - - return SERVER_SUCCESS; + error_code_ = SERVER_NOT_IMPLEMENT; + error_msg_ = "delete group not implemented"; + SERVER_LOG_ERROR << error_msg_; + return SERVER_NOT_IMPLEMENT; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -241,8 +245,10 @@ ServerError AddVectorTask::OnExecute() { group_info.group_id = group_id_; engine::Status stat = DB()->get_group(group_info); if(!stat.ok()) { - SERVER_LOG_ERROR << "Engine failed: " << stat.ToString(); - return SERVER_INVALID_ARGUMENT; + error_code_ = SERVER_GROUP_NOT_EXIST; + error_msg_ = "Engine failed: " + stat.ToString(); + SERVER_LOG_ERROR << error_msg_; + return error_code_; } uint64_t group_dim = group_info.dimension; @@ -250,7 +256,10 @@ ServerError AddVectorTask::OnExecute() { if(group_dim != vec_dim) { SERVER_LOG_ERROR << "Invalid vector dimension: " << vec_dim << " vs. group dimension:" << group_dim; - return SERVER_INVALID_ARGUMENT; + error_code_ = SERVER_INVALID_VECTOR_DIMENSION; + error_msg_ = "Engine failed: " + stat.ToString(); + SERVER_LOG_ERROR << error_msg_; + return error_code_; } std::vector vec_f; @@ -263,8 +272,10 @@ ServerError AddVectorTask::OnExecute() { engine::IDNumbers vector_ids; stat = DB()->add_vectors(group_id_, 1, vec_f.data(), vector_ids); if(!stat.ok()) { - SERVER_LOG_ERROR << "Engine failed: " << stat.ToString(); - return SERVER_UNEXPECTED_ERROR; + error_code_ = SERVER_UNEXPECTED_ERROR; + error_msg_ = "Engine failed: " + stat.ToString(); + SERVER_LOG_ERROR << error_msg_; + return error_code_; } else { if(vector_ids.empty()) { SERVER_LOG_ERROR << "Vector ID not returned"; @@ -289,8 +300,10 @@ ServerError AddVectorTask::OnExecute() { } } catch (std::exception& ex) { - SERVER_LOG_ERROR << ex.what(); - return SERVER_UNEXPECTED_ERROR; + error_code_ = SERVER_UNEXPECTED_ERROR; + error_msg_ = ex.what(); + SERVER_LOG_ERROR << error_msg_; + return error_code_; } return SERVER_SUCCESS; @@ -433,8 +446,10 @@ ServerError AddBatchVectorTask::OnExecute() { group_info.group_id = group_id_; engine::Status stat = DB()->get_group(group_info); if(!stat.ok()) { - SERVER_LOG_ERROR << "Engine failed: " << stat.ToString(); - return SERVER_UNEXPECTED_ERROR; + error_code_ = SERVER_GROUP_NOT_EXIST; + error_msg_ = "Engine failed: " + stat.ToString(); + SERVER_LOG_ERROR << error_msg_; + return error_code_; } rc.Record("check group dimension"); @@ -447,7 +462,10 @@ ServerError AddBatchVectorTask::OnExecute() { if(vec_dim != group_dim) { SERVER_LOG_ERROR << "Invalid vector dimension: " << vec_dim << " vs. group dimension:" << group_dim; - return SERVER_INVALID_ARGUMENT; + error_code_ = SERVER_INVALID_VECTOR_DIMENSION; + error_msg_ = "Engine failed: " + stat.ToString(); + SERVER_LOG_ERROR << error_msg_; + return error_code_; } const double* d_p = GetVecData(i); @@ -462,43 +480,48 @@ ServerError AddBatchVectorTask::OnExecute() { stat = DB()->add_vectors(group_id_, vec_count, vec_f.data(), vector_ids); rc.Record("add vectors to engine"); if(!stat.ok()) { - SERVER_LOG_ERROR << "Engine failed: " << stat.ToString(); + error_code_ = SERVER_UNEXPECTED_ERROR; + error_msg_ = "Engine failed: " + stat.ToString(); + SERVER_LOG_ERROR << error_msg_; + return error_code_; + } + + if(vector_ids.size() < vec_count) { + SERVER_LOG_ERROR << "Vector ID not returned"; + return SERVER_UNEXPECTED_ERROR; } else { - if(vector_ids.size() < vec_count) { - SERVER_LOG_ERROR << "Vector ID not returned"; - return SERVER_UNEXPECTED_ERROR; + tensor_ids_.resize(vector_ids.size()); + if(vec_count < USE_MT) { + ProcessIdMapping(vector_ids, 0, vec_count, tensor_ids_); + rc.Record("built id mapping"); } else { - tensor_ids_.resize(vector_ids.size()); - if(vec_count < USE_MT) { - ProcessIdMapping(vector_ids, 0, vec_count, tensor_ids_); - rc.Record("built id mapping"); - } else { - std::list> threads_list; + std::list> threads_list; - uint64_t begin_index = 0, end_index = USE_MT; - while(end_index < vec_count) { - threads_list.push_back( - GetThreadPool().enqueue(&AddBatchVectorTask::ProcessIdMapping, - this, vector_ids, begin_index, end_index, tensor_ids_)); - begin_index = end_index; - end_index += USE_MT; - if(end_index > vec_count) { - end_index = vec_count; - } + uint64_t begin_index = 0, end_index = USE_MT; + while(end_index < vec_count) { + threads_list.push_back( + GetThreadPool().enqueue(&AddBatchVectorTask::ProcessIdMapping, + this, vector_ids, begin_index, end_index, tensor_ids_)); + begin_index = end_index; + end_index += USE_MT; + if(end_index > vec_count) { + end_index = vec_count; } - - for (std::list>::iterator it = threads_list.begin(); it != threads_list.end(); it++) { - it->wait(); - } - - rc.Record("built id mapping by multi-threads:" + std::to_string(threads_list.size())); } + + for (std::list>::iterator it = threads_list.begin(); it != threads_list.end(); it++) { + it->wait(); + } + + rc.Record("built id mapping by multi-threads:" + std::to_string(threads_list.size())); } } } catch (std::exception& ex) { - SERVER_LOG_ERROR << ex.what(); - return SERVER_UNEXPECTED_ERROR; + error_code_ = SERVER_UNEXPECTED_ERROR; + error_msg_ = ex.what(); + SERVER_LOG_ERROR << error_msg_; + return error_code_; } return SERVER_SUCCESS; @@ -612,8 +635,10 @@ ServerError SearchVectorTask::OnExecute() { group_info.group_id = group_id_; engine::Status stat = DB()->get_group(group_info); if(!stat.ok()) { - SERVER_LOG_ERROR << "Engine failed: " << stat.ToString(); - return SERVER_UNEXPECTED_ERROR; + error_code_ = SERVER_GROUP_NOT_EXIST; + error_msg_ = "Engine failed: " + stat.ToString(); + SERVER_LOG_ERROR << error_msg_; + return error_code_; } uint64_t vec_dim = GetTargetDimension(); @@ -674,8 +699,10 @@ ServerError SearchVectorTask::OnExecute() { } } catch (std::exception& ex) { - SERVER_LOG_ERROR << ex.what(); - return SERVER_UNEXPECTED_ERROR; + error_code_ = SERVER_UNEXPECTED_ERROR; + error_msg_ = ex.what(); + SERVER_LOG_ERROR << error_msg_; + return error_code_; } return SERVER_SUCCESS; diff --git a/cpp/src/thrift/cpp_gen.sh b/cpp/src/thrift/cpp_gen.sh index e279f60ac8..df04e46c99 100755 --- a/cpp/src/thrift/cpp_gen.sh +++ b/cpp/src/thrift/cpp_gen.sh @@ -1,4 +1,4 @@ #!/bin/bash -../../third_party/build/bin/thrift -r --gen cpp ./VectorService.thrift +../../third_party/build/bin/thrift -r --gen cpp ./megasearch.thrift diff --git a/cpp/src/thrift/gen-py/py_sample.py b/cpp/src/thrift/gen-py/py_sample.py index 68b759d755..dee18908cc 100644 --- a/cpp/src/thrift/gen-py/py_sample.py +++ b/cpp/src/thrift/gen-py/py_sample.py @@ -82,8 +82,8 @@ def test_vecwise(): transport.close() time_start = print_time_cost('close connection', time_start) - except Thrift.TException as ex: - print(ex.message) + except VecService.VecException as ex: + print(ex.reason) test_vecwise() \ No newline at end of file diff --git a/cpp/src/thrift/VectorService.thrift b/cpp/src/thrift/megasearch.thrift similarity index 100% rename from cpp/src/thrift/VectorService.thrift rename to cpp/src/thrift/megasearch.thrift diff --git a/cpp/src/thrift/py_gen.sh b/cpp/src/thrift/py_gen.sh index adf2eb04b9..de661cf0c3 100755 --- a/cpp/src/thrift/py_gen.sh +++ b/cpp/src/thrift/py_gen.sh @@ -1,4 +1,4 @@ #!/bin/bash -../../third_party/build/bin/thrift -r --gen py ./VectorService.thrift +../../third_party/build/bin/thrift -r --gen py ./megasearch.thrift diff --git a/cpp/src/utils/Error.h b/cpp/src/utils/Error.h index 83da0cea9f..c354ea2ce1 100644 --- a/cpp/src/utils/Error.h +++ b/cpp/src/utils/Error.h @@ -21,7 +21,7 @@ constexpr ServerError SERVER_ERROR_CODE_BASE = 0x30000; constexpr ServerError ToGlobalServerErrorCode(const ServerError error_code) { - return SERVER_ERROR_CODE_BASE + SERVER_ERROR_CODE_BASE; + return SERVER_ERROR_CODE_BASE + error_code; } constexpr ServerError SERVER_UNEXPECTED_ERROR = ToGlobalServerErrorCode(0x001); @@ -31,6 +31,9 @@ constexpr ServerError SERVER_INVALID_ARGUMENT = ToGlobalServerErrorCode(0x004); constexpr ServerError SERVER_FILE_NOT_FOUND = ToGlobalServerErrorCode(0x005); constexpr ServerError SERVER_NOT_IMPLEMENT = ToGlobalServerErrorCode(0x006); constexpr ServerError SERVER_BLOCKING_QUEUE_EMPTY = ToGlobalServerErrorCode(0x007); +constexpr ServerError SERVER_GROUP_NOT_EXIST = ToGlobalServerErrorCode(0x008); +constexpr ServerError SERVER_INVALID_TIME_RANGE = ToGlobalServerErrorCode(0x009); +constexpr ServerError SERVER_INVALID_VECTOR_DIMENSION = ToGlobalServerErrorCode(0x00a); class ServerException : public std::exception { public: diff --git a/cpp/test_client/src/ClientTest.cpp b/cpp/test_client/src/ClientTest.cpp index 007d932196..c08f4d30f1 100644 --- a/cpp/test_client/src/ClientTest.cpp +++ b/cpp/test_client/src/ClientTest.cpp @@ -126,6 +126,23 @@ TEST(AddVector, CLIENT_TEST) { GetServerAddress(address, port, protocol); client::ClientSession session(address, port, protocol); + //verify get invalid group + try { + std::string id; + zilliz::VecTensor tensor; + session.interface()->add_vector(id, GetGroupID(), tensor); + } catch (zilliz::VecException& ex) { + ASSERT_EQ(ex.code, zilliz::VecErrCode::GROUP_NOT_EXISTS); + } + + try { + VecGroup temp_group; + session.interface()->get_group(temp_group, GetGroupID()); + ASSERT_TRUE(temp_group.id.empty()); + } catch (zilliz::VecException& ex) { + ASSERT_EQ(ex.code, zilliz::VecErrCode::GROUP_NOT_EXISTS); + } + //add group VecGroup group; group.id = GetGroupID(); From 70dda6382023cca79b229ad90f82b551d3874f4b Mon Sep 17 00:00:00 2001 From: "xj.lin" Date: Tue, 14 May 2019 10:26:46 +0800 Subject: [PATCH 08/16] fix topk_2 Former-commit-id: 573d80b6d51407cc80e92989c11e3d74f2531d66 --- cpp/src/db/DBImpl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 3df27ba063..216a9b352d 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -162,9 +162,9 @@ Status DBImpl::search(const std::string& group_id, size_t k, size_t nq, for (auto &item : inverted_table){ if (count == k) break; for (auto &id : item.second){ - if (++count == k) break; output_distence[count] = item.first; output_ids[count] = id; + if (++count == k) break; } } }; From 678f1010a1ad2b5703aa8ccf8e710ad329b55fdd Mon Sep 17 00:00:00 2001 From: groot Date: Tue, 14 May 2019 11:16:18 +0800 Subject: [PATCH 09/16] use rocksdb column family Former-commit-id: dbf59149386a71807ff3ea4498240afd102b5001 --- .gitignore | 1 + cpp/src/CMakeLists.txt | 4 +- cpp/src/server/RocksIdMapper.cpp | 242 ++++++++++++++++++ cpp/src/server/RocksIdMapper.h | 46 ++++ cpp/src/server/VecIdMapper.cpp | 144 +---------- cpp/src/server/VecIdMapper.h | 19 +- cpp/src/server/VecServiceHandler.cpp | 47 ++-- cpp/src/server/VecServiceTask.cpp | 66 +++-- cpp/src/server/VecServiceTask.h | 2 +- cpp/src/server/VecServiceWrapper.cpp | 4 +- cpp/src/thrift/gen-cpp/VecService.h | 2 +- .../thrift/gen-cpp/VectorService_constants.h | 24 -- ...constants.cpp => megasearch_constants.cpp} | 6 +- cpp/src/thrift/gen-cpp/megasearch_constants.h | 24 ++ ...Service_types.cpp => megasearch_types.cpp} | 2 +- ...ctorService_types.h => megasearch_types.h} | 4 +- cpp/src/thrift/gen-py/zilliz/ttypes.py | 12 +- cpp/src/thrift/megasearch.thrift | 12 +- cpp/test_client/CMakeLists.txt | 4 +- cpp/test_client/src/ClientSession.cpp | 4 +- cpp/test_client/src/ClientTest.cpp | 77 ++++-- cpp/unittest/faiss_wrapper/CMakeLists.txt | 1 + cpp/unittest/server/CMakeLists.txt | 1 + cpp/unittest/server/common_test.cpp | 9 +- cpp/unittest/server/idmapper_test.cpp | 59 ++++- 25 files changed, 504 insertions(+), 312 deletions(-) create mode 100644 cpp/src/server/RocksIdMapper.cpp create mode 100644 cpp/src/server/RocksIdMapper.h delete mode 100644 cpp/src/thrift/gen-cpp/VectorService_constants.h rename cpp/src/thrift/gen-cpp/{VectorService_constants.cpp => megasearch_constants.cpp} (55%) create mode 100644 cpp/src/thrift/gen-cpp/megasearch_constants.h rename cpp/src/thrift/gen-cpp/{VectorService_types.cpp => megasearch_types.cpp} (99%) rename cpp/src/thrift/gen-cpp/{VectorService_types.h => megasearch_types.h} (99%) diff --git a/.gitignore b/.gitignore index 87503b8b61..d239dd72a6 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ cmake_build cpp/third_party/thrift-0.12.0/ cpp/third_party/faiss-1.5.1 +cpp/megasearch/ diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index fff8fd9e69..24d382efd8 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -17,8 +17,8 @@ set(license_check_files set(service_files thrift/gen-cpp/VecService.cpp - thrift/gen-cpp/VectorService_constants.cpp - thrift/gen-cpp/VectorService_types.cpp + thrift/gen-cpp/megasearch_constants.cpp + thrift/gen-cpp/megasearch_types.cpp ) set(vecwise_engine_src diff --git a/cpp/src/server/RocksIdMapper.cpp b/cpp/src/server/RocksIdMapper.cpp new file mode 100644 index 0000000000..2dba544243 --- /dev/null +++ b/cpp/src/server/RocksIdMapper.cpp @@ -0,0 +1,242 @@ +/******************************************************************************* + * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved + * Unauthorized copying of this file, via any medium is strictly prohibited. + * Proprietary and confidential. + ******************************************************************************/ + +#include "RocksIdMapper.h" +#include "ServerConfig.h" +#include "utils/Log.h" +#include "utils/CommonUtil.h" + +#include "rocksdb/slice.h" +#include "rocksdb/options.h" + +#include + +namespace zilliz { +namespace vecwise { +namespace server { + +RocksIdMapper::RocksIdMapper() +: db_(nullptr) { + OpenDb(); +} + +RocksIdMapper::~RocksIdMapper() { + CloseDb(); +} + +void RocksIdMapper::OpenDb() { + if(db_) { + return; + } + + ConfigNode& config = ServerConfig::GetInstance().GetConfig(CONFIG_DB); + std::string db_path = config.GetValue(CONFIG_DB_PATH); + db_path += "/id_mapping"; + CommonUtil::CreateDirectory(db_path); + + rocksdb::Options options; + // Optimize RocksDB. This is the easiest way to get RocksDB to perform well + options.IncreaseParallelism(); + options.OptimizeLevelStyleCompaction(); + // create the DB if it's not already present + options.create_if_missing = true; + options.max_open_files = config.GetInt32Value(CONFIG_DB_IDMAPPER_MAX_FILE, 512); + + //load column families + std::vector column_names; + rocksdb::Status s = rocksdb::DB::ListColumnFamilies(options, db_path, &column_names); + if (!s.ok()) { + SERVER_LOG_ERROR << "ID mapper failed to initialize:" << s.ToString(); + } + + if(column_names.empty()) { + column_names.push_back("default"); + } + SERVER_LOG_INFO << "ID mapper has " << std::to_string(column_names.size()) << " groups"; + + std::vector column_families; + for(auto& column_name : column_names) { + rocksdb::ColumnFamilyDescriptor desc; + desc.name = column_name; + column_families.emplace_back(desc); + } + + // open DB + std::vector column_handles; + s = rocksdb::DB::Open(options, db_path, column_families, &column_handles, &db_); + if(!s.ok()) { + SERVER_LOG_ERROR << "ID mapper failed to initialize:" << s.ToString(); + db_ = nullptr; + } + + column_handles_.clear(); + for(auto handler : column_handles) { + column_handles_.insert(std::make_pair(handler->GetName(), handler)); + } +} + +void RocksIdMapper::CloseDb() { + for(auto& iter : column_handles_) { + delete iter.second; + } + column_handles_.clear(); + + if(db_) { + db_->Close(); + delete db_; + } +} + +ServerError RocksIdMapper::Put(const std::string& nid, const std::string& sid, const std::string& group) { + if(db_ == nullptr) { + return SERVER_NULL_POINTER; + } + + rocksdb::Slice key(nid); + rocksdb::Slice value(sid); + if(group.empty()) {//to default group + rocksdb::Status s = db_->Put(rocksdb::WriteOptions(), key, value); + if (!s.ok()) { + SERVER_LOG_ERROR << "ID mapper failed to put:" << s.ToString(); + return SERVER_UNEXPECTED_ERROR; + } + } else { + rocksdb::ColumnFamilyHandle *cfh = nullptr; + if(column_handles_.count(group) == 0) { + try {//add group + rocksdb::Status s = db_->CreateColumnFamily(rocksdb::ColumnFamilyOptions(), group, &cfh); + if (!s.ok()) { + SERVER_LOG_ERROR << "ID mapper failed to create group:" << s.ToString(); + } else { + column_handles_.insert(std::make_pair(group, cfh)); + } + } catch(std::exception& ex) { + std::cout << ex.what() << std::endl; + } + } else { + cfh = column_handles_[group]; + } + + rocksdb::Status s = db_->Put(rocksdb::WriteOptions(), cfh, key, value); + if (!s.ok()) { + SERVER_LOG_ERROR << "ID mapper failed to put:" << s.ToString(); + return SERVER_UNEXPECTED_ERROR; + } + } + + return SERVER_SUCCESS; +} + +ServerError RocksIdMapper::Put(const std::vector& nid, const std::vector& sid, const std::string& group) { + if(nid.size() != sid.size()) { + return SERVER_INVALID_ARGUMENT; + } + + ServerError err = SERVER_SUCCESS; + for(size_t i = 0; i < nid.size(); i++) { + err = Put(nid[i], sid[i], group); + if(err != SERVER_SUCCESS) { + return err; + } + } + + return err; +} + +ServerError RocksIdMapper::Get(const std::string& nid, std::string& sid, const std::string& group) const { + sid = ""; + if(db_ == nullptr) { + return SERVER_NULL_POINTER; + } + + rocksdb::ColumnFamilyHandle *cfh = nullptr; + if(column_handles_.count(group) != 0) { + cfh = column_handles_.at(group); + } + + rocksdb::Slice key(nid); + rocksdb::Status s; + if(cfh){ + s = db_->Get(rocksdb::ReadOptions(), cfh, key, &sid); + } else { + s = db_->Get(rocksdb::ReadOptions(), key, &sid); + } + + if(!s.ok()) { + SERVER_LOG_ERROR << "ID mapper failed to get:" << s.ToString(); + return SERVER_UNEXPECTED_ERROR; + } + + return SERVER_SUCCESS; +} + +ServerError RocksIdMapper::Get(const std::vector& nid, std::vector& sid, const std::string& group) const { + sid.clear(); + + ServerError err = SERVER_SUCCESS; + for(size_t i = 0; i < nid.size(); i++) { + std::string str_id; + ServerError temp_err = Get(nid[i], str_id, group); + if(temp_err != SERVER_SUCCESS) { + sid.push_back(""); + SERVER_LOG_ERROR << "ID mapper failed to get id: " << nid[i]; + err = temp_err; + continue; + } + + sid.push_back(str_id); + } + + return err; +} + +ServerError RocksIdMapper::Delete(const std::string& nid, const std::string& group) { + if(db_ == nullptr) { + return SERVER_NULL_POINTER; + } + + rocksdb::ColumnFamilyHandle *cfh = nullptr; + if(column_handles_.count(group) != 0) { + cfh = column_handles_.at(group); + } + + rocksdb::Slice key(nid); + rocksdb::Status s; + if(cfh){ + s = db_->Delete(rocksdb::WriteOptions(), cfh, key); + } else { + s = db_->Delete(rocksdb::WriteOptions(), key); + } + if(!s.ok()) { + SERVER_LOG_ERROR << "ID mapper failed to delete:" << s.ToString(); + return SERVER_UNEXPECTED_ERROR; + } + + return SERVER_SUCCESS; +} + +ServerError RocksIdMapper::DeleteGroup(const std::string& group) { + if(db_ == nullptr) { + return SERVER_NULL_POINTER; + } + + rocksdb::ColumnFamilyHandle *cfh = nullptr; + if(column_handles_.count(group) != 0) { + cfh = column_handles_.at(group); + } + + if(cfh) { + db_->DropColumnFamily(cfh); + db_->DestroyColumnFamilyHandle(cfh); + column_handles_.erase(group); + } + + return SERVER_SUCCESS; +} + +} +} +} \ No newline at end of file diff --git a/cpp/src/server/RocksIdMapper.h b/cpp/src/server/RocksIdMapper.h new file mode 100644 index 0000000000..8c73155903 --- /dev/null +++ b/cpp/src/server/RocksIdMapper.h @@ -0,0 +1,46 @@ +/******************************************************************************* + * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved + * Unauthorized copying of this file, via any medium is strictly prohibited. + * Proprietary and confidential. + ******************************************************************************/ +#pragma once + +#include "utils/Error.h" +#include "VecIdMapper.h" + +#include "rocksdb/db.h" + +#include +#include +#include + +namespace zilliz { +namespace vecwise { +namespace server { + +class RocksIdMapper : public IVecIdMapper{ +public: + RocksIdMapper(); + ~RocksIdMapper(); + + ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") override; + ServerError Put(const std::vector& nid, const std::vector& sid, const std::string& group = "") override; + + ServerError Get(const std::string& nid, std::string& sid, const std::string& group = "") const override; + ServerError Get(const std::vector& nid, std::vector& sid, const std::string& group = "") const override; + + ServerError Delete(const std::string& nid, const std::string& group = "") override; + ServerError DeleteGroup(const std::string& group) override; + +private: + void OpenDb(); + void CloseDb(); + +private: + rocksdb::DB* db_; + std::unordered_map column_handles_; +}; + +} +} +} diff --git a/cpp/src/server/VecIdMapper.cpp b/cpp/src/server/VecIdMapper.cpp index b0dfcc6db2..ecf5058a4b 100644 --- a/cpp/src/server/VecIdMapper.cpp +++ b/cpp/src/server/VecIdMapper.cpp @@ -5,6 +5,7 @@ ******************************************************************************/ #include "VecIdMapper.h" +#include "RocksIdMapper.h" #include "ServerConfig.h" #include "utils/Log.h" #include "utils/CommonUtil.h" @@ -102,146 +103,9 @@ ServerError SimpleIdMapper::Delete(const std::string& nid, const std::string& gr return SERVER_SUCCESS; } - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -RocksIdMapper::RocksIdMapper() { - ConfigNode& config = ServerConfig::GetInstance().GetConfig(CONFIG_DB); - std::string db_path = config.GetValue(CONFIG_DB_PATH); - db_path += "/id_mapping"; - CommonUtil::CreateDirectory(db_path); - - rocksdb::Options options; - // Optimize RocksDB. This is the easiest way to get RocksDB to perform well - options.IncreaseParallelism(); - options.OptimizeLevelStyleCompaction(); - // create the DB if it's not already present - options.create_if_missing = true; - options.max_open_files = config.GetInt32Value(CONFIG_DB_IDMAPPER_MAX_FILE, 128); - - //load column families - std::vector column_names; - rocksdb::Status s = rocksdb::DB::ListColumnFamilies(options, db_path, &column_names); - if (!s.ok()) { - SERVER_LOG_ERROR << "ID mapper failed to initialize:" << s.ToString(); - } - - std::vector column_families; - for(auto& column_name : column_names) { - rocksdb::ColumnFamilyDescriptor desc; - desc.name = column_name; - column_families.emplace_back(desc); - } - - // open DB - std::vector column_handles; - s = rocksdb::DB::Open(options, db_path, column_families, &column_handles, &db_); - if(!s.ok()) { - SERVER_LOG_ERROR << "ID mapper failed to initialize:" << s.ToString(); - db_ = nullptr; - } - - try { - rocksdb::ColumnFamilyHandle *cf; - s = db_->CreateColumnFamily(rocksdb::ColumnFamilyOptions(), "222", &cf); - if (!s.ok()) { - SERVER_LOG_ERROR << "ID mapper failed to initialize:" << s.ToString(); - } - - std::vector column_families; - s = db_->ListColumnFamilies(options, db_path, &column_families); - if (!s.ok()) { - SERVER_LOG_ERROR << "ID mapper failed to initialize:" << s.ToString(); - } - } catch(std::exception& ex) { - std::cout << ex.what() << std::endl; - } -} -RocksIdMapper::~RocksIdMapper() { - if(db_) { - db_->Close(); - delete db_; - } -} - -ServerError RocksIdMapper::Put(const std::string& nid, const std::string& sid, const std::string& group) { - if(db_ == nullptr) { - return SERVER_NULL_POINTER; - } - - rocksdb::Slice key(nid); - rocksdb::Slice value(sid); - rocksdb::Status s = db_->Put(rocksdb::WriteOptions(), key, value); - if(!s.ok()) { - SERVER_LOG_ERROR << "ID mapper failed to put:" << s.ToString(); - return SERVER_UNEXPECTED_ERROR; - } - - return SERVER_SUCCESS; -} - -ServerError RocksIdMapper::Put(const std::vector& nid, const std::vector& sid, const std::string& group) { - if(nid.size() != sid.size()) { - return SERVER_INVALID_ARGUMENT; - } - - ServerError err = SERVER_SUCCESS; - for(size_t i = 0; i < nid.size(); i++) { - err = Put(nid[i], sid[i]); - if(err != SERVER_SUCCESS) { - return err; - } - } - - return err; -} - -ServerError RocksIdMapper::Get(const std::string& nid, std::string& sid, const std::string& group) const { - if(db_ == nullptr) { - return SERVER_NULL_POINTER; - } - - rocksdb::Slice key(nid); - rocksdb::Status s = db_->Get(rocksdb::ReadOptions(), key, &sid); - if(!s.ok()) { - SERVER_LOG_ERROR << "ID mapper failed to get:" << s.ToString(); - return SERVER_UNEXPECTED_ERROR; - } - - return SERVER_SUCCESS; -} - -ServerError RocksIdMapper::Get(const std::vector& nid, std::vector& sid, const std::string& group) const { - sid.clear(); - - ServerError err = SERVER_SUCCESS; - for(size_t i = 0; i < nid.size(); i++) { - std::string str_id; - ServerError temp_err = Get(nid[i], str_id); - if(temp_err != SERVER_SUCCESS) { - sid.push_back(""); - SERVER_LOG_ERROR << "ID mapper failed to get id: " << nid[i]; - err = temp_err; - continue; - } - - sid.push_back(str_id); - } - - return err; -} - -ServerError RocksIdMapper::Delete(const std::string& nid, const std::string& group) { - if(db_ == nullptr) { - return SERVER_NULL_POINTER; - } - - rocksdb::Slice key(nid); - rocksdb::Status s = db_->Delete(rocksdb::WriteOptions(), key); - if(!s.ok()) { - SERVER_LOG_ERROR << "ID mapper failed to delete:" << s.ToString(); - return SERVER_UNEXPECTED_ERROR; - } - +//not thread-safe +ServerError SimpleIdMapper::DeleteGroup(const std::string& group) { + id_groups_.erase(group); return SERVER_SUCCESS; } diff --git a/cpp/src/server/VecIdMapper.h b/cpp/src/server/VecIdMapper.h index 216218b51e..9bb6d500da 100644 --- a/cpp/src/server/VecIdMapper.h +++ b/cpp/src/server/VecIdMapper.h @@ -33,6 +33,7 @@ public: virtual ServerError Get(const std::vector& nid, std::vector& sid, const std::string& group = "") const = 0; virtual ServerError Delete(const std::string& nid, const std::string& group = "") = 0; + virtual ServerError DeleteGroup(const std::string& group) = 0; }; class SimpleIdMapper : public IVecIdMapper{ @@ -47,29 +48,13 @@ public: ServerError Get(const std::vector& nid, std::vector& sid, const std::string& group = "") const override; ServerError Delete(const std::string& nid, const std::string& group = "") override; + ServerError DeleteGroup(const std::string& group) override; private: using ID_MAPPING = std::unordered_map; mutable std::unordered_map id_groups_; }; -class RocksIdMapper : public IVecIdMapper{ -public: - RocksIdMapper(); - ~RocksIdMapper(); - - ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") override; - ServerError Put(const std::vector& nid, const std::vector& sid, const std::string& group = "") override; - - ServerError Get(const std::string& nid, std::string& sid, const std::string& group = "") const override; - ServerError Get(const std::vector& nid, std::vector& sid, const std::string& group = "") const override; - - ServerError Delete(const std::string& nid, const std::string& group = "") override; - -private: - rocksdb::DB* db_; -}; - } } } diff --git a/cpp/src/server/VecServiceHandler.cpp b/cpp/src/server/VecServiceHandler.cpp index 843b37a8d5..da8a2aa1e8 100644 --- a/cpp/src/server/VecServiceHandler.cpp +++ b/cpp/src/server/VecServiceHandler.cpp @@ -6,7 +6,6 @@ #include "VecServiceHandler.h" #include "VecServiceTask.h" #include "ServerConfig.h" -#include "VecIdMapper.h" #include "utils/Log.h" #include "utils/CommonUtil.h" #include "utils/TimeRecorder.h" @@ -23,12 +22,12 @@ namespace { public: TimeRecordWrapper(const std::string& func_name) : recorder_(func_name), func_name_(func_name) { - SERVER_LOG_TRACE << func_name << " called"; + //SERVER_LOG_TRACE << func_name << " called"; } ~TimeRecordWrapper() { recorder_.Elapse("cost"); - SERVER_LOG_TRACE << func_name_ << " finished"; + //SERVER_LOG_TRACE << func_name_ << " finished"; } private: @@ -98,9 +97,9 @@ namespace { void VecServiceHandler::add_group(const VecGroup &group) { - TimeRecordWrapper rc("add_group()"); - SERVER_LOG_TRACE << "group.id = " << group.id << ", group.dimension = " << group.dimension - << ", group.index_type = " << group.index_type; + std::string info = "add_group() " + group.id + " dimension = " + std::to_string(group.dimension) + + " index_type = " + std::to_string(group.index_type); + TimeRecordWrapper rc(info); BaseTaskPtr task_ptr = AddGroupTask::Create(group.dimension, group.id); ExecTask(task_ptr); @@ -108,8 +107,7 @@ VecServiceHandler::add_group(const VecGroup &group) { void VecServiceHandler::get_group(VecGroup &_return, const std::string &group_id) { - TimeRecordWrapper rc("get_group()"); - SERVER_LOG_TRACE << "group_id = " << group_id; + TimeRecordWrapper rc("get_group() " + group_id); _return.id = group_id; BaseTaskPtr task_ptr = GetGroupTask::Create(group_id, _return.dimension); @@ -118,8 +116,7 @@ VecServiceHandler::get_group(VecGroup &_return, const std::string &group_id) { void VecServiceHandler::del_group(const std::string &group_id) { - TimeRecordWrapper rc("del_group()"); - SERVER_LOG_TRACE << "group_id = " << group_id; + TimeRecordWrapper rc("del_group() " + group_id); BaseTaskPtr task_ptr = DeleteGroupTask::Create(group_id); ExecTask(task_ptr); @@ -128,8 +125,7 @@ VecServiceHandler::del_group(const std::string &group_id) { void VecServiceHandler::add_vector(std::string& _return, const std::string &group_id, const VecTensor &tensor) { - TimeRecordWrapper rc("add_vector()"); - SERVER_LOG_TRACE << "group_id = " << group_id << ", vector size = " << tensor.tensor.size(); + TimeRecordWrapper rc("add_vector() to " + group_id); BaseTaskPtr task_ptr = AddVectorTask::Create(group_id, &tensor, _return); ExecTask(task_ptr); @@ -139,9 +135,7 @@ void VecServiceHandler::add_vector_batch(std::vector & _return, const std::string &group_id, const VecTensorList &tensor_list) { - TimeRecordWrapper rc("add_vector_batch()"); - SERVER_LOG_TRACE << "group_id = " << group_id << ", vector list size = " - << tensor_list.tensor_list.size(); + TimeRecordWrapper rc("add_vector_batch() to " + group_id); BaseTaskPtr task_ptr = AddBatchVectorTask::Create(group_id, &tensor_list, _return); ExecTask(task_ptr); @@ -151,8 +145,7 @@ void VecServiceHandler::add_binary_vector(std::string& _return, const std::string& group_id, const VecBinaryTensor& tensor) { - TimeRecordWrapper rc("add_binary_vector()"); - SERVER_LOG_TRACE << "group_id = " << group_id << ", vector size = " << tensor.tensor.size()/4; + TimeRecordWrapper rc("add_binary_vector() to " + group_id); BaseTaskPtr task_ptr = AddVectorTask::Create(group_id, &tensor, _return); ExecTask(task_ptr); @@ -162,9 +155,7 @@ void VecServiceHandler::add_binary_vector_batch(std::vector & _return, const std::string& group_id, const VecBinaryTensorList& tensor_list) { - TimeRecordWrapper rc("add_binary_vector_batch()"); - SERVER_LOG_TRACE << "group_id = " << group_id << ", vector list size = " - << tensor_list.tensor_list.size(); + TimeRecordWrapper rc("add_binary_vector_batch() to " + group_id); BaseTaskPtr task_ptr = AddBatchVectorTask::Create(group_id, &tensor_list, _return); ExecTask(task_ptr); @@ -176,9 +167,7 @@ VecServiceHandler::search_vector(VecSearchResult &_return, const int64_t top_k, const VecTensor &tensor, const VecSearchFilter& filter) { - TimeRecordWrapper rc("search_vector()"); - SERVER_LOG_TRACE << "group_id = " << group_id << ", top_k = " << top_k - << ", vector dimension = " << tensor.tensor.size(); + TimeRecordWrapper rc("search_vector() in " + group_id); VecTensorList tensor_list; tensor_list.tensor_list.push_back(tensor); @@ -199,9 +188,7 @@ VecServiceHandler::search_vector_batch(VecSearchResultList &_return, const int64_t top_k, const VecTensorList &tensor_list, const VecSearchFilter& filter) { - TimeRecordWrapper rc("search_vector_batch()"); - SERVER_LOG_TRACE << "group_id = " << group_id << ", top_k = " << top_k - << ", vector list size = " << tensor_list.tensor_list.size(); + TimeRecordWrapper rc("search_vector_batch() in " + group_id); BaseTaskPtr task_ptr = SearchVectorTask::Create(group_id, top_k, &tensor_list, filter, _return); ExecTask(task_ptr); @@ -213,9 +200,7 @@ VecServiceHandler::search_binary_vector(VecSearchResult& _return, const int64_t top_k, const VecBinaryTensor& tensor, const VecSearchFilter& filter) { - TimeRecordWrapper rc("search_binary_vector()"); - SERVER_LOG_TRACE << "group_id = " << group_id << ", top_k = " << top_k - << ", vector dimension = " << tensor.tensor.size(); + TimeRecordWrapper rc("search_binary_vector() in " + group_id); VecBinaryTensorList tensor_list; tensor_list.tensor_list.push_back(tensor); @@ -236,9 +221,7 @@ VecServiceHandler::search_binary_vector_batch(VecSearchResultList& _return, const int64_t top_k, const VecBinaryTensorList& tensor_list, const VecSearchFilter& filter) { - TimeRecordWrapper rc("search_binary_vector_batch()"); - SERVER_LOG_TRACE << "group_id = " << group_id << ", top_k = " << top_k - << ", vector list size = " << tensor_list.tensor_list.size(); + TimeRecordWrapper rc("search_binary_vector_batch() in " + group_id); BaseTaskPtr task_ptr = SearchVectorTask::Create(group_id, top_k, &tensor_list, filter, _return); ExecTask(task_ptr); diff --git a/cpp/src/server/VecServiceTask.cpp b/cpp/src/server/VecServiceTask.cpp index 7afdb93f5d..9ebe2c1bdf 100644 --- a/cpp/src/server/VecServiceTask.cpp +++ b/cpp/src/server/VecServiceTask.cpp @@ -91,11 +91,10 @@ ServerError AddGroupTask::OnExecute() { group_info.dimension = (size_t)dimension_; group_info.group_id = group_id_; engine::Status stat = DB()->add_group(group_info); - if(!stat.ok()) { - error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = "Engine failed: " + stat.ToString(); + if(!stat.ok()) {//could exist + SERVER_LOG_ERROR << "Engine failed: " << stat.ToString(); SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SERVER_SUCCESS; } } catch (std::exception& ex) { @@ -161,6 +160,9 @@ ServerError DeleteGroupTask::OnExecute() { error_code_ = SERVER_NOT_IMPLEMENT; error_msg_ = "delete group not implemented"; SERVER_LOG_ERROR << error_msg_; + + //IVecIdMapper::GetInstance()->DeleteGroup(group_id_); + return SERVER_NOT_IMPLEMENT; } @@ -241,27 +243,7 @@ const AttribMap& AddVectorTask::GetVecAttrib() const { ServerError AddVectorTask::OnExecute() { try { - engine::meta::GroupSchema group_info; - group_info.group_id = group_id_; - engine::Status stat = DB()->get_group(group_info); - if(!stat.ok()) { - error_code_ = SERVER_GROUP_NOT_EXIST; - error_msg_ = "Engine failed: " + stat.ToString(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; - } - - uint64_t group_dim = group_info.dimension; uint64_t vec_dim = GetVecDimension(); - if(group_dim != vec_dim) { - SERVER_LOG_ERROR << "Invalid vector dimension: " << vec_dim - << " vs. group dimension:" << group_dim; - error_code_ = SERVER_INVALID_VECTOR_DIMENSION; - error_msg_ = "Engine failed: " + stat.ToString(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; - } - std::vector vec_f; vec_f.resize(vec_dim); const double* d_p = GetVecData(); @@ -270,7 +252,7 @@ ServerError AddVectorTask::OnExecute() { } engine::IDNumbers vector_ids; - stat = DB()->add_vectors(group_id_, 1, vec_f.data(), vector_ids); + engine::Status stat = DB()->add_vectors(group_id_, 1, vec_f.data(), vector_ids); if(!stat.ok()) { error_code_ = SERVER_UNEXPECTED_ERROR; error_msg_ = "Engine failed: " + stat.ToString(); @@ -278,7 +260,8 @@ ServerError AddVectorTask::OnExecute() { return error_code_; } else { if(vector_ids.empty()) { - SERVER_LOG_ERROR << "Vector ID not returned"; + error_msg_ = "Engine failed: " + stat.ToString(); + SERVER_LOG_ERROR << error_msg_; return SERVER_UNEXPECTED_ERROR; } else { std::string uid = GetVecID(); @@ -294,8 +277,8 @@ ServerError AddVectorTask::OnExecute() { attrib[VECTOR_UID] = tensor_id_; std::string attrib_str; AttributeSerializer::Encode(attrib, attrib_str); - IVecIdMapper::GetInstance()->Put(nid, attrib_str); - SERVER_LOG_TRACE << "nid = " << vector_ids[0] << ", uid = " << uid; + IVecIdMapper::GetInstance()->Put(nid, attrib_str, group_id_); + //SERVER_LOG_TRACE << "nid = " << vector_ids[0] << ", uid = " << uid; } } @@ -429,7 +412,7 @@ void AddBatchVectorTask::ProcessIdMapping(engine::IDNumbers& vector_ids, attrib[VECTOR_UID] = uid; std::string attrib_str; AttributeSerializer::Encode(attrib, attrib_str); - IVecIdMapper::GetInstance()->Put(nid, attrib_str); + IVecIdMapper::GetInstance()->Put(nid, attrib_str, group_id_); } } @@ -464,7 +447,6 @@ ServerError AddBatchVectorTask::OnExecute() { << " vs. group dimension:" << group_dim; error_code_ = SERVER_INVALID_VECTOR_DIMENSION; error_msg_ = "Engine failed: " + stat.ToString(); - SERVER_LOG_ERROR << error_msg_; return error_code_; } @@ -645,7 +627,9 @@ ServerError SearchVectorTask::OnExecute() { if(vec_dim != group_info.dimension) { SERVER_LOG_ERROR << "Invalid vector dimension: " << vec_dim << " vs. group dimension:" << group_info.dimension; - return SERVER_INVALID_ARGUMENT; + error_code_ = SERVER_INVALID_VECTOR_DIMENSION; + error_msg_ = "Engine failed: " + stat.ToString(); + return error_code_; } rc.Record("check group dimension"); @@ -679,18 +663,30 @@ ServerError SearchVectorTask::OnExecute() { for(auto id : res) { std::string attrib_str; std::string nid = nid_prefix + std::to_string(id); - IVecIdMapper::GetInstance()->Get(nid, attrib_str); + IVecIdMapper::GetInstance()->Get(nid, attrib_str, group_id_); AttribMap attrib_map; AttributeSerializer::Decode(attrib_str, attrib_map); + AttribMap attrib_return; VecSearchResultItem item; - item.__set_attrib(attrib_map); - item.uid = item.attrib[VECTOR_UID]; + item.uid = attrib_map[VECTOR_UID]; + + if(filter_.return_attribs.empty()) {//return all attributes + attrib_return.swap(attrib_map); + } else {//filter attributes + for(auto& name : filter_.return_attribs) { + if(attrib_map.count(name) == 0) + continue; + + attrib_return[name] = attrib_map[name]; + } + } + item.__set_attrib(attrib_return); item.distance = 0.0;////TODO: return distance v_res.result_list.emplace_back(item); - SERVER_LOG_TRACE << "nid = " << nid << ", uid = " << item.uid; + //SERVER_LOG_TRACE << "nid = " << nid << ", uid = " << item.uid; } result_.result_list.push_back(v_res); diff --git a/cpp/src/server/VecServiceTask.h b/cpp/src/server/VecServiceTask.h index f9fe2a2158..a0c317ccb2 100644 --- a/cpp/src/server/VecServiceTask.h +++ b/cpp/src/server/VecServiceTask.h @@ -10,7 +10,7 @@ #include "utils/AttributeSerializer.h" #include "db/Types.h" -#include "thrift/gen-cpp/VectorService_types.h" +#include "thrift/gen-cpp/megasearch_types.h" #include #include diff --git a/cpp/src/server/VecServiceWrapper.cpp b/cpp/src/server/VecServiceWrapper.cpp index 376f5ef3cc..6e9fd8b3a4 100644 --- a/cpp/src/server/VecServiceWrapper.cpp +++ b/cpp/src/server/VecServiceWrapper.cpp @@ -10,8 +10,8 @@ #include "utils/Log.h" -#include "thrift/gen-cpp/VectorService_types.h" -#include "thrift/gen-cpp/VectorService_constants.h" +#include "thrift/gen-cpp/megasearch_types.h" +#include "thrift/gen-cpp/megasearch_constants.h" #include #include diff --git a/cpp/src/thrift/gen-cpp/VecService.h b/cpp/src/thrift/gen-cpp/VecService.h index 8b56738b20..622196dd41 100644 --- a/cpp/src/thrift/gen-cpp/VecService.h +++ b/cpp/src/thrift/gen-cpp/VecService.h @@ -9,7 +9,7 @@ #include #include -#include "VectorService_types.h" +#include "megasearch_types.h" namespace zilliz { diff --git a/cpp/src/thrift/gen-cpp/VectorService_constants.h b/cpp/src/thrift/gen-cpp/VectorService_constants.h deleted file mode 100644 index 3a8e16b92d..0000000000 --- a/cpp/src/thrift/gen-cpp/VectorService_constants.h +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Autogenerated by Thrift Compiler (0.11.0) - * - * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING - * @generated - */ -#ifndef VectorService_CONSTANTS_H -#define VectorService_CONSTANTS_H - -#include "VectorService_types.h" - -namespace zilliz { - -class VectorServiceConstants { - public: - VectorServiceConstants(); - -}; - -extern const VectorServiceConstants g_VectorService_constants; - -} // namespace - -#endif diff --git a/cpp/src/thrift/gen-cpp/VectorService_constants.cpp b/cpp/src/thrift/gen-cpp/megasearch_constants.cpp similarity index 55% rename from cpp/src/thrift/gen-cpp/VectorService_constants.cpp rename to cpp/src/thrift/gen-cpp/megasearch_constants.cpp index 27ea10ef32..798fbf9e5d 100644 --- a/cpp/src/thrift/gen-cpp/VectorService_constants.cpp +++ b/cpp/src/thrift/gen-cpp/megasearch_constants.cpp @@ -4,13 +4,13 @@ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING * @generated */ -#include "VectorService_constants.h" +#include "megasearch_constants.h" namespace zilliz { -const VectorServiceConstants g_VectorService_constants; +const megasearchConstants g_megasearch_constants; -VectorServiceConstants::VectorServiceConstants() { +megasearchConstants::megasearchConstants() { } } // namespace diff --git a/cpp/src/thrift/gen-cpp/megasearch_constants.h b/cpp/src/thrift/gen-cpp/megasearch_constants.h new file mode 100644 index 0000000000..397b71e9ca --- /dev/null +++ b/cpp/src/thrift/gen-cpp/megasearch_constants.h @@ -0,0 +1,24 @@ +/** + * Autogenerated by Thrift Compiler (0.11.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +#ifndef megasearch_CONSTANTS_H +#define megasearch_CONSTANTS_H + +#include "megasearch_types.h" + +namespace zilliz { + +class megasearchConstants { + public: + megasearchConstants(); + +}; + +extern const megasearchConstants g_megasearch_constants; + +} // namespace + +#endif diff --git a/cpp/src/thrift/gen-cpp/VectorService_types.cpp b/cpp/src/thrift/gen-cpp/megasearch_types.cpp similarity index 99% rename from cpp/src/thrift/gen-cpp/VectorService_types.cpp rename to cpp/src/thrift/gen-cpp/megasearch_types.cpp index 5fefd60fc5..ecc016504e 100644 --- a/cpp/src/thrift/gen-cpp/VectorService_types.cpp +++ b/cpp/src/thrift/gen-cpp/megasearch_types.cpp @@ -4,7 +4,7 @@ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING * @generated */ -#include "VectorService_types.h" +#include "megasearch_types.h" #include #include diff --git a/cpp/src/thrift/gen-cpp/VectorService_types.h b/cpp/src/thrift/gen-cpp/megasearch_types.h similarity index 99% rename from cpp/src/thrift/gen-cpp/VectorService_types.h rename to cpp/src/thrift/gen-cpp/megasearch_types.h index d9285efd80..e6a68eecb8 100644 --- a/cpp/src/thrift/gen-cpp/VectorService_types.h +++ b/cpp/src/thrift/gen-cpp/megasearch_types.h @@ -4,8 +4,8 @@ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING * @generated */ -#ifndef VectorService_TYPES_H -#define VectorService_TYPES_H +#ifndef megasearch_TYPES_H +#define megasearch_TYPES_H #include diff --git a/cpp/src/thrift/gen-py/zilliz/ttypes.py b/cpp/src/thrift/gen-py/zilliz/ttypes.py index 74addabdc3..bfe014667d 100644 --- a/cpp/src/thrift/gen-py/zilliz/ttypes.py +++ b/cpp/src/thrift/gen-py/zilliz/ttypes.py @@ -743,9 +743,9 @@ class VecSearchResultList(object): class VecDateTime(object): """ - second; Seconds. [0-60] (1 leap second) - minute; Minutes. [0-59] - hour; Hours. [0-23] + second; Seconds. [0-59] reserved + minute; Minutes. [0-59] reserved + hour; Hours. [0-23] reserved day; Day. [1-31] month; Month. [0-11] year; Year - 1900. @@ -874,9 +874,9 @@ class VecDateTime(object): class VecTimeRange(object): """ time_begin; time range begin - begine_closed; true means '[', false means '(' + begine_closed; true means '[', false means '(' reserved time_end; set to true to return tensor double array - end_closed; time range end + end_closed; time range end reserved Attributes: - time_begin @@ -977,7 +977,7 @@ class VecTimeRange(object): class VecSearchFilter(object): """ - attrib_filter; search condition, for example: "color=red" + attrib_filter; reserved time_ranges; search condition, for example: "date between 1999-02-12 and 2008-10-14" return_attribs; specify required attribute names diff --git a/cpp/src/thrift/megasearch.thrift b/cpp/src/thrift/megasearch.thrift index e14b58bb1a..e008e4b19d 100644 --- a/cpp/src/thrift/megasearch.thrift +++ b/cpp/src/thrift/megasearch.thrift @@ -69,9 +69,9 @@ struct VecSearchResultList { } /** - * second; Seconds. [0-60] (1 leap second) - * minute; Minutes. [0-59] - * hour; Hours. [0-23] + * second; Seconds. [0-59] reserved + * minute; Minutes. [0-59] reserved + * hour; Hours. [0-23] reserved * day; Day. [1-31] * month; Month. [0-11] * year; Year - 1900. @@ -87,9 +87,9 @@ struct VecDateTime { /** * time_begin; time range begin - * begine_closed; true means '[', false means '(' + * begine_closed; true means '[', false means '(' reserved * time_end; set to true to return tensor double array - * end_closed; time range end + * end_closed; time range end reserved */ struct VecTimeRange { 1: required VecDateTime time_begin; @@ -99,7 +99,7 @@ struct VecTimeRange { } /** - * attrib_filter; search condition, for example: "color=red" + * attrib_filter; reserved * time_ranges; search condition, for example: "date between 1999-02-12 and 2008-10-14" * return_attribs; specify required attribute names */ diff --git a/cpp/test_client/CMakeLists.txt b/cpp/test_client/CMakeLists.txt index 08dd60d08b..df918d734e 100644 --- a/cpp/test_client/CMakeLists.txt +++ b/cpp/test_client/CMakeLists.txt @@ -16,8 +16,8 @@ set(util_files set(service_files ../src/thrift/gen-cpp/VecService.cpp - ../src/thrift/gen-cpp/VectorService_constants.cpp - ../src/thrift/gen-cpp/VectorService_types.cpp) + ../src/thrift/gen-cpp/megasearch_constants.cpp + ../src/thrift/gen-cpp/megasearch_types.cpp) diff --git a/cpp/test_client/src/ClientSession.cpp b/cpp/test_client/src/ClientSession.cpp index 57aa71d4b4..05d60f8c7e 100644 --- a/cpp/test_client/src/ClientSession.cpp +++ b/cpp/test_client/src/ClientSession.cpp @@ -6,8 +6,8 @@ #include "ClientSession.h" #include "Log.h" -#include "thrift/gen-cpp/VectorService_types.h" -#include "thrift/gen-cpp/VectorService_constants.h" +#include "thrift/gen-cpp/megasearch_types.h" +#include "thrift/gen-cpp/megasearch_constants.h" #include diff --git a/cpp/test_client/src/ClientTest.cpp b/cpp/test_client/src/ClientTest.cpp index c08f4d30f1..8ff48e963d 100644 --- a/cpp/test_client/src/ClientTest.cpp +++ b/cpp/test_client/src/ClientTest.cpp @@ -21,7 +21,7 @@ using namespace zilliz::vecwise::client; namespace { static const int32_t VEC_DIMENSION = 256; - static const int64_t BATCH_COUNT = 1000; + static const int64_t BATCH_COUNT = 10000; static const int64_t REPEAT_COUNT = 1; static const int64_t TOP_K = 10; @@ -130,15 +130,18 @@ TEST(AddVector, CLIENT_TEST) { try { std::string id; zilliz::VecTensor tensor; + for(int32_t i = 0; i < VEC_DIMENSION; i++) { + tensor.tensor.push_back(0.5); + } session.interface()->add_vector(id, GetGroupID(), tensor); } catch (zilliz::VecException& ex) { - ASSERT_EQ(ex.code, zilliz::VecErrCode::GROUP_NOT_EXISTS); + ASSERT_EQ(ex.code, zilliz::VecErrCode::ILLEGAL_ARGUMENT); } try { VecGroup temp_group; session.interface()->get_group(temp_group, GetGroupID()); - ASSERT_TRUE(temp_group.id.empty()); + //ASSERT_TRUE(temp_group.id.empty()); } catch (zilliz::VecException& ex) { ASSERT_EQ(ex.code, zilliz::VecErrCode::GROUP_NOT_EXISTS); } @@ -231,7 +234,6 @@ TEST(SearchVector, CLIENT_TEST) { //search vector { const int32_t anchor_index = 100; - server::TimeRecorder rc("Search top_k"); VecTensor tensor; for (int32_t i = 0; i < VEC_DIMENSION; i++) { tensor.tensor.push_back((double) (i + anchor_index)); @@ -249,35 +251,60 @@ TEST(SearchVector, CLIENT_TEST) { time_ranges.emplace_back(range); filter.__set_time_ranges(time_ranges); - //do search - session.interface()->search_vector(res, GetGroupID(), TOP_K, tensor, filter); + //normal search + { + server::TimeRecorder rc("Search top_k"); + session.interface()->search_vector(res, GetGroupID(), TOP_K, tensor, filter); + rc.Elapse("done!"); - //build result - std::cout << "Search result: " << std::endl; - for(VecSearchResultItem& item : res.result_list) { - std::cout << "\t" << item.uid << std::endl; + //build result + std::cout << "Search result: " << std::endl; + for (VecSearchResultItem &item : res.result_list) { + std::cout << "\t" << item.uid << std::endl; - ASSERT_TRUE(item.attrib.count(TEST_ATTRIB_NUM) != 0); - ASSERT_TRUE(item.attrib.count(TEST_ATTRIB_COMMENT) != 0); - ASSERT_TRUE(!item.attrib[TEST_ATTRIB_COMMENT].empty()); + ASSERT_TRUE(item.attrib.count(TEST_ATTRIB_NUM) != 0); + ASSERT_TRUE(item.attrib.count(TEST_ATTRIB_COMMENT) != 0); + ASSERT_TRUE(!item.attrib[TEST_ATTRIB_COMMENT].empty()); + } + + ASSERT_EQ(res.result_list.size(), (uint64_t) TOP_K); + if (!res.result_list.empty()) { + ASSERT_TRUE(!res.result_list[0].uid.empty()); + } } - rc.Elapse("done!"); - ASSERT_EQ(res.result_list.size(), (uint64_t)TOP_K); - if(!res.result_list.empty()) { - ASSERT_TRUE(!res.result_list[0].uid.empty()); + //filter attribute search + { + std::vector require_attributes = {TEST_ATTRIB_COMMENT}; + filter.__set_return_attribs(require_attributes); + server::TimeRecorder rc("Search top_k with attribute filter"); + session.interface()->search_vector(res, GetGroupID(), TOP_K, tensor, filter); + rc.Elapse("done!"); + + //build result + std::cout << "Search result attributes: " << std::endl; + for (VecSearchResultItem &item : res.result_list) { + ASSERT_EQ(item.attrib.size(), 1UL); + ASSERT_TRUE(item.attrib.count(TEST_ATTRIB_COMMENT) != 0); + ASSERT_TRUE(!item.attrib[TEST_ATTRIB_COMMENT].empty()); + std::cout << "\t" << item.uid << ":" << item.attrib[TEST_ATTRIB_COMMENT] << std::endl; + } + + ASSERT_EQ(res.result_list.size(), (uint64_t) TOP_K); } //empty search - date.day > 0 ? date.day -= 1 : date.day += 1; - range.time_begin = date; - range.time_end = date; - time_ranges.clear(); - time_ranges.emplace_back(range); - filter.__set_time_ranges(time_ranges); - session.interface()->search_vector(res, GetGroupID(), TOP_K, tensor, filter); + { + date.day > 0 ? date.day -= 1 : date.day += 1; + range.time_begin = date; + range.time_end = date; + time_ranges.clear(); + time_ranges.emplace_back(range); + filter.__set_time_ranges(time_ranges); + session.interface()->search_vector(res, GetGroupID(), TOP_K, tensor, filter); - ASSERT_EQ(res.result_list.size(), 0); + ASSERT_EQ(res.result_list.size(), 0); + } } //search binary vector diff --git a/cpp/unittest/faiss_wrapper/CMakeLists.txt b/cpp/unittest/faiss_wrapper/CMakeLists.txt index bdd0d797aa..eb179e6b6a 100644 --- a/cpp/unittest/faiss_wrapper/CMakeLists.txt +++ b/cpp/unittest/faiss_wrapper/CMakeLists.txt @@ -14,6 +14,7 @@ link_directories("/usr/local/cuda/lib64") set(require_files ../../src/server/VecIdMapper.cpp + ../../src/server/RocksIdMapper.cpp ../../src/server/ServerConfig.cpp ../../src/utils/CommonUtil.cpp ../../src/utils/TimeRecorder.cpp diff --git a/cpp/unittest/server/CMakeLists.txt b/cpp/unittest/server/CMakeLists.txt index 7141c45ede..5d081d72c9 100644 --- a/cpp/unittest/server/CMakeLists.txt +++ b/cpp/unittest/server/CMakeLists.txt @@ -16,6 +16,7 @@ aux_source_directory(./ test_srcs) set(require_files ../../src/server/VecIdMapper.cpp + ../../src/server/RocksIdMapper.cpp ../../src/server/ServerConfig.cpp ../../src/utils/CommonUtil.cpp ../../src/utils/TimeRecorder.cpp diff --git a/cpp/unittest/server/common_test.cpp b/cpp/unittest/server/common_test.cpp index cfc07c4196..0880a31677 100644 --- a/cpp/unittest/server/common_test.cpp +++ b/cpp/unittest/server/common_test.cpp @@ -11,12 +11,13 @@ using namespace zilliz::vecwise; TEST(CommonTest, COMMON_TEST) { - std::string path1 = "/tmp/vecwise_test/common_test_12345/"; - std::string path2 = path1 + "abcdef"; - server::ServerError err = server::CommonUtil::CreateDirectory(path2); + std::string path1 = "/tmp/vecwise_test/"; + std::string path2 = path1 + "common_test_12345/"; + std::string path3 = path2 + "abcdef"; + server::ServerError err = server::CommonUtil::CreateDirectory(path3); ASSERT_EQ(err, server::SERVER_SUCCESS); - ASSERT_TRUE(server::CommonUtil::IsDirectoryExit(path2)); + ASSERT_TRUE(server::CommonUtil::IsDirectoryExit(path3)); err = server::CommonUtil::DeleteDirectory(path1); ASSERT_EQ(err, server::SERVER_SUCCESS); diff --git a/cpp/unittest/server/idmapper_test.cpp b/cpp/unittest/server/idmapper_test.cpp index d0831c825b..8580d8b5f3 100644 --- a/cpp/unittest/server/idmapper_test.cpp +++ b/cpp/unittest/server/idmapper_test.cpp @@ -9,36 +9,81 @@ #include "utils/TimeRecorder.h" #include "utils/CommonUtil.h" +#include + using namespace zilliz::vecwise; +namespace { + std::string CurrentTime() { + time_t tt; + time(&tt); + tt = tt + 8 * 3600; + tm *t = gmtime(&tt); + + std::string str = std::to_string(t->tm_year + 1900) + "_" + std::to_string(t->tm_mon + 1) + + "_" + std::to_string(t->tm_mday) + "_" + std::to_string(t->tm_hour) + + "_" + std::to_string(t->tm_min) + "_" + std::to_string(t->tm_sec); + + return str; + } + + std::string GetGroupID() { + static std::string s_id(CurrentTime()); + return s_id; + } +} TEST(IdMapperTest, IDMAPPER_TEST) { server::IVecIdMapper* mapper = server::IVecIdMapper::GetInstance(); + std::string group_id = GetGroupID(); + std::vector nid = {"1", "50", "900", "10000"}; std::vector sid = {"one", "fifty", "nine zero zero", "many"}; - server::ServerError err = mapper->Put(nid, sid); + server::ServerError err = mapper->Put(nid, sid, group_id); ASSERT_EQ(err, server::SERVER_SUCCESS); - err = mapper->Put(nid, std::vector()); + err = mapper->Put(nid, std::vector(), group_id); ASSERT_NE(err, server::SERVER_SUCCESS); std::vector res; - err = mapper->Get(nid, res); + err = mapper->Get(nid, res, group_id); ASSERT_EQ(res.size(), nid.size()); for(size_t i = 0; i < res.size(); i++) { ASSERT_EQ(res[i], sid[i]); } std::string str_id; - err = mapper->Get(nid[1], str_id); + err = mapper->Get(nid[1], str_id, group_id); ASSERT_EQ(str_id, "fifty"); - err = mapper->Delete(nid[2]); + err = mapper->Get(nid[1], str_id); + ASSERT_EQ(str_id, ""); + + err = mapper->Get(nid[2], str_id, group_id); + ASSERT_EQ(str_id, "nine zero zero"); + + err = mapper->Delete(nid[2], group_id); ASSERT_EQ(err, server::SERVER_SUCCESS); - err = mapper->Get(nid[2], str_id); - ASSERT_NE(err, server::SERVER_SUCCESS); + err = mapper->Get(nid[2], str_id, group_id); + ASSERT_EQ(str_id, ""); + + err = mapper->Get(nid[3], str_id, group_id); + ASSERT_EQ(str_id, "many"); + + err = mapper->DeleteGroup(group_id); + ASSERT_EQ(err, server::SERVER_SUCCESS); + + err = mapper->Get(nid[3], str_id, group_id); + ASSERT_EQ(str_id, ""); + + std::string ct = CurrentTime(); + err = mapper->Put("current_time", ct, "time"); + ASSERT_EQ(err, server::SERVER_SUCCESS); + + err = mapper->Get("current_time", str_id, "time"); + ASSERT_EQ(str_id, ct); //test performance nid.clear(); From 1a8a97a9609740789c245159a9987510da628eb5 Mon Sep 17 00:00:00 2001 From: groot Date: Tue, 14 May 2019 11:37:37 +0800 Subject: [PATCH 10/16] no change config Former-commit-id: 792765d33c0a448432830a9b3cdfec5272b378cc --- cpp/conf/server_config.yaml | 14 +++++++------- cpp/conf/vecwise_engine_log.conf | 12 ++++++------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml index d91161830b..14570588bd 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/conf/server_config.yaml @@ -5,7 +5,7 @@ server_config: server_mode: thread_pool #optional: simple, thread_pool db_config: - db_path: /tmp/vecwise + db_path: /var/vecwise db_backend_url: http://127.0.0.1 db_flush_interval: 5 #unit: second idmapper_max_open_file: 128 @@ -13,7 +13,7 @@ db_config: log_config: global: format: "%datetime | %level | %logger | %msg" - filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-global.log" + filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-global.log" enabled: true to_file: true to_standard_output: true @@ -21,22 +21,22 @@ log_config: performance_tracking: false max_log_file_size: 2097152 # throw log files away after 2mb debug: - filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-debug.log" + filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-debug.log" enabled: true warning: - filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-warning.log" + filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-warning.log" trace: - filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-trace.log" + filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-trace.log" verbose: format: "%datetime{%d/%m/%y} | %level-%vlevel | %msg" to_file: false to_standard_output: true error: enabled: false - filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-error.log" + filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-error.log" fatal: enabled: false - filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-fatal.log" + filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-fatal.log" cache_config: cpu_cache_capacity: 16 # unit: GB diff --git a/cpp/conf/vecwise_engine_log.conf b/cpp/conf/vecwise_engine_log.conf index 4935b2fd9a..d0164b4158 100644 --- a/cpp/conf/vecwise_engine_log.conf +++ b/cpp/conf/vecwise_engine_log.conf @@ -1,6 +1,6 @@ * GLOBAL: FORMAT = "%datetime | %level | %logger | %msg" - FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-global.log" + FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-global.log" ENABLED = true TO_FILE = true TO_STANDARD_OUTPUT = true @@ -8,12 +8,12 @@ PERFORMANCE_TRACKING = false MAX_LOG_FILE_SIZE = 2097152 ## Throw log files away after 2MB * DEBUG: - FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-debug.log" + FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-debug.log" ENABLED = true * WARNING: - FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-warning.log" + FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-warning.log" * TRACE: - FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-trace.log" + FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-trace.log" * VERBOSE: FORMAT = "%datetime{%d/%M/%y} | %level-%vlevel | %msg" TO_FILE = false @@ -21,7 +21,7 @@ ## Error logs * ERROR: ENABLED = false - FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-error.log" + FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-error.log" * FATAL: ENABLED = false - FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-fatal.log" \ No newline at end of file + FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-fatal.log" \ No newline at end of file From 202e4fcbd8acb92ea39a9004c48547aa0655ff8d Mon Sep 17 00:00:00 2001 From: groot Date: Tue, 14 May 2019 11:49:32 +0800 Subject: [PATCH 11/16] no change config Former-commit-id: b97f814d0db3052d41eaeb9f7990b271565dbb8f --- cpp/conf/server_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml index 14570588bd..790bca4ba5 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/conf/server_config.yaml @@ -1,5 +1,5 @@ server_config: - address: 127.0.0.1 + address: 0.0.0.0 port: 33001 transfer_protocol: json #optional: binary, compact, json, debug server_mode: thread_pool #optional: simple, thread_pool From cc2edace95d9ebfe814655ad10b39d1c76fdc1d9 Mon Sep 17 00:00:00 2001 From: groot Date: Tue, 14 May 2019 12:03:50 +0800 Subject: [PATCH 12/16] add test infomation Former-commit-id: a5ac7a483a0f21182069dd2727f1832d0f94ae97 --- cpp/test_client/src/ClientTest.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpp/test_client/src/ClientTest.cpp b/cpp/test_client/src/ClientTest.cpp index 8ff48e963d..e4e0a604de 100644 --- a/cpp/test_client/src/ClientTest.cpp +++ b/cpp/test_client/src/ClientTest.cpp @@ -135,6 +135,7 @@ TEST(AddVector, CLIENT_TEST) { } session.interface()->add_vector(id, GetGroupID(), tensor); } catch (zilliz::VecException& ex) { + CLIENT_LOG_ERROR << "request encounter exception: " << ex.what(); ASSERT_EQ(ex.code, zilliz::VecErrCode::ILLEGAL_ARGUMENT); } @@ -143,6 +144,7 @@ TEST(AddVector, CLIENT_TEST) { session.interface()->get_group(temp_group, GetGroupID()); //ASSERT_TRUE(temp_group.id.empty()); } catch (zilliz::VecException& ex) { + CLIENT_LOG_ERROR << "request encounter exception: " << ex.what(); ASSERT_EQ(ex.code, zilliz::VecErrCode::GROUP_NOT_EXISTS); } From d14b7188629d60e0903cbd32771db29f45f51a16 Mon Sep 17 00:00:00 2001 From: groot Date: Tue, 14 May 2019 15:29:37 +0800 Subject: [PATCH 13/16] change api namespace Former-commit-id: ad513035c3db217c61e0eddcfaa633237564311c --- cpp/conf/server_config.yaml | 14 +- cpp/conf/vecwise_engine_log.conf | 12 +- cpp/src/server/VecServiceHandler.cpp | 26 +- cpp/src/server/VecServiceHandler.h | 2 + cpp/src/server/VecServiceTask.h | 1 + cpp/src/thrift/gen-cpp/VecService.cpp | 2 +- cpp/src/thrift/gen-cpp/VecService.h | 2 +- .../gen-cpp/VecService_server.skeleton.cpp | 2 +- .../thrift/gen-cpp/megasearch_constants.cpp | 2 +- cpp/src/thrift/gen-cpp/megasearch_constants.h | 2 +- cpp/src/thrift/gen-cpp/megasearch_types.cpp | 2 +- cpp/src/thrift/gen-cpp/megasearch_types.h | 2 +- cpp/src/thrift/gen-py/py_sample.py | 47 +- .../thrift/gen-py/zilliz/VecService-remote | 187 -- cpp/src/thrift/gen-py/zilliz/VecService.py | 2548 ----------------- cpp/src/thrift/gen-py/zilliz/__init__.py | 1 - cpp/src/thrift/gen-py/zilliz/constants.py | 14 - cpp/src/thrift/gen-py/zilliz/ttypes.py | 1165 -------- cpp/src/thrift/megasearch.thrift | 20 +- cpp/test_client/src/ClientSession.cpp | 2 + cpp/test_client/src/ClientSession.h | 2 +- cpp/test_client/src/ClientTest.cpp | 11 +- 22 files changed, 69 insertions(+), 3997 deletions(-) delete mode 100755 cpp/src/thrift/gen-py/zilliz/VecService-remote delete mode 100644 cpp/src/thrift/gen-py/zilliz/VecService.py delete mode 100644 cpp/src/thrift/gen-py/zilliz/__init__.py delete mode 100644 cpp/src/thrift/gen-py/zilliz/constants.py delete mode 100644 cpp/src/thrift/gen-py/zilliz/ttypes.py diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml index 790bca4ba5..2283dc7a0f 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/conf/server_config.yaml @@ -5,7 +5,7 @@ server_config: server_mode: thread_pool #optional: simple, thread_pool db_config: - db_path: /var/vecwise + db_path: /tmp/vecwise db_backend_url: http://127.0.0.1 db_flush_interval: 5 #unit: second idmapper_max_open_file: 128 @@ -13,7 +13,7 @@ db_config: log_config: global: format: "%datetime | %level | %logger | %msg" - filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-global.log" + filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-global.log" enabled: true to_file: true to_standard_output: true @@ -21,22 +21,22 @@ log_config: performance_tracking: false max_log_file_size: 2097152 # throw log files away after 2mb debug: - filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-debug.log" + filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-debug.log" enabled: true warning: - filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-warning.log" + filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-warning.log" trace: - filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-trace.log" + filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-trace.log" verbose: format: "%datetime{%d/%m/%y} | %level-%vlevel | %msg" to_file: false to_standard_output: true error: enabled: false - filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-error.log" + filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-error.log" fatal: enabled: false - filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-fatal.log" + filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-fatal.log" cache_config: cpu_cache_capacity: 16 # unit: GB diff --git a/cpp/conf/vecwise_engine_log.conf b/cpp/conf/vecwise_engine_log.conf index d0164b4158..4935b2fd9a 100644 --- a/cpp/conf/vecwise_engine_log.conf +++ b/cpp/conf/vecwise_engine_log.conf @@ -1,6 +1,6 @@ * GLOBAL: FORMAT = "%datetime | %level | %logger | %msg" - FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-global.log" + FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-global.log" ENABLED = true TO_FILE = true TO_STANDARD_OUTPUT = true @@ -8,12 +8,12 @@ PERFORMANCE_TRACKING = false MAX_LOG_FILE_SIZE = 2097152 ## Throw log files away after 2MB * DEBUG: - FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-debug.log" + FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-debug.log" ENABLED = true * WARNING: - FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-warning.log" + FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-warning.log" * TRACE: - FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-trace.log" + FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-trace.log" * VERBOSE: FORMAT = "%datetime{%d/%M/%y} | %level-%vlevel | %msg" TO_FILE = false @@ -21,7 +21,7 @@ ## Error logs * ERROR: ENABLED = false - FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-error.log" + FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-error.log" * FATAL: ENABLED = false - FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-fatal.log" \ No newline at end of file + FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-fatal.log" \ No newline at end of file diff --git a/cpp/src/server/VecServiceHandler.cpp b/cpp/src/server/VecServiceHandler.cpp index da8a2aa1e8..c8db470e63 100644 --- a/cpp/src/server/VecServiceHandler.cpp +++ b/cpp/src/server/VecServiceHandler.cpp @@ -17,6 +17,8 @@ namespace zilliz { namespace vecwise { namespace server { +using namespace megasearch; + namespace { class TimeRecordWrapper { public: @@ -38,17 +40,17 @@ namespace { } - const std::map& ErrorMap() { - static const std::map code_map = { - {SERVER_UNEXPECTED_ERROR, zilliz::VecErrCode::ILLEGAL_ARGUMENT}, - {SERVER_NULL_POINTER, zilliz::VecErrCode::ILLEGAL_ARGUMENT}, - {SERVER_INVALID_ARGUMENT, zilliz::VecErrCode::ILLEGAL_ARGUMENT}, - {SERVER_FILE_NOT_FOUND, zilliz::VecErrCode::ILLEGAL_ARGUMENT}, - {SERVER_NOT_IMPLEMENT, zilliz::VecErrCode::ILLEGAL_ARGUMENT}, - {SERVER_BLOCKING_QUEUE_EMPTY, zilliz::VecErrCode::ILLEGAL_ARGUMENT}, - {SERVER_GROUP_NOT_EXIST, zilliz::VecErrCode::GROUP_NOT_EXISTS}, - {SERVER_INVALID_TIME_RANGE, zilliz::VecErrCode::ILLEGAL_TIME_RANGE}, - {SERVER_INVALID_VECTOR_DIMENSION, zilliz::VecErrCode::ILLEGAL_VECTOR_DIMENSION}, + const std::map& ErrorMap() { + static const std::map code_map = { + {SERVER_UNEXPECTED_ERROR, VecErrCode::ILLEGAL_ARGUMENT}, + {SERVER_NULL_POINTER, VecErrCode::ILLEGAL_ARGUMENT}, + {SERVER_INVALID_ARGUMENT, VecErrCode::ILLEGAL_ARGUMENT}, + {SERVER_FILE_NOT_FOUND, VecErrCode::ILLEGAL_ARGUMENT}, + {SERVER_NOT_IMPLEMENT, VecErrCode::ILLEGAL_ARGUMENT}, + {SERVER_BLOCKING_QUEUE_EMPTY, VecErrCode::ILLEGAL_ARGUMENT}, + {SERVER_GROUP_NOT_EXIST, VecErrCode::GROUP_NOT_EXISTS}, + {SERVER_INVALID_TIME_RANGE, VecErrCode::ILLEGAL_TIME_RANGE}, + {SERVER_INVALID_VECTOR_DIMENSION, VecErrCode::ILLEGAL_VECTOR_DIMENSION}, }; return code_map; @@ -82,7 +84,7 @@ namespace { task_ptr->WaitToFinish(); ServerError err = task_ptr->ErrorCode(); if (err != SERVER_SUCCESS) { - zilliz::VecException ex; + VecException ex; ex.__set_code(ErrorMap().at(err)); std::string msg = task_ptr->ErrorMsg(); if(msg.empty()){ diff --git a/cpp/src/server/VecServiceHandler.h b/cpp/src/server/VecServiceHandler.h index d37376bd3c..7a6ccc4b18 100644 --- a/cpp/src/server/VecServiceHandler.h +++ b/cpp/src/server/VecServiceHandler.h @@ -22,6 +22,8 @@ namespace zilliz { namespace vecwise { namespace server { +using namespace megasearch; + class VecServiceHandler : virtual public VecServiceIf { public: VecServiceHandler() { diff --git a/cpp/src/server/VecServiceTask.h b/cpp/src/server/VecServiceTask.h index a0c317ccb2..e046e36083 100644 --- a/cpp/src/server/VecServiceTask.h +++ b/cpp/src/server/VecServiceTask.h @@ -19,6 +19,7 @@ namespace zilliz { namespace vecwise { namespace server { +using namespace megasearch; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class AddGroupTask : public BaseTask { public: diff --git a/cpp/src/thrift/gen-cpp/VecService.cpp b/cpp/src/thrift/gen-cpp/VecService.cpp index d99b2bfcef..c794959057 100644 --- a/cpp/src/thrift/gen-cpp/VecService.cpp +++ b/cpp/src/thrift/gen-cpp/VecService.cpp @@ -6,7 +6,7 @@ */ #include "VecService.h" -namespace zilliz { +namespace megasearch { VecService_add_group_args::~VecService_add_group_args() throw() { diff --git a/cpp/src/thrift/gen-cpp/VecService.h b/cpp/src/thrift/gen-cpp/VecService.h index 622196dd41..a3fdba2472 100644 --- a/cpp/src/thrift/gen-cpp/VecService.h +++ b/cpp/src/thrift/gen-cpp/VecService.h @@ -11,7 +11,7 @@ #include #include "megasearch_types.h" -namespace zilliz { +namespace megasearch { #ifdef _MSC_VER #pragma warning( push ) diff --git a/cpp/src/thrift/gen-cpp/VecService_server.skeleton.cpp b/cpp/src/thrift/gen-cpp/VecService_server.skeleton.cpp index 364743cf46..8727a0e500 100644 --- a/cpp/src/thrift/gen-cpp/VecService_server.skeleton.cpp +++ b/cpp/src/thrift/gen-cpp/VecService_server.skeleton.cpp @@ -12,7 +12,7 @@ using namespace ::apache::thrift::protocol; using namespace ::apache::thrift::transport; using namespace ::apache::thrift::server; -using namespace ::zilliz; +using namespace ::megasearch; class VecServiceHandler : virtual public VecServiceIf { public: diff --git a/cpp/src/thrift/gen-cpp/megasearch_constants.cpp b/cpp/src/thrift/gen-cpp/megasearch_constants.cpp index 798fbf9e5d..06adb7629f 100644 --- a/cpp/src/thrift/gen-cpp/megasearch_constants.cpp +++ b/cpp/src/thrift/gen-cpp/megasearch_constants.cpp @@ -6,7 +6,7 @@ */ #include "megasearch_constants.h" -namespace zilliz { +namespace megasearch { const megasearchConstants g_megasearch_constants; diff --git a/cpp/src/thrift/gen-cpp/megasearch_constants.h b/cpp/src/thrift/gen-cpp/megasearch_constants.h index 397b71e9ca..1f4b9553dd 100644 --- a/cpp/src/thrift/gen-cpp/megasearch_constants.h +++ b/cpp/src/thrift/gen-cpp/megasearch_constants.h @@ -9,7 +9,7 @@ #include "megasearch_types.h" -namespace zilliz { +namespace megasearch { class megasearchConstants { public: diff --git a/cpp/src/thrift/gen-cpp/megasearch_types.cpp b/cpp/src/thrift/gen-cpp/megasearch_types.cpp index ecc016504e..bf4d316366 100644 --- a/cpp/src/thrift/gen-cpp/megasearch_types.cpp +++ b/cpp/src/thrift/gen-cpp/megasearch_types.cpp @@ -11,7 +11,7 @@ #include -namespace zilliz { +namespace megasearch { int _kVecErrCodeValues[] = { VecErrCode::SUCCESS, diff --git a/cpp/src/thrift/gen-cpp/megasearch_types.h b/cpp/src/thrift/gen-cpp/megasearch_types.h index e6a68eecb8..e7e424b2db 100644 --- a/cpp/src/thrift/gen-cpp/megasearch_types.h +++ b/cpp/src/thrift/gen-cpp/megasearch_types.h @@ -18,7 +18,7 @@ #include -namespace zilliz { +namespace megasearch { struct VecErrCode { enum type { diff --git a/cpp/src/thrift/gen-py/py_sample.py b/cpp/src/thrift/gen-py/py_sample.py index dee18908cc..0a4690c1eb 100644 --- a/cpp/src/thrift/gen-py/py_sample.py +++ b/cpp/src/thrift/gen-py/py_sample.py @@ -1,22 +1,15 @@ import time import struct -from zilliz import VecService +from megasearch import VecService -from thrift import Thrift +#Note: pip install thrift from thrift.transport import TSocket from thrift.transport import TTransport from thrift.protocol import TBinaryProtocol, TCompactProtocol, TJSONProtocol -def print_time_cost(desc, t): - time_now = time.time() - print(desc + ' cost ', time_now - t, ' sec') - return time_now - -def test_vecwise(): +def test_megasearch(): try: - time_start = time.time() - #connect transport = TSocket.TSocket('localhost', 33001) transport = TTransport.TBufferedTransport(transport) @@ -24,26 +17,13 @@ def test_vecwise(): client = VecService.Client(protocol) transport.open() - time_start = print_time_cost('connection', time_start) + print("connected"); #add group - group = VecService.VecGroup("py_group_" + time.strftime('%H%M%S'), 256) + group = VecService.VecGroup("test_" + time.strftime('%H%M%S'), 256) client.add_group(group) - time_start = print_time_cost('add group', time_start) - - #build vectors - # vec_list = VecService.VecTensorList([]) - # for i in range(10000): - # vec = VecService.VecTensor("normal_"+str(i), []) - # for k in range(group.dimension): - # vec.tensor.append(k) - # vec_list.tensor_list.append(vec) - #time_start = print_time_cost('build normal vectors', time_start) - - #add vectors - #client.add_vector_batch(group.id, vec_list) - #time_start = print_time_cost('add normal vectors', time_start)) + print("group added"); # build binary vectors bin_vec_list = VecService.VecBinaryTensorList([]) @@ -55,14 +35,12 @@ def test_vecwise(): bin_vec.tensor = struct.pack(str(group.dimension)+"d", *a) bin_vec_list.tensor_list.append(bin_vec) - time_start = print_time_cost('build binary vectors', time_start) - # add vectors client.add_binary_vector_batch(group.id, bin_vec_list) - time_start = print_time_cost('add binary vectors', time_start) - time.sleep(5) - time_start = print_time_cost('sleep 5 seconds', time_start) + wait_storage = 5 + print("wait {} seconds for persisting data".format(wait_storage)) + time.sleep(wait_storage) # search vector a = [] @@ -72,18 +50,19 @@ def test_vecwise(): bin_vec.tensor = struct.pack(str(group.dimension) + "d", *a) filter = VecService.VecSearchFilter() res = VecService.VecSearchResult() + + print("begin search ..."); res = client.search_binary_vector(group.id, 5, bin_vec, filter) - time_start = print_time_cost('search binary vectors', time_start) print('result count: ' + str(len(res.result_list))) for item in res.result_list: print(item.uid) transport.close() - time_start = print_time_cost('close connection', time_start) + print("disconnected"); except VecService.VecException as ex: print(ex.reason) -test_vecwise() \ No newline at end of file +test_megasearch() \ No newline at end of file diff --git a/cpp/src/thrift/gen-py/zilliz/VecService-remote b/cpp/src/thrift/gen-py/zilliz/VecService-remote deleted file mode 100755 index 047136afb1..0000000000 --- a/cpp/src/thrift/gen-py/zilliz/VecService-remote +++ /dev/null @@ -1,187 +0,0 @@ -#!/usr/bin/env python -# -# Autogenerated by Thrift Compiler (0.11.0) -# -# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING -# -# options string: py -# - -import sys -import pprint -if sys.version_info[0] > 2: - from urllib.parse import urlparse -else: - from urlparse import urlparse -from thrift.transport import TTransport, TSocket, TSSLSocket, THttpClient -from thrift.protocol.TBinaryProtocol import TBinaryProtocol - -from zilliz import VecService -from zilliz.ttypes import * - -if len(sys.argv) <= 1 or sys.argv[1] == '--help': - print('') - print('Usage: ' + sys.argv[0] + ' [-h host[:port]] [-u url] [-f[ramed]] [-s[sl]] [-novalidate] [-ca_certs certs] [-keyfile keyfile] [-certfile certfile] function [arg1 [arg2...]]') - print('') - print('Functions:') - print(' void add_group(VecGroup group)') - print(' VecGroup get_group(string group_id)') - print(' void del_group(string group_id)') - print(' string add_vector(string group_id, VecTensor tensor)') - print(' add_vector_batch(string group_id, VecTensorList tensor_list)') - print(' string add_binary_vector(string group_id, VecBinaryTensor tensor)') - print(' add_binary_vector_batch(string group_id, VecBinaryTensorList tensor_list)') - print(' VecSearchResult search_vector(string group_id, i64 top_k, VecTensor tensor, VecSearchFilter filter)') - print(' VecSearchResultList search_vector_batch(string group_id, i64 top_k, VecTensorList tensor_list, VecSearchFilter filter)') - print(' VecSearchResult search_binary_vector(string group_id, i64 top_k, VecBinaryTensor tensor, VecSearchFilter filter)') - print(' VecSearchResultList search_binary_vector_batch(string group_id, i64 top_k, VecBinaryTensorList tensor_list, VecSearchFilter filter)') - print('') - sys.exit(0) - -pp = pprint.PrettyPrinter(indent=2) -host = 'localhost' -port = 9090 -uri = '' -framed = False -ssl = False -validate = True -ca_certs = None -keyfile = None -certfile = None -http = False -argi = 1 - -if sys.argv[argi] == '-h': - parts = sys.argv[argi + 1].split(':') - host = parts[0] - if len(parts) > 1: - port = int(parts[1]) - argi += 2 - -if sys.argv[argi] == '-u': - url = urlparse(sys.argv[argi + 1]) - parts = url[1].split(':') - host = parts[0] - if len(parts) > 1: - port = int(parts[1]) - else: - port = 80 - uri = url[2] - if url[4]: - uri += '?%s' % url[4] - http = True - argi += 2 - -if sys.argv[argi] == '-f' or sys.argv[argi] == '-framed': - framed = True - argi += 1 - -if sys.argv[argi] == '-s' or sys.argv[argi] == '-ssl': - ssl = True - argi += 1 - -if sys.argv[argi] == '-novalidate': - validate = False - argi += 1 - -if sys.argv[argi] == '-ca_certs': - ca_certs = sys.argv[argi+1] - argi += 2 - -if sys.argv[argi] == '-keyfile': - keyfile = sys.argv[argi+1] - argi += 2 - -if sys.argv[argi] == '-certfile': - certfile = sys.argv[argi+1] - argi += 2 - -cmd = sys.argv[argi] -args = sys.argv[argi + 1:] - -if http: - transport = THttpClient.THttpClient(host, port, uri) -else: - if ssl: - socket = TSSLSocket.TSSLSocket(host, port, validate=validate, ca_certs=ca_certs, keyfile=keyfile, certfile=certfile) - else: - socket = TSocket.TSocket(host, port) - if framed: - transport = TTransport.TFramedTransport(socket) - else: - transport = TTransport.TBufferedTransport(socket) -protocol = TBinaryProtocol(transport) -client = VecService.Client(protocol) -transport.open() - -if cmd == 'add_group': - if len(args) != 1: - print('add_group requires 1 args') - sys.exit(1) - pp.pprint(client.add_group(eval(args[0]),)) - -elif cmd == 'get_group': - if len(args) != 1: - print('get_group requires 1 args') - sys.exit(1) - pp.pprint(client.get_group(args[0],)) - -elif cmd == 'del_group': - if len(args) != 1: - print('del_group requires 1 args') - sys.exit(1) - pp.pprint(client.del_group(args[0],)) - -elif cmd == 'add_vector': - if len(args) != 2: - print('add_vector requires 2 args') - sys.exit(1) - pp.pprint(client.add_vector(args[0], eval(args[1]),)) - -elif cmd == 'add_vector_batch': - if len(args) != 2: - print('add_vector_batch requires 2 args') - sys.exit(1) - pp.pprint(client.add_vector_batch(args[0], eval(args[1]),)) - -elif cmd == 'add_binary_vector': - if len(args) != 2: - print('add_binary_vector requires 2 args') - sys.exit(1) - pp.pprint(client.add_binary_vector(args[0], eval(args[1]),)) - -elif cmd == 'add_binary_vector_batch': - if len(args) != 2: - print('add_binary_vector_batch requires 2 args') - sys.exit(1) - pp.pprint(client.add_binary_vector_batch(args[0], eval(args[1]),)) - -elif cmd == 'search_vector': - if len(args) != 4: - print('search_vector requires 4 args') - sys.exit(1) - pp.pprint(client.search_vector(args[0], eval(args[1]), eval(args[2]), eval(args[3]),)) - -elif cmd == 'search_vector_batch': - if len(args) != 4: - print('search_vector_batch requires 4 args') - sys.exit(1) - pp.pprint(client.search_vector_batch(args[0], eval(args[1]), eval(args[2]), eval(args[3]),)) - -elif cmd == 'search_binary_vector': - if len(args) != 4: - print('search_binary_vector requires 4 args') - sys.exit(1) - pp.pprint(client.search_binary_vector(args[0], eval(args[1]), eval(args[2]), eval(args[3]),)) - -elif cmd == 'search_binary_vector_batch': - if len(args) != 4: - print('search_binary_vector_batch requires 4 args') - sys.exit(1) - pp.pprint(client.search_binary_vector_batch(args[0], eval(args[1]), eval(args[2]), eval(args[3]),)) - -else: - print('Unrecognized method %s' % cmd) - sys.exit(1) - -transport.close() diff --git a/cpp/src/thrift/gen-py/zilliz/VecService.py b/cpp/src/thrift/gen-py/zilliz/VecService.py deleted file mode 100644 index 3cc23dfafc..0000000000 --- a/cpp/src/thrift/gen-py/zilliz/VecService.py +++ /dev/null @@ -1,2548 +0,0 @@ -# -# Autogenerated by Thrift Compiler (0.11.0) -# -# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING -# -# options string: py -# - -from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException -from thrift.protocol.TProtocol import TProtocolException -from thrift.TRecursive import fix_spec - -import sys -import logging -from .ttypes import * -from thrift.Thrift import TProcessor -from thrift.transport import TTransport -all_structs = [] - - -class Iface(object): - def add_group(self, group): - """ - group interfaces - - Parameters: - - group - """ - pass - - def get_group(self, group_id): - """ - Parameters: - - group_id - """ - pass - - def del_group(self, group_id): - """ - Parameters: - - group_id - """ - pass - - def add_vector(self, group_id, tensor): - """ - insert vector interfaces - - - Parameters: - - group_id - - tensor - """ - pass - - def add_vector_batch(self, group_id, tensor_list): - """ - Parameters: - - group_id - - tensor_list - """ - pass - - def add_binary_vector(self, group_id, tensor): - """ - Parameters: - - group_id - - tensor - """ - pass - - def add_binary_vector_batch(self, group_id, tensor_list): - """ - Parameters: - - group_id - - tensor_list - """ - pass - - def search_vector(self, group_id, top_k, tensor, filter): - """ - search interfaces - you can use filter to reduce search result - filter.attrib_filter can specify which attribute you need, for example: - set attrib_filter = {"color":""} means you want to get "color" attribute for result vector - set attrib_filter = {"color":"red"} means you want to get vectors which has attribute "color" equals "red" - if filter.time_range is empty, engine will search without time limit - - Parameters: - - group_id - - top_k - - tensor - - filter - """ - pass - - def search_vector_batch(self, group_id, top_k, tensor_list, filter): - """ - Parameters: - - group_id - - top_k - - tensor_list - - filter - """ - pass - - def search_binary_vector(self, group_id, top_k, tensor, filter): - """ - Parameters: - - group_id - - top_k - - tensor - - filter - """ - pass - - def search_binary_vector_batch(self, group_id, top_k, tensor_list, filter): - """ - Parameters: - - group_id - - top_k - - tensor_list - - filter - """ - pass - - -class Client(Iface): - def __init__(self, iprot, oprot=None): - self._iprot = self._oprot = iprot - if oprot is not None: - self._oprot = oprot - self._seqid = 0 - - def add_group(self, group): - """ - group interfaces - - Parameters: - - group - """ - self.send_add_group(group) - self.recv_add_group() - - def send_add_group(self, group): - self._oprot.writeMessageBegin('add_group', TMessageType.CALL, self._seqid) - args = add_group_args() - args.group = group - args.write(self._oprot) - self._oprot.writeMessageEnd() - self._oprot.trans.flush() - - def recv_add_group(self): - iprot = self._iprot - (fname, mtype, rseqid) = iprot.readMessageBegin() - if mtype == TMessageType.EXCEPTION: - x = TApplicationException() - x.read(iprot) - iprot.readMessageEnd() - raise x - result = add_group_result() - result.read(iprot) - iprot.readMessageEnd() - if result.e is not None: - raise result.e - return - - def get_group(self, group_id): - """ - Parameters: - - group_id - """ - self.send_get_group(group_id) - return self.recv_get_group() - - def send_get_group(self, group_id): - self._oprot.writeMessageBegin('get_group', TMessageType.CALL, self._seqid) - args = get_group_args() - args.group_id = group_id - args.write(self._oprot) - self._oprot.writeMessageEnd() - self._oprot.trans.flush() - - def recv_get_group(self): - iprot = self._iprot - (fname, mtype, rseqid) = iprot.readMessageBegin() - if mtype == TMessageType.EXCEPTION: - x = TApplicationException() - x.read(iprot) - iprot.readMessageEnd() - raise x - result = get_group_result() - result.read(iprot) - iprot.readMessageEnd() - if result.success is not None: - return result.success - if result.e is not None: - raise result.e - raise TApplicationException(TApplicationException.MISSING_RESULT, "get_group failed: unknown result") - - def del_group(self, group_id): - """ - Parameters: - - group_id - """ - self.send_del_group(group_id) - self.recv_del_group() - - def send_del_group(self, group_id): - self._oprot.writeMessageBegin('del_group', TMessageType.CALL, self._seqid) - args = del_group_args() - args.group_id = group_id - args.write(self._oprot) - self._oprot.writeMessageEnd() - self._oprot.trans.flush() - - def recv_del_group(self): - iprot = self._iprot - (fname, mtype, rseqid) = iprot.readMessageBegin() - if mtype == TMessageType.EXCEPTION: - x = TApplicationException() - x.read(iprot) - iprot.readMessageEnd() - raise x - result = del_group_result() - result.read(iprot) - iprot.readMessageEnd() - if result.e is not None: - raise result.e - return - - def add_vector(self, group_id, tensor): - """ - insert vector interfaces - - - Parameters: - - group_id - - tensor - """ - self.send_add_vector(group_id, tensor) - return self.recv_add_vector() - - def send_add_vector(self, group_id, tensor): - self._oprot.writeMessageBegin('add_vector', TMessageType.CALL, self._seqid) - args = add_vector_args() - args.group_id = group_id - args.tensor = tensor - args.write(self._oprot) - self._oprot.writeMessageEnd() - self._oprot.trans.flush() - - def recv_add_vector(self): - iprot = self._iprot - (fname, mtype, rseqid) = iprot.readMessageBegin() - if mtype == TMessageType.EXCEPTION: - x = TApplicationException() - x.read(iprot) - iprot.readMessageEnd() - raise x - result = add_vector_result() - result.read(iprot) - iprot.readMessageEnd() - if result.success is not None: - return result.success - if result.e is not None: - raise result.e - raise TApplicationException(TApplicationException.MISSING_RESULT, "add_vector failed: unknown result") - - def add_vector_batch(self, group_id, tensor_list): - """ - Parameters: - - group_id - - tensor_list - """ - self.send_add_vector_batch(group_id, tensor_list) - return self.recv_add_vector_batch() - - def send_add_vector_batch(self, group_id, tensor_list): - self._oprot.writeMessageBegin('add_vector_batch', TMessageType.CALL, self._seqid) - args = add_vector_batch_args() - args.group_id = group_id - args.tensor_list = tensor_list - args.write(self._oprot) - self._oprot.writeMessageEnd() - self._oprot.trans.flush() - - def recv_add_vector_batch(self): - iprot = self._iprot - (fname, mtype, rseqid) = iprot.readMessageBegin() - if mtype == TMessageType.EXCEPTION: - x = TApplicationException() - x.read(iprot) - iprot.readMessageEnd() - raise x - result = add_vector_batch_result() - result.read(iprot) - iprot.readMessageEnd() - if result.success is not None: - return result.success - if result.e is not None: - raise result.e - raise TApplicationException(TApplicationException.MISSING_RESULT, "add_vector_batch failed: unknown result") - - def add_binary_vector(self, group_id, tensor): - """ - Parameters: - - group_id - - tensor - """ - self.send_add_binary_vector(group_id, tensor) - return self.recv_add_binary_vector() - - def send_add_binary_vector(self, group_id, tensor): - self._oprot.writeMessageBegin('add_binary_vector', TMessageType.CALL, self._seqid) - args = add_binary_vector_args() - args.group_id = group_id - args.tensor = tensor - args.write(self._oprot) - self._oprot.writeMessageEnd() - self._oprot.trans.flush() - - def recv_add_binary_vector(self): - iprot = self._iprot - (fname, mtype, rseqid) = iprot.readMessageBegin() - if mtype == TMessageType.EXCEPTION: - x = TApplicationException() - x.read(iprot) - iprot.readMessageEnd() - raise x - result = add_binary_vector_result() - result.read(iprot) - iprot.readMessageEnd() - if result.success is not None: - return result.success - if result.e is not None: - raise result.e - raise TApplicationException(TApplicationException.MISSING_RESULT, "add_binary_vector failed: unknown result") - - def add_binary_vector_batch(self, group_id, tensor_list): - """ - Parameters: - - group_id - - tensor_list - """ - self.send_add_binary_vector_batch(group_id, tensor_list) - return self.recv_add_binary_vector_batch() - - def send_add_binary_vector_batch(self, group_id, tensor_list): - self._oprot.writeMessageBegin('add_binary_vector_batch', TMessageType.CALL, self._seqid) - args = add_binary_vector_batch_args() - args.group_id = group_id - args.tensor_list = tensor_list - args.write(self._oprot) - self._oprot.writeMessageEnd() - self._oprot.trans.flush() - - def recv_add_binary_vector_batch(self): - iprot = self._iprot - (fname, mtype, rseqid) = iprot.readMessageBegin() - if mtype == TMessageType.EXCEPTION: - x = TApplicationException() - x.read(iprot) - iprot.readMessageEnd() - raise x - result = add_binary_vector_batch_result() - result.read(iprot) - iprot.readMessageEnd() - if result.success is not None: - return result.success - if result.e is not None: - raise result.e - raise TApplicationException(TApplicationException.MISSING_RESULT, "add_binary_vector_batch failed: unknown result") - - def search_vector(self, group_id, top_k, tensor, filter): - """ - search interfaces - you can use filter to reduce search result - filter.attrib_filter can specify which attribute you need, for example: - set attrib_filter = {"color":""} means you want to get "color" attribute for result vector - set attrib_filter = {"color":"red"} means you want to get vectors which has attribute "color" equals "red" - if filter.time_range is empty, engine will search without time limit - - Parameters: - - group_id - - top_k - - tensor - - filter - """ - self.send_search_vector(group_id, top_k, tensor, filter) - return self.recv_search_vector() - - def send_search_vector(self, group_id, top_k, tensor, filter): - self._oprot.writeMessageBegin('search_vector', TMessageType.CALL, self._seqid) - args = search_vector_args() - args.group_id = group_id - args.top_k = top_k - args.tensor = tensor - args.filter = filter - args.write(self._oprot) - self._oprot.writeMessageEnd() - self._oprot.trans.flush() - - def recv_search_vector(self): - iprot = self._iprot - (fname, mtype, rseqid) = iprot.readMessageBegin() - if mtype == TMessageType.EXCEPTION: - x = TApplicationException() - x.read(iprot) - iprot.readMessageEnd() - raise x - result = search_vector_result() - result.read(iprot) - iprot.readMessageEnd() - if result.success is not None: - return result.success - if result.e is not None: - raise result.e - raise TApplicationException(TApplicationException.MISSING_RESULT, "search_vector failed: unknown result") - - def search_vector_batch(self, group_id, top_k, tensor_list, filter): - """ - Parameters: - - group_id - - top_k - - tensor_list - - filter - """ - self.send_search_vector_batch(group_id, top_k, tensor_list, filter) - return self.recv_search_vector_batch() - - def send_search_vector_batch(self, group_id, top_k, tensor_list, filter): - self._oprot.writeMessageBegin('search_vector_batch', TMessageType.CALL, self._seqid) - args = search_vector_batch_args() - args.group_id = group_id - args.top_k = top_k - args.tensor_list = tensor_list - args.filter = filter - args.write(self._oprot) - self._oprot.writeMessageEnd() - self._oprot.trans.flush() - - def recv_search_vector_batch(self): - iprot = self._iprot - (fname, mtype, rseqid) = iprot.readMessageBegin() - if mtype == TMessageType.EXCEPTION: - x = TApplicationException() - x.read(iprot) - iprot.readMessageEnd() - raise x - result = search_vector_batch_result() - result.read(iprot) - iprot.readMessageEnd() - if result.success is not None: - return result.success - if result.e is not None: - raise result.e - raise TApplicationException(TApplicationException.MISSING_RESULT, "search_vector_batch failed: unknown result") - - def search_binary_vector(self, group_id, top_k, tensor, filter): - """ - Parameters: - - group_id - - top_k - - tensor - - filter - """ - self.send_search_binary_vector(group_id, top_k, tensor, filter) - return self.recv_search_binary_vector() - - def send_search_binary_vector(self, group_id, top_k, tensor, filter): - self._oprot.writeMessageBegin('search_binary_vector', TMessageType.CALL, self._seqid) - args = search_binary_vector_args() - args.group_id = group_id - args.top_k = top_k - args.tensor = tensor - args.filter = filter - args.write(self._oprot) - self._oprot.writeMessageEnd() - self._oprot.trans.flush() - - def recv_search_binary_vector(self): - iprot = self._iprot - (fname, mtype, rseqid) = iprot.readMessageBegin() - if mtype == TMessageType.EXCEPTION: - x = TApplicationException() - x.read(iprot) - iprot.readMessageEnd() - raise x - result = search_binary_vector_result() - result.read(iprot) - iprot.readMessageEnd() - if result.success is not None: - return result.success - if result.e is not None: - raise result.e - raise TApplicationException(TApplicationException.MISSING_RESULT, "search_binary_vector failed: unknown result") - - def search_binary_vector_batch(self, group_id, top_k, tensor_list, filter): - """ - Parameters: - - group_id - - top_k - - tensor_list - - filter - """ - self.send_search_binary_vector_batch(group_id, top_k, tensor_list, filter) - return self.recv_search_binary_vector_batch() - - def send_search_binary_vector_batch(self, group_id, top_k, tensor_list, filter): - self._oprot.writeMessageBegin('search_binary_vector_batch', TMessageType.CALL, self._seqid) - args = search_binary_vector_batch_args() - args.group_id = group_id - args.top_k = top_k - args.tensor_list = tensor_list - args.filter = filter - args.write(self._oprot) - self._oprot.writeMessageEnd() - self._oprot.trans.flush() - - def recv_search_binary_vector_batch(self): - iprot = self._iprot - (fname, mtype, rseqid) = iprot.readMessageBegin() - if mtype == TMessageType.EXCEPTION: - x = TApplicationException() - x.read(iprot) - iprot.readMessageEnd() - raise x - result = search_binary_vector_batch_result() - result.read(iprot) - iprot.readMessageEnd() - if result.success is not None: - return result.success - if result.e is not None: - raise result.e - raise TApplicationException(TApplicationException.MISSING_RESULT, "search_binary_vector_batch failed: unknown result") - - -class Processor(Iface, TProcessor): - def __init__(self, handler): - self._handler = handler - self._processMap = {} - self._processMap["add_group"] = Processor.process_add_group - self._processMap["get_group"] = Processor.process_get_group - self._processMap["del_group"] = Processor.process_del_group - self._processMap["add_vector"] = Processor.process_add_vector - self._processMap["add_vector_batch"] = Processor.process_add_vector_batch - self._processMap["add_binary_vector"] = Processor.process_add_binary_vector - self._processMap["add_binary_vector_batch"] = Processor.process_add_binary_vector_batch - self._processMap["search_vector"] = Processor.process_search_vector - self._processMap["search_vector_batch"] = Processor.process_search_vector_batch - self._processMap["search_binary_vector"] = Processor.process_search_binary_vector - self._processMap["search_binary_vector_batch"] = Processor.process_search_binary_vector_batch - - def process(self, iprot, oprot): - (name, type, seqid) = iprot.readMessageBegin() - if name not in self._processMap: - iprot.skip(TType.STRUCT) - iprot.readMessageEnd() - x = TApplicationException(TApplicationException.UNKNOWN_METHOD, 'Unknown function %s' % (name)) - oprot.writeMessageBegin(name, TMessageType.EXCEPTION, seqid) - x.write(oprot) - oprot.writeMessageEnd() - oprot.trans.flush() - return - else: - self._processMap[name](self, seqid, iprot, oprot) - return True - - def process_add_group(self, seqid, iprot, oprot): - args = add_group_args() - args.read(iprot) - iprot.readMessageEnd() - result = add_group_result() - try: - self._handler.add_group(args.group) - msg_type = TMessageType.REPLY - except TTransport.TTransportException: - raise - except VecException as e: - msg_type = TMessageType.REPLY - result.e = e - except TApplicationException as ex: - logging.exception('TApplication exception in handler') - msg_type = TMessageType.EXCEPTION - result = ex - except Exception: - logging.exception('Unexpected exception in handler') - msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') - oprot.writeMessageBegin("add_group", msg_type, seqid) - result.write(oprot) - oprot.writeMessageEnd() - oprot.trans.flush() - - def process_get_group(self, seqid, iprot, oprot): - args = get_group_args() - args.read(iprot) - iprot.readMessageEnd() - result = get_group_result() - try: - result.success = self._handler.get_group(args.group_id) - msg_type = TMessageType.REPLY - except TTransport.TTransportException: - raise - except VecException as e: - msg_type = TMessageType.REPLY - result.e = e - except TApplicationException as ex: - logging.exception('TApplication exception in handler') - msg_type = TMessageType.EXCEPTION - result = ex - except Exception: - logging.exception('Unexpected exception in handler') - msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') - oprot.writeMessageBegin("get_group", msg_type, seqid) - result.write(oprot) - oprot.writeMessageEnd() - oprot.trans.flush() - - def process_del_group(self, seqid, iprot, oprot): - args = del_group_args() - args.read(iprot) - iprot.readMessageEnd() - result = del_group_result() - try: - self._handler.del_group(args.group_id) - msg_type = TMessageType.REPLY - except TTransport.TTransportException: - raise - except VecException as e: - msg_type = TMessageType.REPLY - result.e = e - except TApplicationException as ex: - logging.exception('TApplication exception in handler') - msg_type = TMessageType.EXCEPTION - result = ex - except Exception: - logging.exception('Unexpected exception in handler') - msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') - oprot.writeMessageBegin("del_group", msg_type, seqid) - result.write(oprot) - oprot.writeMessageEnd() - oprot.trans.flush() - - def process_add_vector(self, seqid, iprot, oprot): - args = add_vector_args() - args.read(iprot) - iprot.readMessageEnd() - result = add_vector_result() - try: - result.success = self._handler.add_vector(args.group_id, args.tensor) - msg_type = TMessageType.REPLY - except TTransport.TTransportException: - raise - except VecException as e: - msg_type = TMessageType.REPLY - result.e = e - except TApplicationException as ex: - logging.exception('TApplication exception in handler') - msg_type = TMessageType.EXCEPTION - result = ex - except Exception: - logging.exception('Unexpected exception in handler') - msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') - oprot.writeMessageBegin("add_vector", msg_type, seqid) - result.write(oprot) - oprot.writeMessageEnd() - oprot.trans.flush() - - def process_add_vector_batch(self, seqid, iprot, oprot): - args = add_vector_batch_args() - args.read(iprot) - iprot.readMessageEnd() - result = add_vector_batch_result() - try: - result.success = self._handler.add_vector_batch(args.group_id, args.tensor_list) - msg_type = TMessageType.REPLY - except TTransport.TTransportException: - raise - except VecException as e: - msg_type = TMessageType.REPLY - result.e = e - except TApplicationException as ex: - logging.exception('TApplication exception in handler') - msg_type = TMessageType.EXCEPTION - result = ex - except Exception: - logging.exception('Unexpected exception in handler') - msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') - oprot.writeMessageBegin("add_vector_batch", msg_type, seqid) - result.write(oprot) - oprot.writeMessageEnd() - oprot.trans.flush() - - def process_add_binary_vector(self, seqid, iprot, oprot): - args = add_binary_vector_args() - args.read(iprot) - iprot.readMessageEnd() - result = add_binary_vector_result() - try: - result.success = self._handler.add_binary_vector(args.group_id, args.tensor) - msg_type = TMessageType.REPLY - except TTransport.TTransportException: - raise - except VecException as e: - msg_type = TMessageType.REPLY - result.e = e - except TApplicationException as ex: - logging.exception('TApplication exception in handler') - msg_type = TMessageType.EXCEPTION - result = ex - except Exception: - logging.exception('Unexpected exception in handler') - msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') - oprot.writeMessageBegin("add_binary_vector", msg_type, seqid) - result.write(oprot) - oprot.writeMessageEnd() - oprot.trans.flush() - - def process_add_binary_vector_batch(self, seqid, iprot, oprot): - args = add_binary_vector_batch_args() - args.read(iprot) - iprot.readMessageEnd() - result = add_binary_vector_batch_result() - try: - result.success = self._handler.add_binary_vector_batch(args.group_id, args.tensor_list) - msg_type = TMessageType.REPLY - except TTransport.TTransportException: - raise - except VecException as e: - msg_type = TMessageType.REPLY - result.e = e - except TApplicationException as ex: - logging.exception('TApplication exception in handler') - msg_type = TMessageType.EXCEPTION - result = ex - except Exception: - logging.exception('Unexpected exception in handler') - msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') - oprot.writeMessageBegin("add_binary_vector_batch", msg_type, seqid) - result.write(oprot) - oprot.writeMessageEnd() - oprot.trans.flush() - - def process_search_vector(self, seqid, iprot, oprot): - args = search_vector_args() - args.read(iprot) - iprot.readMessageEnd() - result = search_vector_result() - try: - result.success = self._handler.search_vector(args.group_id, args.top_k, args.tensor, args.filter) - msg_type = TMessageType.REPLY - except TTransport.TTransportException: - raise - except VecException as e: - msg_type = TMessageType.REPLY - result.e = e - except TApplicationException as ex: - logging.exception('TApplication exception in handler') - msg_type = TMessageType.EXCEPTION - result = ex - except Exception: - logging.exception('Unexpected exception in handler') - msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') - oprot.writeMessageBegin("search_vector", msg_type, seqid) - result.write(oprot) - oprot.writeMessageEnd() - oprot.trans.flush() - - def process_search_vector_batch(self, seqid, iprot, oprot): - args = search_vector_batch_args() - args.read(iprot) - iprot.readMessageEnd() - result = search_vector_batch_result() - try: - result.success = self._handler.search_vector_batch(args.group_id, args.top_k, args.tensor_list, args.filter) - msg_type = TMessageType.REPLY - except TTransport.TTransportException: - raise - except VecException as e: - msg_type = TMessageType.REPLY - result.e = e - except TApplicationException as ex: - logging.exception('TApplication exception in handler') - msg_type = TMessageType.EXCEPTION - result = ex - except Exception: - logging.exception('Unexpected exception in handler') - msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') - oprot.writeMessageBegin("search_vector_batch", msg_type, seqid) - result.write(oprot) - oprot.writeMessageEnd() - oprot.trans.flush() - - def process_search_binary_vector(self, seqid, iprot, oprot): - args = search_binary_vector_args() - args.read(iprot) - iprot.readMessageEnd() - result = search_binary_vector_result() - try: - result.success = self._handler.search_binary_vector(args.group_id, args.top_k, args.tensor, args.filter) - msg_type = TMessageType.REPLY - except TTransport.TTransportException: - raise - except VecException as e: - msg_type = TMessageType.REPLY - result.e = e - except TApplicationException as ex: - logging.exception('TApplication exception in handler') - msg_type = TMessageType.EXCEPTION - result = ex - except Exception: - logging.exception('Unexpected exception in handler') - msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') - oprot.writeMessageBegin("search_binary_vector", msg_type, seqid) - result.write(oprot) - oprot.writeMessageEnd() - oprot.trans.flush() - - def process_search_binary_vector_batch(self, seqid, iprot, oprot): - args = search_binary_vector_batch_args() - args.read(iprot) - iprot.readMessageEnd() - result = search_binary_vector_batch_result() - try: - result.success = self._handler.search_binary_vector_batch(args.group_id, args.top_k, args.tensor_list, args.filter) - msg_type = TMessageType.REPLY - except TTransport.TTransportException: - raise - except VecException as e: - msg_type = TMessageType.REPLY - result.e = e - except TApplicationException as ex: - logging.exception('TApplication exception in handler') - msg_type = TMessageType.EXCEPTION - result = ex - except Exception: - logging.exception('Unexpected exception in handler') - msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') - oprot.writeMessageBegin("search_binary_vector_batch", msg_type, seqid) - result.write(oprot) - oprot.writeMessageEnd() - oprot.trans.flush() - -# HELPER FUNCTIONS AND STRUCTURES - - -class add_group_args(object): - """ - Attributes: - - group - """ - - - def __init__(self, group=None,): - self.group = group - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 2: - if ftype == TType.STRUCT: - self.group = VecGroup() - self.group.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('add_group_args') - if self.group is not None: - oprot.writeFieldBegin('group', TType.STRUCT, 2) - self.group.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(add_group_args) -add_group_args.thrift_spec = ( - None, # 0 - None, # 1 - (2, TType.STRUCT, 'group', [VecGroup, None], None, ), # 2 -) - - -class add_group_result(object): - """ - Attributes: - - e - """ - - - def __init__(self, e=None,): - self.e = e - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.STRUCT: - self.e = VecException() - self.e.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('add_group_result') - if self.e is not None: - oprot.writeFieldBegin('e', TType.STRUCT, 1) - self.e.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(add_group_result) -add_group_result.thrift_spec = ( - None, # 0 - (1, TType.STRUCT, 'e', [VecException, None], None, ), # 1 -) - - -class get_group_args(object): - """ - Attributes: - - group_id - """ - - - def __init__(self, group_id=None,): - self.group_id = group_id - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 2: - if ftype == TType.STRING: - self.group_id = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('get_group_args') - if self.group_id is not None: - oprot.writeFieldBegin('group_id', TType.STRING, 2) - oprot.writeString(self.group_id.encode('utf-8') if sys.version_info[0] == 2 else self.group_id) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(get_group_args) -get_group_args.thrift_spec = ( - None, # 0 - None, # 1 - (2, TType.STRING, 'group_id', 'UTF8', None, ), # 2 -) - - -class get_group_result(object): - """ - Attributes: - - success - - e - """ - - - def __init__(self, success=None, e=None,): - self.success = success - self.e = e - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 0: - if ftype == TType.STRUCT: - self.success = VecGroup() - self.success.read(iprot) - else: - iprot.skip(ftype) - elif fid == 1: - if ftype == TType.STRUCT: - self.e = VecException() - self.e.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('get_group_result') - if self.success is not None: - oprot.writeFieldBegin('success', TType.STRUCT, 0) - self.success.write(oprot) - oprot.writeFieldEnd() - if self.e is not None: - oprot.writeFieldBegin('e', TType.STRUCT, 1) - self.e.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(get_group_result) -get_group_result.thrift_spec = ( - (0, TType.STRUCT, 'success', [VecGroup, None], None, ), # 0 - (1, TType.STRUCT, 'e', [VecException, None], None, ), # 1 -) - - -class del_group_args(object): - """ - Attributes: - - group_id - """ - - - def __init__(self, group_id=None,): - self.group_id = group_id - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 2: - if ftype == TType.STRING: - self.group_id = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('del_group_args') - if self.group_id is not None: - oprot.writeFieldBegin('group_id', TType.STRING, 2) - oprot.writeString(self.group_id.encode('utf-8') if sys.version_info[0] == 2 else self.group_id) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(del_group_args) -del_group_args.thrift_spec = ( - None, # 0 - None, # 1 - (2, TType.STRING, 'group_id', 'UTF8', None, ), # 2 -) - - -class del_group_result(object): - """ - Attributes: - - e - """ - - - def __init__(self, e=None,): - self.e = e - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.STRUCT: - self.e = VecException() - self.e.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('del_group_result') - if self.e is not None: - oprot.writeFieldBegin('e', TType.STRUCT, 1) - self.e.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(del_group_result) -del_group_result.thrift_spec = ( - None, # 0 - (1, TType.STRUCT, 'e', [VecException, None], None, ), # 1 -) - - -class add_vector_args(object): - """ - Attributes: - - group_id - - tensor - """ - - - def __init__(self, group_id=None, tensor=None,): - self.group_id = group_id - self.tensor = tensor - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 2: - if ftype == TType.STRING: - self.group_id = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.STRUCT: - self.tensor = VecTensor() - self.tensor.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('add_vector_args') - if self.group_id is not None: - oprot.writeFieldBegin('group_id', TType.STRING, 2) - oprot.writeString(self.group_id.encode('utf-8') if sys.version_info[0] == 2 else self.group_id) - oprot.writeFieldEnd() - if self.tensor is not None: - oprot.writeFieldBegin('tensor', TType.STRUCT, 3) - self.tensor.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(add_vector_args) -add_vector_args.thrift_spec = ( - None, # 0 - None, # 1 - (2, TType.STRING, 'group_id', 'UTF8', None, ), # 2 - (3, TType.STRUCT, 'tensor', [VecTensor, None], None, ), # 3 -) - - -class add_vector_result(object): - """ - Attributes: - - success - - e - """ - - - def __init__(self, success=None, e=None,): - self.success = success - self.e = e - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 0: - if ftype == TType.STRING: - self.success = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - else: - iprot.skip(ftype) - elif fid == 1: - if ftype == TType.STRUCT: - self.e = VecException() - self.e.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('add_vector_result') - if self.success is not None: - oprot.writeFieldBegin('success', TType.STRING, 0) - oprot.writeString(self.success.encode('utf-8') if sys.version_info[0] == 2 else self.success) - oprot.writeFieldEnd() - if self.e is not None: - oprot.writeFieldBegin('e', TType.STRUCT, 1) - self.e.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(add_vector_result) -add_vector_result.thrift_spec = ( - (0, TType.STRING, 'success', 'UTF8', None, ), # 0 - (1, TType.STRUCT, 'e', [VecException, None], None, ), # 1 -) - - -class add_vector_batch_args(object): - """ - Attributes: - - group_id - - tensor_list - """ - - - def __init__(self, group_id=None, tensor_list=None,): - self.group_id = group_id - self.tensor_list = tensor_list - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 2: - if ftype == TType.STRING: - self.group_id = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.STRUCT: - self.tensor_list = VecTensorList() - self.tensor_list.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('add_vector_batch_args') - if self.group_id is not None: - oprot.writeFieldBegin('group_id', TType.STRING, 2) - oprot.writeString(self.group_id.encode('utf-8') if sys.version_info[0] == 2 else self.group_id) - oprot.writeFieldEnd() - if self.tensor_list is not None: - oprot.writeFieldBegin('tensor_list', TType.STRUCT, 3) - self.tensor_list.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(add_vector_batch_args) -add_vector_batch_args.thrift_spec = ( - None, # 0 - None, # 1 - (2, TType.STRING, 'group_id', 'UTF8', None, ), # 2 - (3, TType.STRUCT, 'tensor_list', [VecTensorList, None], None, ), # 3 -) - - -class add_vector_batch_result(object): - """ - Attributes: - - success - - e - """ - - - def __init__(self, success=None, e=None,): - self.success = success - self.e = e - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 0: - if ftype == TType.LIST: - self.success = [] - (_etype88, _size85) = iprot.readListBegin() - for _i89 in range(_size85): - _elem90 = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - self.success.append(_elem90) - iprot.readListEnd() - else: - iprot.skip(ftype) - elif fid == 1: - if ftype == TType.STRUCT: - self.e = VecException() - self.e.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('add_vector_batch_result') - if self.success is not None: - oprot.writeFieldBegin('success', TType.LIST, 0) - oprot.writeListBegin(TType.STRING, len(self.success)) - for iter91 in self.success: - oprot.writeString(iter91.encode('utf-8') if sys.version_info[0] == 2 else iter91) - oprot.writeListEnd() - oprot.writeFieldEnd() - if self.e is not None: - oprot.writeFieldBegin('e', TType.STRUCT, 1) - self.e.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(add_vector_batch_result) -add_vector_batch_result.thrift_spec = ( - (0, TType.LIST, 'success', (TType.STRING, 'UTF8', False), None, ), # 0 - (1, TType.STRUCT, 'e', [VecException, None], None, ), # 1 -) - - -class add_binary_vector_args(object): - """ - Attributes: - - group_id - - tensor - """ - - - def __init__(self, group_id=None, tensor=None,): - self.group_id = group_id - self.tensor = tensor - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 2: - if ftype == TType.STRING: - self.group_id = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.STRUCT: - self.tensor = VecBinaryTensor() - self.tensor.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('add_binary_vector_args') - if self.group_id is not None: - oprot.writeFieldBegin('group_id', TType.STRING, 2) - oprot.writeString(self.group_id.encode('utf-8') if sys.version_info[0] == 2 else self.group_id) - oprot.writeFieldEnd() - if self.tensor is not None: - oprot.writeFieldBegin('tensor', TType.STRUCT, 3) - self.tensor.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(add_binary_vector_args) -add_binary_vector_args.thrift_spec = ( - None, # 0 - None, # 1 - (2, TType.STRING, 'group_id', 'UTF8', None, ), # 2 - (3, TType.STRUCT, 'tensor', [VecBinaryTensor, None], None, ), # 3 -) - - -class add_binary_vector_result(object): - """ - Attributes: - - success - - e - """ - - - def __init__(self, success=None, e=None,): - self.success = success - self.e = e - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 0: - if ftype == TType.STRING: - self.success = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - else: - iprot.skip(ftype) - elif fid == 1: - if ftype == TType.STRUCT: - self.e = VecException() - self.e.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('add_binary_vector_result') - if self.success is not None: - oprot.writeFieldBegin('success', TType.STRING, 0) - oprot.writeString(self.success.encode('utf-8') if sys.version_info[0] == 2 else self.success) - oprot.writeFieldEnd() - if self.e is not None: - oprot.writeFieldBegin('e', TType.STRUCT, 1) - self.e.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(add_binary_vector_result) -add_binary_vector_result.thrift_spec = ( - (0, TType.STRING, 'success', 'UTF8', None, ), # 0 - (1, TType.STRUCT, 'e', [VecException, None], None, ), # 1 -) - - -class add_binary_vector_batch_args(object): - """ - Attributes: - - group_id - - tensor_list - """ - - - def __init__(self, group_id=None, tensor_list=None,): - self.group_id = group_id - self.tensor_list = tensor_list - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 2: - if ftype == TType.STRING: - self.group_id = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.STRUCT: - self.tensor_list = VecBinaryTensorList() - self.tensor_list.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('add_binary_vector_batch_args') - if self.group_id is not None: - oprot.writeFieldBegin('group_id', TType.STRING, 2) - oprot.writeString(self.group_id.encode('utf-8') if sys.version_info[0] == 2 else self.group_id) - oprot.writeFieldEnd() - if self.tensor_list is not None: - oprot.writeFieldBegin('tensor_list', TType.STRUCT, 3) - self.tensor_list.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(add_binary_vector_batch_args) -add_binary_vector_batch_args.thrift_spec = ( - None, # 0 - None, # 1 - (2, TType.STRING, 'group_id', 'UTF8', None, ), # 2 - (3, TType.STRUCT, 'tensor_list', [VecBinaryTensorList, None], None, ), # 3 -) - - -class add_binary_vector_batch_result(object): - """ - Attributes: - - success - - e - """ - - - def __init__(self, success=None, e=None,): - self.success = success - self.e = e - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 0: - if ftype == TType.LIST: - self.success = [] - (_etype95, _size92) = iprot.readListBegin() - for _i96 in range(_size92): - _elem97 = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - self.success.append(_elem97) - iprot.readListEnd() - else: - iprot.skip(ftype) - elif fid == 1: - if ftype == TType.STRUCT: - self.e = VecException() - self.e.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('add_binary_vector_batch_result') - if self.success is not None: - oprot.writeFieldBegin('success', TType.LIST, 0) - oprot.writeListBegin(TType.STRING, len(self.success)) - for iter98 in self.success: - oprot.writeString(iter98.encode('utf-8') if sys.version_info[0] == 2 else iter98) - oprot.writeListEnd() - oprot.writeFieldEnd() - if self.e is not None: - oprot.writeFieldBegin('e', TType.STRUCT, 1) - self.e.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(add_binary_vector_batch_result) -add_binary_vector_batch_result.thrift_spec = ( - (0, TType.LIST, 'success', (TType.STRING, 'UTF8', False), None, ), # 0 - (1, TType.STRUCT, 'e', [VecException, None], None, ), # 1 -) - - -class search_vector_args(object): - """ - Attributes: - - group_id - - top_k - - tensor - - filter - """ - - - def __init__(self, group_id=None, top_k=None, tensor=None, filter=None,): - self.group_id = group_id - self.top_k = top_k - self.tensor = tensor - self.filter = filter - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 2: - if ftype == TType.STRING: - self.group_id = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.I64: - self.top_k = iprot.readI64() - else: - iprot.skip(ftype) - elif fid == 4: - if ftype == TType.STRUCT: - self.tensor = VecTensor() - self.tensor.read(iprot) - else: - iprot.skip(ftype) - elif fid == 5: - if ftype == TType.STRUCT: - self.filter = VecSearchFilter() - self.filter.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('search_vector_args') - if self.group_id is not None: - oprot.writeFieldBegin('group_id', TType.STRING, 2) - oprot.writeString(self.group_id.encode('utf-8') if sys.version_info[0] == 2 else self.group_id) - oprot.writeFieldEnd() - if self.top_k is not None: - oprot.writeFieldBegin('top_k', TType.I64, 3) - oprot.writeI64(self.top_k) - oprot.writeFieldEnd() - if self.tensor is not None: - oprot.writeFieldBegin('tensor', TType.STRUCT, 4) - self.tensor.write(oprot) - oprot.writeFieldEnd() - if self.filter is not None: - oprot.writeFieldBegin('filter', TType.STRUCT, 5) - self.filter.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(search_vector_args) -search_vector_args.thrift_spec = ( - None, # 0 - None, # 1 - (2, TType.STRING, 'group_id', 'UTF8', None, ), # 2 - (3, TType.I64, 'top_k', None, None, ), # 3 - (4, TType.STRUCT, 'tensor', [VecTensor, None], None, ), # 4 - (5, TType.STRUCT, 'filter', [VecSearchFilter, None], None, ), # 5 -) - - -class search_vector_result(object): - """ - Attributes: - - success - - e - """ - - - def __init__(self, success=None, e=None,): - self.success = success - self.e = e - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 0: - if ftype == TType.STRUCT: - self.success = VecSearchResult() - self.success.read(iprot) - else: - iprot.skip(ftype) - elif fid == 1: - if ftype == TType.STRUCT: - self.e = VecException() - self.e.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('search_vector_result') - if self.success is not None: - oprot.writeFieldBegin('success', TType.STRUCT, 0) - self.success.write(oprot) - oprot.writeFieldEnd() - if self.e is not None: - oprot.writeFieldBegin('e', TType.STRUCT, 1) - self.e.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(search_vector_result) -search_vector_result.thrift_spec = ( - (0, TType.STRUCT, 'success', [VecSearchResult, None], None, ), # 0 - (1, TType.STRUCT, 'e', [VecException, None], None, ), # 1 -) - - -class search_vector_batch_args(object): - """ - Attributes: - - group_id - - top_k - - tensor_list - - filter - """ - - - def __init__(self, group_id=None, top_k=None, tensor_list=None, filter=None,): - self.group_id = group_id - self.top_k = top_k - self.tensor_list = tensor_list - self.filter = filter - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 2: - if ftype == TType.STRING: - self.group_id = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.I64: - self.top_k = iprot.readI64() - else: - iprot.skip(ftype) - elif fid == 4: - if ftype == TType.STRUCT: - self.tensor_list = VecTensorList() - self.tensor_list.read(iprot) - else: - iprot.skip(ftype) - elif fid == 5: - if ftype == TType.STRUCT: - self.filter = VecSearchFilter() - self.filter.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('search_vector_batch_args') - if self.group_id is not None: - oprot.writeFieldBegin('group_id', TType.STRING, 2) - oprot.writeString(self.group_id.encode('utf-8') if sys.version_info[0] == 2 else self.group_id) - oprot.writeFieldEnd() - if self.top_k is not None: - oprot.writeFieldBegin('top_k', TType.I64, 3) - oprot.writeI64(self.top_k) - oprot.writeFieldEnd() - if self.tensor_list is not None: - oprot.writeFieldBegin('tensor_list', TType.STRUCT, 4) - self.tensor_list.write(oprot) - oprot.writeFieldEnd() - if self.filter is not None: - oprot.writeFieldBegin('filter', TType.STRUCT, 5) - self.filter.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(search_vector_batch_args) -search_vector_batch_args.thrift_spec = ( - None, # 0 - None, # 1 - (2, TType.STRING, 'group_id', 'UTF8', None, ), # 2 - (3, TType.I64, 'top_k', None, None, ), # 3 - (4, TType.STRUCT, 'tensor_list', [VecTensorList, None], None, ), # 4 - (5, TType.STRUCT, 'filter', [VecSearchFilter, None], None, ), # 5 -) - - -class search_vector_batch_result(object): - """ - Attributes: - - success - - e - """ - - - def __init__(self, success=None, e=None,): - self.success = success - self.e = e - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 0: - if ftype == TType.STRUCT: - self.success = VecSearchResultList() - self.success.read(iprot) - else: - iprot.skip(ftype) - elif fid == 1: - if ftype == TType.STRUCT: - self.e = VecException() - self.e.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('search_vector_batch_result') - if self.success is not None: - oprot.writeFieldBegin('success', TType.STRUCT, 0) - self.success.write(oprot) - oprot.writeFieldEnd() - if self.e is not None: - oprot.writeFieldBegin('e', TType.STRUCT, 1) - self.e.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(search_vector_batch_result) -search_vector_batch_result.thrift_spec = ( - (0, TType.STRUCT, 'success', [VecSearchResultList, None], None, ), # 0 - (1, TType.STRUCT, 'e', [VecException, None], None, ), # 1 -) - - -class search_binary_vector_args(object): - """ - Attributes: - - group_id - - top_k - - tensor - - filter - """ - - - def __init__(self, group_id=None, top_k=None, tensor=None, filter=None,): - self.group_id = group_id - self.top_k = top_k - self.tensor = tensor - self.filter = filter - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 2: - if ftype == TType.STRING: - self.group_id = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.I64: - self.top_k = iprot.readI64() - else: - iprot.skip(ftype) - elif fid == 4: - if ftype == TType.STRUCT: - self.tensor = VecBinaryTensor() - self.tensor.read(iprot) - else: - iprot.skip(ftype) - elif fid == 5: - if ftype == TType.STRUCT: - self.filter = VecSearchFilter() - self.filter.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('search_binary_vector_args') - if self.group_id is not None: - oprot.writeFieldBegin('group_id', TType.STRING, 2) - oprot.writeString(self.group_id.encode('utf-8') if sys.version_info[0] == 2 else self.group_id) - oprot.writeFieldEnd() - if self.top_k is not None: - oprot.writeFieldBegin('top_k', TType.I64, 3) - oprot.writeI64(self.top_k) - oprot.writeFieldEnd() - if self.tensor is not None: - oprot.writeFieldBegin('tensor', TType.STRUCT, 4) - self.tensor.write(oprot) - oprot.writeFieldEnd() - if self.filter is not None: - oprot.writeFieldBegin('filter', TType.STRUCT, 5) - self.filter.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(search_binary_vector_args) -search_binary_vector_args.thrift_spec = ( - None, # 0 - None, # 1 - (2, TType.STRING, 'group_id', 'UTF8', None, ), # 2 - (3, TType.I64, 'top_k', None, None, ), # 3 - (4, TType.STRUCT, 'tensor', [VecBinaryTensor, None], None, ), # 4 - (5, TType.STRUCT, 'filter', [VecSearchFilter, None], None, ), # 5 -) - - -class search_binary_vector_result(object): - """ - Attributes: - - success - - e - """ - - - def __init__(self, success=None, e=None,): - self.success = success - self.e = e - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 0: - if ftype == TType.STRUCT: - self.success = VecSearchResult() - self.success.read(iprot) - else: - iprot.skip(ftype) - elif fid == 1: - if ftype == TType.STRUCT: - self.e = VecException() - self.e.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('search_binary_vector_result') - if self.success is not None: - oprot.writeFieldBegin('success', TType.STRUCT, 0) - self.success.write(oprot) - oprot.writeFieldEnd() - if self.e is not None: - oprot.writeFieldBegin('e', TType.STRUCT, 1) - self.e.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(search_binary_vector_result) -search_binary_vector_result.thrift_spec = ( - (0, TType.STRUCT, 'success', [VecSearchResult, None], None, ), # 0 - (1, TType.STRUCT, 'e', [VecException, None], None, ), # 1 -) - - -class search_binary_vector_batch_args(object): - """ - Attributes: - - group_id - - top_k - - tensor_list - - filter - """ - - - def __init__(self, group_id=None, top_k=None, tensor_list=None, filter=None,): - self.group_id = group_id - self.top_k = top_k - self.tensor_list = tensor_list - self.filter = filter - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 2: - if ftype == TType.STRING: - self.group_id = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.I64: - self.top_k = iprot.readI64() - else: - iprot.skip(ftype) - elif fid == 4: - if ftype == TType.STRUCT: - self.tensor_list = VecBinaryTensorList() - self.tensor_list.read(iprot) - else: - iprot.skip(ftype) - elif fid == 5: - if ftype == TType.STRUCT: - self.filter = VecSearchFilter() - self.filter.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('search_binary_vector_batch_args') - if self.group_id is not None: - oprot.writeFieldBegin('group_id', TType.STRING, 2) - oprot.writeString(self.group_id.encode('utf-8') if sys.version_info[0] == 2 else self.group_id) - oprot.writeFieldEnd() - if self.top_k is not None: - oprot.writeFieldBegin('top_k', TType.I64, 3) - oprot.writeI64(self.top_k) - oprot.writeFieldEnd() - if self.tensor_list is not None: - oprot.writeFieldBegin('tensor_list', TType.STRUCT, 4) - self.tensor_list.write(oprot) - oprot.writeFieldEnd() - if self.filter is not None: - oprot.writeFieldBegin('filter', TType.STRUCT, 5) - self.filter.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(search_binary_vector_batch_args) -search_binary_vector_batch_args.thrift_spec = ( - None, # 0 - None, # 1 - (2, TType.STRING, 'group_id', 'UTF8', None, ), # 2 - (3, TType.I64, 'top_k', None, None, ), # 3 - (4, TType.STRUCT, 'tensor_list', [VecBinaryTensorList, None], None, ), # 4 - (5, TType.STRUCT, 'filter', [VecSearchFilter, None], None, ), # 5 -) - - -class search_binary_vector_batch_result(object): - """ - Attributes: - - success - - e - """ - - - def __init__(self, success=None, e=None,): - self.success = success - self.e = e - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 0: - if ftype == TType.STRUCT: - self.success = VecSearchResultList() - self.success.read(iprot) - else: - iprot.skip(ftype) - elif fid == 1: - if ftype == TType.STRUCT: - self.e = VecException() - self.e.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('search_binary_vector_batch_result') - if self.success is not None: - oprot.writeFieldBegin('success', TType.STRUCT, 0) - self.success.write(oprot) - oprot.writeFieldEnd() - if self.e is not None: - oprot.writeFieldBegin('e', TType.STRUCT, 1) - self.e.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(search_binary_vector_batch_result) -search_binary_vector_batch_result.thrift_spec = ( - (0, TType.STRUCT, 'success', [VecSearchResultList, None], None, ), # 0 - (1, TType.STRUCT, 'e', [VecException, None], None, ), # 1 -) -fix_spec(all_structs) -del all_structs - diff --git a/cpp/src/thrift/gen-py/zilliz/__init__.py b/cpp/src/thrift/gen-py/zilliz/__init__.py deleted file mode 100644 index fa8ce63771..0000000000 --- a/cpp/src/thrift/gen-py/zilliz/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__all__ = ['ttypes', 'constants', 'VecService'] diff --git a/cpp/src/thrift/gen-py/zilliz/constants.py b/cpp/src/thrift/gen-py/zilliz/constants.py deleted file mode 100644 index 0c217ceda6..0000000000 --- a/cpp/src/thrift/gen-py/zilliz/constants.py +++ /dev/null @@ -1,14 +0,0 @@ -# -# Autogenerated by Thrift Compiler (0.11.0) -# -# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING -# -# options string: py -# - -from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException -from thrift.protocol.TProtocol import TProtocolException -from thrift.TRecursive import fix_spec - -import sys -from .ttypes import * diff --git a/cpp/src/thrift/gen-py/zilliz/ttypes.py b/cpp/src/thrift/gen-py/zilliz/ttypes.py deleted file mode 100644 index bfe014667d..0000000000 --- a/cpp/src/thrift/gen-py/zilliz/ttypes.py +++ /dev/null @@ -1,1165 +0,0 @@ -# -# Autogenerated by Thrift Compiler (0.11.0) -# -# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING -# -# options string: py -# - -from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException -from thrift.protocol.TProtocol import TProtocolException -from thrift.TRecursive import fix_spec - -import sys - -from thrift.transport import TTransport -all_structs = [] - - -class VecErrCode(object): - SUCCESS = 0 - ILLEGAL_ARGUMENT = 1 - GROUP_NOT_EXISTS = 2 - ILLEGAL_TIME_RANGE = 3 - ILLEGAL_VECTOR_DIMENSION = 4 - OUT_OF_MEMORY = 5 - - _VALUES_TO_NAMES = { - 0: "SUCCESS", - 1: "ILLEGAL_ARGUMENT", - 2: "GROUP_NOT_EXISTS", - 3: "ILLEGAL_TIME_RANGE", - 4: "ILLEGAL_VECTOR_DIMENSION", - 5: "OUT_OF_MEMORY", - } - - _NAMES_TO_VALUES = { - "SUCCESS": 0, - "ILLEGAL_ARGUMENT": 1, - "GROUP_NOT_EXISTS": 2, - "ILLEGAL_TIME_RANGE": 3, - "ILLEGAL_VECTOR_DIMENSION": 4, - "OUT_OF_MEMORY": 5, - } - - -class VecException(TException): - """ - Attributes: - - code - - reason - """ - - - def __init__(self, code=None, reason=None,): - self.code = code - self.reason = reason - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.I32: - self.code = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 2: - if ftype == TType.STRING: - self.reason = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('VecException') - if self.code is not None: - oprot.writeFieldBegin('code', TType.I32, 1) - oprot.writeI32(self.code) - oprot.writeFieldEnd() - if self.reason is not None: - oprot.writeFieldBegin('reason', TType.STRING, 2) - oprot.writeString(self.reason.encode('utf-8') if sys.version_info[0] == 2 else self.reason) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __str__(self): - return repr(self) - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - - -class VecGroup(object): - """ - Attributes: - - id - - dimension - - index_type - """ - - - def __init__(self, id=None, dimension=None, index_type=None,): - self.id = id - self.dimension = dimension - self.index_type = index_type - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.STRING: - self.id = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - else: - iprot.skip(ftype) - elif fid == 2: - if ftype == TType.I32: - self.dimension = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.I32: - self.index_type = iprot.readI32() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('VecGroup') - if self.id is not None: - oprot.writeFieldBegin('id', TType.STRING, 1) - oprot.writeString(self.id.encode('utf-8') if sys.version_info[0] == 2 else self.id) - oprot.writeFieldEnd() - if self.dimension is not None: - oprot.writeFieldBegin('dimension', TType.I32, 2) - oprot.writeI32(self.dimension) - oprot.writeFieldEnd() - if self.index_type is not None: - oprot.writeFieldBegin('index_type', TType.I32, 3) - oprot.writeI32(self.index_type) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - if self.id is None: - raise TProtocolException(message='Required field id is unset!') - if self.dimension is None: - raise TProtocolException(message='Required field dimension is unset!') - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - - -class VecTensor(object): - """ - Attributes: - - uid - - tensor - - attrib - """ - - - def __init__(self, uid=None, tensor=None, attrib=None,): - self.uid = uid - self.tensor = tensor - self.attrib = attrib - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.STRING: - self.uid = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - else: - iprot.skip(ftype) - elif fid == 2: - if ftype == TType.LIST: - self.tensor = [] - (_etype3, _size0) = iprot.readListBegin() - for _i4 in range(_size0): - _elem5 = iprot.readDouble() - self.tensor.append(_elem5) - iprot.readListEnd() - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.MAP: - self.attrib = {} - (_ktype7, _vtype8, _size6) = iprot.readMapBegin() - for _i10 in range(_size6): - _key11 = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - _val12 = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - self.attrib[_key11] = _val12 - iprot.readMapEnd() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('VecTensor') - if self.uid is not None: - oprot.writeFieldBegin('uid', TType.STRING, 1) - oprot.writeString(self.uid.encode('utf-8') if sys.version_info[0] == 2 else self.uid) - oprot.writeFieldEnd() - if self.tensor is not None: - oprot.writeFieldBegin('tensor', TType.LIST, 2) - oprot.writeListBegin(TType.DOUBLE, len(self.tensor)) - for iter13 in self.tensor: - oprot.writeDouble(iter13) - oprot.writeListEnd() - oprot.writeFieldEnd() - if self.attrib is not None: - oprot.writeFieldBegin('attrib', TType.MAP, 3) - oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.attrib)) - for kiter14, viter15 in self.attrib.items(): - oprot.writeString(kiter14.encode('utf-8') if sys.version_info[0] == 2 else kiter14) - oprot.writeString(viter15.encode('utf-8') if sys.version_info[0] == 2 else viter15) - oprot.writeMapEnd() - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - if self.uid is None: - raise TProtocolException(message='Required field uid is unset!') - if self.tensor is None: - raise TProtocolException(message='Required field tensor is unset!') - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - - -class VecTensorList(object): - """ - Attributes: - - tensor_list - """ - - - def __init__(self, tensor_list=None,): - self.tensor_list = tensor_list - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.LIST: - self.tensor_list = [] - (_etype19, _size16) = iprot.readListBegin() - for _i20 in range(_size16): - _elem21 = VecTensor() - _elem21.read(iprot) - self.tensor_list.append(_elem21) - iprot.readListEnd() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('VecTensorList') - if self.tensor_list is not None: - oprot.writeFieldBegin('tensor_list', TType.LIST, 1) - oprot.writeListBegin(TType.STRUCT, len(self.tensor_list)) - for iter22 in self.tensor_list: - iter22.write(oprot) - oprot.writeListEnd() - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - if self.tensor_list is None: - raise TProtocolException(message='Required field tensor_list is unset!') - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - - -class VecBinaryTensor(object): - """ - Attributes: - - uid - - tensor - - attrib - """ - - - def __init__(self, uid=None, tensor=None, attrib=None,): - self.uid = uid - self.tensor = tensor - self.attrib = attrib - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.STRING: - self.uid = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - else: - iprot.skip(ftype) - elif fid == 2: - if ftype == TType.STRING: - self.tensor = iprot.readBinary() - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.MAP: - self.attrib = {} - (_ktype24, _vtype25, _size23) = iprot.readMapBegin() - for _i27 in range(_size23): - _key28 = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - _val29 = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - self.attrib[_key28] = _val29 - iprot.readMapEnd() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('VecBinaryTensor') - if self.uid is not None: - oprot.writeFieldBegin('uid', TType.STRING, 1) - oprot.writeString(self.uid.encode('utf-8') if sys.version_info[0] == 2 else self.uid) - oprot.writeFieldEnd() - if self.tensor is not None: - oprot.writeFieldBegin('tensor', TType.STRING, 2) - oprot.writeBinary(self.tensor) - oprot.writeFieldEnd() - if self.attrib is not None: - oprot.writeFieldBegin('attrib', TType.MAP, 3) - oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.attrib)) - for kiter30, viter31 in self.attrib.items(): - oprot.writeString(kiter30.encode('utf-8') if sys.version_info[0] == 2 else kiter30) - oprot.writeString(viter31.encode('utf-8') if sys.version_info[0] == 2 else viter31) - oprot.writeMapEnd() - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - if self.uid is None: - raise TProtocolException(message='Required field uid is unset!') - if self.tensor is None: - raise TProtocolException(message='Required field tensor is unset!') - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - - -class VecBinaryTensorList(object): - """ - Attributes: - - tensor_list - """ - - - def __init__(self, tensor_list=None,): - self.tensor_list = tensor_list - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.LIST: - self.tensor_list = [] - (_etype35, _size32) = iprot.readListBegin() - for _i36 in range(_size32): - _elem37 = VecBinaryTensor() - _elem37.read(iprot) - self.tensor_list.append(_elem37) - iprot.readListEnd() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('VecBinaryTensorList') - if self.tensor_list is not None: - oprot.writeFieldBegin('tensor_list', TType.LIST, 1) - oprot.writeListBegin(TType.STRUCT, len(self.tensor_list)) - for iter38 in self.tensor_list: - iter38.write(oprot) - oprot.writeListEnd() - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - if self.tensor_list is None: - raise TProtocolException(message='Required field tensor_list is unset!') - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - - -class VecSearchResultItem(object): - """ - Attributes: - - uid - - distance - - attrib - """ - - - def __init__(self, uid=None, distance=None, attrib=None,): - self.uid = uid - self.distance = distance - self.attrib = attrib - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.STRING: - self.uid = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - else: - iprot.skip(ftype) - elif fid == 2: - if ftype == TType.DOUBLE: - self.distance = iprot.readDouble() - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.MAP: - self.attrib = {} - (_ktype40, _vtype41, _size39) = iprot.readMapBegin() - for _i43 in range(_size39): - _key44 = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - _val45 = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - self.attrib[_key44] = _val45 - iprot.readMapEnd() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('VecSearchResultItem') - if self.uid is not None: - oprot.writeFieldBegin('uid', TType.STRING, 1) - oprot.writeString(self.uid.encode('utf-8') if sys.version_info[0] == 2 else self.uid) - oprot.writeFieldEnd() - if self.distance is not None: - oprot.writeFieldBegin('distance', TType.DOUBLE, 2) - oprot.writeDouble(self.distance) - oprot.writeFieldEnd() - if self.attrib is not None: - oprot.writeFieldBegin('attrib', TType.MAP, 3) - oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.attrib)) - for kiter46, viter47 in self.attrib.items(): - oprot.writeString(kiter46.encode('utf-8') if sys.version_info[0] == 2 else kiter46) - oprot.writeString(viter47.encode('utf-8') if sys.version_info[0] == 2 else viter47) - oprot.writeMapEnd() - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - if self.uid is None: - raise TProtocolException(message='Required field uid is unset!') - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - - -class VecSearchResult(object): - """ - Attributes: - - result_list - """ - - - def __init__(self, result_list=None,): - self.result_list = result_list - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.LIST: - self.result_list = [] - (_etype51, _size48) = iprot.readListBegin() - for _i52 in range(_size48): - _elem53 = VecSearchResultItem() - _elem53.read(iprot) - self.result_list.append(_elem53) - iprot.readListEnd() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('VecSearchResult') - if self.result_list is not None: - oprot.writeFieldBegin('result_list', TType.LIST, 1) - oprot.writeListBegin(TType.STRUCT, len(self.result_list)) - for iter54 in self.result_list: - iter54.write(oprot) - oprot.writeListEnd() - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - - -class VecSearchResultList(object): - """ - Attributes: - - result_list - """ - - - def __init__(self, result_list=None,): - self.result_list = result_list - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.LIST: - self.result_list = [] - (_etype58, _size55) = iprot.readListBegin() - for _i59 in range(_size55): - _elem60 = VecSearchResult() - _elem60.read(iprot) - self.result_list.append(_elem60) - iprot.readListEnd() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('VecSearchResultList') - if self.result_list is not None: - oprot.writeFieldBegin('result_list', TType.LIST, 1) - oprot.writeListBegin(TType.STRUCT, len(self.result_list)) - for iter61 in self.result_list: - iter61.write(oprot) - oprot.writeListEnd() - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - - -class VecDateTime(object): - """ - second; Seconds. [0-59] reserved - minute; Minutes. [0-59] reserved - hour; Hours. [0-23] reserved - day; Day. [1-31] - month; Month. [0-11] - year; Year - 1900. - - Attributes: - - year - - month - - day - - hour - - minute - - second - """ - - - def __init__(self, year=None, month=None, day=None, hour=None, minute=None, second=None,): - self.year = year - self.month = month - self.day = day - self.hour = hour - self.minute = minute - self.second = second - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.I32: - self.year = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 2: - if ftype == TType.I32: - self.month = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.I32: - self.day = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 4: - if ftype == TType.I32: - self.hour = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 5: - if ftype == TType.I32: - self.minute = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 6: - if ftype == TType.I32: - self.second = iprot.readI32() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('VecDateTime') - if self.year is not None: - oprot.writeFieldBegin('year', TType.I32, 1) - oprot.writeI32(self.year) - oprot.writeFieldEnd() - if self.month is not None: - oprot.writeFieldBegin('month', TType.I32, 2) - oprot.writeI32(self.month) - oprot.writeFieldEnd() - if self.day is not None: - oprot.writeFieldBegin('day', TType.I32, 3) - oprot.writeI32(self.day) - oprot.writeFieldEnd() - if self.hour is not None: - oprot.writeFieldBegin('hour', TType.I32, 4) - oprot.writeI32(self.hour) - oprot.writeFieldEnd() - if self.minute is not None: - oprot.writeFieldBegin('minute', TType.I32, 5) - oprot.writeI32(self.minute) - oprot.writeFieldEnd() - if self.second is not None: - oprot.writeFieldBegin('second', TType.I32, 6) - oprot.writeI32(self.second) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - if self.year is None: - raise TProtocolException(message='Required field year is unset!') - if self.month is None: - raise TProtocolException(message='Required field month is unset!') - if self.day is None: - raise TProtocolException(message='Required field day is unset!') - if self.hour is None: - raise TProtocolException(message='Required field hour is unset!') - if self.minute is None: - raise TProtocolException(message='Required field minute is unset!') - if self.second is None: - raise TProtocolException(message='Required field second is unset!') - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - - -class VecTimeRange(object): - """ - time_begin; time range begin - begine_closed; true means '[', false means '(' reserved - time_end; set to true to return tensor double array - end_closed; time range end reserved - - Attributes: - - time_begin - - begine_closed - - time_end - - end_closed - """ - - - def __init__(self, time_begin=None, begine_closed=None, time_end=None, end_closed=None,): - self.time_begin = time_begin - self.begine_closed = begine_closed - self.time_end = time_end - self.end_closed = end_closed - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.STRUCT: - self.time_begin = VecDateTime() - self.time_begin.read(iprot) - else: - iprot.skip(ftype) - elif fid == 2: - if ftype == TType.BOOL: - self.begine_closed = iprot.readBool() - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.STRUCT: - self.time_end = VecDateTime() - self.time_end.read(iprot) - else: - iprot.skip(ftype) - elif fid == 4: - if ftype == TType.BOOL: - self.end_closed = iprot.readBool() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('VecTimeRange') - if self.time_begin is not None: - oprot.writeFieldBegin('time_begin', TType.STRUCT, 1) - self.time_begin.write(oprot) - oprot.writeFieldEnd() - if self.begine_closed is not None: - oprot.writeFieldBegin('begine_closed', TType.BOOL, 2) - oprot.writeBool(self.begine_closed) - oprot.writeFieldEnd() - if self.time_end is not None: - oprot.writeFieldBegin('time_end', TType.STRUCT, 3) - self.time_end.write(oprot) - oprot.writeFieldEnd() - if self.end_closed is not None: - oprot.writeFieldBegin('end_closed', TType.BOOL, 4) - oprot.writeBool(self.end_closed) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - if self.time_begin is None: - raise TProtocolException(message='Required field time_begin is unset!') - if self.begine_closed is None: - raise TProtocolException(message='Required field begine_closed is unset!') - if self.time_end is None: - raise TProtocolException(message='Required field time_end is unset!') - if self.end_closed is None: - raise TProtocolException(message='Required field end_closed is unset!') - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - - -class VecSearchFilter(object): - """ - attrib_filter; reserved - time_ranges; search condition, for example: "date between 1999-02-12 and 2008-10-14" - return_attribs; specify required attribute names - - Attributes: - - attrib_filter - - time_ranges - - return_attribs - """ - - - def __init__(self, attrib_filter=None, time_ranges=None, return_attribs=None,): - self.attrib_filter = attrib_filter - self.time_ranges = time_ranges - self.return_attribs = return_attribs - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.MAP: - self.attrib_filter = {} - (_ktype63, _vtype64, _size62) = iprot.readMapBegin() - for _i66 in range(_size62): - _key67 = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - _val68 = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - self.attrib_filter[_key67] = _val68 - iprot.readMapEnd() - else: - iprot.skip(ftype) - elif fid == 2: - if ftype == TType.LIST: - self.time_ranges = [] - (_etype72, _size69) = iprot.readListBegin() - for _i73 in range(_size69): - _elem74 = VecTimeRange() - _elem74.read(iprot) - self.time_ranges.append(_elem74) - iprot.readListEnd() - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.LIST: - self.return_attribs = [] - (_etype78, _size75) = iprot.readListBegin() - for _i79 in range(_size75): - _elem80 = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - self.return_attribs.append(_elem80) - iprot.readListEnd() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('VecSearchFilter') - if self.attrib_filter is not None: - oprot.writeFieldBegin('attrib_filter', TType.MAP, 1) - oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.attrib_filter)) - for kiter81, viter82 in self.attrib_filter.items(): - oprot.writeString(kiter81.encode('utf-8') if sys.version_info[0] == 2 else kiter81) - oprot.writeString(viter82.encode('utf-8') if sys.version_info[0] == 2 else viter82) - oprot.writeMapEnd() - oprot.writeFieldEnd() - if self.time_ranges is not None: - oprot.writeFieldBegin('time_ranges', TType.LIST, 2) - oprot.writeListBegin(TType.STRUCT, len(self.time_ranges)) - for iter83 in self.time_ranges: - iter83.write(oprot) - oprot.writeListEnd() - oprot.writeFieldEnd() - if self.return_attribs is not None: - oprot.writeFieldBegin('return_attribs', TType.LIST, 3) - oprot.writeListBegin(TType.STRING, len(self.return_attribs)) - for iter84 in self.return_attribs: - oprot.writeString(iter84.encode('utf-8') if sys.version_info[0] == 2 else iter84) - oprot.writeListEnd() - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(VecException) -VecException.thrift_spec = ( - None, # 0 - (1, TType.I32, 'code', None, None, ), # 1 - (2, TType.STRING, 'reason', 'UTF8', None, ), # 2 -) -all_structs.append(VecGroup) -VecGroup.thrift_spec = ( - None, # 0 - (1, TType.STRING, 'id', 'UTF8', None, ), # 1 - (2, TType.I32, 'dimension', None, None, ), # 2 - (3, TType.I32, 'index_type', None, None, ), # 3 -) -all_structs.append(VecTensor) -VecTensor.thrift_spec = ( - None, # 0 - (1, TType.STRING, 'uid', 'UTF8', None, ), # 1 - (2, TType.LIST, 'tensor', (TType.DOUBLE, None, False), None, ), # 2 - (3, TType.MAP, 'attrib', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 3 -) -all_structs.append(VecTensorList) -VecTensorList.thrift_spec = ( - None, # 0 - (1, TType.LIST, 'tensor_list', (TType.STRUCT, [VecTensor, None], False), None, ), # 1 -) -all_structs.append(VecBinaryTensor) -VecBinaryTensor.thrift_spec = ( - None, # 0 - (1, TType.STRING, 'uid', 'UTF8', None, ), # 1 - (2, TType.STRING, 'tensor', 'BINARY', None, ), # 2 - (3, TType.MAP, 'attrib', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 3 -) -all_structs.append(VecBinaryTensorList) -VecBinaryTensorList.thrift_spec = ( - None, # 0 - (1, TType.LIST, 'tensor_list', (TType.STRUCT, [VecBinaryTensor, None], False), None, ), # 1 -) -all_structs.append(VecSearchResultItem) -VecSearchResultItem.thrift_spec = ( - None, # 0 - (1, TType.STRING, 'uid', 'UTF8', None, ), # 1 - (2, TType.DOUBLE, 'distance', None, None, ), # 2 - (3, TType.MAP, 'attrib', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 3 -) -all_structs.append(VecSearchResult) -VecSearchResult.thrift_spec = ( - None, # 0 - (1, TType.LIST, 'result_list', (TType.STRUCT, [VecSearchResultItem, None], False), None, ), # 1 -) -all_structs.append(VecSearchResultList) -VecSearchResultList.thrift_spec = ( - None, # 0 - (1, TType.LIST, 'result_list', (TType.STRUCT, [VecSearchResult, None], False), None, ), # 1 -) -all_structs.append(VecDateTime) -VecDateTime.thrift_spec = ( - None, # 0 - (1, TType.I32, 'year', None, None, ), # 1 - (2, TType.I32, 'month', None, None, ), # 2 - (3, TType.I32, 'day', None, None, ), # 3 - (4, TType.I32, 'hour', None, None, ), # 4 - (5, TType.I32, 'minute', None, None, ), # 5 - (6, TType.I32, 'second', None, None, ), # 6 -) -all_structs.append(VecTimeRange) -VecTimeRange.thrift_spec = ( - None, # 0 - (1, TType.STRUCT, 'time_begin', [VecDateTime, None], None, ), # 1 - (2, TType.BOOL, 'begine_closed', None, None, ), # 2 - (3, TType.STRUCT, 'time_end', [VecDateTime, None], None, ), # 3 - (4, TType.BOOL, 'end_closed', None, None, ), # 4 -) -all_structs.append(VecSearchFilter) -VecSearchFilter.thrift_spec = ( - None, # 0 - (1, TType.MAP, 'attrib_filter', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 1 - (2, TType.LIST, 'time_ranges', (TType.STRUCT, [VecTimeRange, None], False), None, ), # 2 - (3, TType.LIST, 'return_attribs', (TType.STRING, 'UTF8', False), None, ), # 3 -) -fix_spec(all_structs) -del all_structs diff --git a/cpp/src/thrift/megasearch.thrift b/cpp/src/thrift/megasearch.thrift index e008e4b19d..3565f3c5df 100644 --- a/cpp/src/thrift/megasearch.thrift +++ b/cpp/src/thrift/megasearch.thrift @@ -3,16 +3,16 @@ * Unauthorized copying of this file, via any medium is strictly prohibited. * Proprietary and confidential. ******************************************************************************/ -namespace cl zilliz -namespace cpp zilliz -namespace py zilliz -namespace d zilliz -namespace dart zilliz -namespace java zilliz -namespace perl zilliz -namespace php zilliz -namespace haxe zilliz -namespace netcore zilliz +namespace cl megasearch +namespace cpp megasearch +namespace py megasearch +namespace d megasearch +namespace dart megasearch +namespace java megasearch +namespace perl megasearch +namespace php megasearch +namespace haxe megasearch +namespace netcore megasearch enum VecErrCode { SUCCESS = 0, diff --git a/cpp/test_client/src/ClientSession.cpp b/cpp/test_client/src/ClientSession.cpp index 05d60f8c7e..dac71d1ff5 100644 --- a/cpp/test_client/src/ClientSession.cpp +++ b/cpp/test_client/src/ClientSession.cpp @@ -26,6 +26,8 @@ namespace zilliz { namespace vecwise { namespace client { +using namespace megasearch; + using namespace ::apache::thrift; using namespace ::apache::thrift::protocol; using namespace ::apache::thrift::transport; diff --git a/cpp/test_client/src/ClientSession.h b/cpp/test_client/src/ClientSession.h index 1cb4911aa2..44cf54adb2 100644 --- a/cpp/test_client/src/ClientSession.h +++ b/cpp/test_client/src/ClientSession.h @@ -13,7 +13,7 @@ namespace zilliz { namespace vecwise { namespace client { -using VecServiceClientPtr = std::shared_ptr; +using VecServiceClientPtr = std::shared_ptr; class ClientSession { public: diff --git a/cpp/test_client/src/ClientTest.cpp b/cpp/test_client/src/ClientTest.cpp index e4e0a604de..3a11e8bcc3 100644 --- a/cpp/test_client/src/ClientTest.cpp +++ b/cpp/test_client/src/ClientTest.cpp @@ -15,6 +15,7 @@ #include +using namespace megasearch; using namespace zilliz; using namespace zilliz::vecwise; using namespace zilliz::vecwise::client; @@ -129,23 +130,23 @@ TEST(AddVector, CLIENT_TEST) { //verify get invalid group try { std::string id; - zilliz::VecTensor tensor; + VecTensor tensor; for(int32_t i = 0; i < VEC_DIMENSION; i++) { tensor.tensor.push_back(0.5); } session.interface()->add_vector(id, GetGroupID(), tensor); - } catch (zilliz::VecException& ex) { + } catch (VecException& ex) { CLIENT_LOG_ERROR << "request encounter exception: " << ex.what(); - ASSERT_EQ(ex.code, zilliz::VecErrCode::ILLEGAL_ARGUMENT); + ASSERT_EQ(ex.code, VecErrCode::ILLEGAL_ARGUMENT); } try { VecGroup temp_group; session.interface()->get_group(temp_group, GetGroupID()); //ASSERT_TRUE(temp_group.id.empty()); - } catch (zilliz::VecException& ex) { + } catch (VecException& ex) { CLIENT_LOG_ERROR << "request encounter exception: " << ex.what(); - ASSERT_EQ(ex.code, zilliz::VecErrCode::GROUP_NOT_EXISTS); + ASSERT_EQ(ex.code, VecErrCode::GROUP_NOT_EXISTS); } //add group From 12502fbfbe9d433f47a1ecea1f26163d6e9dcbef Mon Sep 17 00:00:00 2001 From: "xj.lin" Date: Tue, 14 May 2019 15:36:27 +0800 Subject: [PATCH 14/16] update build file Former-commit-id: c8b7f83f2f018c5e8396f4b8a9c8f38d59f0e546 From 59544b03418cb55c1d28ea0db24900a3fd140d76 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Wed, 15 May 2019 11:47:21 +0800 Subject: [PATCH 15/16] fix(db): fix cache error after load file from disk Former-commit-id: 23dd81d168caa49451634ef7a97c6fa618c92957 --- cpp/src/db/FaissExecutionEngine.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cpp/src/db/FaissExecutionEngine.cpp b/cpp/src/db/FaissExecutionEngine.cpp index cc0f1046d6..9f70318a6b 100644 --- a/cpp/src/db/FaissExecutionEngine.cpp +++ b/cpp/src/db/FaissExecutionEngine.cpp @@ -64,13 +64,16 @@ Status FaissExecutionEngine::Serialize() { template Status FaissExecutionEngine::Load() { auto index = zilliz::vecwise::cache::CpuCacheMgr::GetInstance()->GetIndex(location_); + bool to_cache; if (!index) { index = read_index(location_); - Cache(); LOG(DEBUG) << "Disk io from: " << location_; } pIndex_ = index->data(); + if (to_cache) { + Cache(); + } return Status::OK(); } From a6853a5726bfe9e18a165a6d1d70078e09be4e47 Mon Sep 17 00:00:00 2001 From: Xu Peng Date: Wed, 15 May 2019 13:21:58 +0800 Subject: [PATCH 16/16] fix(db): fix cache error after load file from disk part 2 Former-commit-id: e684913260dee0f4e5c39729f12e29925c1fb118 --- cpp/src/db/FaissExecutionEngine.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/src/db/FaissExecutionEngine.cpp b/cpp/src/db/FaissExecutionEngine.cpp index 9f70318a6b..605b979481 100644 --- a/cpp/src/db/FaissExecutionEngine.cpp +++ b/cpp/src/db/FaissExecutionEngine.cpp @@ -64,9 +64,10 @@ Status FaissExecutionEngine::Serialize() { template Status FaissExecutionEngine::Load() { auto index = zilliz::vecwise::cache::CpuCacheMgr::GetInstance()->GetIndex(location_); - bool to_cache; + bool to_cache = false; if (!index) { index = read_index(location_); + to_cache = true; LOG(DEBUG) << "Disk io from: " << location_; }