diff --git a/.gitignore b/.gitignore index 87503b8b61..d239dd72a6 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ cmake_build cpp/third_party/thrift-0.12.0/ cpp/third_party/faiss-1.5.1 +cpp/megasearch/ diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index d5dbdcc05e..61fb44d429 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -83,8 +83,8 @@ add_custom_target(Clean-All COMMAND ${CMAKE_BUILD_TOOL} clean) #install install(FILES - start_server.sh - stop_server.sh + scripts/start_server.sh + scripts/stop_server.sh DESTINATION scripts) install(FILES diff --git a/cpp/README.md b/cpp/README.md index d8cb62991b..c870304ea4 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -23,6 +23,7 @@ cmake_build/src/libvecwise_engine.a is the static library cd [sourcecode path]/cpp ./build.sh -t Debug ./build.sh -t Release + ./build.sh -g # Build GPU version #### To build unittest: diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml index cd66e0efb5..bb109d5352 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/conf/server_config.yaml @@ -1,5 +1,5 @@ server_config: - address: 127.0.0.1 + address: 0.0.0.0 port: 33001 transfer_protocol: json #optional: binary, compact, json, debug server_mode: thread_pool #optional: simple, thread_pool diff --git a/cpp/scripts/start_server.sh b/cpp/scripts/start_server.sh new file mode 100755 index 0000000000..6f5db9490c --- /dev/null +++ b/cpp/scripts/start_server.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +../bin/vecwise_server -c ../conf/server_config.yaml -l ../conf/vecwise_engine_log.conf + diff --git a/cpp/scripts/stop_server.sh b/cpp/scripts/stop_server.sh new file mode 100755 index 0000000000..89bd9398e0 --- /dev/null +++ b/cpp/scripts/stop_server.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +function kill_progress() +{ + kill -s SIGUSR2 $(pgrep $1) + + sleep 2 +} + +STATUS=$(kill_progress "vecwise_server" ) + +if [[ ${STATUS} == "false" ]];then + echo "vecwise_server closed abnormally!" +else + echo "vecwise_server closed successfully!" +fi diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 61918b562b..d7978db37d 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -23,8 +23,8 @@ set(license_generator_src set(service_files thrift/gen-cpp/VecService.cpp - thrift/gen-cpp/VectorService_constants.cpp - thrift/gen-cpp/VectorService_types.cpp + thrift/gen-cpp/megasearch_constants.cpp + thrift/gen-cpp/megasearch_types.cpp ) set(vecwise_engine_src diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index f977222535..216a9b352d 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -107,7 +107,7 @@ Status DBImpl::search(const std::string& group_id, size_t k, size_t nq, using SearchResult = std::pair, std::vector>; std::vector batchresult(nq); // allocate nq cells. - auto cluster = [&](long *nns, float *dis) -> void { + auto cluster = [&](long *nns, float *dis, const int& k) -> void { for (int i = 0; i < nq; ++i) { auto f_begin = batchresult[i].first.cbegin(); auto s_begin = batchresult[i].second.cbegin(); @@ -134,8 +134,10 @@ Status DBImpl::search(const std::string& group_id, size_t k, size_t nq, search_set_size += file_size; LOG(DEBUG) << "Search file_type " << file.file_type << " Of Size: " << file_size << " M"; - index.Search(nq, vectors, k, output_distence, output_ids); - cluster(output_ids, output_distence); // cluster to each query + + int inner_k = index.Count() < k ? index.Count() : k; + index.Search(nq, vectors, inner_k, output_distence, output_ids); + cluster(output_ids, output_distence, inner_k); // cluster to each query memset(output_distence, 0, k * nq * sizeof(float)); memset(output_ids, 0, k * nq * sizeof(long)); } @@ -145,15 +147,25 @@ Status DBImpl::search(const std::string& group_id, size_t k, size_t nq, const int &k, float *output_distence, long *output_ids) -> void { - std::map inverted_table; + std::map> inverted_table; for (int i = 0; i < input_data.size(); ++i) { - inverted_table[input_data[i]] = i; + if (inverted_table.count(input_data[i]) == 1) { + auto& ori_vec = inverted_table[input_data[i]]; + ori_vec.push_back(i); + } + else { + inverted_table[input_data[i]] = std::vector{i}; + } } int count = 0; - for (auto it = inverted_table.begin(); it != inverted_table.end() && count < k; ++it, ++count) { - output_distence[count] = it->first; - output_ids[count] = it->second; + for (auto &item : inverted_table){ + if (count == k) break; + for (auto &id : item.second){ + output_distence[count] = item.first; + output_ids[count] = id; + if (++count == k) break; + } } }; auto cluster_topk = [&]() -> void { @@ -161,8 +173,11 @@ Status DBImpl::search(const std::string& group_id, size_t k, size_t nq, for (auto &result_pair : batchresult) { auto &dis = result_pair.second; auto &nns = result_pair.first; + topk_cpu(dis, k, output_distence, output_ids); - for (int i = 0; i < k; ++i) { + + int inner_k = dis.size() < k ? dis.size() : k; + for (int i = 0; i < inner_k; ++i) { res.emplace_back(nns[output_ids[i]]); // mapping } results.push_back(res); // append to result list diff --git a/cpp/src/db/FaissExecutionEngine.cpp b/cpp/src/db/FaissExecutionEngine.cpp index cc0f1046d6..605b979481 100644 --- a/cpp/src/db/FaissExecutionEngine.cpp +++ b/cpp/src/db/FaissExecutionEngine.cpp @@ -64,13 +64,17 @@ Status FaissExecutionEngine::Serialize() { template Status FaissExecutionEngine::Load() { auto index = zilliz::vecwise::cache::CpuCacheMgr::GetInstance()->GetIndex(location_); + bool to_cache = false; if (!index) { index = read_index(location_); - Cache(); + to_cache = true; LOG(DEBUG) << "Disk io from: " << location_; } pIndex_ = index->data(); + if (to_cache) { + Cache(); + } return Status::OK(); } diff --git a/cpp/src/server/RocksIdMapper.cpp b/cpp/src/server/RocksIdMapper.cpp new file mode 100644 index 0000000000..2dba544243 --- /dev/null +++ b/cpp/src/server/RocksIdMapper.cpp @@ -0,0 +1,242 @@ +/******************************************************************************* + * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved + * Unauthorized copying of this file, via any medium is strictly prohibited. + * Proprietary and confidential. + ******************************************************************************/ + +#include "RocksIdMapper.h" +#include "ServerConfig.h" +#include "utils/Log.h" +#include "utils/CommonUtil.h" + +#include "rocksdb/slice.h" +#include "rocksdb/options.h" + +#include + +namespace zilliz { +namespace vecwise { +namespace server { + +RocksIdMapper::RocksIdMapper() +: db_(nullptr) { + OpenDb(); +} + +RocksIdMapper::~RocksIdMapper() { + CloseDb(); +} + +void RocksIdMapper::OpenDb() { + if(db_) { + return; + } + + ConfigNode& config = ServerConfig::GetInstance().GetConfig(CONFIG_DB); + std::string db_path = config.GetValue(CONFIG_DB_PATH); + db_path += "/id_mapping"; + CommonUtil::CreateDirectory(db_path); + + rocksdb::Options options; + // Optimize RocksDB. This is the easiest way to get RocksDB to perform well + options.IncreaseParallelism(); + options.OptimizeLevelStyleCompaction(); + // create the DB if it's not already present + options.create_if_missing = true; + options.max_open_files = config.GetInt32Value(CONFIG_DB_IDMAPPER_MAX_FILE, 512); + + //load column families + std::vector column_names; + rocksdb::Status s = rocksdb::DB::ListColumnFamilies(options, db_path, &column_names); + if (!s.ok()) { + SERVER_LOG_ERROR << "ID mapper failed to initialize:" << s.ToString(); + } + + if(column_names.empty()) { + column_names.push_back("default"); + } + SERVER_LOG_INFO << "ID mapper has " << std::to_string(column_names.size()) << " groups"; + + std::vector column_families; + for(auto& column_name : column_names) { + rocksdb::ColumnFamilyDescriptor desc; + desc.name = column_name; + column_families.emplace_back(desc); + } + + // open DB + std::vector column_handles; + s = rocksdb::DB::Open(options, db_path, column_families, &column_handles, &db_); + if(!s.ok()) { + SERVER_LOG_ERROR << "ID mapper failed to initialize:" << s.ToString(); + db_ = nullptr; + } + + column_handles_.clear(); + for(auto handler : column_handles) { + column_handles_.insert(std::make_pair(handler->GetName(), handler)); + } +} + +void RocksIdMapper::CloseDb() { + for(auto& iter : column_handles_) { + delete iter.second; + } + column_handles_.clear(); + + if(db_) { + db_->Close(); + delete db_; + } +} + +ServerError RocksIdMapper::Put(const std::string& nid, const std::string& sid, const std::string& group) { + if(db_ == nullptr) { + return SERVER_NULL_POINTER; + } + + rocksdb::Slice key(nid); + rocksdb::Slice value(sid); + if(group.empty()) {//to default group + rocksdb::Status s = db_->Put(rocksdb::WriteOptions(), key, value); + if (!s.ok()) { + SERVER_LOG_ERROR << "ID mapper failed to put:" << s.ToString(); + return SERVER_UNEXPECTED_ERROR; + } + } else { + rocksdb::ColumnFamilyHandle *cfh = nullptr; + if(column_handles_.count(group) == 0) { + try {//add group + rocksdb::Status s = db_->CreateColumnFamily(rocksdb::ColumnFamilyOptions(), group, &cfh); + if (!s.ok()) { + SERVER_LOG_ERROR << "ID mapper failed to create group:" << s.ToString(); + } else { + column_handles_.insert(std::make_pair(group, cfh)); + } + } catch(std::exception& ex) { + std::cout << ex.what() << std::endl; + } + } else { + cfh = column_handles_[group]; + } + + rocksdb::Status s = db_->Put(rocksdb::WriteOptions(), cfh, key, value); + if (!s.ok()) { + SERVER_LOG_ERROR << "ID mapper failed to put:" << s.ToString(); + return SERVER_UNEXPECTED_ERROR; + } + } + + return SERVER_SUCCESS; +} + +ServerError RocksIdMapper::Put(const std::vector& nid, const std::vector& sid, const std::string& group) { + if(nid.size() != sid.size()) { + return SERVER_INVALID_ARGUMENT; + } + + ServerError err = SERVER_SUCCESS; + for(size_t i = 0; i < nid.size(); i++) { + err = Put(nid[i], sid[i], group); + if(err != SERVER_SUCCESS) { + return err; + } + } + + return err; +} + +ServerError RocksIdMapper::Get(const std::string& nid, std::string& sid, const std::string& group) const { + sid = ""; + if(db_ == nullptr) { + return SERVER_NULL_POINTER; + } + + rocksdb::ColumnFamilyHandle *cfh = nullptr; + if(column_handles_.count(group) != 0) { + cfh = column_handles_.at(group); + } + + rocksdb::Slice key(nid); + rocksdb::Status s; + if(cfh){ + s = db_->Get(rocksdb::ReadOptions(), cfh, key, &sid); + } else { + s = db_->Get(rocksdb::ReadOptions(), key, &sid); + } + + if(!s.ok()) { + SERVER_LOG_ERROR << "ID mapper failed to get:" << s.ToString(); + return SERVER_UNEXPECTED_ERROR; + } + + return SERVER_SUCCESS; +} + +ServerError RocksIdMapper::Get(const std::vector& nid, std::vector& sid, const std::string& group) const { + sid.clear(); + + ServerError err = SERVER_SUCCESS; + for(size_t i = 0; i < nid.size(); i++) { + std::string str_id; + ServerError temp_err = Get(nid[i], str_id, group); + if(temp_err != SERVER_SUCCESS) { + sid.push_back(""); + SERVER_LOG_ERROR << "ID mapper failed to get id: " << nid[i]; + err = temp_err; + continue; + } + + sid.push_back(str_id); + } + + return err; +} + +ServerError RocksIdMapper::Delete(const std::string& nid, const std::string& group) { + if(db_ == nullptr) { + return SERVER_NULL_POINTER; + } + + rocksdb::ColumnFamilyHandle *cfh = nullptr; + if(column_handles_.count(group) != 0) { + cfh = column_handles_.at(group); + } + + rocksdb::Slice key(nid); + rocksdb::Status s; + if(cfh){ + s = db_->Delete(rocksdb::WriteOptions(), cfh, key); + } else { + s = db_->Delete(rocksdb::WriteOptions(), key); + } + if(!s.ok()) { + SERVER_LOG_ERROR << "ID mapper failed to delete:" << s.ToString(); + return SERVER_UNEXPECTED_ERROR; + } + + return SERVER_SUCCESS; +} + +ServerError RocksIdMapper::DeleteGroup(const std::string& group) { + if(db_ == nullptr) { + return SERVER_NULL_POINTER; + } + + rocksdb::ColumnFamilyHandle *cfh = nullptr; + if(column_handles_.count(group) != 0) { + cfh = column_handles_.at(group); + } + + if(cfh) { + db_->DropColumnFamily(cfh); + db_->DestroyColumnFamilyHandle(cfh); + column_handles_.erase(group); + } + + return SERVER_SUCCESS; +} + +} +} +} \ No newline at end of file diff --git a/cpp/src/server/RocksIdMapper.h b/cpp/src/server/RocksIdMapper.h new file mode 100644 index 0000000000..8c73155903 --- /dev/null +++ b/cpp/src/server/RocksIdMapper.h @@ -0,0 +1,46 @@ +/******************************************************************************* + * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved + * Unauthorized copying of this file, via any medium is strictly prohibited. + * Proprietary and confidential. + ******************************************************************************/ +#pragma once + +#include "utils/Error.h" +#include "VecIdMapper.h" + +#include "rocksdb/db.h" + +#include +#include +#include + +namespace zilliz { +namespace vecwise { +namespace server { + +class RocksIdMapper : public IVecIdMapper{ +public: + RocksIdMapper(); + ~RocksIdMapper(); + + ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") override; + ServerError Put(const std::vector& nid, const std::vector& sid, const std::string& group = "") override; + + ServerError Get(const std::string& nid, std::string& sid, const std::string& group = "") const override; + ServerError Get(const std::vector& nid, std::vector& sid, const std::string& group = "") const override; + + ServerError Delete(const std::string& nid, const std::string& group = "") override; + ServerError DeleteGroup(const std::string& group) override; + +private: + void OpenDb(); + void CloseDb(); + +private: + rocksdb::DB* db_; + std::unordered_map column_handles_; +}; + +} +} +} diff --git a/cpp/src/server/VecIdMapper.cpp b/cpp/src/server/VecIdMapper.cpp index 2a50d4db2d..ecf5058a4b 100644 --- a/cpp/src/server/VecIdMapper.cpp +++ b/cpp/src/server/VecIdMapper.cpp @@ -5,6 +5,7 @@ ******************************************************************************/ #include "VecIdMapper.h" +#include "RocksIdMapper.h" #include "ServerConfig.h" #include "utils/Log.h" #include "utils/CommonUtil.h" @@ -13,6 +14,8 @@ #include "rocksdb/slice.h" #include "rocksdb/options.h" +#include + namespace zilliz { namespace vecwise { namespace server { @@ -36,26 +39,33 @@ SimpleIdMapper::~SimpleIdMapper() { } -ServerError SimpleIdMapper::Put(const std::string& nid, const std::string& sid) { - ids_[nid] = sid; +//not thread-safe +ServerError SimpleIdMapper::Put(const std::string& nid, const std::string& sid, const std::string& group) { + ID_MAPPING& mapping = id_groups_[group]; + mapping[nid] = sid; return SERVER_SUCCESS; } -ServerError SimpleIdMapper::Put(const std::vector& nid, const std::vector& sid) { +//not thread-safe +ServerError SimpleIdMapper::Put(const std::vector& nid, const std::vector& sid, const std::string& group) { if(nid.size() != sid.size()) { return SERVER_INVALID_ARGUMENT; } + ID_MAPPING& mapping = id_groups_[group]; for(size_t i = 0; i < nid.size(); i++) { - ids_[nid[i]] = sid[i]; + mapping[nid[i]] = sid[i]; } return SERVER_SUCCESS; } -ServerError SimpleIdMapper::Get(const std::string& nid, std::string& sid) const { - auto iter = ids_.find(nid); - if(iter == ids_.end()) { +//not thread-safe +ServerError SimpleIdMapper::Get(const std::string& nid, std::string& sid, const std::string& group) const { + ID_MAPPING& mapping = id_groups_[group]; + + auto iter = mapping.find(nid); + if(iter == mapping.end()) { return SERVER_INVALID_ARGUMENT; } @@ -64,13 +74,16 @@ ServerError SimpleIdMapper::Get(const std::string& nid, std::string& sid) const return SERVER_SUCCESS; } -ServerError SimpleIdMapper::Get(const std::vector& nid, std::vector& sid) const { +//not thread-safe +ServerError SimpleIdMapper::Get(const std::vector& nid, std::vector& sid, const std::string& group) const { sid.clear(); + ID_MAPPING& mapping = id_groups_[group]; + ServerError err = SERVER_SUCCESS; for(size_t i = 0; i < nid.size(); i++) { - auto iter = ids_.find(nid[i]); - if(iter == ids_.end()) { + auto iter = mapping.find(nid[i]); + if(iter == mapping.end()) { sid.push_back(""); SERVER_LOG_ERROR << "ID mapper failed to find id: " << nid[i]; err = SERVER_INVALID_ARGUMENT; @@ -83,120 +96,16 @@ ServerError SimpleIdMapper::Get(const std::vector& nid, std::vector return err; } -ServerError SimpleIdMapper::Delete(const std::string& nid) { - ids_.erase(nid); +//not thread-safe +ServerError SimpleIdMapper::Delete(const std::string& nid, const std::string& group) { + ID_MAPPING& mapping = id_groups_[group]; + mapping.erase(nid); return SERVER_SUCCESS; } - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -RocksIdMapper::RocksIdMapper() { - ConfigNode& config = ServerConfig::GetInstance().GetConfig(CONFIG_DB); - std::string db_path = config.GetValue(CONFIG_DB_PATH); - db_path += "/id_mapping"; - CommonUtil::CreateDirectory(db_path); - - rocksdb::Options options; - // Optimize RocksDB. This is the easiest way to get RocksDB to perform well - options.IncreaseParallelism(); - options.OptimizeLevelStyleCompaction(); - // create the DB if it's not already present - options.create_if_missing = true; - options.max_open_files = config.GetInt32Value(CONFIG_DB_IDMAPPER_MAX_FILE, 128); - - // open DB - rocksdb::Status s = rocksdb::DB::Open(options, db_path, &db_); - if(!s.ok()) { - SERVER_LOG_ERROR << "ID mapper failed to initialize:" << s.ToString(); - db_ = nullptr; - } -} -RocksIdMapper::~RocksIdMapper() { - if(db_) { - db_->Close(); - delete db_; - } -} - -ServerError RocksIdMapper::Put(const std::string& nid, const std::string& sid) { - if(db_ == nullptr) { - return SERVER_NULL_POINTER; - } - - rocksdb::Slice key(nid); - rocksdb::Slice value(sid); - rocksdb::Status s = db_->Put(rocksdb::WriteOptions(), key, value); - if(!s.ok()) { - SERVER_LOG_ERROR << "ID mapper failed to put:" << s.ToString(); - return SERVER_UNEXPECTED_ERROR; - } - - return SERVER_SUCCESS; -} - -ServerError RocksIdMapper::Put(const std::vector& nid, const std::vector& sid) { - if(nid.size() != sid.size()) { - return SERVER_INVALID_ARGUMENT; - } - - ServerError err = SERVER_SUCCESS; - for(size_t i = 0; i < nid.size(); i++) { - err = Put(nid[i], sid[i]); - if(err != SERVER_SUCCESS) { - return err; - } - } - - return err; -} - -ServerError RocksIdMapper::Get(const std::string& nid, std::string& sid) const { - if(db_ == nullptr) { - return SERVER_NULL_POINTER; - } - - rocksdb::Slice key(nid); - rocksdb::Status s = db_->Get(rocksdb::ReadOptions(), key, &sid); - if(!s.ok()) { - SERVER_LOG_ERROR << "ID mapper failed to get:" << s.ToString(); - return SERVER_UNEXPECTED_ERROR; - } - - return SERVER_SUCCESS; -} - -ServerError RocksIdMapper::Get(const std::vector& nid, std::vector& sid) const { - sid.clear(); - - ServerError err = SERVER_SUCCESS; - for(size_t i = 0; i < nid.size(); i++) { - std::string str_id; - ServerError temp_err = Get(nid[i], str_id); - if(temp_err != SERVER_SUCCESS) { - sid.push_back(""); - SERVER_LOG_ERROR << "ID mapper failed to get id: " << nid[i]; - err = temp_err; - continue; - } - - sid.push_back(str_id); - } - - return err; -} - -ServerError RocksIdMapper::Delete(const std::string& nid) { - if(db_ == nullptr) { - return SERVER_NULL_POINTER; - } - - rocksdb::Slice key(nid); - rocksdb::Status s = db_->Delete(rocksdb::WriteOptions(), key); - if(!s.ok()) { - SERVER_LOG_ERROR << "ID mapper failed to delete:" << s.ToString(); - return SERVER_UNEXPECTED_ERROR; - } - +//not thread-safe +ServerError SimpleIdMapper::DeleteGroup(const std::string& group) { + id_groups_.erase(group); return SERVER_SUCCESS; } diff --git a/cpp/src/server/VecIdMapper.h b/cpp/src/server/VecIdMapper.h index b376c12638..9bb6d500da 100644 --- a/cpp/src/server/VecIdMapper.h +++ b/cpp/src/server/VecIdMapper.h @@ -25,14 +25,15 @@ public: virtual ~IVecIdMapper(){} - virtual ServerError Put(const std::string& nid, const std::string& sid) = 0; - virtual ServerError Put(const std::vector& nid, const std::vector& sid) = 0; + virtual ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") = 0; + virtual ServerError Put(const std::vector& nid, const std::vector& sid, const std::string& group = "") = 0; - virtual ServerError Get(const std::string& nid, std::string& sid) const = 0; + virtual ServerError Get(const std::string& nid, std::string& sid, const std::string& group = "") const = 0; //NOTE: the 'sid' will be cleared at begin of the function - virtual ServerError Get(const std::vector& nid, std::vector& sid) const = 0; + virtual ServerError Get(const std::vector& nid, std::vector& sid, const std::string& group = "") const = 0; - virtual ServerError Delete(const std::string& nid) = 0; + virtual ServerError Delete(const std::string& nid, const std::string& group = "") = 0; + virtual ServerError DeleteGroup(const std::string& group) = 0; }; class SimpleIdMapper : public IVecIdMapper{ @@ -40,33 +41,18 @@ public: SimpleIdMapper(); ~SimpleIdMapper(); - ServerError Put(const std::string& nid, const std::string& sid) override; - ServerError Put(const std::vector& nid, const std::vector& sid) override; + ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") override; + ServerError Put(const std::vector& nid, const std::vector& sid, const std::string& group = "") override; - ServerError Get(const std::string& nid, std::string& sid) const override; - ServerError Get(const std::vector& nid, std::vector& sid) const override; + ServerError Get(const std::string& nid, std::string& sid, const std::string& group = "") const override; + ServerError Get(const std::vector& nid, std::vector& sid, const std::string& group = "") const override; - ServerError Delete(const std::string& nid) override; + ServerError Delete(const std::string& nid, const std::string& group = "") override; + ServerError DeleteGroup(const std::string& group) override; private: - std::unordered_map ids_; -}; - -class RocksIdMapper : public IVecIdMapper{ -public: - RocksIdMapper(); - ~RocksIdMapper(); - - ServerError Put(const std::string& nid, const std::string& sid) override; - ServerError Put(const std::vector& nid, const std::vector& sid) override; - - ServerError Get(const std::string& nid, std::string& sid) const override; - ServerError Get(const std::vector& nid, std::vector& sid) const override; - - ServerError Delete(const std::string& nid) override; - -private: - rocksdb::DB* db_; + using ID_MAPPING = std::unordered_map; + mutable std::unordered_map id_groups_; }; } diff --git a/cpp/src/server/VecServiceHandler.cpp b/cpp/src/server/VecServiceHandler.cpp index de5942cd8b..c8db470e63 100644 --- a/cpp/src/server/VecServiceHandler.cpp +++ b/cpp/src/server/VecServiceHandler.cpp @@ -6,7 +6,6 @@ #include "VecServiceHandler.h" #include "VecServiceTask.h" #include "ServerConfig.h" -#include "VecIdMapper.h" #include "utils/Log.h" #include "utils/CommonUtil.h" #include "utils/TimeRecorder.h" @@ -18,17 +17,19 @@ namespace zilliz { namespace vecwise { namespace server { +using namespace megasearch; + namespace { class TimeRecordWrapper { public: TimeRecordWrapper(const std::string& func_name) : recorder_(func_name), func_name_(func_name) { - SERVER_LOG_TRACE << func_name << " called"; + //SERVER_LOG_TRACE << func_name << " called"; } ~TimeRecordWrapper() { recorder_.Elapse("cost"); - SERVER_LOG_TRACE << func_name_ << " finished"; + //SERVER_LOG_TRACE << func_name_ << " finished"; } private: @@ -38,87 +39,128 @@ namespace { void TimeRecord(const std::string& func_name) { } + + const std::map& ErrorMap() { + static const std::map code_map = { + {SERVER_UNEXPECTED_ERROR, VecErrCode::ILLEGAL_ARGUMENT}, + {SERVER_NULL_POINTER, VecErrCode::ILLEGAL_ARGUMENT}, + {SERVER_INVALID_ARGUMENT, VecErrCode::ILLEGAL_ARGUMENT}, + {SERVER_FILE_NOT_FOUND, VecErrCode::ILLEGAL_ARGUMENT}, + {SERVER_NOT_IMPLEMENT, VecErrCode::ILLEGAL_ARGUMENT}, + {SERVER_BLOCKING_QUEUE_EMPTY, VecErrCode::ILLEGAL_ARGUMENT}, + {SERVER_GROUP_NOT_EXIST, VecErrCode::GROUP_NOT_EXISTS}, + {SERVER_INVALID_TIME_RANGE, VecErrCode::ILLEGAL_TIME_RANGE}, + {SERVER_INVALID_VECTOR_DIMENSION, VecErrCode::ILLEGAL_VECTOR_DIMENSION}, + }; + + return code_map; + } + + const std::map& ErrorMessage() { + static const std::map msg_map = { + {SERVER_UNEXPECTED_ERROR, "unexpected error occurs"}, + {SERVER_NULL_POINTER, "null pointer error"}, + {SERVER_INVALID_ARGUMENT, "invalid argument"}, + {SERVER_FILE_NOT_FOUND, "file not found"}, + {SERVER_NOT_IMPLEMENT, "not implemented"}, + {SERVER_BLOCKING_QUEUE_EMPTY, "queue empty"}, + {SERVER_GROUP_NOT_EXIST, "group not exist"}, + {SERVER_INVALID_TIME_RANGE, "invalid time range"}, + {SERVER_INVALID_VECTOR_DIMENSION, "invalid vector dimension"}, + }; + + return msg_map; + } + + void ExecTask(BaseTaskPtr& task_ptr) { + if(task_ptr == nullptr) { + return; + } + + VecServiceScheduler& scheduler = VecServiceScheduler::GetInstance(); + scheduler.ExecuteTask(task_ptr); + + if(!task_ptr->IsAsync()) { + task_ptr->WaitToFinish(); + ServerError err = task_ptr->ErrorCode(); + if (err != SERVER_SUCCESS) { + VecException ex; + ex.__set_code(ErrorMap().at(err)); + std::string msg = task_ptr->ErrorMsg(); + if(msg.empty()){ + msg = ErrorMessage().at(err); + } + ex.__set_reason(msg); + throw ex; + } + } + } } void VecServiceHandler::add_group(const VecGroup &group) { - TimeRecordWrapper rc("add_group()"); - SERVER_LOG_TRACE << "group.id = " << group.id << ", group.dimension = " << group.dimension - << ", group.index_type = " << group.index_type; + std::string info = "add_group() " + group.id + " dimension = " + std::to_string(group.dimension) + + " index_type = " + std::to_string(group.index_type); + TimeRecordWrapper rc(info); BaseTaskPtr task_ptr = AddGroupTask::Create(group.dimension, group.id); - VecServiceScheduler& scheduler = VecServiceScheduler::GetInstance(); - scheduler.ExecuteTask(task_ptr); + ExecTask(task_ptr); } void VecServiceHandler::get_group(VecGroup &_return, const std::string &group_id) { - TimeRecordWrapper rc("get_group()"); - SERVER_LOG_TRACE << "group_id = " << group_id; + TimeRecordWrapper rc("get_group() " + group_id); _return.id = group_id; BaseTaskPtr task_ptr = GetGroupTask::Create(group_id, _return.dimension); - VecServiceScheduler& scheduler = VecServiceScheduler::GetInstance(); - scheduler.ExecuteTask(task_ptr); + ExecTask(task_ptr); } void VecServiceHandler::del_group(const std::string &group_id) { - TimeRecordWrapper rc("del_group()"); - SERVER_LOG_TRACE << "group_id = " << group_id; + TimeRecordWrapper rc("del_group() " + group_id); BaseTaskPtr task_ptr = DeleteGroupTask::Create(group_id); - VecServiceScheduler& scheduler = VecServiceScheduler::GetInstance(); - scheduler.ExecuteTask(task_ptr); + ExecTask(task_ptr); } void VecServiceHandler::add_vector(std::string& _return, const std::string &group_id, const VecTensor &tensor) { - TimeRecordWrapper rc("add_vector()"); - SERVER_LOG_TRACE << "group_id = " << group_id << ", vector size = " << tensor.tensor.size(); + TimeRecordWrapper rc("add_vector() to " + group_id); BaseTaskPtr task_ptr = AddVectorTask::Create(group_id, &tensor, _return); - VecServiceScheduler& scheduler = VecServiceScheduler::GetInstance(); - scheduler.ExecuteTask(task_ptr); + ExecTask(task_ptr); } void VecServiceHandler::add_vector_batch(std::vector & _return, const std::string &group_id, const VecTensorList &tensor_list) { - TimeRecordWrapper rc("add_vector_batch()"); - SERVER_LOG_TRACE << "group_id = " << group_id << ", vector list size = " - << tensor_list.tensor_list.size(); + TimeRecordWrapper rc("add_vector_batch() to " + group_id); BaseTaskPtr task_ptr = AddBatchVectorTask::Create(group_id, &tensor_list, _return); - VecServiceScheduler& scheduler = VecServiceScheduler::GetInstance(); - scheduler.ExecuteTask(task_ptr); + ExecTask(task_ptr); } void VecServiceHandler::add_binary_vector(std::string& _return, const std::string& group_id, const VecBinaryTensor& tensor) { - TimeRecordWrapper rc("add_binary_vector()"); - SERVER_LOG_TRACE << "group_id = " << group_id << ", vector size = " << tensor.tensor.size()/4; + TimeRecordWrapper rc("add_binary_vector() to " + group_id); BaseTaskPtr task_ptr = AddVectorTask::Create(group_id, &tensor, _return); - VecServiceScheduler& scheduler = VecServiceScheduler::GetInstance(); - scheduler.ExecuteTask(task_ptr); + ExecTask(task_ptr); } void VecServiceHandler::add_binary_vector_batch(std::vector & _return, const std::string& group_id, const VecBinaryTensorList& tensor_list) { - TimeRecordWrapper rc("add_binary_vector_batch()"); - SERVER_LOG_TRACE << "group_id = " << group_id << ", vector list size = " - << tensor_list.tensor_list.size(); + TimeRecordWrapper rc("add_binary_vector_batch() to " + group_id); BaseTaskPtr task_ptr = AddBatchVectorTask::Create(group_id, &tensor_list, _return); - VecServiceScheduler& scheduler = VecServiceScheduler::GetInstance(); - scheduler.ExecuteTask(task_ptr); + ExecTask(task_ptr); } void @@ -127,16 +169,13 @@ VecServiceHandler::search_vector(VecSearchResult &_return, const int64_t top_k, const VecTensor &tensor, const VecSearchFilter& filter) { - TimeRecordWrapper rc("search_vector()"); - SERVER_LOG_TRACE << "group_id = " << group_id << ", top_k = " << top_k - << ", vector dimension = " << tensor.tensor.size(); + TimeRecordWrapper rc("search_vector() in " + group_id); VecTensorList tensor_list; tensor_list.tensor_list.push_back(tensor); VecSearchResultList result; BaseTaskPtr task_ptr = SearchVectorTask::Create(group_id, top_k, &tensor_list, filter, result); - VecServiceScheduler& scheduler = VecServiceScheduler::GetInstance(); - scheduler.ExecuteTask(task_ptr); + ExecTask(task_ptr); if(!result.result_list.empty()) { _return = result.result_list[0]; @@ -151,13 +190,10 @@ VecServiceHandler::search_vector_batch(VecSearchResultList &_return, const int64_t top_k, const VecTensorList &tensor_list, const VecSearchFilter& filter) { - TimeRecordWrapper rc("search_vector_batch()"); - SERVER_LOG_TRACE << "group_id = " << group_id << ", top_k = " << top_k - << ", vector list size = " << tensor_list.tensor_list.size(); + TimeRecordWrapper rc("search_vector_batch() in " + group_id); BaseTaskPtr task_ptr = SearchVectorTask::Create(group_id, top_k, &tensor_list, filter, _return); - VecServiceScheduler& scheduler = VecServiceScheduler::GetInstance(); - scheduler.ExecuteTask(task_ptr); + ExecTask(task_ptr); } void @@ -166,16 +202,13 @@ VecServiceHandler::search_binary_vector(VecSearchResult& _return, const int64_t top_k, const VecBinaryTensor& tensor, const VecSearchFilter& filter) { - TimeRecordWrapper rc("search_binary_vector()"); - SERVER_LOG_TRACE << "group_id = " << group_id << ", top_k = " << top_k - << ", vector dimension = " << tensor.tensor.size(); + TimeRecordWrapper rc("search_binary_vector() in " + group_id); VecBinaryTensorList tensor_list; tensor_list.tensor_list.push_back(tensor); VecSearchResultList result; BaseTaskPtr task_ptr = SearchVectorTask::Create(group_id, top_k, &tensor_list, filter, result); - VecServiceScheduler& scheduler = VecServiceScheduler::GetInstance(); - scheduler.ExecuteTask(task_ptr); + ExecTask(task_ptr); if(!result.result_list.empty()) { _return = result.result_list[0]; @@ -190,13 +223,10 @@ VecServiceHandler::search_binary_vector_batch(VecSearchResultList& _return, const int64_t top_k, const VecBinaryTensorList& tensor_list, const VecSearchFilter& filter) { - TimeRecordWrapper rc("search_binary_vector_batch()"); - SERVER_LOG_TRACE << "group_id = " << group_id << ", top_k = " << top_k - << ", vector list size = " << tensor_list.tensor_list.size(); + TimeRecordWrapper rc("search_binary_vector_batch() in " + group_id); BaseTaskPtr task_ptr = SearchVectorTask::Create(group_id, top_k, &tensor_list, filter, _return); - VecServiceScheduler& scheduler = VecServiceScheduler::GetInstance(); - scheduler.ExecuteTask(task_ptr); + ExecTask(task_ptr); } diff --git a/cpp/src/server/VecServiceHandler.h b/cpp/src/server/VecServiceHandler.h index d37376bd3c..7a6ccc4b18 100644 --- a/cpp/src/server/VecServiceHandler.h +++ b/cpp/src/server/VecServiceHandler.h @@ -22,6 +22,8 @@ namespace zilliz { namespace vecwise { namespace server { +using namespace megasearch; + class VecServiceHandler : virtual public VecServiceIf { public: VecServiceHandler() { diff --git a/cpp/src/server/VecServiceScheduler.h b/cpp/src/server/VecServiceScheduler.h index ffcecbce5b..65b234c84d 100644 --- a/cpp/src/server/VecServiceScheduler.h +++ b/cpp/src/server/VecServiceScheduler.h @@ -27,6 +27,7 @@ public: std::string TaskGroup() const { return task_group_; } ServerError ErrorCode() const { return error_code_; } + std::string ErrorMsg() const { return error_msg_; } bool IsAsync() const { return async_; } @@ -41,6 +42,7 @@ protected: bool async_; bool done_; ServerError error_code_; + std::string error_msg_; }; using BaseTaskPtr = std::shared_ptr; diff --git a/cpp/src/server/VecServiceTask.cpp b/cpp/src/server/VecServiceTask.cpp index 9eda08495a..9ebe2c1bdf 100644 --- a/cpp/src/server/VecServiceTask.cpp +++ b/cpp/src/server/VecServiceTask.cpp @@ -91,13 +91,16 @@ ServerError AddGroupTask::OnExecute() { group_info.dimension = (size_t)dimension_; group_info.group_id = group_id_; engine::Status stat = DB()->add_group(group_info); - if(!stat.ok()) { + if(!stat.ok()) {//could exist SERVER_LOG_ERROR << "Engine failed: " << stat.ToString(); - return SERVER_UNEXPECTED_ERROR; + SERVER_LOG_ERROR << error_msg_; + return SERVER_SUCCESS; } } catch (std::exception& ex) { - SERVER_LOG_ERROR << ex.what(); + error_code_ = SERVER_UNEXPECTED_ERROR; + error_msg_ = ex.what(); + SERVER_LOG_ERROR << error_msg_; return SERVER_UNEXPECTED_ERROR; } @@ -124,14 +127,18 @@ ServerError GetGroupTask::OnExecute() { group_info.group_id = group_id_; engine::Status stat = DB()->get_group(group_info); if(!stat.ok()) { - SERVER_LOG_ERROR << "Engine failed: " << stat.ToString(); - return SERVER_UNEXPECTED_ERROR; + error_code_ = SERVER_GROUP_NOT_EXIST; + error_msg_ = "Engine failed: " + stat.ToString(); + SERVER_LOG_ERROR << error_msg_; + return error_code_; } else { dimension_ = (int32_t)group_info.dimension; } } catch (std::exception& ex) { - SERVER_LOG_ERROR << ex.what(); + error_code_ = SERVER_UNEXPECTED_ERROR; + error_msg_ = ex.what(); + SERVER_LOG_ERROR << error_msg_; return SERVER_UNEXPECTED_ERROR; } @@ -150,14 +157,13 @@ BaseTaskPtr DeleteGroupTask::Create(const std::string& group_id) { } ServerError DeleteGroupTask::OnExecute() { - try { + error_code_ = SERVER_NOT_IMPLEMENT; + error_msg_ = "delete group not implemented"; + SERVER_LOG_ERROR << error_msg_; + //IVecIdMapper::GetInstance()->DeleteGroup(group_id_); - } catch (std::exception& ex) { - SERVER_LOG_ERROR << ex.what(); - } - - return SERVER_SUCCESS; + return SERVER_NOT_IMPLEMENT; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -237,22 +243,7 @@ const AttribMap& AddVectorTask::GetVecAttrib() const { ServerError AddVectorTask::OnExecute() { try { - engine::meta::GroupSchema group_info; - group_info.group_id = group_id_; - engine::Status stat = DB()->get_group(group_info); - if(!stat.ok()) { - SERVER_LOG_ERROR << "Engine failed: " << stat.ToString(); - return SERVER_INVALID_ARGUMENT; - } - - uint64_t group_dim = group_info.dimension; uint64_t vec_dim = GetVecDimension(); - if(group_dim != vec_dim) { - SERVER_LOG_ERROR << "Invalid vector dimension: " << vec_dim - << " vs. group dimension:" << group_dim; - return SERVER_INVALID_ARGUMENT; - } - std::vector vec_f; vec_f.resize(vec_dim); const double* d_p = GetVecData(); @@ -261,13 +252,16 @@ ServerError AddVectorTask::OnExecute() { } engine::IDNumbers vector_ids; - stat = DB()->add_vectors(group_id_, 1, vec_f.data(), vector_ids); + engine::Status stat = DB()->add_vectors(group_id_, 1, vec_f.data(), vector_ids); if(!stat.ok()) { - SERVER_LOG_ERROR << "Engine failed: " << stat.ToString(); - return SERVER_UNEXPECTED_ERROR; + error_code_ = SERVER_UNEXPECTED_ERROR; + error_msg_ = "Engine failed: " + stat.ToString(); + SERVER_LOG_ERROR << error_msg_; + return error_code_; } else { if(vector_ids.empty()) { - SERVER_LOG_ERROR << "Vector ID not returned"; + error_msg_ = "Engine failed: " + stat.ToString(); + SERVER_LOG_ERROR << error_msg_; return SERVER_UNEXPECTED_ERROR; } else { std::string uid = GetVecID(); @@ -283,14 +277,16 @@ ServerError AddVectorTask::OnExecute() { attrib[VECTOR_UID] = tensor_id_; std::string attrib_str; AttributeSerializer::Encode(attrib, attrib_str); - IVecIdMapper::GetInstance()->Put(nid, attrib_str); - SERVER_LOG_TRACE << "nid = " << vector_ids[0] << ", uid = " << uid; + IVecIdMapper::GetInstance()->Put(nid, attrib_str, group_id_); + //SERVER_LOG_TRACE << "nid = " << vector_ids[0] << ", uid = " << uid; } } } catch (std::exception& ex) { - SERVER_LOG_ERROR << ex.what(); - return SERVER_UNEXPECTED_ERROR; + error_code_ = SERVER_UNEXPECTED_ERROR; + error_msg_ = ex.what(); + SERVER_LOG_ERROR << error_msg_; + return error_code_; } return SERVER_SUCCESS; @@ -416,7 +412,7 @@ void AddBatchVectorTask::ProcessIdMapping(engine::IDNumbers& vector_ids, attrib[VECTOR_UID] = uid; std::string attrib_str; AttributeSerializer::Encode(attrib, attrib_str); - IVecIdMapper::GetInstance()->Put(nid, attrib_str); + IVecIdMapper::GetInstance()->Put(nid, attrib_str, group_id_); } } @@ -433,8 +429,10 @@ ServerError AddBatchVectorTask::OnExecute() { group_info.group_id = group_id_; engine::Status stat = DB()->get_group(group_info); if(!stat.ok()) { - SERVER_LOG_ERROR << "Engine failed: " << stat.ToString(); - return SERVER_UNEXPECTED_ERROR; + error_code_ = SERVER_GROUP_NOT_EXIST; + error_msg_ = "Engine failed: " + stat.ToString(); + SERVER_LOG_ERROR << error_msg_; + return error_code_; } rc.Record("check group dimension"); @@ -447,7 +445,9 @@ ServerError AddBatchVectorTask::OnExecute() { if(vec_dim != group_dim) { SERVER_LOG_ERROR << "Invalid vector dimension: " << vec_dim << " vs. group dimension:" << group_dim; - return SERVER_INVALID_ARGUMENT; + error_code_ = SERVER_INVALID_VECTOR_DIMENSION; + error_msg_ = "Engine failed: " + stat.ToString(); + return error_code_; } const double* d_p = GetVecData(i); @@ -462,43 +462,48 @@ ServerError AddBatchVectorTask::OnExecute() { stat = DB()->add_vectors(group_id_, vec_count, vec_f.data(), vector_ids); rc.Record("add vectors to engine"); if(!stat.ok()) { - SERVER_LOG_ERROR << "Engine failed: " << stat.ToString(); + error_code_ = SERVER_UNEXPECTED_ERROR; + error_msg_ = "Engine failed: " + stat.ToString(); + SERVER_LOG_ERROR << error_msg_; + return error_code_; + } + + if(vector_ids.size() < vec_count) { + SERVER_LOG_ERROR << "Vector ID not returned"; + return SERVER_UNEXPECTED_ERROR; } else { - if(vector_ids.size() < vec_count) { - SERVER_LOG_ERROR << "Vector ID not returned"; - return SERVER_UNEXPECTED_ERROR; + tensor_ids_.resize(vector_ids.size()); + if(vec_count < USE_MT) { + ProcessIdMapping(vector_ids, 0, vec_count, tensor_ids_); + rc.Record("built id mapping"); } else { - tensor_ids_.resize(vector_ids.size()); - if(vec_count < USE_MT) { - ProcessIdMapping(vector_ids, 0, vec_count, tensor_ids_); - rc.Record("built id mapping"); - } else { - std::list> threads_list; + std::list> threads_list; - uint64_t begin_index = 0, end_index = USE_MT; - while(end_index < vec_count) { - threads_list.push_back( - GetThreadPool().enqueue(&AddBatchVectorTask::ProcessIdMapping, - this, vector_ids, begin_index, end_index, tensor_ids_)); - begin_index = end_index; - end_index += USE_MT; - if(end_index > vec_count) { - end_index = vec_count; - } + uint64_t begin_index = 0, end_index = USE_MT; + while(end_index < vec_count) { + threads_list.push_back( + GetThreadPool().enqueue(&AddBatchVectorTask::ProcessIdMapping, + this, vector_ids, begin_index, end_index, tensor_ids_)); + begin_index = end_index; + end_index += USE_MT; + if(end_index > vec_count) { + end_index = vec_count; } - - for (std::list>::iterator it = threads_list.begin(); it != threads_list.end(); it++) { - it->wait(); - } - - rc.Record("built id mapping by multi-threads:" + std::to_string(threads_list.size())); } + + for (std::list>::iterator it = threads_list.begin(); it != threads_list.end(); it++) { + it->wait(); + } + + rc.Record("built id mapping by multi-threads:" + std::to_string(threads_list.size())); } } } catch (std::exception& ex) { - SERVER_LOG_ERROR << ex.what(); - return SERVER_UNEXPECTED_ERROR; + error_code_ = SERVER_UNEXPECTED_ERROR; + error_msg_ = ex.what(); + SERVER_LOG_ERROR << error_msg_; + return error_code_; } return SERVER_SUCCESS; @@ -612,15 +617,19 @@ ServerError SearchVectorTask::OnExecute() { group_info.group_id = group_id_; engine::Status stat = DB()->get_group(group_info); if(!stat.ok()) { - SERVER_LOG_ERROR << "Engine failed: " << stat.ToString(); - return SERVER_UNEXPECTED_ERROR; + error_code_ = SERVER_GROUP_NOT_EXIST; + error_msg_ = "Engine failed: " + stat.ToString(); + SERVER_LOG_ERROR << error_msg_; + return error_code_; } uint64_t vec_dim = GetTargetDimension(); if(vec_dim != group_info.dimension) { SERVER_LOG_ERROR << "Invalid vector dimension: " << vec_dim << " vs. group dimension:" << group_info.dimension; - return SERVER_INVALID_ARGUMENT; + error_code_ = SERVER_INVALID_VECTOR_DIMENSION; + error_msg_ = "Engine failed: " + stat.ToString(); + return error_code_; } rc.Record("check group dimension"); @@ -654,18 +663,30 @@ ServerError SearchVectorTask::OnExecute() { for(auto id : res) { std::string attrib_str; std::string nid = nid_prefix + std::to_string(id); - IVecIdMapper::GetInstance()->Get(nid, attrib_str); + IVecIdMapper::GetInstance()->Get(nid, attrib_str, group_id_); AttribMap attrib_map; AttributeSerializer::Decode(attrib_str, attrib_map); + AttribMap attrib_return; VecSearchResultItem item; - item.__set_attrib(attrib_map); - item.uid = item.attrib[VECTOR_UID]; + item.uid = attrib_map[VECTOR_UID]; + + if(filter_.return_attribs.empty()) {//return all attributes + attrib_return.swap(attrib_map); + } else {//filter attributes + for(auto& name : filter_.return_attribs) { + if(attrib_map.count(name) == 0) + continue; + + attrib_return[name] = attrib_map[name]; + } + } + item.__set_attrib(attrib_return); item.distance = 0.0;////TODO: return distance v_res.result_list.emplace_back(item); - SERVER_LOG_TRACE << "nid = " << nid << ", uid = " << item.uid; + //SERVER_LOG_TRACE << "nid = " << nid << ", uid = " << item.uid; } result_.result_list.push_back(v_res); @@ -674,8 +695,10 @@ ServerError SearchVectorTask::OnExecute() { } } catch (std::exception& ex) { - SERVER_LOG_ERROR << ex.what(); - return SERVER_UNEXPECTED_ERROR; + error_code_ = SERVER_UNEXPECTED_ERROR; + error_msg_ = ex.what(); + SERVER_LOG_ERROR << error_msg_; + return error_code_; } return SERVER_SUCCESS; diff --git a/cpp/src/server/VecServiceTask.h b/cpp/src/server/VecServiceTask.h index f9fe2a2158..e046e36083 100644 --- a/cpp/src/server/VecServiceTask.h +++ b/cpp/src/server/VecServiceTask.h @@ -10,7 +10,7 @@ #include "utils/AttributeSerializer.h" #include "db/Types.h" -#include "thrift/gen-cpp/VectorService_types.h" +#include "thrift/gen-cpp/megasearch_types.h" #include #include @@ -19,6 +19,7 @@ namespace zilliz { namespace vecwise { namespace server { +using namespace megasearch; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class AddGroupTask : public BaseTask { public: diff --git a/cpp/src/server/VecServiceWrapper.cpp b/cpp/src/server/VecServiceWrapper.cpp index 376f5ef3cc..6e9fd8b3a4 100644 --- a/cpp/src/server/VecServiceWrapper.cpp +++ b/cpp/src/server/VecServiceWrapper.cpp @@ -10,8 +10,8 @@ #include "utils/Log.h" -#include "thrift/gen-cpp/VectorService_types.h" -#include "thrift/gen-cpp/VectorService_constants.h" +#include "thrift/gen-cpp/megasearch_types.h" +#include "thrift/gen-cpp/megasearch_constants.h" #include #include diff --git a/cpp/src/thrift/cpp_gen.sh b/cpp/src/thrift/cpp_gen.sh index e279f60ac8..df04e46c99 100755 --- a/cpp/src/thrift/cpp_gen.sh +++ b/cpp/src/thrift/cpp_gen.sh @@ -1,4 +1,4 @@ #!/bin/bash -../../third_party/build/bin/thrift -r --gen cpp ./VectorService.thrift +../../third_party/build/bin/thrift -r --gen cpp ./megasearch.thrift diff --git a/cpp/src/thrift/gen-cpp/VecService.cpp b/cpp/src/thrift/gen-cpp/VecService.cpp index d99b2bfcef..c794959057 100644 --- a/cpp/src/thrift/gen-cpp/VecService.cpp +++ b/cpp/src/thrift/gen-cpp/VecService.cpp @@ -6,7 +6,7 @@ */ #include "VecService.h" -namespace zilliz { +namespace megasearch { VecService_add_group_args::~VecService_add_group_args() throw() { diff --git a/cpp/src/thrift/gen-cpp/VecService.h b/cpp/src/thrift/gen-cpp/VecService.h index 8b56738b20..a3fdba2472 100644 --- a/cpp/src/thrift/gen-cpp/VecService.h +++ b/cpp/src/thrift/gen-cpp/VecService.h @@ -9,9 +9,9 @@ #include #include -#include "VectorService_types.h" +#include "megasearch_types.h" -namespace zilliz { +namespace megasearch { #ifdef _MSC_VER #pragma warning( push ) diff --git a/cpp/src/thrift/gen-cpp/VecService_server.skeleton.cpp b/cpp/src/thrift/gen-cpp/VecService_server.skeleton.cpp index 364743cf46..8727a0e500 100644 --- a/cpp/src/thrift/gen-cpp/VecService_server.skeleton.cpp +++ b/cpp/src/thrift/gen-cpp/VecService_server.skeleton.cpp @@ -12,7 +12,7 @@ using namespace ::apache::thrift::protocol; using namespace ::apache::thrift::transport; using namespace ::apache::thrift::server; -using namespace ::zilliz; +using namespace ::megasearch; class VecServiceHandler : virtual public VecServiceIf { public: diff --git a/cpp/src/thrift/gen-cpp/VectorService_constants.cpp b/cpp/src/thrift/gen-cpp/VectorService_constants.cpp deleted file mode 100644 index 27ea10ef32..0000000000 --- a/cpp/src/thrift/gen-cpp/VectorService_constants.cpp +++ /dev/null @@ -1,17 +0,0 @@ -/** - * Autogenerated by Thrift Compiler (0.11.0) - * - * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING - * @generated - */ -#include "VectorService_constants.h" - -namespace zilliz { - -const VectorServiceConstants g_VectorService_constants; - -VectorServiceConstants::VectorServiceConstants() { -} - -} // namespace - diff --git a/cpp/src/thrift/gen-cpp/VectorService_constants.h b/cpp/src/thrift/gen-cpp/VectorService_constants.h deleted file mode 100644 index 3a8e16b92d..0000000000 --- a/cpp/src/thrift/gen-cpp/VectorService_constants.h +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Autogenerated by Thrift Compiler (0.11.0) - * - * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING - * @generated - */ -#ifndef VectorService_CONSTANTS_H -#define VectorService_CONSTANTS_H - -#include "VectorService_types.h" - -namespace zilliz { - -class VectorServiceConstants { - public: - VectorServiceConstants(); - -}; - -extern const VectorServiceConstants g_VectorService_constants; - -} // namespace - -#endif diff --git a/cpp/src/thrift/gen-cpp/megasearch_constants.cpp b/cpp/src/thrift/gen-cpp/megasearch_constants.cpp new file mode 100644 index 0000000000..06adb7629f --- /dev/null +++ b/cpp/src/thrift/gen-cpp/megasearch_constants.cpp @@ -0,0 +1,17 @@ +/** + * Autogenerated by Thrift Compiler (0.11.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +#include "megasearch_constants.h" + +namespace megasearch { + +const megasearchConstants g_megasearch_constants; + +megasearchConstants::megasearchConstants() { +} + +} // namespace + diff --git a/cpp/src/thrift/gen-cpp/megasearch_constants.h b/cpp/src/thrift/gen-cpp/megasearch_constants.h new file mode 100644 index 0000000000..1f4b9553dd --- /dev/null +++ b/cpp/src/thrift/gen-cpp/megasearch_constants.h @@ -0,0 +1,24 @@ +/** + * Autogenerated by Thrift Compiler (0.11.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +#ifndef megasearch_CONSTANTS_H +#define megasearch_CONSTANTS_H + +#include "megasearch_types.h" + +namespace megasearch { + +class megasearchConstants { + public: + megasearchConstants(); + +}; + +extern const megasearchConstants g_megasearch_constants; + +} // namespace + +#endif diff --git a/cpp/src/thrift/gen-cpp/VectorService_types.cpp b/cpp/src/thrift/gen-cpp/megasearch_types.cpp similarity index 99% rename from cpp/src/thrift/gen-cpp/VectorService_types.cpp rename to cpp/src/thrift/gen-cpp/megasearch_types.cpp index 5fefd60fc5..bf4d316366 100644 --- a/cpp/src/thrift/gen-cpp/VectorService_types.cpp +++ b/cpp/src/thrift/gen-cpp/megasearch_types.cpp @@ -4,14 +4,14 @@ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING * @generated */ -#include "VectorService_types.h" +#include "megasearch_types.h" #include #include #include -namespace zilliz { +namespace megasearch { int _kVecErrCodeValues[] = { VecErrCode::SUCCESS, diff --git a/cpp/src/thrift/gen-cpp/VectorService_types.h b/cpp/src/thrift/gen-cpp/megasearch_types.h similarity index 99% rename from cpp/src/thrift/gen-cpp/VectorService_types.h rename to cpp/src/thrift/gen-cpp/megasearch_types.h index d9285efd80..e7e424b2db 100644 --- a/cpp/src/thrift/gen-cpp/VectorService_types.h +++ b/cpp/src/thrift/gen-cpp/megasearch_types.h @@ -4,8 +4,8 @@ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING * @generated */ -#ifndef VectorService_TYPES_H -#define VectorService_TYPES_H +#ifndef megasearch_TYPES_H +#define megasearch_TYPES_H #include @@ -18,7 +18,7 @@ #include -namespace zilliz { +namespace megasearch { struct VecErrCode { enum type { diff --git a/cpp/src/thrift/gen-py/py_sample.py b/cpp/src/thrift/gen-py/py_sample.py index 68b759d755..0a4690c1eb 100644 --- a/cpp/src/thrift/gen-py/py_sample.py +++ b/cpp/src/thrift/gen-py/py_sample.py @@ -1,22 +1,15 @@ import time import struct -from zilliz import VecService +from megasearch import VecService -from thrift import Thrift +#Note: pip install thrift from thrift.transport import TSocket from thrift.transport import TTransport from thrift.protocol import TBinaryProtocol, TCompactProtocol, TJSONProtocol -def print_time_cost(desc, t): - time_now = time.time() - print(desc + ' cost ', time_now - t, ' sec') - return time_now - -def test_vecwise(): +def test_megasearch(): try: - time_start = time.time() - #connect transport = TSocket.TSocket('localhost', 33001) transport = TTransport.TBufferedTransport(transport) @@ -24,26 +17,13 @@ def test_vecwise(): client = VecService.Client(protocol) transport.open() - time_start = print_time_cost('connection', time_start) + print("connected"); #add group - group = VecService.VecGroup("py_group_" + time.strftime('%H%M%S'), 256) + group = VecService.VecGroup("test_" + time.strftime('%H%M%S'), 256) client.add_group(group) - time_start = print_time_cost('add group', time_start) - - #build vectors - # vec_list = VecService.VecTensorList([]) - # for i in range(10000): - # vec = VecService.VecTensor("normal_"+str(i), []) - # for k in range(group.dimension): - # vec.tensor.append(k) - # vec_list.tensor_list.append(vec) - #time_start = print_time_cost('build normal vectors', time_start) - - #add vectors - #client.add_vector_batch(group.id, vec_list) - #time_start = print_time_cost('add normal vectors', time_start)) + print("group added"); # build binary vectors bin_vec_list = VecService.VecBinaryTensorList([]) @@ -55,14 +35,12 @@ def test_vecwise(): bin_vec.tensor = struct.pack(str(group.dimension)+"d", *a) bin_vec_list.tensor_list.append(bin_vec) - time_start = print_time_cost('build binary vectors', time_start) - # add vectors client.add_binary_vector_batch(group.id, bin_vec_list) - time_start = print_time_cost('add binary vectors', time_start) - time.sleep(5) - time_start = print_time_cost('sleep 5 seconds', time_start) + wait_storage = 5 + print("wait {} seconds for persisting data".format(wait_storage)) + time.sleep(wait_storage) # search vector a = [] @@ -72,18 +50,19 @@ def test_vecwise(): bin_vec.tensor = struct.pack(str(group.dimension) + "d", *a) filter = VecService.VecSearchFilter() res = VecService.VecSearchResult() + + print("begin search ..."); res = client.search_binary_vector(group.id, 5, bin_vec, filter) - time_start = print_time_cost('search binary vectors', time_start) print('result count: ' + str(len(res.result_list))) for item in res.result_list: print(item.uid) transport.close() - time_start = print_time_cost('close connection', time_start) + print("disconnected"); - except Thrift.TException as ex: - print(ex.message) + except VecService.VecException as ex: + print(ex.reason) -test_vecwise() \ No newline at end of file +test_megasearch() \ No newline at end of file diff --git a/cpp/src/thrift/gen-py/zilliz/VecService-remote b/cpp/src/thrift/gen-py/zilliz/VecService-remote deleted file mode 100755 index 047136afb1..0000000000 --- a/cpp/src/thrift/gen-py/zilliz/VecService-remote +++ /dev/null @@ -1,187 +0,0 @@ -#!/usr/bin/env python -# -# Autogenerated by Thrift Compiler (0.11.0) -# -# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING -# -# options string: py -# - -import sys -import pprint -if sys.version_info[0] > 2: - from urllib.parse import urlparse -else: - from urlparse import urlparse -from thrift.transport import TTransport, TSocket, TSSLSocket, THttpClient -from thrift.protocol.TBinaryProtocol import TBinaryProtocol - -from zilliz import VecService -from zilliz.ttypes import * - -if len(sys.argv) <= 1 or sys.argv[1] == '--help': - print('') - print('Usage: ' + sys.argv[0] + ' [-h host[:port]] [-u url] [-f[ramed]] [-s[sl]] [-novalidate] [-ca_certs certs] [-keyfile keyfile] [-certfile certfile] function [arg1 [arg2...]]') - print('') - print('Functions:') - print(' void add_group(VecGroup group)') - print(' VecGroup get_group(string group_id)') - print(' void del_group(string group_id)') - print(' string add_vector(string group_id, VecTensor tensor)') - print(' add_vector_batch(string group_id, VecTensorList tensor_list)') - print(' string add_binary_vector(string group_id, VecBinaryTensor tensor)') - print(' add_binary_vector_batch(string group_id, VecBinaryTensorList tensor_list)') - print(' VecSearchResult search_vector(string group_id, i64 top_k, VecTensor tensor, VecSearchFilter filter)') - print(' VecSearchResultList search_vector_batch(string group_id, i64 top_k, VecTensorList tensor_list, VecSearchFilter filter)') - print(' VecSearchResult search_binary_vector(string group_id, i64 top_k, VecBinaryTensor tensor, VecSearchFilter filter)') - print(' VecSearchResultList search_binary_vector_batch(string group_id, i64 top_k, VecBinaryTensorList tensor_list, VecSearchFilter filter)') - print('') - sys.exit(0) - -pp = pprint.PrettyPrinter(indent=2) -host = 'localhost' -port = 9090 -uri = '' -framed = False -ssl = False -validate = True -ca_certs = None -keyfile = None -certfile = None -http = False -argi = 1 - -if sys.argv[argi] == '-h': - parts = sys.argv[argi + 1].split(':') - host = parts[0] - if len(parts) > 1: - port = int(parts[1]) - argi += 2 - -if sys.argv[argi] == '-u': - url = urlparse(sys.argv[argi + 1]) - parts = url[1].split(':') - host = parts[0] - if len(parts) > 1: - port = int(parts[1]) - else: - port = 80 - uri = url[2] - if url[4]: - uri += '?%s' % url[4] - http = True - argi += 2 - -if sys.argv[argi] == '-f' or sys.argv[argi] == '-framed': - framed = True - argi += 1 - -if sys.argv[argi] == '-s' or sys.argv[argi] == '-ssl': - ssl = True - argi += 1 - -if sys.argv[argi] == '-novalidate': - validate = False - argi += 1 - -if sys.argv[argi] == '-ca_certs': - ca_certs = sys.argv[argi+1] - argi += 2 - -if sys.argv[argi] == '-keyfile': - keyfile = sys.argv[argi+1] - argi += 2 - -if sys.argv[argi] == '-certfile': - certfile = sys.argv[argi+1] - argi += 2 - -cmd = sys.argv[argi] -args = sys.argv[argi + 1:] - -if http: - transport = THttpClient.THttpClient(host, port, uri) -else: - if ssl: - socket = TSSLSocket.TSSLSocket(host, port, validate=validate, ca_certs=ca_certs, keyfile=keyfile, certfile=certfile) - else: - socket = TSocket.TSocket(host, port) - if framed: - transport = TTransport.TFramedTransport(socket) - else: - transport = TTransport.TBufferedTransport(socket) -protocol = TBinaryProtocol(transport) -client = VecService.Client(protocol) -transport.open() - -if cmd == 'add_group': - if len(args) != 1: - print('add_group requires 1 args') - sys.exit(1) - pp.pprint(client.add_group(eval(args[0]),)) - -elif cmd == 'get_group': - if len(args) != 1: - print('get_group requires 1 args') - sys.exit(1) - pp.pprint(client.get_group(args[0],)) - -elif cmd == 'del_group': - if len(args) != 1: - print('del_group requires 1 args') - sys.exit(1) - pp.pprint(client.del_group(args[0],)) - -elif cmd == 'add_vector': - if len(args) != 2: - print('add_vector requires 2 args') - sys.exit(1) - pp.pprint(client.add_vector(args[0], eval(args[1]),)) - -elif cmd == 'add_vector_batch': - if len(args) != 2: - print('add_vector_batch requires 2 args') - sys.exit(1) - pp.pprint(client.add_vector_batch(args[0], eval(args[1]),)) - -elif cmd == 'add_binary_vector': - if len(args) != 2: - print('add_binary_vector requires 2 args') - sys.exit(1) - pp.pprint(client.add_binary_vector(args[0], eval(args[1]),)) - -elif cmd == 'add_binary_vector_batch': - if len(args) != 2: - print('add_binary_vector_batch requires 2 args') - sys.exit(1) - pp.pprint(client.add_binary_vector_batch(args[0], eval(args[1]),)) - -elif cmd == 'search_vector': - if len(args) != 4: - print('search_vector requires 4 args') - sys.exit(1) - pp.pprint(client.search_vector(args[0], eval(args[1]), eval(args[2]), eval(args[3]),)) - -elif cmd == 'search_vector_batch': - if len(args) != 4: - print('search_vector_batch requires 4 args') - sys.exit(1) - pp.pprint(client.search_vector_batch(args[0], eval(args[1]), eval(args[2]), eval(args[3]),)) - -elif cmd == 'search_binary_vector': - if len(args) != 4: - print('search_binary_vector requires 4 args') - sys.exit(1) - pp.pprint(client.search_binary_vector(args[0], eval(args[1]), eval(args[2]), eval(args[3]),)) - -elif cmd == 'search_binary_vector_batch': - if len(args) != 4: - print('search_binary_vector_batch requires 4 args') - sys.exit(1) - pp.pprint(client.search_binary_vector_batch(args[0], eval(args[1]), eval(args[2]), eval(args[3]),)) - -else: - print('Unrecognized method %s' % cmd) - sys.exit(1) - -transport.close() diff --git a/cpp/src/thrift/gen-py/zilliz/VecService.py b/cpp/src/thrift/gen-py/zilliz/VecService.py deleted file mode 100644 index 3cc23dfafc..0000000000 --- a/cpp/src/thrift/gen-py/zilliz/VecService.py +++ /dev/null @@ -1,2548 +0,0 @@ -# -# Autogenerated by Thrift Compiler (0.11.0) -# -# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING -# -# options string: py -# - -from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException -from thrift.protocol.TProtocol import TProtocolException -from thrift.TRecursive import fix_spec - -import sys -import logging -from .ttypes import * -from thrift.Thrift import TProcessor -from thrift.transport import TTransport -all_structs = [] - - -class Iface(object): - def add_group(self, group): - """ - group interfaces - - Parameters: - - group - """ - pass - - def get_group(self, group_id): - """ - Parameters: - - group_id - """ - pass - - def del_group(self, group_id): - """ - Parameters: - - group_id - """ - pass - - def add_vector(self, group_id, tensor): - """ - insert vector interfaces - - - Parameters: - - group_id - - tensor - """ - pass - - def add_vector_batch(self, group_id, tensor_list): - """ - Parameters: - - group_id - - tensor_list - """ - pass - - def add_binary_vector(self, group_id, tensor): - """ - Parameters: - - group_id - - tensor - """ - pass - - def add_binary_vector_batch(self, group_id, tensor_list): - """ - Parameters: - - group_id - - tensor_list - """ - pass - - def search_vector(self, group_id, top_k, tensor, filter): - """ - search interfaces - you can use filter to reduce search result - filter.attrib_filter can specify which attribute you need, for example: - set attrib_filter = {"color":""} means you want to get "color" attribute for result vector - set attrib_filter = {"color":"red"} means you want to get vectors which has attribute "color" equals "red" - if filter.time_range is empty, engine will search without time limit - - Parameters: - - group_id - - top_k - - tensor - - filter - """ - pass - - def search_vector_batch(self, group_id, top_k, tensor_list, filter): - """ - Parameters: - - group_id - - top_k - - tensor_list - - filter - """ - pass - - def search_binary_vector(self, group_id, top_k, tensor, filter): - """ - Parameters: - - group_id - - top_k - - tensor - - filter - """ - pass - - def search_binary_vector_batch(self, group_id, top_k, tensor_list, filter): - """ - Parameters: - - group_id - - top_k - - tensor_list - - filter - """ - pass - - -class Client(Iface): - def __init__(self, iprot, oprot=None): - self._iprot = self._oprot = iprot - if oprot is not None: - self._oprot = oprot - self._seqid = 0 - - def add_group(self, group): - """ - group interfaces - - Parameters: - - group - """ - self.send_add_group(group) - self.recv_add_group() - - def send_add_group(self, group): - self._oprot.writeMessageBegin('add_group', TMessageType.CALL, self._seqid) - args = add_group_args() - args.group = group - args.write(self._oprot) - self._oprot.writeMessageEnd() - self._oprot.trans.flush() - - def recv_add_group(self): - iprot = self._iprot - (fname, mtype, rseqid) = iprot.readMessageBegin() - if mtype == TMessageType.EXCEPTION: - x = TApplicationException() - x.read(iprot) - iprot.readMessageEnd() - raise x - result = add_group_result() - result.read(iprot) - iprot.readMessageEnd() - if result.e is not None: - raise result.e - return - - def get_group(self, group_id): - """ - Parameters: - - group_id - """ - self.send_get_group(group_id) - return self.recv_get_group() - - def send_get_group(self, group_id): - self._oprot.writeMessageBegin('get_group', TMessageType.CALL, self._seqid) - args = get_group_args() - args.group_id = group_id - args.write(self._oprot) - self._oprot.writeMessageEnd() - self._oprot.trans.flush() - - def recv_get_group(self): - iprot = self._iprot - (fname, mtype, rseqid) = iprot.readMessageBegin() - if mtype == TMessageType.EXCEPTION: - x = TApplicationException() - x.read(iprot) - iprot.readMessageEnd() - raise x - result = get_group_result() - result.read(iprot) - iprot.readMessageEnd() - if result.success is not None: - return result.success - if result.e is not None: - raise result.e - raise TApplicationException(TApplicationException.MISSING_RESULT, "get_group failed: unknown result") - - def del_group(self, group_id): - """ - Parameters: - - group_id - """ - self.send_del_group(group_id) - self.recv_del_group() - - def send_del_group(self, group_id): - self._oprot.writeMessageBegin('del_group', TMessageType.CALL, self._seqid) - args = del_group_args() - args.group_id = group_id - args.write(self._oprot) - self._oprot.writeMessageEnd() - self._oprot.trans.flush() - - def recv_del_group(self): - iprot = self._iprot - (fname, mtype, rseqid) = iprot.readMessageBegin() - if mtype == TMessageType.EXCEPTION: - x = TApplicationException() - x.read(iprot) - iprot.readMessageEnd() - raise x - result = del_group_result() - result.read(iprot) - iprot.readMessageEnd() - if result.e is not None: - raise result.e - return - - def add_vector(self, group_id, tensor): - """ - insert vector interfaces - - - Parameters: - - group_id - - tensor - """ - self.send_add_vector(group_id, tensor) - return self.recv_add_vector() - - def send_add_vector(self, group_id, tensor): - self._oprot.writeMessageBegin('add_vector', TMessageType.CALL, self._seqid) - args = add_vector_args() - args.group_id = group_id - args.tensor = tensor - args.write(self._oprot) - self._oprot.writeMessageEnd() - self._oprot.trans.flush() - - def recv_add_vector(self): - iprot = self._iprot - (fname, mtype, rseqid) = iprot.readMessageBegin() - if mtype == TMessageType.EXCEPTION: - x = TApplicationException() - x.read(iprot) - iprot.readMessageEnd() - raise x - result = add_vector_result() - result.read(iprot) - iprot.readMessageEnd() - if result.success is not None: - return result.success - if result.e is not None: - raise result.e - raise TApplicationException(TApplicationException.MISSING_RESULT, "add_vector failed: unknown result") - - def add_vector_batch(self, group_id, tensor_list): - """ - Parameters: - - group_id - - tensor_list - """ - self.send_add_vector_batch(group_id, tensor_list) - return self.recv_add_vector_batch() - - def send_add_vector_batch(self, group_id, tensor_list): - self._oprot.writeMessageBegin('add_vector_batch', TMessageType.CALL, self._seqid) - args = add_vector_batch_args() - args.group_id = group_id - args.tensor_list = tensor_list - args.write(self._oprot) - self._oprot.writeMessageEnd() - self._oprot.trans.flush() - - def recv_add_vector_batch(self): - iprot = self._iprot - (fname, mtype, rseqid) = iprot.readMessageBegin() - if mtype == TMessageType.EXCEPTION: - x = TApplicationException() - x.read(iprot) - iprot.readMessageEnd() - raise x - result = add_vector_batch_result() - result.read(iprot) - iprot.readMessageEnd() - if result.success is not None: - return result.success - if result.e is not None: - raise result.e - raise TApplicationException(TApplicationException.MISSING_RESULT, "add_vector_batch failed: unknown result") - - def add_binary_vector(self, group_id, tensor): - """ - Parameters: - - group_id - - tensor - """ - self.send_add_binary_vector(group_id, tensor) - return self.recv_add_binary_vector() - - def send_add_binary_vector(self, group_id, tensor): - self._oprot.writeMessageBegin('add_binary_vector', TMessageType.CALL, self._seqid) - args = add_binary_vector_args() - args.group_id = group_id - args.tensor = tensor - args.write(self._oprot) - self._oprot.writeMessageEnd() - self._oprot.trans.flush() - - def recv_add_binary_vector(self): - iprot = self._iprot - (fname, mtype, rseqid) = iprot.readMessageBegin() - if mtype == TMessageType.EXCEPTION: - x = TApplicationException() - x.read(iprot) - iprot.readMessageEnd() - raise x - result = add_binary_vector_result() - result.read(iprot) - iprot.readMessageEnd() - if result.success is not None: - return result.success - if result.e is not None: - raise result.e - raise TApplicationException(TApplicationException.MISSING_RESULT, "add_binary_vector failed: unknown result") - - def add_binary_vector_batch(self, group_id, tensor_list): - """ - Parameters: - - group_id - - tensor_list - """ - self.send_add_binary_vector_batch(group_id, tensor_list) - return self.recv_add_binary_vector_batch() - - def send_add_binary_vector_batch(self, group_id, tensor_list): - self._oprot.writeMessageBegin('add_binary_vector_batch', TMessageType.CALL, self._seqid) - args = add_binary_vector_batch_args() - args.group_id = group_id - args.tensor_list = tensor_list - args.write(self._oprot) - self._oprot.writeMessageEnd() - self._oprot.trans.flush() - - def recv_add_binary_vector_batch(self): - iprot = self._iprot - (fname, mtype, rseqid) = iprot.readMessageBegin() - if mtype == TMessageType.EXCEPTION: - x = TApplicationException() - x.read(iprot) - iprot.readMessageEnd() - raise x - result = add_binary_vector_batch_result() - result.read(iprot) - iprot.readMessageEnd() - if result.success is not None: - return result.success - if result.e is not None: - raise result.e - raise TApplicationException(TApplicationException.MISSING_RESULT, "add_binary_vector_batch failed: unknown result") - - def search_vector(self, group_id, top_k, tensor, filter): - """ - search interfaces - you can use filter to reduce search result - filter.attrib_filter can specify which attribute you need, for example: - set attrib_filter = {"color":""} means you want to get "color" attribute for result vector - set attrib_filter = {"color":"red"} means you want to get vectors which has attribute "color" equals "red" - if filter.time_range is empty, engine will search without time limit - - Parameters: - - group_id - - top_k - - tensor - - filter - """ - self.send_search_vector(group_id, top_k, tensor, filter) - return self.recv_search_vector() - - def send_search_vector(self, group_id, top_k, tensor, filter): - self._oprot.writeMessageBegin('search_vector', TMessageType.CALL, self._seqid) - args = search_vector_args() - args.group_id = group_id - args.top_k = top_k - args.tensor = tensor - args.filter = filter - args.write(self._oprot) - self._oprot.writeMessageEnd() - self._oprot.trans.flush() - - def recv_search_vector(self): - iprot = self._iprot - (fname, mtype, rseqid) = iprot.readMessageBegin() - if mtype == TMessageType.EXCEPTION: - x = TApplicationException() - x.read(iprot) - iprot.readMessageEnd() - raise x - result = search_vector_result() - result.read(iprot) - iprot.readMessageEnd() - if result.success is not None: - return result.success - if result.e is not None: - raise result.e - raise TApplicationException(TApplicationException.MISSING_RESULT, "search_vector failed: unknown result") - - def search_vector_batch(self, group_id, top_k, tensor_list, filter): - """ - Parameters: - - group_id - - top_k - - tensor_list - - filter - """ - self.send_search_vector_batch(group_id, top_k, tensor_list, filter) - return self.recv_search_vector_batch() - - def send_search_vector_batch(self, group_id, top_k, tensor_list, filter): - self._oprot.writeMessageBegin('search_vector_batch', TMessageType.CALL, self._seqid) - args = search_vector_batch_args() - args.group_id = group_id - args.top_k = top_k - args.tensor_list = tensor_list - args.filter = filter - args.write(self._oprot) - self._oprot.writeMessageEnd() - self._oprot.trans.flush() - - def recv_search_vector_batch(self): - iprot = self._iprot - (fname, mtype, rseqid) = iprot.readMessageBegin() - if mtype == TMessageType.EXCEPTION: - x = TApplicationException() - x.read(iprot) - iprot.readMessageEnd() - raise x - result = search_vector_batch_result() - result.read(iprot) - iprot.readMessageEnd() - if result.success is not None: - return result.success - if result.e is not None: - raise result.e - raise TApplicationException(TApplicationException.MISSING_RESULT, "search_vector_batch failed: unknown result") - - def search_binary_vector(self, group_id, top_k, tensor, filter): - """ - Parameters: - - group_id - - top_k - - tensor - - filter - """ - self.send_search_binary_vector(group_id, top_k, tensor, filter) - return self.recv_search_binary_vector() - - def send_search_binary_vector(self, group_id, top_k, tensor, filter): - self._oprot.writeMessageBegin('search_binary_vector', TMessageType.CALL, self._seqid) - args = search_binary_vector_args() - args.group_id = group_id - args.top_k = top_k - args.tensor = tensor - args.filter = filter - args.write(self._oprot) - self._oprot.writeMessageEnd() - self._oprot.trans.flush() - - def recv_search_binary_vector(self): - iprot = self._iprot - (fname, mtype, rseqid) = iprot.readMessageBegin() - if mtype == TMessageType.EXCEPTION: - x = TApplicationException() - x.read(iprot) - iprot.readMessageEnd() - raise x - result = search_binary_vector_result() - result.read(iprot) - iprot.readMessageEnd() - if result.success is not None: - return result.success - if result.e is not None: - raise result.e - raise TApplicationException(TApplicationException.MISSING_RESULT, "search_binary_vector failed: unknown result") - - def search_binary_vector_batch(self, group_id, top_k, tensor_list, filter): - """ - Parameters: - - group_id - - top_k - - tensor_list - - filter - """ - self.send_search_binary_vector_batch(group_id, top_k, tensor_list, filter) - return self.recv_search_binary_vector_batch() - - def send_search_binary_vector_batch(self, group_id, top_k, tensor_list, filter): - self._oprot.writeMessageBegin('search_binary_vector_batch', TMessageType.CALL, self._seqid) - args = search_binary_vector_batch_args() - args.group_id = group_id - args.top_k = top_k - args.tensor_list = tensor_list - args.filter = filter - args.write(self._oprot) - self._oprot.writeMessageEnd() - self._oprot.trans.flush() - - def recv_search_binary_vector_batch(self): - iprot = self._iprot - (fname, mtype, rseqid) = iprot.readMessageBegin() - if mtype == TMessageType.EXCEPTION: - x = TApplicationException() - x.read(iprot) - iprot.readMessageEnd() - raise x - result = search_binary_vector_batch_result() - result.read(iprot) - iprot.readMessageEnd() - if result.success is not None: - return result.success - if result.e is not None: - raise result.e - raise TApplicationException(TApplicationException.MISSING_RESULT, "search_binary_vector_batch failed: unknown result") - - -class Processor(Iface, TProcessor): - def __init__(self, handler): - self._handler = handler - self._processMap = {} - self._processMap["add_group"] = Processor.process_add_group - self._processMap["get_group"] = Processor.process_get_group - self._processMap["del_group"] = Processor.process_del_group - self._processMap["add_vector"] = Processor.process_add_vector - self._processMap["add_vector_batch"] = Processor.process_add_vector_batch - self._processMap["add_binary_vector"] = Processor.process_add_binary_vector - self._processMap["add_binary_vector_batch"] = Processor.process_add_binary_vector_batch - self._processMap["search_vector"] = Processor.process_search_vector - self._processMap["search_vector_batch"] = Processor.process_search_vector_batch - self._processMap["search_binary_vector"] = Processor.process_search_binary_vector - self._processMap["search_binary_vector_batch"] = Processor.process_search_binary_vector_batch - - def process(self, iprot, oprot): - (name, type, seqid) = iprot.readMessageBegin() - if name not in self._processMap: - iprot.skip(TType.STRUCT) - iprot.readMessageEnd() - x = TApplicationException(TApplicationException.UNKNOWN_METHOD, 'Unknown function %s' % (name)) - oprot.writeMessageBegin(name, TMessageType.EXCEPTION, seqid) - x.write(oprot) - oprot.writeMessageEnd() - oprot.trans.flush() - return - else: - self._processMap[name](self, seqid, iprot, oprot) - return True - - def process_add_group(self, seqid, iprot, oprot): - args = add_group_args() - args.read(iprot) - iprot.readMessageEnd() - result = add_group_result() - try: - self._handler.add_group(args.group) - msg_type = TMessageType.REPLY - except TTransport.TTransportException: - raise - except VecException as e: - msg_type = TMessageType.REPLY - result.e = e - except TApplicationException as ex: - logging.exception('TApplication exception in handler') - msg_type = TMessageType.EXCEPTION - result = ex - except Exception: - logging.exception('Unexpected exception in handler') - msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') - oprot.writeMessageBegin("add_group", msg_type, seqid) - result.write(oprot) - oprot.writeMessageEnd() - oprot.trans.flush() - - def process_get_group(self, seqid, iprot, oprot): - args = get_group_args() - args.read(iprot) - iprot.readMessageEnd() - result = get_group_result() - try: - result.success = self._handler.get_group(args.group_id) - msg_type = TMessageType.REPLY - except TTransport.TTransportException: - raise - except VecException as e: - msg_type = TMessageType.REPLY - result.e = e - except TApplicationException as ex: - logging.exception('TApplication exception in handler') - msg_type = TMessageType.EXCEPTION - result = ex - except Exception: - logging.exception('Unexpected exception in handler') - msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') - oprot.writeMessageBegin("get_group", msg_type, seqid) - result.write(oprot) - oprot.writeMessageEnd() - oprot.trans.flush() - - def process_del_group(self, seqid, iprot, oprot): - args = del_group_args() - args.read(iprot) - iprot.readMessageEnd() - result = del_group_result() - try: - self._handler.del_group(args.group_id) - msg_type = TMessageType.REPLY - except TTransport.TTransportException: - raise - except VecException as e: - msg_type = TMessageType.REPLY - result.e = e - except TApplicationException as ex: - logging.exception('TApplication exception in handler') - msg_type = TMessageType.EXCEPTION - result = ex - except Exception: - logging.exception('Unexpected exception in handler') - msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') - oprot.writeMessageBegin("del_group", msg_type, seqid) - result.write(oprot) - oprot.writeMessageEnd() - oprot.trans.flush() - - def process_add_vector(self, seqid, iprot, oprot): - args = add_vector_args() - args.read(iprot) - iprot.readMessageEnd() - result = add_vector_result() - try: - result.success = self._handler.add_vector(args.group_id, args.tensor) - msg_type = TMessageType.REPLY - except TTransport.TTransportException: - raise - except VecException as e: - msg_type = TMessageType.REPLY - result.e = e - except TApplicationException as ex: - logging.exception('TApplication exception in handler') - msg_type = TMessageType.EXCEPTION - result = ex - except Exception: - logging.exception('Unexpected exception in handler') - msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') - oprot.writeMessageBegin("add_vector", msg_type, seqid) - result.write(oprot) - oprot.writeMessageEnd() - oprot.trans.flush() - - def process_add_vector_batch(self, seqid, iprot, oprot): - args = add_vector_batch_args() - args.read(iprot) - iprot.readMessageEnd() - result = add_vector_batch_result() - try: - result.success = self._handler.add_vector_batch(args.group_id, args.tensor_list) - msg_type = TMessageType.REPLY - except TTransport.TTransportException: - raise - except VecException as e: - msg_type = TMessageType.REPLY - result.e = e - except TApplicationException as ex: - logging.exception('TApplication exception in handler') - msg_type = TMessageType.EXCEPTION - result = ex - except Exception: - logging.exception('Unexpected exception in handler') - msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') - oprot.writeMessageBegin("add_vector_batch", msg_type, seqid) - result.write(oprot) - oprot.writeMessageEnd() - oprot.trans.flush() - - def process_add_binary_vector(self, seqid, iprot, oprot): - args = add_binary_vector_args() - args.read(iprot) - iprot.readMessageEnd() - result = add_binary_vector_result() - try: - result.success = self._handler.add_binary_vector(args.group_id, args.tensor) - msg_type = TMessageType.REPLY - except TTransport.TTransportException: - raise - except VecException as e: - msg_type = TMessageType.REPLY - result.e = e - except TApplicationException as ex: - logging.exception('TApplication exception in handler') - msg_type = TMessageType.EXCEPTION - result = ex - except Exception: - logging.exception('Unexpected exception in handler') - msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') - oprot.writeMessageBegin("add_binary_vector", msg_type, seqid) - result.write(oprot) - oprot.writeMessageEnd() - oprot.trans.flush() - - def process_add_binary_vector_batch(self, seqid, iprot, oprot): - args = add_binary_vector_batch_args() - args.read(iprot) - iprot.readMessageEnd() - result = add_binary_vector_batch_result() - try: - result.success = self._handler.add_binary_vector_batch(args.group_id, args.tensor_list) - msg_type = TMessageType.REPLY - except TTransport.TTransportException: - raise - except VecException as e: - msg_type = TMessageType.REPLY - result.e = e - except TApplicationException as ex: - logging.exception('TApplication exception in handler') - msg_type = TMessageType.EXCEPTION - result = ex - except Exception: - logging.exception('Unexpected exception in handler') - msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') - oprot.writeMessageBegin("add_binary_vector_batch", msg_type, seqid) - result.write(oprot) - oprot.writeMessageEnd() - oprot.trans.flush() - - def process_search_vector(self, seqid, iprot, oprot): - args = search_vector_args() - args.read(iprot) - iprot.readMessageEnd() - result = search_vector_result() - try: - result.success = self._handler.search_vector(args.group_id, args.top_k, args.tensor, args.filter) - msg_type = TMessageType.REPLY - except TTransport.TTransportException: - raise - except VecException as e: - msg_type = TMessageType.REPLY - result.e = e - except TApplicationException as ex: - logging.exception('TApplication exception in handler') - msg_type = TMessageType.EXCEPTION - result = ex - except Exception: - logging.exception('Unexpected exception in handler') - msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') - oprot.writeMessageBegin("search_vector", msg_type, seqid) - result.write(oprot) - oprot.writeMessageEnd() - oprot.trans.flush() - - def process_search_vector_batch(self, seqid, iprot, oprot): - args = search_vector_batch_args() - args.read(iprot) - iprot.readMessageEnd() - result = search_vector_batch_result() - try: - result.success = self._handler.search_vector_batch(args.group_id, args.top_k, args.tensor_list, args.filter) - msg_type = TMessageType.REPLY - except TTransport.TTransportException: - raise - except VecException as e: - msg_type = TMessageType.REPLY - result.e = e - except TApplicationException as ex: - logging.exception('TApplication exception in handler') - msg_type = TMessageType.EXCEPTION - result = ex - except Exception: - logging.exception('Unexpected exception in handler') - msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') - oprot.writeMessageBegin("search_vector_batch", msg_type, seqid) - result.write(oprot) - oprot.writeMessageEnd() - oprot.trans.flush() - - def process_search_binary_vector(self, seqid, iprot, oprot): - args = search_binary_vector_args() - args.read(iprot) - iprot.readMessageEnd() - result = search_binary_vector_result() - try: - result.success = self._handler.search_binary_vector(args.group_id, args.top_k, args.tensor, args.filter) - msg_type = TMessageType.REPLY - except TTransport.TTransportException: - raise - except VecException as e: - msg_type = TMessageType.REPLY - result.e = e - except TApplicationException as ex: - logging.exception('TApplication exception in handler') - msg_type = TMessageType.EXCEPTION - result = ex - except Exception: - logging.exception('Unexpected exception in handler') - msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') - oprot.writeMessageBegin("search_binary_vector", msg_type, seqid) - result.write(oprot) - oprot.writeMessageEnd() - oprot.trans.flush() - - def process_search_binary_vector_batch(self, seqid, iprot, oprot): - args = search_binary_vector_batch_args() - args.read(iprot) - iprot.readMessageEnd() - result = search_binary_vector_batch_result() - try: - result.success = self._handler.search_binary_vector_batch(args.group_id, args.top_k, args.tensor_list, args.filter) - msg_type = TMessageType.REPLY - except TTransport.TTransportException: - raise - except VecException as e: - msg_type = TMessageType.REPLY - result.e = e - except TApplicationException as ex: - logging.exception('TApplication exception in handler') - msg_type = TMessageType.EXCEPTION - result = ex - except Exception: - logging.exception('Unexpected exception in handler') - msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') - oprot.writeMessageBegin("search_binary_vector_batch", msg_type, seqid) - result.write(oprot) - oprot.writeMessageEnd() - oprot.trans.flush() - -# HELPER FUNCTIONS AND STRUCTURES - - -class add_group_args(object): - """ - Attributes: - - group - """ - - - def __init__(self, group=None,): - self.group = group - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 2: - if ftype == TType.STRUCT: - self.group = VecGroup() - self.group.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('add_group_args') - if self.group is not None: - oprot.writeFieldBegin('group', TType.STRUCT, 2) - self.group.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(add_group_args) -add_group_args.thrift_spec = ( - None, # 0 - None, # 1 - (2, TType.STRUCT, 'group', [VecGroup, None], None, ), # 2 -) - - -class add_group_result(object): - """ - Attributes: - - e - """ - - - def __init__(self, e=None,): - self.e = e - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.STRUCT: - self.e = VecException() - self.e.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('add_group_result') - if self.e is not None: - oprot.writeFieldBegin('e', TType.STRUCT, 1) - self.e.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(add_group_result) -add_group_result.thrift_spec = ( - None, # 0 - (1, TType.STRUCT, 'e', [VecException, None], None, ), # 1 -) - - -class get_group_args(object): - """ - Attributes: - - group_id - """ - - - def __init__(self, group_id=None,): - self.group_id = group_id - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 2: - if ftype == TType.STRING: - self.group_id = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('get_group_args') - if self.group_id is not None: - oprot.writeFieldBegin('group_id', TType.STRING, 2) - oprot.writeString(self.group_id.encode('utf-8') if sys.version_info[0] == 2 else self.group_id) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(get_group_args) -get_group_args.thrift_spec = ( - None, # 0 - None, # 1 - (2, TType.STRING, 'group_id', 'UTF8', None, ), # 2 -) - - -class get_group_result(object): - """ - Attributes: - - success - - e - """ - - - def __init__(self, success=None, e=None,): - self.success = success - self.e = e - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 0: - if ftype == TType.STRUCT: - self.success = VecGroup() - self.success.read(iprot) - else: - iprot.skip(ftype) - elif fid == 1: - if ftype == TType.STRUCT: - self.e = VecException() - self.e.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('get_group_result') - if self.success is not None: - oprot.writeFieldBegin('success', TType.STRUCT, 0) - self.success.write(oprot) - oprot.writeFieldEnd() - if self.e is not None: - oprot.writeFieldBegin('e', TType.STRUCT, 1) - self.e.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(get_group_result) -get_group_result.thrift_spec = ( - (0, TType.STRUCT, 'success', [VecGroup, None], None, ), # 0 - (1, TType.STRUCT, 'e', [VecException, None], None, ), # 1 -) - - -class del_group_args(object): - """ - Attributes: - - group_id - """ - - - def __init__(self, group_id=None,): - self.group_id = group_id - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 2: - if ftype == TType.STRING: - self.group_id = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('del_group_args') - if self.group_id is not None: - oprot.writeFieldBegin('group_id', TType.STRING, 2) - oprot.writeString(self.group_id.encode('utf-8') if sys.version_info[0] == 2 else self.group_id) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(del_group_args) -del_group_args.thrift_spec = ( - None, # 0 - None, # 1 - (2, TType.STRING, 'group_id', 'UTF8', None, ), # 2 -) - - -class del_group_result(object): - """ - Attributes: - - e - """ - - - def __init__(self, e=None,): - self.e = e - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.STRUCT: - self.e = VecException() - self.e.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('del_group_result') - if self.e is not None: - oprot.writeFieldBegin('e', TType.STRUCT, 1) - self.e.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(del_group_result) -del_group_result.thrift_spec = ( - None, # 0 - (1, TType.STRUCT, 'e', [VecException, None], None, ), # 1 -) - - -class add_vector_args(object): - """ - Attributes: - - group_id - - tensor - """ - - - def __init__(self, group_id=None, tensor=None,): - self.group_id = group_id - self.tensor = tensor - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 2: - if ftype == TType.STRING: - self.group_id = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.STRUCT: - self.tensor = VecTensor() - self.tensor.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('add_vector_args') - if self.group_id is not None: - oprot.writeFieldBegin('group_id', TType.STRING, 2) - oprot.writeString(self.group_id.encode('utf-8') if sys.version_info[0] == 2 else self.group_id) - oprot.writeFieldEnd() - if self.tensor is not None: - oprot.writeFieldBegin('tensor', TType.STRUCT, 3) - self.tensor.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(add_vector_args) -add_vector_args.thrift_spec = ( - None, # 0 - None, # 1 - (2, TType.STRING, 'group_id', 'UTF8', None, ), # 2 - (3, TType.STRUCT, 'tensor', [VecTensor, None], None, ), # 3 -) - - -class add_vector_result(object): - """ - Attributes: - - success - - e - """ - - - def __init__(self, success=None, e=None,): - self.success = success - self.e = e - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 0: - if ftype == TType.STRING: - self.success = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - else: - iprot.skip(ftype) - elif fid == 1: - if ftype == TType.STRUCT: - self.e = VecException() - self.e.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('add_vector_result') - if self.success is not None: - oprot.writeFieldBegin('success', TType.STRING, 0) - oprot.writeString(self.success.encode('utf-8') if sys.version_info[0] == 2 else self.success) - oprot.writeFieldEnd() - if self.e is not None: - oprot.writeFieldBegin('e', TType.STRUCT, 1) - self.e.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(add_vector_result) -add_vector_result.thrift_spec = ( - (0, TType.STRING, 'success', 'UTF8', None, ), # 0 - (1, TType.STRUCT, 'e', [VecException, None], None, ), # 1 -) - - -class add_vector_batch_args(object): - """ - Attributes: - - group_id - - tensor_list - """ - - - def __init__(self, group_id=None, tensor_list=None,): - self.group_id = group_id - self.tensor_list = tensor_list - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 2: - if ftype == TType.STRING: - self.group_id = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.STRUCT: - self.tensor_list = VecTensorList() - self.tensor_list.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('add_vector_batch_args') - if self.group_id is not None: - oprot.writeFieldBegin('group_id', TType.STRING, 2) - oprot.writeString(self.group_id.encode('utf-8') if sys.version_info[0] == 2 else self.group_id) - oprot.writeFieldEnd() - if self.tensor_list is not None: - oprot.writeFieldBegin('tensor_list', TType.STRUCT, 3) - self.tensor_list.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(add_vector_batch_args) -add_vector_batch_args.thrift_spec = ( - None, # 0 - None, # 1 - (2, TType.STRING, 'group_id', 'UTF8', None, ), # 2 - (3, TType.STRUCT, 'tensor_list', [VecTensorList, None], None, ), # 3 -) - - -class add_vector_batch_result(object): - """ - Attributes: - - success - - e - """ - - - def __init__(self, success=None, e=None,): - self.success = success - self.e = e - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 0: - if ftype == TType.LIST: - self.success = [] - (_etype88, _size85) = iprot.readListBegin() - for _i89 in range(_size85): - _elem90 = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - self.success.append(_elem90) - iprot.readListEnd() - else: - iprot.skip(ftype) - elif fid == 1: - if ftype == TType.STRUCT: - self.e = VecException() - self.e.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('add_vector_batch_result') - if self.success is not None: - oprot.writeFieldBegin('success', TType.LIST, 0) - oprot.writeListBegin(TType.STRING, len(self.success)) - for iter91 in self.success: - oprot.writeString(iter91.encode('utf-8') if sys.version_info[0] == 2 else iter91) - oprot.writeListEnd() - oprot.writeFieldEnd() - if self.e is not None: - oprot.writeFieldBegin('e', TType.STRUCT, 1) - self.e.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(add_vector_batch_result) -add_vector_batch_result.thrift_spec = ( - (0, TType.LIST, 'success', (TType.STRING, 'UTF8', False), None, ), # 0 - (1, TType.STRUCT, 'e', [VecException, None], None, ), # 1 -) - - -class add_binary_vector_args(object): - """ - Attributes: - - group_id - - tensor - """ - - - def __init__(self, group_id=None, tensor=None,): - self.group_id = group_id - self.tensor = tensor - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 2: - if ftype == TType.STRING: - self.group_id = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.STRUCT: - self.tensor = VecBinaryTensor() - self.tensor.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('add_binary_vector_args') - if self.group_id is not None: - oprot.writeFieldBegin('group_id', TType.STRING, 2) - oprot.writeString(self.group_id.encode('utf-8') if sys.version_info[0] == 2 else self.group_id) - oprot.writeFieldEnd() - if self.tensor is not None: - oprot.writeFieldBegin('tensor', TType.STRUCT, 3) - self.tensor.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(add_binary_vector_args) -add_binary_vector_args.thrift_spec = ( - None, # 0 - None, # 1 - (2, TType.STRING, 'group_id', 'UTF8', None, ), # 2 - (3, TType.STRUCT, 'tensor', [VecBinaryTensor, None], None, ), # 3 -) - - -class add_binary_vector_result(object): - """ - Attributes: - - success - - e - """ - - - def __init__(self, success=None, e=None,): - self.success = success - self.e = e - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 0: - if ftype == TType.STRING: - self.success = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - else: - iprot.skip(ftype) - elif fid == 1: - if ftype == TType.STRUCT: - self.e = VecException() - self.e.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('add_binary_vector_result') - if self.success is not None: - oprot.writeFieldBegin('success', TType.STRING, 0) - oprot.writeString(self.success.encode('utf-8') if sys.version_info[0] == 2 else self.success) - oprot.writeFieldEnd() - if self.e is not None: - oprot.writeFieldBegin('e', TType.STRUCT, 1) - self.e.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(add_binary_vector_result) -add_binary_vector_result.thrift_spec = ( - (0, TType.STRING, 'success', 'UTF8', None, ), # 0 - (1, TType.STRUCT, 'e', [VecException, None], None, ), # 1 -) - - -class add_binary_vector_batch_args(object): - """ - Attributes: - - group_id - - tensor_list - """ - - - def __init__(self, group_id=None, tensor_list=None,): - self.group_id = group_id - self.tensor_list = tensor_list - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 2: - if ftype == TType.STRING: - self.group_id = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.STRUCT: - self.tensor_list = VecBinaryTensorList() - self.tensor_list.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('add_binary_vector_batch_args') - if self.group_id is not None: - oprot.writeFieldBegin('group_id', TType.STRING, 2) - oprot.writeString(self.group_id.encode('utf-8') if sys.version_info[0] == 2 else self.group_id) - oprot.writeFieldEnd() - if self.tensor_list is not None: - oprot.writeFieldBegin('tensor_list', TType.STRUCT, 3) - self.tensor_list.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(add_binary_vector_batch_args) -add_binary_vector_batch_args.thrift_spec = ( - None, # 0 - None, # 1 - (2, TType.STRING, 'group_id', 'UTF8', None, ), # 2 - (3, TType.STRUCT, 'tensor_list', [VecBinaryTensorList, None], None, ), # 3 -) - - -class add_binary_vector_batch_result(object): - """ - Attributes: - - success - - e - """ - - - def __init__(self, success=None, e=None,): - self.success = success - self.e = e - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 0: - if ftype == TType.LIST: - self.success = [] - (_etype95, _size92) = iprot.readListBegin() - for _i96 in range(_size92): - _elem97 = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - self.success.append(_elem97) - iprot.readListEnd() - else: - iprot.skip(ftype) - elif fid == 1: - if ftype == TType.STRUCT: - self.e = VecException() - self.e.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('add_binary_vector_batch_result') - if self.success is not None: - oprot.writeFieldBegin('success', TType.LIST, 0) - oprot.writeListBegin(TType.STRING, len(self.success)) - for iter98 in self.success: - oprot.writeString(iter98.encode('utf-8') if sys.version_info[0] == 2 else iter98) - oprot.writeListEnd() - oprot.writeFieldEnd() - if self.e is not None: - oprot.writeFieldBegin('e', TType.STRUCT, 1) - self.e.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(add_binary_vector_batch_result) -add_binary_vector_batch_result.thrift_spec = ( - (0, TType.LIST, 'success', (TType.STRING, 'UTF8', False), None, ), # 0 - (1, TType.STRUCT, 'e', [VecException, None], None, ), # 1 -) - - -class search_vector_args(object): - """ - Attributes: - - group_id - - top_k - - tensor - - filter - """ - - - def __init__(self, group_id=None, top_k=None, tensor=None, filter=None,): - self.group_id = group_id - self.top_k = top_k - self.tensor = tensor - self.filter = filter - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 2: - if ftype == TType.STRING: - self.group_id = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.I64: - self.top_k = iprot.readI64() - else: - iprot.skip(ftype) - elif fid == 4: - if ftype == TType.STRUCT: - self.tensor = VecTensor() - self.tensor.read(iprot) - else: - iprot.skip(ftype) - elif fid == 5: - if ftype == TType.STRUCT: - self.filter = VecSearchFilter() - self.filter.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('search_vector_args') - if self.group_id is not None: - oprot.writeFieldBegin('group_id', TType.STRING, 2) - oprot.writeString(self.group_id.encode('utf-8') if sys.version_info[0] == 2 else self.group_id) - oprot.writeFieldEnd() - if self.top_k is not None: - oprot.writeFieldBegin('top_k', TType.I64, 3) - oprot.writeI64(self.top_k) - oprot.writeFieldEnd() - if self.tensor is not None: - oprot.writeFieldBegin('tensor', TType.STRUCT, 4) - self.tensor.write(oprot) - oprot.writeFieldEnd() - if self.filter is not None: - oprot.writeFieldBegin('filter', TType.STRUCT, 5) - self.filter.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(search_vector_args) -search_vector_args.thrift_spec = ( - None, # 0 - None, # 1 - (2, TType.STRING, 'group_id', 'UTF8', None, ), # 2 - (3, TType.I64, 'top_k', None, None, ), # 3 - (4, TType.STRUCT, 'tensor', [VecTensor, None], None, ), # 4 - (5, TType.STRUCT, 'filter', [VecSearchFilter, None], None, ), # 5 -) - - -class search_vector_result(object): - """ - Attributes: - - success - - e - """ - - - def __init__(self, success=None, e=None,): - self.success = success - self.e = e - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 0: - if ftype == TType.STRUCT: - self.success = VecSearchResult() - self.success.read(iprot) - else: - iprot.skip(ftype) - elif fid == 1: - if ftype == TType.STRUCT: - self.e = VecException() - self.e.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('search_vector_result') - if self.success is not None: - oprot.writeFieldBegin('success', TType.STRUCT, 0) - self.success.write(oprot) - oprot.writeFieldEnd() - if self.e is not None: - oprot.writeFieldBegin('e', TType.STRUCT, 1) - self.e.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(search_vector_result) -search_vector_result.thrift_spec = ( - (0, TType.STRUCT, 'success', [VecSearchResult, None], None, ), # 0 - (1, TType.STRUCT, 'e', [VecException, None], None, ), # 1 -) - - -class search_vector_batch_args(object): - """ - Attributes: - - group_id - - top_k - - tensor_list - - filter - """ - - - def __init__(self, group_id=None, top_k=None, tensor_list=None, filter=None,): - self.group_id = group_id - self.top_k = top_k - self.tensor_list = tensor_list - self.filter = filter - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 2: - if ftype == TType.STRING: - self.group_id = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.I64: - self.top_k = iprot.readI64() - else: - iprot.skip(ftype) - elif fid == 4: - if ftype == TType.STRUCT: - self.tensor_list = VecTensorList() - self.tensor_list.read(iprot) - else: - iprot.skip(ftype) - elif fid == 5: - if ftype == TType.STRUCT: - self.filter = VecSearchFilter() - self.filter.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('search_vector_batch_args') - if self.group_id is not None: - oprot.writeFieldBegin('group_id', TType.STRING, 2) - oprot.writeString(self.group_id.encode('utf-8') if sys.version_info[0] == 2 else self.group_id) - oprot.writeFieldEnd() - if self.top_k is not None: - oprot.writeFieldBegin('top_k', TType.I64, 3) - oprot.writeI64(self.top_k) - oprot.writeFieldEnd() - if self.tensor_list is not None: - oprot.writeFieldBegin('tensor_list', TType.STRUCT, 4) - self.tensor_list.write(oprot) - oprot.writeFieldEnd() - if self.filter is not None: - oprot.writeFieldBegin('filter', TType.STRUCT, 5) - self.filter.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(search_vector_batch_args) -search_vector_batch_args.thrift_spec = ( - None, # 0 - None, # 1 - (2, TType.STRING, 'group_id', 'UTF8', None, ), # 2 - (3, TType.I64, 'top_k', None, None, ), # 3 - (4, TType.STRUCT, 'tensor_list', [VecTensorList, None], None, ), # 4 - (5, TType.STRUCT, 'filter', [VecSearchFilter, None], None, ), # 5 -) - - -class search_vector_batch_result(object): - """ - Attributes: - - success - - e - """ - - - def __init__(self, success=None, e=None,): - self.success = success - self.e = e - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 0: - if ftype == TType.STRUCT: - self.success = VecSearchResultList() - self.success.read(iprot) - else: - iprot.skip(ftype) - elif fid == 1: - if ftype == TType.STRUCT: - self.e = VecException() - self.e.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('search_vector_batch_result') - if self.success is not None: - oprot.writeFieldBegin('success', TType.STRUCT, 0) - self.success.write(oprot) - oprot.writeFieldEnd() - if self.e is not None: - oprot.writeFieldBegin('e', TType.STRUCT, 1) - self.e.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(search_vector_batch_result) -search_vector_batch_result.thrift_spec = ( - (0, TType.STRUCT, 'success', [VecSearchResultList, None], None, ), # 0 - (1, TType.STRUCT, 'e', [VecException, None], None, ), # 1 -) - - -class search_binary_vector_args(object): - """ - Attributes: - - group_id - - top_k - - tensor - - filter - """ - - - def __init__(self, group_id=None, top_k=None, tensor=None, filter=None,): - self.group_id = group_id - self.top_k = top_k - self.tensor = tensor - self.filter = filter - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 2: - if ftype == TType.STRING: - self.group_id = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.I64: - self.top_k = iprot.readI64() - else: - iprot.skip(ftype) - elif fid == 4: - if ftype == TType.STRUCT: - self.tensor = VecBinaryTensor() - self.tensor.read(iprot) - else: - iprot.skip(ftype) - elif fid == 5: - if ftype == TType.STRUCT: - self.filter = VecSearchFilter() - self.filter.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('search_binary_vector_args') - if self.group_id is not None: - oprot.writeFieldBegin('group_id', TType.STRING, 2) - oprot.writeString(self.group_id.encode('utf-8') if sys.version_info[0] == 2 else self.group_id) - oprot.writeFieldEnd() - if self.top_k is not None: - oprot.writeFieldBegin('top_k', TType.I64, 3) - oprot.writeI64(self.top_k) - oprot.writeFieldEnd() - if self.tensor is not None: - oprot.writeFieldBegin('tensor', TType.STRUCT, 4) - self.tensor.write(oprot) - oprot.writeFieldEnd() - if self.filter is not None: - oprot.writeFieldBegin('filter', TType.STRUCT, 5) - self.filter.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(search_binary_vector_args) -search_binary_vector_args.thrift_spec = ( - None, # 0 - None, # 1 - (2, TType.STRING, 'group_id', 'UTF8', None, ), # 2 - (3, TType.I64, 'top_k', None, None, ), # 3 - (4, TType.STRUCT, 'tensor', [VecBinaryTensor, None], None, ), # 4 - (5, TType.STRUCT, 'filter', [VecSearchFilter, None], None, ), # 5 -) - - -class search_binary_vector_result(object): - """ - Attributes: - - success - - e - """ - - - def __init__(self, success=None, e=None,): - self.success = success - self.e = e - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 0: - if ftype == TType.STRUCT: - self.success = VecSearchResult() - self.success.read(iprot) - else: - iprot.skip(ftype) - elif fid == 1: - if ftype == TType.STRUCT: - self.e = VecException() - self.e.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('search_binary_vector_result') - if self.success is not None: - oprot.writeFieldBegin('success', TType.STRUCT, 0) - self.success.write(oprot) - oprot.writeFieldEnd() - if self.e is not None: - oprot.writeFieldBegin('e', TType.STRUCT, 1) - self.e.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(search_binary_vector_result) -search_binary_vector_result.thrift_spec = ( - (0, TType.STRUCT, 'success', [VecSearchResult, None], None, ), # 0 - (1, TType.STRUCT, 'e', [VecException, None], None, ), # 1 -) - - -class search_binary_vector_batch_args(object): - """ - Attributes: - - group_id - - top_k - - tensor_list - - filter - """ - - - def __init__(self, group_id=None, top_k=None, tensor_list=None, filter=None,): - self.group_id = group_id - self.top_k = top_k - self.tensor_list = tensor_list - self.filter = filter - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 2: - if ftype == TType.STRING: - self.group_id = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.I64: - self.top_k = iprot.readI64() - else: - iprot.skip(ftype) - elif fid == 4: - if ftype == TType.STRUCT: - self.tensor_list = VecBinaryTensorList() - self.tensor_list.read(iprot) - else: - iprot.skip(ftype) - elif fid == 5: - if ftype == TType.STRUCT: - self.filter = VecSearchFilter() - self.filter.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('search_binary_vector_batch_args') - if self.group_id is not None: - oprot.writeFieldBegin('group_id', TType.STRING, 2) - oprot.writeString(self.group_id.encode('utf-8') if sys.version_info[0] == 2 else self.group_id) - oprot.writeFieldEnd() - if self.top_k is not None: - oprot.writeFieldBegin('top_k', TType.I64, 3) - oprot.writeI64(self.top_k) - oprot.writeFieldEnd() - if self.tensor_list is not None: - oprot.writeFieldBegin('tensor_list', TType.STRUCT, 4) - self.tensor_list.write(oprot) - oprot.writeFieldEnd() - if self.filter is not None: - oprot.writeFieldBegin('filter', TType.STRUCT, 5) - self.filter.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(search_binary_vector_batch_args) -search_binary_vector_batch_args.thrift_spec = ( - None, # 0 - None, # 1 - (2, TType.STRING, 'group_id', 'UTF8', None, ), # 2 - (3, TType.I64, 'top_k', None, None, ), # 3 - (4, TType.STRUCT, 'tensor_list', [VecBinaryTensorList, None], None, ), # 4 - (5, TType.STRUCT, 'filter', [VecSearchFilter, None], None, ), # 5 -) - - -class search_binary_vector_batch_result(object): - """ - Attributes: - - success - - e - """ - - - def __init__(self, success=None, e=None,): - self.success = success - self.e = e - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 0: - if ftype == TType.STRUCT: - self.success = VecSearchResultList() - self.success.read(iprot) - else: - iprot.skip(ftype) - elif fid == 1: - if ftype == TType.STRUCT: - self.e = VecException() - self.e.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('search_binary_vector_batch_result') - if self.success is not None: - oprot.writeFieldBegin('success', TType.STRUCT, 0) - self.success.write(oprot) - oprot.writeFieldEnd() - if self.e is not None: - oprot.writeFieldBegin('e', TType.STRUCT, 1) - self.e.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(search_binary_vector_batch_result) -search_binary_vector_batch_result.thrift_spec = ( - (0, TType.STRUCT, 'success', [VecSearchResultList, None], None, ), # 0 - (1, TType.STRUCT, 'e', [VecException, None], None, ), # 1 -) -fix_spec(all_structs) -del all_structs - diff --git a/cpp/src/thrift/gen-py/zilliz/__init__.py b/cpp/src/thrift/gen-py/zilliz/__init__.py deleted file mode 100644 index fa8ce63771..0000000000 --- a/cpp/src/thrift/gen-py/zilliz/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__all__ = ['ttypes', 'constants', 'VecService'] diff --git a/cpp/src/thrift/gen-py/zilliz/constants.py b/cpp/src/thrift/gen-py/zilliz/constants.py deleted file mode 100644 index 0c217ceda6..0000000000 --- a/cpp/src/thrift/gen-py/zilliz/constants.py +++ /dev/null @@ -1,14 +0,0 @@ -# -# Autogenerated by Thrift Compiler (0.11.0) -# -# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING -# -# options string: py -# - -from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException -from thrift.protocol.TProtocol import TProtocolException -from thrift.TRecursive import fix_spec - -import sys -from .ttypes import * diff --git a/cpp/src/thrift/gen-py/zilliz/ttypes.py b/cpp/src/thrift/gen-py/zilliz/ttypes.py deleted file mode 100644 index 74addabdc3..0000000000 --- a/cpp/src/thrift/gen-py/zilliz/ttypes.py +++ /dev/null @@ -1,1165 +0,0 @@ -# -# Autogenerated by Thrift Compiler (0.11.0) -# -# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING -# -# options string: py -# - -from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException -from thrift.protocol.TProtocol import TProtocolException -from thrift.TRecursive import fix_spec - -import sys - -from thrift.transport import TTransport -all_structs = [] - - -class VecErrCode(object): - SUCCESS = 0 - ILLEGAL_ARGUMENT = 1 - GROUP_NOT_EXISTS = 2 - ILLEGAL_TIME_RANGE = 3 - ILLEGAL_VECTOR_DIMENSION = 4 - OUT_OF_MEMORY = 5 - - _VALUES_TO_NAMES = { - 0: "SUCCESS", - 1: "ILLEGAL_ARGUMENT", - 2: "GROUP_NOT_EXISTS", - 3: "ILLEGAL_TIME_RANGE", - 4: "ILLEGAL_VECTOR_DIMENSION", - 5: "OUT_OF_MEMORY", - } - - _NAMES_TO_VALUES = { - "SUCCESS": 0, - "ILLEGAL_ARGUMENT": 1, - "GROUP_NOT_EXISTS": 2, - "ILLEGAL_TIME_RANGE": 3, - "ILLEGAL_VECTOR_DIMENSION": 4, - "OUT_OF_MEMORY": 5, - } - - -class VecException(TException): - """ - Attributes: - - code - - reason - """ - - - def __init__(self, code=None, reason=None,): - self.code = code - self.reason = reason - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.I32: - self.code = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 2: - if ftype == TType.STRING: - self.reason = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('VecException') - if self.code is not None: - oprot.writeFieldBegin('code', TType.I32, 1) - oprot.writeI32(self.code) - oprot.writeFieldEnd() - if self.reason is not None: - oprot.writeFieldBegin('reason', TType.STRING, 2) - oprot.writeString(self.reason.encode('utf-8') if sys.version_info[0] == 2 else self.reason) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __str__(self): - return repr(self) - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - - -class VecGroup(object): - """ - Attributes: - - id - - dimension - - index_type - """ - - - def __init__(self, id=None, dimension=None, index_type=None,): - self.id = id - self.dimension = dimension - self.index_type = index_type - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.STRING: - self.id = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - else: - iprot.skip(ftype) - elif fid == 2: - if ftype == TType.I32: - self.dimension = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.I32: - self.index_type = iprot.readI32() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('VecGroup') - if self.id is not None: - oprot.writeFieldBegin('id', TType.STRING, 1) - oprot.writeString(self.id.encode('utf-8') if sys.version_info[0] == 2 else self.id) - oprot.writeFieldEnd() - if self.dimension is not None: - oprot.writeFieldBegin('dimension', TType.I32, 2) - oprot.writeI32(self.dimension) - oprot.writeFieldEnd() - if self.index_type is not None: - oprot.writeFieldBegin('index_type', TType.I32, 3) - oprot.writeI32(self.index_type) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - if self.id is None: - raise TProtocolException(message='Required field id is unset!') - if self.dimension is None: - raise TProtocolException(message='Required field dimension is unset!') - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - - -class VecTensor(object): - """ - Attributes: - - uid - - tensor - - attrib - """ - - - def __init__(self, uid=None, tensor=None, attrib=None,): - self.uid = uid - self.tensor = tensor - self.attrib = attrib - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.STRING: - self.uid = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - else: - iprot.skip(ftype) - elif fid == 2: - if ftype == TType.LIST: - self.tensor = [] - (_etype3, _size0) = iprot.readListBegin() - for _i4 in range(_size0): - _elem5 = iprot.readDouble() - self.tensor.append(_elem5) - iprot.readListEnd() - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.MAP: - self.attrib = {} - (_ktype7, _vtype8, _size6) = iprot.readMapBegin() - for _i10 in range(_size6): - _key11 = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - _val12 = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - self.attrib[_key11] = _val12 - iprot.readMapEnd() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('VecTensor') - if self.uid is not None: - oprot.writeFieldBegin('uid', TType.STRING, 1) - oprot.writeString(self.uid.encode('utf-8') if sys.version_info[0] == 2 else self.uid) - oprot.writeFieldEnd() - if self.tensor is not None: - oprot.writeFieldBegin('tensor', TType.LIST, 2) - oprot.writeListBegin(TType.DOUBLE, len(self.tensor)) - for iter13 in self.tensor: - oprot.writeDouble(iter13) - oprot.writeListEnd() - oprot.writeFieldEnd() - if self.attrib is not None: - oprot.writeFieldBegin('attrib', TType.MAP, 3) - oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.attrib)) - for kiter14, viter15 in self.attrib.items(): - oprot.writeString(kiter14.encode('utf-8') if sys.version_info[0] == 2 else kiter14) - oprot.writeString(viter15.encode('utf-8') if sys.version_info[0] == 2 else viter15) - oprot.writeMapEnd() - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - if self.uid is None: - raise TProtocolException(message='Required field uid is unset!') - if self.tensor is None: - raise TProtocolException(message='Required field tensor is unset!') - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - - -class VecTensorList(object): - """ - Attributes: - - tensor_list - """ - - - def __init__(self, tensor_list=None,): - self.tensor_list = tensor_list - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.LIST: - self.tensor_list = [] - (_etype19, _size16) = iprot.readListBegin() - for _i20 in range(_size16): - _elem21 = VecTensor() - _elem21.read(iprot) - self.tensor_list.append(_elem21) - iprot.readListEnd() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('VecTensorList') - if self.tensor_list is not None: - oprot.writeFieldBegin('tensor_list', TType.LIST, 1) - oprot.writeListBegin(TType.STRUCT, len(self.tensor_list)) - for iter22 in self.tensor_list: - iter22.write(oprot) - oprot.writeListEnd() - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - if self.tensor_list is None: - raise TProtocolException(message='Required field tensor_list is unset!') - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - - -class VecBinaryTensor(object): - """ - Attributes: - - uid - - tensor - - attrib - """ - - - def __init__(self, uid=None, tensor=None, attrib=None,): - self.uid = uid - self.tensor = tensor - self.attrib = attrib - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.STRING: - self.uid = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - else: - iprot.skip(ftype) - elif fid == 2: - if ftype == TType.STRING: - self.tensor = iprot.readBinary() - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.MAP: - self.attrib = {} - (_ktype24, _vtype25, _size23) = iprot.readMapBegin() - for _i27 in range(_size23): - _key28 = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - _val29 = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - self.attrib[_key28] = _val29 - iprot.readMapEnd() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('VecBinaryTensor') - if self.uid is not None: - oprot.writeFieldBegin('uid', TType.STRING, 1) - oprot.writeString(self.uid.encode('utf-8') if sys.version_info[0] == 2 else self.uid) - oprot.writeFieldEnd() - if self.tensor is not None: - oprot.writeFieldBegin('tensor', TType.STRING, 2) - oprot.writeBinary(self.tensor) - oprot.writeFieldEnd() - if self.attrib is not None: - oprot.writeFieldBegin('attrib', TType.MAP, 3) - oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.attrib)) - for kiter30, viter31 in self.attrib.items(): - oprot.writeString(kiter30.encode('utf-8') if sys.version_info[0] == 2 else kiter30) - oprot.writeString(viter31.encode('utf-8') if sys.version_info[0] == 2 else viter31) - oprot.writeMapEnd() - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - if self.uid is None: - raise TProtocolException(message='Required field uid is unset!') - if self.tensor is None: - raise TProtocolException(message='Required field tensor is unset!') - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - - -class VecBinaryTensorList(object): - """ - Attributes: - - tensor_list - """ - - - def __init__(self, tensor_list=None,): - self.tensor_list = tensor_list - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.LIST: - self.tensor_list = [] - (_etype35, _size32) = iprot.readListBegin() - for _i36 in range(_size32): - _elem37 = VecBinaryTensor() - _elem37.read(iprot) - self.tensor_list.append(_elem37) - iprot.readListEnd() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('VecBinaryTensorList') - if self.tensor_list is not None: - oprot.writeFieldBegin('tensor_list', TType.LIST, 1) - oprot.writeListBegin(TType.STRUCT, len(self.tensor_list)) - for iter38 in self.tensor_list: - iter38.write(oprot) - oprot.writeListEnd() - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - if self.tensor_list is None: - raise TProtocolException(message='Required field tensor_list is unset!') - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - - -class VecSearchResultItem(object): - """ - Attributes: - - uid - - distance - - attrib - """ - - - def __init__(self, uid=None, distance=None, attrib=None,): - self.uid = uid - self.distance = distance - self.attrib = attrib - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.STRING: - self.uid = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - else: - iprot.skip(ftype) - elif fid == 2: - if ftype == TType.DOUBLE: - self.distance = iprot.readDouble() - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.MAP: - self.attrib = {} - (_ktype40, _vtype41, _size39) = iprot.readMapBegin() - for _i43 in range(_size39): - _key44 = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - _val45 = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - self.attrib[_key44] = _val45 - iprot.readMapEnd() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('VecSearchResultItem') - if self.uid is not None: - oprot.writeFieldBegin('uid', TType.STRING, 1) - oprot.writeString(self.uid.encode('utf-8') if sys.version_info[0] == 2 else self.uid) - oprot.writeFieldEnd() - if self.distance is not None: - oprot.writeFieldBegin('distance', TType.DOUBLE, 2) - oprot.writeDouble(self.distance) - oprot.writeFieldEnd() - if self.attrib is not None: - oprot.writeFieldBegin('attrib', TType.MAP, 3) - oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.attrib)) - for kiter46, viter47 in self.attrib.items(): - oprot.writeString(kiter46.encode('utf-8') if sys.version_info[0] == 2 else kiter46) - oprot.writeString(viter47.encode('utf-8') if sys.version_info[0] == 2 else viter47) - oprot.writeMapEnd() - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - if self.uid is None: - raise TProtocolException(message='Required field uid is unset!') - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - - -class VecSearchResult(object): - """ - Attributes: - - result_list - """ - - - def __init__(self, result_list=None,): - self.result_list = result_list - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.LIST: - self.result_list = [] - (_etype51, _size48) = iprot.readListBegin() - for _i52 in range(_size48): - _elem53 = VecSearchResultItem() - _elem53.read(iprot) - self.result_list.append(_elem53) - iprot.readListEnd() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('VecSearchResult') - if self.result_list is not None: - oprot.writeFieldBegin('result_list', TType.LIST, 1) - oprot.writeListBegin(TType.STRUCT, len(self.result_list)) - for iter54 in self.result_list: - iter54.write(oprot) - oprot.writeListEnd() - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - - -class VecSearchResultList(object): - """ - Attributes: - - result_list - """ - - - def __init__(self, result_list=None,): - self.result_list = result_list - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.LIST: - self.result_list = [] - (_etype58, _size55) = iprot.readListBegin() - for _i59 in range(_size55): - _elem60 = VecSearchResult() - _elem60.read(iprot) - self.result_list.append(_elem60) - iprot.readListEnd() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('VecSearchResultList') - if self.result_list is not None: - oprot.writeFieldBegin('result_list', TType.LIST, 1) - oprot.writeListBegin(TType.STRUCT, len(self.result_list)) - for iter61 in self.result_list: - iter61.write(oprot) - oprot.writeListEnd() - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - - -class VecDateTime(object): - """ - second; Seconds. [0-60] (1 leap second) - minute; Minutes. [0-59] - hour; Hours. [0-23] - day; Day. [1-31] - month; Month. [0-11] - year; Year - 1900. - - Attributes: - - year - - month - - day - - hour - - minute - - second - """ - - - def __init__(self, year=None, month=None, day=None, hour=None, minute=None, second=None,): - self.year = year - self.month = month - self.day = day - self.hour = hour - self.minute = minute - self.second = second - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.I32: - self.year = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 2: - if ftype == TType.I32: - self.month = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.I32: - self.day = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 4: - if ftype == TType.I32: - self.hour = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 5: - if ftype == TType.I32: - self.minute = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 6: - if ftype == TType.I32: - self.second = iprot.readI32() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('VecDateTime') - if self.year is not None: - oprot.writeFieldBegin('year', TType.I32, 1) - oprot.writeI32(self.year) - oprot.writeFieldEnd() - if self.month is not None: - oprot.writeFieldBegin('month', TType.I32, 2) - oprot.writeI32(self.month) - oprot.writeFieldEnd() - if self.day is not None: - oprot.writeFieldBegin('day', TType.I32, 3) - oprot.writeI32(self.day) - oprot.writeFieldEnd() - if self.hour is not None: - oprot.writeFieldBegin('hour', TType.I32, 4) - oprot.writeI32(self.hour) - oprot.writeFieldEnd() - if self.minute is not None: - oprot.writeFieldBegin('minute', TType.I32, 5) - oprot.writeI32(self.minute) - oprot.writeFieldEnd() - if self.second is not None: - oprot.writeFieldBegin('second', TType.I32, 6) - oprot.writeI32(self.second) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - if self.year is None: - raise TProtocolException(message='Required field year is unset!') - if self.month is None: - raise TProtocolException(message='Required field month is unset!') - if self.day is None: - raise TProtocolException(message='Required field day is unset!') - if self.hour is None: - raise TProtocolException(message='Required field hour is unset!') - if self.minute is None: - raise TProtocolException(message='Required field minute is unset!') - if self.second is None: - raise TProtocolException(message='Required field second is unset!') - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - - -class VecTimeRange(object): - """ - time_begin; time range begin - begine_closed; true means '[', false means '(' - time_end; set to true to return tensor double array - end_closed; time range end - - Attributes: - - time_begin - - begine_closed - - time_end - - end_closed - """ - - - def __init__(self, time_begin=None, begine_closed=None, time_end=None, end_closed=None,): - self.time_begin = time_begin - self.begine_closed = begine_closed - self.time_end = time_end - self.end_closed = end_closed - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.STRUCT: - self.time_begin = VecDateTime() - self.time_begin.read(iprot) - else: - iprot.skip(ftype) - elif fid == 2: - if ftype == TType.BOOL: - self.begine_closed = iprot.readBool() - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.STRUCT: - self.time_end = VecDateTime() - self.time_end.read(iprot) - else: - iprot.skip(ftype) - elif fid == 4: - if ftype == TType.BOOL: - self.end_closed = iprot.readBool() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('VecTimeRange') - if self.time_begin is not None: - oprot.writeFieldBegin('time_begin', TType.STRUCT, 1) - self.time_begin.write(oprot) - oprot.writeFieldEnd() - if self.begine_closed is not None: - oprot.writeFieldBegin('begine_closed', TType.BOOL, 2) - oprot.writeBool(self.begine_closed) - oprot.writeFieldEnd() - if self.time_end is not None: - oprot.writeFieldBegin('time_end', TType.STRUCT, 3) - self.time_end.write(oprot) - oprot.writeFieldEnd() - if self.end_closed is not None: - oprot.writeFieldBegin('end_closed', TType.BOOL, 4) - oprot.writeBool(self.end_closed) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - if self.time_begin is None: - raise TProtocolException(message='Required field time_begin is unset!') - if self.begine_closed is None: - raise TProtocolException(message='Required field begine_closed is unset!') - if self.time_end is None: - raise TProtocolException(message='Required field time_end is unset!') - if self.end_closed is None: - raise TProtocolException(message='Required field end_closed is unset!') - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - - -class VecSearchFilter(object): - """ - attrib_filter; search condition, for example: "color=red" - time_ranges; search condition, for example: "date between 1999-02-12 and 2008-10-14" - return_attribs; specify required attribute names - - Attributes: - - attrib_filter - - time_ranges - - return_attribs - """ - - - def __init__(self, attrib_filter=None, time_ranges=None, return_attribs=None,): - self.attrib_filter = attrib_filter - self.time_ranges = time_ranges - self.return_attribs = return_attribs - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.MAP: - self.attrib_filter = {} - (_ktype63, _vtype64, _size62) = iprot.readMapBegin() - for _i66 in range(_size62): - _key67 = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - _val68 = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - self.attrib_filter[_key67] = _val68 - iprot.readMapEnd() - else: - iprot.skip(ftype) - elif fid == 2: - if ftype == TType.LIST: - self.time_ranges = [] - (_etype72, _size69) = iprot.readListBegin() - for _i73 in range(_size69): - _elem74 = VecTimeRange() - _elem74.read(iprot) - self.time_ranges.append(_elem74) - iprot.readListEnd() - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.LIST: - self.return_attribs = [] - (_etype78, _size75) = iprot.readListBegin() - for _i79 in range(_size75): - _elem80 = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - self.return_attribs.append(_elem80) - iprot.readListEnd() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('VecSearchFilter') - if self.attrib_filter is not None: - oprot.writeFieldBegin('attrib_filter', TType.MAP, 1) - oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.attrib_filter)) - for kiter81, viter82 in self.attrib_filter.items(): - oprot.writeString(kiter81.encode('utf-8') if sys.version_info[0] == 2 else kiter81) - oprot.writeString(viter82.encode('utf-8') if sys.version_info[0] == 2 else viter82) - oprot.writeMapEnd() - oprot.writeFieldEnd() - if self.time_ranges is not None: - oprot.writeFieldBegin('time_ranges', TType.LIST, 2) - oprot.writeListBegin(TType.STRUCT, len(self.time_ranges)) - for iter83 in self.time_ranges: - iter83.write(oprot) - oprot.writeListEnd() - oprot.writeFieldEnd() - if self.return_attribs is not None: - oprot.writeFieldBegin('return_attribs', TType.LIST, 3) - oprot.writeListBegin(TType.STRING, len(self.return_attribs)) - for iter84 in self.return_attribs: - oprot.writeString(iter84.encode('utf-8') if sys.version_info[0] == 2 else iter84) - oprot.writeListEnd() - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(VecException) -VecException.thrift_spec = ( - None, # 0 - (1, TType.I32, 'code', None, None, ), # 1 - (2, TType.STRING, 'reason', 'UTF8', None, ), # 2 -) -all_structs.append(VecGroup) -VecGroup.thrift_spec = ( - None, # 0 - (1, TType.STRING, 'id', 'UTF8', None, ), # 1 - (2, TType.I32, 'dimension', None, None, ), # 2 - (3, TType.I32, 'index_type', None, None, ), # 3 -) -all_structs.append(VecTensor) -VecTensor.thrift_spec = ( - None, # 0 - (1, TType.STRING, 'uid', 'UTF8', None, ), # 1 - (2, TType.LIST, 'tensor', (TType.DOUBLE, None, False), None, ), # 2 - (3, TType.MAP, 'attrib', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 3 -) -all_structs.append(VecTensorList) -VecTensorList.thrift_spec = ( - None, # 0 - (1, TType.LIST, 'tensor_list', (TType.STRUCT, [VecTensor, None], False), None, ), # 1 -) -all_structs.append(VecBinaryTensor) -VecBinaryTensor.thrift_spec = ( - None, # 0 - (1, TType.STRING, 'uid', 'UTF8', None, ), # 1 - (2, TType.STRING, 'tensor', 'BINARY', None, ), # 2 - (3, TType.MAP, 'attrib', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 3 -) -all_structs.append(VecBinaryTensorList) -VecBinaryTensorList.thrift_spec = ( - None, # 0 - (1, TType.LIST, 'tensor_list', (TType.STRUCT, [VecBinaryTensor, None], False), None, ), # 1 -) -all_structs.append(VecSearchResultItem) -VecSearchResultItem.thrift_spec = ( - None, # 0 - (1, TType.STRING, 'uid', 'UTF8', None, ), # 1 - (2, TType.DOUBLE, 'distance', None, None, ), # 2 - (3, TType.MAP, 'attrib', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 3 -) -all_structs.append(VecSearchResult) -VecSearchResult.thrift_spec = ( - None, # 0 - (1, TType.LIST, 'result_list', (TType.STRUCT, [VecSearchResultItem, None], False), None, ), # 1 -) -all_structs.append(VecSearchResultList) -VecSearchResultList.thrift_spec = ( - None, # 0 - (1, TType.LIST, 'result_list', (TType.STRUCT, [VecSearchResult, None], False), None, ), # 1 -) -all_structs.append(VecDateTime) -VecDateTime.thrift_spec = ( - None, # 0 - (1, TType.I32, 'year', None, None, ), # 1 - (2, TType.I32, 'month', None, None, ), # 2 - (3, TType.I32, 'day', None, None, ), # 3 - (4, TType.I32, 'hour', None, None, ), # 4 - (5, TType.I32, 'minute', None, None, ), # 5 - (6, TType.I32, 'second', None, None, ), # 6 -) -all_structs.append(VecTimeRange) -VecTimeRange.thrift_spec = ( - None, # 0 - (1, TType.STRUCT, 'time_begin', [VecDateTime, None], None, ), # 1 - (2, TType.BOOL, 'begine_closed', None, None, ), # 2 - (3, TType.STRUCT, 'time_end', [VecDateTime, None], None, ), # 3 - (4, TType.BOOL, 'end_closed', None, None, ), # 4 -) -all_structs.append(VecSearchFilter) -VecSearchFilter.thrift_spec = ( - None, # 0 - (1, TType.MAP, 'attrib_filter', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 1 - (2, TType.LIST, 'time_ranges', (TType.STRUCT, [VecTimeRange, None], False), None, ), # 2 - (3, TType.LIST, 'return_attribs', (TType.STRING, 'UTF8', False), None, ), # 3 -) -fix_spec(all_structs) -del all_structs diff --git a/cpp/src/thrift/VectorService.thrift b/cpp/src/thrift/megasearch.thrift similarity index 87% rename from cpp/src/thrift/VectorService.thrift rename to cpp/src/thrift/megasearch.thrift index e14b58bb1a..3565f3c5df 100644 --- a/cpp/src/thrift/VectorService.thrift +++ b/cpp/src/thrift/megasearch.thrift @@ -3,16 +3,16 @@ * Unauthorized copying of this file, via any medium is strictly prohibited. * Proprietary and confidential. ******************************************************************************/ -namespace cl zilliz -namespace cpp zilliz -namespace py zilliz -namespace d zilliz -namespace dart zilliz -namespace java zilliz -namespace perl zilliz -namespace php zilliz -namespace haxe zilliz -namespace netcore zilliz +namespace cl megasearch +namespace cpp megasearch +namespace py megasearch +namespace d megasearch +namespace dart megasearch +namespace java megasearch +namespace perl megasearch +namespace php megasearch +namespace haxe megasearch +namespace netcore megasearch enum VecErrCode { SUCCESS = 0, @@ -69,9 +69,9 @@ struct VecSearchResultList { } /** - * second; Seconds. [0-60] (1 leap second) - * minute; Minutes. [0-59] - * hour; Hours. [0-23] + * second; Seconds. [0-59] reserved + * minute; Minutes. [0-59] reserved + * hour; Hours. [0-23] reserved * day; Day. [1-31] * month; Month. [0-11] * year; Year - 1900. @@ -87,9 +87,9 @@ struct VecDateTime { /** * time_begin; time range begin - * begine_closed; true means '[', false means '(' + * begine_closed; true means '[', false means '(' reserved * time_end; set to true to return tensor double array - * end_closed; time range end + * end_closed; time range end reserved */ struct VecTimeRange { 1: required VecDateTime time_begin; @@ -99,7 +99,7 @@ struct VecTimeRange { } /** - * attrib_filter; search condition, for example: "color=red" + * attrib_filter; reserved * time_ranges; search condition, for example: "date between 1999-02-12 and 2008-10-14" * return_attribs; specify required attribute names */ diff --git a/cpp/src/thrift/py_gen.sh b/cpp/src/thrift/py_gen.sh index adf2eb04b9..de661cf0c3 100755 --- a/cpp/src/thrift/py_gen.sh +++ b/cpp/src/thrift/py_gen.sh @@ -1,4 +1,4 @@ #!/bin/bash -../../third_party/build/bin/thrift -r --gen py ./VectorService.thrift +../../third_party/build/bin/thrift -r --gen py ./megasearch.thrift diff --git a/cpp/src/utils/Error.h b/cpp/src/utils/Error.h index 8684f8607f..d7904585d9 100644 --- a/cpp/src/utils/Error.h +++ b/cpp/src/utils/Error.h @@ -21,7 +21,7 @@ constexpr ServerError SERVER_ERROR_CODE_BASE = 0x30000; constexpr ServerError ToGlobalServerErrorCode(const ServerError error_code) { - return SERVER_ERROR_CODE_BASE + SERVER_ERROR_CODE_BASE; + return SERVER_ERROR_CODE_BASE + error_code; } constexpr ServerError SERVER_UNEXPECTED_ERROR = ToGlobalServerErrorCode(0x001); @@ -31,8 +31,11 @@ constexpr ServerError SERVER_INVALID_ARGUMENT = ToGlobalServerErrorCode(0x004); constexpr ServerError SERVER_FILE_NOT_FOUND = ToGlobalServerErrorCode(0x005); constexpr ServerError SERVER_NOT_IMPLEMENT = ToGlobalServerErrorCode(0x006); constexpr ServerError SERVER_BLOCKING_QUEUE_EMPTY = ToGlobalServerErrorCode(0x007); -constexpr ServerError SERVER_LICENSE_VALIDATION_FAIL = ToGlobalServerErrorCode(0x008); -constexpr ServerError SERVER_LICENSE_FILE_NOT_EXIST = ToGlobalServerErrorCode(0x009); +constexpr ServerError SERVER_GROUP_NOT_EXIST = ToGlobalServerErrorCode(0x008); +constexpr ServerError SERVER_INVALID_TIME_RANGE = ToGlobalServerErrorCode(0x009); +constexpr ServerError SERVER_INVALID_VECTOR_DIMENSION = ToGlobalServerErrorCode(0x00a); +constexpr ServerError SERVER_LICENSE_VALIDATION_FAIL = ToGlobalServerErrorCode(0x00b); +constexpr ServerError SERVER_LICENSE_FILE_NOT_EXIST = ToGlobalServerErrorCode(0x00c); class ServerException : public std::exception { public: diff --git a/cpp/test_client/CMakeLists.txt b/cpp/test_client/CMakeLists.txt index 08dd60d08b..df918d734e 100644 --- a/cpp/test_client/CMakeLists.txt +++ b/cpp/test_client/CMakeLists.txt @@ -16,8 +16,8 @@ set(util_files set(service_files ../src/thrift/gen-cpp/VecService.cpp - ../src/thrift/gen-cpp/VectorService_constants.cpp - ../src/thrift/gen-cpp/VectorService_types.cpp) + ../src/thrift/gen-cpp/megasearch_constants.cpp + ../src/thrift/gen-cpp/megasearch_types.cpp) diff --git a/cpp/test_client/src/ClientSession.cpp b/cpp/test_client/src/ClientSession.cpp index 57aa71d4b4..dac71d1ff5 100644 --- a/cpp/test_client/src/ClientSession.cpp +++ b/cpp/test_client/src/ClientSession.cpp @@ -6,8 +6,8 @@ #include "ClientSession.h" #include "Log.h" -#include "thrift/gen-cpp/VectorService_types.h" -#include "thrift/gen-cpp/VectorService_constants.h" +#include "thrift/gen-cpp/megasearch_types.h" +#include "thrift/gen-cpp/megasearch_constants.h" #include @@ -26,6 +26,8 @@ namespace zilliz { namespace vecwise { namespace client { +using namespace megasearch; + using namespace ::apache::thrift; using namespace ::apache::thrift::protocol; using namespace ::apache::thrift::transport; diff --git a/cpp/test_client/src/ClientSession.h b/cpp/test_client/src/ClientSession.h index 1cb4911aa2..44cf54adb2 100644 --- a/cpp/test_client/src/ClientSession.h +++ b/cpp/test_client/src/ClientSession.h @@ -13,7 +13,7 @@ namespace zilliz { namespace vecwise { namespace client { -using VecServiceClientPtr = std::shared_ptr; +using VecServiceClientPtr = std::shared_ptr; class ClientSession { public: diff --git a/cpp/test_client/src/ClientTest.cpp b/cpp/test_client/src/ClientTest.cpp index 007d932196..3a11e8bcc3 100644 --- a/cpp/test_client/src/ClientTest.cpp +++ b/cpp/test_client/src/ClientTest.cpp @@ -15,13 +15,14 @@ #include +using namespace megasearch; using namespace zilliz; using namespace zilliz::vecwise; using namespace zilliz::vecwise::client; namespace { static const int32_t VEC_DIMENSION = 256; - static const int64_t BATCH_COUNT = 1000; + static const int64_t BATCH_COUNT = 10000; static const int64_t REPEAT_COUNT = 1; static const int64_t TOP_K = 10; @@ -126,6 +127,28 @@ TEST(AddVector, CLIENT_TEST) { GetServerAddress(address, port, protocol); client::ClientSession session(address, port, protocol); + //verify get invalid group + try { + std::string id; + VecTensor tensor; + for(int32_t i = 0; i < VEC_DIMENSION; i++) { + tensor.tensor.push_back(0.5); + } + session.interface()->add_vector(id, GetGroupID(), tensor); + } catch (VecException& ex) { + CLIENT_LOG_ERROR << "request encounter exception: " << ex.what(); + ASSERT_EQ(ex.code, VecErrCode::ILLEGAL_ARGUMENT); + } + + try { + VecGroup temp_group; + session.interface()->get_group(temp_group, GetGroupID()); + //ASSERT_TRUE(temp_group.id.empty()); + } catch (VecException& ex) { + CLIENT_LOG_ERROR << "request encounter exception: " << ex.what(); + ASSERT_EQ(ex.code, VecErrCode::GROUP_NOT_EXISTS); + } + //add group VecGroup group; group.id = GetGroupID(); @@ -214,7 +237,6 @@ TEST(SearchVector, CLIENT_TEST) { //search vector { const int32_t anchor_index = 100; - server::TimeRecorder rc("Search top_k"); VecTensor tensor; for (int32_t i = 0; i < VEC_DIMENSION; i++) { tensor.tensor.push_back((double) (i + anchor_index)); @@ -232,35 +254,60 @@ TEST(SearchVector, CLIENT_TEST) { time_ranges.emplace_back(range); filter.__set_time_ranges(time_ranges); - //do search - session.interface()->search_vector(res, GetGroupID(), TOP_K, tensor, filter); + //normal search + { + server::TimeRecorder rc("Search top_k"); + session.interface()->search_vector(res, GetGroupID(), TOP_K, tensor, filter); + rc.Elapse("done!"); - //build result - std::cout << "Search result: " << std::endl; - for(VecSearchResultItem& item : res.result_list) { - std::cout << "\t" << item.uid << std::endl; + //build result + std::cout << "Search result: " << std::endl; + for (VecSearchResultItem &item : res.result_list) { + std::cout << "\t" << item.uid << std::endl; - ASSERT_TRUE(item.attrib.count(TEST_ATTRIB_NUM) != 0); - ASSERT_TRUE(item.attrib.count(TEST_ATTRIB_COMMENT) != 0); - ASSERT_TRUE(!item.attrib[TEST_ATTRIB_COMMENT].empty()); + ASSERT_TRUE(item.attrib.count(TEST_ATTRIB_NUM) != 0); + ASSERT_TRUE(item.attrib.count(TEST_ATTRIB_COMMENT) != 0); + ASSERT_TRUE(!item.attrib[TEST_ATTRIB_COMMENT].empty()); + } + + ASSERT_EQ(res.result_list.size(), (uint64_t) TOP_K); + if (!res.result_list.empty()) { + ASSERT_TRUE(!res.result_list[0].uid.empty()); + } } - rc.Elapse("done!"); - ASSERT_EQ(res.result_list.size(), (uint64_t)TOP_K); - if(!res.result_list.empty()) { - ASSERT_TRUE(!res.result_list[0].uid.empty()); + //filter attribute search + { + std::vector require_attributes = {TEST_ATTRIB_COMMENT}; + filter.__set_return_attribs(require_attributes); + server::TimeRecorder rc("Search top_k with attribute filter"); + session.interface()->search_vector(res, GetGroupID(), TOP_K, tensor, filter); + rc.Elapse("done!"); + + //build result + std::cout << "Search result attributes: " << std::endl; + for (VecSearchResultItem &item : res.result_list) { + ASSERT_EQ(item.attrib.size(), 1UL); + ASSERT_TRUE(item.attrib.count(TEST_ATTRIB_COMMENT) != 0); + ASSERT_TRUE(!item.attrib[TEST_ATTRIB_COMMENT].empty()); + std::cout << "\t" << item.uid << ":" << item.attrib[TEST_ATTRIB_COMMENT] << std::endl; + } + + ASSERT_EQ(res.result_list.size(), (uint64_t) TOP_K); } //empty search - date.day > 0 ? date.day -= 1 : date.day += 1; - range.time_begin = date; - range.time_end = date; - time_ranges.clear(); - time_ranges.emplace_back(range); - filter.__set_time_ranges(time_ranges); - session.interface()->search_vector(res, GetGroupID(), TOP_K, tensor, filter); + { + date.day > 0 ? date.day -= 1 : date.day += 1; + range.time_begin = date; + range.time_end = date; + time_ranges.clear(); + time_ranges.emplace_back(range); + filter.__set_time_ranges(time_ranges); + session.interface()->search_vector(res, GetGroupID(), TOP_K, tensor, filter); - ASSERT_EQ(res.result_list.size(), 0); + ASSERT_EQ(res.result_list.size(), 0); + } } //search binary vector diff --git a/cpp/unittest/faiss_wrapper/CMakeLists.txt b/cpp/unittest/faiss_wrapper/CMakeLists.txt index bdd0d797aa..eb179e6b6a 100644 --- a/cpp/unittest/faiss_wrapper/CMakeLists.txt +++ b/cpp/unittest/faiss_wrapper/CMakeLists.txt @@ -14,6 +14,7 @@ link_directories("/usr/local/cuda/lib64") set(require_files ../../src/server/VecIdMapper.cpp + ../../src/server/RocksIdMapper.cpp ../../src/server/ServerConfig.cpp ../../src/utils/CommonUtil.cpp ../../src/utils/TimeRecorder.cpp diff --git a/cpp/unittest/server/CMakeLists.txt b/cpp/unittest/server/CMakeLists.txt index 7141c45ede..5d081d72c9 100644 --- a/cpp/unittest/server/CMakeLists.txt +++ b/cpp/unittest/server/CMakeLists.txt @@ -16,6 +16,7 @@ aux_source_directory(./ test_srcs) set(require_files ../../src/server/VecIdMapper.cpp + ../../src/server/RocksIdMapper.cpp ../../src/server/ServerConfig.cpp ../../src/utils/CommonUtil.cpp ../../src/utils/TimeRecorder.cpp diff --git a/cpp/unittest/server/common_test.cpp b/cpp/unittest/server/common_test.cpp index cfc07c4196..0880a31677 100644 --- a/cpp/unittest/server/common_test.cpp +++ b/cpp/unittest/server/common_test.cpp @@ -11,12 +11,13 @@ using namespace zilliz::vecwise; TEST(CommonTest, COMMON_TEST) { - std::string path1 = "/tmp/vecwise_test/common_test_12345/"; - std::string path2 = path1 + "abcdef"; - server::ServerError err = server::CommonUtil::CreateDirectory(path2); + std::string path1 = "/tmp/vecwise_test/"; + std::string path2 = path1 + "common_test_12345/"; + std::string path3 = path2 + "abcdef"; + server::ServerError err = server::CommonUtil::CreateDirectory(path3); ASSERT_EQ(err, server::SERVER_SUCCESS); - ASSERT_TRUE(server::CommonUtil::IsDirectoryExit(path2)); + ASSERT_TRUE(server::CommonUtil::IsDirectoryExit(path3)); err = server::CommonUtil::DeleteDirectory(path1); ASSERT_EQ(err, server::SERVER_SUCCESS); diff --git a/cpp/unittest/server/idmapper_test.cpp b/cpp/unittest/server/idmapper_test.cpp index 80bbb2e402..8580d8b5f3 100644 --- a/cpp/unittest/server/idmapper_test.cpp +++ b/cpp/unittest/server/idmapper_test.cpp @@ -9,44 +9,93 @@ #include "utils/TimeRecorder.h" #include "utils/CommonUtil.h" +#include + using namespace zilliz::vecwise; +namespace { + std::string CurrentTime() { + time_t tt; + time(&tt); + tt = tt + 8 * 3600; + tm *t = gmtime(&tt); + + std::string str = std::to_string(t->tm_year + 1900) + "_" + std::to_string(t->tm_mon + 1) + + "_" + std::to_string(t->tm_mday) + "_" + std::to_string(t->tm_hour) + + "_" + std::to_string(t->tm_min) + "_" + std::to_string(t->tm_sec); + + return str; + } + + std::string GetGroupID() { + static std::string s_id(CurrentTime()); + return s_id; + } +} TEST(IdMapperTest, IDMAPPER_TEST) { server::IVecIdMapper* mapper = server::IVecIdMapper::GetInstance(); + std::string group_id = GetGroupID(); + std::vector nid = {"1", "50", "900", "10000"}; std::vector sid = {"one", "fifty", "nine zero zero", "many"}; - server::ServerError err = mapper->Put(nid, sid); + server::ServerError err = mapper->Put(nid, sid, group_id); ASSERT_EQ(err, server::SERVER_SUCCESS); - err = mapper->Put(nid, std::vector()); + err = mapper->Put(nid, std::vector(), group_id); ASSERT_NE(err, server::SERVER_SUCCESS); std::vector res; - err = mapper->Get(nid, res); + err = mapper->Get(nid, res, group_id); ASSERT_EQ(res.size(), nid.size()); for(size_t i = 0; i < res.size(); i++) { ASSERT_EQ(res[i], sid[i]); } std::string str_id; - err = mapper->Get(nid[1], str_id); + err = mapper->Get(nid[1], str_id, group_id); ASSERT_EQ(str_id, "fifty"); - err = mapper->Delete(nid[2]); + err = mapper->Get(nid[1], str_id); + ASSERT_EQ(str_id, ""); + + err = mapper->Get(nid[2], str_id, group_id); + ASSERT_EQ(str_id, "nine zero zero"); + + err = mapper->Delete(nid[2], group_id); ASSERT_EQ(err, server::SERVER_SUCCESS); - err = mapper->Get(nid[2], str_id); - ASSERT_NE(err, server::SERVER_SUCCESS); + err = mapper->Get(nid[2], str_id, group_id); + ASSERT_EQ(str_id, ""); + + err = mapper->Get(nid[3], str_id, group_id); + ASSERT_EQ(str_id, "many"); + + err = mapper->DeleteGroup(group_id); + ASSERT_EQ(err, server::SERVER_SUCCESS); + + err = mapper->Get(nid[3], str_id, group_id); + ASSERT_EQ(str_id, ""); + + std::string ct = CurrentTime(); + err = mapper->Put("current_time", ct, "time"); + ASSERT_EQ(err, server::SERVER_SUCCESS); + + err = mapper->Get("current_time", str_id, "time"); + ASSERT_EQ(str_id, ct); //test performance nid.clear(); sid.clear(); const int64_t count = 1000000; - for(int64_t i = 0; i < count; i++) { - nid.push_back(std::to_string(i+100000)); - sid.push_back("val_" + std::to_string(i)); + { + server::TimeRecorder rc("prepare id data"); + for (int64_t i = 0; i < count; i++) { + nid.push_back(std::to_string(i + 100000)); + sid.push_back("val_" + std::to_string(i)); + } + rc.Record("done!"); } {