mirror of
https://gitee.com/milvus-io/milvus.git
synced 2026-01-07 03:13:22 +08:00
Merge remote-tracking branch 'main/master' into caiyd_1547_rename_storage_file
This commit is contained in:
commit
67fad34065
@ -17,7 +17,7 @@ Please mark all change in change log and use the issue from GitHub
|
||||
- \#1022 Check if partition name is valid
|
||||
- \#1028 check if table exists when show partitions
|
||||
- \#1029 check if table exists when try to delete partition
|
||||
- \#1066 optimize http insert and search speed
|
||||
- \#1066 Optimize http insert and search speed
|
||||
- \#1022 Check if partition name is legal
|
||||
- \#1028 Check if table exists when show partitions
|
||||
- \#1029 Check if table exists when try to delete partition
|
||||
@ -108,7 +108,9 @@ Please mark all change in change log and use the issue from GitHub
|
||||
- \#1448 General proto api for NNS libraries
|
||||
- \#1480 Add return code for AVX512 selection
|
||||
- \#1524 Update config "preload_table" description
|
||||
- \#1537 Optimize raw vector and uids read/write
|
||||
- \#1544 Update resources name in HTTP module
|
||||
- \#1546 Move Config.cpp to config directory
|
||||
- \#1567 Update yaml config description
|
||||
|
||||
## Task
|
||||
|
||||
@ -153,9 +153,18 @@ $ docker run -it -p 19530:19530 -d milvusdb/milvus-cpu-build-env:latest
|
||||
|
||||
Start a GPU container:
|
||||
|
||||
- For nvidia docker 2:
|
||||
|
||||
```shell
|
||||
$ docker run --runtime=nvidia -it -p 19530:19530 -d milvusdb/milvus-gpu-build-env:latest
|
||||
```
|
||||
|
||||
- For nvidia container toolkit:
|
||||
|
||||
```shell
|
||||
docker run --gpus all -it -p 19530:19530 -d milvusdb/milvus-gpu-build-env:latest
|
||||
```
|
||||
|
||||
To enter the container:
|
||||
|
||||
```shell
|
||||
|
||||
@ -31,7 +31,7 @@ Milvus is an open source vector similarity search engine. Built with heterogeneo
|
||||
|
||||
For more detailed introduction of Milvus and its architecture, see [Milvus overview](https://www.milvus.io/docs/about_milvus/overview.md).
|
||||
|
||||
Keep up-to-date with newest releases and latest updates by reading Milvus [release notes](https://www.milvus.io/docs/v0.6.0/releases/v0.6.0.md).
|
||||
Keep up-to-date with newest releases and latest updates by reading Milvus [release notes](https://www.milvus.io/docs/master/releases/release_notes.md).
|
||||
|
||||
## Roadmap
|
||||
|
||||
@ -64,11 +64,11 @@ See our [test reports](https://github.com/milvus-io/milvus/tree/master/docs) for
|
||||
|
||||
See the [Milvus install guide](https://www.milvus.io/docs/guides/get_started/install_milvus/install_milvus.md) for using Docker containers. To install Milvus from source code, see [build from source](INSTALL.md).
|
||||
|
||||
To edit Milvus settings, read [Milvus configuration](https://www.milvus.io/docs/v0.6.0/reference/milvus_config.md).
|
||||
To edit Milvus settings, read [Milvus configuration](https://www.milvus.io/docs/reference/milvus_config.md).
|
||||
|
||||
### Try your first Milvus program
|
||||
|
||||
Try running a program with Milvus using [Python](https://www.milvus.io/docs/guides/get_started/example_code.md), [Java](https://github.com/milvus-io/milvus-sdk-java/tree/master/examples), or [C++ example code](https://github.com/milvus-io/milvus/tree/master/sdk/examples).
|
||||
Try running a program with Milvus using [Python](https://www.milvus.io/docs/guides/get_started/example_code.md), [Java](https://github.com/milvus-io/milvus-sdk-java/tree/master/examples), [Go](https://github.com/milvus-io/milvus-sdk-go/tree/master/examples), or [C++ example code](https://github.com/milvus-io/milvus/tree/master/sdk/examples).
|
||||
|
||||
|
||||
## Contribution guidelines
|
||||
|
||||
@ -31,7 +31,7 @@ Milvus 是一款开源的特征向量相似性搜索引擎。基于异构众核
|
||||
|
||||
若要了解 Milvus 详细介绍和整体架构,请访问 [Milvus 简介](https://www.milvus.io/cn/docs/about_milvus/overview.md)。
|
||||
|
||||
通过 [版本发布说明](https://www.milvus.io/cn/docs/v0.6.0/releases/v0.6.0.md) 获取最新版本的功能和更新。
|
||||
通过 [版本发布说明](https://www.milvus.io/cn/docs/master/releases/release_notes.md) 获取最新版本的功能和更新。
|
||||
|
||||
## 路线图
|
||||
|
||||
@ -100,7 +100,7 @@ Milvus 可以广泛应用于各类场景。下图展示了基于 Milvus 和 VGG
|
||||
|
||||
- [Milvus Facebook](https://www.facebook.com/io.milvus.5)
|
||||
|
||||
- [Milvus 设计文档](design.md)
|
||||
- [Milvus 设计文档](DESIGN.md)
|
||||
|
||||
## 许可协议
|
||||
|
||||
|
||||
@ -31,7 +31,7 @@ Milvusは特徴ベクトルにむかうオーペンソース高性能類似性
|
||||
|
||||
[基本紹介](https://www.milvus.io/docs/about_milvus/overview.md)を参考して詳しい情報を手に入れます。
|
||||
|
||||
Milvus [リリースノート](https://www.milvus.io/docs/v0.6.0/releases/v0.6.0.md)を読んで最新バージョンや更新情報を手に入れます。
|
||||
Milvus [リリースノート](https://www.milvus.io/docs/master/releases/release_notes.md)を読んで最新バージョンや更新情報を手に入れます。
|
||||
|
||||
# ロードマップ
|
||||
|
||||
@ -105,7 +105,7 @@ Milvusをコンフィグするために、[Milvusコンフィグ](https://www.mi
|
||||
|
||||
- [Milvus Facebook](https://www.facebook.com/io.milvus.5)
|
||||
|
||||
- [Milvus デザイン文書](design.md)
|
||||
- [Milvus デザイン文書](DESIGN.md)
|
||||
|
||||
|
||||
## ライセンス
|
||||
|
||||
2
core/src/cache/CpuCacheMgr.cpp
vendored
2
core/src/cache/CpuCacheMgr.cpp
vendored
@ -10,7 +10,7 @@
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
#include "cache/CpuCacheMgr.h"
|
||||
#include "server/Config.h"
|
||||
#include "config/Config.h"
|
||||
#include "utils/Log.h"
|
||||
|
||||
#include <fiu-local.h>
|
||||
|
||||
2
core/src/cache/GpuCacheMgr.cpp
vendored
2
core/src/cache/GpuCacheMgr.cpp
vendored
@ -10,7 +10,7 @@
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
#include "cache/GpuCacheMgr.h"
|
||||
#include "server/Config.h"
|
||||
#include "config/Config.h"
|
||||
#include "utils/Log.h"
|
||||
|
||||
#include <fiu-local.h>
|
||||
|
||||
@ -19,15 +19,88 @@
|
||||
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
#include <algorithm>
|
||||
|
||||
#include <boost/filesystem.hpp>
|
||||
|
||||
#include "utils/Exception.h"
|
||||
#include "utils/Log.h"
|
||||
#include "utils/TimeRecorder.h"
|
||||
|
||||
namespace milvus {
|
||||
namespace codec {
|
||||
|
||||
void
|
||||
DefaultVectorsFormat::read_vectors_internal(const std::string& file_path, off_t offset, size_t num,
|
||||
std::vector<uint8_t>& raw_vectors) {
|
||||
int rv_fd = open(file_path.c_str(), O_RDONLY, 00664);
|
||||
if (rv_fd == -1) {
|
||||
std::string err_msg = "Failed to open file: " + file_path + ", error: " + std::strerror(errno);
|
||||
ENGINE_LOG_ERROR << err_msg;
|
||||
throw Exception(SERVER_CANNOT_CREATE_FILE, err_msg);
|
||||
}
|
||||
|
||||
size_t num_bytes;
|
||||
if (::read(rv_fd, &num_bytes, sizeof(size_t)) == -1) {
|
||||
std::string err_msg = "Failed to read from file: " + file_path + ", error: " + std::strerror(errno);
|
||||
ENGINE_LOG_ERROR << err_msg;
|
||||
throw Exception(SERVER_WRITE_ERROR, err_msg);
|
||||
}
|
||||
|
||||
num = std::min(num, num_bytes - offset);
|
||||
|
||||
offset += sizeof(size_t); // Beginning of file is num_bytes
|
||||
int off = lseek(rv_fd, offset, SEEK_SET);
|
||||
if (off == -1) {
|
||||
std::string err_msg = "Failed to seek file: " + file_path + ", error: " + std::strerror(errno);
|
||||
ENGINE_LOG_ERROR << err_msg;
|
||||
throw Exception(SERVER_WRITE_ERROR, err_msg);
|
||||
}
|
||||
|
||||
raw_vectors.resize(num / sizeof(uint8_t));
|
||||
if (::read(rv_fd, raw_vectors.data(), num) == -1) {
|
||||
std::string err_msg = "Failed to read from file: " + file_path + ", error: " + std::strerror(errno);
|
||||
ENGINE_LOG_ERROR << err_msg;
|
||||
throw Exception(SERVER_WRITE_ERROR, err_msg);
|
||||
}
|
||||
|
||||
if (::close(rv_fd) == -1) {
|
||||
std::string err_msg = "Failed to close file: " + file_path + ", error: " + std::strerror(errno);
|
||||
ENGINE_LOG_ERROR << err_msg;
|
||||
throw Exception(SERVER_WRITE_ERROR, err_msg);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
DefaultVectorsFormat::read_uids_internal(const std::string& file_path, std::vector<segment::doc_id_t>& uids) {
|
||||
int uid_fd = open(file_path.c_str(), O_RDONLY, 00664);
|
||||
if (uid_fd == -1) {
|
||||
std::string err_msg = "Failed to open file: " + file_path + ", error: " + std::strerror(errno);
|
||||
ENGINE_LOG_ERROR << err_msg;
|
||||
throw Exception(SERVER_CANNOT_CREATE_FILE, err_msg);
|
||||
}
|
||||
|
||||
size_t num_bytes;
|
||||
if (::read(uid_fd, &num_bytes, sizeof(size_t)) == -1) {
|
||||
std::string err_msg = "Failed to read from file: " + file_path + ", error: " + std::strerror(errno);
|
||||
ENGINE_LOG_ERROR << err_msg;
|
||||
throw Exception(SERVER_WRITE_ERROR, err_msg);
|
||||
}
|
||||
|
||||
uids.resize(num_bytes / sizeof(segment::doc_id_t));
|
||||
if (::read(uid_fd, uids.data(), num_bytes) == -1) {
|
||||
std::string err_msg = "Failed to read from file: " + file_path + ", error: " + std::strerror(errno);
|
||||
ENGINE_LOG_ERROR << err_msg;
|
||||
throw Exception(SERVER_WRITE_ERROR, err_msg);
|
||||
}
|
||||
|
||||
if (::close(uid_fd) == -1) {
|
||||
std::string err_msg = "Failed to close file: " + file_path + ", error: " + std::strerror(errno);
|
||||
ENGINE_LOG_ERROR << err_msg;
|
||||
throw Exception(SERVER_WRITE_ERROR, err_msg);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
DefaultVectorsFormat::read(const store::DirectoryPtr& directory_ptr, segment::VectorsPtr& vectors_read) {
|
||||
const std::lock_guard<std::mutex> lock(mutex_);
|
||||
@ -47,68 +120,15 @@ DefaultVectorsFormat::read(const store::DirectoryPtr& directory_ptr, segment::Ve
|
||||
for (; it != it_end; ++it) {
|
||||
const auto& path = it->path();
|
||||
if (path.extension().string() == raw_vector_extension_) {
|
||||
int rv_fd = open(path.c_str(), O_RDONLY, 00664);
|
||||
if (rv_fd == -1) {
|
||||
std::string err_msg = "Failed to open file: " + path.string() + ", error: " + std::strerror(errno);
|
||||
ENGINE_LOG_ERROR << err_msg;
|
||||
throw Exception(SERVER_CANNOT_CREATE_FILE, err_msg);
|
||||
}
|
||||
|
||||
size_t num_bytes;
|
||||
if (::read(rv_fd, &num_bytes, sizeof(size_t)) == -1) {
|
||||
std::string err_msg = "Failed to read from file: " + path.string() + ", error: " + std::strerror(errno);
|
||||
ENGINE_LOG_ERROR << err_msg;
|
||||
throw Exception(SERVER_WRITE_ERROR, err_msg);
|
||||
}
|
||||
|
||||
std::vector<uint8_t> vector_list;
|
||||
vector_list.resize(num_bytes);
|
||||
if (::read(rv_fd, vector_list.data(), num_bytes) == -1) {
|
||||
std::string err_msg = "Failed to read from file: " + path.string() + ", error: " + std::strerror(errno);
|
||||
ENGINE_LOG_ERROR << err_msg;
|
||||
throw Exception(SERVER_WRITE_ERROR, err_msg);
|
||||
}
|
||||
|
||||
read_vectors_internal(path.string(), 0, INT64_MAX, vector_list);
|
||||
vectors_read->AddData(vector_list);
|
||||
vectors_read->SetName(path.stem().string());
|
||||
|
||||
if (::close(rv_fd) == -1) {
|
||||
std::string err_msg = "Failed to close file: " + path.string() + ", error: " + std::strerror(errno);
|
||||
ENGINE_LOG_ERROR << err_msg;
|
||||
throw Exception(SERVER_WRITE_ERROR, err_msg);
|
||||
}
|
||||
}
|
||||
if (path.extension().string() == user_id_extension_) {
|
||||
int uid_fd = open(path.c_str(), O_RDONLY, 00664);
|
||||
if (uid_fd == -1) {
|
||||
std::string err_msg = "Failed to open file: " + path.string() + ", error: " + std::strerror(errno);
|
||||
ENGINE_LOG_ERROR << err_msg;
|
||||
throw Exception(SERVER_CANNOT_CREATE_FILE, err_msg);
|
||||
}
|
||||
|
||||
size_t num_bytes;
|
||||
if (::read(uid_fd, &num_bytes, sizeof(size_t)) == -1) {
|
||||
std::string err_msg = "Failed to read from file: " + path.string() + ", error: " + std::strerror(errno);
|
||||
ENGINE_LOG_ERROR << err_msg;
|
||||
throw Exception(SERVER_WRITE_ERROR, err_msg);
|
||||
}
|
||||
|
||||
auto count = num_bytes / sizeof(segment::doc_id_t);
|
||||
std::vector<segment::doc_id_t> uids;
|
||||
uids.resize(count);
|
||||
if (::read(uid_fd, uids.data(), num_bytes) == -1) {
|
||||
std::string err_msg = "Failed to read from file: " + path.string() + ", error: " + std::strerror(errno);
|
||||
ENGINE_LOG_ERROR << err_msg;
|
||||
throw Exception(SERVER_WRITE_ERROR, err_msg);
|
||||
}
|
||||
|
||||
read_uids_internal(path.string(), uids);
|
||||
vectors_read->AddUids(uids);
|
||||
|
||||
if (::close(uid_fd) == -1) {
|
||||
std::string err_msg = "Failed to close file: " + path.string() + ", error: " + std::strerror(errno);
|
||||
ENGINE_LOG_ERROR << err_msg;
|
||||
throw Exception(SERVER_WRITE_ERROR, err_msg);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -122,28 +142,7 @@ DefaultVectorsFormat::write(const store::DirectoryPtr& directory_ptr, const segm
|
||||
const std::string rv_file_path = dir_path + "/" + vectors->GetName() + raw_vector_extension_;
|
||||
const std::string uid_file_path = dir_path + "/" + vectors->GetName() + user_id_extension_;
|
||||
|
||||
/*
|
||||
FILE* rv_file = fopen(rv_file_path.c_str(), "wb");
|
||||
if (rv_file == nullptr) {
|
||||
std::string err_msg = "Failed to open file: " + rv_file_path;
|
||||
ENGINE_LOG_ERROR << err_msg;
|
||||
throw Exception(SERVER_CANNOT_CREATE_FILE, err_msg);
|
||||
}
|
||||
|
||||
fwrite((void*)(it.second->GetData()), sizeof(char), it.second->GetNumBytes(), rv_file);
|
||||
fclose(rv_file);
|
||||
|
||||
|
||||
FILE* uid_file = fopen(uid_file_path.c_str(), "wb");
|
||||
if (uid_file == nullptr) {
|
||||
std::string err_msg = "Failed to open file: " + uid_file_path;
|
||||
ENGINE_LOG_ERROR << err_msg;
|
||||
throw Exception(SERVER_CANNOT_CREATE_FILE, err_msg);
|
||||
}
|
||||
|
||||
fwrite((void*)(it.second->GetUids()), sizeof it.second->GetUids()[0], it.second->GetCount(), uid_file);
|
||||
fclose(rv_file);
|
||||
*/
|
||||
TimeRecorder rc("write vectors");
|
||||
|
||||
int rv_fd = open(rv_file_path.c_str(), O_WRONLY | O_TRUNC | O_CREAT, 00664);
|
||||
if (rv_fd == -1) {
|
||||
@ -151,14 +150,6 @@ DefaultVectorsFormat::write(const store::DirectoryPtr& directory_ptr, const segm
|
||||
ENGINE_LOG_ERROR << err_msg;
|
||||
throw Exception(SERVER_CANNOT_CREATE_FILE, err_msg);
|
||||
}
|
||||
int uid_fd = open(uid_file_path.c_str(), O_WRONLY | O_TRUNC | O_CREAT, 00664);
|
||||
if (uid_fd == -1) {
|
||||
std::string err_msg = "Failed to open file: " + uid_file_path + ", error: " + std::strerror(errno);
|
||||
ENGINE_LOG_ERROR << err_msg;
|
||||
throw Exception(SERVER_CANNOT_CREATE_FILE, err_msg);
|
||||
}
|
||||
|
||||
auto start = std::chrono::high_resolution_clock::now();
|
||||
|
||||
size_t rv_num_bytes = vectors->GetData().size() * sizeof(uint8_t);
|
||||
if (::write(rv_fd, &rv_num_bytes, sizeof(size_t)) == -1) {
|
||||
@ -177,12 +168,14 @@ DefaultVectorsFormat::write(const store::DirectoryPtr& directory_ptr, const segm
|
||||
throw Exception(SERVER_WRITE_ERROR, err_msg);
|
||||
}
|
||||
|
||||
auto end = std::chrono::high_resolution_clock::now();
|
||||
std::chrono::duration<double> diff = end - start;
|
||||
ENGINE_LOG_DEBUG << "Writing raw vectors took " << diff.count() << " s";
|
||||
|
||||
start = std::chrono::high_resolution_clock::now();
|
||||
rc.RecordSection("write rv done");
|
||||
|
||||
int uid_fd = open(uid_file_path.c_str(), O_WRONLY | O_TRUNC | O_CREAT, 00664);
|
||||
if (uid_fd == -1) {
|
||||
std::string err_msg = "Failed to open file: " + uid_file_path + ", error: " + std::strerror(errno);
|
||||
ENGINE_LOG_ERROR << err_msg;
|
||||
throw Exception(SERVER_CANNOT_CREATE_FILE, err_msg);
|
||||
}
|
||||
size_t uid_num_bytes = vectors->GetUids().size() * sizeof(segment::doc_id_t);
|
||||
if (::write(uid_fd, &uid_num_bytes, sizeof(size_t)) == -1) {
|
||||
std::string err_msg = "Failed to write to file" + rv_file_path + ", error: " + std::strerror(errno);
|
||||
@ -199,9 +192,8 @@ DefaultVectorsFormat::write(const store::DirectoryPtr& directory_ptr, const segm
|
||||
ENGINE_LOG_ERROR << err_msg;
|
||||
throw Exception(SERVER_WRITE_ERROR, err_msg);
|
||||
}
|
||||
end = std::chrono::high_resolution_clock::now();
|
||||
diff = end - start;
|
||||
ENGINE_LOG_DEBUG << "Writing uids took " << diff.count() << " s";
|
||||
|
||||
rc.RecordSection("write uids done");
|
||||
}
|
||||
|
||||
void
|
||||
@ -223,30 +215,7 @@ DefaultVectorsFormat::read_uids(const store::DirectoryPtr& directory_ptr, std::v
|
||||
for (; it != it_end; ++it) {
|
||||
const auto& path = it->path();
|
||||
if (path.extension().string() == user_id_extension_) {
|
||||
int uid_fd = open(path.c_str(), O_RDONLY, 00664);
|
||||
if (uid_fd == -1) {
|
||||
std::string err_msg = "Failed to open file: " + path.string() + ", error: " + std::strerror(errno);
|
||||
ENGINE_LOG_ERROR << err_msg;
|
||||
throw Exception(SERVER_CANNOT_CREATE_FILE, err_msg);
|
||||
}
|
||||
size_t num_bytes;
|
||||
if (::read(uid_fd, &num_bytes, sizeof(size_t)) == -1) {
|
||||
std::string err_msg = "Failed to read from file: " + path.string() + ", error: " + std::strerror(errno);
|
||||
ENGINE_LOG_ERROR << err_msg;
|
||||
throw Exception(SERVER_WRITE_ERROR, err_msg);
|
||||
}
|
||||
auto count = num_bytes / sizeof(segment::doc_id_t);
|
||||
uids.resize(count);
|
||||
if (::read(uid_fd, uids.data(), num_bytes) == -1) {
|
||||
std::string err_msg = "Failed to read from file: " + path.string() + ", error: " + std::strerror(errno);
|
||||
ENGINE_LOG_ERROR << err_msg;
|
||||
throw Exception(SERVER_WRITE_ERROR, err_msg);
|
||||
}
|
||||
if (::close(uid_fd) == -1) {
|
||||
std::string err_msg = "Failed to close file: " + path.string() + ", error: " + std::strerror(errno);
|
||||
ENGINE_LOG_ERROR << err_msg;
|
||||
throw Exception(SERVER_WRITE_ERROR, err_msg);
|
||||
}
|
||||
read_uids_internal(path.string(), uids);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -271,34 +240,7 @@ DefaultVectorsFormat::read_vectors(const store::DirectoryPtr& directory_ptr, off
|
||||
for (; it != it_end; ++it) {
|
||||
const auto& path = it->path();
|
||||
if (path.extension().string() == raw_vector_extension_) {
|
||||
int rv_fd = open(path.c_str(), O_RDONLY, 00664);
|
||||
if (rv_fd == -1) {
|
||||
std::string err_msg = "Failed to open file: " + path.string() + ", error: " + std::strerror(errno);
|
||||
ENGINE_LOG_ERROR << err_msg;
|
||||
throw Exception(SERVER_CANNOT_CREATE_FILE, err_msg);
|
||||
}
|
||||
|
||||
offset += sizeof(size_t); // Beginning of file is num_bytes
|
||||
int off = lseek(rv_fd, offset, SEEK_SET);
|
||||
if (off == -1) {
|
||||
std::string err_msg = "Failed to seek file: " + path.string() + ", error: " + std::strerror(errno);
|
||||
ENGINE_LOG_ERROR << err_msg;
|
||||
throw Exception(SERVER_WRITE_ERROR, err_msg);
|
||||
}
|
||||
|
||||
raw_vectors.resize(num_bytes);
|
||||
|
||||
if (::read(rv_fd, raw_vectors.data(), num_bytes) == -1) {
|
||||
std::string err_msg = "Failed to read from file: " + path.string() + ", error: " + std::strerror(errno);
|
||||
ENGINE_LOG_ERROR << err_msg;
|
||||
throw Exception(SERVER_WRITE_ERROR, err_msg);
|
||||
}
|
||||
|
||||
if (::close(rv_fd) == -1) {
|
||||
std::string err_msg = "Failed to close file: " + path.string() + ", error: " + std::strerror(errno);
|
||||
ENGINE_LOG_ERROR << err_msg;
|
||||
throw Exception(SERVER_WRITE_ERROR, err_msg);
|
||||
}
|
||||
read_vectors_internal(path.string(), offset, num_bytes, raw_vectors);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -32,17 +32,16 @@ class DefaultVectorsFormat : public VectorsFormat {
|
||||
DefaultVectorsFormat() = default;
|
||||
|
||||
void
|
||||
read(const store::DirectoryPtr& directory_ptr, segment::VectorsPtr& vectors_read) override;
|
||||
read(const store::DirectoryPtr&, segment::VectorsPtr&) override;
|
||||
|
||||
void
|
||||
write(const store::DirectoryPtr& directory_ptr, const segment::VectorsPtr& vectors) override;
|
||||
write(const store::DirectoryPtr&, const segment::VectorsPtr&) override;
|
||||
|
||||
void
|
||||
read_uids(const store::DirectoryPtr& directory_ptr, std::vector<segment::doc_id_t>& uids) override;
|
||||
read_vectors(const store::DirectoryPtr&, off_t, size_t, std::vector<uint8_t>&) override;
|
||||
|
||||
void
|
||||
read_vectors(const store::DirectoryPtr& directory_ptr, off_t offset, size_t num_bytes,
|
||||
std::vector<uint8_t>& raw_vectors) override;
|
||||
read_uids(const store::DirectoryPtr&, std::vector<segment::doc_id_t>&) override;
|
||||
|
||||
// No copy and move
|
||||
DefaultVectorsFormat(const DefaultVectorsFormat&) = delete;
|
||||
@ -53,6 +52,13 @@ class DefaultVectorsFormat : public VectorsFormat {
|
||||
DefaultVectorsFormat&
|
||||
operator=(DefaultVectorsFormat&&) = delete;
|
||||
|
||||
private:
|
||||
void
|
||||
read_vectors_internal(const std::string&, off_t, size_t, std::vector<uint8_t>&);
|
||||
|
||||
void
|
||||
read_uids_internal(const std::string&, std::vector<segment::doc_id_t>&);
|
||||
|
||||
private:
|
||||
std::mutex mutex_;
|
||||
|
||||
|
||||
@ -25,8 +25,8 @@
|
||||
#include <unordered_set>
|
||||
#include <vector>
|
||||
|
||||
#include "config/Config.h"
|
||||
#include "config/YamlConfigMgr.h"
|
||||
#include "server/Config.h"
|
||||
#include "server/DBWrapper.h"
|
||||
#include "thirdparty/nlohmann/json.hpp"
|
||||
#include "utils/CommonUtil.h"
|
||||
@ -10,8 +10,7 @@
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
#include "config/handler/CacheConfigHandler.h"
|
||||
|
||||
#include "server/Config.h"
|
||||
#include "config/Config.h"
|
||||
|
||||
namespace milvus {
|
||||
namespace server {
|
||||
|
||||
@ -13,7 +13,8 @@
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "server/Config.h"
|
||||
#include "config/Config.h"
|
||||
#include "utils/Log.h"
|
||||
|
||||
namespace milvus {
|
||||
namespace server {
|
||||
|
||||
@ -15,8 +15,8 @@
|
||||
#include <limits>
|
||||
#include <string>
|
||||
|
||||
#include "config/Config.h"
|
||||
#include "config/handler/ConfigHandler.h"
|
||||
#include "server/Config.h"
|
||||
|
||||
namespace milvus {
|
||||
namespace server {
|
||||
|
||||
@ -9,12 +9,12 @@
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
#ifdef MILVUS_GPU_VERSION
|
||||
#include "config/handler/GpuSearchConfigHandler.h"
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "server/Config.h"
|
||||
#include "config/Config.h"
|
||||
#include "config/handler/GpuSearchConfigHandler.h"
|
||||
|
||||
namespace milvus {
|
||||
namespace server {
|
||||
|
||||
@ -18,7 +18,7 @@
|
||||
#include <regex>
|
||||
#include <vector>
|
||||
|
||||
#include "server/Config.h"
|
||||
#include "config/Config.h"
|
||||
#include "storage/s3/S3ClientWrapper.h"
|
||||
#include "utils/CommonUtil.h"
|
||||
#include "utils/Log.h"
|
||||
|
||||
@ -20,11 +20,11 @@
|
||||
|
||||
#include "cache/CpuCacheMgr.h"
|
||||
#include "cache/GpuCacheMgr.h"
|
||||
#include "config/Config.h"
|
||||
#include "db/Utils.h"
|
||||
#include "knowhere/common/Config.h"
|
||||
#include "metrics/Metrics.h"
|
||||
#include "scheduler/Utils.h"
|
||||
#include "server/Config.h"
|
||||
#include "utils/CommonUtil.h"
|
||||
#include "utils/Exception.h"
|
||||
#include "utils/Log.h"
|
||||
|
||||
@ -19,11 +19,11 @@
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "config/Config.h"
|
||||
#include "config/handler/CacheConfigHandler.h"
|
||||
#include "db/insert/MemManager.h"
|
||||
#include "db/insert/MemTable.h"
|
||||
#include "db/meta/Meta.h"
|
||||
#include "server/Config.h"
|
||||
#include "utils/Status.h"
|
||||
|
||||
namespace milvus {
|
||||
|
||||
@ -16,7 +16,7 @@
|
||||
#include <algorithm>
|
||||
#include <memory>
|
||||
|
||||
#include "server/Config.h"
|
||||
#include "config/Config.h"
|
||||
#include "utils/CommonUtil.h"
|
||||
#include "utils/Exception.h"
|
||||
#include "utils/Log.h"
|
||||
|
||||
@ -18,7 +18,7 @@
|
||||
|
||||
#include <faiss/gpu/StandardGpuResources.h>
|
||||
|
||||
#include "src/utils/BlockingQueue.h"
|
||||
#include "utils/BlockingQueue.h"
|
||||
|
||||
namespace knowhere {
|
||||
|
||||
|
||||
@ -10,7 +10,7 @@
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
#include "metrics/Metrics.h"
|
||||
#include "server/Config.h"
|
||||
#include "config/Config.h"
|
||||
#ifdef MILVUS_WITH_PROMETHEUS
|
||||
#include "metrics/prometheus/PrometheusMetrics.h"
|
||||
#endif
|
||||
|
||||
@ -11,8 +11,8 @@
|
||||
|
||||
#include "metrics/prometheus/PrometheusMetrics.h"
|
||||
#include "cache/GpuCacheMgr.h"
|
||||
#include "config/Config.h"
|
||||
#include "metrics/SystemInfo.h"
|
||||
#include "server/Config.h"
|
||||
#include "utils/Log.h"
|
||||
|
||||
#include <string>
|
||||
|
||||
@ -12,7 +12,7 @@
|
||||
#include "scheduler/SchedInst.h"
|
||||
#include "ResourceFactory.h"
|
||||
#include "Utils.h"
|
||||
#include "server/Config.h"
|
||||
#include "config/Config.h"
|
||||
|
||||
#include <fiu-local.h>
|
||||
#include <set>
|
||||
|
||||
@ -24,7 +24,6 @@
|
||||
#include "optimizer/FaissIVFSQ8Pass.h"
|
||||
#include "optimizer/FallbackPass.h"
|
||||
#include "optimizer/Optimizer.h"
|
||||
#include "server/Config.h"
|
||||
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
|
||||
@ -11,11 +11,10 @@
|
||||
|
||||
#include <list>
|
||||
#include <random>
|
||||
#include "../Algorithm.h"
|
||||
#include "Action.h"
|
||||
#include "cache/GpuCacheMgr.h"
|
||||
#include "scheduler/Algorithm.h"
|
||||
#include "scheduler/action/Action.h"
|
||||
#include "scheduler/tasklabel/SpecResLabel.h"
|
||||
#include "src/cache/GpuCacheMgr.h"
|
||||
#include "src/server/Config.h"
|
||||
|
||||
namespace milvus {
|
||||
namespace scheduler {
|
||||
|
||||
@ -11,11 +11,11 @@
|
||||
#ifdef MILVUS_GPU_VERSION
|
||||
#include "scheduler/optimizer/FaissFlatPass.h"
|
||||
#include "cache/GpuCacheMgr.h"
|
||||
#include "config/Config.h"
|
||||
#include "scheduler/SchedInst.h"
|
||||
#include "scheduler/Utils.h"
|
||||
#include "scheduler/task/SearchTask.h"
|
||||
#include "scheduler/tasklabel/SpecResLabel.h"
|
||||
#include "server/Config.h"
|
||||
#include "utils/Log.h"
|
||||
|
||||
#include <fiu-local.h>
|
||||
|
||||
@ -11,11 +11,11 @@
|
||||
#ifdef MILVUS_GPU_VERSION
|
||||
#include "scheduler/optimizer/FaissIVFFlatPass.h"
|
||||
#include "cache/GpuCacheMgr.h"
|
||||
#include "config/Config.h"
|
||||
#include "scheduler/SchedInst.h"
|
||||
#include "scheduler/Utils.h"
|
||||
#include "scheduler/task/SearchTask.h"
|
||||
#include "scheduler/tasklabel/SpecResLabel.h"
|
||||
#include "server/Config.h"
|
||||
#include "utils/Log.h"
|
||||
|
||||
namespace milvus {
|
||||
|
||||
@ -11,11 +11,11 @@
|
||||
#ifdef MILVUS_GPU_VERSION
|
||||
#include "scheduler/optimizer/FaissIVFPQPass.h"
|
||||
#include "cache/GpuCacheMgr.h"
|
||||
#include "config/Config.h"
|
||||
#include "scheduler/SchedInst.h"
|
||||
#include "scheduler/Utils.h"
|
||||
#include "scheduler/task/SearchTask.h"
|
||||
#include "scheduler/tasklabel/SpecResLabel.h"
|
||||
#include "server/Config.h"
|
||||
#include "utils/Log.h"
|
||||
|
||||
#include <fiu-local.h>
|
||||
|
||||
@ -11,11 +11,11 @@
|
||||
#ifdef MILVUS_GPU_VERSION
|
||||
#include "scheduler/optimizer/FaissIVFSQ8HPass.h"
|
||||
#include "cache/GpuCacheMgr.h"
|
||||
#include "config/Config.h"
|
||||
#include "scheduler/SchedInst.h"
|
||||
#include "scheduler/Utils.h"
|
||||
#include "scheduler/task/SearchTask.h"
|
||||
#include "scheduler/tasklabel/SpecResLabel.h"
|
||||
#include "server/Config.h"
|
||||
#include "utils/Log.h"
|
||||
|
||||
namespace milvus {
|
||||
|
||||
@ -11,11 +11,11 @@
|
||||
#ifdef MILVUS_GPU_VERSION
|
||||
#include "scheduler/optimizer/FaissIVFSQ8Pass.h"
|
||||
#include "cache/GpuCacheMgr.h"
|
||||
#include "config/Config.h"
|
||||
#include "scheduler/SchedInst.h"
|
||||
#include "scheduler/Utils.h"
|
||||
#include "scheduler/task/SearchTask.h"
|
||||
#include "scheduler/tasklabel/SpecResLabel.h"
|
||||
#include "server/Config.h"
|
||||
#include "utils/Log.h"
|
||||
|
||||
namespace milvus {
|
||||
|
||||
@ -15,8 +15,8 @@
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "config/Config.h"
|
||||
#include "db/DBFactory.h"
|
||||
#include "server/Config.h"
|
||||
#include "server/DBWrapper.h"
|
||||
#include "utils/CommonUtil.h"
|
||||
#include "utils/Log.h"
|
||||
|
||||
@ -15,9 +15,9 @@
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include "config/Config.h"
|
||||
#include "metrics/Metrics.h"
|
||||
#include "scheduler/SchedInst.h"
|
||||
#include "server/Config.h"
|
||||
#include "server/DBWrapper.h"
|
||||
#include "server/grpc_impl/GrpcServer.h"
|
||||
#include "server/web_impl/WebServer.h"
|
||||
|
||||
@ -10,6 +10,7 @@
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
#include "server/delivery/request/CmdRequest.h"
|
||||
#include "config/Config.h"
|
||||
#include "metrics/SystemInfo.h"
|
||||
#include "scheduler/SchedInst.h"
|
||||
#include "utils/Log.h"
|
||||
|
||||
@ -10,8 +10,8 @@
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
#include "server/delivery/request/CreateIndexRequest.h"
|
||||
#include "config/Config.h"
|
||||
#include "db/Utils.h"
|
||||
#include "server/Config.h"
|
||||
#include "server/DBWrapper.h"
|
||||
#include "utils/Log.h"
|
||||
#include "utils/TimeRecorder.h"
|
||||
|
||||
@ -19,7 +19,7 @@
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "server/Config.h"
|
||||
#include "config/Config.h"
|
||||
#include "server/DBWrapper.h"
|
||||
#include "utils/CommonUtil.h"
|
||||
#include "utils/Log.h"
|
||||
|
||||
@ -16,7 +16,6 @@
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include "server/Config.h"
|
||||
#include "tracing/TextMapCarrier.h"
|
||||
#include "tracing/TracerUtil.h"
|
||||
#include "utils/Log.h"
|
||||
|
||||
@ -31,8 +31,8 @@
|
||||
#include <vector>
|
||||
|
||||
#include "GrpcRequestHandler.h"
|
||||
#include "config/Config.h"
|
||||
#include "grpc/gen-milvus/milvus.grpc.pb.h"
|
||||
#include "server/Config.h"
|
||||
#include "server/DBWrapper.h"
|
||||
#include "server/grpc_impl/interceptor/SpanInterceptor.h"
|
||||
#include "utils/Log.h"
|
||||
@ -75,16 +75,9 @@ Status
|
||||
GrpcServer::StartService() {
|
||||
Config& config = Config::GetInstance();
|
||||
std::string address, port;
|
||||
Status s;
|
||||
|
||||
s = config.GetServerConfigAddress(address);
|
||||
if (!s.ok()) {
|
||||
return s;
|
||||
}
|
||||
s = config.GetServerConfigPort(port);
|
||||
if (!s.ok()) {
|
||||
return s;
|
||||
}
|
||||
CONFIG_CHECK(config.GetServerConfigAddress(address));
|
||||
CONFIG_CHECK(config.GetServerConfigPort(port));
|
||||
|
||||
std::string server_address(address + ":" + port);
|
||||
|
||||
|
||||
@ -10,14 +10,12 @@
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
#include <chrono>
|
||||
|
||||
#include <oatpp/network/server/Server.hpp>
|
||||
|
||||
#include "config/Config.h"
|
||||
#include "server/web_impl/WebServer.h"
|
||||
#include "server/web_impl/controller/WebController.hpp"
|
||||
|
||||
#include "server/Config.h"
|
||||
|
||||
namespace milvus {
|
||||
namespace server {
|
||||
namespace web {
|
||||
@ -45,9 +43,8 @@ WebServer::StartService() {
|
||||
|
||||
Config& config = Config::GetInstance();
|
||||
std::string port;
|
||||
Status status;
|
||||
|
||||
status = config.GetServerConfigWebPort(port);
|
||||
CONFIG_CHECK(config.GetServerConfigWebPort(port));
|
||||
|
||||
{
|
||||
AppComponent components = AppComponent(std::stoi(port));
|
||||
|
||||
@ -17,8 +17,8 @@
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "config/Config.h"
|
||||
#include "metrics/SystemInfo.h"
|
||||
#include "server/Config.h"
|
||||
#include "server/delivery/request/BaseRequest.h"
|
||||
#include "server/web_impl/Constants.h"
|
||||
#include "server/web_impl/Types.h"
|
||||
|
||||
@ -22,7 +22,7 @@
|
||||
#include <memory>
|
||||
#include <utility>
|
||||
|
||||
#include "server/Config.h"
|
||||
#include "config/Config.h"
|
||||
#include "storage/s3/S3ClientMock.h"
|
||||
#include "storage/s3/S3ClientWrapper.h"
|
||||
#include "utils/Error.h"
|
||||
|
||||
@ -12,7 +12,7 @@
|
||||
#include "utils/CommonUtil.h"
|
||||
#include "cache/CpuCacheMgr.h"
|
||||
#include "cache/GpuCacheMgr.h"
|
||||
#include "server/Config.h"
|
||||
#include "config/Config.h"
|
||||
#include "utils/Log.h"
|
||||
|
||||
#include <dirent.h>
|
||||
|
||||
@ -8,18 +8,18 @@
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
#include "wrapper/ConfAdapter.h"
|
||||
|
||||
#include <fiu-local.h>
|
||||
|
||||
#include <cmath>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "WrapperException.h"
|
||||
#include "config/Config.h"
|
||||
#include "knowhere/index/vector_index/helpers/IndexParameter.h"
|
||||
#include "server/Config.h"
|
||||
#include "utils/Log.h"
|
||||
|
||||
namespace milvus {
|
||||
|
||||
@ -14,9 +14,9 @@
|
||||
#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h"
|
||||
#endif
|
||||
|
||||
#include "config/Config.h"
|
||||
#include "faiss/FaissHook.h"
|
||||
#include "scheduler/Utils.h"
|
||||
#include "server/Config.h"
|
||||
#include "utils/Log.h"
|
||||
|
||||
#include <fiu-local.h>
|
||||
|
||||
@ -11,7 +11,7 @@
|
||||
|
||||
#include "wrapper/VecIndex.h"
|
||||
|
||||
#include "VecImpl.h"
|
||||
#include "config/Config.h"
|
||||
#include "knowhere/common/Exception.h"
|
||||
#include "knowhere/index/vector_index/IndexBinaryIDMAP.h"
|
||||
#include "knowhere/index/vector_index/IndexBinaryIVF.h"
|
||||
@ -22,7 +22,6 @@
|
||||
#include "knowhere/index/vector_index/IndexIVFSQ.h"
|
||||
#include "knowhere/index/vector_index/IndexNSG.h"
|
||||
#include "knowhere/index/vector_index/IndexSPTAG.h"
|
||||
#include "server/Config.h"
|
||||
#include "storage/disk/DiskIOReader.h"
|
||||
#include "storage/disk/DiskIOWriter.h"
|
||||
#include "storage/s3/S3IOReader.h"
|
||||
@ -31,6 +30,7 @@
|
||||
#include "utils/Log.h"
|
||||
#include "utils/TimeRecorder.h"
|
||||
#include "wrapper/BinVecImpl.h"
|
||||
#include "wrapper/VecImpl.h"
|
||||
|
||||
#ifdef MILVUS_GPU_VERSION
|
||||
#include <cuda.h>
|
||||
|
||||
@ -117,7 +117,7 @@ set(entry_file
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/main.cpp)
|
||||
|
||||
set(helper_files
|
||||
${MILVUS_ENGINE_SRC}/server/Config.cpp
|
||||
${MILVUS_ENGINE_SRC}/config/Config.cpp
|
||||
${MILVUS_ENGINE_SRC}/utils/CommonUtil.cpp
|
||||
${MILVUS_ENGINE_SRC}/utils/TimeRecorder.cpp
|
||||
${MILVUS_ENGINE_SRC}/utils/Status.cpp
|
||||
|
||||
@ -18,6 +18,7 @@
|
||||
#include <thread>
|
||||
|
||||
#include "cache/CpuCacheMgr.h"
|
||||
#include "config/Config.h"
|
||||
#include "db/Constants.h"
|
||||
#include "db/DB.h"
|
||||
#include "db/DBFactory.h"
|
||||
@ -25,7 +26,6 @@
|
||||
#include "db/IDGenerator.h"
|
||||
#include "db/meta/MetaConsts.h"
|
||||
#include "db/utils.h"
|
||||
#include "server/Config.h"
|
||||
#include "utils/CommonUtil.h"
|
||||
|
||||
namespace {
|
||||
|
||||
@ -21,7 +21,7 @@
|
||||
#define private public
|
||||
|
||||
#include "cache/CpuCacheMgr.h"
|
||||
#include "server/Config.h"
|
||||
#include "config/Config.h"
|
||||
#include "metrics/utils.h"
|
||||
#include "db/DB.h"
|
||||
#include "db/meta/SqliteMetaImpl.h"
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
#include "server/Config.h"
|
||||
#include "config/Config.h"
|
||||
#include "metrics/prometheus/PrometheusMetrics.h"
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
@ -17,8 +17,8 @@
|
||||
#include <gtest/gtest.h>
|
||||
#include <cmath>
|
||||
|
||||
#include "config/Config.h"
|
||||
#include "config/YamlConfigMgr.h"
|
||||
#include "server/Config.h"
|
||||
#include "server/utils.h"
|
||||
#include "utils/CommonUtil.h"
|
||||
#include "utils/StringHelpFunctions.h"
|
||||
|
||||
@ -15,6 +15,7 @@
|
||||
#include <boost/filesystem.hpp>
|
||||
#include <thread>
|
||||
|
||||
#include "config/Config.h"
|
||||
#include "server/Server.h"
|
||||
#include "server/grpc_impl/GrpcRequestHandler.h"
|
||||
#include "server/delivery/RequestScheduler.h"
|
||||
@ -26,7 +27,6 @@
|
||||
#include "grpc/gen-status/status.pb.h"
|
||||
#include "scheduler/ResourceFactory.h"
|
||||
#include "scheduler/SchedInst.h"
|
||||
#include "server/Config.h"
|
||||
#include "server/DBWrapper.h"
|
||||
#include "utils/CommonUtil.h"
|
||||
#include "server/grpc_impl/GrpcServer.h"
|
||||
|
||||
@ -21,9 +21,9 @@
|
||||
#include <oatpp/web/client/ApiClient.hpp>
|
||||
#include <oatpp/web/client/HttpRequestExecutor.hpp>
|
||||
|
||||
#include "config/Config.h"
|
||||
#include "scheduler/ResourceFactory.h"
|
||||
#include "scheduler/SchedInst.h"
|
||||
#include "server/Config.h"
|
||||
#include "server/DBWrapper.h"
|
||||
#include "server/delivery/RequestHandler.h"
|
||||
#include "server/delivery/RequestScheduler.h"
|
||||
|
||||
@ -16,8 +16,8 @@
|
||||
#include <fiu-local.h>
|
||||
#include <fiu-control.h>
|
||||
|
||||
#include "config/Config.h"
|
||||
#include "easyloggingpp/easylogging++.h"
|
||||
#include "server/Config.h"
|
||||
#include "storage/s3/S3ClientWrapper.h"
|
||||
#include "storage/s3/S3IOReader.h"
|
||||
#include "storage/s3/S3IOWriter.h"
|
||||
|
||||
@ -12,7 +12,7 @@
|
||||
#include <fstream>
|
||||
#include <string>
|
||||
|
||||
#include "server/Config.h"
|
||||
#include "config/Config.h"
|
||||
#include "storage/utils.h"
|
||||
#include "utils/CommonUtil.h"
|
||||
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
#include "server/Config.h"
|
||||
#include "config/Config.h"
|
||||
#include "wrapper/KnowhereResource.h"
|
||||
#include "wrapper/utils.h"
|
||||
|
||||
|
||||
@ -90,118 +90,118 @@ def milvus(request):
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def table(request, connect):
|
||||
ori_table_name = getattr(request.module, "table_id", "test")
|
||||
table_name = gen_unique_str(ori_table_name)
|
||||
def collection(request, connect):
|
||||
ori_collection_name = getattr(request.module, "collection_id", "test")
|
||||
collection_name = gen_unique_str(ori_collection_name)
|
||||
dim = getattr(request.module, "dim", "128")
|
||||
param = {'table_name': table_name,
|
||||
param = {'collection_name': collection_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.L2}
|
||||
status = connect.create_table(param)
|
||||
status = connect.create_collection(param)
|
||||
# logging.getLogger().info(status)
|
||||
if not status.OK():
|
||||
pytest.exit("Table can not be created, exit pytest ...")
|
||||
pytest.exit("collection can not be created, exit pytest ...")
|
||||
|
||||
def teardown():
|
||||
status, table_names = connect.show_tables()
|
||||
for table_name in table_names:
|
||||
connect.delete_table(table_name)
|
||||
status, collection_names = connect.show_collections()
|
||||
for collection_name in collection_names:
|
||||
connect.drop_collection(collection_name)
|
||||
|
||||
request.addfinalizer(teardown)
|
||||
|
||||
return table_name
|
||||
return collection_name
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def ip_table(request, connect):
|
||||
ori_table_name = getattr(request.module, "table_id", "test")
|
||||
table_name = gen_unique_str(ori_table_name)
|
||||
def ip_collection(request, connect):
|
||||
ori_collection_name = getattr(request.module, "collection_id", "test")
|
||||
collection_name = gen_unique_str(ori_collection_name)
|
||||
dim = getattr(request.module, "dim", "128")
|
||||
param = {'table_name': table_name,
|
||||
param = {'collection_name': collection_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.IP}
|
||||
status = connect.create_table(param)
|
||||
status = connect.create_collection(param)
|
||||
# logging.getLogger().info(status)
|
||||
if not status.OK():
|
||||
pytest.exit("Table can not be created, exit pytest ...")
|
||||
pytest.exit("collection can not be created, exit pytest ...")
|
||||
|
||||
def teardown():
|
||||
status, table_names = connect.show_tables()
|
||||
for table_name in table_names:
|
||||
connect.delete_table(table_name)
|
||||
status, collection_names = connect.show_collections()
|
||||
for collection_name in collection_names:
|
||||
connect.drop_collection(collection_name)
|
||||
|
||||
request.addfinalizer(teardown)
|
||||
|
||||
return table_name
|
||||
return collection_name
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def jac_table(request, connect):
|
||||
ori_table_name = getattr(request.module, "table_id", "test")
|
||||
table_name = gen_unique_str(ori_table_name)
|
||||
def jac_collection(request, connect):
|
||||
ori_collection_name = getattr(request.module, "collection_id", "test")
|
||||
collection_name = gen_unique_str(ori_collection_name)
|
||||
dim = getattr(request.module, "dim", "128")
|
||||
param = {'table_name': table_name,
|
||||
param = {'collection_name': collection_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.JACCARD}
|
||||
status = connect.create_table(param)
|
||||
status = connect.create_collection(param)
|
||||
# logging.getLogger().info(status)
|
||||
if not status.OK():
|
||||
pytest.exit("Table can not be created, exit pytest ...")
|
||||
pytest.exit("collection can not be created, exit pytest ...")
|
||||
|
||||
def teardown():
|
||||
status, table_names = connect.show_tables()
|
||||
for table_name in table_names:
|
||||
connect.delete_table(table_name)
|
||||
status, collection_names = connect.show_collections()
|
||||
for collection_name in collection_names:
|
||||
connect.drop_collection(collection_name)
|
||||
|
||||
request.addfinalizer(teardown)
|
||||
|
||||
return table_name
|
||||
return collection_name
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def ham_table(request, connect):
|
||||
ori_table_name = getattr(request.module, "table_id", "test")
|
||||
table_name = gen_unique_str(ori_table_name)
|
||||
def ham_collection(request, connect):
|
||||
ori_collection_name = getattr(request.module, "collection_id", "test")
|
||||
collection_name = gen_unique_str(ori_collection_name)
|
||||
dim = getattr(request.module, "dim", "128")
|
||||
param = {'table_name': table_name,
|
||||
param = {'collection_name': collection_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.HAMMING}
|
||||
status = connect.create_table(param)
|
||||
status = connect.create_collection(param)
|
||||
# logging.getLogger().info(status)
|
||||
if not status.OK():
|
||||
pytest.exit("Table can not be created, exit pytest ...")
|
||||
pytest.exit("collection can not be created, exit pytest ...")
|
||||
|
||||
def teardown():
|
||||
status, table_names = connect.show_tables()
|
||||
for table_name in table_names:
|
||||
connect.delete_table(table_name)
|
||||
status, collection_names = connect.show_collections()
|
||||
for collection_name in collection_names:
|
||||
connect.drop_collection(collection_name)
|
||||
|
||||
request.addfinalizer(teardown)
|
||||
|
||||
return table_name
|
||||
return collection_name
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def tanimoto_table(request, connect):
|
||||
ori_table_name = getattr(request.module, "table_id", "test")
|
||||
table_name = gen_unique_str(ori_table_name)
|
||||
def tanimoto_collection(request, connect):
|
||||
ori_collection_name = getattr(request.module, "collection_id", "test")
|
||||
collection_name = gen_unique_str(ori_collection_name)
|
||||
dim = getattr(request.module, "dim", "128")
|
||||
param = {'table_name': table_name,
|
||||
param = {'collection_name': collection_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.TANIMOTO}
|
||||
status = connect.create_table(param)
|
||||
status = connect.create_collection(param)
|
||||
# logging.getLogger().info(status)
|
||||
if not status.OK():
|
||||
pytest.exit("Table can not be created, exit pytest ...")
|
||||
pytest.exit("collection can not be created, exit pytest ...")
|
||||
|
||||
def teardown():
|
||||
status, table_names = connect.show_tables()
|
||||
for table_name in table_names:
|
||||
connect.delete_table(table_name)
|
||||
status, collection_names = connect.show_collections()
|
||||
for collection_name in collection_names:
|
||||
connect.drop_collection(collection_name)
|
||||
|
||||
request.addfinalizer(teardown)
|
||||
|
||||
return table_name
|
||||
return collection_name
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
1051
tests/milvus_python_test/test_collection.py
Normal file
1051
tests/milvus_python_test/test_collection.py
Normal file
File diff suppressed because it is too large
Load Diff
644
tests/milvus_python_test/test_collection_count.py
Normal file
644
tests/milvus_python_test/test_collection_count.py
Normal file
@ -0,0 +1,644 @@
|
||||
import pdb
|
||||
import pytest
|
||||
import logging
|
||||
import itertools
|
||||
from time import sleep
|
||||
import threading
|
||||
from multiprocessing import Process
|
||||
from milvus import IndexType, MetricType
|
||||
from utils import *
|
||||
|
||||
dim = 128
|
||||
index_file_size = 10
|
||||
add_time_interval = 3
|
||||
tag = "1970-01-01"
|
||||
nb = 6000
|
||||
|
||||
class TestCollectionCount:
|
||||
"""
|
||||
params means different nb, the nb value may trigger merge, or not
|
||||
"""
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=[
|
||||
1,
|
||||
5000,
|
||||
100000,
|
||||
],
|
||||
)
|
||||
def add_vectors_nb(self, request):
|
||||
yield request.param
|
||||
|
||||
"""
|
||||
generate valid create_index params
|
||||
"""
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=gen_simple_index()
|
||||
)
|
||||
def get_simple_index(self, request, connect):
|
||||
if str(connect._cmd("mode")[1]) == "CPU":
|
||||
if request.param["index_type"] == IndexType.IVF_SQ8H:
|
||||
pytest.skip("sq8h not support in cpu mode")
|
||||
if request.param["index_type"] == IndexType.IVF_PQ:
|
||||
pytest.skip("Skip PQ Temporary")
|
||||
return request.param
|
||||
|
||||
def test_collection_rows_count(self, connect, collection, add_vectors_nb):
|
||||
'''
|
||||
target: test collection rows_count is correct or not
|
||||
method: create collection and add vectors in it,
|
||||
assert the value returned by count_collection method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nb = add_vectors_nb
|
||||
vectors = gen_vectors(nb, dim)
|
||||
res = connect.add_vectors(collection_name=collection, records=vectors)
|
||||
connect.flush([collection])
|
||||
status, res = connect.count_collection(collection)
|
||||
assert res == nb
|
||||
|
||||
def test_collection_rows_count_partition(self, connect, collection, add_vectors_nb):
|
||||
'''
|
||||
target: test collection rows_count is correct or not
|
||||
method: create collection, create partition and add vectors in it,
|
||||
assert the value returned by count_collection method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nb = add_vectors_nb
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status = connect.create_partition(collection, tag)
|
||||
assert status.OK()
|
||||
res = connect.add_vectors(collection_name=collection, records=vectors, partition_tag=tag)
|
||||
connect.flush([collection])
|
||||
status, res = connect.count_collection(collection)
|
||||
assert res == nb
|
||||
|
||||
def test_collection_rows_count_multi_partitions_A(self, connect, collection, add_vectors_nb):
|
||||
'''
|
||||
target: test collection rows_count is correct or not
|
||||
method: create collection, create partitions and add vectors in it,
|
||||
assert the value returned by count_collection method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
new_tag = "new_tag"
|
||||
nb = add_vectors_nb
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status = connect.create_partition(collection, tag)
|
||||
status = connect.create_partition(collection, new_tag)
|
||||
assert status.OK()
|
||||
res = connect.add_vectors(collection_name=collection, records=vectors)
|
||||
connect.flush([collection])
|
||||
status, res = connect.count_collection(collection)
|
||||
assert res == nb
|
||||
|
||||
def test_collection_rows_count_multi_partitions_B(self, connect, collection, add_vectors_nb):
|
||||
'''
|
||||
target: test collection rows_count is correct or not
|
||||
method: create collection, create partitions and add vectors in one of the partitions,
|
||||
assert the value returned by count_collection method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
new_tag = "new_tag"
|
||||
nb = add_vectors_nb
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status = connect.create_partition(collection, tag)
|
||||
status = connect.create_partition(collection, new_tag)
|
||||
assert status.OK()
|
||||
res = connect.add_vectors(collection_name=collection, records=vectors, partition_tag=tag)
|
||||
connect.flush([collection])
|
||||
status, res = connect.count_collection(collection)
|
||||
assert res == nb
|
||||
|
||||
def test_collection_rows_count_multi_partitions_C(self, connect, collection, add_vectors_nb):
|
||||
'''
|
||||
target: test collection rows_count is correct or not
|
||||
method: create collection, create partitions and add vectors in one of the partitions,
|
||||
assert the value returned by count_collection method is equal to length of vectors
|
||||
expected: the collection count is equal to the length of vectors
|
||||
'''
|
||||
new_tag = "new_tag"
|
||||
nb = add_vectors_nb
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status = connect.create_partition(collection, tag)
|
||||
status = connect.create_partition(collection, new_tag)
|
||||
assert status.OK()
|
||||
res = connect.add_vectors(collection_name=collection, records=vectors, partition_tag=tag)
|
||||
res = connect.add_vectors(collection_name=collection, records=vectors, partition_tag=new_tag)
|
||||
connect.flush([collection])
|
||||
status, res = connect.count_collection(collection)
|
||||
assert res == nb * 2
|
||||
|
||||
def test_collection_rows_count_after_index_created(self, connect, collection, get_simple_index):
|
||||
'''
|
||||
target: test count_collection, after index have been created
|
||||
method: add vectors in db, and create index, then calling count_collection with correct params
|
||||
expected: count_collection raise exception
|
||||
'''
|
||||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
nb = 100
|
||||
vectors = gen_vectors(nb, dim)
|
||||
res = connect.add_vectors(collection_name=collection, records=vectors)
|
||||
connect.flush([collection])
|
||||
connect.create_index(collection, index_type, index_param)
|
||||
status, res = connect.count_collection(collection)
|
||||
assert res == nb
|
||||
|
||||
@pytest.mark.level(2)
|
||||
def test_count_without_connection(self, collection, dis_connect):
|
||||
'''
|
||||
target: test count_collection, without connection
|
||||
method: calling count_collection with correct params, with a disconnected instance
|
||||
expected: count_collection raise exception
|
||||
'''
|
||||
with pytest.raises(Exception) as e:
|
||||
status = dis_connect.count_collection(collection)
|
||||
|
||||
def test_collection_rows_count_no_vectors(self, connect, collection):
|
||||
'''
|
||||
target: test collection rows_count is correct or not, if collection is empty
|
||||
method: create collection and no vectors in it,
|
||||
assert the value returned by count_collection method is equal to 0
|
||||
expected: the count is equal to 0
|
||||
'''
|
||||
collection_name = gen_unique_str()
|
||||
param = {'collection_name': collection_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size}
|
||||
connect.create_collection(param)
|
||||
status, res = connect.count_collection(collection)
|
||||
assert res == 0
|
||||
|
||||
# TODO: enable
|
||||
@pytest.mark.level(2)
|
||||
@pytest.mark.timeout(20)
|
||||
def _test_collection_rows_count_multiprocessing(self, connect, collection, args):
|
||||
'''
|
||||
target: test collection rows_count is correct or not with multiprocess
|
||||
method: create collection and add vectors in it,
|
||||
assert the value returned by count_collection method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nq = 2
|
||||
uri = "tcp://%s:%s" % (args["ip"], args["port"])
|
||||
vectors = gen_vectors(nq, dim)
|
||||
res = connect.add_vectors(collection_name=collection, records=vectors)
|
||||
time.sleep(add_time_interval)
|
||||
|
||||
def rows_count(milvus):
|
||||
status, res = milvus.count_collection(collection)
|
||||
logging.getLogger().info(status)
|
||||
assert res == nq
|
||||
|
||||
process_num = 8
|
||||
processes = []
|
||||
for i in range(process_num):
|
||||
milvus = get_milvus(args["handler"])
|
||||
milvus.connect(uri=uri)
|
||||
p = Process(target=rows_count, args=(milvus, ))
|
||||
processes.append(p)
|
||||
p.start()
|
||||
logging.getLogger().info(p)
|
||||
for p in processes:
|
||||
p.join()
|
||||
|
||||
def test_collection_rows_count_multi_collections(self, connect):
|
||||
'''
|
||||
target: test collection rows_count is correct or not with multiple collections of L2
|
||||
method: create collection and add vectors in it,
|
||||
assert the value returned by count_collection method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nq = 100
|
||||
vectors = gen_vectors(nq, dim)
|
||||
collection_list = []
|
||||
for i in range(20):
|
||||
collection_name = gen_unique_str()
|
||||
collection_list.append(collection_name)
|
||||
param = {'collection_name': collection_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.L2}
|
||||
connect.create_collection(param)
|
||||
res = connect.add_vectors(collection_name=collection_name, records=vectors)
|
||||
connect.flush(collection_list)
|
||||
for i in range(20):
|
||||
status, res = connect.count_collection(collection_list[i])
|
||||
assert status.OK()
|
||||
assert res == nq
|
||||
|
||||
|
||||
class TestCollectionCountIP:
|
||||
"""
|
||||
params means different nb, the nb value may trigger merge, or not
|
||||
"""
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=[
|
||||
1,
|
||||
5000,
|
||||
100000,
|
||||
],
|
||||
)
|
||||
def add_vectors_nb(self, request):
|
||||
yield request.param
|
||||
|
||||
"""
|
||||
generate valid create_index params
|
||||
"""
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=gen_simple_index()
|
||||
)
|
||||
def get_simple_index(self, request, connect):
|
||||
if str(connect._cmd("mode")[1]) == "CPU":
|
||||
if request.param["index_type"] == IndexType.IVF_SQ8H:
|
||||
pytest.skip("sq8h not support in CPU mode")
|
||||
if request.param["index_type"] == IndexType.IVF_PQ:
|
||||
pytest.skip("Skip PQ Temporary")
|
||||
return request.param
|
||||
|
||||
def test_collection_rows_count(self, connect, ip_collection, add_vectors_nb):
|
||||
'''
|
||||
target: test collection rows_count is correct or not
|
||||
method: create collection and add vectors in it,
|
||||
assert the value returned by count_collection method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nb = add_vectors_nb
|
||||
vectors = gen_vectors(nb, dim)
|
||||
res = connect.add_vectors(collection_name=ip_collection, records=vectors)
|
||||
connect.flush([ip_collection])
|
||||
status, res = connect.count_collection(ip_collection)
|
||||
assert res == nb
|
||||
|
||||
def test_collection_rows_count_after_index_created(self, connect, ip_collection, get_simple_index):
|
||||
'''
|
||||
target: test count_collection, after index have been created
|
||||
method: add vectors in db, and create index, then calling count_collection with correct params
|
||||
expected: count_collection raise exception
|
||||
'''
|
||||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
nb = 100
|
||||
vectors = gen_vectors(nb, dim)
|
||||
res = connect.add_vectors(collection_name=ip_collection, records=vectors)
|
||||
connect.flush([ip_collection])
|
||||
connect.create_index(ip_collection, index_type, index_param)
|
||||
status, res = connect.count_collection(ip_collection)
|
||||
assert res == nb
|
||||
|
||||
@pytest.mark.level(2)
|
||||
def test_count_without_connection(self, ip_collection, dis_connect):
|
||||
'''
|
||||
target: test count_collection, without connection
|
||||
method: calling count_collection with correct params, with a disconnected instance
|
||||
expected: count_collection raise exception
|
||||
'''
|
||||
with pytest.raises(Exception) as e:
|
||||
status = dis_connect.count_collection(ip_collection)
|
||||
|
||||
def test_collection_rows_count_no_vectors(self, connect, ip_collection):
|
||||
'''
|
||||
target: test collection rows_count is correct or not, if collection is empty
|
||||
method: create collection and no vectors in it,
|
||||
assert the value returned by count_collection method is equal to 0
|
||||
expected: the count is equal to 0
|
||||
'''
|
||||
collection_name = gen_unique_str("test_collection")
|
||||
param = {'collection_name': collection_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size}
|
||||
connect.create_collection(param)
|
||||
status, res = connect.count_collection(ip_collection)
|
||||
assert res == 0
|
||||
|
||||
# TODO: enable
|
||||
@pytest.mark.timeout(60)
|
||||
def _test_collection_rows_count_multiprocessing(self, connect, ip_collection, args):
|
||||
'''
|
||||
target: test collection rows_count is correct or not with multiprocess
|
||||
method: create collection and add vectors in it,
|
||||
assert the value returned by count_collection method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nq = 2
|
||||
uri = "tcp://%s:%s" % (args["ip"], args["port"])
|
||||
vectors = gen_vectors(nq, dim)
|
||||
res = connect.add_vectors(collection_name=ip_collection, records=vectors)
|
||||
time.sleep(add_time_interval)
|
||||
|
||||
def rows_count(milvus):
|
||||
status, res = milvus.count_collection(ip_collection)
|
||||
logging.getLogger().info(status)
|
||||
assert res == nq
|
||||
|
||||
process_num = 8
|
||||
processes = []
|
||||
for i in range(process_num):
|
||||
milvus = get_milvus(args["handler"])
|
||||
milvus.connect(uri=uri)
|
||||
p = Process(target=rows_count, args=(milvus,))
|
||||
processes.append(p)
|
||||
p.start()
|
||||
logging.getLogger().info(p)
|
||||
for p in processes:
|
||||
p.join()
|
||||
|
||||
def test_collection_rows_count_multi_collections(self, connect):
|
||||
'''
|
||||
target: test collection rows_count is correct or not with multiple collections of IP
|
||||
method: create collection and add vectors in it,
|
||||
assert the value returned by count_collection method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nq = 100
|
||||
vectors = gen_vectors(nq, dim)
|
||||
collection_list = []
|
||||
for i in range(20):
|
||||
collection_name = gen_unique_str('test_collection_rows_count_multi_collections')
|
||||
collection_list.append(collection_name)
|
||||
param = {'collection_name': collection_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.IP}
|
||||
connect.create_collection(param)
|
||||
res = connect.add_vectors(collection_name=collection_name, records=vectors)
|
||||
connect.flush(collection_list)
|
||||
for i in range(20):
|
||||
status, res = connect.count_collection(collection_list[i])
|
||||
assert status.OK()
|
||||
assert res == nq
|
||||
|
||||
|
||||
class TestCollectionCountJAC:
|
||||
"""
|
||||
params means different nb, the nb value may trigger merge, or not
|
||||
"""
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=[
|
||||
1,
|
||||
5000,
|
||||
100000,
|
||||
],
|
||||
)
|
||||
def add_vectors_nb(self, request):
|
||||
yield request.param
|
||||
|
||||
"""
|
||||
generate valid create_index params
|
||||
"""
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=gen_simple_index()
|
||||
)
|
||||
def get_jaccard_index(self, request, connect):
|
||||
logging.getLogger().info(request.param)
|
||||
if request.param["index_type"] == IndexType.IVFLAT or request.param["index_type"] == IndexType.FLAT:
|
||||
return request.param
|
||||
else:
|
||||
pytest.skip("Skip index Temporary")
|
||||
|
||||
def test_collection_rows_count(self, connect, jac_collection, add_vectors_nb):
|
||||
'''
|
||||
target: test collection rows_count is correct or not
|
||||
method: create collection and add vectors in it,
|
||||
assert the value returned by count_collection method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nb = add_vectors_nb
|
||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||
res = connect.add_vectors(collection_name=jac_collection, records=vectors)
|
||||
connect.flush([jac_collection])
|
||||
status, res = connect.count_collection(jac_collection)
|
||||
assert res == nb
|
||||
|
||||
def test_collection_rows_count_after_index_created(self, connect, jac_collection, get_jaccard_index):
|
||||
'''
|
||||
target: test count_collection, after index have been created
|
||||
method: add vectors in db, and create index, then calling count_collection with correct params
|
||||
expected: count_collection raise exception
|
||||
'''
|
||||
nb = 100
|
||||
index_param = get_jaccard_index["index_param"]
|
||||
index_type = get_jaccard_index["index_type"]
|
||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||
res = connect.add_vectors(collection_name=jac_collection, records=vectors)
|
||||
connect.flush([jac_collection])
|
||||
connect.create_index(jac_collection, index_type, index_param)
|
||||
status, res = connect.count_collection(jac_collection)
|
||||
assert res == nb
|
||||
|
||||
@pytest.mark.level(2)
|
||||
def test_count_without_connection(self, jac_collection, dis_connect):
|
||||
'''
|
||||
target: test count_collection, without connection
|
||||
method: calling count_collection with correct params, with a disconnected instance
|
||||
expected: count_collection raise exception
|
||||
'''
|
||||
with pytest.raises(Exception) as e:
|
||||
status = dis_connect.count_collection(jac_collection)
|
||||
|
||||
def test_collection_rows_count_no_vectors(self, connect, jac_collection):
|
||||
'''
|
||||
target: test collection rows_count is correct or not, if collection is empty
|
||||
method: create collection and no vectors in it,
|
||||
assert the value returned by count_collection method is equal to 0
|
||||
expected: the count is equal to 0
|
||||
'''
|
||||
collection_name = gen_unique_str("test_collection")
|
||||
param = {'collection_name': collection_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size}
|
||||
connect.create_collection(param)
|
||||
status, res = connect.count_collection(jac_collection)
|
||||
assert res == 0
|
||||
|
||||
def test_collection_rows_count_multi_collections(self, connect):
|
||||
'''
|
||||
target: test collection rows_count is correct or not with multiple collections of IP
|
||||
method: create collection and add vectors in it,
|
||||
assert the value returned by count_collection method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nq = 100
|
||||
tmp, vectors = gen_binary_vectors(nq, dim)
|
||||
collection_list = []
|
||||
for i in range(20):
|
||||
collection_name = gen_unique_str('test_collection_rows_count_multi_collections')
|
||||
collection_list.append(collection_name)
|
||||
param = {'collection_name': collection_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.JACCARD}
|
||||
connect.create_collection(param)
|
||||
res = connect.add_vectors(collection_name=collection_name, records=vectors)
|
||||
connect.flush(collection_list)
|
||||
for i in range(20):
|
||||
status, res = connect.count_collection(collection_list[i])
|
||||
assert status.OK()
|
||||
assert res == nq
|
||||
|
||||
class TestCollectionCountHAM:
|
||||
"""
|
||||
params means different nb, the nb value may trigger merge, or not
|
||||
"""
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=[
|
||||
1,
|
||||
5000,
|
||||
100000,
|
||||
],
|
||||
)
|
||||
def add_vectors_nb(self, request):
|
||||
yield request.param
|
||||
|
||||
"""
|
||||
generate valid create_index params
|
||||
"""
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=gen_simple_index()
|
||||
)
|
||||
def get_hamming_index(self, request, connect):
|
||||
logging.getLogger().info(request.param)
|
||||
if request.param["index_type"] == IndexType.IVFLAT or request.param["index_type"] == IndexType.FLAT:
|
||||
return request.param
|
||||
else:
|
||||
pytest.skip("Skip index Temporary")
|
||||
|
||||
def test_collection_rows_count(self, connect, ham_collection, add_vectors_nb):
|
||||
'''
|
||||
target: test collection rows_count is correct or not
|
||||
method: create collection and add vectors in it,
|
||||
assert the value returned by count_collection method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nb = add_vectors_nb
|
||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||
res = connect.add_vectors(collection_name=ham_collection, records=vectors)
|
||||
connect.flush([ham_collection])
|
||||
status, res = connect.count_collection(ham_collection)
|
||||
assert res == nb
|
||||
|
||||
def test_collection_rows_count_after_index_created(self, connect, ham_collection, get_hamming_index):
|
||||
'''
|
||||
target: test count_collection, after index have been created
|
||||
method: add vectors in db, and create index, then calling count_collection with correct params
|
||||
expected: count_collection raise exception
|
||||
'''
|
||||
nb = 100
|
||||
index_type = get_hamming_index["index_type"]
|
||||
index_param = get_hamming_index["index_param"]
|
||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||
res = connect.add_vectors(collection_name=ham_collection, records=vectors)
|
||||
connect.flush([ham_collection])
|
||||
connect.create_index(ham_collection, index_type, index_param)
|
||||
status, res = connect.count_collection(ham_collection)
|
||||
assert res == nb
|
||||
|
||||
@pytest.mark.level(2)
|
||||
def test_count_without_connection(self, ham_collection, dis_connect):
|
||||
'''
|
||||
target: test count_collection, without connection
|
||||
method: calling count_collection with correct params, with a disconnected instance
|
||||
expected: count_collection raise exception
|
||||
'''
|
||||
with pytest.raises(Exception) as e:
|
||||
status = dis_connect.count_collection(ham_collection)
|
||||
|
||||
def test_collection_rows_count_no_vectors(self, connect, ham_collection):
|
||||
'''
|
||||
target: test collection rows_count is correct or not, if collection is empty
|
||||
method: create collection and no vectors in it,
|
||||
assert the value returned by count_collection method is equal to 0
|
||||
expected: the count is equal to 0
|
||||
'''
|
||||
collection_name = gen_unique_str("test_collection")
|
||||
param = {'collection_name': collection_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size}
|
||||
connect.create_collection(param)
|
||||
status, res = connect.count_collection(ham_collection)
|
||||
assert res == 0
|
||||
|
||||
def test_collection_rows_count_multi_collections(self, connect):
|
||||
'''
|
||||
target: test collection rows_count is correct or not with multiple collections of IP
|
||||
method: create collection and add vectors in it,
|
||||
assert the value returned by count_collection method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nq = 100
|
||||
tmp, vectors = gen_binary_vectors(nq, dim)
|
||||
collection_list = []
|
||||
for i in range(20):
|
||||
collection_name = gen_unique_str('test_collection_rows_count_multi_collections')
|
||||
collection_list.append(collection_name)
|
||||
param = {'collection_name': collection_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.HAMMING}
|
||||
connect.create_collection(param)
|
||||
res = connect.add_vectors(collection_name=collection_name, records=vectors)
|
||||
connect.flush(collection_list)
|
||||
for i in range(20):
|
||||
status, res = connect.count_collection(collection_list[i])
|
||||
assert status.OK()
|
||||
assert res == nq
|
||||
|
||||
|
||||
class TestCollectionCountTANIMOTO:
|
||||
"""
|
||||
params means different nb, the nb value may trigger merge, or not
|
||||
"""
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=[
|
||||
1,
|
||||
5000,
|
||||
100000,
|
||||
],
|
||||
)
|
||||
def add_vectors_nb(self, request):
|
||||
yield request.param
|
||||
|
||||
"""
|
||||
generate valid create_index params
|
||||
"""
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=gen_simple_index()
|
||||
)
|
||||
def get_tanimoto_index(self, request, connect):
|
||||
logging.getLogger().info(request.param)
|
||||
if request.param["index_type"] == IndexType.IVFLAT or request.param["index_type"] == IndexType.FLAT:
|
||||
return request.param
|
||||
else:
|
||||
pytest.skip("Skip index Temporary")
|
||||
|
||||
def test_collection_rows_count(self, connect, tanimoto_collection, add_vectors_nb):
|
||||
'''
|
||||
target: test collection rows_count is correct or not
|
||||
method: create collection and add vectors in it,
|
||||
assert the value returned by count_collection method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nb = add_vectors_nb
|
||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||
res = connect.add_vectors(collection_name=tanimoto_collection, records=vectors)
|
||||
connect.flush([tanimoto_collection])
|
||||
status, res = connect.count_collection(tanimoto_collection)
|
||||
assert status.OK()
|
||||
assert res == nb
|
||||
@ -18,7 +18,7 @@ nb = 6000
|
||||
nlist = 1024
|
||||
|
||||
|
||||
class TestTableInfoBase:
|
||||
class TestCollectionInfoBase:
|
||||
def index_string_convert(self, index_string, index_type):
|
||||
if index_string == "IDMAP" and index_type == IndexType.FLAT:
|
||||
return True
|
||||
@ -30,87 +30,87 @@ class TestTableInfoBase:
|
||||
|
||||
"""
|
||||
******************************************************************
|
||||
The following cases are used to test `table_info` function
|
||||
The following cases are used to test `collection_info` function
|
||||
******************************************************************
|
||||
"""
|
||||
@pytest.mark.timeout(INFO_TIMEOUT)
|
||||
def test_get_table_info_name_None(self, connect, table):
|
||||
def test_get_collection_info_name_None(self, connect, collection):
|
||||
'''
|
||||
target: get table info where table name is None
|
||||
method: call table_info with the table_name: None
|
||||
target: get collection info where collection name is None
|
||||
method: call collection_info with the collection_name: None
|
||||
expected: status not ok
|
||||
'''
|
||||
table_name = None
|
||||
status, info = connect.table_info(table_name)
|
||||
collection_name = None
|
||||
status, info = connect.collection_info(collection_name)
|
||||
assert not status.OK()
|
||||
|
||||
@pytest.mark.timeout(INFO_TIMEOUT)
|
||||
def test_get_table_info_name_not_existed(self, connect, table):
|
||||
def test_get_collection_info_name_not_existed(self, connect, collection):
|
||||
'''
|
||||
target: get table info where table name does not exist
|
||||
method: call table_info with a random table_name, which is not in db
|
||||
target: get collection info where collection name does not exist
|
||||
method: call collection_info with a random collection_name, which is not in db
|
||||
expected: status not ok
|
||||
'''
|
||||
table_name = gen_unique_str("not_existed_table")
|
||||
status, info = connect.table_info(table_name)
|
||||
collection_name = gen_unique_str("not_existed_collection")
|
||||
status, info = connect.collection_info(collection_name)
|
||||
assert not status.OK()
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=gen_invalid_table_names()
|
||||
params=gen_invalid_collection_names()
|
||||
)
|
||||
def get_table_name(self, request):
|
||||
def get_collection_name(self, request):
|
||||
yield request.param
|
||||
|
||||
@pytest.mark.timeout(INFO_TIMEOUT)
|
||||
def test_get_table_info_name_invalid(self, connect, get_table_name):
|
||||
def test_get_collection_info_name_invalid(self, connect, get_collection_name):
|
||||
'''
|
||||
target: get table info where table name is invalid
|
||||
method: call table_info with invalid table_name
|
||||
target: get collection info where collection name is invalid
|
||||
method: call collection_info with invalid collection_name
|
||||
expected: status not ok
|
||||
'''
|
||||
table_name = get_table_name
|
||||
status, info = connect.table_info(table_name)
|
||||
collection_name = get_collection_name
|
||||
status, info = connect.collection_info(collection_name)
|
||||
assert not status.OK()
|
||||
|
||||
@pytest.mark.timeout(INFO_TIMEOUT)
|
||||
def test_get_table_info_table_row_count(self, connect, table):
|
||||
def test_get_collection_info_collection_row_count(self, connect, collection):
|
||||
'''
|
||||
target: get row count with table_info
|
||||
method: add and delete vectors, check count in table info
|
||||
target: get row count with collection_info
|
||||
method: add and delete vectors, check count in collection info
|
||||
expected: status ok, count as expected
|
||||
'''
|
||||
vectors = gen_vector(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(table)
|
||||
status, info = connect.collection_info(collection)
|
||||
assert status.OK()
|
||||
assert info.count == nb
|
||||
# delete a few vectors
|
||||
delete_ids = [ids[0], ids[-1]]
|
||||
status = connect.delete_by_id(table, delete_ids)
|
||||
status = connect.delete_by_id(collection, delete_ids)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(table)
|
||||
status, info = connect.collection_info(collection)
|
||||
assert status.OK()
|
||||
assert info.count == nb - 2
|
||||
|
||||
@pytest.mark.timeout(INFO_TIMEOUT)
|
||||
def test_get_table_info_partition_stats_A(self, connect, table):
|
||||
def test_get_collection_info_partition_stats_A(self, connect, collection):
|
||||
'''
|
||||
target: get partition info in a table
|
||||
method: no partition, call table_info and check partition_stats
|
||||
target: get partition info in a collection
|
||||
method: no partition, call collection_info and check partition_stats
|
||||
expected: status ok, "_default" partition is listed
|
||||
'''
|
||||
vectors = gen_vector(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(table)
|
||||
status, info = connect.collection_info(collection)
|
||||
assert status.OK()
|
||||
logging.getLogger().info(info)
|
||||
assert len(info.partitions_stat) == 1
|
||||
@ -119,19 +119,19 @@ class TestTableInfoBase:
|
||||
|
||||
|
||||
@pytest.mark.timeout(INFO_TIMEOUT)
|
||||
def test_get_table_info_partition_stats_B(self, connect, table):
|
||||
def test_get_collection_info_partition_stats_B(self, connect, collection):
|
||||
'''
|
||||
target: get partition info in a table
|
||||
method: call table_info after partition created and check partition_stats
|
||||
target: get partition info in a collection
|
||||
method: call collection_info after partition created and check partition_stats
|
||||
expected: status ok, vectors added to partition
|
||||
'''
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status = connect.create_partition(table, tag)
|
||||
status, ids = connect.add_vectors(table, vectors, partition_tag=tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
status, ids = connect.add_vectors(collection, vectors, partition_tag=tag)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(table)
|
||||
status, info = connect.collection_info(collection)
|
||||
assert status.OK()
|
||||
logging.getLogger().info(info)
|
||||
assert len(info.partitions_stat) == 2
|
||||
@ -139,23 +139,23 @@ class TestTableInfoBase:
|
||||
assert info.partitions_stat[1].count == nb
|
||||
|
||||
@pytest.mark.timeout(INFO_TIMEOUT)
|
||||
def test_get_table_info_partition_stats_C(self, connect, table):
|
||||
def test_get_collection_info_partition_stats_C(self, connect, collection):
|
||||
'''
|
||||
target: get partition info in a table
|
||||
method: create two partitions, add vectors in one of the partitions, call table_info and check
|
||||
target: get partition info in a collection
|
||||
method: create two partitions, add vectors in one of the partitions, call collection_info and check
|
||||
expected: status ok, vectors added to one partition but not the other
|
||||
'''
|
||||
new_tag = "new_tag"
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
assert status.OK()
|
||||
status = connect.create_partition(table, new_tag)
|
||||
status = connect.create_partition(collection, new_tag)
|
||||
assert status.OK()
|
||||
status, ids = connect.add_vectors(table, vectors, partition_tag=tag)
|
||||
status, ids = connect.add_vectors(collection, vectors, partition_tag=tag)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(table)
|
||||
status, info = connect.collection_info(collection)
|
||||
assert status.OK()
|
||||
logging.getLogger().info(info)
|
||||
for partition in info.partitions_stat:
|
||||
@ -165,25 +165,25 @@ class TestTableInfoBase:
|
||||
assert partition.count == 0
|
||||
|
||||
@pytest.mark.timeout(INFO_TIMEOUT)
|
||||
def test_get_table_info_partition_stats_D(self, connect, table):
|
||||
def test_get_collection_info_partition_stats_D(self, connect, collection):
|
||||
'''
|
||||
target: get partition info in a table
|
||||
method: create two partitions, add vectors in both partitions, call table_info and check
|
||||
target: get partition info in a collection
|
||||
method: create two partitions, add vectors in both partitions, call collection_info and check
|
||||
expected: status ok, vectors added to both partitions
|
||||
'''
|
||||
new_tag = "new_tag"
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
assert status.OK()
|
||||
status = connect.create_partition(table, new_tag)
|
||||
status = connect.create_partition(collection, new_tag)
|
||||
assert status.OK()
|
||||
status, ids = connect.add_vectors(table, vectors, partition_tag=tag)
|
||||
status, ids = connect.add_vectors(collection, vectors, partition_tag=tag)
|
||||
assert status.OK()
|
||||
status, ids = connect.add_vectors(table, vectors, partition_tag=new_tag)
|
||||
status, ids = connect.add_vectors(collection, vectors, partition_tag=new_tag)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(table)
|
||||
status, info = connect.collection_info(collection)
|
||||
assert status.OK()
|
||||
assert info.count == nb * 2
|
||||
for partition in info.partitions_stat:
|
||||
@ -205,24 +205,24 @@ class TestTableInfoBase:
|
||||
return request.param
|
||||
|
||||
@pytest.mark.timeout(INFO_TIMEOUT)
|
||||
def test_get_table_info_after_index_created(self, connect, table, get_simple_index):
|
||||
def test_get_collection_info_after_index_created(self, connect, collection, get_simple_index):
|
||||
'''
|
||||
target: test table info after index created
|
||||
method: create table, add vectors, create index and call table_info
|
||||
target: test collection info after index created
|
||||
method: create collection, add vectors, create index and call collection_info
|
||||
expected: status ok, index created and shown in segments_stat
|
||||
'''
|
||||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
vectors = gen_vector(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status = connect.create_index(table, index_type, index_param)
|
||||
status = connect.create_index(collection, index_type, index_param)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(table)
|
||||
status, info = connect.collection_info(collection)
|
||||
assert status.OK()
|
||||
logging.getLogger().info(info)
|
||||
index_string = info.partitions_stat[0].segments_stat[0].index_name
|
||||
@ -231,24 +231,24 @@ class TestTableInfoBase:
|
||||
assert nb == info.partitions_stat[0].segments_stat[0].count
|
||||
|
||||
@pytest.mark.timeout(INFO_TIMEOUT)
|
||||
def test_get_table_info_after_create_same_index_repeatedly(self, connect, table, get_simple_index):
|
||||
def test_get_collection_info_after_create_same_index_repeatedly(self, connect, collection, get_simple_index):
|
||||
'''
|
||||
target: test table info after index created repeatedly
|
||||
method: create table, add vectors, create index and call table_info multiple times
|
||||
target: test collection info after index created repeatedly
|
||||
method: create collection, add vectors, create index and call collection_info multiple times
|
||||
expected: status ok, index info shown in segments_stat
|
||||
'''
|
||||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
vectors = gen_vector(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status = connect.create_index(table, index_type, index_param)
|
||||
status = connect.create_index(table, index_type, index_param)
|
||||
status = connect.create_index(table, index_type, index_param)
|
||||
status = connect.create_index(collection, index_type, index_param)
|
||||
status = connect.create_index(collection, index_type, index_param)
|
||||
status = connect.create_index(collection, index_type, index_param)
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(table)
|
||||
status, info = connect.collection_info(collection)
|
||||
assert status.OK()
|
||||
logging.getLogger().info(info)
|
||||
index_string = info.partitions_stat[0].segments_stat[0].index_name
|
||||
@ -257,22 +257,22 @@ class TestTableInfoBase:
|
||||
assert nb == info.partitions_stat[0].segments_stat[0].count
|
||||
|
||||
@pytest.mark.timeout(INFO_TIMEOUT)
|
||||
def test_get_table_info_after_create_different_index_repeatedly(self, connect, table, get_simple_index):
|
||||
def test_get_collection_info_after_create_different_index_repeatedly(self, connect, collection, get_simple_index):
|
||||
'''
|
||||
target: test table info after index created repeatedly
|
||||
method: create table, add vectors, create index and call table_info multiple times
|
||||
target: test collection info after index created repeatedly
|
||||
method: create collection, add vectors, create index and call collection_info multiple times
|
||||
expected: status ok, index info shown in segments_stat
|
||||
'''
|
||||
vectors = gen_vector(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
index_param = {"nlist": nlist}
|
||||
for index_type in [IndexType.FLAT, IndexType.IVFLAT, IndexType.IVF_SQ8]:
|
||||
status = connect.create_index(table, index_type, index_param)
|
||||
status = connect.create_index(collection, index_type, index_param)
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(table)
|
||||
status, info = connect.collection_info(collection)
|
||||
assert status.OK()
|
||||
logging.getLogger().info(info)
|
||||
index_string = info.partitions_stat[0].segments_stat[0].index_name
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -285,7 +285,7 @@ class TestConnect:
|
||||
b. data_set not too large incase disconnection happens when data is underd-preparing
|
||||
c. data_set not too small incase disconnection happens when data has already been transferred
|
||||
d. make sure disconnection happens when data is in-transport
|
||||
Expected: Failure, get_table_row_count == 0
|
||||
Expected: Failure, count_collection == 0
|
||||
|
||||
'''
|
||||
pass
|
||||
|
||||
@ -11,7 +11,7 @@ from utils import *
|
||||
|
||||
dim = 128
|
||||
index_file_size = 10
|
||||
table_id = "test_delete"
|
||||
collection_id = "test_delete"
|
||||
DELETE_TIMEOUT = 60
|
||||
nprobe = 1
|
||||
epsilon = 0.001
|
||||
@ -38,7 +38,7 @@ class TestDeleteBase:
|
||||
pytest.skip("Only support CPU mode")
|
||||
return request.param
|
||||
|
||||
def test_delete_vector_search(self, connect, table, get_simple_index):
|
||||
def test_delete_vector_search(self, connect, collection, get_simple_index):
|
||||
'''
|
||||
target: test delete vector
|
||||
method: add vector and delete
|
||||
@ -47,22 +47,22 @@ class TestDeleteBase:
|
||||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
vector = gen_single_vector(dim)
|
||||
status, ids = connect.add_vectors(table, vector)
|
||||
status, ids = connect.add_vectors(collection, vector)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status = connect.delete_by_id(table, ids)
|
||||
status = connect.delete_by_id(collection, ids)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
search_param = get_search_param(index_type)
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
search_param = get_search_param(index_type)
|
||||
status, res = connect.search_vectors(table, top_k, vector, params=search_param)
|
||||
status, res = connect.search_vectors(collection, top_k, vector, params=search_param)
|
||||
logging.getLogger().info(res)
|
||||
assert status.OK()
|
||||
assert len(res) == 0
|
||||
|
||||
def test_delete_vector_multi_same_ids(self, connect, table, get_simple_index):
|
||||
def test_delete_vector_multi_same_ids(self, connect, collection, get_simple_index):
|
||||
'''
|
||||
target: test delete vector, with some same ids
|
||||
method: add vector and delete
|
||||
@ -71,58 +71,58 @@ class TestDeleteBase:
|
||||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
vectors = gen_vectors(nb, dim)
|
||||
connect.add_vectors(table, vectors, ids=[1 for i in range(nb)])
|
||||
status = connect.flush([table])
|
||||
connect.add_vectors(collection, vectors, ids=[1 for i in range(nb)])
|
||||
status = connect.flush([collection])
|
||||
# Bloom filter error
|
||||
assert status.OK()
|
||||
status = connect.delete_by_id(table, [1])
|
||||
status = connect.delete_by_id(collection, [1])
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
search_param = get_search_param(index_type)
|
||||
status, res = connect.search_vectors(table, top_k, [vectors[0]], params=search_param)
|
||||
status, res = connect.search_vectors(collection, top_k, [vectors[0]], params=search_param)
|
||||
logging.getLogger().info(res)
|
||||
assert status.OK()
|
||||
assert len(res) == 0
|
||||
|
||||
def test_delete_vector_table_count(self, connect, table):
|
||||
def test_delete_vector_collection_count(self, connect, collection):
|
||||
'''
|
||||
target: test delete vector
|
||||
method: add vector and delete
|
||||
expected: status ok, vector deleted
|
||||
'''
|
||||
vector = gen_single_vector(dim)
|
||||
status, ids = connect.add_vectors(table, vector)
|
||||
status, ids = connect.add_vectors(collection, vector)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status = connect.delete_by_id(table, ids)
|
||||
status = connect.delete_by_id(collection, ids)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status, res = connect.get_table_row_count(table)
|
||||
status = connect.flush([collection])
|
||||
status, res = connect.count_collection(collection)
|
||||
assert status.OK()
|
||||
assert res == 0
|
||||
|
||||
def test_delete_vector_table_count_no_flush(self, connect, table):
|
||||
def test_delete_vector_collection_count_no_flush(self, connect, collection):
|
||||
'''
|
||||
target: test delete vector
|
||||
method: add vector and delete, no flush(using auto flush)
|
||||
expected: status ok, vector deleted
|
||||
'''
|
||||
vector = gen_single_vector(dim)
|
||||
status, ids = connect.add_vectors(table, vector)
|
||||
status, ids = connect.add_vectors(collection, vector)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status = connect.delete_by_id(table, ids)
|
||||
status = connect.delete_by_id(collection, ids)
|
||||
assert status.OK()
|
||||
time.sleep(2)
|
||||
status, res = connect.get_table_row_count(table)
|
||||
status, res = connect.count_collection(collection)
|
||||
assert status.OK()
|
||||
assert res == 0
|
||||
|
||||
def test_delete_vector_id_not_exised(self, connect, table, get_simple_index):
|
||||
def test_delete_vector_id_not_exised(self, connect, collection, get_simple_index):
|
||||
'''
|
||||
target: test delete vector, params vector_id not existed
|
||||
method: add vector and delete
|
||||
@ -131,34 +131,34 @@ class TestDeleteBase:
|
||||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
vector = gen_single_vector(dim)
|
||||
status, ids = connect.add_vectors(table, vector)
|
||||
status, ids = connect.add_vectors(collection, vector)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status = connect.delete_by_id(table, [0])
|
||||
status = connect.delete_by_id(collection, [0])
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
search_param = get_search_param(index_type)
|
||||
status, res = connect.search_vectors(table, top_k, vector, params=search_param)
|
||||
status, res = connect.search_vectors(collection, top_k, vector, params=search_param)
|
||||
assert status.OK()
|
||||
assert res[0][0].id == ids[0]
|
||||
|
||||
def test_delete_vector_table_not_existed(self, connect, table):
|
||||
def test_delete_vector_collection_not_existed(self, connect, collection):
|
||||
'''
|
||||
target: test delete vector, params table_name not existed
|
||||
target: test delete vector, params collection_name not existed
|
||||
method: add vector and delete
|
||||
expected: status not ok
|
||||
'''
|
||||
vector = gen_single_vector(dim)
|
||||
status, ids = connect.add_vectors(table, vector)
|
||||
status, ids = connect.add_vectors(collection, vector)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
table_new = gen_unique_str()
|
||||
status = connect.delete_by_id(table_new, [0])
|
||||
collection_new = gen_unique_str()
|
||||
status = connect.delete_by_id(collection_new, [0])
|
||||
assert not status.OK()
|
||||
|
||||
def test_add_vectors_delete_vector(self, connect, table, get_simple_index):
|
||||
def test_add_vectors_delete_vector(self, connect, collection, get_simple_index):
|
||||
'''
|
||||
method: add vectors and delete
|
||||
expected: status ok, vectors deleted
|
||||
@ -166,19 +166,19 @@ class TestDeleteBase:
|
||||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
vectors = gen_vector(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
delete_ids = [ids[0], ids[-1]]
|
||||
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
||||
status = connect.delete_by_id(table, delete_ids)
|
||||
status = connect.delete_by_id(collection, delete_ids)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
search_param = get_search_param(index_type)
|
||||
status, res = connect.search_vectors(table, top_k, query_vecs, params=search_param)
|
||||
status, res = connect.search_vectors(collection, top_k, query_vecs, params=search_param)
|
||||
assert status.OK()
|
||||
logging.getLogger().info(res)
|
||||
assert res[0][0].distance > epsilon
|
||||
@ -186,7 +186,7 @@ class TestDeleteBase:
|
||||
assert res[1][0].id == ids[1]
|
||||
assert res[2][0].distance > epsilon
|
||||
|
||||
def test_create_index_after_delete(self, connect, table, get_simple_index):
|
||||
def test_create_index_after_delete(self, connect, collection, get_simple_index):
|
||||
'''
|
||||
method: add vectors and delete, then create index
|
||||
expected: status ok, vectors deleted, index created
|
||||
@ -194,19 +194,19 @@ class TestDeleteBase:
|
||||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
vectors = gen_vector(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
delete_ids = [ids[0], ids[-1]]
|
||||
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
||||
status = connect.delete_by_id(table, delete_ids)
|
||||
status = connect.delete_by_id(collection, delete_ids)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.create_index(table, index_type, index_param)
|
||||
status = connect.flush([collection])
|
||||
status = connect.create_index(collection, index_type, index_param)
|
||||
assert status.OK()
|
||||
search_param = get_search_param(index_type)
|
||||
status, res = connect.search_vectors(table, top_k, query_vecs, params=search_param)
|
||||
status, res = connect.search_vectors(collection, top_k, query_vecs, params=search_param)
|
||||
assert status.OK()
|
||||
logging.getLogger().info(res)
|
||||
assert res[0][0].distance > epsilon
|
||||
@ -214,7 +214,7 @@ class TestDeleteBase:
|
||||
assert res[1][0].id == ids[1]
|
||||
assert res[2][0].distance > epsilon
|
||||
|
||||
def test_add_vector_after_delete(self, connect, table, get_simple_index):
|
||||
def test_add_vector_after_delete(self, connect, collection, get_simple_index):
|
||||
'''
|
||||
method: add vectors and delete, then add vector
|
||||
expected: status ok, vectors deleted, vector added
|
||||
@ -222,22 +222,22 @@ class TestDeleteBase:
|
||||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
vectors = gen_vector(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
delete_ids = [ids[0], ids[-1]]
|
||||
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
||||
status = connect.delete_by_id(table, delete_ids)
|
||||
status = connect.delete_by_id(collection, delete_ids)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status, tmp_ids = connect.add_vectors(table, [vectors[0], vectors[-1]])
|
||||
status = connect.flush([collection])
|
||||
status, tmp_ids = connect.add_vectors(collection, [vectors[0], vectors[-1]])
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
search_param = get_search_param(index_type)
|
||||
status, res = connect.search_vectors(table, top_k, query_vecs, params=search_param)
|
||||
status, res = connect.search_vectors(collection, top_k, query_vecs, params=search_param)
|
||||
assert status.OK()
|
||||
logging.getLogger().info(res)
|
||||
assert res[0][0].id == tmp_ids[0]
|
||||
@ -246,41 +246,41 @@ class TestDeleteBase:
|
||||
assert res[2][0].id == tmp_ids[-1]
|
||||
assert res[2][0].distance < epsilon
|
||||
|
||||
def test_delete_multiable_times(self, connect, table):
|
||||
def test_delete_multiable_times(self, connect, collection):
|
||||
'''
|
||||
method: add vectors and delete id serveral times
|
||||
expected: status ok, vectors deleted, and status ok for next delete operation
|
||||
'''
|
||||
vectors = gen_vector(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
delete_ids = [ids[0], ids[-1]]
|
||||
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
||||
status = connect.delete_by_id(table, delete_ids)
|
||||
status = connect.delete_by_id(collection, delete_ids)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
for i in range(10):
|
||||
status = connect.delete_by_id(table, delete_ids)
|
||||
status = connect.delete_by_id(collection, delete_ids)
|
||||
assert status.OK()
|
||||
|
||||
def test_delete_no_flush_multiable_times(self, connect, table):
|
||||
def test_delete_no_flush_multiable_times(self, connect, collection):
|
||||
'''
|
||||
method: add vectors and delete id serveral times
|
||||
expected: status ok, vectors deleted, and status ok for next delete operation
|
||||
'''
|
||||
vectors = gen_vector(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
delete_ids = [ids[0], ids[-1]]
|
||||
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
||||
status = connect.delete_by_id(table, delete_ids)
|
||||
status = connect.delete_by_id(collection, delete_ids)
|
||||
assert status.OK()
|
||||
for i in range(10):
|
||||
status = connect.delete_by_id(table, delete_ids)
|
||||
status = connect.delete_by_id(collection, delete_ids)
|
||||
assert status.OK()
|
||||
assert status.OK()
|
||||
|
||||
@ -303,7 +303,7 @@ class TestDeleteIndexedVectors:
|
||||
pytest.skip("Only support CPU mode")
|
||||
return request.param
|
||||
|
||||
def test_delete_vectors_after_index_created_search(self, connect, table, get_simple_index):
|
||||
def test_delete_vectors_after_index_created_search(self, connect, collection, get_simple_index):
|
||||
'''
|
||||
target: test delete vector after index created
|
||||
method: add vector, create index and delete vector
|
||||
@ -312,22 +312,22 @@ class TestDeleteIndexedVectors:
|
||||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
vector = gen_single_vector(dim)
|
||||
status, ids = connect.add_vectors(table, vector)
|
||||
status, ids = connect.add_vectors(collection, vector)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status = connect.create_index(table, index_type, index_param)
|
||||
status = connect.create_index(collection, index_type, index_param)
|
||||
assert status.OK()
|
||||
status = connect.delete_by_id(table, ids)
|
||||
status = connect.delete_by_id(collection, ids)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
search_param = get_search_param(index_type)
|
||||
status, res = connect.search_vectors(table, top_k, vector, params=search_param)
|
||||
status, res = connect.search_vectors(collection, top_k, vector, params=search_param)
|
||||
logging.getLogger().info(res)
|
||||
assert status.OK()
|
||||
assert len(res) == 0
|
||||
|
||||
def test_add_vectors_delete_vector(self, connect, table, get_simple_index):
|
||||
def test_add_vectors_delete_vector(self, connect, collection, get_simple_index):
|
||||
'''
|
||||
method: add vectors and delete
|
||||
expected: status ok, vectors deleted
|
||||
@ -335,21 +335,21 @@ class TestDeleteIndexedVectors:
|
||||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
vectors = gen_vector(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status = connect.create_index(table, index_type, index_param)
|
||||
status = connect.create_index(collection, index_type, index_param)
|
||||
assert status.OK()
|
||||
delete_ids = [ids[0], ids[-1]]
|
||||
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
||||
status = connect.delete_by_id(table, delete_ids)
|
||||
status = connect.delete_by_id(collection, delete_ids)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
search_param = get_search_param(index_type)
|
||||
status, res = connect.search_vectors(table, top_k, query_vecs, params=search_param)
|
||||
status, res = connect.search_vectors(collection, top_k, query_vecs, params=search_param)
|
||||
assert status.OK()
|
||||
logging.getLogger().info(res)
|
||||
assert res[0][0].distance > epsilon
|
||||
@ -375,7 +375,7 @@ class TestDeleteBinary:
|
||||
else:
|
||||
pytest.skip("Skip index Temporary")
|
||||
|
||||
def test_delete_vector_search(self, connect, jac_table, get_simple_index):
|
||||
def test_delete_vector_search(self, connect, jac_collection, get_simple_index):
|
||||
'''
|
||||
target: test delete vector
|
||||
method: add vector and delete
|
||||
@ -384,15 +384,15 @@ class TestDeleteBinary:
|
||||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
tmp, vector = gen_binary_vectors(1, dim)
|
||||
status, ids = connect.add_vectors(jac_table, vector)
|
||||
status, ids = connect.add_vectors(jac_collection, vector)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
assert status.OK()
|
||||
status = connect.delete_by_id(jac_table, ids)
|
||||
status = connect.delete_by_id(jac_collection, ids)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
search_param = get_search_param(index_type)
|
||||
status, res = connect.search_vectors(jac_table, top_k, vector, params=search_param)
|
||||
status, res = connect.search_vectors(jac_collection, top_k, vector, params=search_param)
|
||||
logging.getLogger().info(res)
|
||||
assert status.OK()
|
||||
assert len(res) == 0
|
||||
@ -400,25 +400,25 @@ class TestDeleteBinary:
|
||||
assert len(res) == 0
|
||||
|
||||
# TODO: soft delete
|
||||
def test_delete_vector_table_count(self, connect, jac_table):
|
||||
def test_delete_vector_collection_count(self, connect, jac_collection):
|
||||
'''
|
||||
target: test delete vector
|
||||
method: add vector and delete
|
||||
expected: status ok, vector deleted
|
||||
'''
|
||||
tmp, vector = gen_binary_vectors(1, dim)
|
||||
status, ids = connect.add_vectors(jac_table, vector)
|
||||
status, ids = connect.add_vectors(jac_collection, vector)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
assert status.OK()
|
||||
status = connect.delete_by_id(jac_table, ids)
|
||||
status = connect.delete_by_id(jac_collection, ids)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status, res = connect.get_table_row_count(jac_table)
|
||||
status = connect.flush([jac_collection])
|
||||
status, res = connect.count_collection(jac_collection)
|
||||
assert status.OK()
|
||||
assert res == 0
|
||||
|
||||
def test_delete_vector_id_not_exised(self, connect, jac_table, get_simple_index):
|
||||
def test_delete_vector_id_not_exised(self, connect, jac_collection, get_simple_index):
|
||||
'''
|
||||
target: test delete vector, params vector_id not existed
|
||||
method: add vector and delete
|
||||
@ -427,37 +427,37 @@ class TestDeleteBinary:
|
||||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
tmp, vector = gen_binary_vectors(1, dim)
|
||||
status, ids = connect.add_vectors(jac_table, vector)
|
||||
status, ids = connect.add_vectors(jac_collection, vector)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
assert status.OK()
|
||||
status = connect.delete_by_id(jac_table, [0])
|
||||
status = connect.delete_by_id(jac_collection, [0])
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
status = connect.flush([jac_collection])
|
||||
search_param = get_search_param(index_type)
|
||||
status, res = connect.search_vectors(jac_table, top_k, vector, params=search_param)
|
||||
status, res = connect.search_vectors(jac_collection, top_k, vector, params=search_param)
|
||||
assert status.OK()
|
||||
assert res[0][0].id == ids[0]
|
||||
|
||||
def test_delete_vector_table_not_existed(self, connect, jac_table):
|
||||
def test_delete_vector_collection_not_existed(self, connect, jac_collection):
|
||||
'''
|
||||
target: test delete vector, params table_name not existed
|
||||
target: test delete vector, params collection_name not existed
|
||||
method: add vector and delete
|
||||
expected: status not ok
|
||||
'''
|
||||
tmp, vector = gen_binary_vectors(1, dim)
|
||||
status, ids = connect.add_vectors(jac_table, vector)
|
||||
status, ids = connect.add_vectors(jac_collection, vector)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
assert status.OK()
|
||||
table_new = gen_unique_str()
|
||||
status = connect.delete_by_id(table_new, [0])
|
||||
table_new = gen_unique_str()
|
||||
status = connect.delete_by_id(table_new, [0])
|
||||
collection_new = gen_unique_str()
|
||||
status = connect.delete_by_id(collection_new, [0])
|
||||
collection_new = gen_unique_str()
|
||||
status = connect.delete_by_id(collection_new, [0])
|
||||
assert not status.OK()
|
||||
|
||||
def test_add_vectors_delete_vector(self, connect, jac_table, get_simple_index):
|
||||
def test_add_vectors_delete_vector(self, connect, jac_collection, get_simple_index):
|
||||
'''
|
||||
method: add vectors and delete
|
||||
expected: status ok, vectors deleted
|
||||
@ -465,24 +465,24 @@ class TestDeleteBinary:
|
||||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||
status, ids = connect.add_vectors(jac_table, vectors)
|
||||
status, ids = connect.add_vectors(jac_collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
assert status.OK()
|
||||
delete_ids = [ids[0], ids[-1]]
|
||||
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
||||
status = connect.delete_by_id(jac_table, delete_ids)
|
||||
status = connect.delete_by_id(jac_collection, delete_ids)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
search_param = get_search_param(index_type)
|
||||
status, res = connect.search_vectors(jac_table, top_k, query_vecs, params=search_param)
|
||||
status, res = connect.search_vectors(jac_collection, top_k, query_vecs, params=search_param)
|
||||
assert status.OK()
|
||||
logging.getLogger().info(res)
|
||||
assert res[0][0].id != ids[0]
|
||||
assert res[1][0].id == ids[1]
|
||||
assert res[2][0].id != ids[-1]
|
||||
|
||||
def test_add_after_delete_vector(self, connect, jac_table, get_simple_index):
|
||||
def test_add_after_delete_vector(self, connect, jac_collection, get_simple_index):
|
||||
'''
|
||||
method: add vectors and delete, add
|
||||
expected: status ok, vectors added
|
||||
@ -490,20 +490,20 @@ class TestDeleteBinary:
|
||||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||
status, ids = connect.add_vectors(jac_table, vectors)
|
||||
status, ids = connect.add_vectors(jac_collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
assert status.OK()
|
||||
delete_ids = [ids[0], ids[-1]]
|
||||
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
||||
status = connect.delete_by_id(jac_table, delete_ids)
|
||||
status = connect.delete_by_id(jac_collection, delete_ids)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status, tmp_ids = connect.add_vectors(jac_table, [vectors[0], vectors[-1]])
|
||||
status = connect.flush([jac_collection])
|
||||
status, tmp_ids = connect.add_vectors(jac_collection, [vectors[0], vectors[-1]])
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
search_param = get_search_param(index_type)
|
||||
status, res = connect.search_vectors(jac_table, top_k, query_vecs, params=search_param)
|
||||
status, res = connect.search_vectors(jac_collection, top_k, query_vecs, params=search_param)
|
||||
assert status.OK()
|
||||
logging.getLogger().info(res)
|
||||
assert res[0][0].id == tmp_ids[0]
|
||||
@ -526,32 +526,32 @@ class TestDeleteIdsIngalid(object):
|
||||
yield request.param
|
||||
|
||||
@pytest.mark.level(1)
|
||||
def test_delete_vector_id_invalid(self, connect, table, gen_invalid_id):
|
||||
def test_delete_vector_id_invalid(self, connect, collection, gen_invalid_id):
|
||||
invalid_id = gen_invalid_id
|
||||
with pytest.raises(Exception) as e:
|
||||
status = connect.delete_by_id(table, [invalid_id])
|
||||
status = connect.delete_by_id(collection, [invalid_id])
|
||||
|
||||
@pytest.mark.level(2)
|
||||
def test_delete_vector_ids_invalid(self, connect, table, gen_invalid_id):
|
||||
def test_delete_vector_ids_invalid(self, connect, collection, gen_invalid_id):
|
||||
invalid_id = gen_invalid_id
|
||||
with pytest.raises(Exception) as e:
|
||||
status = connect.delete_by_id(table, [1, invalid_id])
|
||||
status = connect.delete_by_id(collection, [1, invalid_id])
|
||||
|
||||
|
||||
class TestTableNameInvalid(object):
|
||||
class TestCollectionNameInvalid(object):
|
||||
"""
|
||||
Test adding vectors with invalid table names
|
||||
Test adding vectors with invalid collection names
|
||||
"""
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=gen_invalid_table_names()
|
||||
params=gen_invalid_collection_names()
|
||||
)
|
||||
def get_table_name(self, request):
|
||||
def get_collection_name(self, request):
|
||||
yield request.param
|
||||
|
||||
@pytest.mark.level(2)
|
||||
def test_delete_vectors_with_invalid_table_name(self, connect, get_table_name):
|
||||
table_name = get_table_name
|
||||
status = connect.delete_by_id(table_name, [1])
|
||||
def test_delete_vectors_with_invalid_collection_name(self, connect, get_collection_name):
|
||||
collection_name = get_collection_name
|
||||
status = connect.delete_by_id(collection_name, [1])
|
||||
assert not status.OK()
|
||||
|
||||
|
||||
@ -10,7 +10,7 @@ from utils import *
|
||||
|
||||
dim = 128
|
||||
index_file_size = 10
|
||||
table_id = "test_flush"
|
||||
collection_id = "test_flush"
|
||||
DELETE_TIMEOUT = 60
|
||||
nprobe = 1
|
||||
tag = "1970-01-01"
|
||||
@ -38,117 +38,117 @@ class TestFlushBase:
|
||||
pytest.skip("Only support CPU mode")
|
||||
return request.param
|
||||
|
||||
def test_flush_table_not_existed(self, connect, table):
|
||||
def test_flush_collection_not_existed(self, connect, collection):
|
||||
'''
|
||||
target: test delete vector, params table_name not existed
|
||||
target: test delete vector, params collection_name not existed
|
||||
method: add vector and delete
|
||||
expected: status not ok
|
||||
'''
|
||||
table_new = gen_unique_str()
|
||||
status = connect.flush([table_new])
|
||||
collection_new = gen_unique_str()
|
||||
status = connect.flush([collection_new])
|
||||
assert not status.OK()
|
||||
|
||||
def test_flush_empty_table(self, connect, table):
|
||||
def test_flush_empty_collection(self, connect, collection):
|
||||
'''
|
||||
method: flush table with no vectors
|
||||
method: flush collection with no vectors
|
||||
expected: status ok
|
||||
'''
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
|
||||
def test_add_partition_flush(self, connect, table):
|
||||
def test_add_partition_flush(self, connect, collection):
|
||||
'''
|
||||
method: add vectors into partition in table, flush serveral times
|
||||
method: add vectors into partition in collection, flush serveral times
|
||||
expected: status ok
|
||||
'''
|
||||
vectors = gen_vector(nb, dim)
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
vectors = gen_vectors(nb, dim)
|
||||
ids = [i for i in range(nb)]
|
||||
status, ids = connect.insert(table, vectors, ids)
|
||||
status = connect.flush([table])
|
||||
result, res = connect.get_table_row_count(table)
|
||||
status, ids = connect.insert(collection, vectors, ids)
|
||||
status = connect.flush([collection])
|
||||
result, res = connect.count_collection(collection)
|
||||
assert res == nb
|
||||
status, ids = connect.insert(table, vectors, ids, partition_tag=tag)
|
||||
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
result, res = connect.get_table_row_count(table)
|
||||
result, res = connect.count_collection(collection)
|
||||
assert res == 2 * nb
|
||||
|
||||
def test_add_partitions_flush(self, connect, table):
|
||||
def test_add_partitions_flush(self, connect, collection):
|
||||
'''
|
||||
method: add vectors into partitions in table, flush one
|
||||
method: add vectors into partitions in collection, flush one
|
||||
expected: status ok
|
||||
'''
|
||||
vectors = gen_vectors(nb, dim)
|
||||
tag_new = gen_unique_str()
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(table, tag_new)
|
||||
status = connect.create_partition(collection, tag)
|
||||
status = connect.create_partition(collection, tag_new)
|
||||
ids = [i for i in range(nb)]
|
||||
status, ids = connect.insert(table, vectors, ids, partition_tag=tag)
|
||||
status = connect.flush([table])
|
||||
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag)
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, ids = connect.insert(table, vectors, ids, partition_tag=tag_new)
|
||||
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag_new)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
result, res = connect.get_table_row_count(table)
|
||||
result, res = connect.count_collection(collection)
|
||||
assert res == 2 * nb
|
||||
|
||||
def test_add_tables_flush(self, connect, table):
|
||||
def test_add_collections_flush(self, connect, collection):
|
||||
'''
|
||||
method: add vectors into tables, flush one
|
||||
method: add vectors into collections, flush one
|
||||
expected: status ok
|
||||
'''
|
||||
vectors = gen_vectors(nb, dim)
|
||||
table_new = gen_unique_str()
|
||||
param = {'table_name': table_new,
|
||||
collection_new = gen_unique_str()
|
||||
param = {'collection_name': collection_new,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.L2}
|
||||
status = connect.create_table(param)
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(table_new, tag)
|
||||
status = connect.create_collection(param)
|
||||
status = connect.create_partition(collection, tag)
|
||||
status = connect.create_partition(collection_new, tag)
|
||||
vectors = gen_vectors(nb, dim)
|
||||
ids = [i for i in range(nb)]
|
||||
status, ids = connect.insert(table, vectors, ids, partition_tag=tag)
|
||||
status, ids = connect.insert(table_new, vectors, ids, partition_tag=tag)
|
||||
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag)
|
||||
status, ids = connect.insert(collection_new, vectors, ids, partition_tag=tag)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([table_new])
|
||||
status = connect.flush([collection])
|
||||
status = connect.flush([collection_new])
|
||||
assert status.OK()
|
||||
result, res = connect.get_table_row_count(table)
|
||||
result, res = connect.count_collection(collection)
|
||||
assert res == nb
|
||||
result, res = connect.get_table_row_count(table_new)
|
||||
result, res = connect.count_collection(collection_new)
|
||||
assert res == nb
|
||||
|
||||
def test_add_flush_multiable_times(self, connect, table):
|
||||
def test_add_flush_multiable_times(self, connect, collection):
|
||||
'''
|
||||
method: add vectors, flush serveral times
|
||||
expected: status ok
|
||||
'''
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
for i in range(10):
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
||||
status, res = connect.search_vectors(table, top_k, query_records=query_vecs)
|
||||
status, res = connect.search_vectors(collection, top_k, query_records=query_vecs)
|
||||
assert status.OK()
|
||||
|
||||
def test_add_flush_auto(self, connect, table):
|
||||
def test_add_flush_auto(self, connect, collection):
|
||||
'''
|
||||
method: add vectors
|
||||
expected: status ok
|
||||
'''
|
||||
vectors = gen_vectors(nb, dim)
|
||||
ids = [i for i in range(nb)]
|
||||
status, ids = connect.add_vectors(table, vectors, ids)
|
||||
status, ids = connect.add_vectors(collection, vectors, ids)
|
||||
assert status.OK()
|
||||
time.sleep(2)
|
||||
status, res = connect.get_table_row_count(table)
|
||||
status, res = connect.count_collection(collection)
|
||||
assert status.OK()
|
||||
assert res == nb
|
||||
|
||||
@ -163,7 +163,7 @@ class TestFlushBase:
|
||||
yield request.param
|
||||
|
||||
# both autoflush / flush
|
||||
def test_add_flush_same_ids(self, connect, table, same_ids):
|
||||
def test_add_flush_same_ids(self, connect, collection, same_ids):
|
||||
'''
|
||||
method: add vectors, with same ids, count(same ids) < 15, > 15
|
||||
expected: status ok
|
||||
@ -173,79 +173,79 @@ class TestFlushBase:
|
||||
for i, item in enumerate(ids):
|
||||
if item <= same_ids:
|
||||
ids[i] = 0
|
||||
status, ids = connect.add_vectors(table, vectors, ids)
|
||||
status, ids = connect.add_vectors(collection, vectors, ids)
|
||||
time.sleep(2)
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_table_row_count(table)
|
||||
status, res = connect.count_collection(collection)
|
||||
assert status.OK()
|
||||
assert res == nb
|
||||
|
||||
def test_delete_flush_multiable_times(self, connect, table):
|
||||
def test_delete_flush_multiable_times(self, connect, collection):
|
||||
'''
|
||||
method: delete vectors, flush serveral times
|
||||
expected: status ok
|
||||
'''
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.delete_by_id(table, [ids[-1]])
|
||||
status = connect.delete_by_id(collection, [ids[-1]])
|
||||
assert status.OK()
|
||||
for i in range(10):
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
||||
status, res = connect.search_vectors(table, top_k, query_records=query_vecs)
|
||||
status, res = connect.search_vectors(collection, top_k, query_records=query_vecs)
|
||||
assert status.OK()
|
||||
|
||||
# TODO: CI fail, LOCAL pass
|
||||
def _test_table_count_during_flush(self, connect, args):
|
||||
def _test_collection_count_during_flush(self, connect, args):
|
||||
'''
|
||||
method: flush table at background, call `get_table_row_count`
|
||||
method: flush collection at background, call `count_collection`
|
||||
expected: status ok
|
||||
'''
|
||||
table = gen_unique_str()
|
||||
collection = gen_unique_str()
|
||||
uri = "tcp://%s:%s" % (args["ip"], args["port"])
|
||||
param = {'table_name': table,
|
||||
param = {'collection_name': collection,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.L2}
|
||||
milvus = get_milvus()
|
||||
milvus.connect(uri=uri)
|
||||
milvus.create_table(param)
|
||||
milvus.create_collection(param)
|
||||
vectors = gen_vector(nb, dim)
|
||||
status, ids = milvus.add_vectors(table, vectors, ids=[i for i in range(nb)])
|
||||
def flush(table_name):
|
||||
status, ids = milvus.add_vectors(collection, vectors, ids=[i for i in range(nb)])
|
||||
def flush(collection_name):
|
||||
milvus = get_milvus()
|
||||
milvus.connect(uri=uri)
|
||||
status = milvus.delete_by_id(table_name, [i for i in range(nb)])
|
||||
status = milvus.delete_by_id(collection_name, [i for i in range(nb)])
|
||||
assert status.OK()
|
||||
status = milvus.flush([table_name])
|
||||
status = milvus.flush([collection_name])
|
||||
assert status.OK()
|
||||
p = Process(target=flush, args=(table, ))
|
||||
p = Process(target=flush, args=(collection, ))
|
||||
p.start()
|
||||
status, res = milvus.get_table_row_count(table)
|
||||
status, res = milvus.count_collection(collection)
|
||||
assert status.OK()
|
||||
p.join()
|
||||
status, res = milvus.get_table_row_count(table)
|
||||
status, res = milvus.count_collection(collection)
|
||||
assert status.OK()
|
||||
logging.getLogger().info(res)
|
||||
assert res == 0
|
||||
|
||||
|
||||
class TestTableNameInvalid(object):
|
||||
class TestCollectionNameInvalid(object):
|
||||
"""
|
||||
Test adding vectors with invalid table names
|
||||
Test adding vectors with invalid collection names
|
||||
"""
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=gen_invalid_table_names()
|
||||
params=gen_invalid_collection_names()
|
||||
)
|
||||
def get_table_name(self, request):
|
||||
def get_collection_name(self, request):
|
||||
yield request.param
|
||||
|
||||
@pytest.mark.level(2)
|
||||
def test_flush_with_invalid_table_name(self, connect, get_table_name):
|
||||
table_name = get_table_name
|
||||
def test_flush_with_invalid_collection_name(self, connect, get_collection_name):
|
||||
collection_name = get_collection_name
|
||||
with pytest.raises(Exception) as e:
|
||||
status, result = connect.flush(table_name)
|
||||
status, result = connect.flush(collection_name)
|
||||
|
||||
@ -11,7 +11,7 @@ from utils import *
|
||||
|
||||
dim = 128
|
||||
index_file_size = 10
|
||||
table_id = "test_get_vector_by_id"
|
||||
collection_id = "test_get_vector_by_id"
|
||||
DELETE_TIMEOUT = 60
|
||||
nprobe = 1
|
||||
tag = "1970-01-01"
|
||||
@ -25,54 +25,54 @@ class TestGetBase:
|
||||
The following cases are used to test `get_vector_by_id` function
|
||||
******************************************************************
|
||||
"""
|
||||
def test_get_vector_A(self, connect, table):
|
||||
def test_get_vector_A(self, connect, collection):
|
||||
'''
|
||||
target: test get_vector_by_id
|
||||
method: add vector, and get
|
||||
expected: status ok, vector returned
|
||||
'''
|
||||
vector = gen_single_vector(dim)
|
||||
status, ids = connect.add_vectors(table, vector)
|
||||
status, ids = connect.add_vectors(collection, vector)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_vector_by_id(table, ids[0])
|
||||
status, res = connect.get_vector_by_id(collection, ids[0])
|
||||
assert status.OK()
|
||||
assert_equal_vector(res, vector[0])
|
||||
|
||||
def test_get_vector_B(self, connect, table):
|
||||
def test_get_vector_B(self, connect, collection):
|
||||
'''
|
||||
target: test get_vector_by_id
|
||||
method: add vector, and get
|
||||
expected: status ok, vector returned
|
||||
'''
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_vector_by_id(table, ids[0])
|
||||
status, res = connect.get_vector_by_id(collection, ids[0])
|
||||
assert status.OK()
|
||||
assert_equal_vector(res, vectors[0])
|
||||
|
||||
def test_get_vector_partition(self, connect, table):
|
||||
def test_get_vector_partition(self, connect, collection):
|
||||
'''
|
||||
target: test get_vector_by_id
|
||||
method: add vector, and get
|
||||
expected: status ok, vector returned
|
||||
'''
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
assert status.OK()
|
||||
status, ids = connect.add_vectors(table, vectors, partition_tag=tag)
|
||||
status, ids = connect.add_vectors(collection, vectors, partition_tag=tag)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_vector_by_id(table, ids[0])
|
||||
status, res = connect.get_vector_by_id(collection, ids[0])
|
||||
assert status.OK()
|
||||
assert_equal_vector(res, vectors[0])
|
||||
|
||||
def test_get_vector_multi_same_ids(self, connect, table):
|
||||
def test_get_vector_multi_same_ids(self, connect, collection):
|
||||
'''
|
||||
target: test get_vector_by_id
|
||||
method: add vectors, with the same id, get vector by the given id
|
||||
@ -81,10 +81,10 @@ class TestGetBase:
|
||||
vectors = gen_vectors(nb, dim)
|
||||
ids = [i for i in range(nb)]
|
||||
ids[1] = 0; ids[-1] = 0
|
||||
status, ids = connect.add_vectors(table, vectors, ids=ids)
|
||||
status = connect.flush([table])
|
||||
status, ids = connect.add_vectors(collection, vectors, ids=ids)
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_vector_by_id(table, 0)
|
||||
status, res = connect.get_vector_by_id(collection, 0)
|
||||
assert status.OK()
|
||||
assert_equal_vector(res, vectors[0])
|
||||
|
||||
@ -101,75 +101,75 @@ class TestGetBase:
|
||||
def get_id(self, request):
|
||||
yield request.param
|
||||
|
||||
def test_get_vector_after_delete(self, connect, table, get_id):
|
||||
def test_get_vector_after_delete(self, connect, collection, get_id):
|
||||
'''
|
||||
target: test get_vector_by_id
|
||||
method: add vectors, and delete, get vector by the given id
|
||||
expected: status ok, get one vector
|
||||
'''
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
id = get_id
|
||||
status = connect.delete_by_id(table, [ids[id]])
|
||||
status = connect.delete_by_id(collection, [ids[id]])
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_vector_by_id(table, ids[id])
|
||||
status, res = connect.get_vector_by_id(collection, ids[id])
|
||||
assert status.OK()
|
||||
assert not res
|
||||
|
||||
def test_get_vector_after_delete_with_partition(self, connect, table, get_id):
|
||||
def test_get_vector_after_delete_with_partition(self, connect, collection, get_id):
|
||||
'''
|
||||
target: test get_vector_by_id
|
||||
method: add vectors into partition, and delete, get vector by the given id
|
||||
expected: status ok, get one vector
|
||||
'''
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status = connect.create_partition(table, tag)
|
||||
status, ids = connect.insert(table, vectors, partition_tag=tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
status, ids = connect.insert(collection, vectors, partition_tag=tag)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
id = get_id
|
||||
status = connect.delete_by_id(table, [ids[id]])
|
||||
status = connect.delete_by_id(collection, [ids[id]])
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_vector_by_id(table, ids[id])
|
||||
status, res = connect.get_vector_by_id(collection, ids[id])
|
||||
assert status.OK()
|
||||
assert not res
|
||||
|
||||
def test_get_vector_id_not_exised(self, connect, table):
|
||||
def test_get_vector_id_not_exised(self, connect, collection):
|
||||
'''
|
||||
target: test get vector, params vector_id not existed
|
||||
method: add vector and get
|
||||
expected: status ok, empty result
|
||||
'''
|
||||
vector = gen_single_vector(dim)
|
||||
status, ids = connect.add_vectors(table, vector)
|
||||
status, ids = connect.add_vectors(collection, vector)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_vector_by_id(table, 1)
|
||||
status, res = connect.get_vector_by_id(collection, 1)
|
||||
assert status.OK()
|
||||
assert not res
|
||||
|
||||
def test_get_vector_table_not_existed(self, connect, table):
|
||||
def test_get_vector_collection_not_existed(self, connect, collection):
|
||||
'''
|
||||
target: test get vector, params table_name not existed
|
||||
target: test get vector, params collection_name not existed
|
||||
method: add vector and get
|
||||
expected: status not ok
|
||||
'''
|
||||
vector = gen_single_vector(dim)
|
||||
status, ids = connect.add_vectors(table, vector)
|
||||
status, ids = connect.add_vectors(collection, vector)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
table_new = gen_unique_str()
|
||||
status, res = connect.get_vector_by_id(table_new, 1)
|
||||
collection_new = gen_unique_str()
|
||||
status, res = connect.get_vector_by_id(collection_new, 1)
|
||||
assert not status.OK()
|
||||
|
||||
|
||||
@ -205,7 +205,7 @@ class TestGetIndexedVectors:
|
||||
def get_id(self, request):
|
||||
yield request.param
|
||||
|
||||
def test_get_vectors_after_index_created(self, connect, table, get_simple_index, get_id):
|
||||
def test_get_vectors_after_index_created(self, connect, collection, get_simple_index, get_id):
|
||||
'''
|
||||
target: test get vector after index created
|
||||
method: add vector, create index and get vector
|
||||
@ -214,20 +214,20 @@ class TestGetIndexedVectors:
|
||||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
vectors = gen_vector(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status = connect.create_index(table, index_type, index_param)
|
||||
status = connect.create_index(collection, index_type, index_param)
|
||||
assert status.OK()
|
||||
id = get_id
|
||||
status, res = connect.get_vector_by_id(table, ids[id])
|
||||
status, res = connect.get_vector_by_id(collection, ids[id])
|
||||
assert status.OK()
|
||||
logging.getLogger().info(res)
|
||||
assert status.OK()
|
||||
assert_equal_vector(res, vectors[id])
|
||||
|
||||
def test_get_vector_after_delete(self, connect, table, get_simple_index, get_id):
|
||||
def test_get_vector_after_delete(self, connect, collection, get_simple_index, get_id):
|
||||
'''
|
||||
target: test get_vector_by_id
|
||||
method: add vectors, and delete, get vector by the given id
|
||||
@ -236,22 +236,22 @@ class TestGetIndexedVectors:
|
||||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status = connect.create_index(table, index_type, index_param)
|
||||
status = connect.create_index(collection, index_type, index_param)
|
||||
assert status.OK()
|
||||
id = get_id
|
||||
status = connect.delete_by_id(table, [ids[id]])
|
||||
status = connect.delete_by_id(collection, [ids[id]])
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_vector_by_id(table, ids[id])
|
||||
status, res = connect.get_vector_by_id(collection, ids[id])
|
||||
assert status.OK()
|
||||
assert not res
|
||||
|
||||
def test_get_vector_partition(self, connect, table, get_simple_index, get_id):
|
||||
def test_get_vector_partition(self, connect, collection, get_simple_index, get_id):
|
||||
'''
|
||||
target: test get_vector_by_id
|
||||
method: add vector, and get
|
||||
@ -260,16 +260,16 @@ class TestGetIndexedVectors:
|
||||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
ids = [i for i in range(nb)]
|
||||
status, ids = connect.add_vectors(table, vectors, ids, partition_tag=tag)
|
||||
status, ids = connect.add_vectors(collection, vectors, ids, partition_tag=tag)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status = connect.create_index(table, index_type, index_param)
|
||||
status = connect.create_index(collection, index_type, index_param)
|
||||
assert status.OK()
|
||||
id = get_id
|
||||
status, res = connect.get_vector_by_id(table, ids[id])
|
||||
status, res = connect.get_vector_by_id(collection, ids[id])
|
||||
assert status.OK()
|
||||
assert_equal_vector(res, vectors[id])
|
||||
|
||||
@ -280,37 +280,37 @@ class TestGetBinary:
|
||||
The following cases are used to test `get_vector_by_id` function
|
||||
******************************************************************
|
||||
"""
|
||||
def test_get_vector_A(self, connect, jac_table):
|
||||
def test_get_vector_A(self, connect, jac_collection):
|
||||
'''
|
||||
target: test get_vector_by_id
|
||||
method: add vector, and get
|
||||
expected: status ok, vector returned
|
||||
'''
|
||||
tmp, vector = gen_binary_vectors(1, dim)
|
||||
status, ids = connect.add_vectors(jac_table, vector)
|
||||
status, ids = connect.add_vectors(jac_collection, vector)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_vector_by_id(jac_table, ids[0])
|
||||
status, res = connect.get_vector_by_id(jac_collection, ids[0])
|
||||
assert status.OK()
|
||||
assert res == vector[0]
|
||||
|
||||
def test_get_vector_B(self, connect, jac_table):
|
||||
def test_get_vector_B(self, connect, jac_collection):
|
||||
'''
|
||||
target: test get_vector_by_id
|
||||
method: add vector, and get
|
||||
expected: status ok, vector returned
|
||||
'''
|
||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||
status, ids = connect.add_vectors(jac_table, vectors)
|
||||
status, ids = connect.add_vectors(jac_collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_vector_by_id(jac_table, ids[0])
|
||||
status, res = connect.get_vector_by_id(jac_collection, ids[0])
|
||||
assert status.OK()
|
||||
assert res == vectors[0]
|
||||
|
||||
def test_get_vector_multi_same_ids(self, connect, jac_table):
|
||||
def test_get_vector_multi_same_ids(self, connect, jac_collection):
|
||||
'''
|
||||
target: test get_vector_by_id
|
||||
method: add vectors, with the same id, get vector by the given id
|
||||
@ -319,56 +319,56 @@ class TestGetBinary:
|
||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||
ids = [i for i in range(nb)]
|
||||
ids[0] = 0; ids[-1] = 0
|
||||
status, ids = connect.add_vectors(jac_table, vectors, ids=ids)
|
||||
status = connect.flush([jac_table])
|
||||
status, ids = connect.add_vectors(jac_collection, vectors, ids=ids)
|
||||
status = connect.flush([jac_collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_vector_by_id(jac_table, 0)
|
||||
status, res = connect.get_vector_by_id(jac_collection, 0)
|
||||
assert status.OK()
|
||||
assert res == vectors[0]
|
||||
|
||||
def test_get_vector_id_not_exised(self, connect, jac_table):
|
||||
def test_get_vector_id_not_exised(self, connect, jac_collection):
|
||||
'''
|
||||
target: test get vector, params vector_id not existed
|
||||
method: add vector and get
|
||||
expected: status ok, empty result
|
||||
'''
|
||||
tmp, vector = gen_binary_vectors(1, dim)
|
||||
status, ids = connect.add_vectors(jac_table, vector)
|
||||
status, ids = connect.add_vectors(jac_collection, vector)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_vector_by_id(jac_table, 1)
|
||||
status, res = connect.get_vector_by_id(jac_collection, 1)
|
||||
assert status.OK()
|
||||
assert not res
|
||||
|
||||
def test_get_vector_table_not_existed(self, connect, jac_table):
|
||||
def test_get_vector_collection_not_existed(self, connect, jac_collection):
|
||||
'''
|
||||
target: test get vector, params table_name not existed
|
||||
target: test get vector, params collection_name not existed
|
||||
method: add vector and get
|
||||
expected: status not ok
|
||||
'''
|
||||
tmp, vector = gen_binary_vectors(1, dim)
|
||||
status, ids = connect.add_vectors(jac_table, vector)
|
||||
status, ids = connect.add_vectors(jac_collection, vector)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
assert status.OK()
|
||||
table_new = gen_unique_str()
|
||||
status, res = connect.get_vector_by_id(table_new, 1)
|
||||
collection_new = gen_unique_str()
|
||||
status, res = connect.get_vector_by_id(collection_new, 1)
|
||||
assert not status.OK()
|
||||
|
||||
def test_get_vector_partition(self, connect, jac_table):
|
||||
def test_get_vector_partition(self, connect, jac_collection):
|
||||
'''
|
||||
target: test get_vector_by_id
|
||||
method: add vector, and get
|
||||
expected: status ok, vector returned
|
||||
'''
|
||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||
status = connect.create_partition(jac_table, tag)
|
||||
status, ids = connect.add_vectors(jac_table, vectors, partition_tag=tag)
|
||||
status = connect.create_partition(jac_collection, tag)
|
||||
status, ids = connect.add_vectors(jac_collection, vectors, partition_tag=tag)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_vector_by_id(jac_table, ids[0])
|
||||
status, res = connect.get_vector_by_id(jac_collection, ids[0])
|
||||
assert status.OK()
|
||||
assert res == vectors[0]
|
||||
|
||||
@ -387,26 +387,26 @@ class TestGetVectorIdIngalid(object):
|
||||
yield request.param
|
||||
|
||||
@pytest.mark.level(2)
|
||||
def test_get_vector_id_invalid(self, connect, table, gen_invalid_id):
|
||||
def test_get_vector_id_invalid(self, connect, collection, gen_invalid_id):
|
||||
invalid_id = gen_invalid_id
|
||||
with pytest.raises(Exception) as e:
|
||||
status = connect.get_vector_by_id(table, invalid_id)
|
||||
status = connect.get_vector_by_id(collection, invalid_id)
|
||||
|
||||
|
||||
class TestTableNameInvalid(object):
|
||||
class TestCollectionNameInvalid(object):
|
||||
"""
|
||||
Test adding vectors with invalid table names
|
||||
Test adding vectors with invalid collection names
|
||||
"""
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=gen_invalid_table_names()
|
||||
params=gen_invalid_collection_names()
|
||||
)
|
||||
def get_table_name(self, request):
|
||||
def get_collection_name(self, request):
|
||||
yield request.param
|
||||
|
||||
@pytest.mark.level(2)
|
||||
def test_get_vectors_with_invalid_table_name(self, connect, get_table_name):
|
||||
table_name = get_table_name
|
||||
def test_get_vectors_with_invalid_collection_name(self, connect, get_collection_name):
|
||||
collection_name = get_collection_name
|
||||
vectors = gen_vectors(1, dim)
|
||||
status, result = connect.get_vector_by_id(table_name, 1)
|
||||
status, result = connect.get_vector_by_id(collection_name, 1)
|
||||
assert not status.OK()
|
||||
|
||||
@ -19,13 +19,13 @@ nb = 6000
|
||||
|
||||
|
||||
class TestGetVectorIdsBase:
|
||||
def get_valid_segment_name(self, connect, table):
|
||||
def get_valid_segment_name(self, connect, collection):
|
||||
vectors = gen_vector(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(table)
|
||||
status, info = connect.collection_info(collection)
|
||||
assert status.OK()
|
||||
logging.getLogger().info(info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
return info.partitions_stat[0].segments_stat[0].segment_name
|
||||
@ -36,88 +36,88 @@ class TestGetVectorIdsBase:
|
||||
******************************************************************
|
||||
"""
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_table_name_None(self, connect, table):
|
||||
def test_get_vector_ids_collection_name_None(self, connect, collection):
|
||||
'''
|
||||
target: get vector ids where table name is None
|
||||
method: call get_vector_ids with the table_name: None
|
||||
target: get vector ids where collection name is None
|
||||
method: call get_vector_ids with the collection_name: None
|
||||
expected: exception raised
|
||||
'''
|
||||
table_name = None
|
||||
segment_name = self.get_valid_segment_name(connect, table)
|
||||
collection_name = None
|
||||
segment_name = self.get_valid_segment_name(connect, collection)
|
||||
with pytest.raises(Exception) as e:
|
||||
status, vector_ids = connect.get_vector_ids(table_name, segment_name)
|
||||
status, vector_ids = connect.get_vector_ids(collection_name, segment_name)
|
||||
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_table_name_not_existed(self, connect, table):
|
||||
def test_get_vector_ids_collection_name_not_existed(self, connect, collection):
|
||||
'''
|
||||
target: get vector ids where table name does not exist
|
||||
method: call get_vector_ids with a random table_name, which is not in db
|
||||
target: get vector ids where collection name does not exist
|
||||
method: call get_vector_ids with a random collection_name, which is not in db
|
||||
expected: status not ok
|
||||
'''
|
||||
table_name = gen_unique_str("not_existed_table")
|
||||
segment_name = self.get_valid_segment_name(connect, table)
|
||||
status, vector_ids = connect.get_vector_ids(table_name, segment_name)
|
||||
collection_name = gen_unique_str("not_existed_collection")
|
||||
segment_name = self.get_valid_segment_name(connect, collection)
|
||||
status, vector_ids = connect.get_vector_ids(collection_name, segment_name)
|
||||
assert not status.OK()
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=gen_invalid_table_names()
|
||||
params=gen_invalid_collection_names()
|
||||
)
|
||||
def get_table_name(self, request):
|
||||
def get_collection_name(self, request):
|
||||
yield request.param
|
||||
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_table_name_invalid(self, connect, table, get_table_name):
|
||||
def test_get_vector_ids_collection_name_invalid(self, connect, collection, get_collection_name):
|
||||
'''
|
||||
target: get vector ids where table name is invalid
|
||||
method: call get_vector_ids with invalid table_name
|
||||
target: get vector ids where collection name is invalid
|
||||
method: call get_vector_ids with invalid collection_name
|
||||
expected: status not ok
|
||||
'''
|
||||
table_name = get_table_name
|
||||
segment_name = self.get_valid_segment_name(connect, table)
|
||||
status, vector_ids = connect.get_vector_ids(table_name, segment_name)
|
||||
collection_name = get_collection_name
|
||||
segment_name = self.get_valid_segment_name(connect, collection)
|
||||
status, vector_ids = connect.get_vector_ids(collection_name, segment_name)
|
||||
assert not status.OK()
|
||||
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_segment_name_None(self, connect, table):
|
||||
def test_get_vector_ids_segment_name_None(self, connect, collection):
|
||||
'''
|
||||
target: get vector ids where segment name is None
|
||||
method: call get_vector_ids with the segment_name: None
|
||||
expected: exception raised
|
||||
'''
|
||||
valid_segment_name = self.get_valid_segment_name(connect, table)
|
||||
valid_segment_name = self.get_valid_segment_name(connect, collection)
|
||||
segment = None
|
||||
with pytest.raises(Exception) as e:
|
||||
status, vector_ids = connect.get_vector_ids(table, segment)
|
||||
status, vector_ids = connect.get_vector_ids(collection, segment)
|
||||
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_segment_name_not_existed(self, connect, table):
|
||||
def test_get_vector_ids_segment_name_not_existed(self, connect, collection):
|
||||
'''
|
||||
target: get vector ids where segment name does not exist
|
||||
method: call get_vector_ids with a random segment name
|
||||
expected: status not ok
|
||||
'''
|
||||
valid_segment_name = self.get_valid_segment_name(connect, table)
|
||||
valid_segment_name = self.get_valid_segment_name(connect, collection)
|
||||
segment = gen_unique_str("not_existed_segment")
|
||||
status, vector_ids = connect.get_vector_ids(table, segment)
|
||||
status, vector_ids = connect.get_vector_ids(collection, segment)
|
||||
logging.getLogger().info(vector_ids)
|
||||
assert not status.OK()
|
||||
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_without_index_A(self, connect, table):
|
||||
def test_get_vector_ids_without_index_A(self, connect, collection):
|
||||
'''
|
||||
target: get vector ids when there is no index
|
||||
method: call get_vector_ids and check if the segment contains vectors
|
||||
expected: status ok
|
||||
'''
|
||||
vectors = gen_vector(10, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(table)
|
||||
status, info = connect.collection_info(collection)
|
||||
assert status.OK()
|
||||
status, vector_ids = connect.get_vector_ids(table, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
status, vector_ids = connect.get_vector_ids(collection, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
# vector_ids should match ids
|
||||
assert len(vector_ids) == 10
|
||||
for i in range(10):
|
||||
@ -125,23 +125,23 @@ class TestGetVectorIdsBase:
|
||||
|
||||
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_without_index_B(self, connect, table):
|
||||
def test_get_vector_ids_without_index_B(self, connect, collection):
|
||||
'''
|
||||
target: get vector ids when there is no index but with partition
|
||||
method: create partition, add vectors to it and call get_vector_ids, check if the segment contains vectors
|
||||
expected: status ok
|
||||
'''
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
assert status.OK()
|
||||
vectors = gen_vector(10, dim)
|
||||
status, ids = connect.add_vectors(table, vectors, partition_tag=tag)
|
||||
status, ids = connect.add_vectors(collection, vectors, partition_tag=tag)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(table)
|
||||
status, info = connect.collection_info(collection)
|
||||
assert status.OK()
|
||||
assert info.partitions_stat[1].tag == tag
|
||||
status, vector_ids = connect.get_vector_ids(table, info.partitions_stat[1].segments_stat[0].segment_name)
|
||||
status, vector_ids = connect.get_vector_ids(collection, info.partitions_stat[1].segments_stat[0].segment_name)
|
||||
# vector_ids should match ids
|
||||
assert len(vector_ids) == 10
|
||||
for i in range(10):
|
||||
@ -160,7 +160,7 @@ class TestGetVectorIdsBase:
|
||||
return request.param
|
||||
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_with_index_A(self, connect, table, get_simple_index):
|
||||
def test_get_vector_ids_with_index_A(self, connect, collection, get_simple_index):
|
||||
'''
|
||||
target: get vector ids when there is index
|
||||
method: call get_vector_ids and check if the segment contains vectors
|
||||
@ -168,65 +168,65 @@ class TestGetVectorIdsBase:
|
||||
'''
|
||||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
status = connect.create_index(table, index_type, index_param)
|
||||
status = connect.create_index(collection, index_type, index_param)
|
||||
assert status.OK()
|
||||
vectors = gen_vector(10, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(table)
|
||||
status, info = connect.collection_info(collection)
|
||||
assert status.OK()
|
||||
status, vector_ids = connect.get_vector_ids(table, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
status, vector_ids = connect.get_vector_ids(collection, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
# vector_ids should match ids
|
||||
assert len(vector_ids) == 10
|
||||
for i in range(10):
|
||||
assert vector_ids[i] == ids[i]
|
||||
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_with_index_B(self, connect, table, get_simple_index):
|
||||
def test_get_vector_ids_with_index_B(self, connect, collection, get_simple_index):
|
||||
'''
|
||||
target: get vector ids when there is index and with partition
|
||||
method: create partition, add vectors to it and call get_vector_ids, check if the segment contains vectors
|
||||
expected: status ok
|
||||
'''
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
assert status.OK()
|
||||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
status = connect.create_index(table, index_type, index_param)
|
||||
status = connect.create_index(collection, index_type, index_param)
|
||||
assert status.OK()
|
||||
vectors = gen_vector(10, dim)
|
||||
status, ids = connect.add_vectors(table, vectors, partition_tag=tag)
|
||||
status, ids = connect.add_vectors(collection, vectors, partition_tag=tag)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(table)
|
||||
status, info = connect.collection_info(collection)
|
||||
assert status.OK()
|
||||
assert info.partitions_stat[1].tag == tag
|
||||
status, vector_ids = connect.get_vector_ids(table, info.partitions_stat[1].segments_stat[0].segment_name)
|
||||
status, vector_ids = connect.get_vector_ids(collection, info.partitions_stat[1].segments_stat[0].segment_name)
|
||||
# vector_ids should match ids
|
||||
assert len(vector_ids) == 10
|
||||
for i in range(10):
|
||||
assert vector_ids[i] == ids[i]
|
||||
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_after_delete_vectors(self, connect, table):
|
||||
def test_get_vector_ids_after_delete_vectors(self, connect, collection):
|
||||
'''
|
||||
target: get vector ids after vectors are deleted
|
||||
method: add vectors and delete a few, call get_vector_ids
|
||||
expected: status ok, vector_ids decreased after vectors deleted
|
||||
'''
|
||||
vectors = gen_vector(2, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
delete_ids = [ids[0]]
|
||||
status = connect.delete_by_id(table, delete_ids)
|
||||
status = connect.flush([table])
|
||||
status = connect.delete_by_id(collection, delete_ids)
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(table)
|
||||
status, info = connect.collection_info(collection)
|
||||
assert status.OK()
|
||||
status, vector_ids = connect.get_vector_ids(table, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
status, vector_ids = connect.get_vector_ids(collection, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
assert len(vector_ids) == 1
|
||||
assert vector_ids[0] == ids[1]
|
||||
|
||||
@ -238,20 +238,20 @@ class TestGetVectorIdsIP:
|
||||
******************************************************************
|
||||
"""
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_without_index_A(self, connect, ip_table):
|
||||
def test_get_vector_ids_without_index_A(self, connect, ip_collection):
|
||||
'''
|
||||
target: get vector ids when there is no index
|
||||
method: call get_vector_ids and check if the segment contains vectors
|
||||
expected: status ok
|
||||
'''
|
||||
vectors = gen_vector(10, dim)
|
||||
status, ids = connect.add_vectors(ip_table, vectors)
|
||||
status, ids = connect.add_vectors(ip_collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([ip_table])
|
||||
status = connect.flush([ip_collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(ip_table)
|
||||
status, info = connect.collection_info(ip_collection)
|
||||
assert status.OK()
|
||||
status, vector_ids = connect.get_vector_ids(ip_table, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
status, vector_ids = connect.get_vector_ids(ip_collection, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
# vector_ids should match ids
|
||||
assert len(vector_ids) == 10
|
||||
for i in range(10):
|
||||
@ -259,23 +259,23 @@ class TestGetVectorIdsIP:
|
||||
|
||||
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_without_index_B(self, connect, ip_table):
|
||||
def test_get_vector_ids_without_index_B(self, connect, ip_collection):
|
||||
'''
|
||||
target: get vector ids when there is no index but with partition
|
||||
method: create partition, add vectors to it and call get_vector_ids, check if the segment contains vectors
|
||||
expected: status ok
|
||||
'''
|
||||
status = connect.create_partition(ip_table, tag)
|
||||
status = connect.create_partition(ip_collection, tag)
|
||||
assert status.OK()
|
||||
vectors = gen_vector(10, dim)
|
||||
status, ids = connect.add_vectors(ip_table, vectors, partition_tag=tag)
|
||||
status, ids = connect.add_vectors(ip_collection, vectors, partition_tag=tag)
|
||||
assert status.OK()
|
||||
status = connect.flush([ip_table])
|
||||
status = connect.flush([ip_collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(ip_table)
|
||||
status, info = connect.collection_info(ip_collection)
|
||||
assert status.OK()
|
||||
assert info.partitions_stat[1].tag == tag
|
||||
status, vector_ids = connect.get_vector_ids(ip_table, info.partitions_stat[1].segments_stat[0].segment_name)
|
||||
status, vector_ids = connect.get_vector_ids(ip_collection, info.partitions_stat[1].segments_stat[0].segment_name)
|
||||
# vector_ids should match ids
|
||||
assert len(vector_ids) == 10
|
||||
for i in range(10):
|
||||
@ -294,7 +294,7 @@ class TestGetVectorIdsIP:
|
||||
return request.param
|
||||
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_with_index_A(self, connect, ip_table, get_simple_index):
|
||||
def test_get_vector_ids_with_index_A(self, connect, ip_collection, get_simple_index):
|
||||
'''
|
||||
target: get vector ids when there is index
|
||||
method: call get_vector_ids and check if the segment contains vectors
|
||||
@ -302,65 +302,65 @@ class TestGetVectorIdsIP:
|
||||
'''
|
||||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
status = connect.create_index(ip_table, index_type, index_param)
|
||||
status = connect.create_index(ip_collection, index_type, index_param)
|
||||
assert status.OK()
|
||||
vectors = gen_vector(10, dim)
|
||||
status, ids = connect.add_vectors(ip_table, vectors)
|
||||
status, ids = connect.add_vectors(ip_collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([ip_table])
|
||||
status = connect.flush([ip_collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(ip_table)
|
||||
status, info = connect.collection_info(ip_collection)
|
||||
assert status.OK()
|
||||
status, vector_ids = connect.get_vector_ids(ip_table, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
status, vector_ids = connect.get_vector_ids(ip_collection, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
# vector_ids should match ids
|
||||
assert len(vector_ids) == 10
|
||||
for i in range(10):
|
||||
assert vector_ids[i] == ids[i]
|
||||
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_with_index_B(self, connect, ip_table, get_simple_index):
|
||||
def test_get_vector_ids_with_index_B(self, connect, ip_collection, get_simple_index):
|
||||
'''
|
||||
target: get vector ids when there is index and with partition
|
||||
method: create partition, add vectors to it and call get_vector_ids, check if the segment contains vectors
|
||||
expected: status ok
|
||||
'''
|
||||
status = connect.create_partition(ip_table, tag)
|
||||
status = connect.create_partition(ip_collection, tag)
|
||||
assert status.OK()
|
||||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
status = connect.create_index(ip_table, index_type, index_param)
|
||||
status = connect.create_index(ip_collection, index_type, index_param)
|
||||
assert status.OK()
|
||||
vectors = gen_vector(10, dim)
|
||||
status, ids = connect.add_vectors(ip_table, vectors, partition_tag=tag)
|
||||
status, ids = connect.add_vectors(ip_collection, vectors, partition_tag=tag)
|
||||
assert status.OK()
|
||||
status = connect.flush([ip_table])
|
||||
status = connect.flush([ip_collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(ip_table)
|
||||
status, info = connect.collection_info(ip_collection)
|
||||
assert status.OK()
|
||||
assert info.partitions_stat[1].tag == tag
|
||||
status, vector_ids = connect.get_vector_ids(ip_table, info.partitions_stat[1].segments_stat[0].segment_name)
|
||||
status, vector_ids = connect.get_vector_ids(ip_collection, info.partitions_stat[1].segments_stat[0].segment_name)
|
||||
# vector_ids should match ids
|
||||
assert len(vector_ids) == 10
|
||||
for i in range(10):
|
||||
assert vector_ids[i] == ids[i]
|
||||
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_after_delete_vectors(self, connect, ip_table):
|
||||
def test_get_vector_ids_after_delete_vectors(self, connect, ip_collection):
|
||||
'''
|
||||
target: get vector ids after vectors are deleted
|
||||
method: add vectors and delete a few, call get_vector_ids
|
||||
expected: status ok, vector_ids decreased after vectors deleted
|
||||
'''
|
||||
vectors = gen_vector(2, dim)
|
||||
status, ids = connect.add_vectors(ip_table, vectors)
|
||||
status, ids = connect.add_vectors(ip_collection, vectors)
|
||||
assert status.OK()
|
||||
delete_ids = [ids[0]]
|
||||
status = connect.delete_by_id(ip_table, delete_ids)
|
||||
status = connect.flush([ip_table])
|
||||
status = connect.delete_by_id(ip_collection, delete_ids)
|
||||
status = connect.flush([ip_collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(ip_table)
|
||||
status, info = connect.collection_info(ip_collection)
|
||||
assert status.OK()
|
||||
status, vector_ids = connect.get_vector_ids(ip_table, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
status, vector_ids = connect.get_vector_ids(ip_collection, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
assert len(vector_ids) == 1
|
||||
assert vector_ids[0] == ids[1]
|
||||
|
||||
@ -372,20 +372,20 @@ class TestGetVectorIdsJAC:
|
||||
******************************************************************
|
||||
"""
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_without_index_A(self, connect, jac_table):
|
||||
def test_get_vector_ids_without_index_A(self, connect, jac_collection):
|
||||
'''
|
||||
target: get vector ids when there is no index
|
||||
method: call get_vector_ids and check if the segment contains vectors
|
||||
expected: status ok
|
||||
'''
|
||||
tmp, vectors = gen_binary_vectors(10, dim)
|
||||
status, ids = connect.add_vectors(jac_table, vectors)
|
||||
status, ids = connect.add_vectors(jac_collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(jac_table)
|
||||
status, info = connect.collection_info(jac_collection)
|
||||
assert status.OK()
|
||||
status, vector_ids = connect.get_vector_ids(jac_table, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
status, vector_ids = connect.get_vector_ids(jac_collection, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
# vector_ids should match ids
|
||||
assert len(vector_ids) == 10
|
||||
for i in range(10):
|
||||
@ -393,23 +393,23 @@ class TestGetVectorIdsJAC:
|
||||
|
||||
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_without_index_B(self, connect, jac_table):
|
||||
def test_get_vector_ids_without_index_B(self, connect, jac_collection):
|
||||
'''
|
||||
target: get vector ids when there is no index but with partition
|
||||
method: create partition, add vectors to it and call get_vector_ids, check if the segment contains vectors
|
||||
expected: status ok
|
||||
'''
|
||||
status = connect.create_partition(jac_table, tag)
|
||||
status = connect.create_partition(jac_collection, tag)
|
||||
assert status.OK()
|
||||
tmp, vectors = gen_binary_vectors(10, dim)
|
||||
status, ids = connect.add_vectors(jac_table, vectors, partition_tag=tag)
|
||||
status, ids = connect.add_vectors(jac_collection, vectors, partition_tag=tag)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(jac_table)
|
||||
status, info = connect.collection_info(jac_collection)
|
||||
assert status.OK()
|
||||
assert info.partitions_stat[1].tag == tag
|
||||
status, vector_ids = connect.get_vector_ids(jac_table, info.partitions_stat[1].segments_stat[0].segment_name)
|
||||
status, vector_ids = connect.get_vector_ids(jac_collection, info.partitions_stat[1].segments_stat[0].segment_name)
|
||||
# vector_ids should match ids
|
||||
assert len(vector_ids) == 10
|
||||
for i in range(10):
|
||||
@ -427,7 +427,7 @@ class TestGetVectorIdsJAC:
|
||||
pytest.skip("Skip index Temporary")
|
||||
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_with_index_A(self, connect, jac_table, get_jaccard_index):
|
||||
def test_get_vector_ids_with_index_A(self, connect, jac_collection, get_jaccard_index):
|
||||
'''
|
||||
target: get vector ids when there is index
|
||||
method: call get_vector_ids and check if the segment contains vectors
|
||||
@ -435,64 +435,64 @@ class TestGetVectorIdsJAC:
|
||||
'''
|
||||
index_param = get_jaccard_index["index_param"]
|
||||
index_type = get_jaccard_index["index_type"]
|
||||
status = connect.create_index(jac_table, index_type, index_param)
|
||||
status = connect.create_index(jac_collection, index_type, index_param)
|
||||
assert status.OK()
|
||||
tmp, vectors = gen_binary_vectors(10, dim)
|
||||
status, ids = connect.add_vectors(jac_table, vectors)
|
||||
status, ids = connect.add_vectors(jac_collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(jac_table)
|
||||
status, info = connect.collection_info(jac_collection)
|
||||
assert status.OK()
|
||||
status, vector_ids = connect.get_vector_ids(jac_table, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
status, vector_ids = connect.get_vector_ids(jac_collection, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
# vector_ids should match ids
|
||||
assert len(vector_ids) == 10
|
||||
for i in range(10):
|
||||
assert vector_ids[i] == ids[i]
|
||||
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_with_index_B(self, connect, jac_table, get_jaccard_index):
|
||||
def test_get_vector_ids_with_index_B(self, connect, jac_collection, get_jaccard_index):
|
||||
'''
|
||||
target: get vector ids when there is index and with partition
|
||||
method: create partition, add vectors to it and call get_vector_ids, check if the segment contains vectors
|
||||
expected: status ok
|
||||
'''
|
||||
status = connect.create_partition(jac_table, tag)
|
||||
status = connect.create_partition(jac_collection, tag)
|
||||
assert status.OK()
|
||||
index_param = get_jaccard_index["index_param"]
|
||||
index_type = get_jaccard_index["index_type"]
|
||||
status = connect.create_index(jac_table, index_type, index_param)
|
||||
status = connect.create_index(jac_collection, index_type, index_param)
|
||||
assert status.OK()
|
||||
tmp, vectors = gen_binary_vectors(10, dim)
|
||||
status, ids = connect.add_vectors(jac_table, vectors, partition_tag=tag)
|
||||
status, ids = connect.add_vectors(jac_collection, vectors, partition_tag=tag)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(jac_table)
|
||||
status, info = connect.collection_info(jac_collection)
|
||||
assert status.OK()
|
||||
assert info.partitions_stat[1].tag == tag
|
||||
status, vector_ids = connect.get_vector_ids(jac_table, info.partitions_stat[1].segments_stat[0].segment_name)
|
||||
status, vector_ids = connect.get_vector_ids(jac_collection, info.partitions_stat[1].segments_stat[0].segment_name)
|
||||
# vector_ids should match ids
|
||||
assert len(vector_ids) == 10
|
||||
for i in range(10):
|
||||
assert vector_ids[i] == ids[i]
|
||||
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_after_delete_vectors(self, connect, jac_table):
|
||||
def test_get_vector_ids_after_delete_vectors(self, connect, jac_collection):
|
||||
'''
|
||||
target: get vector ids after vectors are deleted
|
||||
method: add vectors and delete a few, call get_vector_ids
|
||||
expected: status ok, vector_ids decreased after vectors deleted
|
||||
'''
|
||||
tmp, vectors = gen_binary_vectors(2, dim)
|
||||
status, ids = connect.add_vectors(jac_table, vectors)
|
||||
status, ids = connect.add_vectors(jac_collection, vectors)
|
||||
assert status.OK()
|
||||
delete_ids = [ids[0]]
|
||||
status = connect.delete_by_id(jac_table, delete_ids)
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.delete_by_id(jac_collection, delete_ids)
|
||||
status = connect.flush([jac_collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(jac_table)
|
||||
status, info = connect.collection_info(jac_collection)
|
||||
assert status.OK()
|
||||
status, vector_ids = connect.get_vector_ids(jac_table, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
status, vector_ids = connect.get_vector_ids(jac_collection, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
assert len(vector_ids) == 1
|
||||
assert vector_ids[0] == ids[1]
|
||||
File diff suppressed because it is too large
Load Diff
@ -12,7 +12,7 @@ from utils import *
|
||||
|
||||
dim = 128
|
||||
index_file_size = 10
|
||||
table_id = "test_mix"
|
||||
collection_id = "test_mix"
|
||||
add_interval_time = 2
|
||||
vectors = gen_vectors(10000, dim)
|
||||
vectors = sklearn.preprocessing.normalize(vectors, axis=1, norm='l2')
|
||||
@ -29,35 +29,35 @@ class TestMixBase:
|
||||
# disable
|
||||
def _test_search_during_createIndex(self, args):
|
||||
loops = 10000
|
||||
table = gen_unique_str()
|
||||
collection = gen_unique_str()
|
||||
query_vecs = [vectors[0], vectors[1]]
|
||||
uri = "tcp://%s:%s" % (args["ip"], args["port"])
|
||||
id_0 = 0; id_1 = 0
|
||||
milvus_instance = get_milvus(args["handler"])
|
||||
milvus_instance.connect(uri=uri)
|
||||
milvus_instance.create_table({'table_name': table,
|
||||
milvus_instance.create_collection({'collection_name': collection,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.L2})
|
||||
for i in range(10):
|
||||
status, ids = milvus_instance.add_vectors(table, vectors)
|
||||
status, ids = milvus_instance.add_vectors(collection, vectors)
|
||||
# logging.getLogger().info(ids)
|
||||
if i == 0:
|
||||
id_0 = ids[0]; id_1 = ids[1]
|
||||
def create_index(milvus_instance):
|
||||
logging.getLogger().info("In create index")
|
||||
status = milvus_instance.create_index(table, index_params)
|
||||
status = milvus_instance.create_index(collection, index_params)
|
||||
logging.getLogger().info(status)
|
||||
status, result = milvus_instance.describe_index(table)
|
||||
status, result = milvus_instance.describe_index(collection)
|
||||
logging.getLogger().info(result)
|
||||
def add_vectors(milvus_instance):
|
||||
logging.getLogger().info("In add vectors")
|
||||
status, ids = milvus_instance.add_vectors(table, vectors)
|
||||
status, ids = milvus_instance.add_vectors(collection, vectors)
|
||||
logging.getLogger().info(status)
|
||||
def search(milvus_instance):
|
||||
logging.getLogger().info("In search vectors")
|
||||
for i in range(loops):
|
||||
status, result = milvus_instance.search_vectors(table, top_k, nprobe, query_vecs)
|
||||
status, result = milvus_instance.search_vectors(collection, top_k, nprobe, query_vecs)
|
||||
logging.getLogger().info(status)
|
||||
assert result[0][0].id == id_0
|
||||
assert result[1][0].id == id_1
|
||||
@ -72,82 +72,82 @@ class TestMixBase:
|
||||
p_create.join()
|
||||
|
||||
@pytest.mark.level(2)
|
||||
def test_mix_multi_tables(self, connect):
|
||||
def test_mix_multi_collections(self, connect):
|
||||
'''
|
||||
target: test functions with multiple tables of different metric_types and index_types
|
||||
method: create 60 tables which 30 are L2 and the other are IP, add vectors into them
|
||||
target: test functions with multiple collections of different metric_types and index_types
|
||||
method: create 60 collections which 30 are L2 and the other are IP, add vectors into them
|
||||
and test describe index and search
|
||||
expected: status ok
|
||||
'''
|
||||
nq = 10000
|
||||
table_list = []
|
||||
collection_list = []
|
||||
idx = []
|
||||
index_param = {'nlist': nlist}
|
||||
|
||||
#create table and add vectors
|
||||
#create collection and add vectors
|
||||
for i in range(30):
|
||||
table_name = gen_unique_str('test_mix_multi_tables')
|
||||
table_list.append(table_name)
|
||||
param = {'table_name': table_name,
|
||||
collection_name = gen_unique_str('test_mix_multi_collections')
|
||||
collection_list.append(collection_name)
|
||||
param = {'collection_name': collection_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.L2}
|
||||
connect.create_table(param)
|
||||
status, ids = connect.add_vectors(table_name=table_name, records=vectors)
|
||||
connect.create_collection(param)
|
||||
status, ids = connect.add_vectors(collection_name=collection_name, records=vectors)
|
||||
idx.append(ids[0])
|
||||
idx.append(ids[10])
|
||||
idx.append(ids[20])
|
||||
assert status.OK()
|
||||
for i in range(30):
|
||||
table_name = gen_unique_str('test_mix_multi_tables')
|
||||
table_list.append(table_name)
|
||||
param = {'table_name': table_name,
|
||||
collection_name = gen_unique_str('test_mix_multi_collections')
|
||||
collection_list.append(collection_name)
|
||||
param = {'collection_name': collection_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.IP}
|
||||
connect.create_table(param)
|
||||
status, ids = connect.add_vectors(table_name=table_name, records=vectors)
|
||||
connect.create_collection(param)
|
||||
status, ids = connect.add_vectors(collection_name=collection_name, records=vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table_name])
|
||||
status = connect.flush([collection_name])
|
||||
assert status.OK()
|
||||
idx.append(ids[0])
|
||||
idx.append(ids[10])
|
||||
idx.append(ids[20])
|
||||
assert status.OK()
|
||||
for i in range(10):
|
||||
status = connect.create_index(table_list[i], IndexType.FLAT, index_param)
|
||||
status = connect.create_index(collection_list[i], IndexType.FLAT, index_param)
|
||||
assert status.OK()
|
||||
status = connect.create_index(table_list[30 + i], IndexType.FLAT, index_param)
|
||||
status = connect.create_index(collection_list[30 + i], IndexType.FLAT, index_param)
|
||||
assert status.OK()
|
||||
status = connect.create_index(table_list[10 + i], IndexType.IVFLAT, index_param)
|
||||
status = connect.create_index(collection_list[10 + i], IndexType.IVFLAT, index_param)
|
||||
assert status.OK()
|
||||
status = connect.create_index(table_list[40 + i], IndexType.IVFLAT, index_param)
|
||||
status = connect.create_index(collection_list[40 + i], IndexType.IVFLAT, index_param)
|
||||
assert status.OK()
|
||||
status = connect.create_index(table_list[20 + i], IndexType.IVF_SQ8, index_param)
|
||||
status = connect.create_index(collection_list[20 + i], IndexType.IVF_SQ8, index_param)
|
||||
assert status.OK()
|
||||
status = connect.create_index(table_list[50 + i], IndexType.IVF_SQ8, index_param)
|
||||
status = connect.create_index(collection_list[50 + i], IndexType.IVF_SQ8, index_param)
|
||||
assert status.OK()
|
||||
|
||||
#describe index
|
||||
for i in range(10):
|
||||
status, result = connect.describe_index(table_list[i])
|
||||
status, result = connect.describe_index(collection_list[i])
|
||||
assert result._index_type == IndexType.FLAT
|
||||
status, result = connect.describe_index(table_list[10 + i])
|
||||
status, result = connect.describe_index(collection_list[10 + i])
|
||||
assert result._index_type == IndexType.IVFLAT
|
||||
status, result = connect.describe_index(table_list[20 + i])
|
||||
status, result = connect.describe_index(collection_list[20 + i])
|
||||
assert result._index_type == IndexType.IVF_SQ8
|
||||
status, result = connect.describe_index(table_list[30 + i])
|
||||
status, result = connect.describe_index(collection_list[30 + i])
|
||||
assert result._index_type == IndexType.FLAT
|
||||
status, result = connect.describe_index(table_list[40 + i])
|
||||
status, result = connect.describe_index(collection_list[40 + i])
|
||||
assert result._index_type == IndexType.IVFLAT
|
||||
status, result = connect.describe_index(table_list[50 + i])
|
||||
status, result = connect.describe_index(collection_list[50 + i])
|
||||
assert result._index_type == IndexType.IVF_SQ8
|
||||
|
||||
#search
|
||||
query_vecs = [vectors[0], vectors[10], vectors[20]]
|
||||
for i in range(60):
|
||||
table = table_list[i]
|
||||
status, result = connect.search_vectors(table, top_k, query_records=query_vecs, params={"nprobe": 1})
|
||||
collection = collection_list[i]
|
||||
status, result = connect.search_vectors(collection, top_k, query_records=query_vecs, params={"nprobe": 1})
|
||||
assert status.OK()
|
||||
assert len(result) == len(query_vecs)
|
||||
logging.getLogger().info(i)
|
||||
|
||||
@ -11,7 +11,7 @@ from utils import *
|
||||
|
||||
dim = 128
|
||||
index_file_size = 10
|
||||
table_id = "test_partition"
|
||||
collection_id = "test_partition"
|
||||
ADD_TIMEOUT = 60
|
||||
nprobe = 1
|
||||
tag = "1970-01-01"
|
||||
@ -24,58 +24,58 @@ class TestCreateBase:
|
||||
The following cases are used to test `create_partition` function
|
||||
******************************************************************
|
||||
"""
|
||||
def test_create_partition(self, connect, table):
|
||||
def test_create_partition(self, connect, collection):
|
||||
'''
|
||||
target: test create partition, check status returned
|
||||
method: call function: create_partition
|
||||
expected: status ok
|
||||
'''
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
assert status.OK()
|
||||
|
||||
def test_create_partition_repeat(self, connect, table):
|
||||
def test_create_partition_repeat(self, connect, collection):
|
||||
'''
|
||||
target: test create partition, check status returned
|
||||
method: call function: create_partition
|
||||
expected: status ok
|
||||
'''
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
assert status.OK()
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
assert not status.OK()
|
||||
|
||||
def test_create_partition_table_not_existed(self, connect):
|
||||
def test_create_partition_collection_not_existed(self, connect):
|
||||
'''
|
||||
target: test create partition, its owner table name not existed in db, check status returned
|
||||
target: test create partition, its owner collection name not existed in db, check status returned
|
||||
method: call function: create_partition
|
||||
expected: status not ok
|
||||
'''
|
||||
table_name = gen_unique_str()
|
||||
status = connect.create_partition(table_name, tag)
|
||||
collection_name = gen_unique_str()
|
||||
status = connect.create_partition(collection_name, tag)
|
||||
assert not status.OK()
|
||||
|
||||
def test_create_partition_tag_name_None(self, connect, table):
|
||||
def test_create_partition_tag_name_None(self, connect, collection):
|
||||
'''
|
||||
target: test create partition, tag name set None, check status returned
|
||||
method: call function: create_partition
|
||||
expected: status ok
|
||||
'''
|
||||
tag_name = None
|
||||
status = connect.create_partition(table, tag_name)
|
||||
status = connect.create_partition(collection, tag_name)
|
||||
assert not status.OK()
|
||||
|
||||
def test_create_different_partition_tags(self, connect, table):
|
||||
def test_create_different_partition_tags(self, connect, collection):
|
||||
'''
|
||||
target: test create partition twice with different names
|
||||
method: call function: create_partition, and again
|
||||
expected: status ok
|
||||
'''
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
assert status.OK()
|
||||
tag_name = gen_unique_str()
|
||||
status = connect.create_partition(table, tag_name)
|
||||
status = connect.create_partition(collection, tag_name)
|
||||
assert status.OK()
|
||||
status, res = connect.show_partitions(table)
|
||||
status, res = connect.show_partitions(collection)
|
||||
assert status.OK()
|
||||
tag_list = []
|
||||
for item in res:
|
||||
@ -84,95 +84,95 @@ class TestCreateBase:
|
||||
assert tag_name in tag_list
|
||||
assert "_default" in tag_list
|
||||
|
||||
def test_create_partition_add_vectors_default(self, connect, table):
|
||||
def test_create_partition_add_vectors_default(self, connect, collection):
|
||||
'''
|
||||
target: test create partition, and insert vectors, check status returned
|
||||
method: call function: create_partition
|
||||
expected: status ok
|
||||
'''
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
assert status.OK()
|
||||
nq = 100
|
||||
vectors = gen_vectors(nq, dim)
|
||||
ids = [i for i in range(nq)]
|
||||
status, ids = connect.insert(table, vectors, ids)
|
||||
status, ids = connect.insert(collection, vectors, ids)
|
||||
assert status.OK()
|
||||
|
||||
def test_create_partition_insert_with_tag(self, connect, table):
|
||||
def test_create_partition_insert_with_tag(self, connect, collection):
|
||||
'''
|
||||
target: test create partition, and insert vectors, check status returned
|
||||
method: call function: create_partition
|
||||
expected: status ok
|
||||
'''
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
assert status.OK()
|
||||
nq = 100
|
||||
vectors = gen_vectors(nq, dim)
|
||||
ids = [i for i in range(nq)]
|
||||
status, ids = connect.insert(table, vectors, ids, partition_tag=tag)
|
||||
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag)
|
||||
assert status.OK()
|
||||
|
||||
def test_create_partition_insert_with_tag_not_existed(self, connect, table):
|
||||
def test_create_partition_insert_with_tag_not_existed(self, connect, collection):
|
||||
'''
|
||||
target: test create partition, and insert vectors, check status returned
|
||||
method: call function: create_partition
|
||||
expected: status not ok
|
||||
'''
|
||||
tag_new = "tag_new"
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
assert status.OK()
|
||||
nq = 100
|
||||
vectors = gen_vectors(nq, dim)
|
||||
ids = [i for i in range(nq)]
|
||||
status, ids = connect.insert(table, vectors, ids, partition_tag=tag_new)
|
||||
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag_new)
|
||||
assert not status.OK()
|
||||
|
||||
def test_create_partition_insert_same_tags(self, connect, table):
|
||||
def test_create_partition_insert_same_tags(self, connect, collection):
|
||||
'''
|
||||
target: test create partition, and insert vectors, check status returned
|
||||
method: call function: create_partition
|
||||
expected: status ok
|
||||
'''
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
assert status.OK()
|
||||
nq = 100
|
||||
vectors = gen_vectors(nq, dim)
|
||||
ids = [i for i in range(nq)]
|
||||
status, ids = connect.insert(table, vectors, ids, partition_tag=tag)
|
||||
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag)
|
||||
ids = [(i+100) for i in range(nq)]
|
||||
status, ids = connect.insert(table, vectors, ids, partition_tag=tag)
|
||||
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_table_row_count(table)
|
||||
status, res = connect.count_collection(collection)
|
||||
assert res == nq * 2
|
||||
|
||||
def test_create_partition_insert_same_tags_two_tables(self, connect, table):
|
||||
def test_create_partition_insert_same_tags_two_collections(self, connect, collection):
|
||||
'''
|
||||
target: test create two partitions, and insert vectors with the same tag to each table, check status returned
|
||||
target: test create two partitions, and insert vectors with the same tag to each collection, check status returned
|
||||
method: call function: create_partition
|
||||
expected: status ok, table length is correct
|
||||
expected: status ok, collection length is correct
|
||||
'''
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
assert status.OK()
|
||||
table_new = gen_unique_str()
|
||||
param = {'table_name': table_new,
|
||||
collection_new = gen_unique_str()
|
||||
param = {'collection_name': collection_new,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.L2}
|
||||
status = connect.create_table(param)
|
||||
status = connect.create_partition(table_new, tag)
|
||||
status = connect.create_collection(param)
|
||||
status = connect.create_partition(collection_new, tag)
|
||||
nq = 100
|
||||
vectors = gen_vectors(nq, dim)
|
||||
ids = [i for i in range(nq)]
|
||||
status, ids = connect.insert(table, vectors, ids, partition_tag=tag)
|
||||
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag)
|
||||
ids = [(i+100) for i in range(nq)]
|
||||
status, ids = connect.insert(table_new, vectors, ids, partition_tag=tag)
|
||||
status = connect.flush([table, table_new])
|
||||
status, ids = connect.insert(collection_new, vectors, ids, partition_tag=tag)
|
||||
status = connect.flush([collection, collection_new])
|
||||
assert status.OK()
|
||||
status, res = connect.get_table_row_count(table)
|
||||
status, res = connect.count_collection(collection)
|
||||
assert res == nq
|
||||
status, res = connect.get_table_row_count(table_new)
|
||||
status, res = connect.count_collection(collection_new)
|
||||
assert res == nq
|
||||
|
||||
|
||||
@ -183,35 +183,35 @@ class TestShowBase:
|
||||
The following cases are used to test `show_partitions` function
|
||||
******************************************************************
|
||||
"""
|
||||
def test_show_partitions(self, connect, table):
|
||||
def test_show_partitions(self, connect, collection):
|
||||
'''
|
||||
target: test show partitions, check status and partitions returned
|
||||
method: create partition first, then call function: show_partitions
|
||||
expected: status ok, partition correct
|
||||
'''
|
||||
status = connect.create_partition(table, tag)
|
||||
status, res = connect.show_partitions(table)
|
||||
status = connect.create_partition(collection, tag)
|
||||
status, res = connect.show_partitions(collection)
|
||||
assert status.OK()
|
||||
|
||||
def test_show_partitions_no_partition(self, connect, table):
|
||||
def test_show_partitions_no_partition(self, connect, collection):
|
||||
'''
|
||||
target: test show partitions with table name, check status and partitions returned
|
||||
target: test show partitions with collection name, check status and partitions returned
|
||||
method: call function: show_partitions
|
||||
expected: status ok, partitions correct
|
||||
'''
|
||||
status, res = connect.show_partitions(table)
|
||||
status, res = connect.show_partitions(collection)
|
||||
assert status.OK()
|
||||
|
||||
def test_show_multi_partitions(self, connect, table):
|
||||
def test_show_multi_partitions(self, connect, collection):
|
||||
'''
|
||||
target: test show partitions, check status and partitions returned
|
||||
method: create partitions first, then call function: show_partitions
|
||||
expected: status ok, partitions correct
|
||||
'''
|
||||
tag_new = gen_unique_str()
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(table, tag_new)
|
||||
status, res = connect.show_partitions(table)
|
||||
status = connect.create_partition(collection, tag)
|
||||
status = connect.create_partition(collection, tag_new)
|
||||
status, res = connect.show_partitions(collection)
|
||||
assert status.OK()
|
||||
|
||||
|
||||
@ -222,72 +222,72 @@ class TestDropBase:
|
||||
The following cases are used to test `drop_partition` function
|
||||
******************************************************************
|
||||
"""
|
||||
def test_drop_partition(self, connect, table):
|
||||
def test_drop_partition(self, connect, collection):
|
||||
'''
|
||||
target: test drop partition, check status and partition if existed
|
||||
method: create partitions first, then call function: drop_partition
|
||||
expected: status ok, no partitions in db
|
||||
'''
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.drop_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
status = connect.drop_partition(collection, tag)
|
||||
assert status.OK()
|
||||
status, res = connect.show_partitions(table)
|
||||
status, res = connect.show_partitions(collection)
|
||||
tag_list = []
|
||||
for item in res:
|
||||
tag_list.append(item.tag)
|
||||
assert tag not in tag_list
|
||||
|
||||
def test_drop_partition_tag_not_existed(self, connect, table):
|
||||
def test_drop_partition_tag_not_existed(self, connect, collection):
|
||||
'''
|
||||
target: test drop partition, but tag not existed
|
||||
method: create partitions first, then call function: drop_partition
|
||||
expected: status not ok
|
||||
'''
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
new_tag = "new_tag"
|
||||
status = connect.drop_partition(table, new_tag)
|
||||
status = connect.drop_partition(collection, new_tag)
|
||||
assert not status.OK()
|
||||
|
||||
def test_drop_partition_tag_not_existed_A(self, connect, table):
|
||||
def test_drop_partition_tag_not_existed_A(self, connect, collection):
|
||||
'''
|
||||
target: test drop partition, but table not existed
|
||||
target: test drop partition, but collection not existed
|
||||
method: create partitions first, then call function: drop_partition
|
||||
expected: status not ok
|
||||
'''
|
||||
status = connect.create_partition(table, tag)
|
||||
new_table = gen_unique_str()
|
||||
status = connect.drop_partition(new_table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
new_collection = gen_unique_str()
|
||||
status = connect.drop_partition(new_collection, tag)
|
||||
assert not status.OK()
|
||||
|
||||
def test_drop_partition_repeatedly(self, connect, table):
|
||||
def test_drop_partition_repeatedly(self, connect, collection):
|
||||
'''
|
||||
target: test drop partition twice, check status and partition if existed
|
||||
method: create partitions first, then call function: drop_partition
|
||||
expected: status not ok, no partitions in db
|
||||
'''
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.drop_partition(table, tag)
|
||||
status = connect.drop_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
status = connect.drop_partition(collection, tag)
|
||||
status = connect.drop_partition(collection, tag)
|
||||
time.sleep(2)
|
||||
assert not status.OK()
|
||||
status, res = connect.show_partitions(table)
|
||||
status, res = connect.show_partitions(collection)
|
||||
tag_list = []
|
||||
for item in res:
|
||||
tag_list.append(item.tag)
|
||||
assert tag not in tag_list
|
||||
|
||||
def test_drop_partition_create(self, connect, table):
|
||||
def test_drop_partition_create(self, connect, collection):
|
||||
'''
|
||||
target: test drop partition, and create again, check status
|
||||
method: create partitions first, then call function: drop_partition, create_partition
|
||||
expected: status not ok, partition in db
|
||||
'''
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.drop_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
status = connect.drop_partition(collection, tag)
|
||||
time.sleep(2)
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
assert status.OK()
|
||||
status, res = connect.show_partitions(table)
|
||||
status, res = connect.show_partitions(collection)
|
||||
tag_list = []
|
||||
for item in res:
|
||||
tag_list.append(item.tag)
|
||||
@ -297,47 +297,47 @@ class TestDropBase:
|
||||
class TestNameInvalid(object):
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=gen_invalid_table_names()
|
||||
params=gen_invalid_collection_names()
|
||||
)
|
||||
def get_tag_name(self, request):
|
||||
yield request.param
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=gen_invalid_table_names()
|
||||
params=gen_invalid_collection_names()
|
||||
)
|
||||
def get_table_name(self, request):
|
||||
def get_collection_name(self, request):
|
||||
yield request.param
|
||||
|
||||
def test_drop_partition_with_invalid_table_name(self, connect, table, get_table_name):
|
||||
def test_drop_partition_with_invalid_collection_name(self, connect, collection, get_collection_name):
|
||||
'''
|
||||
target: test drop partition, with invalid table name, check status returned
|
||||
target: test drop partition, with invalid collection name, check status returned
|
||||
method: call function: drop_partition
|
||||
expected: status not ok
|
||||
'''
|
||||
table_name = get_table_name
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.drop_partition(table_name, tag)
|
||||
collection_name = get_collection_name
|
||||
status = connect.create_partition(collection, tag)
|
||||
status = connect.drop_partition(collection_name, tag)
|
||||
assert not status.OK()
|
||||
|
||||
def test_drop_partition_with_invalid_tag_name(self, connect, table, get_tag_name):
|
||||
def test_drop_partition_with_invalid_tag_name(self, connect, collection, get_tag_name):
|
||||
'''
|
||||
target: test drop partition, with invalid tag name, check status returned
|
||||
method: call function: drop_partition
|
||||
expected: status not ok
|
||||
'''
|
||||
tag_name = get_tag_name
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.drop_partition(table, tag_name)
|
||||
status = connect.create_partition(collection, tag)
|
||||
status = connect.drop_partition(collection, tag_name)
|
||||
assert not status.OK()
|
||||
|
||||
def test_show_partitions_with_invalid_table_name(self, connect, table, get_table_name):
|
||||
def test_show_partitions_with_invalid_collection_name(self, connect, collection, get_collection_name):
|
||||
'''
|
||||
target: test show partitions, with invalid table name, check status returned
|
||||
target: test show partitions, with invalid collection name, check status returned
|
||||
method: call function: show_partitions
|
||||
expected: status not ok
|
||||
'''
|
||||
table_name = get_table_name
|
||||
status = connect.create_partition(table, tag)
|
||||
status, res = connect.show_partitions(table_name)
|
||||
collection_name = get_collection_name
|
||||
status = connect.create_partition(collection, tag)
|
||||
status, res = connect.show_partitions(collection_name)
|
||||
assert not status.OK()
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -1,644 +0,0 @@
|
||||
import pdb
|
||||
import pytest
|
||||
import logging
|
||||
import itertools
|
||||
from time import sleep
|
||||
import threading
|
||||
from multiprocessing import Process
|
||||
from milvus import IndexType, MetricType
|
||||
from utils import *
|
||||
|
||||
dim = 128
|
||||
index_file_size = 10
|
||||
add_time_interval = 3
|
||||
tag = "1970-01-01"
|
||||
nb = 6000
|
||||
|
||||
class TestTableCount:
|
||||
"""
|
||||
params means different nb, the nb value may trigger merge, or not
|
||||
"""
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=[
|
||||
1,
|
||||
5000,
|
||||
100000,
|
||||
],
|
||||
)
|
||||
def add_vectors_nb(self, request):
|
||||
yield request.param
|
||||
|
||||
"""
|
||||
generate valid create_index params
|
||||
"""
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=gen_simple_index()
|
||||
)
|
||||
def get_simple_index(self, request, connect):
|
||||
if str(connect._cmd("mode")[1]) == "CPU":
|
||||
if request.param["index_type"] == IndexType.IVF_SQ8H:
|
||||
pytest.skip("sq8h not support in cpu mode")
|
||||
if request.param["index_type"] == IndexType.IVF_PQ:
|
||||
pytest.skip("Skip PQ Temporary")
|
||||
return request.param
|
||||
|
||||
def test_table_rows_count(self, connect, table, add_vectors_nb):
|
||||
'''
|
||||
target: test table rows_count is correct or not
|
||||
method: create table and add vectors in it,
|
||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nb = add_vectors_nb
|
||||
vectors = gen_vectors(nb, dim)
|
||||
res = connect.add_vectors(table_name=table, records=vectors)
|
||||
connect.flush([table])
|
||||
status, res = connect.get_table_row_count(table)
|
||||
assert res == nb
|
||||
|
||||
def test_table_rows_count_partition(self, connect, table, add_vectors_nb):
|
||||
'''
|
||||
target: test table rows_count is correct or not
|
||||
method: create table, create partition and add vectors in it,
|
||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nb = add_vectors_nb
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status = connect.create_partition(table, tag)
|
||||
assert status.OK()
|
||||
res = connect.add_vectors(table_name=table, records=vectors, partition_tag=tag)
|
||||
connect.flush([table])
|
||||
status, res = connect.get_table_row_count(table)
|
||||
assert res == nb
|
||||
|
||||
def test_table_rows_count_multi_partitions_A(self, connect, table, add_vectors_nb):
|
||||
'''
|
||||
target: test table rows_count is correct or not
|
||||
method: create table, create partitions and add vectors in it,
|
||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
new_tag = "new_tag"
|
||||
nb = add_vectors_nb
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(table, new_tag)
|
||||
assert status.OK()
|
||||
res = connect.add_vectors(table_name=table, records=vectors)
|
||||
connect.flush([table])
|
||||
status, res = connect.get_table_row_count(table)
|
||||
assert res == nb
|
||||
|
||||
def test_table_rows_count_multi_partitions_B(self, connect, table, add_vectors_nb):
|
||||
'''
|
||||
target: test table rows_count is correct or not
|
||||
method: create table, create partitions and add vectors in one of the partitions,
|
||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
new_tag = "new_tag"
|
||||
nb = add_vectors_nb
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(table, new_tag)
|
||||
assert status.OK()
|
||||
res = connect.add_vectors(table_name=table, records=vectors, partition_tag=tag)
|
||||
connect.flush([table])
|
||||
status, res = connect.get_table_row_count(table)
|
||||
assert res == nb
|
||||
|
||||
def test_table_rows_count_multi_partitions_C(self, connect, table, add_vectors_nb):
|
||||
'''
|
||||
target: test table rows_count is correct or not
|
||||
method: create table, create partitions and add vectors in one of the partitions,
|
||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
||||
expected: the table count is equal to the length of vectors
|
||||
'''
|
||||
new_tag = "new_tag"
|
||||
nb = add_vectors_nb
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(table, new_tag)
|
||||
assert status.OK()
|
||||
res = connect.add_vectors(table_name=table, records=vectors, partition_tag=tag)
|
||||
res = connect.add_vectors(table_name=table, records=vectors, partition_tag=new_tag)
|
||||
connect.flush([table])
|
||||
status, res = connect.get_table_row_count(table)
|
||||
assert res == nb * 2
|
||||
|
||||
def test_table_rows_count_after_index_created(self, connect, table, get_simple_index):
|
||||
'''
|
||||
target: test get_table_row_count, after index have been created
|
||||
method: add vectors in db, and create index, then calling get_table_row_count with correct params
|
||||
expected: get_table_row_count raise exception
|
||||
'''
|
||||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
nb = 100
|
||||
vectors = gen_vectors(nb, dim)
|
||||
res = connect.add_vectors(table_name=table, records=vectors)
|
||||
connect.flush([table])
|
||||
connect.create_index(table, index_type, index_param)
|
||||
status, res = connect.get_table_row_count(table)
|
||||
assert res == nb
|
||||
|
||||
@pytest.mark.level(2)
|
||||
def test_count_without_connection(self, table, dis_connect):
|
||||
'''
|
||||
target: test get_table_row_count, without connection
|
||||
method: calling get_table_row_count with correct params, with a disconnected instance
|
||||
expected: get_table_row_count raise exception
|
||||
'''
|
||||
with pytest.raises(Exception) as e:
|
||||
status = dis_connect.get_table_row_count(table)
|
||||
|
||||
def test_table_rows_count_no_vectors(self, connect, table):
|
||||
'''
|
||||
target: test table rows_count is correct or not, if table is empty
|
||||
method: create table and no vectors in it,
|
||||
assert the value returned by get_table_row_count method is equal to 0
|
||||
expected: the count is equal to 0
|
||||
'''
|
||||
table_name = gen_unique_str()
|
||||
param = {'table_name': table_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size}
|
||||
connect.create_table(param)
|
||||
status, res = connect.get_table_row_count(table)
|
||||
assert res == 0
|
||||
|
||||
# TODO: enable
|
||||
@pytest.mark.level(2)
|
||||
@pytest.mark.timeout(20)
|
||||
def _test_table_rows_count_multiprocessing(self, connect, table, args):
|
||||
'''
|
||||
target: test table rows_count is correct or not with multiprocess
|
||||
method: create table and add vectors in it,
|
||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nq = 2
|
||||
uri = "tcp://%s:%s" % (args["ip"], args["port"])
|
||||
vectors = gen_vectors(nq, dim)
|
||||
res = connect.add_vectors(table_name=table, records=vectors)
|
||||
time.sleep(add_time_interval)
|
||||
|
||||
def rows_count(milvus):
|
||||
status, res = milvus.get_table_row_count(table)
|
||||
logging.getLogger().info(status)
|
||||
assert res == nq
|
||||
|
||||
process_num = 8
|
||||
processes = []
|
||||
for i in range(process_num):
|
||||
milvus = get_milvus(args["handler"])
|
||||
milvus.connect(uri=uri)
|
||||
p = Process(target=rows_count, args=(milvus, ))
|
||||
processes.append(p)
|
||||
p.start()
|
||||
logging.getLogger().info(p)
|
||||
for p in processes:
|
||||
p.join()
|
||||
|
||||
def test_table_rows_count_multi_tables(self, connect):
|
||||
'''
|
||||
target: test table rows_count is correct or not with multiple tables of L2
|
||||
method: create table and add vectors in it,
|
||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nq = 100
|
||||
vectors = gen_vectors(nq, dim)
|
||||
table_list = []
|
||||
for i in range(20):
|
||||
table_name = gen_unique_str()
|
||||
table_list.append(table_name)
|
||||
param = {'table_name': table_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.L2}
|
||||
connect.create_table(param)
|
||||
res = connect.add_vectors(table_name=table_name, records=vectors)
|
||||
connect.flush(table_list)
|
||||
for i in range(20):
|
||||
status, res = connect.get_table_row_count(table_list[i])
|
||||
assert status.OK()
|
||||
assert res == nq
|
||||
|
||||
|
||||
class TestTableCountIP:
|
||||
"""
|
||||
params means different nb, the nb value may trigger merge, or not
|
||||
"""
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=[
|
||||
1,
|
||||
5000,
|
||||
100000,
|
||||
],
|
||||
)
|
||||
def add_vectors_nb(self, request):
|
||||
yield request.param
|
||||
|
||||
"""
|
||||
generate valid create_index params
|
||||
"""
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=gen_simple_index()
|
||||
)
|
||||
def get_simple_index(self, request, connect):
|
||||
if str(connect._cmd("mode")[1]) == "CPU":
|
||||
if request.param["index_type"] == IndexType.IVF_SQ8H:
|
||||
pytest.skip("sq8h not support in CPU mode")
|
||||
if request.param["index_type"] == IndexType.IVF_PQ:
|
||||
pytest.skip("Skip PQ Temporary")
|
||||
return request.param
|
||||
|
||||
def test_table_rows_count(self, connect, ip_table, add_vectors_nb):
|
||||
'''
|
||||
target: test table rows_count is correct or not
|
||||
method: create table and add vectors in it,
|
||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nb = add_vectors_nb
|
||||
vectors = gen_vectors(nb, dim)
|
||||
res = connect.add_vectors(table_name=ip_table, records=vectors)
|
||||
connect.flush([ip_table])
|
||||
status, res = connect.get_table_row_count(ip_table)
|
||||
assert res == nb
|
||||
|
||||
def test_table_rows_count_after_index_created(self, connect, ip_table, get_simple_index):
|
||||
'''
|
||||
target: test get_table_row_count, after index have been created
|
||||
method: add vectors in db, and create index, then calling get_table_row_count with correct params
|
||||
expected: get_table_row_count raise exception
|
||||
'''
|
||||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
nb = 100
|
||||
vectors = gen_vectors(nb, dim)
|
||||
res = connect.add_vectors(table_name=ip_table, records=vectors)
|
||||
connect.flush([ip_table])
|
||||
connect.create_index(ip_table, index_type, index_param)
|
||||
status, res = connect.get_table_row_count(ip_table)
|
||||
assert res == nb
|
||||
|
||||
@pytest.mark.level(2)
|
||||
def test_count_without_connection(self, ip_table, dis_connect):
|
||||
'''
|
||||
target: test get_table_row_count, without connection
|
||||
method: calling get_table_row_count with correct params, with a disconnected instance
|
||||
expected: get_table_row_count raise exception
|
||||
'''
|
||||
with pytest.raises(Exception) as e:
|
||||
status = dis_connect.get_table_row_count(ip_table)
|
||||
|
||||
def test_table_rows_count_no_vectors(self, connect, ip_table):
|
||||
'''
|
||||
target: test table rows_count is correct or not, if table is empty
|
||||
method: create table and no vectors in it,
|
||||
assert the value returned by get_table_row_count method is equal to 0
|
||||
expected: the count is equal to 0
|
||||
'''
|
||||
table_name = gen_unique_str("test_table")
|
||||
param = {'table_name': table_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size}
|
||||
connect.create_table(param)
|
||||
status, res = connect.get_table_row_count(ip_table)
|
||||
assert res == 0
|
||||
|
||||
# TODO: enable
|
||||
@pytest.mark.timeout(60)
|
||||
def _test_table_rows_count_multiprocessing(self, connect, ip_table, args):
|
||||
'''
|
||||
target: test table rows_count is correct or not with multiprocess
|
||||
method: create table and add vectors in it,
|
||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nq = 2
|
||||
uri = "tcp://%s:%s" % (args["ip"], args["port"])
|
||||
vectors = gen_vectors(nq, dim)
|
||||
res = connect.add_vectors(table_name=ip_table, records=vectors)
|
||||
time.sleep(add_time_interval)
|
||||
|
||||
def rows_count(milvus):
|
||||
status, res = milvus.get_table_row_count(ip_table)
|
||||
logging.getLogger().info(status)
|
||||
assert res == nq
|
||||
|
||||
process_num = 8
|
||||
processes = []
|
||||
for i in range(process_num):
|
||||
milvus = get_milvus(args["handler"])
|
||||
milvus.connect(uri=uri)
|
||||
p = Process(target=rows_count, args=(milvus,))
|
||||
processes.append(p)
|
||||
p.start()
|
||||
logging.getLogger().info(p)
|
||||
for p in processes:
|
||||
p.join()
|
||||
|
||||
def test_table_rows_count_multi_tables(self, connect):
|
||||
'''
|
||||
target: test table rows_count is correct or not with multiple tables of IP
|
||||
method: create table and add vectors in it,
|
||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nq = 100
|
||||
vectors = gen_vectors(nq, dim)
|
||||
table_list = []
|
||||
for i in range(20):
|
||||
table_name = gen_unique_str('test_table_rows_count_multi_tables')
|
||||
table_list.append(table_name)
|
||||
param = {'table_name': table_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.IP}
|
||||
connect.create_table(param)
|
||||
res = connect.add_vectors(table_name=table_name, records=vectors)
|
||||
connect.flush(table_list)
|
||||
for i in range(20):
|
||||
status, res = connect.get_table_row_count(table_list[i])
|
||||
assert status.OK()
|
||||
assert res == nq
|
||||
|
||||
|
||||
class TestTableCountJAC:
|
||||
"""
|
||||
params means different nb, the nb value may trigger merge, or not
|
||||
"""
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=[
|
||||
1,
|
||||
5000,
|
||||
100000,
|
||||
],
|
||||
)
|
||||
def add_vectors_nb(self, request):
|
||||
yield request.param
|
||||
|
||||
"""
|
||||
generate valid create_index params
|
||||
"""
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=gen_simple_index()
|
||||
)
|
||||
def get_jaccard_index(self, request, connect):
|
||||
logging.getLogger().info(request.param)
|
||||
if request.param["index_type"] == IndexType.IVFLAT or request.param["index_type"] == IndexType.FLAT:
|
||||
return request.param
|
||||
else:
|
||||
pytest.skip("Skip index Temporary")
|
||||
|
||||
def test_table_rows_count(self, connect, jac_table, add_vectors_nb):
|
||||
'''
|
||||
target: test table rows_count is correct or not
|
||||
method: create table and add vectors in it,
|
||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nb = add_vectors_nb
|
||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||
res = connect.add_vectors(table_name=jac_table, records=vectors)
|
||||
connect.flush([jac_table])
|
||||
status, res = connect.get_table_row_count(jac_table)
|
||||
assert res == nb
|
||||
|
||||
def test_table_rows_count_after_index_created(self, connect, jac_table, get_jaccard_index):
|
||||
'''
|
||||
target: test get_table_row_count, after index have been created
|
||||
method: add vectors in db, and create index, then calling get_table_row_count with correct params
|
||||
expected: get_table_row_count raise exception
|
||||
'''
|
||||
nb = 100
|
||||
index_param = get_jaccard_index["index_param"]
|
||||
index_type = get_jaccard_index["index_type"]
|
||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||
res = connect.add_vectors(table_name=jac_table, records=vectors)
|
||||
connect.flush([jac_table])
|
||||
connect.create_index(jac_table, index_type, index_param)
|
||||
status, res = connect.get_table_row_count(jac_table)
|
||||
assert res == nb
|
||||
|
||||
@pytest.mark.level(2)
|
||||
def test_count_without_connection(self, jac_table, dis_connect):
|
||||
'''
|
||||
target: test get_table_row_count, without connection
|
||||
method: calling get_table_row_count with correct params, with a disconnected instance
|
||||
expected: get_table_row_count raise exception
|
||||
'''
|
||||
with pytest.raises(Exception) as e:
|
||||
status = dis_connect.get_table_row_count(jac_table)
|
||||
|
||||
def test_table_rows_count_no_vectors(self, connect, jac_table):
|
||||
'''
|
||||
target: test table rows_count is correct or not, if table is empty
|
||||
method: create table and no vectors in it,
|
||||
assert the value returned by get_table_row_count method is equal to 0
|
||||
expected: the count is equal to 0
|
||||
'''
|
||||
table_name = gen_unique_str("test_table")
|
||||
param = {'table_name': table_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size}
|
||||
connect.create_table(param)
|
||||
status, res = connect.get_table_row_count(jac_table)
|
||||
assert res == 0
|
||||
|
||||
def test_table_rows_count_multi_tables(self, connect):
|
||||
'''
|
||||
target: test table rows_count is correct or not with multiple tables of IP
|
||||
method: create table and add vectors in it,
|
||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nq = 100
|
||||
tmp, vectors = gen_binary_vectors(nq, dim)
|
||||
table_list = []
|
||||
for i in range(20):
|
||||
table_name = gen_unique_str('test_table_rows_count_multi_tables')
|
||||
table_list.append(table_name)
|
||||
param = {'table_name': table_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.JACCARD}
|
||||
connect.create_table(param)
|
||||
res = connect.add_vectors(table_name=table_name, records=vectors)
|
||||
connect.flush(table_list)
|
||||
for i in range(20):
|
||||
status, res = connect.get_table_row_count(table_list[i])
|
||||
assert status.OK()
|
||||
assert res == nq
|
||||
|
||||
class TestTableCountHAM:
|
||||
"""
|
||||
params means different nb, the nb value may trigger merge, or not
|
||||
"""
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=[
|
||||
1,
|
||||
5000,
|
||||
100000,
|
||||
],
|
||||
)
|
||||
def add_vectors_nb(self, request):
|
||||
yield request.param
|
||||
|
||||
"""
|
||||
generate valid create_index params
|
||||
"""
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=gen_simple_index()
|
||||
)
|
||||
def get_hamming_index(self, request, connect):
|
||||
logging.getLogger().info(request.param)
|
||||
if request.param["index_type"] == IndexType.IVFLAT or request.param["index_type"] == IndexType.FLAT:
|
||||
return request.param
|
||||
else:
|
||||
pytest.skip("Skip index Temporary")
|
||||
|
||||
def test_table_rows_count(self, connect, ham_table, add_vectors_nb):
|
||||
'''
|
||||
target: test table rows_count is correct or not
|
||||
method: create table and add vectors in it,
|
||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nb = add_vectors_nb
|
||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||
res = connect.add_vectors(table_name=ham_table, records=vectors)
|
||||
connect.flush([ham_table])
|
||||
status, res = connect.get_table_row_count(ham_table)
|
||||
assert res == nb
|
||||
|
||||
def test_table_rows_count_after_index_created(self, connect, ham_table, get_hamming_index):
|
||||
'''
|
||||
target: test get_table_row_count, after index have been created
|
||||
method: add vectors in db, and create index, then calling get_table_row_count with correct params
|
||||
expected: get_table_row_count raise exception
|
||||
'''
|
||||
nb = 100
|
||||
index_type = get_hamming_index["index_type"]
|
||||
index_param = get_hamming_index["index_param"]
|
||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||
res = connect.add_vectors(table_name=ham_table, records=vectors)
|
||||
connect.flush([ham_table])
|
||||
connect.create_index(ham_table, index_type, index_param)
|
||||
status, res = connect.get_table_row_count(ham_table)
|
||||
assert res == nb
|
||||
|
||||
@pytest.mark.level(2)
|
||||
def test_count_without_connection(self, ham_table, dis_connect):
|
||||
'''
|
||||
target: test get_table_row_count, without connection
|
||||
method: calling get_table_row_count with correct params, with a disconnected instance
|
||||
expected: get_table_row_count raise exception
|
||||
'''
|
||||
with pytest.raises(Exception) as e:
|
||||
status = dis_connect.get_table_row_count(ham_table)
|
||||
|
||||
def test_table_rows_count_no_vectors(self, connect, ham_table):
|
||||
'''
|
||||
target: test table rows_count is correct or not, if table is empty
|
||||
method: create table and no vectors in it,
|
||||
assert the value returned by get_table_row_count method is equal to 0
|
||||
expected: the count is equal to 0
|
||||
'''
|
||||
table_name = gen_unique_str("test_table")
|
||||
param = {'table_name': table_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size}
|
||||
connect.create_table(param)
|
||||
status, res = connect.get_table_row_count(ham_table)
|
||||
assert res == 0
|
||||
|
||||
def test_table_rows_count_multi_tables(self, connect):
|
||||
'''
|
||||
target: test table rows_count is correct or not with multiple tables of IP
|
||||
method: create table and add vectors in it,
|
||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nq = 100
|
||||
tmp, vectors = gen_binary_vectors(nq, dim)
|
||||
table_list = []
|
||||
for i in range(20):
|
||||
table_name = gen_unique_str('test_table_rows_count_multi_tables')
|
||||
table_list.append(table_name)
|
||||
param = {'table_name': table_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.HAMMING}
|
||||
connect.create_table(param)
|
||||
res = connect.add_vectors(table_name=table_name, records=vectors)
|
||||
connect.flush(table_list)
|
||||
for i in range(20):
|
||||
status, res = connect.get_table_row_count(table_list[i])
|
||||
assert status.OK()
|
||||
assert res == nq
|
||||
|
||||
|
||||
class TestTableCountTANIMOTO:
|
||||
"""
|
||||
params means different nb, the nb value may trigger merge, or not
|
||||
"""
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=[
|
||||
1,
|
||||
5000,
|
||||
100000,
|
||||
],
|
||||
)
|
||||
def add_vectors_nb(self, request):
|
||||
yield request.param
|
||||
|
||||
"""
|
||||
generate valid create_index params
|
||||
"""
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=gen_simple_index()
|
||||
)
|
||||
def get_tanimoto_index(self, request, connect):
|
||||
logging.getLogger().info(request.param)
|
||||
if request.param["index_type"] == IndexType.IVFLAT or request.param["index_type"] == IndexType.FLAT:
|
||||
return request.param
|
||||
else:
|
||||
pytest.skip("Skip index Temporary")
|
||||
|
||||
def test_table_rows_count(self, connect, tanimoto_table, add_vectors_nb):
|
||||
'''
|
||||
target: test table rows_count is correct or not
|
||||
method: create table and add vectors in it,
|
||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nb = add_vectors_nb
|
||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||
res = connect.add_vectors(table_name=tanimoto_table, records=vectors)
|
||||
connect.flush([tanimoto_table])
|
||||
status, res = connect.get_table_row_count(tanimoto_table)
|
||||
assert status.OK()
|
||||
assert res == nb
|
||||
@ -8,7 +8,7 @@ from milvus import IndexType, MetricType
|
||||
from utils import *
|
||||
|
||||
dim = 128
|
||||
table_id = "test_wal"
|
||||
collection_id = "test_wal"
|
||||
WAL_TIMEOUT = 30
|
||||
nb = 6000
|
||||
add_interval = 1.5
|
||||
@ -21,109 +21,109 @@ class TestWalBase:
|
||||
******************************************************************
|
||||
"""
|
||||
@pytest.mark.timeout(WAL_TIMEOUT)
|
||||
def test_wal_add_vectors(self, connect, table):
|
||||
def test_wal_add_vectors(self, connect, collection):
|
||||
'''
|
||||
target: add vectors in WAL
|
||||
method: add vectors and flush when WAL is enabled
|
||||
expected: status ok, vectors added
|
||||
'''
|
||||
vectors = gen_vector(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_table_row_count(table)
|
||||
status, res = connect.count_collection(collection)
|
||||
assert status.OK()
|
||||
assert res == nb
|
||||
status, res = connect.get_vector_by_id(table, ids[0])
|
||||
status, res = connect.get_vector_by_id(collection, ids[0])
|
||||
logging.getLogger().info(res)
|
||||
assert status.OK()
|
||||
assert_equal_vector(res, vectors[0])
|
||||
|
||||
@pytest.mark.timeout(WAL_TIMEOUT)
|
||||
def test_wal_delete_vectors(self, connect, table):
|
||||
def test_wal_delete_vectors(self, connect, collection):
|
||||
'''
|
||||
target: delete vectors in WAL
|
||||
method: delete vectors and flush when WAL is enabled
|
||||
expected: status ok, vectors deleted
|
||||
'''
|
||||
vectors = gen_vector(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
connect.flush([table])
|
||||
status, res = connect.get_table_row_count(table)
|
||||
connect.flush([collection])
|
||||
status, res = connect.count_collection(collection)
|
||||
assert status.OK()
|
||||
status = connect.delete_by_id(table, ids)
|
||||
status = connect.delete_by_id(collection, ids)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_table_row_count(table)
|
||||
status, res = connect.count_collection(collection)
|
||||
assert status.OK()
|
||||
assert res == 0
|
||||
|
||||
@pytest.mark.timeout(WAL_TIMEOUT)
|
||||
def test_wal_invalid_operation(self, connect, table):
|
||||
def test_wal_invalid_operation(self, connect, collection):
|
||||
'''
|
||||
target: invalid operation in WAL
|
||||
method: add vectors, delete with non-existent ids and flush when WAL is enabled
|
||||
expected: status ok, search with vector have result
|
||||
'''
|
||||
vector = gen_single_vector(dim)
|
||||
status, ids = connect.add_vectors(table, vector)
|
||||
status, ids = connect.add_vectors(collection, vector)
|
||||
assert status.OK()
|
||||
connect.flush([table])
|
||||
status = connect.delete_by_id(table, [0])
|
||||
connect.flush([collection])
|
||||
status = connect.delete_by_id(collection, [0])
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status, res = connect.get_table_row_count(table)
|
||||
status = connect.flush([collection])
|
||||
status, res = connect.count_collection(collection)
|
||||
assert status.OK()
|
||||
assert res == 1
|
||||
|
||||
@pytest.mark.timeout(WAL_TIMEOUT)
|
||||
def test_wal_invalid_operation_B(self, connect, table):
|
||||
def test_wal_invalid_operation_B(self, connect, collection):
|
||||
'''
|
||||
target: invalid operation in WAL
|
||||
method: add vectors, delete with not existed table name when WAL is enabled
|
||||
method: add vectors, delete with not existed collection name when WAL is enabled
|
||||
expected: status not ok
|
||||
'''
|
||||
vectors = gen_vector(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.delete_by_id(table, [0])
|
||||
connect.flush([table])
|
||||
table_new = gen_unique_str()
|
||||
status = connect.delete_by_id(table_new, ids)
|
||||
status = connect.flush([collection])
|
||||
status = connect.delete_by_id(collection, [0])
|
||||
connect.flush([collection])
|
||||
collection_new = gen_unique_str()
|
||||
status = connect.delete_by_id(collection_new, ids)
|
||||
assert not status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_table_row_count(table)
|
||||
status, res = connect.count_collection(collection)
|
||||
assert status.OK()
|
||||
assert res == nb
|
||||
|
||||
@pytest.mark.timeout(WAL_TIMEOUT)
|
||||
def test_wal_server_crashed_recovery(self, connect, table):
|
||||
def test_wal_server_crashed_recovery(self, connect, collection):
|
||||
'''
|
||||
target: test wal when server crashed unexpectedly and restarted
|
||||
method: add vectors, server killed before flush, restarted server and flush
|
||||
expected: status ok, add request is recovered and vectors added
|
||||
'''
|
||||
vector = gen_single_vector(dim)
|
||||
status, ids = connect.add_vectors(table, vector)
|
||||
status, ids = connect.add_vectors(collection, vector)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status, res = connect.get_table_row_count(table)
|
||||
status = connect.flush([collection])
|
||||
status, res = connect.count_collection(collection)
|
||||
assert status.OK()
|
||||
logging.getLogger().info(res) # should be 0 because no auto flush
|
||||
logging.getLogger().info("Stop server and restart")
|
||||
# kill server and restart. auto flush should be set to 15 seconds.
|
||||
# time.sleep(15)
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_table_row_count(table)
|
||||
status, res = connect.count_collection(collection)
|
||||
assert status.OK()
|
||||
assert res == 1
|
||||
status, res = connect.get_vector_by_id(table, ids[0])
|
||||
status, res = connect.get_vector_by_id(collection, ids[0])
|
||||
logging.getLogger().info(res)
|
||||
assert status.OK()
|
||||
assert_equal_vector(res, vector[0])
|
||||
|
||||
@ -146,8 +146,8 @@ def gen_invalid_uris():
|
||||
return uris
|
||||
|
||||
|
||||
def gen_invalid_table_names():
|
||||
table_names = [
|
||||
def gen_invalid_collection_names():
|
||||
collection_names = [
|
||||
"12-s",
|
||||
"12/s",
|
||||
" ",
|
||||
@ -166,7 +166,7 @@ def gen_invalid_table_names():
|
||||
"中文",
|
||||
"a".join("a" for i in range(256))
|
||||
]
|
||||
return table_names
|
||||
return collection_names
|
||||
|
||||
|
||||
def gen_invalid_top_ks():
|
||||
@ -588,7 +588,7 @@ def gen_simple_index():
|
||||
IndexType.IVF_SQ8H,
|
||||
IndexType.IVF_PQ,
|
||||
IndexType.HNSW,
|
||||
# IndexType.RNSG
|
||||
IndexType.RNSG
|
||||
]
|
||||
params = [
|
||||
{"nlist": 1024},
|
||||
@ -597,7 +597,7 @@ def gen_simple_index():
|
||||
{"nlist": 1024},
|
||||
{"nlist": 1024, "m": 16},
|
||||
{"M": 16, "efConstruction": 500},
|
||||
# {"search_length": 100, "out_degree": 40, "candidate_pool_size": 66, "knng": 100}
|
||||
{"search_length": 50, "out_degree": 40, "candidate_pool_size": 100, "knng": 50}
|
||||
]
|
||||
|
||||
index_params = []
|
||||
@ -611,14 +611,14 @@ def get_search_param(index_type):
|
||||
return {"nprobe": 32}
|
||||
elif index_type == IndexType.HNSW:
|
||||
return {"ef": 64}
|
||||
# elif index_type == IndexType.RNSG:
|
||||
# return {"search_length": 100}
|
||||
elif index_type == IndexType.RNSG:
|
||||
return {"search_length": 50}
|
||||
else:
|
||||
logging.getLogger().info("Invalid index_type.")
|
||||
|
||||
|
||||
def assert_has_table(conn, table_name):
|
||||
status, ok = conn.has_table(table_name)
|
||||
def assert_has_collection(conn, collection_name):
|
||||
status, ok = conn.has_collection(collection_name)
|
||||
return status.OK() and ok
|
||||
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user