mirror of
https://gitee.com/milvus-io/milvus.git
synced 2026-01-03 01:12:25 +08:00
* add read/write lock * change compact to ddl queue * add api to get vector data * add flush / merge / compact lock * add api to get vector data * add data size for table info * add db recovery test * add data_size check * change file name to uppercase Signed-off-by: jinhai <hai.jin@zilliz.com> * update wal flush_merge_compact_mutex_ * update wal flush_merge_compact_mutex_ * change requirement * change requirement * upd requirement * add logging * add logging * add logging * add logging * add logging * add logging * add logging * add logging * add logging * delete part * add all size checks * fix bug * update faiss get_vector_by_id * add get_vector case * update get vector by id * update server * fix DBImpl * attempting to fix #1268 * lint * update unit test * fix #1259 * issue 1271 fix wal config * update * fix cases Signed-off-by: del.zhenwu <zhenxiang.li@zilliz.com> * update read / write error message * update read / write error message * [skip ci] get vectors by id from raw files instead faiss * [skip ci] update FilesByType meta * update * fix ci error * update * lint * Hide partition_name parameter * Remove douban pip source Signed-off-by: zhenwu <zw@zilliz.com> * Update epsilon value in test cases Signed-off-by: zhenwu <zw@zilliz.com> * Add default partition * Caiyd crud (#1313) * fix clang format Signed-off-by: yudong.cai <yudong.cai@zilliz.com> * fix unittest build error Signed-off-by: yudong.cai <yudong.cai@zilliz.com> * add faiss_bitset_test Signed-off-by: yudong.cai <yudong.cai@zilliz.com> * avoid user directly operate partition table * fix has table bug * Caiyd crud (#1323) * fix clang format Signed-off-by: yudong.cai <yudong.cai@zilliz.com> * fix unittest build error Signed-off-by: yudong.cai <yudong.cai@zilliz.com> * use compile option -O3 Signed-off-by: yudong.cai <yudong.cai@zilliz.com> * update faiss_bitset_test.cpp Signed-off-by: yudong.cai <yudong.cai@zilliz.com> * change open flags * change OngoingFileChecker to static instance * mark ongoing files when applying deletes * update clean up with ttl * fix centos ci * update * lint * update partition Signed-off-by: zhenwu <zw@zilliz.com> * update delete and flush to include partitions * update * Update cases Signed-off-by: zhenwu <zw@zilliz.com> * Fix test cases crud (#1350) * fix order * add wal case Signed-off-by: sahuang <xiaohaix@student.unimelb.edu.au> * fix wal case Signed-off-by: sahuang <xiaohaix@student.unimelb.edu.au> * fix wal case Signed-off-by: sahuang <xiaohaix@student.unimelb.edu.au> * fix wal case Signed-off-by: sahuang <xiaohaix@student.unimelb.edu.au> * fix invalid operation issue Signed-off-by: sahuang <xiaohaix@student.unimelb.edu.au> * fix invalid operation issue Signed-off-by: sahuang <xiaohaix@student.unimelb.edu.au> * fix bug Signed-off-by: sahuang <xiaohaix@student.unimelb.edu.au> * fix bug Signed-off-by: sahuang <xiaohaix@student.unimelb.edu.au> * crud fix Signed-off-by: sahuang <xiaohaix@student.unimelb.edu.au> * crud fix Signed-off-by: sahuang <xiaohaix@student.unimelb.edu.au> * add table info test cases Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * fix case Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * fix case Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * fix cases Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * fix cases Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * fix cases Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * fix cases Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * fix cases Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> Signed-off-by: JinHai-CN <hai.jin@zilliz.com> * merge cases Signed-off-by: zhenwu <zw@zilliz.com> * Shengjun (#1349) * Add GPU sharing solution on native Kubernetes (#1102) * run hadolint with reviewdog * add LINCENSE in Dockerfile * run hadolint with reviewdog * Reporter of reviewdog command is "github-pr-check" * format Dockerfile * ignore DL3007 in hadolint * clean up old docker images * Add GPU sharing solution on native Kubernetes * nightly test mailer * Fix http server bug (#1096) * refactoring(create_table done) * refactoring * refactor server delivery (insert done) * refactoring server module (count_table done) * server refactor done * cmake pass * refactor server module done. * set grpc response status correctly * format done. * fix redefine ErrorMap() * optimize insert reducing ids data copy * optimize grpc request with reducing data copy * clang format * [skip ci] Refactor server module done. update changlog. prepare for PR * remove explicit and change int32_t to int64_t * add web server * [skip ci] add license in web module * modify header include & comment oatpp environment config * add port configure & create table in handler * modify web url * simple url complation done & add swagger * make sure web url * web functionality done. debuging * add web unittest * web test pass * add web server port * add web server port in template * update unittest cmake file * change web server default port to 19121 * rename method in web module & unittest pass * add search case in unittest for web module * rename some variables * fix bug * unittest pass * web prepare * fix cmd bug(check server status) * update changlog * add web port validate & default set * clang-format pass * add web port test in unittest * add CORS & redirect root to swagger ui * add web status * web table method func cascade test pass * add config url in web module * modify thirdparty cmake to avoid building oatpp test * clang format * update changlog * add constants in web module * reserve Config.cpp * fix constants reference bug * replace web server with async module * modify component to support async * format * developing controller & add test clent into unittest * add web port into demo/server_config * modify thirdparty cmake to allow build test * remove unnecessary comment * add endpoint info in controller * finish web test(bug here) * clang format * add web test cpp to lint exclusions * check null field in GetConfig * add macro RETURN STATUS DTo * fix cmake conflict * fix crash when exit server * remove surplus comments & add http param check * add uri /docs to direct swagger * format * change cmd to system * add default value & unittest in web module * add macros to judge if GPU supported * add macros in unit & add default in index dto & print error message when bind http port fail * format (fix #788) * fix cors bug (not completed) * comment cors * change web framework to simple api * comments optimize * change to simple API * remove comments in controller.hpp * remove EP_COMMON_CMAKE_ARGS in oatpp and oatpp-swagger * add ep cmake args to sqlite * clang-format * change a format * test pass * change name to * fix compiler issue(oatpp-swagger depend on oatpp) * add & in start_server.h * specify lib location with oatpp and oatpp-swagger * add comments * add swagger definition * [skip ci] change http method options status code * remove oatpp swagger(fix #970) * remove comments * check Start web behavior * add default to cpu_cache_capacity * remove swagger component.hpp & /docs url * remove /docs info * remove /docs in unittest * remove space in test rpc * remove repeate info in CHANGLOG * change cache_insert_data default value as a constant * [skip ci] Fix some broken links (#960) * [skip ci] Fix broken link * [skip ci] Fix broken link * [skip ci] Fix broken link * [skip ci] Fix broken links * fix issue 373 (#964) * fix issue 373 * Adjustment format * Adjustment format * Adjustment format * change readme * #966 update NOTICE.md (#967) * remove comments * check Start web behavior * add default to cpu_cache_capacity * remove swagger component.hpp & /docs url * remove /docs info * remove /docs in unittest * remove space in test rpc * remove repeate info in CHANGLOG * change cache_insert_data default value as a constant * adjust web port cofig place * rename web_port variable * change gpu resources invoke way to cmd() * set advanced config name add DEFAULT * change config setting to cmd * modify .. * optimize code * assign TableDto' count default value 0 (fix #995) * check if table exists when show partitions (fix #1028) * check table exists when drop partition (fix #1029) * check if partition name is legal (fix #1022) * modify status code when partition tag is illegal * update changlog * add info to /system url * add binary index and add bin uri & handler method(not completed) * optimize http insert and search time(fix #1066) | add binary vectors support(fix #1067) * fix test partition bug * fix test bug when check insert records * add binary vectors test * add default for offset and page_size * fix uinttest bug * [skip ci] remove comments * optimize web code for PR comments * add new folder named utils * check offset and pagesize (fix #1082) * improve error message if offset or page_size is not legal (fix #1075) * add log into web module * update changlog * check gpu sources setting when assign repeated value (fix #990) * update changlog * clang-format pass * add default handler in http handler * [skip ci] improve error msg when check gpu resources * change check offset way * remove func IsIntStr * add case * change int32 to int64 when check number str * add log in we module(doing) * update test case * add log in web controller Co-authored-by: jielinxu <52057195+jielinxu@users.noreply.github.com> Co-authored-by: JackLCL <53512883+JackLCL@users.noreply.github.com> Co-authored-by: Cai Yudong <yudong.cai@zilliz.com> * Filtering for specific paths in Jenkins CI (#1107) * run hadolint with reviewdog * add LINCENSE in Dockerfile * run hadolint with reviewdog * Reporter of reviewdog command is "github-pr-check" * format Dockerfile * ignore DL3007 in hadolint * clean up old docker images * Add GPU sharing solution on native Kubernetes * nightly test mailer * Filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Fix Filtering for specific paths in Jenkins CI bug (#1109) * run hadolint with reviewdog * add LINCENSE in Dockerfile * run hadolint with reviewdog * Reporter of reviewdog command is "github-pr-check" * format Dockerfile * ignore DL3007 in hadolint * clean up old docker images * Add GPU sharing solution on native Kubernetes * nightly test mailer * Filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Fix Filtering for specific paths in Jenkins CI bug (#1110) * run hadolint with reviewdog * add LINCENSE in Dockerfile * run hadolint with reviewdog * Reporter of reviewdog command is "github-pr-check" * format Dockerfile * ignore DL3007 in hadolint * clean up old docker images * Add GPU sharing solution on native Kubernetes * nightly test mailer * Filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Don't skip ci when triggered by a time (#1113) * run hadolint with reviewdog * add LINCENSE in Dockerfile * run hadolint with reviewdog * Reporter of reviewdog command is "github-pr-check" * format Dockerfile * ignore DL3007 in hadolint * clean up old docker images * Add GPU sharing solution on native Kubernetes * nightly test mailer * Filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Don't skip ci when triggered by a time * Don't skip ci when triggered by a time * Set default sending to Milvus Dev mail group (#1121) * run hadolint with reviewdog * add LINCENSE in Dockerfile * run hadolint with reviewdog * Reporter of reviewdog command is "github-pr-check" * format Dockerfile * ignore DL3007 in hadolint * clean up old docker images * Add GPU sharing solution on native Kubernetes * nightly test mailer * Filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Test filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * Filtering for specific paths in Jenkins CI * No skip ci when triggered by a time * Don't skip ci when triggered by a time * Set default sending to Milvus Dev * Support hnsw (#1131) * add hnsw * add config * format... * format.. * Remove test.template (#1129) * Update framework * remove files * Remove files * Remove ann-acc cases && Update java-sdk cases * change cn to en * [skip ci] remove doc test * [skip ci] change cn to en * Case stability * Add mail notification when test failed * Add main notification * Add main notification * gen milvus instance from utils * Distable case with multiprocess * Add mail notification when nightly test failed * add milvus handler param * add http handler * Remove test.template Co-authored-by: quicksilver <zhifeng.zhang@zilliz.com> * Add doc for the RESTful API / Update contributor number in Milvus readme (#1100) * [skip ci] Update contributor number. * [skip ci] Add RESTful API doc. * [skip ci] Some updates. * [skip ci] Change port to 19121. * [skip ci] Update README.md. Update the descriptions for OPTIONS. * Update README.md Fix a typo. * #1105 update error message when creating IVFSQ8H index without GPU resources (#1117) * [skip ci] Update README (#1104) * remove Nvidia owned files from faiss (#1136) * #1135 remove Nvidia owned files from faiss * Revert "#1135 remove Nvidia owned files from faiss" This reverts commit 3bc007c28c8df5861fdd0452fd64c0e2e719eda2. * #1135 remove Nvidia API implementation * #1135 remove Nvidia owned files from faiss * Update CODE_OF_CONDUCT.md (#1163) * Improve codecov (#1095) * Optimize config test. Dir src/config 99% lines covered * add unittest coverage * optimize cache&config unittest * code format * format * format code * fix merge conflict * cover src/utils unittest * '#831 fix exe_path judge error' * #831 fix exe_path judge error * add some unittest coverage * add some unittest coverage * improve coverage of src/wrapper * improve src/wrapper coverage * *test optimize db/meta unittest * fix bug * *test optimize mysqlMetaImpl unittest * *style: format code * import server& scheduler unittest coverage * handover next work * *test: add some test_meta test case * *format code * *fix: fix typo * feat(codecov): improve code coverage for src/db(#872) * feat(codecov): improve code coverage for src/db/engine(#872) * feat(codecov): improve code coverage(#872) * fix config unittest bug * feat(codecov): improve code coverage core/db/engine(#872) * feat(codecov): improve code coverage core/knowhere * feat(codecov): improve code coverage core/knowhere * feat(codecov): improve code coverage * feat(codecov): fix cpu test some error * feat(codecov): improve code coverage * feat(codecov): rename some fiu * fix(db/meta): fix switch/case default action * feat(codecov): improve code coverage(#872) * fix error caused by merge code * format code * feat(codecov): improve code coverage & format code(#872) * feat(codecov): fix test error(#872) * feat(codecov): fix unittest test_mem(#872) * feat(codecov): fix unittest(#872) * feat(codecov): fix unittest for resource manager(#872) * feat(codecov): code format (#872) * feat(codecov): trigger ci(#872) * fix(RequestScheduler): remove a wrong sleep statement * test(test_rpc): fix rpc test * Fix format issue * Remove unused comments * Fix unit test error Co-authored-by: ABNER-1 <ABNER-1@users.noreply.github.com> Co-authored-by: Jin Hai <hai.jin@zilliz.com> * Support run dev test with http handler in python SDK (#1116) * refactoring(create_table done) * refactoring * refactor server delivery (insert done) * refactoring server module (count_table done) * server refactor done * cmake pass * refactor server module done. * set grpc response status correctly * format done. * fix redefine ErrorMap() * optimize insert reducing ids data copy * optimize grpc request with reducing data copy * clang format * [skip ci] Refactor server module done. update changlog. prepare for PR * remove explicit and change int32_t to int64_t * add web server * [skip ci] add license in web module * modify header include & comment oatpp environment config * add port configure & create table in handler * modify web url * simple url complation done & add swagger * make sure web url * web functionality done. debuging * add web unittest * web test pass * add web server port * add web server port in template * update unittest cmake file * change web server default port to 19121 * rename method in web module & unittest pass * add search case in unittest for web module * rename some variables * fix bug * unittest pass * web prepare * fix cmd bug(check server status) * update changlog * add web port validate & default set * clang-format pass * add web port test in unittest * add CORS & redirect root to swagger ui * add web status * web table method func cascade test pass * add config url in web module * modify thirdparty cmake to avoid building oatpp test * clang format * update changlog * add constants in web module * reserve Config.cpp * fix constants reference bug * replace web server with async module * modify component to support async * format * developing controller & add test clent into unittest * add web port into demo/server_config * modify thirdparty cmake to allow build test * remove unnecessary comment * add endpoint info in controller * finish web test(bug here) * clang format * add web test cpp to lint exclusions * check null field in GetConfig * add macro RETURN STATUS DTo * fix cmake conflict * fix crash when exit server * remove surplus comments & add http param check * add uri /docs to direct swagger * format * change cmd to system * add default value & unittest in web module * add macros to judge if GPU supported * add macros in unit & add default in index dto & print error message when bind http port fail * format (fix #788) * fix cors bug (not completed) * comment cors * change web framework to simple api * comments optimize * change to simple API * remove comments in controller.hpp * remove EP_COMMON_CMAKE_ARGS in oatpp and oatpp-swagger * add ep cmake args to sqlite * clang-format * change a format * test pass * change name to * fix compiler issue(oatpp-swagger depend on oatpp) * add & in start_server.h * specify lib location with oatpp and oatpp-swagger * add comments * add swagger definition * [skip ci] change http method options status code * remove oatpp swagger(fix #970) * remove comments * check Start web behavior * add default to cpu_cache_capacity * remove swagger component.hpp & /docs url * remove /docs info * remove /docs in unittest * remove space in test rpc * remove repeate info in CHANGLOG * change cache_insert_data default value as a constant * [skip ci] Fix some broken links (#960) * [skip ci] Fix broken link * [skip ci] Fix broken link * [skip ci] Fix broken link * [skip ci] Fix broken links * fix issue 373 (#964) * fix issue 373 * Adjustment format * Adjustment format * Adjustment format * change readme * #966 update NOTICE.md (#967) * remove comments * check Start web behavior * add default to cpu_cache_capacity * remove swagger component.hpp & /docs url * remove /docs info * remove /docs in unittest * remove space in test rpc * remove repeate info in CHANGLOG * change cache_insert_data default value as a constant * adjust web port cofig place * rename web_port variable * change gpu resources invoke way to cmd() * set advanced config name add DEFAULT * change config setting to cmd * modify .. * optimize code * assign TableDto' count default value 0 (fix #995) * check if table exists when show partitions (fix #1028) * check table exists when drop partition (fix #1029) * check if partition name is legal (fix #1022) * modify status code when partition tag is illegal * update changlog * add info to /system url * add binary index and add bin uri & handler method(not completed) * optimize http insert and search time(fix #1066) | add binary vectors support(fix #1067) * fix test partition bug * fix test bug when check insert records * add binary vectors test * add default for offset and page_size * fix uinttest bug * [skip ci] remove comments * optimize web code for PR comments * add new folder named utils * check offset and pagesize (fix #1082) * improve error message if offset or page_size is not legal (fix #1075) * add log into web module * update changlog * check gpu sources setting when assign repeated value (fix #990) * update changlog * clang-format pass * add default handler in http handler * [skip ci] improve error msg when check gpu resources * change check offset way * remove func IsIntStr * add case * change int32 to int64 when check number str * add log in we module(doing) * update test case * add log in web controller * remove surplus dot * add preload into /system/ * change get_milvus() to get_milvus(args['handler']) * support load table into memory with http server (fix #1115) * [skip ci] comment surplus dto in VectorDto Co-authored-by: jielinxu <52057195+jielinxu@users.noreply.github.com> Co-authored-by: JackLCL <53512883+JackLCL@users.noreply.github.com> Co-authored-by: Cai Yudong <yudong.cai@zilliz.com> * Fix #1140 (#1162) * fix Signed-off-by: Nicky <nicky.xj.lin@gmail.com> * update... Signed-off-by: Nicky <nicky.xj.lin@gmail.com> * fix2 Signed-off-by: Nicky <nicky.xj.lin@gmail.com> * fix3 Signed-off-by: Nicky <nicky.xj.lin@gmail.com> * update changelog Signed-off-by: Nicky <nicky.xj.lin@gmail.com> * Update INSTALL.md (#1175) * Update INSTALL.md 1. Change image tag and Milvus source code to latest. 2. Fix a typo Signed-off-by: Lu Wang <yamasite@qq.com> * Update INSTALL.md Signed-off-by: lu.wang <yamasite@qq.com> * add Tanimoto ground truth (#1138) * add milvus ground truth * add milvus groundtruth * [skip ci] add milvus ground truth * [skip ci]add tanimoto ground truth * fix mix case bug (#1208) * fix mix case bug Signed-off-by: del.zhenwu <zhenxiang.li@zilliz.com> * Remove case.md Signed-off-by: del.zhenwu <zhenxiang.li@zilliz.com> * Update README.md (#1206) Add LFAI mailing lists. Signed-off-by: Lutkin Wang <yamasite@qq.com> * Add design.md to store links to design docs (#1219) * Update README.md Add link to Milvus design docs Signed-off-by: Lutkin Wang <yamasite@qq.com> * Create design.md Signed-off-by: Lutkin Wang <yamasite@qq.com> * Update design.md Signed-off-by: Lutkin Wang <yamasite@qq.com> * Add troubleshooting info about libmysqlpp.so.3 error (#1225) * Update INSTALL.md Signed-off-by: Lutkin Wang <yamasite@qq.com> * Update INSTALL.md Signed-off-by: Lutkin Wang <yamasite@qq.com> * Update README.md (#1233) Signed-off-by: Lutkin Wang <yamasite@qq.com> * #1240 Update license declaration of each file (#1241) * #1240 Update license declaration of each files Signed-off-by: jinhai <hai.jin@zilliz.com> * #1240 Update CHANGELOG Signed-off-by: jinhai <hai.jin@zilliz.com> * Update README.md (#1258) Add Jenkins master badge. Signed-off-by: Lutkin Wang <yamasite@qq.com> * Update INSTALL.md (#1265) Fix indentation. * support CPU profiling (#1251) * #1250 support CPU profiling Signed-off-by: yudong.cai <yudong.cai@zilliz.com> * #1250 fix code coverage Signed-off-by: yudong.cai <yudong.cai@zilliz.com> * Fix HNSW crash (#1262) * fix Signed-off-by: xiaojun.lin <xiaojun.lin@zilliz.com> * update. Signed-off-by: xiaojun.lin <xiaojun.lin@zilliz.com> * Add troubleshooting information for INSTALL.md and enhance readability (#1274) * Update INSTALL.md 1. Add new troubleshooting message; 2. Enhance readability. Signed-off-by: Lutkin Wang <yamasite@qq.com> * Update INSTALL.md Signed-off-by: Lutkin Wang <yamasite@qq.com> * Update INSTALL.md Signed-off-by: Lutkin Wang <yamasite@qq.com> * Update INSTALL.md Add CentOS link. Signed-off-by: Lutkin Wang <yamasite@qq.com> * Create COMMUNITY.md (#1292) Signed-off-by: Lutkin Wang <yamasite@qq.com> * fix gtest * add copyright * fix gtest * MERGE_NOT_YET * fix lint Co-authored-by: quicksilver <zhifeng.zhang@zilliz.com> Co-authored-by: BossZou <40255591+BossZou@users.noreply.github.com> Co-authored-by: jielinxu <52057195+jielinxu@users.noreply.github.com> Co-authored-by: JackLCL <53512883+JackLCL@users.noreply.github.com> Co-authored-by: Cai Yudong <yudong.cai@zilliz.com> Co-authored-by: Tinkerrr <linxiaojun.cn@outlook.com> Co-authored-by: del-zhenwu <56623710+del-zhenwu@users.noreply.github.com> Co-authored-by: Lutkin Wang <yamasite@qq.com> Co-authored-by: shengjh <46514371+shengjh@users.noreply.github.com> Co-authored-by: ABNER-1 <ABNER-1@users.noreply.github.com> Co-authored-by: Jin Hai <hai.jin@zilliz.com> Co-authored-by: shiyu22 <cshiyu22@gmail.com> * #1302 Get all record IDs in a segment by given a segment id * Remove query time ranges Signed-off-by: zhenwu <zw@zilliz.com> * #1295 let wal enable by default * fix cases Signed-off-by: zhenwu <zw@zilliz.com> * fix partition cases Signed-off-by: zhenwu <zw@zilliz.com> * [skip ci] update test_db * update * fix case bug Signed-off-by: zhenwu <zw@zilliz.com> * lint * fix test case failures * remove some code * Caiyd crud 1 (#1377) * fix clang format Signed-off-by: yudong.cai <yudong.cai@zilliz.com> * fix unittest build error Signed-off-by: yudong.cai <yudong.cai@zilliz.com> * fix build issue when enable profiling Signed-off-by: yudong.cai <yudong.cai@zilliz.com> * fix hastable bug * update bloom filter * update * benchmark * update benchmark * update * update * remove wal record size Signed-off-by: shengjun.li <shengjun.li@zilliz.com> * remove wal record size config Signed-off-by: shengjun.li <shengjun.li@zilliz.com> * update apply deletes: switch to binary search * update sdk_simple Signed-off-by: yudong.cai <yudong.cai@zilliz.com> * update apply deletes: switch to binary search * add test_search_by_id Signed-off-by: zhenwu <zw@zilliz.com> * add more log * flush error with multi same ids Signed-off-by: zhenwu <zw@zilliz.com> * modify wal config Signed-off-by: shengjun.li <shengjun.li@zilliz.com> * update * add binary search_by_id * fix case bug Signed-off-by: zhenwu <zw@zilliz.com> * update cases Signed-off-by: zhenwu <zw@zilliz.com> * fix unit test #1395 * improve merge performance * add uids_ for VectorIndex to improve search performance Signed-off-by: yudong.cai <yudong.cai@zilliz.com> * fix error Signed-off-by: yudong.cai <yudong.cai@zilliz.com> * update * fix search * fix record num Signed-off-by: shengjun.li <shengjun.li@zilliz.com> * refine code * refine code * Add get_vector_ids test cases (#1407) * fix order * add wal case Signed-off-by: sahuang <xiaohaix@student.unimelb.edu.au> * fix wal case Signed-off-by: sahuang <xiaohaix@student.unimelb.edu.au> * fix wal case Signed-off-by: sahuang <xiaohaix@student.unimelb.edu.au> * fix wal case Signed-off-by: sahuang <xiaohaix@student.unimelb.edu.au> * fix invalid operation issue Signed-off-by: sahuang <xiaohaix@student.unimelb.edu.au> * fix invalid operation issue Signed-off-by: sahuang <xiaohaix@student.unimelb.edu.au> * fix bug Signed-off-by: sahuang <xiaohaix@student.unimelb.edu.au> * fix bug Signed-off-by: sahuang <xiaohaix@student.unimelb.edu.au> * crud fix Signed-off-by: sahuang <xiaohaix@student.unimelb.edu.au> * crud fix Signed-off-by: sahuang <xiaohaix@student.unimelb.edu.au> * add table info test cases Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * fix case Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * fix case Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * fix cases Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * fix cases Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * fix cases Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * fix cases Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * fix cases Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> Signed-off-by: JinHai-CN <hai.jin@zilliz.com> * add to compact case Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * add to compact case Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * add to compact case Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * fix case Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * add case and debug compact Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * test pdb Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * test pdb Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * test pdb Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * fix cases Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * update table_info case Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * update table_info case Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * update table_info case Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * update get vector ids case Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * update get vector ids case Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * update get vector ids case Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * update get vector ids case Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * update case Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * update case Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * update case Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * update case Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * update case Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * pdb test Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * pdb test Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * fix case Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * add tests for get_vector_ids Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * fix case Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * add binary and ip Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * fix binary index Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * fix pdb Signed-off-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> * #1408 fix search result in-correct after DeleteById Signed-off-by: yudong.cai <yudong.cai@zilliz.com> * add one case * delete failed segment * update serialize * update serialize * fix case Signed-off-by: zhenwu <zw@zilliz.com> * update * update case assertion Signed-off-by: zhenwu <zw@zilliz.com> * [skip ci] update config * change bloom filter msync flag to async * #1319 add more timing debug info Signed-off-by: yudong.cai <yudong.cai@zilliz.com> * update * update * add normalize Signed-off-by: zhenwu <zw@zilliz.com> * add normalize Signed-off-by: zhenwu <zw@zilliz.com> * add normalize Signed-off-by: zhenwu <zw@zilliz.com> * Fix compiling error Signed-off-by: jinhai <hai.jin@zilliz.com> * support ip (#1383) * support ip Signed-off-by: xiaojun.lin <xiaojun.lin@zilliz.com> * IP result distance sort by descend Signed-off-by: Nicky <nicky.xj.lin@gmail.com> * update Signed-off-by: Nicky <nicky.xj.lin@gmail.com> * format Signed-off-by: xiaojun.lin <xiaojun.lin@zilliz.com> * get table lsn * Remove unused third party Signed-off-by: jinhai <hai.jin@zilliz.com> * Refine code Signed-off-by: jinhai <hai.jin@zilliz.com> * #1319 fix clang format Signed-off-by: yudong.cai <yudong.cai@zilliz.com> * fix wal applied lsn Signed-off-by: shengjun.li <shengjun.li@zilliz.com> * validate partition tag * #1319 improve search performance Signed-off-by: yudong.cai <yudong.cai@zilliz.com> * build error Co-authored-by: Zhiru Zhu <youny626@hotmail.com> Co-authored-by: groot <yihua.mo@zilliz.com> Co-authored-by: Xiaohai Xu <xiaohaix@student.unimelb.edu.au> Co-authored-by: shengjh <46514371+shengjh@users.noreply.github.com> Co-authored-by: del-zhenwu <56623710+del-zhenwu@users.noreply.github.com> Co-authored-by: shengjun.li <49774184+shengjun1985@users.noreply.github.com> Co-authored-by: Cai Yudong <yudong.cai@zilliz.com> Co-authored-by: quicksilver <zhifeng.zhang@zilliz.com> Co-authored-by: BossZou <40255591+BossZou@users.noreply.github.com> Co-authored-by: jielinxu <52057195+jielinxu@users.noreply.github.com> Co-authored-by: JackLCL <53512883+JackLCL@users.noreply.github.com> Co-authored-by: Tinkerrr <linxiaojun.cn@outlook.com> Co-authored-by: Lutkin Wang <yamasite@qq.com> Co-authored-by: ABNER-1 <ABNER-1@users.noreply.github.com> Co-authored-by: shiyu22 <cshiyu22@gmail.com>
583 lines
23 KiB
C++
583 lines
23 KiB
C++
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
|
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
|
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
|
|
|
#include "db/engine/ExecutionEngine.h"
|
|
#include "utils/BlockingQueue.h"
|
|
#include "utils/CommonUtil.h"
|
|
#include "utils/Error.h"
|
|
#include "utils/LogUtil.h"
|
|
#include "utils/SignalUtil.h"
|
|
#include "utils/StringHelpFunctions.h"
|
|
#include "utils/TimeRecorder.h"
|
|
#include "utils/ValidationUtil.h"
|
|
#include "utils/ThreadPool.h"
|
|
|
|
#include <gtest/gtest.h>
|
|
#include <sys/stat.h>
|
|
#include <sys/types.h>
|
|
#include <boost/filesystem.hpp>
|
|
#include <thread>
|
|
#include <src/utils/Exception.h>
|
|
|
|
#include <fiu-local.h>
|
|
#include <fiu-control.h>
|
|
|
|
namespace {
|
|
|
|
static const char* LOG_FILE_PATH = "./milvus/conf/log_config.conf";
|
|
|
|
void
|
|
CopyStatus(milvus::Status& st1, milvus::Status& st2) {
|
|
st1 = st2;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
TEST(UtilTest, EXCEPTION_TEST) {
|
|
std::string err_msg = "failed";
|
|
milvus::server::ServerException ex(milvus::SERVER_UNEXPECTED_ERROR, err_msg);
|
|
ASSERT_EQ(ex.error_code(), milvus::SERVER_UNEXPECTED_ERROR);
|
|
std::string msg = ex.what();
|
|
ASSERT_EQ(msg, err_msg);
|
|
|
|
milvus::Exception ex1(milvus::SERVER_UNEXPECTED_ERROR, err_msg);
|
|
ASSERT_EQ(ex1.code(), milvus::SERVER_UNEXPECTED_ERROR);
|
|
msg = ex1.what();
|
|
ASSERT_EQ(msg, err_msg);
|
|
|
|
std::string empty_err_msg;
|
|
milvus::Exception empty_ex(milvus::SERVER_UNEXPECTED_ERROR, empty_err_msg);
|
|
ASSERT_EQ(empty_ex.code(), milvus::SERVER_UNEXPECTED_ERROR);
|
|
msg = empty_ex.what();
|
|
ASSERT_NE(msg, empty_err_msg);
|
|
}
|
|
|
|
TEST(UtilTest, SIGNAL_TEST) {
|
|
milvus::server::SignalUtil::PrintStacktrace();
|
|
}
|
|
|
|
TEST(UtilTest, COMMON_TEST) {
|
|
uint64_t total_mem = 0, free_mem = 0;
|
|
milvus::server::CommonUtil::GetSystemMemInfo(total_mem, free_mem);
|
|
ASSERT_GT(total_mem, 0);
|
|
ASSERT_GT(free_mem, 0);
|
|
|
|
int64_t thread_cnt = 0;
|
|
milvus::server::CommonUtil::GetSystemAvailableThreads(thread_cnt);
|
|
ASSERT_GT(thread_cnt, 0);
|
|
|
|
fiu_init(0);
|
|
fiu_enable("CommonUtil.GetSystemAvailableThreads.zero_thread", 1, NULL, 0);
|
|
milvus::server::CommonUtil::GetSystemAvailableThreads(thread_cnt);
|
|
ASSERT_GT(thread_cnt, 0);
|
|
fiu_disable("CommonUtil.GetSystemAvailableThreads.zero_thread");
|
|
|
|
std::string empty_path = "";
|
|
std::string path1 = "/tmp/milvus_test/";
|
|
std::string path2 = path1 + "common_test_12345/";
|
|
std::string path3 = path2 + "abcdef";
|
|
milvus::Status status = milvus::server::CommonUtil::CreateDirectory(path3);
|
|
ASSERT_TRUE(status.ok());
|
|
|
|
status = milvus::server::CommonUtil::CreateDirectory(empty_path);
|
|
ASSERT_TRUE(status.ok());
|
|
|
|
// test again
|
|
status = milvus::server::CommonUtil::CreateDirectory(path3);
|
|
ASSERT_TRUE(status.ok());
|
|
|
|
ASSERT_TRUE(milvus::server::CommonUtil::IsDirectoryExist(path3));
|
|
|
|
status = milvus::server::CommonUtil::DeleteDirectory(empty_path);
|
|
ASSERT_TRUE(status.ok());
|
|
|
|
status = milvus::server::CommonUtil::DeleteDirectory(path1);
|
|
ASSERT_TRUE(status.ok());
|
|
// test again
|
|
status = milvus::server::CommonUtil::DeleteDirectory(path1);
|
|
ASSERT_TRUE(status.ok());
|
|
|
|
ASSERT_FALSE(milvus::server::CommonUtil::IsDirectoryExist(path1));
|
|
ASSERT_FALSE(milvus::server::CommonUtil::IsFileExist(path1));
|
|
|
|
std::string exe_path = milvus::server::CommonUtil::GetExePath();
|
|
ASSERT_FALSE(exe_path.empty());
|
|
|
|
fiu_enable("CommonUtil.GetExePath.readlink_fail", 1, NULL, 0);
|
|
exe_path = milvus::server::CommonUtil::GetExePath();
|
|
ASSERT_FALSE(!exe_path.empty());
|
|
fiu_disable("CommonUtil.GetExePath.readlink_fail");
|
|
|
|
fiu_enable("CommonUtil.GetExePath.exe_path_error", 1, NULL, 0);
|
|
exe_path = milvus::server::CommonUtil::GetExePath();
|
|
ASSERT_FALSE(exe_path.empty());
|
|
fiu_disable("CommonUtil.GetExePath.exe_path_error");
|
|
|
|
fiu_enable("CommonUtil.CreateDirectory.create_parent_fail", 1, NULL, 0);
|
|
status = milvus::server::CommonUtil::CreateDirectory(path3);
|
|
ASSERT_FALSE(status.ok());
|
|
fiu_disable("CommonUtil.CreateDirectory.create_parent_fail");
|
|
|
|
fiu_enable("CommonUtil.CreateDirectory.create_dir_fail", 1, NULL, 0);
|
|
status = milvus::server::CommonUtil::CreateDirectory(path3);
|
|
ASSERT_FALSE(status.ok());
|
|
fiu_disable("CommonUtil.CreateDirectory.create_dir_fail");
|
|
|
|
time_t tt;
|
|
time(&tt);
|
|
tm time_struct;
|
|
memset(&time_struct, 0, sizeof(tm));
|
|
milvus::server::CommonUtil::ConvertTime(tt, time_struct);
|
|
ASSERT_GE(time_struct.tm_year, 0);
|
|
ASSERT_GE(time_struct.tm_mon, 0);
|
|
ASSERT_GE(time_struct.tm_mday, 0);
|
|
milvus::server::CommonUtil::ConvertTime(time_struct, tt);
|
|
ASSERT_GT(tt, 0);
|
|
|
|
bool res = milvus::server::CommonUtil::TimeStrToTime("2019-03-23", tt, time_struct);
|
|
ASSERT_EQ(time_struct.tm_year, 119);
|
|
ASSERT_EQ(time_struct.tm_mon, 2);
|
|
ASSERT_EQ(time_struct.tm_mday, 23);
|
|
ASSERT_GT(tt, 0);
|
|
ASSERT_TRUE(res);
|
|
}
|
|
|
|
TEST(UtilTest, STRINGFUNCTIONS_TEST) {
|
|
std::string str = " test str";
|
|
milvus::server::StringHelpFunctions::TrimStringBlank(str);
|
|
ASSERT_EQ(str, "test str");
|
|
|
|
str = "\"test str\"";
|
|
milvus::server::StringHelpFunctions::TrimStringQuote(str, "\"");
|
|
ASSERT_EQ(str, "test str");
|
|
|
|
str = "a,b,c";
|
|
std::vector<std::string> result;
|
|
milvus::server::StringHelpFunctions::SplitStringByDelimeter(str, ",", result);
|
|
ASSERT_EQ(result.size(), 3UL);
|
|
|
|
std::string merge_str;
|
|
milvus::server::StringHelpFunctions::MergeStringWithDelimeter(result, ",", merge_str);
|
|
ASSERT_EQ(merge_str, "a,b,c");
|
|
result.clear();
|
|
milvus::server::StringHelpFunctions::MergeStringWithDelimeter(result, ",", merge_str);
|
|
ASSERT_TRUE(merge_str.empty());
|
|
|
|
auto status = milvus::server::StringHelpFunctions::SplitStringByQuote(str, ",", "\"", result);
|
|
ASSERT_TRUE(status.ok());
|
|
ASSERT_EQ(result.size(), 3UL);
|
|
|
|
result.clear();
|
|
status = milvus::server::StringHelpFunctions::SplitStringByQuote(str, ",", "", result);
|
|
ASSERT_TRUE(status.ok());
|
|
ASSERT_EQ(result.size(), 3UL);
|
|
|
|
str = "55,\"aa,gg,yy\",b";
|
|
result.clear();
|
|
status = milvus::server::StringHelpFunctions::SplitStringByQuote(str, ",", "\"", result);
|
|
ASSERT_TRUE(status.ok());
|
|
ASSERT_EQ(result.size(), 3UL);
|
|
|
|
fiu_init(0);
|
|
fiu_enable("StringHelpFunctions.SplitStringByQuote.invalid_index", 1, NULL, 0);
|
|
result.clear();
|
|
status = milvus::server::StringHelpFunctions::SplitStringByQuote(str, ",", "\"", result);
|
|
ASSERT_FALSE(status.ok());
|
|
fiu_disable("StringHelpFunctions.SplitStringByQuote.invalid_index");
|
|
|
|
fiu_enable("StringHelpFunctions.SplitStringByQuote.index_gt_last", 1, NULL, 0);
|
|
result.clear();
|
|
status = milvus::server::StringHelpFunctions::SplitStringByQuote(str, ",", "\"", result);
|
|
ASSERT_TRUE(status.ok());
|
|
fiu_disable("StringHelpFunctions.SplitStringByQuote.index_gt_last");
|
|
|
|
fiu_enable("StringHelpFunctions.SplitStringByQuote.invalid_index2", 1, NULL, 0);
|
|
result.clear();
|
|
status = milvus::server::StringHelpFunctions::SplitStringByQuote(str, ",", "\"", result);
|
|
ASSERT_FALSE(status.ok());
|
|
fiu_disable("StringHelpFunctions.SplitStringByQuote.invalid_index2");
|
|
|
|
fiu_enable("StringHelpFunctions.SplitStringByQuote.last_is_end", 1, NULL, 0);
|
|
result.clear();
|
|
status = milvus::server::StringHelpFunctions::SplitStringByQuote(str, ",", "\"", result);
|
|
ASSERT_TRUE(status.ok());
|
|
fiu_disable("StringHelpFunctions.SplitStringByQuote.last_is_end2");
|
|
|
|
ASSERT_TRUE(milvus::server::StringHelpFunctions::IsRegexMatch("abc", "abc"));
|
|
ASSERT_TRUE(milvus::server::StringHelpFunctions::IsRegexMatch("a8c", "a\\d."));
|
|
ASSERT_FALSE(milvus::server::StringHelpFunctions::IsRegexMatch("abc", "a\\dc"));
|
|
}
|
|
|
|
TEST(UtilTest, BLOCKINGQUEUE_TEST) {
|
|
milvus::server::BlockingQueue<std::string> bq;
|
|
|
|
static const size_t count = 10;
|
|
bq.SetCapacity(count);
|
|
|
|
for (size_t i = 1; i <= count; i++) {
|
|
std::string id = "No." + std::to_string(i);
|
|
bq.Put(id);
|
|
}
|
|
|
|
ASSERT_EQ(bq.Size(), count);
|
|
ASSERT_FALSE(bq.Empty());
|
|
|
|
std::string str = bq.Front();
|
|
ASSERT_EQ(str, "No.1");
|
|
|
|
str = bq.Back();
|
|
ASSERT_EQ(str, "No." + std::to_string(count));
|
|
|
|
for (size_t i = 1; i <= count; i++) {
|
|
std::string id = "No." + std::to_string(i);
|
|
str = bq.Take();
|
|
ASSERT_EQ(id, str);
|
|
}
|
|
|
|
ASSERT_EQ(bq.Size(), 0);
|
|
}
|
|
|
|
TEST(UtilTest, LOG_TEST) {
|
|
auto status = milvus::server::InitLog(LOG_FILE_PATH);
|
|
ASSERT_TRUE(status.ok());
|
|
|
|
EXPECT_FALSE(el::Loggers::hasFlag(el::LoggingFlag::NewLineForContainer));
|
|
EXPECT_FALSE(el::Loggers::hasFlag(el::LoggingFlag::LogDetailedCrashReason));
|
|
|
|
std::string fname = milvus::server::CommonUtil::GetFileName(LOG_FILE_PATH);
|
|
ASSERT_EQ(fname, "log_config.conf");
|
|
}
|
|
|
|
TEST(UtilTest, TIMERECORDER_TEST) {
|
|
for (int64_t log_level = 0; log_level <= 6; log_level++) {
|
|
if (log_level == 5) {
|
|
continue; // skip fatal
|
|
}
|
|
milvus::TimeRecorder rc("time", log_level);
|
|
rc.RecordSection("end");
|
|
}
|
|
}
|
|
|
|
TEST(UtilTest, TIMERECOREDRAUTO_TEST) {
|
|
milvus::TimeRecorderAuto rc("time");
|
|
rc.RecordSection("end");
|
|
}
|
|
|
|
TEST(UtilTest, STATUS_TEST) {
|
|
auto status = milvus::Status::OK();
|
|
std::string str = status.ToString();
|
|
ASSERT_FALSE(str.empty());
|
|
|
|
status = milvus::Status(milvus::DB_SUCCESS, "success");
|
|
ASSERT_EQ(status.code(), milvus::DB_SUCCESS);
|
|
str = status.ToString();
|
|
ASSERT_FALSE(str.empty());
|
|
|
|
status = milvus::Status(milvus::DB_ERROR, "mistake");
|
|
ASSERT_EQ(status.code(), milvus::DB_ERROR);
|
|
str = status.ToString();
|
|
ASSERT_FALSE(str.empty());
|
|
|
|
status = milvus::Status(milvus::DB_NOT_FOUND, "mistake");
|
|
ASSERT_EQ(status.code(), milvus::DB_NOT_FOUND);
|
|
str = status.ToString();
|
|
ASSERT_FALSE(str.empty());
|
|
|
|
status = milvus::Status(milvus::DB_ALREADY_EXIST, "mistake");
|
|
ASSERT_EQ(status.code(), milvus::DB_ALREADY_EXIST);
|
|
str = status.ToString();
|
|
ASSERT_FALSE(str.empty());
|
|
|
|
status = milvus::Status(milvus::DB_INVALID_PATH, "mistake");
|
|
ASSERT_EQ(status.code(), milvus::DB_INVALID_PATH);
|
|
str = status.ToString();
|
|
ASSERT_FALSE(str.empty());
|
|
|
|
status = milvus::Status(milvus::DB_META_TRANSACTION_FAILED, "mistake");
|
|
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
|
|
str = status.ToString();
|
|
ASSERT_FALSE(str.empty());
|
|
|
|
auto status_copy = milvus::Status::OK();
|
|
CopyStatus(status_copy, status);
|
|
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
|
|
|
|
auto status_ref(status);
|
|
ASSERT_EQ(status_ref.code(), status.code());
|
|
ASSERT_EQ(status_ref.ToString(), status.ToString());
|
|
|
|
auto status_move = std::move(status);
|
|
ASSERT_EQ(status_move.code(), status_ref.code());
|
|
ASSERT_EQ(status_move.ToString(), status_ref.ToString());
|
|
}
|
|
|
|
TEST(ValidationUtilTest, VALIDATE_TABLENAME_TEST) {
|
|
std::string table_name = "Normal123_";
|
|
auto status = milvus::server::ValidationUtil::ValidateTableName(table_name);
|
|
ASSERT_TRUE(status.ok());
|
|
|
|
table_name = "12sds";
|
|
status = milvus::server::ValidationUtil::ValidateTableName(table_name);
|
|
ASSERT_EQ(status.code(), milvus::SERVER_INVALID_TABLE_NAME);
|
|
|
|
table_name = "";
|
|
status = milvus::server::ValidationUtil::ValidateTableName(table_name);
|
|
ASSERT_EQ(status.code(), milvus::SERVER_INVALID_TABLE_NAME);
|
|
|
|
table_name = "_asdasd";
|
|
status = milvus::server::ValidationUtil::ValidateTableName(table_name);
|
|
ASSERT_EQ(status.code(), milvus::SERVER_SUCCESS);
|
|
|
|
table_name = "!@#!@";
|
|
status = milvus::server::ValidationUtil::ValidateTableName(table_name);
|
|
ASSERT_EQ(status.code(), milvus::SERVER_INVALID_TABLE_NAME);
|
|
|
|
table_name = "_!@#!@";
|
|
status = milvus::server::ValidationUtil::ValidateTableName(table_name);
|
|
ASSERT_EQ(status.code(), milvus::SERVER_INVALID_TABLE_NAME);
|
|
|
|
table_name = "中文";
|
|
status = milvus::server::ValidationUtil::ValidateTableName(table_name);
|
|
ASSERT_EQ(status.code(), milvus::SERVER_INVALID_TABLE_NAME);
|
|
|
|
table_name = std::string(10000, 'a');
|
|
status = milvus::server::ValidationUtil::ValidateTableName(table_name);
|
|
ASSERT_EQ(status.code(), milvus::SERVER_INVALID_TABLE_NAME);
|
|
|
|
table_name = "";
|
|
status = milvus::server::ValidationUtil::ValidatePartitionName(table_name);
|
|
ASSERT_EQ(status.code(), milvus::SERVER_INVALID_TABLE_NAME);
|
|
}
|
|
|
|
TEST(ValidationUtilTest, VALIDATE_DIMENSION_TEST) {
|
|
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableDimension(-1).code(),
|
|
milvus::SERVER_INVALID_VECTOR_DIMENSION);
|
|
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableDimension(0).code(),
|
|
milvus::SERVER_INVALID_VECTOR_DIMENSION);
|
|
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableDimension(32769).code(),
|
|
milvus::SERVER_INVALID_VECTOR_DIMENSION);
|
|
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableDimension(32768).code(), milvus::SERVER_SUCCESS);
|
|
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableDimension(1).code(), milvus::SERVER_SUCCESS);
|
|
}
|
|
|
|
TEST(ValidationUtilTest, VALIDATE_INDEX_TEST) {
|
|
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexType((int)milvus::engine::EngineType::INVALID).code(),
|
|
milvus::SERVER_INVALID_INDEX_TYPE);
|
|
for (int i = 1; i <= (int)milvus::engine::EngineType::MAX_VALUE; i++) {
|
|
#ifndef CUSTOMIZATION
|
|
if (i == (int)milvus::engine::EngineType::FAISS_IVFSQ8H) {
|
|
ASSERT_NE(milvus::server::ValidationUtil::ValidateTableIndexType(i).code(), milvus::SERVER_SUCCESS);
|
|
continue;
|
|
}
|
|
#endif
|
|
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexType(i).code(), milvus::SERVER_SUCCESS);
|
|
}
|
|
|
|
ASSERT_EQ(
|
|
milvus::server::ValidationUtil::ValidateTableIndexType((int)milvus::engine::EngineType::MAX_VALUE + 1).code(),
|
|
milvus::SERVER_INVALID_INDEX_TYPE);
|
|
|
|
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexNlist(0).code(), milvus::SERVER_INVALID_INDEX_NLIST);
|
|
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexNlist(100).code(), milvus::SERVER_SUCCESS);
|
|
|
|
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexFileSize(0).code(),
|
|
milvus::SERVER_INVALID_INDEX_FILE_SIZE);
|
|
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexFileSize(100).code(), milvus::SERVER_SUCCESS);
|
|
|
|
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexMetricType(0).code(),
|
|
milvus::SERVER_INVALID_INDEX_METRIC_TYPE);
|
|
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexMetricType(1).code(), milvus::SERVER_SUCCESS);
|
|
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexMetricType(2).code(), milvus::SERVER_SUCCESS);
|
|
}
|
|
|
|
TEST(ValidationUtilTest, VALIDATE_TOPK_TEST) {
|
|
milvus::engine::meta::TableSchema schema;
|
|
ASSERT_EQ(milvus::server::ValidationUtil::ValidateSearchTopk(10, schema).code(), milvus::SERVER_SUCCESS);
|
|
ASSERT_NE(milvus::server::ValidationUtil::ValidateSearchTopk(65536, schema).code(), milvus::SERVER_SUCCESS);
|
|
ASSERT_NE(milvus::server::ValidationUtil::ValidateSearchTopk(0, schema).code(), milvus::SERVER_SUCCESS);
|
|
}
|
|
|
|
TEST(ValidationUtilTest, VALIDATE_NPROBE_TEST) {
|
|
milvus::engine::meta::TableSchema schema;
|
|
schema.nlist_ = 100;
|
|
ASSERT_EQ(milvus::server::ValidationUtil::ValidateSearchNprobe(10, schema).code(), milvus::SERVER_SUCCESS);
|
|
ASSERT_NE(milvus::server::ValidationUtil::ValidateSearchNprobe(0, schema).code(), milvus::SERVER_SUCCESS);
|
|
ASSERT_NE(milvus::server::ValidationUtil::ValidateSearchNprobe(101, schema).code(), milvus::SERVER_SUCCESS);
|
|
}
|
|
|
|
TEST(ValidationUtilTest, VALIDATE_PARTITION_TAGS) {
|
|
std::vector<std::string> partition_tags = {"abc"};
|
|
ASSERT_EQ(milvus::server::ValidationUtil::ValidatePartitionTags(partition_tags).code(), milvus::SERVER_SUCCESS);
|
|
partition_tags.push_back("");
|
|
ASSERT_NE(milvus::server::ValidationUtil::ValidatePartitionTags(partition_tags).code(), milvus::SERVER_SUCCESS);
|
|
std::string ss;
|
|
ss.assign(256, 'a');
|
|
partition_tags = {ss};
|
|
ASSERT_EQ(milvus::server::ValidationUtil::ValidatePartitionTags(partition_tags).code(),
|
|
milvus::SERVER_INVALID_PARTITION_TAG);
|
|
}
|
|
|
|
#ifdef MILVUS_GPU_VERSION
|
|
TEST(ValidationUtilTest, VALIDATE_GPU_TEST) {
|
|
ASSERT_EQ(milvus::server::ValidationUtil::ValidateGpuIndex(0).code(), milvus::SERVER_SUCCESS);
|
|
ASSERT_NE(milvus::server::ValidationUtil::ValidateGpuIndex(100).code(), milvus::SERVER_SUCCESS);
|
|
|
|
fiu_init(0);
|
|
fiu_enable("ValidationUtil.ValidateGpuIndex.get_device_count_fail", 1, NULL, 0);
|
|
ASSERT_NE(milvus::server::ValidationUtil::ValidateGpuIndex(0).code(), milvus::SERVER_SUCCESS);
|
|
fiu_disable("ValidationUtil.ValidateGpuIndex.get_device_count_fail");
|
|
|
|
size_t memory = 0;
|
|
ASSERT_EQ(milvus::server::ValidationUtil::GetGpuMemory(0, memory).code(), milvus::SERVER_SUCCESS);
|
|
ASSERT_NE(milvus::server::ValidationUtil::GetGpuMemory(100, memory).code(), milvus::SERVER_SUCCESS);
|
|
}
|
|
|
|
#endif
|
|
|
|
TEST(ValidationUtilTest, VALIDATE_IPADDRESS_TEST) {
|
|
ASSERT_EQ(milvus::server::ValidationUtil::ValidateIpAddress("127.0.0.1").code(), milvus::SERVER_SUCCESS);
|
|
ASSERT_NE(milvus::server::ValidationUtil::ValidateIpAddress("not ip").code(), milvus::SERVER_SUCCESS);
|
|
|
|
fiu_init(0);
|
|
fiu_enable("ValidationUtil.ValidateIpAddress.error_ip_result", 1, NULL, 0);
|
|
ASSERT_NE(milvus::server::ValidationUtil::ValidateIpAddress("not ip").code(), milvus::SERVER_SUCCESS);
|
|
fiu_disable("ValidationUtil.ValidateIpAddress.error_ip_result");
|
|
}
|
|
|
|
TEST(ValidationUtilTest, VALIDATE_NUMBER_TEST) {
|
|
ASSERT_EQ(milvus::server::ValidationUtil::ValidateStringIsNumber("1234").code(), milvus::SERVER_SUCCESS);
|
|
ASSERT_NE(milvus::server::ValidationUtil::ValidateStringIsNumber("not number").code(), milvus::SERVER_SUCCESS);
|
|
|
|
fiu_init(0);
|
|
fiu_enable("ValidationUtil.ValidateStringIsNumber.throw_exception", 1, NULL, 0);
|
|
ASSERT_NE(milvus::server::ValidationUtil::ValidateStringIsNumber("122").code(), milvus::SERVER_SUCCESS);
|
|
fiu_disable("ValidationUtil.ValidateStringIsNumber.throw_exception");
|
|
}
|
|
|
|
TEST(ValidationUtilTest, VALIDATE_BOOL_TEST) {
|
|
std::string str = "true";
|
|
ASSERT_EQ(milvus::server::ValidationUtil::ValidateStringIsBool(str).code(), milvus::SERVER_SUCCESS);
|
|
str = "not bool";
|
|
ASSERT_NE(milvus::server::ValidationUtil::ValidateStringIsBool(str).code(), milvus::SERVER_SUCCESS);
|
|
}
|
|
|
|
TEST(ValidationUtilTest, VALIDATE_DOUBLE_TEST) {
|
|
ASSERT_EQ(milvus::server::ValidationUtil::ValidateStringIsFloat("2.5").code(), milvus::SERVER_SUCCESS);
|
|
ASSERT_NE(milvus::server::ValidationUtil::ValidateStringIsFloat("not double").code(), milvus::SERVER_SUCCESS);
|
|
}
|
|
|
|
TEST(ValidationUtilTest, VALIDATE_DBURI_TEST) {
|
|
ASSERT_EQ(milvus::server::ValidationUtil::ValidateDbURI("sqlite://:@:/").code(), milvus::SERVER_SUCCESS);
|
|
ASSERT_NE(milvus::server::ValidationUtil::ValidateDbURI("xxx://:@:/").code(), milvus::SERVER_SUCCESS);
|
|
ASSERT_NE(milvus::server::ValidationUtil::ValidateDbURI("not uri").code(), milvus::SERVER_SUCCESS);
|
|
ASSERT_EQ(milvus::server::ValidationUtil::ValidateDbURI("mysql://root:123456@127.0.0.1:3303/milvus").code(),
|
|
milvus::SERVER_SUCCESS);
|
|
ASSERT_NE(milvus::server::ValidationUtil::ValidateDbURI("mysql://root:123456@127.0.0.1:port/milvus").code(),
|
|
milvus::SERVER_SUCCESS);
|
|
}
|
|
|
|
TEST(ValidationUtilTest, VALIDATE_PATH_TEST) {
|
|
ASSERT_TRUE(milvus::server::ValidationUtil::ValidateStoragePath("/home/milvus").ok());
|
|
ASSERT_TRUE(milvus::server::ValidationUtil::ValidateStoragePath("/tmp/milvus").ok());
|
|
ASSERT_TRUE(milvus::server::ValidationUtil::ValidateStoragePath("/tmp/milvus/").ok());
|
|
ASSERT_TRUE(milvus::server::ValidationUtil::ValidateStoragePath("/_tmp/milvus12345").ok());
|
|
ASSERT_TRUE(milvus::server::ValidationUtil::ValidateStoragePath("/tmp-/milvus").ok());
|
|
ASSERT_FALSE(milvus::server::ValidationUtil::ValidateStoragePath("/-tmp/milvus").ok());
|
|
ASSERT_FALSE(milvus::server::ValidationUtil::ValidateStoragePath("/****tmp/milvus").ok());
|
|
ASSERT_FALSE(milvus::server::ValidationUtil::ValidateStoragePath("/tmp--/milvus").ok());
|
|
ASSERT_FALSE(milvus::server::ValidationUtil::ValidateStoragePath("./tmp/milvus").ok());
|
|
ASSERT_FALSE(milvus::server::ValidationUtil::ValidateStoragePath("/tmp space/milvus").ok());
|
|
ASSERT_FALSE(milvus::server::ValidationUtil::ValidateStoragePath("../tmp/milvus").ok());
|
|
ASSERT_FALSE(milvus::server::ValidationUtil::ValidateStoragePath("/tmp//milvus").ok());
|
|
}
|
|
|
|
TEST(UtilTest, ROLLOUTHANDLER_TEST) {
|
|
std::string dir1 = "/tmp/milvus_test";
|
|
std::string dir2 = "/tmp/milvus_test/log_test";
|
|
std::string filename[6] = {"log_global.log", "log_debug.log", "log_warning.log",
|
|
"log_trace.log", "log_error.log", "log_fatal.log"};
|
|
|
|
el::Level list[6] = {el::Level::Global, el::Level::Debug, el::Level::Warning,
|
|
el::Level::Trace, el::Level::Error, el::Level::Fatal};
|
|
|
|
mkdir(dir1.c_str(), S_IRWXU);
|
|
mkdir(dir2.c_str(), S_IRWXU);
|
|
// [&]() {
|
|
//// std::string tmp = dir2 + "/" + filename[0]+"*@%$";
|
|
// std::string tmp = dir2 + "/" + filename[0] + "*$";
|
|
// std::ofstream file;
|
|
// file.open(tmp.c_str());
|
|
// file << "test" << std::endl;
|
|
// milvus::server::RolloutHandler(tmp.c_str(), 0, el::Level::Unknown);
|
|
// tmp.append(".1");
|
|
// std::ifstream file2;
|
|
// file2.open(tmp);
|
|
// std::string tmp2;
|
|
// file2 >> tmp2;
|
|
// ASSERT_EQ(tmp2, "test");
|
|
// }();
|
|
|
|
for (int i = 0; i < 6; ++i) {
|
|
std::string tmp = dir2 + "/" + filename[i];
|
|
|
|
std::ofstream file;
|
|
file.open(tmp.c_str());
|
|
file << "test" << std::endl;
|
|
|
|
milvus::server::RolloutHandler(tmp.c_str(), 0, list[i]);
|
|
|
|
tmp.append(".1");
|
|
std::ifstream file2;
|
|
file2.open(tmp);
|
|
|
|
std::string tmp2;
|
|
file2 >> tmp2;
|
|
ASSERT_EQ(tmp2, "test");
|
|
}
|
|
|
|
[&]() {
|
|
std::string tmp = dir2 + "/" + filename[0];
|
|
std::ofstream file;
|
|
file.open(tmp.c_str());
|
|
file << "test" << std::endl;
|
|
milvus::server::RolloutHandler(tmp.c_str(), 0, el::Level::Unknown);
|
|
tmp.append(".1");
|
|
std::ifstream file2;
|
|
file2.open(tmp);
|
|
std::string tmp2;
|
|
file2 >> tmp2;
|
|
ASSERT_EQ(tmp2, "test");
|
|
}();
|
|
|
|
boost::filesystem::remove_all(dir2);
|
|
}
|
|
|
|
TEST(UtilTest, THREADPOOL_TEST) {
|
|
auto thread_pool_ptr = std::make_unique<milvus::ThreadPool>(3);
|
|
auto fun = [](int i) {
|
|
sleep(1);
|
|
};
|
|
for (int i = 0; i < 10; ++i) {
|
|
thread_pool_ptr->enqueue(fun, i);
|
|
}
|
|
|
|
fiu_init(0);
|
|
fiu_enable("ThreadPool.enqueue.stop_is_true", 1, NULL, 0);
|
|
try {
|
|
thread_pool_ptr->enqueue(fun, -1);
|
|
} catch (std::exception& err) {
|
|
std::cout << "catch an error here" << std::endl;
|
|
}
|
|
fiu_disable("ThreadPool.enqueue.stop_is_true");
|
|
|
|
thread_pool_ptr.reset();
|
|
}
|