diff --git a/CHANGELOG.md b/CHANGELOG.md index f7d2ed5596..9a2abb7beb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ Please mark all change in change log and use the issue from GitHub - \#1649 Fix Milvus crash on old CPU - \#1653 IndexFlat (SSE) and IndexBinaryFlat performance improvement for small NQ - \#1678 Remove CUSTOMIZATION macro +- \#1698 Upgrade mishards to v0.7.0 ## Task diff --git a/shards/all_in_one/all_in_one.yml b/shards/all_in_one/all_in_one.yml index 125b88e4cb..79a6f93cd2 100644 --- a/shards/all_in_one/all_in_one.yml +++ b/shards/all_in_one/all_in_one.yml @@ -3,7 +3,7 @@ services: milvus_wr: runtime: nvidia restart: always - image: milvusdb/milvus:0.6.0-gpu-d120719-2b40dd + image: milvusdb/milvus:0.7.0-gpu-d031120-40c7e8 ports: - "0.0.0.0:19540:19530" volumes: @@ -13,7 +13,7 @@ services: milvus_ro: runtime: nvidia restart: always - image: milvusdb/milvus:0.6.0-gpu-d120719-2b40dd + image: milvusdb/milvus:0.7.0-gpu-d031120-40c7e8 ports: - "0.0.0.0:19541:19530" volumes: diff --git a/shards/all_in_one/ro_server.yml b/shards/all_in_one/ro_server.yml index 00a8f61101..a575b1c2e4 100644 --- a/shards/all_in_one/ro_server.yml +++ b/shards/all_in_one/ro_server.yml @@ -1,41 +1,185 @@ +# Copyright (C) 2019-2020 Zilliz. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under the License. + +version: 0.2 + +#----------------------+------------------------------------------------------------+------------+-----------------+ +# Server Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# address | IP address that Milvus server monitors. | IP | 0.0.0.0 | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# port | Port that Milvus server monitors. Port range (1024, 65535) | Integer | 19530 | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# deploy_mode | Milvus deployment type: | DeployMode | single | +# | single, cluster_readonly, cluster_writable | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# time_zone | Use UTC-x or UTC+x to specify a time zone. | Timezone | UTC+8 | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# web_port | Port that Milvus web server monitors. | Integer | 19121 | +# | Port range (1024, 65535) | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ server_config: - address: 0.0.0.0 # milvus server ip address (IPv4) - port: 19530 # port range: 1025 ~ 65534 - deploy_mode: cluster_readonly # deployment type: single, cluster_readonly, cluster_writable + address: 0.0.0.0 + port: 19530 + deploy_mode: cluster_readonly time_zone: UTC+8 + web_port: 19121 +#----------------------+------------------------------------------------------------+------------+-----------------+ +# DataBase Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# backend_url | URL for metadata storage, using SQLite (for single server | URL | sqlite://:@:/ | +# | Milvus) or MySQL (for distributed cluster Milvus). | | | +# | Format: dialect://username:password@host:port/database | | | +# | Keep 'dialect://:@:/', 'dialect' can be either 'sqlite' or | | | +# | 'mysql', replace other texts with real values. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# preload_table | A comma-separated list of table names that need to be pre- | StringList | | +# | loaded when Milvus server starts up. | | | +# | '*' means preload all existing tables (single-quote or | | | +# | double-quote required). | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# auto_flush_interval | The interval, in seconds, at which Milvus automatically | Integer | 1 (s) | +# | flushes data to disk. | | | +# | 0 means disable the regular flush. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ db_config: - primary_path: /var/lib/milvus # path used to store data and meta - secondary_path: # path used to store data only, split by semicolon + backend_url: sqlite://:@:/ + preload_table: + auto_flush_interval: 1 - backend_url: sqlite://:@:/ # URI format: dialect://username:password@host:port/database - # Keep 'dialect://:@:/', and replace other texts with real values - # Replace 'dialect' with 'mysql' or 'sqlite' - - insert_buffer_size: 1 # GB, maximum insert buffer size allowed - # sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory - - preload_table: # preload data at startup, '*' means load all tables, empty value means no preload - # you can specify preload tables like this: table1,table2,table3 +#----------------------+------------------------------------------------------------+------------+-----------------+ +# Storage Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# primary_path | Primary directory used to save meta data, vector data and | Path | /var/lib/milvus | +# | index data. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# secondary_path | A semicolon-separated list of secondary directories used | Path | | +# | to save vector data and index data. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +storage_config: + primary_path: /var/lib/milvus + secondary_path: +#----------------------+------------------------------------------------------------+------------+-----------------+ +# Metric Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# enable_monitor | Enable monitoring function or not. | Boolean | false | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# address | Pushgateway address | IP | 127.0.0.1 + +#----------------------+------------------------------------------------------------+------------+-----------------+ +# port | Pushgateway port, port range (1024, 65535) | Integer | 9091 | +#----------------------+------------------------------------------------------------+------------+-----------------+ metric_config: - enable_monitor: false # enable monitoring or not - collector: prometheus # prometheus - prometheus_config: - port: 8080 # port prometheus uses to fetch metrics + enable_monitor: false + address: 127.0.0.1 + port: 9091 +#----------------------+------------------------------------------------------------+------------+-----------------+ +# Cache Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# cpu_cache_capacity | The size of CPU memory used for caching data for faster | Integer | 4 (GB) | +# | query. The sum of 'cpu_cache_capacity' and | | | +# | 'insert_buffer_size' must be less than system memory size. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# insert_buffer_size | Buffer size used for data insertion. | Integer | 1 (GB) | +# | The sum of 'insert_buffer_size' and 'cpu_cache_capacity' | | | +# | must be less than system memory size. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# cache_insert_data | Whether to load data to cache for hot query | Boolean | false | +#----------------------+------------------------------------------------------------+------------+-----------------+ cache_config: - cpu_cache_capacity: 4 # GB, CPU memory used for cache - cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered - gpu_cache_capacity: 1 # GB, GPU memory used for cache - gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered - cache_insert_data: false # whether to load inserted data into cache + cpu_cache_capacity: 4 + insert_buffer_size: 1 + cache_insert_data: false +#----------------------+------------------------------------------------------------+------------+-----------------+ +# Engine Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# use_blas_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1100 | +# | compared with 'nq' to decide if OpenBLAS should be used. | | | +# | If nq >= use_blas_threshold, OpenBLAS will be used, search | | | +# | response time will be stable but the search speed will be | | | +# | slower; if nq < use_blas_threshold, AVX or SSE will be | | | +# | used, search speed will be faster but search response time | | | +# | will fluctuate. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# gpu_search_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1000 | +# | compared with 'nq' to decide if the search computation will| | | +# | be executed on GPUs only. | | | +# | If nq >= gpu_search_threshold, the search computation will | | | +# | be executed on GPUs only; | | | +# | if nq < gpu_search_threshold, the search computation will | | | +# | be executed on both CPUs and GPUs. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ engine_config: - use_blas_threshold: 800 # if nq < use_blas_threshold, use SSE, faster with fluctuated response times - # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times + use_blas_threshold: 1100 + gpu_search_threshold: 1000 -resource_config: - search_resources: # define the GPUs used for search computation, valid value: gpux +#----------------------+------------------------------------------------------------+------------+-----------------+ +# GPU Resource Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# enable | Enable GPU resources or not. | Boolean | false | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# cache_capacity | The size of GPU memory per card used for cache. | Integer | 1 (GB) | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# search_resources | The list of GPU devices used for search computation. | DeviceList | gpu0 | +# | Must be in format gpux. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# build_index_resources| The list of GPU devices used for index building. | DeviceList | gpu0 | +# | Must be in format gpux. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +gpu_resource_config: + enable: true + cache_capacity: 1 + search_resources: - gpu0 - index_build_device: gpu0 # GPU used for building index + build_index_resources: + - gpu0 + +#----------------------+------------------------------------------------------------+------------+-----------------+ +# Tracing Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# json_config_path | Absolute path for tracing config file. | Path | | +# | Leave it empty, a no-op tracer will be created. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +tracing_config: + json_config_path: + +#----------------------+------------------------------------------------------------+------------+-----------------+ +# WAL Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# enable | Whether to enable write-ahead logging (WAL) in Milvus. | Boolean | true | +# | If WAL is enabled, Milvus writes all data changes to log | | | +# | files in advance before implementing data changes. WAL | | | +# | ensures the atomicity and durability for Milvus operations.| | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# recovery_error_ignore| Whether to ignore logs with errors that happens during WAL | Boolean | true | +# | recovery. If true, when Milvus restarts for recovery and | | | +# | there are errors in WAL log files, log files with errors | | | +# | are ignored. If false, Milvus does not restart when there | | | +# | are errors in WAL log files. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# buffer_size | Sum total of the read buffer and the write buffer in MBs. | Integer | 256 (MB) | +# | buffer_size must be in range [64, 4096] (MB). | | | +# | If the value you specified is out of range, Milvus | | | +# | automatically uses the boundary value closest to the | | | +# | specified value. It is recommended you set buffer_size to | | | +# | a value greater than the inserted data size of a single | | | +# | insert operation for better performance. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# wal_path | Location of WAL log files. | String | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +wal_config: + enable: true + recovery_error_ignore: true + buffer_size: 256 + wal_path: /var/lib/milvus/wal diff --git a/shards/all_in_one/wr_server.yml b/shards/all_in_one/wr_server.yml index cf0d06e23e..32c1ebaf81 100644 --- a/shards/all_in_one/wr_server.yml +++ b/shards/all_in_one/wr_server.yml @@ -1,41 +1,185 @@ +# Copyright (C) 2019-2020 Zilliz. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under the License. + +version: 0.2 + +#----------------------+------------------------------------------------------------+------------+-----------------+ +# Server Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# address | IP address that Milvus server monitors. | IP | 0.0.0.0 | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# port | Port that Milvus server monitors. Port range (1024, 65535) | Integer | 19530 | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# deploy_mode | Milvus deployment type: | DeployMode | single | +# | single, cluster_readonly, cluster_writable | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# time_zone | Use UTC-x or UTC+x to specify a time zone. | Timezone | UTC+8 | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# web_port | Port that Milvus web server monitors. | Integer | 19121 | +# | Port range (1024, 65535) | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ server_config: - address: 0.0.0.0 # milvus server ip address (IPv4) - port: 19530 # port range: 1025 ~ 65534 - deploy_mode: cluster_writable # deployment type: single, cluster_readonly, cluster_writable + address: 0.0.0.0 + port: 19530 + deploy_mode: cluster_writable time_zone: UTC+8 + web_port: 19121 +#----------------------+------------------------------------------------------------+------------+-----------------+ +# DataBase Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# backend_url | URL for metadata storage, using SQLite (for single server | URL | sqlite://:@:/ | +# | Milvus) or MySQL (for distributed cluster Milvus). | | | +# | Format: dialect://username:password@host:port/database | | | +# | Keep 'dialect://:@:/', 'dialect' can be either 'sqlite' or | | | +# | 'mysql', replace other texts with real values. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# preload_table | A comma-separated list of table names that need to be pre- | StringList | | +# | loaded when Milvus server starts up. | | | +# | '*' means preload all existing tables (single-quote or | | | +# | double-quote required). | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# auto_flush_interval | The interval, in seconds, at which Milvus automatically | Integer | 1 (s) | +# | flushes data to disk. | | | +# | 0 means disable the regular flush. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ db_config: - primary_path: /var/lib/milvus # path used to store data and meta - secondary_path: # path used to store data only, split by semicolon + backend_url: sqlite://:@:/ + preload_table: + auto_flush_interval: 1 - backend_url: sqlite://:@:/ # URI format: dialect://username:password@host:port/database - # Keep 'dialect://:@:/', and replace other texts with real values - # Replace 'dialect' with 'mysql' or 'sqlite' - - insert_buffer_size: 2 # GB, maximum insert buffer size allowed - # sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory - - preload_table: # preload data at startup, '*' means load all tables, empty value means no preload - # you can specify preload tables like this: table1,table2,table3 +#----------------------+------------------------------------------------------------+------------+-----------------+ +# Storage Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# primary_path | Primary directory used to save meta data, vector data and | Path | /var/lib/milvus | +# | index data. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# secondary_path | A semicolon-separated list of secondary directories used | Path | | +# | to save vector data and index data. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +storage_config: + primary_path: /var/lib/milvus + secondary_path: +#----------------------+------------------------------------------------------------+------------+-----------------+ +# Metric Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# enable_monitor | Enable monitoring function or not. | Boolean | false | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# address | Pushgateway address | IP | 127.0.0.1 + +#----------------------+------------------------------------------------------------+------------+-----------------+ +# port | Pushgateway port, port range (1024, 65535) | Integer | 9091 | +#----------------------+------------------------------------------------------------+------------+-----------------+ metric_config: - enable_monitor: false # enable monitoring or not - collector: prometheus # prometheus - prometheus_config: - port: 8080 # port prometheus uses to fetch metrics + enable_monitor: false + address: 127.0.0.1 + port: 9091 +#----------------------+------------------------------------------------------------+------------+-----------------+ +# Cache Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# cpu_cache_capacity | The size of CPU memory used for caching data for faster | Integer | 4 (GB) | +# | query. The sum of 'cpu_cache_capacity' and | | | +# | 'insert_buffer_size' must be less than system memory size. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# insert_buffer_size | Buffer size used for data insertion. | Integer | 1 (GB) | +# | The sum of 'insert_buffer_size' and 'cpu_cache_capacity' | | | +# | must be less than system memory size. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# cache_insert_data | Whether to load data to cache for hot query | Boolean | false | +#----------------------+------------------------------------------------------------+------------+-----------------+ cache_config: - cpu_cache_capacity: 2 # GB, CPU memory used for cache - cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered - gpu_cache_capacity: 2 # GB, GPU memory used for cache - gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered - cache_insert_data: false # whether to load inserted data into cache + cpu_cache_capacity: 4 + insert_buffer_size: 1 + cache_insert_data: false +#----------------------+------------------------------------------------------------+------------+-----------------+ +# Engine Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# use_blas_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1100 | +# | compared with 'nq' to decide if OpenBLAS should be used. | | | +# | If nq >= use_blas_threshold, OpenBLAS will be used, search | | | +# | response time will be stable but the search speed will be | | | +# | slower; if nq < use_blas_threshold, AVX or SSE will be | | | +# | used, search speed will be faster but search response time | | | +# | will fluctuate. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# gpu_search_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1000 | +# | compared with 'nq' to decide if the search computation will| | | +# | be executed on GPUs only. | | | +# | If nq >= gpu_search_threshold, the search computation will | | | +# | be executed on GPUs only; | | | +# | if nq < gpu_search_threshold, the search computation will | | | +# | be executed on both CPUs and GPUs. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ engine_config: - use_blas_threshold: 800 # if nq < use_blas_threshold, use SSE, faster with fluctuated response times - # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times + use_blas_threshold: 1100 + gpu_search_threshold: 1000 -resource_config: - search_resources: # define the GPUs used for search computation, valid value: gpux +#----------------------+------------------------------------------------------------+------------+-----------------+ +# GPU Resource Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# enable | Enable GPU resources or not. | Boolean | false | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# cache_capacity | The size of GPU memory per card used for cache. | Integer | 1 (GB) | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# search_resources | The list of GPU devices used for search computation. | DeviceList | gpu0 | +# | Must be in format gpux. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# build_index_resources| The list of GPU devices used for index building. | DeviceList | gpu0 | +# | Must be in format gpux. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +gpu_resource_config: + enable: true + cache_capacity: 1 + search_resources: - gpu0 - index_build_device: gpu0 # GPU used for building index + build_index_resources: + - gpu0 + +#----------------------+------------------------------------------------------------+------------+-----------------+ +# Tracing Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# json_config_path | Absolute path for tracing config file. | Path | | +# | Leave it empty, a no-op tracer will be created. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +tracing_config: + json_config_path: + +#----------------------+------------------------------------------------------------+------------+-----------------+ +# WAL Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# enable | Whether to enable write-ahead logging (WAL) in Milvus. | Boolean | true | +# | If WAL is enabled, Milvus writes all data changes to log | | | +# | files in advance before implementing data changes. WAL | | | +# | ensures the atomicity and durability for Milvus operations.| | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# recovery_error_ignore| Whether to ignore logs with errors that happens during WAL | Boolean | true | +# | recovery. If true, when Milvus restarts for recovery and | | | +# | there are errors in WAL log files, log files with errors | | | +# | are ignored. If false, Milvus does not restart when there | | | +# | are errors in WAL log files. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# buffer_size | Sum total of the read buffer and the write buffer in MBs. | Integer | 256 (MB) | +# | buffer_size must be in range [64, 4096] (MB). | | | +# | If the value you specified is out of range, Milvus | | | +# | automatically uses the boundary value closest to the | | | +# | specified value. It is recommended you set buffer_size to | | | +# | a value greater than the inserted data size of a single | | | +# | insert operation for better performance. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# wal_path | Location of WAL log files. | String | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +wal_config: + enable: true + recovery_error_ignore: true + buffer_size: 256 + wal_path: /var/lib/milvus/wal diff --git a/shards/all_in_one_with_mysql/all_in_one.yml b/shards/all_in_one_with_mysql/all_in_one.yml index 5343f18e88..fd8f236860 100644 --- a/shards/all_in_one_with_mysql/all_in_one.yml +++ b/shards/all_in_one_with_mysql/all_in_one.yml @@ -18,7 +18,7 @@ services: milvus_wr: runtime: nvidia restart: always - image: milvusdb/milvus:0.6.0-gpu-d120719-2b40dd + image: milvusdb/milvus:0.7.0-gpu-d031120-40c7e8 volumes: - /tmp/milvus/db:/var/lib/milvus/db - ./wr_server.yml:/var/lib/milvus/conf/server_config.yaml @@ -29,7 +29,7 @@ services: milvus_ro: runtime: nvidia restart: always - image: milvusdb/milvus:0.6.0-gpu-d120719-2b40dd + image: milvusdb/milvus:0.7.0-gpu-d031120-40c7e8 volumes: - /tmp/milvus/db:/var/lib/milvus/db - ./ro_server.yml:/var/lib/milvus/conf/server_config.yaml diff --git a/shards/all_in_one_with_mysql/ro_server.yml b/shards/all_in_one_with_mysql/ro_server.yml index 4355fb5aa9..f6a752145a 100644 --- a/shards/all_in_one_with_mysql/ro_server.yml +++ b/shards/all_in_one_with_mysql/ro_server.yml @@ -1,43 +1,186 @@ -version: 0.1 +# Copyright (C) 2019-2020 Zilliz. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under the License. + +version: 0.2 + +#----------------------+------------------------------------------------------------+------------+-----------------+ +# Server Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# address | IP address that Milvus server monitors. | IP | 0.0.0.0 | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# port | Port that Milvus server monitors. Port range (1024, 65535) | Integer | 19530 | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# deploy_mode | Milvus deployment type: | DeployMode | single | +# | single, cluster_readonly, cluster_writable | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# time_zone | Use UTC-x or UTC+x to specify a time zone. | Timezone | UTC+8 | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# web_port | Port that Milvus web server monitors. | Integer | 19121 | +# | Port range (1024, 65535) | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ server_config: - address: 0.0.0.0 # milvus server ip address (IPv4) - port: 19530 # port range: 1025 ~ 65534 - deploy_mode: cluster_readonly # deployment type: single, cluster_readonly, cluster_writable + address: 0.0.0.0 + port: 19530 + deploy_mode: cluster_readonly time_zone: UTC+8 + web_port: 19121 +#----------------------+------------------------------------------------------------+------------+-----------------+ +# DataBase Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# backend_url | URL for metadata storage, using SQLite (for single server | URL | sqlite://:@:/ | +# | Milvus) or MySQL (for distributed cluster Milvus). | | | +# | Format: dialect://username:password@host:port/database | | | +# | Keep 'dialect://:@:/', 'dialect' can be either 'sqlite' or | | | +# | 'mysql', replace other texts with real values. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# preload_table | A comma-separated list of table names that need to be pre- | StringList | | +# | loaded when Milvus server starts up. | | | +# | '*' means preload all existing tables (single-quote or | | | +# | double-quote required). | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# auto_flush_interval | The interval, in seconds, at which Milvus automatically | Integer | 1 (s) | +# | flushes data to disk. | | | +# | 0 means disable the regular flush. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ db_config: - primary_path: /var/lib/milvus # path used to store data and meta - secondary_path: # path used to store data only, split by semicolon - backend_url: mysql://root:milvusroot@milvus-mysql:3306/milvus - # URI format: dialect://username:password@host:port/database - # Keep 'dialect://:@:/', and replace other texts with real values - # Replace 'dialect' with 'mysql' or 'sqlite' + preload_table: + auto_flush_interval: 1 - insert_buffer_size: 1 # GB, maximum insert buffer size allowed - # sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory - - preload_table: # preload data at startup, '*' means load all tables, empty value means no preload - # you can specify preload tables like this: table1,table2,table3 +#----------------------+------------------------------------------------------------+------------+-----------------+ +# Storage Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# primary_path | Primary directory used to save meta data, vector data and | Path | /var/lib/milvus | +# | index data. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# secondary_path | A semicolon-separated list of secondary directories used | Path | | +# | to save vector data and index data. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +storage_config: + primary_path: /var/lib/milvus + secondary_path: +#----------------------+------------------------------------------------------------+------------+-----------------+ +# Metric Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# enable_monitor | Enable monitoring function or not. | Boolean | false | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# address | Pushgateway address | IP | 127.0.0.1 + +#----------------------+------------------------------------------------------------+------------+-----------------+ +# port | Pushgateway port, port range (1024, 65535) | Integer | 9091 | +#----------------------+------------------------------------------------------------+------------+-----------------+ metric_config: - enable_monitor: false # enable monitoring or not - collector: prometheus # prometheus - prometheus_config: - port: 8080 # port prometheus uses to fetch metrics + enable_monitor: false + address: 127.0.0.1 + port: 9091 +#----------------------+------------------------------------------------------------+------------+-----------------+ +# Cache Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# cpu_cache_capacity | The size of CPU memory used for caching data for faster | Integer | 4 (GB) | +# | query. The sum of 'cpu_cache_capacity' and | | | +# | 'insert_buffer_size' must be less than system memory size. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# insert_buffer_size | Buffer size used for data insertion. | Integer | 1 (GB) | +# | The sum of 'insert_buffer_size' and 'cpu_cache_capacity' | | | +# | must be less than system memory size. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# cache_insert_data | Whether to load data to cache for hot query | Boolean | false | +#----------------------+------------------------------------------------------------+------------+-----------------+ cache_config: - cpu_cache_capacity: 4 # GB, CPU memory used for cache - cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered - gpu_cache_capacity: 1 # GB, GPU memory used for cache - gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered - cache_insert_data: false # whether to load inserted data into cache + cpu_cache_capacity: 4 + insert_buffer_size: 1 + cache_insert_data: false +#----------------------+------------------------------------------------------------+------------+-----------------+ +# Engine Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# use_blas_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1100 | +# | compared with 'nq' to decide if OpenBLAS should be used. | | | +# | If nq >= use_blas_threshold, OpenBLAS will be used, search | | | +# | response time will be stable but the search speed will be | | | +# | slower; if nq < use_blas_threshold, AVX or SSE will be | | | +# | used, search speed will be faster but search response time | | | +# | will fluctuate. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# gpu_search_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1000 | +# | compared with 'nq' to decide if the search computation will| | | +# | be executed on GPUs only. | | | +# | If nq >= gpu_search_threshold, the search computation will | | | +# | be executed on GPUs only; | | | +# | if nq < gpu_search_threshold, the search computation will | | | +# | be executed on both CPUs and GPUs. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ engine_config: - use_blas_threshold: 800 # if nq < use_blas_threshold, use SSE, faster with fluctuated response times - # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times + use_blas_threshold: 1100 + gpu_search_threshold: 1000 -resource_config: - search_resources: # define the GPUs used for search computation, valid value: gpux +#----------------------+------------------------------------------------------------+------------+-----------------+ +# GPU Resource Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# enable | Enable GPU resources or not. | Boolean | false | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# cache_capacity | The size of GPU memory per card used for cache. | Integer | 1 (GB) | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# search_resources | The list of GPU devices used for search computation. | DeviceList | gpu0 | +# | Must be in format gpux. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# build_index_resources| The list of GPU devices used for index building. | DeviceList | gpu0 | +# | Must be in format gpux. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +gpu_resource_config: + enable: true + cache_capacity: 1 + search_resources: - gpu0 - index_build_device: gpu0 # GPU used for building index + build_index_resources: + - gpu0 + +#----------------------+------------------------------------------------------------+------------+-----------------+ +# Tracing Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# json_config_path | Absolute path for tracing config file. | Path | | +# | Leave it empty, a no-op tracer will be created. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +tracing_config: + json_config_path: + +#----------------------+------------------------------------------------------------+------------+-----------------+ +# WAL Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# enable | Whether to enable write-ahead logging (WAL) in Milvus. | Boolean | true | +# | If WAL is enabled, Milvus writes all data changes to log | | | +# | files in advance before implementing data changes. WAL | | | +# | ensures the atomicity and durability for Milvus operations.| | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# recovery_error_ignore| Whether to ignore logs with errors that happens during WAL | Boolean | true | +# | recovery. If true, when Milvus restarts for recovery and | | | +# | there are errors in WAL log files, log files with errors | | | +# | are ignored. If false, Milvus does not restart when there | | | +# | are errors in WAL log files. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# buffer_size | Sum total of the read buffer and the write buffer in MBs. | Integer | 256 (MB) | +# | buffer_size must be in range [64, 4096] (MB). | | | +# | If the value you specified is out of range, Milvus | | | +# | automatically uses the boundary value closest to the | | | +# | specified value. It is recommended you set buffer_size to | | | +# | a value greater than the inserted data size of a single | | | +# | insert operation for better performance. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# wal_path | Location of WAL log files. | String | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +wal_config: + enable: true + recovery_error_ignore: true + buffer_size: 256 + wal_path: /var/lib/milvus/wal + diff --git a/shards/all_in_one_with_mysql/wr_server.yml b/shards/all_in_one_with_mysql/wr_server.yml index 9cbb8f9c4c..25860bd183 100644 --- a/shards/all_in_one_with_mysql/wr_server.yml +++ b/shards/all_in_one_with_mysql/wr_server.yml @@ -1,42 +1,186 @@ -version: 0.1 +# Copyright (C) 2019-2020 Zilliz. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under the License. + +version: 0.2 + +#----------------------+------------------------------------------------------------+------------+-----------------+ +# Server Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# address | IP address that Milvus server monitors. | IP | 0.0.0.0 | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# port | Port that Milvus server monitors. Port range (1024, 65535) | Integer | 19530 | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# deploy_mode | Milvus deployment type: | DeployMode | single | +# | single, cluster_readonly, cluster_writable | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# time_zone | Use UTC-x or UTC+x to specify a time zone. | Timezone | UTC+8 | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# web_port | Port that Milvus web server monitors. | Integer | 19121 | +# | Port range (1024, 65535) | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ server_config: - address: 0.0.0.0 # milvus server ip address (IPv4) - port: 19530 # port range: 1025 ~ 65534 - deploy_mode: cluster_writable # deployment type: single, cluster_readonly, cluster_writable + address: 0.0.0.0 + port: 19530 + deploy_mode: cluster_writable time_zone: UTC+8 + web_port: 19121 +#----------------------+------------------------------------------------------------+------------+-----------------+ +# DataBase Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# backend_url | URL for metadata storage, using SQLite (for single server | URL | sqlite://:@:/ | +# | Milvus) or MySQL (for distributed cluster Milvus). | | | +# | Format: dialect://username:password@host:port/database | | | +# | Keep 'dialect://:@:/', 'dialect' can be either 'sqlite' or | | | +# | 'mysql', replace other texts with real values. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# preload_table | A comma-separated list of table names that need to be pre- | StringList | | +# | loaded when Milvus server starts up. | | | +# | '*' means preload all existing tables (single-quote or | | | +# | double-quote required). | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# auto_flush_interval | The interval, in seconds, at which Milvus automatically | Integer | 1 (s) | +# | flushes data to disk. | | | +# | 0 means disable the regular flush. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ db_config: - primary_path: /var/lib/milvus # path used to store data and meta - secondary_path: # path used to store data only, split by semicolon + backend_url: mysql://root:milvusroot@milvus-mysql:3306/milvus + preload_table: + auto_flush_interval: 1 - backend_url: mysql://root:milvusroot@milvus-mysql:3306/milvus # URI format: dialect://username:password@host:port/database - # Keep 'dialect://:@:/', and replace other texts with real values - # Replace 'dialect' with 'mysql' or 'sqlite' - - insert_buffer_size: 2 # GB, maximum insert buffer size allowed - # sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory - - preload_table: # preload data at startup, '*' means load all tables, empty value means no preload - # you can specify preload tables like this: table1,table2,table3 +#----------------------+------------------------------------------------------------+------------+-----------------+ +# Storage Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# primary_path | Primary directory used to save meta data, vector data and | Path | /var/lib/milvus | +# | index data. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# secondary_path | A semicolon-separated list of secondary directories used | Path | | +# | to save vector data and index data. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +storage_config: + primary_path: /var/lib/milvus + secondary_path: +#----------------------+------------------------------------------------------------+------------+-----------------+ +# Metric Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# enable_monitor | Enable monitoring function or not. | Boolean | false | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# address | Pushgateway address | IP | 127.0.0.1 + +#----------------------+------------------------------------------------------------+------------+-----------------+ +# port | Pushgateway port, port range (1024, 65535) | Integer | 9091 | +#----------------------+------------------------------------------------------------+------------+-----------------+ metric_config: - enable_monitor: false # enable monitoring or not - collector: prometheus # prometheus - prometheus_config: - port: 8080 # port prometheus uses to fetch metrics + enable_monitor: false + address: 127.0.0.1 + port: 9091 +#----------------------+------------------------------------------------------------+------------+-----------------+ +# Cache Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# cpu_cache_capacity | The size of CPU memory used for caching data for faster | Integer | 4 (GB) | +# | query. The sum of 'cpu_cache_capacity' and | | | +# | 'insert_buffer_size' must be less than system memory size. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# insert_buffer_size | Buffer size used for data insertion. | Integer | 1 (GB) | +# | The sum of 'insert_buffer_size' and 'cpu_cache_capacity' | | | +# | must be less than system memory size. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# cache_insert_data | Whether to load data to cache for hot query | Boolean | false | +#----------------------+------------------------------------------------------------+------------+-----------------+ cache_config: - cpu_cache_capacity: 2 # GB, CPU memory used for cache - cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered - gpu_cache_capacity: 2 # GB, GPU memory used for cache - gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered - cache_insert_data: false # whether to load inserted data into cache + cpu_cache_capacity: 4 + insert_buffer_size: 1 + cache_insert_data: false +#----------------------+------------------------------------------------------------+------------+-----------------+ +# Engine Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# use_blas_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1100 | +# | compared with 'nq' to decide if OpenBLAS should be used. | | | +# | If nq >= use_blas_threshold, OpenBLAS will be used, search | | | +# | response time will be stable but the search speed will be | | | +# | slower; if nq < use_blas_threshold, AVX or SSE will be | | | +# | used, search speed will be faster but search response time | | | +# | will fluctuate. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# gpu_search_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1000 | +# | compared with 'nq' to decide if the search computation will| | | +# | be executed on GPUs only. | | | +# | If nq >= gpu_search_threshold, the search computation will | | | +# | be executed on GPUs only; | | | +# | if nq < gpu_search_threshold, the search computation will | | | +# | be executed on both CPUs and GPUs. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ engine_config: - use_blas_threshold: 800 # if nq < use_blas_threshold, use SSE, faster with fluctuated response times - # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times + use_blas_threshold: 1100 + gpu_search_threshold: 1000 -resource_config: - search_resources: # define the GPUs used for search computation, valid value: gpux +#----------------------+------------------------------------------------------------+------------+-----------------+ +# GPU Resource Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# enable | Enable GPU resources or not. | Boolean | false | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# cache_capacity | The size of GPU memory per card used for cache. | Integer | 1 (GB) | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# search_resources | The list of GPU devices used for search computation. | DeviceList | gpu0 | +# | Must be in format gpux. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# build_index_resources| The list of GPU devices used for index building. | DeviceList | gpu0 | +# | Must be in format gpux. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +gpu_resource_config: + enable: true + cache_capacity: 1 + search_resources: - gpu0 - index_build_device: gpu0 # GPU used for building index + build_index_resources: + - gpu0 + +#----------------------+------------------------------------------------------------+------------+-----------------+ +# Tracing Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# json_config_path | Absolute path for tracing config file. | Path | | +# | Leave it empty, a no-op tracer will be created. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +tracing_config: + json_config_path: + +#----------------------+------------------------------------------------------------+------------+-----------------+ +# WAL Config | Description | Type | Default | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# enable | Whether to enable write-ahead logging (WAL) in Milvus. | Boolean | true | +# | If WAL is enabled, Milvus writes all data changes to log | | | +# | files in advance before implementing data changes. WAL | | | +# | ensures the atomicity and durability for Milvus operations.| | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# recovery_error_ignore| Whether to ignore logs with errors that happens during WAL | Boolean | true | +# | recovery. If true, when Milvus restarts for recovery and | | | +# | there are errors in WAL log files, log files with errors | | | +# | are ignored. If false, Milvus does not restart when there | | | +# | are errors in WAL log files. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# buffer_size | Sum total of the read buffer and the write buffer in MBs. | Integer | 256 (MB) | +# | buffer_size must be in range [64, 4096] (MB). | | | +# | If the value you specified is out of range, Milvus | | | +# | automatically uses the boundary value closest to the | | | +# | specified value. It is recommended you set buffer_size to | | | +# | a value greater than the inserted data size of a single | | | +# | insert operation for better performance. | | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +# wal_path | Location of WAL log files. | String | | +#----------------------+------------------------------------------------------------+------------+-----------------+ +wal_config: + enable: true + recovery_error_ignore: true + buffer_size: 256 + wal_path: /var/lib/milvus/wal + diff --git a/shards/mishards/exception_codes.py b/shards/mishards/exception_codes.py index bdd4572dd5..7c06f445fd 100644 --- a/shards/mishards/exception_codes.py +++ b/shards/mishards/exception_codes.py @@ -8,3 +8,4 @@ TABLE_NOT_FOUND_CODE = 20001 INVALID_ARGUMENT_CODE = 20002 INVALID_DATE_RANGE_CODE = 20003 INVALID_TOPK_CODE = 20004 +SEARCH_PARAM_LOSS_CODE = 20005 diff --git a/shards/mishards/exceptions.py b/shards/mishards/exceptions.py index 72839f88d2..1661b06263 100644 --- a/shards/mishards/exceptions.py +++ b/shards/mishards/exceptions.py @@ -30,6 +30,10 @@ class InvalidTopKError(BaseException): code = codes.INVALID_TOPK_CODE +class SearchParamError(BaseException): + code = codes.SEARCH_PARAM_LOSS_CODE + + class InvalidArgumentError(BaseException): code = codes.INVALID_ARGUMENT_CODE diff --git a/shards/mishards/grpc_utils/grpc_args_parser.py b/shards/mishards/grpc_utils/grpc_args_parser.py index 59dec55ab9..73c427744a 100644 --- a/shards/mishards/grpc_utils/grpc_args_parser.py +++ b/shards/mishards/grpc_utils/grpc_args_parser.py @@ -1,3 +1,4 @@ +import ujson from milvus import Status from functools import wraps @@ -21,26 +22,30 @@ class GrpcArgsParser(object): @error_status def parse_proto_TableSchema(cls, param): _table_schema = { - 'status': param.status, - 'table_name': param.table_name, + 'collection_name': param.table_name, 'dimension': param.dimension, 'index_file_size': param.index_file_size, 'metric_type': param.metric_type } - return _table_schema + return param.status, _table_schema @classmethod @error_status def parse_proto_TableName(cls, param): return param.table_name + @classmethod + @error_status + def parse_proto_FlushParam(cls, param): + return list(param.table_name_array) + @classmethod @error_status def parse_proto_Index(cls, param): _index = { 'index_type': param.index_type, - 'nlist': param.nlist + 'params': param.extra_params[0].value } return _index @@ -49,12 +54,14 @@ class GrpcArgsParser(object): @error_status def parse_proto_IndexParam(cls, param): _table_name = param.table_name - _status, _index = cls.parse_proto_Index(param.index) + _index_type = param.index_type + _index_param = {} - if not _status.OK(): - raise Exception("Argument parse error") + for params in param.extra_params: + if params.key == 'params': + _index_param = ujson.loads(str(params.value)) - return _table_name, _index + return _table_name, _index_type, _index_param @classmethod @error_status @@ -63,14 +70,6 @@ class GrpcArgsParser(object): return _cmd - @classmethod - @error_status - def parse_proto_Range(cls, param): - _start_value = param.start_value - _end_value = param.end_value - - return _start_value, _end_value - @classmethod @error_status def parse_proto_RowRecord(cls, param): @@ -79,32 +78,52 @@ class GrpcArgsParser(object): @classmethod def parse_proto_PartitionParam(cls, param): _table_name = param.table_name - _partition_name = param.partition_name _tag = param.tag - return _table_name, _partition_name, _tag + return _table_name, _tag @classmethod @error_status def parse_proto_SearchParam(cls, param): _table_name = param.table_name _topk = param.topk - _nprobe = param.nprobe - _status, _range = cls.parse_proto_Range(param.query_range_array) - if not _status.OK(): - raise Exception("Argument parse error") + if len(param.extra_params) == 0: + raise Exception("Search param loss") + _params = ujson.loads(str(param.extra_params[0].value)) - _row_record = param.query_record_array + _query_record_array = [] + if param.query_record_array: + for record in param.query_record_array: + if record.float_data: + _query_record_array.append(list(record.float_data)) + else: + _query_record_array.append(bytes(record.binary_data)) + else: + raise Exception("Search argument parse error: record array is empty") - return _table_name, _row_record, _range, _topk + return _table_name, _query_record_array, _topk, _params @classmethod @error_status - def parse_proto_DeleteByRangeParam(cls, param): + def parse_proto_DeleteByIDParam(cls, param): _table_name = param.table_name - _range = param.range - _start_value = _range.start_value - _end_value = _range.end_value + _id_array = list(param.id_array) - return _table_name, _start_value, _end_value + return _table_name, _id_array + + @classmethod + @error_status + def parse_proto_VectorIdentity(cls, param): + _table_name = param.table_name + _id = param.id + + return _table_name, _id + + @classmethod + @error_status + def parse_proto_GetVectorIDsParam(cls, param): + _table__name = param.table_name + _segment_name = param.segment_name + + return _table__name, _segment_name diff --git a/shards/mishards/models.py b/shards/mishards/models.py index f8c6b55b40..57e03ddd16 100644 --- a/shards/mishards/models.py +++ b/shards/mishards/models.py @@ -23,6 +23,7 @@ class TableFiles(db.Model): id = Column(BigInteger, primary_key=True, autoincrement=True) table_id = Column(String(50)) + segment_id = Column(String(50)) engine_type = Column(Integer) file_id = Column(String(50)) file_type = Column(Integer) @@ -31,6 +32,7 @@ class TableFiles(db.Model): updated_time = Column(BigInteger) created_on = Column(BigInteger) date = Column(Integer) + flush_lsn = Column(Integer) table = relationship( 'Tables', @@ -55,9 +57,10 @@ class Tables(db.Model): created_on = Column(Integer) flag = Column(Integer, default=0) index_file_size = Column(Integer) + index_params = Column(String(50)) engine_type = Column(Integer) - nlist = Column(Integer) metric_type = Column(Integer) + flush_lsn = Column(Integer) def files_to_search(self, date_range=None): cond = or_( diff --git a/shards/mishards/service_handler.py b/shards/mishards/service_handler.py index 332c24e717..701048a2d2 100644 --- a/shards/mishards/service_handler.py +++ b/shards/mishards/service_handler.py @@ -3,13 +3,14 @@ import time import datetime import json from collections import defaultdict +import ujson import multiprocessing from concurrent.futures import ThreadPoolExecutor from milvus.grpc_gen import milvus_pb2, milvus_pb2_grpc, status_pb2 from milvus.grpc_gen.milvus_pb2 import TopKQueryResult -from milvus.client.abstract import Range from milvus.client import types as Types +from milvus import MetricType from mishards import (db, settings, exceptions) from mishards.grpc_utils import mark_grpc_method @@ -106,21 +107,20 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): table_meta, vectors, topk, - nprobe, - range_array=None, + search_params, partition_tags=None, **kwargs): metadata = kwargs.get('metadata', None) - range_array = [ - utilities.range_to_date(r, metadata=metadata) for r in range_array - ] if range_array else None + # range_array = [ + # utilities.range_to_date(r, metadata=metadata) for r in range_array + # ] if range_array else None routing = {} p_span = None if self.tracer.empty else context.get_active_span( ).context with self.tracer.start_span('get_routing', child_of=p_span): routing = self.router.routing(table_id, - range_array=range_array, + # range_array=range_array, partition_tags=partition_tags, metadata=metadata) logger.info('Routing: {}'.format(routing)) @@ -130,10 +130,10 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): rs = [] all_topk_results = [] - def search(addr, table_id, file_ids, vectors, topk, nprobe, **kwargs): + def search(addr, table_id, file_ids, vectors, topk, params, **kwargs): logger.info( - 'Send Search Request: addr={};table_id={};ids={};nq={};topk={};nprobe={}' - .format(addr, table_id, file_ids, len(vectors), topk, nprobe)) + 'Send Search Request: addr={};table_id={};ids={};nq={};topk={};params={}' + .format(addr, table_id, file_ids, len(vectors), topk, params)) conn = self.router.query_conn(addr, metadata=metadata) start = time.time() @@ -143,11 +143,14 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): with self.tracer.start_span('search_{}'.format(addr), child_of=span): - ret = conn.conn.search_vectors_in_files(table_name=table_id, - file_ids=file_ids, - query_records=vectors, - top_k=topk, - nprobe=nprobe) + ret = conn.conn.search_vectors_in_files(collection_name=table_id, + file_ids=file_ids, + query_records=vectors, + top_k=topk, + params=params) + if ret.status.error_code != 0: + logger.error(ret.status) + end = time.time() all_topk_results.append(ret) @@ -155,14 +158,14 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): with self.tracer.start_span('do_search', child_of=p_span) as span: with ThreadPoolExecutor(max_workers=self.max_workers) as pool: for addr, params in routing.items(): - for table_id, file_ids in params.items(): + for sub_table_id, file_ids in params.items(): res = pool.submit(search, addr, table_id, file_ids, vectors, topk, - nprobe, + search_params, span=span) rs.append(res) @@ -177,17 +180,22 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): metadata=metadata) def _create_table(self, table_schema): - return self.router.connection().create_table(table_schema) + return self.router.connection().create_collection(table_schema) @mark_grpc_method def CreateTable(self, request, context): - _status, _table_schema = Parser.parse_proto_TableSchema(request) + _status, unpacks = Parser.parse_proto_TableSchema(request) if not _status.OK(): return status_pb2.Status(error_code=_status.code, reason=_status.message) - logger.info('CreateTable {}'.format(_table_schema['table_name'])) + _status, _table_schema = unpacks + # if _status.error_code == 0: + # logging.warning('[CreateTable] table schema error occurred: {}'.format(_status)) + # return _status + + logger.info('CreateTable {}'.format(_table_schema['collection_name'])) _status = self._create_table(_table_schema) @@ -195,7 +203,7 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): reason=_status.message) def _has_table(self, table_name, metadata=None): - return self.router.connection(metadata=metadata).has_table(table_name) + return self.router.connection(metadata=metadata).has_collection(table_name) @mark_grpc_method def HasTable(self, request, context): @@ -217,14 +225,14 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): @mark_grpc_method def CreatePartition(self, request, context): - _table_name, _partition_name, _tag = Parser.parse_proto_PartitionParam(request) - _status = self.router.connection().create_partition(_table_name, _partition_name, _tag) + _table_name, _tag = Parser.parse_proto_PartitionParam(request) + _status = self.router.connection().create_partition(_table_name, _tag) return status_pb2.Status(error_code=_status.code, reason=_status.message) @mark_grpc_method def DropPartition(self, request, context): - _table_name, _partition_name, _tag = Parser.parse_proto_PartitionParam(request) + _table_name, _tag = Parser.parse_proto_PartitionParam(request) _status = self.router.connection().drop_partition(_table_name, _tag) return status_pb2.Status(error_code=_status.code, @@ -244,13 +252,10 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): return milvus_pb2.PartitionList(status=status_pb2.Status( error_code=_status.code, reason=_status.message), - partition_array=[milvus_pb2.PartitionParam(table_name=param.table_name, - tag=param.tag, - partition_name=param.partition_name) - for param in partition_array]) + partition_tag_array=[param.tag for param in partition_array]) def _delete_table(self, table_name): - return self.router.connection().delete_table(table_name) + return self.router.connection().drop_collection(table_name) @mark_grpc_method def DropTable(self, request, context): @@ -267,8 +272,8 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): return status_pb2.Status(error_code=_status.code, reason=_status.message) - def _create_index(self, table_name, index): - return self.router.connection().create_index(table_name, index) + def _create_index(self, table_name, index_type, param): + return self.router.connection().create_index(table_name, index_type, param) @mark_grpc_method def CreateIndex(self, request, context): @@ -278,12 +283,12 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): return status_pb2.Status(error_code=_status.code, reason=_status.message) - _table_name, _index = unpacks + _table_name, _index_type, _index_param = unpacks logger.info('CreateIndex {}'.format(_table_name)) # TODO: interface create_table incompleted - _status = self._create_index(_table_name, _index) + _status = self._create_index(_table_name, _index_type, _index_param) return status_pb2.Status(error_code=_status.code, reason=_status.message) @@ -305,19 +310,22 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): @mark_grpc_method def Search(self, request, context): + metadata = {'resp_class': milvus_pb2.TopKQueryResult} + table_name = request.table_name topk = request.topk - nprobe = request.nprobe - logger.info('Search {}: topk={} nprobe={}'.format( - table_name, topk, nprobe)) + if len(request.extra_params) == 0: + raise exceptions.SearchParamError(message="Search parma loss", metadata=metadata) + params = ujson.loads(str(request.extra_params[0].value)) - metadata = {'resp_class': milvus_pb2.TopKQueryResult} + logger.info('Search {}: topk={} params={}'.format( + table_name, topk, params)) - if nprobe > self.MAX_NPROBE or nprobe <= 0: - raise exceptions.InvalidArgumentError( - message='Invalid nprobe: {}'.format(nprobe), metadata=metadata) + # if nprobe > self.MAX_NPROBE or nprobe <= 0: + # raise exceptions.InvalidArgumentError( + # message='Invalid nprobe: {}'.format(nprobe), metadata=metadata) if topk > self.MAX_TOPK or topk <= 0: raise exceptions.InvalidTopKError( @@ -327,7 +335,7 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): if not table_meta: status, info = self.router.connection( - metadata=metadata).describe_table(table_name) + metadata=metadata).describe_collection(table_name) if not status.OK(): raise exceptions.TableNotFoundError(table_name, metadata=metadata) @@ -338,27 +346,24 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): start = time.time() query_record_array = [] - - for query_record in request.query_record_array: - query_record_array.append(list(query_record.vector_data)) - - query_range_array = [] - for query_range in request.query_range_array: - query_range_array.append( - Range(query_range.start_value, query_range.end_value)) + if int(table_meta.metric_type) >= MetricType.HAMMING.value: + for query_record in request.query_record_array: + query_record_array.append(bytes(query_record.binary_data)) + else: + for query_record in request.query_record_array: + query_record_array.append(list(query_record.float_data)) status, id_results, dis_results = self._do_query(context, table_name, table_meta, query_record_array, topk, - nprobe, - query_range_array, + params, partition_tags=getattr(request, "partition_tag_array", []), metadata=metadata) now = time.time() - # logger.info('SearchVector takes: {}'.format(now - start)) + logger.info('SearchVector takes: {}'.format(now - start)) topk_result_list = milvus_pb2.TopKQueryResult( status=status_pb2.Status(error_code=status.error_code, @@ -373,7 +378,7 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): raise NotImplemented() def _describe_table(self, table_name, metadata=None): - return self.router.connection(metadata=metadata).describe_table(table_name) + return self.router.connection(metadata=metadata).describe_collection(table_name) @mark_grpc_method def DescribeTable(self, request, context): @@ -405,9 +410,53 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): reason=_status.message), ) + def _table_info(self, table_name, metadata=None): + return self.router.connection(metadata=metadata).collection_info(table_name) + + @mark_grpc_method + def ShowTableInfo(self, request, context): + _status, _table_name = Parser.parse_proto_TableName(request) + + if not _status.OK(): + return milvus_pb2.TableInfo(status=status_pb2.Status( + error_code=_status.code, reason=_status.message), ) + + metadata = {'resp_class': milvus_pb2.TableInfo} + + logger.info('ShowTableInfo {}'.format(_table_name)) + _status, _info = self._table_info(metadata=metadata, table_name=_table_name) + + if _status.OK(): + _table_info = milvus_pb2.TableInfo( + status=status_pb2.Status(error_code=_status.code, + reason=_status.message), + total_row_count=_info.count + ) + + for par_stat in _info.partitions_stat: + _par = milvus_pb2.PartitionStat( + tag=par_stat.tag, + total_row_count=par_stat.count + ) + for seg_stat in par_stat.segments_stat: + _par.segments_stat.add( + segment_name=seg_stat.segment_name, + row_count=seg_stat.count, + index_name=seg_stat.index_name, + data_size=seg_stat.data_size, + ) + + _table_info.partitions_stat.append(_par) + return _table_info + + return milvus_pb2.TableInfo( + status=status_pb2.Status(error_code=_status.code, + reason=_status.message), + ) + def _count_table(self, table_name, metadata=None): return self.router.connection( - metadata=metadata).get_table_row_count(table_name) + metadata=metadata).count_collection(table_name) @mark_grpc_method def CountTable(self, request, context): @@ -432,6 +481,9 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): def _get_server_version(self, metadata=None): return self.router.connection(metadata=metadata).server_version() + def _cmd(self, cmd, metadata=None): + return self.router.connection(metadata=metadata)._cmd(cmd) + @mark_grpc_method def Cmd(self, request, context): _status, _cmd = Parser.parse_proto_Command(request) @@ -449,18 +501,19 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): error_code=status_pb2.SUCCESS), string_reply=json.dumps(stats, indent=2)) - if _cmd == 'version': - _status, _reply = self._get_server_version(metadata=metadata) - else: - _status, _reply = self.router.connection( - metadata=metadata).server_status() + # if _cmd == 'version': + # _status, _reply = self._get_server_version(metadata=metadata) + # else: + # _status, _reply = self.router.connection( + # metadata=metadata).server_status() + _status, _reply = self._cmd(_cmd, metadata=metadata) return milvus_pb2.StringReply(status=status_pb2.Status( error_code=_status.code, reason=_status.message), string_reply=_reply) def _show_tables(self, metadata=None): - return self.router.connection(metadata=metadata).show_tables() + return self.router.connection(metadata=metadata).show_collections() @mark_grpc_method def ShowTables(self, request, context): @@ -472,30 +525,8 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): error_code=_status.code, reason=_status.message), table_names=_results) - def _delete_by_range(self, table_name, start_date, end_date): - return self.router.connection().delete_vectors_by_range(table_name, - start_date, - end_date) - - @mark_grpc_method - def DeleteByRange(self, request, context): - _status, unpacks = \ - Parser.parse_proto_DeleteByRangeParam(request) - - if not _status.OK(): - return status_pb2.Status(error_code=_status.code, - reason=_status.message) - - _table_name, _start_date, _end_date = unpacks - - logger.info('DeleteByRange {}: {} {}'.format(_table_name, _start_date, - _end_date)) - _status = self._delete_by_range(_table_name, _start_date, _end_date) - return status_pb2.Status(error_code=_status.code, - reason=_status.message) - def _preload_table(self, table_name): - return self.router.connection().preload_table(table_name) + return self.router.connection().preload_collection(table_name) @mark_grpc_method def PreloadTable(self, request, context): @@ -531,13 +562,89 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): return milvus_pb2.IndexParam(status=status_pb2.Status( error_code=_status.code, reason=_status.message)) - _index = milvus_pb2.Index(index_type=_index_param._index_type, - nlist=_index_param._nlist) + _index_type = _index_param._index_type - return milvus_pb2.IndexParam(status=status_pb2.Status( + grpc_index = milvus_pb2.IndexParam(status=status_pb2.Status( error_code=_status.code, reason=_status.message), - table_name=_table_name, - index=_index) + table_name=_table_name, index_type=_index_type) + + grpc_index.extra_params.add(key='params', value=ujson.dumps(_index_param._params)) + return grpc_index + + def _get_vector_by_id(self, table_name, vec_id, metadata): + return self.router.connection(metadata=metadata).get_vector_by_id(table_name, vec_id) + + @mark_grpc_method + def GetVectorByID(self, request, context): + _status, unpacks = Parser.parse_proto_VectorIdentity(request) + if not _status.OK(): + return status_pb2.Status(error_code=_status.code, + reason=_status.message) + + metadata = {'resp_class': milvus_pb2.VectorData} + + _table_name, _id = unpacks + logger.info('GetVectorByID {}'.format(_table_name)) + _status, vector = self._get_vector_by_id(_table_name, _id, metadata) + + if not vector: + return milvus_pb2.VectorData(status=status_pb2.Status( + error_code=_status.code, reason=_status.message), ) + + if isinstance(vector, bytes): + records = milvus_pb2.RowRecord(binary_data=vector) + else: + records = milvus_pb2.RowRecord(float_data=vector) + + return milvus_pb2.VectorData(status=status_pb2.Status( + error_code=_status.code, reason=_status.message), + vector_data=records + ) + + def _get_vector_ids(self, table_name, segment_name, metadata): + return self.router.connection(metadata=metadata).get_vector_ids(table_name, segment_name) + + @mark_grpc_method + def GetVectorIDs(self, request, context): + _status, unpacks = Parser.parse_proto_GetVectorIDsParam(request) + + if not _status.OK(): + return status_pb2.Status(error_code=_status.code, + reason=_status.message) + + metadata = {'resp_class': milvus_pb2.VectorIds} + + _table_name, _segment_name = unpacks + logger.info('GetVectorIDs {}'.format(_table_name)) + _status, ids = self._get_vector_ids(_table_name, _segment_name, metadata) + + if not ids: + return milvus_pb2.VectorIds(status=status_pb2.Status( + error_code=_status.code, reason=_status.message), ) + + return milvus_pb2.VectorIds(status=status_pb2.Status( + error_code=_status.code, reason=_status.message), + vector_id_array=ids + ) + + def _delete_by_id(self, table_name, id_array): + return self.router.connection().delete_by_id(table_name, id_array) + + @mark_grpc_method + def DeleteByID(self, request, context): + _status, unpacks = Parser.parse_proto_DeleteByIDParam(request) + + if not _status.OK(): + logging.error('DeleteByID {}'.format(_status.message)) + return status_pb2.Status(error_code=_status.code, + reason=_status.message) + + _table_name, _ids = unpacks + logger.info('DeleteByID {}'.format(_table_name)) + _status = self._delete_by_id(_table_name, _ids) + + return status_pb2.Status(error_code=_status.code, + reason=_status.message) def _drop_index(self, table_name): return self.router.connection().drop_index(table_name) @@ -554,3 +661,35 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): _status = self._drop_index(_table_name) return status_pb2.Status(error_code=_status.code, reason=_status.message) + + def _flush(self, table_names): + return self.router.connection().flush(table_names) + + @mark_grpc_method + def Flush(self, request, context): + _status, _table_names = Parser.parse_proto_FlushParam(request) + + if not _status.OK(): + return status_pb2.Status(error_code=_status.code, + reason=_status.message) + + logger.info('Flush {}'.format(_table_names)) + _status = self._flush(_table_names) + return status_pb2.Status(error_code=_status.code, + reason=_status.message) + + def _compact(self, table_name): + return self.router.connection().compact(table_name) + + @mark_grpc_method + def Compact(self, request, context): + _status, _table_name = Parser.parse_proto_TableName(request) + + if not _status.OK(): + return status_pb2.Status(error_code=_status.code, + reason=_status.message) + + logger.info('Compact {}'.format(_table_name)) + _status = self._compact(_table_name) + return status_pb2.Status(error_code=_status.code, + reason=_status.message) diff --git a/shards/mishards/settings.py b/shards/mishards/settings.py index 730d040de2..493e9f7d66 100644 --- a/shards/mishards/settings.py +++ b/shards/mishards/settings.py @@ -12,7 +12,7 @@ else: env.read_env() -SERVER_VERSIONS = ['0.6.0'] +SERVER_VERSIONS = ['0.7.0'] DEBUG = env.bool('DEBUG', False) MAX_RETRY = env.int('MAX_RETRY', 3) diff --git a/shards/requirements.txt b/shards/requirements.txt index 8f4667f34d..f3a06ac1c7 100644 --- a/shards/requirements.txt +++ b/shards/requirements.txt @@ -14,8 +14,8 @@ py==1.8.0 pyasn1==0.4.7 pyasn1-modules==0.2.6 pylint==2.3.1 -#pymilvus==0.2.5 -pymilvus-test==0.2.42 +#pymilvus==0.2.8 +pymilvus-test==0.3.3 pyparsing==2.4.0 pytest==4.6.3 pytest-level==0.1.1 @@ -30,6 +30,7 @@ requests-oauthlib==1.2.0 rsa==4.0 six==1.12.0 SQLAlchemy==1.3.5 +ujson==2.0.1 urllib3==1.25.8 jaeger-client>=3.4.0 grpcio-opentracing>=1.0