From 2863029665f132bc388376053a1d25111a8e9936 Mon Sep 17 00:00:00 2001 From: "zhenshan.cao" Date: Wed, 9 Sep 2020 18:59:56 +0800 Subject: [PATCH] Fix config Signed-off-by: zhenshan.cao --- conf/conf.go | 74 +++++++++ conf/conf.toml | 0 conf/conf_test.go | 11 ++ conf/config.yaml | 52 ++++++ conf/server_config.template | 188 ---------------------- conf/storage.toml | 1 - go.mod | 1 + proxy/conf/demo/server_config.yaml | 187 --------------------- proxy/conf/log_config.conf | 27 ---- proxy/conf/server_config.template | 188 ---------------------- proxy/conf/server_config.yaml | 109 ------------- proxy/conf/tracing_config.json | 26 --- proxy/src/config/ConfigMgr.cpp | 43 ++--- proxy/src/config/ServerConfig.h | 28 +--- proxy/src/server/Server.cpp | 5 +- proxy/src/server/grpc_impl/GrpcServer.cpp | 2 +- proxy/src/server/init/Directory.cpp | 74 +++++++++ proxy/src/server/init/Directory.h | 42 +++++ proxy/src/server/init/StorageChecker.cpp | 12 -- writer/main.go | 5 +- 20 files changed, 275 insertions(+), 800 deletions(-) create mode 100644 conf/conf.go delete mode 100644 conf/conf.toml create mode 100644 conf/conf_test.go create mode 100644 conf/config.yaml delete mode 100644 conf/server_config.template delete mode 100644 conf/storage.toml delete mode 100644 proxy/conf/demo/server_config.yaml delete mode 100644 proxy/conf/log_config.conf delete mode 100644 proxy/conf/server_config.template delete mode 100644 proxy/conf/server_config.yaml delete mode 100644 proxy/conf/tracing_config.json create mode 100644 proxy/src/server/init/Directory.cpp create mode 100644 proxy/src/server/init/Directory.h diff --git a/conf/conf.go b/conf/conf.go new file mode 100644 index 0000000000..9d69025e88 --- /dev/null +++ b/conf/conf.go @@ -0,0 +1,74 @@ +package conf + +import ( + "io/ioutil" + "github.com/czs007/suvlim/storage/pkg/types" + yaml "gopkg.in/yaml.v2" +) + +// yaml.MapSlice + +type MasterConfig struct { + Address string + Port int32 +} + +type EtcdConfig struct { + Address string + Port int32 + Rootpath string + Segthreshold int64 +} + +type TimeSyncConfig struct { + Interval int32 +} + +type StorageConfig struct { + Driver types.DriverType + Address string + Port int32 + Accesskey string + Secretkey string +} + +type PulsarConfig struct { + Address string + Port int32 +} + +//type ProxyConfig struct { +// Timezone string +// Address string +// Port int32 +//} + +type ServerConfig struct { + Master MasterConfig + Etcd EtcdConfig + Timesync TimeSyncConfig + Storage StorageConfig + Pulsar PulsarConfig + //Proxy ProxyConfig +} + +var Config ServerConfig + +func init() { + load_config() +} + +func load_config() { + //var config ServerConfig + filename := "github.com/czs007/suvlim/conf/config.yaml" + source, err := ioutil.ReadFile(filename) + if err != nil { + panic(err) + } + err = yaml.Unmarshal(source, &Config) + if err != nil { + panic(err) + } + + //fmt.Printf("Result: %v\n", Config) +} diff --git a/conf/conf.toml b/conf/conf.toml deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/conf/conf_test.go b/conf/conf_test.go new file mode 100644 index 0000000000..e133ee5e5e --- /dev/null +++ b/conf/conf_test.go @@ -0,0 +1,11 @@ +package conf + +import ( + "fmt" + "testing" +) + +func TestMain(m *testing.M) { + fmt.Printf("Result: %v\n", Config) +} + diff --git a/conf/config.yaml b/conf/config.yaml new file mode 100644 index 0000000000..d5b85b439b --- /dev/null +++ b/conf/config.yaml @@ -0,0 +1,52 @@ +# Copyright (C) 2019-2020 Zilliz. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under the License. + +master: + address: localhost + port: 6000 + +etcd: + address: localhost + port: 0 + rootpath: a + segthreshold: 10000 + +timesync: + interval: 10 + +storage: + driver: MinIO + address: localhost + port: 0 + accesskey: ab + secretkey: dd + +pulsar: + address: localhost + port: 6650 + +proxy: + timezone: UTC+8 + + network: + address: 0.0.0.0 + port: 19530 + + logs: + level: debug + trace.enable: true + path: /tmp/logs + max_log_file_size: 1024MB + log_rotate_num: 0 + + storage: + path: /var/lib/milvus + auto_flush_interval: 1 diff --git a/conf/server_config.template b/conf/server_config.template deleted file mode 100644 index 2ccab5e200..0000000000 --- a/conf/server_config.template +++ /dev/null @@ -1,188 +0,0 @@ -# Copyright (C) 2019-2020 Zilliz. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under the License. - -version: 0.5 - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# Cluster Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# enable | If running with Mishards, set true, otherwise false. | Boolean | false | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# role | Milvus deployment role: rw / ro | Role | rw | -#----------------------+------------------------------------------------------------+------------+-----------------+ -cluster: - enable: false - role: rw - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# General Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# timezone | Use UTC-x or UTC+x to specify a time zone. | Timezone | UTC+8 | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# meta_uri | URI for metadata storage, using SQLite (for single server | URI | sqlite://:@:/ | -# | Milvus) or MySQL (for distributed cluster Milvus). | | | -# | Format: dialect://username:password@host:port/database | | | -# | Keep 'dialect://:@:/', 'dialect' can be either 'sqlite' or | | | -# | 'mysql', replace other texts with real values. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -general: - timezone: UTC+8 - meta_uri: sqlite://:@:/ - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# Network Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# bind.address | IP address that Milvus server monitors. | IP | 0.0.0.0 | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# bind.port | Port that Milvus server monitors. Port range (1024, 65535) | Integer | 19530 | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# http.enable | Enable HTTP server or not. | Boolean | true | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# http.port | Port that Milvus HTTP server monitors. | Integer | 19121 | -# | Port range (1024, 65535) | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -network: - bind.address: 0.0.0.0 - bind.port: 19530 - http.enable: true - http.port: 19121 - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# Storage Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# path | Path used to save meta data, vector data and index data. | Path | /var/lib/milvus | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# auto_flush_interval | The interval, in seconds, at which Milvus automatically | Integer | 1 (s) | -# | flushes data to disk. | | | -# | 0 means disable the regular flush. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -storage: - path: @MILVUS_DB_PATH@ - auto_flush_interval: 1 - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# WAL Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# enable | Whether to enable write-ahead logging (WAL) in Milvus. | Boolean | true | -# | If WAL is enabled, Milvus writes all data changes to log | | | -# | files in advance before implementing data changes. WAL | | | -# | ensures the atomicity and durability for Milvus operations.| | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# recovery_error_ignore| Whether to ignore logs with errors that happens during WAL | Boolean | false | -# | recovery. If true, when Milvus restarts for recovery and | | | -# | there are errors in WAL log files, log files with errors | | | -# | are ignored. If false, Milvus does not restart when there | | | -# | are errors in WAL log files. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# buffer_size | Sum total of the read buffer and the write buffer in Bytes.| String | 256MB | -# | buffer_size must be in range [64MB, 4096MB]. | | | -# | If the value you specified is out of range, Milvus | | | -# | automatically uses the boundary value closest to the | | | -# | specified value. It is recommended you set buffer_size to | | | -# | a value greater than the inserted data size of a single | | | -# | insert operation for better performance. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# path | Location of WAL log files. | String | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -wal: - enable: true - recovery_error_ignore: false - buffer_size: 256MB - path: @MILVUS_DB_PATH@/wal - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# Cache Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# cache_size | The size of CPU memory used for caching data for faster | String | 4GB | -# | query. The sum of 'cache_size' and 'insert_buffer_size' | | | -# | must be less than system memory size. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# insert_buffer_size | Buffer size used for data insertion. | String | 1GB | -# | The sum of 'insert_buffer_size' and 'cache_size' | | | -# | must be less than system memory size. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# preload_collection | A comma-separated list of collection names that need to | StringList | | -# | be pre-loaded when Milvus server starts up. | | | -# | '*' means preload all existing tables (single-quote or | | | -# | double-quote required). | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -cache: - cache_size: 4GB - insert_buffer_size: 1GB - preload_collection: - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# GPU Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# enable | Use GPU devices or not. | Boolean | false | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# cache_size | The size of GPU memory per card used for cache. | String | 1GB | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# gpu_search_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1000 | -# | compared with 'nq' to decide if the search computation will| | | -# | be executed on GPUs only. | | | -# | If nq >= gpu_search_threshold, the search computation will | | | -# | be executed on GPUs only; | | | -# | if nq < gpu_search_threshold, the search computation will | | | -# | be executed on both CPUs and GPUs. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# search_devices | The list of GPU devices used for search computation. | DeviceList | gpu0 | -# | Must be in format gpux. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# build_index_devices | The list of GPU devices used for index building. | DeviceList | gpu0 | -# | Must be in format gpux. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -gpu: - enable: @GPU_ENABLE@ - cache_size: 1GB - gpu_search_threshold: 1000 - search_devices: - - gpu0 - build_index_devices: - - gpu0 - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# Logs Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# level | Log level in Milvus. Must be one of debug, info, warning, | String | debug | -# | error, fatal | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# trace.enable | Whether to enable trace level logging in Milvus. | Boolean | true | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# path | Absolute path to the folder holding the log files. | String | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# max_log_file_size | The maximum size of each log file, size range | String | 1024MB | -# | [512MB, 4096MB]. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# log_rotate_num | The maximum number of log files that Milvus keeps for each | Integer | 0 | -# | logging level, num range [0, 1024], 0 means unlimited. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -logs: - level: debug - trace.enable: true - path: @MILVUS_DB_PATH@/logs - max_log_file_size: 1024MB - log_rotate_num: 0 - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# Metric Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# enable | Enable monitoring function or not. | Boolean | false | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# address | Pushgateway address | IP | 127.0.0.1 + -#----------------------+------------------------------------------------------------+------------+-----------------+ -# port | Pushgateway port, port range (1024, 65535) | Integer | 9091 | -#----------------------+------------------------------------------------------------+------------+-----------------+ -metric: - enable: false - address: 127.0.0.1 - port: 9091 - diff --git a/conf/storage.toml b/conf/storage.toml deleted file mode 100644 index 08208e61da..0000000000 --- a/conf/storage.toml +++ /dev/null @@ -1 +0,0 @@ -driver="S3" diff --git a/go.mod b/go.mod index b630eba116..ad39f82d2e 100644 --- a/go.mod +++ b/go.mod @@ -43,6 +43,7 @@ require ( google.golang.org/grpc/examples v0.0.0-20200828165940-d8ef479ab79a // indirect google.golang.org/protobuf v1.25.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 + gopkg.in/yaml.v2 v2.3.0 sigs.k8s.io/yaml v1.2.0 // indirect ) diff --git a/proxy/conf/demo/server_config.yaml b/proxy/conf/demo/server_config.yaml deleted file mode 100644 index b9626eb952..0000000000 --- a/proxy/conf/demo/server_config.yaml +++ /dev/null @@ -1,187 +0,0 @@ -# Copyright (C) 2019-2020 Zilliz. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under the License. - -version: 0.5 - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# Cluster Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# enable | If runinng with Mishards, set true, otherwise false. | Boolean | false | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# role | Milvus deployment role: rw / ro | role | rw | -#----------------------+------------------------------------------------------------+------------+-----------------+ -cluster: - enable: false - role: rw - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# General Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# time_zone | Use UTC-x or UTC+x to specify a time zone. | Timezone | UTC+8 | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# meta_uri | URI for metadata storage, using SQLite (for single server | URL | sqlite://:@:/ | -# | Milvus) or MySQL (for distributed cluster Milvus). | | | -# | Format: dialect://username:password@host:port/database | | | -# | Keep 'dialect://:@:/', 'dialect' can be either 'sqlite' or | | | -# | 'mysql', replace other texts with real values. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -general: - timezone: UTC+8 - meta_uri: sqlite://:@:/ - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# Network Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# bind.address | IP address that Milvus server monitors. | IP | 0.0.0.0 | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# bind.port | Port that Milvus server monitors. Port range (1024, 65535) | Integer | 19530 | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# http.enable | Enable web server or not. | Boolean | true | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# http.port | Port that Milvus web server monitors. | Integer | 19121 | -# | Port range (1024, 65535) | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -network: - bind.address: 0.0.0.0 - bind.port: 19530 - http.enable: true - http.port: 19121 - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# Storage Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# path | Path used to save meta data, vector data and index data. | Path | /var/lib/milvus | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# auto_flush_interval | The interval, in seconds, at which Milvus automatically | Integer | 1 (s) | -# | flushes data to disk. | | | -# | 0 means disable the regular flush. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -storage: - path: /var/lib/milvus - auto_flush_interval: 1 - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# WAL Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# enable | Whether to enable write-ahead logging (WAL) in Milvus. | Boolean | true | -# | If WAL is enabled, Milvus writes all data changes to log | | | -# | files in advance before implementing data changes. WAL | | | -# | ensures the atomicity and durability for Milvus operations.| | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# recovery_error_ignore| Whether to ignore logs with errors that happens during WAL | Boolean | false | -# | recovery. If true, when Milvus restarts for recovery and | | | -# | there are errors in WAL log files, log files with errors | | | -# | are ignored. If false, Milvus does not restart when there | | | -# | are errors in WAL log files. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# buffer_size | Sum total of the read buffer and the write buffer in MBs. | Integer | 256 (MB) | -# | buffer_size must be in range [64, 4096] (MB). | | | -# | If the value you specified is out of range, Milvus | | | -# | automatically uses the boundary value closest to the | | | -# | specified value. It is recommended you set buffer_size to | | | -# | a value greater than the inserted data size of a single | | | -# | insert operation for better performance. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# path | Location of WAL log files. | String | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -wal: - enable: true - recovery_error_ignore: false - buffer_size: 256MB - path: /var/lib/milvus/wal - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# Cache Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# cache_size | The size of CPU memory used for caching data for faster | Integer | 4 (GB) | -# | query. The sum of 'cpu_cache_capacity' and | | | -# | 'insert_buffer_size' must be less than system memory size. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# insert_buffer_size | Buffer size used for data insertion. | Integer | 1 (GB) | -# | The sum of 'insert_buffer_size' and 'cpu_cache_capacity' | | | -# | must be less than system memory size. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# preload_collection | A comma-separated list of collection names that need to | StringList | | -# | be pre-loaded when Milvus server starts up. | | | -# | '*' means preload all existing tables (single-quote or | | | -# | double-quote required). | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -cache: - cache_size: 4GB - insert_buffer_size: 1GB - preload_collection: - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# GPU Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# enable | Enable GPU resources or not. | Boolean | false | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# cache_size | The size of GPU memory per card used for cache. | Integer | 1 (GB) | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# gpu_search_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1000 | -# | compared with 'nq' to decide if the search computation will| | | -# | be executed on GPUs only. | | | -# | If nq >= gpu_search_threshold, the search computation will | | | -# | be executed on GPUs only; | | | -# | if nq < gpu_search_threshold, the search computation will | | | -# | be executed on both CPUs and GPUs. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# search_resources | The list of GPU devices used for search computation. | DeviceList | gpu0 | -# | Must be in format gpux. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# build_index_resources| The list of GPU devices used for index building. | DeviceList | gpu0 | -# | Must be in format gpux. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -gpu: - enable: false - cache_size: 1GB - gpu_search_threshold: 1000 - search_devices: - - gpu0 - build_index_devices: - - gpu0 - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# Logs Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# level | Log level in Milvus. Must be one of debug, info, warning, | String | debug | -# | error, fatal | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# trace.enable | Whether to enable trace level logging in Milvus. | Boolean | true | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# path | Absolute path to the folder holding the log files. | String | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# max_log_file_size | The maximum size of each log file, size range [512, 4096] | Integer | 1024 (MB) | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# log_rotate_num | The maximum number of log files that Milvus keeps for each | Integer | 0 | -# | logging level, num range [0, 1024], 0 means unlimited. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -logs: - level: debug - trace.enable: true - path: /var/lib/milvus/logs - max_log_file_size: 1024MB - log_rotate_num: 0 - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# Metric Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# enable | Enable monitoring function or not. | Boolean | false | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# address | Pushgateway address | IP | 127.0.0.1 + -#----------------------+------------------------------------------------------------+------------+-----------------+ -# port | Pushgateway port, port range (1024, 65535) | Integer | 9091 | -#----------------------+------------------------------------------------------------+------------+-----------------+ -metric: - enable: false - address: 127.0.0.1 - port: 9091 - diff --git a/proxy/conf/log_config.conf b/proxy/conf/log_config.conf deleted file mode 100644 index 574ccc2b5f..0000000000 --- a/proxy/conf/log_config.conf +++ /dev/null @@ -1,27 +0,0 @@ -* GLOBAL: - FORMAT = "%datetime | %level | %logger | %msg" - FILENAME = "/usr/local/logs/milvus-%datetime{%y-%M-%d-%H:%m}-global.log" - ENABLED = true - TO_FILE = true - TO_STANDARD_OUTPUT = false - SUBSECOND_PRECISION = 3 - PERFORMANCE_TRACKING = false - MAX_LOG_FILE_SIZE = 209715200 ## Throw log files away after 200MB -* DEBUG: - FILENAME = "/usr/local/logs/milvus-%datetime{%y-%M-%d-%H:%m}-debug.log" - ENABLED = true -* WARNING: - FILENAME = "/usr/local/logs/milvus-%datetime{%y-%M-%d-%H:%m}-warning.log" -* TRACE: - FILENAME = "/usr/local/logs/milvus-%datetime{%y-%M-%d-%H:%m}-trace.log" -* VERBOSE: - FORMAT = "%datetime{%d/%M/%y} | %level-%vlevel | %msg" - TO_FILE = false - TO_STANDARD_OUTPUT = false -## Error logs -* ERROR: - ENABLED = true - FILENAME = "/usr/local/logs/milvus-%datetime{%y-%M-%d-%H:%m}-error.log" -* FATAL: - ENABLED = true - FILENAME = "/usr/local/logs/milvus-%datetime{%y-%M-%d-%H:%m}-fatal.log" diff --git a/proxy/conf/server_config.template b/proxy/conf/server_config.template deleted file mode 100644 index 2ccab5e200..0000000000 --- a/proxy/conf/server_config.template +++ /dev/null @@ -1,188 +0,0 @@ -# Copyright (C) 2019-2020 Zilliz. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under the License. - -version: 0.5 - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# Cluster Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# enable | If running with Mishards, set true, otherwise false. | Boolean | false | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# role | Milvus deployment role: rw / ro | Role | rw | -#----------------------+------------------------------------------------------------+------------+-----------------+ -cluster: - enable: false - role: rw - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# General Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# timezone | Use UTC-x or UTC+x to specify a time zone. | Timezone | UTC+8 | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# meta_uri | URI for metadata storage, using SQLite (for single server | URI | sqlite://:@:/ | -# | Milvus) or MySQL (for distributed cluster Milvus). | | | -# | Format: dialect://username:password@host:port/database | | | -# | Keep 'dialect://:@:/', 'dialect' can be either 'sqlite' or | | | -# | 'mysql', replace other texts with real values. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -general: - timezone: UTC+8 - meta_uri: sqlite://:@:/ - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# Network Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# bind.address | IP address that Milvus server monitors. | IP | 0.0.0.0 | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# bind.port | Port that Milvus server monitors. Port range (1024, 65535) | Integer | 19530 | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# http.enable | Enable HTTP server or not. | Boolean | true | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# http.port | Port that Milvus HTTP server monitors. | Integer | 19121 | -# | Port range (1024, 65535) | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -network: - bind.address: 0.0.0.0 - bind.port: 19530 - http.enable: true - http.port: 19121 - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# Storage Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# path | Path used to save meta data, vector data and index data. | Path | /var/lib/milvus | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# auto_flush_interval | The interval, in seconds, at which Milvus automatically | Integer | 1 (s) | -# | flushes data to disk. | | | -# | 0 means disable the regular flush. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -storage: - path: @MILVUS_DB_PATH@ - auto_flush_interval: 1 - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# WAL Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# enable | Whether to enable write-ahead logging (WAL) in Milvus. | Boolean | true | -# | If WAL is enabled, Milvus writes all data changes to log | | | -# | files in advance before implementing data changes. WAL | | | -# | ensures the atomicity and durability for Milvus operations.| | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# recovery_error_ignore| Whether to ignore logs with errors that happens during WAL | Boolean | false | -# | recovery. If true, when Milvus restarts for recovery and | | | -# | there are errors in WAL log files, log files with errors | | | -# | are ignored. If false, Milvus does not restart when there | | | -# | are errors in WAL log files. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# buffer_size | Sum total of the read buffer and the write buffer in Bytes.| String | 256MB | -# | buffer_size must be in range [64MB, 4096MB]. | | | -# | If the value you specified is out of range, Milvus | | | -# | automatically uses the boundary value closest to the | | | -# | specified value. It is recommended you set buffer_size to | | | -# | a value greater than the inserted data size of a single | | | -# | insert operation for better performance. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# path | Location of WAL log files. | String | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -wal: - enable: true - recovery_error_ignore: false - buffer_size: 256MB - path: @MILVUS_DB_PATH@/wal - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# Cache Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# cache_size | The size of CPU memory used for caching data for faster | String | 4GB | -# | query. The sum of 'cache_size' and 'insert_buffer_size' | | | -# | must be less than system memory size. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# insert_buffer_size | Buffer size used for data insertion. | String | 1GB | -# | The sum of 'insert_buffer_size' and 'cache_size' | | | -# | must be less than system memory size. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# preload_collection | A comma-separated list of collection names that need to | StringList | | -# | be pre-loaded when Milvus server starts up. | | | -# | '*' means preload all existing tables (single-quote or | | | -# | double-quote required). | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -cache: - cache_size: 4GB - insert_buffer_size: 1GB - preload_collection: - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# GPU Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# enable | Use GPU devices or not. | Boolean | false | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# cache_size | The size of GPU memory per card used for cache. | String | 1GB | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# gpu_search_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1000 | -# | compared with 'nq' to decide if the search computation will| | | -# | be executed on GPUs only. | | | -# | If nq >= gpu_search_threshold, the search computation will | | | -# | be executed on GPUs only; | | | -# | if nq < gpu_search_threshold, the search computation will | | | -# | be executed on both CPUs and GPUs. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# search_devices | The list of GPU devices used for search computation. | DeviceList | gpu0 | -# | Must be in format gpux. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# build_index_devices | The list of GPU devices used for index building. | DeviceList | gpu0 | -# | Must be in format gpux. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -gpu: - enable: @GPU_ENABLE@ - cache_size: 1GB - gpu_search_threshold: 1000 - search_devices: - - gpu0 - build_index_devices: - - gpu0 - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# Logs Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# level | Log level in Milvus. Must be one of debug, info, warning, | String | debug | -# | error, fatal | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# trace.enable | Whether to enable trace level logging in Milvus. | Boolean | true | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# path | Absolute path to the folder holding the log files. | String | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# max_log_file_size | The maximum size of each log file, size range | String | 1024MB | -# | [512MB, 4096MB]. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# log_rotate_num | The maximum number of log files that Milvus keeps for each | Integer | 0 | -# | logging level, num range [0, 1024], 0 means unlimited. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -logs: - level: debug - trace.enable: true - path: @MILVUS_DB_PATH@/logs - max_log_file_size: 1024MB - log_rotate_num: 0 - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# Metric Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# enable | Enable monitoring function or not. | Boolean | false | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# address | Pushgateway address | IP | 127.0.0.1 + -#----------------------+------------------------------------------------------------+------------+-----------------+ -# port | Pushgateway port, port range (1024, 65535) | Integer | 9091 | -#----------------------+------------------------------------------------------------+------------+-----------------+ -metric: - enable: false - address: 127.0.0.1 - port: 9091 - diff --git a/proxy/conf/server_config.yaml b/proxy/conf/server_config.yaml deleted file mode 100644 index ac8dfdfcea..0000000000 --- a/proxy/conf/server_config.yaml +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright (C) 2019-2020 Zilliz. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under the License. - -version: 0.5 - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# General Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# timezone | Use UTC-x or UTC+x to specify a time zone. | Timezone | UTC+8 | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# meta_uri | URI for metadata storage, using SQLite (for single server | URI | sqlite://:@:/ | -# | Milvus) or MySQL (for distributed cluster Milvus). | | | -# | Format: dialect://username:password@host:port/database | | | -# | Keep 'dialect://:@:/', 'dialect' can be either 'sqlite' or | | | -# | 'mysql', replace other texts with real values. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -general: - timezone: UTC+8 - meta_uri: sqlite://:@:/ - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# Network Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# bind.address | IP address that Milvus server monitors. | IP | 0.0.0.0 | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# bind.port | Port that Milvus server monitors. Port range (1024, 65535) | Integer | 19530 | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# http.enable | Enable HTTP server or not. | Boolean | true | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# http.port | Port that Milvus HTTP server monitors. | Integer | 19121 | -# | Port range (1024, 65535) | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -network: - bind.address: 0.0.0.0 - bind.port: 19530 - http.enable: true - http.port: 19121 - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# Storage Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# path | Path used to save meta data, vector data and index data. | Path | /var/lib/milvus | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# auto_flush_interval | The interval, in seconds, at which Milvus automatically | Integer | 1 (s) | -# | flushes data to disk. | | | -# | 0 means disable the regular flush. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -storage: - path: /tmp - auto_flush_interval: 1 - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# Cache Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# cache_size | The size of CPU memory used for caching data for faster | String | 4GB | -# | query. The sum of 'cache_size' and 'insert_buffer_size' | | | -# | must be less than system memory size. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# insert_buffer_size | Buffer size used for data insertion. | String | 1GB | -# | The sum of 'insert_buffer_size' and 'cache_size' | | | -# | must be less than system memory size. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# preload_collection | A comma-separated list of collection names that need to | StringList | | -# | be pre-loaded when Milvus server starts up. | | | -# | '*' means preload all existing tables (single-quote or | | | -# | double-quote required). | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -cache: - cache_size: 4GB - insert_buffer_size: 1GB - preload_collection: - -#----------------------+------------------------------------------------------------+------------+-----------------+ -# Logs Config | Description | Type | Default | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# level | Log level in Milvus. Must be one of debug, info, warning, | String | debug | -# | error, fatal | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# trace.enable | Whether to enable trace level logging in Milvus. | Boolean | true | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# path | Absolute path to the folder holding the log files. | String | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# max_log_file_size | The maximum size of each log file, size range | String | 1024MB | -# | [512MB, 4096MB]. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -# log_rotate_num | The maximum number of log files that Milvus keeps for each | Integer | 0 | -# | logging level, num range [0, 1024], 0 means unlimited. | | | -#----------------------+------------------------------------------------------------+------------+-----------------+ -logs: - level: debug - trace.enable: true - path: /tmp/logs - max_log_file_size: 1024MB - log_rotate_num: 0 - -metric: - level: debug - -pulsar: - address: localhost - port: 6650 diff --git a/proxy/conf/tracing_config.json b/proxy/conf/tracing_config.json deleted file mode 100644 index 3a80614390..0000000000 --- a/proxy/conf/tracing_config.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "host": "127.0.0.1", - "port": "5666", - "tracer_library": "/path/to/shared_tracing_library", - "tracer_configuration": { - "service_name": "milvus_server", - "sampler": { - "type": "const", - "param": "1" - }, - "disabled": false, - "reporter": { - "localAgentHostPort": "127.0.0.1:6831" - }, - "headers": { - "jaegerDebugHeader": "jaeger_debug_header", - "jaegerBaggageHeader": "jarger_baggage_header", - "TraceContextHeaderName": "trace_context_header_name", - "traceBaggageHeaderPrefix": "trace_baggage_header_prefix" - }, - "baggage_restrictions": { - "denyBaggageOnInitializationFailure": false, - "hostPort": "" - } - } -} diff --git a/proxy/src/config/ConfigMgr.cpp b/proxy/src/config/ConfigMgr.cpp index c3d5b5c046..22e830a69e 100644 --- a/proxy/src/config/ConfigMgr.cpp +++ b/proxy/src/config/ConfigMgr.cpp @@ -68,22 +68,16 @@ ConfigMgr ConfigMgr::instance; ConfigMgr::ConfigMgr() { config_list_ = { - /* version */ - {"version", CreateStringConfig("version", false, &config.version.value, "unknown", nullptr, nullptr)}, /* general */ - {"general.timezone", - CreateStringConfig("general.timezone", false, &config.general.timezone.value, "UTC+8", nullptr, nullptr)}, + {"timezone", + CreateStringConfig("timezone", false, &config.timezone.value, "UTC+8", nullptr, nullptr)}, /* network */ - {"network.bind.address", CreateStringConfig("network.bind.address", false, &config.network.bind.address.value, + {"network.address", CreateStringConfig("network.address", false, &config.network.address.value, "0.0.0.0", nullptr, nullptr)}, - {"network.bind.port", CreateIntegerConfig("network.bind.port", false, 0, 65535, &config.network.bind.port.value, + {"network.port", CreateIntegerConfig("network.port", false, 0, 65535, &config.network.port.value, 19530, nullptr, nullptr)}, - {"network.http.enable", - CreateBoolConfig("network.http.enable", false, &config.network.http.enable.value, true, nullptr, nullptr)}, - {"network.http.port", CreateIntegerConfig("network.http.port", false, 0, 65535, &config.network.http.port.value, - 19121, nullptr, nullptr)}, /* pulsar */ @@ -92,27 +86,6 @@ ConfigMgr::ConfigMgr() { {"pulsar.port", CreateIntegerConfig("pulsar.port", false, 0, 65535, &config.pulsar.port.value, 6650, nullptr, nullptr)}, - /* storage */ - {"storage.path", - CreateStringConfig("storage.path", false, &config.storage.path.value, "/var/lib/milvus", nullptr, nullptr)}, - {"storage.auto_flush_interval", - CreateIntegerConfig("storage.auto_flush_interval", true, 0, std::numeric_limits::max(), - &config.storage.auto_flush_interval.value, 1, nullptr, nullptr)}, - - - /* cache */ - {"cache.cache_size", CreateSizeConfig("cache.cache_size", true, 0, std::numeric_limits::max(), - &config.cache.cache_size.value, 4 * GB, nullptr, nullptr)}, - {"cache.cpu_cache_threshold", - CreateFloatingConfig("cache.cpu_cache_threshold", false, 0.0, 1.0, &config.cache.cpu_cache_threshold.value, - 0.7, nullptr, nullptr)}, - {"cache.insert_buffer_size", - CreateSizeConfig("cache.insert_buffer_size", false, 0, std::numeric_limits::max(), - &config.cache.insert_buffer_size.value, 1 * GB, nullptr, nullptr)}, - {"cache.cache_insert_data", CreateBoolConfig("cache.cache_insert_data", false, - &config.cache.cache_insert_data.value, false, nullptr, nullptr)}, - {"cache.preload_collection", CreateStringConfig("cache.preload_collection", false, - &config.cache.preload_collection.value, "", nullptr, nullptr)}, /* log */ {"logs.level", CreateStringConfig("logs.level", false, &config.logs.level.value, "debug", nullptr, nullptr)}, @@ -162,13 +135,19 @@ ConfigMgr::Load(const std::string& path) { auto yaml = YAML::LoadFile(path); /* make it flattened */ std::unordered_map flattened; - Flatten(yaml, flattened, ""); + // auto proxy_yaml = yaml["porxy"]; + auto other_yaml = YAML::Node{}; + other_yaml["pulsar"] = yaml["pulsar"]; + Flatten(yaml["proxy"], flattened, ""); + Flatten(other_yaml, flattened, ""); + // Flatten(yaml["proxy"], flattened, ""); /* update config */ for (auto& it : flattened) Set(it.first, it.second, false); } void ConfigMgr::Set(const std::string& name, const std::string& value, bool update) { + std::cout<<"InSet Config "<< name <; using Floating = ConfigValue; - String version{"unknown"}; - - struct General { - String timezone{"unknown"}; - } general; + String timezone{"unknown"}; struct Network { - struct Bind { - String address{"unknown"}; - Integer port{0}; - } bind; - struct Http { - Bool enable{false}; - Integer port{0}; - } http; + String address{"unknown"}; + Integer port{0}; } network; struct Pulsar{ @@ -86,18 +76,6 @@ struct ServerConfig { Integer port{6650}; }pulsar; - struct Storage { - String path{"unknown"}; - Integer auto_flush_interval{0}; - } storage; - - struct Cache { - Integer cache_size{0}; - Floating cpu_cache_threshold{0.0}; - Integer insert_buffer_size{0}; - Bool cache_insert_data{false}; - String preload_collection{"unknown"}; - } cache; struct Engine { Integer build_index_threshold{4096}; diff --git a/proxy/src/server/Server.cpp b/proxy/src/server/Server.cpp index a7b1898a29..0f05bb39be 100644 --- a/proxy/src/server/Server.cpp +++ b/proxy/src/server/Server.cpp @@ -24,6 +24,7 @@ // #include "scheduler/SchedInst.h" #include "server/grpc_impl/GrpcServer.h" #include "server/init/CpuChecker.h" +#include "server/init/Directory.h" // #include "server/init/GpuChecker.h" #include "server/init/StorageChecker.h" #include "src/version.h" @@ -146,7 +147,7 @@ Server::Start() { tracing_config_path.empty() ? tracing::TracerUtil::InitGlobal() : tracing::TracerUtil::InitGlobal(tracing_config_path); - auto time_zone = config.general.timezone(); + auto time_zone = config.timezone(); if (time_zone.length() == 3) { time_zone = "CUT"; @@ -170,6 +171,8 @@ Server::Start() { STATUS_CHECK(LogMgr::InitLog(config.logs.trace.enable(), config.logs.level(), config.logs.path(), config.logs.max_log_file_size(), config.logs.log_rotate_num())); + STATUS_CHECK(Directory::Initialize(config.logs.path())); + // print version information LOG_SERVER_INFO_ << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << ", built at " << BUILD_TIME; #ifdef MILVUS_GPU_VERSION diff --git a/proxy/src/server/grpc_impl/GrpcServer.cpp b/proxy/src/server/grpc_impl/GrpcServer.cpp index 1b0c7ca8da..2c3c73ea57 100644 --- a/proxy/src/server/grpc_impl/GrpcServer.cpp +++ b/proxy/src/server/grpc_impl/GrpcServer.cpp @@ -84,7 +84,7 @@ Status GrpcServer::StartService() { SetThreadName("grpcserv_thread"); - std::string server_address(config.network.bind.address() + ":" + std::to_string(config.network.bind.port())); + std::string server_address(config.network.address() + ":" + std::to_string(config.network.port())); ::grpc::ServerBuilder builder; builder.SetOption(std::unique_ptr<::grpc::ServerBuilderOption>(new NoReusePortOption)); diff --git a/proxy/src/server/init/Directory.cpp b/proxy/src/server/init/Directory.cpp new file mode 100644 index 0000000000..6b868a2e85 --- /dev/null +++ b/proxy/src/server/init/Directory.cpp @@ -0,0 +1,74 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License. + +#include "server/init/Directory.h" + +#include +#include +#include +#include + +#include "config/ServerConfig.h" + +namespace milvus::server { + +Status +Directory::Initialize(const std::string& path) { + try { + init(path); + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + return Status::OK(); +} + + +Status +Directory::Access(const std::string& path) { + try { + access_check(path); + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + return Status::OK(); +} + +void +Directory::init(const std::string& path) { + if (path.empty()) { + return; + } + try { + // Returns True if a new directory was created, otherwise false. + boost::filesystem::create_directories(path); + } catch (std::exception& ex) { + std::string msg = "Cannot create directory: " + path + ", reason: " + ex.what(); + throw std::runtime_error(msg); + } catch (...) { + std::string msg = "Cannot create directory: " + path; + throw std::runtime_error(msg); + } +} + +void +Directory::access_check(const std::string& path) { + if (path.empty()) { + return; + } + int ret = access(path.c_str(), F_OK | R_OK | W_OK); + if (0 != ret) { + std::string msg = "Cannot access path: " + path + ", error(" + std::to_string(errno) + + "): " + std::string(strerror(errno)) + "."; + throw std::runtime_error(msg); + } +} + +} // namespace milvus::server diff --git a/proxy/src/server/init/Directory.h b/proxy/src/server/init/Directory.h new file mode 100644 index 0000000000..0e405c3d77 --- /dev/null +++ b/proxy/src/server/init/Directory.h @@ -0,0 +1,42 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License. + +#pragma once + +#include + +#include "utils/Status.h" + +namespace milvus::server { + +class Directory { + public: + static Status + Initialize(const std::string& path); + + static Status + Lock(const std::string& path); + + static Status + Access(const std::string& path); + + private: + static void + init(const std::string& path); + + static void + lock(const std::string& path); + + static void + access_check(const std::string& path); +}; + +} // namespace milvus::server diff --git a/proxy/src/server/init/StorageChecker.cpp b/proxy/src/server/init/StorageChecker.cpp index 8f6f8ffb9e..939c67ca59 100644 --- a/proxy/src/server/init/StorageChecker.cpp +++ b/proxy/src/server/init/StorageChecker.cpp @@ -37,18 +37,6 @@ StorageChecker::CheckStoragePermission() { return Status(SERVER_UNEXPECTED_ERROR, err_msg); } - /* Check db directory write permission */ - const std::string& primary_path = config.storage.path(); - - ret = access(primary_path.c_str(), F_OK | R_OK | W_OK); - if (0 != ret) { - std::string err_msg = " Access DB storage path " + primary_path + " fail. " + strerror(errno) + - "(code: " + std::to_string(errno) + ")"; - LOG_SERVER_FATAL_ << err_msg; - std::cerr << err_msg << std::endl; - return Status(SERVER_UNEXPECTED_ERROR, err_msg); - } - return Status::OK(); } diff --git a/writer/main.go b/writer/main.go index 5b4267f55c..0ffb0e9df7 100644 --- a/writer/main.go +++ b/writer/main.go @@ -4,9 +4,9 @@ import ( "context" "fmt" "github.com/czs007/suvlim/storage/pkg" - "github.com/czs007/suvlim/storage/pkg/types" "github.com/czs007/suvlim/writer/message_client" "github.com/czs007/suvlim/writer/write_node" + "github.com/czs007/suvlim/conf" "log" "sync" "time" @@ -16,14 +16,13 @@ func main() { mc := message_client.MessageClient{} mc.InitClient("pulsar://localhost:6650") - //mc.InitClient("pulsar://192.168.2.18:6650") //TODO::close client / consumer/ producer //mc.Close() go mc.ReceiveMessage() wg := sync.WaitGroup{} ctx := context.Background() - kv, err := storage.NewStore(ctx, types.MinIODriver) + kv, err := storage.NewStore(ctx, conf.Config.Storage.Driver) // if err != nil, should retry link if err != nil { log.Fatal(err)