diff --git a/core/src/server/Server.cpp b/core/src/server/Server.cpp index f8b77d7406..261e235d47 100644 --- a/core/src/server/Server.cpp +++ b/core/src/server/Server.cpp @@ -10,7 +10,6 @@ // or implied. See the License for the specific language governing permissions and limitations under the License. #include "server/Server.h" -#include "server/init/InstanceLockCheck.h" #include #include @@ -26,8 +25,10 @@ #include "server/DBWrapper.h" #include "server/grpc_impl/GrpcServer.h" #include "server/init/CpuChecker.h" +#include "server/init/Directory.h" #include "server/init/GpuChecker.h" #include "server/init/StorageChecker.h" +#include "server/init/Timezone.h" #include "server/web_impl/WebServer.h" #include "src/version.h" //#include "storage/s3/S3ClientWrapper.h" @@ -156,72 +157,25 @@ Server::Start() { tracing_config_path.empty() ? tracing::TracerUtil::InitGlobal() : tracing::TracerUtil::InitGlobal(tracing_config_path); - auto time_zone = config.general.timezone(); - - if (time_zone.length() == 3) { - time_zone = "CUT"; - } else { - int time_bias = std::stoi(time_zone.substr(3, std::string::npos)); - if (time_bias == 0) { - time_zone = "CUT"; - } else if (time_bias > 0) { - time_zone = "CUT" + std::to_string(-time_bias); - } else { - time_zone = "CUT+" + std::to_string(-time_bias); - } - } - - if (setenv("TZ", time_zone.c_str(), 1) != 0) { - return Status(SERVER_UNEXPECTED_ERROR, "Fail to setenv"); - } - tzset(); + STATUS_CHECK(Timezone::SetTimezone(config.general.timezone())); /* log path is defined in Config file, so InitLog must be called after LoadConfig */ STATUS_CHECK(LogMgr::InitLog(config.logs.trace.enable(), config.logs.level(), config.logs.path(), config.logs.max_log_file_size(), config.logs.log_rotate_num())); - bool cluster_enable = config.cluster.enable(); - auto cluster_role = config.cluster.role(); + auto wal_path = config.wal.enable() ? config.wal.path() : ""; + STATUS_CHECK(Directory::Initialize(config.storage.path(), wal_path)); - Status s; - if ((not cluster_enable) || cluster_role == ClusterRole::RW) { - try { - // True if a new directory was created, otherwise false. - boost::filesystem::create_directories(config.storage.path()); - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, "Cannot create db directory, " + std::string(ex.what())); - } catch (...) { - return Status(SERVER_UNEXPECTED_ERROR, "Cannot create db directory"); - } + /* TODO: add a invisible config */ + if (true) { + bool cluster_enable = config.cluster.enable(); + auto cluster_role = config.cluster.role(); - s = InstanceLockCheck::Check(config.storage.path()); - if (!s.ok()) { - if (not cluster_enable) { - std::cerr << "single instance lock db path failed." << s.message() << std::endl; - } else { - std::cerr << cluster_role << " instance lock db path failed." << s.message() << std::endl; - } - return s; - } + std::cout << "Running on " + RunningMode(cluster_enable, (ClusterRole)cluster_role) + " mode." << std::endl; - if (config.wal.enable()) { - std::string wal_path = config.wal.path(); - - try { - // True if a new directory was created, otherwise false. - boost::filesystem::create_directories(wal_path); - } catch (...) { - return Status(SERVER_UNEXPECTED_ERROR, "Cannot create wal directory"); - } - s = InstanceLockCheck::Check(wal_path); - if (!s.ok()) { - if (not cluster_enable) { - std::cerr << "single instance lock wal path failed." << s.message() << std::endl; - } else { - std::cerr << cluster_role << " instance lock wal path failed." << s.message() << std::endl; - } - return s; - } + /* Only read-only mode do not lock directories */ + if ((not cluster_enable) || cluster_role == ClusterRole::RW) { + STATUS_CHECK(Directory::Lock(config.storage.path(), wal_path)); } } @@ -328,6 +282,21 @@ Server::StopService() { engine::KnowhereResource::Finalize(); } +std::string +Server::RunningMode(bool cluster_enable, ClusterRole cluster_role) { + if (cluster_enable) { + if (cluster_role == ClusterRole::RW) { + return "RW"; + } else if (cluster_role == ClusterRole::RO) { + return "RO"; + } else { + return "Unknown"; + } + } else { + return "single"; + } +} + void Server::LogConfigInFile(const std::string& path) { // TODO(yhz): Check if file exists diff --git a/core/src/server/Server.h b/core/src/server/Server.h index 0718fdfcda..e486a81ad9 100644 --- a/core/src/server/Server.h +++ b/core/src/server/Server.h @@ -44,6 +44,9 @@ class Server { StopService(); private: + static std::string + RunningMode(bool cluster_enable, ClusterRole cluster_role); + static void LogConfigInFile(const std::string& path); diff --git a/core/src/server/init/Directory.cpp b/core/src/server/init/Directory.cpp new file mode 100644 index 0000000000..1397de58be --- /dev/null +++ b/core/src/server/init/Directory.cpp @@ -0,0 +1,103 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License. + +#include "server/init/Directory.h" + +#include +#include +#include +#include +#include + +#include "config/ServerConfig.h" + +namespace milvus::server { +Status +Directory::Initialize(const std::string& storage_path, const std::string& wal_path) { + try { + init(storage_path); + if (not wal_path.empty()) { + init(wal_path); + } + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + return Status::OK(); +} + +Status +Directory::Lock(const std::string& storage_path, const std::string& wal_path) { + try { + lock(storage_path); + if (not wal_path.empty()) { + lock(wal_path); + } + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + return Status::OK(); +} + +void +Directory::init(const std::string& path) { + try { + // Returns True if a new directory was created, otherwise false. + boost::filesystem::create_directories(path); + } catch (std::exception& ex) { + std::string msg = "Cannot create directory: " + path + ", reason: " + ex.what(); + throw std::runtime_error(msg); + } catch (...) { + std::string msg = "Cannot create directory: " + path; + throw std::runtime_error(msg); + } +} + +void +Directory::lock(const std::string& path) { + std::string lock_path = path + "/lock"; + auto fd = open(lock_path.c_str(), O_RDWR | O_CREAT | O_NOFOLLOW, 0640); + fiu_do_on("Directory.lock.fd", fd = -1); + if (fd < 0) { + std::string msg = "Cannot lock file: " + lock_path + ", reason: "; + if (errno == EROFS) { + // Not using locking for read-only lock file + msg += "Lock file is read-only."; + } else { + msg += strerror(errno); + } + throw std::runtime_error(msg); + } + + // Acquire a write lock + struct flock fl; + // exclusive lock + fl.l_type = F_WRLCK; + fl.l_whence = SEEK_SET; + fl.l_start = 0; + fl.l_len = 0; + auto fcl = fcntl(fd, F_SETLK, &fl); + fiu_do_on("Directory.lock.fcntl", fcl = -1); + if (fcl == -1) { + std::string msg = "Cannot lock file: " + lock_path + ", reason: "; + if (errno == EACCES || errno == EAGAIN) { + msg += "Permission denied."; + } else if (errno == ENOLCK) { + // Not using locking for nfs mounted lock file + msg += "Using nfs."; + } else { + msg += std::string(strerror(errno)) + "."; + } + close(fd); + throw std::runtime_error(msg); + } +} + +} // namespace milvus::server diff --git a/core/src/server/init/Directory.h b/core/src/server/init/Directory.h new file mode 100644 index 0000000000..86f6a8ed77 --- /dev/null +++ b/core/src/server/init/Directory.h @@ -0,0 +1,36 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License. + +#pragma once + +#include + +#include "utils/Status.h" + +namespace milvus::server { + +class Directory { + public: + static Status + Initialize(const std::string& storage_path, const std::string& wal_path); + + static Status + Lock(const std::string& storage_path, const std::string& wal_path); + + private: + static void + init(const std::string& path); + + static void + lock(const std::string& path); +}; + +} // namespace milvus::server diff --git a/core/src/server/init/InstanceLockCheck.cpp b/core/src/server/init/InstanceLockCheck.cpp deleted file mode 100644 index 9d8fb8515d..0000000000 --- a/core/src/server/init/InstanceLockCheck.cpp +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#include "server/init/InstanceLockCheck.h" - -#include -#include -#include - -#include - -#include "utils/Log.h" - -namespace milvus { -namespace server { - -Status -InstanceLockCheck::Check(const std::string& path) { - std::string lock_path = path + "/lock"; - auto fd = open(lock_path.c_str(), O_RDWR | O_CREAT | O_NOFOLLOW, 0640); - fiu_do_on("InstanceLockCheck.Check.fd", fd = -1); - if (fd < 0) { - std::string msg; - if (errno == EROFS) { - // Not using locking for read-only lock file - msg += "Lock file is read-only."; - } - msg += "Could not open file: " + lock_path + ", " + strerror(errno); - return Status(SERVER_UNEXPECTED_ERROR, msg); - } - - // Acquire a write lock - struct flock fl; - // exclusive lock - fl.l_type = F_WRLCK; - fl.l_whence = SEEK_SET; - fl.l_start = 0; - fl.l_len = 0; - auto fcl = fcntl(fd, F_SETLK, &fl); - fiu_do_on("InstanceLockCheck.Check.fcntl", fcl = -1); - if (fcl == -1) { - std::string msg = "Can't lock file: " + lock_path + ", due to "; - if (errno == EACCES || errno == EAGAIN) { - msg += "permission denied. "; - } else if (errno == ENOLCK) { - // Not using locking for nfs mounted lock file - msg += "using nfs. "; - } else { - msg += std::string(strerror(errno)) + ". "; - } - close(fd); - return Status(SERVER_UNEXPECTED_ERROR, msg); - } - - LOG_SERVER_INFO_ << "InstanceLockCheck passed."; - - return Status::OK(); -} - -} // namespace server -} // namespace milvus diff --git a/core/src/server/init/Timezone.cpp b/core/src/server/init/Timezone.cpp new file mode 100644 index 0000000000..3dd880391f --- /dev/null +++ b/core/src/server/init/Timezone.cpp @@ -0,0 +1,40 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License. + +#include "server/init/Timezone.h" + +#include + +namespace milvus::server { + +Status +Timezone::SetTimezone(std::string time_zone) { + if (time_zone.length() == 3) { + time_zone = "CUT"; + } else { + int time_bias = std::stoi(time_zone.substr(3, std::string::npos)); + if (time_bias == 0) { + time_zone = "CUT"; + } else if (time_bias > 0) { + time_zone = "CUT" + std::to_string(-time_bias); + } else { + time_zone = "CUT+" + std::to_string(-time_bias); + } + } + + if (setenv("TZ", time_zone.c_str(), 1) != 0) { + return Status(SERVER_UNEXPECTED_ERROR, "Fail to setenv"); + } + tzset(); + + return Status::OK(); +} +} // namespace milvus::server diff --git a/core/src/server/init/InstanceLockCheck.h b/core/src/server/init/Timezone.h similarity index 80% rename from core/src/server/init/InstanceLockCheck.h rename to core/src/server/init/Timezone.h index 14f88343d6..960af2255f 100644 --- a/core/src/server/init/InstanceLockCheck.h +++ b/core/src/server/init/Timezone.h @@ -12,16 +12,14 @@ #pragma once #include + #include "utils/Status.h" -namespace milvus { -namespace server { +namespace milvus::server { -class InstanceLockCheck { +class Timezone { public: static Status - Check(const std::string& path); -}; // InstanceLockCheck - -} // namespace server -} // namespace milvus + SetTimezone(std::string timezone); +}; +} // namespace milvus::server