From 77aecc31bcd83b6bb59b35f85e728d4e9fd47279 Mon Sep 17 00:00:00 2001 From: "shengjun.li" Date: Mon, 28 Sep 2020 22:23:45 +0800 Subject: [PATCH] Let bitset all functions inline (#3904) Signed-off-by: shengjun.li --- .../faiss/utils/ConcurrentBitset.cpp | 217 ------------------ .../thirdparty/faiss/utils/ConcurrentBitset.h | 191 ++++++++++++--- 2 files changed, 161 insertions(+), 247 deletions(-) delete mode 100644 core/src/index/thirdparty/faiss/utils/ConcurrentBitset.cpp diff --git a/core/src/index/thirdparty/faiss/utils/ConcurrentBitset.cpp b/core/src/index/thirdparty/faiss/utils/ConcurrentBitset.cpp deleted file mode 100644 index 2f935fba93..0000000000 --- a/core/src/index/thirdparty/faiss/utils/ConcurrentBitset.cpp +++ /dev/null @@ -1,217 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include -#include "ConcurrentBitset.h" - -namespace faiss { - -ConcurrentBitset::ConcurrentBitset(id_type_t capacity, uint8_t init_value) : capacity_(capacity), bitset_(((capacity + 8 - 1) >> 3)) { - if (init_value) { - memset(mutable_data(), init_value, (capacity + 8 - 1) >> 3); - } -} - -ConcurrentBitset& -ConcurrentBitset::operator&=(const ConcurrentBitset& bitset) { - auto u8_1 = mutable_data(); - auto u8_2 = bitset.data(); - auto u64_1 = reinterpret_cast(u8_1); - auto u64_2 = reinterpret_cast(u8_2); - - size_t n8 = bitset_.size(); - size_t n64 = n8 / 8; - - for (size_t i = 0; i < n64; i++) { - u64_1[i] &= u64_2[i]; - } - - size_t remain = n8 % 8; - u8_1 += n64 * 8; - u8_2 += n64 * 8; - for (size_t i = 0; i < remain; i++) { - u8_1[i] &= u8_2[i]; - } - - return *this; -} - -std::shared_ptr -ConcurrentBitset::operator&(const ConcurrentBitset& bitset) const { - auto result_bitset = std::make_shared(bitset.capacity()); - - auto result_8 = result_bitset->mutable_data(); - auto result_64 = reinterpret_cast(result_8); - - auto u8_1 = data(); - auto u8_2 = bitset.data(); - auto u64_1 = reinterpret_cast(u8_1); - auto u64_2 = reinterpret_cast(u8_2); - - size_t n8 = bitset_.size(); - size_t n64 = n8 / 8; - - for (size_t i = 0; i < n64; i++) { - result_64[i] = u64_1[i] & u64_2[i]; - } - - size_t remain = n8 % 8; - u8_1 += n64 * 8; - u8_2 += n64 * 8; - result_8 += n64 * 8; - for (size_t i = 0; i < remain; i++) { - result_8[i] = u8_1[i] & u8_2[i]; - } - - - return result_bitset; -} - -ConcurrentBitset& -ConcurrentBitset::operator|=(const ConcurrentBitset& bitset) { - auto u8_1 = mutable_data(); - auto u8_2 = bitset.data(); - auto u64_1 = reinterpret_cast(u8_1); - auto u64_2 = reinterpret_cast(u8_2); - - size_t n8 = bitset_.size(); - size_t n64 = n8 / 8; - - for (size_t i = 0; i < n64; i++) { - u64_1[i] |= u64_2[i]; - } - - size_t remain = n8 % 8; - u8_1 += n64 * 8; - u8_2 += n64 * 8; - for (size_t i = 0; i < remain; i++) { - u8_1[i] |= u8_2[i]; - } - - return *this; -} - -std::shared_ptr -ConcurrentBitset::operator|(const ConcurrentBitset& bitset) const { - auto result_bitset = std::make_shared(bitset.capacity()); - - auto result_8 = result_bitset->mutable_data(); - auto result_64 = reinterpret_cast(result_8); - - auto u8_1 = data(); - auto u8_2 = bitset.data(); - auto u64_1 = reinterpret_cast(u8_1); - auto u64_2 = reinterpret_cast(u8_2); - - size_t n8 = bitset_.size(); - size_t n64 = n8 / 8; - - for (size_t i = 0; i < n64; i++) { - result_64[i] = u64_1[i] | u64_2[i]; - } - - size_t remain = n8 % 8; - u8_1 += n64 * 8; - u8_2 += n64 * 8; - result_8 += n64 * 8; - for (size_t i = 0; i < remain; i++) { - result_8[i] = u8_1[i] | u8_2[i]; - } - - return result_bitset; -} - -ConcurrentBitset& -ConcurrentBitset::operator^=(const ConcurrentBitset& bitset) { - auto u8_1 = mutable_data(); - auto u8_2 = bitset.data(); - auto u64_1 = reinterpret_cast(u8_1); - auto u64_2 = reinterpret_cast(u8_2); - - size_t n8 = bitset_.size(); - size_t n64 = n8 / 8; - - for (size_t i = 0; i < n64; i++) { - u64_1[i] &= u64_2[i]; - } - - size_t remain = n8 % 8; - u8_1 += n64 * 8; - u8_2 += n64 * 8; - for (size_t i = 0; i < remain; i++) { - u8_1[i] ^= u8_2[i]; - } - - return *this; -} - -ConcurrentBitset& -ConcurrentBitset::negate() { - auto u8_1 = mutable_data(); - auto u64_1 = reinterpret_cast(u8_1); - - size_t n8 = bitset_.size(); - size_t n64 = n8 / 8; - - for (size_t i = 0; i < n64; i++) { - u64_1[i] = ~u64_1[i]; - } - - size_t remain = n8 % 8; - u8_1 += n64 * 8; - for (size_t i = 0; i < remain; i++) { - u8_1[i] = ~u8_1[i]; - } - - return *this; -} - -bool -ConcurrentBitset::test(id_type_t id) { - return bitset_[id >> 3].load() & (0x1 << (id & 0x7)); -} - -void -ConcurrentBitset::set(id_type_t id) { - bitset_[id >> 3].fetch_or(0x1 << (id & 0x7)); -} - -void -ConcurrentBitset::clear(id_type_t id) { - bitset_[id >> 3].fetch_and(~(0x1 << (id & 0x7))); -} - -size_t -ConcurrentBitset::capacity() const { - return capacity_; -} - -size_t -ConcurrentBitset::size() const { - return ((capacity_ + 8 - 1) >> 3); -} - -const uint8_t* -ConcurrentBitset::data() const { - return reinterpret_cast(bitset_.data()); -} - -uint8_t* -ConcurrentBitset::mutable_data() { - return reinterpret_cast(bitset_.data()); -} -} // namespace faiss diff --git a/core/src/index/thirdparty/faiss/utils/ConcurrentBitset.h b/core/src/index/thirdparty/faiss/utils/ConcurrentBitset.h index 156ecc5963..82a277805f 100644 --- a/core/src/index/thirdparty/faiss/utils/ConcurrentBitset.h +++ b/core/src/index/thirdparty/faiss/utils/ConcurrentBitset.h @@ -1,24 +1,19 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at +// Copyright (C) 2019-2020 Zilliz. All rights reserved. // -// http://www.apache.org/licenses/LICENSE-2.0 +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at // -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License. #pragma once #include #include +#include #include namespace faiss { @@ -27,49 +22,185 @@ class ConcurrentBitset { public: using id_type_t = int64_t; - explicit ConcurrentBitset(id_type_t size, uint8_t init_value = 0); + explicit ConcurrentBitset(size_t count, uint8_t init_value = 0) + : count_(count), bitset_(((count + 8 - 1) >> 3)) { + if (init_value) { + memset(mutable_data(), init_value, (count + 8 - 1) >> 3); + } + } ConcurrentBitset& - operator&=(const ConcurrentBitset& bitset); + operator&=(const ConcurrentBitset& bitset) { + auto u8_1 = mutable_data(); + auto u8_2 = bitset.data(); + auto u64_1 = reinterpret_cast(u8_1); + auto u64_2 = reinterpret_cast(u8_2); + + size_t n8 = bitset_.size(); + size_t n64 = n8 / 8; + + for (size_t i = 0; i < n64; i++) { + u64_1[i] &= u64_2[i]; + } + + size_t remain = n8 % 8; + u8_1 += n64 * 8; + u8_2 += n64 * 8; + for (size_t i = 0; i < remain; i++) { + u8_1[i] &= u8_2[i]; + } + + return *this; + } std::shared_ptr - operator&(const ConcurrentBitset& bitset) const; + operator&(const ConcurrentBitset& bitset) const { + auto result_bitset = std::make_shared(bitset.count()); + + auto result_8 = result_bitset->mutable_data(); + auto result_64 = reinterpret_cast(result_8); + + auto u8_1 = data(); + auto u8_2 = bitset.data(); + auto u64_1 = reinterpret_cast(u8_1); + auto u64_2 = reinterpret_cast(u8_2); + + size_t n8 = bitset_.size(); + size_t n64 = n8 / 8; + + for (size_t i = 0; i < n64; i++) { + result_64[i] = u64_1[i] & u64_2[i]; + } + + size_t remain = n8 % 8; + u8_1 += n64 * 8; + u8_2 += n64 * 8; + result_8 += n64 * 8; + for (size_t i = 0; i < remain; i++) { + result_8[i] = u8_1[i] & u8_2[i]; + } + + + return result_bitset; + } ConcurrentBitset& - operator|=(const ConcurrentBitset& bitset); + operator|=(const ConcurrentBitset& bitset) { + auto u8_1 = mutable_data(); + auto u8_2 = bitset.data(); + auto u64_1 = reinterpret_cast(u8_1); + auto u64_2 = reinterpret_cast(u8_2); + + size_t n8 = bitset_.size(); + size_t n64 = n8 / 8; + + for (size_t i = 0; i < n64; i++) { + u64_1[i] |= u64_2[i]; + } + + size_t remain = n8 % 8; + u8_1 += n64 * 8; + u8_2 += n64 * 8; + for (size_t i = 0; i < remain; i++) { + u8_1[i] |= u8_2[i]; + } + + return *this; + } std::shared_ptr - operator|(const ConcurrentBitset& bitset) const; + operator|(const ConcurrentBitset& bitset) const { + auto result_bitset = std::make_shared(bitset.count()); + + auto result_8 = result_bitset->mutable_data(); + auto result_64 = reinterpret_cast(result_8); + + auto u8_1 = data(); + auto u8_2 = bitset.data(); + auto u64_1 = reinterpret_cast(u8_1); + auto u64_2 = reinterpret_cast(u8_2); + + size_t n8 = bitset_.size(); + size_t n64 = n8 / 8; + + for (size_t i = 0; i < n64; i++) { + result_64[i] = u64_1[i] | u64_2[i]; + } + + size_t remain = n8 % 8; + u8_1 += n64 * 8; + u8_2 += n64 * 8; + result_8 += n64 * 8; + for (size_t i = 0; i < remain; i++) { + result_8[i] = u8_1[i] | u8_2[i]; + } + + return result_bitset; + } ConcurrentBitset& - operator^=(const ConcurrentBitset& bitset); + negate() { + auto u8_1 = mutable_data(); + auto u64_1 = reinterpret_cast(u8_1); + + size_t n8 = bitset_.size(); + size_t n64 = n8 / 8; + + for (size_t i = 0; i < n64; i++) { + u64_1[i] = ~u64_1[i]; + } + + size_t remain = n8 % 8; + u8_1 += n64 * 8; + for (size_t i = 0; i < remain; i++) { + u8_1[i] = ~u8_1[i]; + } + + return *this; + } - ConcurrentBitset& - negate(); bool - test(id_type_t id); + test(id_type_t id) { + unsigned char mask = (unsigned char)(0x01) << (id & 0x07); + return (bitset_[id >> 3].load() & mask); + } void - set(id_type_t id); + set(id_type_t id) { + unsigned char mask = (unsigned char)(0x01) << (id & 0x07); + bitset_[id >> 3].fetch_or(mask); + } + void - clear(id_type_t id); + clear(id_type_t id) { + unsigned char mask = (unsigned char)(0x01) << (id & 0x07); + bitset_[id >> 3].fetch_and(~mask); + } size_t - capacity() const; + count() const { + return count_; + } size_t - size() const; + size() const { + return ((count_ + 8 - 1) >> 3); + } const uint8_t* - data() const; + data() const { + return reinterpret_cast(bitset_.data()); + } uint8_t* - mutable_data(); + mutable_data() { + return reinterpret_cast(bitset_.data()); + } private: - size_t capacity_; + size_t count_; std::vector> bitset_; };