Add Generator for visitor pattern (#89)

Signed-off-by: FluorineDog <guilin.gou@zilliz.com>

internal/core/run_clang_format.sh

@@ -7,7 +7,7 @@ fi
 CorePath=$1
 
 formatThis() {
-    find "$1" | grep -E "(*\.cpp|*\.h|*\.cc)$" | grep -v "/thirdparty" | grep -v "\.pb\." | xargs clang-format-10 -i
+    find "$1" | grep -E "(*\.cpp|*\.h|*\.cc)$" | grep -v "gen_tools/templates" | grep -v "/thirdparty" | grep -v "\.pb\." | xargs clang-format-10 -i
 }
 
 formatThis "${CorePath}/src"

internal/core/src/query/CMakeLists.txt

@@ -1,6 +1,8 @@
 # TODO
 set(MILVUS_QUERY_SRCS
-        BinaryQuery.cpp
+        deprecated/BinaryQuery.cpp
+        generated/PlanNode.cpp
+        generated/Expr.cpp
         Parser.cpp
         )
 add_library(milvus_query ${MILVUS_QUERY_SRCS})

internal/core/src/query/Expr.h

@@ -54,15 +54,6 @@ struct BoolBinaryExpr : BinaryExpr {
     accept(ExprVisitor&) override;
 };
 
-// // TODO: not enabled in sprint 1
-// struct ArthmeticBinaryOpExpr : BinaryExpr {
-//     enum class OpType { Add, Sub, Multiply, Divide };
-//     OpType op_type_;
-//  public:
-//     void
-//     accept(ExprVisitor&) override;
-// };
-
 using FieldId = int64_t;
 
 struct TermExpr : Expr {

internal/core/src/query/Parser.h

@@ -1,8 +1,8 @@
 #pragma once
 #include "pb/service_msg.pb.h"
-#include "query/BooleanQuery.h"
+#include "query/deprecated/BooleanQuery.h"
-#include "query/BinaryQuery.h"
+#include "query/deprecated/BinaryQuery.h"
-#include "query/GeneralQuery.h"
+#include "query/deprecated/GeneralQuery.h"
 
 namespace milvus::wtf {
 

internal/core/src/query/PlanNode.h

@@ -4,53 +4,53 @@
 #include <any>
 #include <string>
 #include <optional>
-#include "Predicate.h"
+#include "Expr.h"
 namespace milvus::query {
-class QueryNodeVisitor;
+class PlanNodeVisitor;
 
-enum class QueryNodeType {
+enum class PlanNodeType {
     kInvalid = 0,
     kScan,
     kANNS,
 };
 
 // Base of all Nodes
-struct QueryNode {
+struct PlanNode {
-    QueryNodeType node_type;
+    PlanNodeType node_type;
 
  public:
-    virtual ~QueryNode() = default;
+    virtual ~PlanNode() = default;
     virtual void
-    accept(QueryNodeVisitor&) = 0;
+    accept(PlanNodeVisitor&) = 0;
 };
 
-using QueryNodePtr = std::unique_ptr<QueryNode>;
+using PlanNodePtr = std::unique_ptr<PlanNode>;
 
-struct VectorQueryNode : QueryNode {
+struct VectorPlanNode : PlanNode {
-    std::optional<QueryNodePtr> child_;
+    std::optional<PlanNodePtr> child_;
     int64_t num_queries_;
     int64_t dim_;
     FieldId field_id_;
 
  public:
     virtual void
-    accept(QueryNodeVisitor&) = 0;
+    accept(PlanNodeVisitor&) = 0;
 };
 
-struct FloatVectorANNS : VectorQueryNode {
+struct FloatVectorANNS : VectorPlanNode {
     std::shared_ptr<float> data;
     std::string metric_type_;  // TODO: use enum
  public:
     void
-    accept(QueryNodeVisitor&) override;
+    accept(PlanNodeVisitor&) override;
 };
 
-struct BinaryVectorANNS : VectorQueryNode {
+struct BinaryVectorANNS : VectorPlanNode {
     std::shared_ptr<uint8_t> data;
     std::string metric_type_;  // TODO: use enum
  public:
     void
-    accept(QueryNodeVisitor&) override;
+    accept(PlanNodeVisitor&) override;
 };
 
 }  // namespace milvus::query

internal/core/src/query/QueryVistor.h

@@ -1 +0,0 @@
-#pragma once

internal/core/src/query/deprecated/BinaryQuery.cpp

@@ -16,7 +16,7 @@
 #include <utility>
 #include <vector>
 
-#include "query/BinaryQuery.h"
+#include "BinaryQuery.h"
 
 namespace milvus {
 namespace query_old {

internal/core/src/query/deprecated/ValidationUtil.cpp

@@ -9,7 +9,7 @@
 // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 // or implied. See the License for the specific language governing permissions and limitations under the License.
 
-#include "query/ValidationUtil.h"
+#include "ValidationUtil.h"
 #include "config/ServerConfig.h"
 //#include "db/Constants.h"
 //#include "db/Utils.h"

internal/core/src/query/generated/Expr.cpp

@@ -0,0 +1,28 @@
+#pragma once
+// Generated File
+// DO NOT EDIT
+#include "query/Expr.h"
+#include "ExprVisitor.h"
+
+namespace milvus::query {
+void
+BoolUnaryExpr::accept(ExprVisitor& visitor) {
+    visitor.visit(*this);
+}
+
+void
+BoolBinaryExpr::accept(ExprVisitor& visitor) {
+    visitor.visit(*this);
+}
+
+void
+TermExpr::accept(ExprVisitor& visitor) {
+    visitor.visit(*this);
+}
+
+void
+RangeExpr::accept(ExprVisitor& visitor) {
+    visitor.visit(*this);
+}
+
+}  // namespace milvus::query

internal/core/src/query/generated/ExprVisitor.h

@@ -0,0 +1,23 @@
+#pragma once
+// Generated File
+// DO NOT EDIT
+#include "query/Expr.h"
+namespace milvus::query {
+class ExprVisitor {
+ public:
+    virtual ~ExprVisitor() = 0;
+
+ public:
+    virtual void
+    visit(BoolUnaryExpr&) = 0;
+
+    virtual void
+    visit(BoolBinaryExpr&) = 0;
+
+    virtual void
+    visit(TermExpr&) = 0;
+
+    virtual void
+    visit(RangeExpr&) = 0;
+};
+}  // namespace milvus::query

internal/core/src/query/generated/PlanNode.cpp

@@ -0,0 +1,18 @@
+#pragma once
+// Generated File
+// DO NOT EDIT
+#include "query/PlanNode.h"
+#include "PlanNodeVisitor.h"
+
+namespace milvus::query {
+void
+FloatVectorANNS::accept(PlanNodeVisitor& visitor) {
+    visitor.visit(*this);
+}
+
+void
+BinaryVectorANNS::accept(PlanNodeVisitor& visitor) {
+    visitor.visit(*this);
+}
+
+}  // namespace milvus::query

internal/core/src/query/generated/PlanNodeVisitor.h

@@ -0,0 +1,17 @@
+#pragma once
+// Generated File
+// DO NOT EDIT
+#include "query/PlanNode.h"
+namespace milvus::query {
+class PlanNodeVisitor {
+ public:
+    virtual ~PlanNodeVisitor() = 0;
+
+ public:
+    virtual void
+    visit(FloatVectorANNS&) = 0;
+
+    virtual void
+    visit(BinaryVectorANNS&) = 0;
+};
+}  // namespace milvus::query

internal/core/src/query/generated/ShowExprVisitor.cpp

@@ -0,0 +1,25 @@
+#error TODO: copy this file out, and modify the content.
+#include "query/generated/ShowExprVisitor.h"
+
+namespace milvus::query {
+void
+ShowExprVisitor::visit(BoolUnaryExpr& expr) {
+    // TODO
+}
+
+void
+ShowExprVisitor::visit(BoolBinaryExpr& expr) {
+    // TODO
+}
+
+void
+ShowExprVisitor::visit(TermExpr& expr) {
+    // TODO
+}
+
+void
+ShowExprVisitor::visit(RangeExpr& expr) {
+    // TODO
+}
+
+}  // namespace milvus::query

internal/core/src/query/generated/ShowExprVisitor.h

@@ -0,0 +1,22 @@
+#pragma once
+// Generated File
+// DO NOT EDIT
+#include "ExprVisitor.h"
+namespace milvus::query {
+class ShowExprVisitor : ExprVisitor {
+ public:
+    virtual void
+    visit(BoolUnaryExpr& expr) override;
+
+    virtual void
+    visit(BoolBinaryExpr& expr) override;
+
+    virtual void
+    visit(TermExpr& expr) override;
+
+    virtual void
+    visit(RangeExpr& expr) override;
+
+ public:
+};
+}  // namespace milvus::query

internal/core/src/query/generated/ShowPlanNodeVisitor.cpp

@@ -0,0 +1,15 @@
+#error TODO: copy this file out, and modify the content.
+#include "query/generated/ShowPlanNodeVisitor.h"
+
+namespace milvus::query {
+void
+ShowPlanNodeVisitor::visit(FloatVectorANNS& node) {
+    // TODO
+}
+
+void
+ShowPlanNodeVisitor::visit(BinaryVectorANNS& node) {
+    // TODO
+}
+
+}  // namespace milvus::query

internal/core/src/query/generated/ShowPlanNodeVisitor.h

@@ -0,0 +1,16 @@
+#pragma once
+// Generated File
+// DO NOT EDIT
+#include "PlanNodeVisitor.h"
+namespace milvus::query {
+class ShowPlanNodeVisitor : PlanNodeVisitor {
+ public:
+    virtual void
+    visit(FloatVectorANNS& node) override;
+
+    virtual void
+    visit(BinaryVectorANNS& node) override;
+
+ public:
+};
+}  // namespace milvus::query

internal/core/src/segcore/SegmentBase.h

@@ -6,7 +6,7 @@
 #include "segcore/SegmentDefs.h"
 // #include "knowhere/index/Index.h"
 // #include "knowhere/index/IndexType.h"
-#include "query/GeneralQuery.h"
+#include "query/deprecated/GeneralQuery.h"
 
 namespace milvus {
 namespace segcore {

internal/core/src/segcore/SegmentNaive.h

@@ -11,7 +11,7 @@
 #include "ConcurrentVector.h"
 #include "segcore/SegmentBase.h"
 // #include "knowhere/index/structured_index/StructuredIndex.h"
-#include "query/GeneralQuery.h"
+#include "query/deprecated/GeneralQuery.h"
 #include "utils/Status.h"
 #include "segcore/DeletedRecord.h"
 #include "EasyAssert.h"

internal/core/src/segcore/SegmentSmallIndex.h

@@ -11,7 +11,7 @@
 #include "ConcurrentVector.h"
 #include "segcore/SegmentBase.h"
 // #include "knowhere/index/structured_index/StructuredIndex.h"
-#include "query/GeneralQuery.h"
+#include "query/deprecated/GeneralQuery.h"
 #include "utils/Status.h"
 #include "segcore/DeletedRecord.h"
 #include "EasyAssert.h"

internal/core/unittest/test_query.cpp

@@ -1,7 +1,9 @@
 #include <gtest/gtest.h>
 #include "query/Parser.h"
-#include "query/Predicate.h"
+#include "query/Expr.h"
-#include "query/QueryNode.h"
+#include "query/PlanNode.h"
+#include "query/generated/ExprVisitor.h"
+#include "query/generated/PlanNodeVisitor.h"
 
 TEST(Query, Naive) {
     SUCCEED();

tools/core_gen/all_generate.py

@@ -0,0 +1,54 @@
+#!python
+# from gen_base_visitor import *
+# from gen_node import *
+from assemble import *
+from meta_gen import *
+
+def gen_file(rootfile, template, output, **kwargs):
+    namespace, root_base, struct_name = meta_gen(readfile(rootfile))
+    vc = assemble(readfile(template), namespace=namespace, root_base=root_base, struct_name=struct_name, **kwargs)
+    file = open(output, 'w')
+    file.write(vc)
+
+if __name__ == "__main__":
+    query_path = "../../internal/core/src/query/"
+    output_path = query_path + "generated/"
+    
+    
+    node_names = ["Expr", "PlanNode"]
+    visitor_info = {
+        'Expr': [{
+            'visitor_name': "ShowExprVisitor",
+            "ctor_and_member": ' public:',
+            "parameter_name": 'expr',
+        }],
+        'PlanNode': [{
+            'visitor_name': "ShowPlanNodeVisitor",
+            "ctor_and_member": ' public:',
+            "parameter_name": 'node',
+        }]
+    }
+    
+    for name in node_names:
+        rootfile = query_path + name + ".h"
+
+        template = 'templates/visitor_base.h'
+        output = output_path + name + 'Visitor.h'
+        gen_file(rootfile, template, output)
+
+        template = 'templates/node_def.cpp'
+        output = output_path + name + '.cpp'
+        gen_file(rootfile, template, output)
+
+        for info in visitor_info[name]:
+            vis_name = info['visitor_name']
+            template = 'templates/visitor_derived.h'
+            output = output_path + vis_name + '.h'
+            gen_file(rootfile, template, output, **info)
+
+            vis_name = info['visitor_name']
+            template = 'templates/visitor_derived.cpp'
+            output = output_path + vis_name + '.cpp'
+            gen_file(rootfile, template, output, **info)
+    print("Done")
+

tools/core_gen/assemble.py

@@ -0,0 +1,39 @@
+#!python
+from meta_gen import *
+import copy
+import sys
+import re
+
+def assemble(template, **kwargs):
+    pattern = re.compile("@@@@(.*?)\n((.|\n)*?)\n####", re.MULTILINE)
+    temp_info = pattern.findall(template)
+    # print(temp_info)
+    mapping = dict()
+    rep_map = dict()
+
+    # drop repetive field from mapping
+    for k, v in kwargs.items():
+        if isinstance(v, list):
+            rep_map[k] = v
+        else: 
+            mapping[k] = v
+
+    for k, v, _ in temp_info:
+        info = k.split("@")
+        new_v = replace_all(v, **mapping)
+        assert(1 <= len(info) <= 2)  
+        if len(info) == 2:
+            k = info[0]
+            rep = info[1]
+            new_v = "\n\n".join([new_v.replace("@@" + rep + "@@", rep_v) for rep_v in rep_map[rep]])
+        mapping[k] = new_v
+    return mapping["main"]
+
+
+# if __name__ == "__main__":
+#     assert(len(sys.argv) == 2)
+#     root_file = sys.argv[1]
+#     namespace, root_base, struct_name = meta_gen(readfile(root_file))
+#     gen_all(readfile("templates/node_full.cpp"), namespace=namespace, root_base=root_base, struct_name=struct_name)
+
+

tools/core_gen/meta_gen.py

@@ -0,0 +1,62 @@
+#!python
+import re
+import sys
+
+def eprint(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+
+def readfile(filename):
+    file = open(filename)
+    content = file.read()
+    return content
+
+def replace_all(template, **kwargs):
+    for k, v in kwargs.items():
+        template = template.replace("@@" + k + "@@", v)
+    return template
+
+
+def meta_gen(content):
+    namespace_pattern = re.compile(r"namespace(.*){")
+    results = namespace_pattern.findall(content)
+    assert(len(results) == 1)
+    namespace = results[0].strip()
+
+    struct_pattern = re.compile(r"struct (.*?){((.|\n)*?)^};", re.MULTILINE)
+    results = struct_pattern.findall(content)
+
+    body_pattern = re.compile(r"accept\((.*)Visitor ?& ?\) (.*?);")
+    # print(results)
+    # print(len(results[0]))
+
+    root_base = None
+    override_structs = []
+    for (title, body, _) in results:
+        pack = title.replace(' ', '').split(':')
+
+        if len(pack) == 1:
+            pack.append(None)
+        struct_name, base_name = pack
+        if not base_name:
+            root_base = struct_name
+        body_res = body_pattern.findall(body)
+        if len(body_res) != 1:
+            eprint(struct_name)
+            eprint(body_res)
+            eprint(body)
+            assert(false)
+        visitor_name, state = body_res[0]
+        assert(visitor_name == root_base) 
+        if state.strip() == 'override':
+            override_structs.append(struct_name) 
+        # print(body_res)
+    return namespace, root_base, override_structs
+
+if __name__ == "__main__":
+    assert(len(sys.argv) == 2)
+    file = open(sys.argv[1])
+    content = file.read()
+    namespace, root_base, override_structs = meta_gen(content)
+    eprint(namespace)
+    eprint(root_base)
+    eprint(override_structs)

tools/core_gen/templates/node_def.cpp

@@ -0,0 +1,20 @@
+@@@@body@struct_name
+void
+@@struct_name@@::accept(@@root_base@@Visitor& visitor) {
+    visitor.visit(*this);
+}
+####
+
+@@@@main
+#pragma once
+// Generated File
+// DO NOT EDIT
+#include "query/@@root_base@@.h"
+#include "@@root_base@@Visitor.h"
+
+namespace @@namespace@@ {
+@@body@@
+
+}  // namespace @@namespace@@
+
+####

tools/core_gen/templates/visitor_base.h

@@ -0,0 +1,20 @@
+@@@@body@struct_name
+    virtual void
+    visit(@@struct_name@@&) = 0;
+####
+@@@@main
+#pragma once
+// Generated File
+// DO NOT EDIT
+#include "query/@@root_base@@.h"
+namespace @@namespace@@ {
+class @@root_base@@Visitor {
+ public:
+    virtual ~@@root_base@@Visitor() = 0;
+
+ public:
+@@body@@
+};
+}  // namespace @@namespace@@
+
+####

tools/core_gen/templates/visitor_derived.cpp

@@ -0,0 +1,18 @@
+@@@@func_list@struct_name
+void
+@@visitor_name@@::visit(@@struct_name@@& @@parameter_name@@) {
+    // TODO
+}
+####
+
+
+@@@@main
+#error TODO: copy this file out, and modify the content.
+#include "query/generated/@@visitor_name@@.h"
+
+namespace @@namespace@@ {
+@@func_list@@
+
+}  // namespace @@namespace@@
+
+####

tools/core_gen/templates/visitor_derived.h

@@ -0,0 +1,22 @@
+@@@@base_visitor
+@@root_base@@Visitor
+####
+@@@@body@struct_name
+    virtual void
+    visit(@@struct_name@@& @@parameter_name@@) override;
+####
+@@@@main
+#pragma once
+// Generated File
+// DO NOT EDIT
+#include "@@base_visitor@@.h"
+namespace @@namespace@@ {
+class @@visitor_name@@ : @@base_visitor@@ {
+ public:
+@@body@@
+
+@@ctor_and_member@@
+};
+}  // namespace @@namespace@@
+
+####