Merge operation

Signed-off-by: neza2017 <yefu.chen@zilliz.com>

cmd/storage/benchmark.go

@@ -1,315 +0,0 @@
-package main
-
-import (
-	"context"
-	"crypto/md5"
-	"flag"
-	"fmt"
-	"log"
-	"math/rand"
-	"os"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/pivotal-golang/bytefmt"
-	"github.com/zilliztech/milvus-distributed/internal/storage"
-	storagetype "github.com/zilliztech/milvus-distributed/internal/storage/type"
-)
-
-// Global variables
-var durationSecs, threads, loops, numVersion, batchOpSize int
-var valueSize uint64
-var valueData []byte
-var batchValueData [][]byte
-var counter, totalKeyCount, keyNum int32
-var endTime, setFinish, getFinish, deleteFinish time.Time
-var totalKeys [][]byte
-
-var logFileName = "benchmark.log"
-var logFile *os.File
-
-var store storagetype.Store
-var wg sync.WaitGroup
-
-func runSet() {
-	for time.Now().Before(endTime) {
-		num := atomic.AddInt32(&keyNum, 1)
-		key := []byte(fmt.Sprint("key", num))
-		for ver := 1; ver <= numVersion; ver++ {
-			atomic.AddInt32(&counter, 1)
-			err := store.PutRow(context.Background(), key, valueData, "empty", uint64(ver))
-			if err != nil {
-				log.Fatalf("Error setting key %s, %s", key, err.Error())
-				//atomic.AddInt32(&setCount, -1)
-			}
-		}
-	}
-	// Remember last done time
-	setFinish = time.Now()
-	wg.Done()
-}
-
-func runBatchSet() {
-	for time.Now().Before(endTime) {
-		num := atomic.AddInt32(&keyNum, int32(batchOpSize))
-		keys := make([][]byte, batchOpSize)
-		versions := make([]uint64, batchOpSize)
-		batchSuffix := make([]string, batchOpSize)
-		for n := batchOpSize; n > 0; n-- {
-			keys[n-1] = []byte(fmt.Sprint("key", num-int32(n)))
-		}
-		for ver := 1; ver <= numVersion; ver++ {
-			atomic.AddInt32(&counter, 1)
-			err := store.PutRows(context.Background(), keys, batchValueData, batchSuffix, versions)
-			if err != nil {
-				log.Fatalf("Error setting batch keys %s %s", keys, err.Error())
-				//atomic.AddInt32(&batchSetCount, -1)
-			}
-		}
-	}
-	setFinish = time.Now()
-	wg.Done()
-}
-
-func runGet() {
-	for time.Now().Before(endTime) {
-		num := atomic.AddInt32(&counter, 1)
-		//num := atomic.AddInt32(&keyNum, 1)
-		//key := []byte(fmt.Sprint("key", num))
-		num = num % totalKeyCount
-		key := totalKeys[num]
-		_, err := store.GetRow(context.Background(), key, uint64(numVersion))
-		if err != nil {
-			log.Fatalf("Error getting key %s, %s", key, err.Error())
-			//atomic.AddInt32(&getCount, -1)
-		}
-	}
-	// Remember last done time
-	getFinish = time.Now()
-	wg.Done()
-}
-
-func runBatchGet() {
-	for time.Now().Before(endTime) {
-		num := atomic.AddInt32(&keyNum, int32(batchOpSize))
-		//keys := make([][]byte, batchOpSize)
-		//for n := batchOpSize; n > 0; n-- {
-		//	keys[n-1] = []byte(fmt.Sprint("key", num-int32(n)))
-		//}
-		end := num % totalKeyCount
-		if end < int32(batchOpSize) {
-			end = int32(batchOpSize)
-		}
-		start := end - int32(batchOpSize)
-		keys := totalKeys[start:end]
-		versions := make([]uint64, batchOpSize)
-		for i := range versions {
-			versions[i] = uint64(numVersion)
-		}
-		atomic.AddInt32(&counter, 1)
-		_, err := store.GetRows(context.Background(), keys, versions)
-		if err != nil {
-			log.Fatalf("Error getting key %s, %s", keys, err.Error())
-			//atomic.AddInt32(&batchGetCount, -1)
-		}
-	}
-	// Remember last done time
-	getFinish = time.Now()
-	wg.Done()
-}
-
-func runDelete() {
-	for time.Now().Before(endTime) {
-		num := atomic.AddInt32(&counter, 1)
-		//num := atomic.AddInt32(&keyNum, 1)
-		//key := []byte(fmt.Sprint("key", num))
-		num = num % totalKeyCount
-		key := totalKeys[num]
-		err := store.DeleteRow(context.Background(), key, uint64(numVersion))
-		if err != nil {
-			log.Fatalf("Error getting key %s, %s", key, err.Error())
-			//atomic.AddInt32(&deleteCount, -1)
-		}
-	}
-	// Remember last done time
-	deleteFinish = time.Now()
-	wg.Done()
-}
-
-func runBatchDelete() {
-	for time.Now().Before(endTime) {
-		num := atomic.AddInt32(&keyNum, int32(batchOpSize))
-		//keys := make([][]byte, batchOpSize)
-		//for n := batchOpSize; n > 0; n-- {
-		//	keys[n-1] = []byte(fmt.Sprint("key", num-int32(n)))
-		//}
-		end := num % totalKeyCount
-		if end < int32(batchOpSize) {
-			end = int32(batchOpSize)
-		}
-		start := end - int32(batchOpSize)
-		keys := totalKeys[start:end]
-		atomic.AddInt32(&counter, 1)
-		versions := make([]uint64, batchOpSize)
-		for i := range versions {
-			versions[i] = uint64(numVersion)
-		}
-		err := store.DeleteRows(context.Background(), keys, versions)
-		if err != nil {
-			log.Fatalf("Error getting key %s, %s", keys, err.Error())
-			//atomic.AddInt32(&batchDeleteCount, -1)
-		}
-	}
-	// Remember last done time
-	getFinish = time.Now()
-	wg.Done()
-}
-
-func main() {
-	// Parse command line
-	myflag := flag.NewFlagSet("myflag", flag.ExitOnError)
-	myflag.IntVar(&durationSecs, "d", 5, "Duration of each test in seconds")
-	myflag.IntVar(&threads, "t", 1, "Number of threads to run")
-	myflag.IntVar(&loops, "l", 1, "Number of times to repeat test")
-	var sizeArg string
-	var storeType string
-	myflag.StringVar(&sizeArg, "z", "1k", "Size of objects in bytes with postfix K, M, and G")
-	myflag.StringVar(&storeType, "s", "s3", "Storage type, tikv or minio or s3")
-	myflag.IntVar(&numVersion, "v", 1, "Max versions for each key")
-	myflag.IntVar(&batchOpSize, "b", 100, "Batch operation kv pair number")
-
-	if err := myflag.Parse(os.Args[1:]); err != nil {
-		os.Exit(1)
-	}
-
-	// Check the arguments
-	var err error
-	if valueSize, err = bytefmt.ToBytes(sizeArg); err != nil {
-		log.Fatalf("Invalid -z argument for object size: %v", err)
-	}
-	var option = storagetype.Option{TikvAddress: "localhost:2379", Type: storeType, BucketName: "zilliz-hz"}
-
-	store, err = storage.NewStore(context.Background(), option)
-	if err != nil {
-		log.Fatalf("Error when creating storage " + err.Error())
-	}
-	logFile, err = os.OpenFile(logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0777)
-	if err != nil {
-		log.Fatalf("Prepare log file error, " + err.Error())
-	}
-
-	// Echo the parameters
-	log.Printf("Benchmark log will write to file %s\n", logFile.Name())
-	fmt.Fprintf(logFile, "Parameters: duration=%d, threads=%d, loops=%d, valueSize=%s, batchSize=%d, versions=%d\n", durationSecs, threads, loops, sizeArg, batchOpSize, numVersion)
-	// Init test data
-	valueData = make([]byte, valueSize)
-	rand.Read(valueData)
-	hasher := md5.New()
-	hasher.Write(valueData)
-
-	batchValueData = make([][]byte, batchOpSize)
-	for i := range batchValueData {
-		batchValueData[i] = make([]byte, valueSize)
-		rand.Read(batchValueData[i])
-		hasher := md5.New()
-		hasher.Write(batchValueData[i])
-	}
-
-	// Loop running the tests
-	for loop := 1; loop <= loops; loop++ {
-
-		// reset counters
-		counter = 0
-		keyNum = 0
-		totalKeyCount = 0
-		totalKeys = nil
-
-		// Run the batchSet case
-		// key seq start from setCount
-		counter = 0
-		startTime := time.Now()
-		endTime = startTime.Add(time.Second * time.Duration(durationSecs))
-		for n := 1; n <= threads; n++ {
-			wg.Add(1)
-			go runBatchSet()
-		}
-		wg.Wait()
-
-		setTime := setFinish.Sub(startTime).Seconds()
-		bps := float64(uint64(counter)*valueSize*uint64(batchOpSize)) / setTime
-		fmt.Fprintf(logFile, "Loop %d: BATCH PUT time %.1f secs, batchs = %d, kv pairs = %d, speed = %sB/sec, %.1f operations/sec, %.1f kv/sec.\n",
-			loop, setTime, counter, counter*int32(batchOpSize), bytefmt.ByteSize(uint64(bps)), float64(counter)/setTime, float64(counter*int32(batchOpSize))/setTime)
-		// Record all test keys
-		//totalKeyCount = keyNum
-		//totalKeys = make([][]byte, totalKeyCount)
-		//for i := int32(0); i < totalKeyCount; i++ {
-		//	totalKeys[i] = []byte(fmt.Sprint("key", i))
-		//}
-		//
-		//// Run the get case
-		//counter = 0
-		//startTime = time.Now()
-		//endTime = startTime.Add(time.Second * time.Duration(durationSecs))
-		//for n := 1; n <= threads; n++ {
-		//	wg.Add(1)
-		//	go runGet()
-		//}
-		//wg.Wait()
-		//
-		//getTime := getFinish.Sub(startTime).Seconds()
-		//bps = float64(uint64(counter)*valueSize) / getTime
-		//fmt.Fprint(logFile, fmt.Sprintf("Loop %d: GET time %.1f secs, kv pairs = %d, speed = %sB/sec, %.1f operations/sec, %.1f kv/sec.\n",
-		//	loop, getTime, counter, bytefmt.ByteSize(uint64(bps)), float64(counter)/getTime, float64(counter)/getTime))
-
-		// Run the batchGet case
-		//counter = 0
-		//startTime = time.Now()
-		//endTime = startTime.Add(time.Second * time.Duration(durationSecs))
-		//for n := 1; n <= threads; n++ {
-		//	wg.Add(1)
-		//	go runBatchGet()
-		//}
-		//wg.Wait()
-		//
-		//getTime = getFinish.Sub(startTime).Seconds()
-		//bps = float64(uint64(counter)*valueSize*uint64(batchOpSize)) / getTime
-		//fmt.Fprint(logFile, fmt.Sprintf("Loop %d: BATCH GET time %.1f secs, batchs = %d, kv pairs = %d, speed = %sB/sec, %.1f operations/sec, %.1f kv/sec.\n",
-		//	loop, getTime, counter, counter*int32(batchOpSize), bytefmt.ByteSize(uint64(bps)), float64(counter)/getTime, float64(counter * int32(batchOpSize))/getTime))
-		//
-		//// Run the delete case
-		//counter = 0
-		//startTime = time.Now()
-		//endTime = startTime.Add(time.Second * time.Duration(durationSecs))
-		//for n := 1; n <= threads; n++ {
-		//	wg.Add(1)
-		//	go runDelete()
-		//}
-		//wg.Wait()
-		//
-		//deleteTime := deleteFinish.Sub(startTime).Seconds()
-		//bps = float64(uint64(counter)*valueSize) / deleteTime
-		//fmt.Fprint(logFile, fmt.Sprintf("Loop %d: Delete time %.1f secs, kv pairs = %d, %.1f operations/sec, %.1f kv/sec.\n",
-		//	loop, deleteTime, counter, float64(counter)/deleteTime, float64(counter)/deleteTime))
-		//
-		//// Run the batchDelete case
-		//counter = 0
-		//startTime = time.Now()
-		//endTime = startTime.Add(time.Second * time.Duration(durationSecs))
-		//for n := 1; n <= threads; n++ {
-		//	wg.Add(1)
-		//	go runBatchDelete()
-		//}
-		//wg.Wait()
-		//
-		//deleteTime = setFinish.Sub(startTime).Seconds()
-		//bps = float64(uint64(counter)*valueSize*uint64(batchOpSize)) / setTime
-		//fmt.Fprint(logFile, fmt.Sprintf("Loop %d: BATCH DELETE time %.1f secs, batchs = %d, kv pairs = %d, %.1f operations/sec, %.1f kv/sec.\n",
-		//	loop, setTime, counter, counter*int32(batchOpSize), float64(counter)/setTime, float64(counter * int32(batchOpSize))/setTime))
-
-		// Print line mark
-		lineMark := "\n"
-		fmt.Fprint(logFile, lineMark)
-	}
-	log.Print("Benchmark test done.")
-}

deployments/docker/docker-compose.yml

@@ -36,6 +36,14 @@ services:
     networks:
       - milvus
 
+  jaeger:
+    image: jaegertracing/all-in-one:latest
+    ports:
+      - "6831:6831/udp"
+      - "16686:16686"
+    networks:
+      - milvus
+
 networks:
   milvus:
 

docker-compose.yml

@@ -83,5 +83,10 @@ services:
     networks:
       - milvus
 
+  jaeger:
+    image: jaegertracing/all-in-one:latest
+    networks:
+      - milvus
+
 networks:
   milvus:

go.mod

@@ -4,14 +4,17 @@ go 1.15
 
 require (
 	code.cloudfoundry.org/bytefmt v0.0.0-20200131002437-cf55d5288a48 // indirect
+	github.com/HdrHistogram/hdrhistogram-go v1.0.1 // indirect
 	github.com/apache/pulsar-client-go v0.1.1
-	github.com/aws/aws-sdk-go v1.30.8
+	github.com/apache/thrift v0.13.0
+	github.com/aws/aws-sdk-go v1.30.8 // indirect
 	github.com/coreos/etcd v3.3.25+incompatible // indirect
-	github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548
+	github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
 	github.com/frankban/quicktest v1.10.2 // indirect
 	github.com/fsnotify/fsnotify v1.4.9 // indirect
 	github.com/git-hooks/git-hooks v1.3.1 // indirect
 	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/golang/mock v1.3.1
 	github.com/golang/protobuf v1.3.2
 	github.com/google/btree v1.0.0
 	github.com/klauspost/compress v1.10.11 // indirect
@@ -20,12 +23,12 @@ require (
 	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
 	github.com/onsi/ginkgo v1.12.1 // indirect
 	github.com/onsi/gomega v1.10.0 // indirect
-	github.com/opentracing/opentracing-go v1.2.0 // indirect
+	github.com/opentracing/opentracing-go v1.2.0
 	github.com/pierrec/lz4 v2.5.2+incompatible // indirect
 	github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712 // indirect
 	github.com/pingcap/errors v0.11.4 // indirect
 	github.com/pingcap/log v0.0.0-20200828042413-fce0951f1463 // indirect
-	github.com/pivotal-golang/bytefmt v0.0.0-20200131002437-cf55d5288a48
+	github.com/pivotal-golang/bytefmt v0.0.0-20200131002437-cf55d5288a48 // indirect
 	github.com/prometheus/client_golang v1.5.1 // indirect
 	github.com/prometheus/common v0.10.0 // indirect
 	github.com/prometheus/procfs v0.1.3 // indirect
@@ -35,7 +38,9 @@ require (
 	github.com/spf13/cast v1.3.0
 	github.com/spf13/viper v1.7.1
 	github.com/stretchr/testify v1.6.1
-	github.com/tikv/client-go v0.0.0-20200824032810-95774393107b
+	github.com/tikv/client-go v0.0.0-20200824032810-95774393107b // indirect
+	github.com/uber/jaeger-client-go v2.25.0+incompatible
+	github.com/uber/jaeger-lib v2.4.0+incompatible // indirect
 	github.com/urfave/cli v1.22.5 // indirect
 	github.com/yahoo/athenz v1.9.16 // indirect
 	go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738
@@ -50,7 +55,7 @@ require (
 	google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150 // indirect
 	google.golang.org/grpc v1.31.0
 	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
-	gopkg.in/yaml.v2 v2.3.0
+	gopkg.in/yaml.v2 v2.3.0 // indirect
 	honnef.co/go/tools v0.0.1-2020.1.4 // indirect
 	sigs.k8s.io/yaml v1.2.0 // indirect
 )

go.sum

@@ -15,6 +15,8 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/HdrHistogram/hdrhistogram-go v1.0.1 h1:GX8GAYDuhlFQnI2fRDHQhTlkHMz8bEn0jTI6LJU0mpw=
+github.com/HdrHistogram/hdrhistogram-go v1.0.1/go.mod h1:BWJ+nMSHY3L41Zj7CA3uXnloDp7xxV0YvstAE7nKTaM=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
@@ -24,6 +26,8 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1C
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/apache/pulsar-client-go v0.1.1 h1:v/kU+2ZCC6yFIcbZrFtWa9/nvVzVr18L+xYJUvZSxEQ=
 github.com/apache/pulsar-client-go v0.1.1/go.mod h1:mlxC65KL1BLhGO2bnT9zWMttVzR2czVPb27D477YpyU=
+github.com/apache/thrift v0.13.0 h1:5hryIiq9gtn+MiLVn0wP37kb/uTeRZgN08WoCsAhIhI=
+github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/ardielle/ardielle-go v1.5.2 h1:TilHTpHIQJ27R1Tl/iITBzMwiUGSlVfiVhwDNGM3Zj4=
 github.com/ardielle/ardielle-go v1.5.2/go.mod h1:I4hy1n795cUhaVt/ojz83SNVCYIGsAFAONtv2Dr7HUI=
 github.com/ardielle/ardielle-tools v1.5.4/go.mod h1:oZN+JRMnqGiIhrzkRN9l26Cej9dEx4jeNG6A+AdkShk=
@@ -117,6 +121,7 @@ github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18h
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
 github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -343,6 +348,7 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx
 github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/protocolbuffers/protobuf v3.14.0+incompatible h1:8r0H76h/Q/lEnFFY60AuM23NOnaDMi6bd7zuboSYM+o=
 github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446 h1:/NRJ5vAYoqz+7sG51ubIDHXeWO8DlTSrToPu6q11ziA=
 github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
@@ -403,6 +409,12 @@ github.com/tikv/client-go v0.0.0-20200824032810-95774393107b/go.mod h1:K0NcdVNrX
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/uber/jaeger-client-go v1.6.0 h1:3+zLlq+4npI5fg8IsgAje3YsP7TcEdNzJScyqFIzxEQ=
+github.com/uber/jaeger-client-go v2.25.0+incompatible h1:IxcNZ7WRY1Y3G4poYlx24szfsn/3LvK9QHCq9oQw8+U=
+github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
+github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo=
+github.com/uber/jaeger-lib v2.4.0+incompatible h1:fY7QsGQWiCt8pajv4r7JEvmATdCVaWxXbjwyYwsNaLQ=
+github.com/uber/jaeger-lib v2.4.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
 github.com/ugorji/go v1.1.2/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ=
 github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43/go.mod h1:iT03XoTwV7xq/+UGwKO3UbC1nNNlopQiY61beSdrtOA=
 github.com/unrolled/render v1.0.0 h1:XYtvhA3UkpB7PqkvhUFYmpKD55OudoIeygcfus4vcd4=

internal/core/src/common/FieldMeta.h

@@ -18,7 +18,7 @@
 
 namespace milvus {
 inline int
-field_sizeof(DataType data_type, int dim = 1) {
+datatype_sizeof(DataType data_type, int dim = 1) {
     switch (data_type) {
         case DataType::BOOL:
             return sizeof(bool);
@@ -78,7 +78,7 @@ datatype_name(DataType data_type) {
 }
 
 inline bool
-field_is_vector(DataType datatype) {
+datatype_is_vector(DataType datatype) {
     return datatype == DataType::VECTOR_BINARY || datatype == DataType::VECTOR_FLOAT;
 }
 
@@ -119,9 +119,9 @@ struct FieldMeta {
     int
     get_sizeof() const {
         if (is_vector()) {
-            return field_sizeof(type_, get_dim());
+            return datatype_sizeof(type_, get_dim());
         } else {
-            return field_sizeof(type_, 1);
+            return datatype_sizeof(type_, 1);
         }
     }
 

internal/core/src/common/Schema.cpp

@@ -50,7 +50,7 @@ Schema::ParseFrom(const milvus::proto::schema::CollectionSchema& schema_proto) {
             schema->primary_key_offset_opt_ = schema->size();
         }
 
-        if (field_is_vector(data_type)) {
+        if (datatype_is_vector(data_type)) {
             auto type_map = RepeatedKeyValToMap(child.type_params());
             auto index_map = RepeatedKeyValToMap(child.index_params());
             if (!index_map.count("metric_type")) {

internal/core/src/common/Types.h

@@ -14,13 +14,15 @@
 #include <faiss/MetricType.h>
 #include <string>
 #include <boost/align/aligned_allocator.hpp>
+#include <memory>
 #include <vector>
 
 namespace milvus {
 using Timestamp = uint64_t;  // TODO: use TiKV-like timestamp
 using engine::DataType;
 using engine::FieldElementType;
-using engine::QueryResult;
+using engine::idx_t;
+
 using MetricType = faiss::MetricType;
 
 MetricType
@@ -39,4 +41,33 @@ constexpr std::false_type always_false{};
 template <typename T>
 using aligned_vector = std::vector<T, boost::alignment::aligned_allocator<T, 512>>;
 
+///////////////////////////////////////////////////////////////////////////////////////////////////
+struct QueryResult {
+    QueryResult() = default;
+    QueryResult(uint64_t num_queries, uint64_t topK) : topK_(topK), num_queries_(num_queries) {
+        auto count = get_row_count();
+        result_distances_.resize(count);
+        internal_seg_offsets_.resize(count);
+    }
+
+    [[nodiscard]] uint64_t
+    get_row_count() const {
+        return topK_ * num_queries_;
+    }
+
+ public:
+    uint64_t num_queries_;
+    uint64_t topK_;
+    uint64_t seg_id_;
+    std::vector<float> result_distances_;
+
+ public:
+    // TODO(gexi): utilize these field
+    std::vector<int64_t> internal_seg_offsets_;
+    std::vector<int64_t> result_offsets_;
+    std::vector<std::vector<char>> row_data_;
+};
+
+using QueryResultPtr = std::shared_ptr<QueryResult>;
+
 }  // namespace milvus

internal/core/src/indexbuilder/IndexWrapper.cpp

@@ -55,6 +55,7 @@ IndexWrapper::parse_impl(const std::string& serialized_params_str, knowhere::Con
     }
 
     auto stoi_closure = [](const std::string& s) -> int { return std::stoi(s); };
+    auto stof_closure = [](const std::string& s) -> int { return std::stof(s); };
 
     /***************************** meta *******************************/
     check_parameter<int>(conf, milvus::knowhere::meta::DIM, stoi_closure, std::nullopt);
@@ -88,7 +89,7 @@ IndexWrapper::parse_impl(const std::string& serialized_params_str, knowhere::Con
     check_parameter<int>(conf, milvus::knowhere::IndexParams::edge_size, stoi_closure, std::nullopt);
 
     /************************** NGT Search Params *****************************/
-    check_parameter<int>(conf, milvus::knowhere::IndexParams::epsilon, stoi_closure, std::nullopt);
+    check_parameter<float>(conf, milvus::knowhere::IndexParams::epsilon, stof_closure, std::nullopt);
     check_parameter<int>(conf, milvus::knowhere::IndexParams::max_search_edges, stoi_closure, std::nullopt);
 
     /************************** NGT_PANNG Params *****************************/
@@ -274,6 +275,12 @@ IndexWrapper::QueryWithParam(const knowhere::DatasetPtr& dataset, const char* se
 
 std::unique_ptr<IndexWrapper::QueryResult>
 IndexWrapper::QueryImpl(const knowhere::DatasetPtr& dataset, const knowhere::Config& conf) {
+    auto load_raw_data_closure = [&]() { LoadRawData(); };  // hide this pointer
+    auto index_type = get_index_type();
+    if (is_in_nm_list(index_type)) {
+        std::call_once(raw_data_loaded_, load_raw_data_closure);
+    }
+
     auto res = index_->Query(dataset, conf, nullptr);
     auto ids = res->Get<int64_t*>(milvus::knowhere::meta::IDS);
     auto distances = res->Get<float*>(milvus::knowhere::meta::DISTANCE);
@@ -291,5 +298,19 @@ IndexWrapper::QueryImpl(const knowhere::DatasetPtr& dataset, const knowhere::Con
     return std::move(query_res);
 }
 
+void
+IndexWrapper::LoadRawData() {
+    auto index_type = get_index_type();
+    if (is_in_nm_list(index_type)) {
+        auto bs = index_->Serialize(config_);
+        auto bptr = std::make_shared<milvus::knowhere::Binary>();
+        auto deleter = [&](uint8_t*) {};  // avoid repeated deconstruction
+        bptr->data = std::shared_ptr<uint8_t[]>(static_cast<uint8_t*>(raw_data_.data()), deleter);
+        bptr->size = raw_data_.size();
+        bs.Append(RAW_DATA, bptr);
+        index_->Load(bs);
+    }
+}
+
 }  // namespace indexbuilder
 }  // namespace milvus

internal/core/src/indexbuilder/IndexWrapper.h

@@ -66,6 +66,9 @@ class IndexWrapper {
     void
     StoreRawData(const knowhere::DatasetPtr& dataset);
 
+    void
+    LoadRawData();
+
     template <typename T>
     void
     check_parameter(knowhere::Config& conf,
@@ -92,6 +95,7 @@ class IndexWrapper {
     milvus::json index_config_;
     knowhere::Config config_;
     std::vector<uint8_t> raw_data_;
+    std::once_flag raw_data_loaded_;
 };
 
 }  // namespace indexbuilder

internal/core/src/query/CMakeLists.txt

@@ -4,13 +4,16 @@ set(MILVUS_QUERY_SRCS
         generated/PlanNode.cpp
         generated/Expr.cpp
         visitors/ShowPlanNodeVisitor.cpp
-        visitors/ExecPlanNodeVisitor.cpp
         visitors/ShowExprVisitor.cpp
+        visitors/ExecPlanNodeVisitor.cpp
         visitors/ExecExprVisitor.cpp
+        visitors/VerifyPlanNodeVisitor.cpp
+        visitors/VerifyExprVisitor.cpp
         Plan.cpp
         Search.cpp
         SearchOnSealed.cpp
-        BruteForceSearch.cpp
+        SearchBruteForce.cpp
+        SubQueryResult.cpp
         )
 add_library(milvus_query ${MILVUS_QUERY_SRCS})
-target_link_libraries(milvus_query milvus_proto milvus_utils)
+target_link_libraries(milvus_query milvus_proto milvus_utils knowhere)

internal/core/src/query/Plan.cpp

@@ -21,6 +21,7 @@
 #include <boost/align/aligned_allocator.hpp>
 #include <boost/algorithm/string.hpp>
 #include <algorithm>
+#include "query/generated/VerifyPlanNodeVisitor.h"
 
 namespace milvus::query {
 
@@ -106,7 +107,7 @@ Parser::ParseRangeNode(const Json& out_body) {
     auto field_name = out_iter.key();
     auto body = out_iter.value();
     auto data_type = schema[field_name].get_data_type();
-    Assert(!field_is_vector(data_type));
+    Assert(!datatype_is_vector(data_type));
 
     switch (data_type) {
         case DataType::BOOL:
@@ -138,6 +139,8 @@ Parser::CreatePlanImpl(const std::string& dsl_str) {
     if (predicate != nullptr) {
         vec_node->predicate_ = std::move(predicate);
     }
+    VerifyPlanNodeVisitor verifier;
+    vec_node->accept(verifier);
 
     auto plan = std::make_unique<Plan>(schema);
     plan->tag2field_ = std::move(tag2field_);
@@ -152,7 +155,7 @@ Parser::ParseTermNode(const Json& out_body) {
     auto field_name = out_iter.key();
     auto body = out_iter.value();
     auto data_type = schema[field_name].get_data_type();
-    Assert(!field_is_vector(data_type));
+    Assert(!datatype_is_vector(data_type));
     switch (data_type) {
         case DataType::BOOL: {
             return ParseTermNodeImpl<bool>(field_name, body);

internal/core/src/query/Search.cpp

@@ -16,7 +16,7 @@
 
 #include <faiss/utils/distances.h>
 #include "utils/tools.h"
-#include "query/BruteForceSearch.h"
+#include "query/SearchBruteForce.h"
 
 namespace milvus::query {
 
@@ -34,13 +34,13 @@ create_bitmap_view(std::optional<const BitmapSimple*> bitmaps_opt, int64_t chunk
 }
 
 Status
-QueryBruteForceImpl(const segcore::SegmentSmallIndex& segment,
+FloatSearch(const segcore::SegmentSmallIndex& segment,
-                    const query::QueryInfo& info,
+            const query::QueryInfo& info,
-                    const float* query_data,
+            const float* query_data,
-                    int64_t num_queries,
+            int64_t num_queries,
-                    Timestamp timestamp,
+            Timestamp timestamp,
-                    std::optional<const BitmapSimple*> bitmaps_opt,
+            std::optional<const BitmapSimple*> bitmaps_opt,
-                    QueryResult& results) {
+            QueryResult& results) {
     auto& schema = segment.get_schema();
     auto& indexing_record = segment.get_indexing_record();
     auto& record = segment.get_insert_record();
@@ -75,6 +75,7 @@ QueryBruteForceImpl(const segcore::SegmentSmallIndex& segment,
     const auto& indexing_entry = indexing_record.get_vec_entry(vecfield_offset);
     auto search_conf = indexing_entry.get_search_conf(topK);
 
+    // TODO: use sub_qr
     for (int chunk_id = 0; chunk_id < max_indexed_id; ++chunk_id) {
         auto indexing = indexing_entry.get_vec_indexing(chunk_id);
         auto dataset = knowhere::GenDataset(num_queries, dim, query_data);
@@ -99,10 +100,12 @@ QueryBruteForceImpl(const segcore::SegmentSmallIndex& segment,
     Assert(vec_chunk_size == indexing_entry.get_chunk_size());
     auto max_chunk = upper_div(ins_barrier, vec_chunk_size);
 
+    // TODO: use sub_qr
     for (int chunk_id = max_indexed_id; chunk_id < max_chunk; ++chunk_id) {
         std::vector<int64_t> buf_uids(total_count, -1);
         std::vector<float> buf_dis(total_count, std::numeric_limits<float>::max());
 
+        // should be not visitable
         faiss::float_maxheap_array_t buf = {(size_t)num_queries, (size_t)topK, buf_uids.data(), buf_dis.data()};
         auto& chunk = vec_ptr->get_chunk(chunk_id);
 
@@ -112,6 +115,7 @@ QueryBruteForceImpl(const segcore::SegmentSmallIndex& segment,
         auto nsize = element_end - element_begin;
 
         auto bitmap_view = create_bitmap_view(bitmaps_opt, chunk_id);
+        // TODO: make it wrapped
         faiss::knn_L2sqr(query_data, chunk.data(), dim, num_queries, nsize, &buf, bitmap_view);
 
         Assert(buf_uids.size() == total_count);
@@ -134,13 +138,13 @@ QueryBruteForceImpl(const segcore::SegmentSmallIndex& segment,
 }
 
 Status
-BinaryQueryBruteForceImpl(const segcore::SegmentSmallIndex& segment,
+BinarySearch(const segcore::SegmentSmallIndex& segment,
-                          const query::QueryInfo& info,
+             const query::QueryInfo& info,
-                          const uint8_t* query_data,
+             const uint8_t* query_data,
-                          int64_t num_queries,
+             int64_t num_queries,
-                          Timestamp timestamp,
+             Timestamp timestamp,
-                          std::optional<const BitmapSimple*> bitmaps_opt,
+             std::optional<const BitmapSimple*> bitmaps_opt,
-                          QueryResult& results) {
+             QueryResult& results) {
     auto& schema = segment.get_schema();
     auto& indexing_record = segment.get_indexing_record();
     auto& record = segment.get_insert_record();
@@ -169,8 +173,8 @@ BinaryQueryBruteForceImpl(const segcore::SegmentSmallIndex& segment,
     auto total_count = topK * num_queries;
 
     // step 3: small indexing search
-    std::vector<int64_t> final_uids(total_count, -1);
+    // TODO: this is too intrusive
-    std::vector<float> final_dis(total_count, std::numeric_limits<float>::max());
+    // TODO: use QuerySubResult instead
     query::dataset::BinaryQueryDataset query_dataset{metric_type, num_queries, topK, code_size, query_data};
 
     using segcore::BinaryVector;
@@ -181,30 +185,27 @@ BinaryQueryBruteForceImpl(const segcore::SegmentSmallIndex& segment,
 
     auto vec_chunk_size = vec_ptr->get_chunk_size();
     auto max_chunk = upper_div(ins_barrier, vec_chunk_size);
+    SubQueryResult final_result(num_queries, topK, metric_type);
     for (int chunk_id = max_indexed_id; chunk_id < max_chunk; ++chunk_id) {
-        std::vector<int64_t> buf_uids(total_count, -1);
-        std::vector<float> buf_dis(total_count, std::numeric_limits<float>::max());
-
         auto& chunk = vec_ptr->get_chunk(chunk_id);
         auto element_begin = chunk_id * vec_chunk_size;
         auto element_end = std::min(ins_barrier, (chunk_id + 1) * vec_chunk_size);
         auto nsize = element_end - element_begin;
 
         auto bitmap_view = create_bitmap_view(bitmaps_opt, chunk_id);
-        BinarySearchBruteForce(query_dataset, chunk.data(), nsize, buf_dis.data(), buf_uids.data(), bitmap_view);
+        auto sub_result = BinarySearchBruteForce(query_dataset, chunk.data(), nsize, bitmap_view);
 
         // convert chunk uid to segment uid
-        for (auto& x : buf_uids) {
+        for (auto& x : sub_result.mutable_labels()) {
             if (x != -1) {
                 x += chunk_id * vec_chunk_size;
             }
         }
-
+        final_result.merge(sub_result);
-        segcore::merge_into(num_queries, topK, final_dis.data(), final_uids.data(), buf_dis.data(), buf_uids.data());
     }
 
-    results.result_distances_ = std::move(final_dis);
+    results.result_distances_ = std::move(final_result.mutable_values());
-    results.internal_seg_offsets_ = std::move(final_uids);
+    results.internal_seg_offsets_ = std::move(final_result.mutable_labels());
     results.topK_ = topK;
     results.num_queries_ = num_queries;
 

internal/core/src/query/Search.h

@@ -14,27 +14,29 @@
 #include "segcore/SegmentSmallIndex.h"
 #include <deque>
 #include <boost/dynamic_bitset.hpp>
+#include "query/SubQueryResult.h"
 
 namespace milvus::query {
 using BitmapChunk = boost::dynamic_bitset<>;
 using BitmapSimple = std::deque<BitmapChunk>;
 
+// TODO: merge these two search into one
 // note: c++17 don't support optional ref
 Status
-QueryBruteForceImpl(const segcore::SegmentSmallIndex& segment,
+FloatSearch(const segcore::SegmentSmallIndex& segment,
-                    const QueryInfo& info,
+            const QueryInfo& info,
-                    const float* query_data,
+            const float* query_data,
-                    int64_t num_queries,
+            int64_t num_queries,
-                    Timestamp timestamp,
+            Timestamp timestamp,
-                    std::optional<const BitmapSimple*> bitmap_opt,
+            std::optional<const BitmapSimple*> bitmap_opt,
-                    QueryResult& results);
+            QueryResult& results);
 
 Status
-BinaryQueryBruteForceImpl(const segcore::SegmentSmallIndex& segment,
+BinarySearch(const segcore::SegmentSmallIndex& segment,
-                          const query::QueryInfo& info,
+             const query::QueryInfo& info,
-                          const uint8_t* query_data,
+             const uint8_t* query_data,
-                          int64_t num_queries,
+             int64_t num_queries,
-                          Timestamp timestamp,
+             Timestamp timestamp,
-                          std::optional<const BitmapSimple*> bitmaps_opt,
+             std::optional<const BitmapSimple*> bitmaps_opt,
-                          QueryResult& results);
+             QueryResult& results);
 }  // namespace milvus::query

internal/core/src/query/SearchBruteForce.cpp

@@ -9,58 +9,16 @@
 // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 // or implied. See the License for the specific language governing permissions and limitations under the License
 
-#include "BruteForceSearch.h"
+#include "SearchBruteForce.h"
 #include <vector>
 #include <common/Types.h>
 #include <boost/dynamic_bitset.hpp>
 #include <queue>
+#include "SubQueryResult.h"
 
 namespace milvus::query {
 
-void
+SubQueryResult
-BinarySearchBruteForceNaive(MetricType metric_type,
-                            int64_t code_size,
-                            const uint8_t* binary_chunk,
-                            int64_t chunk_size,
-                            int64_t topk,
-                            int64_t num_queries,
-                            const uint8_t* query_data,
-                            float* result_distances,
-                            idx_t* result_labels,
-                            faiss::ConcurrentBitsetPtr bitset) {
-    // THIS IS A NAIVE IMPLEMENTATION, ready for optimize
-    Assert(metric_type == faiss::METRIC_Jaccard);
-    Assert(code_size % 4 == 0);
-
-    using T = std::tuple<float, int>;
-
-    for (int64_t q = 0; q < num_queries; ++q) {
-        auto query_ptr = query_data + code_size * q;
-        auto query = boost::dynamic_bitset(query_ptr, query_ptr + code_size);
-        std::vector<T> max_heap(topk + 1, std::make_tuple(std::numeric_limits<float>::max(), -1));
-
-        for (int64_t i = 0; i < chunk_size; ++i) {
-            auto element_ptr = binary_chunk + code_size * i;
-            auto element = boost::dynamic_bitset(element_ptr, element_ptr + code_size);
-            auto the_and = (query & element).count();
-            auto the_or = (query | element).count();
-            auto distance = the_or ? (float)(the_or - the_and) / the_or : 0;
-            if (distance < std::get<0>(max_heap[0])) {
-                max_heap[topk] = std::make_tuple(distance, i);
-                std::push_heap(max_heap.begin(), max_heap.end());
-                std::pop_heap(max_heap.begin(), max_heap.end());
-            }
-        }
-        std::sort(max_heap.begin(), max_heap.end());
-        for (int k = 0; k < topk; ++k) {
-            auto info = max_heap[k];
-            result_distances[k + q * topk] = std::get<0>(info);
-            result_labels[k + q * topk] = std::get<1>(info);
-        }
-    }
-}
-
-void
 BinarySearchBruteForceFast(MetricType metric_type,
                            int64_t code_size,
                            const uint8_t* binary_chunk,
@@ -68,9 +26,11 @@ BinarySearchBruteForceFast(MetricType metric_type,
                            int64_t topk,
                            int64_t num_queries,
                            const uint8_t* query_data,
-                           float* result_distances,
+                           const faiss::BitsetView& bitset) {
-                           idx_t* result_labels,
+    SubQueryResult sub_result(num_queries, topk, metric_type);
-                           faiss::ConcurrentBitsetPtr bitset) {
+    float* result_distances = sub_result.get_values();
+    idx_t* result_labels = sub_result.get_labels();
+
     const idx_t block_size = chunk_size;
     bool use_heap = true;
 
@@ -132,18 +92,26 @@ BinarySearchBruteForceFast(MetricType metric_type,
     } else {
         PanicInfo("Unsupported metric type");
     }
+    return sub_result;
 }
 
 void
+FloatSearchBruteForceFast(MetricType metric_type,
+                          const float* chunk_data,
+                          int64_t chunk_size,
+                          float* result_distances,
+                          idx_t* result_labels,
+                          const faiss::BitsetView& bitset) {
+    // TODO
+}
+
+SubQueryResult
 BinarySearchBruteForce(const dataset::BinaryQueryDataset& query_dataset,
                        const uint8_t* binary_chunk,
                        int64_t chunk_size,
-                       float* result_distances,
+                       const faiss::BitsetView& bitset) {
-                       idx_t* result_labels,
-                       faiss::ConcurrentBitsetPtr bitset) {
     // TODO: refactor the internal function
-    BinarySearchBruteForceFast(query_dataset.metric_type, query_dataset.code_size, binary_chunk, chunk_size,
+    return BinarySearchBruteForceFast(query_dataset.metric_type, query_dataset.code_size, binary_chunk, chunk_size,
-                               query_dataset.topk, query_dataset.num_queries, query_dataset.query_data,
+                                      query_dataset.topk, query_dataset.num_queries, query_dataset.query_data, bitset);
-                               result_distances, result_labels, bitset);
 }
 }  // namespace milvus::query

internal/core/src/query/SearchBruteForce.h

@@ -13,6 +13,7 @@
 #include <faiss/utils/BinaryDistance.h>
 #include "segcore/ConcurrentVector.h"
 #include "common/Schema.h"
+#include "query/SubQueryResult.h"
 
 namespace milvus::query {
 using MetricType = faiss::MetricType;
@@ -28,12 +29,10 @@ struct BinaryQueryDataset {
 
 }  // namespace dataset
 
-void
+SubQueryResult
 BinarySearchBruteForce(const dataset::BinaryQueryDataset& query_dataset,
                        const uint8_t* binary_chunk,
                        int64_t chunk_size,
-                       float* result_distances,
+                       const faiss::BitsetView& bitset = nullptr);
-                       idx_t* result_labels,
-                       faiss::ConcurrentBitsetPtr bitset = nullptr);
 
 }  // namespace milvus::query

internal/core/src/query/SubQueryResult.cpp

@@ -0,0 +1,77 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License
+
+#include "utils/EasyAssert.h"
+#include "query/SubQueryResult.h"
+#include "segcore/Reduce.h"
+
+namespace milvus::query {
+
+template <bool is_desc>
+void
+SubQueryResult::merge_impl(const SubQueryResult& right) {
+    Assert(num_queries_ == right.num_queries_);
+    Assert(topk_ == right.topk_);
+    Assert(metric_type_ == right.metric_type_);
+    Assert(is_desc == is_descending(metric_type_));
+
+    for (int64_t qn = 0; qn < num_queries_; ++qn) {
+        auto offset = qn * topk_;
+
+        int64_t* __restrict__ left_labels = this->get_labels() + offset;
+        float* __restrict__ left_values = this->get_values() + offset;
+
+        auto right_labels = right.get_labels() + offset;
+        auto right_values = right.get_values() + offset;
+
+        std::vector<float> buf_values(topk_);
+        std::vector<int64_t> buf_labels(topk_);
+
+        auto lit = 0;  // left iter
+        auto rit = 0;  // right iter
+
+        for (auto buf_iter = 0; buf_iter < topk_; ++buf_iter) {
+            auto left_v = left_values[lit];
+            auto right_v = right_values[rit];
+            // optimize out at compiling
+            if (is_desc ? (left_v >= right_v) : (left_v <= right_v)) {
+                buf_values[buf_iter] = left_values[lit];
+                buf_labels[buf_iter] = left_labels[lit];
+                ++lit;
+            } else {
+                buf_values[buf_iter] = right_values[rit];
+                buf_labels[buf_iter] = right_labels[rit];
+                ++rit;
+            }
+        }
+        std::copy_n(buf_values.data(), topk_, left_values);
+        std::copy_n(buf_labels.data(), topk_, left_labels);
+    }
+}
+
+void
+SubQueryResult::merge(const SubQueryResult& sub_result) {
+    Assert(metric_type_ == sub_result.metric_type_);
+    if (is_descending(metric_type_)) {
+        this->merge_impl<true>(sub_result);
+    } else {
+        this->merge_impl<false>(sub_result);
+    }
+}
+
+SubQueryResult
+SubQueryResult::merge(const SubQueryResult& left, const SubQueryResult& right) {
+    auto left_copy = left;
+    left_copy.merge(right);
+    return left_copy;
+}
+
+}  // namespace milvus::query

internal/core/src/query/SubQueryResult.h

@@ -0,0 +1,98 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License
+
+#pragma once
+#include "common/Types.h"
+#include <limits>
+#include <vector>
+namespace milvus::query {
+
+class SubQueryResult {
+ public:
+    SubQueryResult(int64_t num_queries, int64_t topk, MetricType metric_type)
+        : metric_type_(metric_type),
+          num_queries_(num_queries),
+          topk_(topk),
+          labels_(num_queries * topk, -1),
+          values_(num_queries * topk, init_value(metric_type)) {
+    }
+
+ public:
+    static constexpr float
+    init_value(MetricType metric_type) {
+        return (is_descending(metric_type) ? -1 : 1) * std::numeric_limits<float>::max();
+    }
+
+    static constexpr bool
+    is_descending(MetricType metric_type) {
+        // TODO
+        if (metric_type == MetricType::METRIC_INNER_PRODUCT) {
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+ public:
+    int64_t
+    get_num_queries() const {
+        return num_queries_;
+    }
+    int64_t
+    get_topk() const {
+        return topk_;
+    }
+
+    const int64_t*
+    get_labels() const {
+        return labels_.data();
+    }
+    int64_t*
+    get_labels() {
+        return labels_.data();
+    }
+    const float*
+    get_values() const {
+        return values_.data();
+    }
+    float*
+    get_values() {
+        return values_.data();
+    }
+    auto&
+    mutable_labels() {
+        return labels_;
+    }
+    auto&
+    mutable_values() {
+        return values_;
+    }
+
+    static SubQueryResult
+    merge(const SubQueryResult& left, const SubQueryResult& right);
+
+    void
+    merge(const SubQueryResult& sub_result);
+
+ private:
+    template <bool is_desc>
+    void
+    merge_impl(const SubQueryResult& sub_result);
+
+ private:
+    int64_t num_queries_;
+    int64_t topk_;
+    MetricType metric_type_;
+    std::vector<int64_t> labels_;
+    std::vector<float> values_;
+};
+
+}  // namespace milvus::query

internal/core/src/query/generated/ExecExprVisitor.h

@@ -21,7 +21,7 @@
 #include "ExprVisitor.h"
 
 namespace milvus::query {
-class ExecExprVisitor : ExprVisitor {
+class ExecExprVisitor : public ExprVisitor {
  public:
     void
     visit(BoolUnaryExpr& expr) override;

internal/core/src/query/generated/ExecPlanNodeVisitor.h

@@ -19,7 +19,7 @@
 #include "PlanNodeVisitor.h"
 
 namespace milvus::query {
-class ExecPlanNodeVisitor : PlanNodeVisitor {
+class ExecPlanNodeVisitor : public PlanNodeVisitor {
  public:
     void
     visit(FloatVectorANNS& node) override;

internal/core/src/query/generated/ShowExprVisitor.h

@@ -19,7 +19,7 @@
 #include "ExprVisitor.h"
 
 namespace milvus::query {
-class ShowExprVisitor : ExprVisitor {
+class ShowExprVisitor : public ExprVisitor {
  public:
     void
     visit(BoolUnaryExpr& expr) override;

internal/core/src/query/generated/ShowPlanNodeVisitor.h

@@ -20,7 +20,7 @@
 #include "PlanNodeVisitor.h"
 
 namespace milvus::query {
-class ShowPlanNodeVisitor : PlanNodeVisitor {
+class ShowPlanNodeVisitor : public PlanNodeVisitor {
  public:
     void
     visit(FloatVectorANNS& node) override;

internal/core/src/query/generated/VerifyExprVisitor.cpp

@@ -0,0 +1,36 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License
+
+#error TODO: copy this file out, and modify the content.
+#include "query/generated/VerifyExprVisitor.h"
+
+namespace milvus::query {
+void
+VerifyExprVisitor::visit(BoolUnaryExpr& expr) {
+    // TODO
+}
+
+void
+VerifyExprVisitor::visit(BoolBinaryExpr& expr) {
+    // TODO
+}
+
+void
+VerifyExprVisitor::visit(TermExpr& expr) {
+    // TODO
+}
+
+void
+VerifyExprVisitor::visit(RangeExpr& expr) {
+    // TODO
+}
+
+}  // namespace milvus::query

internal/core/src/query/generated/VerifyExprVisitor.h

@@ -0,0 +1,40 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License
+
+#pragma once
+// Generated File
+// DO NOT EDIT
+#include <optional>
+#include <boost/dynamic_bitset.hpp>
+#include <utility>
+#include <deque>
+#include "segcore/SegmentSmallIndex.h"
+#include "query/ExprImpl.h"
+#include "ExprVisitor.h"
+
+namespace milvus::query {
+class VerifyExprVisitor : public ExprVisitor {
+ public:
+    void
+    visit(BoolUnaryExpr& expr) override;
+
+    void
+    visit(BoolBinaryExpr& expr) override;
+
+    void
+    visit(TermExpr& expr) override;
+
+    void
+    visit(RangeExpr& expr) override;
+
+ public:
+};
+}  // namespace milvus::query

internal/core/src/query/generated/VerifyPlanNodeVisitor.cpp

@@ -0,0 +1,26 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License
+
+#error TODO: copy this file out, and modify the content.
+#include "query/generated/VerifyPlanNodeVisitor.h"
+
+namespace milvus::query {
+void
+VerifyPlanNodeVisitor::visit(FloatVectorANNS& node) {
+    // TODO
+}
+
+void
+VerifyPlanNodeVisitor::visit(BinaryVectorANNS& node) {
+    // TODO
+}
+
+}  // namespace milvus::query

internal/core/src/query/generated/VerifyPlanNodeVisitor.h

@@ -0,0 +1,37 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License
+
+#pragma once
+// Generated File
+// DO NOT EDIT
+#include "utils/Json.h"
+#include "query/PlanImpl.h"
+#include "segcore/SegmentBase.h"
+#include <utility>
+#include "PlanNodeVisitor.h"
+
+namespace milvus::query {
+class VerifyPlanNodeVisitor : public PlanNodeVisitor {
+ public:
+    void
+    visit(FloatVectorANNS& node) override;
+
+    void
+    visit(BinaryVectorANNS& node) override;
+
+ public:
+    using RetType = QueryResult;
+    VerifyPlanNodeVisitor() = default;
+
+ private:
+    std::optional<RetType> ret_;
+};
+}  // namespace milvus::query

internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp

@@ -79,7 +79,7 @@ ExecPlanNodeVisitor::visit(FloatVectorANNS& node) {
         SearchOnSealed(segment->get_schema(), sealed_indexing, node.query_info_, src_data, num_queries, timestamp_,
                        bitset_pack, ret);
     } else {
-        QueryBruteForceImpl(*segment, node.query_info_, src_data, num_queries, timestamp_, bitset_pack, ret);
+        FloatSearch(*segment, node.query_info_, src_data, num_queries, timestamp_, bitset_pack, ret);
     }
 
     ret_ = ret;
@@ -104,7 +104,7 @@ ExecPlanNodeVisitor::visit(BinaryVectorANNS& node) {
         bitset_pack = &bitmap_holder;
     }
 
-    BinaryQueryBruteForceImpl(*segment, node.query_info_, src_data, num_queries, timestamp_, bitset_pack, ret);
+    BinarySearch(*segment, node.query_info_, src_data, num_queries, timestamp_, bitset_pack, ret);
     ret_ = ret;
 }
 

internal/core/src/query/visitors/ShowExprVisitor.cpp

@@ -112,7 +112,7 @@ TermExtract(const TermExpr& expr_raw) {
 void
 ShowExprVisitor::visit(TermExpr& expr) {
     Assert(!ret_.has_value());
-    Assert(field_is_vector(expr.data_type_) == false);
+    Assert(datatype_is_vector(expr.data_type_) == false);
     auto terms = [&] {
         switch (expr.data_type_) {
             case DataType::BOOL:
@@ -161,7 +161,7 @@ ConditionExtract(const RangeExpr& expr_raw) {
 void
 ShowExprVisitor::visit(RangeExpr& expr) {
     Assert(!ret_.has_value());
-    Assert(field_is_vector(expr.data_type_) == false);
+    Assert(datatype_is_vector(expr.data_type_) == false);
     auto conditions = [&] {
         switch (expr.data_type_) {
             case DataType::BOOL:

internal/core/src/query/visitors/VerifyExprVisitor.cpp

@@ -0,0 +1,35 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License
+
+#include "query/generated/VerifyExprVisitor.h"
+
+namespace milvus::query {
+void
+VerifyExprVisitor::visit(BoolUnaryExpr& expr) {
+    // TODO
+}
+
+void
+VerifyExprVisitor::visit(BoolBinaryExpr& expr) {
+    // TODO
+}
+
+void
+VerifyExprVisitor::visit(TermExpr& expr) {
+    // TODO
+}
+
+void
+VerifyExprVisitor::visit(RangeExpr& expr) {
+    // TODO
+}
+
+}  // namespace milvus::query

internal/core/src/query/visitors/VerifyPlanNodeVisitor.cpp

@@ -0,0 +1,85 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License
+
+#include "query/generated/VerifyPlanNodeVisitor.h"
+#include "knowhere/index/vector_index/ConfAdapterMgr.h"
+#include "segcore/SegmentSmallIndex.h"
+#include "knowhere/index/vector_index/ConfAdapter.h"
+#include "knowhere/index/vector_index/helpers/IndexParameter.h"
+
+namespace milvus::query {
+
+#if 1
+namespace impl {
+// THIS CONTAINS EXTRA BODY FOR VISITOR
+// WILL BE USED BY GENERATOR UNDER suvlim/core_gen/
+class VerifyPlanNodeVisitor : PlanNodeVisitor {
+ public:
+    using RetType = QueryResult;
+    VerifyPlanNodeVisitor() = default;
+
+ private:
+    std::optional<RetType> ret_;
+};
+}  // namespace impl
+#endif
+
+static knowhere::IndexType
+InferIndexType(const Json& search_params) {
+    // ivf -> nprobe
+    // nsg -> search_length
+    // hnsw/rhnsw/*pq/*sq -> ef
+    // annoy -> search_k
+    // ngtpanng / ngtonng -> max_search_edges / epsilon
+    static const std::map<std::string, knowhere::IndexType> key_list = [] {
+        std::map<std::string, knowhere::IndexType> list;
+        namespace ip = knowhere::IndexParams;
+        namespace ie = knowhere::IndexEnum;
+        list.emplace(ip::nprobe, ie::INDEX_FAISS_IVFFLAT);
+        list.emplace(ip::search_length, ie::INDEX_NSG);
+        list.emplace(ip::ef, ie::INDEX_HNSW);
+        list.emplace(ip::search_k, ie::INDEX_ANNOY);
+        list.emplace(ip::max_search_edges, ie::INDEX_NGTONNG);
+        list.emplace(ip::epsilon, ie::INDEX_NGTONNG);
+        return list;
+    }();
+    auto dbg_str = search_params.dump();
+    for (auto& kv : search_params.items()) {
+        std::string key = kv.key();
+        if (key_list.count(key)) {
+            return key_list.at(key);
+        }
+    }
+    PanicInfo("failed to infer index type");
+}
+
+void
+VerifyPlanNodeVisitor::visit(FloatVectorANNS& node) {
+    auto& search_params = node.query_info_.search_params_;
+    auto inferred_type = InferIndexType(search_params);
+    auto adapter = knowhere::AdapterMgr::GetInstance().GetAdapter(inferred_type);
+    auto index_mode = knowhere::IndexMode::MODE_CPU;
+
+    // mock the api, topk will be passed from placeholder
+    auto params_copy = search_params;
+    params_copy[knowhere::meta::TOPK] = 10;
+
+    // NOTE: the second parameter is not checked in knowhere, may be redundant
+    auto passed = adapter->CheckSearch(params_copy, inferred_type, index_mode);
+    AssertInfo(passed, "invalid search params");
+}
+
+void
+VerifyPlanNodeVisitor::visit(BinaryVectorANNS& node) {
+    // TODO
+}
+
+}  // namespace milvus::query

internal/core/src/segcore/CMakeLists.txt

@@ -24,5 +24,6 @@ target_link_libraries(milvus_segcore
         dl backtrace
         milvus_common
         milvus_query
+        milvus_utils
         )
 

internal/core/src/segcore/SegmentBase.h

@@ -13,7 +13,7 @@
 #include <vector>
 
 #include "IndexMeta.h"
-#include "utils/Types.h"
+#include "common/Types.h"
 #include "common/Schema.h"
 #include <memory>
 

internal/core/src/segcore/SegmentNaive.cpp

@@ -274,19 +274,14 @@ SegmentNaive::QueryImpl(query::QueryDeprecatedPtr query_info, Timestamp timestam
     auto distances = final->Get<float*>(knowhere::meta::DISTANCE);
 
     auto total_num = num_queries * topK;
-    result.result_ids_.resize(total_num);
     result.result_distances_.resize(total_num);
 
     result.num_queries_ = num_queries;
     result.topK_ = topK;
 
-    std::copy_n(ids, total_num, result.result_ids_.data());
+    std::copy_n(ids, total_num, result.internal_seg_offsets_.data());
     std::copy_n(distances, total_num, result.result_distances_.data());
 
-    for (auto& id : result.result_ids_) {
-        id = record_.uids_[id];
-    }
-
     return Status::OK();
 }
 
@@ -347,7 +342,7 @@ SegmentNaive::QuerySlowImpl(query::QueryDeprecatedPtr query_info, Timestamp time
     result.topK_ = topK;
     auto row_num = topK * num_queries;
 
-    result.result_ids_.resize(row_num);
+    result.internal_seg_offsets_.resize(row_num);
     result.result_distances_.resize(row_num);
 
     for (int q_id = 0; q_id < num_queries; ++q_id) {
@@ -356,7 +351,7 @@ SegmentNaive::QuerySlowImpl(query::QueryDeprecatedPtr query_info, Timestamp time
             auto dst_id = topK - 1 - i + q_id * topK;
             auto [dis, offset] = records[q_id].top();
             records[q_id].pop();
-            result.result_ids_[dst_id] = record_.uids_[offset];
+            result.internal_seg_offsets_[dst_id] = offset;
             result.result_distances_[dst_id] = dis;
         }
     }

internal/core/src/segcore/SegmentSmallIndex.cpp

@@ -349,19 +349,12 @@ SegmentSmallIndex::FillTargetEntry(const query::Plan* plan, QueryResult& results
     Assert(results.result_offsets_.size() == size);
     Assert(results.row_data_.size() == 0);
 
-    // TODO: deprecate
-    results.result_ids_.clear();
-    results.result_ids_.resize(size);
-
     if (plan->schema_.get_is_auto_id()) {
         auto& uids = record_.uids_;
         for (int64_t i = 0; i < size; ++i) {
             auto seg_offset = results.internal_seg_offsets_[i];
             auto row_id = seg_offset == -1 ? -1 : uids[seg_offset];
 
-            // TODO: deprecate
-            results.result_ids_[i] = row_id;
-
             std::vector<char> blob(sizeof(row_id));
             memcpy(blob.data(), &row_id, sizeof(row_id));
             results.row_data_.emplace_back(std::move(blob));
@@ -377,9 +370,6 @@ SegmentSmallIndex::FillTargetEntry(const query::Plan* plan, QueryResult& results
             auto seg_offset = results.internal_seg_offsets_[i];
             auto row_id = seg_offset == -1 ? -1 : uids->operator[](seg_offset);
 
-            // TODO: deprecate
-            results.result_ids_[i] = row_id;
-
             std::vector<char> blob(sizeof(row_id));
             memcpy(blob.data(), &row_id, sizeof(row_id));
             results.row_data_.emplace_back(std::move(blob));

internal/core/src/segcore/reduce_c.cpp

@@ -14,10 +14,10 @@
 #include "segcore/reduce_c.h"
 
 #include "segcore/Reduce.h"
-#include "utils/Types.h"
+#include "common/Types.h"
 #include "pb/service_msg.pb.h"
 
-using SearchResult = milvus::engine::QueryResult;
+using SearchResult = milvus::QueryResult;
 
 int
 MergeInto(int64_t num_queries, int64_t topk, float* distances, int64_t* uids, float* new_distances, int64_t* new_uids) {

internal/core/src/segcore/segment_c.cpp

@@ -165,7 +165,7 @@ CStatus
 FillTargetEntry(CSegmentBase c_segment, CPlan c_plan, CQueryResult c_result) {
     auto segment = (milvus::segcore::SegmentBase*)c_segment;
     auto plan = (milvus::query::Plan*)c_plan;
-    auto result = (milvus::engine::QueryResult*)c_result;
+    auto result = (milvus::QueryResult*)c_result;
 
     auto status = CStatus();
     try {

internal/core/src/utils/Types.h

@@ -136,45 +136,5 @@ struct AttrsData {
     IDNumbers id_array_;
 };
 
-///////////////////////////////////////////////////////////////////////////////////////////////////
-struct QueryResult {
-    QueryResult() = default;
-    QueryResult(uint64_t num_queries, uint64_t topK) : topK_(topK), num_queries_(num_queries) {
-        auto count = get_row_count();
-        result_distances_.resize(count);
-        internal_seg_offsets_.resize(count);
-
-        // TODO: deprecated
-        result_ids_.resize(count);
-    }
-
-    [[nodiscard]] uint64_t
-    get_row_count() const {
-        return topK_ * num_queries_;
-    }
-
-    uint64_t num_queries_;
-    uint64_t topK_;
-    //    uint64_t total_row_count_;  // total_row_count_ = topK * num_queries_
-
-    // vector<tuple<Score, SegId, Offset>> data_reduced;
-
-    // vector<tuple<Score, SegId, Offset, RawData>>
-
-    // map<SegId, vector<tuple<DataOffset, ResLoc>>>
-    uint64_t seg_id_;
-    std::vector<float> result_distances_;
-
-    // TODO(gexi): utilize these field
-    std::vector<int64_t> internal_seg_offsets_;
-    std::vector<int64_t> result_offsets_;
-    std::vector<std::vector<char>> row_data_;
-
-    // TODO: deprecated, use row_data directly
-    std::vector<idx_t> result_ids_;
-};
-
-using QueryResultPtr = std::shared_ptr<QueryResult>;
-
 }  // namespace engine
 }  // namespace milvus

internal/core/unittest/CMakeLists.txt

@@ -14,6 +14,7 @@ set(MILVUS_TEST_FILES
         test_binary.cpp
         test_index_wrapper.cpp
         test_sealed.cpp
+        test_reduce.cpp
         )
 add_executable(all_tests
         ${MILVUS_TEST_FILES}
@@ -24,10 +25,8 @@ target_link_libraries(all_tests
         gtest_main
         milvus_segcore
         milvus_indexbuilder
-        knowhere
         log
         pthread
-        milvus_utils
         )
 
 install (TARGETS all_tests DESTINATION unittest)

internal/core/unittest/test_c_api.cpp

@@ -137,7 +137,7 @@ TEST(CApiTest, SearchTest) {
     auto offset = PreInsert(segment, N);
 
     auto ins_res = Insert(segment, offset, N, uids.data(), timestamps.data(), raw_data.data(), (int)line_sizeof, N);
-    assert(ins_res.error_code == Success);
+    ASSERT_EQ(ins_res.error_code, Success);
 
     const char* dsl_string = R"(
     {
@@ -176,11 +176,11 @@ TEST(CApiTest, SearchTest) {
     void* plan = nullptr;
 
     auto status = CreatePlan(collection, dsl_string, &plan);
-    assert(status.error_code == Success);
+    ASSERT_EQ(status.error_code, Success);
 
     void* placeholderGroup = nullptr;
     status = ParsePlaceholderGroup(plan, blob.data(), blob.length(), &placeholderGroup);
-    assert(status.error_code == Success);
+    ASSERT_EQ(status.error_code, Success);
 
     std::vector<CPlaceholderGroup> placeholderGroups;
     placeholderGroups.push_back(placeholderGroup);
@@ -189,7 +189,7 @@ TEST(CApiTest, SearchTest) {
 
     CQueryResult search_result;
     auto res = Search(segment, plan, placeholderGroups.data(), timestamps.data(), 1, &search_result);
-    assert(res.error_code == Success);
+    ASSERT_EQ(res.error_code, Success);
 
     DeletePlan(plan);
     DeletePlaceholderGroup(placeholderGroup);

internal/core/unittest/test_index_wrapper.cpp

@@ -11,6 +11,8 @@
 
 #include <tuple>
 #include <map>
+#include <limits>
+#include <math.h>
 #include <gtest/gtest.h>
 #include <google/protobuf/text_format.h>
 
@@ -41,16 +43,16 @@ generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowh
     if (index_type == milvus::knowhere::IndexEnum::INDEX_FAISS_IDMAP) {
         return milvus::knowhere::Config{
             {milvus::knowhere::meta::DIM, DIM},
-            // {milvus::knowhere::meta::TOPK, K},
+            {milvus::knowhere::meta::TOPK, K},
             {milvus::knowhere::Metric::TYPE, metric_type},
             {milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
         };
     } else if (index_type == milvus::knowhere::IndexEnum::INDEX_FAISS_IVFPQ) {
         return milvus::knowhere::Config{
             {milvus::knowhere::meta::DIM, DIM},
-            // {milvus::knowhere::meta::TOPK, K},
+            {milvus::knowhere::meta::TOPK, K},
             {milvus::knowhere::IndexParams::nlist, 100},
-            // {milvus::knowhere::IndexParams::nprobe, 4},
+            {milvus::knowhere::IndexParams::nprobe, 4},
             {milvus::knowhere::IndexParams::m, 4},
             {milvus::knowhere::IndexParams::nbits, 8},
             {milvus::knowhere::Metric::TYPE, metric_type},
@@ -59,9 +61,9 @@ generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowh
     } else if (index_type == milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT) {
         return milvus::knowhere::Config{
             {milvus::knowhere::meta::DIM, DIM},
-            // {milvus::knowhere::meta::TOPK, K},
+            {milvus::knowhere::meta::TOPK, K},
             {milvus::knowhere::IndexParams::nlist, 100},
-            // {milvus::knowhere::IndexParams::nprobe, 4},
+            {milvus::knowhere::IndexParams::nprobe, 4},
             {milvus::knowhere::Metric::TYPE, metric_type},
             {milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
 #ifdef MILVUS_GPU_VERSION
@@ -71,9 +73,9 @@ generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowh
     } else if (index_type == milvus::knowhere::IndexEnum::INDEX_FAISS_IVFSQ8) {
         return milvus::knowhere::Config{
             {milvus::knowhere::meta::DIM, DIM},
-            // {milvus::knowhere::meta::TOPK, K},
+            {milvus::knowhere::meta::TOPK, K},
             {milvus::knowhere::IndexParams::nlist, 100},
-            // {milvus::knowhere::IndexParams::nprobe, 4},
+            {milvus::knowhere::IndexParams::nprobe, 4},
             {milvus::knowhere::IndexParams::nbits, 8},
             {milvus::knowhere::Metric::TYPE, metric_type},
             {milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
@@ -84,9 +86,9 @@ generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowh
     } else if (index_type == milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT) {
         return milvus::knowhere::Config{
             {milvus::knowhere::meta::DIM, DIM},
-            // {milvus::knowhere::meta::TOPK, K},
+            {milvus::knowhere::meta::TOPK, K},
             {milvus::knowhere::IndexParams::nlist, 100},
-            // {milvus::knowhere::IndexParams::nprobe, 4},
+            {milvus::knowhere::IndexParams::nprobe, 4},
             {milvus::knowhere::IndexParams::m, 4},
             {milvus::knowhere::IndexParams::nbits, 8},
             {milvus::knowhere::Metric::TYPE, metric_type},
@@ -95,13 +97,14 @@ generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowh
     } else if (index_type == milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP) {
         return milvus::knowhere::Config{
             {milvus::knowhere::meta::DIM, DIM},
-            // {milvus::knowhere::meta::TOPK, K},
+            {milvus::knowhere::meta::TOPK, K},
             {milvus::knowhere::Metric::TYPE, metric_type},
         };
     } else if (index_type == milvus::knowhere::IndexEnum::INDEX_NSG) {
         return milvus::knowhere::Config{
             {milvus::knowhere::meta::DIM, DIM},
             {milvus::knowhere::IndexParams::nlist, 163},
+            {milvus::knowhere::meta::TOPK, K},
             {milvus::knowhere::IndexParams::nprobe, 8},
             {milvus::knowhere::IndexParams::knng, 20},
             {milvus::knowhere::IndexParams::search_length, 40},
@@ -127,17 +130,14 @@ generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowh
 #endif
     } else if (index_type == milvus::knowhere::IndexEnum::INDEX_HNSW) {
         return milvus::knowhere::Config{
-            {milvus::knowhere::meta::DIM, DIM},
+            {milvus::knowhere::meta::DIM, DIM},       {milvus::knowhere::meta::TOPK, K},
-            // {milvus::knowhere::meta::TOPK, 10},
+            {milvus::knowhere::IndexParams::M, 16},   {milvus::knowhere::IndexParams::efConstruction, 200},
-            {milvus::knowhere::IndexParams::M, 16},
+            {milvus::knowhere::IndexParams::ef, 200}, {milvus::knowhere::Metric::TYPE, metric_type},
-            {milvus::knowhere::IndexParams::efConstruction, 200},
-            {milvus::knowhere::IndexParams::ef, 200},
-            {milvus::knowhere::Metric::TYPE, metric_type},
         };
     } else if (index_type == milvus::knowhere::IndexEnum::INDEX_ANNOY) {
         return milvus::knowhere::Config{
             {milvus::knowhere::meta::DIM, DIM},
-            // {milvus::knowhere::meta::TOPK, 10},
+            {milvus::knowhere::meta::TOPK, K},
             {milvus::knowhere::IndexParams::n_trees, 4},
             {milvus::knowhere::IndexParams::search_k, 100},
             {milvus::knowhere::Metric::TYPE, metric_type},
@@ -146,7 +146,7 @@ generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowh
     } else if (index_type == milvus::knowhere::IndexEnum::INDEX_RHNSWFlat) {
         return milvus::knowhere::Config{
             {milvus::knowhere::meta::DIM, DIM},
-            // {milvus::knowhere::meta::TOPK, 10},
+            {milvus::knowhere::meta::TOPK, K},
             {milvus::knowhere::IndexParams::M, 16},
             {milvus::knowhere::IndexParams::efConstruction, 200},
             {milvus::knowhere::IndexParams::ef, 200},
@@ -156,7 +156,7 @@ generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowh
     } else if (index_type == milvus::knowhere::IndexEnum::INDEX_RHNSWPQ) {
         return milvus::knowhere::Config{
             {milvus::knowhere::meta::DIM, DIM},
-            // {milvus::knowhere::meta::TOPK, 10},
+            {milvus::knowhere::meta::TOPK, K},
             {milvus::knowhere::IndexParams::M, 16},
             {milvus::knowhere::IndexParams::efConstruction, 200},
             {milvus::knowhere::IndexParams::ef, 200},
@@ -167,7 +167,7 @@ generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowh
     } else if (index_type == milvus::knowhere::IndexEnum::INDEX_RHNSWSQ) {
         return milvus::knowhere::Config{
             {milvus::knowhere::meta::DIM, DIM},
-            // {milvus::knowhere::meta::TOPK, 10},
+            {milvus::knowhere::meta::TOPK, K},
             {milvus::knowhere::IndexParams::M, 16},
             {milvus::knowhere::IndexParams::efConstruction, 200},
             {milvus::knowhere::IndexParams::ef, 200},
@@ -177,7 +177,7 @@ generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowh
     } else if (index_type == milvus::knowhere::IndexEnum::INDEX_NGTPANNG) {
         return milvus::knowhere::Config{
             {milvus::knowhere::meta::DIM, DIM},
-            // {milvus::knowhere::meta::TOPK, 10},
+            {milvus::knowhere::meta::TOPK, K},
             {milvus::knowhere::Metric::TYPE, metric_type},
             {milvus::knowhere::IndexParams::edge_size, 10},
             {milvus::knowhere::IndexParams::epsilon, 0.1},
@@ -189,7 +189,7 @@ generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowh
     } else if (index_type == milvus::knowhere::IndexEnum::INDEX_NGTONNG) {
         return milvus::knowhere::Config{
             {milvus::knowhere::meta::DIM, DIM},
-            // {milvus::knowhere::meta::TOPK, 10},
+            {milvus::knowhere::meta::TOPK, K},
             {milvus::knowhere::Metric::TYPE, metric_type},
             {milvus::knowhere::IndexParams::edge_size, 20},
             {milvus::knowhere::IndexParams::epsilon, 0.1},
@@ -234,6 +234,99 @@ GenDataset(int64_t N, const milvus::knowhere::MetricType& metric_type, bool is_b
         return milvus::segcore::DataGen(schema, N);
     }
 }
+
+using QueryResultPtr = std::unique_ptr<milvus::indexbuilder::IndexWrapper::QueryResult>;
+void
+PrintQueryResult(const QueryResultPtr& result) {
+    auto nq = result->nq;
+    auto k = result->topk;
+
+    std::stringstream ss_id;
+    std::stringstream ss_dist;
+
+    for (auto i = 0; i < nq; i++) {
+        for (auto j = 0; j < k; ++j) {
+            ss_id << result->ids[i * k + j] << " ";
+            ss_dist << result->distances[i * k + j] << " ";
+        }
+        ss_id << std::endl;
+        ss_dist << std::endl;
+    }
+    std::cout << "id\n" << ss_id.str() << std::endl;
+    std::cout << "dist\n" << ss_dist.str() << std::endl;
+}
+
+float
+L2(const float* point_a, const float* point_b, int dim) {
+    float dis = 0;
+    for (auto i = 0; i < dim; i++) {
+        auto c_a = point_a[i];
+        auto c_b = point_b[i];
+        dis += pow(c_b - c_a, 2);
+    }
+    return dis;
+}
+
+int hamming_weight(uint8_t n) {
+    int count=0;
+    while(n != 0){
+        count += n&1;
+        n >>= 1;
+    }
+    return count;
+}
+float
+Jaccard(const uint8_t* point_a, const uint8_t* point_b, int dim) {
+    float dis;
+    int len = dim / 8;
+    float intersection = 0;
+    float union_num = 0;
+    for (int i = 0; i < len; i++) {
+        intersection += hamming_weight(point_a[i] & point_b[i]);
+        union_num += hamming_weight(point_a[i] | point_b[i]);
+    }
+    dis = 1 - (intersection / union_num);
+    return dis;
+}
+
+float
+CountDistance(const void* point_a,
+              const void* point_b,
+              int dim,
+              const milvus::knowhere::MetricType& metric,
+              bool is_binary = false) {
+    if (point_a == nullptr || point_b == nullptr) {
+        return std::numeric_limits<float>::max();
+    }
+    if (metric == milvus::knowhere::Metric::L2) {
+        return L2(static_cast<const float*>(point_a), static_cast<const float*>(point_b), dim);
+    } else if (metric == milvus::knowhere::Metric::JACCARD) {
+        return Jaccard(static_cast<const uint8_t*>(point_a), static_cast<const uint8_t*>(point_b), dim);
+    } else {
+        return std::numeric_limits<float>::max();
+    }
+}
+
+void
+CheckDistances(const QueryResultPtr& result,
+               const milvus::knowhere::DatasetPtr& base_dataset,
+               const milvus::knowhere::DatasetPtr& query_dataset,
+               const milvus::knowhere::MetricType& metric,
+               const float threshold = 1.0e-5) {
+    auto base_vecs = base_dataset->Get<float*>(milvus::knowhere::meta::TENSOR);
+    auto query_vecs = query_dataset->Get<float*>(milvus::knowhere::meta::TENSOR);
+    auto dim = base_dataset->Get<int64_t>(milvus::knowhere::meta::DIM);
+    auto nq = result->nq;
+    auto k = result->topk;
+    for (auto i = 0; i < nq; i++) {
+        for (auto j = 0; j < k; ++j) {
+            auto dis = result->distances[i * k + j];
+            auto id = result->ids[i * k + j];
+            auto count_dis = CountDistance(query_vecs + i * dim, base_vecs + id * dim, dim, metric);
+            // assert(std::abs(dis - count_dis) < threshold);
+        }
+    }
+}
 }  // namespace
 
 using Param = std::pair<milvus::knowhere::IndexType, milvus::knowhere::MetricType>;
@@ -247,8 +340,26 @@ class IndexWrapperTest : public ::testing::TestWithParam<Param> {
         metric_type = param.second;
         std::tie(type_params, index_params) = generate_params(index_type, metric_type);
 
-        std::map<std::string, bool> is_binary_map = {{milvus::knowhere::IndexEnum::INDEX_FAISS_IVFPQ, false},
+        std::map<std::string, bool> is_binary_map = {
-                                                     {milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, true}};
+            {milvus::knowhere::IndexEnum::INDEX_FAISS_IDMAP, false},
+            {milvus::knowhere::IndexEnum::INDEX_FAISS_IVFPQ, false},
+            {milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT, false},
+            {milvus::knowhere::IndexEnum::INDEX_FAISS_IVFSQ8, false},
+            {milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, true},
+            {milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP, true},
+#ifdef MILVUS_SUPPORT_SPTAG
+            {milvus::knowhere::IndexEnum::INDEX_SPTAG_KDT_RNT, false},
+            {milvus::knowhere::IndexEnum::INDEX_SPTAG_BKT_RNT, false},
+#endif
+            {milvus::knowhere::IndexEnum::INDEX_HNSW, false},
+            {milvus::knowhere::IndexEnum::INDEX_ANNOY, false},
+            {milvus::knowhere::IndexEnum::INDEX_RHNSWFlat, false},
+            {milvus::knowhere::IndexEnum::INDEX_RHNSWPQ, false},
+            {milvus::knowhere::IndexEnum::INDEX_RHNSWSQ, false},
+            {milvus::knowhere::IndexEnum::INDEX_NGTPANNG, false},
+            {milvus::knowhere::IndexEnum::INDEX_NGTONNG, false},
+            {milvus::knowhere::IndexEnum::INDEX_NSG, false},
+        };
 
         is_binary = is_binary_map[index_type];
 
@@ -262,9 +373,13 @@ class IndexWrapperTest : public ::testing::TestWithParam<Param> {
         if (!is_binary) {
             xb_data = dataset.get_col<float>(0);
             xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data());
+            xq_data = dataset.get_col<float>(0);
+            xq_dataset = milvus::knowhere::GenDataset(NQ, DIM, xq_data.data());
         } else {
             xb_bin_data = dataset.get_col<uint8_t>(0);
             xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_bin_data.data());
+            xq_bin_data = dataset.get_col<uint8_t>(0);
+            xq_dataset = milvus::knowhere::GenDataset(NQ, DIM, xq_bin_data.data());
         }
     }
 
@@ -282,6 +397,9 @@ class IndexWrapperTest : public ::testing::TestWithParam<Param> {
     std::vector<float> xb_data;
     std::vector<uint8_t> xb_bin_data;
     std::vector<milvus::knowhere::IDType> ids;
+    milvus::knowhere::DatasetPtr xq_dataset;
+    std::vector<float> xq_data;
+    std::vector<uint8_t> xq_bin_data;
 };
 
 TEST(PQ, Build) {
@@ -308,6 +426,47 @@ TEST(IVFFLATNM, Build) {
     ASSERT_NO_THROW(index->AddWithoutIds(xb_dataset, conf));
 }
 
+TEST(IVFFLATNM, Query) {
+    auto index_type = milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT;
+    auto metric_type = milvus::knowhere::Metric::L2;
+    auto conf = generate_conf(index_type, metric_type);
+    auto index = milvus::knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type);
+    auto dataset = GenDataset(NB, metric_type, false);
+    auto xb_data = dataset.get_col<float>(0);
+    auto xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data());
+    ASSERT_NO_THROW(index->Train(xb_dataset, conf));
+    ASSERT_NO_THROW(index->AddWithoutIds(xb_dataset, conf));
+    auto bs = index->Serialize(conf);
+    auto bptr = std::make_shared<milvus::knowhere::Binary>();
+    bptr->data = std::shared_ptr<uint8_t[]>((uint8_t*)xb_data.data(), [&](uint8_t*) {});
+    bptr->size = DIM * NB * sizeof(float);
+    bs.Append(RAW_DATA, bptr);
+    index->Load(bs);
+    auto xq_data = dataset.get_col<float>(0);
+    auto xq_dataset = milvus::knowhere::GenDataset(NQ, DIM, xq_data.data());
+    auto result = index->Query(xq_dataset, conf, nullptr);
+}
+
+TEST(NSG, Query) {
+    auto index_type = milvus::knowhere::IndexEnum::INDEX_NSG;
+    auto metric_type = milvus::knowhere::Metric::L2;
+    auto conf = generate_conf(index_type, metric_type);
+    auto index = milvus::knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type);
+    auto dataset = GenDataset(NB, metric_type, false);
+    auto xb_data = dataset.get_col<float>(0);
+    auto xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data());
+    index->BuildAll(xb_dataset, conf);
+    auto bs = index->Serialize(conf);
+    auto bptr = std::make_shared<milvus::knowhere::Binary>();
+    bptr->data = std::shared_ptr<uint8_t[]>((uint8_t*)xb_data.data(), [&](uint8_t*) {});
+    bptr->size = DIM * NB * sizeof(float);
+    bs.Append(RAW_DATA, bptr);
+    index->Load(bs);
+    auto xq_data = dataset.get_col<float>(0);
+    auto xq_dataset = milvus::knowhere::GenDataset(NQ, DIM, xq_data.data());
+    auto result = index->Query(xq_dataset, conf, nullptr);
+}
+
 TEST(BINFLAT, Build) {
     auto index_type = milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT;
     auto metric_type = milvus::knowhere::Metric::JACCARD;
@@ -485,12 +644,7 @@ TEST_P(IndexWrapperTest, Dim) {
 TEST_P(IndexWrapperTest, BuildWithoutIds) {
     auto index =
         std::make_unique<milvus::indexbuilder::IndexWrapper>(type_params_str.c_str(), index_params_str.c_str());
-
+    ASSERT_NO_THROW(index->BuildWithoutIds(xb_dataset));
-    if (milvus::indexbuilder::is_in_need_id_list(index_type)) {
-        ASSERT_ANY_THROW(index->BuildWithoutIds(xb_dataset));
-    } else {
-        ASSERT_NO_THROW(index->BuildWithoutIds(xb_dataset));
-    }
 }
 
 TEST_P(IndexWrapperTest, Codec) {
@@ -511,3 +665,16 @@ TEST_P(IndexWrapperTest, Codec) {
         ASSERT_EQ(strcmp(binary.data, copy_binary.data), 0);
     }
 }
+
+TEST_P(IndexWrapperTest, Query) {
+    auto index_wrapper =
+        std::make_unique<milvus::indexbuilder::IndexWrapper>(type_params_str.c_str(), index_params_str.c_str());
+
+    index_wrapper->BuildWithoutIds(xb_dataset);
+
+    std::unique_ptr<milvus::indexbuilder::IndexWrapper::QueryResult> query_result = index_wrapper->Query(xq_dataset);
+    ASSERT_EQ(query_result->topk, K);
+    ASSERT_EQ(query_result->nq, NQ);
+    ASSERT_EQ(query_result->distances.size(), query_result->topk * query_result->nq);
+    ASSERT_EQ(query_result->ids.size(), query_result->topk * query_result->nq);
+}

internal/core/unittest/test_indexing.cpp

@@ -33,7 +33,7 @@
 #include "test_utils/Timer.h"
 #include "segcore/Reduce.h"
 #include "test_utils/DataGen.h"
-#include "query/BruteForceSearch.h"
+#include "query/SearchBruteForce.h"
 
 using std::cin;
 using std::cout;
@@ -245,8 +245,6 @@ TEST(Indexing, BinaryBruteForce) {
     schema->AddField("vecbin", DataType::VECTOR_BINARY, dim, MetricType::METRIC_Jaccard);
     schema->AddField("age", DataType::INT64);
     auto dataset = DataGen(schema, N, 10);
-    vector<float> distances(result_count);
-    vector<int64_t> ids(result_count);
     auto bin_vec = dataset.get_col<uint8_t>(0);
     auto line_sizeof = schema->operator[](0).get_sizeof();
     auto query_data = 1024 * line_sizeof + bin_vec.data();
@@ -258,13 +256,13 @@ TEST(Indexing, BinaryBruteForce) {
         query_data                          //
     };
 
-    query::BinarySearchBruteForce(query_dataset, bin_vec.data(), N, distances.data(), ids.data());
+    auto sub_result = query::BinarySearchBruteForce(query_dataset, bin_vec.data(), N);
 
     QueryResult qr;
     qr.num_queries_ = num_queries;
     qr.topK_ = topk;
-    qr.internal_seg_offsets_ = ids;
+    qr.internal_seg_offsets_ = std::move(sub_result.mutable_labels());
-    qr.result_distances_ = distances;
+    qr.result_distances_ = std::move(sub_result.mutable_values());
 
     auto json = QueryResultToJson(qr);
     auto ref = json::parse(R"(

internal/core/unittest/test_query.cpp

@@ -402,7 +402,7 @@ TEST(Query, FillSegment) {
     pb::schema::CollectionSchema proto;
     proto.set_name("col");
     proto.set_description("asdfhsalkgfhsadg");
-    proto.set_autoid(true);
+    proto.set_autoid(false);
 
     {
         auto field = proto.add_fields();
@@ -425,7 +425,7 @@ TEST(Query, FillSegment) {
         field->set_fieldid(101);
         field->set_is_primary_key(true);
         field->set_description("asdgfsagf");
-        field->set_data_type(pb::schema::DataType::INT32);
+        field->set_data_type(pb::schema::DataType::INT64);
     }
 
     auto schema = Schema::ParseFrom(proto);
@@ -466,18 +466,17 @@ TEST(Query, FillSegment) {
     result.result_offsets_.resize(topk * num_queries);
     segment->FillTargetEntry(plan.get(), result);
 
-    // TODO: deprecated result_ids_
-    ASSERT_EQ(result.result_ids_, result.internal_seg_offsets_);
-
     auto ans = result.row_data_;
     ASSERT_EQ(ans.size(), topk * num_queries);
     int64_t std_index = 0;
+    auto std_vec = dataset.get_col<int64_t>(1);
     for (auto& vec : ans) {
         ASSERT_EQ(vec.size(), sizeof(int64_t));
         int64_t val;
         memcpy(&val, vec.data(), sizeof(int64_t));
-        auto std_val = result.result_ids_[std_index];
+        auto internal_offset = result.internal_seg_offsets_[std_index];
-        ASSERT_EQ(val, std_val);
+        auto std_val = std_vec[internal_offset];
+        ASSERT_EQ(val, std_val) << "io:" << internal_offset;
         ++std_index;
     }
 }

internal/core/unittest/test_reduce.cpp

@@ -0,0 +1,120 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License
+
+#include <gtest/gtest.h>
+#include "query/SubQueryResult.h"
+#include <vector>
+#include <queue>
+#include <random>
+
+using namespace milvus;
+using namespace milvus::query;
+
+TEST(Reduce, SubQueryResult) {
+    int64_t num_queries = 512;
+    int64_t topk = 32;
+    int64_t iteration = 50;
+    constexpr int64_t limit = 100000000L;
+    auto metric_type = MetricType::METRIC_L2;
+    using queue_type = std::priority_queue<int64_t>;
+
+    std::vector<queue_type> ref_results(num_queries);
+    for (auto& ref_result : ref_results) {
+        for (int i = 0; i < topk; ++i) {
+            ref_result.push(limit);
+        }
+    }
+    std::default_random_engine e(42);
+    SubQueryResult final_result(num_queries, topk, metric_type);
+    for (int i = 0; i < iteration; ++i) {
+        std::vector<int64_t> labels;
+        std::vector<float> values;
+        for (int n = 0; n < num_queries; ++n) {
+            for (int k = 0; k < topk; ++k) {
+                auto gen_x = e() % limit;
+                ref_results[n].push(gen_x);
+                ref_results[n].pop();
+                labels.push_back(gen_x);
+                values.push_back(gen_x);
+            }
+            std::sort(labels.begin() + n * topk, labels.begin() + n * topk + topk);
+            std::sort(values.begin() + n * topk, values.begin() + n * topk + topk);
+        }
+        SubQueryResult sub_result(num_queries, topk, metric_type);
+        sub_result.mutable_values() = values;
+        sub_result.mutable_labels() = labels;
+        final_result.merge(sub_result);
+    }
+
+    for (int n = 0; n < num_queries; ++n) {
+        ASSERT_EQ(ref_results[n].size(), topk);
+        for (int k = 0; k < topk; ++k) {
+            auto ref_x = ref_results[n].top();
+            ref_results[n].pop();
+            auto index = n * topk + topk - 1 - k;
+            auto label = final_result.get_labels()[index];
+            auto value = final_result.get_values()[index];
+            ASSERT_EQ(label, ref_x);
+            ASSERT_EQ(value, ref_x);
+        }
+    }
+}
+
+TEST(Reduce, SubQueryResultDesc) {
+    int64_t num_queries = 512;
+    int64_t topk = 32;
+    int64_t iteration = 50;
+    constexpr int64_t limit = 100000000L;
+    constexpr int64_t init_value = 0;
+    auto metric_type = MetricType::METRIC_INNER_PRODUCT;
+    using queue_type = std::priority_queue<int64_t, std::vector<int64_t>, std::greater<int64_t>>;
+
+    std::vector<queue_type> ref_results(num_queries);
+    for (auto& ref_result : ref_results) {
+        for (int i = 0; i < topk; ++i) {
+            ref_result.push(init_value);
+        }
+    }
+    std::default_random_engine e(42);
+    SubQueryResult final_result(num_queries, topk, metric_type);
+    for (int i = 0; i < iteration; ++i) {
+        std::vector<int64_t> labels;
+        std::vector<float> values;
+        for (int n = 0; n < num_queries; ++n) {
+            for (int k = 0; k < topk; ++k) {
+                auto gen_x = e() % limit;
+                ref_results[n].push(gen_x);
+                ref_results[n].pop();
+                labels.push_back(gen_x);
+                values.push_back(gen_x);
+            }
+            std::sort(labels.begin() + n * topk, labels.begin() + n * topk + topk, std::greater<int64_t>());
+            std::sort(values.begin() + n * topk, values.begin() + n * topk + topk, std::greater<float>());
+        }
+        SubQueryResult sub_result(num_queries, topk, metric_type);
+        sub_result.mutable_values() = values;
+        sub_result.mutable_labels() = labels;
+        final_result.merge(sub_result);
+    }
+
+    for (int n = 0; n < num_queries; ++n) {
+        ASSERT_EQ(ref_results[n].size(), topk);
+        for (int k = 0; k < topk; ++k) {
+            auto ref_x = ref_results[n].top();
+            ref_results[n].pop();
+            auto index = n * topk + topk - 1 - k;
+            auto label = final_result.get_labels()[index];
+            auto value = final_result.get_values()[index];
+            ASSERT_EQ(label, ref_x);
+            ASSERT_EQ(value, ref_x);
+        }
+    }
+}

internal/indexbuilder/indexbuilder.go

@@ -11,9 +11,6 @@ import (
 
 	miniokv "github.com/zilliztech/milvus-distributed/internal/kv/minio"
 
-	"github.com/minio/minio-go/v7"
-	"github.com/minio/minio-go/v7/pkg/credentials"
-
 	"go.etcd.io/etcd/clientv3"
 
 	"github.com/zilliztech/milvus-distributed/internal/allocator"
@@ -71,19 +68,16 @@ func CreateBuilder(ctx context.Context) (*Builder, error) {
 
 	idAllocator, err := allocator.NewIDAllocator(b.loopCtx, Params.MasterAddress)
 
-	minIOEndPoint := Params.MinIOAddress
+	option := &miniokv.Option{
-	minIOAccessKeyID := Params.MinIOAccessKeyID
+		Address:           Params.MinIOAddress,
-	minIOSecretAccessKey := Params.MinIOSecretAccessKey
+		AccessKeyID:       Params.MinIOAccessKeyID,
-	minIOUseSSL := Params.MinIOUseSSL
+		SecretAccessKeyID: Params.MinIOSecretAccessKey,
-	minIOClient, err := minio.New(minIOEndPoint, &minio.Options{
+		UseSSL:            Params.MinIOUseSSL,
-		Creds:  credentials.NewStaticV4(minIOAccessKeyID, minIOSecretAccessKey, ""),
+		BucketName:        Params.MinioBucketName,
-		Secure: minIOUseSSL,
+		CreateBucket:      true,
-	})
-	if err != nil {
-		return nil, err
 	}
 
-	b.kv, err = miniokv.NewMinIOKV(b.loopCtx, minIOClient, Params.MinioBucketName)
+	b.kv, err = miniokv.NewMinIOKV(b.loopCtx, option)
 	if err != nil {
 		return nil, err
 	}

internal/kv/minio/minio_kv.go

@@ -2,11 +2,15 @@ package miniokv
 
 import (
 	"context"
+	"fmt"
+
 	"io"
 	"log"
 	"strings"
 
 	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/zilliztech/milvus-distributed/internal/errors"
 )
 
 type MinIOKV struct {
@@ -15,24 +19,46 @@ type MinIOKV struct {
 	bucketName  string
 }
 
-// NewMinIOKV creates a new MinIO kv.
+type Option struct {
-func NewMinIOKV(ctx context.Context, client *minio.Client, bucketName string) (*MinIOKV, error) {
+	Address           string
+	AccessKeyID       string
+	BucketName        string
+	SecretAccessKeyID string
+	UseSSL            bool
+	CreateBucket      bool // when bucket not existed, create it
+}
 
-	bucketExists, err := client.BucketExists(ctx, bucketName)
+func NewMinIOKV(ctx context.Context, option *Option) (*MinIOKV, error) {
+	minIOClient, err := minio.New(option.Address, &minio.Options{
+		Creds:  credentials.NewStaticV4(option.AccessKeyID, option.SecretAccessKeyID, ""),
+		Secure: option.UseSSL,
+	})
 	if err != nil {
 		return nil, err
 	}
 
-	if !bucketExists {
+	bucketExists, err := minIOClient.BucketExists(ctx, option.BucketName)
-		err = client.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{})
+	if err != nil {
-		if err != nil {
+		return nil, err
-			return nil, err
+	}
+
+	if option.CreateBucket {
+		if !bucketExists {
+			err = minIOClient.MakeBucket(ctx, option.BucketName, minio.MakeBucketOptions{})
+			if err != nil {
+				return nil, err
+			}
+		}
+	} else {
+		if !bucketExists {
+			return nil, errors.New(fmt.Sprintf("Bucket %s not Existed.", option.BucketName))
 		}
 	}
+
 	return &MinIOKV{
 		ctx:         ctx,
-		minioClient: client,
+		minioClient: minIOClient,
-		bucketName:  bucketName,
+		bucketName:  option.BucketName,
 	}, nil
 }
 

internal/kv/minio/minio_kv_test.go

@@ -5,8 +5,6 @@ import (
 	"strconv"
 	"testing"
 
-	"github.com/minio/minio-go/v7"
-	"github.com/minio/minio-go/v7/pkg/credentials"
 	miniokv "github.com/zilliztech/milvus-distributed/internal/kv/minio"
 	"github.com/zilliztech/milvus-distributed/internal/util/paramtable"
 
@@ -15,24 +13,31 @@ import (
 
 var Params paramtable.BaseTable
 
-func TestMinIOKV_Load(t *testing.T) {
+func newMinIOKVClient(ctx context.Context, bucketName string) (*miniokv.MinIOKV, error) {
-	Params.Init()
 	endPoint, _ := Params.Load("_MinioAddress")
 	accessKeyID, _ := Params.Load("minio.accessKeyID")
 	secretAccessKey, _ := Params.Load("minio.secretAccessKey")
 	useSSLStr, _ := Params.Load("minio.useSSL")
+	useSSL, _ := strconv.ParseBool(useSSLStr)
+	option := &miniokv.Option{
+		Address:           endPoint,
+		AccessKeyID:       accessKeyID,
+		SecretAccessKeyID: secretAccessKey,
+		UseSSL:            useSSL,
+		BucketName:        bucketName,
+		CreateBucket:      true,
+	}
+	client, err := miniokv.NewMinIOKV(ctx, option)
+	return client, err
+}
+
+func TestMinIOKV_Load(t *testing.T) {
+	Params.Init()
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	useSSL, _ := strconv.ParseBool(useSSLStr)
-
-	minioClient, err := minio.New(endPoint, &minio.Options{
-		Creds:  credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
-		Secure: useSSL,
-	})
-	assert.Nil(t, err)
 
 	bucketName := "fantastic-tech-test"
-	MinIOKV, err := miniokv.NewMinIOKV(ctx, minioClient, bucketName)
+	MinIOKV, err := newMinIOKVClient(ctx, bucketName)
 	assert.Nil(t, err)
 	defer MinIOKV.RemoveWithPrefix("")
 
@@ -79,25 +84,14 @@ func TestMinIOKV_Load(t *testing.T) {
 }
 
 func TestMinIOKV_MultiSave(t *testing.T) {
+	Params.Init()
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	Params.Init()
-	endPoint, _ := Params.Load("_MinioAddress")
-	accessKeyID, _ := Params.Load("minio.accessKeyID")
-	secretAccessKey, _ := Params.Load("minio.secretAccessKey")
-	useSSLStr, _ := Params.Load("minio.useSSL")
-	useSSL, _ := strconv.ParseBool(useSSLStr)
-
-	minioClient, err := minio.New(endPoint, &minio.Options{
-		Creds:  credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
-		Secure: useSSL,
-	})
-	assert.Nil(t, err)
-
 	bucketName := "fantastic-tech-test"
-	MinIOKV, err := miniokv.NewMinIOKV(ctx, minioClient, bucketName)
+	MinIOKV, err := newMinIOKVClient(ctx, bucketName)
 	assert.Nil(t, err)
+
 	defer MinIOKV.RemoveWithPrefix("")
 
 	err = MinIOKV.Save("key_1", "111")
@@ -117,25 +111,13 @@ func TestMinIOKV_MultiSave(t *testing.T) {
 }
 
 func TestMinIOKV_Remove(t *testing.T) {
+	Params.Init()
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	Params.Init()
-	endPoint, _ := Params.Load("_MinioAddress")
-	accessKeyID, _ := Params.Load("minio.accessKeyID")
-	secretAccessKey, _ := Params.Load("minio.secretAccessKey")
-	useSSLStr, _ := Params.Load("minio.useSSL")
-	useSSL, _ := strconv.ParseBool(useSSLStr)
-
-	minioClient, err := minio.New(endPoint, &minio.Options{
-		Creds:  credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
-		Secure: useSSL,
-	})
-	assert.Nil(t, err)
-
 	bucketName := "fantastic-tech-test"
-	MinIOKV, err := miniokv.NewMinIOKV(ctx, minioClient, bucketName)
+	MinIOKV, err := newMinIOKVClient(ctx, bucketName)
 	assert.Nil(t, err)
 	defer MinIOKV.RemoveWithPrefix("")
 

internal/master/client.go

@@ -9,19 +9,25 @@ import (
 )
 
 type WriteNodeClient interface {
-	FlushSegment(segmentID UniqueID) error
+	FlushSegment(segmentID UniqueID, collectionID UniqueID, partitionTag string, timestamp Timestamp) error
 	DescribeSegment(segmentID UniqueID) (*writerclient.SegmentDescription, error)
 	GetInsertBinlogPaths(segmentID UniqueID) (map[UniqueID][]string, error)
 }
 
 type MockWriteNodeClient struct {
-	segmentID UniqueID
+	segmentID    UniqueID
-	flushTime time.Time
+	flushTime    time.Time
+	partitionTag string
+	timestamp    Timestamp
+	collectionID UniqueID
 }
 
-func (m *MockWriteNodeClient) FlushSegment(segmentID UniqueID) error {
+func (m *MockWriteNodeClient) FlushSegment(segmentID UniqueID, collectionID UniqueID, partitionTag string, timestamp Timestamp) error {
 	m.flushTime = time.Now()
 	m.segmentID = segmentID
+	m.collectionID = collectionID
+	m.partitionTag = partitionTag
+	m.timestamp = timestamp
 	return nil
 }
 

internal/master/flush_scheduler.go

@@ -17,11 +17,12 @@ type FlushScheduler struct {
 	segmentDescribeChan chan UniqueID
 	indexBuilderSch     persistenceScheduler
 
-	ctx    context.Context
+	ctx                context.Context
-	cancel context.CancelFunc
+	cancel             context.CancelFunc
+	globalTSOAllocator func() (Timestamp, error)
 }
 
-func NewFlushScheduler(ctx context.Context, client WriteNodeClient, metaTable *metaTable, buildScheduler *IndexBuildScheduler) *FlushScheduler {
+func NewFlushScheduler(ctx context.Context, client WriteNodeClient, metaTable *metaTable, buildScheduler *IndexBuildScheduler, globalTSOAllocator func() (Timestamp, error)) *FlushScheduler {
 	ctx2, cancel := context.WithCancel(ctx)
 
 	return &FlushScheduler{
@@ -32,12 +33,23 @@ func NewFlushScheduler(ctx context.Context, client WriteNodeClient, metaTable *m
 		segmentDescribeChan: make(chan UniqueID, 100),
 		ctx:                 ctx2,
 		cancel:              cancel,
+		globalTSOAllocator:  globalTSOAllocator,
 	}
 }
 
 func (scheduler *FlushScheduler) schedule(id interface{}) error {
 	segmentID := id.(UniqueID)
-	err := scheduler.client.FlushSegment(segmentID)
+	segmentMeta, err := scheduler.metaTable.GetSegmentByID(segmentID)
+	if err != nil {
+		return err
+	}
+
+	ts, err := scheduler.globalTSOAllocator()
+	if err != nil {
+		return err
+	}
+	// todo set corrent timestamp
+	err = scheduler.client.FlushSegment(segmentID, segmentMeta.CollectionID, segmentMeta.PartitionTag, ts)
 	log.Printf("flush segment %d", segmentID)
 	if err != nil {
 		return err

internal/master/master.go

@@ -193,7 +193,7 @@ func CreateServer(ctx context.Context) (*Master, error) {
 
 	m.indexLoadSch = NewIndexLoadScheduler(ctx, loadIndexClient, m.metaTable)
 	m.indexBuildSch = NewIndexBuildScheduler(ctx, buildIndexClient, m.metaTable, m.indexLoadSch)
-	m.flushSch = NewFlushScheduler(ctx, flushClient, m.metaTable, m.indexBuildSch)
+	m.flushSch = NewFlushScheduler(ctx, flushClient, m.metaTable, m.indexBuildSch, func() (Timestamp, error) { return m.tsoAllocator.AllocOne() })
 
 	m.segmentAssigner = NewSegmentAssigner(ctx, metakv,
 		func() (Timestamp, error) { return m.tsoAllocator.AllocOne() },
@@ -218,6 +218,7 @@ func CreateServer(ctx context.Context) (*Master, error) {
 
 	m.grpcServer = grpc.NewServer()
 	masterpb.RegisterMasterServer(m.grpcServer, m)
+
 	return m, nil
 }
 

internal/master/master_test.go

@@ -110,6 +110,7 @@ func TestMaster(t *testing.T) {
 
 	conn, err := grpc.DialContext(ctx, Params.Address, grpc.WithInsecure(), grpc.WithBlock())
 	require.Nil(t, err)
+
 	cli := masterpb.NewMasterClient(conn)
 
 	t.Run("TestConfigTask", func(t *testing.T) {
@@ -886,12 +887,6 @@ func TestMaster(t *testing.T) {
 		var k2sMsgstream ms.MsgStream = k2sMs
 		assert.True(t, receiveTimeTickMsg(&k2sMsgstream))
 
-		conn, err := grpc.DialContext(ctx, Params.Address, grpc.WithInsecure(), grpc.WithBlock())
-		assert.Nil(t, err)
-		defer conn.Close()
-
-		cli := masterpb.NewMasterClient(conn)
-
 		sch := schemapb.CollectionSchema{
 			Name:        "name" + strconv.FormatUint(rand.Uint64(), 10),
 			Description: "test collection",

internal/master/persistence_scheduler_test.go

@@ -59,7 +59,11 @@ func TestPersistenceScheduler(t *testing.T) {
 	//Init scheduler
 	indexLoadSch := NewIndexLoadScheduler(ctx, loadIndexClient, meta)
 	indexBuildSch := NewIndexBuildScheduler(ctx, buildIndexClient, meta, indexLoadSch)
-	flushSch := NewFlushScheduler(ctx, flushClient, meta, indexBuildSch)
+	cnt := 0
+	flushSch := NewFlushScheduler(ctx, flushClient, meta, indexBuildSch, func() (Timestamp, error) {
+		cnt++
+		return Timestamp(cnt), nil
+	})
 
 	//scheduler start
 	err = indexLoadSch.Start()

internal/msgstream/msg.go

@@ -1,6 +1,8 @@
 package msgstream
 
 import (
+	"context"
+
 	"github.com/golang/protobuf/proto"
 	internalPb "github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
 )
@@ -8,6 +10,8 @@ import (
 type MsgType = internalPb.MsgType
 
 type TsMsg interface {
+	GetContext() context.Context
+	SetContext(context.Context)
 	BeginTs() Timestamp
 	EndTs() Timestamp
 	Type() MsgType
@@ -17,6 +21,7 @@ type TsMsg interface {
 }
 
 type BaseMsg struct {
+	ctx            context.Context
 	BeginTimestamp Timestamp
 	EndTimestamp   Timestamp
 	HashValues     []uint32
@@ -44,6 +49,14 @@ func (it *InsertMsg) Type() MsgType {
 	return it.MsgType
 }
 
+func (it *InsertMsg) GetContext() context.Context {
+	return it.ctx
+}
+
+func (it *InsertMsg) SetContext(ctx context.Context) {
+	it.ctx = ctx
+}
+
 func (it *InsertMsg) Marshal(input TsMsg) ([]byte, error) {
 	insertMsg := input.(*InsertMsg)
 	insertRequest := &insertMsg.InsertRequest
@@ -88,6 +101,13 @@ func (fl *FlushMsg) Type() MsgType {
 	return fl.GetMsgType()
 }
 
+func (fl *FlushMsg) GetContext() context.Context {
+	return fl.ctx
+}
+func (fl *FlushMsg) SetContext(ctx context.Context) {
+	fl.ctx = ctx
+}
+
 func (fl *FlushMsg) Marshal(input TsMsg) ([]byte, error) {
 	flushMsgTask := input.(*FlushMsg)
 	flushMsg := &flushMsgTask.FlushMsg
@@ -121,6 +141,14 @@ func (dt *DeleteMsg) Type() MsgType {
 	return dt.MsgType
 }
 
+func (dt *DeleteMsg) GetContext() context.Context {
+	return dt.ctx
+}
+
+func (dt *DeleteMsg) SetContext(ctx context.Context) {
+	dt.ctx = ctx
+}
+
 func (dt *DeleteMsg) Marshal(input TsMsg) ([]byte, error) {
 	deleteTask := input.(*DeleteMsg)
 	deleteRequest := &deleteTask.DeleteRequest
@@ -165,6 +193,14 @@ func (st *SearchMsg) Type() MsgType {
 	return st.MsgType
 }
 
+func (st *SearchMsg) GetContext() context.Context {
+	return st.ctx
+}
+
+func (st *SearchMsg) SetContext(ctx context.Context) {
+	st.ctx = ctx
+}
+
 func (st *SearchMsg) Marshal(input TsMsg) ([]byte, error) {
 	searchTask := input.(*SearchMsg)
 	searchRequest := &searchTask.SearchRequest
@@ -198,6 +234,14 @@ func (srt *SearchResultMsg) Type() MsgType {
 	return srt.MsgType
 }
 
+func (srt *SearchResultMsg) GetContext() context.Context {
+	return srt.ctx
+}
+
+func (srt *SearchResultMsg) SetContext(ctx context.Context) {
+	srt.ctx = ctx
+}
+
 func (srt *SearchResultMsg) Marshal(input TsMsg) ([]byte, error) {
 	searchResultTask := input.(*SearchResultMsg)
 	searchResultRequest := &searchResultTask.SearchResult
@@ -231,6 +275,14 @@ func (tst *TimeTickMsg) Type() MsgType {
 	return tst.MsgType
 }
 
+func (tst *TimeTickMsg) GetContext() context.Context {
+	return tst.ctx
+}
+
+func (tst *TimeTickMsg) SetContext(ctx context.Context) {
+	tst.ctx = ctx
+}
+
 func (tst *TimeTickMsg) Marshal(input TsMsg) ([]byte, error) {
 	timeTickTask := input.(*TimeTickMsg)
 	timeTick := &timeTickTask.TimeTickMsg
@@ -264,6 +316,14 @@ func (qs *QueryNodeStatsMsg) Type() MsgType {
 	return qs.MsgType
 }
 
+func (qs *QueryNodeStatsMsg) GetContext() context.Context {
+	return qs.ctx
+}
+
+func (qs *QueryNodeStatsMsg) SetContext(ctx context.Context) {
+	qs.ctx = ctx
+}
+
 func (qs *QueryNodeStatsMsg) Marshal(input TsMsg) ([]byte, error) {
 	queryNodeSegStatsTask := input.(*QueryNodeStatsMsg)
 	queryNodeSegStats := &queryNodeSegStatsTask.QueryNodeStats
@@ -305,6 +365,14 @@ func (cc *CreateCollectionMsg) Type() MsgType {
 	return cc.MsgType
 }
 
+func (cc *CreateCollectionMsg) GetContext() context.Context {
+	return cc.ctx
+}
+
+func (cc *CreateCollectionMsg) SetContext(ctx context.Context) {
+	cc.ctx = ctx
+}
+
 func (cc *CreateCollectionMsg) Marshal(input TsMsg) ([]byte, error) {
 	createCollectionMsg := input.(*CreateCollectionMsg)
 	createCollectionRequest := &createCollectionMsg.CreateCollectionRequest
@@ -337,6 +405,13 @@ type DropCollectionMsg struct {
 func (dc *DropCollectionMsg) Type() MsgType {
 	return dc.MsgType
 }
+func (dc *DropCollectionMsg) GetContext() context.Context {
+	return dc.ctx
+}
+
+func (dc *DropCollectionMsg) SetContext(ctx context.Context) {
+	dc.ctx = ctx
+}
 
 func (dc *DropCollectionMsg) Marshal(input TsMsg) ([]byte, error) {
 	dropCollectionMsg := input.(*DropCollectionMsg)
@@ -361,111 +436,20 @@ func (dc *DropCollectionMsg) Unmarshal(input []byte) (TsMsg, error) {
 	return dropCollectionMsg, nil
 }
 
-/////////////////////////////////////////HasCollection//////////////////////////////////////////
-type HasCollectionMsg struct {
-	BaseMsg
-	internalPb.HasCollectionRequest
-}
-
-func (hc *HasCollectionMsg) Type() MsgType {
-	return hc.MsgType
-}
-
-func (hc *HasCollectionMsg) Marshal(input TsMsg) ([]byte, error) {
-	hasCollectionMsg := input.(*HasCollectionMsg)
-	hasCollectionRequest := &hasCollectionMsg.HasCollectionRequest
-	mb, err := proto.Marshal(hasCollectionRequest)
-	if err != nil {
-		return nil, err
-	}
-	return mb, nil
-}
-
-func (hc *HasCollectionMsg) Unmarshal(input []byte) (TsMsg, error) {
-	hasCollectionRequest := internalPb.HasCollectionRequest{}
-	err := proto.Unmarshal(input, &hasCollectionRequest)
-	if err != nil {
-		return nil, err
-	}
-	hasCollectionMsg := &HasCollectionMsg{HasCollectionRequest: hasCollectionRequest}
-	hasCollectionMsg.BeginTimestamp = hasCollectionMsg.Timestamp
-	hasCollectionMsg.EndTimestamp = hasCollectionMsg.Timestamp
-
-	return hasCollectionMsg, nil
-}
-
-/////////////////////////////////////////DescribeCollection//////////////////////////////////////////
-type DescribeCollectionMsg struct {
-	BaseMsg
-	internalPb.DescribeCollectionRequest
-}
-
-func (dc *DescribeCollectionMsg) Type() MsgType {
-	return dc.MsgType
-}
-
-func (dc *DescribeCollectionMsg) Marshal(input TsMsg) ([]byte, error) {
-	describeCollectionMsg := input.(*DescribeCollectionMsg)
-	describeCollectionRequest := &describeCollectionMsg.DescribeCollectionRequest
-	mb, err := proto.Marshal(describeCollectionRequest)
-	if err != nil {
-		return nil, err
-	}
-	return mb, nil
-}
-
-func (dc *DescribeCollectionMsg) Unmarshal(input []byte) (TsMsg, error) {
-	describeCollectionRequest := internalPb.DescribeCollectionRequest{}
-	err := proto.Unmarshal(input, &describeCollectionRequest)
-	if err != nil {
-		return nil, err
-	}
-	describeCollectionMsg := &DescribeCollectionMsg{DescribeCollectionRequest: describeCollectionRequest}
-	describeCollectionMsg.BeginTimestamp = describeCollectionMsg.Timestamp
-	describeCollectionMsg.EndTimestamp = describeCollectionMsg.Timestamp
-
-	return describeCollectionMsg, nil
-}
-
-/////////////////////////////////////////ShowCollection//////////////////////////////////////////
-type ShowCollectionMsg struct {
-	BaseMsg
-	internalPb.ShowCollectionRequest
-}
-
-func (sc *ShowCollectionMsg) Type() MsgType {
-	return sc.MsgType
-}
-
-func (sc *ShowCollectionMsg) Marshal(input TsMsg) ([]byte, error) {
-	showCollectionMsg := input.(*ShowCollectionMsg)
-	showCollectionRequest := &showCollectionMsg.ShowCollectionRequest
-	mb, err := proto.Marshal(showCollectionRequest)
-	if err != nil {
-		return nil, err
-	}
-	return mb, nil
-}
-
-func (sc *ShowCollectionMsg) Unmarshal(input []byte) (TsMsg, error) {
-	showCollectionRequest := internalPb.ShowCollectionRequest{}
-	err := proto.Unmarshal(input, &showCollectionRequest)
-	if err != nil {
-		return nil, err
-	}
-	showCollectionMsg := &ShowCollectionMsg{ShowCollectionRequest: showCollectionRequest}
-	showCollectionMsg.BeginTimestamp = showCollectionMsg.Timestamp
-	showCollectionMsg.EndTimestamp = showCollectionMsg.Timestamp
-
-	return showCollectionMsg, nil
-}
-
 /////////////////////////////////////////CreatePartition//////////////////////////////////////////
 type CreatePartitionMsg struct {
 	BaseMsg
 	internalPb.CreatePartitionRequest
 }
 
+func (cc *CreatePartitionMsg) GetContext() context.Context {
+	return cc.ctx
+}
+
+func (cc *CreatePartitionMsg) SetContext(ctx context.Context) {
+	cc.ctx = ctx
+}
+
 func (cc *CreatePartitionMsg) Type() MsgType {
 	return cc.MsgType
 }
@@ -499,6 +483,14 @@ type DropPartitionMsg struct {
 	internalPb.DropPartitionRequest
 }
 
+func (dc *DropPartitionMsg) GetContext() context.Context {
+	return dc.ctx
+}
+
+func (dc *DropPartitionMsg) SetContext(ctx context.Context) {
+	dc.ctx = ctx
+}
+
 func (dc *DropPartitionMsg) Type() MsgType {
 	return dc.MsgType
 }
@@ -526,105 +518,6 @@ func (dc *DropPartitionMsg) Unmarshal(input []byte) (TsMsg, error) {
 	return dropPartitionMsg, nil
 }
 
-/////////////////////////////////////////HasPartition//////////////////////////////////////////
-type HasPartitionMsg struct {
-	BaseMsg
-	internalPb.HasPartitionRequest
-}
-
-func (hc *HasPartitionMsg) Type() MsgType {
-	return hc.MsgType
-}
-
-func (hc *HasPartitionMsg) Marshal(input TsMsg) ([]byte, error) {
-	hasPartitionMsg := input.(*HasPartitionMsg)
-	hasPartitionRequest := &hasPartitionMsg.HasPartitionRequest
-	mb, err := proto.Marshal(hasPartitionRequest)
-	if err != nil {
-		return nil, err
-	}
-	return mb, nil
-}
-
-func (hc *HasPartitionMsg) Unmarshal(input []byte) (TsMsg, error) {
-	hasPartitionRequest := internalPb.HasPartitionRequest{}
-	err := proto.Unmarshal(input, &hasPartitionRequest)
-	if err != nil {
-		return nil, err
-	}
-	hasPartitionMsg := &HasPartitionMsg{HasPartitionRequest: hasPartitionRequest}
-	hasPartitionMsg.BeginTimestamp = hasPartitionMsg.Timestamp
-	hasPartitionMsg.EndTimestamp = hasPartitionMsg.Timestamp
-
-	return hasPartitionMsg, nil
-}
-
-/////////////////////////////////////////DescribePartition//////////////////////////////////////////
-type DescribePartitionMsg struct {
-	BaseMsg
-	internalPb.DescribePartitionRequest
-}
-
-func (dc *DescribePartitionMsg) Type() MsgType {
-	return dc.MsgType
-}
-
-func (dc *DescribePartitionMsg) Marshal(input TsMsg) ([]byte, error) {
-	describePartitionMsg := input.(*DescribePartitionMsg)
-	describePartitionRequest := &describePartitionMsg.DescribePartitionRequest
-	mb, err := proto.Marshal(describePartitionRequest)
-	if err != nil {
-		return nil, err
-	}
-	return mb, nil
-}
-
-func (dc *DescribePartitionMsg) Unmarshal(input []byte) (TsMsg, error) {
-	describePartitionRequest := internalPb.DescribePartitionRequest{}
-	err := proto.Unmarshal(input, &describePartitionRequest)
-	if err != nil {
-		return nil, err
-	}
-	describePartitionMsg := &DescribePartitionMsg{DescribePartitionRequest: describePartitionRequest}
-	describePartitionMsg.BeginTimestamp = describePartitionMsg.Timestamp
-	describePartitionMsg.EndTimestamp = describePartitionMsg.Timestamp
-
-	return describePartitionMsg, nil
-}
-
-/////////////////////////////////////////ShowPartition//////////////////////////////////////////
-type ShowPartitionMsg struct {
-	BaseMsg
-	internalPb.ShowPartitionRequest
-}
-
-func (sc *ShowPartitionMsg) Type() MsgType {
-	return sc.MsgType
-}
-
-func (sc *ShowPartitionMsg) Marshal(input TsMsg) ([]byte, error) {
-	showPartitionMsg := input.(*ShowPartitionMsg)
-	showPartitionRequest := &showPartitionMsg.ShowPartitionRequest
-	mb, err := proto.Marshal(showPartitionRequest)
-	if err != nil {
-		return nil, err
-	}
-	return mb, nil
-}
-
-func (sc *ShowPartitionMsg) Unmarshal(input []byte) (TsMsg, error) {
-	showPartitionRequest := internalPb.ShowPartitionRequest{}
-	err := proto.Unmarshal(input, &showPartitionRequest)
-	if err != nil {
-		return nil, err
-	}
-	showPartitionMsg := &ShowPartitionMsg{ShowPartitionRequest: showPartitionRequest}
-	showPartitionMsg.BeginTimestamp = showPartitionMsg.Timestamp
-	showPartitionMsg.EndTimestamp = showPartitionMsg.Timestamp
-
-	return showPartitionMsg, nil
-}
-
 /////////////////////////////////////////LoadIndex//////////////////////////////////////////
 type LoadIndexMsg struct {
 	BaseMsg
@@ -635,6 +528,14 @@ func (lim *LoadIndexMsg) Type() MsgType {
 	return lim.MsgType
 }
 
+func (lim *LoadIndexMsg) GetContext() context.Context {
+	return lim.ctx
+}
+
+func (lim *LoadIndexMsg) SetContext(ctx context.Context) {
+	lim.ctx = ctx
+}
+
 func (lim *LoadIndexMsg) Marshal(input TsMsg) ([]byte, error) {
 	loadIndexMsg := input.(*LoadIndexMsg)
 	loadIndexRequest := &loadIndexMsg.LoadIndex

internal/msgstream/msgstream.go

@@ -4,9 +4,13 @@ import (
 	"context"
 	"log"
 	"reflect"
+	"strings"
 	"sync"
 	"time"
 
+	"github.com/opentracing/opentracing-go"
+	"github.com/opentracing/opentracing-go/ext"
+
 	"github.com/apache/pulsar-client-go/pulsar"
 	"github.com/golang/protobuf/proto"
 
@@ -151,6 +155,29 @@ func (ms *PulsarMsgStream) Close() {
 	}
 }
 
+type propertiesReaderWriter struct {
+	ppMap map[string]string
+}
+
+func (ppRW *propertiesReaderWriter) Set(key, val string) {
+	// The GRPC HPACK implementation rejects any uppercase keys here.
+	//
+	// As such, since the HTTP_HEADERS format is case-insensitive anyway, we
+	// blindly lowercase the key (which is guaranteed to work in the
+	// Inject/Extract sense per the OpenTracing spec).
+	key = strings.ToLower(key)
+	ppRW.ppMap[key] = val
+}
+
+func (ppRW *propertiesReaderWriter) ForeachKey(handler func(key, val string) error) error {
+	for k, val := range ppRW.ppMap {
+		if err := handler(k, val); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (ms *PulsarMsgStream) Produce(msgPack *MsgPack) error {
 	tsMsgs := msgPack.Msgs
 	if len(tsMsgs) <= 0 {
@@ -200,12 +227,41 @@ func (ms *PulsarMsgStream) Produce(msgPack *MsgPack) error {
 			if err != nil {
 				return err
 			}
+
+			msg := &pulsar.ProducerMessage{Payload: mb}
+			var child opentracing.Span
+			if v.Msgs[i].Type() == internalPb.MsgType_kInsert || v.Msgs[i].Type() == internalPb.MsgType_kSearch {
+				tracer := opentracing.GlobalTracer()
+				ctx := v.Msgs[i].GetContext()
+				if ctx == nil {
+					ctx = context.Background()
+				}
+
+				if parent := opentracing.SpanFromContext(ctx); parent != nil {
+					child = tracer.StartSpan("start send pulsar msg",
+						opentracing.FollowsFrom(parent.Context()))
+				} else {
+					child = tracer.StartSpan("start send pulsar msg")
+				}
+				child.SetTag("hash keys", v.Msgs[i].HashKeys())
+				child.SetTag("start time", v.Msgs[i].BeginTs())
+				child.SetTag("end time", v.Msgs[i].EndTs())
+				msg.Properties = make(map[string]string)
+				err = tracer.Inject(child.Context(), opentracing.TextMap, &propertiesReaderWriter{msg.Properties})
+				if err != nil {
+					return err
+				}
+			}
+
 			if _, err := (*ms.producers[k]).Send(
 				context.Background(),
-				&pulsar.ProducerMessage{Payload: mb},
+				msg,
 			); err != nil {
 				return err
 			}
+			if child != nil {
+				child.Finish()
+			}
 		}
 	}
 	return nil
@@ -218,10 +274,34 @@ func (ms *PulsarMsgStream) Broadcast(msgPack *MsgPack) error {
 		if err != nil {
 			return err
 		}
+		msg := &pulsar.ProducerMessage{Payload: mb}
+		if v.Type() == internalPb.MsgType_kInsert || v.Type() == internalPb.MsgType_kSearch {
+			tracer := opentracing.GlobalTracer()
+			ctx := v.GetContext()
+			if ctx == nil {
+				ctx = context.Background()
+			}
+			var child opentracing.Span
+			if parent := opentracing.SpanFromContext(ctx); parent != nil {
+				child = tracer.StartSpan("start send pulsar msg",
+					opentracing.FollowsFrom(parent.Context()))
+			} else {
+				child = tracer.StartSpan("start send pulsar msg, start time: %d")
+			}
+			child.SetTag("hash keys", v.HashKeys())
+			child.SetTag("start time", v.BeginTs())
+			child.SetTag("end time", v.EndTs())
+			msg.Properties = make(map[string]string)
+			err = tracer.Inject(child.Context(), opentracing.TextMap, &propertiesReaderWriter{msg.Properties})
+			if err != nil {
+				return err
+			}
+			child.Finish()
+		}
 		for i := 0; i < producerLen; i++ {
 			if _, err := (*ms.producers[i]).Send(
 				context.Background(),
-				&pulsar.ProducerMessage{Payload: mb},
+				msg,
 			); err != nil {
 				return err
 			}
@@ -258,6 +338,7 @@ func (ms *PulsarMsgStream) bufMsgPackToChannel() {
 	for {
 		select {
 		case <-ms.ctx.Done():
+			log.Println("done")
 			return
 		default:
 			tsMsgList := make([]TsMsg, 0)
@@ -270,6 +351,7 @@ func (ms *PulsarMsgStream) bufMsgPackToChannel() {
 				}
 
 				pulsarMsg, ok := value.Interface().(pulsar.ConsumerMessage)
+
 				if !ok {
 					log.Printf("type assertion failed, not consumer message type")
 					continue
@@ -283,6 +365,21 @@ func (ms *PulsarMsgStream) bufMsgPackToChannel() {
 					continue
 				}
 				tsMsg, err := ms.unmarshal.Unmarshal(pulsarMsg.Payload(), headerMsg.MsgType)
+				if tsMsg.Type() == internalPb.MsgType_kInsert || tsMsg.Type() == internalPb.MsgType_kSearch {
+					tracer := opentracing.GlobalTracer()
+					spanContext, err := tracer.Extract(opentracing.HTTPHeaders, &propertiesReaderWriter{pulsarMsg.Properties()})
+					if err != nil {
+						log.Println("extract message err")
+						log.Println(err.Error())
+					}
+					span := opentracing.StartSpan("pulsar msg received",
+						ext.RPCServerOption(spanContext))
+					span.SetTag("hash keys", tsMsg.HashKeys())
+					span.SetTag("start time", tsMsg.BeginTs())
+					span.SetTag("end time", tsMsg.EndTs())
+					tsMsg.SetContext(opentracing.ContextWithSpan(context.Background(), span))
+					span.Finish()
+				}
 				if err != nil {
 					log.Printf("Failed to unmarshal tsMsg, error = %v", err)
 					continue
@@ -420,6 +517,23 @@ func (ms *PulsarTtMsgStream) findTimeTick(channelIndex int,
 			if err != nil {
 				log.Printf("Failed to unmarshal, error = %v", err)
 			}
+
+			if tsMsg.Type() == internalPb.MsgType_kInsert || tsMsg.Type() == internalPb.MsgType_kSearch {
+				tracer := opentracing.GlobalTracer()
+				spanContext, err := tracer.Extract(opentracing.HTTPHeaders, &propertiesReaderWriter{pulsarMsg.Properties()})
+				if err != nil {
+					log.Println("extract message err")
+					log.Println(err.Error())
+				}
+				span := opentracing.StartSpan("pulsar msg received",
+					ext.RPCServerOption(spanContext))
+				span.SetTag("hash keys", tsMsg.HashKeys())
+				span.SetTag("start time", tsMsg.BeginTs())
+				span.SetTag("end time", tsMsg.EndTs())
+				tsMsg.SetContext(opentracing.ContextWithSpan(context.Background(), span))
+				span.Finish()
+			}
+
 			if headerMsg.MsgType == internalPb.MsgType_kTimeTick {
 				eofMsgMap[channelIndex] = tsMsg.(*TimeTickMsg).Timestamp
 				return
@@ -500,7 +614,7 @@ func insertRepackFunc(tsMsgs []TsMsg, hashKeys [][]int32) (map[int32]*MsgPack, e
 	result := make(map[int32]*MsgPack)
 	for i, request := range tsMsgs {
 		if request.Type() != internalPb.MsgType_kInsert {
-			return nil, errors.New(string("msg's must be Insert"))
+			return nil, errors.New("msg's must be Insert")
 		}
 		insertRequest := request.(*InsertMsg)
 		keys := hashKeys[i]
@@ -511,7 +625,7 @@ func insertRepackFunc(tsMsgs []TsMsg, hashKeys [][]int32) (map[int32]*MsgPack, e
 		keysLen := len(keys)
 
 		if keysLen != timestampLen || keysLen != rowIDLen || keysLen != rowDataLen {
-			return nil, errors.New(string("the length of hashValue, timestamps, rowIDs, RowData are not equal"))
+			return nil, errors.New("the length of hashValue, timestamps, rowIDs, RowData are not equal")
 		}
 		for index, key := range keys {
 			_, ok := result[key]
@@ -534,6 +648,9 @@ func insertRepackFunc(tsMsgs []TsMsg, hashKeys [][]int32) (map[int32]*MsgPack, e
 			}
 
 			insertMsg := &InsertMsg{
+				BaseMsg: BaseMsg{
+					ctx: request.GetContext(),
+				},
 				InsertRequest: sliceRequest,
 			}
 			result[key].Msgs = append(result[key].Msgs, insertMsg)
@@ -546,7 +663,7 @@ func deleteRepackFunc(tsMsgs []TsMsg, hashKeys [][]int32) (map[int32]*MsgPack, e
 	result := make(map[int32]*MsgPack)
 	for i, request := range tsMsgs {
 		if request.Type() != internalPb.MsgType_kDelete {
-			return nil, errors.New(string("msg's must be Delete"))
+			return nil, errors.New("msg's must be Delete")
 		}
 		deleteRequest := request.(*DeleteMsg)
 		keys := hashKeys[i]
@@ -556,7 +673,7 @@ func deleteRepackFunc(tsMsgs []TsMsg, hashKeys [][]int32) (map[int32]*MsgPack, e
 		keysLen := len(keys)
 
 		if keysLen != timestampLen || keysLen != primaryKeysLen {
-			return nil, errors.New(string("the length of hashValue, timestamps, primaryKeys are not equal"))
+			return nil, errors.New("the length of hashValue, timestamps, primaryKeys are not equal")
 		}
 
 		for index, key := range keys {
@@ -590,7 +707,7 @@ func defaultRepackFunc(tsMsgs []TsMsg, hashKeys [][]int32) (map[int32]*MsgPack,
 	for i, request := range tsMsgs {
 		keys := hashKeys[i]
 		if len(keys) != 1 {
-			return nil, errors.New(string("len(msg.hashValue) must equal 1"))
+			return nil, errors.New("len(msg.hashValue) must equal 1")
 		}
 		key := keys[0]
 		_, ok := result[key]

internal/proto/internal_msg.proto

@@ -272,6 +272,8 @@ message FlushMsg {
     MsgType msg_type = 1;
     int64 segmentID = 2;
     uint64 timestamp = 3;
+    int64 collectionID = 4;
+    string partitionTag = 5;
 }
 
 

internal/proto/internalpb/internal_msg.pb.go

@@ -1881,6 +1881,8 @@ type FlushMsg struct {
 	MsgType              MsgType  `protobuf:"varint,1,opt,name=msg_type,json=msgType,proto3,enum=milvus.proto.internal.MsgType" json:"msg_type,omitempty"`
 	SegmentID            int64    `protobuf:"varint,2,opt,name=segmentID,proto3" json:"segmentID,omitempty"`
 	Timestamp            uint64   `protobuf:"varint,3,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
+	CollectionID         int64    `protobuf:"varint,4,opt,name=collectionID,proto3" json:"collectionID,omitempty"`
+	PartitionTag         string   `protobuf:"bytes,5,opt,name=partitionTag,proto3" json:"partitionTag,omitempty"`
 	XXX_NoUnkeyedLiteral struct{} `json:"-"`
 	XXX_unrecognized     []byte   `json:"-"`
 	XXX_sizecache        int32    `json:"-"`
@@ -1932,6 +1934,20 @@ func (m *FlushMsg) GetTimestamp() uint64 {
 	return 0
 }
 
+func (m *FlushMsg) GetCollectionID() int64 {
+	if m != nil {
+		return m.CollectionID
+	}
+	return 0
+}
+
+func (m *FlushMsg) GetPartitionTag() string {
+	if m != nil {
+		return m.PartitionTag
+	}
+	return ""
+}
+
 type Key2Seg struct {
 	RowID                int64    `protobuf:"varint,1,opt,name=rowID,proto3" json:"rowID,omitempty"`
 	PrimaryKey           int64    `protobuf:"varint,2,opt,name=primary_key,json=primaryKey,proto3" json:"primary_key,omitempty"`
@@ -2661,121 +2677,122 @@ func init() {
 func init() { proto.RegisterFile("internal_msg.proto", fileDescriptor_7eb37f6b80b23116) }
 
 var fileDescriptor_7eb37f6b80b23116 = []byte{
-	// 1852 bytes of a gzipped FileDescriptorProto
+	// 1867 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x99, 0xcd, 0x6f, 0x23, 0x49,
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x59, 0xcd, 0x6f, 0x24, 0x47,
-	0x15, 0xc0, 0xb7, 0xbb, 0xfd, 0x11, 0x3f, 0x3b, 0x4e, 0x4f, 0x25, 0x99, 0xf1, 0xec, 0x2c, 0x3b,
+	0x15, 0x4f, 0x77, 0xcf, 0x87, 0xe7, 0xcd, 0x78, 0xdc, 0x5b, 0xb6, 0x77, 0x67, 0xb3, 0x21, 0xeb,
-	0x99, 0x1e, 0xc4, 0x86, 0x45, 0x24, 0x90, 0xe1, 0xc0, 0xde, 0x20, 0xb1, 0x96, 0x35, 0x43, 0x46,
+	0xed, 0x45, 0xc4, 0x04, 0x61, 0x83, 0x97, 0x03, 0xb9, 0x81, 0x3d, 0x0a, 0x19, 0x16, 0xaf, 0x4c,
-	0xa1, 0x13, 0x2d, 0x12, 0x5a, 0xa9, 0xd5, 0xb1, 0x5f, 0xec, 0x52, 0x7f, 0x39, 0x55, 0xed, 0x49,
+	0xdb, 0x0a, 0x12, 0x8a, 0xd4, 0x6a, 0xcf, 0x3c, 0xcf, 0x94, 0xfa, 0x6b, 0x5c, 0xd5, 0xb3, 0xf6,
-	0x3c, 0x07, 0x24, 0xc4, 0x9c, 0x11, 0x1f, 0xe2, 0xc0, 0x8d, 0x3b, 0xec, 0x8a, 0x05, 0xf1, 0x3f,
+	0xec, 0x81, 0x0b, 0x7b, 0x46, 0x7c, 0x88, 0x03, 0x37, 0xee, 0x90, 0x88, 0x80, 0xf8, 0x1f, 0x08,
-	0xf0, 0x29, 0x24, 0xfe, 0x0b, 0x38, 0x80, 0xc4, 0x2e, 0x07, 0x6e, 0xa8, 0xaa, 0xfa, 0xc3, 0x9d,
+	0x1f, 0x42, 0xe2, 0xbf, 0x80, 0x03, 0x48, 0x24, 0x1c, 0xb8, 0xa1, 0xaa, 0xea, 0x8f, 0x69, 0x7b,
-	0xd8, 0x4e, 0xb4, 0xc9, 0x2c, 0xb3, 0xda, 0xb9, 0x75, 0x3d, 0x57, 0x57, 0xbf, 0xf7, 0x7b, 0x1f,
+	0x66, 0x6c, 0xc5, 0xde, 0xb0, 0x51, 0xf6, 0xd6, 0xf5, 0xba, 0xba, 0xea, 0xbd, 0xdf, 0x7b, 0xef,
-	0x55, 0xf5, 0x0c, 0x84, 0x86, 0x31, 0xb2, 0xd0, 0xf5, 0x9d, 0x80, 0xf7, 0x37, 0x86, 0x2c, 0x8a,
+	0x57, 0xf5, 0x5e, 0x03, 0xa1, 0x61, 0x8c, 0x2c, 0x74, 0x7d, 0x27, 0xe0, 0xfd, 0x8d, 0x21, 0x8b,
-	0x23, 0xb2, 0x1a, 0x50, 0xff, 0xf1, 0x88, 0xab, 0xd1, 0x46, 0x3a, 0xe1, 0xe5, 0x46, 0x37, 0x0a,
+	0xe2, 0x88, 0xac, 0x06, 0xd4, 0x7f, 0x3c, 0xe2, 0x6a, 0xb4, 0x91, 0x4e, 0x78, 0xb9, 0xd1, 0x8d,
-	0x82, 0x28, 0x54, 0xe2, 0x97, 0x6f, 0x70, 0x64, 0x8f, 0x69, 0x17, 0xf3, 0xf7, 0xac, 0x10, 0x6a,
+	0x82, 0x20, 0x0a, 0x95, 0xf8, 0xe5, 0x1b, 0x1c, 0xd9, 0x63, 0xda, 0xc5, 0xfc, 0x3b, 0x2b, 0x84,
-	0x9d, 0xb6, 0x8d, 0xc7, 0x23, 0xe4, 0x31, 0xb9, 0x09, 0x95, 0x21, 0x22, 0xeb, 0xb4, 0x5b, 0xda,
+	0x5a, 0xa7, 0x6d, 0xe3, 0xf1, 0x08, 0x79, 0x4c, 0x6e, 0x42, 0x65, 0x88, 0xc8, 0x3a, 0xed, 0x96,
-	0x9a, 0xb6, 0x6e, 0xd8, 0xc9, 0x88, 0x3c, 0x80, 0x12, 0x8b, 0x7c, 0x6c, 0xe9, 0x6b, 0xda, 0x7a,
+	0xb6, 0xa6, 0xad, 0x1b, 0x76, 0x32, 0x22, 0x0f, 0xa0, 0xc4, 0x22, 0x1f, 0x5b, 0xfa, 0x9a, 0xb6,
-	0x73, 0xeb, 0xee, 0xc6, 0xd4, 0x6f, 0x6d, 0xec, 0x21, 0x32, 0x3b, 0xf2, 0xd1, 0x96, 0x93, 0xc9,
+	0xde, 0xdc, 0xba, 0xbb, 0x31, 0x75, 0xaf, 0x8d, 0x3d, 0x44, 0x66, 0x47, 0x3e, 0xda, 0x72, 0x32,
-	0x0a, 0x94, 0xbb, 0xd1, 0x28, 0x8c, 0x5b, 0xc6, 0x9a, 0xb6, 0xbe, 0x68, 0xab, 0x81, 0xd5, 0x07,
+	0x59, 0x81, 0x72, 0x37, 0x1a, 0x85, 0x71, 0xcb, 0x58, 0xd3, 0xd6, 0x17, 0x6d, 0x35, 0xb0, 0xfa,
-	0x10, 0xdf, 0xe3, 0xc3, 0x28, 0xe4, 0x48, 0x1e, 0x40, 0x85, 0xc7, 0x6e, 0x3c, 0xe2, 0xf2, 0x83,
+	0x00, 0x62, 0x3f, 0x3e, 0x8c, 0x42, 0x8e, 0xe4, 0x01, 0x54, 0x78, 0xec, 0xc6, 0x23, 0x2e, 0x37,
-	0xf5, 0xad, 0x3b, 0xc5, 0xa5, 0x13, 0xe5, 0xf7, 0xe5, 0x14, 0x3b, 0x99, 0x4a, 0x9a, 0xa0, 0x77,
+	0xac, 0x6f, 0xdd, 0x29, 0x2e, 0x9d, 0x28, 0xbf, 0x2f, 0xa7, 0xd8, 0xc9, 0x54, 0xd2, 0x04, 0xbd,
-	0xda, 0x52, 0x17, 0xc3, 0xd6, 0x3b, 0xed, 0x19, 0x1f, 0x8a, 0x00, 0x0e, 0x78, 0xf4, 0x31, 0x5a,
+	0xd3, 0x96, 0xba, 0x18, 0xb6, 0xde, 0x69, 0xcf, 0xd8, 0x28, 0x02, 0x38, 0xe0, 0xd1, 0x27, 0x68,
-	0xf6, 0x18, 0xea, 0xf2, 0x83, 0x57, 0x31, 0xed, 0x15, 0xa8, 0xc5, 0x34, 0x40, 0x1e, 0xbb, 0xc1,
+	0xd9, 0x63, 0xa8, 0xcb, 0x0d, 0xaf, 0x62, 0xda, 0x2b, 0x50, 0x8b, 0x69, 0x80, 0x3c, 0x76, 0x83,
-	0x50, 0xea, 0x54, 0xb2, 0x73, 0xc1, 0x8c, 0xef, 0x3e, 0xd5, 0xa0, 0xb1, 0x8f, 0xfd, 0xdc, 0x8b,
+	0xa1, 0xd4, 0xa9, 0x64, 0xe7, 0x82, 0x19, 0xfb, 0x3e, 0xd5, 0xa0, 0xb1, 0x8f, 0xfd, 0xdc, 0x8b,
-	0xd9, 0x34, 0x6d, 0x62, 0x9a, 0x58, 0xba, 0x3b, 0x70, 0xc3, 0x10, 0xfd, 0x04, 0x5e, 0xd9, 0xce,
+	0xd9, 0x34, 0x6d, 0x62, 0x9a, 0x58, 0xba, 0x3b, 0x70, 0xc3, 0x10, 0xfd, 0x04, 0xbc, 0xb2, 0x9d,
-	0x05, 0xe4, 0x0e, 0xd4, 0xba, 0x91, 0xef, 0x3b, 0xa1, 0x1b, 0xa0, 0x5c, 0xbe, 0x66, 0x2f, 0x08,
+	0x0b, 0xc8, 0x1d, 0xa8, 0x75, 0x23, 0xdf, 0x77, 0x42, 0x37, 0x40, 0xb9, 0x7c, 0xcd, 0x5e, 0x10,
-	0xc1, 0x23, 0x37, 0x40, 0x72, 0x1f, 0x16, 0x87, 0x2e, 0x8b, 0x69, 0x4c, 0xa3, 0xd0, 0x89, 0xdd,
+	0x82, 0x47, 0x6e, 0x80, 0xe4, 0x3e, 0x2c, 0x0e, 0x5d, 0x16, 0xd3, 0x98, 0x46, 0xa1, 0x13, 0xbb,
-	0x7e, 0xab, 0x24, 0x27, 0x34, 0x32, 0xe1, 0x81, 0xdb, 0xb7, 0xde, 0xd3, 0x80, 0x7c, 0x9d, 0x73,
+	0xfd, 0x56, 0x49, 0x4e, 0x68, 0x64, 0xc2, 0x03, 0xb7, 0x6f, 0xbd, 0xa7, 0x01, 0xf9, 0x26, 0xe7,
-	0xda, 0x0f, 0x0b, 0xca, 0x5c, 0x2b, 0xf8, 0x87, 0xb0, 0x34, 0x44, 0xe6, 0x24, 0x6a, 0x3b, 0x0c,
+	0xb4, 0x1f, 0x16, 0x94, 0xb9, 0x56, 0xe0, 0x1f, 0xc2, 0xd2, 0x10, 0x99, 0x93, 0xa8, 0xed, 0x30,
-	0x8f, 0x5b, 0xc6, 0x9a, 0xb1, 0x5e, 0xdf, 0xba, 0x3f, 0xe3, 0xfd, 0x49, 0x55, 0xec, 0xc5, 0x21,
+	0x3c, 0x6e, 0x19, 0x6b, 0xc6, 0x7a, 0x7d, 0xeb, 0xfe, 0x8c, 0xef, 0x27, 0x55, 0xb1, 0x17, 0x87,
-	0xb2, 0x1d, 0xf5, 0xaa, 0x8d, 0xc7, 0xd6, 0x07, 0x1a, 0x2c, 0xc9, 0xdf, 0x95, 0xd6, 0x01, 0x86,
+	0xc8, 0x76, 0xd4, 0xa7, 0x36, 0x1e, 0x5b, 0x1f, 0x6a, 0xb0, 0x24, 0xdf, 0x2b, 0xad, 0x03, 0x0c,
-	0x12, 0x1d, 0x17, 0xa2, 0x44, 0x59, 0x35, 0xb8, 0x00, 0xdd, 0x54, 0xaf, 0x14, 0x81, 0x96, 0x2e,
+	0x25, 0x74, 0x5c, 0x88, 0x12, 0x65, 0xd5, 0xe0, 0x02, 0xe8, 0xa6, 0x7a, 0xa5, 0x08, 0x68, 0xe9,
-	0x02, 0x5a, 0x3e, 0x0f, 0x94, 0xdc, 0x85, 0x3a, 0x9e, 0x0e, 0x29, 0x43, 0x47, 0x44, 0x40, 0xab,
+	0x22, 0x40, 0xcb, 0xe7, 0x01, 0x25, 0x77, 0xa1, 0x8e, 0xa7, 0x43, 0xca, 0xd0, 0x11, 0x11, 0xd0,
-	0x22, 0xa3, 0x01, 0x94, 0xe8, 0x80, 0x06, 0x93, 0x11, 0x56, 0xbd, 0x74, 0x84, 0x59, 0x1c, 0x96,
+	0xaa, 0xc8, 0x68, 0x00, 0x25, 0x3a, 0xa0, 0xc1, 0x64, 0x84, 0x55, 0x2f, 0x1d, 0x61, 0x16, 0x87,
-	0x0b, 0x5e, 0x4a, 0xa2, 0xf5, 0x1d, 0xb8, 0x39, 0x49, 0xd6, 0xcd, 0x90, 0xb4, 0x34, 0x09, 0xf8,
+	0xe5, 0x82, 0x97, 0x92, 0x68, 0x7d, 0x07, 0x6e, 0x4e, 0x22, 0xeb, 0x66, 0x90, 0xb4, 0x34, 0x09,
-	0x73, 0xf3, 0x00, 0xe7, 0x00, 0xed, 0x95, 0x9c, 0x71, 0x2e, 0xb5, 0xfe, 0xab, 0xc1, 0xad, 0x1d,
+	0xf0, 0x17, 0xe6, 0x01, 0x9c, 0x03, 0x68, 0xaf, 0xe4, 0x18, 0xe7, 0x52, 0xeb, 0xbf, 0x1a, 0xdc,
-	0x86, 0x6e, 0x8c, 0x3b, 0x91, 0xef, 0x63, 0x57, 0x98, 0x98, 0x06, 0xc8, 0x1b, 0xb0, 0x10, 0xf0,
+	0xda, 0x61, 0xe8, 0xc6, 0xb8, 0x13, 0xf9, 0x3e, 0x76, 0x85, 0x89, 0x69, 0x80, 0xbc, 0x01, 0x0b,
-	0xbe, 0x13, 0x8f, 0x87, 0x28, 0xa9, 0x37, 0xb7, 0x5e, 0x9d, 0xf1, 0xad, 0x5d, 0xde, 0x3f, 0x18,
+	0x01, 0xef, 0x3b, 0xf1, 0x78, 0x88, 0x12, 0xf5, 0xe6, 0xd6, 0xab, 0x33, 0xf6, 0xda, 0xe5, 0xfd,
-	0x0f, 0xd1, 0xae, 0x06, 0xea, 0x81, 0x58, 0xd0, 0xe8, 0x66, 0xeb, 0x65, 0x25, 0xa1, 0x20, 0x13,
+	0x83, 0xf1, 0x10, 0xed, 0x6a, 0xa0, 0x1e, 0x88, 0x05, 0x8d, 0x6e, 0xb6, 0x5e, 0x46, 0x09, 0x05,
-	0xde, 0x61, 0x78, 0xdc, 0x69, 0x4b, 0xef, 0x18, 0xb6, 0x1a, 0x14, 0xf3, 0xac, 0x74, 0x36, 0xcf,
+	0x99, 0xf0, 0x0e, 0xc3, 0xe3, 0x4e, 0x5b, 0x7a, 0xc7, 0xb0, 0xd5, 0xa0, 0x98, 0x67, 0xa5, 0xb3,
-	0x5a, 0x50, 0x1d, 0xb2, 0xe8, 0x74, 0xdc, 0x69, 0x4b, 0xc7, 0x18, 0x76, 0x3a, 0x24, 0x5f, 0x86,
+	0x79, 0xd6, 0x82, 0xea, 0x90, 0x45, 0xa7, 0xe3, 0x4e, 0x5b, 0x3a, 0xc6, 0xb0, 0xd3, 0x21, 0xf9,
-	0x0a, 0xef, 0x0e, 0x30, 0x70, 0xa5, 0x3b, 0xea, 0x5b, 0xb7, 0xa7, 0x22, 0xdf, 0xf6, 0xa3, 0x43,
+	0x2a, 0x54, 0x78, 0x77, 0x80, 0x81, 0x2b, 0xdd, 0x51, 0xdf, 0xba, 0x3d, 0x15, 0xf2, 0x6d, 0x3f,
-	0x3b, 0x99, 0x68, 0xfd, 0x54, 0x87, 0xd5, 0x36, 0x8b, 0x86, 0x9f, 0x70, 0xcb, 0x77, 0x61, 0x29,
+	0x3a, 0xb4, 0x93, 0x89, 0xd6, 0xcf, 0x74, 0x58, 0x6d, 0xb3, 0x68, 0xf8, 0x29, 0xb7, 0x7c, 0x17,
-	0x5f, 0x5d, 0x45, 0xb5, 0x42, 0xf0, 0xd9, 0xa2, 0xce, 0xc9, 0x0e, 0xb3, 0x91, 0x9b, 0x2b, 0x22,
+	0x96, 0xf2, 0xd5, 0x55, 0x54, 0x2b, 0x08, 0x3e, 0x5f, 0xd4, 0x39, 0x39, 0x61, 0x36, 0x72, 0x73,
-	0xde, 0x6e, 0x76, 0x0b, 0x63, 0xeb, 0x9f, 0x1a, 0xac, 0xbc, 0xe5, 0xf2, 0x6b, 0x85, 0x92, 0x19,
+	0x45, 0xc4, 0xdb, 0xcd, 0x6e, 0x61, 0x6c, 0xfd, 0x53, 0x83, 0x95, 0xb7, 0x5c, 0x7e, 0xad, 0xa0,
-	0xac, 0xcf, 0x34, 0xd8, 0x98, 0x63, 0x70, 0xe9, 0x42, 0x83, 0xcb, 0x57, 0x30, 0xf8, 0x03, 0x0d,
+	0x64, 0x06, 0xeb, 0x33, 0x0d, 0x36, 0xe6, 0x18, 0x5c, 0xba, 0xd0, 0xe0, 0xf2, 0x15, 0x0c, 0xfe,
-	0x6e, 0xb7, 0x91, 0x77, 0x19, 0x3d, 0xc4, 0x4f, 0x8f, 0xd5, 0xbf, 0xd0, 0x60, 0x75, 0x7f, 0x10,
+	0x50, 0x83, 0xdb, 0x6d, 0xe4, 0x5d, 0x46, 0x0f, 0xf1, 0xb3, 0x63, 0xf5, 0x2f, 0x35, 0x58, 0xdd,
-	0x9d, 0x3c, 0xbf, 0x16, 0x5b, 0xbf, 0xd5, 0xe1, 0xa6, 0xaa, 0x4d, 0x7b, 0x69, 0xf5, 0xfd, 0x98,
+	0x1f, 0x44, 0x27, 0xcf, 0xaf, 0xc5, 0xd6, 0xef, 0x74, 0xb8, 0xa9, 0xb8, 0x69, 0x2f, 0x65, 0xdf,
-	0x12, 0x74, 0x0d, 0xea, 0x59, 0xc1, 0xcf, 0xd2, 0x74, 0x52, 0x94, 0x5b, 0x5a, 0x9a, 0x69, 0x69,
+	0x4f, 0x28, 0x41, 0xd7, 0xa0, 0x9e, 0x11, 0x7e, 0x96, 0xa6, 0x93, 0xa2, 0xdc, 0xd2, 0xd2, 0x4c,
-	0x79, 0x8e, 0xa5, 0x95, 0xa2, 0x6f, 0xbf, 0x09, 0xcd, 0x7c, 0xd7, 0x91, 0xae, 0x55, 0xfb, 0xc6,
+	0x4b, 0xcb, 0x73, 0x2c, 0xad, 0x14, 0x7d, 0xfb, 0x6d, 0x68, 0xe6, 0xa7, 0x8e, 0x74, 0xad, 0x3a,
-	0xfd, 0xe9, 0xae, 0xcd, 0x70, 0x48, 0xcf, 0xe6, 0x1b, 0x96, 0x74, 0xec, 0xfb, 0x3a, 0xac, 0x88,
+	0x37, 0xee, 0x4f, 0x77, 0x6d, 0x06, 0x87, 0xf4, 0x6c, 0x7e, 0x60, 0x49, 0xc7, 0xbe, 0xaf, 0xc3,
-	0xaa, 0xf6, 0x82, 0xd9, 0xe5, 0x99, 0xfd, 0x43, 0x83, 0xe5, 0xb7, 0x5c, 0x7e, 0x9d, 0xc8, 0xae,
+	0x8a, 0x60, 0xb5, 0x17, 0x98, 0x5d, 0x1e, 0xb3, 0x7f, 0x68, 0xb0, 0xfc, 0x96, 0xcb, 0xaf, 0x13,
-	0x37, 0xf9, 0xcf, 0x1b, 0x5b, 0xfe, 0xc8, 0xc6, 0xfe, 0x4b, 0x83, 0x56, 0x5a, 0xef, 0x3e, 0x1d,
+	0xb2, 0xeb, 0x4d, 0xfe, 0xf3, 0xc6, 0x96, 0x3f, 0xb6, 0xb1, 0xff, 0xd2, 0xa0, 0x95, 0xf2, 0xdd,
-	0x16, 0x8b, 0x2d, 0x4d, 0xd4, 0xba, 0xe7, 0xd7, 0xda, 0x6b, 0x2e, 0xee, 0xff, 0xd6, 0x61, 0xb1,
+	0x67, 0xc3, 0x62, 0x71, 0xa4, 0x09, 0xae, 0x7b, 0x7e, 0xad, 0xbd, 0x66, 0x72, 0xff, 0xb7, 0x0e,
-	0x13, 0x72, 0x64, 0xf1, 0x33, 0xb3, 0xf4, 0xb5, 0xf3, 0x1a, 0xab, 0xcb, 0xc9, 0x19, 0x5d, 0x2e,
+	0x8b, 0x9d, 0x90, 0x23, 0x8b, 0x9f, 0x99, 0xa5, 0xaf, 0x9d, 0xd7, 0x58, 0x15, 0x27, 0x67, 0x74,
-	0x75, 0x45, 0x11, 0xdc, 0x38, 0xf6, 0xc5, 0x89, 0x34, 0x3b, 0xdf, 0xe4, 0x82, 0xe2, 0x29, 0x5f,
+	0xb9, 0x54, 0x89, 0x22, 0x70, 0xe3, 0xd8, 0x17, 0x37, 0xd2, 0xec, 0x7e, 0x93, 0x0b, 0x8a, 0xb7,
-	0x95, 0x81, 0x89, 0x53, 0xfe, 0x04, 0xd5, 0x6a, 0x91, 0xea, 0xab, 0x00, 0x19, 0x7c, 0xde, 0x5a,
+	0x7c, 0x45, 0x03, 0x13, 0xb7, 0xfc, 0x09, 0x54, 0xab, 0x45, 0x54, 0x5f, 0x05, 0xc8, 0xc0, 0xe7,
-	0x58, 0x33, 0xc4, 0x31, 0x3d, 0x97, 0x88, 0x1b, 0x10, 0x8b, 0x4e, 0x3a, 0x6d, 0xde, 0xaa, 0xad,
+	0xad, 0x85, 0x35, 0x43, 0x5c, 0xd3, 0x73, 0x89, 0xa8, 0x80, 0x58, 0x74, 0xd2, 0x69, 0xf3, 0x56,
-	0x19, 0xe2, 0x06, 0xa4, 0x46, 0xe4, 0x2b, 0xb0, 0xc0, 0xa2, 0x13, 0xa7, 0xe7, 0xc6, 0x6e, 0x0b,
+	0x6d, 0xcd, 0x10, 0x15, 0x90, 0x1a, 0x91, 0xaf, 0xc1, 0x02, 0x8b, 0x4e, 0x9c, 0x9e, 0x1b, 0xbb,
-	0xe4, 0x21, 0x7b, 0xce, 0x69, 0xb2, 0xca, 0xa2, 0x93, 0xb6, 0x1b, 0xbb, 0xd6, 0x53, 0x1d, 0x16,
+	0x2d, 0x90, 0x97, 0xec, 0x39, 0xb7, 0xc9, 0x2a, 0x8b, 0x4e, 0xda, 0x6e, 0xec, 0x5a, 0x4f, 0x75,
-	0xdb, 0xe8, 0x63, 0x8c, 0xff, 0x7f, 0xe8, 0x05, 0x62, 0xa5, 0x39, 0xc4, 0xca, 0xf3, 0x88, 0x55,
+	0x58, 0x6c, 0xa3, 0x8f, 0x31, 0xfe, 0xff, 0x41, 0x2f, 0x20, 0x56, 0x9a, 0x83, 0x58, 0x79, 0x1e,
-	0xce, 0x11, 0xbb, 0x07, 0x8d, 0x21, 0xa3, 0x81, 0xcb, 0xc6, 0x8e, 0x87, 0x63, 0x71, 0xbd, 0x31,
+	0x62, 0x95, 0x73, 0x88, 0xdd, 0x83, 0xc6, 0x90, 0xd1, 0xc0, 0x65, 0x63, 0xc7, 0xc3, 0xb1, 0x28,
-	0x64, 0x95, 0x57, 0xb2, 0x87, 0x38, 0xe6, 0xd6, 0x87, 0x1a, 0x2c, 0xee, 0xa3, 0xcb, 0xba, 0x83,
+	0x6f, 0x0c, 0xc9, 0xf2, 0x4a, 0xf6, 0x10, 0xc7, 0xdc, 0xfa, 0x48, 0x83, 0xc5, 0x7d, 0x74, 0x59,
-	0x67, 0x86, 0x61, 0x42, 0x7f, 0xa3, 0xa8, 0xff, 0xfc, 0x33, 0xf4, 0xe7, 0xc1, 0x64, 0xc8, 0x47,
+	0x77, 0xf0, 0xcc, 0x60, 0x98, 0xd0, 0xdf, 0x28, 0xea, 0x3f, 0xff, 0x0e, 0xfd, 0x45, 0x30, 0x19,
-	0x7e, 0xec, 0xe4, 0x70, 0x14, 0x80, 0x25, 0x25, 0xdf, 0xc9, 0x10, 0x6d, 0x42, 0xf9, 0x78, 0x84,
+	0xf2, 0x91, 0x1f, 0x3b, 0x39, 0x38, 0x0a, 0x80, 0x25, 0x25, 0xdf, 0xc9, 0x20, 0xda, 0x84, 0xf2,
-	0x6c, 0x7c, 0xf1, 0x6d, 0x42, 0xcd, 0xb3, 0xfe, 0xa6, 0x81, 0xb9, 0x3f, 0xe6, 0x3b, 0x51, 0x78,
+	0xf1, 0x08, 0xd9, 0xf8, 0xe2, 0x6a, 0x42, 0xcd, 0xb3, 0xfe, 0xa6, 0x81, 0xb9, 0x3f, 0xe6, 0x3b,
-	0x44, 0xfb, 0xcf, 0x9d, 0xe5, 0x04, 0x4a, 0xd2, 0x5f, 0xe5, 0x35, 0x63, 0xbd, 0x66, 0xcb, 0x67,
+	0x51, 0x78, 0x44, 0xfb, 0xcf, 0x9d, 0xe5, 0x04, 0x4a, 0xd2, 0x5f, 0xe5, 0x35, 0x63, 0xbd, 0x66,
-	0xe1, 0x4b, 0x0f, 0xc7, 0xce, 0x90, 0xe1, 0x11, 0x3d, 0x45, 0xe5, 0xed, 0x9a, 0x5d, 0xf7, 0x70,
+	0xcb, 0x67, 0xe1, 0x4b, 0x0f, 0xc7, 0xce, 0x90, 0xe1, 0x11, 0x3d, 0x45, 0xe5, 0xed, 0x9a, 0x5d,
-	0xbc, 0x97, 0x88, 0xac, 0xbf, 0xea, 0xd0, 0x48, 0x7d, 0x29, 0xf8, 0x5c, 0xc5, 0xa0, 0xfc, 0x4e,
+	0xf7, 0x70, 0xbc, 0x97, 0x88, 0xac, 0xbf, 0xea, 0xd0, 0x48, 0x7d, 0x29, 0xf0, 0xb9, 0x8a, 0x41,
-	0xac, 0x5f, 0xbe, 0xeb, 0x32, 0xfd, 0xa6, 0x34, 0xbb, 0x8e, 0xde, 0x83, 0x86, 0x74, 0x87, 0x13,
+	0x79, 0x4d, 0xac, 0x5f, 0xbe, 0xeb, 0x32, 0xbd, 0x52, 0x9a, 0xcd, 0xa3, 0xf7, 0xa0, 0x21, 0xdd,
-	0x46, 0x3d, 0xcc, 0xbc, 0x5b, 0x97, 0xb2, 0x47, 0x52, 0x54, 0x04, 0x55, 0xb9, 0x4c, 0x88, 0x54,
+	0xe1, 0x84, 0x51, 0x0f, 0x33, 0xef, 0xd6, 0xa5, 0xec, 0x91, 0x14, 0x15, 0x81, 0xaa, 0x5c, 0x26,
-	0xa7, 0x87, 0x08, 0x81, 0xd2, 0x80, 0xc6, 0xaa, 0xae, 0x34, 0x6c, 0xf9, 0x4c, 0xee, 0x42, 0x3d,
+	0x44, 0xaa, 0xd3, 0x43, 0x84, 0x40, 0x69, 0x40, 0x63, 0xc5, 0x2b, 0x0d, 0x5b, 0x3e, 0x93, 0xbb,
-	0xc0, 0x98, 0xd1, 0xae, 0x42, 0x54, 0x93, 0xc9, 0x09, 0x4a, 0x24, 0x28, 0x58, 0xdf, 0x83, 0xfa,
+	0x50, 0x0f, 0x30, 0x66, 0xb4, 0xab, 0x20, 0xaa, 0xc9, 0xe4, 0x04, 0x25, 0x12, 0x28, 0x58, 0x3f,
-	0x01, 0x0d, 0xf0, 0x80, 0x76, 0xbd, 0x5d, 0xde, 0xbf, 0x0a, 0xcf, 0xbc, 0x7d, 0xa3, 0x17, 0xda,
+	0x80, 0xfa, 0x01, 0x0d, 0xf0, 0x80, 0x76, 0xbd, 0x5d, 0xde, 0xbf, 0x0a, 0x9e, 0x79, 0xfb, 0x46,
-	0x37, 0x73, 0xb7, 0x20, 0xeb, 0xfb, 0x1a, 0x2c, 0xbc, 0xe9, 0x8f, 0xf8, 0xe0, 0x8a, 0x5f, 0x2f,
+	0x2f, 0xb4, 0x6f, 0xe6, 0x1e, 0x41, 0xd6, 0x07, 0x1a, 0x2c, 0xbc, 0xe9, 0x8f, 0xf8, 0xe0, 0x8a,
-	0x14, 0x6c, 0x7d, 0x4a, 0xc1, 0x9e, 0xa3, 0xc3, 0xcf, 0x35, 0xa8, 0x3e, 0xc4, 0xf1, 0xd6, 0x3e,
+	0xbb, 0x17, 0x08, 0x5b, 0x9f, 0x42, 0xd8, 0x73, 0x8e, 0xc1, 0xb3, 0xb7, 0xc9, 0xd2, 0x94, 0xdb,
-	0xf6, 0xa5, 0x83, 0x45, 0xd1, 0x4d, 0xdb, 0x3a, 0x72, 0x20, 0x30, 0x4e, 0x94, 0x99, 0x64, 0x7d,
+	0xa4, 0x05, 0x85, 0x03, 0x62, 0x5a, 0x1b, 0xc6, 0xfa, 0x85, 0x06, 0xd5, 0x87, 0x38, 0xde, 0xda,
-	0xc8, 0xab, 0xcc, 0x05, 0xfb, 0xec, 0x6d, 0x58, 0xa0, 0xdc, 0x79, 0xec, 0xfa, 0xb4, 0x27, 0x03,
+	0xc7, 0xbe, 0x0c, 0x14, 0x41, 0xde, 0x69, 0x7b, 0x48, 0x0e, 0x84, 0x3b, 0x26, 0xe8, 0x2a, 0xd1,
-	0x64, 0xc1, 0xae, 0x52, 0xfe, 0xb6, 0x18, 0x8a, 0x02, 0x97, 0xa9, 0xa9, 0xd2, 0xc1, 0xb0, 0x27,
+	0x13, 0x72, 0xb6, 0xba, 0x40, 0xd1, 0xdb, 0xb0, 0x40, 0xb9, 0xf3, 0xd8, 0xf5, 0x69, 0x4f, 0x2a,
-	0x24, 0xd6, 0x3b, 0x00, 0x89, 0x6a, 0x02, 0x50, 0x16, 0x7e, 0xda, 0x64, 0xf8, 0x7d, 0x15, 0xaa,
+	0xb9, 0x60, 0x57, 0x29, 0x7f, 0x5b, 0x0c, 0x05, 0x51, 0x66, 0xe6, 0xaa, 0xb4, 0x32, 0xec, 0x09,
-	0x1e, 0x8e, 0xb7, 0x38, 0xf6, 0x5b, 0xba, 0xdc, 0x1d, 0x66, 0x51, 0x4b, 0x56, 0xb2, 0xd3, 0xe9,
+	0x89, 0xf5, 0x0e, 0x40, 0xa2, 0x9a, 0x00, 0x3a, 0x0b, 0x63, 0x6d, 0x32, 0x8c, 0xbf, 0x0e, 0x55,
-	0xd6, 0x0f, 0x74, 0xa8, 0x7d, 0x2b, 0x72, 0x7b, 0x9d, 0xb0, 0x87, 0xa7, 0xcf, 0x14, 0xff, 0x11,
+	0x0f, 0xc7, 0x5b, 0x1c, 0xfb, 0x2d, 0x5d, 0x9e, 0x32, 0xb3, 0xd0, 0x4f, 0x56, 0xb2, 0xd3, 0xe9,
-	0x45, 0xbf, 0xf7, 0x28, 0xdf, 0x20, 0x72, 0x81, 0xc8, 0x1e, 0x39, 0xc8, 0xb3, 0x27, 0x19, 0x0a,
+	0xd6, 0x0f, 0x75, 0xa8, 0x7d, 0x27, 0x72, 0x7b, 0x9d, 0xb0, 0x87, 0xa7, 0xcf, 0xd4, 0x8d, 0x47,
-	0xec, 0x54, 0x68, 0xe6, 0x0c, 0xdd, 0x78, 0x90, 0x16, 0x0b, 0x90, 0xa2, 0x3d, 0x21, 0x21, 0x6d,
+	0x14, 0xfd, 0xde, 0xa3, 0xfc, 0xa0, 0xc9, 0x05, 0x22, 0x0b, 0xe5, 0x20, 0xcf, 0xc2, 0x64, 0x28,
-	0x68, 0xa4, 0x13, 0x98, 0x1b, 0xa8, 0x92, 0x51, 0xdf, 0xba, 0x37, 0x35, 0x93, 0x1f, 0xe2, 0xf8,
+	0x60, 0xa7, 0x42, 0x33, 0x67, 0xe8, 0xc6, 0x83, 0x94, 0x74, 0x40, 0x8a, 0xf6, 0x84, 0x84, 0xb4,
-	0x6d, 0xd7, 0x1f, 0xe1, 0x9e, 0x4b, 0x99, 0x5d, 0x4f, 0x16, 0x11, 0x6f, 0x59, 0x4f, 0x35, 0x00,
+	0xa1, 0x91, 0x4e, 0x60, 0x6e, 0xa0, 0xa8, 0xa7, 0xbe, 0x75, 0x6f, 0x2a, 0x23, 0x3c, 0xc4, 0xf1,
-	0x49, 0x40, 0x24, 0xfb, 0xf9, 0x45, 0xb5, 0x8f, 0xb2, 0x28, 0xf9, 0x12, 0xac, 0x84, 0xa3, 0xc0,
+	0xdb, 0xae, 0x3f, 0xc2, 0x3d, 0x97, 0x32, 0xbb, 0x9e, 0x2c, 0x22, 0xbe, 0xb2, 0x9e, 0x6a, 0x00,
-	0x61, 0xe8, 0xbb, 0x31, 0xf6, 0x9c, 0x04, 0x06, 0x4f, 0xe0, 0x90, 0x70, 0x14, 0xd8, 0xea, 0xa7,
+	0x12, 0x01, 0x41, 0x1a, 0xe7, 0x17, 0xd5, 0x3e, 0xce, 0xa2, 0xe4, 0x2b, 0xb0, 0x12, 0x8e, 0x02,
-	0xfd, 0xe4, 0x17, 0xeb, 0x87, 0x1a, 0xc0, 0x9b, 0xc2, 0x72, 0xa5, 0xc6, 0xd9, 0x3b, 0x8e, 0x36,
+	0x87, 0xa1, 0xef, 0xc6, 0xd8, 0x73, 0x12, 0x30, 0x78, 0x02, 0x0e, 0x09, 0x47, 0x81, 0xad, 0x5e,
-	0xe5, 0x8e, 0x33, 0x81, 0x4e, 0x2f, 0xa2, 0xdb, 0x4e, 0xd1, 0x89, 0xc2, 0xc5, 0x93, 0xde, 0xe7,
+	0xed, 0x27, 0x6f, 0xac, 0x1f, 0x69, 0x00, 0x6f, 0x0a, 0xcb, 0x95, 0x1a, 0x67, 0xa3, 0x5b, 0x9b,
-	0xbd, 0x19, 0xee, 0xcc, 0x8d, 0x4f, 0xe8, 0xca, 0x67, 0xeb, 0x67, 0xaa, 0x5d, 0x2c, 0xb4, 0x53,
+	0x12, 0xdd, 0x13, 0xd0, 0xe9, 0x45, 0xe8, 0xb6, 0x53, 0xe8, 0x04, 0x01, 0xf2, 0xa4, 0x87, 0x7a,
-	0x2a, 0x15, 0xbc, 0xac, 0x9d, 0xf5, 0xb2, 0xac, 0x35, 0x41, 0xc4, 0xc6, 0x0e, 0xa7, 0x4f, 0x30,
+	0x6f, 0x86, 0x3b, 0x73, 0xe3, 0x13, 0x74, 0xe5, 0xb3, 0xf5, 0x73, 0xd5, 0x76, 0x16, 0xda, 0x29,
-	0x4d, 0x12, 0x25, 0xda, 0xa7, 0x4f, 0x50, 0xa4, 0x81, 0x44, 0x12, 0x9d, 0xf0, 0x74, 0xb7, 0x10,
+	0x95, 0x0a, 0x5e, 0xd6, 0xce, 0x7a, 0x59, 0x72, 0x56, 0x10, 0xb1, 0xb1, 0xc3, 0xe9, 0x13, 0x4c,
-	0x18, 0xa2, 0x13, 0x4e, 0xbe, 0x00, 0x37, 0x18, 0x76, 0x31, 0x8c, 0xfd, 0xb1, 0x13, 0x44, 0x3d,
+	0x93, 0x44, 0x89, 0xf6, 0xe9, 0x13, 0x14, 0x69, 0x20, 0x21, 0x89, 0x4e, 0x78, 0x7a, 0xea, 0x08,
-	0x7a, 0x44, 0x31, 0x4d, 0x15, 0x33, 0xfd, 0x61, 0x37, 0x91, 0x5b, 0x7f, 0xd7, 0xa0, 0xf9, 0xed,
+	0x18, 0xa2, 0x13, 0x4e, 0xbe, 0x04, 0x37, 0x18, 0x76, 0x31, 0x8c, 0xfd, 0xb1, 0x13, 0x44, 0x3d,
-	0xb4, 0x82, 0x2a, 0xcd, 0x9e, 0x41, 0xdd, 0xfa, 0x9a, 0x34, 0xb6, 0xc0, 0x6f, 0x4e, 0xef, 0x38,
+	0x7a, 0x44, 0x31, 0x4d, 0x15, 0x33, 0x7d, 0xb1, 0x9b, 0xc8, 0xad, 0xbf, 0x6b, 0xd0, 0xfc, 0x6e,
-	0x83, 0x64, 0x2f, 0x70, 0xec, 0x2b, 0xa5, 0xb6, 0xa1, 0x2e, 0xdd, 0x91, 0xac, 0x51, 0x9a, 0xeb,
+	0xca, 0xc4, 0x4a, 0xb3, 0x67, 0xc0, 0x7f, 0xdf, 0x90, 0xc6, 0x16, 0xf0, 0x9b, 0xd3, 0x83, 0xce,
-	0x83, 0xdc, 0xf3, 0x36, 0x1c, 0x65, 0xcf, 0xd6, 0x2f, 0x75, 0x20, 0xaa, 0xe7, 0x20, 0x9d, 0xf4,
+	0x40, 0xb2, 0x17, 0x38, 0xf6, 0x95, 0x52, 0xdb, 0x50, 0x97, 0xee, 0x48, 0xd6, 0x28, 0xcd, 0xf5,
-	0xdc, 0x5d, 0x14, 0x5e, 0x9b, 0x7e, 0x51, 0x38, 0x7f, 0x02, 0xfc, 0x0c, 0x28, 0xb3, 0xf2, 0x86,
+	0x41, 0xee, 0x79, 0x1b, 0x8e, 0xb2, 0x67, 0xeb, 0x57, 0x3a, 0x10, 0xd5, 0xbb, 0x90, 0x4e, 0x7a,
-	0x60, 0xa1, 0x08, 0xb4, 0xa1, 0x81, 0xa7, 0x31, 0x73, 0xd3, 0xa4, 0xab, 0x5e, 0x3a, 0xe9, 0xe4,
+	0xee, 0x0a, 0x8e, 0xd7, 0xa6, 0x17, 0x1c, 0xe7, 0x6f, 0x92, 0x9f, 0x03, 0x65, 0x56, 0xde, 0x58,
-	0x6b, 0x49, 0x26, 0xbf, 0xab, 0xc3, 0x4a, 0x7a, 0x95, 0x7c, 0xc1, 0xeb, 0x62, 0x5e, 0xbf, 0xd3,
+	0x2c, 0x90, 0x40, 0x1b, 0x1a, 0x78, 0x1a, 0x33, 0x37, 0x4d, 0xba, 0xea, 0xa5, 0x93, 0x4e, 0x7e,
-	0xe1, 0x95, 0x02, 0xaf, 0x3d, 0x16, 0xf5, 0x19, 0x72, 0xfe, 0x82, 0xdb, 0x3c, 0x6e, 0xaf, 0xff,
+	0x96, 0x64, 0xf2, 0xbb, 0x3a, 0xac, 0xa4, 0x25, 0xe9, 0x0b, 0xbc, 0x2e, 0xc6, 0xeb, 0xf7, 0x3a,
+	0xbc, 0x52, 0xc0, 0x6b, 0x8f, 0x45, 0x7d, 0x86, 0x9c, 0xbf, 0xc0, 0x6d, 0x1e, 0x6e, 0xaf, 0xff,
 	0xc5, 0x80, 0x6a, 0x62, 0x30, 0xa9, 0x41, 0xd9, 0x7b, 0x14, 0x85, 0x68, 0xbe, 0x44, 0x56, 0xe1,
-	0x86, 0x77, 0xf6, 0xcf, 0x0b, 0xb3, 0x47, 0x96, 0x61, 0xc9, 0x2b, 0xf6, 0xf5, 0x4d, 0x24, 0x04,
+	0x86, 0x77, 0xf6, 0x27, 0x88, 0xd9, 0x23, 0xcb, 0xb0, 0xe4, 0x15, 0xff, 0x0f, 0x98, 0x48, 0x08,
-	0x9a, 0x5e, 0xa1, 0xad, 0x6d, 0x1e, 0x91, 0x5b, 0xb0, 0xec, 0x9d, 0xef, 0xfc, 0x9a, 0x62, 0xdf,
+	0x34, 0xbd, 0x42, 0x7b, 0xdc, 0x3c, 0x22, 0xb7, 0x60, 0xd9, 0x3b, 0xdf, 0x41, 0x36, 0xc5, 0xb9,
-	0x37, 0xbd, 0x62, 0x73, 0x94, 0x9b, 0x03, 0xb9, 0xc4, 0x37, 0x30, 0xce, 0x4e, 0xf9, 0xdc, 0xa4,
+	0x6f, 0x7a, 0xc5, 0x26, 0x2b, 0x37, 0x07, 0x72, 0x89, 0x6f, 0x61, 0x9c, 0x55, 0x0b, 0xdc, 0xa4,
-	0x64, 0x15, 0x4c, 0xef, 0x4c, 0x8f, 0xd2, 0xfc, 0xbd, 0x46, 0x96, 0xa1, 0xe9, 0x15, 0x9a, 0x70,
+	0x64, 0x15, 0x4c, 0xef, 0x4c, 0xaf, 0xd3, 0xfc, 0x83, 0x46, 0x96, 0xa1, 0xe9, 0x15, 0x9a, 0x79,
-	0xe6, 0x1f, 0x34, 0x42, 0x60, 0xd1, 0x9b, 0xec, 0x32, 0x99, 0x7f, 0xd4, 0xc8, 0x2d, 0x20, 0xde,
+	0xe6, 0x07, 0x1a, 0x21, 0xb0, 0xe8, 0x4d, 0x76, 0xab, 0xcc, 0x3f, 0x6a, 0xe4, 0x16, 0x10, 0xef,
-	0xb9, 0x66, 0x8c, 0xf9, 0x27, 0x8d, 0xac, 0xc0, 0x92, 0x57, 0xe8, 0x59, 0x70, 0xf3, 0xcf, 0x1a,
+	0x5c, 0x53, 0xc7, 0xfc, 0x93, 0x46, 0x56, 0x60, 0xc9, 0x2b, 0xf4, 0x3e, 0xb8, 0xf9, 0x67, 0x8d,
-	0xb9, 0x01, 0x0d, 0x6f, 0xa2, 0x3c, 0x99, 0xbf, 0xd2, 0xd5, 0xa7, 0x26, 0x63, 0xca, 0x7c, 0x57,
+	0xdc, 0x80, 0x86, 0x37, 0x41, 0x4f, 0xe6, 0xaf, 0x75, 0xb5, 0xd5, 0x64, 0x4c, 0x99, 0xef, 0xea,
-	0x27, 0x77, 0xe0, 0xa6, 0x37, 0x35, 0xd0, 0xcc, 0xf7, 0x74, 0xd2, 0x80, 0xaa, 0xa7, 0xba, 0x03,
+	0xe4, 0x0e, 0xdc, 0xf4, 0xa6, 0x06, 0x9a, 0xf9, 0x9e, 0x4e, 0x1a, 0x50, 0xf5, 0x54, 0x97, 0xc1,
-	0xe6, 0x8f, 0x0c, 0x39, 0x52, 0xd7, 0x56, 0xf3, 0xc7, 0x06, 0xa9, 0x43, 0xc5, 0x93, 0xe7, 0x43,
+	0xfc, 0xb1, 0x21, 0x47, 0xaa, 0xfc, 0x35, 0x7f, 0x62, 0x90, 0x3a, 0x54, 0x3c, 0x79, 0xcf, 0x34,
-	0xf3, 0x27, 0xea, 0x27, 0x75, 0xfc, 0x37, 0x3f, 0x34, 0xa4, 0xfa, 0x93, 0x97, 0x01, 0xf3, 0x3f,
+	0x7f, 0xaa, 0x5e, 0xa9, 0x32, 0xc2, 0xfc, 0xc8, 0x90, 0xea, 0x4f, 0x16, 0x15, 0xe6, 0x7f, 0x0c,
-	0x06, 0x69, 0x42, 0xcd, 0x4b, 0xcf, 0xb3, 0xe6, 0xaf, 0x6b, 0x52, 0xeb, 0xe2, 0x56, 0x61, 0xbe,
+	0xd2, 0x84, 0x9a, 0x97, 0xde, 0x8b, 0xcd, 0xdf, 0xd4, 0xa4, 0xd6, 0xc5, 0xa3, 0xc2, 0x7c, 0xbf,
-	0x5f, 0x23, 0x4b, 0x00, 0x5e, 0x76, 0xec, 0x31, 0x7f, 0x53, 0x7b, 0xfd, 0x0d, 0x58, 0x48, 0xff,
+	0x46, 0x96, 0x00, 0xbc, 0xec, 0xda, 0x63, 0xfe, 0xb6, 0xf6, 0xfa, 0x1b, 0xb0, 0x90, 0xfe, 0x87,
-	0x3f, 0x24, 0x00, 0x95, 0x5d, 0x97, 0xc7, 0xc8, 0xcc, 0x97, 0xc4, 0xb3, 0x8d, 0x6e, 0x0f, 0x99,
+	0x24, 0x00, 0x95, 0x5d, 0x97, 0xc7, 0xc8, 0xcc, 0x97, 0xc4, 0xb3, 0x8d, 0x6e, 0x0f, 0x99, 0xa9,
-	0xa9, 0x89, 0xe7, 0xef, 0x30, 0x2a, 0xe4, 0xba, 0xf0, 0xf9, 0x9e, 0x08, 0x4c, 0xd3, 0xd8, 0x6e,
+	0x89, 0xe7, 0xef, 0x31, 0x2a, 0xe4, 0xba, 0xf0, 0xf9, 0x9e, 0x08, 0x4c, 0xd3, 0xd8, 0x6e, 0x7f,
-	0x7f, 0x77, 0xbb, 0x4f, 0xe3, 0xc1, 0xe8, 0x50, 0x44, 0xcd, 0xe6, 0x13, 0xea, 0xfb, 0xf4, 0x49,
+	0x7f, 0xbb, 0x4f, 0xe3, 0xc1, 0xe8, 0x50, 0x44, 0xcd, 0xe6, 0x13, 0xea, 0xfb, 0xf4, 0x49, 0x8c,
-	0x8c, 0xdd, 0xc1, 0xa6, 0x8a, 0xa8, 0x2f, 0xf6, 0x28, 0x8f, 0x19, 0x3d, 0x1c, 0xc5, 0xd8, 0xdb,
+	0xdd, 0xc1, 0xa6, 0x8a, 0xa8, 0x2f, 0xf7, 0x28, 0x8f, 0x19, 0x3d, 0x1c, 0xc5, 0xd8, 0xdb, 0x4c,
-	0x4c, 0x93, 0x65, 0x53, 0x86, 0x59, 0x36, 0x1c, 0x1e, 0x1e, 0x56, 0xa4, 0xe4, 0xc1, 0xff, 0x02,
+	0x93, 0x65, 0x53, 0x86, 0x59, 0x36, 0x1c, 0x1e, 0x1e, 0x56, 0xa4, 0xe4, 0xc1, 0xff, 0x02, 0x00,
-	0x00, 0x00, 0xff, 0xff, 0xc1, 0x89, 0xef, 0x28, 0xb0, 0x1f, 0x00, 0x00,
+	0x00, 0xff, 0xff, 0xc4, 0x5d, 0xb1, 0xba, 0xf8, 0x1f, 0x00, 0x00,
 }

internal/proxy/proxy.go

@@ -2,6 +2,8 @@ package proxy
 
 import (
 	"context"
+	"fmt"
+	"io"
 	"log"
 	"math/rand"
 	"net"
@@ -9,6 +11,10 @@ import (
 	"sync"
 	"time"
 
+	"github.com/opentracing/opentracing-go"
+	"github.com/uber/jaeger-client-go"
+	"github.com/uber/jaeger-client-go/config"
+
 	"google.golang.org/grpc"
 
 	"github.com/zilliztech/milvus-distributed/internal/allocator"
@@ -39,6 +45,9 @@ type Proxy struct {
 	manipulationMsgStream *msgstream.PulsarMsgStream
 	queryMsgStream        *msgstream.PulsarMsgStream
 
+	tracer opentracing.Tracer
+	closer io.Closer
+
 	// Add callback functions at different stages
 	startCallbacks []func()
 	closeCallbacks []func()
@@ -51,11 +60,28 @@ func Init() {
 func CreateProxy(ctx context.Context) (*Proxy, error) {
 	rand.Seed(time.Now().UnixNano())
 	ctx1, cancel := context.WithCancel(ctx)
+	var err error
 	p := &Proxy{
 		proxyLoopCtx:    ctx1,
 		proxyLoopCancel: cancel,
 	}
 
+	cfg := &config.Configuration{
+		ServiceName: "tracing",
+		Sampler: &config.SamplerConfig{
+			Type:  "const",
+			Param: 1,
+		},
+		Reporter: &config.ReporterConfig{
+			LogSpans: true,
+		},
+	}
+	p.tracer, p.closer, err = cfg.NewTracer(config.Logger(jaeger.StdLogger))
+	if err != nil {
+		panic(fmt.Sprintf("ERROR: cannot init Jaeger: %v\n", err))
+	}
+	opentracing.SetGlobalTracer(p.tracer)
+
 	pulsarAddress := Params.PulsarAddress()
 
 	p.queryMsgStream = msgstream.NewPulsarMsgStream(p.proxyLoopCtx, Params.MsgStreamSearchBufSize())
@@ -198,6 +224,8 @@ func (p *Proxy) stopProxyLoop() {
 	p.tick.Close()
 
 	p.proxyLoopWg.Wait()
+
+	p.closer.Close()
 }
 
 // Close closes the server.

internal/proxy/repack_func.go

@@ -182,6 +182,7 @@ func insertRepackFunc(tsMsgs []msgstream.TsMsg,
 			insertMsg := &msgstream.InsertMsg{
 				InsertRequest: sliceRequest,
 			}
+			insertMsg.SetContext(request.GetContext())
 			if together { // all rows with same hash value are accumulated to only one message
 				if len(result[key].Msgs) <= 0 {
 					result[key].Msgs = append(result[key].Msgs, insertMsg)

internal/proxy/task.go

@@ -7,6 +7,9 @@ import (
 	"math"
 	"strconv"
 
+	"github.com/opentracing/opentracing-go"
+	oplog "github.com/opentracing/opentracing-go/log"
+
 	"github.com/golang/protobuf/proto"
 	"github.com/zilliztech/milvus-distributed/internal/allocator"
 	"github.com/zilliztech/milvus-distributed/internal/msgstream"
@@ -74,12 +77,21 @@ func (it *InsertTask) Type() internalpb.MsgType {
 }
 
 func (it *InsertTask) PreExecute() error {
+	span := opentracing.StartSpan("InsertTask preExecute")
+	defer span.Finish()
+	it.ctx = opentracing.ContextWithSpan(it.ctx, span)
+	span.SetTag("hash keys", it.ReqID)
+	span.SetTag("start time", it.BeginTs())
 	collectionName := it.BaseInsertTask.CollectionName
 	if err := ValidateCollectionName(collectionName); err != nil {
+		span.LogFields(oplog.Error(err))
+		span.Finish()
 		return err
 	}
 	partitionTag := it.BaseInsertTask.PartitionTag
 	if err := ValidatePartitionTag(partitionTag, true); err != nil {
+		span.LogFields(oplog.Error(err))
+		span.Finish()
 		return err
 	}
 
@@ -87,22 +99,36 @@ func (it *InsertTask) PreExecute() error {
 }
 
 func (it *InsertTask) Execute() error {
+	span, ctx := opentracing.StartSpanFromContext(it.ctx, "InsertTask Execute")
+	defer span.Finish()
+	it.ctx = ctx
+	span.SetTag("hash keys", it.ReqID)
+	span.SetTag("start time", it.BeginTs())
 	collectionName := it.BaseInsertTask.CollectionName
+	span.LogFields(oplog.String("collection_name", collectionName))
 	if !globalMetaCache.Hit(collectionName) {
 		err := globalMetaCache.Sync(collectionName)
 		if err != nil {
+			span.LogFields(oplog.Error(err))
+			span.Finish()
 			return err
 		}
 	}
 	description, err := globalMetaCache.Get(collectionName)
 	if err != nil || description == nil {
+		span.LogFields(oplog.Error(err))
+		span.Finish()
 		return err
 	}
 	autoID := description.Schema.AutoID
+	span.LogFields(oplog.Bool("auto_id", autoID))
 	var rowIDBegin UniqueID
 	var rowIDEnd UniqueID
 	rowNums := len(it.BaseInsertTask.RowData)
 	rowIDBegin, rowIDEnd, _ = it.rowIDAllocator.Alloc(uint32(rowNums))
+	span.LogFields(oplog.Int("rowNums", rowNums),
+		oplog.Int("rowIDBegin", int(rowIDBegin)),
+		oplog.Int("rowIDEnd", int(rowIDEnd)))
 	it.BaseInsertTask.RowIDs = make([]UniqueID, rowNums)
 	for i := rowIDBegin; i < rowIDEnd; i++ {
 		offset := i - rowIDBegin
@@ -125,6 +151,8 @@ func (it *InsertTask) Execute() error {
 		EndTs:   it.EndTs(),
 		Msgs:    make([]msgstream.TsMsg, 1),
 	}
+	tsMsg.SetContext(it.ctx)
+	span.LogFields(oplog.String("send msg", "send msg"))
 	msgPack.Msgs[0] = tsMsg
 	err = it.manipulationMsgStream.Produce(msgPack)
 
@@ -138,11 +166,14 @@ func (it *InsertTask) Execute() error {
 	if err != nil {
 		it.result.Status.ErrorCode = commonpb.ErrorCode_UNEXPECTED_ERROR
 		it.result.Status.Reason = err.Error()
+		span.LogFields(oplog.Error(err))
 	}
 	return nil
 }
 
 func (it *InsertTask) PostExecute() error {
+	span, _ := opentracing.StartSpanFromContext(it.ctx, "InsertTask postExecute")
+	defer span.Finish()
 	return nil
 }
 
@@ -352,24 +383,38 @@ func (qt *QueryTask) SetTs(ts Timestamp) {
 }
 
 func (qt *QueryTask) PreExecute() error {
+	span := opentracing.StartSpan("InsertTask preExecute")
+	defer span.Finish()
+	qt.ctx = opentracing.ContextWithSpan(qt.ctx, span)
+	span.SetTag("hash keys", qt.ReqID)
+	span.SetTag("start time", qt.BeginTs())
+
 	collectionName := qt.query.CollectionName
 	if !globalMetaCache.Hit(collectionName) {
 		err := globalMetaCache.Sync(collectionName)
 		if err != nil {
+			span.LogFields(oplog.Error(err))
+			span.Finish()
 			return err
 		}
 	}
 	_, err := globalMetaCache.Get(collectionName)
 	if err != nil { // err is not nil if collection not exists
+		span.LogFields(oplog.Error(err))
+		span.Finish()
 		return err
 	}
 
 	if err := ValidateCollectionName(qt.query.CollectionName); err != nil {
+		span.LogFields(oplog.Error(err))
+		span.Finish()
 		return err
 	}
 
 	for _, tag := range qt.query.PartitionTags {
 		if err := ValidatePartitionTag(tag, false); err != nil {
+			span.LogFields(oplog.Error(err))
+			span.Finish()
 			return err
 		}
 	}
@@ -379,6 +424,8 @@ func (qt *QueryTask) PreExecute() error {
 	}
 	queryBytes, err := proto.Marshal(qt.query)
 	if err != nil {
+		span.LogFields(oplog.Error(err))
+		span.Finish()
 		return err
 	}
 	qt.Query = &commonpb.Blob{
@@ -388,6 +435,10 @@ func (qt *QueryTask) PreExecute() error {
 }
 
 func (qt *QueryTask) Execute() error {
+	span, ctx := opentracing.StartSpanFromContext(qt.ctx, "InsertTask Execute")
+	defer span.Finish()
+	span.SetTag("hash keys", qt.ReqID)
+	span.SetTag("start time", qt.BeginTs())
 	var tsMsg msgstream.TsMsg = &msgstream.SearchMsg{
 		SearchRequest: qt.SearchRequest,
 		BaseMsg: msgstream.BaseMsg{
@@ -401,20 +452,28 @@ func (qt *QueryTask) Execute() error {
 		EndTs:   qt.Timestamp,
 		Msgs:    make([]msgstream.TsMsg, 1),
 	}
+	tsMsg.SetContext(ctx)
 	msgPack.Msgs[0] = tsMsg
 	err := qt.queryMsgStream.Produce(msgPack)
 	log.Printf("[Proxy] length of searchMsg: %v", len(msgPack.Msgs))
 	if err != nil {
+		span.LogFields(oplog.Error(err))
+		span.Finish()
 		log.Printf("[Proxy] send search request failed: %v", err)
 	}
 	return err
 }
 
 func (qt *QueryTask) PostExecute() error {
+	span, _ := opentracing.StartSpanFromContext(qt.ctx, "InsertTask postExecute")
+	span.SetTag("hash keys", qt.ReqID)
+	span.SetTag("start time", qt.BeginTs())
 	for {
 		select {
 		case <-qt.ctx.Done():
 			log.Print("wait to finish failed, timeout!")
+			span.LogFields(oplog.String("wait to finish failed, timeout", "wait to finish failed, timeout"))
+			span.Finish()
 			return errors.New("wait to finish failed, timeout")
 		case searchResults := <-qt.resultBuf:
 			filterSearchResult := make([]*internalpb.SearchResult, 0)
@@ -435,6 +494,8 @@ func (qt *QueryTask) PostExecute() error {
 						Reason:    filterReason,
 					},
 				}
+				span.LogFields(oplog.Error(errors.New(filterReason)))
+				span.Finish()
 				return errors.New(filterReason)
 			}
 
@@ -465,6 +526,7 @@ func (qt *QueryTask) PostExecute() error {
 						Reason:    filterReason,
 					},
 				}
+				span.Finish()
 				return nil
 			}
 
@@ -476,6 +538,7 @@ func (qt *QueryTask) PostExecute() error {
 						Reason:    filterReason,
 					},
 				}
+				span.Finish()
 				return nil
 			}
 
@@ -526,10 +589,13 @@ func (qt *QueryTask) PostExecute() error {
 				reducedHitsBs, err := proto.Marshal(reducedHits)
 				if err != nil {
 					log.Println("marshal error")
+					span.LogFields(oplog.Error(err))
+					span.Finish()
 					return err
 				}
 				qt.result.Hits = append(qt.result.Hits, reducedHitsBs)
 			}
+			span.Finish()
 			return nil
 		}
 	}
@@ -637,7 +703,10 @@ func (dct *DescribeCollectionTask) PreExecute() error {
 func (dct *DescribeCollectionTask) Execute() error {
 	var err error
 	dct.result, err = dct.masterClient.DescribeCollection(dct.ctx, &dct.DescribeCollectionRequest)
-	globalMetaCache.Update(dct.CollectionName.CollectionName, dct.result)
+	if err != nil {
+		return err
+	}
+	err = globalMetaCache.Update(dct.CollectionName.CollectionName, dct.result)
 	return err
 }
 

internal/querynode/data_sync_service.go

@@ -48,7 +48,6 @@ func (dsService *dataSyncService) initNodes() {
 
 	var insertNode Node = newInsertNode(dsService.replica)
 	var serviceTimeNode Node = newServiceTimeNode(dsService.replica)
-	var gcNode Node = newGCNode(dsService.replica)
 
 	dsService.fg.AddNode(&dmStreamNode)
 	dsService.fg.AddNode(&ddStreamNode)
@@ -58,7 +57,6 @@ func (dsService *dataSyncService) initNodes() {
 
 	dsService.fg.AddNode(&insertNode)
 	dsService.fg.AddNode(&serviceTimeNode)
-	dsService.fg.AddNode(&gcNode)
 
 	// dmStreamNode
 	var err = dsService.fg.SetEdges(dmStreamNode.Name(),
@@ -108,17 +106,9 @@ func (dsService *dataSyncService) initNodes() {
 	// serviceTimeNode
 	err = dsService.fg.SetEdges(serviceTimeNode.Name(),
 		[]string{insertNode.Name()},
-		[]string{gcNode.Name()},
+		[]string{},
 	)
 	if err != nil {
 		log.Fatal("set edges failed in node:", serviceTimeNode.Name())
 	}
-
-	// gcNode
-	err = dsService.fg.SetEdges(gcNode.Name(),
-		[]string{serviceTimeNode.Name()},
-		[]string{})
-	if err != nil {
-		log.Fatal("set edges failed in node:", gcNode.Name())
-	}
 }

internal/querynode/flow_graph_dd_node.go

@@ -44,11 +44,6 @@ func (ddNode *ddNode) Operate(in []*Msg) []*Msg {
 		},
 	}
 	ddNode.ddMsg = &ddMsg
-	gcRecord := gcRecord{
-		collections: make([]UniqueID, 0),
-		partitions:  make([]partitionWithID, 0),
-	}
-	ddNode.ddMsg.gcRecord = &gcRecord
 
 	// sort tsMessages
 	tsMessages := msMsg.TsMessages()
@@ -120,11 +115,11 @@ func (ddNode *ddNode) createCollection(msg *msgstream.CreateCollectionMsg) {
 func (ddNode *ddNode) dropCollection(msg *msgstream.DropCollectionMsg) {
 	collectionID := msg.CollectionID
 
-	//err := ddNode.replica.removeCollection(collectionID)
+	err := ddNode.replica.removeCollection(collectionID)
-	//if err != nil {
+	if err != nil {
-	//	log.Println(err)
+		log.Println(err)
-	//	return
+		return
-	//}
+	}
 
 	collectionName := msg.CollectionName.CollectionName
 	ddNode.ddMsg.collectionRecords[collectionName] = append(ddNode.ddMsg.collectionRecords[collectionName],
@@ -132,8 +127,6 @@ func (ddNode *ddNode) dropCollection(msg *msgstream.DropCollectionMsg) {
 			createOrDrop: false,
 			timestamp:    msg.Timestamp,
 		})
-
-	ddNode.ddMsg.gcRecord.collections = append(ddNode.ddMsg.gcRecord.collections, collectionID)
 }
 
 func (ddNode *ddNode) createPartition(msg *msgstream.CreatePartitionMsg) {
@@ -157,22 +150,17 @@ func (ddNode *ddNode) dropPartition(msg *msgstream.DropPartitionMsg) {
 	collectionID := msg.CollectionID
 	partitionTag := msg.PartitionName.Tag
 
-	//err := ddNode.replica.removePartition(collectionID, partitionTag)
+	err := ddNode.replica.removePartition(collectionID, partitionTag)
-	//if err != nil {
+	if err != nil {
-	//	log.Println(err)
+		log.Println(err)
-	//	return
+		return
-	//}
+	}
 
 	ddNode.ddMsg.partitionRecords[partitionTag] = append(ddNode.ddMsg.partitionRecords[partitionTag],
 		metaOperateRecord{
 			createOrDrop: false,
 			timestamp:    msg.Timestamp,
 		})
-
-	ddNode.ddMsg.gcRecord.partitions = append(ddNode.ddMsg.gcRecord.partitions, partitionWithID{
-		partitionTag: partitionTag,
-		collectionID: collectionID,
-	})
 }
 
 func newDDNode(replica collectionReplica) *ddNode {

internal/querynode/flow_graph_filter_dm_node.go

@@ -2,7 +2,6 @@ package querynode
 
 import (
 	"log"
-	"math"
 
 	"github.com/zilliztech/milvus-distributed/internal/msgstream"
 	"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
@@ -60,7 +59,6 @@ func (fdmNode *filterDmNode) Operate(in []*Msg) []*Msg {
 		}
 	}
 
-	iMsg.gcRecord = ddMsg.gcRecord
 	var res Msg = &iMsg
 	return []*Msg{&res}
 }
@@ -83,35 +81,17 @@ func (fdmNode *filterDmNode) filterInvalidInsertMessage(msg *msgstream.InsertMsg
 		log.Println("Error, misaligned messages detected")
 		return nil
 	}
-
 	tmpTimestamps := make([]Timestamp, 0)
 	tmpRowIDs := make([]int64, 0)
 	tmpRowData := make([]*commonpb.Blob, 0)
-
+	targetTimestamp := records[len(records)-1].timestamp
-	// calculate valid time range
-	timeBegin := Timestamp(0)
-	timeEnd := Timestamp(math.MaxUint64)
-	for _, record := range records {
-		if record.createOrDrop && timeBegin < record.timestamp {
-			timeBegin = record.timestamp
-		}
-		if !record.createOrDrop && timeEnd > record.timestamp {
-			timeEnd = record.timestamp
-		}
-	}
-
 	for i, t := range msg.Timestamps {
-		if t >= timeBegin && t <= timeEnd {
+		if t >= targetTimestamp {
 			tmpTimestamps = append(tmpTimestamps, t)
 			tmpRowIDs = append(tmpRowIDs, msg.RowIDs[i])
 			tmpRowData = append(tmpRowData, msg.RowData[i])
 		}
 	}
-
-	if len(tmpRowIDs) <= 0 {
-		return nil
-	}
-
 	msg.Timestamps = tmpTimestamps
 	msg.RowIDs = tmpRowIDs
 	msg.RowData = tmpRowData

internal/querynode/flow_graph_gc_node.go

@@ -1,61 +0,0 @@
-package querynode
-
-import (
-	"log"
-)
-
-type gcNode struct {
-	BaseNode
-	replica collectionReplica
-}
-
-func (gcNode *gcNode) Name() string {
-	return "gcNode"
-}
-
-func (gcNode *gcNode) Operate(in []*Msg) []*Msg {
-	//fmt.Println("Do gcNode operation")
-
-	if len(in) != 1 {
-		log.Println("Invalid operate message input in gcNode, input length = ", len(in))
-		// TODO: add error handling
-	}
-
-	gcMsg, ok := (*in[0]).(*gcMsg)
-	if !ok {
-		log.Println("type assertion failed for gcMsg")
-		// TODO: add error handling
-	}
-
-	// drop collections
-	for _, collectionID := range gcMsg.gcRecord.collections {
-		err := gcNode.replica.removeCollection(collectionID)
-		if err != nil {
-			log.Println(err)
-		}
-	}
-
-	// drop partitions
-	for _, partition := range gcMsg.gcRecord.partitions {
-		err := gcNode.replica.removePartition(partition.collectionID, partition.partitionTag)
-		if err != nil {
-			log.Println(err)
-		}
-	}
-
-	return nil
-}
-
-func newGCNode(replica collectionReplica) *gcNode {
-	maxQueueLength := Params.FlowGraphMaxQueueLength
-	maxParallelism := Params.FlowGraphMaxParallelism
-
-	baseNode := BaseNode{}
-	baseNode.SetMaxQueueLength(maxQueueLength)
-	baseNode.SetMaxParallelism(maxParallelism)
-
-	return &gcNode{
-		BaseNode: baseNode,
-		replica:  replica,
-	}
-}

internal/querynode/flow_graph_insert_node.go

@@ -90,7 +90,6 @@ func (iNode *insertNode) Operate(in []*Msg) []*Msg {
 	wg.Wait()
 
 	var res Msg = &serviceTimeMsg{
-		gcRecord:  iMsg.gcRecord,
 		timeRange: iMsg.timeRange,
 	}
 	return []*Msg{&res}

internal/querynode/flow_graph_message.go

@@ -16,7 +16,6 @@ type key2SegMsg struct {
 type ddMsg struct {
 	collectionRecords map[string][]metaOperateRecord
 	partitionRecords  map[string][]metaOperateRecord
-	gcRecord          *gcRecord
 	timeRange         TimeRange
 }
 
@@ -27,7 +26,6 @@ type metaOperateRecord struct {
 
 type insertMsg struct {
 	insertMessages []*msgstream.InsertMsg
-	gcRecord       *gcRecord
 	timeRange      TimeRange
 }
 
@@ -37,12 +35,6 @@ type deleteMsg struct {
 }
 
 type serviceTimeMsg struct {
-	gcRecord  *gcRecord
-	timeRange TimeRange
-}
-
-type gcMsg struct {
-	gcRecord  *gcRecord
 	timeRange TimeRange
 }
 
@@ -63,39 +55,42 @@ type DeletePreprocessData struct {
 	count         int32
 }
 
-// TODO: replace partitionWithID by partition id
-type partitionWithID struct {
-	partitionTag string
-	collectionID UniqueID
-}
-
-type gcRecord struct {
-	// collections and partitions to be dropped
-	collections []UniqueID
-	// TODO: use partition id
-	partitions []partitionWithID
-}
-
 func (ksMsg *key2SegMsg) TimeTick() Timestamp {
 	return ksMsg.timeRange.timestampMax
 }
 
+func (ksMsg *key2SegMsg) DownStreamNodeIdx() int {
+	return 0
+}
+
 func (suMsg *ddMsg) TimeTick() Timestamp {
 	return suMsg.timeRange.timestampMax
 }
 
+func (suMsg *ddMsg) DownStreamNodeIdx() int {
+	return 0
+}
+
 func (iMsg *insertMsg) TimeTick() Timestamp {
 	return iMsg.timeRange.timestampMax
 }
 
+func (iMsg *insertMsg) DownStreamNodeIdx() int {
+	return 0
+}
+
 func (dMsg *deleteMsg) TimeTick() Timestamp {
 	return dMsg.timeRange.timestampMax
 }
 
+func (dMsg *deleteMsg) DownStreamNodeIdx() int {
+	return 0
+}
+
 func (stMsg *serviceTimeMsg) TimeTick() Timestamp {
 	return stMsg.timeRange.timestampMax
 }
 
-func (gcMsg *gcMsg) TimeTick() Timestamp {
+func (stMsg *serviceTimeMsg) DownStreamNodeIdx() int {
-	return gcMsg.timeRange.timestampMax
+	return 0
 }

internal/querynode/flow_graph_service_time_node.go

@@ -30,12 +30,7 @@ func (stNode *serviceTimeNode) Operate(in []*Msg) []*Msg {
 	// update service time
 	stNode.replica.getTSafe().set(serviceTimeMsg.timeRange.timestampMax)
 	//fmt.Println("update tSafe to:", getPhysicalTime(serviceTimeMsg.timeRange.timestampMax))
-
+	return nil
-	var res Msg = &gcMsg{
-		gcRecord:  serviceTimeMsg.gcRecord,
-		timeRange: serviceTimeMsg.timeRange,
-	}
-	return []*Msg{&res}
 }
 
 func newServiceTimeNode(replica collectionReplica) *serviceTimeNode {

internal/querynode/load_index_service.go

@@ -11,9 +11,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/minio/minio-go/v7"
-	"github.com/minio/minio-go/v7/pkg/credentials"
-
 	minioKV "github.com/zilliztech/milvus-distributed/internal/kv/minio"
 	"github.com/zilliztech/milvus-distributed/internal/msgstream"
 	"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
@@ -38,16 +35,17 @@ type loadIndexService struct {
 func newLoadIndexService(ctx context.Context, replica collectionReplica) *loadIndexService {
 	ctx1, cancel := context.WithCancel(ctx)
 
-	// init minio
+	option := &minioKV.Option{
-	minioClient, err := minio.New(Params.MinioEndPoint, &minio.Options{
+		Address:           Params.MinioEndPoint,
-		Creds:  credentials.NewStaticV4(Params.MinioAccessKeyID, Params.MinioSecretAccessKey, ""),
+		AccessKeyID:       Params.MinioAccessKeyID,
-		Secure: Params.MinioUseSSLStr,
+		SecretAccessKeyID: Params.MinioSecretAccessKey,
-	})
+		UseSSL:            Params.MinioUseSSLStr,
-	if err != nil {
+		CreateBucket:      true,
-		panic(err)
+		BucketName:        Params.MinioBucketName,
 	}
 
-	MinioKV, err := minioKV.NewMinIOKV(ctx1, minioClient, Params.MinioBucketName)
+	// TODO: load bucketName from config
+	MinioKV, err := minioKV.NewMinIOKV(ctx1, option)
 	if err != nil {
 		panic(err)
 	}

internal/querynode/load_index_service_test.go

@@ -5,8 +5,6 @@ import (
 	"sort"
 	"testing"
 
-	"github.com/minio/minio-go/v7"
-	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/zilliztech/milvus-distributed/internal/indexbuilder"
@@ -68,13 +66,16 @@ func TestLoadIndexService(t *testing.T) {
 	binarySet, err := index.Serialize()
 	assert.Equal(t, err, nil)
 
-	//save index to minio
+	option := &minioKV.Option{
-	minioClient, err := minio.New(Params.MinioEndPoint, &minio.Options{
+		Address:           Params.MinioEndPoint,
-		Creds:  credentials.NewStaticV4(Params.MinioAccessKeyID, Params.MinioSecretAccessKey, ""),
+		AccessKeyID:       Params.MinioAccessKeyID,
-		Secure: Params.MinioUseSSLStr,
+		SecretAccessKeyID: Params.MinioSecretAccessKey,
-	})
+		UseSSL:            Params.MinioUseSSLStr,
-	assert.Equal(t, err, nil)
+		BucketName:        Params.MinioBucketName,
-	minioKV, err := minioKV.NewMinIOKV(node.queryNodeLoopCtx, minioClient, Params.MinioBucketName)
+		CreateBucket:      true,
+	}
+
+	minioKV, err := minioKV.NewMinIOKV(node.queryNodeLoopCtx, option)
 	assert.Equal(t, err, nil)
 	indexPaths := make([]string, 0)
 	for _, index := range binarySet {

internal/storage/internal/S3/S3_test.go

@@ -1,134 +0,0 @@
-package s3driver
-
-import (
-	"context"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	storagetype "github.com/zilliztech/milvus-distributed/internal/storage/type"
-)
-
-var option = storagetype.Option{BucketName: "zilliz-hz"}
-var ctx = context.Background()
-var client, err = NewS3Driver(ctx, option)
-
-func TestS3Driver_PutRowAndGetRow(t *testing.T) {
-	err = client.PutRow(ctx, []byte("bar"), []byte("abcdefghijklmnoopqrstuvwxyz"), "SegmentA", 1)
-	assert.Nil(t, err)
-	err = client.PutRow(ctx, []byte("bar"), []byte("djhfkjsbdfbsdughorsgsdjhgoisdgh"), "SegmentA", 2)
-	assert.Nil(t, err)
-	err = client.PutRow(ctx, []byte("bar"), []byte("123854676ershdgfsgdfk,sdhfg;sdi8"), "SegmentB", 3)
-	assert.Nil(t, err)
-	err = client.PutRow(ctx, []byte("bar1"), []byte("testkeybarorbar_1"), "SegmentC", 3)
-	assert.Nil(t, err)
-	object, _ := client.GetRow(ctx, []byte("bar"), 1)
-	assert.Equal(t, "abcdefghijklmnoopqrstuvwxyz", string(object))
-	object, _ = client.GetRow(ctx, []byte("bar"), 2)
-	assert.Equal(t, "djhfkjsbdfbsdughorsgsdjhgoisdgh", string(object))
-	object, _ = client.GetRow(ctx, []byte("bar"), 5)
-	assert.Equal(t, "123854676ershdgfsgdfk,sdhfg;sdi8", string(object))
-	object, _ = client.GetRow(ctx, []byte("bar1"), 5)
-	assert.Equal(t, "testkeybarorbar_1", string(object))
-}
-
-func TestS3Driver_DeleteRow(t *testing.T) {
-	err = client.DeleteRow(ctx, []byte("bar"), 5)
-	assert.Nil(t, err)
-	object, _ := client.GetRow(ctx, []byte("bar"), 6)
-	assert.Nil(t, object)
-	err = client.DeleteRow(ctx, []byte("bar1"), 5)
-	assert.Nil(t, err)
-	object2, _ := client.GetRow(ctx, []byte("bar1"), 6)
-	assert.Nil(t, object2)
-}
-
-func TestS3Driver_GetSegments(t *testing.T) {
-	err = client.PutRow(ctx, []byte("seg"), []byte("abcdefghijklmnoopqrstuvwxyz"), "SegmentA", 1)
-	assert.Nil(t, err)
-	err = client.PutRow(ctx, []byte("seg"), []byte("djhfkjsbdfbsdughorsgsdjhgoisdgh"), "SegmentA", 2)
-	assert.Nil(t, err)
-	err = client.PutRow(ctx, []byte("seg"), []byte("123854676ershdgfsgdfk,sdhfg;sdi8"), "SegmentB", 3)
-	assert.Nil(t, err)
-	err = client.PutRow(ctx, []byte("seg2"), []byte("testkeybarorbar_1"), "SegmentC", 1)
-	assert.Nil(t, err)
-
-	segements, err := client.GetSegments(ctx, []byte("seg"), 4)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(segements))
-	if segements[0] == "SegmentA" {
-		assert.Equal(t, "SegmentA", segements[0])
-		assert.Equal(t, "SegmentB", segements[1])
-	} else {
-		assert.Equal(t, "SegmentB", segements[0])
-		assert.Equal(t, "SegmentA", segements[1])
-	}
-}
-
-func TestS3Driver_PutRowsAndGetRows(t *testing.T) {
-	keys := [][]byte{[]byte("foo"), []byte("bar")}
-	values := [][]byte{[]byte("The key is foo!"), []byte("The key is bar!")}
-	segments := []string{"segmentA", "segmentB"}
-	timestamps := []uint64{1, 2}
-	err = client.PutRows(ctx, keys, values, segments, timestamps)
-	assert.Nil(t, err)
-
-	objects, err := client.GetRows(ctx, keys, timestamps)
-	assert.Nil(t, err)
-	assert.Equal(t, "The key is foo!", string(objects[0]))
-	assert.Equal(t, "The key is bar!", string(objects[1]))
-}
-
-func TestS3Driver_DeleteRows(t *testing.T) {
-	keys := [][]byte{[]byte("foo"), []byte("bar")}
-	timestamps := []uint64{3, 3}
-	err := client.DeleteRows(ctx, keys, timestamps)
-	assert.Nil(t, err)
-
-	objects, err := client.GetRows(ctx, keys, timestamps)
-	assert.Nil(t, err)
-	assert.Nil(t, objects[0])
-	assert.Nil(t, objects[1])
-}
-
-func TestS3Driver_PutLogAndGetLog(t *testing.T) {
-	err = client.PutLog(ctx, []byte("insert"), []byte("This is insert log!"), 1, 11)
-	assert.Nil(t, err)
-	err = client.PutLog(ctx, []byte("delete"), []byte("This is delete log!"), 2, 10)
-	assert.Nil(t, err)
-	err = client.PutLog(ctx, []byte("update"), []byte("This is update log!"), 3, 9)
-	assert.Nil(t, err)
-	err = client.PutLog(ctx, []byte("select"), []byte("This is select log!"), 4, 8)
-	assert.Nil(t, err)
-
-	channels := []int{5, 8, 9, 10, 11, 12, 13}
-	logValues, err := client.GetLog(ctx, 0, 5, channels)
-	assert.Nil(t, err)
-	assert.Equal(t, "This is select log!", string(logValues[0]))
-	assert.Equal(t, "This is update log!", string(logValues[1]))
-	assert.Equal(t, "This is delete log!", string(logValues[2]))
-	assert.Equal(t, "This is insert log!", string(logValues[3]))
-}
-
-func TestS3Driver_Segment(t *testing.T) {
-	err := client.PutSegmentIndex(ctx, "segmentA", []byte("This is segmentA's index!"))
-	assert.Nil(t, err)
-
-	segmentIndex, err := client.GetSegmentIndex(ctx, "segmentA")
-	assert.Equal(t, "This is segmentA's index!", string(segmentIndex))
-	assert.Nil(t, err)
-
-	err = client.DeleteSegmentIndex(ctx, "segmentA")
-	assert.Nil(t, err)
-}
-
-func TestS3Driver_SegmentDL(t *testing.T) {
-	err := client.PutSegmentDL(ctx, "segmentB", []byte("This is segmentB's delete log!"))
-	assert.Nil(t, err)
-
-	segmentDL, err := client.GetSegmentDL(ctx, "segmentB")
-	assert.Nil(t, err)
-	assert.Equal(t, "This is segmentB's delete log!", string(segmentDL))
-
-	err = client.DeleteSegmentDL(ctx, "segmentB")
-	assert.Nil(t, err)
-}

internal/storage/internal/S3/s3_engine.go

@@ -1,173 +0,0 @@
-package s3driver
-
-import (
-	"bytes"
-	"context"
-	"io"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/s3"
-	. "github.com/zilliztech/milvus-distributed/internal/storage/type"
-)
-
-type S3Store struct {
-	client *s3.S3
-}
-
-func NewS3Store(config aws.Config) (*S3Store, error) {
-	sess := session.Must(session.NewSession(&config))
-	service := s3.New(sess)
-
-	return &S3Store{
-		client: service,
-	}, nil
-}
-
-func (s *S3Store) Put(ctx context.Context, key Key, value Value) error {
-	_, err := s.client.PutObjectWithContext(ctx, &s3.PutObjectInput{
-		Bucket: aws.String(bucketName),
-		Key:    aws.String(string(key)),
-		Body:   bytes.NewReader(value),
-	})
-
-	//sess := session.Must(session.NewSessionWithOptions(session.Options{
-	//	SharedConfigState: session.SharedConfigEnable,
-	//}))
-	//uploader := s3manager.NewUploader(sess)
-	//
-	//_, err := uploader.Upload(&s3manager.UploadInput{
-	//	Bucket: aws.String(bucketName),
-	//	Key:    aws.String(string(key)),
-	//	Body:   bytes.NewReader(value),
-	//})
-
-	return err
-}
-
-func (s *S3Store) Get(ctx context.Context, key Key) (Value, error) {
-	object, err := s.client.GetObjectWithContext(ctx, &s3.GetObjectInput{
-		Bucket: aws.String(bucketName),
-		Key:    aws.String(string(key)),
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	//TODO: get size
-	size := 256 * 1024
-	buf := make([]byte, size)
-	n, err := object.Body.Read(buf)
-	if err != nil && err != io.EOF {
-		return nil, err
-	}
-	return buf[:n], nil
-}
-
-func (s *S3Store) GetByPrefix(ctx context.Context, prefix Key, keyOnly bool) ([]Key, []Value, error) {
-	objectsOutput, err := s.client.ListObjectsWithContext(ctx, &s3.ListObjectsInput{
-		Bucket: aws.String(bucketName),
-		Prefix: aws.String(string(prefix)),
-	})
-
-	var objectsKeys []Key
-	var objectsValues []Value
-
-	if objectsOutput != nil && err == nil {
-		for _, object := range objectsOutput.Contents {
-			objectsKeys = append(objectsKeys, []byte(*object.Key))
-			if !keyOnly {
-				value, err := s.Get(ctx, []byte(*object.Key))
-				if err != nil {
-					return nil, nil, err
-				}
-				objectsValues = append(objectsValues, value)
-			}
-		}
-	} else {
-		return nil, nil, err
-	}
-
-	return objectsKeys, objectsValues, nil
-
-}
-
-func (s *S3Store) Scan(ctx context.Context, keyStart Key, keyEnd Key, limit int, keyOnly bool) ([]Key, []Value, error) {
-	var keys []Key
-	var values []Value
-	limitCount := uint(limit)
-	objects, err := s.client.ListObjectsWithContext(ctx, &s3.ListObjectsInput{
-		Bucket: aws.String(bucketName),
-		Prefix: aws.String(string(keyStart)),
-	})
-	if err == nil && objects != nil {
-		for _, object := range objects.Contents {
-			if *object.Key >= string(keyEnd) {
-				keys = append(keys, []byte(*object.Key))
-				if !keyOnly {
-					value, err := s.Get(ctx, []byte(*object.Key))
-					if err != nil {
-						return nil, nil, err
-					}
-					values = append(values, value)
-				}
-				limitCount--
-				if limitCount <= 0 {
-					break
-				}
-			}
-		}
-	}
-
-	return keys, values, err
-}
-
-func (s *S3Store) Delete(ctx context.Context, key Key) error {
-	_, err := s.client.DeleteObjectWithContext(ctx, &s3.DeleteObjectInput{
-		Bucket: aws.String(bucketName),
-		Key:    aws.String(string(key)),
-	})
-	return err
-}
-
-func (s *S3Store) DeleteByPrefix(ctx context.Context, prefix Key) error {
-
-	objects, err := s.client.ListObjectsWithContext(ctx, &s3.ListObjectsInput{
-		Bucket: aws.String(bucketName),
-		Prefix: aws.String(string(prefix)),
-	})
-
-	if objects != nil && err == nil {
-		for _, object := range objects.Contents {
-			_, err := s.client.DeleteObjectWithContext(ctx, &s3.DeleteObjectInput{
-				Bucket: aws.String(bucketName),
-				Key:    object.Key,
-			})
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (s *S3Store) DeleteRange(ctx context.Context, keyStart Key, keyEnd Key) error {
-
-	objects, err := s.client.ListObjectsWithContext(ctx, &s3.ListObjectsInput{
-		Bucket: aws.String(bucketName),
-		Prefix: aws.String(string(keyStart)),
-	})
-
-	if objects != nil && err == nil {
-		for _, object := range objects.Contents {
-			if *object.Key > string(keyEnd) {
-				_, err := s.client.DeleteObjectWithContext(ctx, &s3.DeleteObjectInput{
-					Bucket: aws.String(bucketName),
-					Key:    object.Key,
-				})
-				return err
-			}
-		}
-	}
-
-	return nil
-}

internal/storage/internal/S3/s3_store.go

@@ -1,339 +0,0 @@
-package s3driver
-
-import (
-	"context"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/endpoints"
-	"github.com/zilliztech/milvus-distributed/internal/storage/internal/minio/codec"
-	. "github.com/zilliztech/milvus-distributed/internal/storage/type"
-)
-
-type S3Driver struct {
-	driver *S3Store
-}
-
-var bucketName string
-
-func NewS3Driver(ctx context.Context, option Option) (*S3Driver, error) {
-	// to-do read conf
-
-	bucketName = option.BucketName
-
-	S3Client, err := NewS3Store(aws.Config{
-		Region: aws.String(endpoints.CnNorthwest1RegionID)})
-
-	if err != nil {
-		return nil, err
-	}
-
-	return &S3Driver{
-		S3Client,
-	}, nil
-}
-
-func (s *S3Driver) put(ctx context.Context, key Key, value Value, timestamp Timestamp, suffix string) error {
-	minioKey, err := codec.MvccEncode(key, timestamp, suffix)
-	if err != nil {
-		return err
-	}
-
-	err = s.driver.Put(ctx, minioKey, value)
-	return err
-}
-
-func (s *S3Driver) scanLE(ctx context.Context, key Key, timestamp Timestamp, keyOnly bool) ([]Timestamp, []Key, []Value, error) {
-	keyEnd, err := codec.MvccEncode(key, timestamp, "")
-	if err != nil {
-		return nil, nil, nil, err
-	}
-
-	keys, values, err := s.driver.Scan(ctx, key, []byte(keyEnd), -1, keyOnly)
-	if err != nil {
-		return nil, nil, nil, err
-	}
-
-	var timestamps []Timestamp
-	for _, key := range keys {
-		_, timestamp, _, _ := codec.MvccDecode(key)
-		timestamps = append(timestamps, timestamp)
-	}
-
-	return timestamps, keys, values, nil
-}
-
-func (s *S3Driver) scanGE(ctx context.Context, key Key, timestamp Timestamp, keyOnly bool) ([]Timestamp, []Key, []Value, error) {
-	keyStart, err := codec.MvccEncode(key, timestamp, "")
-	if err != nil {
-		return nil, nil, nil, err
-	}
-
-	keys, values, err := s.driver.Scan(ctx, key, keyStart, -1, keyOnly)
-	if err != nil {
-		return nil, nil, nil, err
-	}
-
-	var timestamps []Timestamp
-	for _, key := range keys {
-		_, timestamp, _, _ := codec.MvccDecode(key)
-		timestamps = append(timestamps, timestamp)
-	}
-
-	return timestamps, keys, values, nil
-}
-
-//scan(ctx context.Context, key Key, start Timestamp, end Timestamp, withValue bool) ([]Timestamp, []Key, []Value, error)
-func (s *S3Driver) deleteLE(ctx context.Context, key Key, timestamp Timestamp) error {
-	keyEnd, err := codec.MvccEncode(key, timestamp, "delete")
-	if err != nil {
-		return err
-	}
-	err = s.driver.DeleteRange(ctx, key, keyEnd)
-	return err
-}
-func (s *S3Driver) deleteGE(ctx context.Context, key Key, timestamp Timestamp) error {
-	keys, _, err := s.driver.GetByPrefix(ctx, key, true)
-	if err != nil {
-		return err
-	}
-	keyStart, err := codec.MvccEncode(key, timestamp, "")
-	if err != nil {
-		panic(err)
-	}
-	err = s.driver.DeleteRange(ctx, []byte(keyStart), keys[len(keys)-1])
-	return err
-}
-func (s *S3Driver) deleteRange(ctx context.Context, key Key, start Timestamp, end Timestamp) error {
-	keyStart, err := codec.MvccEncode(key, start, "")
-	if err != nil {
-		return err
-	}
-	keyEnd, err := codec.MvccEncode(key, end, "")
-	if err != nil {
-		return err
-	}
-	err = s.driver.DeleteRange(ctx, keyStart, keyEnd)
-	return err
-}
-
-func (s *S3Driver) GetRow(ctx context.Context, key Key, timestamp Timestamp) (Value, error) {
-	minioKey, err := codec.MvccEncode(key, timestamp, "")
-	if err != nil {
-		return nil, err
-	}
-
-	keys, values, err := s.driver.Scan(ctx, append(key, byte('_')), minioKey, 1, false)
-	if values == nil || keys == nil {
-		return nil, err
-	}
-
-	_, _, suffix, err := codec.MvccDecode(keys[0])
-	if err != nil {
-		return nil, err
-	}
-	if suffix == "delete" {
-		return nil, nil
-	}
-
-	return values[0], err
-}
-func (s *S3Driver) GetRows(ctx context.Context, keys []Key, timestamps []Timestamp) ([]Value, error) {
-	var values []Value
-	for i, key := range keys {
-		value, err := s.GetRow(ctx, key, timestamps[i])
-		if err != nil {
-			return nil, err
-		}
-		values = append(values, value)
-	}
-	return values, nil
-}
-
-func (s *S3Driver) PutRow(ctx context.Context, key Key, value Value, segment string, timestamp Timestamp) error {
-	minioKey, err := codec.MvccEncode(key, timestamp, segment)
-	if err != nil {
-		return err
-	}
-	err = s.driver.Put(ctx, minioKey, value)
-	return err
-}
-func (s *S3Driver) PutRows(ctx context.Context, keys []Key, values []Value, segments []string, timestamps []Timestamp) error {
-	maxThread := 100
-	batchSize := 1
-	keysLength := len(keys)
-
-	if keysLength/batchSize > maxThread {
-		batchSize = keysLength / maxThread
-	}
-
-	batchNums := keysLength / batchSize
-
-	if keysLength%batchSize != 0 {
-		batchNums = keysLength/batchSize + 1
-	}
-
-	errCh := make(chan error)
-	f := func(ctx2 context.Context, keys2 []Key, values2 []Value, segments2 []string, timestamps2 []Timestamp) {
-		for i := 0; i < len(keys2); i++ {
-			err := s.PutRow(ctx2, keys2[i], values2[i], segments2[i], timestamps2[i])
-			errCh <- err
-		}
-	}
-	for i := 0; i < batchNums; i++ {
-		j := i
-		go func() {
-			start, end := j*batchSize, (j+1)*batchSize
-			if len(keys) < end {
-				end = len(keys)
-			}
-			f(ctx, keys[start:end], values[start:end], segments[start:end], timestamps[start:end])
-		}()
-	}
-
-	for i := 0; i < len(keys); i++ {
-		if err := <-errCh; err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (s *S3Driver) GetSegments(ctx context.Context, key Key, timestamp Timestamp) ([]string, error) {
-	keyEnd, err := codec.MvccEncode(key, timestamp, "")
-	if err != nil {
-		return nil, err
-	}
-	keys, _, err := s.driver.Scan(ctx, append(key, byte('_')), keyEnd, -1, true)
-	if err != nil {
-		return nil, err
-	}
-	segmentsSet := map[string]bool{}
-	for _, key := range keys {
-		_, _, segment, err := codec.MvccDecode(key)
-		if err != nil {
-			panic("must no error")
-		}
-		if segment != "delete" {
-			segmentsSet[segment] = true
-		}
-	}
-
-	var segments []string
-	for k, v := range segmentsSet {
-		if v {
-			segments = append(segments, k)
-		}
-	}
-	return segments, err
-}
-
-func (s *S3Driver) DeleteRow(ctx context.Context, key Key, timestamp Timestamp) error {
-	minioKey, err := codec.MvccEncode(key, timestamp, "delete")
-	if err != nil {
-		return err
-	}
-	value := []byte("0")
-	err = s.driver.Put(ctx, minioKey, value)
-	return err
-}
-
-func (s *S3Driver) DeleteRows(ctx context.Context, keys []Key, timestamps []Timestamp) error {
-	maxThread := 100
-	batchSize := 1
-	keysLength := len(keys)
-
-	if keysLength/batchSize > maxThread {
-		batchSize = keysLength / maxThread
-	}
-
-	batchNums := keysLength / batchSize
-
-	if keysLength%batchSize != 0 {
-		batchNums = keysLength/batchSize + 1
-	}
-
-	errCh := make(chan error)
-	f := func(ctx2 context.Context, keys2 []Key, timestamps2 []Timestamp) {
-		for i := 0; i < len(keys2); i++ {
-			err := s.DeleteRow(ctx2, keys2[i], timestamps2[i])
-			errCh <- err
-		}
-	}
-	for i := 0; i < batchNums; i++ {
-		j := i
-		go func() {
-			start, end := j*batchSize, (j+1)*batchSize
-			if len(keys) < end {
-				end = len(keys)
-			}
-			f(ctx, keys[start:end], timestamps[start:end])
-		}()
-	}
-
-	for i := 0; i < len(keys); i++ {
-		if err := <-errCh; err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (s *S3Driver) PutLog(ctx context.Context, key Key, value Value, timestamp Timestamp, channel int) error {
-	logKey := codec.LogEncode(key, timestamp, channel)
-	err := s.driver.Put(ctx, logKey, value)
-	return err
-}
-
-func (s *S3Driver) GetLog(ctx context.Context, start Timestamp, end Timestamp, channels []int) ([]Value, error) {
-	keys, values, err := s.driver.GetByPrefix(ctx, []byte("log_"), false)
-	if err != nil {
-		return nil, err
-	}
-
-	var resultValues []Value
-	for i, key := range keys {
-		_, ts, channel, err := codec.LogDecode(string(key))
-		if err != nil {
-			return nil, err
-		}
-		if ts >= start && ts <= end {
-			for j := 0; j < len(channels); j++ {
-				if channel == channels[j] {
-					resultValues = append(resultValues, values[i])
-				}
-			}
-		}
-	}
-
-	return resultValues, nil
-}
-
-func (s *S3Driver) GetSegmentIndex(ctx context.Context, segment string) (SegmentIndex, error) {
-
-	return s.driver.Get(ctx, codec.SegmentEncode(segment, "index"))
-}
-
-func (s *S3Driver) PutSegmentIndex(ctx context.Context, segment string, index SegmentIndex) error {
-
-	return s.driver.Put(ctx, codec.SegmentEncode(segment, "index"), index)
-}
-
-func (s *S3Driver) DeleteSegmentIndex(ctx context.Context, segment string) error {
-
-	return s.driver.Delete(ctx, codec.SegmentEncode(segment, "index"))
-}
-
-func (s *S3Driver) GetSegmentDL(ctx context.Context, segment string) (SegmentDL, error) {
-
-	return s.driver.Get(ctx, codec.SegmentEncode(segment, "DL"))
-}
-
-func (s *S3Driver) PutSegmentDL(ctx context.Context, segment string, log SegmentDL) error {
-
-	return s.driver.Put(ctx, codec.SegmentEncode(segment, "DL"), log)
-}
-
-func (s *S3Driver) DeleteSegmentDL(ctx context.Context, segment string) error {
-
-	return s.driver.Delete(ctx, codec.SegmentEncode(segment, "DL"))
-}

internal/storage/internal/minio/codec/codec.go

@@ -1,101 +0,0 @@
-package codec
-
-import (
-	"errors"
-	"fmt"
-)
-
-func MvccEncode(key []byte, ts uint64, suffix string) ([]byte, error) {
-	return []byte(string(key) + "_" + fmt.Sprintf("%016x", ^ts) + "_" + suffix), nil
-}
-
-func MvccDecode(key []byte) (string, uint64, string, error) {
-	if len(key) < 16 {
-		return "", 0, "", errors.New("insufficient bytes to decode value")
-	}
-
-	suffixIndex := 0
-	TSIndex := 0
-	undersCount := 0
-	for i := len(key) - 1; i > 0; i-- {
-		if key[i] == byte('_') {
-			undersCount++
-			if undersCount == 1 {
-				suffixIndex = i + 1
-			}
-			if undersCount == 2 {
-				TSIndex = i + 1
-				break
-			}
-		}
-	}
-	if suffixIndex == 0 || TSIndex == 0 {
-		return "", 0, "", errors.New("key is wrong formatted")
-	}
-
-	var TS uint64
-	_, err := fmt.Sscanf(string(key[TSIndex:suffixIndex-1]), "%x", &TS)
-	TS = ^TS
-	if err != nil {
-		return "", 0, "", err
-	}
-
-	return string(key[0 : TSIndex-1]), TS, string(key[suffixIndex:]), nil
-}
-
-func LogEncode(key []byte, ts uint64, channel int) []byte {
-	suffix := string(key) + "_" + fmt.Sprintf("%d", channel)
-	logKey, err := MvccEncode([]byte("log"), ts, suffix)
-	if err != nil {
-		return nil
-	}
-	return logKey
-}
-
-func LogDecode(logKey string) (string, uint64, int, error) {
-	if len(logKey) < 16 {
-		return "", 0, 0, errors.New("insufficient bytes to decode value")
-	}
-
-	channelIndex := 0
-	keyIndex := 0
-	TSIndex := 0
-	undersCount := 0
-
-	for i := len(logKey) - 1; i > 0; i-- {
-		if logKey[i] == '_' {
-			undersCount++
-			if undersCount == 1 {
-				channelIndex = i + 1
-			}
-			if undersCount == 2 {
-				keyIndex = i + 1
-			}
-			if undersCount == 3 {
-				TSIndex = i + 1
-				break
-			}
-		}
-	}
-	if channelIndex == 0 || TSIndex == 0 || keyIndex == 0 || logKey[:TSIndex-1] != "log" {
-		return "", 0, 0, errors.New("key is wrong formatted")
-	}
-
-	var TS uint64
-	var channel int
-	_, err := fmt.Sscanf(logKey[TSIndex:keyIndex-1], "%x", &TS)
-	if err != nil {
-		return "", 0, 0, err
-	}
-	TS = ^TS
-
-	_, err = fmt.Sscanf(logKey[channelIndex:], "%d", &channel)
-	if err != nil {
-		return "", 0, 0, err
-	}
-	return logKey[keyIndex : channelIndex-1], TS, channel, nil
-}
-
-func SegmentEncode(segment string, suffix string) []byte {
-	return []byte(segment + "_" + suffix)
-}

internal/storage/internal/minio/minio_store.go

@@ -1,361 +0,0 @@
-package miniodriver
-
-import (
-	"context"
-
-	"github.com/minio/minio-go/v7"
-	"github.com/minio/minio-go/v7/pkg/credentials"
-	"github.com/zilliztech/milvus-distributed/internal/storage/internal/minio/codec"
-	storageType "github.com/zilliztech/milvus-distributed/internal/storage/type"
-)
-
-type MinioDriver struct {
-	driver *minioStore
-}
-
-var bucketName string
-
-func NewMinioDriver(ctx context.Context, option storageType.Option) (*MinioDriver, error) {
-	// to-do read conf
-	var endPoint = "localhost:9000"
-	var accessKeyID = "testminio"
-	var secretAccessKey = "testminio"
-	var useSSL = false
-
-	bucketName := option.BucketName
-
-	minioClient, err := minio.New(endPoint, &minio.Options{
-		Creds:  credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
-		Secure: useSSL,
-	})
-
-	if err != nil {
-		return nil, err
-	}
-
-	bucketExists, err := minioClient.BucketExists(ctx, bucketName)
-	if err != nil {
-		return nil, err
-	}
-
-	if !bucketExists {
-		err = minioClient.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{})
-		if err != nil {
-			return nil, err
-		}
-	}
-	return &MinioDriver{
-		&minioStore{
-			client: minioClient,
-		},
-	}, nil
-}
-
-func (s *MinioDriver) put(ctx context.Context, key storageType.Key, value storageType.Value, timestamp storageType.Timestamp, suffix string) error {
-	minioKey, err := codec.MvccEncode(key, timestamp, suffix)
-	if err != nil {
-		return err
-	}
-
-	err = s.driver.Put(ctx, minioKey, value)
-	return err
-}
-
-func (s *MinioDriver) scanLE(ctx context.Context, key storageType.Key, timestamp storageType.Timestamp, keyOnly bool) ([]storageType.Timestamp, []storageType.Key, []storageType.Value, error) {
-	keyEnd, err := codec.MvccEncode(key, timestamp, "")
-	if err != nil {
-		return nil, nil, nil, err
-	}
-
-	keys, values, err := s.driver.Scan(ctx, key, []byte(keyEnd), -1, keyOnly)
-	if err != nil {
-		return nil, nil, nil, err
-	}
-
-	var timestamps []storageType.Timestamp
-	for _, key := range keys {
-		_, timestamp, _, _ := codec.MvccDecode(key)
-		timestamps = append(timestamps, timestamp)
-	}
-
-	return timestamps, keys, values, nil
-}
-
-func (s *MinioDriver) scanGE(ctx context.Context, key storageType.Key, timestamp storageType.Timestamp, keyOnly bool) ([]storageType.Timestamp, []storageType.Key, []storageType.Value, error) {
-	keyStart, err := codec.MvccEncode(key, timestamp, "")
-	if err != nil {
-		return nil, nil, nil, err
-	}
-
-	keys, values, err := s.driver.Scan(ctx, key, keyStart, -1, keyOnly)
-	if err != nil {
-		return nil, nil, nil, err
-	}
-
-	var timestamps []storageType.Timestamp
-	for _, key := range keys {
-		_, timestamp, _, _ := codec.MvccDecode(key)
-		timestamps = append(timestamps, timestamp)
-	}
-
-	return timestamps, keys, values, nil
-}
-
-//scan(ctx context.Context, key storageType.Key, start storageType.Timestamp, end storageType.Timestamp, withValue bool) ([]storageType.Timestamp, []storageType.Key, []storageType.Value, error)
-func (s *MinioDriver) deleteLE(ctx context.Context, key storageType.Key, timestamp storageType.Timestamp) error {
-	keyEnd, err := codec.MvccEncode(key, timestamp, "delete")
-	if err != nil {
-		return err
-	}
-	err = s.driver.DeleteRange(ctx, key, keyEnd)
-	return err
-}
-func (s *MinioDriver) deleteGE(ctx context.Context, key storageType.Key, timestamp storageType.Timestamp) error {
-	keys, _, err := s.driver.GetByPrefix(ctx, key, true)
-	if err != nil {
-		return err
-	}
-	keyStart, err := codec.MvccEncode(key, timestamp, "")
-	if err != nil {
-		panic(err)
-	}
-	err = s.driver.DeleteRange(ctx, keyStart, keys[len(keys)-1])
-	if err != nil {
-		panic(err)
-	}
-	return nil
-}
-func (s *MinioDriver) deleteRange(ctx context.Context, key storageType.Key, start storageType.Timestamp, end storageType.Timestamp) error {
-	keyStart, err := codec.MvccEncode(key, start, "")
-	if err != nil {
-		return err
-	}
-	keyEnd, err := codec.MvccEncode(key, end, "")
-	if err != nil {
-		return err
-	}
-	err = s.driver.DeleteRange(ctx, keyStart, keyEnd)
-	return err
-}
-
-func (s *MinioDriver) GetRow(ctx context.Context, key storageType.Key, timestamp storageType.Timestamp) (storageType.Value, error) {
-	minioKey, err := codec.MvccEncode(key, timestamp, "")
-	if err != nil {
-		return nil, err
-	}
-
-	keys, values, err := s.driver.Scan(ctx, append(key, byte('_')), minioKey, 1, false)
-	if values == nil || keys == nil {
-		return nil, err
-	}
-
-	_, _, suffix, err := codec.MvccDecode(keys[0])
-	if err != nil {
-		return nil, err
-	}
-	if suffix == "delete" {
-		return nil, nil
-	}
-
-	return values[0], err
-}
-func (s *MinioDriver) GetRows(ctx context.Context, keys []storageType.Key, timestamps []storageType.Timestamp) ([]storageType.Value, error) {
-	var values []storageType.Value
-	for i, key := range keys {
-		value, err := s.GetRow(ctx, key, timestamps[i])
-		if err != nil {
-			return nil, err
-		}
-		values = append(values, value)
-	}
-	return values, nil
-}
-
-func (s *MinioDriver) PutRow(ctx context.Context, key storageType.Key, value storageType.Value, segment string, timestamp storageType.Timestamp) error {
-	minioKey, err := codec.MvccEncode(key, timestamp, segment)
-	if err != nil {
-		return err
-	}
-	err = s.driver.Put(ctx, minioKey, value)
-	return err
-}
-func (s *MinioDriver) PutRows(ctx context.Context, keys []storageType.Key, values []storageType.Value, segments []string, timestamps []storageType.Timestamp) error {
-	maxThread := 100
-	batchSize := 1
-	keysLength := len(keys)
-
-	if keysLength/batchSize > maxThread {
-		batchSize = keysLength / maxThread
-	}
-
-	batchNums := keysLength / batchSize
-
-	if keysLength%batchSize != 0 {
-		batchNums = keysLength/batchSize + 1
-	}
-
-	errCh := make(chan error)
-	f := func(ctx2 context.Context, keys2 []storageType.Key, values2 []storageType.Value, segments2 []string, timestamps2 []storageType.Timestamp) {
-		for i := 0; i < len(keys2); i++ {
-			err := s.PutRow(ctx2, keys2[i], values2[i], segments2[i], timestamps2[i])
-			errCh <- err
-		}
-	}
-	for i := 0; i < batchNums; i++ {
-		j := i
-		go func() {
-			start, end := j*batchSize, (j+1)*batchSize
-			if len(keys) < end {
-				end = len(keys)
-			}
-			f(ctx, keys[start:end], values[start:end], segments[start:end], timestamps[start:end])
-		}()
-	}
-
-	for i := 0; i < len(keys); i++ {
-		if err := <-errCh; err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (s *MinioDriver) GetSegments(ctx context.Context, key storageType.Key, timestamp storageType.Timestamp) ([]string, error) {
-	keyEnd, err := codec.MvccEncode(key, timestamp, "")
-	if err != nil {
-		return nil, err
-	}
-	keys, _, err := s.driver.Scan(ctx, append(key, byte('_')), keyEnd, -1, true)
-	if err != nil {
-		return nil, err
-	}
-	segmentsSet := map[string]bool{}
-	for _, key := range keys {
-		_, _, segment, err := codec.MvccDecode(key)
-		if err != nil {
-			panic("must no error")
-		}
-		if segment != "delete" {
-			segmentsSet[segment] = true
-		}
-	}
-
-	var segments []string
-	for k, v := range segmentsSet {
-		if v {
-			segments = append(segments, k)
-		}
-	}
-	return segments, err
-}
-
-func (s *MinioDriver) DeleteRow(ctx context.Context, key storageType.Key, timestamp storageType.Timestamp) error {
-	minioKey, err := codec.MvccEncode(key, timestamp, "delete")
-	if err != nil {
-		return err
-	}
-	value := []byte("0")
-	err = s.driver.Put(ctx, minioKey, value)
-	return err
-}
-
-func (s *MinioDriver) DeleteRows(ctx context.Context, keys []storageType.Key, timestamps []storageType.Timestamp) error {
-	maxThread := 100
-	batchSize := 1
-	keysLength := len(keys)
-
-	if keysLength/batchSize > maxThread {
-		batchSize = keysLength / maxThread
-	}
-
-	batchNums := keysLength / batchSize
-
-	if keysLength%batchSize != 0 {
-		batchNums = keysLength/batchSize + 1
-	}
-
-	errCh := make(chan error)
-	f := func(ctx2 context.Context, keys2 []storageType.Key, timestamps2 []storageType.Timestamp) {
-		for i := 0; i < len(keys2); i++ {
-			err := s.DeleteRow(ctx2, keys2[i], timestamps2[i])
-			errCh <- err
-		}
-	}
-	for i := 0; i < batchNums; i++ {
-		j := i
-		go func() {
-			start, end := j*batchSize, (j+1)*batchSize
-			if len(keys) < end {
-				end = len(keys)
-			}
-			f(ctx, keys[start:end], timestamps[start:end])
-		}()
-	}
-
-	for i := 0; i < len(keys); i++ {
-		if err := <-errCh; err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (s *MinioDriver) PutLog(ctx context.Context, key storageType.Key, value storageType.Value, timestamp storageType.Timestamp, channel int) error {
-	logKey := codec.LogEncode(key, timestamp, channel)
-	err := s.driver.Put(ctx, logKey, value)
-	return err
-}
-
-func (s *MinioDriver) GetLog(ctx context.Context, start storageType.Timestamp, end storageType.Timestamp, channels []int) ([]storageType.Value, error) {
-	keys, values, err := s.driver.GetByPrefix(ctx, []byte("log_"), false)
-	if err != nil {
-		return nil, err
-	}
-
-	var resultValues []storageType.Value
-	for i, key := range keys {
-		_, ts, channel, err := codec.LogDecode(string(key))
-		if err != nil {
-			return nil, err
-		}
-		if ts >= start && ts <= end {
-			for j := 0; j < len(channels); j++ {
-				if channel == channels[j] {
-					resultValues = append(resultValues, values[i])
-				}
-			}
-		}
-	}
-
-	return resultValues, nil
-}
-
-func (s *MinioDriver) GetSegmentIndex(ctx context.Context, segment string) (storageType.SegmentIndex, error) {
-
-	return s.driver.Get(ctx, codec.SegmentEncode(segment, "index"))
-}
-
-func (s *MinioDriver) PutSegmentIndex(ctx context.Context, segment string, index storageType.SegmentIndex) error {
-
-	return s.driver.Put(ctx, codec.SegmentEncode(segment, "index"), index)
-}
-
-func (s *MinioDriver) DeleteSegmentIndex(ctx context.Context, segment string) error {
-
-	return s.driver.Delete(ctx, codec.SegmentEncode(segment, "index"))
-}
-
-func (s *MinioDriver) GetSegmentDL(ctx context.Context, segment string) (storageType.SegmentDL, error) {
-
-	return s.driver.Get(ctx, codec.SegmentEncode(segment, "DL"))
-}
-
-func (s *MinioDriver) PutSegmentDL(ctx context.Context, segment string, log storageType.SegmentDL) error {
-
-	return s.driver.Put(ctx, codec.SegmentEncode(segment, "DL"), log)
-}
-
-func (s *MinioDriver) DeleteSegmentDL(ctx context.Context, segment string) error {
-
-	return s.driver.Delete(ctx, codec.SegmentEncode(segment, "DL"))
-}

internal/storage/internal/minio/minio_storeEngine.go

@@ -1,130 +0,0 @@
-package miniodriver
-
-import (
-	"bytes"
-	"context"
-	"io"
-
-	"github.com/minio/minio-go/v7"
-	. "github.com/zilliztech/milvus-distributed/internal/storage/type"
-)
-
-type minioStore struct {
-	client *minio.Client
-}
-
-func (s *minioStore) Put(ctx context.Context, key Key, value Value) error {
-	reader := bytes.NewReader(value)
-	_, err := s.client.PutObject(ctx, bucketName, string(key), reader, int64(len(value)), minio.PutObjectOptions{})
-
-	if err != nil {
-		return err
-	}
-
-	return err
-}
-
-func (s *minioStore) Get(ctx context.Context, key Key) (Value, error) {
-	object, err := s.client.GetObject(ctx, bucketName, string(key), minio.GetObjectOptions{})
-	if err != nil {
-		return nil, err
-	}
-
-	size := 256 * 1024
-	buf := make([]byte, size)
-	n, err := object.Read(buf)
-	if err != nil && err != io.EOF {
-		return nil, err
-	}
-	return buf[:n], nil
-}
-
-func (s *minioStore) GetByPrefix(ctx context.Context, prefix Key, keyOnly bool) ([]Key, []Value, error) {
-	objects := s.client.ListObjects(ctx, bucketName, minio.ListObjectsOptions{Prefix: string(prefix)})
-
-	var objectsKeys []Key
-	var objectsValues []Value
-
-	for object := range objects {
-		objectsKeys = append(objectsKeys, []byte(object.Key))
-		if !keyOnly {
-			value, err := s.Get(ctx, []byte(object.Key))
-			if err != nil {
-				return nil, nil, err
-			}
-			objectsValues = append(objectsValues, value)
-		}
-	}
-
-	return objectsKeys, objectsValues, nil
-
-}
-
-func (s *minioStore) Scan(ctx context.Context, keyStart Key, keyEnd Key, limit int, keyOnly bool) ([]Key, []Value, error) {
-	var keys []Key
-	var values []Value
-	limitCount := uint(limit)
-	for object := range s.client.ListObjects(ctx, bucketName, minio.ListObjectsOptions{Prefix: string(keyStart)}) {
-		if object.Key >= string(keyEnd) {
-			keys = append(keys, []byte(object.Key))
-			if !keyOnly {
-				value, err := s.Get(ctx, []byte(object.Key))
-				if err != nil {
-					return nil, nil, err
-				}
-				values = append(values, value)
-			}
-			limitCount--
-			if limitCount <= 0 {
-				break
-			}
-		}
-	}
-
-	return keys, values, nil
-}
-
-func (s *minioStore) Delete(ctx context.Context, key Key) error {
-	err := s.client.RemoveObject(ctx, bucketName, string(key), minio.RemoveObjectOptions{})
-	return err
-}
-
-func (s *minioStore) DeleteByPrefix(ctx context.Context, prefix Key) error {
-	objectsCh := make(chan minio.ObjectInfo)
-
-	go func() {
-		defer close(objectsCh)
-
-		for object := range s.client.ListObjects(ctx, bucketName, minio.ListObjectsOptions{Prefix: string(prefix)}) {
-			objectsCh <- object
-		}
-	}()
-
-	for rErr := range s.client.RemoveObjects(ctx, bucketName, objectsCh, minio.RemoveObjectsOptions{GovernanceBypass: true}) {
-		if rErr.Err != nil {
-			return rErr.Err
-		}
-	}
-	return nil
-}
-
-func (s *minioStore) DeleteRange(ctx context.Context, keyStart Key, keyEnd Key) error {
-	objectsCh := make(chan minio.ObjectInfo)
-
-	go func() {
-		defer close(objectsCh)
-
-		for object := range s.client.ListObjects(ctx, bucketName, minio.ListObjectsOptions{Prefix: string(keyStart)}) {
-			if object.Key <= string(keyEnd) {
-				objectsCh <- object
-			}
-		}
-	}()
-
-	for rErr := range s.client.RemoveObjects(ctx, bucketName, objectsCh, minio.RemoveObjectsOptions{GovernanceBypass: true}) {
-		if rErr.Err != nil {
-			return rErr.Err
-		}
-	}
-	return nil
-}

internal/storage/internal/minio/minio_test.go

@@ -1,134 +0,0 @@
-package miniodriver
-
-import (
-	"context"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	storagetype "github.com/zilliztech/milvus-distributed/internal/storage/type"
-)
-
-var option = storagetype.Option{BucketName: "zilliz-hz"}
-var ctx = context.Background()
-var client, err = NewMinioDriver(ctx, option)
-
-func TestMinioDriver_PutRowAndGetRow(t *testing.T) {
-	err = client.PutRow(ctx, []byte("bar"), []byte("abcdefghijklmnoopqrstuvwxyz"), "SegmentA", 1)
-	assert.Nil(t, err)
-	err = client.PutRow(ctx, []byte("bar"), []byte("djhfkjsbdfbsdughorsgsdjhgoisdgh"), "SegmentA", 2)
-	assert.Nil(t, err)
-	err = client.PutRow(ctx, []byte("bar"), []byte("123854676ershdgfsgdfk,sdhfg;sdi8"), "SegmentB", 3)
-	assert.Nil(t, err)
-	err = client.PutRow(ctx, []byte("bar1"), []byte("testkeybarorbar_1"), "SegmentC", 3)
-	assert.Nil(t, err)
-	object, _ := client.GetRow(ctx, []byte("bar"), 5)
-	assert.Equal(t, "abcdefghijklmnoopqrstuvwxyz", string(object))
-	object, _ = client.GetRow(ctx, []byte("bar"), 2)
-	assert.Equal(t, "djhfkjsbdfbsdughorsgsdjhgoisdgh", string(object))
-	object, _ = client.GetRow(ctx, []byte("bar"), 5)
-	assert.Equal(t, "123854676ershdgfsgdfk,sdhfg;sdi8", string(object))
-	object, _ = client.GetRow(ctx, []byte("bar1"), 5)
-	assert.Equal(t, "testkeybarorbar_1", string(object))
-}
-
-func TestMinioDriver_DeleteRow(t *testing.T) {
-	err = client.DeleteRow(ctx, []byte("bar"), 5)
-	assert.Nil(t, err)
-	object, _ := client.GetRow(ctx, []byte("bar"), 6)
-	assert.Nil(t, object)
-	err = client.DeleteRow(ctx, []byte("bar1"), 5)
-	assert.Nil(t, err)
-	object2, _ := client.GetRow(ctx, []byte("bar1"), 6)
-	assert.Nil(t, object2)
-}
-
-func TestMinioDriver_GetSegments(t *testing.T) {
-	err = client.PutRow(ctx, []byte("seg"), []byte("abcdefghijklmnoopqrstuvwxyz"), "SegmentA", 1)
-	assert.Nil(t, err)
-	err = client.PutRow(ctx, []byte("seg"), []byte("djhfkjsbdfbsdughorsgsdjhgoisdgh"), "SegmentA", 2)
-	assert.Nil(t, err)
-	err = client.PutRow(ctx, []byte("seg"), []byte("123854676ershdgfsgdfk,sdhfg;sdi8"), "SegmentB", 3)
-	assert.Nil(t, err)
-	err = client.PutRow(ctx, []byte("seg2"), []byte("testkeybarorbar_1"), "SegmentC", 1)
-	assert.Nil(t, err)
-
-	segements, err := client.GetSegments(ctx, []byte("seg"), 4)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(segements))
-	if segements[0] == "SegmentA" {
-		assert.Equal(t, "SegmentA", segements[0])
-		assert.Equal(t, "SegmentB", segements[1])
-	} else {
-		assert.Equal(t, "SegmentB", segements[0])
-		assert.Equal(t, "SegmentA", segements[1])
-	}
-}
-
-func TestMinioDriver_PutRowsAndGetRows(t *testing.T) {
-	keys := [][]byte{[]byte("foo"), []byte("bar")}
-	values := [][]byte{[]byte("The key is foo!"), []byte("The key is bar!")}
-	segments := []string{"segmentA", "segmentB"}
-	timestamps := []uint64{1, 2}
-	err = client.PutRows(ctx, keys, values, segments, timestamps)
-	assert.Nil(t, err)
-
-	objects, err := client.GetRows(ctx, keys, timestamps)
-	assert.Nil(t, err)
-	assert.Equal(t, "The key is foo!", string(objects[0]))
-	assert.Equal(t, "The key is bar!", string(objects[1]))
-}
-
-func TestMinioDriver_DeleteRows(t *testing.T) {
-	keys := [][]byte{[]byte("foo"), []byte("bar")}
-	timestamps := []uint64{3, 3}
-	err := client.DeleteRows(ctx, keys, timestamps)
-	assert.Nil(t, err)
-
-	objects, err := client.GetRows(ctx, keys, timestamps)
-	assert.Nil(t, err)
-	assert.Nil(t, objects[0])
-	assert.Nil(t, objects[1])
-}
-
-func TestMinioDriver_PutLogAndGetLog(t *testing.T) {
-	err = client.PutLog(ctx, []byte("insert"), []byte("This is insert log!"), 1, 11)
-	assert.Nil(t, err)
-	err = client.PutLog(ctx, []byte("delete"), []byte("This is delete log!"), 2, 10)
-	assert.Nil(t, err)
-	err = client.PutLog(ctx, []byte("update"), []byte("This is update log!"), 3, 9)
-	assert.Nil(t, err)
-	err = client.PutLog(ctx, []byte("select"), []byte("This is select log!"), 4, 8)
-	assert.Nil(t, err)
-
-	channels := []int{5, 8, 9, 10, 11, 12, 13}
-	logValues, err := client.GetLog(ctx, 0, 5, channels)
-	assert.Nil(t, err)
-	assert.Equal(t, "This is select log!", string(logValues[0]))
-	assert.Equal(t, "This is update log!", string(logValues[1]))
-	assert.Equal(t, "This is delete log!", string(logValues[2]))
-	assert.Equal(t, "This is insert log!", string(logValues[3]))
-}
-
-func TestMinioDriver_Segment(t *testing.T) {
-	err := client.PutSegmentIndex(ctx, "segmentA", []byte("This is segmentA's index!"))
-	assert.Nil(t, err)
-
-	segmentIndex, err := client.GetSegmentIndex(ctx, "segmentA")
-	assert.Equal(t, "This is segmentA's index!", string(segmentIndex))
-	assert.Nil(t, err)
-
-	err = client.DeleteSegmentIndex(ctx, "segmentA")
-	assert.Nil(t, err)
-}
-
-func TestMinioDriver_SegmentDL(t *testing.T) {
-	err := client.PutSegmentDL(ctx, "segmentB", []byte("This is segmentB's delete log!"))
-	assert.Nil(t, err)
-
-	segmentDL, err := client.GetSegmentDL(ctx, "segmentB")
-	assert.Nil(t, err)
-	assert.Equal(t, "This is segmentB's delete log!", string(segmentDL))
-
-	err = client.DeleteSegmentDL(ctx, "segmentB")
-	assert.Nil(t, err)
-}

internal/storage/internal/tikv/codec/codec.go

@@ -1,62 +0,0 @@
-package codec
-
-import (
-	"encoding/binary"
-	"errors"
-
-	"github.com/tikv/client-go/codec"
-)
-
-var (
-	Delimiter        = byte('_')
-	DelimiterPlusOne = Delimiter + 0x01
-	DeleteMark       = byte('d')
-	SegmentIndexMark = byte('i')
-	SegmentDLMark    = byte('d')
-)
-
-// EncodeKey append timestamp, delimiter, and suffix string
-// to one slice key.
-// Note: suffix string should not contains Delimiter
-func EncodeKey(key []byte, timestamp uint64, suffix string) []byte {
-	//TODO: should we encode key to memory comparable
-	ret := EncodeDelimiter(key, Delimiter)
-	ret = codec.EncodeUintDesc(ret, timestamp)
-	return append(ret, suffix...)
-}
-
-func DecodeKey(key []byte) ([]byte, uint64, string, error) {
-	if len(key) < 8 {
-		return nil, 0, "", errors.New("insufficient bytes to decode value")
-	}
-
-	lenDeKey := 0
-	for i := len(key) - 1; i > 0; i-- {
-		if key[i] == Delimiter {
-			lenDeKey = i
-			break
-		}
-	}
-
-	if lenDeKey == 0 || lenDeKey+8 > len(key) {
-		return nil, 0, "", errors.New("insufficient bytes to decode value")
-	}
-
-	tsBytes := key[lenDeKey+1 : lenDeKey+9]
-	ts := binary.BigEndian.Uint64(tsBytes)
-	suffix := string(key[lenDeKey+9:])
-	key = key[:lenDeKey-1]
-	return key, ^ts, suffix, nil
-}
-
-// EncodeDelimiter append a delimiter byte to slice b, and return the appended slice.
-func EncodeDelimiter(b []byte, delimiter byte) []byte {
-	return append(b, delimiter)
-}
-
-func EncodeSegment(segName []byte, segType byte) []byte {
-	segmentKey := []byte("segment")
-	segmentKey = append(segmentKey, Delimiter)
-	segmentKey = append(segmentKey, segName...)
-	return append(segmentKey, Delimiter, segType)
-}

internal/storage/internal/tikv/tikv_store.go

@@ -1,389 +0,0 @@
-package tikvdriver
-
-import (
-	"context"
-	"errors"
-	"strconv"
-	"strings"
-
-	"github.com/tikv/client-go/config"
-	"github.com/tikv/client-go/rawkv"
-	. "github.com/zilliztech/milvus-distributed/internal/storage/internal/tikv/codec"
-	. "github.com/zilliztech/milvus-distributed/internal/storage/type"
-	storagetype "github.com/zilliztech/milvus-distributed/internal/storage/type"
-)
-
-func keyAddOne(key Key) Key {
-	if key == nil {
-		return nil
-	}
-	lenKey := len(key)
-	ret := make(Key, lenKey)
-	copy(ret, key)
-	ret[lenKey-1] += 0x01
-	return ret
-}
-
-type tikvEngine struct {
-	client *rawkv.Client
-	conf   config.Config
-}
-
-func (e tikvEngine) Put(ctx context.Context, key Key, value Value) error {
-	return e.client.Put(ctx, key, value)
-}
-
-func (e tikvEngine) BatchPut(ctx context.Context, keys []Key, values []Value) error {
-	return e.client.BatchPut(ctx, keys, values)
-}
-
-func (e tikvEngine) Get(ctx context.Context, key Key) (Value, error) {
-	return e.client.Get(ctx, key)
-}
-
-func (e tikvEngine) GetByPrefix(ctx context.Context, prefix Key, keyOnly bool) (keys []Key, values []Value, err error) {
-	startKey := prefix
-	endKey := keyAddOne(prefix)
-	limit := e.conf.Raw.MaxScanLimit
-	for {
-		ks, vs, err := e.Scan(ctx, startKey, endKey, limit, keyOnly)
-		if err != nil {
-			return keys, values, err
-		}
-		keys = append(keys, ks...)
-		values = append(values, vs...)
-		if len(ks) < limit {
-			break
-		}
-		// update the start key, and exclude the start key
-		startKey = append(ks[len(ks)-1], '\000')
-	}
-	return
-}
-
-func (e tikvEngine) Scan(ctx context.Context, startKey Key, endKey Key, limit int, keyOnly bool) ([]Key, []Value, error) {
-	return e.client.Scan(ctx, startKey, endKey, limit, rawkv.ScanOption{KeyOnly: keyOnly})
-}
-
-func (e tikvEngine) Delete(ctx context.Context, key Key) error {
-	return e.client.Delete(ctx, key)
-}
-
-func (e tikvEngine) DeleteByPrefix(ctx context.Context, prefix Key) error {
-	startKey := prefix
-	endKey := keyAddOne(prefix)
-	return e.client.DeleteRange(ctx, startKey, endKey)
-}
-
-func (e tikvEngine) DeleteRange(ctx context.Context, startKey Key, endKey Key) error {
-	return e.client.DeleteRange(ctx, startKey, endKey)
-}
-
-func (e tikvEngine) Close() error {
-	return e.client.Close()
-}
-
-type TikvStore struct {
-	engine *tikvEngine
-}
-
-func NewTikvStore(ctx context.Context, option storagetype.Option) (*TikvStore, error) {
-
-	conf := config.Default()
-	client, err := rawkv.NewClient(ctx, []string{option.TikvAddress}, conf)
-	if err != nil {
-		return nil, err
-	}
-	return &TikvStore{
-		&tikvEngine{
-			client: client,
-			conf:   conf,
-		},
-	}, nil
-}
-
-func (s *TikvStore) Name() string {
-	return "TiKV storage"
-}
-
-func (s *TikvStore) put(ctx context.Context, key Key, value Value, timestamp Timestamp, suffix string) error {
-	return s.engine.Put(ctx, EncodeKey(key, timestamp, suffix), value)
-}
-
-func (s *TikvStore) scanLE(ctx context.Context, key Key, timestamp Timestamp, keyOnly bool) ([]Timestamp, []Key, []Value, error) {
-	panic("implement me")
-}
-
-func (s *TikvStore) scanGE(ctx context.Context, key Key, timestamp Timestamp, keyOnly bool) ([]Timestamp, []Key, []Value, error) {
-	panic("implement me")
-}
-
-func (s *TikvStore) scan(ctx context.Context, key Key, start Timestamp, end Timestamp, keyOnly bool) ([]Timestamp, []Key, []Value, error) {
-	//startKey := EncodeKey(key, start, "")
-	//endKey := EncodeKey(EncodeDelimiter(key, DelimiterPlusOne), end, "")
-	//return s.engine.Scan(ctx, startKey, endKey, -1, keyOnly)
-	panic("implement me")
-}
-
-func (s *TikvStore) deleteLE(ctx context.Context, key Key, timestamp Timestamp) error {
-	panic("implement me")
-}
-
-func (s *TikvStore) deleteGE(ctx context.Context, key Key, timestamp Timestamp) error {
-	panic("implement me")
-}
-
-func (s *TikvStore) deleteRange(ctx context.Context, key Key, start Timestamp, end Timestamp) error {
-	panic("implement me")
-}
-
-func (s *TikvStore) GetRow(ctx context.Context, key Key, timestamp Timestamp) (Value, error) {
-	startKey := EncodeKey(key, timestamp, "")
-	endKey := EncodeDelimiter(key, DelimiterPlusOne)
-	keys, values, err := s.engine.Scan(ctx, startKey, endKey, 1, false)
-	if err != nil || keys == nil {
-		return nil, err
-	}
-	_, _, suffix, err := DecodeKey(keys[0])
-	if err != nil {
-		return nil, err
-	}
-	// key is marked deleted
-	if suffix == string(DeleteMark) {
-		return nil, nil
-	}
-	return values[0], nil
-}
-
-// TODO: how to spilt keys to some batches
-var batchSize = 100
-
-type kvPair struct {
-	key   Key
-	value Value
-	err   error
-}
-
-func batchKeys(keys []Key) [][]Key {
-	keysLen := len(keys)
-	numBatch := (keysLen-1)/batchSize + 1
-	batches := make([][]Key, numBatch)
-
-	for i := 0; i < numBatch; i++ {
-		batchStart := i * batchSize
-		batchEnd := batchStart + batchSize
-		// the last batch
-		if i == numBatch-1 {
-			batchEnd = keysLen
-		}
-		batches[i] = keys[batchStart:batchEnd]
-	}
-	return batches
-}
-
-func (s *TikvStore) GetRows(ctx context.Context, keys []Key, timestamps []Timestamp) ([]Value, error) {
-	if len(keys) != len(timestamps) {
-		return nil, errors.New("the len of keys is not equal to the len of timestamps")
-	}
-
-	batches := batchKeys(keys)
-	ch := make(chan kvPair, len(keys))
-	ctx, cancel := context.WithCancel(ctx)
-	defer cancel()
-
-	for n, b := range batches {
-		batch := b
-		numBatch := n
-		go func() {
-			for i, key := range batch {
-				select {
-				case <-ctx.Done():
-					return
-				default:
-					v, err := s.GetRow(ctx, key, timestamps[numBatch*batchSize+i])
-					ch <- kvPair{
-						key:   key,
-						value: v,
-						err:   err,
-					}
-				}
-			}
-		}()
-	}
-
-	var err error
-	var values []Value
-	kvMap := make(map[string]Value)
-	for i := 0; i < len(keys); i++ {
-		kv := <-ch
-		if kv.err != nil {
-			cancel()
-			if err == nil {
-				err = kv.err
-			}
-		}
-		kvMap[string(kv.key)] = kv.value
-	}
-	for _, key := range keys {
-		values = append(values, kvMap[string(key)])
-	}
-	return values, err
-}
-
-func (s *TikvStore) PutRow(ctx context.Context, key Key, value Value, segment string, timestamp Timestamp) error {
-	return s.put(ctx, key, value, timestamp, segment)
-}
-
-func (s *TikvStore) PutRows(ctx context.Context, keys []Key, values []Value, segments []string, timestamps []Timestamp) error {
-	if len(keys) != len(values) {
-		return errors.New("the len of keys is not equal to the len of values")
-	}
-	if len(keys) != len(timestamps) {
-		return errors.New("the len of keys is not equal to the len of timestamps")
-	}
-
-	encodedKeys := make([]Key, len(keys))
-	for i, key := range keys {
-		encodedKeys[i] = EncodeKey(key, timestamps[i], segments[i])
-	}
-	return s.engine.BatchPut(ctx, encodedKeys, values)
-}
-
-func (s *TikvStore) DeleteRow(ctx context.Context, key Key, timestamp Timestamp) error {
-	return s.put(ctx, key, Value{0x00}, timestamp, string(DeleteMark))
-}
-
-func (s *TikvStore) DeleteRows(ctx context.Context, keys []Key, timestamps []Timestamp) error {
-	encodeKeys := make([]Key, len(keys))
-	values := make([]Value, len(keys))
-	for i, key := range keys {
-		encodeKeys[i] = EncodeKey(key, timestamps[i], string(DeleteMark))
-		values[i] = Value{0x00}
-	}
-	return s.engine.BatchPut(ctx, encodeKeys, values)
-}
-
-//func (s *TikvStore) DeleteRows(ctx context.Context, keys []Key, timestamp Timestamp) error {
-//	batches := batchKeys(keys)
-//	ch := make(chan error, len(batches))
-//	ctx, cancel := context.WithCancel(ctx)
-//
-//	for _, b := range batches {
-//		batch := b
-//		go func() {
-//			for _, key := range batch {
-//				select {
-//				case <-ctx.Done():
-//					return
-//				default:
-//					ch <- s.DeleteRow(ctx, key, timestamp)
-//				}
-//			}
-//		}()
-//	}
-//
-//	var err error
-//	for i := 0; i < len(keys); i++ {
-//		if e := <-ch; e != nil {
-//			cancel()
-//			if err == nil {
-//				err = e
-//			}
-//		}
-//	}
-//	return err
-//}
-
-func (s *TikvStore) PutLog(ctx context.Context, key Key, value Value, timestamp Timestamp, channel int) error {
-	suffix := string(EncodeDelimiter(key, DelimiterPlusOne)) + strconv.Itoa(channel)
-	return s.put(ctx, Key("log"), value, timestamp, suffix)
-}
-
-func (s *TikvStore) GetLog(ctx context.Context, start Timestamp, end Timestamp, channels []int) (logs []Value, err error) {
-	key := Key("log")
-	startKey := EncodeKey(key, end, "")
-	endKey := EncodeKey(key, start, "")
-	// TODO: use for loop to ensure get all keys
-	keys, values, err := s.engine.Scan(ctx, startKey, endKey, s.engine.conf.Raw.MaxScanLimit, false)
-	if err != nil || keys == nil {
-		return nil, err
-	}
-
-	for i, key := range keys {
-		_, _, suffix, err := DecodeKey(key)
-		log := values[i]
-		if err != nil {
-			return logs, err
-		}
-
-		// no channels filter
-		if len(channels) == 0 {
-			logs = append(logs, log)
-		}
-		slice := strings.Split(suffix, string(DelimiterPlusOne))
-		channel, err := strconv.Atoi(slice[len(slice)-1])
-		if err != nil {
-			panic(err)
-		}
-		for _, item := range channels {
-			if item == channel {
-				logs = append(logs, log)
-				break
-			}
-		}
-	}
-	return
-}
-
-func (s *TikvStore) GetSegmentIndex(ctx context.Context, segment string) (SegmentIndex, error) {
-	return s.engine.Get(ctx, EncodeSegment([]byte(segment), SegmentIndexMark))
-}
-
-func (s *TikvStore) PutSegmentIndex(ctx context.Context, segment string, index SegmentIndex) error {
-	return s.engine.Put(ctx, EncodeSegment([]byte(segment), SegmentIndexMark), index)
-}
-
-func (s *TikvStore) DeleteSegmentIndex(ctx context.Context, segment string) error {
-	return s.engine.Delete(ctx, EncodeSegment([]byte(segment), SegmentIndexMark))
-}
-
-func (s *TikvStore) GetSegmentDL(ctx context.Context, segment string) (SegmentDL, error) {
-	return s.engine.Get(ctx, EncodeSegment([]byte(segment), SegmentDLMark))
-}
-
-func (s *TikvStore) PutSegmentDL(ctx context.Context, segment string, log SegmentDL) error {
-	return s.engine.Put(ctx, EncodeSegment([]byte(segment), SegmentDLMark), log)
-}
-
-func (s *TikvStore) DeleteSegmentDL(ctx context.Context, segment string) error {
-	return s.engine.Delete(ctx, EncodeSegment([]byte(segment), SegmentDLMark))
-}
-
-func (s *TikvStore) GetSegments(ctx context.Context, key Key, timestamp Timestamp) ([]string, error) {
-	keys, _, err := s.engine.GetByPrefix(ctx, EncodeDelimiter(key, Delimiter), true)
-	if err != nil {
-		return nil, err
-	}
-	segmentsSet := map[string]bool{}
-	for _, key := range keys {
-		_, ts, segment, err := DecodeKey(key)
-		if err != nil {
-			panic("must no error")
-		}
-		if ts <= timestamp && segment != string(DeleteMark) {
-			segmentsSet[segment] = true
-		}
-	}
-
-	var segments []string
-	for k, v := range segmentsSet {
-		if v {
-			segments = append(segments, k)
-		}
-	}
-	return segments, err
-}
-
-func (s *TikvStore) Close() error {
-	return s.engine.Close()
-}

internal/storage/internal/tikv/tikv_test.go

@@ -1,293 +0,0 @@
-package tikvdriver
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"math"
-	"os"
-	"sort"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	. "github.com/zilliztech/milvus-distributed/internal/storage/internal/tikv/codec"
-	. "github.com/zilliztech/milvus-distributed/internal/storage/type"
-)
-
-//var store TikvStore
-var store *TikvStore
-var option = Option{TikvAddress: "localhost:2379"}
-
-func TestMain(m *testing.M) {
-	store, _ = NewTikvStore(context.Background(), option)
-	exitCode := m.Run()
-	_ = store.Close()
-	os.Exit(exitCode)
-}
-
-func TestTikvEngine_Prefix(t *testing.T) {
-	ctx := context.Background()
-	prefix := Key("key")
-	engine := store.engine
-	value := Value("value")
-
-	// Put some key with same prefix
-	key := prefix
-	err := engine.Put(ctx, key, value)
-	require.Nil(t, err)
-	key = EncodeKey(prefix, 0, "")
-	err = engine.Put(ctx, key, value)
-	assert.Nil(t, err)
-
-	// Get by prefix
-	ks, _, err := engine.GetByPrefix(ctx, prefix, true)
-	assert.Equal(t, 2, len(ks))
-	assert.Nil(t, err)
-
-	// Delete by prefix
-	err = engine.DeleteByPrefix(ctx, prefix)
-	assert.Nil(t, err)
-	ks, _, err = engine.GetByPrefix(ctx, prefix, true)
-	assert.Equal(t, 0, len(ks))
-	assert.Nil(t, err)
-
-	//Test large amount keys
-	num := engine.conf.Raw.MaxScanLimit + 1
-	keys := make([]Key, num)
-	values := make([]Value, num)
-	for i := 0; i < num; i++ {
-		key = EncodeKey(prefix, uint64(i), "")
-		keys[i] = key
-		values[i] = value
-	}
-	err = engine.BatchPut(ctx, keys, values)
-	assert.Nil(t, err)
-
-	ks, _, err = engine.GetByPrefix(ctx, prefix, true)
-	assert.Nil(t, err)
-	assert.Equal(t, num, len(ks))
-	err = engine.DeleteByPrefix(ctx, prefix)
-	assert.Nil(t, err)
-}
-
-func TestTikvStore_Row(t *testing.T) {
-	ctx := context.Background()
-	key := Key("key")
-
-	// Add same row with different timestamp
-	err := store.PutRow(ctx, key, Value("value0"), "segment0", 0)
-	assert.Nil(t, err)
-	err = store.PutRow(ctx, key, Value("value1"), "segment0", 2)
-	assert.Nil(t, err)
-
-	// Get most recent row using key and timestamp
-	v, err := store.GetRow(ctx, key, 3)
-	assert.Nil(t, err)
-	assert.Equal(t, Value("value1"), v)
-	v, err = store.GetRow(ctx, key, 2)
-	assert.Nil(t, err)
-	assert.Equal(t, Value("value1"), v)
-	v, err = store.GetRow(ctx, key, 1)
-	assert.Nil(t, err)
-	assert.Equal(t, Value("value0"), v)
-
-	// Add a different row, but with same prefix
-	key1 := Key("key_y")
-	err = store.PutRow(ctx, key1, Value("valuey"), "segment0", 2)
-	assert.Nil(t, err)
-
-	// Get most recent row using key and timestamp
-	v, err = store.GetRow(ctx, key, 3)
-	assert.Nil(t, err)
-	assert.Equal(t, Value("value1"), v)
-	v, err = store.GetRow(ctx, key1, 3)
-	assert.Nil(t, err)
-	assert.Equal(t, Value("valuey"), v)
-
-	// Delete a row
-	err = store.DeleteRow(ctx, key, 4)
-	assert.Nil(t, err)
-	v, err = store.GetRow(ctx, key, 5)
-	assert.Nil(t, err)
-	assert.Nil(t, v)
-
-	// Clear test data
-	err = store.engine.DeleteByPrefix(ctx, key)
-	assert.Nil(t, err)
-	k, va, err := store.engine.GetByPrefix(ctx, key, false)
-	assert.Nil(t, err)
-	assert.Nil(t, k)
-	assert.Nil(t, va)
-}
-
-func TestTikvStore_BatchRow(t *testing.T) {
-	ctx := context.Background()
-
-	// Prepare test data
-	size := 0
-	var testKeys []Key
-	var testValues []Value
-	var segments []string
-	var timestamps []Timestamp
-	for i := 0; size/store.engine.conf.Raw.MaxBatchPutSize < 1; i++ {
-		key := fmt.Sprint("key", i)
-		size += len(key)
-		testKeys = append(testKeys, []byte(key))
-		value := fmt.Sprint("value", i)
-		size += len(value)
-		testValues = append(testValues, []byte(value))
-		segments = append(segments, "test")
-		v, err := store.GetRow(ctx, Key(key), math.MaxUint64)
-		assert.Nil(t, v)
-		assert.Nil(t, err)
-	}
-
-	// Batch put rows
-	for range testKeys {
-		timestamps = append(timestamps, 1)
-	}
-	err := store.PutRows(ctx, testKeys, testValues, segments, timestamps)
-	assert.Nil(t, err)
-
-	// Batch get rows
-	for i := range timestamps {
-		timestamps[i] = 2
-	}
-	checkValues, err := store.GetRows(ctx, testKeys, timestamps)
-	assert.NotNil(t, checkValues)
-	assert.Nil(t, err)
-	assert.Equal(t, len(checkValues), len(testValues))
-	for i := range testKeys {
-		assert.Equal(t, testValues[i], checkValues[i])
-	}
-
-	// Delete all test rows
-	for i := range timestamps {
-		timestamps[i] = math.MaxUint64
-	}
-	err = store.DeleteRows(ctx, testKeys, timestamps)
-	assert.Nil(t, err)
-	// Ensure all test row is deleted
-	for i := range timestamps {
-		timestamps[i] = math.MaxUint64
-	}
-	checkValues, err = store.GetRows(ctx, testKeys, timestamps)
-	assert.Nil(t, err)
-	for _, value := range checkValues {
-		assert.Nil(t, value)
-	}
-
-	// Clean test data
-	err = store.engine.DeleteByPrefix(ctx, Key("key"))
-	assert.Nil(t, err)
-}
-
-func TestTikvStore_GetSegments(t *testing.T) {
-	ctx := context.Background()
-	key := Key("key")
-
-	// Put rows
-	err := store.PutRow(ctx, key, Value{0}, "a", 1)
-	assert.Nil(t, err)
-	err = store.PutRow(ctx, key, Value{0}, "a", 2)
-	assert.Nil(t, err)
-	err = store.PutRow(ctx, key, Value{0}, "c", 3)
-	assert.Nil(t, err)
-
-	// Get segments
-	segs, err := store.GetSegments(ctx, key, 2)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(segs))
-	assert.Equal(t, "a", segs[0])
-
-	segs, err = store.GetSegments(ctx, key, 3)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(segs))
-
-	// Clean test data
-	err = store.engine.DeleteByPrefix(ctx, key)
-	assert.Nil(t, err)
-}
-
-func TestTikvStore_Log(t *testing.T) {
-	ctx := context.Background()
-
-	// Put some log
-	err := store.PutLog(ctx, Key("key1"), Value("value1"), 1, 1)
-	assert.Nil(t, err)
-	err = store.PutLog(ctx, Key("key1"), Value("value1_1"), 1, 2)
-	assert.Nil(t, err)
-	err = store.PutLog(ctx, Key("key2"), Value("value2"), 2, 1)
-	assert.Nil(t, err)
-
-	// Check log
-	log, err := store.GetLog(ctx, 0, 2, []int{1, 2})
-	if err != nil {
-		panic(err)
-	}
-	sort.Slice(log, func(i, j int) bool {
-		return bytes.Compare(log[i], log[j]) == -1
-	})
-	assert.Equal(t, log[0], Value("value1"))
-	assert.Equal(t, log[1], Value("value1_1"))
-	assert.Equal(t, log[2], Value("value2"))
-
-	// Delete test data
-	err = store.engine.DeleteByPrefix(ctx, Key("log"))
-	assert.Nil(t, err)
-}
-
-func TestTikvStore_SegmentIndex(t *testing.T) {
-	ctx := context.Background()
-
-	// Put segment index
-	err := store.PutSegmentIndex(ctx, "segment0", []byte("index0"))
-	assert.Nil(t, err)
-	err = store.PutSegmentIndex(ctx, "segment1", []byte("index1"))
-	assert.Nil(t, err)
-
-	// Get segment index
-	index, err := store.GetSegmentIndex(ctx, "segment0")
-	assert.Nil(t, err)
-	assert.Equal(t, []byte("index0"), index)
-	index, err = store.GetSegmentIndex(ctx, "segment1")
-	assert.Nil(t, err)
-	assert.Equal(t, []byte("index1"), index)
-
-	// Delete segment index
-	err = store.DeleteSegmentIndex(ctx, "segment0")
-	assert.Nil(t, err)
-	err = store.DeleteSegmentIndex(ctx, "segment1")
-	assert.Nil(t, err)
-	index, err = store.GetSegmentIndex(ctx, "segment0")
-	assert.Nil(t, err)
-	assert.Nil(t, index)
-}
-
-func TestTikvStore_DeleteSegmentDL(t *testing.T) {
-	ctx := context.Background()
-
-	// Put segment delete log
-	err := store.PutSegmentDL(ctx, "segment0", []byte("index0"))
-	assert.Nil(t, err)
-	err = store.PutSegmentDL(ctx, "segment1", []byte("index1"))
-	assert.Nil(t, err)
-
-	// Get segment delete log
-	index, err := store.GetSegmentDL(ctx, "segment0")
-	assert.Nil(t, err)
-	assert.Equal(t, []byte("index0"), index)
-	index, err = store.GetSegmentDL(ctx, "segment1")
-	assert.Nil(t, err)
-	assert.Equal(t, []byte("index1"), index)
-
-	// Delete segment delete log
-	err = store.DeleteSegmentDL(ctx, "segment0")
-	assert.Nil(t, err)
-	err = store.DeleteSegmentDL(ctx, "segment1")
-	assert.Nil(t, err)
-	index, err = store.GetSegmentDL(ctx, "segment0")
-	assert.Nil(t, err)
-	assert.Nil(t, index)
-}

internal/storage/storage.go

@@ -1,39 +0,0 @@
-package storage
-
-import (
-	"context"
-	"errors"
-
-	S3Driver "github.com/zilliztech/milvus-distributed/internal/storage/internal/S3"
-	minIODriver "github.com/zilliztech/milvus-distributed/internal/storage/internal/minio"
-	tikvDriver "github.com/zilliztech/milvus-distributed/internal/storage/internal/tikv"
-	storagetype "github.com/zilliztech/milvus-distributed/internal/storage/type"
-)
-
-func NewStore(ctx context.Context, option storagetype.Option) (storagetype.Store, error) {
-	var err error
-	var store storagetype.Store
-	switch option.Type {
-	case storagetype.TIKVDriver:
-		store, err = tikvDriver.NewTikvStore(ctx, option)
-		if err != nil {
-			panic(err.Error())
-		}
-		return store, nil
-	case storagetype.MinIODriver:
-		store, err = minIODriver.NewMinioDriver(ctx, option)
-		if err != nil {
-			//panic(err.Error())
-			return nil, err
-		}
-		return store, nil
-	case storagetype.S3DRIVER:
-		store, err = S3Driver.NewS3Driver(ctx, option)
-		if err != nil {
-			//panic(err.Error())
-			return nil, err
-		}
-		return store, nil
-	}
-	return nil, errors.New("unsupported driver")
-}

internal/storage/type/storagetype.go

@@ -1,79 +0,0 @@
-package storagetype
-
-import (
-	"context"
-
-	"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
-)
-
-type Key = []byte
-type Value = []byte
-type Timestamp = typeutil.Timestamp
-type DriverType = string
-type SegmentIndex = []byte
-type SegmentDL = []byte
-
-type Option struct {
-	Type        DriverType
-	TikvAddress string
-	BucketName  string
-}
-
-const (
-	MinIODriver DriverType = "MinIO"
-	TIKVDriver  DriverType = "TIKV"
-	S3DRIVER    DriverType = "S3"
-)
-
-/*
-type Store interface {
-	Get(ctx context.Context, key Key, timestamp Timestamp) (Value, error)
-	BatchGet(ctx context.Context, keys [] Key, timestamp Timestamp) ([]Value, error)
-	Set(ctx context.Context, key Key, v Value, timestamp Timestamp) error
-	BatchSet(ctx context.Context, keys []Key, v []Value, timestamp Timestamp) error
-	Delete(ctx context.Context, key Key, timestamp Timestamp) error
-	BatchDelete(ctx context.Context, keys []Key, timestamp Timestamp) error
-	Close() error
-}
-*/
-
-type storeEngine interface {
-	Put(ctx context.Context, key Key, value Value) error
-	Get(ctx context.Context, key Key) (Value, error)
-	GetByPrefix(ctx context.Context, prefix Key, keyOnly bool) ([]Key, []Value, error)
-	Scan(ctx context.Context, startKey Key, endKey Key, limit int, keyOnly bool) ([]Key, []Value, error)
-	Delete(ctx context.Context, key Key) error
-	DeleteByPrefix(ctx context.Context, prefix Key) error
-	DeleteRange(ctx context.Context, keyStart Key, keyEnd Key) error
-}
-
-type Store interface {
-	//put(ctx context.Context, key Key, value Value, timestamp Timestamp, suffix string) error
-	//scanLE(ctx context.Context, key Key, timestamp Timestamp, keyOnly bool) ([]Timestamp, []Key, []Value, error)
-	//scanGE(ctx context.Context, key Key, timestamp Timestamp, keyOnly bool) ([]Timestamp, []Key, []Value, error)
-	//deleteLE(ctx context.Context, key Key, timestamp Timestamp) error
-	//deleteGE(ctx context.Context, key Key, timestamp Timestamp) error
-	//deleteRange(ctx context.Context, key Key, start Timestamp, end Timestamp) error
-
-	GetRow(ctx context.Context, key Key, timestamp Timestamp) (Value, error)
-	GetRows(ctx context.Context, keys []Key, timestamps []Timestamp) ([]Value, error)
-
-	PutRow(ctx context.Context, key Key, value Value, segment string, timestamp Timestamp) error
-	PutRows(ctx context.Context, keys []Key, values []Value, segments []string, timestamps []Timestamp) error
-
-	GetSegments(ctx context.Context, key Key, timestamp Timestamp) ([]string, error)
-
-	DeleteRow(ctx context.Context, key Key, timestamp Timestamp) error
-	DeleteRows(ctx context.Context, keys []Key, timestamps []Timestamp) error
-
-	PutLog(ctx context.Context, key Key, value Value, timestamp Timestamp, channel int) error
-	GetLog(ctx context.Context, start Timestamp, end Timestamp, channels []int) ([]Value, error)
-
-	GetSegmentIndex(ctx context.Context, segment string) (SegmentIndex, error)
-	PutSegmentIndex(ctx context.Context, segment string, index SegmentIndex) error
-	DeleteSegmentIndex(ctx context.Context, segment string) error
-
-	GetSegmentDL(ctx context.Context, segment string) (SegmentDL, error)
-	PutSegmentDL(ctx context.Context, segment string, log SegmentDL) error
-	DeleteSegmentDL(ctx context.Context, segment string) error
-}

internal/util/flowgraph/input_node.go

@@ -1,8 +1,12 @@
 package flowgraph
 
 import (
+	"fmt"
 	"log"
 
+	"github.com/opentracing/opentracing-go"
+	"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
+
 	"github.com/zilliztech/milvus-distributed/internal/msgstream"
 )
 
@@ -25,11 +29,32 @@ func (inNode *InputNode) InStream() *msgstream.MsgStream {
 }
 
 // empty input and return one *Msg
-func (inNode *InputNode) Operate(in []*Msg) []*Msg {
+func (inNode *InputNode) Operate([]*Msg) []*Msg {
 	//fmt.Println("Do InputNode operation")
-
 	msgPack := (*inNode.inStream).Consume()
 
+	var childs []opentracing.Span
+	tracer := opentracing.GlobalTracer()
+	if tracer != nil && msgPack != nil {
+		for _, msg := range msgPack.Msgs {
+			if msg.Type() == internalpb.MsgType_kInsert || msg.Type() == internalpb.MsgType_kSearch {
+				var child opentracing.Span
+				ctx := msg.GetContext()
+				if parent := opentracing.SpanFromContext(ctx); parent != nil {
+					child = tracer.StartSpan(fmt.Sprintf("through msg input node, start time = %d", msg.BeginTs()),
+						opentracing.FollowsFrom(parent.Context()))
+				} else {
+					child = tracer.StartSpan(fmt.Sprintf("through msg input node, start time = %d", msg.BeginTs()))
+				}
+				child.SetTag("hash keys", msg.HashKeys())
+				child.SetTag("start time", msg.BeginTs())
+				child.SetTag("end time", msg.EndTs())
+				msg.SetContext(opentracing.ContextWithSpan(ctx, child))
+				childs = append(childs, child)
+			}
+		}
+	}
+
 	// TODO: add status
 	if msgPack == nil {
 		log.Println("null msg pack")
@@ -42,6 +67,10 @@ func (inNode *InputNode) Operate(in []*Msg) []*Msg {
 		timestampMax: msgPack.EndTs,
 	}
 
+	for _, child := range childs {
+		child.Finish()
+	}
+
 	return []*Msg{&msgStreamMsg}
 }
 

internal/util/flowgraph/message.go

@@ -4,6 +4,7 @@ import "github.com/zilliztech/milvus-distributed/internal/msgstream"
 
 type Msg interface {
 	TimeTick() Timestamp
+	DownStreamNodeIdx() int
 }
 
 type MsgStreamMsg struct {

internal/writenode/client/client.go

@@ -48,7 +48,7 @@ type SegmentDescription struct {
 	CloseTime Timestamp
 }
 
-func (c *Client) FlushSegment(segmentID UniqueID) error {
+func (c *Client) FlushSegment(segmentID UniqueID, collectionID UniqueID, partitionTag string, timestamp Timestamp) error {
 	baseMsg := msgstream.BaseMsg{
 		BeginTimestamp: 0,
 		EndTimestamp:   0,
@@ -56,9 +56,11 @@ func (c *Client) FlushSegment(segmentID UniqueID) error {
 	}
 
 	flushMsg := internalPb.FlushMsg{
-		MsgType:   internalPb.MsgType_kFlush,
+		MsgType:      internalPb.MsgType_kFlush,
-		SegmentID: segmentID,
+		SegmentID:    segmentID,
-		Timestamp: Timestamp(0),
+		CollectionID: collectionID,
+		PartitionTag: partitionTag,
+		Timestamp:    timestamp,
 	}
 
 	fMsg := &msgstream.FlushMsg{

internal/writenode/collection.go

@@ -0,0 +1,37 @@
+package writenode
+
+import (
+	"log"
+
+	"github.com/golang/protobuf/proto"
+	"github.com/zilliztech/milvus-distributed/internal/proto/schemapb"
+)
+
+type Collection struct {
+	schema *schemapb.CollectionSchema
+	id     UniqueID
+}
+
+func (c *Collection) Name() string {
+	return c.schema.Name
+}
+
+func (c *Collection) ID() UniqueID {
+	return c.id
+}
+
+func newCollection(collectionID UniqueID, schemaStr string) *Collection {
+
+	var schema schemapb.CollectionSchema
+	err := proto.UnmarshalText(schemaStr, &schema)
+	if err != nil {
+		log.Println(err)
+		return nil
+	}
+
+	var newCollection = &Collection{
+		schema: &schema,
+		id:     collectionID,
+	}
+	return newCollection
+}

internal/writenode/collection_replica.go

@@ -0,0 +1,94 @@
+package writenode
+
+import (
+	"strconv"
+	"sync"
+
+	"github.com/zilliztech/milvus-distributed/internal/errors"
+)
+
+type collectionReplica interface {
+
+	// collection
+	getCollectionNum() int
+	addCollection(collectionID UniqueID, schemaBlob string) error
+	removeCollection(collectionID UniqueID) error
+	getCollectionByID(collectionID UniqueID) (*Collection, error)
+	getCollectionByName(collectionName string) (*Collection, error)
+	hasCollection(collectionID UniqueID) bool
+}
+
+type collectionReplicaImpl struct {
+	mu          sync.RWMutex
+	collections []*Collection
+}
+
+//----------------------------------------------------------------------------------------------------- collection
+func (colReplica *collectionReplicaImpl) getCollectionNum() int {
+	colReplica.mu.RLock()
+	defer colReplica.mu.RUnlock()
+
+	return len(colReplica.collections)
+}
+
+func (colReplica *collectionReplicaImpl) addCollection(collectionID UniqueID, schemaBlob string) error {
+	colReplica.mu.Lock()
+	defer colReplica.mu.Unlock()
+
+	var newCollection = newCollection(collectionID, schemaBlob)
+	colReplica.collections = append(colReplica.collections, newCollection)
+
+	return nil
+}
+
+func (colReplica *collectionReplicaImpl) removeCollection(collectionID UniqueID) error {
+	colReplica.mu.Lock()
+	defer colReplica.mu.Unlock()
+
+	tmpCollections := make([]*Collection, 0)
+	for _, col := range colReplica.collections {
+		if col.ID() != collectionID {
+			tmpCollections = append(tmpCollections, col)
+		}
+	}
+	colReplica.collections = tmpCollections
+	return nil
+}
+
+func (colReplica *collectionReplicaImpl) getCollectionByID(collectionID UniqueID) (*Collection, error) {
+	colReplica.mu.RLock()
+	defer colReplica.mu.RUnlock()
+
+	for _, collection := range colReplica.collections {
+		if collection.ID() == collectionID {
+			return collection, nil
+		}
+	}
+
+	return nil, errors.New("cannot find collection, id = " + strconv.FormatInt(collectionID, 10))
+}
+
+func (colReplica *collectionReplicaImpl) getCollectionByName(collectionName string) (*Collection, error) {
+	colReplica.mu.RLock()
+	defer colReplica.mu.RUnlock()
+
+	for _, collection := range colReplica.collections {
+		if collection.Name() == collectionName {
+			return collection, nil
+		}
+	}
+
+	return nil, errors.New("Cannot found collection: " + collectionName)
+}
+
+func (colReplica *collectionReplicaImpl) hasCollection(collectionID UniqueID) bool {
+	colReplica.mu.RLock()
+	defer colReplica.mu.RUnlock()
+
+	for _, col := range colReplica.collections {
+		if col.ID() == collectionID {
+			return true
+		}
+	}
+	return false
+}

internal/writenode/collection_replica_test.go

@@ -0,0 +1,153 @@
+package writenode
+
+import (
+	"testing"
+
+	"github.com/golang/protobuf/proto"
+	"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
+	"github.com/zilliztech/milvus-distributed/internal/proto/etcdpb"
+	"github.com/zilliztech/milvus-distributed/internal/proto/schemapb"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func newReplica() collectionReplica {
+	collections := make([]*Collection, 0)
+
+	var replica collectionReplica = &collectionReplicaImpl{
+		collections: collections,
+	}
+	return replica
+}
+
+func genTestCollectionMeta(collectionName string, collectionID UniqueID) *etcdpb.CollectionMeta {
+	fieldVec := schemapb.FieldSchema{
+		FieldID:      UniqueID(100),
+		Name:         "vec",
+		IsPrimaryKey: false,
+		DataType:     schemapb.DataType_VECTOR_FLOAT,
+		TypeParams: []*commonpb.KeyValuePair{
+			{
+				Key:   "dim",
+				Value: "16",
+			},
+		},
+		IndexParams: []*commonpb.KeyValuePair{
+			{
+				Key:   "metric_type",
+				Value: "L2",
+			},
+		},
+	}
+
+	fieldInt := schemapb.FieldSchema{
+		FieldID:      UniqueID(101),
+		Name:         "age",
+		IsPrimaryKey: false,
+		DataType:     schemapb.DataType_INT32,
+	}
+
+	schema := schemapb.CollectionSchema{
+		Name:   collectionName,
+		AutoID: true,
+		Fields: []*schemapb.FieldSchema{
+			&fieldVec, &fieldInt,
+		},
+	}
+
+	collectionMeta := etcdpb.CollectionMeta{
+		ID:            collectionID,
+		Schema:        &schema,
+		CreateTime:    Timestamp(0),
+		SegmentIDs:    []UniqueID{0},
+		PartitionTags: []string{"default"},
+	}
+
+	return &collectionMeta
+}
+
+func initTestMeta(t *testing.T, replica collectionReplica, collectionName string, collectionID UniqueID, segmentID UniqueID) {
+	collectionMeta := genTestCollectionMeta(collectionName, collectionID)
+
+	schemaBlob := proto.MarshalTextString(collectionMeta.Schema)
+	assert.NotEqual(t, "", schemaBlob)
+
+	var err = replica.addCollection(collectionMeta.ID, schemaBlob)
+	assert.NoError(t, err)
+
+	collection, err := replica.getCollectionByName(collectionName)
+	assert.NoError(t, err)
+	assert.Equal(t, collection.Name(), collectionName)
+	assert.Equal(t, collection.ID(), collectionID)
+	assert.Equal(t, replica.getCollectionNum(), 1)
+
+}
+
+//----------------------------------------------------------------------------------------------------- collection
+func TestCollectionReplica_getCollectionNum(t *testing.T) {
+	replica := newReplica()
+	initTestMeta(t, replica, "collection0", 0, 0)
+	assert.Equal(t, replica.getCollectionNum(), 1)
+}
+
+func TestCollectionReplica_addCollection(t *testing.T) {
+	replica := newReplica()
+	initTestMeta(t, replica, "collection0", 0, 0)
+}
+
+func TestCollectionReplica_removeCollection(t *testing.T) {
+	replica := newReplica()
+	initTestMeta(t, replica, "collection0", 0, 0)
+	assert.Equal(t, replica.getCollectionNum(), 1)
+
+	err := replica.removeCollection(0)
+	assert.NoError(t, err)
+	assert.Equal(t, replica.getCollectionNum(), 0)
+}
+
+func TestCollectionReplica_getCollectionByID(t *testing.T) {
+	replica := newReplica()
+	collectionName := "collection0"
+	collectionID := UniqueID(0)
+	initTestMeta(t, replica, collectionName, collectionID, 0)
+	targetCollection, err := replica.getCollectionByID(collectionID)
+	assert.NoError(t, err)
+	assert.NotNil(t, targetCollection)
+	assert.Equal(t, targetCollection.Name(), collectionName)
+	assert.Equal(t, targetCollection.ID(), collectionID)
+}
+
+func TestCollectionReplica_getCollectionByName(t *testing.T) {
+	replica := newReplica()
+	collectionName := "collection0"
+	collectionID := UniqueID(0)
+	initTestMeta(t, replica, collectionName, collectionID, 0)
+
+	targetCollection, err := replica.getCollectionByName(collectionName)
+	assert.NoError(t, err)
+	assert.NotNil(t, targetCollection)
+	assert.Equal(t, targetCollection.Name(), collectionName)
+	assert.Equal(t, targetCollection.ID(), collectionID)
+
+}
+
+func TestCollectionReplica_hasCollection(t *testing.T) {
+	replica := newReplica()
+	collectionName := "collection0"
+	collectionID := UniqueID(0)
+	initTestMeta(t, replica, collectionName, collectionID, 0)
+
+	hasCollection := replica.hasCollection(collectionID)
+	assert.Equal(t, hasCollection, true)
+	hasCollection = replica.hasCollection(UniqueID(1))
+	assert.Equal(t, hasCollection, false)
+
+}
+
+func TestCollectionReplica_freeAll(t *testing.T) {
+	replica := newReplica()
+	collectionName := "collection0"
+	collectionID := UniqueID(0)
+	initTestMeta(t, replica, collectionName, collectionID, 0)
+
+}

internal/writenode/collection_test.go

@@ -0,0 +1,34 @@
+package writenode
+
+import (
+	"testing"
+
+	"github.com/golang/protobuf/proto"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCollection_newCollection(t *testing.T) {
+	collectionName := "collection0"
+	collectionID := UniqueID(0)
+	collectionMeta := genTestCollectionMeta(collectionName, collectionID)
+
+	schemaBlob := proto.MarshalTextString(collectionMeta.Schema)
+	assert.NotEqual(t, "", schemaBlob)
+
+	collection := newCollection(collectionMeta.ID, schemaBlob)
+	assert.Equal(t, collection.Name(), collectionName)
+	assert.Equal(t, collection.ID(), collectionID)
+}
+
+func TestCollection_deleteCollection(t *testing.T) {
+	collectionName := "collection0"
+	collectionID := UniqueID(0)
+	collectionMeta := genTestCollectionMeta(collectionName, collectionID)
+
+	schemaBlob := proto.MarshalTextString(collectionMeta.Schema)
+	assert.NotEqual(t, "", schemaBlob)
+
+	collection := newCollection(collectionMeta.ID, schemaBlob)
+	assert.Equal(t, collection.Name(), collectionName)
+	assert.Equal(t, collection.ID(), collectionID)
+}

internal/writenode/data_sync_service.go

@@ -12,16 +12,18 @@ type dataSyncService struct {
 	fg         *flowgraph.TimeTickedFlowGraph
 	ddChan     chan *ddlFlushSyncMsg
 	insertChan chan *insertFlushSyncMsg
+	replica    collectionReplica
 }
 
 func newDataSyncService(ctx context.Context,
-	ddChan chan *ddlFlushSyncMsg, insertChan chan *insertFlushSyncMsg) *dataSyncService {
+	ddChan chan *ddlFlushSyncMsg, insertChan chan *insertFlushSyncMsg, replica collectionReplica) *dataSyncService {
 
 	return &dataSyncService{
 		ctx:        ctx,
 		fg:         nil,
 		ddChan:     ddChan,
 		insertChan: insertChan,
+		replica:    replica,
 	}
 }
 
@@ -46,8 +48,8 @@ func (dsService *dataSyncService) initNodes() {
 
 	var filterDmNode Node = newFilteredDmNode()
 
-	var ddNode Node = newDDNode(dsService.ctx, dsService.ddChan)
+	var ddNode Node = newDDNode(dsService.ctx, dsService.ddChan, dsService.replica)
-	var insertBufferNode Node = newInsertBufferNode(dsService.ctx, dsService.insertChan)
+	var insertBufferNode Node = newInsertBufferNode(dsService.ctx, dsService.insertChan, dsService.replica)
 
 	dsService.fg.AddNode(&dmStreamNode)
 	dsService.fg.AddNode(&ddStreamNode)

internal/writenode/data_sync_service_test.go

@@ -197,7 +197,8 @@ func TestDataSyncService_Start(t *testing.T) {
 	assert.NoError(t, err)
 
 	// dataSync
-	node.dataSyncService = newDataSyncService(node.ctx, nil, nil)
+	replica := newReplica()
+	node.dataSyncService = newDataSyncService(node.ctx, nil, nil, replica)
 	go node.dataSyncService.start()
 
 	node.Close()

internal/writenode/flow_graph_dd_node.go

@@ -9,9 +9,6 @@ import (
 	"strconv"
 
 	"github.com/golang/protobuf/proto"
-	"github.com/minio/minio-go/v7"
-	"github.com/minio/minio-go/v7/pkg/credentials"
-
 	"github.com/zilliztech/milvus-distributed/internal/allocator"
 	"github.com/zilliztech/milvus-distributed/internal/kv"
 	miniokv "github.com/zilliztech/milvus-distributed/internal/kv/minio"
@@ -30,6 +27,7 @@ type ddNode struct {
 
 	idAllocator *allocator.IDAllocator
 	kv          kv.Base
+	replica     collectionReplica
 }
 
 type ddData struct {
@@ -228,6 +226,15 @@ func (ddNode *ddNode) createCollection(msg *msgstream.CreateCollectionMsg) {
 		log.Println(err)
 		return
 	}
+
+	schemaStr := proto.MarshalTextString(&schema)
+	// add collection
+	err = ddNode.replica.addCollection(collectionID, schemaStr)
+	if err != nil {
+		log.Println(err)
+		return
+	}
+
 	collectionName := schema.Name
 	ddNode.ddMsg.collectionRecords[collectionName] = append(ddNode.ddMsg.collectionRecords[collectionName],
 		metaOperateRecord{
@@ -252,6 +259,11 @@ func (ddNode *ddNode) createCollection(msg *msgstream.CreateCollectionMsg) {
 func (ddNode *ddNode) dropCollection(msg *msgstream.DropCollectionMsg) {
 	collectionID := msg.CollectionID
 
+	err := ddNode.replica.removeCollection(collectionID)
+	if err != nil {
+		log.Println(err)
+	}
+
 	// remove collection
 	if _, ok := ddNode.ddRecords.collectionRecords[collectionID]; !ok {
 		err := errors.New("cannot found collection " + strconv.FormatInt(collectionID, 10))
@@ -347,7 +359,7 @@ func (ddNode *ddNode) dropPartition(msg *msgstream.DropPartitionMsg) {
 	ddNode.ddBuffer.ddData[collectionID].eventTypes = append(ddNode.ddBuffer.ddData[collectionID].eventTypes, storage.DropPartitionEventType)
 }
 
-func newDDNode(ctx context.Context, outCh chan *ddlFlushSyncMsg) *ddNode {
+func newDDNode(ctx context.Context, outCh chan *ddlFlushSyncMsg, replica collectionReplica) *ddNode {
 	maxQueueLength := Params.FlowGraphMaxQueueLength
 	maxParallelism := Params.FlowGraphMaxParallelism
 
@@ -360,19 +372,16 @@ func newDDNode(ctx context.Context, outCh chan *ddlFlushSyncMsg) *ddNode {
 		partitionRecords:  make(map[UniqueID]interface{}),
 	}
 
-	minIOEndPoint := Params.MinioAddress
-	minIOAccessKeyID := Params.MinioAccessKeyID
-	minIOSecretAccessKey := Params.MinioSecretAccessKey
-	minIOUseSSL := Params.MinioUseSSL
-	minIOClient, err := minio.New(minIOEndPoint, &minio.Options{
-		Creds:  credentials.NewStaticV4(minIOAccessKeyID, minIOSecretAccessKey, ""),
-		Secure: minIOUseSSL,
-	})
-	if err != nil {
-		panic(err)
-	}
 	bucketName := Params.MinioBucketName
-	minioKV, err := miniokv.NewMinIOKV(ctx, minIOClient, bucketName)
+	option := &miniokv.Option{
+		Address:           Params.MinioAddress,
+		AccessKeyID:       Params.MinioAccessKeyID,
+		SecretAccessKeyID: Params.MinioSecretAccessKey,
+		UseSSL:            Params.MinioUseSSL,
+		BucketName:        bucketName,
+		CreateBucket:      true,
+	}
+	minioKV, err := miniokv.NewMinIOKV(ctx, option)
 	if err != nil {
 		panic(err)
 	}
@@ -397,5 +406,6 @@ func newDDNode(ctx context.Context, outCh chan *ddlFlushSyncMsg) *ddNode {
 
 		idAllocator: idAllocator,
 		kv:          minioKV,
+		replica:     replica,
 	}
 }

internal/writenode/flow_graph_dd_node_test.go

@@ -45,8 +45,8 @@ func TestFlowGraphDDNode_Operate(t *testing.T) {
 	go fService.start()
 
 	Params.FlushDdBufSize = 4
-
+	replica := newReplica()
-	ddNode := newDDNode(ctx, ddChan)
+	ddNode := newDDNode(ctx, ddChan, replica)
 
 	colID := UniqueID(0)
 	colName := "col-test-0"

internal/writenode/flow_graph_filter_dm_node.go

@@ -1,8 +1,11 @@
 package writenode
 
 import (
+	"context"
 	"log"
 
+	"github.com/opentracing/opentracing-go"
+
 	"github.com/zilliztech/milvus-distributed/internal/msgstream"
 	"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
 	internalPb "github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
@@ -31,11 +34,34 @@ func (fdmNode *filterDmNode) Operate(in []*Msg) []*Msg {
 		// TODO: add error handling
 	}
 
+	var childs []opentracing.Span
+	tracer := opentracing.GlobalTracer()
+	if tracer != nil {
+		for _, msg := range msgStreamMsg.TsMessages() {
+			if msg.Type() == internalPb.MsgType_kInsert {
+				var child opentracing.Span
+				ctx := msg.GetContext()
+				if parent := opentracing.SpanFromContext(ctx); parent != nil {
+					child = tracer.StartSpan("pass filter node",
+						opentracing.FollowsFrom(parent.Context()))
+				} else {
+					child = tracer.StartSpan("pass filter node")
+				}
+				child.SetTag("hash keys", msg.HashKeys())
+				child.SetTag("start time", msg.BeginTs())
+				child.SetTag("end time", msg.EndTs())
+				msg.SetContext(opentracing.ContextWithSpan(ctx, child))
+				childs = append(childs, child)
+			}
+		}
+	}
+
 	ddMsg, ok := (*in[1]).(*ddMsg)
 	if !ok {
 		log.Println("type assertion failed for ddMsg")
 		// TODO: add error handling
 	}
+
 	fdmNode.ddMsg = ddMsg
 
 	var iMsg = insertMsg{
@@ -56,11 +82,20 @@ func (fdmNode *filterDmNode) Operate(in []*Msg) []*Msg {
 		}
 	}
 
-	for _, msg := range msgStreamMsg.TsMessages() {
+	for key, msg := range msgStreamMsg.TsMessages() {
 		switch msg.Type() {
 		case internalPb.MsgType_kInsert:
+			var ctx2 context.Context
+			if childs != nil {
+				if childs[key] != nil {
+					ctx2 = opentracing.ContextWithSpan(msg.GetContext(), childs[key])
+				} else {
+					ctx2 = context.Background()
+				}
+			}
 			resMsg := fdmNode.filterInvalidInsertMessage(msg.(*msgstream.InsertMsg))
 			if resMsg != nil {
+				resMsg.SetContext(ctx2)
 				iMsg.insertMessages = append(iMsg.insertMessages, resMsg)
 			}
 		// case internalPb.MsgType_kDelete:
@@ -69,8 +104,11 @@ func (fdmNode *filterDmNode) Operate(in []*Msg) []*Msg {
 			log.Println("Non supporting message type:", msg.Type())
 		}
 	}
-
 	var res Msg = &iMsg
+
+	for _, child := range childs {
+		child.Finish()
+	}
 	return []*Msg{&res}
 }
 

internal/writenode/flow_graph_insert_buffer_node.go

@@ -4,18 +4,17 @@ import (
 	"bytes"
 	"context"
 	"encoding/binary"
+	"fmt"
 	"log"
 	"path"
 	"strconv"
-	"time"
 	"unsafe"
 
-	"github.com/golang/protobuf/proto"
+	"github.com/opentracing/opentracing-go"
-	"github.com/minio/minio-go/v7"
+	oplog "github.com/opentracing/opentracing-go/log"
-	"github.com/minio/minio-go/v7/pkg/credentials"
+
 	"github.com/zilliztech/milvus-distributed/internal/allocator"
 	"github.com/zilliztech/milvus-distributed/internal/kv"
-	etcdkv "github.com/zilliztech/milvus-distributed/internal/kv/etcd"
 	miniokv "github.com/zilliztech/milvus-distributed/internal/kv/minio"
 	"github.com/zilliztech/milvus-distributed/internal/msgstream"
 	"github.com/zilliztech/milvus-distributed/internal/proto/etcdpb"
@@ -23,7 +22,6 @@ import (
 	"github.com/zilliztech/milvus-distributed/internal/proto/schemapb"
 	"github.com/zilliztech/milvus-distributed/internal/storage"
 	"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
-	"go.etcd.io/etcd/clientv3"
 )
 
 const (
@@ -37,13 +35,13 @@ type (
 
 	insertBufferNode struct {
 		BaseNode
-		kvClient                      *etcdkv.EtcdKV
 		insertBuffer                  *insertBuffer
 		minIOKV                       kv.Base
 		minioPrifex                   string
 		idAllocator                   *allocator.IDAllocator
 		outCh                         chan *insertFlushSyncMsg
 		pulsarWriteNodeTimeTickStream *msgstream.PulsarMsgStream
+		replica                       collectionReplica
 	}
 
 	insertBuffer struct {
@@ -102,11 +100,23 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg {
 	// iMsg is insertMsg
 	// 1. iMsg -> buffer
 	for _, msg := range iMsg.insertMessages {
+		ctx := msg.GetContext()
+		var span opentracing.Span
+		if ctx != nil {
+			span, _ = opentracing.StartSpanFromContext(ctx, fmt.Sprintf("insert buffer node, start time = %d", msg.BeginTs()))
+		} else {
+			span = opentracing.StartSpan(fmt.Sprintf("insert buffer node, start time = %d", msg.BeginTs()))
+		}
+		span.SetTag("hash keys", msg.HashKeys())
+		span.SetTag("start time", msg.BeginTs())
+		span.SetTag("end time", msg.EndTs())
 		if len(msg.RowIDs) != len(msg.Timestamps) || len(msg.RowIDs) != len(msg.RowData) {
 			log.Println("Error: misaligned messages detected")
 			continue
 		}
 		currentSegID := msg.GetSegmentID()
+		collectionName := msg.GetCollectionName()
+		span.LogFields(oplog.Int("segment id", int(currentSegID)))
 
 		idata, ok := ibNode.insertBuffer.insertData[currentSegID]
 		if !ok {
@@ -115,17 +125,35 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg {
 			}
 		}
 
+		// Timestamps
+		_, ok = idata.Data[1].(*storage.Int64FieldData)
+		if !ok {
+			idata.Data[1] = &storage.Int64FieldData{
+				Data:    []int64{},
+				NumRows: 0,
+			}
+		}
+		tsData := idata.Data[1].(*storage.Int64FieldData)
+		for _, ts := range msg.Timestamps {
+			tsData.Data = append(tsData.Data, int64(ts))
+		}
+		tsData.NumRows += len(msg.Timestamps)
+		span.LogFields(oplog.Int("tsData numRows", tsData.NumRows))
+
 		// 1.1 Get CollectionMeta from etcd
-		segMeta, collMeta, err := ibNode.getMeta(currentSegID)
+		collection, err := ibNode.replica.getCollectionByName(collectionName)
+		//collSchema, err := ibNode.getCollectionSchemaByName(collectionName)
 		if err != nil {
 			// GOOSE TODO add error handler
 			log.Println("Get meta wrong:", err)
 			continue
 		}
 
+		collectionID := collection.ID()
+		collSchema := collection.schema
 		// 1.2 Get Fields
 		var pos int = 0 // Record position of blob
-		for _, field := range collMeta.Schema.Fields {
+		for _, field := range collSchema.Fields {
 			switch field.DataType {
 			case schemapb.DataType_VECTOR_FLOAT:
 				var dim int
@@ -360,9 +388,11 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg {
 
 		// 1.3 store in buffer
 		ibNode.insertBuffer.insertData[currentSegID] = idata
+		span.LogFields(oplog.String("store in buffer", "store in buffer"))
 
 		// 1.4 if full
 		//   1.4.1 generate binlogs
+		span.LogFields(oplog.String("generate binlogs", "generate binlogs"))
 		if ibNode.insertBuffer.full(currentSegID) {
 			log.Printf(". Insert Buffer full, auto flushing (%v) rows of data...", ibNode.insertBuffer.size(currentSegID))
 			// partitionTag -> partitionID
@@ -372,7 +402,10 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg {
 				log.Println("partitionTag to partitionID wrong")
 				// TODO GOOSE add error handler
 			}
-
+			collMeta := &etcdpb.CollectionMeta{
+				Schema: collSchema,
+				ID:     collectionID,
+			}
 			inCodec := storage.NewInsertCodec(collMeta)
 
 			// buffer data to binlogs
@@ -388,7 +421,7 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg {
 			log.Println(".. Clearing buffer")
 
 			//   1.5.2 binLogs -> minIO/S3
-			collIDStr := strconv.FormatInt(segMeta.GetCollectionID(), 10)
+			collIDStr := strconv.FormatInt(collectionID, 10)
 			partitionIDStr := strconv.FormatInt(partitionID, 10)
 			segIDStr := strconv.FormatInt(currentSegID, 10)
 			keyPrefix := path.Join(ibNode.minioPrifex, collIDStr, partitionIDStr, segIDStr)
@@ -428,6 +461,7 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg {
 				ibNode.outCh <- inBinlogMsg
 			}
 		}
+		span.Finish()
 	}
 
 	if len(iMsg.insertMessages) > 0 {
@@ -447,20 +481,24 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg {
 	for _, msg := range iMsg.flushMessages {
 		currentSegID := msg.GetSegmentID()
 		flushTs := msg.GetTimestamp()
-
+		partitionTag := msg.GetPartitionTag()
+		collectionID := msg.GetCollectionID()
 		log.Printf(". Receiving flush message segID(%v)...", currentSegID)
 
 		if ibNode.insertBuffer.size(currentSegID) > 0 {
 			log.Println(".. Buffer not empty, flushing ...")
-			segMeta, collMeta, err := ibNode.getMeta(currentSegID)
+			collSchema, err := ibNode.getCollectionSchemaByID(collectionID)
 			if err != nil {
 				// GOOSE TODO add error handler
 				log.Println("Get meta wrong: ", err)
 			}
+			collMeta := &etcdpb.CollectionMeta{
+				Schema: collSchema,
+				ID:     collectionID,
+			}
 			inCodec := storage.NewInsertCodec(collMeta)
 
 			// partitionTag -> partitionID
-			partitionTag := segMeta.GetPartitionTag()
 			partitionID, err := typeutil.Hash32String(partitionTag)
 			if err != nil {
 				// GOOSE TODO add error handler
@@ -478,7 +516,7 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg {
 			delete(ibNode.insertBuffer.insertData, currentSegID)
 
 			//   binLogs -> minIO/S3
-			collIDStr := strconv.FormatInt(segMeta.GetCollectionID(), 10)
+			collIDStr := strconv.FormatInt(collectionID, 10)
 			partitionIDStr := strconv.FormatInt(partitionID, 10)
 			segIDStr := strconv.FormatInt(currentSegID, 10)
 			keyPrefix := path.Join(ibNode.minioPrifex, collIDStr, partitionIDStr, segIDStr)
@@ -537,31 +575,20 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg {
 	return nil
 }
 
-func (ibNode *insertBufferNode) getMeta(segID UniqueID) (*etcdpb.SegmentMeta, *etcdpb.CollectionMeta, error) {
+func (ibNode *insertBufferNode) getCollectionSchemaByID(collectionID UniqueID) (*schemapb.CollectionSchema, error) {
+	ret, err := ibNode.replica.getCollectionByID(collectionID)
+	if err != nil {
+		return nil, err
+	}
+	return ret.schema, nil
+}
 
-	segMeta := &etcdpb.SegmentMeta{}
+func (ibNode *insertBufferNode) getCollectionSchemaByName(collectionName string) (*schemapb.CollectionSchema, error) {
-
+	ret, err := ibNode.replica.getCollectionByName(collectionName)
-	key := path.Join(SegmentPrefix, strconv.FormatInt(segID, 10))
-	value, err := ibNode.kvClient.Load(key)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
-	err = proto.UnmarshalText(value, segMeta)
+	return ret.schema, nil
-	if err != nil {
-		return nil, nil, err
-	}
-
-	collMeta := &etcdpb.CollectionMeta{}
-	key = path.Join(CollectionPrefix, strconv.FormatInt(segMeta.GetCollectionID(), 10))
-	value, err = ibNode.kvClient.Load(key)
-	if err != nil {
-		return nil, nil, err
-	}
-	err = proto.UnmarshalText(value, collMeta)
-	if err != nil {
-		return nil, nil, err
-	}
-	return segMeta, collMeta, nil
 }
 
 func (ibNode *insertBufferNode) writeHardTimeTick(ts Timestamp) error {
@@ -582,7 +609,7 @@ func (ibNode *insertBufferNode) writeHardTimeTick(ts Timestamp) error {
 	return ibNode.pulsarWriteNodeTimeTickStream.Produce(&msgPack)
 }
 
-func newInsertBufferNode(ctx context.Context, outCh chan *insertFlushSyncMsg) *insertBufferNode {
+func newInsertBufferNode(ctx context.Context, outCh chan *insertFlushSyncMsg, replica collectionReplica) *insertBufferNode {
 	maxQueueLength := Params.FlowGraphMaxQueueLength
 	maxParallelism := Params.FlowGraphMaxParallelism
 
@@ -596,34 +623,18 @@ func newInsertBufferNode(ctx context.Context, outCh chan *insertFlushSyncMsg) *i
 		maxSize:    maxSize,
 	}
 
-	// EtcdKV
-	ETCDAddr := Params.EtcdAddress
-	MetaRootPath := Params.MetaRootPath
-	log.Println("metaRootPath: ", MetaRootPath)
-	cli, err := clientv3.New(clientv3.Config{
-		Endpoints:   []string{ETCDAddr},
-		DialTimeout: 5 * time.Second,
-	})
-	if err != nil {
-		panic(err)
-	}
-	kvClient := etcdkv.NewEtcdKV(cli, MetaRootPath)
-
 	// MinIO
-	minioendPoint := Params.MinioAddress
-	miniioAccessKeyID := Params.MinioAccessKeyID
-	miniioSecretAccessKey := Params.MinioSecretAccessKey
-	minioUseSSL := Params.MinioUseSSL
-	minioBucketName := Params.MinioBucketName
 
-	minioClient, err := minio.New(minioendPoint, &minio.Options{
+	option := &miniokv.Option{
-		Creds:  credentials.NewStaticV4(miniioAccessKeyID, miniioSecretAccessKey, ""),
+		Address:           Params.MinioAddress,
-		Secure: minioUseSSL,
+		AccessKeyID:       Params.MinioAccessKeyID,
-	})
+		SecretAccessKeyID: Params.MinioSecretAccessKey,
-	if err != nil {
+		UseSSL:            Params.MinioUseSSL,
-		panic(err)
+		CreateBucket:      true,
+		BucketName:        Params.MinioBucketName,
 	}
-	minIOKV, err := miniokv.NewMinIOKV(ctx, minioClient, minioBucketName)
+
+	minIOKV, err := miniokv.NewMinIOKV(ctx, option)
 	if err != nil {
 		panic(err)
 	}
@@ -644,12 +655,12 @@ func newInsertBufferNode(ctx context.Context, outCh chan *insertFlushSyncMsg) *i
 
 	return &insertBufferNode{
 		BaseNode:                      baseNode,
-		kvClient:                      kvClient,
 		insertBuffer:                  iBuffer,
 		minIOKV:                       minIOKV,
 		minioPrifex:                   minioPrefix,
 		idAllocator:                   idAllocator,
 		outCh:                         outCh,
 		pulsarWriteNodeTimeTickStream: wTt,
+		replica:                       replica,
 	}
 }

internal/writenode/flow_graph_insert_buffer_node_test.go

@@ -47,7 +47,8 @@ func TestFlowGraphInputBufferNode_Operate(t *testing.T) {
 	go fService.start()
 
 	// Params.FlushInsertBufSize = 2
-	iBNode := newInsertBufferNode(ctx, insertChan)
+	replica := newReplica()
+	iBNode := newInsertBufferNode(ctx, insertChan, replica)
 
 	newMeta()
 	inMsg := genInsertMsg()

internal/writenode/flow_graph_message.go

@@ -46,14 +46,30 @@ func (ksMsg *key2SegMsg) TimeTick() Timestamp {
 	return ksMsg.timeRange.timestampMax
 }
 
+func (ksMsg *key2SegMsg) DownStreamNodeIdx() int {
+	return 0
+}
+
 func (suMsg *ddMsg) TimeTick() Timestamp {
 	return suMsg.timeRange.timestampMax
 }
 
+func (suMsg *ddMsg) DownStreamNodeIdx() int {
+	return 0
+}
+
 func (iMsg *insertMsg) TimeTick() Timestamp {
 	return iMsg.timeRange.timestampMax
 }
 
+func (iMsg *insertMsg) DownStreamNodeIdx() int {
+	return 0
+}
+
 func (dMsg *deleteMsg) TimeTick() Timestamp {
 	return dMsg.timeRange.timestampMax
 }
+
+func (dMsg *deleteMsg) DownStreamNodeIdx() int {
+	return 0
+}

internal/writenode/meta_service.go

@@ -0,0 +1,135 @@
+package writenode
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"path"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/golang/protobuf/proto"
+	"go.etcd.io/etcd/clientv3"
+
+	etcdkv "github.com/zilliztech/milvus-distributed/internal/kv/etcd"
+	"github.com/zilliztech/milvus-distributed/internal/proto/etcdpb"
+)
+
+type metaService struct {
+	ctx     context.Context
+	kvBase  *etcdkv.EtcdKV
+	replica collectionReplica
+}
+
+func newMetaService(ctx context.Context, replica collectionReplica) *metaService {
+	ETCDAddr := Params.EtcdAddress
+	MetaRootPath := Params.MetaRootPath
+
+	cli, _ := clientv3.New(clientv3.Config{
+		Endpoints:   []string{ETCDAddr},
+		DialTimeout: 5 * time.Second,
+	})
+
+	return &metaService{
+		ctx:     ctx,
+		kvBase:  etcdkv.NewEtcdKV(cli, MetaRootPath),
+		replica: replica,
+	}
+}
+
+func (mService *metaService) start() {
+	// init from meta
+	err := mService.loadCollections()
+	if err != nil {
+		log.Fatal("metaService loadCollections failed")
+	}
+}
+
+func GetCollectionObjID(key string) string {
+	ETCDRootPath := Params.MetaRootPath
+
+	prefix := path.Join(ETCDRootPath, CollectionPrefix) + "/"
+	return strings.TrimPrefix(key, prefix)
+}
+
+func isCollectionObj(key string) bool {
+	ETCDRootPath := Params.MetaRootPath
+
+	prefix := path.Join(ETCDRootPath, CollectionPrefix) + "/"
+	prefix = strings.TrimSpace(prefix)
+	index := strings.Index(key, prefix)
+
+	return index == 0
+}
+
+func isSegmentObj(key string) bool {
+	ETCDRootPath := Params.MetaRootPath
+
+	prefix := path.Join(ETCDRootPath, SegmentPrefix) + "/"
+	prefix = strings.TrimSpace(prefix)
+	index := strings.Index(key, prefix)
+
+	return index == 0
+}
+
+func printCollectionStruct(obj *etcdpb.CollectionMeta) {
+	v := reflect.ValueOf(obj)
+	v = reflect.Indirect(v)
+	typeOfS := v.Type()
+
+	for i := 0; i < v.NumField(); i++ {
+		if typeOfS.Field(i).Name == "GrpcMarshalString" {
+			continue
+		}
+		fmt.Printf("Field: %s\tValue: %v\n", typeOfS.Field(i).Name, v.Field(i).Interface())
+	}
+}
+
+func (mService *metaService) processCollectionCreate(id string, value string) {
+	//println(fmt.Sprintf("Create Collection:$%s$", id))
+
+	col := mService.collectionUnmarshal(value)
+	if col != nil {
+		schema := col.Schema
+		schemaBlob := proto.MarshalTextString(schema)
+		err := mService.replica.addCollection(col.ID, schemaBlob)
+		if err != nil {
+			log.Println(err)
+		}
+	}
+}
+
+func (mService *metaService) loadCollections() error {
+	keys, values, err := mService.kvBase.LoadWithPrefix(CollectionPrefix)
+	if err != nil {
+		return err
+	}
+
+	for i := range keys {
+		objID := GetCollectionObjID(keys[i])
+		mService.processCollectionCreate(objID, values[i])
+	}
+
+	return nil
+}
+
+//----------------------------------------------------------------------- Unmarshal and Marshal
+func (mService *metaService) collectionUnmarshal(value string) *etcdpb.CollectionMeta {
+	col := etcdpb.CollectionMeta{}
+	err := proto.UnmarshalText(value, &col)
+	if err != nil {
+		log.Println(err)
+		return nil
+	}
+	return &col
+}
+
+func (mService *metaService) collectionMarshal(col *etcdpb.CollectionMeta) string {
+	value := proto.MarshalTextString(col)
+	if value == "" {
+		log.Println("marshal collection failed")
+		return ""
+	}
+	return value
+}

internal/writenode/meta_service_test.go

@@ -0,0 +1,100 @@
+package writenode
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMetaService_start(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	replica := newReplica()
+
+	metaService := newMetaService(ctx, replica)
+
+	metaService.start()
+}
+
+func TestMetaService_getCollectionObjId(t *testing.T) {
+	var key = "/collection/collection0"
+	var collectionObjID1 = GetCollectionObjID(key)
+
+	assert.Equal(t, collectionObjID1, "/collection/collection0")
+
+	key = "fakeKey"
+	var collectionObjID2 = GetCollectionObjID(key)
+
+	assert.Equal(t, collectionObjID2, "fakeKey")
+}
+
+func TestMetaService_isCollectionObj(t *testing.T) {
+	var key = Params.MetaRootPath + "/collection/collection0"
+	var b1 = isCollectionObj(key)
+
+	assert.Equal(t, b1, true)
+
+	key = Params.MetaRootPath + "/segment/segment0"
+	var b2 = isCollectionObj(key)
+
+	assert.Equal(t, b2, false)
+}
+
+func TestMetaService_processCollectionCreate(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	replica := newReplica()
+	metaService := newMetaService(ctx, replica)
+	defer cancel()
+	id := "0"
+	value := `schema: <
+				name: "test"
+				fields: <
+				fieldID:100
+				name: "vec"
+				data_type: VECTOR_FLOAT
+				type_params: <
+				  key: "dim"
+				  value: "16"
+				>
+				index_params: <
+				  key: "metric_type"
+				  value: "L2"
+				>
+				>
+				fields: <
+				fieldID:101
+				name: "age"
+				data_type: INT32
+				type_params: <
+				  key: "dim"
+				  value: "1"
+				>
+				>
+				>
+				segmentIDs: 0
+				partition_tags: "default"
+				`
+
+	metaService.processCollectionCreate(id, value)
+
+	collectionNum := replica.getCollectionNum()
+	assert.Equal(t, collectionNum, 1)
+
+	collection, err := replica.getCollectionByName("test")
+	assert.NoError(t, err)
+	assert.Equal(t, collection.ID(), UniqueID(0))
+}
+
+func TestMetaService_loadCollections(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	replica := newReplica()
+
+	metaService := newMetaService(ctx, replica)
+
+	err2 := (*metaService).loadCollections()
+	assert.Nil(t, err2)
+}

internal/writenode/write_node.go

@@ -2,6 +2,12 @@ package writenode
 
 import (
 	"context"
+	"fmt"
+	"io"
+
+	"github.com/opentracing/opentracing-go"
+	"github.com/uber/jaeger-client-go"
+	"github.com/uber/jaeger-client-go/config"
 )
 
 type WriteNode struct {
@@ -9,15 +15,27 @@ type WriteNode struct {
 	WriteNodeID      uint64
 	dataSyncService  *dataSyncService
 	flushSyncService *flushSyncService
+	metaService      *metaService
+	replica          collectionReplica
+	tracer           opentracing.Tracer
+	closer           io.Closer
 }
 
 func NewWriteNode(ctx context.Context, writeNodeID uint64) *WriteNode {
 
+	collections := make([]*Collection, 0)
+
+	var replica collectionReplica = &collectionReplicaImpl{
+		collections: collections,
+	}
+
 	node := &WriteNode{
 		ctx:              ctx,
 		WriteNodeID:      writeNodeID,
 		dataSyncService:  nil,
 		flushSyncService: nil,
+		metaService:      nil,
+		replica:          replica,
 	}
 
 	return node
@@ -28,6 +46,22 @@ func Init() {
 }
 
 func (node *WriteNode) Start() error {
+	cfg := &config.Configuration{
+		ServiceName: "tracing",
+		Sampler: &config.SamplerConfig{
+			Type:  "const",
+			Param: 1,
+		},
+		Reporter: &config.ReporterConfig{
+			LogSpans: true,
+		},
+	}
+	var err error
+	node.tracer, node.closer, err = cfg.NewTracer(config.Logger(jaeger.StdLogger))
+	if err != nil {
+		panic(fmt.Sprintf("ERROR: cannot init Jaeger: %v\n", err))
+	}
+	opentracing.SetGlobalTracer(node.tracer)
 
 	// TODO GOOSE Init Size??
 	chanSize := 100
@@ -35,10 +69,12 @@ func (node *WriteNode) Start() error {
 	insertChan := make(chan *insertFlushSyncMsg, chanSize)
 	node.flushSyncService = newFlushSyncService(node.ctx, ddChan, insertChan)
 
-	node.dataSyncService = newDataSyncService(node.ctx, ddChan, insertChan)
+	node.dataSyncService = newDataSyncService(node.ctx, ddChan, insertChan, node.replica)
+	node.metaService = newMetaService(node.ctx, node.replica)
 
 	go node.dataSyncService.start()
 	go node.flushSyncService.start()
+	node.metaService.start()
 	return nil
 }
 

tools/core_gen/all_generate.py

@@ -58,6 +58,10 @@ if __name__ == "__main__":
                 'visitor_name': "ExecExprVisitor",
                 "parameter_name": 'expr',
             },
+            {
+                'visitor_name': "VerifyExprVisitor",
+                "parameter_name": 'expr',
+            },
         ],
         'PlanNode': [
             {
@@ -68,7 +72,10 @@ if __name__ == "__main__":
                 'visitor_name': "ExecPlanNodeVisitor",
                 "parameter_name": 'node',
             },
-
+            {
+                'visitor_name': "VerifyPlanNodeVisitor",
+                "parameter_name": 'node',
+            },
         ]
     }
     extract_extra_body(visitor_info, query_path)

tools/core_gen/templates/visitor_derived.h

@@ -13,7 +13,7 @@
 #include "@@base_visitor@@.h"
 
 namespace @@namespace@@ {
-class @@visitor_name@@ : @@base_visitor@@ {
+class @@visitor_name@@ : public @@base_visitor@@ {
  public:
 @@body@@