From 12e49781e6ca83729328429a7b9a8c674d0bc379 Mon Sep 17 00:00:00 2001 From: xige-16 Date: Tue, 29 Sep 2020 10:55:48 +0800 Subject: [PATCH] Fix error when cover string to bytes in write node Signed-off-by: xige-16 --- sdk/examples/simple/create_collection.cpp | 2 +- sdk/examples/simple/insert.cpp | 2 +- sdk/examples/simple/search.cpp | 2 +- writer/main.go | 38 ----------------------- writer/write_node/writer_node.go | 13 +++----- 5 files changed, 8 insertions(+), 49 deletions(-) diff --git a/sdk/examples/simple/create_collection.cpp b/sdk/examples/simple/create_collection.cpp index a020938cf6..5b69c11657 100644 --- a/sdk/examples/simple/create_collection.cpp +++ b/sdk/examples/simple/create_collection.cpp @@ -5,7 +5,7 @@ #include #include "utils/Utils.h" -const int DIM = 128; +const int DIM = 512; int main(int argc , char**argv) { diff --git a/sdk/examples/simple/insert.cpp b/sdk/examples/simple/insert.cpp index e5c19fb90c..ff048f4e12 100644 --- a/sdk/examples/simple/insert.cpp +++ b/sdk/examples/simple/insert.cpp @@ -22,7 +22,7 @@ #include int N = 6000000; -int DIM = 128; +int DIM = 512; int LOOP = 2000; int ID_START = 0; diff --git a/sdk/examples/simple/search.cpp b/sdk/examples/simple/search.cpp index ec88dba7fc..35fde2e39b 100644 --- a/sdk/examples/simple/search.cpp +++ b/sdk/examples/simple/search.cpp @@ -20,7 +20,7 @@ const int TOP_K = 10; const int LOOP = 1000; -const int DIM = 128; +const int DIM = 512; std::default_random_engine eng(42); diff --git a/writer/main.go b/writer/main.go index 63d1515817..e44020e406 100644 --- a/writer/main.go +++ b/writer/main.go @@ -44,17 +44,6 @@ func main() { const Debug = true if Debug { - //var shouldBenchmark = false - //var start time.Time - //var LogRecords int64 - //var logFlag int64 - //var logString = "" - //logFile, err := os.OpenFile("writenode.benchmark", os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0777) - //defer logFile.Close() - //if err != nil { - // log.Fatalf("Prepare log file error, " + err.Error()) - //} - const CountInsertMsgBaseline = 1000 * 1000 var BaselineCounter int64 = 0 @@ -63,13 +52,6 @@ func main() { break } msgLength := wn.MessageClient.PrepareBatchMsg() - // wait until first 100,000 rows are successfully wrote - //if wn.MsgCounter.InsertCounter >= CountMsgNum && shouldBenchmark == false { - // shouldBenchmark = true - // wn.MsgCounter.InsertCounter = 0 - // wn.MsgCounter.InsertedRecordSize = 0 - // start = time.Now() - //} if wn.MsgCounter.InsertCounter/CountInsertMsgBaseline != BaselineCounter { wn.WriteWriterLog() @@ -80,26 +62,6 @@ func main() { wn.DoWriteNode(ctx) fmt.Println("write node do a batch message, storage len: ", msgLength) } - - // Test insert time - // ignore if less than 1000 records per time interval - //if shouldBenchmark && wn.MsgCounter.InsertCounter > 1000 { - // LogRecords += msgCounter.InsertCounter - // timeSince := time.Since(start) - // if timeSince >= timeInterval { - // speed := wn.MsgCounter.InsertedRecordSize / timeInterval.Seconds() / MB - // logString = fmt.Sprintln("============> Insert", wn.MsgCounter.InsertCounter, "records, cost:", timeSince, "speed:", speed, "M/s", "<============") - // newFlag := LogRecords / (10000 * 100) - // if newFlag != logFlag { - // logFlag = newFlag - // fmt.Fprintln(logFile, logString) - // logString = "" - // } - // wn.MsgCounter.InsertCounter = 0 - // wn.MsgCounter.InsertedRecordSize = 0 - // start = time.Now() - // } - //} } } diff --git a/writer/write_node/writer_node.go b/writer/write_node/writer_node.go index 6c7c013e5f..d45cd0fb71 100644 --- a/writer/write_node/writer_node.go +++ b/writer/write_node/writer_node.go @@ -2,6 +2,7 @@ package write_node import ( "context" + "encoding/binary" "encoding/json" "fmt" "github.com/czs007/suvlim/conf" @@ -82,23 +83,19 @@ func (wn *WriteNode) InsertBatchData(ctx context.Context, data []*msgpb.InsertOr var suffixKeys []string var binaryData [][]byte var timeStamp []uint64 + byteArr := make([]byte, 8) + intData := uint64(0) + binary.BigEndian.PutUint64(byteArr, intData) for i := 0; i < len(data); i++ { prefixKey = data[i].CollectionName + "-" + strconv.FormatUint(uint64(data[i].Uid), 10) suffixKey = strconv.FormatUint(uint64(data[i].SegmentId), 10) prefixKeys = append(prefixKeys, []byte(prefixKey)) suffixKeys = append(suffixKeys, suffixKey) - binaryData = append(binaryData, []byte(data[i].String())) + binaryData = append(binaryData, byteArr) timeStamp = append(timeStamp, uint64(data[i].Timestamp)) } - //wn.WriterLog(len(data)) - - //wn.MsgCounter.InsertCounter += int64(len(timeStamp)) - //if len(timeStamp) > 0 { - // // assume each record is same size - // wn.MsgCounter.InsertedRecordSize += float64(len(timeStamp) * len(data[0].RowsData.Blob)) - //} error := (*wn.KvStore).PutRows(ctx, prefixKeys, binaryData, suffixKeys, timeStamp) if error != nil { fmt.Println("Can't insert data!")