From ae43230703054c5c5a2df49a9bd82399d6629540 Mon Sep 17 00:00:00 2001 From: Zhen Ye Date: Thu, 15 May 2025 20:10:23 +0800 Subject: [PATCH] enhance: set jemalloc prof disable by default (#41850) issue: #40730 - add assertion for insert message - add more buffer for seal notifier Signed-off-by: chyezh --- build/docker/milvus/amazonlinux2023/Dockerfile | 2 +- build/docker/milvus/gpu/ubuntu20.04/Dockerfile | 2 +- build/docker/milvus/gpu/ubuntu22.04/Dockerfile | 2 +- build/docker/milvus/rockylinux8/Dockerfile | 2 +- build/docker/milvus/ubuntu20.04/Dockerfile | 2 +- build/docker/milvus/ubuntu22.04/Dockerfile | 2 +- .../server/wal/interceptors/redo/redo_interceptor.go | 4 ++++ .../server/wal/interceptors/shard/stats/stats_seal_worker.go | 2 +- scripts/start_cluster.sh | 2 +- scripts/start_standalone.sh | 2 +- 10 files changed, 13 insertions(+), 9 deletions(-) diff --git a/build/docker/milvus/amazonlinux2023/Dockerfile b/build/docker/milvus/amazonlinux2023/Dockerfile index c1902f3dab..a9ab05b6e4 100644 --- a/build/docker/milvus/amazonlinux2023/Dockerfile +++ b/build/docker/milvus/amazonlinux2023/Dockerfile @@ -32,7 +32,7 @@ COPY --chown=root:root --chmod=774 ./lib/ /milvus/lib/ ENV PATH=/milvus/bin:$PATH ENV LD_LIBRARY_PATH=/milvus/lib:$LD_LIBRARY_PATH:/usr/lib ENV LD_PRELOAD=${MILVUS_ASAN_LIB}:/milvus/lib/libjemalloc.so -ENV MALLOC_CONF=background_thread:true,prof:true +ENV MALLOC_CONF=background_thread:true ENTRYPOINT ["/tini", "--"] diff --git a/build/docker/milvus/gpu/ubuntu20.04/Dockerfile b/build/docker/milvus/gpu/ubuntu20.04/Dockerfile index 643104201b..1ea3764691 100644 --- a/build/docker/milvus/gpu/ubuntu20.04/Dockerfile +++ b/build/docker/milvus/gpu/ubuntu20.04/Dockerfile @@ -23,4 +23,4 @@ COPY --chown=root:root --chmod=774 ./lib/ /milvus/lib/ ENV PATH=/milvus/bin:$PATH ENV LD_LIBRARY_PATH=/milvus/lib:$LD_LIBRARY_PATH:/usr/lib ENV LD_PRELOAD=/milvus/lib/libjemalloc.so -ENV MALLOC_CONF=background_thread:true,prof:true +ENV MALLOC_CONF=background_thread:true diff --git a/build/docker/milvus/gpu/ubuntu22.04/Dockerfile b/build/docker/milvus/gpu/ubuntu22.04/Dockerfile index f6604d631b..342c2fbbbe 100644 --- a/build/docker/milvus/gpu/ubuntu22.04/Dockerfile +++ b/build/docker/milvus/gpu/ubuntu22.04/Dockerfile @@ -19,7 +19,7 @@ COPY --chown=root:root --chmod=774 ./lib/ /milvus/lib/ ENV PATH=/milvus/bin:$PATH ENV LD_LIBRARY_PATH=/milvus/lib:$LD_LIBRARY_PATH:/usr/lib ENV LD_PRELOAD=/milvus/lib/libjemalloc.so -ENV MALLOC_CONF=background_thread:true,prof:true +ENV MALLOC_CONF=background_thread:true ENTRYPOINT ["/tini", "--"] diff --git a/build/docker/milvus/rockylinux8/Dockerfile b/build/docker/milvus/rockylinux8/Dockerfile index 0eb8220186..89cb91a747 100644 --- a/build/docker/milvus/rockylinux8/Dockerfile +++ b/build/docker/milvus/rockylinux8/Dockerfile @@ -36,7 +36,7 @@ COPY ./lib/ /milvus/lib/ ENV PATH=/milvus/bin:$PATH ENV LD_LIBRARY_PATH=/milvus/lib:$LD_LIBRARY_PATH:/usr/lib ENV LD_PRELOAD=${MILVUS_ASAN_LIB}:/milvus/lib/libjemalloc.so -ENV MALLOC_CONF=background_thread:true,prof:true +ENV MALLOC_CONF=background_thread:true ENTRYPOINT ["/tini", "--"] diff --git a/build/docker/milvus/ubuntu20.04/Dockerfile b/build/docker/milvus/ubuntu20.04/Dockerfile index 06d38d09a3..8fb208605b 100644 --- a/build/docker/milvus/ubuntu20.04/Dockerfile +++ b/build/docker/milvus/ubuntu20.04/Dockerfile @@ -33,7 +33,7 @@ COPY --chown=root:root --chmod=774 ./lib/ /milvus/lib/ ENV PATH=/milvus/bin:$PATH ENV LD_LIBRARY_PATH=/milvus/lib:$LD_LIBRARY_PATH:/usr/lib ENV LD_PRELOAD=${MILVUS_ASAN_LIB}:/milvus/lib/libjemalloc.so -ENV MALLOC_CONF=background_thread:true,prof:true +ENV MALLOC_CONF=background_thread:true ENTRYPOINT ["/tini", "--"] diff --git a/build/docker/milvus/ubuntu22.04/Dockerfile b/build/docker/milvus/ubuntu22.04/Dockerfile index b4b32e6523..ed69987c2a 100644 --- a/build/docker/milvus/ubuntu22.04/Dockerfile +++ b/build/docker/milvus/ubuntu22.04/Dockerfile @@ -33,7 +33,7 @@ COPY --chown=root:root --chmod=774 ./lib/ /milvus/lib/ ENV PATH=/milvus/bin:$PATH ENV LD_LIBRARY_PATH=/milvus/lib:$LD_LIBRARY_PATH:/usr/lib ENV LD_PRELOAD=${MILVUS_ASAN_LIB}:/milvus/lib/libjemalloc.so -ENV MALLOC_CONF=background_thread:true,prof:true +ENV MALLOC_CONF=background_thread:true ENTRYPOINT ["/tini", "--"] diff --git a/internal/streamingnode/server/wal/interceptors/redo/redo_interceptor.go b/internal/streamingnode/server/wal/interceptors/redo/redo_interceptor.go index df3e46358f..37e1c51bfd 100644 --- a/internal/streamingnode/server/wal/interceptors/redo/redo_interceptor.go +++ b/internal/streamingnode/server/wal/interceptors/redo/redo_interceptor.go @@ -48,6 +48,10 @@ func (r *redoAppendInterceptor) waitUntilGrowingSegmentReady(ctx context.Context if msg.MessageType() == message.MessageTypeInsert { insertMessage := message.MustAsMutableInsertMessageV1(msg) h := insertMessage.Header() + if len(h.Partitions) != 1 { + // TODO: We will support multi-partition insert in the future. + panic("insert message should only have one partition") + } for _, partition := range h.Partitions { ready, err := r.shardManager.WaitUntilGrowingSegmentReady(h.CollectionId, partition.PartitionId) if err != nil { diff --git a/internal/streamingnode/server/wal/interceptors/shard/stats/stats_seal_worker.go b/internal/streamingnode/server/wal/interceptors/shard/stats/stats_seal_worker.go index 953034d369..9a047f9898 100644 --- a/internal/streamingnode/server/wal/interceptors/shard/stats/stats_seal_worker.go +++ b/internal/streamingnode/server/wal/interceptors/shard/stats/stats_seal_worker.go @@ -21,7 +21,7 @@ var ( func newSealWorker(statsManager *StatsManager) *sealWorker { w := &sealWorker{ statsManager: statsManager, - sealNotifier: make(chan sealSegmentIDWithPolicy, 1), + sealNotifier: make(chan sealSegmentIDWithPolicy, 100), growingBytesNotifier: syncutil.NewCooldownNotifier[uint64](growingBytesNotifyCooldown, 100), } return w diff --git a/scripts/start_cluster.sh b/scripts/start_cluster.sh index fa12f26eec..5cb3c9089e 100755 --- a/scripts/start_cluster.sh +++ b/scripts/start_cluster.sh @@ -21,7 +21,7 @@ if [[ "$OSTYPE" == "linux-gnu"* ]]; then if test -f "$LIBJEMALLOC"; then #echo "Found $LIBJEMALLOC" export LD_PRELOAD="$LIBJEMALLOC" - export MALLOC_CONF=background_thread:true,prof:true + export MALLOC_CONF=background_thread:true else echo "WARN: Cannot find $LIBJEMALLOC" fi diff --git a/scripts/start_standalone.sh b/scripts/start_standalone.sh index 44e90dce73..5d7f78f725 100755 --- a/scripts/start_standalone.sh +++ b/scripts/start_standalone.sh @@ -21,7 +21,7 @@ if [[ "$OSTYPE" == "linux-gnu"* ]]; then if test -f "$LIBJEMALLOC"; then #echo "Found $LIBJEMALLOC" export LD_PRELOAD="$LIBJEMALLOC" - export MALLOC_CONF=background_thread:true,prof:true + export MALLOC_CONF=background_thread:true else echo "WARN: Cannot find $LIBJEMALLOC" fi