yihao.dai e0113b375e

fix: Fix sort stats generates large binlogs (#42456) ...

Remove the hardcoded batchSize of 100,000 and instead trigger a write
every 64MB based on actual data size. This prevents sort stats from
generating excessively large binlog files.

issue: https://github.com/milvus-io/milvus/issues/42400

---------

Signed-off-by: bigsheeper <yihao.dai@zilliz.com>

2025-06-04 09:56:39 +08:00

..

arrow_util.go

fix: Fix sort stats generates large binlogs (#42456)

2025-06-04 09:56:39 +08:00

azure_object_storage_test.go

enhance: move object storage client creation to pkg package (#40440)

2025-03-12 20:38:07 +08:00

azure_object_storage.go

enhance: move object storage client creation to pkg package (#40440)

2025-03-12 20:38:07 +08:00

binlog_reader.go

enhance: Use v2 package name for pkg module (#39990)

2025-02-22 23:15:58 +08:00

binlog_test.go

enhance: remove not inused DDLCodec (#41485)

2025-04-25 17:26:37 +08:00

binlog_util_test.go

Format the code (#27275)

2023-09-21 09:45:27 +08:00

binlog_util.go

enhance: Use v2 package name for pkg module (#39990)

2025-02-22 23:15:58 +08:00

binlog_writer_test.go

enhance: binlog primary key turn off dict encoding (#34358)

2024-07-17 17:47:44 +08:00

binlog_writer.go

enhance: remove not inused DDLCodec (#41485)

2025-04-25 17:26:37 +08:00

data_codec_test.go

feat: bulk insert support storage v2 (#41843)

2025-05-19 10:34:24 +08:00

data_codec.go

enhance: remove not inused DDLCodec (#41485)

2025-04-25 17:26:37 +08:00

data_sorter_test.go

enhance: Use v2 package name for pkg module (#39990)

2025-02-22 23:15:58 +08:00

data_sorter.go

enhance: Use v2 package name for pkg module (#39990)

2025-02-22 23:15:58 +08:00

delta_data_test.go

enhance: Return deltadata for DeleteCodec.Deserialize (#37214)

2024-10-29 12:04:24 +08:00

delta_data.go

enhance: Add buffer forwarder for stream delta loading (#40559)

2025-03-17 15:24:10 +08:00

event_data.go

enhance: Use v2 package name for pkg module (#39990)

2025-02-22 23:15:58 +08:00

event_header.go

enhance: Use v2 package name for pkg module (#39990)

2025-02-22 23:15:58 +08:00

event_reader.go

fix: Update logging context and upgrade dependencies (#41318)

2025-04-23 10:52:38 +08:00

event_test.go

enhance: Use v2 package name for pkg module (#39990)

2025-02-22 23:15:58 +08:00

event_writer_test.go

enhance: Use v2 package name for pkg module (#39990)

2025-02-22 23:15:58 +08:00

event_writer.go

enhance: Use v2 package name for pkg module (#39990)

2025-02-22 23:15:58 +08:00

factory.go

enhance: move object storage client creation to pkg package (#40440)

2025-03-12 20:38:07 +08:00

field_stats_test.go

enhance: Use v2 package name for pkg module (#39990)

2025-02-22 23:15:58 +08:00

field_stats.go

fix: Update logging context and upgrade dependencies (#41318)

2025-04-23 10:52:38 +08:00

field_value_test.go

enhance: Use internal json(sonic) to replace std json lib (#37708)

2024-11-18 10:46:31 +08:00

field_value.go

fix: Update logging context and upgrade dependencies (#41318)

2025-04-23 10:52:38 +08:00

gcp_native_object_storage_test.go

enhance: move object storage client creation to pkg package (#40440)

2025-03-12 20:38:07 +08:00

gcp_native_object_storage.go

enhance: move object storage client creation to pkg package (#40440)

2025-03-12 20:38:07 +08:00

index_data_codec_test.go

enhance: Use v2 package name for pkg module (#39990)

2025-02-22 23:15:58 +08:00

index_data_codec.go

fix: Update logging context and upgrade dependencies (#41318)

2025-04-23 10:52:38 +08:00

insert_data_test.go

enhance: Use v2 package name for pkg module (#39990)

2025-02-22 23:15:58 +08:00

insert_data.go

enhance: Use v2 package name for pkg module (#39990)

2025-02-22 23:15:58 +08:00

local_chunk_manager_test.go

enhance: move object storage client creation to pkg package (#40440)

2025-03-12 20:38:07 +08:00

local_chunk_manager.go

enhance: move object storage client creation to pkg package (#40440)

2025-03-12 20:38:07 +08:00

minio_object_storage_test.go

enhance: move object storage client creation to pkg package (#40440)

2025-03-12 20:38:07 +08:00

minio_object_storage.go

enhance: move object storage client creation to pkg package (#40440)

2025-03-12 20:38:07 +08:00

OWNERS

[skip ci]Update OWNERS files (#11898)

2021-11-16 15:41:11 +08:00

partition_stats_test.go

feat: Define FieldValue, FieldStats and PartitionStats (#30286)

2024-03-06 20:42:37 -08:00

partition_stats.go

enhance: Use internal json(sonic) to replace std json lib (#37708)

2024-11-18 10:46:31 +08:00

payload_reader_test.go

enhance: upgrade go arrow version from 12.0.1 to 17.0.0 (#39916)

2025-02-25 10:30:02 +08:00

payload_reader.go

fix: Update logging context and upgrade dependencies (#41318)

2025-04-23 10:52:38 +08:00

payload_test.go

enhance: upgrade go arrow version from 12.0.1 to 17.0.0 (#39916)

2025-02-25 10:30:02 +08:00

payload_writer_test.go

enhance: binlog primary key turn off dict encoding (#34358)

2024-07-17 17:47:44 +08:00

payload_writer.go

enhance: upgrade go arrow version from 12.0.1 to 17.0.0 (#39916)

2025-02-25 10:30:02 +08:00

payload.go

enhance: upgrade go arrow version from 12.0.1 to 17.0.0 (#39916)

2025-02-25 10:30:02 +08:00

pk_statistics.go

enhance: Use v2 package name for pkg module (#39990)

2025-02-22 23:15:58 +08:00

primary_key_test.go

enhance: Use internal json(sonic) to replace std json lib (#37708)

2024-11-18 10:46:31 +08:00

primary_key.go

fix: Update logging context and upgrade dependencies (#41318)

2025-04-23 10:52:38 +08:00

primary_keys_test.go

enhance: Add PrimaryKeys interface to reduce memory usage (#30405)

2024-02-01 09:57:11 +08:00

primary_keys.go

enhance: Add buffer forwarder for stream delta loading (#40559)

2025-03-17 15:24:10 +08:00

print_binlog_test.go

enhance: remove not inused DDLCodec (#41485)

2025-04-25 17:26:37 +08:00

print_binlog.go

enhance: Use v2 package name for pkg module (#39990)

2025-02-22 23:15:58 +08:00

remote_chunk_manager_test.go

enhance: move object storage client creation to pkg package (#40440)

2025-03-12 20:38:07 +08:00

remote_chunk_manager.go

enhance: move object storage client creation to pkg package (#40440)

2025-03-12 20:38:07 +08:00

rw_test.go

enhance: [StorageV2] fix issues integrating basic RW operations (#41834)

2025-05-15 09:52:23 +08:00

rw.go

enhance: [StorageV2] Make packed reader use correct path (#41919)

2025-05-20 10:36:23 +08:00

schema_test.go

feat: storage v2 serde reader and writer (#39667)

2025-02-11 16:00:46 +08:00

schema.go

enhance: update packed reader api (#41055)

2025-04-09 10:18:26 +08:00

serde_events_test.go

feat: compaction to support add field (#40415)

2025-03-18 11:32:12 +08:00

serde_events_v2_test.go

enhance: [StorageV2] fix issues integrating basic RW operations (#41834)

2025-05-15 09:52:23 +08:00

serde_events_v2.go

enhance: [StorageV2] fix issues integrating basic RW operations (#41834)

2025-05-15 09:52:23 +08:00

serde_events.go

fix: Update logging context and upgrade dependencies (#41318)

2025-04-23 10:52:38 +08:00

serde_test.go

enhance: Remove not inuse binlog iterator (#41359)

2025-04-24 12:04:38 +08:00

serde.go

feat: bulk insert support storage v2 (#41843)

2025-05-19 10:34:24 +08:00

sort_test.go

fix: Fix sort stats generates large binlogs (#42456)

2025-06-04 09:56:39 +08:00

sort.go

fix: Fix sort stats generates large binlogs (#42456)

2025-06-04 09:56:39 +08:00

stats_test.go

enhance: Use v2 package name for pkg module (#39990)

2025-02-22 23:15:58 +08:00

stats.go

fix: bm25 search failed when avgdl == nan (#41502)

2025-04-27 17:34:38 +08:00

storage_test.go

enhance: Use v2 package name for pkg module (#39990)

2025-02-22 23:15:58 +08:00

types.go

enhance: use WalkWithPrefix api for oss, enable piplined file gc (#31740)

2024-04-25 20:41:27 +08:00

unsafe_test.go

[skip e2e]Update license for storage unsafe (#14452)

2021-12-28 20:03:56 +08:00

unsafe.go

enhance: support null in go payload (#32296)

2024-06-19 17:08:00 +08:00

utils_test.go

enhance: Use v2 package name for pkg module (#39990)

2025-02-22 23:15:58 +08:00

utils.go

fix: Update logging context and upgrade dependencies (#41318)

2025-04-23 10:52:38 +08:00