Improve docker-compose config for Pulsar (#7335)

- fix graceful shutdown by using "exec bin/pulsar" when starting in a shell script
  so that OS signals can get delivered to Pulsar process to fix graceful shutdown
- simplify and fix configuration issues
- disable Pulsar Functions worker and related key-value storage service so that
  resource usage is reduced
- upgrade to use Pulsar 2.7.3 since 2.6.x isn't actively maintained

Signed-off-by: Lari Hotari <lari@hotari.net>

deployments/docker/cluster/docker-compose.yml

@@ -10,17 +10,21 @@ services:
 
   pulsar:
     container_name: milvus-pulsar
-    image: apachepulsar/pulsar:2.6.1
+    image: apachepulsar/pulsar:2.7.3
     volumes:
       - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/pulsar:/pulsar/data
-    command: >
+    environment:
-      /bin/sh -c "
+      # bin/apply-config-from-env.py script will modify the configuration file based on the environment variables
-                  echo "" >> /pulsar/conf/standalone.conf && \
+      # nettyMaxFrameSizeBytes must be calculated from maxMessageSize + 10240 (padding)
-                  echo "maxMessageSize=104857600" >> /pulsar/conf/standalone.conf && \
+      - nettyMaxFrameSizeBytes=104867840 # this is 104857600 + 10240 (padding)
-                  echo "" >> /pulsar/conf/standalone.conf && \
+      - defaultRetentionTimeInMinutes=10080
-                  echo "nettyMaxFrameSizeBytes=104857600" >> /pulsar/conf/standalone.conf && \
+      # maxMessageSize is missing from standalone.conf, must use PULSAR_PREFIX_ to get it configured
-                  sed -i 's/^defaultRetentionTimeInMinutes=.*/defaultRetentionTimeInMinutes=10080/' /pulsar/conf/broker.conf && \
+      - PULSAR_PREFIX_maxMessageSize=104857600
-                  bin/pulsar standalone"
+      - PULSAR_GC=-XX:+UseG1GC
+    command: |
+      /bin/bash -c 
+      "bin/apply-config-from-env.py conf/standalone.conf
+      && exec bin/pulsar standalone --no-functions-worker --no-stream-storage"
 
   minio:
     container_name: milvus-minio

deployments/docker/dev/docker-compose.yml

@@ -10,15 +10,19 @@ services:
       - "4001:4001"
 
   pulsar:
-    image: apachepulsar/pulsar:2.6.1
+    image: apachepulsar/pulsar:2.7.3
-    command: >
+    environment:
-      /bin/sh -c "
+      # bin/apply-config-from-env.py script will modify the configuration file based on the environment variables
-                  echo "" >> /pulsar/conf/standalone.conf &&
+      # nettyMaxFrameSizeBytes must be calculated from maxMessageSize + 10240 (padding)
-                  echo "maxMessageSize=104857600" >> /pulsar/conf/standalone.conf &&
+      - nettyMaxFrameSizeBytes=104867840 # this is 104857600 + 10240 (padding)
-                  echo "" >> /pulsar/conf/standalone.conf &&
+      - defaultRetentionTimeInMinutes=10080
-                  echo "nettyMaxFrameSizeBytes=104857600" >> /pulsar/conf/standalone.conf &&
+      # maxMessageSize is missing from standalone.conf, must use PULSAR_PREFIX_ to get it configured
-                  sed -i 's/^defaultRetentionTimeInMinutes=.*/defaultRetentionTimeInMinutes=10080/' /pulsar/conf/broker.conf && \
+      - PULSAR_PREFIX_maxMessageSize=104857600
-                  bin/pulsar standalone"
+      - PULSAR_GC=-XX:+UseG1GC
+    command: |
+      /bin/bash -c 
+      "bin/apply-config-from-env.py conf/standalone.conf
+      && exec bin/pulsar standalone --no-functions-worker --no-stream-storage"
     ports:
       - "6650:6650"
       - "18080:8080"

docker-compose.yml

@@ -41,8 +41,8 @@ services:
     command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379
 
   pulsar:
-    image: apachepulsar/pulsar:2.6.1
+    image: apachepulsar/pulsar:2.7.3
-    command: bin/pulsar standalone
+    command: bin/pulsar standalone --no-functions-worker --no-stream-storage
 
   minio:
     image: minio/minio:RELEASE.2020-12-03T00-03-10Z