Fryuni opened a new issue, #13330:
URL: https://github.com/apache/druid/issues/13330
### Affected Version
24.0.0
### Description
Cluster size:
- 5 Coordinators
- 5 Overlords
- 20 Middle-Managers
- 6 Historicals
- 4 Brokers
- 3 Routers
Configurations in use:
<details>
<summary>Middle-Manager</summary>
```json
{
"awt.toolkit": "sun.awt.X11.XToolkit",
"druid.coordinator.loadqueuepeon.type": "http",
"druid.discovery.k8s.clusterIdentifier": "druid",
"druid.discovery.type": "k8s",
"druid.emitter": "noop",
"druid.emmiter.logging.logLevel": "warn",
"druid.extensions.loadList": "[ \"druid-kubernetes-extensions\",
\"druid-basic-security\", \"prometheus-emitter\",
\"postgresql-metadata-storage\", \"druid-google-extensions\", \"druid-pac4j\",
\"druid-kafka-indexing-service\", \"druid-lookups-cached-global\",
\"druid-virtual-columns\", \"druid-avro-extensions\", \"druid-datasketches\",
\"druid-moving-average-query\", \"druid-time-min-max\" ]",
"druid.generic.useDefaultValueForNull": "true",
"druid.google.bucket": "***-druid-deepstorage",
"druid.host": "10.28.8.24",
"druid.indexer.fork.property.druid.processing.buffer.sizeBytes": "100MiB",
"druid.indexer.fork.property.druid.processing.numMergeBuffers": "2",
"druid.indexer.fork.property.druid.processing.numThreads": "1",
"druid.indexer.logs.bucket": "***-druid-deepstorage",
"druid.indexer.logs.kill.durationToRetain": "604800000",
"druid.indexer.logs.kill.enabled": "false",
"druid.indexer.logs.prefix": "taskLogs",
"druid.indexer.logs.type": "google",
"druid.indexer.runner.javaOptsArray":
"[\"-XX:MaxDirectMemorySize=3072m\",\"-Xmx1024m\",\"-Xms128m\"]",
"druid.indexer.runner.type": "httpRemote",
"druid.javascript.enabled": "false",
"druid.lookup.enableLookupSyncOnStartup": "true",
"druid.lookup.lookupTier": "broker",
"druid.metadata.storage.connector.connectURI":
"jdbc:postgresql://***.svc.cluster.local:5432/druid",
"druid.metadata.storage.connector.createTables": "true",
"druid.metadata.storage.type": "postgresql",
"druid.port": "8088",
"druid.processing.numMergeBuffers": "2",
"druid.processing.numThreads": "1",
"druid.query.groupBy.applyLimitPushDownToSegment": "true",
"druid.query.groupBy.maxOnDiskStorage": "1073741824",
"druid.realtime.cache.populateCache": "true",
"druid.realtime.cache.useCache": "true",
"druid.selectors.coordinator.serviceName": "druid/coordinator",
"druid.selectors.indexing.serviceName": "druid/overlord",
"druid.server.http.numThreads": "5",
"druid.serverview.type": "http",
"druid.service": "druid/middlemanager",
"druid.storage.type": "google",
"druid.worker.capacity": "1",
"druid.zk.service.enabled": "false",
"file.encoding": "UTF-8",
"file.encoding.pkg": "sun.io",
"file.separator": "/",
"java.awt.graphicsenv": "sun.awt.X11GraphicsEnvironment",
"java.awt.printerjob": "sun.print.PSPrinterJob",
"java.class.path":
"/tmp/conf/druid/cluster/_common:/tmp/conf/druid/cluster/data/middleManager:lib/zstd-jni-1.5.2-3.jar:lib/jackson-dataformat-smile-2.10.5.jar:lib/commons-beanutils-1.9.4.jar:lib/classmate-1.1.0.jar:lib/log4j-jul-2.18.0.jar:lib/javax.inject-1.jar:lib/avatica-metrics-1.17.0.jar:lib/avatica-server-1.17.0.jar:lib/aopalliance-1.0.jar:lib/aether-util-0.9.0.M2.jar:lib/jetty-servlet-9.4.48.v20220622.jar:lib/spymemcached-2.12.3.jar:lib/zookeeper-jute-3.5.9.jar:lib/guava-16.0.1.jar:lib/google-oauth-client-1.26.0.jar:lib/druid-core-24.0.0.jar:lib/validation-api-1.1.0.Final.jar:lib/janino-3.0.11.jar:lib/httpclient-4.5.13.jar:lib/accessors-smart-1.2.jar:lib/async-http-client-2.5.3.jar:lib/joda-time-2.10.5.jar:lib/json-smart-2.3.jar:lib/guice-servlet-4.1.0.jar:lib/vavr-match-0.10.2.jar:lib/curator-client-4.3.0.jar:lib/calcite-linq4j-1.21.0.jar:lib/commons-collections4-4.2.jar:lib/curator-recipes-4.3.0.jar:lib/netty-transport-native-epoll-4.1.68.Final-linux-x86_64.jar:lib/agg
designer-algorithm-6.0.jar:lib/jna-4.5.1.jar:lib/jackson-jaxrs-json-provider-2.10.5.jar:lib/asm-9.3.jar:lib/netty-reactive-streams-2.0.0.jar:lib/metrics-core-4.0.0.jar:lib/commons-collections-3.2.2.jar:lib/guice-assistedinject-4.1.0.jar:lib/derby-10.14.2.0.jar:lib/fastutil-8.5.4.jar:lib/google-api-client-1.26.0.jar:lib/netty-buffer-4.1.68.Final.jar:lib/druid-sql-24.0.0.jar:lib/log4j-slf4j-impl-2.18.0.jar:lib/commons-compiler-3.0.11.jar:lib/caffeine-2.8.0.jar:lib/jackson-module-guice-2.10.5.jar:lib/jetty-security-9.4.48.v20220622.jar:lib/maven-repository-metadata-3.1.1.jar:lib/commons-io-2.11.0.jar:lib/jackson-jq-0.0.10.jar:lib/asm-commons-9.3.jar:lib/compress-lzf-1.0.4.jar:lib/commons-codec-1.13.jar:lib/protobuf-java-3.11.0.jar:lib/config-magic-0.9.jar:lib/airline-io-2.8.4.jar:lib/audience-annotations-0.5.0.jar:lib/fastutil-extra-8.5.4.jar:lib/txw2-2.3.1.jar:lib/jersey-core-1.19.4.jar:lib/avatica-core-1.17.0.jar:lib/netty-3.10.6.Final.jar:lib/fastutil-core-8.5.4.jar:lib/druid-indexi
ng-hadoop-24.0.0.jar:lib/jetty-client-9.4.48.v20220622.jar:lib/druid-indexing-service-24.0.0.jar:lib/commons-compress-1.21.jar:lib/reactive-streams-1.0.2.jar:lib/maven-model-builder-3.1.1.jar:lib/javax.el-api-3.0.0.jar:lib/netty-resolver-dns-4.1.68.Final.jar:lib/hibernate-validator-5.2.5.Final.jar:lib/druid-aws-common-24.0.0.jar:lib/sigar-1.6.5.132.jar:lib/maven-settings-3.1.1.jar:lib/log4j-1.2-api-2.18.0.jar:lib/resilience4j-bulkhead-1.3.1.jar:lib/icu4j-55.1.jar:lib/jackson-jaxrs-base-2.10.5.jar:lib/stax-ex-1.8.jar:lib/aether-connector-okhttp-0.0.9.jar:lib/javax.activation-api-1.2.0.jar:lib/netty-codec-4.1.68.Final.jar:lib/calcite-core-1.21.0.jar:lib/netty-codec-http-4.1.68.Final.jar:lib/aws-java-sdk-sts-1.12.264.jar:lib/jersey-servlet-1.19.4.jar:lib/json-path-2.3.0.jar:lib/druid-server-24.0.0.jar:lib/jetty-proxy-9.4.48.v20220622.jar:lib/lz4-java-1.8.0.jar:lib/jaxb-runtime-2.3.1.jar:lib/maven-artifact-3.6.0.jar:lib/jackson-module-jaxb-annotations-2.10.5.jar:lib/antlr4-runtime-4.5.1
.jar:lib/curator-framework-4.3.0.jar:lib/druid-console-24.0.0.jar:lib/datasketches-java-3.2.0.jar:lib/RoaringBitmap-0.9.0.jar:lib/guice-4.1.0.jar:lib/log4j-api-2.18.0.jar:lib/jakarta.inject-api-1.0.3.jar:lib/netty-codec-socks-4.1.68.Final.jar:lib/jackson-annotations-2.10.5.jar:lib/derbyclient-10.14.2.0.jar:lib/aether-connector-file-0.9.0.M2.jar:lib/aws-java-sdk-ec2-1.12.264.jar:lib/disruptor-3.3.6.jar:lib/datasketches-memory-2.0.0.jar:lib/wagon-provider-api-2.4.jar:lib/xz-1.8.jar:lib/jetty-util-ajax-9.4.48.v20220622.jar:lib/commons-dbcp2-2.0.1.jar:lib/netty-handler-proxy-4.1.68.Final.jar:lib/javax.activation-1.2.0.jar:lib/jcodings-1.0.43.jar:lib/cron-scheduler-0.1.jar:lib/jsr305-2.0.1.jar:lib/aws-java-sdk-kms-1.12.264.jar:lib/jakarta.xml.bind-api-2.3.2.jar:lib/FastInfoset-1.2.15.jar:lib/jackson-datatype-joda-2.10.5.jar:lib/jcl-over-slf4j-1.7.36.jar:lib/druid-services-24.0.0.jar:lib/jetty-server-9.4.48.v20220622.jar:lib/curator-x-discovery-4.3.0.jar:lib/jvm-attach-api-1.5.jar:lib/jac
kson-jaxrs-smile-provider-2.10.5.jar:lib/maven-settings-builder-3.1.1.jar:lib/javax.el-3.0.0.jar:lib/google-http-client-1.26.0.jar:lib/rhino-1.7.11.jar:lib/jdbi-2.63.1.jar:lib/extendedset-24.0.0.jar:lib/netty-handler-4.1.68.Final.jar:lib/error_prone_annotations-2.11.0.jar:lib/commons-lang-2.6.jar:lib/jetty-util-9.4.48.v20220622.jar:lib/aether-api-0.9.0.M2.jar:lib/maven-aether-provider-3.1.1.jar:lib/commons-lang3-3.8.1.jar:lib/plexus-interpolation-1.19.jar:lib/httpcore-4.4.11.jar:lib/commons-pool2-2.2.jar:lib/druid-hll-24.0.0.jar:lib/jetty-continuation-9.4.48.v20220622.jar:lib/joni-2.1.27.jar:lib/istack-commons-runtime-3.0.7.jar:lib/aws-java-sdk-s3-1.12.264.jar:lib/ion-java-1.0.2.jar:lib/netty-common-4.1.68.Final.jar:lib/netty-transport-4.1.68.Final.jar:lib/jetty-rewrite-9.4.48.v20220622.jar:lib/jersey-server-1.19.4.jar:lib/derbynet-10.14.2.0.jar:lib/jetty-servlets-9.4.48.v20220622.jar:lib/tesla-aether-0.0.5.jar:lib/jmespath-java-1.12.264.jar:lib/zookeeper-3.5.9.jar:lib/async-http-cl
ient-netty-utils-2.5.3.jar:lib/druid-processing-24.0.0.jar:lib/jetty-http-9.4.48.v20220622.jar:lib/slf4j-api-1.7.36.jar:lib/jackson-dataformat-cbor-2.10.5.jar:lib/jakarta.activation-api-1.2.1.jar:lib/aws-java-sdk-core-1.12.264.jar:lib/jaxb-api-2.3.1.jar:lib/j2objc-annotations-1.1.jar:lib/aether-spi-0.9.0.M2.jar:lib/jsr311-api-1.1.1.jar:lib/commons-math3-3.6.1.jar:lib/ipaddress-5.3.4.jar:lib/jersey-guice-1.19.4.jar:lib/airline-2.8.4.jar:lib/jackson-datatype-guava-2.10.5.jar:lib/commons-logging-1.1.1.jar:lib/asm-tree-9.3.jar:lib/netty-resolver-4.1.68.Final.jar:lib/jackson-core-2.10.5.jar:lib/commons-text-1.3.jar:lib/resilience4j-core-1.3.1.jar:lib/maven-model-3.1.1.jar:lib/netty-transport-native-unix-common-4.1.68.Final.jar:lib/checker-qual-2.5.7.jar:lib/okhttp-1.0.2.jar:lib/guice-multibindings-4.1.0.jar:lib/javax.servlet-api-3.1.0.jar:lib/jetty-io-9.4.48.v20220622.jar:lib/vavr-0.10.2.jar:lib/google-http-client-jackson2-1.26.0.jar:lib/netty-codec-dns-4.1.68.Final.jar:lib/opencsv-4.6.j
ar:lib/jackson-databind-2.10.5.1.jar:lib/aether-impl-0.9.0.M2.jar:lib/plexus-utils-3.0.24.jar:lib/jboss-logging-3.2.1.Final.jar:lib/esri-geometry-api-2.2.0.jar:lib/asm-analysis-9.3.jar:lib/druid-gcp-common-24.0.0.jar:lib/log4j-core-2.18.0.jar:lib/shims-0.9.0.jar:",
"java.class.version": "52.0",
"java.endorsed.dirs": "/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/endorsed",
"java.ext.dirs":
"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/ext:/usr/java/packages/lib/ext",
"java.home": "/usr/lib/jvm/java-8-openjdk-amd64/jre",
"java.io.tmpdir": "/druid/data",
"java.library.path":
"/usr/java/packages/lib/amd64:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni:/lib:/usr/lib",
"java.runtime.name": "OpenJDK Runtime Environment",
"java.runtime.version": "1.8.0_275-8u275-b01-1~deb9u1-b01",
"java.specification.name": "Java Platform API Specification",
"java.specification.vendor": "Oracle Corporation",
"java.specification.version": "1.8",
"java.util.logging.manager": "org.apache.logging.log4j.jul.LogManager",
"java.vendor": "Oracle Corporation",
"java.vendor.url": "http://java.oracle.com/";,
"java.vendor.url.bug": "http://bugreport.sun.com/bugreport/";,
"java.version": "1.8.0_275",
"java.vm.info": "mixed mode",
"java.vm.name": "OpenJDK 64-Bit Server VM",
"java.vm.specification.name": "Java Virtual Machine Specification",
"java.vm.specification.vendor": "Oracle Corporation",
"java.vm.specification.version": "1.8",
"java.vm.vendor": "Oracle Corporation",
"java.vm.version": "25.275-b01",
"line.separator": "\n",
"log4j.shutdownCallbackRegistry":
"org.apache.druid.common.config.Log4jShutdown",
"log4j.shutdownHookEnabled": "true",
"log4j2.is.webapp": "false",
"net.spy.log.LoggerImpl": "net.spy.memcached.compat.log.SLF4JLogger",
"org.jboss.logging.provider": "slf4j",
"os.arch": "amd64",
"os.name": "Linux",
"os.version": "5.4.202+",
"path.separator": ":",
"sun.arch.data.model": "64",
"sun.boot.class.path":
"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/resources.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/rt.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/sunrsasign.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jsse.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jce.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/charsets.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jfr.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/classes",
"sun.boot.library.path": "/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64",
"sun.cpu.endian": "little",
"sun.cpu.isalist": "",
"sun.io.unicode.encoding": "UnicodeLittle",
"sun.java.command": "org.apache.druid.cli.Main server middleManager",
"sun.java.launcher": "SUN_STANDARD",
"sun.jnu.encoding": "UTF-8",
"sun.management.compiler": "HotSpot 64-Bit Tiered Compilers",
"sun.os.patch.level": "unknown",
"user.dir": "/opt/druid",
"user.home": "/root",
"user.language": "en",
"user.name": "root",
"user.timezone": "UTC"
}
```
</details>
<details>
<summary>Coordinator/Overlord</summary>
```json
{
"awt.toolkit": "sun.awt.X11.XToolkit",
"druid.coordinator.asOverlord.enabled": "true",
"druid.coordinator.asOverlord.overlordService": "druid/overlord",
"druid.coordinator.balancer.strategy": "cachingCost",
"druid.coordinator.kill.durationToRetain": "P180D",
"druid.coordinator.kill.on": "false",
"druid.coordinator.loadqueuepeon.type": "http",
"druid.coordinator.period": "PT30S",
"druid.coordinator.period.indexingPeriod": "PT5M",
"druid.coordinator.startDelay": "PT30S",
"druid.discovery.k8s.clusterIdentifier": "druid",
"druid.discovery.type": "k8s",
"druid.emitter": "prometheus",
"druid.emitter.prometheus.addServiceAsLabel": "true",
"druid.emitter.prometheus.port": "9999",
"druid.emitter.prometheus.strategy": "exporter",
"druid.emmiter.logging.logLevel": "warn",
"druid.extensions.loadList": "[ \"druid-kubernetes-extensions\",
\"druid-basic-security\", \"prometheus-emitter\",
\"postgresql-metadata-storage\", \"druid-google-extensions\", \"druid-pac4j\",
\"druid-kafka-indexing-service\", \"druid-lookups-cached-global\",
\"druid-virtual-columns\", \"druid-avro-extensions\", \"druid-datasketches\",
\"druid-moving-average-query\", \"druid-time-min-max\" ]",
"druid.generic.useDefaultValueForNull": "true",
"druid.google.bucket": "***-druid-deepstorage",
"druid.host": "10.28.27.21",
"druid.indexer.logs.bucket": "***-druid-deepstorage",
"druid.indexer.logs.kill.durationToRetain": "604800000",
"druid.indexer.logs.kill.enabled": "false",
"druid.indexer.logs.prefix": "taskLogs",
"druid.indexer.logs.type": "google",
"druid.indexer.queue.startDelay": "PT30S",
"druid.indexer.runner.type": "httpRemote",
"druid.indexer.storage.type": "metadata",
"druid.javascript.enabled": "false",
"druid.lookup.enableLookupSyncOnStartup": "true",
"druid.lookup.lookupTier": "broker",
"druid.metadata.storage.connector.connectURI":
"jdbc:postgresql://***.svc.cluster.local:5432/druid",
"druid.metadata.storage.connector.createTables": "true",
"druid.metadata.storage.type": "postgresql",
"druid.peon.mode": "remote",
"druid.port": "8081",
"druid.query.groupBy.applyLimitPushDownToSegment": "true",
"druid.query.groupBy.maxOnDiskStorage": "1073741824",
"druid.selectors.coordinator.serviceName": "druid/coordinator",
"druid.selectors.indexing.serviceName": "druid/overlord",
"druid.serverview.type": "http",
"druid.service": "druid/coordinator",
"druid.storage.type": "google",
"druid.zk.service.enabled": "false",
"file.encoding": "UTF-8",
"file.encoding.pkg": "sun.io",
"file.separator": "/",
"java.awt.graphicsenv": "sun.awt.X11GraphicsEnvironment",
"java.awt.printerjob": "sun.print.PSPrinterJob",
"java.class.path":
"/tmp/conf/druid/cluster/_common:/tmp/conf/druid/cluster/master/coordinator-overlord:lib/zstd-jni-1.5.2-3.jar:lib/jackson-dataformat-smile-2.10.5.jar:lib/commons-beanutils-1.9.4.jar:lib/classmate-1.1.0.jar:lib/log4j-jul-2.18.0.jar:lib/javax.inject-1.jar:lib/avatica-metrics-1.17.0.jar:lib/avatica-server-1.17.0.jar:lib/aopalliance-1.0.jar:lib/aether-util-0.9.0.M2.jar:lib/jetty-servlet-9.4.48.v20220622.jar:lib/spymemcached-2.12.3.jar:lib/zookeeper-jute-3.5.9.jar:lib/guava-16.0.1.jar:lib/google-oauth-client-1.26.0.jar:lib/druid-core-24.0.0.jar:lib/validation-api-1.1.0.Final.jar:lib/janino-3.0.11.jar:lib/httpclient-4.5.13.jar:lib/accessors-smart-1.2.jar:lib/async-http-client-2.5.3.jar:lib/joda-time-2.10.5.jar:lib/json-smart-2.3.jar:lib/guice-servlet-4.1.0.jar:lib/vavr-match-0.10.2.jar:lib/curator-client-4.3.0.jar:lib/calcite-linq4j-1.21.0.jar:lib/commons-collections4-4.2.jar:lib/curator-recipes-4.3.0.jar:lib/netty-transport-native-epoll-4.1.68.Final-linux-x86_64.ja
r:lib/aggdesigner-algorithm-6.0.jar:lib/jna-4.5.1.jar:lib/jackson-jaxrs-json-provider-2.10.5.jar:lib/asm-9.3.jar:lib/netty-reactive-streams-2.0.0.jar:lib/metrics-core-4.0.0.jar:lib/commons-collections-3.2.2.jar:lib/guice-assistedinject-4.1.0.jar:lib/derby-10.14.2.0.jar:lib/fastutil-8.5.4.jar:lib/google-api-client-1.26.0.jar:lib/netty-buffer-4.1.68.Final.jar:lib/druid-sql-24.0.0.jar:lib/log4j-slf4j-impl-2.18.0.jar:lib/commons-compiler-3.0.11.jar:lib/caffeine-2.8.0.jar:lib/jackson-module-guice-2.10.5.jar:lib/jetty-security-9.4.48.v20220622.jar:lib/maven-repository-metadata-3.1.1.jar:lib/commons-io-2.11.0.jar:lib/jackson-jq-0.0.10.jar:lib/asm-commons-9.3.jar:lib/compress-lzf-1.0.4.jar:lib/commons-codec-1.13.jar:lib/protobuf-java-3.11.0.jar:lib/config-magic-0.9.jar:lib/airline-io-2.8.4.jar:lib/audience-annotations-0.5.0.jar:lib/fastutil-extra-8.5.4.jar:lib/txw2-2.3.1.jar:lib/jersey-core-1.19.4.jar:lib/avatica-core-1.17.0.jar:lib/netty-3.10.6.Final.jar:lib/fastutil-core-8.5.4.jar:lib/dru
id-indexing-hadoop-24.0.0.jar:lib/jetty-client-9.4.48.v20220622.jar:lib/druid-indexing-service-24.0.0.jar:lib/commons-compress-1.21.jar:lib/reactive-streams-1.0.2.jar:lib/maven-model-builder-3.1.1.jar:lib/javax.el-api-3.0.0.jar:lib/netty-resolver-dns-4.1.68.Final.jar:lib/hibernate-validator-5.2.5.Final.jar:lib/druid-aws-common-24.0.0.jar:lib/sigar-1.6.5.132.jar:lib/maven-settings-3.1.1.jar:lib/log4j-1.2-api-2.18.0.jar:lib/resilience4j-bulkhead-1.3.1.jar:lib/icu4j-55.1.jar:lib/jackson-jaxrs-base-2.10.5.jar:lib/stax-ex-1.8.jar:lib/aether-connector-okhttp-0.0.9.jar:lib/javax.activation-api-1.2.0.jar:lib/netty-codec-4.1.68.Final.jar:lib/calcite-core-1.21.0.jar:lib/netty-codec-http-4.1.68.Final.jar:lib/aws-java-sdk-sts-1.12.264.jar:lib/jersey-servlet-1.19.4.jar:lib/json-path-2.3.0.jar:lib/druid-server-24.0.0.jar:lib/jetty-proxy-9.4.48.v20220622.jar:lib/lz4-java-1.8.0.jar:lib/jaxb-runtime-2.3.1.jar:lib/maven-artifact-3.6.0.jar:lib/jackson-module-jaxb-annotations-2.10.5.jar:lib/antlr4-runt
ime-4.5.1.jar:lib/curator-framework-4.3.0.jar:lib/druid-console-24.0.0.jar:lib/datasketches-java-3.2.0.jar:lib/RoaringBitmap-0.9.0.jar:lib/guice-4.1.0.jar:lib/log4j-api-2.18.0.jar:lib/jakarta.inject-api-1.0.3.jar:lib/netty-codec-socks-4.1.68.Final.jar:lib/jackson-annotations-2.10.5.jar:lib/derbyclient-10.14.2.0.jar:lib/aether-connector-file-0.9.0.M2.jar:lib/aws-java-sdk-ec2-1.12.264.jar:lib/disruptor-3.3.6.jar:lib/datasketches-memory-2.0.0.jar:lib/wagon-provider-api-2.4.jar:lib/xz-1.8.jar:lib/jetty-util-ajax-9.4.48.v20220622.jar:lib/commons-dbcp2-2.0.1.jar:lib/netty-handler-proxy-4.1.68.Final.jar:lib/javax.activation-1.2.0.jar:lib/jcodings-1.0.43.jar:lib/cron-scheduler-0.1.jar:lib/jsr305-2.0.1.jar:lib/aws-java-sdk-kms-1.12.264.jar:lib/jakarta.xml.bind-api-2.3.2.jar:lib/FastInfoset-1.2.15.jar:lib/jackson-datatype-joda-2.10.5.jar:lib/jcl-over-slf4j-1.7.36.jar:lib/druid-services-24.0.0.jar:lib/jetty-server-9.4.48.v20220622.jar:lib/curator-x-discovery-4.3.0.jar:lib/jvm-attach-api-1.5.ja
r:lib/jackson-jaxrs-smile-provider-2.10.5.jar:lib/maven-settings-builder-3.1.1.jar:lib/javax.el-3.0.0.jar:lib/google-http-client-1.26.0.jar:lib/rhino-1.7.11.jar:lib/jdbi-2.63.1.jar:lib/extendedset-24.0.0.jar:lib/netty-handler-4.1.68.Final.jar:lib/error_prone_annotations-2.11.0.jar:lib/commons-lang-2.6.jar:lib/jetty-util-9.4.48.v20220622.jar:lib/aether-api-0.9.0.M2.jar:lib/maven-aether-provider-3.1.1.jar:lib/commons-lang3-3.8.1.jar:lib/plexus-interpolation-1.19.jar:lib/httpcore-4.4.11.jar:lib/commons-pool2-2.2.jar:lib/druid-hll-24.0.0.jar:lib/jetty-continuation-9.4.48.v20220622.jar:lib/joni-2.1.27.jar:lib/istack-commons-runtime-3.0.7.jar:lib/aws-java-sdk-s3-1.12.264.jar:lib/ion-java-1.0.2.jar:lib/netty-common-4.1.68.Final.jar:lib/netty-transport-4.1.68.Final.jar:lib/jetty-rewrite-9.4.48.v20220622.jar:lib/jersey-server-1.19.4.jar:lib/derbynet-10.14.2.0.jar:lib/jetty-servlets-9.4.48.v20220622.jar:lib/tesla-aether-0.0.5.jar:lib/jmespath-java-1.12.264.jar:lib/zookeeper-3.5.9.jar:lib/asyn
c-http-client-netty-utils-2.5.3.jar:lib/druid-processing-24.0.0.jar:lib/jetty-http-9.4.48.v20220622.jar:lib/slf4j-api-1.7.36.jar:lib/jackson-dataformat-cbor-2.10.5.jar:lib/jakarta.activation-api-1.2.1.jar:lib/aws-java-sdk-core-1.12.264.jar:lib/jaxb-api-2.3.1.jar:lib/j2objc-annotations-1.1.jar:lib/aether-spi-0.9.0.M2.jar:lib/jsr311-api-1.1.1.jar:lib/commons-math3-3.6.1.jar:lib/ipaddress-5.3.4.jar:lib/jersey-guice-1.19.4.jar:lib/airline-2.8.4.jar:lib/jackson-datatype-guava-2.10.5.jar:lib/commons-logging-1.1.1.jar:lib/asm-tree-9.3.jar:lib/netty-resolver-4.1.68.Final.jar:lib/jackson-core-2.10.5.jar:lib/commons-text-1.3.jar:lib/resilience4j-core-1.3.1.jar:lib/maven-model-3.1.1.jar:lib/netty-transport-native-unix-common-4.1.68.Final.jar:lib/checker-qual-2.5.7.jar:lib/okhttp-1.0.2.jar:lib/guice-multibindings-4.1.0.jar:lib/javax.servlet-api-3.1.0.jar:lib/jetty-io-9.4.48.v20220622.jar:lib/vavr-0.10.2.jar:lib/google-http-client-jackson2-1.26.0.jar:lib/netty-codec-dns-4.1.68.Final.jar:lib/open
csv-4.6.jar:lib/jackson-databind-2.10.5.1.jar:lib/aether-impl-0.9.0.M2.jar:lib/plexus-utils-3.0.24.jar:lib/jboss-logging-3.2.1.Final.jar:lib/esri-geometry-api-2.2.0.jar:lib/asm-analysis-9.3.jar:lib/druid-gcp-common-24.0.0.jar:lib/log4j-core-2.18.0.jar:lib/shims-0.9.0.jar:",
"java.class.version": "52.0",
"java.endorsed.dirs": "/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/endorsed",
"java.ext.dirs":
"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/ext:/usr/java/packages/lib/ext",
"java.home": "/usr/lib/jvm/java-8-openjdk-amd64/jre",
"java.io.tmpdir": "/druid/data",
"java.library.path":
"/usr/java/packages/lib/amd64:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni:/lib:/usr/lib",
"java.runtime.name": "OpenJDK Runtime Environment",
"java.runtime.version": "1.8.0_275-8u275-b01-1~deb9u1-b01",
"java.specification.name": "Java Platform API Specification",
"java.specification.vendor": "Oracle Corporation",
"java.specification.version": "1.8",
"java.util.logging.manager": "org.apache.logging.log4j.jul.LogManager",
"java.vendor": "Oracle Corporation",
"java.vendor.url": "http://java.oracle.com/";,
"java.vendor.url.bug": "http://bugreport.sun.com/bugreport/";,
"java.version": "1.8.0_275",
"java.vm.info": "mixed mode",
"java.vm.name": "OpenJDK 64-Bit Server VM",
"java.vm.specification.name": "Java Virtual Machine Specification",
"java.vm.specification.vendor": "Oracle Corporation",
"java.vm.specification.version": "1.8",
"java.vm.vendor": "Oracle Corporation",
"java.vm.version": "25.275-b01",
"line.separator": "\n",
"log4j.shutdownCallbackRegistry":
"org.apache.druid.common.config.Log4jShutdown",
"log4j.shutdownHookEnabled": "true",
"log4j2.is.webapp": "false",
"net.spy.log.LoggerImpl": "net.spy.memcached.compat.log.SLF4JLogger",
"org.jboss.logging.provider": "slf4j",
"os.arch": "amd64",
"os.name": "Linux",
"os.version": "5.4.202+",
"path.separator": ":",
"sun.arch.data.model": "64",
"sun.boot.class.path":
"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/resources.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/rt.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/sunrsasign.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jsse.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jce.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/charsets.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jfr.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/classes",
"sun.boot.library.path": "/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64",
"sun.cpu.endian": "little",
"sun.cpu.isalist": "",
"sun.io.unicode.encoding": "UnicodeLittle",
"sun.java.command": "org.apache.druid.cli.Main server coordinator",
"sun.java.launcher": "SUN_STANDARD",
"sun.jnu.encoding": "UTF-8",
"sun.management.compiler": "HotSpot 64-Bit Tiered Compilers",
"sun.os.patch.level": "unknown",
"user.dir": "/opt/druid",
"user.home": "/root",
"user.language": "en",
"user.name": "root",
"user.timezone": "UTC"
}
```
</details>
There are no error messages, but this is the log from the Middle-Manager and
the Coordinator/Overlord leader. The other Middle-Managers have the same logs.
<details>
<summary>Middle-Manager</summary>
```
2022-11-08T17:56:41+0000 startup service middleManager
Setting druid.host=10.28.1.28 in
/tmp/conf/druid/cluster/data/middleManager/runtime.properties
2022-11-08T17:56:49,272 WARN [main]
org.apache.druid.k8s.discovery.K8sDiscoveryConfig - IF coordinator pods run in
multiple namespaces, then you MUST provide
coordinatorLeaderElectionConfigMapNamespace
2022-11-08T17:56:49,276 WARN [main]
org.apache.druid.k8s.discovery.K8sDiscoveryConfig - IF overlord pods run in
multiple namespaces, then you MUST provide
overlordLeaderElectionConfigMapNamespace
2022-11-08T17:56:52,417 WARN [main]
org.eclipse.jetty.server.handler.gzip.GzipHandler - minGzipSize of 0 is
inefficient for short content, break even is size 23
2022-11-08T17:56:56,269 WARN [qtp546936087-66]
org.apache.druid.indexing.common.config.TaskConfig - Batch processing mode
argument value is null or not valid:[null], defaulting to[CLOSED_SEGMENTS]
```
</details>
<details>
<summary>Coordinator/Overlord</summary>
```
2022-11-08T14:21:20+0000 startup service coordinator
Setting druid.host=10.28.27.21 in
/tmp/conf/druid/cluster/master/coordinator-overlord/runtime.properties
2022-11-08T14:21:25,787 WARN [main]
org.apache.druid.k8s.discovery.K8sDiscoveryConfig - IF coordinator pods run in
multiple namespaces, then you MUST provide
coordinatorLeaderElectionConfigMapNamespace
2022-11-08T14:21:25,792 WARN [main]
org.apache.druid.k8s.discovery.K8sDiscoveryConfig - IF overlord pods run in
multiple namespaces, then you MUST provide
overlordLeaderElectionConfigMapNamespace
2022-11-08T14:21:28,176 WARN [main]
org.eclipse.jetty.server.handler.gzip.GzipHandler - minGzipSize of 0 is
inefficient for short content, break even is size 23
2022-11-08T14:21:29,694 WARN
[K8sDruidNodeDiscoveryProvider-ListenerExecutor]
org.apache.druid.discovery.DruidNodeDiscoveryProvider$ServiceDruidNodeDiscovery
- Node[DiscoveryDruidNode{druidNode=DruidNode{serviceName='druid/broker',
host='10.28.4.25', bindOnHost=false, port=-1, plaintextPort=8088,
enablePlaintextPort=true, tlsPort=-1, enableTlsPort=false}, nodeRole='BROKER',
services={lookupNodeService=LookupNodeService{lookupTier='broker'}}}]
discovered but doesn't have service[dataNodeService]. Ignored.
2022-11-08T14:21:29,695 WARN
[K8sDruidNodeDiscoveryProvider-ListenerExecutor]
org.apache.druid.discovery.DruidNodeDiscoveryProvider$ServiceDruidNodeDiscovery
- Node[DiscoveryDruidNode{druidNode=DruidNode{serviceName='druid/broker',
host='10.28.8.20', bindOnHost=false, port=-1, plaintextPort=8088,
enablePlaintextPort=true, tlsPort=-1, enableTlsPort=false}, nodeRole='BROKER',
services={lookupNodeService=LookupNodeService{lookupTier='broker'}}}]
discovered but doesn't have service[dataNodeService]. Ignored.
2022-11-08T14:21:29,695 WARN
[K8sDruidNodeDiscoveryProvider-ListenerExecutor]
org.apache.druid.discovery.DruidNodeDiscoveryProvider$ServiceDruidNodeDiscovery
- Node[DiscoveryDruidNode{druidNode=DruidNode{serviceName='druid/broker',
host='10.28.15.3', bindOnHost=false, port=-1, plaintextPort=8088,
enablePlaintextPort=true, tlsPort=-1, enableTlsPort=false}, nodeRole='BROKER',
services={lookupNodeService=LookupNodeService{lookupTier='broker'}}}]
discovered but doesn't have service[dataNodeService]. Ignored.
2022-11-08T14:21:29,695 WARN
[K8sDruidNodeDiscoveryProvider-ListenerExecutor]
org.apache.druid.discovery.DruidNodeDiscoveryProvider$ServiceDruidNodeDiscovery
- Node[DiscoveryDruidNode{druidNode=DruidNode{serviceName='druid/broker',
host='10.28.2.24', bindOnHost=false, port=-1, plaintextPort=8088,
enablePlaintextPort=true, tlsPort=-1, enableTlsPort=false}, nodeRole='BROKER',
services={lookupNodeService=LookupNodeService{lookupTier='broker'}}}]
discovered but doesn't have service[dataNodeService]. Ignored.
2022-11-08T14:21:29,696 WARN
[K8sDruidNodeDiscoveryProvider-ListenerExecutor]
org.apache.druid.discovery.DruidNodeDiscoveryProvider$ServiceDruidNodeDiscovery
- Node[DiscoveryDruidNode{druidNode=DruidNode{serviceName='druid/broker',
host='10.28.0.12', bindOnHost=false, port=-1, plaintextPort=8088,
enablePlaintextPort=true, tlsPort=-1, enableTlsPort=false}, nodeRole='BROKER',
services={lookupNodeService=LookupNodeService{lookupTier='broker'}}}]
discovered but doesn't have service[dataNodeService]. Ignored.
```
</details>
This seems to be the same symptom as [this
thread](https://www.druidforum.org/t/middlemanagers-keep-disappearing/8001) on
the forum.
This error doesn't happen when using ZooKeeper for leader election, but
using ZooKeeper brings our whole cluster down whenever something forcefully
breaks the connection between the Coordinators and ZooKeeper due to #13167
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
