soullkk opened a new issue, #13972: URL: https://github.com/apache/druid/issues/13972
### Peon errors when network address is ipv6 2023-03-22 10:48:58,524 ERROR [task-runner-0-priority-0][][org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskRunner] Encountered exception in run() before persisting. java.io.IOException: org.apache.druid.rpc.RpcException: Service [OVERLORD] redirected too many times [3] to invalid url https://[fec0:125:1401:1615::2110:246]:26203/druid/indexer/v1/action at org.apache.druid.indexing.common.actions.RemoteTaskActionClient.submit(RemoteTaskActionClient.java:95) ~[druid-indexing-service-24.0.1-h0.gdd.sop.r4.jar:?] at org.apache.druid.indexing.appenderator.ActionBasedSegmentAllocator.allocate(ActionBasedSegmentAllocator.java:57) ~[druid-indexing-service-24.0.1-h0.gdd.sop.r4.jar:?] at org.apache.druid.segment.realtime.appenderator.BaseAppenderatorDriver.getSegment(BaseAppenderatorDriver.java:338) ~[druid-server-24.0.1-h0.gdd.sop.r4.jar:?] at org.apache.druid.segment.realtime.appenderator.BaseAppenderatorDriver.append(BaseAppenderatorDriver.java:407) ~[druid-server-24.0.1-h0.gdd.sop.r4.jar:?] at org.apache.druid.segment.realtime.appenderator.StreamAppenderatorDriver.add(StreamAppenderatorDriver.java:192) ~[druid-server-24.0.1-h0.gdd.sop.r4.jar:?] at org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskRunner.runInternal(SeekableStreamIndexTaskRunner.java:655) ~[druid-indexing-service-24.0.1-h0.gdd.sop.r4.jar:?] at org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskRunner.run(SeekableStreamIndexTaskRunner.java:267) ~[druid-indexing-service-24.0.1-h0.gdd.sop.r4.jar:?] at org.apache.druid.indexing.seekablestream.SeekableStreamIndexTask.run(SeekableStreamIndexTask.java:151) ~[druid-indexing-service-24.0.1-h0.gdd.sop.r4.jar:?] at org.apache.druid.indexing.overlord.SingleTaskBackgroundRunner$SingleTaskBackgroundRunnerCallable.call(SingleTaskBackgroundRunner.java:477) ~[druid-indexing-service-24.0.1-h0.gdd.sop.r4.jar:?] at org.apache.druid.indexing.overlord.SingleTaskBackgroundRunner$SingleTaskBackgroundRunnerCallable.call(SingleTaskBackgroundRunner.java:449) ~[druid-indexing-service-24.0.1-h0.gdd.sop.r4.jar:?] at com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:125) ~[guava-27.0-jre.jar:?] at com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:57) ~[guava-27.0-jre.jar:?] at com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:78) ~[guava-27.0-jre.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_352] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_352] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_352] Caused by: org.apache.druid.rpc.RpcException: Service [OVERLORD] redirected too many times [3] to invalid url https://[fec0:125:1401:1615::2110:246]:26203/druid/indexer/v1/action at org.apache.druid.rpc.ServiceClientImpl$1.onSuccess(ServiceClientImpl.java:206) ~[druid-server-24.0.1-h0.gdd.sop.r4.jar:?] at org.apache.druid.rpc.ServiceClientImpl$1.onSuccess(ServiceClientImpl.java:168) ~[druid-server-24.0.1-h0.gdd.sop.r4.jar:?] at com.google.common.util.concurrent.Futures$CallbackListener.run(Futures.java:1062) ~[guava-27.0-jre.jar:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_352] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_352] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) ~[?:1.8.0_352] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) ~[?:1.8.0_352] ... 3 more 2023-03-22 10:48:58,657 ERROR [task-runner-0-priority-0][][org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskRunner] Encountered exception while running task. java.io.IOException: org.apache.druid.rpc.RpcException: Service [OVERLORD] redirected too many times [3] to invalid url https://[fec0:125:1401:1615::2110:246]:26203/druid/indexer/v1/action at org.apache.druid.indexing.common.actions.RemoteTaskActionClient.submit(RemoteTaskActionClient.java:95) ~[druid-indexing-service-24.0.1-h0.gdd.sop.r4.jar:?] at org.apache.druid.indexing.appenderator.ActionBasedSegmentAllocator.allocate(ActionBasedSegmentAllocator.java:57) ~[druid-indexing-service-24.0.1-h0.gdd.sop.r4.jar:?] at org.apache.druid.segment.realtime.appenderator.BaseAppenderatorDriver.getSegment(BaseAppenderatorDriver.java:338) ~[druid-server-24.0.1-h0.gdd.sop.r4.jar:?] at org.apache.druid.segment.realtime.appenderator.BaseAppenderatorDriver.append(BaseAppenderatorDriver.java:407) ~[druid-server-24.0.1-h0.gdd.sop.r4.jar:?] at org.apache.druid.segment.realtime.appenderator.StreamAppenderatorDriver.add(StreamAppenderatorDriver.java:192) ~[druid-server-24.0.1-h0.gdd.sop.r4.jar:?] at org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskRunner.runInternal(SeekableStreamIndexTaskRunner.java:655) ~[druid-indexing-service-24.0.1-h0.gdd.sop.r4.jar:?] at org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskRunner.run(SeekableStreamIndexTaskRunner.java:267) ~[druid-indexing-service-24.0.1-h0.gdd.sop.r4.jar:?] at org.apache.druid.indexing.seekablestream.SeekableStreamIndexTask.run(SeekableStreamIndexTask.java:151) ~[druid-indexing-service-24.0.1-h0.gdd.sop.r4.jar:?] at org.apache.druid.indexing.overlord.SingleTaskBackgroundRunner$SingleTaskBackgroundRunnerCallable.call(SingleTaskBackgroundRunner.java:477) ~[druid-indexing-service-24.0.1-h0.gdd.sop.r4.jar:?] at org.apache.druid.indexing.overlord.SingleTaskBackgroundRunner$SingleTaskBackgroundRunnerCallable.call(SingleTaskBackgroundRunner.java:449) ~[druid-indexing-service-24.0.1-h0.gdd.sop.r4.jar:?] at com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:125) ~[guava-27.0-jre.jar:?] at com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:57) ~[guava-27.0-jre.jar:?] at com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:78) ~[guava-27.0-jre.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_352] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_352] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_352] Caused by: org.apache.druid.rpc.RpcException: Service [OVERLORD] redirected too many times [3] to invalid url https://[fec0:125:1401:1615::2110:246]:26203/druid/indexer/v1/action at org.apache.druid.rpc.ServiceClientImpl$1.onSuccess(ServiceClientImpl.java:206) ~[druid-server-24.0.1-h0.gdd.sop.r4.jar:?] at org.apache.druid.rpc.ServiceClientImpl$1.onSuccess(ServiceClientImpl.java:168) ~[druid-server-24.0.1-h0.gdd.sop.r4.jar:?] at com.google.common.util.concurrent.Futures$CallbackListener.run(Futures.java:1062) ~[guava-27.0-jre.jar:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_352] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_352] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) ~[?:1.8.0_352] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) ~[?:1.8.0_352] ... 3 more ### Affected Version druid 24.0.1 version ### Description Please include as much detailed information about the problem as possible. - total 4 nodes in cluster - use ipv6 for network address When obtaining the master node IP of overlord，i found that the redirected ip is [xxxxxx: 227]，however，The ip of http request is [xxxxxx:246]. I added a log in ServiceClientImpl#tryRequest() to confirm this.  2023-03-23 11:33:47,337 WARN [ServiceClientFactory-0][][org.apache.druid.rpc.ServiceClientImpl] test0323 redirect url is [https://[xxxxxx:227]:26203/druid/indexer/v1/action], request url is [https://[xxxxxx:246]:26203/druid/indexer/v1/action] 2023-03-23 11:33:47,372 WARN [ServiceClientFactory-2][][org.apache.druid.rpc.ServiceClientImpl] test0323 redirect url is [https://[xxxxxx:227]:26203/druid/indexer/v1/action], request url is [https://[xxxxxx:246]:26203/druid/indexer/v1/action] 2023-03-23 11:33:47,386 WARN [ServiceClientFactory-0][][org.apache.druid.rpc.ServiceClientImpl] test0323 redirect url is [https://[xxxxxx:227]:26203/druid/indexer/v1/action], request url is [https://[xxxxxx:246]:26203/druid/indexer/v1/action] Why not to use [xxxxxx:227]？when to pick serviceLocation，preferred serviceLocation and serviceLocation in the list of ServiceLocations are not equal. So it returned the first one in the list. i add a log in ServiceClinetImpl#pick()  2023-03-23 11:33:47,345 INFO [ServiceClientFactory-0][][org.apache.druid.rpc.ServiceClientImpl] test0323 pick: ServiceLocation{host='xxxxxx:246', plaintextPort=-1, tlsPort=26203, basePath=''} equals ServiceLocation{host='[xxxxxx:227]', plaintextPort=-1, tlsPort=26203, basePath=''}, result: false 2023-03-23 11:33:47,345 INFO [ServiceClientFactory-0][][org.apache.druid.rpc.ServiceClientImpl] test0323 pick: ServiceLocation{host='xxxxxx:1141', plaintextPort=-1, tlsPort=26203, basePath=''} equals ServiceLocation{host='[xxxxxx:227]', plaintextPort=-1, tlsPort=26203, basePath=''}, result: false 2023-03-23 11:33:47,346 INFO [ServiceClientFactory-0][][org.apache.druid.rpc.ServiceClientImpl] test0323 pick: ServiceLocation{host='xxxxxx:227', plaintextPort=-1, tlsPort=26203, basePath=''} equals ServiceLocation{host='[xxxxxx:227]', plaintextPort=-1, tlsPort=26203, basePath=''}, result: false We can find that the host of prefered serviceLocation has brackets, but the host in the list of ServiceLocations does not，so it caused too many times [3] invalid url to redirected -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
