#general
@e-ramirez: @e-ramirez has joined the channel
#random
@e-ramirez: @e-ramirez has joined the channel
#troubleshooting
@e-ramirez: @e-ramirez has joined the channel
@fabricio.dutra87: Hi team, I'm trying to setup MinIO to be used as deep store for Pinot. After including the extra configs for controller and server I'm getting this kind of error on controller which refuses connection to port 9000. Does anyone have any ideia how to fix it? controller: ``` name: controller port: 9000 replicaCount: 1 persistence: enabled: true accessMode: ReadWriteOnce size: 1G mountPath: /var/pinot/controller/data storageClass: "csi-cinder-high-speed" data: #dir: /var/pinot/controller/data #dir:
@fx19880617: what’s the error msg you are getting from the pinot?
@fabricio.dutra87: ```2021/02/21 23:02:56.570 ERROR [AddTableCommand] [main] Got Exception to upload Pinot Schema: sch_users_json shaded.org.apache.http.conn.HttpHostConnectException: Connect to 10.2.4.133:9000 [/10.2.4.133] failed: Connection refused (Connection refused) at shaded.org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:156) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:374) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:393) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:108) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.common.utils.FileUploadDownloadClient.sendRequest(FileUploadDownloadClient.java:383) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.common.utils.FileUploadDownloadClient.addSchema(FileUploadDownloadClient.java:448) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.command.AddTableCommand.uploadSchema(AddTableCommand.java:135) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.command.AddTableCommand.execute(AddTableCommand.java:170) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.PinotAdministrator.execute(PinotAdministrator.java:164) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.PinotAdministrator.main(PinotAdministrator.java:184) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] Caused by: java.net.ConnectException: Connection refused (Connection refused) at java.net.PlainSocketImpl.socketConnect(Native Method) ~[?:1.8.0_282] at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350) ~[?:1.8.0_282] at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206) ~[?:1.8.0_282] at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188) ~[?:1.8.0_282] at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) ~[?:1.8.0_282] at java.net.Socket.connect(Socket.java:607) ~[?:1.8.0_282] at shaded.org.apache.http.conn.socket.PlainConnectionSocketFactory.connectSocket(PlainConnectionSocketFactory.java:75) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:142) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] ... 15 more 2021/02/21 23:02:56.577 ERROR [PinotAdministrator] [main] Exception caught: shaded.org.apache.http.conn.HttpHostConnectException: Connect to 10.2.4.133:9000 [/10.2.4.133] failed: Connection refused (Connection refused) at shaded.org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:156) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:374) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:393) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:108) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.common.utils.FileUploadDownloadClient.sendRequest(FileUploadDownloadClient.java:383) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.common.utils.FileUploadDownloadClient.addSchema(FileUploadDownloadClient.java:448) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.command.AddTableCommand.uploadSchema(AddTableCommand.java:135) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.command.AddTableCommand.execute(AddTableCommand.java:170) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.PinotAdministrator.execute(PinotAdministrator.java:164) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.PinotAdministrator.main(PinotAdministrator.java:184) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] Caused by: java.net.ConnectException: Connection refused (Connection refused) at java.net.PlainSocketImpl.socketConnect(Native Method) ~[?:1.8.0_282] at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350) ~[?:1.8.0_282] at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206) ~[?:1.8.0_282] at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188) ~[?:1.8.0_282] at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) ~[?:1.8.0_282] at java.net.Socket.connect(Socket.java:607) ~[?:1.8.0_282] at shaded.org.apache.http.conn.socket.PlainConnectionSocketFactory.connectSocket(PlainConnectionSocketFactory.java:75) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:142) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] ... 15 more```
@fx19880617: is this a valid ip: `10.2.4.133:9000` ?
@fx19880617: did you try the controller external service ip？
@fabricio.dutra87: yes, that's the internal ip and I didnt try the external one yet. How can I test it?
@fx19880617: where did you run the add table cmd
@fabricio.dutra87: inside the controller: bin/pinot-admin.sh AddTable \ -schemaFile /opt/pinot/sch_users_json_v2.json \ -tableConfigFile /opt/pinot/realtime_users_json_events_v2.json \ -exec
@fx19880617: ok, then can you ping this ip from the controller container?
@fx19880617: also this means your controller is actually up running ?
@fabricio.dutra87: yes I can ping it and controller up and running
@fx19880617: Can you try this : ```bin/pinot-admin.sh AddTable \ -schemaFile /opt/pinot/sch_users_json_v2.json \ -tableConfigFile /opt/pinot/realtime_users_json_events_v2.json \ -controllerHost pinot-controller \ -exec```
@fabricio.dutra87: still got the error: bin/pinot-admin.sh AddTable \ -schemaFile /opt/pinot/sch_users_json_v3.json \ -tableConfigFile /opt/pinot/realtime_users_json_events_v3.json \ -controllerHost pinot-controller \ -exec
@fabricio.dutra87: ```2021/02/21 23:48:18.090 ERROR [AddTableCommand] [main] Got Exception to upload Pinot Schema: sch_users_json_v3 shaded.org.apache.http.conn.HttpHostConnectException: Connect to pinot-controller:9000 [pinot-controller/10.3.116.202] failed: Connection refused (Connection refused) at shaded.org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:156) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:374) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:393) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:108) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.common.utils.FileUploadDownloadClient.sendRequest(FileUploadDownloadClient.java:383) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.common.utils.FileUploadDownloadClient.addSchema(FileUploadDownloadClient.java:448) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.command.AddTableCommand.uploadSchema(AddTableCommand.java:135) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.command.AddTableCommand.execute(AddTableCommand.java:170) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.PinotAdministrator.execute(PinotAdministrator.java:164) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.PinotAdministrator.main(PinotAdministrator.java:184) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] Caused by: java.net.ConnectException: Connection refused (Connection refused) at java.net.PlainSocketImpl.socketConnect(Native Method) ~[?:1.8.0_282] at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350) ~[?:1.8.0_282] at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206) ~[?:1.8.0_282] at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188) ~[?:1.8.0_282] at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) ~[?:1.8.0_282] at java.net.Socket.connect(Socket.java:607) ~[?:1.8.0_282] at shaded.org.apache.http.conn.socket.PlainConnectionSocketFactory.connectSocket(PlainConnectionSocketFactory.java:75) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:142) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] ... 15 more 2021/02/21 23:48:18.095 ERROR [PinotAdministrator] [main] Exception caught: shaded.org.apache.http.conn.HttpHostConnectException: Connect to pinot-controller:9000 [pinot-controller/10.3.116.202] failed: Connection refused (Connection refused) at shaded.org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:156) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:374) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:393) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:108) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.common.utils.FileUploadDownloadClient.sendRequest(FileUploadDownloadClient.java:383) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.common.utils.FileUploadDownloadClient.addSchema(FileUploadDownloadClient.java:448) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.command.AddTableCommand.uploadSchema(AddTableCommand.java:135) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.command.AddTableCommand.execute(AddTableCommand.java:170) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.PinotAdministrator.execute(PinotAdministrator.java:164) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.PinotAdministrator.main(PinotAdministrator.java:184) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] Caused by: java.net.ConnectException: Connection refused (Connection refused) at java.net.PlainSocketImpl.socketConnect(Native Method) ~[?:1.8.0_282] at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350) ~[?:1.8.0_282] at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206) ~[?:1.8.0_282] at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188) ~[?:1.8.0_282] at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) ~[?:1.8.0_282] at java.net.Socket.connect(Socket.java:607) ~[?:1.8.0_282] at shaded.org.apache.http.conn.socket.PlainConnectionSocketFactory.connectSocket(PlainConnectionSocketFactory.java:75) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:142) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] ... 15 more```
@fx19880617: hmm
@fx19880617: can you show me your k8s cluster status
@fx19880617: `kubectl get all -n <pinot-namespace>`
@fx19880617: I somehow feel the controller is not really up
@fx19880617: do you have logs for controller?
@fabricio.dutra87: also important to remember is that it runs fine when removing all the extra configs and working with the default deep store
@fx19880617: ic
@fx19880617: then I guess the issue is minio fs
@fx19880617: can pinot controller access the minio ip?
@fabricio.dutra87: yes it can, and also after including this extra configs I cannot access controller UI on port 9000
@fx19880617: ic, can you provide this pinot controller log?
@fx19880617: in side the controller container, there is a file pinotController.log
@fx19880617: can you check if there is any logs from S3PinotFs?
@fabricio.dutra87: it shows the same thing
@fabricio.dutra87: ```2021/02/21 23:51:06.852 INFO [PluginManager] [main] Successfully Loaded plugin [pinot-batch-ingestion-hadoop] from dir [/opt/pinot/plugins/pinot-batch-ingestion/pinot-batch-ingestion-hadoop] 2021/02/21 23:51:06.918 INFO [AddTableCommand] [main] Executing command: AddTable -tableConfigFile /opt/pinot/realtime_users_json_events_v3.json -schemaFile /opt/pinot/sch_users_json_v3.json -controllerProtocol http -controllerHost pinot-controller-0 -controllerPort 9000 -exec 2021/02/21 23:51:07.638 ERROR [AddTableCommand] [main] Got Exception to upload Pinot Schema: sch_users_json_v3 shaded.org.apache.http.conn.HttpHostConnectException: Connect to pinot-controller-0:9000 [pinot-controller-0/10.2.4.133] failed: Connection refused (Connection refused) at shaded.org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:156) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:374) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:393) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:108) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.common.utils.FileUploadDownloadClient.sendRequest(FileUploadDownloadClient.java:383) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.common.utils.FileUploadDownloadClient.addSchema(FileUploadDownloadClient.java:448) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.command.AddTableCommand.uploadSchema(AddTableCommand.java:135) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.command.AddTableCommand.execute(AddTableCommand.java:170) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.PinotAdministrator.execute(PinotAdministrator.java:164) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.PinotAdministrator.main(PinotAdministrator.java:184) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] Caused by: java.net.ConnectException: Connection refused (Connection refused) at java.net.PlainSocketImpl.socketConnect(Native Method) ~[?:1.8.0_282] at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350) ~[?:1.8.0_282] at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206) ~[?:1.8.0_282] at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188) ~[?:1.8.0_282] at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) ~[?:1.8.0_282] at java.net.Socket.connect(Socket.java:607) ~[?:1.8.0_282] at shaded.org.apache.http.conn.socket.PlainConnectionSocketFactory.connectSocket(PlainConnectionSocketFactory.java:75) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:142) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] ... 15 more 2021/02/21 23:51:07.644 ERROR [PinotAdministrator] [main] Exception caught: shaded.org.apache.http.conn.HttpHostConnectException: Connect to pinot-controller-0:9000 [pinot-controller-0/10.2.4.133] failed: Connection refused (Connection refused) at shaded.org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:156) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:374) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:393) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at shaded.org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:108) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.common.utils.FileUploadDownloadClient.sendRequest(FileUploadDownloadClient.java:383) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.common.utils.FileUploadDownloadClient.addSchema(FileUploadDownloadClient.java:448) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SN#```
@fabricio.dutra87: and checking the k8s logs for the controller there's a bunch of warns there but no errors
@fabricio.dutra87: ```2021/02/21 22:51:07.220 WARN [TopStateHandoffReportStage] [HelixController-pipeline-default-pinot-(90285d38_DEFAULT)] Event 90285d38_DEFAULT : Cannot confirm top state missing start time. Use the current system time as the start time. 2021/02/21 22:51:07.220 WARN [TopStateHandoffReportStage] [HelixController-pipeline-default-pinot-(90285d38_DEFAULT)] Event 90285d38_DEFAULT : Cannot confirm top state missing start time. Use the current system time as the start time. 2021/02/21 22:51:07.234 WARN [AutoRebalancer] [HelixController-pipeline-default-pinot-(90285d38_DEFAULT)] Resource leadControllerResource has tag controller but no live participants have this tag 2021/02/21 22:51:07.236 WARN [BestPossibleStateCalcStage] [HelixController-pipeline-default-pinot-(90285d38_DEFAULT)] Event 90285d38_DEFAULT : Failed to calculate best possible states for leadControllerResource 2021/02/21 22:51:07.236 WARN [BestPossibleStateCalcStage] [HelixController-async_tasks-pinot] Event 90285d38_DEFAULT : Failed to calculate best possible states for 1 resources. 2021/02/21 22:51:07.420 WARN [AutoRebalancer] [HelixController-pipeline-default-pinot-(2652243f_DEFAULT)] Resource leadControllerResource has tag controller but no live participants have this tag 2021/02/21 22:51:07.420 WARN [BestPossibleStateCalcStage] [HelixController-pipeline-default-pinot-(2652243f_DEFAULT)] Event 2652243f_DEFAULT : Failed to calculate best possible states for leadControllerResource 2021/02/21 22:51:07.420 WARN [BestPossibleStateCalcStage] [HelixController-async_tasks-pinot] Event 2652243f_DEFAULT : Failed to calculate best possible states for 1 resources. 2021/02/21 22:51:07.496 WARN [AutoRebalancer] [HelixController-pipeline-default-pinot-(5f02e036_DEFAULT)] Resource leadControllerResource has tag controller but no live participants have this tag 2021/02/21 22:51:07.496 WARN [BestPossibleStateCalcStage] [HelixController-pipeline-default-pinot-(5f02e036_DEFAULT)] Event 5f02e036_DEFAULT : Failed to calculate best possible states for leadControllerResource 2021/02/21 22:51:07.497 WARN [BestPossibleStateCalcStage] [HelixController-async_tasks-pinot] Event 5f02e036_DEFAULT : Failed to calculate best possible states for 1 resources. 2021/02/21 22:51:07.550 WARN [AutoRebalancer] [HelixController-pipeline-default-pinot-(cf4d6967_DEFAULT)] Resource leadControllerResource has tag controller but no live participants have this tag 2021/02/21 22:51:07.550 WARN [BestPossibleStateCalcStage] [HelixController-pipeline-default-pinot-(cf4d6967_DEFAULT)] Event cf4d6967_DEFAULT : Failed to calculate best possible states for leadControllerResource 2021/02/21 22:51:07.550 WARN [BestPossibleStateCalcStage] [HelixController-async_tasks-pinot] Event cf4d6967_DEFAULT : Failed to calculate best possible states for 1 resources. 2021/02/21 22:51:07.618 WARN [AutoRebalancer] [HelixController-pipeline-default-pinot-(18405170_DEFAULT)] Resource leadControllerResource has tag controller but no live participants have this tag 2021/02/21 22:51:07.618 WARN [BestPossibleStateCalcStage] [HelixController-pipeline-default-pinot-(18405170_DEFAULT)] Event 18405170_DEFAULT : Failed to calculate best possible states for leadControllerResource 2021/02/21 22:51:07.619 WARN [BestPossibleStateCalcStage] [HelixController-async_tasks-pinot] Event 18405170_DEFAULT : Failed to calculate best possible states for 1 resources. 2021/02/21 22:51:26.649 WARN [AutoRebalancer] [HelixController-pipeline-default-pinot-(25237c3c_DEFAULT)] Resource leadControllerResource has tag controller but no live participants have this tag 2021/02/21 22:51:26.652 WARN [BestPossibleStateCalcStage] [HelixController-pipeline-default-pinot-(25237c3c_DEFAULT)] Event 25237c3c_DEFAULT : Failed to calculate best possible states for leadControllerResource 2021/02/21 22:51:26.652 WARN [BestPossibleStateCalcStage] [HelixController-async_tasks-pinot] Event 25237c3c_DEFAULT : Failed to calculate best possible states for 1 resources. 2021/02/21 22:51:26.798 WARN [ZkBaseDataAccessor] [HelixController-pipeline-default-pinot-(fead5ddd_DEFAULT)] Fail to read record for paths: {/pinot/INSTANCES/Minion_pinot-minion-0.pinot-minion-headless.datastore.svc.cluster.local_9514/MESSAGES/a8a58002-ffdd-419f-a341-2bf7e4bc7e45=-101} 2021/02/21 22:51:26.828 WARN [AutoRebalancer] [HelixController-pipeline-default-pinot-(fead5ddd_DEFAULT)] Resource leadControllerResource has tag controller but no live participants have this tag 2021/02/21 22:51:26.829 WARN [BestPossibleStateCalcStage] [HelixController-pipeline-default-pinot-(fead5ddd_DEFAULT)] Event fead5ddd_DEFAULT : Failed to calculate best possible states for leadControllerResource 2021/02/21 22:51:26.830 WARN [BestPossibleStateCalcStage] [HelixController-async_tasks-pinot] Event fead5ddd_DEFAULT : Failed to calculate best possible states for 1 resources. 2021/02/21 22:51:26.923 WARN [AutoRebalancer] [HelixController-pipeline-default-pinot-(78df67c8_DEFAULT)] Resource leadControllerResource has tag controller but no live participants have this tag 2021/02/21 22:51:26.923 WARN [BestPossibleStateCalcStage] [HelixController-pipeline-default-pinot-(78df67c8_DEFAULT)] Event 78df67c8_DEFAULT : Failed to calculate best possible states for leadControllerResource 2021/02/21 22:51:26.923 WARN [BestPossibleStateCalcStage] [HelixController-async_tasks-pinot] Event 78df67c8_DEFAULT : Failed to calculate best possible states for 1 resources. 2021/02/21 22:51:26.986 WARN [AutoRebalancer] [HelixController-pipeline-default-pinot-(7392b981_DEFAULT)] Resource leadControllerResource has tag controller but no live participants have this tag 2021/02/21 22:51:26.986 WARN [BestPossibleStateCalcStage] [HelixController-pipeline-default-pinot-(7392b981_DEFAULT)] Event 7392b981_DEFAULT : Failed to calculate best possible states for leadControllerResource 2021/02/21 22:51:26.986 WARN [BestPossibleStateCalcStage] [HelixController-async_tasks-pinot] Event 7392b981_DEFAULT : Failed to calculate best possible states for 1 resources. 2021/02/21 22:51:38.298 WARN [AutoRebalancer] [HelixController-pipeline-default-pinot-(30ca2ff1_DEFAULT)] Resource leadControllerResource has tag controller but no live participants have this tag 2021/02/21 22:51:38.301 WARN [BestPossibleStateCalcStage] [HelixController-pipeline-default-pinot-(30ca2ff1_DEFAULT)] Event 30ca2ff1_DEFAULT : Failed to calculate best possible states for leadControllerResource 2021/02/21 22:51:38.301 WARN [BestPossibleStateCalcStage] [HelixController-async_tasks-pinot] Event 30ca2ff1_DEFAULT : Failed to calculate best possible states for 1 resources. 2021/02/21 22:51:38.444 WARN [ZkBaseDataAccessor] [HelixController-pipeline-default-pinot-(3cf6ec26_DEFAULT)] Fail to read record for paths: {/pinot/INSTANCES/Broker_pinot-broker-0.pinot-broker-headless.datastore.svc.cluster.local_8099/MESSAGES/032f358e-e0b8-4fa9-88d7-732f10887ff5=-101} 2021/02/21 22:51:38.460 WARN [AutoRebalancer] [HelixController-pipeline-default-pinot-(3cf6ec26_DEFAULT)] Resource leadControllerResource has tag controller but no live participants have this tag 2021/02/21 22:51:38.466 WARN [BestPossibleStateCalcStage] [HelixController-pipeline-default-pinot-(3cf6ec26_DEFAULT)] Event 3cf6ec26_DEFAULT : Failed to calculate best possible states for leadControllerResource 2021/02/21 22:51:38.466 WARN [BestPossibleStateCalcStage] [HelixController-async_tasks-pinot] Event 3cf6ec26_DEFAULT : Failed to calculate best possible states for 1 resources. 2021/02/21 22:51:38.539 WARN [AutoRebalancer] [HelixController-pipeline-default-pinot-(519c9bd3_DEFAULT)] Resource leadControllerResource has tag controller but no live participants have this tag 2021/02/21 22:51:38.540 WARN [BestPossibleStateCalcStage] [HelixController-pipeline-default-pinot-(519c9bd3_DEFAULT)] Event 519c9bd3_DEFAULT : Failed to calculate best possible states for leadControllerResource```
@fx19880617: hmm
@fx19880617: can you try to restart the controller
@fx19880617: and check the pinotController.log inside the container
@fx19880617: I think the container log is overrided by the bin/pinot-admin.sh cmd
@fabricio.dutra87: lets see
@fabricio.dutra87: I found an error there maybe if I change the pinot.server.segment.fetcher.protocols to only http?
@fabricio.dutra87: ```2021/02/22 00:14:50.254 ERROR [PinotFSFactory] [main] Could not instantiate file system for class org.apache.pinot.plugin.filesystem.S3PinotFS with scheme s3 java.lang.IllegalArgumentException: null at shaded.com.google.common.base.Preconditions.checkArgument(Preconditions.java:108) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.plugin.filesystem.S3PinotFS.init(S3PinotFS.java:95) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.spi.filesystem.PinotFSFactory.register(PinotFSFactory.java:54) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.spi.filesystem.PinotFSFactory.init(PinotFSFactory.java:74) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.controller.ControllerStarter.initPinotFSFactory(ControllerStarter.java:483) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.controller.ControllerStarter.setUpPinotController(ControllerStarter.java:333) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.controller.ControllerStarter.start(ControllerStarter.java:283) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.service.PinotServiceManager.startController(PinotServiceManager.java:116) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.service.PinotServiceManager.startRole(PinotServiceManager.java:91) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.command.StartServiceManagerCommand.lambda$startBootstrapServices$0(StartServiceManagerCommand.java:234) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.command.StartServiceManagerCommand.startPinotService(StartServiceManagerCommand.java:286) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.command.StartServiceManagerCommand.startBootstrapServices(StartServiceManagerCommand.java:233) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.command.StartServiceManagerCommand.execute(StartServiceManagerCommand.java:183) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.command.StartControllerCommand.execute(StartControllerCommand.java:130) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.PinotAdministrator.execute(PinotAdministrator.java:164) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.PinotAdministrator.main(PinotAdministrator.java:184) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] 2021/02/22 00:14:50.259 ERROR [StartServiceManagerCommand] [main] Failed to start a Pinot [CONTROLLER] at 2.957 since launch java.lang.RuntimeException: java.lang.IllegalArgumentException at org.apache.pinot.spi.filesystem.PinotFSFactory.register(PinotFSFactory.java:58) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.spi.filesystem.PinotFSFactory.init(PinotFSFactory.java:74) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.controller.ControllerStarter.initPinotFSFactory(ControllerStarter.java:483) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.controller.ControllerStarter.setUpPinotController(ControllerStarter.java:333) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.controller.ControllerStarter.start(ControllerStarter.java:283) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.service.PinotServiceManager.startController(PinotServiceManager.java:116) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.service.PinotServiceManager.startRole(PinotServiceManager.java:91) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.command.StartServiceManagerCommand.lambda$startBootstrapServices$0(StartServiceManagerCommand.java:234) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.command.StartServiceManagerCommand.startPinotService(StartServiceManagerCommand.java:286) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.command.StartServiceManagerCommand.startBootstrapServices(StartServiceManagerCommand.java:233) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.command.StartServiceManagerCommand.execute(StartServiceManagerCommand.java:183) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.command.StartControllerCommand.execute(StartControllerCommand.java:130) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.PinotAdministrator.execute(PinotAdministrator.java:164) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.PinotAdministrator.main(PinotAdministrator.java:184) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] Caused by: java.lang.IllegalArgumentException at shaded.com.google.common.base.Preconditions.checkArgument(Preconditions.java:108) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.plugin.filesystem.S3PinotFS.init(S3PinotFS.java:95) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.spi.filesystem.PinotFSFactory.register(PinotFSFactory.java:54) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] ... 13 more```
@fx19880617: ``` Preconditions.checkArgument(!isNullOrEmpty(config.getProperty(REGION)));```
@fx19880617: I think it’s missing the config for region
@fx19880617: ```pinot.controller.storage.factory.s3.region=us-east-1```
@fabricio.dutra87: hmmm
@fx19880617: can you try to add this to controller extra config and restart controller
@fabricio.dutra87: just did it but still got no luck
@fabricio.dutra87: ```java.lang.RuntimeException: java.net.URISyntaxException: Illegal character in scheme name at index 0: 10.3.120.223:9000 at org.apache.pinot.plugin.filesystem.S3PinotFS.init(S3PinotFS.java:141) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.spi.filesystem.PinotFSFactory.register(PinotFSFactory.java:54) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.spi.filesystem.PinotFSFactory.init(PinotFSFactory.java:74) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.controller.ControllerStarter.initPinotFSFactory(ControllerStarter.java:483) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.controller.ControllerStarter.setUpPinotController(ControllerStarter.java:333) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.controller.ControllerStarter.start(ControllerStarter.java:283) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.service.PinotServiceManager.startController(PinotServiceManager.java:116) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.service.PinotServiceManager.startRole(PinotServiceManager.java:91) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.command.StartServiceManagerCommand.lambda$startBootstrapServices$0(StartServiceManagerCommand.java:234) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.command.StartServiceManagerCommand.startPinotService(StartServiceManagerCommand.java:286) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.command.StartServiceManagerCommand.startBootstrapServices(StartServiceManagerCommand.java:233) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.command.StartServiceManagerCommand.execute(StartServiceManagerCommand.java:183) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.command.StartControllerCommand.execute(StartControllerCommand.java:130) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.PinotAdministrator.execute(PinotAdministrator.java:164) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.PinotAdministrator.main(PinotAdministrator.java:184) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] Caused by: java.net.URISyntaxException: Illegal character in scheme name at index 0: 10.3.120.223:9000 at java.net.URI$Parser.fail(URI.java:2847) ~[?:1.8.0_282] at java.net.URI$Parser.checkChars(URI.java:3020) ~[?:1.8.0_282] at java.net.URI$Parser.checkChar(URI.java:3030) ~[?:1.8.0_282] at java.net.URI$Parser.parse(URI.java:3046) ~[?:1.8.0_282] at java.net.URI.<init>(URI.java:588) ~[?:1.8.0_282] at org.apache.pinot.plugin.filesystem.S3PinotFS.init(S3PinotFS.java:139) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] ... 14 more 2021/02/22 00:29:30.074 ERROR [StartServiceManagerCommand] [main] Failed to start a Pinot [CONTROLLER] at 2.752 since launch java.lang.RuntimeException: java.lang.RuntimeException: java.net.URISyntaxException: Illegal character in scheme name at index 0: 10.3.120.223:9000 at org.apache.pinot.spi.filesystem.PinotFSFactory.register(PinotFSFactory.java:58) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.spi.filesystem.PinotFSFactory.init(PinotFSFactory.java:74) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.controller.ControllerStarter.initPinotFSFactory(ControllerStarter.java:483) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.controller.ControllerStarter.setUpPinotController(ControllerStarter.java:333) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.controller.ControllerStarter.start(ControllerStarter.java:283) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.service.PinotServiceManager.startController(PinotServiceManager.java:116) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.service.PinotServiceManager.startRole(PinotServiceManager.java:91) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.command.StartServiceManagerCommand.lambda$startBootstrapServices$0(StartServiceManagerCommand.java:234) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.command.StartServiceManagerCommand.startPinotService(StartServiceManagerCommand.java:286) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.command.StartServiceManagerCommand.startBootstrapServices(StartServiceManagerCommand.java:233) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.command.StartServiceManagerCommand.execute(StartServiceManagerCommand.java:183) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.command.StartControllerCommand.execute(StartControllerCommand.java:130) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.PinotAdministrator.execute(PinotAdministrator.java:164) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.tools.admin.PinotAdministrator.main(PinotAdministrator.java:184) [pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] Caused by: java.lang.RuntimeException: java.net.URISyntaxException: Illegal character in scheme name at index 0: 10.3.120.223:9000 at org.apache.pinot.plugin.filesystem.S3PinotFS.init(S3PinotFS.java:141) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.spi.filesystem.PinotFSFactory.register(PinotFSFactory.java:54) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] ... 13 more Caused by: java.net.URISyntaxException: Illegal character in scheme name at index 0: 10.3.120.223:9000 at java.net.URI$Parser.fail(URI.java:2847) ~[?:1.8.0_282] at java.net.URI$Parser.checkChars(URI.java:3020) ~[?:1.8.0_282] at java.net.URI$Parser.checkChar(URI.java:3030) ~[?:1.8.0_282] at java.net.URI$Parser.parse(URI.java:3046) ~[?:1.8.0_282] at java.net.URI.<init>(URI.java:588) ~[?:1.8.0_282] at org.apache.pinot.plugin.filesystem.S3PinotFS.init(S3PinotFS.java:139) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] at org.apache.pinot.spi.filesystem.PinotFSFactory.register(PinotFSFactory.java:54) ~[pinot-all-0.7.0-SNAPSHOT-jar-with-dependencies.jar:0.7.0-SNAPSHOT-d74224a14b07bbe3e5c4909b51856886e29510df] ... 13 more```
@fx19880617:
@fabricio.dutra87: wow after changing that is finally working
@fabricio.dutra87: really appreciate your help!!
--------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@pinot.apache.org For additional commands, e-mail: dev-h...@pinot.apache.org
