[
https://issues.apache.org/jira/browse/CARBONDATA-3177?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Ryan Sun updated CARBONDATA-3177:
---------------------------------
Description:
Hey team,
As per the carbondata-presto integration reference, the carbondata-store is
configured with specified namenode address, e.g.
hdfs://namenode:9000/test/carbondata.
However we leverage namenode federation so the hdfs entry is configured with
nameservice, e.g. hdfs://nameservice1/test/carbondata. The nameservice
information is configured in hdfs-site.xml. It seems that carbondata connector
has no way to load this configuration file. So it fails with exception message:
{code:java}
Query 20181217_142352_01851_paya2 failed: java.lang.IllegalArgumentException:
java.net.UnknownHostException: nameservice1
java.lang.RuntimeException: java.lang.IllegalArgumentException:
java.net.UnknownHostException: nameservice1
at
org.apache.carbondata.presto.impl.CarbonTableReader.updateCarbonFile(CarbonTableReader.java:204)
at
org.apache.carbondata.presto.impl.CarbonTableReader.updateSchemaList(CarbonTableReader.java:216)
at
org.apache.carbondata.presto.impl.CarbonTableReader.getSchemaNames(CarbonTableReader.java:189)
at
org.apache.carbondata.presto.CarbondataMetadata.listSchemaNamesInternal(CarbondataMetadata.java:86)
at
org.apache.carbondata.presto.CarbondataMetadata.getTableMetadata(CarbondataMetadata.java:135)
at
org.apache.carbondata.presto.CarbondataMetadata.getTableMetadataInternal(CarbondataMetadata.java:240)
at
org.apache.carbondata.presto.CarbondataMetadata.getTableMetadata(CarbondataMetadata.java:232)
at
com.facebook.presto.spi.connector.classloader.ClassLoaderSafeConnectorMetadata.getTableMetadata(ClassLoaderSafeConnectorMetadata.java:162)
at
com.facebook.presto.metadata.MetadataManager.getTableMetadata(MetadataManager.java:423)
at
com.facebook.presto.sql.analyzer.StatementAnalyzer$Visitor.visitTable(StatementAnalyzer.java:857)
at
com.facebook.presto.sql.analyzer.StatementAnalyzer$Visitor.visitTable(StatementAnalyzer.java:262)
at com.facebook.presto.sql.tree.Table.accept(Table.java:53)
at com.facebook.presto.sql.tree.AstVisitor.process(AstVisitor.java:27)
at
com.facebook.presto.sql.analyzer.StatementAnalyzer$Visitor.process(StatementAnalyzer.java:276)
at
com.facebook.presto.sql.analyzer.StatementAnalyzer$Visitor.analyzeFrom(StatementAnalyzer.java:1780)
at
com.facebook.presto.sql.analyzer.StatementAnalyzer$Visitor.visitQuerySpecification(StatementAnalyzer.java:962)
at
com.facebook.presto.sql.analyzer.StatementAnalyzer$Visitor.visitQuerySpecification(StatementAnalyzer.java:262)
at
com.facebook.presto.sql.tree.QuerySpecification.accept(QuerySpecification.java:127)
at com.facebook.presto.sql.tree.AstVisitor.process(AstVisitor.java:27)
at
com.facebook.presto.sql.analyzer.StatementAnalyzer$Visitor.process(StatementAnalyzer.java:276)
at
com.facebook.presto.sql.analyzer.StatementAnalyzer$Visitor.process(StatementAnalyzer.java:286)
at
com.facebook.presto.sql.analyzer.StatementAnalyzer$Visitor.visitQuery(StatementAnalyzer.java:683)
at
com.facebook.presto.sql.analyzer.StatementAnalyzer$Visitor.visitQuery(StatementAnalyzer.java:262)
at com.facebook.presto.sql.tree.Query.accept(Query.java:94)
at com.facebook.presto.sql.tree.AstVisitor.process(AstVisitor.java:27)
at
com.facebook.presto.sql.analyzer.StatementAnalyzer$Visitor.process(StatementAnalyzer.java:276)
at
com.facebook.presto.sql.analyzer.StatementAnalyzer.analyze(StatementAnalyzer.java:248)
at com.facebook.presto.sql.analyzer.Analyzer.analyze(Analyzer.java:76)
at com.facebook.presto.sql.analyzer.Analyzer.analyze(Analyzer.java:68)
at
com.facebook.presto.execution.SqlQueryExecution.<init>(SqlQueryExecution.java:206)
at
com.facebook.presto.execution.SqlQueryExecution.<init>(SqlQueryExecution.java:96)
at
com.facebook.presto.execution.SqlQueryExecution$SqlQueryExecutionFactory.createQueryExecution(SqlQueryExecution.java:752)
at
com.facebook.presto.execution.SqlQueryManager.createQueryInternal(SqlQueryManager.java:361)
at
com.facebook.presto.execution.SqlQueryManager.lambda$createQuery$4(SqlQueryManager.java:303)
at
com.facebook.presto.$gen.Presto_0_214_9_g36965f8_dirty__0_214_100_2____20181217_102607_1.run(Unknown
Source)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.IllegalArgumentException: java.net.UnknownHostException:
nameservice1
at
org.apache.hadoop.security.SecurityUtil.buildTokenService(SecurityUtil.java:378)
at
org.apache.hadoop.hdfs.NameNodeProxies.createNonHAProxy(NameNodeProxies.java:310)
at org.apache.hadoop.hdfs.NameNodeProxies.createProxy(NameNodeProxies.java:176)
at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:678)
at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:619)
at
org.apache.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem.java:149)
at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2669)
at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:94)
at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2703)
at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2685)
at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:373)
at org.apache.hadoop.fs.Path.getFileSystem(Path.java:295)
at
org.apache.carbondata.core.datastore.filesystem.AbstractDFSCarbonFile.<init>(AbstractDFSCarbonFile.java:74)
at
org.apache.carbondata.core.datastore.filesystem.AbstractDFSCarbonFile.<init>(AbstractDFSCarbonFile.java:66)
at
org.apache.carbondata.core.datastore.filesystem.HDFSCarbonFile.<init>(HDFSCarbonFile.java:41)
at
org.apache.carbondata.core.datastore.impl.DefaultFileTypeProvider.getCarbonFile(DefaultFileTypeProvider.java:51)
at
org.apache.carbondata.core.datastore.impl.FileFactory.getCarbonFile(FileFactory.java:102)
at
org.apache.carbondata.presto.impl.CarbonTableReader.updateCarbonFile(CarbonTableReader.java:202)
... 39 more
{code}
Any suggestion or workaround is appreciated.
Thanks
Best regards,
Ryan
was:
Hey team,
As per the carbondata-presto integration reference, the carbondata-store is
configured with specified namenode address, e.g.
hdfs://namenode:9000/test/carbondata.
However we leverage namenode federation so the hdfs entry is configured with
nameservice, e.g. hdfs://nameservice1/test/carbondata. The nameservice
information is configured in hdfs-site.xml. It seems that carbondata connector
has no way to load this configuration file. So it fails with exception message:
java.lang.IllegalArgumentException: java.net.UnknownHostException: nameservice1
Any suggestion or workaround is appreciated.
Thanks
Best regards,
Ryan
> Nameservice support for presto on carbondata
> --------------------------------------------
>
> Key: CARBONDATA-3177
> URL: https://issues.apache.org/jira/browse/CARBONDATA-3177
> Project: CarbonData
> Issue Type: Improvement
> Affects Versions: 1.5.1
> Reporter: Ryan Sun
> Priority: Major
>
> Hey team,
>
> As per the carbondata-presto integration reference, the carbondata-store is
> configured with specified namenode address, e.g.
> hdfs://namenode:9000/test/carbondata.
>
> However we leverage namenode federation so the hdfs entry is configured with
> nameservice, e.g. hdfs://nameservice1/test/carbondata. The nameservice
> information is configured in hdfs-site.xml. It seems that carbondata
> connector has no way to load this configuration file. So it fails with
> exception message:
> {code:java}
> Query 20181217_142352_01851_paya2 failed: java.lang.IllegalArgumentException:
> java.net.UnknownHostException: nameservice1
> java.lang.RuntimeException: java.lang.IllegalArgumentException:
> java.net.UnknownHostException: nameservice1
> at
> org.apache.carbondata.presto.impl.CarbonTableReader.updateCarbonFile(CarbonTableReader.java:204)
> at
> org.apache.carbondata.presto.impl.CarbonTableReader.updateSchemaList(CarbonTableReader.java:216)
> at
> org.apache.carbondata.presto.impl.CarbonTableReader.getSchemaNames(CarbonTableReader.java:189)
> at
> org.apache.carbondata.presto.CarbondataMetadata.listSchemaNamesInternal(CarbondataMetadata.java:86)
> at
> org.apache.carbondata.presto.CarbondataMetadata.getTableMetadata(CarbondataMetadata.java:135)
> at
> org.apache.carbondata.presto.CarbondataMetadata.getTableMetadataInternal(CarbondataMetadata.java:240)
> at
> org.apache.carbondata.presto.CarbondataMetadata.getTableMetadata(CarbondataMetadata.java:232)
> at
> com.facebook.presto.spi.connector.classloader.ClassLoaderSafeConnectorMetadata.getTableMetadata(ClassLoaderSafeConnectorMetadata.java:162)
> at
> com.facebook.presto.metadata.MetadataManager.getTableMetadata(MetadataManager.java:423)
> at
> com.facebook.presto.sql.analyzer.StatementAnalyzer$Visitor.visitTable(StatementAnalyzer.java:857)
> at
> com.facebook.presto.sql.analyzer.StatementAnalyzer$Visitor.visitTable(StatementAnalyzer.java:262)
> at com.facebook.presto.sql.tree.Table.accept(Table.java:53)
> at com.facebook.presto.sql.tree.AstVisitor.process(AstVisitor.java:27)
> at
> com.facebook.presto.sql.analyzer.StatementAnalyzer$Visitor.process(StatementAnalyzer.java:276)
> at
> com.facebook.presto.sql.analyzer.StatementAnalyzer$Visitor.analyzeFrom(StatementAnalyzer.java:1780)
> at
> com.facebook.presto.sql.analyzer.StatementAnalyzer$Visitor.visitQuerySpecification(StatementAnalyzer.java:962)
> at
> com.facebook.presto.sql.analyzer.StatementAnalyzer$Visitor.visitQuerySpecification(StatementAnalyzer.java:262)
> at
> com.facebook.presto.sql.tree.QuerySpecification.accept(QuerySpecification.java:127)
> at com.facebook.presto.sql.tree.AstVisitor.process(AstVisitor.java:27)
> at
> com.facebook.presto.sql.analyzer.StatementAnalyzer$Visitor.process(StatementAnalyzer.java:276)
> at
> com.facebook.presto.sql.analyzer.StatementAnalyzer$Visitor.process(StatementAnalyzer.java:286)
> at
> com.facebook.presto.sql.analyzer.StatementAnalyzer$Visitor.visitQuery(StatementAnalyzer.java:683)
> at
> com.facebook.presto.sql.analyzer.StatementAnalyzer$Visitor.visitQuery(StatementAnalyzer.java:262)
> at com.facebook.presto.sql.tree.Query.accept(Query.java:94)
> at com.facebook.presto.sql.tree.AstVisitor.process(AstVisitor.java:27)
> at
> com.facebook.presto.sql.analyzer.StatementAnalyzer$Visitor.process(StatementAnalyzer.java:276)
> at
> com.facebook.presto.sql.analyzer.StatementAnalyzer.analyze(StatementAnalyzer.java:248)
> at com.facebook.presto.sql.analyzer.Analyzer.analyze(Analyzer.java:76)
> at com.facebook.presto.sql.analyzer.Analyzer.analyze(Analyzer.java:68)
> at
> com.facebook.presto.execution.SqlQueryExecution.<init>(SqlQueryExecution.java:206)
> at
> com.facebook.presto.execution.SqlQueryExecution.<init>(SqlQueryExecution.java:96)
> at
> com.facebook.presto.execution.SqlQueryExecution$SqlQueryExecutionFactory.createQueryExecution(SqlQueryExecution.java:752)
> at
> com.facebook.presto.execution.SqlQueryManager.createQueryInternal(SqlQueryManager.java:361)
> at
> com.facebook.presto.execution.SqlQueryManager.lambda$createQuery$4(SqlQueryManager.java:303)
> at
> com.facebook.presto.$gen.Presto_0_214_9_g36965f8_dirty__0_214_100_2____20181217_102607_1.run(Unknown
> Source)
> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> Caused by: java.lang.IllegalArgumentException: java.net.UnknownHostException:
> nameservice1
> at
> org.apache.hadoop.security.SecurityUtil.buildTokenService(SecurityUtil.java:378)
> at
> org.apache.hadoop.hdfs.NameNodeProxies.createNonHAProxy(NameNodeProxies.java:310)
> at
> org.apache.hadoop.hdfs.NameNodeProxies.createProxy(NameNodeProxies.java:176)
> at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:678)
> at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:619)
> at
> org.apache.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem.java:149)
> at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2669)
> at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:94)
> at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2703)
> at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2685)
> at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:373)
> at org.apache.hadoop.fs.Path.getFileSystem(Path.java:295)
> at
> org.apache.carbondata.core.datastore.filesystem.AbstractDFSCarbonFile.<init>(AbstractDFSCarbonFile.java:74)
> at
> org.apache.carbondata.core.datastore.filesystem.AbstractDFSCarbonFile.<init>(AbstractDFSCarbonFile.java:66)
> at
> org.apache.carbondata.core.datastore.filesystem.HDFSCarbonFile.<init>(HDFSCarbonFile.java:41)
> at
> org.apache.carbondata.core.datastore.impl.DefaultFileTypeProvider.getCarbonFile(DefaultFileTypeProvider.java:51)
> at
> org.apache.carbondata.core.datastore.impl.FileFactory.getCarbonFile(FileFactory.java:102)
> at
> org.apache.carbondata.presto.impl.CarbonTableReader.updateCarbonFile(CarbonTableReader.java:202)
> ... 39 more
> {code}
>
> Any suggestion or workaround is appreciated.
>
> Thanks
>
> Best regards,
> Ryan
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
