Hi,
Kindly advise me to resolve the below exception getting while trying to
access the AVRO file on HA cluster from the below code. I have attached
the resource file and pasted too.
*Code: **import* java.io.IOException;
*import* org.apache.avro.file.DataFileReader;
*import* org.apache.avro.file.FileReader;
*import* org.apache.avro.file.SeekableInput;
*import* org.apache.avro.generic.GenericDatumReader;
*import* org.apache.avro.generic.GenericRecord;
*import* org.apache.avro.io.DatumReader;
*import* org.apache.avro.mapred.FsInput;
*import* org.apache.hadoop.conf.Configuration;
*import* org.apache.hadoop.fs.Path;
*public* *static* *void* main(String args[]) *throws* IOException{
Path path = *new* Path(
"hdfs://infaqaha/user/hive/warehouse/avro_table_string/000000_0");
Configuration config = *new* Configuration(); // make this your
*Hadoop* *env* *config*
config.addResource(
"C:\\Users\\mamishra\\Desktop\\hdfs-clientconfig\\hadoop-conf\\hdfs-site.xml"
);
config.addResource(
"C:\\Users\\mamishra\\Desktop\\hdfs-clientconfig\\core-site");
//*Metadata* fetch for each record, we sample only one record in
actual implementation.
SeekableInput input = *new* FsInput(path, config); <-Exception
on this line
DatumReader<GenericRecord> reader = *new*
GenericDatumReader<GenericRecord>();
FileReader<GenericRecord> fileReader = DataFileReader.
*openReader*(input, reader);
fileReader.close(); // also closes underlying FsInput
}
*Exception is : *
Exception in thread "main" *java.lang.IllegalArgumentException*:
*java.net.UnknownHostException*: infaqaha
at org.apache.hadoop.security.SecurityUtil.buildTokenService(
*SecurityUtil.java:374*)
at org.apache.hadoop.hdfs.NameNodeProxies.createNonHAProxy(
*NameNodeProxies.java:312*)
at org.apache.hadoop.hdfs.NameNodeProxies.createProxy(
*NameNodeProxies.java:178*)
at org.apache.hadoop.hdfs.DFSClient.<init>(*DFSClient.java:662*)
at org.apache.hadoop.hdfs.DFSClient.<init>(*DFSClient.java:606*)
at org.apache.hadoop.hdfs.DistributedFileSystem.initialize(
*DistributedFileSystem.java:148*)
at org.apache.hadoop.fs.FileSystem.createFileSystem(
*FileSystem.java:2596*)
at org.apache.hadoop.fs.FileSystem.access$200(*FileSystem.java:91*)
at org.apache.hadoop.fs.FileSystem$Cache.getInternal(
*FileSystem.java:2630*)
at org.apache.hadoop.fs.FileSystem$Cache.get(*FileSystem.java:2612*)
at org.apache.hadoop.fs.FileSystem.get(*FileSystem.java:370*)
at org.apache.hadoop.fs.Path.getFileSystem(*Path.java:296*)
at org.apache.avro.mapred.FsInput.<init>(*FsInput.java:38*)
at parquetAndAvro.CopyOfInfaAvro.main(*CopyOfInfaAvro.java:97*)
Caused by: *java.net.UnknownHostException*: infaqaha
... 14 more
ERROR: JDWP Unable to get JNI 1.2 environment, jvm->GetEnv() return code =
-2
JDWP exit error AGENT_ERROR_NO_JNI_ENV(183): [util.c:840]
*hdfs -site.xml:*
<?xml version="1.0" encoding="UTF-8"?>
<!--Autogenerated by Cloudera Manager-->
<configuration>
<property>
<name>dfs.nameservices</name>
<value>INFAQAHA</value>
</property>
<property>
<name>dfs.client.failover.proxy.provider.INFAQAHA</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<property>
<name>dfs.ha.automatic-failover.enabled.INFAQAHA</name>
<value>true</value>
</property>
<property>
<name>ha.zookeeper.quorum</name>
<value>ivlhdp18.informatica.com:2181</value>
</property>
<property>
<name>dfs.ha.namenodes.INFAQAHA</name>
<value>namenode34,namenode54</value>
</property>
<property>
<name>dfs.namenode.rpc-address.INFAQAHA.namenode34</name>
<value>ivlhdp18.informatica.com:8020</value>
</property>
<property>
<name>dfs.namenode.servicerpc-address.INFAQAHA.namenode34</name>
<value>ivlhdp18.informatica.com:8022</value>
</property>
<property>
<name>dfs.namenode.http-address.INFAQAHA.namenode34</name>
<value>ivlhdp18.informatica.com:50070</value>
</property>
<property>
<name>dfs.namenode.https-address.INFAQAHA.namenode34</name>
<value>ivlhdp18.informatica.com:50470</value>
</property>
<property>
<name>dfs.namenode.rpc-address.INFAQAHA.namenode54</name>
<value>ivlhdp19.informatica.com:8020</value>
</property>
<property>
<name>dfs.namenode.servicerpc-address.INFAQAHA.namenode54</name>
<value>ivlhdp19.informatica.com:8022</value>
</property>
<property>
<name>dfs.namenode.http-address.INFAQAHA.namenode54</name>
<value>ivlhdp19.informatica.com:50070</value>
</property>
<property>
<name>dfs.namenode.https-address.INFAQAHA.namenode54</name>
<value>ivlhdp19.informatica.com:50470</value>
</property>
<property>
<name>dfs.replication</name>
<value>1</value>
</property>
<property>
<name>dfs.blocksize</name>
<value>134217728</value>
</property>
<property>
<name>dfs.client.use.datanode.hostname</name>
<value>false</value>
</property>
<property>
<name>fs.permissions.umask-mode</name>
<value>022</value>
</property>
<property>
<name>dfs.namenode.acls.enabled</name>
<value>false</value>
</property>
<property>
<name>dfs.client.read.shortcircuit</name>
<value>false</value>
</property>
<property>
<name>dfs.domain.socket.path</name>
<value>/var/run/hdfs-sockets/dn</value>
</property>
<property>
<name>dfs.client.read.shortcircuit.skip.checksum</name>
<value>false</value>
</property>
<property>
<name>dfs.client.domain.socket.data.traffic</name>
<value>false</value>
</property>
<property>
<name>dfs.datanode.hdfs-blocks-metadata.enabled</name>
<value>true</value>
</property>
</configuration>
*core-site.xml: *
<?xml version="1.0" encoding="UTF-8"?>
<!--Autogenerated by Cloudera Manager-->
<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://INFAQAHA</value>
</property>
<property>
<name>fs.trash.interval</name>
<value>1</value>
</property>
<property>
<name>io.compression.codecs</name>
<value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.DeflateCodec,org.apache.hadoop.io.compress.SnappyCodec,org.apache.hadoop.io.compress.Lz4Codec</value>
</property>
<property>
<name>hadoop.security.authentication</name>
<value>simple</value>
</property>
<property>
<name>hadoop.security.authorization</name>
<value>false</value>
</property>
<property>
<name>hadoop.rpc.protection</name>
<value>authentication</value>
</property>
<property>
<name>hadoop.ssl.require.client.cert</name>
<value>false</value>
<final>true</final>
</property>
<property>
<name>hadoop.ssl.keystores.factory.class</name>
<value>org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory</value>
<final>true</final>
</property>
<property>
<name>hadoop.ssl.server.conf</name>
<value>ssl-server.xml</value>
<final>true</final>
</property>
<property>
<name>hadoop.ssl.client.conf</name>
<value>ssl-client.xml</value>
<final>true</final>
</property>
<property>
<name>hadoop.security.auth_to_local</name>
<value>DEFAULT</value>
</property>
<property>
<name>hadoop.proxyuser.oozie.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.oozie.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.mapred.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.mapred.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.flume.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.flume.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.HTTP.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.HTTP.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.hive.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.hive.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.hue.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.hue.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.httpfs.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.httpfs.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.hdfs.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.hdfs.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.yarn.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.yarn.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.security.group.mapping</name>
<value>org.apache.hadoop.security.ShellBasedUnixGroupsMapping</value>
</property>
<property>
<name>hadoop.security.instrumentation.requires.admin</name>
<value>false</value>
</property>
</configuration>
Thanks,
Mayank Mishra
<?xml version="1.0" encoding="UTF-8"?>
<!--Autogenerated by Cloudera Manager-->
<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://INFAQAHA</value>
</property>
<property>
<name>fs.trash.interval</name>
<value>1</value>
</property>
<property>
<name>io.compression.codecs</name>
<value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.DeflateCodec,org.apache.hadoop.io.compress.SnappyCodec,org.apache.hadoop.io.compress.Lz4Codec</value>
</property>
<property>
<name>hadoop.security.authentication</name>
<value>simple</value>
</property>
<property>
<name>hadoop.security.authorization</name>
<value>false</value>
</property>
<property>
<name>hadoop.rpc.protection</name>
<value>authentication</value>
</property>
<property>
<name>hadoop.ssl.require.client.cert</name>
<value>false</value>
<final>true</final>
</property>
<property>
<name>hadoop.ssl.keystores.factory.class</name>
<value>org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory</value>
<final>true</final>
</property>
<property>
<name>hadoop.ssl.server.conf</name>
<value>ssl-server.xml</value>
<final>true</final>
</property>
<property>
<name>hadoop.ssl.client.conf</name>
<value>ssl-client.xml</value>
<final>true</final>
</property>
<property>
<name>hadoop.security.auth_to_local</name>
<value>DEFAULT</value>
</property>
<property>
<name>hadoop.proxyuser.oozie.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.oozie.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.mapred.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.mapred.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.flume.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.flume.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.HTTP.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.HTTP.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.hive.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.hive.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.hue.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.hue.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.httpfs.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.httpfs.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.hdfs.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.hdfs.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.yarn.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.yarn.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.security.group.mapping</name>
<value>org.apache.hadoop.security.ShellBasedUnixGroupsMapping</value>
</property>
<property>
<name>hadoop.security.instrumentation.requires.admin</name>
<value>false</value>
</property>
</configuration>
<?xml version="1.0" encoding="UTF-8"?>
<!--Autogenerated by Cloudera Manager-->
<configuration>
<property>
<name>dfs.nameservices</name>
<value>INFAQAHA</value>
</property>
<property>
<name>dfs.client.failover.proxy.provider.INFAQAHA</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<property>
<name>dfs.ha.automatic-failover.enabled.INFAQAHA</name>
<value>true</value>
</property>
<property>
<name>ha.zookeeper.quorum</name>
<value>ivlhdp18.informatica.com:2181</value>
</property>
<property>
<name>dfs.ha.namenodes.INFAQAHA</name>
<value>namenode34,namenode54</value>
</property>
<property>
<name>dfs.namenode.rpc-address.INFAQAHA.namenode34</name>
<value>ivlhdp18.informatica.com:8020</value>
</property>
<property>
<name>dfs.namenode.servicerpc-address.INFAQAHA.namenode34</name>
<value>ivlhdp18.informatica.com:8022</value>
</property>
<property>
<name>dfs.namenode.http-address.INFAQAHA.namenode34</name>
<value>ivlhdp18.informatica.com:50070</value>
</property>
<property>
<name>dfs.namenode.https-address.INFAQAHA.namenode34</name>
<value>ivlhdp18.informatica.com:50470</value>
</property>
<property>
<name>dfs.namenode.rpc-address.INFAQAHA.namenode54</name>
<value>ivlhdp19.informatica.com:8020</value>
</property>
<property>
<name>dfs.namenode.servicerpc-address.INFAQAHA.namenode54</name>
<value>ivlhdp19.informatica.com:8022</value>
</property>
<property>
<name>dfs.namenode.http-address.INFAQAHA.namenode54</name>
<value>ivlhdp19.informatica.com:50070</value>
</property>
<property>
<name>dfs.namenode.https-address.INFAQAHA.namenode54</name>
<value>ivlhdp19.informatica.com:50470</value>
</property>
<property>
<name>dfs.replication</name>
<value>1</value>
</property>
<property>
<name>dfs.blocksize</name>
<value>134217728</value>
</property>
<property>
<name>dfs.client.use.datanode.hostname</name>
<value>false</value>
</property>
<property>
<name>fs.permissions.umask-mode</name>
<value>022</value>
</property>
<property>
<name>dfs.namenode.acls.enabled</name>
<value>false</value>
</property>
<property>
<name>dfs.client.read.shortcircuit</name>
<value>false</value>
</property>
<property>
<name>dfs.domain.socket.path</name>
<value>/var/run/hdfs-sockets/dn</value>
</property>
<property>
<name>dfs.client.read.shortcircuit.skip.checksum</name>
<value>false</value>
</property>
<property>
<name>dfs.client.domain.socket.data.traffic</name>
<value>false</value>
</property>
<property>
<name>dfs.datanode.hdfs-blocks-metadata.enabled</name>
<value>true</value>
</property>
</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
