Hi,
I am getting below exception when i am trying to write tuples into HDFS which
is in a secured Hadoop cluster. Can someone pls share your thoughts and help me
fix the issue
Exception:
---------------
java.lang.RuntimeException: Error preparing HdfsBolt:
java.net.UnknownHostException: <HOSTNAME> at
org.apache.storm.hdfs.bolt.AbstractHdfsBolt.prepare(AbstractHdfsBolt.java:111)
at backtype.storm.daemon.executor$fn__5265$fn__5278.invoke(executor.clj:732) at
backtype.storm.util$async_loop$fn__551.invoke(util.clj:463) at
clojure.lang.AFn.run(AFn.java:24) at java.lang.Thread.run(Thread.java:744)
Caused by: java.lang.IllegalArgumentException: java.net.UnknownHostException:
<HOSTNAME> at
org.apache.hadoop.security.SecurityUtil.buildTokenService(SecurityUtil.java:374)
at
org.apache.hadoop.hdfs.NameNodeProxies.createNonHAProxy(NameNodeProxies.java:312)
at
org.apache.hadoop.hdfs.NameNodeProxies.createProxy(NameNodeProxies.java:178) at
org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:665) at
org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:601) at
org.apache.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem.java:148)
at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2619) at
org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:91) at
org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2653) at
org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2635) at
org.apache.hadoop.fs.FileSystem.get(FileSystem.java:370) at
org.apache.storm.hdfs.bolt.HdfsBolt.doPrepare(HdfsBolt.java:87) at
org.apache.storm.hdfs.bolt.AbstractHdfsBolt.prepare(AbstractHdfsBolt.java:107)
... 4 more Caused by: java.net.UnknownHostException: <HOSTNAME> ... 17 more
Code
-------
HdfsBolt hdfsbolt = new HdfsBolt()
.withFsUrl("hdfs://NAMENODESERVICEID")
.withFileNameFormat(fileNameFormat)
.withRecordFormat(format)
.withRotationPolicy(rotationPolicy)
.withSyncPolicy(syncPolicy)
.withConfigKey("hdfs.config”);
Map<String, Object> map = new HashMap<String, Object>();
map.put("hdfs.keytab.file", “PATH_TO_KEYTAB");
map.put("hdfs.kerberos.principal", “PRINCIPAL");
Config config = new Config();
config.setDebug(true);
config.put(HdfsSecurityUtil.STORM_KEYTAB_FILE_KEY, “PATH_TO_KEYTAB");
config.put(HdfsSecurityUtil.STORM_USER_NAME_KEY, "PRINCIPAL");
config.put("hdfs.config", map);
StormTopology topology = builder.createTopology();
StormSubmitter.submitTopology(“<TOPOLOGY_NAME>", config, topology);
POM.xml
——————
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
<modelVersion>4.0.0</modelVersion>
<groupId>GROUPID</groupId>
<artifactId>ARTIFACTID</artifactId>
<version>1.0-SNAPSHOT</version>
<properties>
<storm.version>0.9.3.2.2.4.0-2633</storm.version>
<hadoop.version>2.6.0.2.2.4.12-2</hadoop.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-core</artifactId>
<version>${storm.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-hdfs</artifactId>
<version>${storm.version}</version>
<scope>compile</scope>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${hadoop.version}</version>
<scope>compile</scope>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version>
<scope>compile</scope>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<repositories>
<repository>
<id>HDPReleases</id>
<name>HDP Releases</name>
<url>http://repo.hortonworks.com/content/repositories/releases/</url>
<layout>default</layout>
</repository>
<repository>
<id>HDP Jetty Hadoop</id>
<name>HDP Jetty Hadoop</name>
<url>http://repo.hortonworks.com/content/repositories/jetty-hadoop/</url>
<layout>default</layout>
</repository>
<repository>
<id>confluent</id>
<url>http://packages.confluent.io/maven</url>
</repository>
</repositories>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>2.5.1</version>
<configuration>
<source>1.7</source>
<target>1.7</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.4.2</version>
<configuration>
<createDependencyReducedPom>true</createDependencyReducedPom>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"
/>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>xml</mainClass>
</transformer>
</transformers>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
--Raja.
