you need to package hdfs-site.xml and core-site.xml from your hadoop
cluster as part of your topology jar.
http://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.3.0/bk_storm-user-guide/content/storm-connectors-secure.html
-Harsha
On Thu, Dec 24, 2015, at 08:48 PM, Raja.Aravapalli wrote:
>
>
>
Hi,
>
>
I am getting below exception when i am trying to write tuples into HDFS
which is in a secured Hadoop cluster. Can someone pls share your
thoughts and help me fix the issue
>
> *Exception: *
> *---------------*
> java.lang.RuntimeException: Error preparing HdfsBolt:
> java.net.UnknownHostException: <HOSTNAME> at
org.apache.storm.hdfs.bolt.AbstractHdfsBolt.prepare(AbstractHdfsBolt.j-
ava:111) at backtype.storm.daemon.executor$fn__5265$fn__5278.invoke(ex-
ecutor.clj:732) at
backtype.storm.util$async_loop$fn__551.invoke(util.clj:463) at
clojure.lang.AFn.run(AFn.java:24) at
java.lang.Thread.run(Thread.java:744) Caused by:
java.lang.IllegalArgumentException: java.net.UnknownHostException:
<HOSTNAME> at org.apache.hadoop.security.SecurityUtil.buildTokenServic-
e(SecurityUtil.java:374) at org.apache.hadoop.hdfs.NameNodeProxies.cre-
ateNonHAProxy(NameNodeProxies.java:312) at org.apache.hadoop.hdfs.Name-
NodeProxies.createProxy(NameNodeProxies.java:178) at
org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:665) at
org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:601) at org.apa-
che.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem-
.java:148) at
org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2619)
at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:91) at
org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2653)
at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2635) at
org.apache.hadoop.fs.FileSystem.get(FileSystem.java:370) at
org.apache.storm.hdfs.bolt.HdfsBolt.doPrepare(HdfsBolt.java:87) at org-
.apache.storm.hdfs.bolt.AbstractHdfsBolt.prepare(AbstractHdfsBolt.java-
:107) ... 4 more Caused by: java.net.UnknownHostException: <HOSTNAME>
... 17 more
>
>
> *Code* *-------* HdfsBolt hdfsbolt = *new *HdfsBolt()
>
.withFsUrl(*"hdfs://NAMENODESERVICEID"*)
>
.withFileNameFormat(fileNameFormat)
>
.withRecordFormat(format)
>
.withRotationPolicy(rotationPolicy)
>
.withSyncPolicy(syncPolicy)
>
.withConfigKey(*"hdfs.config**”*);
>
> Map<String, Object> map = new HashMap<String, Object>();
> map.put("hdfs.keytab.file", “PATH_TO_KEYTAB");
> map.put("hdfs.kerberos.principal", “PRINCIPAL");
>
> Config config = new Config(); config.setDebug(true);
> config.put(HdfsSecurityUtil.STORM_KEYTAB_FILE_KEY, “PATH_TO_KEYTAB");
> config.put(HdfsSecurityUtil.STORM_USER_NAME_KEY, "PRINCIPAL");
> config.put("hdfs.config", map);
>
> StormTopology topology = builder.createTopology();
> StormSubmitter.submitTopology(“<TOPOLOGY_NAME>", config, topology);
>
> *POM.xml* *——————*
>
> <?xml version="1.0" encoding="UTF-8"?> <project xmlns="
> http://maven.apache.org/POM/4.0.0" xmlns:xsi="
> http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="
> http://maven.apache.org/POM/4.0.0
> http://maven.apache.org/xsd/maven-4.0.0.xsd";>
> <modelVersion>4.0.0</modelVersion>
>
> <groupId>GROUPID</groupId> <artifactId>ARTIFACTID</artifactId>
> <version>1.0-SNAPSHOT</version>
>
> <properties> <storm.version>0.9.3.2.2.4.0-2633</storm.version>
> <hadoop.version>2.6.0.2.2.4.12-2</hadoop.version> </properties>
>
> <dependencies> <dependency>
> <groupId>org.apache.storm</groupId> <artifactId>storm-
> core</artifactId> <version>${storm.version}</version>
> <scope>provided</scope> </dependency>
>
> <dependency> <groupId>org.apache.storm</groupId>
> <artifactId>storm-hdfs</artifactId>
> <version>${storm.version}</version> <scope>compile</scope>
> <exclusions> <exclusion>
> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-
> client</artifactId> </exclusion>
> <exclusion> <groupId>org.apache.hadoop</groupId>
> <artifactId>hadoop-hdfs</artifactId> </exclusion>
> <exclusion> <groupId>log4j</groupId>
> <artifactId>log4j</artifactId> </exclusion>
> <exclusion> <groupId>org.slf4j</groupId>
> <artifactId>slf4j-log4j12</artifactId> </exclusion>
> </exclusions> </dependency>
>
> <dependency> <groupId>org.apache.hadoop</groupId>
> <artifactId>hadoop-hdfs</artifactId>
> <version>${hadoop.version}</version> <scope>compile</scope>
> <exclusions> <exclusion>
> <groupId>log4j</groupId>
> <artifactId>log4j</artifactId> </exclusion>
> <exclusion> <groupId>org.slf4j</groupId>
> <artifactId>slf4j-log4j12</artifactId> </exclusion>
> </exclusions> </dependency>
>
> <dependency> <groupId>org.apache.hadoop</groupId>
> <artifactId>hadoop-client</artifactId>
> <version>${hadoop.version}</version> <scope>compile</scope>
> <exclusions> <exclusion>
> <groupId>log4j</groupId>
> <artifactId>log4j</artifactId> </exclusion>
> <exclusion> <groupId>org.slf4j</groupId>
> <artifactId>slf4j-log4j12</artifactId> </exclusion>
> </exclusions> </dependency>
>
> </dependencies>
>
> <repositories> <repository> <id>HDPReleases</id>
> <name>HDP Releases</name> <url>
> http://repo.hortonworks.com/content/repositories/releases/</url>
> <layout>default</layout> </repository> <repository>
> <id>HDP Jetty Hadoop</id> <name>HDP Jetty Hadoop</name>
> <url>
> http://repo.hortonworks.com/content/repositories/jetty-hadoop/</url>
> <layout>default</layout> </repository> <repository>
> <id>confluent</id> <url>
> http://packages.confluent.io/maven</url> </repository>
> </repositories>
>
> <build> <plugins> <plugin>
> <groupId>org.apache.maven.plugins</groupId>
> <artifactId>maven-compiler-
> plugin</artifactId> <version>2.5.1</version>
> <configuration> <source>1.7</source>
> <target>1.7</target> </configuration>
> </plugin> <plugin>
> <groupId>org.apache.maven.plugins</groupId>
> <artifactId>maven-shade-
> plugin</artifactId> <version>2.4.2</version>
> <configuration> <createDependencyReducedPom>true</-
> createDependencyReducedPom> </configuration>
> <executions> <execution>
> <phase>package</phase> <goals>
> <goal>shade</goal> </goals>
> <configuration> <transformers>
> <transformer implementation="org.apache.maven.plugins.shade.resource.-
> ServicesResourceTransformer" />
> <transformer implementation="org.apache.maven.plugins.shade.resource.-
> ManifestResourceTransformer">
> <mainClass>xml</mainClass>
> </transformer> </transformers>
> <filters> <filter>
> <artifact>*:*</artifact> <excludes>
> <exclude>META-INF/*.SF</exclude>
> <exclude>META-INF/*.DSA</exclude>
> <exclude>META-INF/*.RSA</exclude>
> </excludes> </filter>
> </filters> </configuration>
> </execution> </executions> </plugin>
> </plugins> </build> </project>
>
>
> --Raja.
>
