This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new a65aca2 HDDS-922. Create isolated classloder to use ozonefs with any
older hadoop versions. Contributed by Elek, Marton.
a65aca2 is described below
commit a65aca2feff90b28fd5d52cf2da7e7248967bbaf
Author: Márton Elek <[email protected]>
AuthorDate: Thu Feb 7 17:02:03 2019 +0100
HDDS-922. Create isolated classloder to use ozonefs with any older hadoop
versions. Contributed by Elek, Marton.
---
.../hadoop/hdds/conf/OzoneConfiguration.java | 2 +
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 7 +
.../common/src/main/resources/ozone-default.xml | 15 ++
hadoop-hdds/docs/content/OzoneFS.md | 19 +-
.../dist/dev-support/bin/dist-layout-stitching | 4 -
hadoop-ozone/dist/pom.xml | 15 ++
.../src/main/compose/ozonefs/docker-compose.yaml | 29 ++-
hadoop-ozone/ozonefs-lib-legacy/pom.xml | 104 +++++++++
.../src/main/resources/ozonefs.txt | 21 ++
hadoop-ozone/ozonefs-lib/pom.xml | 89 ++++++++
hadoop-ozone/ozonefs/pom.xml | 60 ++----
.../org/apache/hadoop/fs/ozone/BasicKeyInfo.java | 53 +++++
.../hadoop/fs/ozone/FilteredClassLoader.java | 84 ++++++++
.../apache/hadoop/fs/ozone/OzoneClientAdapter.java | 55 +++++
.../hadoop/fs/ozone/OzoneClientAdapterFactory.java | 119 +++++++++++
.../hadoop/fs/ozone/OzoneClientAdapterImpl.java | 235 +++++++++++++++++++++
.../apache/hadoop/fs/ozone/OzoneFSInputStream.java | 16 +-
.../hadoop/fs/ozone/OzoneFSOutputStream.java | 5 +-
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 225 ++++++++------------
hadoop-ozone/pom.xml | 23 +-
20 files changed, 963 insertions(+), 217 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java
index 36d953c..43e6fe7 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java
@@ -47,6 +47,8 @@ public class OzoneConfiguration extends Configuration {
public OzoneConfiguration(Configuration conf) {
super(conf);
+ //load the configuration from the classloader of the original conf.
+ setClassLoader(conf.getClassLoader());
}
public List<Property> readPropertyFromXml(URL url) throws JAXBException {
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index e9a52f8aae..91f53f3 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -371,6 +371,13 @@ public final class OzoneConfigKeys {
public static final boolean OZONE_ACL_ENABLED_DEFAULT =
false;
+ //For technical reasons this is unused and hardcoded to the
+ // OzoneFileSystem.initialize.
+ public static final String OZONE_FS_ISOLATED_CLASSLOADER =
+ "ozone.fs.isolated-classloader";
+
+
+
/**
* There is no need to instantiate this class.
*/
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml
b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 5489936..7ba15f3 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -1806,4 +1806,19 @@
not be renewed.
</description>
</property>
+
+ <property>
+ <name>ozone.fs.isolated-classloader</name>
+ <value></value>
+ <tag>OZONE, OZONEFS</tag>
+ <description>
+ Enable it for older hadoops to separate the classloading of all the
+ Ozone classes. With 'true' value, ozonefs can be used with older
+ hadoop versions as the hadoop3/ozone related classes are loaded by
+ an isolated classloader.
+
+ Default depends from the used jar. true for ozone-filesystem-lib-legacy
+ jar and false for the ozone-filesystem-lib.jar
+ </description>
+ </property>
</configuration>
\ No newline at end of file
diff --git a/hadoop-hdds/docs/content/OzoneFS.md
b/hadoop-hdds/docs/content/OzoneFS.md
index 92c83d8..b7f8a74 100644
--- a/hadoop-hdds/docs/content/OzoneFS.md
+++ b/hadoop-hdds/docs/content/OzoneFS.md
@@ -56,12 +56,11 @@ This will make this bucket to be the default file system
for HDFS dfs commands a
You also need to add the ozone-filesystem.jar file to the classpath:
{{< highlight bash >}}
-export
HADOOP_CLASSPATH=/opt/ozone/share/hadoop/ozonefs/hadoop-ozone-filesystem.jar:$HADOOP_CLASSPATH
+export
HADOOP_CLASSPATH=/opt/ozone/share/ozonefs/lib/hadoop-ozone-filesystem-lib-.*.jar:$HADOOP_CLASSPATH
{{< /highlight >}}
-
Once the default Filesystem has been setup, users can run commands like ls,
put, mkdir, etc.
For example,
@@ -78,3 +77,19 @@ hdfs dfs -mkdir /users
Or put command etc. In other words, all programs like Hive, Spark, and Distcp
will work against this file system.
Please note that any keys created/deleted in the bucket using methods apart
from OzoneFileSystem will show up as diectories and files in the Ozone File
System.
+
+## Legacy mode
+
+There are two ozonefs files which includes all the dependencies:
+
+ * share/ozone/lib/hadoop-ozone-filesystem-lib-VERSION.jar
+ * share/ozone/lib/hadoop-ozone-filesystem-lib-legacy-VERSION.jar
+
+ The first one contains all the required dependency to use ozonefs with a
+ compatible hadoop version (hadoop 3.2 / 3.1).
+
+ The second one contains all the dependency in an internal, separated
directory,
+ and a special class loader is used to load all the classes from the location.
+
+ With this method the hadoop-ozone-filesystem-lib-legacy.jar can be used from
+ any older hadoop version (eg. hadoop 2.7 or spark+hadoop 2.7)
\ No newline at end of file
diff --git a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
index 250a089..dc2819b 100755
--- a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
+++ b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
@@ -106,10 +106,6 @@ run cp
"${ROOT}/hadoop-common-project/hadoop-common/src/main/bin/workers.sh" "sb
run cp "${ROOT}/hadoop-ozone/common/src/main/bin/start-ozone.sh" "sbin/"
run cp "${ROOT}/hadoop-ozone/common/src/main/bin/stop-ozone.sh" "sbin/"
-#shaded ozonefs
-run mkdir -p "./share/hadoop/ozonefs"
-run cp
"${ROOT}/hadoop-ozone/ozonefs/target/hadoop-ozone-filesystem-${HDDS_VERSION}.jar"
"./share/hadoop/ozonefs/hadoop-ozone-filesystem-${HDDS_VERSION}.jar"
-
#shaded datanode service
run mkdir -p "./share/hadoop/ozoneplugin"
run cp
"${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore-service-${HDDS_VERSION}-plugin.jar"
"./share/hadoop/ozoneplugin/hadoop-ozone-datanode-plugin-${HDDS_VERSION}.jar"
diff --git a/hadoop-ozone/dist/pom.xml b/hadoop-ozone/dist/pom.xml
index f625cb1..6224e9d 100644
--- a/hadoop-ozone/dist/pom.xml
+++ b/hadoop-ozone/dist/pom.xml
@@ -84,6 +84,13 @@
</artifactItem>
<artifactItem>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-filesystem</artifactId>
+ <version>${ozone.version}</version>
+ <classifier>classpath</classifier>
+
<destFileName>hadoop-ozone-filesystem.classpath</destFileName>
+ </artifactItem>
+ <artifactItem>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-common</artifactId>
<version>${ozone.version}</version>
<classifier>classpath</classifier>
@@ -185,6 +192,14 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-filesystem-lib</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-filesystem-lib-legacy</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-common</artifactId>
</dependency>
<dependency>
diff --git a/hadoop-ozone/dist/src/main/compose/ozonefs/docker-compose.yaml
b/hadoop-ozone/dist/src/main/compose/ozonefs/docker-compose.yaml
index f155b3b..22055fc 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonefs/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/ozonefs/docker-compose.yaml
@@ -19,12 +19,12 @@ services:
datanode:
image: apache/hadoop-runner
volumes:
- - ../..:/opt/hadoop
+ - ../..:/opt/hadoop
ports:
- - 9864
+ - 9864
command: ["/opt/hadoop/bin/ozone","datanode"]
env_file:
- - ./docker-config
+ - ./docker-config
ozoneManager:
image: apache/hadoop-runner
hostname: ozoneManager
@@ -36,7 +36,7 @@ services:
ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION
WAITFOR: scm:9876
env_file:
- - ./docker-config
+ - ./docker-config
command: ["/opt/hadoop/bin/ozone","om"]
scm:
image: apache/hadoop-runner
@@ -45,16 +45,25 @@ services:
ports:
- 9876
env_file:
- - ./docker-config
+ - ./docker-config
environment:
- ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+ ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
command: ["/opt/hadoop/bin/ozone","scm"]
- hadooplast:
+ hadoop3:
image: flokkr/hadoop:3.1.0
volumes:
- - ../..:/opt/ozone
+ - ../..:/opt/ozone
+ env_file:
+ - ./docker-config
+ environment:
+ HADOOP_CLASSPATH:
/opt/ozone/share/ozone/lib/hadoop-ozone-filesystem-lib-0*.jar
+ command: ["watch","-n","100000","ls"]
+ hadoop2:
+ image: flokkr/hadoop:2.9.0
+ volumes:
+ - ../..:/opt/ozone
env_file:
- - ./docker-config
+ - ./docker-config
environment:
- HADOOP_CLASSPATH: /opt/ozone/share/hadoop/ozonefs/*.jar
+ HADOOP_CLASSPATH:
/opt/ozone/share/ozone/lib/hadoop-ozone-filesystem-lib-legacy*.jar
command: ["watch","-n","100000","ls"]
diff --git a/hadoop-ozone/ozonefs-lib-legacy/pom.xml
b/hadoop-ozone/ozonefs-lib-legacy/pom.xml
new file mode 100644
index 0000000..b4b7636
--- /dev/null
+++ b/hadoop-ozone/ozonefs-lib-legacy/pom.xml
@@ -0,0 +1,104 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone</artifactId>
+ <version>0.4.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>hadoop-ozone-filesystem-lib-legacy</artifactId>
+ <name>Apache Hadoop Ozone FileSystem Legacy Jar Library</name>
+ <description>This projects creates an uberjar from ozonefs with all the
+ dependencies, but the dependencies are located in an isolated subdir
+ and loaded by a custom class loader. Can be used together with Hadoop 2.x
+ </description>
+ <packaging>jar</packaging>
+ <version>0.4.0-SNAPSHOT</version>
+ <properties>
+ <file.encoding>UTF-8</file.encoding>
+ <downloadSources>true</downloadSources>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>include-dependencies</id>
+ <goals>
+ <goal>unpack-dependencies</goal>
+ </goals>
+ <phase>prepare-package</phase>
+ <configuration>
+ <outputDirectory>target/classes/libs</outputDirectory>
+ <includeScope>compile</includeScope>
+ <excludes>META-INF/*.SF</excludes>
+ <excludeArtifactIds>
+
slf4j-api,slf4j-log4j12,log4j-api,log4j-core,log4j,hadoop-ozone-filesystem
+ </excludeArtifactIds>
+ </configuration>
+ </execution>
+ <execution>
+ <id>include-ozonefs</id>
+ <goals>
+ <goal>unpack-dependencies</goal>
+ </goals>
+ <phase>prepare-package</phase>
+ <configuration>
+ <outputDirectory>target/classes</outputDirectory>
+ <includeArtifactIds>hadoop-ozone-filesystem</includeArtifactIds>
+ <includeScope>compile</includeScope>
+ <excludes>META-INF/*.SF</excludes>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>animal-sniffer-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>signature-check</id>
+ <phase></phase>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-filesystem</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hadoop-ozone/ozonefs-lib-legacy/src/main/resources/ozonefs.txt
b/hadoop-ozone/ozonefs-lib-legacy/src/main/resources/ozonefs.txt
new file mode 100644
index 0000000..85c1307
--- /dev/null
+++ b/hadoop-ozone/ozonefs-lib-legacy/src/main/resources/ozonefs.txt
@@ -0,0 +1,21 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+Apache Hadoop Ozone placeholder file.
+
+The usage of the legacy version of the uber jar can be detected based on
+the existence of this file.
diff --git a/hadoop-ozone/ozonefs-lib/pom.xml b/hadoop-ozone/ozonefs-lib/pom.xml
new file mode 100644
index 0000000..c8c5c75
--- /dev/null
+++ b/hadoop-ozone/ozonefs-lib/pom.xml
@@ -0,0 +1,89 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone</artifactId>
+ <version>0.4.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>hadoop-ozone-filesystem-lib</artifactId>
+ <name>Apache Hadoop Ozone FileSystem Single Jar Library</name>
+ <packaging>jar</packaging>
+ <description>This projects creates an uber jar from ozonefs with all the
+ dependencies.
+ </description>
+ <version>0.4.0-SNAPSHOT</version>
+ <properties>
+ <file.encoding>UTF-8</file.encoding>
+ <downloadSources>true</downloadSources>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>unpack-dependencies</goal>
+ </goals>
+ <phase>prepare-package</phase>
+ <configuration>
+ <outputDirectory>target/classes</outputDirectory>
+ <includeScope>compile</includeScope>
+ <excludes>META-INF/*.SF</excludes>
+ <excludeArtifactIds>
+ slf4j-api,slf4j-log4j12,log4j-api,log4j-core,log4j
+ </excludeArtifactIds>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>animal-sniffer-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>signature-check</id>
+ <phase></phase>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-filesystem</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hadoop-ozone/ozonefs/pom.xml b/hadoop-ozone/ozonefs/pom.xml
index 0bf1c3a..95a602c 100644
--- a/hadoop-ozone/ozonefs/pom.xml
+++ b/hadoop-ozone/ozonefs/pom.xml
@@ -45,46 +45,6 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>3.1.1</version>
- <configuration>
- <artifactSet>
- <includes>
- <include>com.google.guava:guava</include>
- <include>org.slf4j:slf4j-api</include>
- <include>com.google.protobuf:protobuf-java</include>
- <include>com.nimbusds:nimbus-jose-jwt</include>
- <include>com.github.stephenc.jcip:jcip-annotations</include>
- <include>com.google.code.findbugs:jsr305</include>
- <include>org.apache.hadoop:hadoop-ozone-client</include>
- <include>org.apache.hadoop:hadoop-hdds-client</include>
- <include>org.apache.hadoop:hadoop-hdds-common</include>
- <include>org.fusesource.leveldbjni:leveldbjni-all</include>
- <include>org.apache.ratis:ratis-server</include>
- <include>org.apache.ratis:ratis-proto-shaded</include>
- <include>com.google.auto.value:auto-value-annotations</include>
- <include>com.squareup:javapoet</include>
- <include>org.jctools:jctools-core</include>
- <include>org.apache.ratis:ratis-common</include>
- <include>org.apache.ratis:ratis-client</include>
- <include>org.apache.ratis:ratis-netty</include>
- <include>org.apache.ratis:ratis-grpc</include>
- <include>org.rocksdb:rocksdbjni</include>
- <include>org.apache.hadoop:hadoop-ozone-common</include>
- </includes>
- </artifactSet>
- </configuration>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
@@ -96,7 +56,7 @@
<configuration>
<!-- build a shellprofile -->
<outputFile>
-
${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-optional.txt
+
${project.basedir}/target/1hadoop-tools-deps/${project.artifactId}.tools-optional.txt
</outputFile>
</configuration>
</execution>
@@ -110,7 +70,17 @@
<include>ITestOzoneContract*.java</include>
</includes>
</configuration>
- </plugin>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>animal-sniffer-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>signature-check</id>
+ <phase></phase>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
@@ -118,17 +88,17 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <scope>provided</scope>
+ <scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
- <scope>provided</scope>
+ <scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs-client</artifactId>
- <scope>provided</scope>
+ <scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
diff --git
a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicKeyInfo.java
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicKeyInfo.java
new file mode 100644
index 0000000..06ebc15
--- /dev/null
+++
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicKeyInfo.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+/**
+ * Minimum set of Ozone key information attributes.
+ * <p>
+ * This class doesn't depend on any other ozone class just on primitive
+ * java types. It could be used easily in the signature of OzoneClientAdapter
+ * as even if a separated class loader is loaded it it won't cause any
+ * dependency problem.
+ */
+public class BasicKeyInfo {
+
+ private String name;
+
+ private long modificationTime;
+
+ private long dataSize;
+
+ public BasicKeyInfo(String name, long modificationTime, long size) {
+ this.name = name;
+ this.modificationTime = modificationTime;
+ this.dataSize = size;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public long getModificationTime() {
+ return modificationTime;
+ }
+
+ public long getDataSize() {
+ return dataSize;
+ }
+}
diff --git
a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/FilteredClassLoader.java
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/FilteredClassLoader.java
new file mode 100644
index 0000000..4627245
--- /dev/null
+++
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/FilteredClassLoader.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Class loader which delegates the loading only for the selected class.
+ *
+ * <p>
+ * By default java classloader delegates first all the class loading to the
+ * parent, and loads the class only if it's not found in the class.
+ * <p>
+ * This simple class loader do the opposit. Everything is loaded with this
+ * class loader without delegation _except_ the few classes which are defined
+ * in the constructor.
+ * <p>
+ * With this method we can use two separated class loader (the original main
+ * classloader and instance of this which loaded separated classes, but the
+ * few selected classes are shared between the two class loaders.
+ * <p>
+ * With this approach it's possible to use any older hadoop version
+ * (main classloader) together with ozonefs (instance of this classloader) as
+ * only the selected classes are selected between the class loaders.
+ */
+public class FilteredClassLoader extends URLClassLoader {
+
+ private final ClassLoader systemClassLoader;
+
+ private final ClassLoader delegate;
+ private Set<String> delegatedClasses = new HashSet<>();
+
+ public FilteredClassLoader(URL[] urls, ClassLoader parent) {
+ super(urls, null);
+ delegatedClasses.add("org.apache.hadoop.fs.ozone.OzoneClientAdapter");
+ delegatedClasses.add("org.apache.hadoop.fs.ozone.BasicKeyInfo");
+ delegatedClasses.add("org.apache.hadoop.fs.ozone.OzoneFSOutputStream");
+ delegatedClasses.add("org.apache.hadoop.fs.Seekable");
+ this.delegate = parent;
+ systemClassLoader = getSystemClassLoader();
+
+ }
+
+ @Override
+ public Class<?> loadClass(String name) throws ClassNotFoundException {
+ if (delegatedClasses.contains(name) ||
+ name.startsWith("org.apache.log4j") ||
+ name.startsWith("org.slf4j")) {
+ return delegate.loadClass(name);
+ }
+ return super.loadClass(name);
+ }
+
+ private Class<?> loadFromSystem(String name) {
+ if (systemClassLoader != null) {
+ try {
+ return systemClassLoader.loadClass(name);
+ } catch (ClassNotFoundException ex) {
+ //no problem
+ return null;
+ }
+ } else {
+ return null;
+ }
+ }
+}
diff --git
a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java
new file mode 100644
index 0000000..59f3f7a
--- /dev/null
+++
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Iterator;
+
+/**
+ * Lightweight adapter to separte hadoop/ozone classes.
+ * <p>
+ * This class contains only the bare minimum Ozone classes in the signature.
+ * It could be loaded by a different classloader because only the objects in
+ * the method signatures should be shared between the classloader.
+ */
+public interface OzoneClientAdapter {
+
+ void close() throws IOException;
+
+ InputStream createInputStream(String key) throws IOException;
+
+ OzoneFSOutputStream createKey(String key) throws IOException;
+
+ void renameKey(String key, String newKeyName) throws IOException;
+
+ BasicKeyInfo getKeyInfo(String keyName);
+
+ boolean isDirectory(BasicKeyInfo key);
+
+ boolean createDirectory(String keyName);
+
+ boolean deleteObject(String keyName);
+
+ long getCreationTime();
+
+ boolean hasNextKey(String key);
+
+ Iterator<BasicKeyInfo> listKeys(String pathKey);
+
+}
diff --git
a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapterFactory.java
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapterFactory.java
new file mode 100644
index 0000000..2aa816a
--- /dev/null
+++
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapterFactory.java
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Creates OzoneClientAdapter with classloader separation.
+ */
+public final class OzoneClientAdapterFactory {
+
+ static final Logger LOG =
+ LoggerFactory.getLogger(OzoneClientAdapterFactory.class);
+
+ private OzoneClientAdapterFactory() {
+ }
+
+ public static OzoneClientAdapter createAdapter(
+ String volumeStr,
+ String bucketStr)
+ throws IOException {
+
+ ClassLoader currentClassLoader = OzoneFileSystem.class.getClassLoader();
+ List<URL> urls = new ArrayList<>();
+
+ findEmbeddedLibsUrl(urls, currentClassLoader);
+
+ findConfigDirUrl(urls, currentClassLoader);
+
+ ClassLoader classLoader =
+ new FilteredClassLoader(urls.toArray(new URL[0]), currentClassLoader);
+
+ try {
+
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(classLoader);
+
+ //this class caches the context classloader during the first load
+ //call it here when the context class loader is set to the isoloated
+ //loader to make sure the grpc class will be loaded by the right
+ //loader
+ Class<?> reflectionUtils =
+ classLoader.loadClass("org.apache.ratis.util.ReflectionUtils");
+ reflectionUtils.getMethod("getClassByName", String.class)
+ .invoke(null, "org.apache.ratis.grpc.GrpcFactory");
+
+ OzoneClientAdapter ozoneClientAdapter = (OzoneClientAdapter) classLoader
+ .loadClass("org.apache.hadoop.fs.ozone.OzoneClientAdapterImpl")
+ .getConstructor(String.class, String.class)
+ .newInstance(
+ volumeStr,
+ bucketStr);
+
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+
+ return ozoneClientAdapter;
+ } catch (Exception e) {
+ LOG.error("Can't initialize the ozoneClientAdapter", e);
+ throw new IOException(
+ "Can't initialize the OzoneClientAdapter implementation", e);
+ }
+
+ }
+
+ private static void findConfigDirUrl(List<URL> urls,
+ ClassLoader currentClassLoader) throws IOException {
+ Enumeration<URL> conf =
+ currentClassLoader.getResources("ozone-site.xml");
+ while (conf.hasMoreElements()) {
+ urls.add(
+ new URL(
+ conf.nextElement().toString().replace("ozone-site.xml", "")));
+
+ }
+ }
+
+ private static void findEmbeddedLibsUrl(List<URL> urls,
+ ClassLoader currentClassloader)
+ throws MalformedURLException {
+
+ //marker file is added to the jar to make it easier to find the URL
+ // for the current jar.
+ String markerFile = "ozonefs.txt";
+ ClassLoader currentClassLoader = OzoneFileSystem.class.getClassLoader();
+
+ URL ozFs = currentClassLoader
+ .getResource(markerFile);
+ String rootPath = ozFs.toString().replace(markerFile, "");
+ urls.add(new URL(rootPath));
+
+ urls.add(new URL(rootPath + "libs/"));
+
+ }
+
+}
diff --git
a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapterImpl.java
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapterImpl.java
new file mode 100644
index 0000000..23d32e2
--- /dev/null
+++
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapterImpl.java
@@ -0,0 +1,235 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.ozone;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.Iterator;
+
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Implementation of the OzoneFileSystem calls.
+ */
+public class OzoneClientAdapterImpl implements OzoneClientAdapter {
+
+ static final Logger LOG =
+ LoggerFactory.getLogger(OzoneClientAdapterImpl.class);
+
+ private OzoneClient ozoneClient;
+ private ObjectStore objectStore;
+ private OzoneVolume volume;
+ private OzoneBucket bucket;
+ private ReplicationType replicationType;
+ private ReplicationFactor replicationFactor;
+
+ public OzoneClientAdapterImpl(String volumeStr, String bucketStr)
+ throws IOException {
+ this(createConf(), volumeStr, bucketStr);
+ }
+
+ private static OzoneConfiguration createConf() {
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+ OzoneConfiguration conf = new OzoneConfiguration();
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ return conf;
+ }
+
+ public OzoneClientAdapterImpl(OzoneConfiguration conf, String volumeStr,
+ String bucketStr) throws IOException {
+ ClassLoader contextClassLoader =
+ Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(null);
+ try {
+ String replicationTypeConf =
+ conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
+ OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT);
+
+ int replicationCountConf = conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
+ OzoneConfigKeys.OZONE_REPLICATION_DEFAULT);
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(conf);
+ objectStore = ozoneClient.getObjectStore();
+ this.volume = objectStore.getVolume(volumeStr);
+ this.bucket = volume.getBucket(bucketStr);
+ this.replicationType = ReplicationType.valueOf(replicationTypeConf);
+ this.replicationFactor = ReplicationFactor.valueOf(replicationCountConf);
+ } finally {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+
+ }
+
+ @Override
+ public void close() throws IOException {
+ ozoneClient.close();
+ }
+
+ @Override
+ public InputStream createInputStream(String key) throws IOException {
+ return bucket.readKey(key).getInputStream();
+ }
+
+ @Override
+ public OzoneFSOutputStream createKey(String key) throws IOException {
+ OzoneOutputStream ozoneOutputStream =
+ bucket.createKey(key, 0, replicationType, replicationFactor,
+ new HashMap<>());
+ return new OzoneFSOutputStream(ozoneOutputStream.getOutputStream());
+ }
+
+ @Override
+ public void renameKey(String key, String newKeyName) throws IOException {
+ bucket.renameKey(key, newKeyName);
+ }
+
+ /**
+ * Helper method to fetch the key metadata info.
+ *
+ * @param keyName key whose metadata information needs to be fetched
+ * @return metadata info of the key
+ */
+ @Override
+ public BasicKeyInfo getKeyInfo(String keyName) {
+ try {
+ OzoneKey key = bucket.getKey(keyName);
+ return new BasicKeyInfo(
+ keyName,
+ key.getModificationTime(),
+ key.getDataSize()
+ );
+ } catch (IOException e) {
+ LOG.trace("Key:{} does not exist", keyName);
+ return null;
+ }
+ }
+
+ /**
+ * Helper method to check if an Ozone key is representing a directory.
+ *
+ * @param key key to be checked as a directory
+ * @return true if key is a directory, false otherwise
+ */
+ @Override
+ public boolean isDirectory(BasicKeyInfo key) {
+ LOG.trace("key name:{} size:{}", key.getName(),
+ key.getDataSize());
+ return key.getName().endsWith(OZONE_URI_DELIMITER)
+ && (key.getDataSize() == 0);
+ }
+
+ /**
+ * Helper method to create an directory specified by key name in bucket.
+ *
+ * @param keyName key name to be created as directory
+ * @return true if the key is created, false otherwise
+ */
+ @Override
+ public boolean createDirectory(String keyName) {
+ try {
+ LOG.trace("creating dir for key:{}", keyName);
+ bucket.createKey(keyName, 0, replicationType, replicationFactor,
+ new HashMap<>()).close();
+ return true;
+ } catch (IOException ioe) {
+ LOG.error("create key failed for key:{}", keyName, ioe);
+ return false;
+ }
+ }
+
+ /**
+ * Helper method to delete an object specified by key name in bucket.
+ *
+ * @param keyName key name to be deleted
+ * @return true if the key is deleted, false otherwise
+ */
+ @Override
+ public boolean deleteObject(String keyName) {
+ LOG.trace("issuing delete for key" + keyName);
+ try {
+ bucket.deleteKey(keyName);
+ return true;
+ } catch (IOException ioe) {
+ LOG.error("delete key failed " + ioe.getMessage());
+ return false;
+ }
+ }
+
+ @Override
+ public long getCreationTime() {
+ return bucket.getCreationTime();
+ }
+
+ @Override
+ public boolean hasNextKey(String key) {
+ return bucket.listKeys(key).hasNext();
+ }
+
+ @Override
+ public Iterator<BasicKeyInfo> listKeys(String pathKey) {
+ return new IteratorAdapter(bucket.listKeys(pathKey));
+ }
+
+ /**
+ * Adapter to conver OzoneKey to a safe and simple Key implementation.
+ */
+ public static class IteratorAdapter implements Iterator<BasicKeyInfo> {
+
+ private Iterator<? extends OzoneKey> original;
+
+ public IteratorAdapter(Iterator<? extends OzoneKey> listKeys) {
+ this.original = listKeys;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return original.hasNext();
+ }
+
+ @Override
+ public BasicKeyInfo next() {
+ OzoneKey next = original.next();
+ if (next == null) {
+ return null;
+ } else {
+ return new BasicKeyInfo(
+ next.getName(),
+ next.getModificationTime(),
+ next.getDataSize()
+ );
+ }
+ }
+ }
+}
diff --git
a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSInputStream.java
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSInputStream.java
index 5df3cff..909b2af 100644
---
a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSInputStream.java
+++
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSInputStream.java
@@ -18,13 +18,13 @@
package org.apache.hadoop.fs.ozone;
+import java.io.IOException;
+import java.io.InputStream;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.FSInputStream;
-import org.apache.hadoop.ozone.client.io.KeyInputStream;
-
-import java.io.IOException;
-import java.io.InputStream;
+import org.apache.hadoop.fs.Seekable;
/**
* The input stream for Ozone file system.
@@ -36,10 +36,10 @@ import java.io.InputStream;
@InterfaceStability.Evolving
public final class OzoneFSInputStream extends FSInputStream {
- private final KeyInputStream inputStream;
+ private final InputStream inputStream;
public OzoneFSInputStream(InputStream inputStream) {
- this.inputStream = (KeyInputStream)inputStream;
+ this.inputStream = inputStream;
}
@Override
@@ -59,12 +59,12 @@ public final class OzoneFSInputStream extends FSInputStream
{
@Override
public void seek(long pos) throws IOException {
- inputStream.seek(pos);
+ ((Seekable) inputStream).seek(pos);
}
@Override
public long getPos() throws IOException {
- return inputStream.getPos();
+ return ((Seekable) inputStream).getPos();
}
@Override
diff --git
a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSOutputStream.java
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSOutputStream.java
index 3670cff..efbf93b 100644
---
a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSOutputStream.java
+++
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSOutputStream.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.fs.ozone;
import java.io.IOException;
import java.io.OutputStream;
-import org.apache.hadoop.ozone.client.io.KeyOutputStream;
/**
@@ -31,10 +30,10 @@ import org.apache.hadoop.ozone.client.io.KeyOutputStream;
*/
public class OzoneFSOutputStream extends OutputStream {
- private final KeyOutputStream outputStream;
+ private final OutputStream outputStream;
public OzoneFSOutputStream(OutputStream outputStream) {
- this.outputStream = (KeyOutputStream)outputStream;
+ this.outputStream = outputStream;
}
@Override
diff --git
a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
index eb9d100..abec436 100644
---
a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
+++
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
@@ -25,20 +25,18 @@ import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
-import java.util.Iterator;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-
-import com.google.common.base.Preconditions;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -47,34 +45,25 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
-import org.apache.hadoop.ozone.client.ObjectStore;
-import org.apache.hadoop.ozone.client.OzoneBucket;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.client.OzoneClientFactory;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.client.OzoneKey;
-import org.apache.hadoop.ozone.client.OzoneVolume;
-import org.apache.hadoop.hdds.client.ReplicationFactor;
-import org.apache.hadoop.hdds.client.ReplicationType;
-import org.apache.http.client.utils.URIBuilder;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Progressable;
-import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import com.google.common.base.Preconditions;
+import org.apache.commons.lang3.StringUtils;
+import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_SCHEME;
import static org.apache.hadoop.fs.ozone.Constants.OZONE_USER_DIR;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
-import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_SCHEME;
+import org.apache.http.client.utils.URIBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* The Ozone Filesystem implementation.
- *
+ * <p>
* This subclass is marked as private as code should not be creating it
* directly; use {@link FileSystem#get(Configuration)} and variants to create
* one. If cast to {@link OzoneFileSystem}, extra methods and features may be
@@ -85,16 +74,15 @@ import static
org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
public class OzoneFileSystem extends FileSystem {
static final Logger LOG = LoggerFactory.getLogger(OzoneFileSystem.class);
- /** The Ozone client for connecting to Ozone server. */
- private OzoneClient ozoneClient;
- private ObjectStore objectStore;
- private OzoneVolume volume;
- private OzoneBucket bucket;
+ /**
+ * The Ozone client for connecting to Ozone server.
+ */
+
private URI uri;
private String userName;
private Path workingDir;
- private ReplicationType replicationType;
- private ReplicationFactor replicationFactor;
+
+ private OzoneClientAdapter adapter;
private static final Pattern URL_SCHEMA_PATTERN =
Pattern.compile("(.+)\\.([^\\.]+)");
@@ -102,11 +90,7 @@ public class OzoneFileSystem extends FileSystem {
@Override
public void initialize(URI name, Configuration conf) throws IOException {
super.initialize(name, conf);
- if(!(conf instanceof OzoneConfiguration)) {
- setConf(new OzoneConfiguration(conf));
- } else {
- setConf(conf);
- }
+ setConf(conf);
Objects.requireNonNull(name.getScheme(), "No scheme provided in " + name);
assert getScheme().equals(name.getScheme());
@@ -125,16 +109,32 @@ public class OzoneFileSystem extends FileSystem {
uri = new URIBuilder().setScheme(OZONE_URI_SCHEME)
.setHost(authority).build();
LOG.trace("Ozone URI for ozfs initialization is " + uri);
- this.ozoneClient = OzoneClientFactory.getRpcClient(getConf());
- objectStore = ozoneClient.getObjectStore();
- this.volume = objectStore.getVolume(volumeStr);
- this.bucket = volume.getBucket(bucketStr);
- this.replicationType = ReplicationType.valueOf(
- getConf().get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
- OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT));
- this.replicationFactor = ReplicationFactor.valueOf(
- getConf().getInt(OzoneConfigKeys.OZONE_REPLICATION,
- OzoneConfigKeys.OZONE_REPLICATION_DEFAULT));
+
+ //isolated is the default for ozonefs-lib-legacy which includes the
+ // /ozonefs.txt, otherwise the default is false. It could be overridden.
+ boolean defaultValue =
+ OzoneFileSystem.class.getClassLoader().getResource("ozonefs.txt")
+ != null;
+
+ //Use string here instead of the constant as constant may not be
available
+ //on the classpath of a hadoop 2.7
+ boolean isolatedClassloader =
+ conf.getBoolean("ozone.fs.isolated-classloader", defaultValue);
+
+ if (isolatedClassloader) {
+ this.adapter =
+ OzoneClientAdapterFactory.createAdapter(volumeStr, bucketStr);
+ } else {
+ OzoneConfiguration ozoneConfiguration;
+ if (conf instanceof OzoneConfiguration) {
+ ozoneConfiguration = (OzoneConfiguration) conf;
+ } else {
+ ozoneConfiguration = new OzoneConfiguration(conf);
+ }
+ this.adapter = new OzoneClientAdapterImpl(ozoneConfiguration,
+ volumeStr, bucketStr);
+ }
+
try {
this.userName =
UserGroupInformation.getCurrentUser().getShortUserName();
@@ -142,7 +142,7 @@ public class OzoneFileSystem extends FileSystem {
this.userName = OZONE_DEFAULT_USER;
}
this.workingDir = new Path(OZONE_USER_DIR, this.userName)
- .makeQualified(this.uri, this.workingDir);
+ .makeQualified(this.uri, this.workingDir);
} catch (URISyntaxException ue) {
final String msg = "Invalid Ozone endpoint " + name;
LOG.error(msg, ue);
@@ -153,7 +153,7 @@ public class OzoneFileSystem extends FileSystem {
@Override
public void close() throws IOException {
try {
- ozoneClient.close();
+ adapter.close();
} finally {
super.close();
}
@@ -179,7 +179,7 @@ public class OzoneFileSystem extends FileSystem {
}
return new FSDataInputStream(
- new OzoneFSInputStream(bucket.readKey(key).getInputStream()));
+ new OzoneFSInputStream(adapter.createInputStream(key)));
}
@Override
@@ -200,19 +200,16 @@ public class OzoneFileSystem extends FileSystem {
throw new FileAlreadyExistsException(f + " already exists");
}
LOG.trace("Overwriting file {}", f);
- deleteObject(key);
+ adapter.deleteObject(key);
}
} catch (FileNotFoundException ignored) {
// this means the file is not found
}
- OzoneOutputStream ozoneOutputStream =
- bucket.createKey(key, 0, replicationType, replicationFactor,
- new HashMap<>());
// We pass null to FSDataOutputStream so it won't count writes that
// are being buffered to a file
return new FSDataOutputStream(
- new OzoneFSOutputStream(ozoneOutputStream.getOutputStream()), null);
+ adapter.createKey(key), null);
}
@Override
@@ -236,7 +233,7 @@ public class OzoneFileSystem extends FileSystem {
@Override
public FSDataOutputStream append(Path f, int bufferSize,
- Progressable progress) throws IOException {
+ Progressable progress) throws IOException {
throw new UnsupportedOperationException("append() Not implemented by the "
+ getClass().getSimpleName() + " FileSystem implementation");
}
@@ -256,7 +253,7 @@ public class OzoneFileSystem extends FileSystem {
@Override
boolean processKey(String key) throws IOException {
String newKeyName = dstKey.concat(key.substring(srcKey.length()));
- bucket.renameKey(key, newKeyName);
+ adapter.renameKey(key, newKeyName);
return true;
}
}
@@ -264,7 +261,7 @@ public class OzoneFileSystem extends FileSystem {
/**
* Check whether the source and destination path are valid and then perform
* rename from source path to destination path.
- *
+ * <p>
* The rename operation is performed by renaming the keys with src as prefix.
* For such keys the prefix is changed from src to dst.
*
@@ -361,6 +358,7 @@ public class OzoneFileSystem extends FileSystem {
private class DeleteIterator extends OzoneListingIterator {
private boolean recursive;
+
DeleteIterator(Path f, boolean recursive)
throws IOException {
super(f);
@@ -379,7 +377,7 @@ public class OzoneFileSystem extends FileSystem {
return true;
} else {
LOG.trace("deleting key:" + key);
- boolean succeed = deleteObject(key);
+ boolean succeed = adapter.deleteObject(key);
// if recursive delete is requested ignore the return value of
// deleteObject and issue deletes for other keys.
return recursive || succeed;
@@ -390,6 +388,7 @@ public class OzoneFileSystem extends FileSystem {
/**
* Deletes the children of the input dir path by iterating though the
* DeleteIterator.
+ *
* @param f directory path to be deleted
* @return true if successfully deletes all required keys, false otherwise
* @throws IOException
@@ -431,7 +430,7 @@ public class OzoneFileSystem extends FileSystem {
result = innerDelete(f, recursive);
} else {
LOG.debug("delete: Path is a file: {}", f);
- result = deleteObject(key);
+ result = adapter.deleteObject(key);
}
if (result) {
@@ -449,6 +448,7 @@ public class OzoneFileSystem extends FileSystem {
/**
* Create a fake parent directory key if it does not already exist and no
* other child of this parent directory exists.
+ *
* @param f path to the fake parent directory
* @throws IOException
*/
@@ -457,12 +457,13 @@ public class OzoneFileSystem extends FileSystem {
if (!key.isEmpty() && !o3Exists(f)) {
LOG.debug("Creating new fake directory at {}", f);
String dirKey = addTrailingSlashIfNeeded(key);
- createDirectory(dirKey);
+ adapter.createDirectory(dirKey);
}
}
/**
* Check if a file or directory exists corresponding to given path.
+ *
* @param f path to file/directory.
* @return true if it exists, false otherwise.
* @throws IOException
@@ -487,7 +488,7 @@ public class OzoneFileSystem extends FileSystem {
new HashMap<>(LISTING_PAGE_SIZE);
private Path f; // the input path
- ListStatusIterator(Path f) throws IOException {
+ ListStatusIterator(Path f) throws IOException {
super(f);
this.f = f;
}
@@ -495,6 +496,7 @@ public class OzoneFileSystem extends FileSystem {
/**
* Add the key to the listStatus result if the key corresponds to the
* input path or is an immediate child of the input path.
+ *
* @param key key to be processed
* @return always returns true
* @throws IOException
@@ -518,7 +520,7 @@ public class OzoneFileSystem extends FileSystem {
if (pathToKey(keyPath.getParent()).equals(pathToKey(f))) {
// This key is an immediate child. Can be file or directory
if (key.endsWith(OZONE_URI_DELIMITER)) {
- // Key is a directory
+ // Key is a directory
addSubDirStatus(keyPath);
} else {
addFileStatus(keyPath);
@@ -537,6 +539,7 @@ public class OzoneFileSystem extends FileSystem {
/**
* Adds the FileStatus of keyPath to final result of listStatus.
+ *
* @param filePath path to the file
* @throws FileNotFoundException
*/
@@ -547,6 +550,7 @@ public class OzoneFileSystem extends FileSystem {
/**
* Adds the FileStatus of the subdir to final result of listStatus, if not
* already included.
+ *
* @param dirPath path to the dir
* @throws FileNotFoundException
*/
@@ -560,9 +564,9 @@ public class OzoneFileSystem extends FileSystem {
/**
* Traverse the parent directory structure of keyPath to determine the
* which parent/ grand-parent/.. is the immediate child of the input path
f.
+ *
* @param keyPath path whose parent directory structure should be
traversed.
* @return immediate child path of the input path f.
- * @return immediate child path of the input path f.
*/
Path getImmediateChildPath(Path keyPath) {
Path path = keyPath;
@@ -610,6 +614,7 @@ public class OzoneFileSystem extends FileSystem {
/**
* Get the username of the FS.
+ *
* @return the short name of the user who instantiated the FS
*/
public String getUsername() {
@@ -648,7 +653,7 @@ public class OzoneFileSystem extends FileSystem {
LOG.trace("creating directory for fpart:{}", fPart);
String key = pathToKey(fPart);
String dirKey = addTrailingSlashIfNeeded(key);
- if (!createDirectory(dirKey)) {
+ if (!adapter.createDirectory(dirKey)) {
// Directory creation failed here,
// rollback and delete newly created directories
LOG.trace("Directory creation failed, path:{}", fPart);
@@ -682,11 +687,11 @@ public class OzoneFileSystem extends FileSystem {
if (key.length() == 0) {
return new FileStatus(0, true, 1, 0,
- bucket.getCreationTime(), qualifiedPath);
+ adapter.getCreationTime(), qualifiedPath);
}
// Check if the key exists
- OzoneKey ozoneKey = getKeyInfo(key);
+ BasicKeyInfo ozoneKey = adapter.getKeyInfo(key);
if (ozoneKey != null) {
LOG.debug("Found exact file for path {}: normal file", f);
return new FileStatus(ozoneKey.getDataSize(), false, 1,
@@ -702,6 +707,7 @@ public class OzoneFileSystem extends FileSystem {
* Get the FileStatus for input directory path.
* They key corresponding to input path is appended with a trailing slash
* to return only the corresponding directory key in the bucket.
+ *
* @param f directory path
* @return FileStatus for the input directory path
* @throws FileNotFoundException
@@ -712,9 +718,9 @@ public class OzoneFileSystem extends FileSystem {
String key = pathToKey(qualifiedPath);
key = addTrailingSlashIfNeeded(key);
- OzoneKey ozoneKey = getKeyInfo(key);
- if(ozoneKey != null) {
- if (isDirectory(ozoneKey)) {
+ BasicKeyInfo ozoneKey = adapter.getKeyInfo(key);
+ if (ozoneKey != null) {
+ if (adapter.isDirectory(ozoneKey)) {
// Key is a directory
LOG.debug("Found file (with /) for path {}: fake directory", f);
} else {
@@ -730,7 +736,7 @@ public class OzoneFileSystem extends FileSystem {
// File or directory corresponding to input path does not exist.
// Check if there exists a key prefixed with this key.
- boolean hasChildren = bucket.listKeys(key).hasNext();
+ boolean hasChildren = adapter.hasNextKey(key);
if (hasChildren) {
return new FileStatus(0, true, 1, 0, 0, 0, FsPermission.getDirDefault(),
getUsername(), getUsername(), qualifiedPath);
@@ -740,72 +746,13 @@ public class OzoneFileSystem extends FileSystem {
}
/**
- * Helper method to fetch the key metadata info.
- * @param key key whose metadata information needs to be fetched
- * @return metadata info of the key
- */
- private OzoneKey getKeyInfo(String key) {
- try {
- return bucket.getKey(key);
- } catch (IOException e) {
- LOG.trace("Key:{} does not exist", key);
- return null;
- }
- }
-
- /**
- * Helper method to check if an Ozone key is representing a directory.
- * @param key key to be checked as a directory
- * @return true if key is a directory, false otherwise
- */
- private boolean isDirectory(OzoneKey key) {
- LOG.trace("key name:{} size:{}", key.getName(),
- key.getDataSize());
- return key.getName().endsWith(OZONE_URI_DELIMITER)
- && (key.getDataSize() == 0);
- }
-
- /**
- * Helper method to create an directory specified by key name in bucket.
- * @param keyName key name to be created as directory
- * @return true if the key is created, false otherwise
- */
- private boolean createDirectory(String keyName) {
- try {
- LOG.trace("creating dir for key:{}", keyName);
- bucket.createKey(keyName, 0, replicationType, replicationFactor,
- new HashMap<>()).close();
- return true;
- } catch (IOException ioe) {
- LOG.error("create key failed for key:{}", keyName, ioe);
- return false;
- }
- }
-
- /**
- * Helper method to delete an object specified by key name in bucket.
- * @param keyName key name to be deleted
- * @return true if the key is deleted, false otherwise
- */
- private boolean deleteObject(String keyName) {
- LOG.trace("issuing delete for key" + keyName);
- try {
- bucket.deleteKey(keyName);
- return true;
- } catch (IOException ioe) {
- LOG.error("delete key failed " + ioe.getMessage());
- return false;
- }
- }
-
- /**
* Turn a path (relative or otherwise) into an Ozone key.
*
* @param path the path of the file.
* @return the key of the object that represents the file.
*/
public String pathToKey(Path path) {
- Objects.requireNonNull(path, "Path can not be null!");
+ Objects.requireNonNull(path, "Path canf not be null!");
if (!path.isAbsolute()) {
path = new Path(workingDir, path);
}
@@ -839,17 +786,17 @@ public class OzoneFileSystem extends FileSystem {
}
/**
- * This class provides an interface to iterate through all the keys in the
- * bucket prefixed with the input path key and process them.
- *
- * Each implementing class should define how the keys should be processed
- * through the processKey() function.
+ * This class provides an interface to iterate through all the keys in the
+ * bucket prefixed with the input path key and process them.
+ * <p>
+ * Each implementing class should define how the keys should be processed
+ * through the processKey() function.
*/
private abstract class OzoneListingIterator {
private final Path path;
private final FileStatus status;
private String pathKey;
- private Iterator<? extends OzoneKey> keyIterator;
+ private Iterator<BasicKeyInfo> keyIterator;
OzoneListingIterator(Path path)
throws IOException {
@@ -859,12 +806,13 @@ public class OzoneFileSystem extends FileSystem {
if (status.isDirectory()) {
this.pathKey = addTrailingSlashIfNeeded(pathKey);
}
- keyIterator = bucket.listKeys(pathKey);
+ keyIterator = adapter.listKeys(pathKey);
}
/**
* The output of processKey determines if further iteration through the
* keys should be done or not.
+ *
* @return true if we should continue iteration of keys, false otherwise.
* @throws IOException
*/
@@ -876,6 +824,7 @@ public class OzoneFileSystem extends FileSystem {
* If for any key, the processKey() returns false, then the iteration is
* stopped and returned with false indicating that all the keys could not
* be processed successfully.
+ *
* @return true if all keys are processed successfully, false otherwise.
* @throws IOException
*/
@@ -884,7 +833,7 @@ public class OzoneFileSystem extends FileSystem {
if (status.isDirectory()) {
LOG.trace("Iterating directory:{}", pathKey);
while (keyIterator.hasNext()) {
- OzoneKey key = keyIterator.next();
+ BasicKeyInfo key = keyIterator.next();
LOG.trace("iterating key:{}", key.getName());
if (!processKey(key.getName())) {
return false;
diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml
index 0dc70b6..12250e7 100644
--- a/hadoop-ozone/pom.xml
+++ b/hadoop-ozone/pom.xml
@@ -11,11 +11,7 @@
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0";
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
-http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+--><project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hadoop</groupId>
@@ -43,8 +39,10 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";>
<module>common</module>
<module>client</module>
<module>ozone-manager</module>
- <module>tools</module>
<module>ozonefs</module>
+ <module>ozonefs-lib</module>
+ <module>ozonefs-lib-legacy</module>
+ <module>tools</module>
<module>integration-test</module>
<module>objectstore-service</module>
<module>datanode</module>
@@ -104,6 +102,16 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-filesystem-lib</artifactId>
+ <version>${ozone.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-filesystem-lib-legacy</artifactId>
+ <version>${ozone.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-integration-test</artifactId>
<version>${ozone.version}</version>
<type>test-jar</type>
@@ -207,6 +215,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-tools</artifactId>
</dependency>
+
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
@@ -304,7 +313,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";>
<groupId>org.codehaus.mojo</groupId>
<artifactId>findbugs-maven-plugin</artifactId>
<configuration>
- <excludeFilterFile combine.self="override"></excludeFilterFile>
+ <excludeFilterFile combine.self="override"/>
</configuration>
</plugin>
<plugin>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
