This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hadoop-api-shim.git
The following commit(s) were added to refs/heads/main by this push:
new 03dac43 shim work
03dac43 is described below
commit 03dac43ff47c91c36d3ab419a2db338024fc6b82
Author: Steve Loughran <ste...@cloudera.com>
AuthorDate: Wed Jun 14 15:35:04 2023 +0100
shim work
* Tuning of shim APIs when wiring up to parquet (javadocs)
* shim-test main jar includes the s3a.xml and abfs.xml files used by
the matching test contract classes for s3a and abfs in the
hadoop test jars. this will set them up for use in integration
tests.
---
LICENSE | 41 ------
hadoop-api-shim-test/pom.xml | 10 ++
hadoop-api-shim-test/src/main/resources/abfs.xml | 85 ++++++++++++
.../src/main/resources/contract/s3a.xml | 145 +++++++++++++++++++++
hadoop-api-shim/pom.xml | 3 +-
.../apache/hadoop/fs/shim/api/VectorFileRange.java | 6 +-
pom.xml | 1 -
7 files changed, 246 insertions(+), 45 deletions(-)
diff --git a/LICENSE b/LICENSE
index b006581..f433b1a 100644
--- a/LICENSE
+++ b/LICENSE
@@ -175,44 +175,3 @@
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
-
---------------------------------------------------------------------------------
-
-This product includes code from Apache Avro.
-
-Copyright: 2014 The Apache Software Foundation.
-Home page: https://avro.apache.org/
-License: http://www.apache.org/licenses/LICENSE-2.0
-
---------------------------------------------------------------------------------
-
-This project includes code from Daniel Lemire's JavaFastPFOR project. The
-"Lemire" bit packing source code produced by parquet-generator is derived from
-the JavaFastPFOR project.
-
-Copyright: 2013 Daniel Lemire
-Home page: http://lemire.me/en/
-Project page: https://github.com/lemire/JavaFastPFOR
-License: Apache License Version 2.0 http://www.apache.org/licenses/LICENSE-2.0
-
---------------------------------------------------------------------------------
-
-This product includes code from Apache Spark.
-
-* dev/merge_parquet_pr.py is based on Spark's dev/merge_spark_pr.py
-
-Copyright: 2014 The Apache Software Foundation.
-Home page: https://spark.apache.org/
-License: http://www.apache.org/licenses/LICENSE-2.0
-
---------------------------------------------------------------------------------
-
-This product includes code from Twitter's ElephantBird project.
-
-* parquet-hadoop's UnmaterializableRecordCounter.java includes code from
- ElephantBird's LzoRecordReader.java
-
-Copyright: 2012-2014 Twitter
-Home page: https://github.com/twitter/elephant-bird
-License: http://www.apache.org/licenses/LICENSE-2.0
-
diff --git a/hadoop-api-shim-test/pom.xml b/hadoop-api-shim-test/pom.xml
index b04ef27..a1875bc 100644
--- a/hadoop-api-shim-test/pom.xml
+++ b/hadoop-api-shim-test/pom.xml
@@ -32,6 +32,16 @@
<description>
Module to test the hadoop-api-shim library against a later version
of hadoop.
+ The main JAR includes resource XML files which allow for the
+ contract test classes of hadoop cloud store modules to be instantiated
+ hadoop-azure: org.apache.hadoop.fs.azurebfs.contract.AbfsFileSystemContract
+ hadop-aws: org.apache.hadoop.fs.contract.s3a.S3AContract
+ +any added later.
+ These have to go into a main jar as -test.jars are always built without
+ the resources entries on the classpath. By including this jar along with
the
+ relevant cloud test modules, it's possible to wire up the test runs as per
+ the normal cloud connector test mechanism
+ (secrets or better an xminclude to the git-excluded
test/resources/auth-keys.xml
</description>
diff --git a/hadoop-api-shim-test/src/main/resources/abfs.xml
b/hadoop-api-shim-test/src/main/resources/abfs.xml
new file mode 100644
index 0000000..f06e5ca
--- /dev/null
+++ b/hadoop-api-shim-test/src/main/resources/abfs.xml
@@ -0,0 +1,85 @@
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<configuration xmlns:xi="http://www.w3.org/2001/XInclude";>
+ <property>
+ <name>fs.contract.test.root-tests-enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-append</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-seek</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.rename-overwrites-dest</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.rename-returns-false-if-source-missing</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.rename-creates-dest-dirs</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-settimes</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-concat</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-getfilestatus</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-unbuffer</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-hflush</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-hsync</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.metadata_updated_on_hsync</name>
+ <value>true</value>
+ </property>
+
+</configuration>
diff --git a/hadoop-api-shim-test/src/main/resources/contract/s3a.xml
b/hadoop-api-shim-test/src/main/resources/contract/s3a.xml
new file mode 100644
index 0000000..4b84397
--- /dev/null
+++ b/hadoop-api-shim-test/src/main/resources/contract/s3a.xml
@@ -0,0 +1,145 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<configuration>
+ <!--
+ S3A is a blobstore, with very different behavior than a
+ classic filesystem.
+ -->
+
+ <property>
+ <name>fs.contract.test.root-tests-enabled</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.test.random-seek-count</name>
+ <value>10</value>
+ </property>
+
+ <property>
+ <name>fs.contract.is-blobstore</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.create-visibility-delayed</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.is-case-sensitive</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.rename-creates-dest-dirs</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.rename-returns-false-if-source-missing</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.rename-overwrites-dest</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.rename-returns-false-if-dest-exists</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.rename-remove-dest-if-empty-dir</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-append</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-atomic-directory-delete</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-atomic-rename</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-block-locality</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-concat</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-getfilestatus</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-seek</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-seek-on-closed-file</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.rejects-seek-past-eof</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-strict-exceptions</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-unix-permissions</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.rename-overwrites-dest</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-unbuffer</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.create-file-under-file-allowed</name>
+ <value>true</value>
+ </property>
+
+</configuration>
diff --git a/hadoop-api-shim/pom.xml b/hadoop-api-shim/pom.xml
index c58d664..7f8e11c 100644
--- a/hadoop-api-shim/pom.xml
+++ b/hadoop-api-shim/pom.xml
@@ -31,7 +31,7 @@
<name>hadoop-api-shim-library</name>
<description>
- A simple hadoop-api-shim-library.
+ The redistributable hadoop-api-shim-library.
</description>
@@ -72,6 +72,7 @@
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
diff --git
a/hadoop-api-shim/src/main/java/org/apache/hadoop/fs/shim/api/VectorFileRange.java
b/hadoop-api-shim/src/main/java/org/apache/hadoop/fs/shim/api/VectorFileRange.java
index 3397390..0bfeb0f 100644
---
a/hadoop-api-shim/src/main/java/org/apache/hadoop/fs/shim/api/VectorFileRange.java
+++
b/hadoop-api-shim/src/main/java/org/apache/hadoop/fs/shim/api/VectorFileRange.java
@@ -25,8 +25,10 @@ import java.util.function.IntFunction;
import org.apache.hadoop.fs.shim.impl.VectorFileRangeImpl;
/**
- * A byte range of a file.
- * This is used for the asynchronous gather read API of
+ * A byte range of a file, based on the Hadoop Vector API class
+ * {@code org.apache.hadoop.fs.FileRange} used in
+ * {@code PositionedReadable.readVectored()}.
+ * In the shim code, it is used in for the vector read operation
* {@link FSDataInputStreamShim#readVectoredRanges(List, IntFunction)}.
*/
public interface VectorFileRange {
diff --git a/pom.xml b/pom.xml
index 4442ca1..822ab47 100644
--- a/pom.xml
+++ b/pom.xml
@@ -23,7 +23,6 @@
<version>1.0-SNAPSHOT</version>
<packaging>pom</packaging>
<name>hadoop-api-shim-project</name>
- <!-- FIXME change it to the project's website -->
<url>https://hadoop.apache.org/</url>
<licenses>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
