This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 752473628d Revert "HDDS-12528. Create new module for S3 integration
tests (#8152)"
752473628d is described below
commit 752473628d89d953db3afdab91ab04e4940f6b43
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Tue Mar 25 18:34:22 2025 +0100
Revert "HDDS-12528. Create new module for S3 integration tests (#8152)"
This reverts commit d44ebf564ce444c6171d0da05913e500a8038b47.
Reason for revert: integration test failing due to disk out of space
---
.../dev-support/checks/_mvn_unit_report.sh | 4 +-
hadoop-ozone/dev-support/checks/unit.sh | 2 +-
.../dev-support/findbugsExcludeFile.xml | 16 ---
hadoop-ozone/integration-test-s3/pom.xml | 121 --------------------
.../src/test/resources/ozone-site.xml | 126 ---------------------
hadoop-ozone/integration-test/pom.xml | 5 +
.../apache/hadoop/ozone/s3/S3ClientFactory.java | 0
.../apache/hadoop/ozone/s3/S3GatewayService.java | 0
.../hadoop/ozone/s3/awssdk/S3SDKTestUtils.java | 2 +-
.../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 0
.../hadoop/ozone/s3/awssdk/v1/TestS3SDKV1.java | 0
.../awssdk/v1/TestS3SDKV1WithRatisStreaming.java | 0
.../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 0
.../hadoop/ozone/s3/awssdk/v2/TestS3SDKV2.java | 0
.../awssdk/v2/TestS3SDKV2WithRatisStreaming.java | 0
.../org/apache/hadoop/utils}/InputSubstream.java | 7 +-
hadoop-ozone/pom.xml | 7 --
17 files changed, 13 insertions(+), 277 deletions(-)
diff --git a/hadoop-ozone/dev-support/checks/_mvn_unit_report.sh
b/hadoop-ozone/dev-support/checks/_mvn_unit_report.sh
index 8136e6ea80..4d27d50728 100755
--- a/hadoop-ozone/dev-support/checks/_mvn_unit_report.sh
+++ b/hadoop-ozone/dev-support/checks/_mvn_unit_report.sh
@@ -38,7 +38,7 @@ cat ${failures} > "${tempfile}"
leaks=${REPORT_DIR}/leaks.txt
if [[ "${CHECK:-unit}" == "integration" ]]; then
- find hadoop-ozone/integration-test* -not -path '*/iteration*' -name
'*-output.txt' -print0 \
+ find hadoop-ozone/integration-test -not -path '*/iteration*' -name
'*-output.txt' -print0 \
| xargs -n1 -0 "grep" -l -E "not closed properly|was not shutdown
properly" \
| awk -F/ '{sub("-output.txt",""); print $NF}' \
> "${leaks}"
@@ -47,7 +47,7 @@ fi
cluster=${REPORT_DIR}/cluster-startup-errors.txt
if [[ "${CHECK:-unit}" == "integration" ]]; then
- find hadoop-ozone/integration-test* -not -path '*/iteration*' -name
'*-output.txt' -print0 \
+ find hadoop-ozone/integration-test -not -path '*/iteration*' -name
'*-output.txt' -print0 \
| xargs -n1 -0 "grep" -l -E "Unable to build MiniOzoneCluster" \
| awk -F/ '{sub("-output.txt",""); print $NF}' \
> "${cluster}"
diff --git a/hadoop-ozone/dev-support/checks/unit.sh
b/hadoop-ozone/dev-support/checks/unit.sh
index f4a0a834d1..09f45795fe 100755
--- a/hadoop-ozone/dev-support/checks/unit.sh
+++ b/hadoop-ozone/dev-support/checks/unit.sh
@@ -17,5 +17,5 @@
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
CHECK=unit
source "${DIR}/junit.sh" \
- -pl
\!:ozone-integration-test,\!:ozone-integration-test-s3,\!:mini-chaos-tests \
+ -pl \!:ozone-integration-test,\!:mini-chaos-tests \
"$@"
diff --git
a/hadoop-ozone/integration-test-s3/dev-support/findbugsExcludeFile.xml
b/hadoop-ozone/integration-test-s3/dev-support/findbugsExcludeFile.xml
deleted file mode 100644
index ee5ed59808..0000000000
--- a/hadoop-ozone/integration-test-s3/dev-support/findbugsExcludeFile.xml
+++ /dev/null
@@ -1,16 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-<FindBugsFilter>
-</FindBugsFilter>
diff --git a/hadoop-ozone/integration-test-s3/pom.xml
b/hadoop-ozone/integration-test-s3/pom.xml
deleted file mode 100644
index b23d389ef8..0000000000
--- a/hadoop-ozone/integration-test-s3/pom.xml
+++ /dev/null
@@ -1,121 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
https://maven.apache.org/xsd/maven-4.0.0.xsd";>
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.ozone</groupId>
- <artifactId>ozone</artifactId>
- <version>2.1.0-SNAPSHOT</version>
- </parent>
- <artifactId>ozone-integration-test-s3</artifactId>
- <version>2.1.0-SNAPSHOT</version>
- <packaging>jar</packaging>
- <name>Apache Ozone S3 Integration Tests</name>
- <description>Apache Ozone Integration Tests with S3 Gateway</description>
-
- <dependencies>
- <dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-core</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-s3</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>jakarta.xml.bind</groupId>
- <artifactId>jakarta.xml.bind-api</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.ozone</groupId>
- <artifactId>hdds-common</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.ozone</groupId>
- <artifactId>hdds-config</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.ozone</groupId>
- <artifactId>hdds-server-framework</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.ozone</groupId>
- <artifactId>hdds-test-utils</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.ozone</groupId>
- <artifactId>ozone-client</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.ozone</groupId>
- <artifactId>ozone-mini-cluster</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.ozone</groupId>
- <artifactId>ozone-s3gateway</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.ratis</groupId>
- <artifactId>ratis-common</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>software.amazon.awssdk</groupId>
- <artifactId>s3</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>com.github.spotbugs</groupId>
- <artifactId>spotbugs-maven-plugin</artifactId>
- <configuration>
-
<excludeFilterFile>${basedir}/dev-support/findbugsExcludeFile.xml</excludeFilterFile>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <configuration>
- <proc>none</proc>
- </configuration>
- </plugin>
- </plugins>
- </build>
-
-</project>
diff --git a/hadoop-ozone/integration-test-s3/src/test/resources/ozone-site.xml
b/hadoop-ozone/integration-test-s3/src/test/resources/ozone-site.xml
deleted file mode 100644
index 2354b6f321..0000000000
--- a/hadoop-ozone/integration-test-s3/src/test/resources/ozone-site.xml
+++ /dev/null
@@ -1,126 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration>
-
- <property>
- <name>ozone.om.transport.class</name>
-
<value>org.apache.hadoop.ozone.om.protocolPB.Hadoop3OmTransportFactory</value>
- </property>
-
- <property>
- <name>ozone.om.s3.grpc.server_enabled</name>
- <value>false</value>
- </property>
-
- <property>
- <name>hdds.container.ratis.num.write.chunk.threads.per.volume</name>
- <value>4</value>
- </property>
-
- <property>
- <name>ozone.scm.handler.count.key</name>
- <value>20</value>
- </property>
-
- <property>
- <name>ozone.om.handler.count.key</name>
- <value>20</value>
- </property>
-
- <property>
- <name>hdds.container.ratis.datastream.enabled</name>
- <value>true</value>
- </property>
-
-
- <property>
- <name>hdds.heartbeat.interval</name>
- <value>1s</value>
- </property>
- <property>
- <name>ozone.scm.heartbeat.thread.interval</name>
- <value>100ms</value>
- </property>
-
- <property>
- <name>ozone.scm.ratis.pipeline.limit</name>
- <value>3</value>
- </property>
-
- <property>
- <name>ozone.scm.close.container.wait.duration</name>
- <value>1s</value>
- </property>
-
- <property>
- <name>ozone.om.snapshot.diff.job.default.wait.time</name>
- <value>1s</value>
- </property>
-
- <property>
- <name>hdds.container.ratis.log.appender.queue.byte-limit
-</name>
- <value>32MB</value>
- </property>
- <property>
- <name>ozone.om.ratis.log.appender.queue.byte-limit</name>
- <value>4MB</value>
- </property>
- <property>
- <name>ozone.scm.ha.ratis.log.appender.queue.byte-limit</name>
- <value>4MB</value>
- </property>
-
- <property>
- <name>ozone.scm.chunk.size</name>
- <value>1MB</value>
- </property>
- <property>
- <name>ozone.scm.block.size</name>
- <value>4MB</value>
- </property>
- <property>
- <name>ozone.client.stream.buffer.flush.size</name>
- <value>1MB</value>
- </property>
- <property>
- <name>ozone.client.stream.buffer.max.size</name>
- <value>2MB</value>
- </property>
- <property>
- <name>ozone.client.stream.buffer.size</name>
- <value>1MB</value>
- </property>
- <property>
- <name>ozone.client.datastream.buffer.flush.size</name>
- <value>4MB</value>
- </property>
- <property>
- <name>ozone.client.datastream.min.packet.size</name>
- <value>256KB</value>
- </property>
- <property>
- <name>ozone.client.datastream.window.size</name>
- <value>8MB</value>
- </property>
-
-</configuration>
diff --git a/hadoop-ozone/integration-test/pom.xml
b/hadoop-ozone/integration-test/pom.xml
index ff899d4e0b..bbd06127c6 100644
--- a/hadoop-ozone/integration-test/pom.xml
+++ b/hadoop-ozone/integration-test/pom.xml
@@ -286,6 +286,11 @@
<artifactId>ozone-mini-cluster</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>s3</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/S3ClientFactory.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/S3ClientFactory.java
similarity index 100%
rename from
hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/S3ClientFactory.java
rename to
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/S3ClientFactory.java
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/S3GatewayService.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/S3GatewayService.java
similarity index 100%
rename from
hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/S3GatewayService.java
rename to
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/S3GatewayService.java
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/S3SDKTestUtils.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/S3SDKTestUtils.java
similarity index 98%
rename from
hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/S3SDKTestUtils.java
rename to
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/S3SDKTestUtils.java
index 33b7788e69..6703e4bbc4 100644
---
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/S3SDKTestUtils.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/S3SDKTestUtils.java
@@ -23,7 +23,7 @@
import java.io.RandomAccessFile;
import java.security.MessageDigest;
import org.apache.commons.lang3.RandomUtils;
-import org.apache.ozone.test.InputSubstream;
+import org.apache.hadoop.utils.InputSubstream;
/**
* Utilities for S3 SDK tests.
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
similarity index 100%
rename from
hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
rename to
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/TestS3SDKV1.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/TestS3SDKV1.java
similarity index 100%
rename from
hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/TestS3SDKV1.java
rename to
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/TestS3SDKV1.java
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/TestS3SDKV1WithRatisStreaming.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/TestS3SDKV1WithRatisStreaming.java
similarity index 100%
rename from
hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/TestS3SDKV1WithRatisStreaming.java
rename to
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/TestS3SDKV1WithRatisStreaming.java
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
similarity index 100%
rename from
hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
rename to
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/TestS3SDKV2.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/TestS3SDKV2.java
similarity index 100%
rename from
hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/TestS3SDKV2.java
rename to
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/TestS3SDKV2.java
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/TestS3SDKV2WithRatisStreaming.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/TestS3SDKV2WithRatisStreaming.java
similarity index 100%
rename from
hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/TestS3SDKV2WithRatisStreaming.java
rename to
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/TestS3SDKV2WithRatisStreaming.java
diff --git
a/hadoop-hdds/test-utils/src/test/java/org/apache/ozone/test/InputSubstream.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/utils/InputSubstream.java
similarity index 96%
rename from
hadoop-hdds/test-utils/src/test/java/org/apache/ozone/test/InputSubstream.java
rename to
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/utils/InputSubstream.java
index 3c7b73aa93..e71689b9ac 100644
---
a/hadoop-hdds/test-utils/src/test/java/org/apache/ozone/test/InputSubstream.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/utils/InputSubstream.java
@@ -15,12 +15,12 @@
* limitations under the License.
*/
-package org.apache.ozone.test;
+package org.apache.hadoop.utils;
+import com.google.common.base.Preconditions;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.util.Objects;
/**
* A filter input stream implementation that exposes a range of the underlying
input stream.
@@ -33,7 +33,8 @@ public class InputSubstream extends FilterInputStream {
private long markedPosition = 0;
public InputSubstream(InputStream in, long skip, long length) {
- super(Objects.requireNonNull(in, "in == null"));
+ super(in);
+ Preconditions.checkNotNull(in);
this.currentPosition = 0;
this.requestedSkipOffset = skip;
this.requestedLength = length;
diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml
index 7298e26ec6..69cfcf4820 100644
--- a/hadoop-ozone/pom.xml
+++ b/hadoop-ozone/pom.xml
@@ -35,7 +35,6 @@
<module>httpfsgateway</module>
<module>insight</module>
<module>integration-test</module>
- <module>integration-test-s3</module>
<module>interface-client</module>
<module>interface-storage</module>
<module>mini-cluster</module>
@@ -148,12 +147,6 @@
<artifactId>hdds-server-framework</artifactId>
<version>${hdds.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.ozone</groupId>
- <artifactId>hdds-server-framework</artifactId>
- <version>${hdds.version}</version>
- <type>test-jar</type>
- </dependency>
<dependency>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-server-scm</artifactId>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
