This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 19ed794 HDDS-3292. Support Hadoop 3.3 (#2356)
19ed794 is described below
commit 19ed79464ca9ed2210ca8ac47a4736fb67d8bd3e
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Wed Jul 7 09:59:45 2021 +0200
HDDS-3292. Support Hadoop 3.3 (#2356)
---
hadoop-hdds/common/pom.xml | 5 +++
hadoop-hdds/hadoop-dependency-test/pom.xml | 9 +++++
hadoop-hdds/server-scm/pom.xml | 3 +-
.../scm/container/TestContainerReportHandler.java | 4 +-
.../hdds/scm/container/TestReplicationManager.java | 46 ++++------------------
hadoop-hdds/tools/pom.xml | 12 +++---
.../scm/cli/datanode/TestListInfoSubcommand.java | 4 +-
hadoop-ozone/dist/src/main/license/jar-report.txt | 4 +-
.../dist/src/main/smoketest/spnego/web.robot | 2 +-
hadoop-ozone/integration-test/pom.xml | 3 +-
.../hadoop/ozone/TestStorageContainerManager.java | 8 ++--
.../hadoop/ozone/shell/TestOzoneShellHA.java | 1 -
pom.xml | 5 +--
13 files changed, 42 insertions(+), 64 deletions(-)
diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml
index c3f29df..4940adc 100644
--- a/hadoop-hdds/common/pom.xml
+++ b/hadoop-hdds/common/pom.xml
@@ -34,6 +34,11 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd";>
<artifactId>hdds-hadoop-dependency-client</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.ozone</groupId>
+ <artifactId>hdds-hadoop-dependency-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>info.picocli</groupId>
<artifactId>picocli</artifactId>
</dependency>
diff --git a/hadoop-hdds/hadoop-dependency-test/pom.xml
b/hadoop-hdds/hadoop-dependency-test/pom.xml
index 34cf949..799fc54 100644
--- a/hadoop-hdds/hadoop-dependency-test/pom.xml
+++ b/hadoop-hdds/hadoop-dependency-test/pom.xml
@@ -54,5 +54,14 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd";>
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.assertj</groupId>
+ <artifactId>assertj-core</artifactId>
+ <scope>compile</scope>
+ </dependency>
</dependencies>
</project>
diff --git a/hadoop-hdds/server-scm/pom.xml b/hadoop-hdds/server-scm/pom.xml
index 1f6f7e9..89edde4 100644
--- a/hadoop-hdds/server-scm/pom.xml
+++ b/hadoop-hdds/server-scm/pom.xml
@@ -118,8 +118,7 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd";>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <version>${mockito1-hadoop.version}</version>
+ <artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
index a72ab61..97e61ec 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hdds.scm.server
.SCMDatanodeHeartbeatDispatcher.ContainerReportFromDatanode;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import
org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException;
+import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -648,7 +649,8 @@ public class TestContainerReportHandler {
new ContainerReportFromDatanode(datanodeOne, containerReport);
reportHandler.onMessage(containerReportFromDatanode, publisher);
- Mockito.verify(publisher, Mockito.times(1));
+ Mockito.verify(publisher, Mockito.times(1))
+ .fireEvent(Mockito.any(), Mockito.any(CommandForDatanode.class));
Assert.assertEquals(0, containerManager.getContainerReplicas(
containerOne.containerID()).size());
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java
index 7d3d457..3bdc6ab 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java
@@ -47,7 +47,6 @@ import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import org.mockito.ArgumentMatcher;
import org.mockito.Mockito;
import java.io.IOException;
@@ -61,7 +60,6 @@ import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
@@ -130,8 +128,8 @@ public class TestReplicationManager {
containerPlacementPolicy = Mockito.mock(PlacementPolicy.class);
Mockito.when(containerPlacementPolicy.chooseDatanodes(
- Mockito.anyListOf(DatanodeDetails.class),
- Mockito.anyListOf(DatanodeDetails.class),
+ Mockito.any(),
+ Mockito.any(),
Mockito.anyInt(), Mockito.anyLong()))
.thenAnswer(invocation -> {
int count = (int) invocation.getArguments()[2];
@@ -141,7 +139,7 @@ public class TestReplicationManager {
});
Mockito.when(containerPlacementPolicy.validateContainerPlacement(
- Mockito.anyListOf(DatanodeDetails.class),
+ Mockito.any(),
Mockito.anyInt()
)).thenAnswer(invocation -> {
return new ContainerPlacementStatusDefault(2, 2, 3);
@@ -730,7 +728,7 @@ public class TestReplicationManager {
// test where there are 3 replicas. When there are 2 or 4 replicas
// the status returned will be healthy.
Mockito.when(containerPlacementPolicy.validateContainerPlacement(
- Mockito.argThat(new ListOfNElements(3)),
+ Mockito.argThat(list -> list.size() == 3),
Mockito.anyInt()
)).thenAnswer(invocation -> {
return new ContainerPlacementStatusDefault(1, 2, 3);
@@ -799,7 +797,7 @@ public class TestReplicationManager {
// Ensure a mis-replicated status is returned for any containers in this
// test where there are exactly 3 replicas checked.
Mockito.when(containerPlacementPolicy.validateContainerPlacement(
- Mockito.argThat(new ListOfNElements(3)),
+ Mockito.argThat(list -> list.size() == 3),
Mockito.anyInt()
)).thenAnswer(
invocation -> new ContainerPlacementStatusDefault(1, 2, 3));
@@ -843,7 +841,7 @@ public class TestReplicationManager {
containerStateManager.updateContainerReplica(id, replicaFour);
Mockito.when(containerPlacementPolicy.validateContainerPlacement(
- Mockito.argThat(new ListOfNElements(3)),
+ Mockito.argThat(list -> list.size() == 3),
Mockito.anyInt()
)).thenAnswer(
invocation -> new ContainerPlacementStatusDefault(2, 2, 3));
@@ -883,8 +881,7 @@ public class TestReplicationManager {
containerStateManager.updateContainerReplica(id, replicaFive);
Mockito.when(containerPlacementPolicy.validateContainerPlacement(
- Mockito.argThat(new FunctionMatcher(list ->
- list != null && ((List) list).size() <= 4)),
+ Mockito.argThat(list -> list != null && list.size() <= 4),
Mockito.anyInt()
)).thenAnswer(
invocation -> new ContainerPlacementStatusDefault(1, 2, 3));
@@ -1188,31 +1185,4 @@ public class TestReplicationManager {
}
}
- static class ListOfNElements extends ArgumentMatcher<List> {
-
- private int expected;
-
- ListOfNElements(int expected) {
- this.expected = expected;
- }
-
- @Override
- public boolean matches(Object argument) {
- return ((List)argument).size() == expected;
- }
- }
-
- static class FunctionMatcher extends ArgumentMatcher<List> {
-
- private Function<Object, Boolean> function;
-
- FunctionMatcher(Function<Object, Boolean> function) {
- this.function = function;
- }
-
- @Override
- public boolean matches(Object argument) {
- return function.apply(argument);
- }
- }
-}
\ No newline at end of file
+}
diff --git a/hadoop-hdds/tools/pom.xml b/hadoop-hdds/tools/pom.xml
index 1ce76e6..0799835 100644
--- a/hadoop-hdds/tools/pom.xml
+++ b/hadoop-hdds/tools/pom.xml
@@ -78,12 +78,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd";>
<groupId>org.xerial</groupId>
<artifactId>sqlite-jdbc</artifactId>
</dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <version>${mockito1-hadoop.version}</version>
- <scope>test</scope>
- </dependency>
-
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
diff --git
a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java
b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java
index b6af473..a05417e 100644
---
a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java
+++
b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java
@@ -68,9 +68,7 @@ public class TestListInfoSubcommand {
public void testDataNodeOperationalStateAndHealthIncludedInOutput()
throws Exception {
ScmClient scmClient = mock(ScmClient.class);
-
Mockito.when(scmClient.queryNode(any(HddsProtos.NodeOperationalState.class),
- any(HddsProtos.NodeState.class), any(HddsProtos.QueryScope.class),
- Mockito.anyString()))
+ Mockito.when(scmClient.queryNode(any(), any(), any(), any()))
.thenAnswer(invocation -> getNodeDetails());
Mockito.when(scmClient.listPipelines())
.thenReturn(new ArrayList<>());
diff --git a/hadoop-ozone/dist/src/main/license/jar-report.txt
b/hadoop-ozone/dist/src/main/license/jar-report.txt
index 06aeeca..df1097a 100644
--- a/hadoop-ozone/dist/src/main/license/jar-report.txt
+++ b/hadoop-ozone/dist/src/main/license/jar-report.txt
@@ -68,6 +68,8 @@ share/ozone/lib/hdds-server-scm.jar
share/ozone/lib/hdds-tools.jar
share/ozone/lib/hadoop-hdfs-client.jar
share/ozone/lib/hadoop-hdfs.jar
+share/ozone/lib/hadoop-shaded-guava.jar
+share/ozone/lib/hadoop-shaded-protobuf_3_7.jar
share/ozone/lib/ozone-client.jar
share/ozone/lib/ozone-common.jar
share/ozone/lib/ozone-csi.jar
@@ -111,7 +113,6 @@ share/ozone/lib/jakarta.validation-api.jar
share/ozone/lib/jakarta.ws.rs-api.jar
share/ozone/lib/jakarta.xml.bind-api.jar
share/ozone/lib/javassist-GA.jar
-share/ozone/lib/javax.activation-api.jar
share/ozone/lib/javax.annotation-api.jar
share/ozone/lib/javax.el-api.jar
share/ozone/lib/javax.inject.jar
@@ -220,6 +221,7 @@ share/ozone/lib/simpleclient.jar
share/ozone/lib/slf4j-api.jar
share/ozone/lib/slf4j-log4j12.jar
share/ozone/lib/snakeyaml.jar
+share/ozone/lib/snappy-java.jar
share/ozone/lib/spring-beans.RELEASE.jar
share/ozone/lib/spring-core.RELEASE.jar
share/ozone/lib/spring-jcl.RELEASE.jar
diff --git a/hadoop-ozone/dist/src/main/smoketest/spnego/web.robot
b/hadoop-ozone/dist/src/main/smoketest/spnego/web.robot
index 4a92bd7..c500d57 100644
--- a/hadoop-ozone/dist/src/main/smoketest/spnego/web.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/spnego/web.robot
@@ -41,7 +41,7 @@ Verify SPNEGO enabled URL
[arguments] ${url}
Run Keyword if '${SECURITY_ENABLED}' == 'true' Execute
kdestroy
${result} = Execute curl --negotiate
-u : -v -s -I ${url}
- Should contain ${result} 401 Unauthorized
+ Should contain ${result} 401 Authentication required
Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit test user
testuser testuser.keytab
${result} = Execute curl --negotiate
-u : -v -s -I ${url}
diff --git a/hadoop-ozone/integration-test/pom.xml
b/hadoop-ozone/integration-test/pom.xml
index 97e72ae..51328e4 100644
--- a/hadoop-ozone/integration-test/pom.xml
+++ b/hadoop-ozone/integration-test/pom.xml
@@ -135,8 +135,7 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd";>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <version>${mockito1-hadoop.version}</version>
+ <artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
index 7763855..1370ee1 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
@@ -820,9 +820,8 @@ public class TestStorageContainerManager {
}
}
- @SuppressWarnings("visibilitymodifier")
- static class CloseContainerCommandMatcher
- extends ArgumentMatcher<CommandForDatanode> {
+ private static class CloseContainerCommandMatcher
+ implements ArgumentMatcher<CommandForDatanode> {
private final CommandForDatanode cmd;
private final UUID uuid;
@@ -833,8 +832,7 @@ public class TestStorageContainerManager {
}
@Override
- public boolean matches(Object argument) {
- CommandForDatanode cmdRight = (CommandForDatanode) argument;
+ public boolean matches(CommandForDatanode cmdRight) {
CloseContainerCommand left = (CloseContainerCommand) cmd.getCommand();
CloseContainerCommand right =
(CloseContainerCommand) cmdRight.getCommand();
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
index 162ed8a..155738b 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
@@ -139,7 +139,6 @@ public class TestOzoneShellHA {
.setOMServiceId(omServiceId)
.setNumOfOzoneManagers(numOfOMs)
.build();
- conf.setQuietMode(false);
cluster.waitForClusterToBeReady();
}
diff --git a/pom.xml b/pom.xml
index 2cdca42..0097604 100644
--- a/pom.xml
+++ b/pom.xml
@@ -62,7 +62,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xs
</organization>
<properties>
- <hadoop.version>3.2.2</hadoop.version>
+ <hadoop.version>3.3.1</hadoop.version>
<!-- version for hdds/ozone components -->
<hdds.version>${ozone.version}</hdds.version>
@@ -161,7 +161,6 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xs
<hikari.version>2.4.12</hikari.version>
<mssql.version>6.2.1.jre7</mssql.version>
<okhttp.version>2.7.5</okhttp.version>
- <mockito1-hadoop.version>1.8.5</mockito1-hadoop.version>
<mockito1-powermock.version>1.10.19</mockito1-powermock.version>
<mockito2.version>2.28.2</mockito2.version>
<hamcrest.version>1.3</hamcrest.version>
@@ -1564,7 +1563,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xs
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
- <version>3.8.0</version>
+ <version>3.12.2</version>
<scope>test</scope>
</dependency>
<dependency>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
