[06/50] [abbrv] hadoop git commit: HDDS-518. Implement PutObject Rest endpoint. Contributed by chencan.
HDDS-518. Implement PutObject Rest endpoint. Contributed by chencan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ae268b48
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ae268b48
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ae268b48
Branch: refs/heads/HDFS-12943
Commit: ae268b48943dcce1adacd4bf01231fcb21d76f3a
Parents: cdad91c
Author: Márton Elek
Authored: Sat Oct 13 15:55:57 2018 +0200
Committer: Márton Elek
Committed: Sat Oct 13 15:55:57 2018 +0200
--
.../hadoop/ozone/s3/object/PutObject.java | 53 +---
.../hadoop/ozone/s3/object/TestPutObject.java | 89
2 files changed, 131 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae268b48/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/PutObject.java
--
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/PutObject.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/PutObject.java
index 53e63d8..84c25eb 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/PutObject.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/PutObject.java
@@ -17,45 +17,76 @@
*/
package org.apache.hadoop.ozone.s3.object;
+import javax.ws.rs.DefaultValue;
import javax.ws.rs.HeaderParam;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
import java.io.IOException;
import java.io.InputStream;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.s3.EndpointBase;
import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.ozone.s3.EndpointBase;
+import org.apache.http.HttpStatus;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* File upload.
*/
-@Path("/{volume}/{bucket}/{path:.+}")
+@Path("/{bucket}/{path:.+}")
public class PutObject extends EndpointBase {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(PutObject.class);
+
@PUT
@Produces(MediaType.APPLICATION_XML)
- public void put(
- @PathParam("volume") String volumeName,
+ public Response put(
+ @Context HttpHeaders headers,
@PathParam("bucket") String bucketName,
@PathParam("path") String keyPath,
+ @DefaultValue("STAND_ALONE" ) @QueryParam("replicationType")
+ ReplicationType replicationType,
+ @DefaultValue("ONE") @QueryParam("replicationFactor")
+ ReplicationFactor replicationFactor,
+ @DefaultValue("32 * 1024 * 1024") @QueryParam("chunkSize")
+ String chunkSize,
@HeaderParam("Content-Length") long length,
InputStream body) throws IOException {
-OzoneBucket bucket = getBucket(volumeName, bucketName);
+try {
+ Configuration config = new OzoneConfiguration();
+ config.set(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY, chunkSize);
+
+ OzoneBucket bucket = getVolume(getOzoneVolumeName(bucketName))
+ .getBucket(bucketName);
+ OzoneOutputStream output = bucket
+ .createKey(keyPath, length, replicationType, replicationFactor);
-OzoneOutputStream output = bucket
-.createKey(keyPath, length, ReplicationType.STAND_ALONE,
-ReplicationFactor.ONE);
+ IOUtils.copy(body, output);
+ output.close();
-IOUtils.copy(body, output);
-output.close();
+ return Response.ok().status(HttpStatus.SC_OK)
+ .header("Content-Length", length)
+ .build();
+} catch (IOException ex) {
+ LOG.error("Exception occurred in PutObject", ex);
+ throw ex;
+}
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae268b48/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/object/TestPutObject.java
--
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/object/TestPutObject.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/object/TestPutObject.java
new file mode 100644
index 000..a8f0648
--- /dev/null
+++
b/hadoop-ozone/s3gateway/src/te
[15/50] [abbrv] hadoop git commit: YARN-8836. Add tags and attributes in resource definition. Contributed by Weiwei Yang.
YARN-8836. Add tags and attributes in resource definition. Contributed by
Weiwei Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8e5365e2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8e5365e2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8e5365e2
Branch: refs/heads/HDFS-12943
Commit: 8e5365e277a184ff65f2f6bca2bf037d1a9f3fd0
Parents: 5033deb
Author: Sunil G
Authored: Mon Oct 15 15:38:42 2018 +0530
Committer: Sunil G
Committed: Mon Oct 15 15:38:42 2018 +0530
--
.../yarn/api/records/ResourceInformation.java | 68 +-
.../yarn/util/resource/ResourceUtils.java | 9 +-
.../src/main/proto/yarn_protos.proto| 2 +
.../yarn/conf/TestResourceInformation.java | 44 +++
.../yarn/api/records/impl/pb/ProtoUtils.java| 28
.../api/records/impl/pb/ResourcePBImpl.java | 23
.../hadoop/yarn/api/TestResourcePBImpl.java | 128 ++-
.../resource-types/resource-types-5.xml | 53
8 files changed, 350 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e5365e2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
index c83c3a2..057e94e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
@@ -19,11 +19,15 @@
package org.apache.hadoop.yarn.api.records;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
import org.apache.hadoop.yarn.util.UnitsConversionUtil;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
+import java.util.Set;
/**
* Class to encapsulate information about a Resource - the name of the
resource,
@@ -37,6 +41,8 @@ public class ResourceInformation implements
Comparable {
private long value;
private long minimumAllocation;
private long maximumAllocation;
+ private Set tags = new HashSet<>();
+ private Map attributes = new HashMap<>();
// Known resource types
public static final String MEMORY_URI = "memory-mb";
@@ -185,6 +191,42 @@ public class ResourceInformation implements
Comparable {
}
/**
+ * Get the attributes of the resource.
+ * @return resource attributes
+ */
+ public Map getAttributes() {
+return attributes;
+ }
+
+ /**
+ * Set a map of attributes to the resource.
+ * @param attributes resource attributes
+ */
+ public void setAttributes(Map attributes) {
+if (attributes != null) {
+ this.attributes = attributes;
+}
+ }
+
+ /**
+ * Get resource tags.
+ * @return resource tags
+ */
+ public Set getTags() {
+return this.tags;
+ }
+
+ /**
+ * Add tags to the resource.
+ * @param tags resource tags
+ */
+ public void setTags(Set tags) {
+if (tags != null) {
+ this.tags = tags;
+}
+ }
+
+ /**
* Create a new instance of ResourceInformation from another object.
*
* @param other the object from which the new object should be created
@@ -199,6 +241,15 @@ public class ResourceInformation implements
Comparable {
public static ResourceInformation newInstance(String name, String units,
long value, ResourceTypes type, long minimumAllocation,
long maximumAllocation) {
+return ResourceInformation.newInstance(name, units, value, type,
+minimumAllocation, maximumAllocation,
+ImmutableSet.of(), ImmutableMap.of());
+ }
+
+ public static ResourceInformation newInstance(String name, String units,
+ long value, ResourceTypes type, long minimumAllocation,
+ long maximumAllocation,
+ Set tags, Map attributes) {
ResourceInformation ret = new ResourceInformation();
ret.setName(name);
ret.setResourceType(type);
@@ -206,6 +257,8 @@ public class ResourceInformation implements
Comparable {
ret.setValue(value);
ret.setMinimumAllocation(minimumAllocation);
ret.setMaximumAllocation(maximumAllocation);
+ret.setTags(tags);
+ret.setAttributes(attributes);
return ret;
}
@@ -258,13 +311,16 @@ public class ResourceInformation implements
Comparable {
dst.
[01/50] [abbrv] hadoop git commit: HDDS-587. Add new classes for pipeline management. Contributed by Lokesh Jain.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-12943 bba8aa3d9 -> 3a5b78ac4
HDDS-587. Add new classes for pipeline management. Contributed by Lokesh Jain.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5c8e023b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5c8e023b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5c8e023b
Branch: refs/heads/HDFS-12943
Commit: 5c8e023ba32da3e65193f6ced354efe830dba75d
Parents: c07b95b
Author: Nandakumar
Authored: Sat Oct 13 03:22:20 2018 +0530
Committer: Nandakumar
Committed: Sat Oct 13 03:22:20 2018 +0530
--
.../hadoop/hdds/scm/pipeline/Pipeline.java | 211
.../hadoop/hdds/scm/pipeline/PipelineID.java| 80 ++
.../hadoop/hdds/scm/pipeline/package-info.java | 24 ++
.../hdds/scm/pipeline/PipelineFactory.java | 56 +
.../hdds/scm/pipeline/PipelineManager.java | 58 +
.../hdds/scm/pipeline/PipelineProvider.java | 35 +++
.../hdds/scm/pipeline/PipelineStateManager.java | 179 ++
.../hdds/scm/pipeline/PipelineStateMap.java | 212
.../scm/pipeline/RatisPipelineProvider.java | 135 ++
.../hdds/scm/pipeline/SCMPipelineManager.java | 226 +
.../scm/pipeline/SimplePipelineProvider.java| 80 ++
.../hadoop/hdds/scm/pipeline/package-info.java | 24 ++
.../scm/pipeline/TestPipelineStateManager.java | 246 +++
.../scm/pipeline/TestRatisPipelineProvider.java | 104
.../pipeline/TestSimplePipelineProvider.java| 102
15 files changed, 1772 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c8e023b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
--
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
new file mode 100644
index 000..b58a001
--- /dev/null
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
@@ -0,0 +1,211 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.pipeline;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Represents a group of datanodes which store a container.
+ */
+public final class Pipeline {
+
+ private final PipelineID id;
+ private final ReplicationType type;
+ private final ReplicationFactor factor;
+
+ private LifeCycleState state;
+ private List nodes;
+
+ private Pipeline(PipelineID id, ReplicationType type,
+ ReplicationFactor factor, LifeCycleState state,
+ List nodes) {
+this.id = id;
+this.type = type;
+this.factor = factor;
+this.state = state;
+this.nodes = nodes;
+ }
+
+ /**
+ * Returns the ID of this pipeline.
+ *
+ * @return PipelineID
+ */
+ public PipelineID getID() {
+return id;
+ }
+
+ /**
+ * Returns the type.
+ *
+ * @return type - Simple or Ratis.
+ */
+ public ReplicationType getType() {
+return type;
+ }
+
+ /**
+ * Returns the factor.
+ *
+ * @return type - Simple or Ratis.
+ */
+ public ReplicationFactor getFactor() {
+return factor;
+ }
+
+ /**
+ * Returns the State of the pipeline.
+ *
+ * @return - LifeCycleStates.
+ */
+ public LifeCycleState getLifeCycleState() {
+return state;
+ }
+
+ /**
+ * Retu
[09/50] [abbrv] hadoop git commit: HDDS-629. Make ApplyTransaction calls in ContainerStateMachine idempotent. Contributed by Shashikant Banerjee.
HDDS-629. Make ApplyTransaction calls in ContainerStateMachine idempotent.
Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0473b680
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0473b680
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0473b680
Branch: refs/heads/HDFS-12943
Commit: 0473b6817cfe4f03acdcb0eedc78b509244f
Parents: 9227f3d
Author: Jitendra Pandey
Authored: Sat Oct 13 12:12:08 2018 -0700
Committer: Jitendra Pandey
Committed: Sat Oct 13 12:12:08 2018 -0700
--
.../org/apache/hadoop/ozone/OzoneConsts.java| 1 +
.../apache/hadoop/utils/MetadataKeyFilters.java | 3 +-
.../common/helpers/ContainerReport.java | 205 ---
.../common/helpers/KeyValueContainerReport.java | 117 ---
.../container/common/interfaces/Container.java | 5 +
.../container/keyvalue/KeyValueContainer.java | 9 +-
.../keyvalue/KeyValueContainerData.java | 16 ++
.../keyvalue/impl/BlockManagerImpl.java | 21 +-
.../container/ozoneimpl/ContainerReader.java| 7 +
.../StorageContainerDatanodeProtocol.proto | 1 +
.../ozone/om/helpers/OmKeyLocationInfo.java | 4 +
.../hadoop/ozone/client/rpc/TestBCSID.java | 147 +
12 files changed, 208 insertions(+), 328 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0473b680/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
--
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index 923271c..8ccc648 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -113,6 +113,7 @@ public final class OzoneConsts {
public static final String DELETING_KEY_PREFIX = "#deleting#";
public static final String DELETED_KEY_PREFIX = "#deleted#";
public static final String DELETE_TRANSACTION_KEY_PREFIX = "#delTX#";
+ public static final String BLOCK_COMMIT_SEQUENCE_ID_PREFIX = "#BCSID";
/**
* OM LevelDB prefixes.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0473b680/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java
--
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java
index a3430f8..04c87ae 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java
@@ -42,7 +42,8 @@ public final class MetadataKeyFilters {
new MetadataKeyFilters.KeyPrefixFilter()
.addFilter(OzoneConsts.DELETING_KEY_PREFIX, true)
.addFilter(OzoneConsts.DELETED_KEY_PREFIX, true)
- .addFilter(OzoneConsts.DELETE_TRANSACTION_KEY_PREFIX, true);
+ .addFilter(OzoneConsts.DELETE_TRANSACTION_KEY_PREFIX, true)
+ .addFilter(OzoneConsts.BLOCK_COMMIT_SEQUENCE_ID_PREFIX, true);
private MetadataKeyFilters() {
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0473b680/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerReport.java
--
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerReport.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerReport.java
deleted file mode 100644
index a4c1f2f..000
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerReport.java
+++ /dev/null
@@ -1,205 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific langu
[13/50] [abbrv] hadoop git commit: HADOOP-15849. Upgrade netty version to 3.10.6.
HADOOP-15849. Upgrade netty version to 3.10.6. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8853fc8a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8853fc8a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8853fc8a Branch: refs/heads/HDFS-12943 Commit: 8853fc8a55b07ecdc5ce8d85278b822e5675d97a Parents: 603649d Author: Xiao Chen Authored: Sat Oct 13 20:21:36 2018 -0700 Committer: Xiao Chen Committed: Sat Oct 13 20:22:02 2018 -0700 -- hadoop-project/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8853fc8a/hadoop-project/pom.xml -- diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index b850c7c..4cdbcfb 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -961,7 +961,7 @@ io.netty netty -3.10.5.Final +3.10.6.Final - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[38/50] [abbrv] hadoop git commit: YARN-8448. AM HTTPS Support for AM communication with RMWeb proxy. (Contributed by Robert Kanter)
YARN-8448. AM HTTPS Support for AM communication with RMWeb proxy. (Contributed
by Robert Kanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c2288ac4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c2288ac4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c2288ac4
Branch: refs/heads/HDFS-12943
Commit: c2288ac45b748b4119442c46147ccc324926c340
Parents: d59ca43
Author: Haibo Chen
Authored: Tue Oct 16 13:36:26 2018 -0700
Committer: Haibo Chen
Committed: Tue Oct 16 13:36:26 2018 -0700
--
.../org/apache/hadoop/security/Credentials.java | 1 +
.../hadoop/security/ssl/KeyStoreTestUtil.java | 21 +-
.../hadoop/yarn/api/ApplicationConstants.java | 22 +
.../hadoop/yarn/conf/YarnConfiguration.java | 20 +
.../src/main/resources/yarn-default.xml | 18 +
.../yarn/server/security/AMSecretKeys.java | 45 ++
.../nodemanager/DefaultContainerExecutor.java | 14 +
.../nodemanager/LinuxContainerExecutor.java | 4 +
.../launcher/ContainerLaunch.java | 50 ++
.../launcher/ContainerRelaunch.java | 23 +
.../runtime/DefaultLinuxContainerRuntime.java | 14 +-
.../runtime/DockerLinuxContainerRuntime.java| 36 +-
.../runtime/LinuxContainerRuntimeConstants.java | 4 +
.../executor/ContainerStartContext.java | 24 +
.../impl/container-executor.c | 121 -
.../impl/container-executor.h | 17 +-
.../main/native/container-executor/impl/main.c | 35 +-
.../main/native/container-executor/impl/util.h | 6 +-
.../test/test-container-executor.c | 228 +---
.../TestDefaultContainerExecutor.java | 157 ++
.../TestLinuxContainerExecutorWithMocks.java| 115 ++--
.../launcher/TestContainerLaunch.java | 149 ++
.../launcher/TestContainerRelaunch.java | 32 +-
.../runtime/TestDockerContainerRuntime.java | 225
.../resourcemanager/RMActiveServiceContext.java | 15 +
.../yarn/server/resourcemanager/RMContext.java | 5 +
.../server/resourcemanager/RMContextImpl.java | 11 +
.../server/resourcemanager/ResourceManager.java | 9 +
.../resourcemanager/amlauncher/AMLauncher.java | 29 ++
.../security/ProxyCAManager.java| 68 +++
.../TestApplicationMasterLauncher.java | 86 ++-
.../security/TestProxyCAManager.java| 51 ++
.../hadoop/yarn/server/webproxy/ProxyCA.java| 408 +++
.../yarn/server/webproxy/WebAppProxy.java | 1 +
.../server/webproxy/WebAppProxyServlet.java | 89 +++-
.../yarn/server/webproxy/TestProxyCA.java | 518 +++
.../server/webproxy/TestWebAppProxyServlet.java | 58 ++-
37 files changed, 2406 insertions(+), 323 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2288ac4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
index 6a9527a..4fafa4a 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
@@ -59,6 +59,7 @@ import org.slf4j.LoggerFactory;
@InterfaceAudience.Public
@InterfaceStability.Evolving
public class Credentials implements Writable {
+
public enum SerializedFormat {
WRITABLE((byte) 0x00),
PROTOBUF((byte) 0x01);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2288ac4/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java
index 898c94e..1870b22 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java
@@ -25,6 +25,7 @@ import
org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
import org.apache.hadoop.test.GenericTestUtils;
+import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
@@ -50,6 +51,7 @@ import java.security.NoSuchProviderException;
import java.se
[11/50] [abbrv] hadoop git commit: HDDS-603. Add BlockCommitSequenceId field per Container and expose it in Container Reports. Contributed by Shashikant Banerjee.
HDDS-603. Add BlockCommitSequenceId field per Container and expose it in
Container Reports. Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5209c750
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5209c750
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5209c750
Branch: refs/heads/HDFS-12943
Commit: 5209c7503bee7849d134c16213133b4fa0c330f6
Parents: 22f37af
Author: Jitendra Pandey
Authored: Sat Oct 13 12:15:42 2018 -0700
Committer: Jitendra Pandey
Committed: Sat Oct 13 12:15:42 2018 -0700
--
.../org/apache/hadoop/ozone/OzoneConsts.java| 1 +
.../apache/hadoop/utils/MetadataKeyFilters.java | 3 +-
.../common/helpers/ContainerReport.java | 205 ---
.../common/helpers/KeyValueContainerReport.java | 117 ---
.../container/common/interfaces/Container.java | 5 +
.../container/keyvalue/KeyValueContainer.java | 9 +-
.../keyvalue/KeyValueContainerData.java | 16 ++
.../keyvalue/impl/BlockManagerImpl.java | 21 +-
.../container/ozoneimpl/ContainerReader.java| 7 +
.../StorageContainerDatanodeProtocol.proto | 1 +
.../ozone/om/helpers/OmKeyLocationInfo.java | 4 +
.../hadoop/ozone/client/rpc/TestBCSID.java | 147 +
12 files changed, 208 insertions(+), 328 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5209c750/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
--
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index 923271c..8ccc648 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -113,6 +113,7 @@ public final class OzoneConsts {
public static final String DELETING_KEY_PREFIX = "#deleting#";
public static final String DELETED_KEY_PREFIX = "#deleted#";
public static final String DELETE_TRANSACTION_KEY_PREFIX = "#delTX#";
+ public static final String BLOCK_COMMIT_SEQUENCE_ID_PREFIX = "#BCSID";
/**
* OM LevelDB prefixes.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5209c750/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java
--
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java
index a3430f8..04c87ae 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java
@@ -42,7 +42,8 @@ public final class MetadataKeyFilters {
new MetadataKeyFilters.KeyPrefixFilter()
.addFilter(OzoneConsts.DELETING_KEY_PREFIX, true)
.addFilter(OzoneConsts.DELETED_KEY_PREFIX, true)
- .addFilter(OzoneConsts.DELETE_TRANSACTION_KEY_PREFIX, true);
+ .addFilter(OzoneConsts.DELETE_TRANSACTION_KEY_PREFIX, true)
+ .addFilter(OzoneConsts.BLOCK_COMMIT_SEQUENCE_ID_PREFIX, true);
private MetadataKeyFilters() {
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5209c750/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerReport.java
--
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerReport.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerReport.java
deleted file mode 100644
index a4c1f2f..000
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerReport.java
+++ /dev/null
@@ -1,205 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for t
[45/50] [abbrv] hadoop git commit: YARN-8842. Expose metrics for custom resource types in QueueMetrics. (Contributed by Szilard Nemeth)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84e22a6a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetricsForCustomResources.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetricsForCustomResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetricsForCustomResources.java new file mode 100644 index 000..76a9849 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetricsForCustomResources.java @@ -0,0 +1,645 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.metrics2.MetricsSource; +import org.apache.hadoop.metrics2.MetricsSystem; +import org.apache.hadoop.metrics2.impl.MetricsSystemImpl; +import org.apache.hadoop.metrics2.lib.MutableCounterLong; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.resourcetypes.ResourceTypesTestHelper; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; + + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler +.QueueMetricsForCustomResources.QueueMetricsCustomResource; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; +import org.junit.Before; +import org.junit.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.function.BiFunction; +import java.util.function.Function; + +import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES; +import static org.apache.hadoop.yarn.resourcetypes.ResourceTypesTestHelper +.extractCustomResourcesAsStrings; +import static org.apache.hadoop.yarn.resourcetypes.ResourceTypesTestHelper.newResource; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler +.ResourceMetricsChecker.ResourceMetricsKey.AGGREGATE_CONTAINERS_ALLOCATED; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler +.ResourceMetricsChecker.ResourceMetricsKey.AGGREGATE_CONTAINERS_RELEASED; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler + .ResourceMetricsChecker.ResourceMetricsKey.AGGREGATE_MEMORY_MB_SECONDS_PREEMPTED; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler + .ResourceMetricsChecker.ResourceMetricsKey.AGGREGATE_VCORE_SECONDS_PREEMPTED; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler +.ResourceMetricsChecker.ResourceMetricsKey.ALLOCATED_CONTAINERS; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler +.ResourceMetricsChecker.ResourceMetricsKey.ALLOCATED_MB; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler +.ResourceMetricsChecker.ResourceMetricsKey.ALLOCATED_V_CORES; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceMetricsChecker.ResourceMetricsKey.AVAILABLE_MB; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceMetricsChecker.ResourceMetricsKey.AVAILABLE_V_CORES; + +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler +.ResourceMetricsChecker.ResourceMetricsKey.PENDING_CONTAINERS; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler +.ResourceMetricsChecker.ResourceMetricsKey.PENDING_MB; +import static org.apache.hadoop
[17/50] [abbrv] hadoop git commit: YARN-8830. SLS tool fix node addition. Contributed by Bibin A Chundatt.
YARN-8830. SLS tool fix node addition. Contributed by Bibin A Chundatt.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b4a38e7b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b4a38e7b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b4a38e7b
Branch: refs/heads/HDFS-12943
Commit: b4a38e7b3e530756fb79d23dd4e218beeb5e3190
Parents: b60ca37
Author: bibinchundatt
Authored: Mon Oct 15 16:10:25 2018 +0530
Committer: bibinchundatt
Committed: Mon Oct 15 16:10:25 2018 +0530
--
.../org/apache/hadoop/yarn/sls/SLSRunner.java | 29
1 file changed, 29 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4a38e7b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
--
diff --git
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
index 1e83e40..1fadd42 100644
---
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
+++
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
@@ -45,6 +45,8 @@ import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
+import org.apache.commons.collections.SetUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
@@ -1029,5 +1031,32 @@ public class SLSRunner extends Configured implements
Tool {
public void setLabels(Set labels) {
this.labels = labels;
}
+
+@Override
+public boolean equals(Object o) {
+ if (this == o) {
+return true;
+ }
+ if (!(o instanceof NodeDetails)) {
+return false;
+ }
+
+ NodeDetails that = (NodeDetails) o;
+
+ return StringUtils.equals(hostname, that.hostname) && (
+ nodeResource == null ?
+ that.nodeResource == null :
+ nodeResource.equals(that.nodeResource)) && SetUtils
+ .isEqualSet(labels, that.labels);
+}
+
+@Override
+public int hashCode() {
+ int result = hostname == null ? 0 : hostname.hashCode();
+ result =
+ 31 * result + (nodeResource == null ? 0 : nodeResource.hashCode());
+ result = 31 * result + (labels == null ? 0 : labels.hashCode());
+ return result;
+}
}
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[23/50] [abbrv] hadoop git commit: HADOOP-15851. Disable wildfly logs to the console. Contributed by Vishwajeet Dusane.
HADOOP-15851. Disable wildfly logs to the console.
Contributed by Vishwajeet Dusane.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ef9dc6c4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ef9dc6c4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ef9dc6c4
Branch: refs/heads/HDFS-12943
Commit: ef9dc6c44c686e836bb25e31ff355cff80572d23
Parents: e13a38f
Author: Steve Loughran
Authored: Mon Oct 15 22:53:55 2018 +0100
Committer: Steve Loughran
Committed: Mon Oct 15 22:53:55 2018 +0100
--
.../org/apache/hadoop/fs/azurebfs/utils/SSLSocketFactoryEx.java | 3 +++
1 file changed, 3 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef9dc6c4/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/SSLSocketFactoryEx.java
--
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/SSLSocketFactoryEx.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/SSLSocketFactoryEx.java
index 00e7786..6d3e4ea 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/SSLSocketFactoryEx.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/SSLSocketFactoryEx.java
@@ -25,6 +25,7 @@ import java.net.SocketException;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
+import java.util.logging.Level;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSocket;
@@ -33,6 +34,7 @@ import javax.net.ssl.SSLSocketFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.wildfly.openssl.OpenSSLProvider;
+import org.wildfly.openssl.SSL;
/**
@@ -118,6 +120,7 @@ public final class SSLSocketFactoryEx extends
SSLSocketFactory {
switch (preferredChannelMode) {
case Default:
try {
+
java.util.logging.Logger.getLogger(SSL.class.getName()).setLevel(Level.WARNING);
ctx = SSLContext.getInstance("openssl.TLS");
ctx.init(null, null, null);
channelMode = SSLChannelMode.OpenSSL;
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[16/50] [abbrv] hadoop git commit: Fix potential FSImage corruption. Contributed by Daryn Sharp.
Fix potential FSImage corruption. Contributed by Daryn Sharp.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b60ca379
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b60ca379
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b60ca379
Branch: refs/heads/HDFS-12943
Commit: b60ca37914b22550e3630fa02742d40697decb31
Parents: 8e5365e
Author: Vinayakumar B
Authored: Mon Oct 15 15:48:26 2018 +0530
Committer: Vinayakumar B
Committed: Mon Oct 15 15:48:26 2018 +0530
--
.../apache/hadoop/hdfs/util/LongBitFormat.java | 8 +
.../server/namenode/AclEntryStatusFormat.java | 109 ++---
.../server/namenode/FSImageFormatPBINode.java | 101
.../server/namenode/FSImageFormatProtobuf.java | 26 ++--
.../namenode/INodeWithAdditionalFields.java | 36 -
.../server/namenode/NameNodeLayoutVersion.java | 3 +-
.../server/namenode/SerialNumberManager.java| 152 +--
.../hdfs/server/namenode/SerialNumberMap.java | 43 +-
.../hdfs/server/namenode/XAttrFormat.java | 95 +++-
.../hdfs/server/namenode/XAttrStorage.java | 11 --
.../tools/offlineImageViewer/FSImageLoader.java | 18 ++-
.../offlineImageViewer/PBImageTextWriter.java | 3 +-
.../offlineImageViewer/PBImageXmlWriter.java| 22 +--
.../hadoop-hdfs/src/main/proto/fsimage.proto| 1 +
.../TestCommitBlockSynchronization.java | 6 +-
15 files changed, 383 insertions(+), 251 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b60ca379/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/LongBitFormat.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/LongBitFormat.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/LongBitFormat.java
index 51ad08f..195fd8c 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/LongBitFormat.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/LongBitFormat.java
@@ -26,6 +26,10 @@ import java.io.Serializable;
public class LongBitFormat implements Serializable {
private static final long serialVersionUID = 1L;
+ public interface Enum {
+int getLength();
+ }
+
private final String NAME;
/** Bit offset */
private final int OFFSET;
@@ -69,4 +73,8 @@ public class LongBitFormat implements Serializable {
public long getMin() {
return MIN;
}
+
+ public int getLength() {
+return LENGTH;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b60ca379/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclEntryStatusFormat.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclEntryStatusFormat.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclEntryStatusFormat.java
index 2c5b23b..e9e3e59 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclEntryStatusFormat.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclEntryStatusFormat.java
@@ -31,25 +31,23 @@ import com.google.common.collect.ImmutableList;
/**
* Class to pack an AclEntry into an integer.
* An ACL entry is represented by a 32-bit integer in Big Endian format.
- * The bits can be divided in four segments:
- * [0:1) || [1:3) || [3:6) || [6:7) || [7:32)
- *
- * [0:1) -- the scope of the entry (AclEntryScope)
- * [1:3) -- the type of the entry (AclEntryType)
- * [3:6) -- the permission of the entry (FsAction)
- * [6:7) -- A flag to indicate whether Named entry or not
- * [7:8) -- Reserved
- * [8:32) -- the name of the entry, which is an ID that points to a
- * string in the StringTableSection.
+ *
+ * Note: this format is used both in-memory and on-disk. Changes will be
+ * incompatible.
+ *
*/
-public enum AclEntryStatusFormat {
+public enum AclEntryStatusFormat implements LongBitFormat.Enum {
+
+ PERMISSION(null, 3),
+ TYPE(PERMISSION.BITS, 2),
+ SCOPE(TYPE.BITS, 1),
+ NAME(SCOPE.BITS, 24);
- SCOPE(null, 1),
- TYPE(SCOPE.BITS, 2),
- PERMISSION(TYPE.BITS, 3),
- NAMED_ENTRY_CHECK(PERMISSION.BITS, 1),
- RESERVED(NAMED_ENTRY_CHECK.BITS, 1),
- NAME(RESERVED.BITS, 24);
+ private static final FsAction[] FSACTION_VALUES = FsAction.values();
+ private static final AclEntryScope[] ACL_ENTRY_SCOPE_VALUES =
+ AclEntryScope.values();
+ private static final AclEntryType[] ACL_ENTRY_TYPE_VALUES =
+ AclEntryType.values();
[37/50] [abbrv] hadoop git commit: YARN-8448. AM HTTPS Support for AM communication with RMWeb proxy. (Contributed by Robert Kanter)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2288ac4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
index a110f10..2304501c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
@@ -37,6 +37,8 @@ import java.io.FileReader;
import java.io.IOException;
import java.io.LineNumberReader;
import java.net.InetSocketAddress;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
@@ -88,12 +90,11 @@ public class TestLinuxContainerExecutorWithMocks {
private static final Logger LOG =
LoggerFactory.getLogger(TestLinuxContainerExecutorWithMocks.class);
- private static final String MOCK_EXECUTOR =
- "./src/test/resources/mock-container-executor";
+ private static final String MOCK_EXECUTOR = "mock-container-executor";
private static final String MOCK_EXECUTOR_WITH_ERROR =
- "./src/test/resources/mock-container-executer-with-error";
+ "mock-container-executer-with-error";
private static final String MOCK_EXECUTOR_WITH_CONFIG_ERROR =
- "./src/test/resources/mock-container-executer-with-configuration-error";
+ "mock-container-executer-with-configuration-error";
private String tmpMockExecutor;
private LinuxContainerExecutor mockExec = null;
@@ -121,11 +122,13 @@ public class TestLinuxContainerExecutorWithMocks {
return ret;
}
- private void setupMockExecutor(String executorPath, Configuration conf)
- throws IOException {
+ private void setupMockExecutor(String executorName, Configuration conf)
+ throws IOException, URISyntaxException {
//we'll always use the tmpMockExecutor - since
// PrivilegedOperationExecutor can only be initialized once.
+URI executorPath = getClass().getClassLoader().getResource(executorName)
+.toURI();
Files.copy(Paths.get(executorPath), Paths.get(tmpMockExecutor),
REPLACE_EXISTING);
@@ -140,7 +143,8 @@ public class TestLinuxContainerExecutorWithMocks {
}
@Before
- public void setup() throws IOException, ContainerExecutionException {
+ public void setup() throws IOException, ContainerExecutionException,
+ URISyntaxException {
assumeNotWindows();
tmpMockExecutor = System.getProperty("test.build.data") +
@@ -172,7 +176,18 @@ public class TestLinuxContainerExecutorWithMocks {
}
@Test
- public void testContainerLaunch()
+ public void testContainerLaunchWithoutHTTPS()
+ throws IOException, ConfigurationException {
+testContainerLaunch(false);
+ }
+
+ @Test
+ public void testContainerLaunchWithHTTPS()
+ throws IOException, ConfigurationException {
+testContainerLaunch(true);
+ }
+
+ private void testContainerLaunch(boolean https)
throws IOException, ConfigurationException {
String appSubmitter = "nobody";
String cmd = String.valueOf(
@@ -193,41 +208,64 @@ public class TestLinuxContainerExecutorWithMocks {
Path scriptPath = new Path("file:///bin/echo");
Path tokensPath = new Path("file:///dev/null");
+Path keystorePath = new Path("file:///dev/null");
+Path truststorePath = new Path("file:///dev/null");
Path workDir = new Path("/tmp");
Path pidFile = new Path(workDir, "pid.txt");
mockExec.activateContainer(cId, pidFile);
-int ret = mockExec.launchContainer(new ContainerStartContext.Builder()
-.setContainer(container)
-.setNmPrivateContainerScriptPath(scriptPath)
-.setNmPrivateTokensPath(tokensPath)
-.setUser(appSubmitter)
-.setAppId(appId)
-.setContainerWorkDir(workDir)
-.setLocalDirs(dirsHandler.getLocalDirs())
-.setLogDirs(dirsHandler.getLogDirs())
-.setFilecacheDirs(new ArrayList<>())
-.setUserLocalDirs(new ArrayList<>())
-.setContainerLocalDirs(new ArrayList<>())
-.setContainerLogDirs(new ArrayList<>())
-.setUserFilecacheDirs(new ArrayList<>())
-.setApplicationLocalDirs(new ArrayList<>())
-.build());
+ContainerStartContext.Builder ctxBuilder =
+new Contai
[30/50] [abbrv] hadoop git commit: YARN-8879. Kerberos principal is needed when submitting a submarine job. Contributed by Zac Zhou.
YARN-8879. Kerberos principal is needed when submitting a submarine job.
Contributed by Zac Zhou.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/753f149f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/753f149f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/753f149f
Branch: refs/heads/HDFS-12943
Commit: 753f149fd3f5acf9a98cfc780d7899e307c19002
Parents: 0c2914e
Author: Sunil G
Authored: Tue Oct 16 22:17:51 2018 +0530
Committer: Sunil G
Committed: Tue Oct 16 22:17:51 2018 +0530
--
.../org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java | 5 +
.../apache/hadoop/yarn/service/utils/TestServiceApiUtil.java | 8
2 files changed, 9 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/753f149f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
index d201c7d..f055981 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
@@ -245,15 +245,12 @@ public class ServiceApiUtil {
public static void validateKerberosPrincipal(
KerberosPrincipal kerberosPrincipal) throws IOException {
-try {
+if (!StringUtils.isEmpty(kerberosPrincipal.getPrincipalName())) {
if (!kerberosPrincipal.getPrincipalName().contains("/")) {
throw new IllegalArgumentException(String.format(
RestApiErrorMessages.ERROR_KERBEROS_PRINCIPAL_NAME_FORMAT,
kerberosPrincipal.getPrincipalName()));
}
-} catch (NullPointerException e) {
- throw new IllegalArgumentException(
- RestApiErrorMessages.ERROR_KERBEROS_PRINCIPAL_MISSING);
}
if (!StringUtils.isEmpty(kerberosPrincipal.getKeytab())) {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/753f149f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/utils/TestServiceApiUtil.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/utils/TestServiceApiUtil.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/utils/TestServiceApiUtil.java
index 1e3c180..4940f8b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/utils/TestServiceApiUtil.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/utils/TestServiceApiUtil.java
@@ -652,6 +652,14 @@ public class TestServiceApiUtil extends ServiceTestUtils {
} catch (IllegalArgumentException e) {
Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
}
+
+kp.setPrincipalName(null);
+kp.setKeytab(null);
+try {
+ ServiceApiUtil.validateKerberosPrincipal(app.getKerberosPrincipal());
+} catch (NullPointerException e) {
+Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+}
}
@Test
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[20/50] [abbrv] hadoop git commit: HDFS-13987. RBF: Review of RandomResolver Class. Contributed by BELUGA BEHR.
HDFS-13987. RBF: Review of RandomResolver Class. Contributed by BELUGA BEHR.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ee1c80ea
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ee1c80ea
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ee1c80ea
Branch: refs/heads/HDFS-12943
Commit: ee1c80ea324459852e8c1c73b2545a261cb02b6f
Parents: f880ff4
Author: Inigo Goiri
Authored: Mon Oct 15 09:51:26 2018 -0700
Committer: Inigo Goiri
Committed: Mon Oct 15 09:51:26 2018 -0700
--
.../resolver/order/RandomResolver.java | 22 +++-
1 file changed, 7 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee1c80ea/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/RandomResolver.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/RandomResolver.java
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/RandomResolver.java
index 022aa48..13643e5 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/RandomResolver.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/RandomResolver.java
@@ -17,15 +17,15 @@
*/
package org.apache.hadoop.hdfs.server.federation.resolver.order;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
import java.util.Set;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.hadoop.hdfs.server.federation.resolver.PathLocation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.Iterables;
/**
* Order the destinations randomly.
@@ -35,10 +35,6 @@ public class RandomResolver implements OrderedResolver {
private static final Logger LOG =
LoggerFactory.getLogger(RandomResolver.class);
-
- /** Random number generator. */
- private static final Random RANDOM = new Random();
-
/**
* Get a random name space from the path.
*
@@ -47,16 +43,12 @@ public class RandomResolver implements OrderedResolver {
* @return Random name space.
*/
public String getFirstNamespace(final String path, final PathLocation loc) {
-if (loc == null) {
- return null;
-}
-Set namespaces = loc.getNamespaces();
-if (namespaces == null || namespaces.isEmpty()) {
+final Set namespaces = (loc == null) ? null : loc.getNamespaces();
+if (CollectionUtils.isEmpty(namespaces)) {
LOG.error("Cannot get namespaces for {}", loc);
return null;
}
-List nssList = new ArrayList<>(namespaces);
-int index = RANDOM.nextInt(nssList.size());
-return nssList.get(index);
+final int index = ThreadLocalRandom.current().nextInt(namespaces.size());
+return Iterables.get(namespaces, index);
}
}
\ No newline at end of file
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[31/50] [abbrv] hadoop git commit: MAPREDUCE-7132. JobSplitWriter prints unnecessary warnings if EC(RS10, 4) is used. Contributed by Peter Bacsko.
MAPREDUCE-7132. JobSplitWriter prints unnecessary warnings if EC(RS10,4) is
used. Contributed by Peter Bacsko.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/25f8fcb0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/25f8fcb0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/25f8fcb0
Branch: refs/heads/HDFS-12943
Commit: 25f8fcb06476938826cdc92858a61124b18cd98d
Parents: 753f149
Author: Xiao Chen
Authored: Tue Oct 16 10:22:47 2018 -0700
Committer: Xiao Chen
Committed: Tue Oct 16 10:23:31 2018 -0700
--
.../org/apache/hadoop/mapreduce/MRConfig.java | 2 +-
.../src/main/resources/mapred-default.xml | 2 +-
.../split/TestJobSplitWriterWithEC.java | 128 +++
3 files changed, 130 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f8fcb0/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java
index e85c893..b4d9149 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java
@@ -78,7 +78,7 @@ public interface MRConfig {
"mapreduce.task.max.status.length";
public static final int PROGRESS_STATUS_LEN_LIMIT_DEFAULT = 512;
- public static final int MAX_BLOCK_LOCATIONS_DEFAULT = 10;
+ public static final int MAX_BLOCK_LOCATIONS_DEFAULT = 15;
public static final String MAX_BLOCK_LOCATIONS_KEY =
"mapreduce.job.max.split.locations";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f8fcb0/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index 9f33d65..e5da41f 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -149,7 +149,7 @@
mapreduce.job.max.split.locations
-10
+15
The max number of block locations to store for each split for
locality calculation.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f8fcb0/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/split/TestJobSplitWriterWithEC.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/split/TestJobSplitWriterWithEC.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/split/TestJobSplitWriterWithEC.java
new file mode 100644
index 000..23f8a40
--- /dev/null
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/split/TestJobSplitWriterWithEC.java
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.split;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.File;
+import java.io.IO
[28/50] [abbrv] hadoop git commit: HDDS-657. Remove {volume} path segments from all the remaining rest endpoints. Contributed by Elek, Marton.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c2914e5/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
--
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
new file mode 100644
index 000..d32bc9f
--- /dev/null
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
@@ -0,0 +1,222 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.s3.endpoint;
+
+import javax.ws.rs.DELETE;
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.HEAD;
+import javax.ws.rs.HeaderParam;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.HttpHeaders;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.Response.ResponseBuilder;
+import javax.ws.rs.core.Response.Status;
+import javax.ws.rs.core.StreamingOutput;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneKeyDetails;
+import org.apache.hadoop.ozone.client.io.OzoneInputStream;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.s3.exception.OS3Exception;
+import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.http.HttpStatus;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Key level rest endpoints.
+ */
+@Path("/{bucket}/{path:.+}")
+public class ObjectEndpoint extends EndpointBase {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ObjectEndpoint.class);
+
+ private List customizableGetHeaders = new ArrayList<>();
+
+ public ObjectEndpoint() {
+customizableGetHeaders.add("Content-Type");
+customizableGetHeaders.add("Content-Language");
+customizableGetHeaders.add("Expires");
+customizableGetHeaders.add("Cache-Control");
+customizableGetHeaders.add("Content-Disposition");
+customizableGetHeaders.add("Content-Encoding");
+ }
+
+ /**
+ * Rest endpoint to upload object to a bucket.
+ *
+ * See: https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html
for
+ * more details.
+ */
+ @PUT
+ public Response put(
+ @Context HttpHeaders headers,
+ @PathParam("bucket") String bucketName,
+ @PathParam("path") String keyPath,
+ @DefaultValue("STAND_ALONE") @QueryParam("replicationType")
+ ReplicationType replicationType,
+ @DefaultValue("ONE") @QueryParam("replicationFactor")
+ ReplicationFactor replicationFactor,
+ @DefaultValue("32 * 1024 * 1024") @QueryParam("chunkSize")
+ String chunkSize,
+ @HeaderParam("Content-Length") long length,
+ InputStream body) throws IOException, OS3Exception {
+
+try {
+ Configuration config = new OzoneConfiguration();
+ config.set(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY, chunkSize);
+
+ OzoneBucket bucket = getBucket(bucketName);
+ OzoneOutputStream output = bucket
+ .createKey(keyPath, length, replicationType, replicationFactor);
+
+ IOUtils.copy(body, output);
+ output.close();
+
+ return Response.ok().status(HttpStatus.SC_OK)
+ .build();
+} catch (IOException ex) {
+ LOG.error("Exception occurred in PutObject", ex);
+ throw ex;
+}
+ }
+
+ /**
+ * Rest endpoint to download object from a bucket.
+ *
+ * See: https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html
for
+ * more details.
+ */
+ @GET
+ public Response get(
+ @
[08/50] [abbrv] hadoop git commit: HDDS-653. TestMetadataStore#testIterator fails on Windows. Contributed by Yiqun Lin.
HDDS-653. TestMetadataStore#testIterator fails on Windows.
Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9227f3d2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9227f3d2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9227f3d2
Branch: refs/heads/HDFS-12943
Commit: 9227f3d22412f94d22f4cae6a4fb3a52ead9b011
Parents: addb846
Author: Anu Engineer
Authored: Sat Oct 13 10:41:06 2018 -0700
Committer: Anu Engineer
Committed: Sat Oct 13 10:41:06 2018 -0700
--
.../src/test/java/org/apache/hadoop/utils/TestMetadataStore.java | 2 ++
1 file changed, 2 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9227f3d2/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestMetadataStore.java
--
diff --git
a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestMetadataStore.java
b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestMetadataStore.java
index 30fc7f3..a91bc80 100644
---
a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestMetadataStore.java
+++
b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestMetadataStore.java
@@ -163,6 +163,8 @@ public class TestMetadataStore {
GenericTestUtils.assertExceptionContains("Store has no more elements",
ex);
}
+dbStore.close();
+dbStore.destroy();
FileUtils.deleteDirectory(dbDir);
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[04/50] [abbrv] hadoop git commit: HDDS-613. Update HeadBucket, DeleteBucket to not to have volume in path. Contributed by Bharat Viswanadham.
HDDS-613. Update HeadBucket, DeleteBucket to not to have volume in path.
Contributed by Bharat Viswanadham.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/28ca5c9d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/28ca5c9d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/28ca5c9d
Branch: refs/heads/HDFS-12943
Commit: 28ca5c9d1647837a1b2480d8935deffc6f68d807
Parents: 3c1fe07
Author: Bharat Viswanadham
Authored: Fri Oct 12 16:59:11 2018 -0700
Committer: Bharat Viswanadham
Committed: Fri Oct 12 16:59:11 2018 -0700
--
.../dist/src/main/smoketest/commonlib.robot | 7
.../dist/src/main/smoketest/s3/bucketv2.robot | 11 ++
.../dist/src/main/smoketest/s3/bucketv4.robot | 11 ++
.../src/main/smoketest/s3/commonawslib.robot| 5 +++
.../hadoop/ozone/s3/bucket/DeleteBucket.java| 7 ++--
.../hadoop/ozone/s3/bucket/HeadBucket.java | 9 ++---
.../hadoop/ozone/client/ObjectStoreStub.java| 36 ++--
.../hadoop/ozone/client/OzoneVolumeStub.java| 12 +--
.../ozone/s3/bucket/TestDeleteBucket.java | 21
.../hadoop/ozone/s3/bucket/TestHeadBucket.java | 15 +++-
10 files changed, 86 insertions(+), 48 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ca5c9d/hadoop-ozone/dist/src/main/smoketest/commonlib.robot
--
diff --git a/hadoop-ozone/dist/src/main/smoketest/commonlib.robot
b/hadoop-ozone/dist/src/main/smoketest/commonlib.robot
index e2620fa..eb3a8bb 100644
--- a/hadoop-ozone/dist/src/main/smoketest/commonlib.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/commonlib.robot
@@ -22,3 +22,10 @@ Execute
Log ${output}
Should Be Equal As Integers ${rc} 0
[return]${output}
+
+Execute and checkrc
+[arguments] ${command}
${expected_error_code}
+${rc} ${output} = Run And Return
Rc And Output ${command}
+Log ${output}
+Should Be Equal As Integers ${rc}
${expected_error_code}
+[return]${output}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ca5c9d/hadoop-ozone/dist/src/main/smoketest/s3/bucketv2.robot
--
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/bucketv2.robot
b/hadoop-ozone/dist/src/main/smoketest/s3/bucketv2.robot
index eb7fa45..f17189b 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/bucketv2.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/bucketv2.robot
@@ -23,6 +23,7 @@ Resourcecommonawslib.robot
${ENDPOINT_URL} http://s3g:9878
${OZONE_TEST} true
${BUCKET} generated
+${NONEXIST-BUCKET}generated1
*** Keywords ***
Install aws s3 cli
@@ -53,3 +54,13 @@ Create Bucket
Should contain ${result} Location
Run Keyword if '${OZONE_TEST}' == 'true'Check Volume
+
+Head Bucket
+${result} = Execute AWSS3APICli head-bucket --bucket ${BUCKET}
+${result} = Execute AWSS3APICli and checkrc head-bucket
--bucket ${NONEXIST-BUCKET} 255
+Should contain ${result}Not Found
+Should contain ${result}404
+Delete Bucket
+${result} = Execute AWSS3APICli head-bucket --bucket ${BUCKET}
+${result} = Execute AWSS3APICli and checkrc delete-bucket
--bucket ${NONEXIST-BUCKET} 255
+Should contain ${result}NoSuchBucket
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ca5c9d/hadoop-ozone/dist/src/main/smoketest/s3/bucketv4.robot
--
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/bucketv4.robot
b/hadoop-ozone/dist/src/main/smoketest/s3/bucketv4.robot
index 5d306aa..1a93690 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/bucketv4.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/bucketv4.robot
@@ -23,6 +23,7 @@ Resourcecommonawslib.robot
${ENDPOINT_URL} http://s3g:9878
${OZONE_TEST} true
${BUCKET} generated
+${NONEXIST-BUCKET}generated1
*** Keywords ***
@@ -58,3 +59,13 @@ Create Bucket
Should contain ${result} Location
Run Keyword if '${OZONE_TEST}' == 'true'Check Volume
+
+Head Bucket
+${result} = Execute AWSS3APICli head-bucket --bucket ${BUCKET}
+
[32/50] [abbrv] hadoop git commit: HDDS-667. Fix TestOzoneFileInterfaces. Contributed by Mukul Kumar Singh.
HDDS-667. Fix TestOzoneFileInterfaces. Contributed by Mukul Kumar Singh.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/53e5173b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/53e5173b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/53e5173b
Branch: refs/heads/HDFS-12943
Commit: 53e5173bd1d970ec1c714568cbdb1c0dfd0fc6fb
Parents: 25f8fcb
Author: Jitendra Pandey
Authored: Tue Oct 16 10:34:16 2018 -0700
Committer: Jitendra Pandey
Committed: Tue Oct 16 10:34:51 2018 -0700
--
.../hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java | 4 ++--
.../test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java | 2 ++
2 files changed, 4 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e5173b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java
--
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java
index 7fa0cfb..67cda9f 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java
@@ -53,6 +53,8 @@ import static
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Res
public class BlockManagerImpl implements BlockManager {
static final Logger LOG = LoggerFactory.getLogger(BlockManagerImpl.class);
+ private static byte[] blockCommitSequenceIdKey =
+ DFSUtil.string2Bytes(OzoneConsts.BLOCK_COMMIT_SEQUENCE_ID_PREFIX);
private Configuration config;
@@ -89,8 +91,6 @@ public class BlockManagerImpl implements BlockManager {
Preconditions.checkNotNull(db, "DB cannot be null here");
long blockCommitSequenceId = data.getBlockCommitSequenceId();
-byte[] blockCommitSequenceIdKey =
-DFSUtil.string2Bytes(OzoneConsts.BLOCK_COMMIT_SEQUENCE_ID_PREFIX);
byte[] blockCommitSequenceIdValue = db.get(blockCommitSequenceIdKey);
// default blockCommitSequenceId for any block is 0. It the putBlock
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e5173b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
--
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index 71a4bef..ae52451 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientFactory;
import org.apache.hadoop.ozone.client.rest.OzoneException;
import org.apache.hadoop.ozone.common.Storage.StorageState;
+import org.apache.hadoop.ozone.container.common.utils.ContainerCache;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.hdds.scm.server.SCMStorage;
@@ -283,6 +284,7 @@ public final class MiniOzoneClusterImpl implements
MiniOzoneCluster {
scm.getClientProtocolServer().getScmInfo().getClusterId()));
stop();
FileUtils.deleteDirectory(baseDir);
+ ContainerCache.getInstance(conf).shutdownCache();
} catch (IOException e) {
LOG.error("Exception while shutting down the cluster.", e);
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[42/50] [abbrv] hadoop git commit: YARN-8892. YARN UI2 doc changes to update security status (verified under security environment). (Sunil G via wangda)
YARN-8892. YARN UI2 doc changes to update security status (verified under security environment). (Sunil G via wangda) Change-Id: I8bc8622936861b8d6de3e42a0b75af86ad8a3961 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/538250db Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/538250db Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/538250db Branch: refs/heads/HDFS-12943 Commit: 538250db26ce0b261bb74053348cddfc2d65cf52 Parents: 143d747 Author: Wangda Tan Authored: Tue Oct 16 13:41:17 2018 -0700 Committer: Wangda Tan Committed: Tue Oct 16 13:51:02 2018 -0700 -- .../hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/538250db/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md index 609ebe1..4c9daed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md @@ -52,4 +52,4 @@ Open your browser, go to `rm-address:8088/ui2` and try it! Notes - -- This UI framework is not verified under security environment, please use with caution under security environment. +This UI framework is verified under security environment as well. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[26/50] [abbrv] hadoop git commit: HDDS-439. 'ozone oz volume create' should default to current logged in user. Contributed by Dinesh Chitlangia.
HDDS-439. 'ozone oz volume create' should default to current logged in user.
Contributed by Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0bf8a110
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0bf8a110
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0bf8a110
Branch: refs/heads/HDFS-12943
Commit: 0bf8a110a56539ee85de0fc24575dd9a4c66d01b
Parents: 2614078
Author: Arpit Agarwal
Authored: Mon Oct 15 19:30:40 2018 -0700
Committer: Arpit Agarwal
Committed: Mon Oct 15 19:30:40 2018 -0700
--
.../hadoop/ozone/ozShell/TestOzoneShell.java| 22
.../web/ozShell/volume/CreateVolumeHandler.java | 7 +--
2 files changed, 27 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bf8a110/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
--
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
index d5f2554..03efa1c 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
@@ -32,6 +32,7 @@ import java.util.List;
import java.util.Random;
import java.util.UUID;
import java.util.stream.Collectors;
+
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.cli.MissingSubcommandException;
@@ -60,6 +61,7 @@ import org.apache.hadoop.ozone.web.response.BucketInfo;
import org.apache.hadoop.ozone.web.response.KeyInfo;
import org.apache.hadoop.ozone.web.response.VolumeInfo;
import org.apache.hadoop.ozone.web.utils.JsonUtils;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
import org.junit.AfterClass;
@@ -258,6 +260,26 @@ public class TestOzoneShell {
exceptionHandler, args);
}
+ /**
+ * Test to create volume without specifying --user or -u.
+ * @throws Exception
+ */
+ @Test
+ public void testCreateVolumeWithoutUser() throws Exception {
+String volumeName = "volume" + RandomStringUtils.randomNumeric(1);
+String[] args = new String[] {"volume", "create", url + "/" + volumeName,
+"--root"};
+
+execute(shell, args);
+
+String truncatedVolumeName =
+volumeName.substring(volumeName.lastIndexOf('/') + 1);
+OzoneVolume volumeInfo = client.getVolumeDetails(truncatedVolumeName);
+assertEquals(truncatedVolumeName, volumeInfo.getName());
+assertEquals(UserGroupInformation.getCurrentUser().getUserName(),
+volumeInfo.getOwner());
+ }
+
@Test
public void testDeleteVolume() throws Exception {
LOG.info("Running testDeleteVolume");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bf8a110/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/CreateVolumeHandler.java
--
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/CreateVolumeHandler.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/CreateVolumeHandler.java
index de0fbaa..84922a7 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/CreateVolumeHandler.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/CreateVolumeHandler.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.ozone.web.ozShell.Handler;
import org.apache.hadoop.ozone.web.ozShell.Shell;
import org.apache.hadoop.ozone.web.utils.JsonUtils;
+import org.apache.hadoop.security.UserGroupInformation;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
import picocli.CommandLine.Parameters;
@@ -45,8 +46,7 @@ public class CreateVolumeHandler extends Handler {
private String uri;
@Option(names = {"--user", "-u"},
- description = "Owner of of the volume", required =
- true)
+ description = "Owner of of the volume")
private String userName;
@Option(names = {"--quota", "-q"},
@@ -64,6 +64,9 @@ public class CreateVolumeHandler extends Handler {
*/
@Override
public Void call() throws Exception {
+if(userName == null) {
+ userName = UserGroupInformation.getCurrentUser().getUserName();
+}
URI ozoneURI = verifyURI(uri);
Path path = Paths.get(ozoneURI.getPath());
-
[40/50] [abbrv] hadoop git commit: YARN-8875. [Submarine] Add documentation for submarine installation script details. (Xun Liu via wangda)
YARN-8875. [Submarine] Add documentation for submarine installation script details. (Xun Liu via wangda) Change-Id: I1c8d39c394e5a30f967ea514919835b951f2c124 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ed08dd3b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ed08dd3b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ed08dd3b Branch: refs/heads/HDFS-12943 Commit: ed08dd3b0c9cec20373e8ca4e34d6526bd759943 Parents: babd144 Author: Wangda Tan Authored: Tue Oct 16 13:36:09 2018 -0700 Committer: Wangda Tan Committed: Tue Oct 16 13:51:01 2018 -0700 -- .../src/site/markdown/HowToInstall.md | 36 +++ .../src/site/markdown/Index.md | 4 +- .../src/site/markdown/InstallationGuide.md | 205 +++ .../src/site/markdown/InstallationScriptCN.md | 242 ++ .../src/site/markdown/InstallationScriptEN.md | 250 +++ .../src/site/markdown/TestAndTroubleshooting.md | 165 .../resources/images/submarine-installer.gif| Bin 0 -> 546547 bytes 7 files changed, 724 insertions(+), 178 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed08dd3b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/HowToInstall.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/HowToInstall.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/HowToInstall.md new file mode 100644 index 000..05d87c1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/HowToInstall.md @@ -0,0 +1,36 @@ + + +# How to Install Dependencies + +Submarine project uses YARN Service, Docker container, and GPU (when GPU hardware available and properly configured). + +That means as an admin, you have to properly setup YARN Service related dependencies, including: +- YARN Registry DNS + +Docker related dependencies, including: +- Docker binary with expected versions. +- Docker network which allows Docker container can talk to each other across different nodes. + +And when GPU wanna to be used: +- GPU Driver. +- Nvidia-docker. + +For your convenience, we provided installation documents to help you to setup your environment. You can always choose to have them installed in your own way. + +Use Submarine installer to install dependencies: [EN](InstallationScriptEN.html) [CN](InstallationScriptCN.html) + +Alternatively, you can follow manual install dependencies: [EN](InstallationGuide.html) [CN](InstallationGuideChineseVersion.html) + +Once you have installed dependencies, please follow following guide to [TestAndTroubleshooting](TestAndTroubleshooting.html). \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed08dd3b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md index 0006f6c..baeaa15 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md @@ -41,6 +41,4 @@ Click below contents if you want to understand more. - [Developer guide](DeveloperGuide.html) -- [Installation guide](InstallationGuide.html) - -- [Installation guide Chinese version](InstallationGuideChineseVersion.html) +- [Installation guides](HowToInstall.html) http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed08dd3b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md index d4f4269..4ef2bda 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md @@ -16,9 +16,11 @@ ## Prerequisites +(Please note that all following prerequisites are jus
[29/50] [abbrv] hadoop git commit: HDDS-657. Remove {volume} path segments from all the remaining rest endpoints. Contributed by Elek, Marton.
HDDS-657. Remove {volume} path segments from all the remaining rest endpoints.
Contributed by Elek, Marton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0c2914e5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0c2914e5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0c2914e5
Branch: refs/heads/HDFS-12943
Commit: 0c2914e582587c066db1b2c1530e57d5c078577a
Parents: 0bf8a11
Author: Márton Elek
Authored: Tue Oct 16 15:14:05 2018 +0200
Committer: Márton Elek
Committed: Tue Oct 16 17:30:53 2018 +0200
--
.../dist/src/main/smoketest/s3/README.md| 27 +++
.../dist/src/main/smoketest/s3/__init__.robot | 21 ++
.../dist/src/main/smoketest/s3/awscli.robot | 63 --
.../dist/src/main/smoketest/s3/awss3.robot | 47
.../dist/src/main/smoketest/s3/buckethead.robot | 34 +++
.../dist/src/main/smoketest/s3/bucketlist.robot | 32 +++
.../dist/src/main/smoketest/s3/bucketv2.robot | 66 --
.../dist/src/main/smoketest/s3/bucketv4.robot | 71 --
.../src/main/smoketest/s3/commonawslib.robot| 43 +++-
.../src/main/smoketest/s3/objectdelete.robot| 72 ++
.../src/main/smoketest/s3/objectputget.robot| 42
hadoop-ozone/dist/src/main/smoketest/test.sh| 5 +-
.../apache/hadoop/ozone/s3/EndpointBase.java| 158 -
.../hadoop/ozone/s3/bucket/DeleteBucket.java| 67 --
.../hadoop/ozone/s3/bucket/HeadBucket.java | 61 -
.../hadoop/ozone/s3/bucket/ListBucket.java | 74 ---
.../ozone/s3/bucket/ListBucketResponse.java | 55 -
.../hadoop/ozone/s3/bucket/PutBucket.java | 79 ---
.../hadoop/ozone/s3/bucket/package-info.java| 30 ---
.../ozone/s3/commontypes/IsoDateAdapter.java| 2 +-
.../ozone/s3/endpoint/BucketEndpoint.java | 199 +
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 213 ++
.../ozone/s3/endpoint/ListBucketResponse.java | 55 +
.../ozone/s3/endpoint/ListObjectResponse.java | 158 +
.../ozone/s3/endpoint/ObjectEndpoint.java | 222 +++
.../hadoop/ozone/s3/endpoint/RootEndpoint.java | 82 +++
.../hadoop/ozone/s3/endpoint/package-info.java | 30 +++
.../hadoop/ozone/s3/object/DeleteObject.java| 51 -
.../hadoop/ozone/s3/object/HeadObject.java | 74 ---
.../hadoop/ozone/s3/object/ListObject.java | 119 --
.../ozone/s3/object/ListObjectResponse.java | 147
.../hadoop/ozone/s3/object/PutObject.java | 92
.../hadoop/ozone/s3/object/package-info.java| 29 ---
.../hadoop/ozone/client/OzoneVolumeStub.java| 3 +-
.../ozone/s3/bucket/TestBucketResponse.java | 40
.../ozone/s3/bucket/TestDeleteBucket.java | 99 -
.../hadoop/ozone/s3/bucket/TestGetBucket.java | 114 --
.../hadoop/ozone/s3/bucket/TestHeadBucket.java | 85 ---
.../hadoop/ozone/s3/bucket/TestListBucket.java | 97
.../hadoop/ozone/s3/bucket/package-info.java| 21 --
.../ozone/s3/endpoint/TestBucketDelete.java | 100 +
.../hadoop/ozone/s3/endpoint/TestBucketGet.java | 115 ++
.../ozone/s3/endpoint/TestBucketHead.java | 71 ++
.../ozone/s3/endpoint/TestBucketResponse.java | 38
.../ozone/s3/endpoint/TestObjectDelete.java | 60 +
.../hadoop/ozone/s3/endpoint/TestObjectGet.java | 80 +++
.../ozone/s3/endpoint/TestObjectHead.java | 96
.../hadoop/ozone/s3/endpoint/TestPutObject.java | 91
.../hadoop/ozone/s3/endpoint/TestRootList.java | 79 +++
.../hadoop/ozone/s3/endpoint/package-info.java | 21 ++
.../ozone/s3/object/TestDeleteObject.java | 56 -
.../hadoop/ozone/s3/object/TestHeadObject.java | 95
.../hadoop/ozone/s3/object/TestPutObject.java | 89
.../hadoop/ozone/s3/object/package-info.java| 21 --
54 files changed, 2032 insertions(+), 1959 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c2914e5/hadoop-ozone/dist/src/main/smoketest/s3/README.md
--
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/README.md
b/hadoop-ozone/dist/src/main/smoketest/s3/README.md
new file mode 100644
index 000..884ba2e
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/README.md
@@ -0,0 +1,27 @@
+
+
+## Ozone S3 Gatway Acceptance Tests
+
+Note: the aws cli based acceptance tests can be cross-checked with the
original AWS s3 endpoint.
+
+You need to
+
+ 1. Create a bucket
+ 2. Configure your local aws cli
+ 3. Set bucket/endpointurl during the robot test execution
+
+```
+robot -v bucket:ozonetest -v OZONE_S3_SET_CREDENTIALS:false -v
ENDPOINT_URL:https://s3.us-east-2.amazona
[27/50] [abbrv] hadoop git commit: HDDS-657. Remove {volume} path segments from all the remaining rest endpoints. Contributed by Elek, Marton.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c2914e5/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/object/TestHeadObject.java
--
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/object/TestHeadObject.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/object/TestHeadObject.java
deleted file mode 100644
index 5b65dac..000
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/object/TestHeadObject.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package org.apache.hadoop.ozone.s3.object;
-
-import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.hadoop.hdds.client.ReplicationFactor;
-import org.apache.hadoop.hdds.client.ReplicationType;
-import org.apache.hadoop.ozone.client.*;
-import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import javax.ws.rs.core.Response;
-import org.apache.hadoop.ozone.s3.exception.OS3Exception;
-import java.io.IOException;
-
-import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
-
-/**
- * Test head object.
- */
-public class TestHeadObject {
- private String volName = "vol1";
- private String bucketName = "b1";
- private OzoneClientStub clientStub;
- private ObjectStore objectStoreStub;
- private HeadObject headObject;
- private OzoneBucket bucket;
-
- @Before
- public void setup() throws IOException {
-//Create client stub and object store stub.
-clientStub = new OzoneClientStub();
-objectStoreStub = clientStub.getObjectStore();
-
-// Create volume and bucket
-objectStoreStub.createVolume(volName);
-OzoneVolume volumeStub = objectStoreStub.getVolume(volName);
-volumeStub.createBucket(bucketName);
-bucket = objectStoreStub.getVolume(volName).getBucket(bucketName);
-
-// Create HeadBucket and setClient to OzoneClientStub
-headObject = new HeadObject();
-headObject.setClient(clientStub);
- }
-
- @Test
- public void testHeadObject() throws Exception {
-//GIVEN
-String value = RandomStringUtils.randomAlphanumeric(32);
-OzoneOutputStream out = bucket.createKey("key1",
-value.getBytes().length, ReplicationType.STAND_ALONE,
-ReplicationFactor.ONE);
-out.write(value.getBytes());
-out.close();
-
-//WHEN
-Response response = headObject.head(volName, bucketName, "key1");
-
-//THEN
-Assert.assertEquals(200, response.getStatus());
-Assert.assertEquals(value.getBytes().length,
-Long.parseLong(response.getHeaderString("Content-Length")));
- }
-
- @Test
- public void testHeadFailByBadName() throws Exception {
-//Head an object that doesn't exist.
-try {
- headObject.head(volName, bucketName, "badKeyName");
-} catch (OS3Exception ex) {
- Assert.assertTrue(ex.getCode().contains("NoSuchObject"));
- Assert.assertTrue(ex.getErrorMessage().contains("object does not
exist"));
- Assert.assertEquals(HTTP_NOT_FOUND, ex.getHttpCode());
-}
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c2914e5/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/object/TestPutObject.java
--
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/object/TestPutObject.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/object/TestPutObject.java
deleted file mode 100644
index a8f0648..000
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/object/TestPutObject.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org
[46/50] [abbrv] hadoop git commit: YARN-8842. Expose metrics for custom resource types in QueueMetrics. (Contributed by Szilard Nemeth)
YARN-8842. Expose metrics for custom resource types in QueueMetrics.
(Contributed by Szilard Nemeth)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/84e22a6a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/84e22a6a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/84e22a6a
Branch: refs/heads/HDFS-12943
Commit: 84e22a6af46db2859d7d2caf192861cae9b6a1a8
Parents: 538250d
Author: Haibo Chen
Authored: Tue Oct 16 14:12:02 2018 -0700
Committer: Haibo Chen
Committed: Tue Oct 16 14:14:30 2018 -0700
--
.../resourcetypes/ResourceTypesTestHelper.java | 22 +
.../resourcemanager/scheduler/QueueMetrics.java | 130 +++-
.../QueueMetricsForCustomResources.java | 158 +
.../scheduler/capacity/CapacityScheduler.java | 5 +-
.../resourcemanager/scheduler/QueueInfo.java| 90 +++
.../scheduler/QueueMetricsTestData.java | 105 +++
.../scheduler/ResourceMetricsChecker.java | 88 ++-
.../scheduler/TestQueueMetrics.java | 250 +++
.../TestQueueMetricsForCustomResources.java | 645 +++
9 files changed, 1325 insertions(+), 168 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84e22a6a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/resourcetypes/ResourceTypesTestHelper.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/resourcetypes/ResourceTypesTestHelper.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/resourcetypes/ResourceTypesTestHelper.java
index 98a8a00..3c3c2cc 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/resourcetypes/ResourceTypesTestHelper.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/resourcetypes/ResourceTypesTestHelper.java
@@ -16,6 +16,7 @@
package org.apache.hadoop.yarn.resourcetypes;
+import com.google.common.collect.Maps;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -24,6 +25,7 @@ import
org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import java.util.stream.Collectors;
/**
* Contains helper methods to create Resource and ResourceInformation objects.
@@ -90,4 +92,24 @@ public final class ResourceTypesTestHelper {
return new ResourceValueAndUnit(value, matcher.group(2));
}
+ public static Map extractCustomResources(Resource res) {
+Map customResources = Maps.newHashMap();
+for (int i = 0; i < res.getResources().length; i++) {
+ ResourceInformation ri = res.getResourceInformation(i);
+ if (!ri.getName().equals(ResourceInformation.MEMORY_URI)
+ && !ri.getName().equals(ResourceInformation.VCORES_URI)) {
+customResources.put(ri.getName(), ri.getValue());
+ }
+}
+return customResources;
+ }
+
+ public static Map extractCustomResourcesAsStrings(
+ Resource res) {
+Map resValues = extractCustomResources(res);
+return resValues.entrySet().stream()
+.collect(Collectors.toMap(
+Map.Entry::getKey, e -> String.valueOf(e.getValue(;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84e22a6a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
index 20a5a1f..1315c2e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
@@ -24,6 +24,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceAudien
[41/50] [abbrv] hadoop git commit: YARN-8798. [Submarine] Job should not be submitted if --input_path option is missing. (Zhankun Tang via wangda)
YARN-8798. [Submarine] Job should not be submitted if --input_path option is
missing. (Zhankun Tang via wangda)
Change-Id: I7ae0e44eb5179b04a6ac861ec1c65f3b18c38f0f
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/143d7477
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/143d7477
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/143d7477
Branch: refs/heads/HDFS-12943
Commit: 143d74775b2b62884090fdd88874134b9eab2888
Parents: 46d6e00
Author: Wangda Tan
Authored: Tue Oct 16 13:39:34 2018 -0700
Committer: Wangda Tan
Committed: Tue Oct 16 13:51:02 2018 -0700
--
.../client/cli/param/RunJobParameters.java | 6 +++
.../client/cli/TestRunJobCliParsing.java| 39
2 files changed, 45 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/143d7477/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java
index 92a1883..d923e0f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java
@@ -62,6 +62,12 @@ public class RunJobParameters extends RunParameters {
if (parsedCommandLine.getOptionValue(CliConstants.N_WORKERS) != null) {
nWorkers = Integer.parseInt(
parsedCommandLine.getOptionValue(CliConstants.N_WORKERS));
+ // Only check null value.
+ // Training job shouldn't ignore INPUT_PATH option
+ // But if nWorkers is 0, INPUT_PATH can be ignored because user can only
run Tensorboard
+ if (null == input && 0 != nWorkers) {
+throw new ParseException("\"--" + CliConstants.INPUT_PATH + "\" is
absent");
+ }
}
int nPS = 0;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/143d7477/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/TestRunJobCliParsing.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/TestRunJobCliParsing.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/TestRunJobCliParsing.java
index 295d6a8..240de06 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/TestRunJobCliParsing.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/TestRunJobCliParsing.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.submarine.client.cli;
+import org.apache.commons.cli.ParseException;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
@@ -136,6 +137,44 @@ public class TestRunJobCliParsing {
}
@Test
+ public void testNoInputPathOptionSpecified() throws Exception {
+RunJobCli runJobCli = new RunJobCli(getMockClientContext());
+String expectedErrorMessage = "\"--" + CliConstants.INPUT_PATH + "\" is
absent";
+String actualMessage = "";
+try {
+ runJobCli.run(
+ new String[]{"--name", "my-job", "--docker_image", "tf-docker:1.1.0",
+ "--checkpoint_path", "hdfs://output",
+ "--num_workers", "1", "--worker_launch_cmd", "python run-job.py",
+ "--worker_resources", "memory=4g,vcores=2", "--tensorboard",
+ "true", "--verbose", "--wait_job_finish"});
+} catch (ParseException e) {
+ actualMessage = e.getMessage();
+ e.printStackTrace();
+}
+Assert.assertEquals(expectedErrorMessage, actualMessage);
+ }
+
+ /**
+ * when only run tensorboard, input_path is not needed
+ * */
+ @Test
+ public void testNoInputPathOptionButOnlyRunTensorboard() throws Exception {
+RunJobCli runJobCli = new RunJobCli(getMoc
[10/50] [abbrv] hadoop git commit: Revert "HDDS-629. Make ApplyTransaction calls in ContainerStateMachine idempotent. Contributed by Shashikant Banerjee."
Revert "HDDS-629. Make ApplyTransaction calls in ContainerStateMachine
idempotent. Contributed by Shashikant Banerjee."
This reverts commit 0473b6817cfe4f03acdcb0eedc78b509244f.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/22f37af9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/22f37af9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/22f37af9
Branch: refs/heads/HDFS-12943
Commit: 22f37af93583e9c63e03f0781ffb903e35544559
Parents: 0473b68
Author: Jitendra Pandey
Authored: Sat Oct 13 12:14:39 2018 -0700
Committer: Jitendra Pandey
Committed: Sat Oct 13 12:14:39 2018 -0700
--
.../org/apache/hadoop/ozone/OzoneConsts.java| 1 -
.../apache/hadoop/utils/MetadataKeyFilters.java | 3 +-
.../common/helpers/ContainerReport.java | 205 +++
.../common/helpers/KeyValueContainerReport.java | 117 +++
.../container/common/interfaces/Container.java | 5 -
.../container/keyvalue/KeyValueContainer.java | 9 +-
.../keyvalue/KeyValueContainerData.java | 16 --
.../keyvalue/impl/BlockManagerImpl.java | 21 +-
.../container/ozoneimpl/ContainerReader.java| 7 -
.../StorageContainerDatanodeProtocol.proto | 1 -
.../ozone/om/helpers/OmKeyLocationInfo.java | 4 -
.../hadoop/ozone/client/rpc/TestBCSID.java | 147 -
12 files changed, 328 insertions(+), 208 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22f37af9/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
--
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index 8ccc648..923271c 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -113,7 +113,6 @@ public final class OzoneConsts {
public static final String DELETING_KEY_PREFIX = "#deleting#";
public static final String DELETED_KEY_PREFIX = "#deleted#";
public static final String DELETE_TRANSACTION_KEY_PREFIX = "#delTX#";
- public static final String BLOCK_COMMIT_SEQUENCE_ID_PREFIX = "#BCSID";
/**
* OM LevelDB prefixes.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22f37af9/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java
--
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java
index 04c87ae..a3430f8 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java
@@ -42,8 +42,7 @@ public final class MetadataKeyFilters {
new MetadataKeyFilters.KeyPrefixFilter()
.addFilter(OzoneConsts.DELETING_KEY_PREFIX, true)
.addFilter(OzoneConsts.DELETED_KEY_PREFIX, true)
- .addFilter(OzoneConsts.DELETE_TRANSACTION_KEY_PREFIX, true)
- .addFilter(OzoneConsts.BLOCK_COMMIT_SEQUENCE_ID_PREFIX, true);
+ .addFilter(OzoneConsts.DELETE_TRANSACTION_KEY_PREFIX, true);
private MetadataKeyFilters() {
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22f37af9/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerReport.java
--
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerReport.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerReport.java
new file mode 100644
index 000..a4c1f2f
--- /dev/null
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerReport.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, ei
[03/50] [abbrv] hadoop git commit: HDDS-445. Create a logger to print out all of the incoming requests. Contributed by Bharat Viswanadham.
HDDS-445. Create a logger to print out all of the incoming requests.
Contributed by Bharat Viswanadham.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3c1fe073
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3c1fe073
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3c1fe073
Branch: refs/heads/HDFS-12943
Commit: 3c1fe073d2fef76676660144e7dce2050761ae64
Parents: ddc9649
Author: Anu Engineer
Authored: Fri Oct 12 16:27:54 2018 -0700
Committer: Anu Engineer
Committed: Fri Oct 12 16:27:54 2018 -0700
--
.../hadoop-common/src/main/conf/log4j.properties | 6 ++
hadoop-ozone/dist/src/main/compose/ozones3/docker-config | 4
2 files changed, 10 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c1fe073/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
--
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
index aeae2b8..0214da3 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
+++ b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
@@ -271,6 +271,12 @@
log4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
#log4j.appender.nodemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-nodemanager-_mm_dd.log
#log4j.appender.nodemanagerrequestlog.RetainDays=3
+#Http Server request logs for Ozone S3Gateway
+log4j.logger.http.requests.s3gateway=INFO,s3gatewayrequestlog
+log4j.appender.s3gatewayrequestlog=org.apache.hadoop.http.HttpRequestLogAppender
+log4j.appender.s3gatewayrequestlog.Filename=${hadoop.log.dir}/jetty-s3gateway-_mm_dd.log
+log4j.appender.s3gatewayrequestlog.RetainDays=3
+
# WebHdfs request log on datanodes
# Specify -Ddatanode.webhdfs.logger=INFO,HTTPDRFA on datanode startup to
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c1fe073/hadoop-ozone/dist/src/main/compose/ozones3/docker-config
--
diff --git a/hadoop-ozone/dist/src/main/compose/ozones3/docker-config
b/hadoop-ozone/dist/src/main/compose/ozones3/docker-config
index 2b22874..dd53d9d 100644
--- a/hadoop-ozone/dist/src/main/compose/ozones3/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozones3/docker-config
@@ -31,6 +31,10 @@
LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{-MM-dd HH
LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN
LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR
+LOG4J.PROPERTIES_log4j.logger.http.requests.s3gateway=INFO,s3gatewayrequestlog
+LOG4J.PROPERTIES_log4j.appender.s3gatewayrequestlog=org.apache.hadoop.http.HttpRequestLogAppender
+LOG4J.PROPERTIES_log4j.appender.s3gatewayrequestlog.Filename=/tmp/jetty-s3gateway-_mm_dd.log
+LOG4J.PROPERTIES_log4j.appender.s3gatewayrequestlog.RetainDays=3
#Enable this variable to print out all hadoop rpc traffic to the stdout. See
http://byteman.jboss.org/ to define your own instrumentation.
#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[43/50] [abbrv] hadoop git commit: YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46d6e001/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh
new file mode 100644
index 000..7b3c2a9
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh
@@ -0,0 +1,123 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## @description check install user
+## @audience public
+## @stabilitystable
+function check_install_user()
+{
+ if [[ $(id -u) -ne 0 ]];then
+echo "This script must be run with a ROOT user!"
+exit # don't call exit_install()
+ fi
+}
+
+## @description exit install
+## @audience public
+## @stabilitystable
+function exit_install()
+{
+ echo "Exit the installation!" | tee -a $LOG
+ exit $1
+}
+
+## @description Check if the IP address format is correct
+## @audience public
+## @stabilitystable
+function valid_ip()
+{
+ local ip=$1
+ local stat=1
+
+ if [[ $ip =~ ^[0-9]{1,3\}.[0-9]{1,3\}.[0-9]{1,3\}.[0-9]{1,3\}$ ]]; then
+OIFS=$IFS
+IFS='.'
+ip=($ip)
+IFS=$OIFS
+
+if [[ ${ip[0]} -le 255 && ${ip[1]} -le 255 && ${ip[2]} -le 255 && ${ip[3]}
-le 255 ]]; then
+ stat=$?
+fi
+ fi
+
+ return $stat
+}
+
+## @description Check if the configuration file configuration is correct
+## @audience public
+## @stabilitystable
+function check_install_conf()
+{
+ echo "Check if the configuration file configuration is correct ..." | tee -a
$LOG
+
+ # check etcd conf
+ hostCount=${#ETCD_HOSTS[@]}
+ if [[ $hostCount -lt 3 && hostCount -ne 0 ]]; then # <>2
+echo "Number of nodes = [$hostCount], must be configured to be greater
than or equal to 3 servers! " | tee -a $LOG
+exit_install
+ fi
+ for ip in ${ETCD_HOSTS[@]}
+ do
+if ! valid_ip $ip; then
+ echo "]ETCD_HOSTS=[$ip], IP address format is incorrect! " | tee -a $LOG
+ exit_install
+fi
+ done
+ echo "Check if the configuration file configuration is correct [ Done ]" |
tee -a $LOG
+}
+
+## @description index by EtcdHosts list
+## @audience public
+## @stabilitystable
+function indexByEtcdHosts() {
+ index=0
+ while [ "$index" -lt "${#ETCD_HOSTS[@]}" ]; do
+if [ "${ETCD_HOSTS[$index]}" = "$1" ]; then
+ echo $index
+ return
+fi
+let "index++"
+ done
+ echo ""
+}
+
+## @description get local IP
+## @audience public
+## @stabilitystable
+function getLocalIP()
+{
+ local _ip _myip _line _nl=$'\n'
+ while IFS=$': \t' read -a _line ;do
+ [ -z "${_line%inet}" ] &&
+ _ip=${_line[${#_line[1]}>4?1:2]} &&
+ [ "${_ip#127.0.0.1}" ] && _myip=$_ip
+done< <(LANG=C /sbin/ifconfig)
+ printf ${1+-v} $1 "%s${_nl:0:$[${#1}>0?0:1]}" $_myip
+}
+
+## @description get ip list
+## @audience public
+## @stabilitystable
+function get_ip_list()
+{
+ array=$(ifconfig | grep inet | grep -v inet6 | grep -v 127 | sed 's/^[
\t]*//g' | cut -d ' ' -f2)
+
+ for ip in ${array[@]}
+ do
+LOCAL_HOST_IP_LIST+=(${ip})
+ done
+}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[33/50] [abbrv] hadoop git commit: HDDS-378. Remove dependencies between hdds/ozone and hdfs proto files. Contributed by Elek, Marton.
HDDS-378. Remove dependencies between hdds/ozone and hdfs proto files.
Contributed by Elek, Marton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5f0b43fa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5f0b43fa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5f0b43fa
Branch: refs/heads/HDFS-12943
Commit: 5f0b43fa93d02c90956d48eb5c6c0b60deeac91c
Parents: 53e5173
Author: Arpit Agarwal
Authored: Tue Oct 16 10:40:00 2018 -0700
Committer: Arpit Agarwal
Committed: Tue Oct 16 10:40:00 2018 -0700
--
hadoop-hdds/common/pom.xml | 6 --
.../main/proto/ScmBlockLocationProtocol.proto | 1 -
.../StorageContainerLocationProtocol.proto | 1 -
hadoop-hdds/container-service/pom.xml | 11 +---
.../apache/hadoop/ozone/client/BucketArgs.java | 2 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 2 +-
.../ozone/client/protocol/ClientProtocol.java | 2 +-
.../hadoop/ozone/client/rest/RestClient.java| 2 +-
.../hadoop/ozone/client/rpc/RpcClient.java | 2 +-
hadoop-ozone/common/pom.xml | 6 --
.../hadoop/hdds/protocol/StorageType.java | 64
.../ozone/client/rest/response/BucketInfo.java | 2 +-
.../hadoop/ozone/om/helpers/OmBucketArgs.java | 7 +--
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 9 ++-
.../hadoop/ozone/web/handlers/BucketArgs.java | 2 +-
.../hadoop/ozone/web/response/BucketInfo.java | 2 +-
.../src/main/proto/OzoneManagerProtocol.proto | 12 +++-
.../apache/hadoop/ozone/web/TestBucketInfo.java | 2 +-
.../TestStorageContainerManagerHelper.java | 2 +-
.../ozone/client/rest/TestOzoneRestClient.java | 2 +-
.../ozone/client/rpc/TestOzoneRpcClient.java| 2 +-
.../om/TestMultipleContainerReadWrite.java | 2 +-
.../hadoop/ozone/om/TestOmBlockVersioning.java | 2 +-
.../hadoop/ozone/om/TestOzoneManager.java | 2 +-
.../web/handlers/BucketProcessTemplate.java | 2 +-
.../hadoop/ozone/om/BucketManagerImpl.java | 2 +-
.../hadoop/ozone/om/S3BucketManagerImpl.java| 2 +-
.../hadoop/ozone/om/TestBucketManagerImpl.java | 2 +-
.../hadoop/ozone/client/OzoneBucketStub.java| 2 +-
.../hadoop/ozone/client/OzoneVolumeStub.java| 2 +-
30 files changed, 102 insertions(+), 57 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f0b43fa/hadoop-hdds/common/pom.xml
--
diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml
index bf2a6b9..65cd1d1 100644
--- a/hadoop-hdds/common/pom.xml
+++ b/hadoop-hdds/common/pom.xml
@@ -218,12 +218,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";>
${basedir}/../../hadoop-common-project/hadoop-common/src/main/proto
-
-
${basedir}/../../hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/
-
-
-
${basedir}/../../hadoop-hdfs-project/hadoop-hdfs/src/main/proto/
-
${basedir}/src/main/proto
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f0b43fa/hadoop-hdds/common/src/main/proto/ScmBlockLocationProtocol.proto
--
diff --git a/hadoop-hdds/common/src/main/proto/ScmBlockLocationProtocol.proto
b/hadoop-hdds/common/src/main/proto/ScmBlockLocationProtocol.proto
index 9b4e0ac..01a0dde 100644
--- a/hadoop-hdds/common/src/main/proto/ScmBlockLocationProtocol.proto
+++ b/hadoop-hdds/common/src/main/proto/ScmBlockLocationProtocol.proto
@@ -28,7 +28,6 @@ option java_generic_services = true;
option java_generate_equals_and_hash = true;
package hadoop.hdds;
-import "hdfs.proto";
import "hdds.proto";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f0b43fa/hadoop-hdds/common/src/main/proto/StorageContainerLocationProtocol.proto
--
diff --git
a/hadoop-hdds/common/src/main/proto/StorageContainerLocationProtocol.proto
b/hadoop-hdds/common/src/main/proto/StorageContainerLocationProtocol.proto
index fb01d6a..49d1975 100644
--- a/hadoop-hdds/common/src/main/proto/StorageContainerLocationProtocol.proto
+++ b/hadoop-hdds/common/src/main/proto/StorageContainerLocationProtocol.proto
@@ -28,7 +28,6 @@ option java_generic_services = true;
option java_generate_equals_and_hash = true;
package hadoop.hdds;
-import "hdfs.proto";
import "hdds.proto";
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f0b43fa/hadoop-hdds/container-service/pom.xml
--
diff --git a/hadoop-hdds/container-service/po
[34/50] [abbrv] hadoop git commit: HDFS-13993. TestDataNodeVolumeFailure#testTolerateVolumeFailuresAfterAddingMoreVolumes is flaky. Contributed by Ayush Saxena.
HDFS-13993.
TestDataNodeVolumeFailure#testTolerateVolumeFailuresAfterAddingMoreVolumes is
flaky. Contributed by Ayush Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f90c64e6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f90c64e6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f90c64e6
Branch: refs/heads/HDFS-12943
Commit: f90c64e6242facf38c2baedeeda42e4a8293e642
Parents: 5f0b43f
Author: Inigo Goiri
Authored: Tue Oct 16 11:22:57 2018 -0700
Committer: Inigo Goiri
Committed: Tue Oct 16 11:22:57 2018 -0700
--
.../hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java | 1 +
1 file changed, 1 insertion(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f90c64e6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java
index 7d04942..b70a356 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java
@@ -389,6 +389,7 @@ public class TestDataNodeVolumeFailure {
DataNodeTestUtils.injectDataDirFailure(dn0Vol2);
DataNodeTestUtils.waitForDiskError(dn0,
DataNodeTestUtils.getVolume(dn0, dn0Vol2));
+dn0.checkDiskError();
assertFalse(dn0.shouldRun());
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[14/50] [abbrv] hadoop git commit: HDDS-519. Implement ListBucket REST endpoint. Contributed by LiXin Ge.
HDDS-519. Implement ListBucket REST endpoint. Contributed by LiXin Ge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5033deb1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5033deb1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5033deb1
Branch: refs/heads/HDFS-12943
Commit: 5033deb13b7f393d165e282b0c3b9e1ee1390bb2
Parents: 8853fc8
Author: Márton Elek
Authored: Sun Oct 14 11:29:50 2018 +0200
Committer: Márton Elek
Committed: Sun Oct 14 11:30:18 2018 +0200
--
.../hadoop/ozone/s3/bucket/ListBucket.java | 74 +++
.../ozone/s3/bucket/ListBucketResponse.java | 55 +++
.../ozone/s3/commontypes/BucketMetadata.java| 53 +++
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 6 +-
.../hadoop/ozone/client/OzoneVolumeStub.java| 7 +-
.../hadoop/ozone/s3/bucket/TestListBucket.java | 97
6 files changed, 290 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5033deb1/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/bucket/ListBucket.java
--
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/bucket/ListBucket.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/bucket/ListBucket.java
new file mode 100644
index 000..962b8a6
--- /dev/null
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/bucket/ListBucket.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.s3.bucket;
+
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.s3.EndpointBase;
+import org.apache.hadoop.ozone.s3.commontypes.BucketMetadata;
+import org.apache.hadoop.ozone.s3.exception.OS3Exception;
+import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.*;
+import javax.ws.rs.core.MediaType;
+import java.io.IOException;
+import java.time.Instant;
+import java.util.Iterator;
+
+/**
+ * List Object Rest endpoint.
+ */
+@Path("/{volume}")
+public class ListBucket extends EndpointBase {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ListBucket.class);
+
+ @GET
+ @Produces(MediaType.APPLICATION_XML)
+ public ListBucketResponse get(@PathParam("volume") String volumeName)
+ throws OS3Exception, IOException {
+OzoneVolume volume;
+try {
+ volume = getVolume(volumeName);
+} catch (NotFoundException ex) {
+ LOG.error("Exception occurred in ListBucket: volume {} not found.",
+ volumeName, ex);
+ OS3Exception os3Exception = S3ErrorTable.newError(S3ErrorTable
+ .NO_SUCH_VOLUME, S3ErrorTable.Resource.VOLUME);
+ throw os3Exception;
+} catch (IOException e) {
+ throw e;
+}
+
+Iterator volABucketIter = volume.listBuckets(null);
+ListBucketResponse response = new ListBucketResponse();
+
+while(volABucketIter.hasNext()) {
+ OzoneBucket next = volABucketIter.next();
+ BucketMetadata bucketMetadata = new BucketMetadata();
+ bucketMetadata.setName(next.getName());
+ bucketMetadata.setCreationDate(
+ Instant.ofEpochMilli(next.getCreationTime()));
+ response.addBucket(bucketMetadata);
+}
+
+return response;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5033deb1/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/bucket/ListBucketResponse.java
--
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/bucket/ListBucketResponse.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/bucket/ListBucketResponse.java
new file mode 100644
index 000..1f117dd
--- /dev/null
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/bucket/ListBu
[21/50] [abbrv] hadoop git commit: HADOOP-14445. Addendum: Use DelegationTokenIssuer to create KMS delegation tokens that can authenticate to all KMS instances.
HADOOP-14445. Addendum: Use DelegationTokenIssuer to create KMS delegation tokens that can authenticate to all KMS instances. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b6fc72a0 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b6fc72a0 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b6fc72a0 Branch: refs/heads/HDFS-12943 Commit: b6fc72a0250ac3f2341ebe8a14d19b073e6224c8 Parents: ee1c80e Author: Xiao Chen Authored: Mon Oct 15 10:50:27 2018 -0700 Committer: Xiao Chen Committed: Mon Oct 15 10:51:55 2018 -0700 -- .../KeyProviderDelegationTokenExtension.java| 2 +- .../crypto/key/KeyProviderTokenIssuer.java | 2 +- .../java/org/apache/hadoop/fs/FileSystem.java | 2 +- .../security/token/DelegationTokenIssuer.java | 3 +- .../apache/hadoop/fs/TestFilterFileSystem.java | 2 +- .../org/apache/hadoop/fs/TestHarFileSystem.java | 2 +- .../hadoop/hdfs/DistributedFileSystem.java | 2 +- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 11 ++- .../apache/hadoop/hdfs/TestEncryptionZones.java | 32 +++- 9 files changed, 34 insertions(+), 24 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6fc72a0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java index 29c5bcd..05d99ed 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java @@ -22,7 +22,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.org.apache.hadoop.security.token.DelegationTokenIssuer; +import org.apache.hadoop.security.token.DelegationTokenIssuer; import java.io.IOException; http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6fc72a0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderTokenIssuer.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderTokenIssuer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderTokenIssuer.java index 81caff4..187bee6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderTokenIssuer.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderTokenIssuer.java @@ -22,7 +22,7 @@ import java.net.URI; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.security.token.org.apache.hadoop.security.token.DelegationTokenIssuer; +import org.apache.hadoop.security.token.DelegationTokenIssuer; /** * File systems that support Encryption Zones have to implement this interface. http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6fc72a0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index 3d40b6a..fe4159b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -64,7 +64,7 @@ import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.org.apache.hadoop.security.token.DelegationTokenIssuer; +import org.apache.hadoop.security.token.DelegationTokenIssuer; import org.apache.hadoop.util.ClassUtil; import org.apache.hadoop.util.DataChecksum; import org.apache.hadoop.util.Progressable; http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6fc72a0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/org/a
[44/50] [abbrv] hadoop git commit: YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda)
YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda) Change-Id: I46e8d9fd32c7745c313030da62da41486a77b3ea Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/46d6e001 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/46d6e001 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/46d6e001 Branch: refs/heads/HDFS-12943 Commit: 46d6e0016610ced51a76189daeb3ad0e3dbbf94c Parents: ed08dd3 Author: Wangda Tan Authored: Tue Oct 16 13:36:59 2018 -0700 Committer: Wangda Tan Committed: Tue Oct 16 13:51:02 2018 -0700 -- .../resources/assemblies/hadoop-yarn-dist.xml | 8 + .../installation/install.conf | 74 .../installation/install.sh | 116 + .../package/calico/calico-node.service | 50 +++ .../installation/package/calico/calicoctl.cfg | 22 + .../installation/package/docker/daemon.json | 23 + .../installation/package/docker/docker.service | 35 ++ .../installation/package/etcd/etcd.service | 40 ++ .../package/hadoop/container-executor.cfg | 41 ++ .../installation/package/submarine/submarine.sh | 25 ++ .../installation/scripts/calico.sh | 224 ++ .../installation/scripts/docker.sh | 166 +++ .../installation/scripts/download-server.sh | 42 ++ .../installation/scripts/environment.sh | 213 + .../installation/scripts/etcd.sh| 152 +++ .../installation/scripts/hadoop.sh | 117 + .../installation/scripts/menu.sh| 444 +++ .../installation/scripts/nvidia-docker.sh | 99 + .../installation/scripts/nvidia.sh | 120 + .../installation/scripts/submarine.sh | 38 ++ .../installation/scripts/utils.sh | 123 + 21 files changed, 2172 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/46d6e001/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml -- diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml index a2ea08c..9c401e8 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml @@ -66,6 +66,14 @@ 0755 + hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation + /share/hadoop/yarn/submarine-installer + +**/* + + 0755 + + hadoop-yarn/conf etc/hadoop http://git-wip-us.apache.org/repos/asf/hadoop/blob/46d6e001/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf new file mode 100644 index 000..82dcf61 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# install config # + +# DNS +LOCAL_DNS_HOST="172.17.0.9" # /etc/resolv.conf +YARN_DNS_HOST="10.196.69.173" # yarn dns server ip address + +# etcd hosts list +ETCD_HOSTS=(10.196.69.173 10.196.69.174 10.196.69.175) + +# docker registry ip:port +DOCKER_REGISTRY="10.120.196.232:5000" + +# Start the http download service on the specified server, +# Will download all the dependencies in the http server, +# Run the install script on other servers. +# Automatically download dependencies from http, +# Solve the problem that all servers are slow to download online. +# At the same time, you can also manually download the dependenc
[22/50] [abbrv] hadoop git commit: HDDS-629. Make ApplyTransaction calls in ContainerStateMachine idempotent. Contributed by Shashikant Banerjee.
HDDS-629. Make ApplyTransaction calls in ContainerStateMachine idempotent.
Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e13a38f4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e13a38f4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e13a38f4
Branch: refs/heads/HDFS-12943
Commit: e13a38f4bc358666e64687636cf7b025bce83b46
Parents: b6fc72a
Author: Jitendra Pandey
Authored: Mon Oct 15 11:52:38 2018 -0700
Committer: Jitendra Pandey
Committed: Mon Oct 15 11:52:38 2018 -0700
--
.../container/keyvalue/KeyValueHandler.java | 10 +-
.../container/keyvalue/helpers/ChunkUtils.java | 11 +-
.../keyvalue/impl/BlockManagerImpl.java | 24 +++-
.../keyvalue/impl/ChunkManagerImpl.java | 43 +++
.../TestContainerStateMachineIdempotency.java | 121 +++
.../common/impl/TestContainerPersistence.java | 17 +--
6 files changed, 203 insertions(+), 23 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e13a38f4/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
--
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index 4c87b19..da77f1c 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -241,9 +241,12 @@ public class KeyValueHandler extends Handler {
newContainer.create(volumeSet, volumeChoosingPolicy, scmID);
containerSet.addContainer(newContainer);
} else {
-throw new StorageContainerException("Container already exists with " +
-"container Id " + containerID, ContainerProtos.Result
-.CONTAINER_EXISTS);
+
+// The create container request for an already existing container can
+// arrive in case the ContainerStateMachine reapplies the transaction
+// on datanode restart. Just log a warning msg here.
+LOG.warn("Container already exists." +
+"container Id " + containerID);
}
} catch (StorageContainerException ex) {
return ContainerUtils.logAndReturnError(LOG, ex, request);
@@ -370,6 +373,7 @@ public class KeyValueHandler extends Handler {
/**
* Handles Close Container Request. An open container is closed.
+ * Close Container call is idempotent.
*/
ContainerCommandResponseProto handleCloseContainer(
ContainerCommandRequestProto request, KeyValueContainer kvContainer) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e13a38f4/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
--
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
index 8bdae0f..20598d9 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
@@ -231,21 +231,18 @@ public final class ChunkUtils {
*
* @param chunkFile - chunkFile to write data into.
* @param info - chunk info.
- * @return boolean isOverwrite
- * @throws StorageContainerException
+ * @return true if the chunkFile exists and chunkOffset < chunkFile length,
+ * false otherwise.
*/
public static boolean validateChunkForOverwrite(File chunkFile,
- ChunkInfo info) throws StorageContainerException {
+ ChunkInfo info) {
Logger log = LoggerFactory.getLogger(ChunkManagerImpl.class);
if (isOverWriteRequested(chunkFile, info)) {
if (!isOverWritePermitted(info)) {
-log.error("Rejecting write chunk request. Chunk overwrite " +
+log.warn("Duplicate write chunk request. Chunk overwrite " +
"without explicit request. {}", info.toString());
-throw new StorageContainerException("Rejecting write chunk request. " +
-"OverWrite flag required." + info.toString(),
-OVERWRITE_FLAG_REQUIRED);
}
return true;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e13a38f4/hadoop-hdds/container-service/src/main/java/org/apache/hadoo
[50/50] [abbrv] hadoop git commit: HDFS-13925. Unit Test for transitioning between different states. Contributed by Sherwood Zheng.
HDFS-13925. Unit Test for transitioning between different states. Contributed
by Sherwood Zheng.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3a5b78ac
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3a5b78ac
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3a5b78ac
Branch: refs/heads/HDFS-12943
Commit: 3a5b78ac47ae8bca747e530941728d573975104c
Parents: 8094c38
Author: Konstantin V Shvachko
Authored: Tue Oct 16 16:35:02 2018 -0700
Committer: Konstantin V Shvachko
Committed: Tue Oct 16 16:35:02 2018 -0700
--
.../server/namenode/ha/TestObserverNode.java| 23
1 file changed, 23 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a5b78ac/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java
index 28fd330..b18c5b8 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.namenode.ha;
import static
org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter.getServiceState;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.anyLong;
@@ -30,11 +31,13 @@ import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
import java.util.List;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
+import org.apache.hadoop.ha.ServiceFailedException;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.Block;
@@ -100,6 +103,26 @@ public class TestObserverNode {
}
@Test
+ public void testNoActiveToObserver() throws Exception {
+try {
+ dfsCluster.transitionToObserver(0);
+} catch (ServiceFailedException e) {
+ return;
+}
+fail("active cannot be transitioned to observer");
+ }
+
+ @Test
+ public void testNoObserverToActive() throws Exception {
+try {
+ dfsCluster.transitionToActive(2);
+} catch (ServiceFailedException e) {
+ return;
+}
+fail("observer cannot be transitioned to active");
+ }
+
+ @Test
public void testSimpleRead() throws Exception {
Path testPath2 = new Path(testPath, "test2");
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[35/50] [abbrv] hadoop git commit: HADOOP-15826. @Retries annotation of putObject() call & uses wrong. Contributed by Steve Loughran and Ewan Higgs.
HADOOP-15826. @Retries annotation of putObject() call & uses wrong.
Contributed by Steve Loughran and Ewan Higgs.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d59ca43b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d59ca43b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d59ca43b
Branch: refs/heads/HDFS-12943
Commit: d59ca43bff8a457ce7ab62a61acd89aacbe71b93
Parents: f90c64e
Author: Steve Loughran
Authored: Tue Oct 16 20:02:54 2018 +0100
Committer: Steve Loughran
Committed: Tue Oct 16 20:02:54 2018 +0100
--
.../src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 5 -
.../java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java | 4 ++--
2 files changed, 6 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d59ca43b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
--
diff --git
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index df0ec5d..3c432fc 100644
---
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -2449,11 +2449,14 @@ public class S3AFileSystem extends FileSystem
implements StreamCapabilities {
* Wait for an upload to complete.
* If the waiting for completion is interrupted, the upload will be
* aborted before an {@code InterruptedIOException} is thrown.
- * @param upload upload to wait for
+ * If the upload (or its result collection) failed, this is where
+ * the failure is raised as an AWS exception
* @param key destination key
+ * @param uploadInfo upload to wait for
* @return the upload result
* @throws InterruptedIOException if the blocking was interrupted.
*/
+ @Retries.OnceRaw
UploadResult waitForUploadCompletion(String key, UploadInfo uploadInfo)
throws InterruptedIOException {
Upload upload = uploadInfo.getUpload();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d59ca43b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java
--
diff --git
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java
index a85a87f..a5f6817 100644
---
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java
+++
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java
@@ -436,7 +436,7 @@ public class WriteOperationHelper {
* @return the result of the operation
* @throws IOException on problems
*/
- @Retries.OnceTranslated
+ @Retries.RetryTranslated
public UploadResult uploadObject(PutObjectRequest putObjectRequest)
throws IOException {
// no retry; rely on xfer manager logic
@@ -451,7 +451,7 @@ public class WriteOperationHelper {
* @throws IOException on problems
* @param destKey destination key
*/
- @Retries.RetryTranslated
+ @Retries.OnceTranslated
public void revertCommit(String destKey) throws IOException {
once("revert commit", destKey,
() -> {
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[24/50] [abbrv] hadoop git commit: HADOOP-15853. TestConfigurationDeprecation leaves behind a temp file, resulting in a license issue (ayushtkn via rkanter)
HADOOP-15853. TestConfigurationDeprecation leaves behind a temp file, resulting
in a license issue (ayushtkn via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7fe1a40a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7fe1a40a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7fe1a40a
Branch: refs/heads/HDFS-12943
Commit: 7fe1a40a6ba692ce5907b96db3a7cb3639c091bd
Parents: ef9dc6c
Author: Robert Kanter
Authored: Mon Oct 15 15:02:37 2018 -0700
Committer: Robert Kanter
Committed: Mon Oct 15 15:02:37 2018 -0700
--
.../java/org/apache/hadoop/conf/TestConfigurationDeprecation.java | 1 +
1 file changed, 1 insertion(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7fe1a40a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java
index 4014b60..efb8131 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java
@@ -77,6 +77,7 @@ public class TestConfigurationDeprecation {
new File(CONFIG).delete();
new File(CONFIG2).delete();
new File(CONFIG3).delete();
+new File(CONFIG4).delete();
}
private void startConfig() throws IOException{
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[47/50] [abbrv] hadoop git commit: HDDS-665. Add hdds.datanode.dir to docker-config. Contributed by Bharat Viswanadham.
HDDS-665. Add hdds.datanode.dir to docker-config. Contributed by Bharat Viswanadham. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/657032f5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/657032f5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/657032f5 Branch: refs/heads/HDFS-12943 Commit: 657032f5ddd2cdf111fc89c96672fe08b483cbfc Parents: 84e22a6 Author: Bharat Viswanadham Authored: Tue Oct 16 15:29:53 2018 -0700 Committer: Bharat Viswanadham Committed: Tue Oct 16 15:29:53 2018 -0700 -- hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config | 1 + hadoop-ozone/dist/src/main/compose/ozone/docker-config| 1 + hadoop-ozone/dist/src/main/compose/ozonefs/docker-config | 1 + hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config| 2 ++ hadoop-ozone/dist/src/main/compose/ozones3/docker-config | 2 ++ hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config | 1 + 6 files changed, 8 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/657032f5/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config -- diff --git a/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config index 3b2819f..9729aef 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config @@ -24,6 +24,7 @@ OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata OZONE-SITE.XML_ozone.scm.client.address=scm OZONE-SITE.XML_ozone.replication=1 OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService +OZONE-SITE.XML_hdds.datanode.dir=/data/hdds HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000 http://git-wip-us.apache.org/repos/asf/hadoop/blob/657032f5/hadoop-ozone/dist/src/main/compose/ozone/docker-config -- diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-config b/hadoop-ozone/dist/src/main/compose/ozone/docker-config index f2c8db1..86257ff 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-config @@ -24,6 +24,7 @@ OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata OZONE-SITE.XML_ozone.handler.type=distributed OZONE-SITE.XML_ozone.scm.client.address=scm OZONE-SITE.XML_ozone.replication=1 +OZONE-SITE.XML_hdds.datanode.dir=/data/hdds HDFS-SITE.XML_rpc.metrics.quantile.enable=true HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout http://git-wip-us.apache.org/repos/asf/hadoop/blob/657032f5/hadoop-ozone/dist/src/main/compose/ozonefs/docker-config -- diff --git a/hadoop-ozone/dist/src/main/compose/ozonefs/docker-config b/hadoop-ozone/dist/src/main/compose/ozonefs/docker-config index 4ff7f56..675dcba 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonefs/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonefs/docker-config @@ -25,6 +25,7 @@ OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata OZONE-SITE.XML_ozone.handler.type=distributed OZONE-SITE.XML_ozone.scm.client.address=scm OZONE-SITE.XML_ozone.scm.heartbeat.interval=3s +OZONE-SITE.XML_hdds.datanode.dir=/data/hdds HDFS-SITE.XML_rpc.metrics.quantile.enable=true OZONE-SITE.XML_ozone.replication=1 HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 http://git-wip-us.apache.org/repos/asf/hadoop/blob/657032f5/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config -- diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config b/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config index f1c0147..a814c39 100644 --- a/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config @@ -23,6 +23,8 @@ OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata OZONE-SITE.XML_ozone.handler.type=distributed OZONE-SITE.XML_ozone.scm.client.address=scm +OZONE-SITE.XML_hdds.datanode.dir=/data/hdds + HDFS-SITE.XML_rpc.metrics.quantile.enable=true HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 OZONE-SITE.XML_ozone.replication=1 http://git-wip-us.apache.org/repos/asf/hadoop/blob/657032f5/hadoop-ozone/dist/src/main/compose/ozones3/docker-config -- diff --git a/hadoop-ozone/dist/src/main/compose/ozones3/docker-config b/hadoop-ozone/dist/src/main/compos
[19/50] [abbrv] hadoop git commit: YARN-8775. TestDiskFailures.testLocalDirsFailures sometimes can fail on concurrent File modifications. (Contributed by Antal Bálint Steinbach)
YARN-8775. TestDiskFailures.testLocalDirsFailures sometimes can fail on
concurrent File modifications. (Contributed by Antal Bálint Steinbach)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f880ff41
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f880ff41
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f880ff41
Branch: refs/heads/HDFS-12943
Commit: f880ff418c07965b333c750805c8de77a067b158
Parents: fa94d37
Author: Haibo Chen
Authored: Mon Oct 15 09:37:20 2018 -0700
Committer: Haibo Chen
Committed: Mon Oct 15 09:37:20 2018 -0700
--
.../nodemanager/LocalDirsHandlerService.java| 5 ++-
.../hadoop/yarn/server/TestDiskFailures.java| 38
2 files changed, 19 insertions(+), 24 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f880ff41/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
index 6eabd0d..b2bb4e3 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
@@ -27,6 +27,8 @@ import java.util.List;
import java.util.Set;
import java.util.Timer;
import java.util.TimerTask;
+
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.util.DiskChecker.DiskErrorException;
import org.apache.hadoop.util.DiskValidator;
import org.apache.hadoop.util.DiskValidatorFactory;
@@ -493,7 +495,8 @@ public class LocalDirsHandlerService extends
AbstractService {
}
- private void checkDirs() {
+ @VisibleForTesting
+ public void checkDirs() {
boolean disksStatusChange = false;
Set failedLocalDirsPreCheck =
new HashSet(localDirs.getFailedDirs());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f880ff41/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestDiskFailures.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestDiskFailures.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestDiskFailures.java
index e9de20a..23bb039 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestDiskFailures.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestDiskFailures.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
@@ -56,7 +55,12 @@ public class TestDiskFailures {
private static final Logger LOG =
LoggerFactory.getLogger(TestDiskFailures.class);
- private static final long DISK_HEALTH_CHECK_INTERVAL = 1000;//1 sec
+ /*
+ * Set disk check interval high enough so that it never runs during the test.
+ * Checks will be called manually if necessary.
+ */
+ private static final long TOO_HIGH_DISK_HEALTH_CHECK_INTERVAL =
+ 1000 * 60 * 60 * 24;
private static FileContext localFS = null;
private static final File testDir = new File("target",
@@ -146,9 +150,10 @@ public class TestDiskFailures {
: YarnConfiguration.NM_LOG_DIRS;
Configuration conf = new Configuration();
-// set disk health check interval to a small value (say 1 sec).
+// set disk health check interval to a large value to effectively disable
+// disk health check done internally in LocalDirsHandlerService"
conf.setLong(YarnC
[07/50] [abbrv] hadoop git commit: HDDS-654. Add ServiceLoader resource for OzoneFileSystem. Contributed by Jitendra Nath Pandey.
HDDS-654. Add ServiceLoader resource for OzoneFileSystem. Contributed by
Jitendra Nath Pandey.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/addb8460
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/addb8460
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/addb8460
Branch: refs/heads/HDFS-12943
Commit: addb84600fa2f9e010768c642db3fdc2ec497d52
Parents: ae268b4
Author: Arpit Agarwal
Authored: Sat Oct 13 09:19:45 2018 -0700
Committer: Arpit Agarwal
Committed: Sat Oct 13 09:19:45 2018 -0700
--
.../services/org.apache.hadoop.fs.FileSystem| 16
.../hadoop/fs/ozone/TestOzoneFileSystem.java| 39
2 files changed, 55 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/addb8460/hadoop-ozone/ozonefs/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
--
diff --git
a/hadoop-ozone/ozonefs/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
b/hadoop-ozone/ozonefs/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
new file mode 100644
index 000..0368002
--- /dev/null
+++
b/hadoop-ozone/ozonefs/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.hadoop.fs.ozone.OzoneFileSystem
http://git-wip-us.apache.org/repos/asf/hadoop/blob/addb8460/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
--
diff --git
a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
new file mode 100644
index 000..ba5253f
--- /dev/null
+++
b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+
+/**
+ * Ozone file system tests that are not covered by contract tests.
+ */
+public class TestOzoneFileSystem {
+
+ @Test
+ public void testOzoneFsServiceLoader() throws IOException {
+Assert.assertEquals(
+FileSystem.getFileSystemClass(OzoneConsts.OZONE_URI_SCHEME, null),
+OzoneFileSystem.class);
+ }
+}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[02/50] [abbrv] hadoop git commit: HDDS-646. TestChunkStreams.testErrorReadGroupInputStream fails. Contributed by Nanda kumar.
HDDS-646. TestChunkStreams.testErrorReadGroupInputStream fails. Contributed by
Nanda kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ddc96493
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ddc96493
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ddc96493
Branch: refs/heads/HDFS-12943
Commit: ddc964932817b4c4e4f4dc848dae764d5285e875
Parents: 5c8e023
Author: Arpit Agarwal
Authored: Fri Oct 12 15:06:42 2018 -0700
Committer: Arpit Agarwal
Committed: Fri Oct 12 15:06:42 2018 -0700
--
.../src/test/java/org/apache/hadoop/ozone/om/TestChunkStreams.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddc96493/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestChunkStreams.java
--
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestChunkStreams.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestChunkStreams.java
index 7ff9d63..177694c 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestChunkStreams.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestChunkStreams.java
@@ -226,7 +226,7 @@ public class TestChunkStreams {
// read following 300 bytes, but only 200 left
len = groupInputStream.read(resBuf, 340, 260);
- assertEquals(5, groupInputStream.getCurrentStreamIndex());
+ assertEquals(4, groupInputStream.getCurrentStreamIndex());
assertEquals(0, groupInputStream.getRemainingOfIndex(4));
assertEquals(160, len);
assertEquals(dataString, new String(resBuf).substring(0, 500));
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[25/50] [abbrv] hadoop git commit: HDDS-490. Improve om and scm start up options . Contributed by Namit Maheshwari.
HDDS-490. Improve om and scm start up options . Contributed by Namit Maheshwari.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2614078b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2614078b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2614078b
Branch: refs/heads/HDFS-12943
Commit: 2614078b213e03085635b4455ed391438eef8db2
Parents: 7fe1a40
Author: Arpit Agarwal
Authored: Mon Oct 15 16:45:08 2018 -0700
Committer: Arpit Agarwal
Committed: Mon Oct 15 16:45:08 2018 -0700
--
.../scm/server/StorageContainerManager.java | 8 +-
.../dist/src/main/compose/ozonescripts/start.sh | 4 +-
hadoop-ozone/docs/content/RealCluster.md| 10 +-
.../org/apache/hadoop/ozone/om/TestOmInit.java | 103 +++
.../apache/hadoop/ozone/om/OzoneManager.java| 14 +--
5 files changed, 121 insertions(+), 18 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2614078b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
--
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index ce2725f..9626105 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -927,10 +927,10 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
* Startup options.
*/
public enum StartupOption {
-INIT("-init"),
-CLUSTERID("-clusterid"),
-GENCLUSTERID("-genclusterid"),
-REGULAR("-regular"),
+INIT("--init"),
+CLUSTERID("--clusterid"),
+GENCLUSTERID("--genclusterid"),
+REGULAR("--regular"),
HELP("-help");
private final String name;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2614078b/hadoop-ozone/dist/src/main/compose/ozonescripts/start.sh
--
diff --git a/hadoop-ozone/dist/src/main/compose/ozonescripts/start.sh
b/hadoop-ozone/dist/src/main/compose/ozonescripts/start.sh
index 3358b07..9540eb9 100755
--- a/hadoop-ozone/dist/src/main/compose/ozonescripts/start.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozonescripts/start.sh
@@ -15,10 +15,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
docker-compose ps | grep datanode | awk '{print $1}' | xargs -n1 docker
inspect --format '{{ .Config.Hostname }}' > ../../etc/hadoop/workers
-docker-compose exec scm /opt/hadoop/bin/ozone scm -init
+docker-compose exec scm /opt/hadoop/bin/ozone scm --init
docker-compose exec scm /opt/hadoop/sbin/start-ozone.sh
#We need a running SCM for om objectstore creation
#TODO create a utility to wait for the startup
sleep 10
-docker-compose exec om /opt/hadoop/bin/ozone om -createObjectStore
+docker-compose exec om /opt/hadoop/bin/ozone om --init
docker-compose exec scm /opt/hadoop/sbin/start-ozone.sh
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2614078b/hadoop-ozone/docs/content/RealCluster.md
--
diff --git a/hadoop-ozone/docs/content/RealCluster.md
b/hadoop-ozone/docs/content/RealCluster.md
index 9d86c84..78dd46e 100644
--- a/hadoop-ozone/docs/content/RealCluster.md
+++ b/hadoop-ozone/docs/content/RealCluster.md
@@ -26,7 +26,7 @@ menu:
Before we boot up the Ozone cluster, we need to initialize both SCM and Ozone
Manager.
{{< highlight bash >}}
-ozone scm -init
+ozone scm --init
{{< /highlight >}}
This allows SCM to create the cluster Identity and initialize its state.
The ```init``` command is similar to Namenode format. Init command is executed
only once, that allows SCM to create all the required on-disk structures to
work correctly.
@@ -37,7 +37,7 @@ ozone --daemon start scm
Once we know SCM is up and running, we can create an Object Store for our use.
This is done by running the following command.
{{< highlight bash >}}
-ozone om -createObjectStore
+ozone om --init
{{< /highlight >}}
@@ -50,7 +50,7 @@ ozone --daemon start om
At this point Ozone's name services, the Ozone manager, and the block service
SCM is both running.
**Please note**: If SCM is not running
-```createObjectStore``` command will fail. SCM start will fail if on-disk data
structures are missing. So please make sure you have done both ```init``` and
```createObjectStore``` commands.
+```om --init``` command will fail. SCM start will fail if on-disk data
structures are missing
[48/50] [abbrv] hadoop git commit: YARN-8810. Fixed a YARN service bug in comparing ConfigFile object. Contributed by Chandni Singh
YARN-8810. Fixed a YARN service bug in comparing ConfigFile object.
Contributed by Chandni Singh
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3bfd214a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3bfd214a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3bfd214a
Branch: refs/heads/HDFS-12943
Commit: 3bfd214a59a60263aff67850c4d646c64fd76a01
Parents: 657032f
Author: Eric Yang
Authored: Tue Oct 16 18:54:40 2018 -0400
Committer: Eric Yang
Committed: Tue Oct 16 18:54:40 2018 -0400
--
.../yarn/service/UpgradeComponentsFinder.java | 2 +-
.../yarn/service/api/records/ConfigFile.java| 3 +-
.../TestDefaultUpgradeComponentsFinder.java | 40 ++--
3 files changed, 40 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3bfd214a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/UpgradeComponentsFinder.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/UpgradeComponentsFinder.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/UpgradeComponentsFinder.java
index 19ff6db..96a34f4 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/UpgradeComponentsFinder.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/UpgradeComponentsFinder.java
@@ -88,7 +88,7 @@ public interface UpgradeComponentsFinder {
}
if (!Objects.equals(currentDef.getConfiguration(),
- currentDef.getConfiguration())) {
+ targetDef.getConfiguration())) {
return targetDef.getComponents();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3bfd214a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
index cd9dc84..1cdae86 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
@@ -199,7 +199,8 @@ public class ConfigFile implements Serializable {
ConfigFile configFile = (ConfigFile) o;
return Objects.equals(this.type, configFile.type)
&& Objects.equals(this.destFile, configFile.destFile)
-&& Objects.equals(this.srcFile, configFile.srcFile);
+&& Objects.equals(this.srcFile, configFile.srcFile)
+&& Objects.equals(this.properties, configFile.properties);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3bfd214a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestDefaultUpgradeComponentsFinder.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestDefaultUpgradeComponentsFinder.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestDefaultUpgradeComponentsFinder.java
index b0a01b3..304e740 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestDefaultUpgradeComponentsFinder.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestDefaultUpgradeComponentsFinder.java
@
[18/50] [abbrv] hadoop git commit: YARN-8869. YARN Service Client might not work correctly with RM REST API for Kerberos authentication. Contributed by Eric Yang.
YARN-8869. YARN Service Client might not work correctly with RM REST API for
Kerberos authentication. Contributed by Eric Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fa94d370
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fa94d370
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fa94d370
Branch: refs/heads/HDFS-12943
Commit: fa94d370b6e3cba9c7560c09b517583e6652f103
Parents: b4a38e7
Author: Sunil G
Authored: Mon Oct 15 21:21:57 2018 +0530
Committer: Sunil G
Committed: Mon Oct 15 21:21:57 2018 +0530
--
.../yarn/service/client/ApiServiceClient.java | 19 ++--
.../client/TestSecureApiServiceClient.java | 113 ++-
2 files changed, 120 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa94d370/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
index b7a1541..851acbd 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
@@ -70,7 +70,6 @@ import org.slf4j.LoggerFactory;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.api.client.WebResource.Builder;
import com.sun.jersey.api.client.config.ClientConfig;
import com.sun.jersey.api.client.config.DefaultClientConfig;
@@ -144,7 +143,7 @@ public class ApiServiceClient extends AppAdminClient {
/**
* Calculate Resource Manager address base on working REST API.
*/
- private String getRMWebAddress() {
+ String getRMWebAddress() {
Configuration conf = getConfig();
String scheme = "http://";;
String path = "/app/v1/services/version";
@@ -156,8 +155,7 @@ public class ApiServiceClient extends AppAdminClient {
.get("yarn.resourcemanager.webapp.https.address");
}
boolean useKerberos = UserGroupInformation.isSecurityEnabled();
-List rmServers = RMHAUtils
-.getRMHAWebappAddresses(new YarnConfiguration(conf));
+List rmServers = getRMHAWebAddresses(conf);
for (String host : rmServers) {
try {
Client client = Client.create();
@@ -175,16 +173,16 @@ public class ApiServiceClient extends AppAdminClient {
LOG.debug("Fail to resolve username: {}", e);
}
}
-WebResource webResource = client
-.resource(sb.toString());
+Builder builder = client
+.resource(sb.toString()).type(MediaType.APPLICATION_JSON);
if (useKerberos) {
String[] server = host.split(":");
String challenge = generateToken(server[0]);
- webResource.header(HttpHeaders.AUTHORIZATION, "Negotiate " +
+ builder.header(HttpHeaders.AUTHORIZATION, "Negotiate " +
challenge);
LOG.debug("Authorization: Negotiate {}", challenge);
}
-ClientResponse test = webResource.get(ClientResponse.class);
+ClientResponse test = builder.get(ClientResponse.class);
if (test.getStatus() == 200) {
rmAddress = host;
break;
@@ -197,6 +195,11 @@ public class ApiServiceClient extends AppAdminClient {
return scheme+rmAddress;
}
+ List getRMHAWebAddresses(Configuration conf) {
+return RMHAUtils
+.getRMHAWebappAddresses(new YarnConfiguration(conf));
+ }
+
/**
* Compute active resource manager API service location.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa94d370/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java
b/hadoop-yarn-project/hadoop-ya
[36/50] [abbrv] hadoop git commit: YARN-8448. AM HTTPS Support for AM communication with RMWeb proxy. (Contributed by Robert Kanter)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2288ac4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyCA.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyCA.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyCA.java
new file mode 100644
index 000..26760d3
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyCA.java
@@ -0,0 +1,408 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.webproxy;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.http.conn.ssl.DefaultHostnameVerifier;
+import org.apache.http.conn.util.PublicSuffixMatcherLoader;
+import org.bouncycastle.asn1.x500.X500Name;
+import org.bouncycastle.asn1.x509.AlgorithmIdentifier;
+import org.bouncycastle.asn1.x509.BasicConstraints;
+import org.bouncycastle.asn1.x509.Extension;
+import org.bouncycastle.asn1.x509.SubjectPublicKeyInfo;
+import org.bouncycastle.cert.X509CertificateHolder;
+import org.bouncycastle.cert.X509v3CertificateBuilder;
+import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter;
+import org.bouncycastle.cert.jcajce.JcaX509ExtensionUtils;
+import org.bouncycastle.crypto.util.PrivateKeyFactory;
+import org.bouncycastle.jce.provider.BouncyCastleProvider;
+import org.bouncycastle.operator.ContentSigner;
+import org.bouncycastle.operator.DefaultDigestAlgorithmIdentifierFinder;
+import org.bouncycastle.operator.DefaultSignatureAlgorithmIdentifierFinder;
+import org.bouncycastle.operator.OperatorCreationException;
+import org.bouncycastle.operator.bc.BcRSAContentSignerBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.KeyManager;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLPeerUnverifiedException;
+import javax.net.ssl.SSLSession;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.TrustManagerFactory;
+import javax.net.ssl.X509KeyManager;
+import javax.net.ssl.X509TrustManager;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.math.BigInteger;
+import java.net.Socket;
+import java.security.GeneralSecurityException;
+import java.security.InvalidKeyException;
+import java.security.Key;
+import java.security.KeyPair;
+import java.security.KeyPairGenerator;
+import java.security.KeyStore;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.Principal;
+import java.security.PrivateKey;
+import java.security.PublicKey;
+import java.security.SecureRandom;
+import java.security.Security;
+import java.security.SignatureException;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.GregorianCalendar;
+import java.util.Random;
+import java.util.UUID;
+
+/**
+ * Allows for the generation and acceptance of specialized HTTPS Certificates
to
+ * be used for HTTPS communication between the AMs and the RM Proxy.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class ProxyCA {
+ private static final Logger LOG = LoggerFactory.getLogger(ProxyCA.class);
+
+ private X509Certificate caCert;
+ private KeyPair caKeyPair;
+ private KeyStore childTrustStore;
+ private final Random srand;
+ private X509TrustManager defaultTrustManager;
+ private X509KeyManager x509KeyManager;
+ private HostnameVerifier ho
[05/50] [abbrv] hadoop git commit: HDDS-616. Collect all the robot test outputs and return with the right exit code. Contributed by Elek, Marton.
HDDS-616. Collect all the robot test outputs and return with the right exit
code. Contributed by Elek, Marton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cdad91c0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cdad91c0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cdad91c0
Branch: refs/heads/HDFS-12943
Commit: cdad91c03985317ec7b1807246709444ff6d5da0
Parents: 28ca5c9
Author: Márton Elek
Authored: Thu Oct 11 11:30:07 2018 +0200
Committer: Márton Elek
Committed: Sat Oct 13 08:40:49 2018 +0200
--
hadoop-ozone/dist/src/main/smoketest/test.sh | 25 ---
1 file changed, 22 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cdad91c0/hadoop-ozone/dist/src/main/smoketest/test.sh
--
diff --git a/hadoop-ozone/dist/src/main/smoketest/test.sh
b/hadoop-ozone/dist/src/main/smoketest/test.sh
index f3c2224..a6dc1f1 100755
--- a/hadoop-ozone/dist/src/main/smoketest/test.sh
+++ b/hadoop-ozone/dist/src/main/smoketest/test.sh
@@ -17,19 +17,35 @@
set -e
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+RESULT_DIR=result
+#delete previous results
+rm -rf "${DIR:?}/$RESULT_DIR"
+mkdir -p "$DIR/$RESULT_DIR"
+#Should be writeable from the docker containers where user is different.
+chmod ogu+w "$DIR/$RESULT_DIR"
execute_tests(){
- COMPOSE_FILE=$DIR/../compose/$1/docker-compose.yaml
+ COMPOSE_DIR=$1
+ COMPOSE_FILE=$DIR/../compose/$COMPOSE_DIR/docker-compose.yaml
TESTS=$2
- echo "Executing test ${TESTS[*]} with $COMPOSE_FILE"
+ echo "-"
+ echo "Executing test(s): [${TESTS[*]}]"
+ echo ""
+ echo " Cluster type: $COMPOSE_DIR"
+ echo " Compose file: $COMPOSE_FILE"
+ echo " Output dir:$DIR/$RESULT_DIR"
+ echo " Command to rerun: ./test.sh --keep --env $COMPOSE_DIR $TESTS"
+ echo "-"
docker-compose -f "$COMPOSE_FILE" down
docker-compose -f "$COMPOSE_FILE" up -d
echo "Waiting 30s for cluster start up..."
sleep 30
for TEST in "${TESTS[@]}"; do
+ TITLE="Ozone $TEST tests with $COMPOSE_DIR cluster"
set +e
- docker-compose -f "$COMPOSE_FILE" exec datanode python -m robot
"smoketest/$TEST"
+ docker-compose -f "$COMPOSE_FILE" exec datanode python -m robot --log
NONE --report NONE "${OZONE_ROBOT_OPTS[@]}" --output
"smoketest/$RESULT_DIR/robot-$COMPOSE_DIR-${TEST//\//_/}.xml" --logtitle
"$TITLE" --reporttitle "$TITLE" "smoketest/$TEST"
set -e
+ docker-compose -f "$COMPOSE_FILE" logs >
"$DIR/$RESULT_DIR/docker-$COMPOSE_DIR-${TEST//\//_/}.log"
done
if [ "$KEEP_RUNNING" = false ]; then
docker-compose -f "$COMPOSE_FILE" down
@@ -99,3 +115,6 @@ if [ "$RUN_ALL" = true ]; then
else
execute_tests "$DOCKERENV" "${POSITIONAL[@]}"
fi
+
+#Generate the combined output and return with the right exit code (note: robot
= execute test, rebot = generate output)
+docker run --rm -it -v "$DIR/..:/opt/hadoop" apache/hadoop-runner rebot -d
"smoketest/$RESULT_DIR" "smoketest/$RESULT_DIR/robot-*.xml"
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[12/50] [abbrv] hadoop git commit: HDDS-579. ContainerStateMachine should fail subsequent transactions per container in case one fails. Contributed by Shashikant Banerjee.
HDDS-579. ContainerStateMachine should fail subsequent transactions per
container in case one fails. Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/603649d3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/603649d3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/603649d3
Branch: refs/heads/HDFS-12943
Commit: 603649d3a9ff12b725d06f5f317966de9a59fe70
Parents: 5209c75
Author: Jitendra Pandey
Authored: Sat Oct 13 19:15:01 2018 -0700
Committer: Jitendra Pandey
Committed: Sat Oct 13 19:15:01 2018 -0700
--
.../main/proto/DatanodeContainerProtocol.proto | 4 +-
.../container/common/impl/HddsDispatcher.java | 63 +--
.../container/keyvalue/KeyValueHandler.java | 20 +-
.../StorageContainerDatanodeProtocol.proto | 1 +
.../rpc/TestContainerStateMachineFailures.java | 185 +++
5 files changed, 242 insertions(+), 31 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/603649d3/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
--
diff --git a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
index 662df8f..da55db3 100644
--- a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
+++ b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
@@ -139,6 +139,7 @@ enum Result {
CONTAINER_CHECKSUM_ERROR = 33;
UNKNOWN_CONTAINER_TYPE = 34;
BLOCK_NOT_COMMITTED = 35;
+ CONTAINER_UNHEALTHY = 36;
}
/**
@@ -161,7 +162,8 @@ enum ContainerLifeCycleState {
OPEN = 1;
CLOSING = 2;
CLOSED = 3;
-INVALID = 4;
+UNHEALTHY = 4;
+INVALID = 5;
}
message ContainerCommandRequestProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/603649d3/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
--
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
index bb5002a..1849841 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
@@ -142,6 +142,26 @@ public class HddsDispatcher implements ContainerDispatcher
{
responseProto = handler.handle(msg, container);
if (responseProto != null) {
metrics.incContainerOpsLatencies(cmdType, System.nanoTime() - startTime);
+
+ // If the request is of Write Type and the container operation
+ // is unsuccessful, it implies the applyTransaction on the container
+ // failed. All subsequent transactions on the container should fail and
+ // hence replica will be marked unhealthy here. In this case, a close
+ // container action will be sent to SCM to close the container.
+ if (!HddsUtils.isReadOnly(msg)
+ && responseProto.getResult() != ContainerProtos.Result.SUCCESS) {
+// If the container is open and the container operation has failed,
+// it should be first marked unhealthy and the initiate the close
+// container action. This also implies this is the first transaction
+// which has failed, so the container is marked unhealthy right here.
+// Once container is marked unhealthy, all the subsequent write
+// transactions will fail with UNHEALTHY_CONTAINER exception.
+if (container.getContainerState() == ContainerLifeCycleState.OPEN) {
+ container.getContainerData()
+ .setState(ContainerLifeCycleState.UNHEALTHY);
+ sendCloseContainerActionIfNeeded(container);
+}
+ }
return responseProto;
} else {
return ContainerUtils.unsupportedRequest(msg);
@@ -149,31 +169,46 @@ public class HddsDispatcher implements
ContainerDispatcher {
}
/**
- * If the container usage reaches the close threshold we send Close
- * ContainerAction to SCM.
- *
+ * If the container usage reaches the close threshold or the container is
+ * marked unhealthy we send Close ContainerAction to SCM.
* @param container current state of container
*/
private void sendCloseContainerActionIfNeeded(Container container) {
// We have to find a more efficient way to close a container.
-Boolean isOpen = Optional.ofNullable(container)
+boolean isSpaceFull = isContainerFull(container);
+boolean shouldClose = is
[39/50] [abbrv] hadoop git commit: MAPREDUCE-7150. Optimize collections used by MR JHS to reduce its memory. (Contributed by Misha Dmitriev)
MAPREDUCE-7150. Optimize collections used by MR JHS to reduce its memory.
(Contributed by Misha Dmitriev)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/babd1449
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/babd1449
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/babd1449
Branch: refs/heads/HDFS-12943
Commit: babd1449bf8898f44c434c852e67240721c0eb00
Parents: c2288ac
Author: Haibo Chen
Authored: Tue Oct 16 13:44:41 2018 -0700
Committer: Haibo Chen
Committed: Tue Oct 16 13:44:41 2018 -0700
--
.../counters/FileSystemCounterGroup.java| 56
.../mapreduce/jobhistory/JobHistoryParser.java | 2 +-
.../hadoop/mapreduce/v2/hs/CompletedTask.java | 5 +-
.../mapreduce/v2/hs/CompletedTaskAttempt.java | 2 +-
4 files changed, 38 insertions(+), 27 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/babd1449/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FileSystemCounterGroup.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FileSystemCounterGroup.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FileSystemCounterGroup.java
index 046368e..ed7f271 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FileSystemCounterGroup.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FileSystemCounterGroup.java
@@ -61,8 +61,9 @@ public abstract class FileSystemCounterGroup
// C[] would need Array.newInstance which requires a Class reference.
// Just a few local casts probably worth not having to carry it around.
- private final Map map =
-new ConcurrentSkipListMap();
+ // Initialized lazily, since in some situations millions of empty maps can
+ // waste a substantial (e.g. 4% as we observed) portion of the heap
+ private Map map;
private String displayName;
private static final Joiner NAME_JOINER = Joiner.on('_');
@@ -214,6 +215,9 @@ public abstract class FileSystemCounterGroup
@SuppressWarnings("unchecked")
public synchronized C findCounter(String scheme, FileSystemCounter key) {
final String canonicalScheme = checkScheme(scheme);
+if (map == null) {
+ map = new ConcurrentSkipListMap<>();
+}
Object[] counters = map.get(canonicalScheme);
int ord = key.ordinal();
if (counters == null) {
@@ -247,10 +251,12 @@ public abstract class FileSystemCounterGroup
protected abstract C newCounter(String scheme, FileSystemCounter key);
@Override
- public int size() {
+ public synchronized int size() {
int n = 0;
-for (Object[] counters : map.values()) {
- n += numSetCounters(counters);
+if (map != null) {
+ for (Object[] counters : map.values()) {
+n += numSetCounters(counters);
+ }
}
return n;
}
@@ -271,19 +277,23 @@ public abstract class FileSystemCounterGroup
* FileSystemGroup ::= #scheme (scheme #counter (key value)*)*
*/
@Override
- public void write(DataOutput out) throws IOException {
-WritableUtils.writeVInt(out, map.size()); // #scheme
-for (Map.Entry entry : map.entrySet()) {
- WritableUtils.writeString(out, entry.getKey()); // scheme
- // #counter for the above scheme
- WritableUtils.writeVInt(out, numSetCounters(entry.getValue()));
- for (Object counter : entry.getValue()) {
-if (counter == null) continue;
-@SuppressWarnings("unchecked")
-FSCounter c = (FSCounter) ((Counter)counter).getUnderlyingCounter();
-WritableUtils.writeVInt(out, c.key.ordinal()); // key
-WritableUtils.writeVLong(out, c.getValue());// value
+ public synchronized void write(DataOutput out) throws IOException {
+if (map != null) {
+ WritableUtils.writeVInt(out, map.size()); // #scheme
+ for (Map.Entry entry : map.entrySet()) {
+WritableUtils.writeString(out, entry.getKey()); // scheme
+// #counter for the above scheme
+WritableUtils.writeVInt(out, numSetCounters(entry.getValue()));
+for (Object counter : entry.getValue()) {
+ if (counter == null) continue;
+ @SuppressWarnings("unchecked")
+ FSCounter c = (FSCounter) ((Counter) counter).getUnderlyingCounter();
+ WritableUtils.writeVInt(out, c.key.ordinal()); // key
+ WritableUtils.writeVLong(out, c.getValue());// value
+
[49/50] [abbrv] hadoop git commit: Merge branch 'trunk' into HDFS-12943
Merge branch 'trunk' into HDFS-12943 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8094c38e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8094c38e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8094c38e Branch: refs/heads/HDFS-12943 Commit: 8094c38efdbfd11519ce518636cb630ac50c5f93 Parents: bba8aa3 3bfd214 Author: Konstantin V Shvachko Authored: Tue Oct 16 16:32:13 2018 -0700 Committer: Konstantin V Shvachko Committed: Tue Oct 16 16:32:13 2018 -0700 -- README.txt | 2 + .../resources/assemblies/hadoop-yarn-dist.xml | 8 + .../src/main/conf/log4j.properties | 6 + .../KeyProviderDelegationTokenExtension.java| 71 +- .../crypto/key/KeyProviderTokenIssuer.java | 4 +- .../crypto/key/kms/KMSClientProvider.java | 220 +-- .../key/kms/LoadBalancingKMSClientProvider.java | 75 ++- .../java/org/apache/hadoop/fs/FileSystem.java | 75 +-- .../org/apache/hadoop/security/Credentials.java | 1 + .../web/DelegationTokenAuthenticatedURL.java| 25 +- .../security/token/DelegationTokenIssuer.java | 111 .../java/org/apache/hadoop/util/KMSUtil.java| 13 +- .../conf/TestConfigurationDeprecation.java | 1 + ...TestKeyProviderDelegationTokenExtension.java | 20 +- .../crypto/key/kms/TestKMSClientProvider.java | 138 .../kms/TestLoadBalancingKMSClientProvider.java | 63 +- .../apache/hadoop/fs/TestFilterFileSystem.java | 3 + .../org/apache/hadoop/fs/TestHarFileSystem.java | 3 + .../hadoop/security/ssl/KeyStoreTestUtil.java | 21 +- .../hadoop/crypto/key/kms/server/TestKMS.java | 349 -- .../hadoop/hdds/scm/XceiverClientManager.java | 17 +- .../scm/client/ContainerOperationClient.java| 15 +- hadoop-hdds/common/pom.xml | 6 - .../apache/hadoop/hdds/scm/ScmConfigKeys.java | 2 +- .../hadoop/hdds/scm/pipeline/Pipeline.java | 211 ++ .../hadoop/hdds/scm/pipeline/PipelineID.java| 80 +++ .../hadoop/hdds/scm/pipeline/package-info.java | 24 + .../org/apache/hadoop/ozone/OzoneConsts.java| 1 + .../apache/hadoop/utils/MetadataKeyFilters.java | 3 +- .../hadoop/utils/MetadataStoreBuilder.java | 38 +- .../main/proto/DatanodeContainerProtocol.proto | 4 +- .../main/proto/ScmBlockLocationProtocol.proto | 1 - .../StorageContainerLocationProtocol.proto | 1 - .../apache/hadoop/utils/TestMetadataStore.java | 2 + .../hadoop/utils/TestRocksDBStoreMBean.java | 19 +- hadoop-hdds/container-service/pom.xml | 11 +- .../common/helpers/ContainerReport.java | 205 -- .../common/helpers/KeyValueContainerReport.java | 117 .../container/common/impl/HddsDispatcher.java | 63 +- .../container/common/interfaces/Container.java | 5 + .../container/common/utils/ContainerCache.java | 6 +- .../container/keyvalue/KeyValueContainer.java | 9 +- .../keyvalue/KeyValueContainerData.java | 16 + .../container/keyvalue/KeyValueHandler.java | 30 +- .../container/keyvalue/helpers/BlockUtils.java | 3 +- .../container/keyvalue/helpers/ChunkUtils.java | 11 +- .../keyvalue/impl/BlockManagerImpl.java | 43 +- .../keyvalue/impl/ChunkManagerImpl.java | 43 ++ .../container/ozoneimpl/ContainerReader.java| 7 + .../StorageContainerDatanodeProtocol.proto | 2 + .../hdds/scm/pipeline/PipelineFactory.java | 56 ++ .../hdds/scm/pipeline/PipelineManager.java | 58 ++ .../hdds/scm/pipeline/PipelineProvider.java | 35 + .../hdds/scm/pipeline/PipelineStateManager.java | 179 + .../hdds/scm/pipeline/PipelineStateMap.java | 212 ++ .../scm/pipeline/RatisPipelineProvider.java | 135 .../hdds/scm/pipeline/SCMPipelineManager.java | 226 +++ .../scm/pipeline/SimplePipelineProvider.java| 80 +++ .../hadoop/hdds/scm/pipeline/package-info.java | 24 + .../scm/server/StorageContainerManager.java | 8 +- .../java/org/apache/hadoop/hdfs/DFSClient.java | 11 +- .../hadoop/hdfs/DistributedFileSystem.java | 14 +- .../org/apache/hadoop/hdfs/HdfsKMSUtil.java | 60 +- .../apache/hadoop/hdfs/util/LongBitFormat.java | 8 + .../hadoop/hdfs/web/WebHdfsFileSystem.java | 29 +- .../resolver/order/RandomResolver.java | 22 +- .../main/webapps/router/federationhealth.html | 1 - .../server/namenode/AclEntryStatusFormat.java | 109 ++-- .../server/namenode/FSImageFormatPBINode.java | 101 +-- .../server/namenode/FSImageFormatProtobuf.java | 26 +- .../namenode/INodeWithAdditionalFields.java | 36 +- .../server/namenode/NameNodeLayoutVersion.java | 3 +- .../server/namenode/SerialNumberManager.java| 152 - .../hdfs/server/namenode/SerialNumberMap.java | 43 +- .../hdfs/server/namenode/XAttrFormat.java |
[01/50] [abbrv] hadoop git commit: HDDS-415. 'ozone om' with incorrect argument first logs all the STARTUP_MSG. Contributed by Namit Maheshwari.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-12943 c2e0e9a02 -> c377e3ca4
HDDS-415. 'ozone om' with incorrect argument first logs all the STARTUP_MSG.
Contributed by Namit Maheshwari.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e0846271
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e0846271
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e0846271
Branch: refs/heads/HDFS-12943
Commit: e08462715003a7680e6e9479d1842e4f0bb3c007
Parents: 5d084d7
Author: Márton Elek
Authored: Thu Sep 13 13:47:59 2018 +0200
Committer: Márton Elek
Committed: Thu Sep 13 14:01:53 2018 +0200
--
.../src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java| 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0846271/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
--
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 348d0c0..b5bd361 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -258,7 +258,6 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
hParser.printGenericCommandUsage(System.err);
System.exit(1);
}
- StringUtils.startupShutdownMessage(OzoneManager.class, argv, LOG);
OzoneManager om = createOm(hParser.getRemainingArgs(), conf);
if (om != null) {
om.start();
@@ -298,6 +297,7 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
}
switch (startOpt) {
case CREATEOBJECTSTORE:
+ StringUtils.startupShutdownMessage(OzoneManager.class, argv, LOG);
terminate(omInit(conf) ? 0 : 1);
return null;
case HELP:
@@ -305,6 +305,7 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
terminate(0);
return null;
default:
+ StringUtils.startupShutdownMessage(OzoneManager.class, argv, LOG);
return new OzoneManager(conf);
}
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[40/50] [abbrv] hadoop git commit: HDDS-463. Fix the release packaging of the ozone distribution. Contributed by Elek Marton.
HDDS-463. Fix the release packaging of the ozone distribution. Contributed by
Elek Marton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3d89c3e7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3d89c3e7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3d89c3e7
Branch: refs/heads/HDFS-12943
Commit: 3d89c3e73eba280b8780228fcd097809271b4c8a
Parents: 8af8453
Author: Bharat Viswanadham
Authored: Mon Sep 17 11:49:09 2018 -0700
Committer: Bharat Viswanadham
Committed: Mon Sep 17 11:49:09 2018 -0700
--
dev-support/bin/ozone-dist-layout-stitching | 11 +++-
.../assemblies/hadoop-src-with-hdds.xml | 56
.../assemblies/hadoop-src-with-hdsl.xml | 56
hadoop-dist/src/main/ozone/README.txt | 51 ++
4 files changed, 116 insertions(+), 58 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d89c3e7/dev-support/bin/ozone-dist-layout-stitching
--
diff --git a/dev-support/bin/ozone-dist-layout-stitching
b/dev-support/bin/ozone-dist-layout-stitching
index b4d94b3..8f1f169 100755
--- a/dev-support/bin/ozone-dist-layout-stitching
+++ b/dev-support/bin/ozone-dist-layout-stitching
@@ -122,7 +122,6 @@ run mkdir "ozone-${HDDS_VERSION}"
run cd "ozone-${HDDS_VERSION}"
run cp -p "${ROOT}/LICENSE.txt" .
run cp -p "${ROOT}/NOTICE.txt" .
-run cp -p "${ROOT}/README.txt" .
# Copy hadoop-common first so that it have always have all dependencies.
# Remaining projects will copy only libraries which are not present already in
'share' directory.
@@ -162,6 +161,14 @@ cp -r
"${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./
rm sbin/*all.sh
rm sbin/*all.cmd
+#remove test and java sources
+find . -name "*tests.jar" | xargs rm
+find . -name "*sources.jar" | xargs rm
+find . -name jdiff -type d | xargs rm -rf
+
+#add ozone specific readme
+
+run cp "${ROOT}/hadoop-dist/src/main/ozone/README.txt" README.txt
#Copy docker compose files
run cp -p -r "${ROOT}/hadoop-dist/src/main/compose" .
@@ -169,5 +176,5 @@ mkdir -p ./share/hadoop/mapreduce
mkdir -p ./share/hadoop/yarn
mkdir -p ./share/hadoop/hdfs
echo
-echo "Hadoop Ozone dist layout available at: ${BASEDIR}/ozone"
+echo "Hadoop Ozone dist layout available at: ${BASEDIR}/ozone-${HDDS_VERSION}"
echo
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d89c3e7/hadoop-assemblies/src/main/resources/assemblies/hadoop-src-with-hdds.xml
--
diff --git
a/hadoop-assemblies/src/main/resources/assemblies/hadoop-src-with-hdds.xml
b/hadoop-assemblies/src/main/resources/assemblies/hadoop-src-with-hdds.xml
new file mode 100644
index 000..b1e039f
--- /dev/null
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-src-with-hdds.xml
@@ -0,0 +1,56 @@
+
+http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3
http://maven.apache.org/xsd/assembly-1.1.3.xsd";>
+ hadoop-src
+
+tar.gz
+
+ true
+
+
+ .
+
+LICENCE.txt
+README.txt
+NOTICE.txt
+
+
+
+ .
+ true
+
+.git/**
+**/.gitignore
+**/.svn
+**/*.iws
+**/*.ipr
+**/*.iml
+**/.classpath
+**/.project
+**/.settings
+**/target/**
+
+**/*.log
+**/build/**
+**/file:/**
+**/SecurityAuth.audit*
+
+
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d89c3e7/hadoop-assemblies/src/main/resources/assemblies/hadoop-src-with-hdsl.xml
--
diff --git
a/hadoop-assemblies/src/main/resources/assemblies/hadoop-src-with-hdsl.xml
b/hadoop-assemblies/src/main/resources/assemblies/hadoop-src-with-hdsl.xml
deleted file mode 100644
index b1e039f..000
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-src-with-hdsl.xml
+++ /dev/null
@@ -1,56 +0,0 @@
-
-http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3";
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
-
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3
http://maven.apache.org/xsd/assembly-1.1.3.xsd";>
- hadoop-src
-
-tar.gz
-
- true
-
-
- .
-
-LICENCE.txt
-README.txt
-NOTICE.txt
-
-
-
- .
- true
-
-.git/**
-**/.gitignore
-**/.svn
-**/*.iws
-**/*.ipr
-**/*.iml
-**/.classpath
-
[18/50] [abbrv] hadoop git commit: YARN-8045. Reduce log output from container status calls. Contributed by Craig Condit
YARN-8045. Reduce log output from container status calls. Contributed by Craig
Condit
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/144a55f0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/144a55f0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/144a55f0
Branch: refs/heads/HDFS-12943
Commit: 144a55f0e3ba302327baf2e98d1e07b953dcbbfd
Parents: 78902f0
Author: Shane Kumpf
Authored: Fri Sep 14 10:41:55 2018 -0600
Committer: Shane Kumpf
Committed: Fri Sep 14 10:41:55 2018 -0600
--
.../containermanager/ContainerManagerImpl.java | 29 +++-
1 file changed, 28 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/144a55f0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index b89e2dd..27a7c80 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -1474,10 +1474,37 @@ public class ContainerManagerImpl extends
CompositeService implements
}
}
ContainerStatus containerStatus = container.cloneAndGetContainerStatus();
-LOG.info("Returning " + containerStatus);
+logContainerStatus("Returning ", containerStatus);
return containerStatus;
}
+ private void logContainerStatus(String prefix, ContainerStatus status) {
+StringBuilder sb = new StringBuilder();
+sb.append(prefix);
+sb.append("ContainerStatus: [");
+sb.append("ContainerId: ");
+sb.append(status.getContainerId()).append(", ");
+sb.append("ExecutionType: ");
+sb.append(status.getExecutionType()).append(", ");
+sb.append("State: ");
+sb.append(status.getState()).append(", ");
+sb.append("Capability: ");
+sb.append(status.getCapability()).append(", ");
+sb.append("Diagnostics: ");
+sb.append(LOG.isDebugEnabled() ? status.getDiagnostics() : "...");
+sb.append(", ");
+sb.append("ExitStatus: ");
+sb.append(status.getExitStatus()).append(", ");
+sb.append("IP: ");
+sb.append(status.getIPs()).append(", ");
+sb.append("Host: ");
+sb.append(status.getHost()).append(", ");
+sb.append("ContainerSubState: ");
+sb.append(status.getContainerSubState());
+sb.append("]");
+LOG.info(sb.toString());
+ }
+
@Private
@VisibleForTesting
protected void authorizeGetAndStopContainerRequest(ContainerId containerId,
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[06/50] [abbrv] hadoop git commit: YARN-8680. YARN NM: Implement Iterable Abstraction for LocalResourceTracker state. Contributed by Pradeep Ambati
YARN-8680. YARN NM: Implement Iterable Abstraction for LocalResourceTracker
state. Contributed by Pradeep Ambati
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/250b5001
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/250b5001
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/250b5001
Branch: refs/heads/HDFS-12943
Commit: 250b50018e8c94d8ca83ff981b01f26bb68c0842
Parents: e1b242a
Author: Jason Lowe
Authored: Thu Sep 13 13:28:54 2018 -0500
Committer: Jason Lowe
Committed: Thu Sep 13 13:28:54 2018 -0500
--
.../localizer/ResourceLocalizationService.java | 87 +++---
.../recovery/NMLeveldbStateStoreService.java| 173 +++-
.../recovery/NMStateStoreService.java | 29 +-
.../recovery/NMMemoryStateStoreService.java | 18 +-
.../TestNMLeveldbStateStoreService.java | 269 ---
5 files changed, 418 insertions(+), 158 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/250b5001/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index d9b887f..71f48ac 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -308,63 +308,66 @@ public class ResourceLocalizationService extends
CompositeService
String user = userEntry.getKey();
RecoveredUserResources userResources = userEntry.getValue();
trackerState = userResources.getPrivateTrackerState();
-if (!trackerState.isEmpty()) {
- LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
- null, dispatcher, true, super.getConfig(), stateStore,
- dirsHandler);
- LocalResourcesTracker oldTracker = privateRsrc.putIfAbsent(user,
- tracker);
- if (oldTracker != null) {
-tracker = oldTracker;
- }
- recoverTrackerResources(tracker, trackerState);
+LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
+null, dispatcher, true, super.getConfig(), stateStore,
+dirsHandler);
+LocalResourcesTracker oldTracker = privateRsrc.putIfAbsent(user,
+tracker);
+if (oldTracker != null) {
+ tracker = oldTracker;
}
+recoverTrackerResources(tracker, trackerState);
for (Map.Entry appEntry :
userResources.getAppTrackerStates().entrySet()) {
trackerState = appEntry.getValue();
- if (!trackerState.isEmpty()) {
-ApplicationId appId = appEntry.getKey();
-String appIdStr = appId.toString();
-LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
-appId, dispatcher, false, super.getConfig(), stateStore,
-dirsHandler);
-LocalResourcesTracker oldTracker = appRsrc.putIfAbsent(appIdStr,
-tracker);
-if (oldTracker != null) {
- tracker = oldTracker;
-}
-recoverTrackerResources(tracker, trackerState);
+ ApplicationId appId = appEntry.getKey();
+ String appIdStr = appId.toString();
+ LocalResourcesTracker tracker1 = new LocalResourcesTrackerImpl(user,
+ appId, dispatcher, false, super.getConfig(), stateStore,
+ dirsHandler);
+ LocalResourcesTracker oldTracker1 = appRsrc.putIfAbsent(appIdStr,
+ tracker1);
+ if (oldTracker1 != null) {
+tracker1 = oldTracker1;
}
+ recoverTrackerResources(tracker1, trackerState);
}
}
}
}
private void recoverTrackerResources(LocalResourcesTracker tracker,
- LocalResourceTrackerState state) throws URISyntaxException {
-for (LocalizedResourceProto proto : state.getLocalizedResources()) {
- LocalResource rsrc = new LocalResourcePBI
[12/50] [abbrv] hadoop git commit: HADOOP-15733. Correct the log when Invalid emptier Interval configured. Contributed by Ayush Saxena
HADOOP-15733. Correct the log when Invalid emptier Interval configured.
Contributed by Ayush Saxena
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ef5c776a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ef5c776a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ef5c776a
Branch: refs/heads/HDFS-12943
Commit: ef5c776a42697fb3958a77646b759e2a6edfe48e
Parents: 291dcf2
Author: Brahma Reddy Battula
Authored: Fri Sep 14 07:32:27 2018 +0530
Committer: Brahma Reddy Battula
Committed: Fri Sep 14 07:32:27 2018 +0530
--
.../src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef5c776a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
index 9c6a685..6e101a2 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
@@ -249,7 +249,7 @@ public class TrashPolicyDefault extends TrashPolicy {
LOG.info("Namenode trash configuration: Deletion interval = "
+ (deletionInterval / MSECS_PER_MINUTE)
+ " minutes, Emptier interval = "
- + (emptierInterval / MSECS_PER_MINUTE) + " minutes.");
+ + (this.emptierInterval / MSECS_PER_MINUTE) + " minutes.");
}
@Override
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[34/50] [abbrv] hadoop git commit: YARN-8715. Make allocation tags in the placement spec optional for node-attributes. Contributed by Weiwei Yang.
YARN-8715. Make allocation tags in the placement spec optional for
node-attributes. Contributed by Weiwei Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/33d8327c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/33d8327c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/33d8327c
Branch: refs/heads/HDFS-12943
Commit: 33d8327cffdc483b538aec3022fd8730b85babdb
Parents: 95231f1
Author: Sunil G
Authored: Mon Sep 17 10:07:45 2018 +0530
Committer: Sunil G
Committed: Mon Sep 17 10:07:45 2018 +0530
--
.../constraint/PlacementConstraintParser.java | 44 ++--
.../resource/TestPlacementConstraintParser.java | 22 ++
.../distributedshell/ApplicationMaster.java | 20 +++--
3 files changed, 78 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33d8327c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/constraint/PlacementConstraintParser.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/constraint/PlacementConstraintParser.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/constraint/PlacementConstraintParser.java
index 93fd706..de9419a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/constraint/PlacementConstraintParser.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/constraint/PlacementConstraintParser.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.yarn.util.constraint;
+import com.google.common.base.Strings;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.yarn.api.records.NodeAttributeOpCode;
@@ -589,6 +590,14 @@ public final class PlacementConstraintParser {
this.num = number;
}
+public static SourceTags emptySourceTags() {
+ return new SourceTags("", 0);
+}
+
+public boolean isEmpty() {
+ return Strings.isNullOrEmpty(tag) && num == 0;
+}
+
public String getTag() {
return this.tag;
}
@@ -692,20 +701,47 @@ public final class PlacementConstraintParser {
// foo=4,Pn
String[] splitted = specStr.split(
String.valueOf(EXPRESSION_VAL_DELIM), 2);
- if (splitted.length != 2) {
+ final SourceTags st;
+ final String exprs;
+ if (splitted.length == 1) {
+// source tags not specified
+exprs = splitted[0];
+st = SourceTags.emptySourceTags();
+ } else if (splitted.length == 2) {
+exprs = splitted[1];
+String tagAlloc = splitted[0];
+st = SourceTags.parseFrom(tagAlloc);
+ } else {
throw new PlacementConstraintParseException(
"Unexpected placement constraint expression " + specStr);
}
- String tagAlloc = splitted[0];
- SourceTags st = SourceTags.parseFrom(tagAlloc);
- String exprs = splitted[1];
AbstractConstraint constraint =
PlacementConstraintParser.parseExpression(exprs);
result.put(st, constraint.build());
}
+// Validation
+Set sourceTagSet = result.keySet();
+if (sourceTagSet.stream()
+.filter(sourceTags -> sourceTags.isEmpty())
+.findAny()
+.isPresent()) {
+ // Source tags, e.g foo=3, is optional for a node-attribute constraint,
+ // but when source tags is absent, the parser only accept single
+ // constraint expression to avoid ambiguous semantic. This is because
+ // DS AM is requesting number of containers per the number specified
+ // in the source tags, we do overwrite when there is no source tags
+ // with num_containers argument from commandline. If that is partially
+ // missed in the constraints, we don't know if it is ought to
+ // overwritten or not.
+ if (result.size() != 1) {
+throw new PlacementConstraintParseException(
+"Source allocation tags is required for a multi placement"
++ " constraint expression.");
+ }
+}
return result;
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33d8327c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintParser.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintParser.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api
[22/50] [abbrv] hadoop git commit: HDDS-454. TestChunkStreams#testErrorReadGroupInputStream & TestChunkStreams#testReadGroupInputStream are failing. Contributed by chencan.
HDDS-454. TestChunkStreams#testErrorReadGroupInputStream &
TestChunkStreams#testReadGroupInputStream are failing. Contributed by chencan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5470de42
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5470de42
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5470de42
Branch: refs/heads/HDFS-12943
Commit: 5470de420b2660e3042a5fec566ecc6b0d8ff93a
Parents: c1df308
Author: Nanda kumar
Authored: Sat Sep 15 01:38:13 2018 +0530
Committer: Nanda kumar
Committed: Sat Sep 15 01:38:57 2018 +0530
--
.../org/apache/hadoop/ozone/om/TestChunkStreams.java | 14 ++
1 file changed, 10 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5470de42/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestChunkStreams.java
--
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestChunkStreams.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestChunkStreams.java
index 7ce916a..7ff9d63 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestChunkStreams.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestChunkStreams.java
@@ -121,6 +121,7 @@ public class TestChunkStreams {
int tempOffset = offset;
ChunkInputStream in =
new ChunkInputStream(null, null, null, new ArrayList<>(), null) {
+ private long pos = 0;
private ByteArrayInputStream in =
new ByteArrayInputStream(buf, tempOffset, 100);
@@ -131,7 +132,7 @@ public class TestChunkStreams {
@Override
public long getPos() throws IOException {
-throw new UnsupportedOperationException();
+return pos;
}
@Override
@@ -147,7 +148,9 @@ public class TestChunkStreams {
@Override
public int read(byte[] b, int off, int len) throws IOException {
-return in.read(b, off, len);
+int readLen = in.read(b, off, len);
+pos += readLen;
+return readLen;
}
};
inputStreams.add(in);
@@ -175,6 +178,7 @@ public class TestChunkStreams {
int tempOffset = offset;
ChunkInputStream in =
new ChunkInputStream(null, null, null, new ArrayList<>(), null) {
+ private long pos = 0;
private ByteArrayInputStream in =
new ByteArrayInputStream(buf, tempOffset, 100);
@@ -185,7 +189,7 @@ public class TestChunkStreams {
@Override
public long getPos() throws IOException {
-throw new UnsupportedOperationException();
+return pos;
}
@Override
@@ -201,7 +205,9 @@ public class TestChunkStreams {
@Override
public int read(byte[] b, int off, int len) throws IOException {
-return in.read(b, off, len);
+int readLen = in.read(b, off, len);
+pos += readLen;
+return readLen;
}
};
inputStreams.add(in);
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[11/50] [abbrv] hadoop git commit: HDDS-423. Introduce an ozone specific log4j.properties. Contributed by Elek, Marton.
HDDS-423. Introduce an ozone specific log4j.properties. Contributed by Elek,
Marton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/291dcf22
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/291dcf22
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/291dcf22
Branch: refs/heads/HDFS-12943
Commit: 291dcf2247f8f5f92a3290734a2ed081a99ccbc3
Parents: a1de8cb
Author: Hanisha Koneru
Authored: Thu Sep 13 16:37:44 2018 -0700
Committer: Hanisha Koneru
Committed: Thu Sep 13 16:37:44 2018 -0700
--
.../src/main/conf/log4j.properties | 23 ---
.../src/main/compose/ozone/docker-config| 3 +
.../src/main/compose/ozoneperf/docker-config| 2 +
.../common/src/main/conf/log4j.properties | 156 +++
.../src/test/acceptance/basic/docker-config | 3 +-
.../src/test/acceptance/ozonefs/docker-config | 3 +-
6 files changed, 165 insertions(+), 25 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/291dcf22/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
--
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
index 5783013..aeae2b8 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
+++ b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
@@ -305,29 +305,6 @@
log4j.appender.FSLOGGER.MaxFileSize=${hadoop.log.maxfilesize}
log4j.appender.FSLOGGER.MaxBackupIndex=${hadoop.log.maxbackupindex}
#
-# Add a logger for ozone that is separate from the Datanode.
-#
-log4j.logger.org.apache.hadoop.ozone=DEBUG,OZONE,FILE
-
-# Do not log into datanode logs. Remove this line to have single log.
-log4j.additivity.org.apache.hadoop.ozone=false
-
-# For development purposes, log both to console and log file.
-log4j.appender.OZONE=org.apache.log4j.ConsoleAppender
-log4j.appender.OZONE.Threshold=info
-log4j.appender.OZONE.layout=org.apache.log4j.PatternLayout
-log4j.appender.OZONE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p \
- %X{component} %X{function} %X{resource} %X{user} %X{request} - %m%n
-
-# Real ozone logger that writes to ozone.log
-log4j.appender.FILE=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.FILE.File=${hadoop.log.dir}/ozone.log
-log4j.appender.FILE.Threshold=debug
-log4j.appender.FILE.layout=org.apache.log4j.PatternLayout
-log4j.appender.FILE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p \
-(%F:%L) %X{function} %X{resource} %X{user} %X{request} - \
-%m%n
-#
# Fair scheduler state dump
#
# Use following logger to dump the state to a separate file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/291dcf22/hadoop-dist/src/main/compose/ozone/docker-config
--
diff --git a/hadoop-dist/src/main/compose/ozone/docker-config
b/hadoop-dist/src/main/compose/ozone/docker-config
index 21127f8..0def70e 100644
--- a/hadoop-dist/src/main/compose/ozone/docker-config
+++ b/hadoop-dist/src/main/compose/ozone/docker-config
@@ -29,6 +29,9 @@ LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{-MM-dd
HH:mm:ss} %-5p %c{1}:%L - %m%n
+LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN
+
#Enable this variable to print out all hadoop rpc traffic to the stdout. See
http://byteman.jboss.org/ to define your own instrumentation.
#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
http://git-wip-us.apache.org/repos/asf/hadoop/blob/291dcf22/hadoop-dist/src/main/compose/ozoneperf/docker-config
--
diff --git a/hadoop-dist/src/main/compose/ozoneperf/docker-config
b/hadoop-dist/src/main/compose/ozoneperf/docker-config
index 2539950..309adee 100644
--- a/hadoop-dist/src/main/compose/ozoneperf/docker-config
+++ b/hadoop-dist/src/main/compose/ozoneperf/docker-config
@@ -29,4 +29,6 @@
LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{-MM-dd
HH:mm:ss} %-5p %c{1}:%L - %m%n
HADOOP_OPTS=-javaagent:/opt/jmxpromo.jar=port=0:consulHost=consul:consulMode=node
+LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+LOG4J.PROPERTIES_log4j.
[28/50] [abbrv] hadoop git commit: HDDS-449. Add a NULL check to protect DeadNodeHandler#onMessage. Contributed by LiXin Ge.
HDDS-449. Add a NULL check to protect DeadNodeHandler#onMessage. Contributed by
LiXin Ge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a65c3ea9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a65c3ea9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a65c3ea9
Branch: refs/heads/HDFS-12943
Commit: a65c3ea91cad7e8b453976bab2165ea4a3c6daf9
Parents: 985f3bf
Author: Márton Elek
Authored: Sat Sep 15 13:35:00 2018 +0200
Committer: Márton Elek
Committed: Sat Sep 15 13:35:21 2018 +0200
--
.../java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java | 5 +
1 file changed, 5 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a65c3ea9/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java
--
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java
index d694a10..7fda67d 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java
@@ -57,6 +57,11 @@ public class DeadNodeHandler implements
EventHandler {
EventPublisher publisher) {
Set containers =
node2ContainerMap.getContainers(datanodeDetails.getUuid());
+if (containers == null) {
+ LOG.info("There's no containers in dead datanode {}, no replica will be"
+ + " removed from the in-memory state.", datanodeDetails.getUuid());
+ return;
+}
LOG.info(
"Datanode {} is dead. Removing replications from the in-memory
state.",
datanodeDetails.getUuid());
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[48/50] [abbrv] hadoop git commit: Merge commit 'eca1a4bfe952fc184fe90dde50bac9b0e5293568' into HDFS-12943
Merge commit 'eca1a4bfe952fc184fe90dde50bac9b0e5293568' into HDFS-12943 # Conflicts: # hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java # hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileInputStream.java Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9ade422b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9ade422b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9ade422b Branch: refs/heads/HDFS-12943 Commit: 9ade422b3fb585fe4cbecdc241d6642dcd510e8e Parents: e97b104 eca1a4b Author: Konstantin V Shvachko Authored: Mon Sep 17 17:49:26 2018 -0700 Committer: Konstantin V Shvachko Committed: Mon Sep 17 18:50:23 2018 -0700 -- .../java/org/apache/hadoop/hdfs/DFSClient.java | 4 +-- .../org/apache/hadoop/hdfs/DFSUtilClient.java | 2 +- .../java/org/apache/hadoop/hdfs/PeerCache.java | 8 ++--- .../hdfs/client/impl/BlockReaderFactory.java| 12 +++ .../client/impl/BlockReaderLocalLegacy.java | 2 +- .../hdfs/shortcircuit/ShortCircuitCache.java| 4 +-- .../hdfs/shortcircuit/ShortCircuitReplica.java | 2 +- .../apache/hadoop/hdfs/util/IOUtilsClient.java | 3 +- .../hadoop/hdfs/nfs/nfs3/OpenFileCtx.java | 4 +-- .../hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java | 6 ++-- .../java/org/apache/hadoop/hdfs/DFSUtil.java| 7 +++-- .../org/apache/hadoop/hdfs/HdfsDtFetcher.java | 7 +++-- .../org/apache/hadoop/hdfs/NameNodeProxies.java | 7 +++-- .../apache/hadoop/hdfs/SWebHdfsDtFetcher.java | 7 +++-- .../apache/hadoop/hdfs/WebHdfsDtFetcher.java| 7 +++-- .../hadoop/hdfs/net/DomainPeerServer.java | 6 ++-- .../apache/hadoop/hdfs/net/TcpPeerServer.java | 6 ++-- .../hdfs/qjournal/client/AsyncLoggerSet.java| 6 ++-- .../qjournal/client/QuorumJournalManager.java | 6 ++-- .../qjournal/server/GetJournalEditServlet.java | 7 +++-- .../hadoop/hdfs/qjournal/server/Journal.java| 12 +++ .../hdfs/qjournal/server/JournalNode.java | 10 +++--- .../qjournal/server/JournalNodeRpcServer.java | 4 +-- .../qjournal/server/JournaledEditsCache.java| 2 +- .../token/block/BlockTokenSecretManager.java| 7 +++-- .../DelegationTokenSecretManager.java | 8 ++--- .../hadoop/hdfs/server/balancer/Balancer.java | 8 ++--- .../hadoop/hdfs/server/balancer/Dispatcher.java | 6 ++-- .../hdfs/server/balancer/NameNodeConnector.java | 7 +++-- .../AvailableSpaceBlockPlacementPolicy.java | 8 ++--- .../server/blockmanagement/DatanodeManager.java | 6 ++-- .../server/blockmanagement/HostFileManager.java | 7 +++-- .../hadoop/hdfs/server/common/JspHelper.java| 6 ++-- .../hdfs/server/common/MetricsLoggerTask.java | 6 ++-- .../apache/hadoop/hdfs/server/common/Util.java | 7 +++-- .../hdfs/server/datanode/DirectoryScanner.java | 7 +++-- .../server/datanode/ProfilingFileIoEvents.java | 7 +++-- .../server/datanode/ShortCircuitRegistry.java | 7 +++-- .../AvailableSpaceVolumeChoosingPolicy.java | 7 +++-- .../RoundRobinVolumeChoosingPolicy.java | 7 +++-- .../datanode/fsdataset/impl/BlockPoolSlice.java | 8 ++--- .../impl/FsDatasetAsyncDiskService.java | 7 +++-- .../impl/RamDiskAsyncLazyPersistService.java| 7 +++-- .../fsdataset/impl/RamDiskReplicaTracker.java | 7 +++-- .../server/datanode/web/DatanodeHttpServer.java | 6 ++-- .../web/RestCsrfPreventionFilterHandler.java| 4 +-- .../datanode/web/SimpleHttpProxyHandler.java| 4 +-- .../web/webhdfs/DataNodeUGIProvider.java| 6 ++-- .../datanode/web/webhdfs/ExceptionHandler.java | 4 +-- .../server/datanode/web/webhdfs/HdfsWriter.java | 8 ++--- .../datanode/web/webhdfs/WebHdfsHandler.java| 10 +++--- .../apache/hadoop/hdfs/server/mover/Mover.java | 12 +++ .../hadoop/hdfs/server/namenode/CachePool.java | 2 -- .../hdfs/server/namenode/CheckpointConf.java| 7 +++-- .../hdfs/server/namenode/Checkpointer.java | 8 ++--- .../ContentSummaryComputationContext.java | 8 ++--- .../hadoop/hdfs/server/namenode/DfsServlet.java | 7 +++-- .../namenode/EditLogBackupOutputStream.java | 7 +++-- .../server/namenode/EditLogFileInputStream.java | 8 ++--- .../namenode/EditLogFileOutputStream.java | 11 --- .../hdfs/server/namenode/EditsDoubleBuffer.java | 7 +++-- .../hdfs/server/namenode/FSEditLogAsync.java| 8 ++--- .../hdfs/server/namenode/FSEditLogLoader.java | 7 +++-- .../hadoop/hdfs/server/namenode/FSImage.java| 9 +++--- .../hdfs/server/namenode/FSImageFormat.java | 6 ++-- .../server/namenode/FSImageFormatPBINode.java | 7 +++-- ...FSImagePreTransactionalStorageInspector.java | 10 +++--- .../FSImageTransactionalStorageInspector.java | 6 ++-- .../hdfs/server/namenode/FSNamesyste
[09/50] [abbrv] hadoop git commit: HDDS-414. Fix sbin/stop-ozone.sh to stop Ozone daemons. Contributed by Elek, Marton.
HDDS-414. Fix sbin/stop-ozone.sh to stop Ozone daemons. Contributed by Elek, Marton. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4441fe92 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4441fe92 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4441fe92 Branch: refs/heads/HDFS-12943 Commit: 4441fe9201223d02b8aecccd36333bdccc9c0d0b Parents: 76a0fdf Author: Hanisha Koneru Authored: Thu Sep 13 13:34:22 2018 -0700 Committer: Hanisha Koneru Committed: Thu Sep 13 13:34:22 2018 -0700 -- .../compose/ozonescripts/.ssh/authorized_keys | 16 .../src/main/compose/ozonescripts/.ssh/config | 18 + .../main/compose/ozonescripts/.ssh/environment | 16 .../src/main/compose/ozonescripts/.ssh/id_rsa | 42 .../main/compose/ozonescripts/.ssh/id_rsa.pub | 16 .../src/main/compose/ozonescripts/Dockerfile| 33 +++ .../src/main/compose/ozonescripts/README.md | 38 ++ .../compose/ozonescripts/docker-compose.yaml| 42 .../src/main/compose/ozonescripts/docker-config | 37 + hadoop-dist/src/main/compose/ozonescripts/ps.sh | 17 .../src/main/compose/ozonescripts/start.sh | 24 +++ .../src/main/compose/ozonescripts/stop.sh | 17 hadoop-ozone/common/src/main/bin/start-ozone.sh | 0 hadoop-ozone/common/src/main/bin/stop-ozone.sh | 16 14 files changed, 325 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4441fe92/hadoop-dist/src/main/compose/ozonescripts/.ssh/authorized_keys -- diff --git a/hadoop-dist/src/main/compose/ozonescripts/.ssh/authorized_keys b/hadoop-dist/src/main/compose/ozonescripts/.ssh/authorized_keys new file mode 100644 index 000..ae39052 --- /dev/null +++ b/hadoop-dist/src/main/compose/ozonescripts/.ssh/authorized_keys @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +ssh-rsa B3NzaC1yc2EDAQABAAABAQDgEmLpYm4BrWtq1KG9hhZXCZgGrETntu0eNTo21U3VKc9nH9/ot7M6lAawsFcT9uXu4b58PTlnfvwH/TATlCFjC8n0Z7SOx+FU6L3Sn8URh9HaX4L0tF8u87oCAD4dBrUGhhB36eiuH9dBBWly6RKffYJvrjatbc7GxBO/e5OSUMtqk/DSVKksmBhZxutrKivCNjDish9ViGIf8b5yS/MlEGmaVKApik1fJ5iOlloM/GgpB60YV/hbqfCecbWgeiM1gK92gdOcA/Wx1C7fj8BSI5iDSE6eZeF80gM3421lvyPDWyVhFaGbka4rXBX/fb9QSRBA9RTqhRKAEmAIf49H hadoop@cdae967fa87a http://git-wip-us.apache.org/repos/asf/hadoop/blob/4441fe92/hadoop-dist/src/main/compose/ozonescripts/.ssh/config -- diff --git a/hadoop-dist/src/main/compose/ozonescripts/.ssh/config b/hadoop-dist/src/main/compose/ozonescripts/.ssh/config new file mode 100644 index 000..6506916 --- /dev/null +++ b/hadoop-dist/src/main/compose/ozonescripts/.ssh/config @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +Host * + UserKnownHostsFile /dev/null + StrictHostKeyChecking no http://git-wip-us.apache.org/repos/asf/hadoop/blob/4441fe92/hadoop-dist/src/main/compose/ozonescripts/.ssh/environment -- diff --git a/hadoop-dist/src/main/compose/ozonescripts/.ssh/environment b/hadoop-dist/src/main/compose/ozone
[31/50] [abbrv] hadoop git commit: HDDS-469. Rename 'ozone oz' to 'ozone sh'. Contributed by Arpit Agarwal.
HDDS-469. Rename 'ozone oz' to 'ozone sh'. Contributed by Arpit Agarwal.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/82fbbd54
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/82fbbd54
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/82fbbd54
Branch: refs/heads/HDFS-12943
Commit: 82fbbd546c74785bcb5317e592ab2fc1c47ef59b
Parents: 87e2c0f
Author: Arpit Agarwal
Authored: Sat Sep 15 22:12:47 2018 -0700
Committer: Arpit Agarwal
Committed: Sat Sep 15 22:12:47 2018 -0700
--
.../src/test/acceptance/basic/ozone-shell.robot | 36 ++--
.../src/test/acceptance/ozonefs/ozonefs.robot | 6 ++--
.../acceptance/ozonefs/ozonesinglenode.robot| 16 -
hadoop-ozone/common/src/main/bin/ozone | 12 +++
hadoop-ozone/docs/content/CommandShell.md | 34 +-
.../hadoop/ozone/ozShell/TestOzoneShell.java| 16 -
.../apache/hadoop/ozone/web/ozShell/Shell.java | 4 +--
7 files changed, 62 insertions(+), 62 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82fbbd54/hadoop-ozone/acceptance-test/src/test/acceptance/basic/ozone-shell.robot
--
diff --git
a/hadoop-ozone/acceptance-test/src/test/acceptance/basic/ozone-shell.robot
b/hadoop-ozone/acceptance-test/src/test/acceptance/basic/ozone-shell.robot
index cc88a8a..1bd1eb6 100644
--- a/hadoop-ozone/acceptance-test/src/test/acceptance/basic/ozone-shell.robot
+++ b/hadoop-ozone/acceptance-test/src/test/acceptance/basic/ozone-shell.robot
@@ -48,39 +48,39 @@ RpcClient without scheme
*** Keywords ***
Test ozone shell
[arguments] ${protocol} ${server} ${volume}
-${result} = Execute on datanodeozone oz volume create
${protocol}${server}/${volume} --user bilbo --quota 100TB --root
+${result} = Execute on datanodeozone sh volume create
${protocol}${server}/${volume} --user bilbo --quota 100TB --root
Should not contain ${result} Failed
Should contain ${result} Creating Volume:
${volume}
-${result} = Execute on datanodeozone oz volume list
${protocol}${server}/ --user bilbo | grep -Ev
'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] |
select(.volumeName=="${volume}")'
+${result} = Execute on datanodeozone sh volume list
${protocol}${server}/ --user bilbo | grep -Ev
'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] |
select(.volumeName=="${volume}")'
Should contain ${result} createdOn
-${result} = Execute on datanodeozone oz volume list
--user bilbo | grep -Ev 'Removed|DEBUG|ERROR|INFO|TRACE|WARN' | jq -r '.[] |
select(.volumeName=="${volume}")'
+${result} = Execute on datanodeozone sh volume list
--user bilbo | grep -Ev 'Removed|DEBUG|ERROR|INFO|TRACE|WARN' | jq -r '.[] |
select(.volumeName=="${volume}")'
Should contain ${result} createdOn
-Execute on datanodeozone oz volume update
${protocol}${server}/${volume} --user bill --quota 10TB
-${result} = Execute on datanodeozone oz volume info
${protocol}${server}/${volume} | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE'
| jq -r '. | select(.volumeName=="${volume}") | .owner | .name'
+Execute on datanodeozone sh volume update
${protocol}${server}/${volume} --user bill --quota 10TB
+${result} = Execute on datanodeozone sh volume info
${protocol}${server}/${volume} | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE'
| jq -r '. | select(.volumeName=="${volume}") | .owner | .name'
Should Be Equal ${result} bill
-${result} = Execute on datanodeozone oz volume info
${protocol}${server}/${volume} | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE'
| jq -r '. | select(.volumeName=="${volume}") | .quota | .size'
+${result} = Execute on datanodeozone sh volume info
${protocol}${server}/${volume} | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE'
| jq -r '. | select(.volumeName=="${volume}") | .quota | .size'
Should Be Equal ${result} 10
-Execute on datanodeozone oz bucket create
${protocol}${server}/${volume}/bb1
-${result} = Execute on datanodeozone oz bucket info
${protocol}${server}/${volume}/bb1 | grep -Ev
'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.bucketName=="bb1") |
.storageType'
+Execute on datanodeozone s
[46/50] [abbrv] hadoop git commit: Merge commit '9af96d4ed4b6f80d3ca53a2b003d2ef768650dd4' into HDFS-12943
Merge commit '9af96d4ed4b6f80d3ca53a2b003d2ef768650dd4' into HDFS-12943 # Conflicts: # hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1363eff6 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1363eff6 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1363eff6 Branch: refs/heads/HDFS-12943 Commit: 1363eff69c36c4f2085194b59a86370505cc00cd Parents: 94d7f90 9af96d4 Author: Konstantin V Shvachko Authored: Mon Sep 17 17:39:11 2018 -0700 Committer: Konstantin V Shvachko Committed: Mon Sep 17 17:39:11 2018 -0700 -- .../hadoop-common/src/main/bin/start-all.cmd| 104 +-- .../hadoop-common/src/main/bin/stop-all.cmd | 104 +-- .../org/apache/hadoop/http/IsActiveServlet.java | 71 + .../apache/hadoop/http/TestIsActiveServlet.java | 95 + .../router/IsRouterActiveServlet.java | 37 +++ .../federation/router/RouterHttpServer.java | 9 ++ .../src/site/markdown/HDFSRouterFederation.md | 2 +- .../hadoop-hdfs/src/main/bin/hdfs-config.cmd| 86 +++ .../hadoop-hdfs/src/main/bin/start-dfs.cmd | 82 +++ .../hadoop-hdfs/src/main/bin/stop-dfs.cmd | 82 +++ .../namenode/IsNameNodeActiveServlet.java | 33 ++ .../server/namenode/NameNodeHttpServer.java | 3 + .../markdown/HDFSHighAvailabilityWithQJM.md | 8 ++ hadoop-mapreduce-project/bin/mapred-config.cmd | 86 +++ .../hadoop-streaming/src/test/bin/cat.cmd | 36 +++ .../hadoop-streaming/src/test/bin/xargs_cat.cmd | 36 +++ .../hadoop-yarn/bin/start-yarn.cmd | 94 - .../hadoop-yarn/bin/stop-yarn.cmd | 94 - .../IsResourceManagerActiveServlet.java | 38 +++ .../server/resourcemanager/ResourceManager.java | 5 + .../resourcemanager/webapp/RMWebAppFilter.java | 3 +- .../src/site/markdown/ResourceManagerHA.md | 5 + 22 files changed, 709 insertions(+), 404 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1363eff6/hadoop-common-project/hadoop-common/src/main/bin/start-all.cmd -- diff --cc hadoop-common-project/hadoop-common/src/main/bin/start-all.cmd index 9f65b5d,9f65b5d..805cfbe --- a/hadoop-common-project/hadoop-common/src/main/bin/start-all.cmd +++ b/hadoop-common-project/hadoop-common/src/main/bin/start-all.cmd @@@ -1,52 -1,52 +1,52 @@@ --@echo off --@rem Licensed to the Apache Software Foundation (ASF) under one or more --@rem contributor license agreements. See the NOTICE file distributed with --@rem this work for additional information regarding copyright ownership. --@rem The ASF licenses this file to You under the Apache License, Version 2.0 --@rem (the "License"); you may not use this file except in compliance with --@rem the License. You may obtain a copy of the License at --@rem --@rem http://www.apache.org/licenses/LICENSE-2.0 --@rem --@rem Unless required by applicable law or agreed to in writing, software --@rem distributed under the License is distributed on an "AS IS" BASIS, --@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --@rem See the License for the specific language governing permissions and --@rem limitations under the License. -- --setlocal enabledelayedexpansion -- --@rem Start all hadoop daemons. Run this on master node. -- --echo This script is Deprecated. Instead use start-dfs.cmd and start-yarn.cmd -- --if not defined HADOOP_BIN_PATH ( -- set HADOOP_BIN_PATH=%~dp0 --) -- --if "%HADOOP_BIN_PATH:~-1%" == "\" ( -- set HADOOP_BIN_PATH=%HADOOP_BIN_PATH:~0,-1% --) -- --set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec --if not defined HADOOP_LIBEXEC_DIR ( -- set HADOOP_LIBEXEC_DIR=%DEFAULT_LIBEXEC_DIR% --) -- --call %HADOOP_LIBEXEC_DIR%\hadoop-config.cmd %* --if "%1" == "--config" ( -- shift -- shift --) -- --@rem start hdfs daemons if hdfs is present --if exist %HADOOP_HDFS_HOME%\sbin\start-dfs.cmd ( -- call %HADOOP_HDFS_HOME%\sbin\start-dfs.cmd --config %HADOOP_CONF_DIR% --) -- --@rem start yarn daemons if yarn is present --if exist %HADOOP_YARN_HOME%\sbin\start-yarn.cmd ( -- call %HADOOP_YARN_HOME%\sbin\start-yarn.cmd --config %HADOOP_CONF_DIR% --) -- --endlocal ++@echo off ++@rem Licensed to the Apache Software Foundation (ASF) under one or more ++@rem contributor license agreements. See the NOTICE file distributed with ++@rem this work for additional information regarding copyright ownership. ++@rem The ASF licenses this file to You under the Apache License, Version 2.0 ++@rem (the "License"); you may not use this file except
[39/50] [abbrv] hadoop git commit: HDDS-435. Enhance the existing ozone documentation. Contributed by Elek, Marton.
HDDS-435. Enhance the existing ozone documentation.
Contributed by Elek, Marton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8af84535
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8af84535
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8af84535
Branch: refs/heads/HDFS-12943
Commit: 8af8453589232695c01b362872c0bd83686b2184
Parents: fdf5a3f
Author: Anu Engineer
Authored: Mon Sep 17 10:46:28 2018 -0700
Committer: Anu Engineer
Committed: Mon Sep 17 10:46:28 2018 -0700
--
dev-support/bin/ozone-dist-layout-stitching | 2 +
hadoop-ozone/common/src/main/bin/ozone | 1 -
hadoop-ozone/docs/config.toml | 23 --
hadoop-ozone/docs/config.yaml | 41 +++
hadoop-ozone/docs/content/BucketCommands.md | 106 ++
hadoop-ozone/docs/content/BuildingSources.md| 37 ++
hadoop-ozone/docs/content/CommandShell.md | 232 +---
hadoop-ozone/docs/content/Concepts.md | 101 +
hadoop-ozone/docs/content/Dozone.md | 107 ++
hadoop-ozone/docs/content/Freon.md | 61 +++
hadoop-ozone/docs/content/GettingStarted.md | 369 ---
hadoop-ozone/docs/content/Hdds.md | 49 +++
hadoop-ozone/docs/content/JavaApi.md| 152
hadoop-ozone/docs/content/KeyCommands.md| 111 ++
hadoop-ozone/docs/content/Metrics.md| 170 -
hadoop-ozone/docs/content/OzoneFS.md| 64
hadoop-ozone/docs/content/OzoneManager.md | 61 +++
hadoop-ozone/docs/content/RealCluster.md| 71
hadoop-ozone/docs/content/Rest.md | 32 +-
hadoop-ozone/docs/content/RunningViaDocker.md | 70
hadoop-ozone/docs/content/RunningWithHDFS.md| 61 +++
hadoop-ozone/docs/content/SCMCLI.md | 26 ++
hadoop-ozone/docs/content/Settings.md | 139 +++
hadoop-ozone/docs/content/VolumeCommands.md | 100 +
hadoop-ozone/docs/content/_index.md | 86 +
hadoop-ozone/docs/pom.xml | 4 -
.../ozonedoc/layouts/_default/single.html | 3 +
.../ozonedoc/layouts/partials/header.html | 4 +-
.../ozonedoc/layouts/partials/sidebar.html | 8 +-
.../themes/ozonedoc/static/css/ozonedoc.css | 14 +-
30 files changed, 1493 insertions(+), 812 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8af84535/dev-support/bin/ozone-dist-layout-stitching
--
diff --git a/dev-support/bin/ozone-dist-layout-stitching
b/dev-support/bin/ozone-dist-layout-stitching
index 128ce10..b4d94b3 100755
--- a/dev-support/bin/ozone-dist-layout-stitching
+++ b/dev-support/bin/ozone-dist-layout-stitching
@@ -156,6 +156,8 @@ cp
"${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore-ser
# Optional documentation, could be missing
cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs"
./share/hadoop/ozone/webapps/ozoneManager/
cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs"
./share/hadoop/hdds/webapps/scm/
+cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./
+
rm sbin/*all.sh
rm sbin/*all.cmd
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8af84535/hadoop-ozone/common/src/main/bin/ozone
--
diff --git a/hadoop-ozone/common/src/main/bin/ozone
b/hadoop-ozone/common/src/main/bin/ozone
index 5ad6801..6bf8b01 100755
--- a/hadoop-ozone/common/src/main/bin/ozone
+++ b/hadoop-ozone/common/src/main/bin/ozone
@@ -32,7 +32,6 @@ function hadoop_usage
hadoop_add_option "--loglevel level" "set the log4j level for this command"
hadoop_add_option "--workers" "turn on worker mode"
-
hadoop_add_subcommand "classpath" client "prints the class path needed to
get the hadoop jar and the required libraries"
hadoop_add_subcommand "datanode" daemon "run a HDDS datanode"
hadoop_add_subcommand "envvars" client "display computed Hadoop environment
variables"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8af84535/hadoop-ozone/docs/config.toml
--
diff --git a/hadoop-ozone/docs/config.toml b/hadoop-ozone/docs/config.toml
deleted file mode 100644
index eed74a9..000
--- a/hadoop-ozone/docs/config.toml
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in complian
[32/50] [abbrv] hadoop git commit: HDDS-470. Ozone acceptance tests are failing. Contributed by Elek, Marton.
HDDS-470. Ozone acceptance tests are failing. Contributed by Elek, Marton.
(cherry picked from commit dca8d0c2615d142bca55d367a0bc988ce9860368)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/07385f88
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/07385f88
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/07385f88
Branch: refs/heads/HDFS-12943
Commit: 07385f886ed534aba527820c0bda4dcf410e05f6
Parents: 82fbbd5
Author: Arpit Agarwal
Authored: Sun Sep 16 14:31:09 2018 -0700
Committer: Arpit Agarwal
Committed: Sun Sep 16 14:31:29 2018 -0700
--
.../test/acceptance/ozonefs/ozonesinglenode.robot | 18 +-
1 file changed, 9 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07385f88/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/ozonesinglenode.robot
--
diff --git
a/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/ozonesinglenode.robot
b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/ozonesinglenode.robot
index b718bc9..15ad5bb 100644
---
a/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/ozonesinglenode.robot
+++
b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/ozonesinglenode.robot
@@ -14,7 +14,7 @@
# limitations under the License.
*** Settings ***
-Documentation Ozonefs Single Node Test
+Documentation Ozone Single Node Test
Library OperatingSystem
Suite Setup Startup Ozone cluster with size 1
Suite Teardown Teardown Ozone cluster
@@ -27,23 +27,23 @@ ${PROJECTDIR} ${CURDIR}/../../../../../..
*** Test Cases ***
Create volume and bucket
-Execute on datanodeozone sh -createVolume
http://ozoneManager/fstest -user bilbo -quota 100TB -root
-Execute on datanodeozone sh -createBucket
http://ozoneManager/fstest/bucket1
+Execute on datanodeozone sh volume create
http://ozoneManager/fstest --user bilbo --quota 100TB --root
+Execute on datanodeozone sh bucket create
http://ozoneManager/fstest/bucket1
Check volume from ozonefs
${result} = Execute on datanode ozone fs -ls
o3://bucket1.fstest/
Create directory from ozonefs
Execute on datanode ozone fs -mkdir
-p o3://bucket1.fstest/testdir/deep
-${result} = Execute on ozoneManager ozone sh
-listKey o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+${result} = Execute on ozoneManager ozone sh key
list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
Should contain${result}
testdir/deep
Test key handling
-Execute on datanodeozone sh -putKey
o3://ozoneManager/fstest/bucket1/key1 -file NOTICE.txt -replicationFactor 1
+Execute on datanodeozone sh key put
o3://ozoneManager/fstest/bucket1/key1 NOTICE.txt --replication ONE
Execute on datanoderm -f NOTICE.txt.1
-Execute on datanodeozone sh -getKey
o3://ozoneManager/fstest/bucket1/key1 -file NOTICE.txt.1
+Execute on datanodeozone sh key get
o3://ozoneManager/fstest/bucket1/key1 NOTICE.txt.1
Execute on datanodels -l NOTICE.txt.1
-${result} = Execute on datanodeozone sh -infoKey
o3://ozoneManager/fstest/bucket1/key1 | grep -Ev
'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.keyName=="key1")'
+${result} = Execute on datanodeozone sh key info
o3://ozoneManager/fstest/bucket1/key1 | grep -Ev
'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.keyName=="key1")'
Should contain ${result} createdOn
-${result} = Execute on datanodeozone sh -listKey
o3://ozoneManager/fstest/bucket1 | grep -Ev
'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.keyName=="key1") |
.keyName'
+${result} = Execute on datanodeozone sh key list
o3://ozoneManager/fstest/bucket1 | grep -Ev
'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.keyName=="key1") |
.keyName'
Should Be Equal ${result} key1
-Execute on datanodeozone sh -deleteKey
o3://ozoneManager/fstest/bucket1/key1 -v
+Execute on datanodeozone sh key delete
o3://ozoneManager/fstest/bucket1/key1
---
[14/50] [abbrv] hadoop git commit: YARN-8720. CapacityScheduler does not enforce max resource allocation check at queue level. Contributed by Tarun Parimi.
YARN-8720. CapacityScheduler does not enforce max resource allocation check at
queue level. Contributed by Tarun Parimi.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f1a893fd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f1a893fd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f1a893fd
Branch: refs/heads/HDFS-12943
Commit: f1a893fdbc2dbe949cae786f08bdb2651b88d673
Parents: 568ebec
Author: Weiwei Yang
Authored: Fri Sep 14 16:33:51 2018 +0800
Committer: Weiwei Yang
Committed: Fri Sep 14 16:33:51 2018 +0800
--
.../hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java | 3 ++-
.../apache/hadoop/yarn/server/resourcemanager/RMAppManager.java | 5 +++--
.../hadoop/yarn/server/resourcemanager/TestAppManager.java | 3 +++
.../hadoop/yarn/server/resourcemanager/TestClientRMService.java | 3 +++
4 files changed, 11 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1a893fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
index 4cd5925..b2c5ef3 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
@@ -231,7 +231,8 @@ final class DefaultAMSProcessor implements
ApplicationMasterServiceProcessor {
}
}
-Resource maximumCapacity = getScheduler().getMaximumResourceCapability();
+Resource maximumCapacity =
+getScheduler().getMaximumResourceCapability(app.getQueue());
// sanity check
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1a893fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index ee78c08..d0f2ce6 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -566,10 +566,11 @@ public class RMAppManager implements
EventHandler,
}
// Normalize all requests
+String queue = submissionContext.getQueue();
for (ResourceRequest amReq : amReqs) {
SchedulerUtils.normalizeAndValidateRequest(amReq,
- scheduler.getMaximumResourceCapability(),
- submissionContext.getQueue(), scheduler, isRecovery, rmContext);
+ scheduler.getMaximumResourceCapability(queue),
+ queue, scheduler, isRecovery, rmContext);
amReq.setCapability(
scheduler.getNormalizedResource(amReq.getCapability()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1a893fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index 27e87bd..0bd5372 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++
[20/50] [abbrv] hadoop git commit: HDDS-452. 'ozone scm' with incorrect argument first logs all the STARTUP_MSG. Contributed by Namit Maheshwari.
HDDS-452. 'ozone scm' with incorrect argument first logs all the STARTUP_MSG.
Contributed by Namit Maheshwari.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/446cb830
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/446cb830
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/446cb830
Branch: refs/heads/HDFS-12943
Commit: 446cb8301ebd651879dfd403d5fa342b4dfaf6e3
Parents: 0c8a43b
Author: Jitendra Pandey
Authored: Fri Sep 14 11:38:50 2018 -0700
Committer: Jitendra Pandey
Committed: Fri Sep 14 11:39:36 2018 -0700
--
.../hadoop/hdds/scm/server/StorageContainerManager.java | 8 ++--
1 file changed, 6 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/446cb830/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
--
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 28a4983..b3408a4 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -361,8 +361,6 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
hParser.printGenericCommandUsage(System.err);
System.exit(1);
}
- StringUtils.startupShutdownMessage(StorageContainerManager.class, argv,
- LOG);
StorageContainerManager scm = createSCM(hParser.getRemainingArgs(),
conf);
if (scm != null) {
scm.start();
@@ -395,9 +393,13 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
}
switch (startOpt) {
case INIT:
+ StringUtils.startupShutdownMessage(StorageContainerManager.class, argv,
+ LOG);
terminate(scmInit(conf) ? 0 : 1);
return null;
case GENCLUSTERID:
+ StringUtils.startupShutdownMessage(StorageContainerManager.class, argv,
+ LOG);
System.out.println("Generating new cluster id:");
System.out.println(StorageInfo.newClusterID());
terminate(0);
@@ -407,6 +409,8 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
terminate(0);
return null;
default:
+ StringUtils.startupShutdownMessage(StorageContainerManager.class, argv,
+ LOG);
return new StorageContainerManager(conf);
}
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[16/50] [abbrv] hadoop git commit: YARN-8706. Allow additional flag in docker inspect call. Contributed by Chandni Singh
YARN-8706. Allow additional flag in docker inspect call.
Contributed by Chandni Singh
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/99237607
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/99237607
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/99237607
Branch: refs/heads/HDFS-12943
Commit: 99237607bf73e97b06eeb3455aa1327bfab4d5d2
Parents: a12f12f
Author: Eric Yang
Authored: Fri Sep 14 11:46:59 2018 -0400
Committer: Eric Yang
Committed: Fri Sep 14 11:46:59 2018 -0400
--
.../src/main/native/container-executor/impl/utils/docker-util.c | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99237607/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
index 3884abf..ee80798 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
@@ -558,7 +558,8 @@ cleanup:
int get_docker_inspect_command(const char *command_file, const struct
configuration *conf, args *args) {
const char *valid_format_strings[] = { "{{.State.Status}}",
-
"{{range(.NetworkSettings.Networks)}}{{.IPAddress}},{{end}}{{.Config.Hostname}}"
};
+
"{{range(.NetworkSettings.Networks)}}{{.IPAddress}},{{end}}{{.Config.Hostname}}",
+ "{{.State.Status}},{{.Config.StopSignal}}"};
int ret = 0, i = 0, valid_format = 0;
char *format = NULL, *container_name = NULL, *tmp_buffer = NULL;
struct configuration command_config = {0, NULL};
@@ -578,7 +579,7 @@ int get_docker_inspect_command(const char *command_file,
const struct configurat
ret = INVALID_DOCKER_INSPECT_FORMAT;
goto free_and_exit;
}
- for (i = 0; i < 2; ++i) {
+ for (i = 0; i < 3; ++i) {
if (strcmp(format, valid_format_strings[i]) == 0) {
valid_format = 1;
break;
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[47/50] [abbrv] hadoop git commit: Merge commit 'b3161c4dd9367c68b30528a63c03756eaa32aaf9' into HDFS-12943
Merge commit 'b3161c4dd9367c68b30528a63c03756eaa32aaf9' into HDFS-12943 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e97b104b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e97b104b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e97b104b Branch: refs/heads/HDFS-12943 Commit: e97b104b7b33f2770a95aee0fbded7a3d3966873 Parents: 1363eff b3161c4 Author: Konstantin V Shvachko Authored: Mon Sep 17 17:43:13 2018 -0700 Committer: Konstantin V Shvachko Committed: Mon Sep 17 17:43:13 2018 -0700 -- .../hadoop/metrics2/annotation/Metric.java | 5 + .../metrics2/lib/MutableMetricsFactory.java | 4 + .../apache/hadoop/hdds/scm/XceiverClient.java | 8 +- .../hadoop/hdds/scm/XceiverClientGrpc.java | 9 +- .../hadoop/hdds/scm/XceiverClientRatis.java | 36 ++- .../scm/client/ContainerOperationClient.java| 2 +- .../org/apache/hadoop/hdds/HddsConfigKeys.java | 11 + .../apache/hadoop/hdds/scm/ScmConfigKeys.java | 17 +- .../hadoop/hdds/scm/XceiverClientSpi.java | 10 +- .../apache/hadoop/ozone/OzoneConfigKeys.java| 12 + .../main/java/org/apache/ratis/RatisHelper.java | 22 +- .../common/src/main/resources/ozone-default.xml | 31 ++- .../container/common/impl/HddsDispatcher.java | 6 +- .../common/statemachine/StateContext.java | 45 .../states/endpoint/HeartbeatEndpointTask.java | 28 +++ .../server/ratis/ContainerStateMachine.java | 17 +- .../server/ratis/XceiverServerRatis.java| 205 - .../container/ozoneimpl/OzoneContainer.java | 2 +- .../StorageContainerDatanodeProtocol.proto | 26 +++ .../hdds/scm/container/ContainerMapping.java| 109 +++-- .../scm/container/ContainerStateManager.java| 5 +- .../hadoop/hdds/scm/container/Mapping.java | 14 ++ .../scm/container/closer/ContainerCloser.java | 194 .../hadoop/hdds/scm/events/SCMEvents.java | 24 +- .../hadoop/hdds/scm/node/StaleNodeHandler.java | 16 +- .../hdds/scm/pipelines/Node2PipelineMap.java| 34 +-- .../pipelines/PipelineActionEventHandler.java | 60 + .../scm/pipelines/PipelineCloseHandler.java | 38 .../hdds/scm/pipelines/PipelineManager.java | 10 +- .../hdds/scm/pipelines/PipelineSelector.java| 46 ++-- .../scm/pipelines/ratis/RatisManagerImpl.java | 14 +- .../standalone/StandaloneManagerImpl.java | 7 +- .../server/SCMDatanodeHeartbeatDispatcher.java | 23 ++ .../scm/server/StorageContainerManager.java | 13 +- .../scm/container/TestContainerMapping.java | 43 .../container/closer/TestContainerCloser.java | 228 --- .../mapreduce/v2/hs/HistoryFileManager.java | 12 +- .../mapreduce/v2/hs/TestHistoryFileManager.java | 52 + .../ozone/om/helpers/OmKeyLocationInfo.java | 10 + .../hdds/scm/pipeline/TestNode2PipelineMap.java | 6 +- .../hdds/scm/pipeline/TestNodeFailure.java | 126 ++ .../hdds/scm/pipeline/TestPipelineClose.java| 6 +- .../apache/hadoop/ozone/MiniOzoneCluster.java | 15 ++ .../hadoop/ozone/MiniOzoneClusterImpl.java | 21 ++ .../transport/server/ratis/TestCSMMetrics.java | 3 +- .../container/server/TestContainerServer.java | 3 +- .../server/TestContainerStateMachine.java | 2 +- .../hadoop/ozone/om/TestOzoneManager.java | 26 ++- .../hadoop/ozone/om/VolumeManagerImpl.java | 2 +- 49 files changed, 936 insertions(+), 722 deletions(-) -- - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[03/50] [abbrv] hadoop git commit: HDDS-233. Update ozone to latest ratis snapshot build(0.3.0-50588bd-SNAPSHOT). Contributed by Shashikant Banerjee.
HDDS-233. Update ozone to latest ratis snapshot build(0.3.0-50588bd-SNAPSHOT).
Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c9e0b69a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c9e0b69a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c9e0b69a
Branch: refs/heads/HDFS-12943
Commit: c9e0b69ab3b8e70b804e325ebe8901c2be98edca
Parents: f4bda5e
Author: Mukul Kumar Singh
Authored: Thu Sep 13 19:00:55 2018 +0530
Committer: Mukul Kumar Singh
Committed: Thu Sep 13 19:01:07 2018 +0530
--
.../hadoop/hdds/scm/XceiverClientRatis.java | 97
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 6 ++
.../apache/hadoop/ozone/OzoneConfigKeys.java| 6 ++
.../main/java/org/apache/ratis/RatisHelper.java | 45 +++--
.../common/src/main/resources/ozone-default.xml | 8 ++
.../server/ratis/XceiverServerRatis.java| 24 +++--
.../apache/hadoop/ozone/RatisTestHelper.java| 13 +++
.../transport/server/ratis/TestCSMMetrics.java | 16 +---
.../container/server/TestContainerServer.java | 13 +--
hadoop-project/pom.xml | 2 +-
10 files changed, 127 insertions(+), 103 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9e0b69a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
--
diff --git
a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
index 499f94d..f0db7b5 100644
---
a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
+++
b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.hdds.scm;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.HddsUtils;
+import org.apache.hadoop.io.MultipleIOException;
+import org.apache.ratis.retry.RetryPolicy;
import org.apache.ratis.shaded.com.google.protobuf
.InvalidProtocolBufferException;
import org.apache.hadoop.conf.Configuration;
@@ -35,15 +37,17 @@ import org.apache.ratis.RatisHelper;
import org.apache.ratis.client.RaftClient;
import org.apache.ratis.protocol.RaftClientReply;
import org.apache.ratis.protocol.RaftGroup;
-import org.apache.ratis.protocol.RaftGroupId;
import org.apache.ratis.protocol.RaftPeer;
import org.apache.ratis.rpc.RpcType;
import org.apache.ratis.rpc.SupportedRpcType;
import org.apache.ratis.shaded.com.google.protobuf.ByteString;
+import org.apache.ratis.util.CheckedBiConsumer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.CompletableFuture;
@@ -65,50 +69,48 @@ public final class XceiverClientRatis extends
XceiverClientSpi {
ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT);
final int maxOutstandingRequests =
HddsClientUtils.getMaxOutstandingRequests(ozoneConf);
+final RetryPolicy retryPolicy = RatisHelper.createRetryPolicy(ozoneConf);
return new XceiverClientRatis(pipeline,
-SupportedRpcType.valueOfIgnoreCase(rpcType), maxOutstandingRequests);
+SupportedRpcType.valueOfIgnoreCase(rpcType), maxOutstandingRequests,
+retryPolicy);
}
private final Pipeline pipeline;
private final RpcType rpcType;
private final AtomicReference client = new AtomicReference<>();
private final int maxOutstandingRequests;
+ private final RetryPolicy retryPolicy;
/**
* Constructs a client.
*/
private XceiverClientRatis(Pipeline pipeline, RpcType rpcType,
- int maxOutStandingChunks) {
+ int maxOutStandingChunks, RetryPolicy retryPolicy) {
super();
this.pipeline = pipeline;
this.rpcType = rpcType;
this.maxOutstandingRequests = maxOutStandingChunks;
+this.retryPolicy = retryPolicy;
}
/**
* {@inheritDoc}
*/
- public void createPipeline()
- throws IOException {
-RaftGroupId groupId = pipeline.getId().getRaftGroupID();
-RaftGroup group = RatisHelper.newRaftGroup(groupId,
pipeline.getMachines());
-LOG.debug("initializing pipeline:{} with nodes:{}",
-pipeline.getId(), group.getPeers());
-reinitialize(pipeline.getMachines(), RatisHelper.emptyRaftGroup(), group);
+ public void createPipeline() throws IOException {
+final RaftGroup group = RatisHelper.newRaftGroup(pipeline);
+LOG.debug("creating pipeline:{} with {}", pipeline.getId(), group);
+callRatisRpc(pipeline.getMachines(),
+(raftClient, peer) -> raftCli
[04/50] [abbrv] hadoop git commit: YARN-8729. Node status updater thread could be lost after it is restarted. Contributed by Tao Yang.
YARN-8729. Node status updater thread could be lost after it is restarted.
Contributed by Tao Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/39c1ea1e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/39c1ea1e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/39c1ea1e
Branch: refs/heads/HDFS-12943
Commit: 39c1ea1ed454b6c61f0985fc951f20913ed964fb
Parents: c9e0b69
Author: Weiwei Yang
Authored: Thu Sep 13 22:21:35 2018 +0800
Committer: Weiwei Yang
Committed: Thu Sep 13 22:21:35 2018 +0800
--
.../hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39c1ea1e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index df76ed7..3bb9f92 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -326,8 +326,8 @@ public class NodeStatusUpdaterImpl extends AbstractService
implements
statusUpdater.join();
registerWithRM();
statusUpdater = new Thread(statusUpdaterRunnable, "Node Status
Updater");
-statusUpdater.start();
this.isStopped = false;
+statusUpdater.start();
LOG.info("NodeStatusUpdater thread is reRegistered and restarted");
} catch (Exception e) {
String errorMessage = "Unexpected error rebooting NodeStatusUpdater";
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[17/50] [abbrv] hadoop git commit: YARN-8748. Javadoc warnings within the nodemanager package. Contributed by Craig Condit
YARN-8748. Javadoc warnings within the nodemanager package. Contributed by
Craig Condit
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/78902f02
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/78902f02
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/78902f02
Branch: refs/heads/HDFS-12943
Commit: 78902f0250e2c6d3dea7f2b5b1fcf086a80aa727
Parents: 9923760
Author: Shane Kumpf
Authored: Fri Sep 14 10:28:36 2018 -0600
Committer: Shane Kumpf
Committed: Fri Sep 14 10:28:36 2018 -0600
--
.../server/nodemanager/ContainerExecutor.java | 3 ++-
.../TrafficControlBandwidthHandlerImpl.java | 2 +-
.../JavaSandboxLinuxContainerRuntime.java | 28 +---
3 files changed, 22 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78902f02/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index ba272e2..98cc2a4 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -53,6 +53,7 @@ import
org.apache.hadoop.yarn.exceptions.ConfigurationException;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
+import
org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.ShellScriptBuilder;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
import
org.apache.hadoop.yarn.server.nodemanager.executor.ContainerPrepareContext;
import org.apache.hadoop.yarn.server.nodemanager.util.NodeManagerHardwareUtils;
@@ -88,7 +89,7 @@ public abstract class ContainerExecutor implements
Configurable {
/**
* The relative path to which debug information will be written.
*
- * @see ContainerLaunch.ShellScriptBuilder#listDebugInformation
+ * @see ShellScriptBuilder#listDebugInformation
*/
public static final String DIRECTORY_CONTENTS = "directory.info";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78902f02/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficControlBandwidthHandlerImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficControlBandwidthHandlerImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficControlBandwidthHandlerImpl.java
index c04e935..a65de02 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficControlBandwidthHandlerImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficControlBandwidthHandlerImpl.java
@@ -241,7 +241,7 @@ public class TrafficControlBandwidthHandlerImpl
* Cleanup operations once container is completed - deletes cgroup and
* removes traffic shaping rule(s).
* @param containerId of the container that was completed.
- * @return
+ * @return null
* @throws ResourceHandlerException
*/
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78902f02/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java
[30/50] [abbrv] hadoop git commit: HDDS-465. Suppress group mapping lookup warnings for ozone. Contributed by Xiaoyu Yao.
HDDS-465. Suppress group mapping lookup warnings for ozone. Contributed by
Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/87e2c0f4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/87e2c0f4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/87e2c0f4
Branch: refs/heads/HDFS-12943
Commit: 87e2c0f4258f2e46183f796d7d904c0b27030df0
Parents: c9fa081
Author: Nanda kumar
Authored: Sat Sep 15 23:14:57 2018 +0530
Committer: Nanda kumar
Committed: Sat Sep 15 23:14:57 2018 +0530
--
hadoop-dist/src/main/compose/ozone/docker-config | 1 +
hadoop-dist/src/main/compose/ozoneperf/docker-config | 1 +
hadoop-hdds/common/src/main/conf/log4j.properties| 1 +
3 files changed, 3 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87e2c0f4/hadoop-dist/src/main/compose/ozone/docker-config
--
diff --git a/hadoop-dist/src/main/compose/ozone/docker-config
b/hadoop-dist/src/main/compose/ozone/docker-config
index 0def70e..0bf76a3 100644
--- a/hadoop-dist/src/main/compose/ozone/docker-config
+++ b/hadoop-dist/src/main/compose/ozone/docker-config
@@ -31,6 +31,7 @@
LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{-MM-dd
HH:mm:ss} %-5p %c{1}:%L - %m%n
LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN
+LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR
#Enable this variable to print out all hadoop rpc traffic to the stdout. See
http://byteman.jboss.org/ to define your own instrumentation.
#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87e2c0f4/hadoop-dist/src/main/compose/ozoneperf/docker-config
--
diff --git a/hadoop-dist/src/main/compose/ozoneperf/docker-config
b/hadoop-dist/src/main/compose/ozoneperf/docker-config
index 309adee..acfdb86 100644
--- a/hadoop-dist/src/main/compose/ozoneperf/docker-config
+++ b/hadoop-dist/src/main/compose/ozoneperf/docker-config
@@ -31,4 +31,5 @@
LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{-MM-dd HH
HADOOP_OPTS=-javaagent:/opt/jmxpromo.jar=port=0:consulHost=consul:consulMode=node
LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN
+LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87e2c0f4/hadoop-hdds/common/src/main/conf/log4j.properties
--
diff --git a/hadoop-hdds/common/src/main/conf/log4j.properties
b/hadoop-hdds/common/src/main/conf/log4j.properties
index 87c8da8..663e254 100644
--- a/hadoop-hdds/common/src/main/conf/log4j.properties
+++ b/hadoop-hdds/common/src/main/conf/log4j.properties
@@ -154,3 +154,4 @@ log4j.logger.org.apache.commons.beanutils=WARN
log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
log4j.logger.org.apache.ratis.conf.ConfUtils=WARN
+log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[13/50] [abbrv] hadoop git commit: YARN-8772. Annotation javax.annotation.Generated has moved
YARN-8772. Annotation javax.annotation.Generated has moved
Signed-off-by: Akira Ajisaka
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/568ebecd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/568ebecd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/568ebecd
Branch: refs/heads/HDFS-12943
Commit: 568ebecdf49d0919db1a8d856043c10b76326c34
Parents: ef5c776
Author: Andrew Purtell
Authored: Thu Sep 13 01:27:02 2018 +
Committer: Akira Ajisaka
Committed: Fri Sep 14 15:02:52 2018 +0900
--
.../java/org/apache/hadoop/yarn/service/api/records/Artifact.java | 1 -
.../org/apache/hadoop/yarn/service/api/records/Component.java | 1 -
.../org/apache/hadoop/yarn/service/api/records/ConfigFile.java| 1 -
.../org/apache/hadoop/yarn/service/api/records/Configuration.java | 1 -
.../org/apache/hadoop/yarn/service/api/records/Container.java | 1 -
.../java/org/apache/hadoop/yarn/service/api/records/Error.java| 1 -
.../apache/hadoop/yarn/service/api/records/KerberosPrincipal.java | 2 --
.../hadoop/yarn/service/api/records/PlacementConstraint.java | 3 ---
.../apache/hadoop/yarn/service/api/records/PlacementPolicy.java | 3 ---
.../apache/hadoop/yarn/service/api/records/PlacementScope.java| 3 ---
.../org/apache/hadoop/yarn/service/api/records/PlacementType.java | 3 ---
.../apache/hadoop/yarn/service/api/records/ReadinessCheck.java| 1 -
.../java/org/apache/hadoop/yarn/service/api/records/Resource.java | 1 -
.../hadoop/yarn/service/api/records/ResourceInformation.java | 2 --
.../java/org/apache/hadoop/yarn/service/api/records/Service.java | 1 -
.../org/apache/hadoop/yarn/service/api/records/ServiceState.java | 1 -
.../org/apache/hadoop/yarn/service/api/records/ServiceStatus.java | 1 -
17 files changed, 27 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/568ebecd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
index ce062cc..bba5791 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
@@ -38,7 +38,6 @@ import org.apache.hadoop.classification.InterfaceStability;
@InterfaceAudience.Public
@InterfaceStability.Unstable
@ApiModel(description = "Artifact of an service component")
-@javax.annotation.Generated(value = "class
io.swagger.codegen.languages.JavaClientCodegen", date =
"2016-06-02T08:15:05.615-07:00")
@JsonInclude(JsonInclude.Include.NON_NULL)
public class Artifact implements Serializable {
private static final long serialVersionUID = 3608929500111099035L;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/568ebecd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java
index 0481123..9f64b29 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java
@@ -49,7 +49,6 @@ import org.apache.hadoop.classification.InterfaceStability;
@InterfaceAudience.Public
@InterfaceStability.Unstable
@ApiModel(description = "One or more components of the service. If the service
is HBase say, then the component can be a simple role like master or
regionserver. If the service is a complex business webapp then
[24/50] [abbrv] hadoop git commit: HDDS-419. ChunkInputStream bulk read api does not read from all the chunks. Contributed by Lokesh Jain and Mukul Kumar.
HDDS-419. ChunkInputStream bulk read api does not read from all the chunks.
Contributed by Lokesh Jain and Mukul Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6f037468
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6f037468
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6f037468
Branch: refs/heads/HDFS-12943
Commit: 6f037468bce7bbda6b9fc01166f2c61ae40b690b
Parents: 488806b
Author: Xiaoyu Yao
Authored: Fri Sep 14 13:34:29 2018 -0700
Committer: Xiaoyu Yao
Committed: Fri Sep 14 13:45:38 2018 -0700
--
.../hdds/scm/storage/ChunkInputStream.java | 30 +++-
.../ozone/client/io/ChunkGroupInputStream.java | 25
.../hadoop/ozone/freon/TestDataValidate.java| 4 +--
.../hadoop/ozone/freon/RandomKeyGenerator.java | 7 +++--
4 files changed, 42 insertions(+), 24 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f037468/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkInputStream.java
--
diff --git
a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkInputStream.java
b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkInputStream.java
index a969b68..a483197 100644
---
a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkInputStream.java
+++
b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkInputStream.java
@@ -121,12 +121,17 @@ public class ChunkInputStream extends InputStream
implements Seekable {
return 0;
}
checkOpen();
-int available = prepareRead(len);
-if (available == EOF) {
- return EOF;
+int total = 0;
+while (len > 0) {
+ int available = prepareRead(len);
+ if (available == EOF) {
+return total != 0 ? total : EOF;
+ }
+ buffers.get(bufferIndex).get(b, off + total, available);
+ len -= available;
+ total += available;
}
-buffers.get(bufferIndex).get(b, off, available);
-return available;
+return total;
}
@Override
@@ -196,13 +201,20 @@ public class ChunkInputStream extends InputStream
implements Seekable {
// next chunk
chunkIndex += 1;
final ReadChunkResponseProto readChunkResponse;
+final ChunkInfo chunkInfo = chunks.get(chunkIndex);
try {
- readChunkResponse = ContainerProtocolCalls.readChunk(xceiverClient,
- chunks.get(chunkIndex), blockID, traceID);
+ readChunkResponse = ContainerProtocolCalls
+ .readChunk(xceiverClient, chunkInfo, blockID, traceID);
} catch (IOException e) {
throw new IOException("Unexpected OzoneException: " + e.toString(), e);
}
ByteString byteString = readChunkResponse.getData();
+if (byteString.size() != chunkInfo.getLen()) {
+ // Bytes read from chunk should be equal to chunk size.
+ throw new IOException(String
+ .format("Inconsistent read for chunk=%s len=%d bytesRead=%d",
+ chunkInfo.getChunkName(), chunkInfo.getLen(),
byteString.size()));
+}
buffers = byteString.asReadOnlyByteBufferList();
bufferIndex = 0;
}
@@ -260,4 +272,8 @@ public class ChunkInputStream extends InputStream
implements Seekable {
public boolean seekToNewSource(long targetPos) throws IOException {
return false;
}
+
+ public BlockID getBlockID() {
+return blockID;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f037468/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java
--
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java
index 742cfcc..94966f6 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java
@@ -115,19 +115,20 @@ public class ChunkGroupInputStream extends InputStream
implements Seekable {
return totalReadLen == 0 ? EOF : totalReadLen;
}
ChunkInputStreamEntry current = streamEntries.get(currentStreamIndex);
- int readLen = Math.min(len, (int)current.getRemaining());
- int actualLen = current.read(b, off, readLen);
- // this means the underlying stream has nothing at all, return
- if (actualLen == EOF) {
-return totalReadLen > 0 ? totalReadLen : EOF;
+ int numBytesToRead = Math.min(len, (int)current.getRemaining());
+ int numBytesRead = current.read(b, off, numBytesToRead);
+ if
[10/50] [abbrv] hadoop git commit: HDFS-13838. WebHdfsFileSystem.getFileStatus() won't return correct "snapshot enabled" status. Contributed by Siyao Meng.
HDFS-13838. WebHdfsFileSystem.getFileStatus() won't return correct "snapshot
enabled" status. Contributed by Siyao Meng.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a1de8cba
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a1de8cba
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a1de8cba
Branch: refs/heads/HDFS-12943
Commit: a1de8cbac5fb9af403db2a02814575f0940d5f39
Parents: 4441fe9
Author: Wei-Chiu Chuang
Authored: Fri Sep 14 05:22:56 2018 +0800
Committer: Wei-Chiu Chuang
Committed: Fri Sep 14 05:22:56 2018 +0800
--
.../java/org/apache/hadoop/hdfs/web/JsonUtilClient.java | 4
.../java/org/apache/hadoop/hdfs/web/TestWebHDFS.java| 12
2 files changed, 16 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1de8cba/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
index 9bb1846..a685573 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
@@ -133,6 +133,7 @@ class JsonUtilClient {
Boolean aclBit = (Boolean) m.get("aclBit");
Boolean encBit = (Boolean) m.get("encBit");
Boolean erasureBit = (Boolean) m.get("ecBit");
+Boolean snapshotEnabledBit = (Boolean) m.get("snapshotEnabled");
EnumSet f =
EnumSet.noneOf(HdfsFileStatus.Flags.class);
if (aclBit != null && aclBit) {
@@ -144,6 +145,9 @@ class JsonUtilClient {
if (erasureBit != null && erasureBit) {
f.add(HdfsFileStatus.Flags.HAS_EC);
}
+if (snapshotEnabledBit != null && snapshotEnabledBit) {
+ f.add(HdfsFileStatus.Flags.SNAPSHOT_ENABLED);
+}
Map ecPolicyObj = (Map) m.get("ecPolicyObj");
ErasureCodingPolicy ecPolicy = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1de8cba/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
index bb1c398..5d33220 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
@@ -482,6 +482,9 @@ public class TestWebHDFS {
// allow snapshots on /bar using webhdfs
webHdfs.allowSnapshot(bar);
+ // check if snapshot status is enabled
+ assertTrue(dfs.getFileStatus(bar).isSnapshotEnabled());
+ assertTrue(webHdfs.getFileStatus(bar).isSnapshotEnabled());
webHdfs.createSnapshot(bar, "s1");
final Path s1path = SnapshotTestHelper.getSnapshotRoot(bar, "s1");
Assert.assertTrue(webHdfs.exists(s1path));
@@ -491,15 +494,24 @@ public class TestWebHDFS {
assertEquals(bar, snapshottableDirs[0].getFullPath());
dfs.deleteSnapshot(bar, "s1");
dfs.disallowSnapshot(bar);
+ // check if snapshot status is disabled
+ assertFalse(dfs.getFileStatus(bar).isSnapshotEnabled());
+ assertFalse(webHdfs.getFileStatus(bar).isSnapshotEnabled());
snapshottableDirs = dfs.getSnapshottableDirListing();
assertNull(snapshottableDirs);
// disallow snapshots on /bar using webhdfs
dfs.allowSnapshot(bar);
+ // check if snapshot status is enabled, again
+ assertTrue(dfs.getFileStatus(bar).isSnapshotEnabled());
+ assertTrue(webHdfs.getFileStatus(bar).isSnapshotEnabled());
snapshottableDirs = dfs.getSnapshottableDirListing();
assertEquals(1, snapshottableDirs.length);
assertEquals(bar, snapshottableDirs[0].getFullPath());
webHdfs.disallowSnapshot(bar);
+ // check if snapshot status is disabled, again
+ assertFalse(dfs.getFileStatus(bar).isSnapshotEnabled());
+ assertFalse(webHdfs.getFileStatus(bar).isSnapshotEnabled());
snapshottableDirs = dfs.getSnapshottableDirListing();
assertNull(snapshottableDirs);
try {
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[07/50] [abbrv] hadoop git commit: MAPREDUCE-7133. History Server task attempts REST API returns invalid data. Contributed by Oleksandr Shevchenko
MAPREDUCE-7133. History Server task attempts REST API returns invalid data.
Contributed by Oleksandr Shevchenko
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2886024a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2886024a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2886024a
Branch: refs/heads/HDFS-12943
Commit: 2886024ac3a8613ecc27f1595b278ce6fc2d03ba
Parents: 250b500
Author: Jason Lowe
Authored: Thu Sep 13 14:41:38 2018 -0500
Committer: Jason Lowe
Committed: Thu Sep 13 14:41:38 2018 -0500
--
.../hadoop/mapreduce/v2/app/webapp/dao/TaskAttemptsInfo.java | 8 +---
.../mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java | 3 +++
.../src/site/markdown/MapredAppMasterRest.md | 2 +-
3 files changed, 9 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2886024a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/TaskAttemptsInfo.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/TaskAttemptsInfo.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/TaskAttemptsInfo.java
index c92488f..6f188d9 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/TaskAttemptsInfo.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/TaskAttemptsInfo.java
@@ -19,12 +19,10 @@ package org.apache.hadoop.mapreduce.v2.app.webapp.dao;
import java.util.ArrayList;
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElementRef;
import javax.xml.bind.annotation.XmlRootElement;
@XmlRootElement(name = "taskAttempts")
-@XmlAccessorType(XmlAccessType.FIELD)
public class TaskAttemptsInfo {
protected ArrayList taskAttempt = new
ArrayList();
@@ -36,6 +34,10 @@ public class TaskAttemptsInfo {
taskAttempt.add(taskattemptInfo);
}
+ // XmlElementRef annotation should be used to identify the exact type of a
list element
+ // otherwise metadata will be added to XML attributes,
+ // it can lead to incorrect JSON marshaling
+ @XmlElementRef
public ArrayList getTaskAttempts() {
return taskAttempt;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2886024a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java
index d92c275..32d054f 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.mapreduce.v2.app.webapp;
import static
org.apache.hadoop.yarn.webapp.WebServicesTestUtils.assertResponseStatusCode;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -483,6 +484,8 @@ public class TestAMWebServicesAttempts extends
JerseyTestBase {
Boolean found = false;
for (int i = 0; i < nodes.getLength(); i++) {
Element element = (Element) nodes.item(i);
+assertFalse("task attempt should not contain any attributes, it can
lead to incorrect JSON marshaling",
+element.hasAttributes());
if (attid.matches(WebServicesTestUtils.getXmlString(element, "id"))) {
found = true;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2886024a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredAppMasterRest.md
--
diff --git
a
[25/50] [abbrv] hadoop git commit: HDDS-466. Handle null in argv of StorageContainerManager#createSCM. Contributed by Nanda kumar
HDDS-466. Handle null in argv of StorageContainerManager#createSCM. Contributed
by Nanda kumar
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b0a659c8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b0a659c8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b0a659c8
Branch: refs/heads/HDFS-12943
Commit: b0a659c8c0500c47f974518ec98ae44c283db9f4
Parents: 6f03746
Author: Nanda kumar
Authored: Sat Sep 15 02:34:33 2018 +0530
Committer: Nanda kumar
Committed: Sat Sep 15 02:34:33 2018 +0530
--
.../apache/hadoop/hdds/scm/server/StorageContainerManager.java| 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0a659c8/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
--
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index b3408a4..60796c7 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -376,9 +376,10 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
out.println(USAGE + "\n");
}
- public static StorageContainerManager createSCM(String[] argv,
+ public static StorageContainerManager createSCM(String[] args,
OzoneConfiguration conf)
throws IOException {
+String[] argv = (args == null) ? new String[0] : args;
if (!HddsUtils.isHddsEnabled(conf)) {
System.err.println(
"SCM cannot be started in secure mode or when " + OZONE_ENABLED + ""
+
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[23/50] [abbrv] hadoop git commit: MAPREDUCE-7140. Refactoring TaskAttemptInfo to separate Map and Reduce tasks. Contributed by Oleksandr Shevchenko
MAPREDUCE-7140. Refactoring TaskAttemptInfo to separate Map and Reduce tasks.
Contributed by Oleksandr Shevchenko
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/488806ba
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/488806ba
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/488806ba
Branch: refs/heads/HDFS-12943
Commit: 488806baca6d84c12b24532ddeacf6d249c2136b
Parents: 5470de4
Author: Jason Lowe
Authored: Fri Sep 14 15:04:18 2018 -0500
Committer: Jason Lowe
Committed: Fri Sep 14 15:10:27 2018 -0500
--
.../mapreduce/v2/app/webapp/AMWebServices.java | 9 +++--
.../v2/app/webapp/JAXBContextResolver.java | 7 ++--
.../mapreduce/v2/app/webapp/TaskPage.java | 3 +-
.../v2/app/webapp/dao/MapTaskAttemptInfo.java | 39
.../app/webapp/dao/ReduceTaskAttemptInfo.java | 11 +++---
.../v2/app/webapp/dao/TaskAttemptInfo.java | 14 +++
.../v2/app/webapp/dao/TaskAttemptsInfo.java | 10 ++---
.../mapreduce/v2/hs/webapp/HsTaskPage.java | 3 +-
.../mapreduce/v2/hs/webapp/HsTasksBlock.java| 5 ++-
.../mapreduce/v2/hs/webapp/HsWebServices.java | 9 +++--
.../v2/hs/webapp/JAXBContextResolver.java | 4 +-
11 files changed, 77 insertions(+), 37 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/488806ba/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebServices.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebServices.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebServices.java
index f477d31..fe3ace8 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebServices.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebServices.java
@@ -65,6 +65,7 @@ import
org.apache.hadoop.mapreduce.v2.app.webapp.dao.JobTaskAttemptCounterInfo;
import org.apache.hadoop.mapreduce.v2.app.webapp.dao.JobTaskAttemptState;
import org.apache.hadoop.mapreduce.v2.app.webapp.dao.JobTaskCounterInfo;
import org.apache.hadoop.mapreduce.v2.app.webapp.dao.JobsInfo;
+import org.apache.hadoop.mapreduce.v2.app.webapp.dao.MapTaskAttemptInfo;
import org.apache.hadoop.mapreduce.v2.app.webapp.dao.ReduceTaskAttemptInfo;
import org.apache.hadoop.mapreduce.v2.app.webapp.dao.TaskAttemptInfo;
import org.apache.hadoop.mapreduce.v2.app.webapp.dao.TaskAttemptsInfo;
@@ -396,9 +397,9 @@ public class AMWebServices {
for (TaskAttempt ta : task.getAttempts().values()) {
if (ta != null) {
if (task.getType() == TaskType.REDUCE) {
- attempts.add(new ReduceTaskAttemptInfo(ta, task.getType()));
+ attempts.add(new ReduceTaskAttemptInfo(ta));
} else {
- attempts.add(new TaskAttemptInfo(ta, task.getType(), true));
+ attempts.add(new MapTaskAttemptInfo(ta, true));
}
}
}
@@ -419,9 +420,9 @@ public class AMWebServices {
Task task = getTaskFromTaskIdString(tid, job);
TaskAttempt ta = getTaskAttemptFromTaskAttemptString(attId, task);
if (task.getType() == TaskType.REDUCE) {
- return new ReduceTaskAttemptInfo(ta, task.getType());
+ return new ReduceTaskAttemptInfo(ta);
} else {
- return new TaskAttemptInfo(ta, task.getType(), true);
+ return new MapTaskAttemptInfo(ta, true);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/488806ba/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JAXBContextResolver.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JAXBContextResolver.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JAXBContextResolver.java
index 88c7d86..625eb4e 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JAXBContextResolver.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JAXBContextResolver.java
@@ -42,8 +42,8 @@ import
org.apache.hadoo
[08/50] [abbrv] hadoop git commit: HDDS-456. TestOzoneShell#init is breaking due to Null Pointer Exception. Contributed by Dinesh Chitlangia.
HDDS-456. TestOzoneShell#init is breaking due to Null Pointer Exception.
Contributed by Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/76a0fdfe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/76a0fdfe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/76a0fdfe
Branch: refs/heads/HDFS-12943
Commit: 76a0fdfed9e3b972e0904503dcb0901c546c1d8c
Parents: 2886024
Author: Arpit Agarwal
Authored: Thu Sep 13 13:00:35 2018 -0700
Committer: Arpit Agarwal
Committed: Thu Sep 13 13:00:35 2018 -0700
--
.../src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java| 3 +++
1 file changed, 3 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76a0fdfe/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
--
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index b5bd361..71cc6ba 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -305,6 +305,9 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
terminate(0);
return null;
default:
+ if (argv == null) {
+argv = new String[]{};
+ }
StringUtils.startupShutdownMessage(OzoneManager.class, argv, LOG);
return new OzoneManager(conf);
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[33/50] [abbrv] hadoop git commit: HDDS-362. Modify functions impacted by SCM chill mode in ScmBlockLocationProtocol. Contributed by Ajay Kumar.
HDDS-362. Modify functions impacted by SCM chill mode in
ScmBlockLocationProtocol. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/95231f17
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/95231f17
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/95231f17
Branch: refs/heads/HDFS-12943
Commit: 95231f1749301b011fe48c9399953f774c40513d
Parents: 07385f8
Author: Xiaoyu Yao
Authored: Sun Sep 16 17:55:46 2018 -0700
Committer: Xiaoyu Yao
Committed: Sun Sep 16 17:55:46 2018 -0700
--
hadoop-hdds/common/src/main/proto/hdds.proto| 7 ++
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 49 +--
.../replication/ReplicationActivityStatus.java | 55 +---
.../hadoop/hdds/scm/events/SCMEvents.java | 2 +
.../hdds/scm/server/ChillModePrecheck.java | 54
.../apache/hadoop/hdds/scm/server/Precheck.java | 29 +++
.../hdds/scm/server/SCMChillModeManager.java| 49 ++-
.../scm/server/StorageContainerManager.java | 7 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 89 +++-
.../TestReplicationActivityStatus.java | 63 ++
10 files changed, 360 insertions(+), 44 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95231f17/hadoop-hdds/common/src/main/proto/hdds.proto
--
diff --git a/hadoop-hdds/common/src/main/proto/hdds.proto
b/hadoop-hdds/common/src/main/proto/hdds.proto
index 89c928b..41f1851 100644
--- a/hadoop-hdds/common/src/main/proto/hdds.proto
+++ b/hadoop-hdds/common/src/main/proto/hdds.proto
@@ -171,6 +171,13 @@ enum ReplicationFactor {
THREE = 3;
}
+enum ScmOps {
+allocateBlock = 1;
+keyBlocksInfoList = 2;
+getScmInfo = 3;
+deleteBlock = 4;
+}
+
/**
* Block ID that uniquely identify a block by SCM.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95231f17/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
--
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
index e4e33c7..8322b73 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
@@ -18,6 +18,7 @@ package org.apache.hadoop.hdds.scm.block;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ScmOps;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.Mapping;
import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
@@ -28,6 +29,9 @@ import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
+import org.apache.hadoop.hdds.scm.server.ChillModePrecheck;
+import org.apache.hadoop.hdds.scm.server.Precheck;
+import org.apache.hadoop.hdds.server.events.EventHandler;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hadoop.hdds.client.BlockID;
@@ -61,7 +65,8 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys
.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT;
/** Block Manager manages the block access for SCM. */
-public class BlockManagerImpl implements BlockManager, BlockmanagerMXBean {
+public class BlockManagerImpl implements EventHandler,
+BlockManager, BlockmanagerMXBean {
private static final Logger LOG =
LoggerFactory.getLogger(BlockManagerImpl.class);
// TODO : FIX ME : Hard coding the owner.
@@ -80,6 +85,7 @@ public class BlockManagerImpl implements BlockManager,
BlockmanagerMXBean {
private final int containerProvisionBatchSize;
private final Random rand;
private ObjectName mxBean;
+ private ChillModePrecheck chillModePrecheck;
/**
* Constructor.
@@ -125,6 +131,7 @@ public class BlockManagerImpl implements BlockManager,
BlockmanagerMXBean {
blockDeletingService =
new SCMBlockDeletingService(deletedBlockLog, containerManager,
nodeManager, eventPublisher, svcInterval, serviceTimeout, conf);
+chillModePrecheck = new ChillModePrecheck();
}
/**
@@ -187,19 +194,13 @@ public class BlockManagerImpl implements BlockManager,
BlockmanagerMXBean {
ReplicationType type, Repl
[15/50] [abbrv] hadoop git commit: HDDS-438. 'ozone oz' should print usage when command or sub-command is missing. Contributed by Dinesh Chitlangia.
HDDS-438. 'ozone oz' should print usage when command or sub-command is missing.
Contributed by Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a12f12f1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a12f12f1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a12f12f1
Branch: refs/heads/HDFS-12943
Commit: a12f12f1af22a28bce2d361a506394761cf53aa7
Parents: f1a893f
Author: Arpit Agarwal
Authored: Fri Sep 14 07:02:28 2018 -0700
Committer: Arpit Agarwal
Committed: Fri Sep 14 07:02:28 2018 -0700
--
.../org/apache/hadoop/hdds/cli/GenericCli.java | 5 +-
.../hdds/cli/MissingSubcommandException.java| 10 ++-
.../hadoop/ozone/ozShell/TestOzoneShell.java| 69 ++--
.../web/ozShell/bucket/BucketCommands.java | 3 +-
.../ozone/web/ozShell/keys/KeyCommands.java | 3 +-
.../web/ozShell/volume/VolumeCommands.java | 3 +-
6 files changed, 82 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a12f12f1/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/GenericCli.java
--
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/GenericCli.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/GenericCli.java
index 9a0be44..e56810c 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/GenericCli.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/GenericCli.java
@@ -67,11 +67,14 @@ public class GenericCli implements Callable,
GenericParentCommand {
} else {
System.err.println(error.getMessage().split("\n")[0]);
}
+if(error instanceof MissingSubcommandException){
+ System.err.println(((MissingSubcommandException) error).getUsage());
+}
}
@Override
public Void call() throws Exception {
-throw new MissingSubcommandException();
+throw new MissingSubcommandException(cmd.getUsageMessage());
}
public OzoneConfiguration createOzoneConfiguration() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a12f12f1/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/MissingSubcommandException.java
--
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/MissingSubcommandException.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/MissingSubcommandException.java
index bf3818f..9f0c494 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/MissingSubcommandException.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/MissingSubcommandException.java
@@ -22,8 +22,14 @@ package org.apache.hadoop.hdds.cli;
*/
public class MissingSubcommandException extends RuntimeException {
- public MissingSubcommandException() {
-super("Please select a subcommand");
+ private String usage;
+
+ public MissingSubcommandException(String usage) {
+super("Incomplete command");
+this.usage = usage;
}
+ public String getUsage() {
+return usage;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a12f12f1/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
--
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
index f872865..c80030e 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
@@ -33,6 +33,7 @@ import java.util.UUID;
import java.util.stream.Collectors;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.cli.MissingSubcommandException;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -236,10 +237,10 @@ public class TestOzoneShell {
assertEquals(userName, volumeInfo.getOwner());
}
- private void execute(Shell shell, String[] args) {
+ private void execute(Shell ozoneShell, String[] args) {
List arguments = new ArrayList(Arrays.asList(args));
LOG.info("Executing shell command with args {}", arguments);
-CommandLine cmd = shell.getCmd();
+CommandLine cmd = ozoneShell.getCmd();
IExceptionHandler2> exceptionHandler =
new IExceptionHandler2>() {
@@ -310,6 +311,29 @@ public class TestOzoneShell {
}
@Test
+ public void testShellIncompleteCom
[05/50] [abbrv] hadoop git commit: HDFS-13914. Fix DN UI logs link broken when https is enabled after HDFS-13902. Contributed by Jianfei Jiang.
HDFS-13914. Fix DN UI logs link broken when https is enabled after HDFS-13902. Contributed by Jianfei Jiang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e1b242a9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e1b242a9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e1b242a9 Branch: refs/heads/HDFS-12943 Commit: e1b242a9844d6438c93cdc2ab2443d042b4d5aea Parents: 39c1ea1 Author: Inigo Goiri Authored: Thu Sep 13 09:16:58 2018 -0700 Committer: Inigo Goiri Committed: Thu Sep 13 09:16:58 2018 -0700 -- .../hadoop-hdfs/src/main/webapps/datanode/datanode.html| 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e1b242a9/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html index ba3eadc..0fdf552 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html @@ -36,7 +36,7 @@ Utilities -Logs +Logs Metrics Configuration Process Thread Dump - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[19/50] [abbrv] hadoop git commit: HDDS-429. StorageContainerManager lock optimization. Contributed by Nanda Kumar.
HDDS-429. StorageContainerManager lock optimization.
Contributed by Nanda Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0c8a43b9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0c8a43b9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0c8a43b9
Branch: refs/heads/HDFS-12943
Commit: 0c8a43b9ec77b3ba7b2bb4c8aa863b1deba3bc7b
Parents: 144a55f
Author: Anu Engineer
Authored: Fri Sep 14 10:08:06 2018 -0700
Committer: Anu Engineer
Committed: Fri Sep 14 10:08:06 2018 -0700
--
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 229 ++-
.../scm/container/states/ContainerStateMap.java | 177 --
2 files changed, 226 insertions(+), 180 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c8a43b9/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
--
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
index 82d9a28..e4e33c7 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
@@ -16,7 +16,6 @@
*/
package org.apache.hadoop.hdds.scm.block;
-import java.util.UUID;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
@@ -45,8 +44,8 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes
.CHILL_MODE_EXCEPTION;
@@ -72,7 +71,7 @@ public class BlockManagerImpl implements BlockManager,
BlockmanagerMXBean {
private final NodeManager nodeManager;
private final Mapping containerManager;
- private final Lock lock;
+ private final ReadWriteLock lock;
private final long containerSize;
private final DeletedBlockLog deletedBlockLog;
@@ -108,7 +107,7 @@ public class BlockManagerImpl implements BlockManager,
BlockmanagerMXBean {
ScmConfigKeys.OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE,
ScmConfigKeys.OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE_DEFAULT);
rand = new Random();
-this.lock = new ReentrantLock();
+this.lock = new ReentrantReadWriteLock();
mxBean = MBeans.register("BlockManager", "BlockManagerImpl", this);
@@ -155,29 +154,22 @@ public class BlockManagerImpl implements BlockManager,
BlockmanagerMXBean {
* @param factor - how many copies needed for this container.
* @throws IOException
*/
- private void preAllocateContainers(int count, ReplicationType type,
- ReplicationFactor factor, String owner)
+ private synchronized void preAllocateContainers(int count,
+ ReplicationType type, ReplicationFactor factor, String owner)
throws IOException {
-lock.lock();
-try {
- for (int i = 0; i < count; i++) {
-ContainerWithPipeline containerWithPipeline;
-try {
- // TODO: Fix this later when Ratis is made the Default.
- containerWithPipeline = containerManager.allocateContainer(
- type, factor, owner);
+for (int i = 0; i < count; i++) {
+ ContainerWithPipeline containerWithPipeline;
+ try {
+// TODO: Fix this later when Ratis is made the Default.
+containerWithPipeline = containerManager.allocateContainer(
+type, factor, owner);
- if (containerWithPipeline == null) {
-LOG.warn("Unable to allocate container.");
-continue;
- }
-} catch (IOException ex) {
- LOG.warn("Unable to allocate container: {}", ex);
- continue;
+if (containerWithPipeline == null) {
+ LOG.warn("Unable to allocate container.");
}
+ } catch (IOException ex) {
+LOG.warn("Unable to allocate container: {}", ex);
}
-} finally {
- lock.unlock();
}
}
@@ -208,46 +200,61 @@ public class BlockManagerImpl implements BlockManager,
BlockmanagerMXBean {
CHILL_MODE_EXCEPTION);
}
-lock.lock();
-try {
- /*
- Here is the high level logic.
-
- 1. First we check if there are containers in ALLOCATED state,
- that is
-SCM has allocated them in the SCM namespace but the
-
[50/50] [abbrv] hadoop git commit: HDFS-13778. [SBN read] TestStateAlignmentContextWithHA should use real ObserverReadProxyProvider instead of AlignmentContextProxyProvider. Contributed by Konstantin
conf, uri, xface, factory);
-
- // Create but DON'T set in HAProxyFactory.
- this.alignmentContext = (spy != null ? spy : new ClientGSIContext());
-
- AC_LIST.add(alignmentContext);
+ AC_LIST.add((ClientGSIContext) getAlignmentContext());
}
}
@@ -121,23 +93,21 @@ public class TestStateAlignmentContextWithHA {
CONF.setInt(HdfsClientConfigKeys.Retry.WINDOW_BASE_KEY, 10);
CONF.setBoolean("fs.hdfs.impl.disable.cache", true);
-MiniDFSNNTopology.NSConf nsConf = new
MiniDFSNNTopology.NSConf(NAMESERVICE);
-nsConf.addNN(new MiniDFSNNTopology.NNConf("nn1"));
-nsConf.addNN(new MiniDFSNNTopology.NNConf("nn2"));
-
cluster = new MiniDFSCluster.Builder(CONF).numDataNodes(NUMDATANODES)
-
.nnTopology(MiniDFSNNTopology.simpleHATopology().addNameservice(nsConf))
+.nnTopology(MiniDFSNNTopology.simpleHATopology(3))
.build();
cluster.waitActive();
cluster.transitionToActive(0);
+cluster.transitionToObserver(2);
+
+String nameservice = HATestUtil.getLogicalHostname(cluster);
+HATestUtil.setFailoverConfigurations(cluster, CONF, nameservice, 0);
+CONF.set(HdfsClientConfigKeys.Failover.PROXY_PROVIDER_KEY_PREFIX +
+"." + nameservice, ORPPwithAlignmentContexts.class.getName());
}
@Before
public void before() throws IOException, URISyntaxException {
-killWorkers();
-HATestUtil.setFailoverConfigurations(cluster, CONF, NAMESERVICE, 0);
-CONF.set(HdfsClientConfigKeys.Failover.PROXY_PROVIDER_KEY_PREFIX +
-"." + NAMESERVICE, AlignmentContextProxyProvider.class.getName());
dfs = (DistributedFileSystem) FileSystem.get(CONF);
}
@@ -151,6 +121,7 @@ public class TestStateAlignmentContextWithHA {
@After
public void after() throws IOException {
+killWorkers();
cluster.transitionToStandby(1);
cluster.transitionToActive(0);
active = 0;
@@ -160,26 +131,6 @@ public class TestStateAlignmentContextWithHA {
dfs = null;
}
AC_LIST.clear();
-spy = null;
- }
-
- /**
- * This test checks if after a client writes we can see the state id in
- * updated via the response.
- */
- @Test
- public void testNoStateOnConfiguredProxyProvider() throws Exception {
-Configuration confCopy = new Configuration(CONF);
-confCopy.set(HdfsClientConfigKeys.Failover.PROXY_PROVIDER_KEY_PREFIX +
-"." + NAMESERVICE, SpyConfiguredContextProxyProvider.class.getName());
-
-try (DistributedFileSystem clearDfs =
- (DistributedFileSystem) FileSystem.get(confCopy)) {
- ClientGSIContext clientState = getContext(1);
- assertThat(clientState.getLastSeenStateId(), is(Long.MIN_VALUE));
- DFSTestUtil.writeFile(clearDfs, new Path("/testFileNoState"),
"no_state");
- assertThat(clientState.getLastSeenStateId(), is(Long.MIN_VALUE));
-}
}
/**
@@ -234,48 +185,6 @@ public class TestStateAlignmentContextWithHA {
}
/**
- * This test mocks an AlignmentContext and ensures that DFSClient
- * writes its lastSeenStateId into RPC requests.
- */
- @Test
- public void testClientSendsState() throws Exception {
-ClientGSIContext alignmentContext = new ClientGSIContext();
-ClientGSIContext spiedAlignContext = Mockito.spy(alignmentContext);
-spy = spiedAlignContext;
-
-try (DistributedFileSystem clearDfs =
- (DistributedFileSystem) FileSystem.get(CONF)) {
-
- // Collect RpcRequestHeaders for verification later.
- final List headers =
- new ArrayList<>();
- Mockito.doAnswer(a -> {
-Object[] arguments = a.getArguments();
-RpcHeaderProtos.RpcRequestHeaderProto.Builder header =
-(RpcHeaderProtos.RpcRequestHeaderProto.Builder) arguments[0];
-headers.add(header);
-return a.callRealMethod();
- }).when(spiedAlignContext).updateRequestState(Mockito.any());
-
- DFSTestUtil.writeFile(clearDfs, new Path("/testFile4"), "shv");
-
- // Ensure first header and last header have different state.
- assertThat(headers.size() > 1, is(true));
- assertThat(headers.get(0).getStateId(),
- is(not(headers.get(headers.size() - 1;
-
- // Ensure collected RpcRequestHeaders are in increasing order.
- long lastHeader = headers.get(0).getStateId();
- for (RpcHeaderProtos.RpcRequestHeaderProto.Builder header :
- headers.subList(1, headers.size())) {
-long currentHeader = header.getStateId();
-assertThat(currentHeader >= lastHeader, is(true));
-lastHeader = header.getStateId();
- }
-}
- }
-
- /**
* This test checks if after a client writes we can see the state id in
* updated via the response.
*/
@@ -310,14 +219,22 @@ public class TestStateAlignmentContextWithHA {
@Test(timeout=30)
public void testMultiClientStatesWithRandom
[49/50] [abbrv] hadoop git commit: Merge branch 'trunk' into HDFS-12943
Merge branch 'trunk' into HDFS-12943 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/30d1cd9f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/30d1cd9f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/30d1cd9f Branch: refs/heads/HDFS-12943 Commit: 30d1cd9f9338456f2a7cb80c9b8270ab0fe46ca1 Parents: 9ade422 0a26c52 Author: Konstantin V Shvachko Authored: Mon Sep 17 18:59:40 2018 -0700 Committer: Konstantin V Shvachko Committed: Mon Sep 17 18:59:40 2018 -0700 -- dev-support/bin/create-release |4 +- dev-support/bin/ozone-dist-layout-stitching | 27 +- dev-support/bin/yetus-wrapper |2 +- dev-support/docker/Dockerfile | 212 +- .../assemblies/hadoop-src-with-hdds.xml | 56 + .../assemblies/hadoop-src-with-hdsl.xml | 56 - hadoop-common-project/hadoop-common/pom.xml |2 + .../src/main/conf/log4j.properties | 23 - .../apache/hadoop/crypto/CryptoStreamUtils.java | 21 +- .../apache/hadoop/fs/TrashPolicyDefault.java|2 +- .../main/java/org/apache/hadoop/ha/HAAdmin.java |7 +- .../org/apache/hadoop/io/nativeio/NativeIO.java | 15 +- .../org/apache/hadoop/util/CleanerUtil.java | 199 + .../hadoop/util/curator/ZKCuratorManager.java | 10 +- .../src/main/resources/core-default.xml | 24 +- .../markdown/release/0.1.0/CHANGELOG.0.1.0.md | 101 + .../markdown/release/0.1.0/CHANGES.0.1.0.md | 101 - .../markdown/release/0.1.1/CHANGELOG.0.1.1.md | 39 + .../markdown/release/0.1.1/CHANGES.0.1.1.md | 39 - .../markdown/release/0.10.0/CHANGELOG.0.10.0.md | 101 + .../markdown/release/0.10.0/CHANGES.0.10.0.md | 101 - .../markdown/release/0.10.1/CHANGELOG.0.10.1.md | 49 + .../markdown/release/0.10.1/CHANGES.0.10.1.md | 49 - .../markdown/release/0.11.0/CHANGELOG.0.11.0.md | 96 + .../markdown/release/0.11.0/CHANGES.0.11.0.md | 96 - .../markdown/release/0.11.1/CHANGELOG.0.11.1.md | 34 + .../markdown/release/0.11.1/CHANGES.0.11.1.md | 34 - .../markdown/release/0.11.2/CHANGELOG.0.11.2.md | 33 + .../markdown/release/0.11.2/CHANGES.0.11.2.md | 33 - .../markdown/release/0.12.0/CHANGELOG.0.12.0.md | 113 + .../markdown/release/0.12.0/CHANGES.0.12.0.md | 113 - .../markdown/release/0.12.1/CHANGELOG.0.12.1.md | 59 + .../markdown/release/0.12.1/CHANGES.0.12.1.md | 59 - .../markdown/release/0.12.2/CHANGELOG.0.12.2.md | 34 + .../markdown/release/0.12.2/CHANGES.0.12.2.md | 34 - .../markdown/release/0.12.3/CHANGELOG.0.12.3.md | 38 + .../markdown/release/0.12.3/CHANGES.0.12.3.md | 38 - .../markdown/release/0.13.0/CHANGELOG.0.13.0.md | 173 + .../markdown/release/0.13.0/CHANGES.0.13.0.md | 173 - .../markdown/release/0.14.0/CHANGELOG.0.14.0.md | 214 + .../markdown/release/0.14.0/CHANGES.0.14.0.md | 214 - .../markdown/release/0.14.1/CHANGELOG.0.14.1.md | 33 + .../markdown/release/0.14.1/CHANGES.0.14.1.md | 33 - .../markdown/release/0.14.2/CHANGELOG.0.14.2.md | 40 + .../markdown/release/0.14.2/CHANGES.0.14.2.md | 40 - .../markdown/release/0.14.3/CHANGELOG.0.14.3.md | 34 + .../markdown/release/0.14.3/CHANGES.0.14.3.md | 34 - .../markdown/release/0.14.4/CHANGELOG.0.14.4.md | 39 + .../markdown/release/0.14.4/CHANGES.0.14.4.md | 39 - .../markdown/release/0.15.0/CHANGELOG.0.15.0.md | 190 + .../markdown/release/0.15.0/CHANGES.0.15.0.md | 190 - .../markdown/release/0.15.1/CHANGELOG.0.15.1.md | 49 + .../markdown/release/0.15.1/CHANGES.0.15.1.md | 49 - .../markdown/release/0.15.2/CHANGELOG.0.15.2.md | 51 + .../markdown/release/0.15.2/CHANGES.0.15.2.md | 51 - .../markdown/release/0.15.3/CHANGELOG.0.15.3.md | 35 + .../markdown/release/0.15.3/CHANGES.0.15.3.md | 35 - .../markdown/release/0.15.4/CHANGELOG.0.15.4.md | 31 + .../markdown/release/0.15.4/CHANGES.0.15.4.md | 31 - .../markdown/release/0.16.0/CHANGELOG.0.16.0.md | 225 ++ .../markdown/release/0.16.0/CHANGES.0.16.0.md | 225 -- .../markdown/release/0.16.1/CHANGELOG.0.16.1.md | 94 + .../markdown/release/0.16.1/CHANGES.0.16.1.md | 94 - .../markdown/release/0.16.2/CHANGELOG.0.16.2.md | 59 + .../markdown/release/0.16.2/CHANGES.0.16.2.md | 59 - .../markdown/release/0.16.3/CHANGELOG.0.16.3.md | 37 + .../markdown/release/0.16.3/CHANGES.0.16.3.md | 37 - .../markdown/release/0.16.4/CHANGELOG.0.16.4.md | 34 + .../markdown/release/0.16.4/CHANGES.0.16.4.md | 34 - .../markdown/release/0.17.0/CHANGELOG.0.17.0.md | 259 ++ .../markdown/release/0.17.0/CHANGES.0.17.0.md | 259 -- .../markdown/release/0.17.1/CHANGELOG.0.17.1.md | 44 + .../markdown/release/0.17.1/CHANGES.0.17.1.md | 44 - .../markdown/release/0.17.2/CHANGELOG.0.17.2.md | 43 + .../markdown/release/0.17.2/CHANGES.0.17.2.md | 43 - .../mark
[26/50] [abbrv] hadoop git commit: HDDS-446. Provide shaded artifact to start ozone service as a datanode plugin. Contributed by Elek Marton
HDDS-446. Provide shaded artifact to start ozone service as a datanode plugin.
Contributed by Elek Marton
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b95aa560
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b95aa560
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b95aa560
Branch: refs/heads/HDFS-12943
Commit: b95aa560628b527b1534a7f148bd61605deb90b3
Parents: b0a659c
Author: Bharat Viswanadham
Authored: Fri Sep 14 14:11:53 2018 -0700
Committer: Bharat Viswanadham
Committed: Fri Sep 14 14:12:03 2018 -0700
--
dev-support/bin/ozone-dist-layout-stitching | 8 +++
hadoop-dist/src/main/compose/ozone-hdfs/.env| 17 +
.../main/compose/ozone-hdfs/docker-compose.yaml | 60
.../src/main/compose/ozone-hdfs/docker-config | 75
hadoop-ozone/objectstore-service/pom.xml| 52 ++
5 files changed, 212 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b95aa560/dev-support/bin/ozone-dist-layout-stitching
--
diff --git a/dev-support/bin/ozone-dist-layout-stitching
b/dev-support/bin/ozone-dist-layout-stitching
index d91c7ef..128ce10 100755
--- a/dev-support/bin/ozone-dist-layout-stitching
+++ b/dev-support/bin/ozone-dist-layout-stitching
@@ -143,8 +143,16 @@ run copy
"${ROOT}/hadoop-ozone/ozone-manager/target/hadoop-ozone-ozone-manager-$
run copy
"${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore-service-${HDDS_VERSION}"
.
run copy
"${ROOT}/hadoop-ozone/client/target/hadoop-ozone-client-${HDDS_VERSION}" .
run copy
"${ROOT}/hadoop-ozone/tools/target/hadoop-ozone-tools-${HDDS_VERSION}" .
+
+#shaded ozonefs
mkdir -p "./share/hadoop/ozonefs"
cp
"${ROOT}/hadoop-ozone/ozonefs/target/hadoop-ozone-filesystem-${HDDS_VERSION}.jar"
"./share/hadoop/ozonefs/hadoop-ozone-filesystem.jar"
+
+#shaded datanode service
+mkdir -p "./share/hadoop/ozoneplugin"
+cp
"${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore-service-${HDDS_VERSION}-plugin.jar"
"./share/hadoop/ozoneplugin/hadoop-ozone-datanode-plugin.jar"
+
+
# Optional documentation, could be missing
cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs"
./share/hadoop/ozone/webapps/ozoneManager/
cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs"
./share/hadoop/hdds/webapps/scm/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b95aa560/hadoop-dist/src/main/compose/ozone-hdfs/.env
--
diff --git a/hadoop-dist/src/main/compose/ozone-hdfs/.env
b/hadoop-dist/src/main/compose/ozone-hdfs/.env
new file mode 100644
index 000..c437513
--- /dev/null
+++ b/hadoop-dist/src/main/compose/ozone-hdfs/.env
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HADOOP_VERSION=3.1.0
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b95aa560/hadoop-dist/src/main/compose/ozone-hdfs/docker-compose.yaml
--
diff --git a/hadoop-dist/src/main/compose/ozone-hdfs/docker-compose.yaml
b/hadoop-dist/src/main/compose/ozone-hdfs/docker-compose.yaml
new file mode 100644
index 000..ed18e5c
--- /dev/null
+++ b/hadoop-dist/src/main/compose/ozone-hdfs/docker-compose.yaml
@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF AN
[38/50] [abbrv] hadoop git commit: HDDS-435. Enhance the existing ozone documentation. Contributed by Elek, Marton.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8af84535/hadoop-ozone/docs/content/Settings.md
--
diff --git a/hadoop-ozone/docs/content/Settings.md
b/hadoop-ozone/docs/content/Settings.md
new file mode 100644
index 000..bfa644e
--- /dev/null
+++ b/hadoop-ozone/docs/content/Settings.md
@@ -0,0 +1,139 @@
+---
+title: Configuration
+weight: 1
+menu:
+ main:
+ parent: Starting
+ weight: 2
+---
+
+
+
+
+
+If you are feeling adventurous, you can setup ozone in a real cluster.
+Setting up a real cluster requires us to understand the components of Ozone.
+Ozone is designed to work concurrently with HDFS. However, Ozone is also
+capable of running independently. The components of ozone are the same in both
approaches.
+
+## Ozone Components
+
+1. Ozone Manager - Is the server that is in charge of the namespace of Ozone.
Ozone Manager is responsible for all volume, bucket and key operations.
+2. Storage Container Manager - Acts as the block manager. Ozone Manager
+requests blocks from SCM, to which clients can write data.
+3. Datanodes - Ozone data node code runs inside the HDFS datanode or in the
independent deployment case runs an ozone datanode daemon.
+
+
+
+
+## Setting up an Ozone only cluster
+
+* Please untar the ozone-0.2.1-SNAPSHOT to the directory where you are going
+to run Ozone from. We need Ozone jars on all machines in the cluster. So you
+need to do this on all machines in the cluster.
+
+* Ozone relies on a configuration file called ```ozone-site.xml```. To
+generate a template that you can replace with proper values, please run the
+following command. This will generate a template called ```ozone-site.xml``` at
+the specified path (directory).
+
+{{< highlight bash >}}
+ozone genconf -output
+{{< /highlight >}}
+
+Let us look at the settings inside the generated file (ozone-site.xml) and
+how they control ozone. Once the right values are defined, this file
+needs to be copied to ```ozone directory/etc/Hadoop```.
+
+
+* **ozone.enabled** This is the most critical setting for ozone.
+Ozone is a work in progress and users have to enable this service explicitly.
+By default, Ozone is disabled. Setting this flag to `true` enables ozone in the
+HDFS or Ozone cluster.
+
+Here is an example,
+
+{{< highlight xml >}}
+
+ ozone.enabled
+ True
+
+{{< /highlight >}}
+
+* **ozone.metadata.dirs** Allows Administrators to specify where the
+ metadata must reside. Usually you pick your fastest disk (SSD if
+ you have them on your nodes). OzoneManager, SCM and datanode will write the
+ metadata to this path. This is a required setting, if this is missing Ozone
+ will fail to come up.
+
+ Here is an example,
+
+{{< highlight xml >}}
+
+ ozone.metadata.dirs
+ /data/disk1/meta
+
+{{< /highlight >}}
+
+* **ozone.scm.names** Storage container manager(SCM) is a distributed block
+ service which is used by ozone. This property allows data nodes to discover
+ SCM's address. Data nodes send heartbeat to SCM.
+ Until HA feature is complete, we configure ozone.scm.names to be a
+ single machine.
+
+ Here is an example,
+
+ {{< highlight xml >}}
+
+ozone.scm.names
+scm.hadoop.apache.org
+
+ {{< /highlight >}}
+
+ * **ozone.scm.datanode.id** Data nodes generate a Unique ID called Datanode
+ ID. This identity is written to the file specified by this path. *Data nodes
+ will create this path if it doesn't exist already.*
+
+Here is an example,
+{{< highlight xml >}}
+
+ ozone.scm.datanode.id
+ /data/disk1/meta/node/datanode.id
+
+{{< /highlight >}}
+
+* **ozone.om.address** OM server address. This is used by OzoneClient and
+Ozone File System.
+
+Here is an example,
+{{< highlight xml >}}
+
+ ozone.om.address
+ ozonemanager.hadoop.apache.org
+
+{{< /highlight >}}
+
+
+### Ozone Settings Summary
+
+| Setting| Value| Comment |
+||--|--|
+| ozone.enabled | true | This enables
SCM and containers in HDFS cluster.|
+| ozone.metadata.dirs| file path| The metadata
will be stored here.|
+| ozone.scm.names| SCM server name |
Hostname:port or IP:port address of SCM. |
+| ozone.scm.block.client.address | SCM server name and port | Used by
services like OM |
+| ozone.scm.client.address | SCM server name and port | Used by
client-side |
+| ozone.scm.datanode.address | SCM server name and port | Used by
datanode to talk to SCM |
+| ozone.om.address
[02/50] [abbrv] hadoop git commit: YARN-8630. ATSv2 REST APIs should honor filter-entity-list-by-user in non-secure cluster when ACls are enabled. Contributed by Rohith Sharma K S.
YARN-8630. ATSv2 REST APIs should honor filter-entity-list-by-user in
non-secure cluster when ACls are enabled. Contributed by Rohith Sharma K S.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f4bda5e8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f4bda5e8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f4bda5e8
Branch: refs/heads/HDFS-12943
Commit: f4bda5e8e9fee6c5a0dda7c79ef14e73aec20e7e
Parents: e084627
Author: Sunil G
Authored: Thu Sep 13 17:47:02 2018 +0530
Committer: Sunil G
Committed: Thu Sep 13 17:47:21 2018 +0530
--
.../reader/TimelineReaderWebServices.java| 4 ++--
.../reader/TestTimelineReaderWebServicesBasicAcl.java| 11 ---
2 files changed, 10 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4bda5e8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
index b10b705..3a4ea2e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
@@ -3532,9 +3532,9 @@ public class TimelineReaderWebServices {
static boolean checkAccess(TimelineReaderManager readerManager,
UserGroupInformation ugi, String entityUser) {
if (isDisplayEntityPerUserFilterEnabled(readerManager.getConfig())) {
- if (ugi != null && !validateAuthUserWithEntityUser(readerManager, ugi,
+ if (!validateAuthUserWithEntityUser(readerManager, ugi,
entityUser)) {
-String userName = ugi.getShortUserName();
+String userName = ugi == null ? null : ugi.getShortUserName();
String msg = "User " + userName
+ " is not allowed to read TimelineService V2 data.";
throw new ForbiddenException(msg);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4bda5e8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesBasicAcl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesBasicAcl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesBasicAcl.java
index 6651457..6ad4427 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesBasicAcl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesBasicAcl.java
@@ -88,9 +88,14 @@ public class TestTimelineReaderWebServicesBasicAcl {
Assert.assertFalse(TimelineReaderWebServices
.validateAuthUserWithEntityUser(manager, null, user1));
-// true because ugi is null
-Assert.assertTrue(
-TimelineReaderWebServices.checkAccess(manager, null, user1));
+// false because ugi is null in non-secure cluster. User must pass
+// ?user.name as query params in REST end points.
+try {
+ TimelineReaderWebServices.checkAccess(manager, null, user1);
+ Assert.fail("user1Ugi is not allowed to view user1");
+} catch (ForbiddenException e) {
+ // expected
+}
// incoming ugi is admin asking for entity owner user1
Assert.assertTrue(
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[21/50] [abbrv] hadoop git commit: HDDS-389. Remove XceiverServer and XceiverClient and related classes. Contributed by chencan.
HDDS-389. Remove XceiverServer and XceiverClient and related classes.
Contributed by chencan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c1df3084
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c1df3084
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c1df3084
Branch: refs/heads/HDFS-12943
Commit: c1df3084ffd062a41f05601b928430a0b1a0db47
Parents: 446cb83
Author: Nanda kumar
Authored: Sat Sep 15 00:18:52 2018 +0530
Committer: Nanda kumar
Committed: Sat Sep 15 00:20:19 2018 +0530
--
.../apache/hadoop/hdds/scm/XceiverClient.java | 209 ---
.../hadoop/hdds/scm/XceiverClientHandler.java | 202 --
.../hdds/scm/XceiverClientInitializer.java | 74 ---
.../common/transport/server/XceiverServer.java | 140 -
.../transport/server/XceiverServerHandler.java | 82
.../server/XceiverServerInitializer.java| 64 --
.../hadoop/ozone/TestMiniOzoneCluster.java | 4 +-
.../container/metrics/TestContainerMetrics.java | 21 +-
.../container/server/TestContainerServer.java | 54 ++---
9 files changed, 36 insertions(+), 814 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1df3084/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClient.java
--
diff --git
a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClient.java
b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClient.java
deleted file mode 100644
index 5f2fe26..000
---
a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClient.java
+++ /dev/null
@@ -1,209 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.ratis.shaded.io.netty.bootstrap.Bootstrap;
-import org.apache.ratis.shaded.io.netty.channel.Channel;
-import org.apache.ratis.shaded.io.netty.channel.EventLoopGroup;
-import org.apache.ratis.shaded.io.netty.channel.nio.NioEventLoopGroup;
-import org.apache.ratis.shaded.io.netty.channel.socket.nio.NioSocketChannel;
-import org.apache.ratis.shaded.io.netty.handler.logging.LogLevel;
-import org.apache.ratis.shaded.io.netty.handler.logging.LoggingHandler;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Semaphore;
-
-/**
- * A Client for the storageContainer protocol.
- */
-public class XceiverClient extends XceiverClientSpi {
- static final Logger LOG = LoggerFactory.getLogger(XceiverClient.class);
- private final Pipeline pipeline;
- private final Configuration config;
- private Channel channel;
- private Bootstrap b;
- private EventLoopGroup group;
- private final Semaphore semaphore;
- private boolean closed = false;
-
- /**
- * Constructs a client that can communicate with the Container framework on
- * data nodes.
- *
- * @param pipeline - Pipeline that defines the machines.
- * @param config -- Ozone Config
- */
- public XceiverClient(Pipeline pipeline, Configuration config) {
-super();
-Preconditions.checkNotNull(pipeline);
-Preconditions.checkNotNull(config);
-this.pipeline = pipeline;
-this.config = config;
-this.semaphore =
-new Semaphore(HddsClientUtils.getMaxOutstandingRequests(config));
- }
-
- @Override
- public void c
[35/50] [abbrv] hadoop git commit: YARN-8782. Fix exception message in Resource.throwExceptionWhenArrayOutOfBound. Contributed by Gergely Pollak.
YARN-8782. Fix exception message in Resource.throwExceptionWhenArrayOutOfBound.
Contributed by Gergely Pollak.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9a265fa6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9a265fa6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9a265fa6
Branch: refs/heads/HDFS-12943
Commit: 9a265fa673ef1b8774cfd69c76cdd29bf344e79d
Parents: 33d8327
Author: Weiwei Yang
Authored: Mon Sep 17 22:15:24 2018 +0800
Committer: Weiwei Yang
Committed: Mon Sep 17 22:15:24 2018 +0800
--
.../src/main/java/org/apache/hadoop/yarn/api/records/Resource.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a265fa6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
index 1a7252d..7740354 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
@@ -397,7 +397,7 @@ public abstract class Resource implements
Comparable {
protected void throwExceptionWhenArrayOutOfBound(int index) {
String exceptionMsg = String.format(
"Trying to access ResourceInformation for given index=%d. "
-+ "Acceptable index range is [0,%d), please check double check "
++ "Acceptable index range is [0,%d), please double check "
+ "configured resources in resource-types.xml",
index, ResourceUtils.getNumberOfKnownResourceTypes());
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[29/50] [abbrv] hadoop git commit: HDDS-462. Optimize ContainerStateMap#getMatchingContainerIDs in SCM. Contributed by Nanda kumar.
HDDS-462. Optimize ContainerStateMap#getMatchingContainerIDs in SCM.
Contributed by Nanda kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c9fa0818
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c9fa0818
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c9fa0818
Branch: refs/heads/HDFS-12943
Commit: c9fa081897df34dba1c2989f597e67a1f384a4e3
Parents: a65c3ea
Author: Nanda kumar
Authored: Sat Sep 15 23:11:39 2018 +0530
Committer: Nanda kumar
Committed: Sat Sep 15 23:11:39 2018 +0530
--
.../hadoop/hdds/scm/container/ContainerID.java | 26 +++--
.../container/common/helpers/ContainerInfo.java | 10 +-
.../scm/container/ContainerStateManager.java| 4 +-
.../scm/container/states/ContainerQueryKey.java | 110 +++
.../scm/container/states/ContainerStateMap.java | 42 ++-
.../scm/node/states/TestNode2ContainerMap.java | 7 +-
.../genesis/BenchMarkContainerStateMap.java | 24 +++-
.../genesis/BenchMarkDatanodeDispatcher.java| 42 +++
.../apache/hadoop/ozone/genesis/Genesis.java| 9 +-
9 files changed, 224 insertions(+), 50 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9fa0818/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerID.java
--
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerID.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerID.java
index 9845c04..49af297 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerID.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerID.java
@@ -19,7 +19,9 @@
package org.apache.hadoop.hdds.scm.container;
import com.google.common.base.Preconditions;
-import org.apache.commons.math3.util.MathUtils;
+import org.apache.commons.lang3.builder.CompareToBuilder;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
/**
* Container ID is an integer that is a value between 1..MAX_CONTAINER ID.
@@ -48,7 +50,6 @@ public class ContainerID implements Comparable {
* @return ContainerID.
*/
public static ContainerID valueof(long containerID) {
-Preconditions.checkState(containerID > 0);
return new ContainerID(containerID);
}
@@ -66,28 +67,37 @@ public class ContainerID implements Comparable {
if (this == o) {
return true;
}
+
if (o == null || getClass() != o.getClass()) {
return false;
}
ContainerID that = (ContainerID) o;
-return id == that.id;
+return new EqualsBuilder()
+.append(getId(), that.getId())
+.isEquals();
}
@Override
public int hashCode() {
-return MathUtils.hash(id);
+return new HashCodeBuilder(61, 71)
+.append(getId())
+.toHashCode();
}
@Override
public int compareTo(Object o) {
Preconditions.checkNotNull(o);
-if (o instanceof ContainerID) {
- return Long.compare(((ContainerID) o).getId(), this.getId());
+if(getClass() != o.getClass()) {
+ throw new ClassCastException("ContainerID class expected. found:" +
+ o.getClass().toString());
}
-throw new IllegalArgumentException("Object O, should be an instance " +
-"of ContainerID");
+
+ContainerID that = (ContainerID) o;
+return new CompareToBuilder()
+.append(this.getId(), that.getId())
+.build();
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9fa0818/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java
--
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java
index ed0e0aa..5abcd14 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java
@@ -106,6 +106,13 @@ public class ContainerInfo implements
Comparator,
this.replicationType = repType;
}
+ public ContainerInfo(ContainerInfo info) {
+this(info.getContainerID(), info.getState(), info.getPipelineID(),
+info.getAllocatedBytes(), info.getUsedBytes(), info.getNumberOfKeys(),
+info.getStateEnterTime(), info.getOwner(),
+info.getDeleteTransactionId(), info.getReplicationFactor(),
+info.getReplicationType());
+ }
/**
[43/50] [abbrv] hadoop git commit: HDFS-13844. Fix the fmt_bytes function in the dfs-dust.js. Contributed by yanghuafeng.
HDFS-13844. Fix the fmt_bytes function in the dfs-dust.js. Contributed by
yanghuafeng.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d1541932
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d1541932
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d1541932
Branch: refs/heads/HDFS-12943
Commit: d1541932dbf2efd09da251b23c8825ce97f9c86c
Parents: 23a6137
Author: Inigo Goiri
Authored: Mon Sep 17 14:37:25 2018 -0700
Committer: Inigo Goiri
Committed: Mon Sep 17 14:42:03 2018 -0700
--
.../hadoop-hdfs/src/main/webapps/static/dfs-dust.js | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1541932/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dfs-dust.js
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dfs-dust.js
b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dfs-dust.js
index a572282..316a994 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dfs-dust.js
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dfs-dust.js
@@ -20,7 +20,7 @@
var filters = {
'fmt_bytes': function (v) {
- var UNITS = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'ZB'];
+ var UNITS = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB'];
var prev = 0, i = 0;
while (Math.floor(v) > 0 && i < UNITS.length) {
prev = v;
@@ -28,7 +28,7 @@
i += 1;
}
- if (i > 0 && i < UNITS.length) {
+ if (i > 0) {
v = prev;
i -= 1;
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[27/50] [abbrv] hadoop git commit: HDDS-409. Ozone acceptance-test and integration-test packages have undefined hadoop component. Contributed by Dinesh Chitlangia.
HDDS-409. Ozone acceptance-test and integration-test packages have undefined hadoop component. Contributed by Dinesh Chitlangia. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/985f3bf3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/985f3bf3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/985f3bf3 Branch: refs/heads/HDFS-12943 Commit: 985f3bf3fb2e9ba6cccf0420cd91cb4b9394d750 Parents: b95aa56 Author: Márton Elek Authored: Sat Sep 15 13:16:59 2018 +0200 Committer: Márton Elek Committed: Sat Sep 15 13:21:32 2018 +0200 -- hadoop-ozone/acceptance-test/pom.xml | 6 ++ hadoop-ozone/integration-test/pom.xml | 5 + 2 files changed, 11 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/985f3bf3/hadoop-ozone/acceptance-test/pom.xml -- diff --git a/hadoop-ozone/acceptance-test/pom.xml b/hadoop-ozone/acceptance-test/pom.xml index fc11c07..a60d4b0 100644 --- a/hadoop-ozone/acceptance-test/pom.xml +++ b/hadoop-ozone/acceptance-test/pom.xml @@ -27,6 +27,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> Apache Hadoop Ozone Acceptance Tests Apache Hadoop Ozone Acceptance Tests pom + + +ozone +true + + ozone-acceptance-test http://git-wip-us.apache.org/repos/asf/hadoop/blob/985f3bf3/hadoop-ozone/integration-test/pom.xml -- diff --git a/hadoop-ozone/integration-test/pom.xml b/hadoop-ozone/integration-test/pom.xml index d7a3bc0..993e91f 100644 --- a/hadoop-ozone/integration-test/pom.xml +++ b/hadoop-ozone/integration-test/pom.xml @@ -28,6 +28,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> Apache Hadoop Ozone Integration Tests jar + +ozone +true + + - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[44/50] [abbrv] hadoop git commit: HDDS-487. Doc files are missing ASF license headers. Contributed by Namit Maheshwari.
HDDS-487. Doc files are missing ASF license headers. Contributed by Namit
Maheshwari.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0a26c521
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0a26c521
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0a26c521
Branch: refs/heads/HDFS-12943
Commit: 0a26c521f0e5d3a2f7d40e07f11fe0a26765bc41
Parents: d154193
Author: Arpit Agarwal
Authored: Mon Sep 17 16:21:10 2018 -0700
Committer: Arpit Agarwal
Committed: Mon Sep 17 16:21:10 2018 -0700
--
hadoop-ozone/docs/README.md | 13 +++-
hadoop-ozone/docs/archetypes/default.md | 13 +++-
hadoop-ozone/docs/content/BucketCommands.md | 16 +++
hadoop-ozone/docs/content/BuildingSources.md| 17
hadoop-ozone/docs/content/CommandShell.md | 13 +++-
hadoop-ozone/docs/content/Concepts.md | 21 +---
hadoop-ozone/docs/content/Dozone.md | 13 +++-
hadoop-ozone/docs/content/Freon.md | 13 +++-
hadoop-ozone/docs/content/Hdds.md | 16 +++
hadoop-ozone/docs/content/JavaApi.md| 16 +++
hadoop-ozone/docs/content/KeyCommands.md| 16 +++
hadoop-ozone/docs/content/OzoneFS.md| 16 +++
hadoop-ozone/docs/content/OzoneManager.md | 16 +++
hadoop-ozone/docs/content/RealCluster.md| 13 +++-
hadoop-ozone/docs/content/Rest.md | 13 +++-
hadoop-ozone/docs/content/RunningViaDocker.md | 13 +++-
hadoop-ozone/docs/content/RunningWithHDFS.md| 16 +++
hadoop-ozone/docs/content/SCMCLI.md | 13 +++-
hadoop-ozone/docs/content/Settings.md | 13 +++-
hadoop-ozone/docs/content/VolumeCommands.md | 16 +++
hadoop-ozone/docs/content/_index.md | 13 +++-
hadoop-ozone/docs/static/NOTES.md | 13 +++-
.../ozonedoc/layouts/_default/single.html | 13 +++-
.../docs/themes/ozonedoc/layouts/index.html | 16 +++
.../ozonedoc/layouts/partials/footer.html | 13 +++-
.../ozonedoc/layouts/partials/header.html | 13 +++-
.../ozonedoc/layouts/partials/navbar.html | 13 +++-
.../ozonedoc/layouts/partials/sidebar.html | 13 +++-
28 files changed, 311 insertions(+), 92 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a26c521/hadoop-ozone/docs/README.md
--
diff --git a/hadoop-ozone/docs/README.md b/hadoop-ozone/docs/README.md
index 426789f..85817a7 100644
--- a/hadoop-ozone/docs/README.md
+++ b/hadoop-ozone/docs/README.md
@@ -1,15 +1,18 @@
# Hadoop Ozone/HDDS docs
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a26c521/hadoop-ozone/docs/archetypes/default.md
--
diff --git a/hadoop-ozone/docs/archetypes/default.md
b/hadoop-ozone/docs/archetypes/default.md
index e67e68a..f4cc999 100644
--- a/hadoop-ozone/docs/archetypes/default.md
+++ b/hadoop-ozone/docs/archetypes/default.md
@@ -3,15 +3,18 @@ title: "{{ replace .Name "-" " " | title }}"
menu: main
---
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a26c521/hadoop-ozone/docs/content/BucketCommands.md
--
diff --git a/hadoop-ozone/docs/content/BucketCommands.md
b/hadoop-ozone/docs/content/BucketCommands.md
index dad11e3..3ab3505 100644
--- a/hadoop-ozone/docs/content/BucketCommands.md
+++ b/hadoop-ozone/docs/content/BucketCommands.md
@@ -5,6 +5,22 @@ menu:
parent: Client
weight: 3
---
+
Ozone shell supports the following bucket commands.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a26c521/hadoop-ozone/docs/content/BuildingSources.md
--
diff --git a/hadoop-ozone/docs/content/BuildingSources.md
b/hadoop-ozone/docs/content/BuildingSources.md
index 2cad55b..1953f47 100644
--- a/hadoop-ozone/docs/content/BuildingSources.md
+++ b/hadoop-ozone/docs/content/BuildingSources.md
@@ -6,6 +6,23 @@ menu:
parent: Starting
weight: 5
---
+
+
***This is a guide on how to build the ozone sources. If you are not
planning to build sources yourself, you can safely skip this page.***
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a26c521/hadoop-ozone/docs/content/CommandShell.md
--
diff --git a/hadoop-ozone/docs/content/CommandShell.md
b/hadoop-ozone/docs/content/CommandShell.md
index bfb2d39..74
[36/50] [abbrv] hadoop git commit: HDDS-399. Persist open pipeline information across SCM restart. Contributed by Mukul Kumar Singh.
HDDS-399. Persist open pipeline information across SCM restart. Contributed by
Mukul Kumar Singh.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/84693669
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/84693669
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/84693669
Branch: refs/heads/HDFS-12943
Commit: 846936698b2c8c50662e43534ac999df82066a8b
Parents: 9a265fa
Author: Nanda kumar
Authored: Mon Sep 17 21:51:54 2018 +0530
Committer: Nanda kumar
Committed: Mon Sep 17 21:51:54 2018 +0530
--
.../scm/container/common/helpers/Pipeline.java | 24 ++
.../org/apache/hadoop/ozone/OzoneConsts.java| 2 +
.../hdds/scm/container/ContainerMapping.java| 24 +-
.../scm/container/ContainerStateManager.java| 25 +-
.../scm/container/states/ContainerStateMap.java | 38 ---
.../hdds/scm/pipelines/PipelineManager.java | 148 +--
.../hdds/scm/pipelines/PipelineSelector.java| 249 +--
.../scm/pipelines/PipelineStateManager.java | 136 ++
.../scm/pipelines/ratis/RatisManagerImpl.java | 8 +-
.../standalone/StandaloneManagerImpl.java | 8 +-
.../container/TestContainerReportHandler.java | 3 +-
.../container/TestContainerStateManager.java| 4 +-
.../hdds/scm/node/TestDeadNodeHandler.java | 4 +-
.../TestContainerStateManagerIntegration.java | 10 +-
.../hdds/scm/pipeline/TestNode2PipelineMap.java | 22 +-
.../hdds/scm/pipeline/TestPipelineClose.java| 15 +-
.../hdds/scm/pipeline/TestSCMRestart.java | 101
.../apache/hadoop/ozone/MiniOzoneCluster.java | 5 +-
.../hadoop/ozone/MiniOzoneClusterImpl.java | 8 +-
19 files changed, 510 insertions(+), 324 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84693669/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/Pipeline.java
--
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/Pipeline.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/Pipeline.java
index 6757262..ef148e5 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/Pipeline.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/Pipeline.java
@@ -86,6 +86,30 @@ public class Pipeline {
datanodes = new TreeMap<>();
}
+ @Override
+ public int hashCode() {
+return id.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+if (this == o) {
+ return true;
+}
+if (o == null || getClass() != o.getClass()) {
+ return false;
+}
+
+Pipeline that = (Pipeline) o;
+
+return id.equals(that.id)
+&& factor.equals(that.factor)
+&& type.equals(that.type)
+&& lifeCycleState.equals(that.lifeCycleState)
+&& leaderID.equals(that.leaderID);
+
+ }
+
/**
* Gets pipeline object from protobuf.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84693669/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
--
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index bf4508b..0a15ec8 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -90,7 +90,9 @@ public final class OzoneConsts {
* level DB names used by SCM and data nodes.
*/
public static final String CONTAINER_DB_SUFFIX = "container.db";
+ public static final String PIPELINE_DB_SUFFIX = "pipeline.db";
public static final String SCM_CONTAINER_DB = "scm-" + CONTAINER_DB_SUFFIX;
+ public static final String SCM_PIPELINE_DB = "scm-" + PIPELINE_DB_SUFFIX;
public static final String DN_CONTAINER_DB = "-dn-"+ CONTAINER_DB_SUFFIX;
public static final String DELETED_BLOCK_DB = "deletedBlock.db";
public static final String OM_DB_NAME = "om.db";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84693669/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
--
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
index 5678205..11cc9ee 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.
[45/50] [abbrv] hadoop git commit: Merge commit 'e780556ae9229fe7a90817eb4e5449d7eed35dd8' into HDFS-12943
Merge commit 'e780556ae9229fe7a90817eb4e5449d7eed35dd8' into HDFS-12943 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/94d7f90e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/94d7f90e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/94d7f90e Branch: refs/heads/HDFS-12943 Commit: 94d7f90e93beca22476504b0cdb3e6bd05f05888 Parents: c2e0e9a e780556 Author: Konstantin V Shvachko Authored: Mon Sep 17 17:26:28 2018 -0700 Committer: Konstantin V Shvachko Committed: Mon Sep 17 17:26:28 2018 -0700 -- .../org/apache/hadoop/http/HttpServer2.java | 26 +- .../main/java/org/apache/hadoop/ipc/Client.java | 14 + .../org/apache/hadoop/http/TestHttpServer.java | 14 + .../java/org/apache/hadoop/ipc/TestIPC.java | 45 ++ .../src/main/resources/kms-default.xml | 7 + .../src/main/compose/ozone/docker-config| 37 ++ .../common/dev-support/findbugsExcludeFile.xml | 4 + .../org/apache/hadoop/hdds/cli/GenericCli.java | 82 +++ .../hadoop/hdds/cli/HddsVersionProvider.java| 35 ++ .../apache/hadoop/hdds/cli/package-info.java| 22 + .../helpers/BlockNotCommittedException.java | 36 ++ .../scm/storage/ContainerProtocolCalls.java | 5 + .../apache/hadoop/ozone/OzoneConfigKeys.java| 11 + .../org/apache/hadoop/ozone/OzoneConsts.java| 39 +- .../apache/hadoop/ozone/audit/package-info.java | 6 +- .../org/apache/hadoop/utils/RocksDBStore.java | 2 +- .../org/apache/hadoop/utils/db/DBStore.java | 22 + .../org/apache/hadoop/utils/db/RDBStore.java| 26 +- .../main/proto/DatanodeContainerProtocol.proto | 12 +- .../common/src/main/resources/ozone-default.xml | 30 +- .../container/common/interfaces/Handler.java| 15 +- .../statemachine/DatanodeStateMachine.java | 25 +- .../ReplicateContainerCommandHandler.java | 124 - .../transport/server/XceiverServerGrpc.java | 12 +- .../container/keyvalue/KeyValueContainer.java | 3 +- .../container/keyvalue/KeyValueHandler.java | 100 ++-- .../container/ozoneimpl/OzoneContainer.java | 9 +- .../replication/ContainerDownloader.java| 40 ++ .../replication/ContainerReplicationSource.java | 49 ++ .../replication/ContainerStreamingOutput.java | 45 ++ .../replication/GrpcReplicationClient.java | 169 ++ .../replication/GrpcReplicationService.java | 130 + .../OnDemandContainerReplicationSource.java | 76 +++ .../replication/SimpleContainerDownloader.java | 121 .../container/replication/package-info.java | 21 + .../TestReplicateContainerCommandHandler.java | 146 + .../commandhandler/package-info.java| 22 + .../apache/hadoop/hdds/server/ServerUtils.java | 5 + .../hadoop/hdds/server/events/EventWatcher.java | 20 +- hadoop-hdds/pom.xml | 5 + .../scm/command/CommandStatusReportHandler.java | 6 +- .../container/CloseContainerEventHandler.java | 26 + .../scm/container/CloseContainerWatcher.java| 100 .../hdds/scm/container/ContainerMapping.java| 13 + .../hadoop/hdds/scm/events/SCMEvents.java | 11 + .../hadoop/hdds/scm/node/DeadNodeHandler.java | 41 +- .../hdds/scm/node/states/Node2ContainerMap.java | 29 +- .../scm/server/StorageContainerManager.java | 11 +- .../hdds/scm/node/TestDeadNodeHandler.java | 112 .../container/TestCloseContainerWatcher.java| 287 ++ .../hadoop/hdds/scm/cli/OzoneBaseCLI.java | 43 -- .../hdds/scm/cli/OzoneCommandHandler.java | 87 --- .../apache/hadoop/hdds/scm/cli/ResultCode.java | 31 -- .../org/apache/hadoop/hdds/scm/cli/SCMCLI.java | 246 +++-- .../cli/container/CloseContainerHandler.java| 85 --- .../hdds/scm/cli/container/CloseSubcommand.java | 54 ++ .../cli/container/ContainerCommandHandler.java | 128 - .../cli/container/CreateContainerHandler.java | 67 --- .../scm/cli/container/CreateSubcommand.java | 65 +++ .../cli/container/DeleteContainerHandler.java | 95 .../scm/cli/container/DeleteSubcommand.java | 60 ++ .../scm/cli/container/InfoContainerHandler.java | 114 .../hdds/scm/cli/container/InfoSubcommand.java | 94 .../scm/cli/container/ListContainerHandler.java | 117 .../hdds/scm/cli/container/ListSubcommand.java | 83 +++ .../hdds/scm/cli/container/package-info.java| 3 + .../hadoop/hdds/scm/cli/package-info.java | 12 +- .../java/org/apache/hadoop/hdfs/DFSClient.java | 4 + .../src/main/resources/httpfs-default.xml | 7 + .../federation/resolver/MountTableResolver.java | 39 +- .../server/federation/router/RBFConfigKeys.java | 4 + .../federation/router/RouterRpcServer.java | 2 +- .../hdfs/tools/federation/RouterAdmin.java | 55 +- .../src/main/resources/hdfs-rbf-default.xml | 8 + .../resolve
[41/50] [abbrv] hadoop git commit: HADOOP-15754. s3guard: testDynamoTableTagging should clear existing config. Contributed by Gabor Bota.
HADOOP-15754. s3guard: testDynamoTableTagging should clear existing config.
Contributed by Gabor Bota.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/26d0c63a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/26d0c63a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/26d0c63a
Branch: refs/heads/HDFS-12943
Commit: 26d0c63a1e2eea6558fca2c55c134c02ecc93bf8
Parents: 3d89c3e
Author: Steve Loughran
Authored: Mon Sep 17 22:40:08 2018 +0100
Committer: Steve Loughran
Committed: Mon Sep 17 22:40:08 2018 +0100
--
.../fs/s3a/s3guard/AbstractS3GuardToolTestBase.java | 8 +---
.../fs/s3a/s3guard/ITestDynamoDBMetadataStore.java | 6 ++
.../fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java| 16 +++-
3 files changed, 26 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26d0c63a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
--
diff --git
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
index 242f616..632676f 100644
---
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
+++
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
@@ -300,16 +300,18 @@ public abstract class AbstractS3GuardToolTestBase extends
AbstractS3ATestBase {
@Test
public void testSetCapacityFailFastOnReadWriteOfZero() throws Exception{
Configuration conf = getConfiguration();
+String bucket = getFileSystem().getBucket();
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, getFileSystem().getBucket());
S3GuardTool.SetCapacity cmdR = new S3GuardTool.SetCapacity(conf);
-String[] argsR = new String[]{cmdR.getName(), "-read", "0",
"s3a://bucket"};
+String[] argsR =
+new String[]{cmdR.getName(), "-read", "0", "s3a://" + bucket};
intercept(IllegalArgumentException.class,
S3GuardTool.SetCapacity.READ_CAP_INVALID, () -> cmdR.run(argsR));
S3GuardTool.SetCapacity cmdW = new S3GuardTool.SetCapacity(conf);
-String[] argsW = new String[]{cmdW.getName(), "-write", "0",
-"s3a://bucket"};
+String[] argsW =
+new String[]{cmdW.getName(), "-write", "0", "s3a://" + bucket};
intercept(IllegalArgumentException.class,
S3GuardTool.SetCapacity.WRITE_CAP_INVALID, () -> cmdW.run(argsW));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26d0c63a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestDynamoDBMetadataStore.java
--
diff --git
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestDynamoDBMetadataStore.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestDynamoDBMetadataStore.java
index a8425bf..5355910 100644
---
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestDynamoDBMetadataStore.java
+++
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestDynamoDBMetadataStore.java
@@ -630,6 +630,12 @@ public class ITestDynamoDBMetadataStore extends
MetadataStoreTestBase {
@Test
public void testTableTagging() throws IOException {
final Configuration conf = getFileSystem().getConf();
+
+// clear all table tagging config before this test
+conf.getPropsWithPrefix(S3GUARD_DDB_TABLE_TAG).keySet().forEach(
+propKey -> conf.unset(S3GUARD_DDB_TABLE_TAG + propKey)
+);
+
String tableName = "testTableTagging-" + UUID.randomUUID();
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName);
conf.set(S3GUARD_DDB_TABLE_CREATE_KEY, "true");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26d0c63a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
--
diff --git
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
index 66a8239..65e2619 100644
---
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
+++
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
@@ -44,7 +44,9 @@ import org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.Destroy;
import org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.Init;
import org.apache.hadoop.test.L
[42/50] [abbrv] hadoop git commit: HDDS-475. Block Allocation returns same BlockID on different keys creation. Contributed by Nanda Kumar.
HDDS-475. Block Allocation returns same BlockID on different keys creation.
Contributed by Nanda Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/23a6137a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/23a6137a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/23a6137a
Branch: refs/heads/HDFS-12943
Commit: 23a6137a40b34742714e0140268c491e1da6db6d
Parents: 26d0c63
Author: Anu Engineer
Authored: Mon Sep 17 14:08:39 2018 -0700
Committer: Anu Engineer
Committed: Mon Sep 17 14:41:17 2018 -0700
--
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 202 +++
1 file changed, 117 insertions(+), 85 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23a6137a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
--
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
index 8322b73..3405b0d 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
@@ -48,8 +48,6 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes
.CHILL_MODE_EXCEPTION;
@@ -76,7 +74,6 @@ public class BlockManagerImpl implements
EventHandler,
private final NodeManager nodeManager;
private final Mapping containerManager;
- private final ReadWriteLock lock;
private final long containerSize;
private final DeletedBlockLog deletedBlockLog;
@@ -113,7 +110,6 @@ public class BlockManagerImpl implements
EventHandler,
ScmConfigKeys.OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE,
ScmConfigKeys.OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE_DEFAULT);
rand = new Random();
-this.lock = new ReentrantReadWriteLock();
mxBean = MBeans.register("BlockManager", "BlockManagerImpl", this);
@@ -223,74 +219,29 @@ public class BlockManagerImpl implements
EventHandler,
ContainerWithPipeline containerWithPipeline;
-lock.readLock().lock();
-try {
- // This is to optimize performance, if the below condition is evaluated
- // to false, then we can be sure that there are no containers in
- // ALLOCATED state.
- // This can result in false positive, but it will never be false
negative.
- // How can this result in false positive? We check if there are any
- // containers in ALLOCATED state, this check doesn't care about the
- // USER of the containers. So there might be cases where a different
- // USER has few containers in ALLOCATED state, which will result in
- // false positive.
- if (!containerManager.getStateManager().getContainerStateMap()
- .getContainerIDsByState(HddsProtos.LifeCycleState.ALLOCATED)
- .isEmpty()) {
-// Since the above check can result in false positive, we have to do
-// the actual check and find out if there are containers in ALLOCATED
-// state matching our criteria.
-synchronized (this) {
- // Using containers from ALLOCATED state should be done within
- // synchronized block (or) write lock. Since we already hold a
- // read lock, we will end up in deadlock situation if we take
- // write lock here.
- containerWithPipeline = containerManager
- .getMatchingContainerWithPipeline(size, owner, type, factor,
- HddsProtos.LifeCycleState.ALLOCATED);
- if (containerWithPipeline != null) {
-containerManager.updateContainerState(
-containerWithPipeline.getContainerInfo().getContainerID(),
-HddsProtos.LifeCycleEvent.CREATE);
-return newBlock(containerWithPipeline,
-HddsProtos.LifeCycleState.ALLOCATED);
- }
-}
- }
-
- // Since we found no allocated containers that match our criteria, let us
- // look for OPEN containers that match the criteria.
- containerWithPipeline = containerManager
- .getMatchingContainerWithPipeline(size, owner, type, factor,
- HddsProtos.LifeCycleState.OPEN);
- if (containerWithPipeline != null) {
-return newBlock(containerWithPipeline, HddsProtos.LifeCycleState.OPEN);
- }
-
- // We found neither ALLOCATED or OPEN Containers. This gen
[37/50] [abbrv] hadoop git commit: HDFS-13919. Documentation: Improper formatting in Disk Balancer for Settings. Contributed by Ayush Saxena.
HDFS-13919. Documentation: Improper formatting in Disk Balancer for Settings. Contributed by Ayush Saxena. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fdf5a3fd Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fdf5a3fd Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fdf5a3fd Branch: refs/heads/HDFS-12943 Commit: fdf5a3fd63a24b2cb2acafbc30ae4f993ff33145 Parents: 8469366 Author: Anu Engineer Authored: Mon Sep 17 10:08:23 2018 -0700 Committer: Anu Engineer Committed: Mon Sep 17 10:08:23 2018 -0700 -- .../hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5a3fd/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md index ed0233a..5dd6ffc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md @@ -127,6 +127,7 @@ There is a set of diskbalancer settings that can be controlled via hdfs-site.xml |`dfs.disk.balancer.block.tolerance.percent`| The tolerance percent specifies when we have reached a good enough value for any copy step. For example, if you specify 10% then getting close to 10% of the target value is good enough.| |`dfs.disk.balancer.plan.threshold.percent`| The percentage threshold value for volume Data Density in a plan. If the absolute value of volume Data Density which is out of threshold value in a node, it means that the volumes corresponding to the disks should do the balancing in the plan. The default value is 10.| |`dfs.disk.balancer.plan.valid.interval`| Maximum amount of time disk balancer plan is valid. Supports the following suffixes (case insensitive): ms(millis), s(sec), m(min), h(hour), d(day) to specify the time (such as 2s, 2m, 1h, etc.). If no suffix is specified then milliseconds is assumed. Default value is 1d| + Debugging - - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
