HDDS-242. Introduce NEW_NODE, STALE_NODE and DEAD_NODE event and corresponding event handlers in SCM. Contributed by Nanda Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a47ec5da Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a47ec5da Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a47ec5da Branch: refs/heads/YARN-7402 Commit: a47ec5dac4a1cdfec788ce3296b4f610411911ea Parents: 4e59b92 Author: Anu Engineer <aengin...@apache.org> Authored: Tue Jul 10 15:58:47 2018 -0700 Committer: Anu Engineer <aengin...@apache.org> Committed: Tue Jul 10 15:58:47 2018 -0700 ---------------------------------------------------------------------- .../scm/container/ContainerReportHandler.java | 47 ++++++++++++++++++ .../hadoop/hdds/scm/node/DeadNodeHandler.java | 42 ++++++++++++++++ .../hadoop/hdds/scm/node/NewNodeHandler.java | 50 +++++++++++++++++++ .../hadoop/hdds/scm/node/NodeReportHandler.java | 42 ++++++++++++++++ .../hadoop/hdds/scm/node/StaleNodeHandler.java | 42 ++++++++++++++++ .../common/src/main/bin/ozone-config.sh | 51 ++++++++++++++++++++ 6 files changed, 274 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a47ec5da/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java new file mode 100644 index 0000000..486162e --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.container; + +import org.apache.hadoop.hdds.scm.node.states.Node2ContainerMap; +import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher + .ContainerReportFromDatanode; +import org.apache.hadoop.hdds.server.events.EventHandler; +import org.apache.hadoop.hdds.server.events.EventPublisher; + +/** + * Handles container reports from datanode. + */ +public class ContainerReportHandler implements + EventHandler<ContainerReportFromDatanode> { + + private final Mapping containerMapping; + private final Node2ContainerMap node2ContainerMap; + + public ContainerReportHandler(Mapping containerMapping, + Node2ContainerMap node2ContainerMap) { + this.containerMapping = containerMapping; + this.node2ContainerMap = node2ContainerMap; + } + + @Override + public void onMessage(ContainerReportFromDatanode containerReportFromDatanode, + EventPublisher publisher) { + // TODO: process container report. + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a47ec5da/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java new file mode 100644 index 0000000..427aef8 --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.node; + +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.scm.node.states.Node2ContainerMap; +import org.apache.hadoop.hdds.server.events.EventHandler; +import org.apache.hadoop.hdds.server.events.EventPublisher; + +/** + * Handles Dead Node event. + */ +public class DeadNodeHandler implements EventHandler<DatanodeDetails> { + + private final Node2ContainerMap node2ContainerMap; + + public DeadNodeHandler(Node2ContainerMap node2ContainerMap) { + this.node2ContainerMap = node2ContainerMap; + } + + @Override + public void onMessage(DatanodeDetails datanodeDetails, + EventPublisher publisher) { + //TODO: add logic to handle dead node. + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a47ec5da/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NewNodeHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NewNodeHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NewNodeHandler.java new file mode 100644 index 0000000..79b75a5 --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NewNodeHandler.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.node; + +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.scm.exceptions.SCMException; +import org.apache.hadoop.hdds.scm.node.states.Node2ContainerMap; +import org.apache.hadoop.hdds.server.events.EventHandler; +import org.apache.hadoop.hdds.server.events.EventPublisher; + +import java.util.Collections; + +/** + * Handles New Node event. + */ +public class NewNodeHandler implements EventHandler<DatanodeDetails> { + + private final Node2ContainerMap node2ContainerMap; + + public NewNodeHandler(Node2ContainerMap node2ContainerMap) { + this.node2ContainerMap = node2ContainerMap; + } + + @Override + public void onMessage(DatanodeDetails datanodeDetails, + EventPublisher publisher) { + try { + node2ContainerMap.insertNewDatanode(datanodeDetails.getUuid(), + Collections.emptySet()); + } catch (SCMException e) { + // TODO: log exception message. + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a47ec5da/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeReportHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeReportHandler.java new file mode 100644 index 0000000..aa78d53 --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeReportHandler.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.node; + +import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher + .NodeReportFromDatanode; +import org.apache.hadoop.hdds.server.events.EventHandler; +import org.apache.hadoop.hdds.server.events.EventPublisher; + +/** + * Handles Node Reports from datanode. + */ +public class NodeReportHandler implements EventHandler<NodeReportFromDatanode> { + + private final NodeManager nodeManager; + + public NodeReportHandler(NodeManager nodeManager) { + this.nodeManager = nodeManager; + } + + @Override + public void onMessage(NodeReportFromDatanode nodeReportFromDatanode, + EventPublisher publisher) { + //TODO: process node report. + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a47ec5da/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java new file mode 100644 index 0000000..b37dd93 --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.node; + +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.scm.node.states.Node2ContainerMap; +import org.apache.hadoop.hdds.server.events.EventHandler; +import org.apache.hadoop.hdds.server.events.EventPublisher; + +/** + * Handles Stale node event. + */ +public class StaleNodeHandler implements EventHandler<DatanodeDetails> { + + private final Node2ContainerMap node2ContainerMap; + + public StaleNodeHandler(Node2ContainerMap node2ContainerMap) { + this.node2ContainerMap = node2ContainerMap; + } + + @Override + public void onMessage(DatanodeDetails datanodeDetails, + EventPublisher publisher) { + //TODO: logic to handle stale node. + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a47ec5da/hadoop-ozone/common/src/main/bin/ozone-config.sh ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/src/main/bin/ozone-config.sh b/hadoop-ozone/common/src/main/bin/ozone-config.sh new file mode 100755 index 0000000..83f30ce --- /dev/null +++ b/hadoop-ozone/common/src/main/bin/ozone-config.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# included in all the ozone scripts with source command +# should not be executed directly + +function hadoop_subproject_init +{ + if [[ -z "${HADOOP_OZONE_ENV_PROCESSED}" ]]; then + if [[ -e "${HADOOP_CONF_DIR}/hdfs-env.sh" ]]; then + . "${HADOOP_CONF_DIR}/hdfs-env.sh" + export HADOOP_OZONES_ENV_PROCESSED=true + fi + fi + HADOOP_OZONE_HOME="${HADOOP_OZONE_HOME:-$HADOOP_HOME}" + +} + +if [[ -z "${HADOOP_LIBEXEC_DIR}" ]]; then + _hd_this="${BASH_SOURCE-$0}" + HADOOP_LIBEXEC_DIR=$(cd -P -- "$(dirname -- "${_hd_this}")" >/dev/null && pwd -P) +fi + +# shellcheck source=./hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh + +if [[ -n "${HADOOP_COMMON_HOME}" ]] && + [[ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" ]]; then + . "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" +elif [[ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then + . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" +elif [ -e "${HADOOP_HOME}/libexec/hadoop-config.sh" ]; then + . "${HADOOP_HOME}/libexec/hadoop-config.sh" +else + echo "ERROR: Hadoop common not found." 2>&1 + exit 1 +fi + --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
