http://git-wip-us.apache.org/repos/asf/hadoop/blob/4aa34324/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCPerformanceMonitor.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCPerformanceMonitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCPerformanceMonitor.java deleted file mode 100644 index 547ebb5..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCPerformanceMonitor.java +++ /dev/null @@ -1,221 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs.server.federation.metrics; - -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ThreadFactory; - -import javax.management.NotCompliantMBeanException; -import javax.management.ObjectName; -import javax.management.StandardMBean; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.server.federation.router.RouterRpcMonitor; -import org.apache.hadoop.hdfs.server.federation.router.RouterRpcServer; -import org.apache.hadoop.hdfs.server.federation.store.StateStoreService; -import org.apache.hadoop.metrics2.util.MBeans; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.util.concurrent.ThreadFactoryBuilder; - -/** - * Customizable RPC performance monitor. Receives events from the RPC server - * and aggregates them via JMX. - */ -public class FederationRPCPerformanceMonitor implements RouterRpcMonitor { - - private static final Logger LOG = - LoggerFactory.getLogger(FederationRPCPerformanceMonitor.class); - - - /** Time for an operation to be received in the Router. */ - private static final ThreadLocal<Long> START_TIME = new ThreadLocal<>(); - /** Time for an operation to be send to the Namenode. */ - private static final ThreadLocal<Long> PROXY_TIME = new ThreadLocal<>(); - - /** Configuration for the performance monitor. */ - private Configuration conf; - /** RPC server for the Router. */ - private RouterRpcServer server; - /** State Store. */ - private StateStoreService store; - - /** JMX interface to monitor the RPC metrics. */ - private FederationRPCMetrics metrics; - private ObjectName registeredBean; - - /** Thread pool for logging stats. */ - private ExecutorService executor; - - - @Override - public void init(Configuration configuration, RouterRpcServer rpcServer, - StateStoreService stateStore) { - - this.conf = configuration; - this.server = rpcServer; - this.store = stateStore; - - // Create metrics - this.metrics = FederationRPCMetrics.create(conf, server); - - // Create thread pool - ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setNameFormat("Federation RPC Performance Monitor-%d").build(); - this.executor = Executors.newFixedThreadPool(1, threadFactory); - - // Adding JMX interface - try { - StandardMBean bean = - new StandardMBean(this.metrics, FederationRPCMBean.class); - registeredBean = MBeans.register("Router", "FederationRPC", bean); - LOG.info("Registered FederationRPCMBean: {}", registeredBean); - } catch (NotCompliantMBeanException e) { - throw new RuntimeException("Bad FederationRPCMBean setup", e); - } - } - - @Override - public void close() { - if (registeredBean != null) { - MBeans.unregister(registeredBean); - registeredBean = null; - } - if (this.executor != null) { - this.executor.shutdown(); - } - } - - /** - * Resets all RPC service performance counters to their defaults. - */ - public void resetPerfCounters() { - if (registeredBean != null) { - MBeans.unregister(registeredBean); - registeredBean = null; - } - if (metrics != null) { - FederationRPCMetrics.reset(); - metrics = null; - } - init(conf, server, store); - } - - @Override - public void startOp() { - START_TIME.set(this.getNow()); - } - - @Override - public long proxyOp() { - PROXY_TIME.set(this.getNow()); - long processingTime = getProcessingTime(); - if (processingTime >= 0) { - metrics.addProcessingTime(processingTime); - } - return Thread.currentThread().getId(); - } - - @Override - public void proxyOpComplete(boolean success) { - if (success) { - long proxyTime = getProxyTime(); - if (proxyTime >= 0) { - metrics.addProxyTime(proxyTime); - } - } - } - - @Override - public void proxyOpFailureStandby() { - metrics.incrProxyOpFailureStandby(); - } - - @Override - public void proxyOpFailureCommunicate() { - metrics.incrProxyOpFailureCommunicate(); - } - - @Override - public void proxyOpNotImplemented() { - metrics.incrProxyOpNotImplemented(); - } - - @Override - public void proxyOpRetries() { - metrics.incrProxyOpRetries(); - } - - @Override - public void routerFailureStateStore() { - metrics.incrRouterFailureStateStore(); - } - - @Override - public void routerFailureSafemode() { - metrics.incrRouterFailureSafemode(); - } - - @Override - public void routerFailureReadOnly() { - metrics.incrRouterFailureReadOnly(); - } - - @Override - public void routerFailureLocked() { - metrics.incrRouterFailureLocked(); - } - - /** - * Get current time. - * @return Current time in nanoseconds. - */ - private long getNow() { - return System.nanoTime(); - } - - /** - * Get time between we receiving the operation and sending it to the Namenode. - * @return Processing time in nanoseconds. - */ - private long getProcessingTime() { - if (START_TIME.get() != null && START_TIME.get() > 0 && - PROXY_TIME.get() != null && PROXY_TIME.get() > 0) { - return PROXY_TIME.get() - START_TIME.get(); - } - return -1; - } - - /** - * Get time between now and when the operation was forwarded to the Namenode. - * @return Current proxy time in nanoseconds. - */ - private long getProxyTime() { - if (PROXY_TIME.get() != null && PROXY_TIME.get() > 0) { - return getNow() - PROXY_TIME.get(); - } - return -1; - } - - @Override - public FederationRPCMetrics getRPCMetrics() { - return this.metrics; - } -} \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4aa34324/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java deleted file mode 100644 index 93e9ea0..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java +++ /dev/null @@ -1,637 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs.server.federation.metrics; - -import static org.apache.hadoop.util.Time.now; - -import java.io.IOException; -import java.lang.management.ManagementFactory; -import java.lang.reflect.Method; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -import javax.management.NotCompliantMBeanException; -import javax.management.ObjectName; -import javax.management.StandardMBean; - -import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; -import org.apache.hadoop.hdfs.DFSUtilClient; -import org.apache.hadoop.hdfs.protocol.DatanodeInfo; -import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; -import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo; -import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole; -import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo; -import org.apache.hadoop.hdfs.server.federation.router.Router; -import org.apache.hadoop.hdfs.server.federation.router.RouterRpcServer; -import org.apache.hadoop.hdfs.server.federation.store.MembershipStore; -import org.apache.hadoop.hdfs.server.federation.store.StateStoreService; -import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamespaceInfoRequest; -import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamespaceInfoResponse; -import org.apache.hadoop.hdfs.server.namenode.NameNodeMXBean; -import org.apache.hadoop.hdfs.server.namenode.NameNodeStatusMXBean; -import org.apache.hadoop.hdfs.server.namenode.metrics.FSNamesystemMBean; -import org.apache.hadoop.ipc.StandbyException; -import org.apache.hadoop.metrics2.util.MBeans; -import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.util.VersionInfo; -import org.mortbay.util.ajax.JSON; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Expose the Namenode metrics as the Router was one. - */ -public class NamenodeBeanMetrics - implements FSNamesystemMBean, NameNodeMXBean, NameNodeStatusMXBean { - - private static final Logger LOG = - LoggerFactory.getLogger(NamenodeBeanMetrics.class); - - private final Router router; - - /** FSNamesystem bean. */ - private ObjectName fsBeanName; - /** FSNamesystemState bean. */ - private ObjectName fsStateBeanName; - /** NameNodeInfo bean. */ - private ObjectName nnInfoBeanName; - /** NameNodeStatus bean. */ - private ObjectName nnStatusBeanName; - - - public NamenodeBeanMetrics(Router router) { - this.router = router; - - try { - // TODO this needs to be done with the Metrics from FSNamesystem - StandardMBean bean = new StandardMBean(this, FSNamesystemMBean.class); - this.fsBeanName = MBeans.register("NameNode", "FSNamesystem", bean); - LOG.info("Registered FSNamesystem MBean: {}", this.fsBeanName); - } catch (NotCompliantMBeanException e) { - throw new RuntimeException("Bad FSNamesystem MBean setup", e); - } - - try { - StandardMBean bean = new StandardMBean(this, FSNamesystemMBean.class); - this.fsStateBeanName = - MBeans.register("NameNode", "FSNamesystemState", bean); - LOG.info("Registered FSNamesystemState MBean: {}", this.fsStateBeanName); - } catch (NotCompliantMBeanException e) { - throw new RuntimeException("Bad FSNamesystemState MBean setup", e); - } - - try { - StandardMBean bean = new StandardMBean(this, NameNodeMXBean.class); - this.nnInfoBeanName = MBeans.register("NameNode", "NameNodeInfo", bean); - LOG.info("Registered NameNodeInfo MBean: {}", this.nnInfoBeanName); - } catch (NotCompliantMBeanException e) { - throw new RuntimeException("Bad NameNodeInfo MBean setup", e); - } - - try { - StandardMBean bean = new StandardMBean(this, NameNodeStatusMXBean.class); - this.nnStatusBeanName = - MBeans.register("NameNode", "NameNodeStatus", bean); - LOG.info("Registered NameNodeStatus MBean: {}", this.nnStatusBeanName); - } catch (NotCompliantMBeanException e) { - throw new RuntimeException("Bad NameNodeStatus MBean setup", e); - } - } - - /** - * De-register the JMX interfaces. - */ - public void close() { - if (fsStateBeanName != null) { - MBeans.unregister(fsStateBeanName); - fsStateBeanName = null; - } - if (nnInfoBeanName != null) { - MBeans.unregister(nnInfoBeanName); - nnInfoBeanName = null; - } - // Remove the NameNode status bean - if (nnStatusBeanName != null) { - MBeans.unregister(nnStatusBeanName); - nnStatusBeanName = null; - } - } - - private FederationMetrics getFederationMetrics() { - return this.router.getMetrics(); - } - - ///////////////////////////////////////////////////////// - // NameNodeMXBean - ///////////////////////////////////////////////////////// - - @Override - public String getVersion() { - return VersionInfo.getVersion() + ", r" + VersionInfo.getRevision(); - } - - @Override - public String getSoftwareVersion() { - return VersionInfo.getVersion(); - } - - @Override - public long getUsed() { - return getFederationMetrics().getUsedCapacity(); - } - - @Override - public long getFree() { - return getFederationMetrics().getRemainingCapacity(); - } - - @Override - public long getTotal() { - return getFederationMetrics().getTotalCapacity(); - } - - @Override - public String getSafemode() { - // We assume that the global federated view is never in safe mode - return ""; - } - - @Override - public boolean isUpgradeFinalized() { - // We assume the upgrade is always finalized in a federated biew - return true; - } - - @Override - public RollingUpgradeInfo.Bean getRollingUpgradeStatus() { - return null; - } - - @Override - public long getNonDfsUsedSpace() { - return 0; - } - - @Override - public float getPercentUsed() { - return DFSUtilClient.getPercentUsed(getCapacityUsed(), getCapacityTotal()); - } - - @Override - public float getPercentRemaining() { - return DFSUtilClient.getPercentUsed( - getCapacityRemaining(), getCapacityTotal()); - } - - @Override - public long getCacheUsed() { - return 0; - } - - @Override - public long getCacheCapacity() { - return 0; - } - - @Override - public long getBlockPoolUsedSpace() { - return 0; - } - - @Override - public float getPercentBlockPoolUsed() { - return 0; - } - - @Override - public long getTotalBlocks() { - return getFederationMetrics().getNumBlocks(); - } - - @Override - public long getNumberOfMissingBlocks() { - return getFederationMetrics().getNumOfMissingBlocks(); - } - - @Override - public long getPendingReplicationBlocks() { - return getFederationMetrics().getNumOfBlocksPendingReplication(); - } - - @Override - public long getUnderReplicatedBlocks() { - return getFederationMetrics().getNumOfBlocksUnderReplicated(); - } - - @Override - public long getPendingDeletionBlocks() { - return getFederationMetrics().getNumOfBlocksPendingDeletion(); - } - - @Override - public long getScheduledReplicationBlocks() { - return -1; - } - - @Override - public long getNumberOfMissingBlocksWithReplicationFactorOne() { - return 0; - } - - @Override - public String getCorruptFiles() { - return "N/A"; - } - - @Override - public int getThreads() { - return ManagementFactory.getThreadMXBean().getThreadCount(); - } - - @Override - public String getLiveNodes() { - return this.getNodes(DatanodeReportType.LIVE); - } - - @Override - public String getDeadNodes() { - return this.getNodes(DatanodeReportType.DEAD); - } - - @Override - public String getDecomNodes() { - return this.getNodes(DatanodeReportType.DECOMMISSIONING); - } - - /** - * Get all the nodes in the federation from a particular type. - * TODO this is expensive, we may want to cache it. - * @param type Type of the datanodes to check. - * @return JSON with the nodes. - */ - private String getNodes(DatanodeReportType type) { - final Map<String, Map<String, Object>> info = new HashMap<>(); - try { - RouterRpcServer rpcServer = this.router.getRpcServer(); - DatanodeInfo[] datanodes = rpcServer.getDatanodeReport(type); - for (DatanodeInfo node : datanodes) { - Map<String, Object> innerinfo = new HashMap<>(); - innerinfo.put("infoAddr", node.getInfoAddr()); - innerinfo.put("infoSecureAddr", node.getInfoSecureAddr()); - innerinfo.put("xferaddr", node.getXferAddr()); - innerinfo.put("location", node.getNetworkLocation()); - innerinfo.put("lastContact", getLastContact(node)); - innerinfo.put("usedSpace", node.getDfsUsed()); - innerinfo.put("adminState", node.getAdminState().toString()); - innerinfo.put("nonDfsUsedSpace", node.getNonDfsUsed()); - innerinfo.put("capacity", node.getCapacity()); - innerinfo.put("numBlocks", -1); // node.numBlocks() - innerinfo.put("version", (node.getSoftwareVersion() == null ? - "UNKNOWN" : node.getSoftwareVersion())); - innerinfo.put("used", node.getDfsUsed()); - innerinfo.put("remaining", node.getRemaining()); - innerinfo.put("blockScheduled", -1); // node.getBlocksScheduled() - innerinfo.put("blockPoolUsed", node.getBlockPoolUsed()); - innerinfo.put("blockPoolUsedPercent", node.getBlockPoolUsedPercent()); - innerinfo.put("volfails", -1); // node.getVolumeFailures() - info.put(node.getHostName() + ":" + node.getXferPort(), - Collections.unmodifiableMap(innerinfo)); - } - } catch (StandbyException e) { - LOG.error("Cannot get {} nodes, Router in safe mode", type); - } catch (IOException e) { - LOG.error("Cannot get " + type + " nodes", e); - } - return JSON.toString(info); - } - - @Override - public String getClusterId() { - try { - return getNamespaceInfo("getClusterId").toString(); - } catch (IOException e) { - LOG.error("Cannot fetch cluster ID metrics {}", e.getMessage()); - return ""; - } - } - - @Override - public String getBlockPoolId() { - try { - return getNamespaceInfo("getBlockPoolId").toString(); - } catch (IOException e) { - LOG.error("Cannot fetch block pool ID metrics {}", e.getMessage()); - return ""; - } - } - - /** - * Build a set of unique values found in all namespaces. - * - * @param getterName String name of the appropriate FederationNamespaceInfo - * getter function - * @return Set of unique string values found in all discovered namespaces. - * @throws IOException if the query could not be executed. - */ - public Collection<String> getNamespaceInfo(String getterName) - throws IOException { - - StateStoreService stateStore = router.getStateStore(); - MembershipStore membershipStore = - stateStore.getRegisteredRecordStore(MembershipStore.class); - - GetNamespaceInfoRequest request = GetNamespaceInfoRequest.newInstance(); - GetNamespaceInfoResponse response = - membershipStore.getNamespaceInfo(request); - Set<FederationNamespaceInfo> namespacesInfo = response.getNamespaceInfo(); - - Set<String> ret = new HashSet<String>(); - for (FederationNamespaceInfo namespace : namespacesInfo) { - try { - Method m = FederationNamespaceInfo.class.getDeclaredMethod(getterName); - String data = (String) m.invoke(namespace); - ret.add(data); - } catch (SecurityException | ReflectiveOperationException ex) { - throw new IOException( - "Cannot invoke " + getterName + " from " + namespace); - } - } - return ret; - } - - @Override - public String getNameDirStatuses() { - return "N/A"; - } - - @Override - public String getNodeUsage() { - return "N/A"; - } - - @Override - public String getNameJournalStatus() { - return "N/A"; - } - - @Override - public String getJournalTransactionInfo() { - return "N/A"; - } - - @Override - public long getNNStartedTimeInMillis() { - return this.router.getStartTime(); - } - - @Deprecated - @Override - public String getNNStarted() { - return new Date(this.router.getStartTime()).toString(); - } - - @Override - public String getCompileInfo() { - return VersionInfo.getDate() + " by " + VersionInfo.getUser() + - " from " + VersionInfo.getBranch(); - } - - @Override - public int getDistinctVersionCount() { - return 0; - } - - @Override - public Map<String, Integer> getDistinctVersions() { - return null; - } - - ///////////////////////////////////////////////////////// - // FSNamesystemMBean - ///////////////////////////////////////////////////////// - - @Override - public String getFSState() { - // We assume is not in safe mode - return "Operational"; - } - - @Override - public long getBlocksTotal() { - return this.getTotalBlocks(); - } - - @Override - public long getCapacityTotal() { - return this.getTotal(); - } - - @Override - public long getCapacityRemaining() { - return this.getFree(); - } - - @Override - public long getCapacityUsed() { - return this.getUsed(); - } - - @Override - public long getFilesTotal() { - return getFederationMetrics().getNumFiles(); - } - - @Deprecated - @Override - public long getTotalFiles() { - return getFederationMetrics().getNumFiles(); - } - - @Override - public int getTotalLoad() { - return -1; - } - - @Override - public int getNumLiveDataNodes() { - return this.router.getMetrics().getNumLiveNodes(); - } - - @Override - public int getNumDeadDataNodes() { - return this.router.getMetrics().getNumDeadNodes(); - } - - @Override - public int getNumStaleDataNodes() { - return -1; - } - - @Override - public int getNumDecomLiveDataNodes() { - return this.router.getMetrics().getNumDecomLiveNodes(); - } - - @Override - public int getNumDecomDeadDataNodes() { - return this.router.getMetrics().getNumDecomDeadNodes(); - } - - @Override - public int getNumDecommissioningDataNodes() { - return this.router.getMetrics().getNumDecommissioningNodes(); - } - - @Override - public int getNumInMaintenanceLiveDataNodes() { - return 0; - } - - @Override - public int getNumInMaintenanceDeadDataNodes() { - return 0; - } - - @Override - public int getNumEnteringMaintenanceDataNodes() { - return 0; - } - - @Override - public int getVolumeFailuresTotal() { - return 0; - } - - @Override - public long getEstimatedCapacityLostTotal() { - return 0; - } - - @Override - public String getSnapshotStats() { - return null; - } - - @Override - public long getMaxObjects() { - return 0; - } - - @Override - public long getBlockDeletionStartTime() { - return -1; - } - - @Override - public int getNumStaleStorages() { - return -1; - } - - @Override - public String getTopUserOpCounts() { - return "N/A"; - } - - @Override - public int getFsLockQueueLength() { - return 0; - } - - @Override - public long getTotalSyncCount() { - return 0; - } - - @Override - public String getTotalSyncTimes() { - return ""; - } - - private long getLastContact(DatanodeInfo node) { - return (now() - node.getLastUpdate()) / 1000; - } - - ///////////////////////////////////////////////////////// - // NameNodeStatusMXBean - ///////////////////////////////////////////////////////// - - @Override - public String getNNRole() { - return NamenodeRole.NAMENODE.toString(); - } - - @Override - public String getState() { - return HAServiceState.ACTIVE.toString(); - } - - @Override - public String getHostAndPort() { - return NetUtils.getHostPortString(router.getRpcServerAddress()); - } - - @Override - public boolean isSecurityEnabled() { - return false; - } - - @Override - public long getLastHATransitionTime() { - return 0; - } - - @Override - public long getBytesWithFutureGenerationStamps() { - return 0; - } - - @Override - public String getSlowPeersReport() { - return "N/A"; - } - - @Override - public String getSlowDisksReport() { - return "N/A"; - } - - @Override - public long getNumberOfSnapshottableDirs() { - return 0; - } - - @Override - public String getEnteringMaintenanceNodes() { - return "N/A"; - } - - @Override - public String getNameDirSize() { - return "N/A"; - } - - @Override - public int getNumEncryptionZones() { - return 0; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/4aa34324/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMBean.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMBean.java deleted file mode 100644 index 5e4ccab..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMBean.java +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs.server.federation.metrics; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; - -/** - * JMX interface for the State Store metrics. - */ [email protected] [email protected] -public interface StateStoreMBean { - - long getReadOps(); - - double getReadAvg(); - - long getWriteOps(); - - double getWriteAvg(); - - long getFailureOps(); - - double getFailureAvg(); - - long getRemoveOps(); - - double getRemoveAvg(); -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/4aa34324/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java deleted file mode 100644 index 09253a2..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java +++ /dev/null @@ -1,149 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs.server.federation.metrics; - -import static org.apache.hadoop.metrics2.impl.MsInfo.ProcessName; -import static org.apache.hadoop.metrics2.impl.MsInfo.SessionId; - -import java.util.HashMap; -import java.util.Map; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.metrics2.MetricsSystem; -import org.apache.hadoop.metrics2.annotation.Metric; -import org.apache.hadoop.metrics2.annotation.Metrics; -import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; -import org.apache.hadoop.metrics2.lib.MetricsRegistry; -import org.apache.hadoop.metrics2.lib.MutableGaugeInt; -import org.apache.hadoop.metrics2.lib.MutableRate; - -import com.google.common.annotations.VisibleForTesting; - -/** - * Implementations of the JMX interface for the State Store metrics. - */ -@Metrics(name = "StateStoreActivity", about = "Router metrics", - context = "dfs") -public final class StateStoreMetrics implements StateStoreMBean { - - private final MetricsRegistry registry = new MetricsRegistry("router"); - - @Metric("GET transactions") - private MutableRate reads; - @Metric("PUT transactions") - private MutableRate writes; - @Metric("REMOVE transactions") - private MutableRate removes; - @Metric("Failed transactions") - private MutableRate failures; - - private Map<String, MutableGaugeInt> cacheSizes; - - private StateStoreMetrics(Configuration conf) { - registry.tag(SessionId, "RouterSession"); - registry.tag(ProcessName, "Router"); - cacheSizes = new HashMap<>(); - } - - public static StateStoreMetrics create(Configuration conf) { - MetricsSystem ms = DefaultMetricsSystem.instance(); - return ms.register(new StateStoreMetrics(conf)); - } - - public void shutdown() { - DefaultMetricsSystem.shutdown(); - reset(); - } - - public void addRead(long latency) { - reads.add(latency); - } - - public long getReadOps() { - return reads.lastStat().numSamples(); - } - - public double getReadAvg() { - return reads.lastStat().mean(); - } - - public void addWrite(long latency) { - writes.add(latency); - } - - public long getWriteOps() { - return writes.lastStat().numSamples(); - } - - public double getWriteAvg() { - return writes.lastStat().mean(); - } - - public void addFailure(long latency) { - failures.add(latency); - } - - public long getFailureOps() { - return failures.lastStat().numSamples(); - } - - public double getFailureAvg() { - return failures.lastStat().mean(); - } - - public void addRemove(long latency) { - removes.add(latency); - } - - public long getRemoveOps() { - return removes.lastStat().numSamples(); - } - - public double getRemoveAvg() { - return removes.lastStat().mean(); - } - - /** - * Set the size of the cache for a State Store interface. - * - * @param name Name of the record to cache. - * @param size Number of records. - */ - public void setCacheSize(String name, int size) { - String counterName = "Cache" + name + "Size"; - MutableGaugeInt counter = cacheSizes.get(counterName); - if (counter == null) { - counter = registry.newGauge(counterName, name, size); - cacheSizes.put(counterName, counter); - } - counter.set(size); - } - - @VisibleForTesting - public void reset() { - reads.resetMinMax(); - writes.resetMinMax(); - removes.resetMinMax(); - failures.resetMinMax(); - - reads.lastStat().reset(); - writes.lastStat().reset(); - removes.lastStat().reset(); - failures.lastStat().reset(); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/4aa34324/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/package-info.java deleted file mode 100644 index c56c823..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/package-info.java +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Report metrics for Router-based Federation. - */ [email protected] [email protected] -package org.apache.hadoop.hdfs.server.federation.metrics; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/4aa34324/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java deleted file mode 100644 index 1773b34..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java +++ /dev/null @@ -1,118 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs.server.federation.resolver; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.List; -import java.util.Set; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; - -/** - * Locates the most active NN for a given nameservice ID or blockpool ID. This - * interface is used by the {@link org.apache.hadoop.hdfs.server.federation. - * router.RouterRpcServer RouterRpcServer} to: - * <ul> - * <li>Determine the target NN for a given subcluster. - * <li>List of all namespaces discovered/active in the federation. - * <li>Update the currently active NN empirically. - * </ul> - * The interface is also used by the {@link org.apache.hadoop.hdfs.server. - * federation.router.NamenodeHeartbeatService NamenodeHeartbeatService} to - * register a discovered NN. - */ [email protected] [email protected] -public interface ActiveNamenodeResolver { - - /** - * Report a successful, active NN address for a nameservice or blockPool. - * - * @param ns Nameservice identifier. - * @param successfulAddress The address the successful responded to the - * command. - * @throws IOException If the state store cannot be accessed. - */ - void updateActiveNamenode( - String ns, InetSocketAddress successfulAddress) throws IOException; - - /** - * Returns a prioritized list of the most recent cached registration entries - * for a single nameservice ID. - * Returns an empty list if none are found. Returns entries in preference of: - * <ul> - * <li>The most recent ACTIVE NN - * <li>The most recent STANDBY NN - * <li>The most recent UNAVAILABLE NN - * </ul> - * - * @param nameserviceId Nameservice identifier. - * @return Prioritized list of namenode contexts. - * @throws IOException If the state store cannot be accessed. - */ - List<? extends FederationNamenodeContext> - getNamenodesForNameserviceId(String nameserviceId) throws IOException; - - /** - * Returns a prioritized list of the most recent cached registration entries - * for a single block pool ID. - * Returns an empty list if none are found. Returns entries in preference of: - * <ul> - * <li>The most recent ACTIVE NN - * <li>The most recent STANDBY NN - * <li>The most recent UNAVAILABLE NN - * </ul> - * - * @param blockPoolId Block pool identifier for the nameservice. - * @return Prioritized list of namenode contexts. - * @throws IOException If the state store cannot be accessed. - */ - List<? extends FederationNamenodeContext> - getNamenodesForBlockPoolId(String blockPoolId) throws IOException; - - /** - * Register a namenode in the State Store. - * - * @param report Namenode status report. - * @return True if the node was registered and successfully committed to the - * data store. - * @throws IOException Throws exception if the namenode could not be - * registered. - */ - boolean registerNamenode(NamenodeStatusReport report) throws IOException; - - /** - * Get a list of all namespaces that are registered and active in the - * federation. - * - * @return List of name spaces in the federation - * @throws IOException Throws exception if the namespace list is not - * available. - */ - Set<FederationNamespaceInfo> getNamespaces() throws IOException; - - /** - * Assign a unique identifier for the parent router service. - * Required to report the status to the namenode resolver. - * - * @param routerId Unique string identifier for the router. - */ - void setRouterId(String routerId); -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/4aa34324/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeContext.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeContext.java deleted file mode 100644 index 68ef02a..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeContext.java +++ /dev/null @@ -1,87 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs.server.federation.resolver; - -/** - * Interface for a discovered NN and its current server endpoints. - */ -public interface FederationNamenodeContext { - - /** - * Get the RPC server address of the namenode. - * - * @return RPC server address in the form of host:port. - */ - String getRpcAddress(); - - /** - * Get the Service RPC server address of the namenode. - * - * @return Service RPC server address in the form of host:port. - */ - String getServiceAddress(); - - /** - * Get the Lifeline RPC server address of the namenode. - * - * @return Lifeline RPC server address in the form of host:port. - */ - String getLifelineAddress(); - - /** - * Get the HTTP server address of the namenode. - * - * @return HTTP address in the form of host:port. - */ - String getWebAddress(); - - /** - * Get the unique key representing the namenode. - * - * @return Combination of the nameservice and the namenode IDs. - */ - String getNamenodeKey(); - - /** - * Identifier for the nameservice/namespace. - * - * @return Namenode nameservice identifier. - */ - String getNameserviceId(); - - /** - * Identifier for the namenode. - * - * @return String - */ - String getNamenodeId(); - - /** - * The current state of the namenode (active, standby, etc). - * - * @return FederationNamenodeServiceState State of the namenode. - */ - FederationNamenodeServiceState getState(); - - /** - * The update date. - * - * @return Long with the update date. - */ - long getDateModified(); -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/4aa34324/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeServiceState.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeServiceState.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeServiceState.java deleted file mode 100644 index c773f82..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeServiceState.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs.server.federation.resolver; - -import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; - -/** - * Namenode state in the federation. The order of this enum is used to evaluate - * NN priority for RPC calls. - */ -public enum FederationNamenodeServiceState { - ACTIVE, // HAServiceState.ACTIVE or operational. - STANDBY, // HAServiceState.STANDBY. - UNAVAILABLE, // When the namenode cannot be reached. - EXPIRED; // When the last update is too old. - - public static FederationNamenodeServiceState getState(HAServiceState state) { - switch(state) { - case ACTIVE: - return FederationNamenodeServiceState.ACTIVE; - case STANDBY: - return FederationNamenodeServiceState.STANDBY; - case INITIALIZING: - return FederationNamenodeServiceState.UNAVAILABLE; - case STOPPING: - return FederationNamenodeServiceState.UNAVAILABLE; - default: - return FederationNamenodeServiceState.UNAVAILABLE; - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/4aa34324/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java deleted file mode 100644 index edcd308..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamespaceInfo.java +++ /dev/null @@ -1,73 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs.server.federation.resolver; - -import org.apache.hadoop.hdfs.server.federation.router.RemoteLocationContext; - -/** - * Represents information about a single nameservice/namespace in a federated - * HDFS cluster. - */ -public class FederationNamespaceInfo extends RemoteLocationContext { - - /** Block pool identifier. */ - private final String blockPoolId; - /** Cluster identifier. */ - private final String clusterId; - /** Nameservice identifier. */ - private final String nameserviceId; - - public FederationNamespaceInfo(String bpId, String clId, String nsId) { - this.blockPoolId = bpId; - this.clusterId = clId; - this.nameserviceId = nsId; - } - - @Override - public String getNameserviceId() { - return this.nameserviceId; - } - - @Override - public String getDest() { - return this.nameserviceId; - } - - /** - * The HDFS cluster id for this namespace. - * - * @return Cluster identifier. - */ - public String getClusterId() { - return this.clusterId; - } - - /** - * The HDFS block pool id for this namespace. - * - * @return Block pool identifier. - */ - public String getBlockPoolId() { - return this.blockPoolId; - } - - @Override - public String toString() { - return this.nameserviceId + "->" + this.blockPoolId + ":" + this.clusterId; - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/4aa34324/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FileSubclusterResolver.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FileSubclusterResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FileSubclusterResolver.java deleted file mode 100644 index af9f493..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FileSubclusterResolver.java +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hdfs.server.federation.resolver; - -import java.io.IOException; -import java.util.List; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; - -/** - * Interface to map a file path in the global name space to a specific - * subcluster and path in an HDFS name space. - * <p> - * Each path in the global/federated namespace may map to 1-N different HDFS - * locations. Each location specifies a single nameservice and a single HDFS - * path. The behavior is similar to MergeFS and Nfly and allows the merger - * of multiple HDFS locations into a single path. See HADOOP-8298 and - * HADOOP-12077 - * <p> - * For example, a directory listing will fetch listings for each destination - * path and combine them into a single set of results. - * <p> - * When multiple destinations are available for a path, the destinations are - * prioritized in a consistent manner. This allows the proxy server to - * guess the best/most likely destination and attempt it first. - */ [email protected] [email protected] -public interface FileSubclusterResolver { - - /** - * Get the destinations for a global path. Results are from the mount table - * cache. If multiple destinations are available, the first result is the - * highest priority destination. - * - * @param path Global path. - * @return Location in a destination namespace or null if it does not exist. - * @throws IOException Throws exception if the data is not available. - */ - PathLocation getDestinationForPath(String path) throws IOException; - - /** - * Get a list of mount points for a path. Results are from the mount table - * cache. - * - * @return List of mount points present at this path or zero-length list if - * none are found. - * @throws IOException Throws exception if the data is not available. - */ - List<String> getMountPoints(String path) throws IOException; - - /** - * Get the default namespace for the cluster. - * - * @return Default namespace identifier. - */ - String getDefaultNamespace(); -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/4aa34324/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java deleted file mode 100644 index 98ddd22..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java +++ /dev/null @@ -1,316 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs.server.federation.resolver; - -import static org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState.ACTIVE; -import static org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState.EXPIRED; -import static org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState.UNAVAILABLE; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.server.federation.store.MembershipStore; -import org.apache.hadoop.hdfs.server.federation.store.StateStoreCache; -import org.apache.hadoop.hdfs.server.federation.store.StateStoreService; -import org.apache.hadoop.hdfs.server.federation.store.StateStoreUnavailableException; -import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsRequest; -import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegistrationsResponse; -import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamespaceInfoRequest; -import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamespaceInfoResponse; -import org.apache.hadoop.hdfs.server.federation.store.protocol.NamenodeHeartbeatRequest; -import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateNamenodeRegistrationRequest; -import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState; -import org.apache.hadoop.hdfs.server.federation.store.records.MembershipStats; -import org.apache.hadoop.util.Time; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Implements a cached lookup of the most recently active namenode for a - * particular nameservice. Relies on the {@link StateStoreService} to - * discover available nameservices and namenodes. - */ -public class MembershipNamenodeResolver - implements ActiveNamenodeResolver, StateStoreCache { - - private static final Logger LOG = - LoggerFactory.getLogger(MembershipNamenodeResolver.class); - - /** Reference to the State Store. */ - private final StateStoreService stateStore; - /** Membership State Store interface. */ - private MembershipStore membershipInterface; - - /** Parent router ID. */ - private String routerId; - - /** Cached lookup of NN for nameservice. Invalidated on cache refresh. */ - private Map<String, List<? extends FederationNamenodeContext>> cacheNS; - /** Cached lookup of NN for block pool. Invalidated on cache refresh. */ - private Map<String, List<? extends FederationNamenodeContext>> cacheBP; - - - public MembershipNamenodeResolver( - Configuration conf, StateStoreService store) throws IOException { - this.stateStore = store; - - this.cacheNS = new ConcurrentHashMap<>(); - this.cacheBP = new ConcurrentHashMap<>(); - - if (this.stateStore != null) { - // Request cache updates from the state store - this.stateStore.registerCacheExternal(this); - } - } - - private synchronized MembershipStore getMembershipStore() throws IOException { - if (this.membershipInterface == null) { - this.membershipInterface = this.stateStore.getRegisteredRecordStore( - MembershipStore.class); - if (this.membershipInterface == null) { - throw new IOException("State Store does not have an interface for " + - MembershipStore.class.getSimpleName()); - } - } - return this.membershipInterface; - } - - @Override - public boolean loadCache(boolean force) { - // Our cache depends on the store, update it first - try { - MembershipStore membership = getMembershipStore(); - membership.loadCache(force); - } catch (IOException e) { - LOG.error("Cannot update membership from the State Store", e); - } - - // Force refresh of active NN cache - cacheBP.clear(); - cacheNS.clear(); - return true; - } - - @Override - public void updateActiveNamenode( - final String nsId, final InetSocketAddress address) throws IOException { - - // Called when we have an RPC miss and successful hit on an alternate NN. - // Temporarily update our cache, it will be overwritten on the next update. - try { - MembershipState partial = MembershipState.newInstance(); - String rpcAddress = address.getHostName() + ":" + address.getPort(); - partial.setRpcAddress(rpcAddress); - partial.setNameserviceId(nsId); - - GetNamenodeRegistrationsRequest request = - GetNamenodeRegistrationsRequest.newInstance(partial); - - MembershipStore membership = getMembershipStore(); - GetNamenodeRegistrationsResponse response = - membership.getNamenodeRegistrations(request); - List<MembershipState> records = response.getNamenodeMemberships(); - - if (records != null && records.size() == 1) { - MembershipState record = records.get(0); - UpdateNamenodeRegistrationRequest updateRequest = - UpdateNamenodeRegistrationRequest.newInstance( - record.getNameserviceId(), record.getNamenodeId(), ACTIVE); - membership.updateNamenodeRegistration(updateRequest); - } - } catch (StateStoreUnavailableException e) { - LOG.error("Cannot update {} as active, State Store unavailable", address); - } - } - - @Override - public List<? extends FederationNamenodeContext> getNamenodesForNameserviceId( - final String nsId) throws IOException { - - List<? extends FederationNamenodeContext> ret = cacheNS.get(nsId); - if (ret == null) { - try { - MembershipState partial = MembershipState.newInstance(); - partial.setNameserviceId(nsId); - GetNamenodeRegistrationsRequest request = - GetNamenodeRegistrationsRequest.newInstance(partial); - - final List<MembershipState> result = - getRecentRegistrationForQuery(request, true, false); - if (result == null || result.isEmpty()) { - LOG.error("Cannot locate eligible NNs for {}", nsId); - return null; - } else { - cacheNS.put(nsId, result); - ret = result; - } - } catch (StateStoreUnavailableException e) { - LOG.error("Cannot get active NN for {}, State Store unavailable", nsId); - } - } - if (ret == null) { - return null; - } - return Collections.unmodifiableList(ret); - } - - @Override - public List<? extends FederationNamenodeContext> getNamenodesForBlockPoolId( - final String bpId) throws IOException { - - List<? extends FederationNamenodeContext> ret = cacheBP.get(bpId); - if (ret == null) { - try { - MembershipState partial = MembershipState.newInstance(); - partial.setBlockPoolId(bpId); - GetNamenodeRegistrationsRequest request = - GetNamenodeRegistrationsRequest.newInstance(partial); - - final List<MembershipState> result = - getRecentRegistrationForQuery(request, true, false); - if (result == null || result.isEmpty()) { - LOG.error("Cannot locate eligible NNs for {}", bpId); - } else { - cacheBP.put(bpId, result); - ret = result; - } - } catch (StateStoreUnavailableException e) { - LOG.error("Cannot get active NN for {}, State Store unavailable", bpId); - return null; - } - } - if (ret == null) { - return null; - } - return Collections.unmodifiableList(ret); - } - - @Override - public boolean registerNamenode(NamenodeStatusReport report) - throws IOException { - - if (this.routerId == null) { - LOG.warn("Cannot register namenode, router ID is not known {}", report); - return false; - } - - MembershipState record = MembershipState.newInstance( - routerId, report.getNameserviceId(), report.getNamenodeId(), - report.getClusterId(), report.getBlockPoolId(), report.getRpcAddress(), - report.getServiceAddress(), report.getLifelineAddress(), - report.getWebAddress(), report.getState(), report.getSafemode()); - - if (report.statsValid()) { - MembershipStats stats = MembershipStats.newInstance(); - stats.setNumOfFiles(report.getNumFiles()); - stats.setNumOfBlocks(report.getNumBlocks()); - stats.setNumOfBlocksMissing(report.getNumBlocksMissing()); - stats.setNumOfBlocksPendingReplication( - report.getNumOfBlocksPendingReplication()); - stats.setNumOfBlocksUnderReplicated( - report.getNumOfBlocksUnderReplicated()); - stats.setNumOfBlocksPendingDeletion( - report.getNumOfBlocksPendingDeletion()); - stats.setAvailableSpace(report.getAvailableSpace()); - stats.setTotalSpace(report.getTotalSpace()); - stats.setNumOfDecommissioningDatanodes( - report.getNumDecommissioningDatanodes()); - stats.setNumOfActiveDatanodes(report.getNumLiveDatanodes()); - stats.setNumOfDeadDatanodes(report.getNumDeadDatanodes()); - stats.setNumOfDecomActiveDatanodes(report.getNumDecomLiveDatanodes()); - stats.setNumOfDecomDeadDatanodes(report.getNumDecomDeadDatanodes()); - record.setStats(stats); - } - - if (report.getState() != UNAVAILABLE) { - // Set/update our last contact time - record.setLastContact(Time.now()); - } - - NamenodeHeartbeatRequest request = NamenodeHeartbeatRequest.newInstance(); - request.setNamenodeMembership(record); - return getMembershipStore().namenodeHeartbeat(request).getResult(); - } - - @Override - public Set<FederationNamespaceInfo> getNamespaces() throws IOException { - GetNamespaceInfoRequest request = GetNamespaceInfoRequest.newInstance(); - GetNamespaceInfoResponse response = - getMembershipStore().getNamespaceInfo(request); - return response.getNamespaceInfo(); - } - - /** - * Picks the most relevant record registration that matches the query. Return - * registrations matching the query in this preference: 1) Most recently - * updated ACTIVE registration 2) Most recently updated STANDBY registration - * (if showStandby) 3) Most recently updated UNAVAILABLE registration (if - * showUnavailable). EXPIRED registrations are ignored. - * - * @param request The select query for NN registrations. - * @param addUnavailable include UNAVAILABLE registrations. - * @param addExpired include EXPIRED registrations. - * @return List of memberships or null if no registrations that - * both match the query AND the selected states. - * @throws IOException - */ - private List<MembershipState> getRecentRegistrationForQuery( - GetNamenodeRegistrationsRequest request, boolean addUnavailable, - boolean addExpired) throws IOException { - - // Retrieve a list of all registrations that match this query. - // This may include all NN records for a namespace/blockpool, including - // duplicate records for the same NN from different routers. - MembershipStore membershipStore = getMembershipStore(); - GetNamenodeRegistrationsResponse response = - membershipStore.getNamenodeRegistrations(request); - - List<MembershipState> memberships = response.getNamenodeMemberships(); - if (!addExpired || !addUnavailable) { - Iterator<MembershipState> iterator = memberships.iterator(); - while (iterator.hasNext()) { - MembershipState membership = iterator.next(); - if (membership.getState() == EXPIRED && !addExpired) { - iterator.remove(); - } else if (membership.getState() == UNAVAILABLE && !addUnavailable) { - iterator.remove(); - } - } - } - - List<MembershipState> priorityList = new ArrayList<>(); - priorityList.addAll(memberships); - Collections.sort(priorityList, new NamenodePriorityComparator()); - - LOG.debug("Selected most recent NN {} for query", priorityList); - return priorityList; - } - - @Override - public void setRouterId(String router) { - this.routerId = router; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/4aa34324/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableManager.java deleted file mode 100644 index c2e4a5b..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableManager.java +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs.server.federation.resolver; - -import java.io.IOException; - -import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest; -import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse; -import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest; -import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse; -import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest; -import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse; -import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest; -import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryResponse; - -/** - * Manage a mount table. - */ -public interface MountTableManager { - - /** - * Add an entry to the mount table. - * - * @param request Fully populated request object. - * @return True if the mount table entry was successfully committed to the - * data store. - * @throws IOException Throws exception if the data store is not initialized. - */ - AddMountTableEntryResponse addMountTableEntry( - AddMountTableEntryRequest request) throws IOException; - - /** - * Updates an existing entry in the mount table. - * - * @param request Fully populated request object. - * @return True if the mount table entry was successfully committed to the - * data store. - * @throws IOException Throws exception if the data store is not initialized. - */ - UpdateMountTableEntryResponse updateMountTableEntry( - UpdateMountTableEntryRequest request) throws IOException; - - /** - * Remove an entry from the mount table. - * - * @param request Fully populated request object. - * @return True the mount table entry was removed from the data store. - * @throws IOException Throws exception if the data store is not initialized. - */ - RemoveMountTableEntryResponse removeMountTableEntry( - RemoveMountTableEntryRequest request) throws IOException; - - /** - * List all mount table entries present at or below the path. Fetches from the - * state store. - * - * @param request Fully populated request object. - * - * @return List of all mount table entries under the path. Zero-length list if - * none are found. - * @throws IOException Throws exception if the data store cannot be queried. - */ - GetMountTableEntriesResponse getMountTableEntries( - GetMountTableEntriesRequest request) throws IOException; -} \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
