http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestRDBTableStore.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestRDBTableStore.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestRDBTableStore.java deleted file mode 100644 index cd25548..0000000 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestRDBTableStore.java +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.utils.db; - -import org.apache.commons.lang3.RandomStringUtils; -import org.apache.hadoop.hdfs.DFSUtil; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.rocksdb.ColumnFamilyOptions; -import org.rocksdb.DBOptions; -import org.rocksdb.RocksDB; -import org.rocksdb.Statistics; -import org.rocksdb.StatsLevel; -import org.rocksdb.WriteBatch; - -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; - -/** - * Tests for RocksDBTable Store. - */ -public class TestRDBTableStore { - private static int count = 0; - private final List<String> families = - Arrays.asList(DFSUtil.bytes2String(RocksDB.DEFAULT_COLUMN_FAMILY), - "First", "Second", "Third", - "Fourth", "Fifth", - "Sixth"); - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - private RDBStore rdbStore = null; - private DBOptions options = null; - - @Before - public void setUp() throws Exception { - options = new DBOptions(); - options.setCreateIfMissing(true); - options.setCreateMissingColumnFamilies(true); - - Statistics statistics = new Statistics(); - statistics.setStatsLevel(StatsLevel.ALL); - options = options.setStatistics(statistics); - - Set<TableConfig> configSet = new HashSet<>(); - for(String name : families) { - TableConfig newConfig = new TableConfig(name, new ColumnFamilyOptions()); - configSet.add(newConfig); - } - rdbStore = new RDBStore(folder.newFolder(), options, configSet); - } - - @After - public void tearDown() throws Exception { - if (rdbStore != null) { - rdbStore.close(); - } - } - - @Test - public void toIOException() { - } - - @Test - public void getHandle() throws Exception { - try (Table testTable = rdbStore.getTable("First")) { - Assert.assertNotNull(testTable); - Assert.assertNotNull(testTable.getHandle()); - } - } - - @Test - public void putGetAndEmpty() throws Exception { - try (Table testTable = rdbStore.getTable("First")) { - byte[] key = - RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); - byte[] value = - RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); - testTable.put(key, value); - Assert.assertFalse(testTable.isEmpty()); - byte[] readValue = testTable.get(key); - Assert.assertArrayEquals(value, readValue); - } - try (Table secondTable = rdbStore.getTable("Second")) { - Assert.assertTrue(secondTable.isEmpty()); - } - } - - @Test - public void delete() throws Exception { - List<byte[]> deletedKeys = new LinkedList<>(); - List<byte[]> validKeys = new LinkedList<>(); - byte[] value = - RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); - for (int x = 0; x < 100; x++) { - deletedKeys.add( - RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8)); - } - - for (int x = 0; x < 100; x++) { - validKeys.add( - RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8)); - } - - // Write all the keys and delete the keys scheduled for delete. - //Assert we find only expected keys in the Table. - try (Table testTable = rdbStore.getTable("Fourth")) { - for (int x = 0; x < deletedKeys.size(); x++) { - testTable.put(deletedKeys.get(x), value); - testTable.delete(deletedKeys.get(x)); - } - - for (int x = 0; x < validKeys.size(); x++) { - testTable.put(validKeys.get(x), value); - } - - for (int x = 0; x < validKeys.size(); x++) { - Assert.assertNotNull(testTable.get(validKeys.get(0))); - } - - for (int x = 0; x < deletedKeys.size(); x++) { - Assert.assertNull(testTable.get(deletedKeys.get(0))); - } - } - } - - @Test - public void writeBatch() throws Exception { - WriteBatch batch = new WriteBatch(); - try (Table testTable = rdbStore.getTable("Fifth")) { - byte[] key = - RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); - byte[] value = - RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); - batch.put(testTable.getHandle(), key, value); - testTable.writeBatch(batch); - Assert.assertNotNull(testTable.get(key)); - } - batch.close(); - } - - private static boolean consume(Table.KeyValue keyValue) { - count++; - Assert.assertNotNull(keyValue.getKey()); - return true; - } - - @Test - public void forEachAndIterator() throws Exception { - final int iterCount = 100; - try (Table testTable = rdbStore.getTable("Sixth")) { - for (int x = 0; x < iterCount; x++) { - byte[] key = - RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); - byte[] value = - RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8); - testTable.put(key, value); - } - int localCount = 0; - try (TableIterator<Table.KeyValue> iter = testTable.iterator()) { - while (iter.hasNext()) { - Table.KeyValue keyValue = iter.next(); - localCount++; - } - - Assert.assertEquals(iterCount, localCount); - iter.seekToFirst(); - iter.forEachRemaining(TestRDBTableStore::consume); - Assert.assertEquals(iterCount, count); - - } - } - } -} \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/package-info.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/package-info.java deleted file mode 100644 index f06855e..0000000 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/package-info.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -/** - * Tests for the DB Utilities. - */ -package org.apache.hadoop.utils.db; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/common/src/test/resources/log4j2.properties ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/test/resources/log4j2.properties b/hadoop-hdds/common/src/test/resources/log4j2.properties deleted file mode 100644 index cef69e1..0000000 --- a/hadoop-hdds/common/src/test/resources/log4j2.properties +++ /dev/null @@ -1,76 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with this -# work for additional information regarding copyright ownership. The ASF -# licenses this file to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# <p> -# http://www.apache.org/licenses/LICENSE-2.0 -# <p> -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS,WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# -name=PropertiesConfig - -# Checks for config change periodically and reloads -monitorInterval=5 - -filter=read, write -# filter.read.onMatch = DENY avoids logging all READ events -# filter.read.onMatch = ACCEPT permits logging all READ events -# The above two settings ignore the log levels in configuration -# filter.read.onMatch = NEUTRAL permits logging of only those READ events -# which are attempted at log level equal or greater than log level specified -# in the configuration -filter.read.type = MarkerFilter -filter.read.marker = READ -filter.read.onMatch = DENY -filter.read.onMismatch = NEUTRAL - -# filter.write.onMatch = DENY avoids logging all WRITE events -# filter.write.onMatch = ACCEPT permits logging all WRITE events -# The above two settings ignore the log levels in configuration -# filter.write.onMatch = NEUTRAL permits logging of only those WRITE events -# which are attempted at log level equal or greater than log level specified -# in the configuration -filter.write.type = MarkerFilter -filter.write.marker = WRITE -filter.write.onMatch = NEUTRAL -filter.write.onMismatch = NEUTRAL - -# Log Levels are organized from most specific to least: -# OFF (most specific, no logging) -# FATAL (most specific, little data) -# ERROR -# WARN -# INFO -# DEBUG -# TRACE (least specific, a lot of data) -# ALL (least specific, all data) - -appenders = console, audit -appender.console.type = Console -appender.console.name = STDOUT -appender.console.layout.type = PatternLayout -appender.console.layout.pattern = %-5level | %c{1} | %msg%n - -appender.audit.type = File -appender.audit.name = AUDITLOG -appender.audit.fileName=audit.log -appender.audit.layout.type=PatternLayout -appender.audit.layout.pattern= %-5level | %c{1} | %msg%n - -loggers=audit -logger.audit.type=AsyncLogger -logger.audit.name=OMAudit -logger.audit.level = INFO -logger.audit.appenderRefs = audit -logger.audit.appenderRef.file.ref = AUDITLOG - -rootLogger.level = INFO -rootLogger.appenderRefs = stdout -rootLogger.appenderRef.stdout.ref = STDOUT http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/common/src/test/resources/test.db.ini ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/test/resources/test.db.ini b/hadoop-hdds/common/src/test/resources/test.db.ini deleted file mode 100644 index 6666cd2..0000000 --- a/hadoop-hdds/common/src/test/resources/test.db.ini +++ /dev/null @@ -1,145 +0,0 @@ -# This is a RocksDB option file. -# -# A typical RocksDB options file has four sections, which are -# Version section, DBOptions section, at least one CFOptions -# section, and one TableOptions section for each column family. -# The RocksDB options file in general follows the basic INI -# file format with the following extensions / modifications: -# -# * Escaped characters -# We escaped the following characters: -# - \n -- line feed - new line -# - \r -- carriage return -# - \\ -- backslash \ -# - \: -- colon symbol : -# - \# -- hash tag # -# * Comments -# We support # style comments. Comments can appear at the ending -# part of a line. -# * Statements -# A statement is of the form option_name = value. -# Each statement contains a '=', where extra white-spaces -# are supported. However, we don't support multi-lined statement. -# Furthermore, each line can only contain at most one statement. -# * Sections -# Sections are of the form [SecitonTitle "SectionArgument"], -# where section argument is optional. -# * List -# We use colon-separated string to represent a list. -# For instance, n1:n2:n3:n4 is a list containing four values. -# -# Below is an example of a RocksDB options file: - - -#----------------------IMPORTANT------------------------------------# -### FAKE VALUES FOR TESTING ONLY ### DO NOT USE THESE FOR PRODUCTION. -#----------------------IMPORTANT------------------------------------# -[DBOptions] - stats_dump_period_sec=600 - max_manifest_file_size=551615 - bytes_per_sync=8388608 - delayed_write_rate=2097152 - WAL_ttl_seconds=0 - WAL_size_limit_MB=0 - max_subcompactions=1 - wal_dir= - wal_bytes_per_sync=0 - db_write_buffer_size=0 - keep_log_file_num=1000 - table_cache_numshardbits=4 - max_file_opening_threads=1 - writable_file_max_buffer_size=1048576 - random_access_max_buffer_size=1048576 - use_fsync=false - max_total_wal_size=0 - max_open_files=-1 - skip_stats_update_on_db_open=false - max_background_compactions=16 - manifest_preallocation_size=4194304 - max_background_flushes=7 - is_fd_close_on_exec=true - max_log_file_size=0 - advise_random_on_open=true - create_missing_column_families=false - paranoid_checks=true - delete_obsolete_files_period_micros=21600000000 - log_file_time_to_roll=0 - compaction_readahead_size=0 - create_if_missing=false - use_adaptive_mutex=false - enable_thread_tracking=false - allow_fallocate=true - error_if_exists=false - recycle_log_file_num=0 - skip_log_error_on_recovery=false - db_log_dir= - new_table_reader_for_compaction_inputs=true - allow_mmap_reads=false - allow_mmap_writes=false - use_direct_reads=false - use_direct_writes=false - - -[CFOptions "default"] - compaction_style=kCompactionStyleLevel - compaction_filter=nullptr - num_levels=6 - table_factory=BlockBasedTable - comparator=leveldb.BytewiseComparator - max_sequential_skip_in_iterations=8 - soft_rate_limit=0.000000 - max_bytes_for_level_base=1073741824 - memtable_prefix_bloom_probes=6 - memtable_prefix_bloom_bits=0 - memtable_prefix_bloom_huge_page_tlb_size=0 - max_successive_merges=0 - arena_block_size=16777216 - min_write_buffer_number_to_merge=1 - target_file_size_multiplier=1 - source_compaction_factor=1 - max_bytes_for_level_multiplier=8 - max_bytes_for_level_multiplier_additional=2:3:5 - compaction_filter_factory=nullptr - max_write_buffer_number=8 - level0_stop_writes_trigger=20 - compression=kSnappyCompression - level0_file_num_compaction_trigger=4 - purge_redundant_kvs_while_flush=true - max_write_buffer_number_to_maintain=0 - memtable_factory=SkipListFactory - max_grandparent_overlap_factor=8 - expanded_compaction_factor=25 - hard_pending_compaction_bytes_limit=137438953472 - inplace_update_num_locks=10000 - level_compaction_dynamic_level_bytes=true - level0_slowdown_writes_trigger=12 - filter_deletes=false - verify_checksums_in_compaction=true - min_partial_merge_operands=2 - paranoid_file_checks=false - target_file_size_base=134217728 - optimize_filters_for_hits=false - merge_operator=PutOperator - compression_per_level=kNoCompression:kNoCompression:kNoCompression:kSnappyCompression:kSnappyCompression:kSnappyCompression - compaction_measure_io_stats=false - prefix_extractor=nullptr - bloom_locality=0 - write_buffer_size=134217728 - disable_auto_compactions=false - inplace_update_support=false - -[TableOptions/BlockBasedTable "default"] - format_version=2 - whole_key_filtering=true - no_block_cache=false - checksum=kCRC32c - filter_policy=rocksdb.BuiltinBloomFilter - block_size_deviation=10 - block_size=8192 - block_restart_interval=16 - cache_index_and_filter_blocks=false - pin_l0_filter_and_index_blocks_in_cache=false - pin_top_level_index_and_filter=false - index_type=kBinarySearch - hash_index_allow_collision=true - flush_block_policy_factory=FlushBlockBySizePolicyFactory \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/dev-support/findbugsExcludeFile.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/dev-support/findbugsExcludeFile.xml b/hadoop-hdds/container-service/dev-support/findbugsExcludeFile.xml deleted file mode 100644 index 3571a89..0000000 --- a/hadoop-hdds/container-service/dev-support/findbugsExcludeFile.xml +++ /dev/null @@ -1,21 +0,0 @@ -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<FindBugsFilter> - <Match> - <Package name="org.apache.hadoop.hdds.protocol.proto"/> - </Match> -</FindBugsFilter> http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/pom.xml b/hadoop-hdds/container-service/pom.xml deleted file mode 100644 index 54c5fad..0000000 --- a/hadoop-hdds/container-service/pom.xml +++ /dev/null @@ -1,109 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. See accompanying LICENSE file. ---> -<project xmlns="http://maven.apache.org/POM/4.0.0"; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 -http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-hdds</artifactId> - <version>0.3.0-SNAPSHOT</version> - </parent> - <artifactId>hadoop-hdds-container-service</artifactId> - <version>0.3.0-SNAPSHOT</version> - <description>Apache Hadoop Distributed Data Store Container Service</description> - <name>Apache Hadoop HDDS Container Service</name> - <packaging>jar</packaging> - - <dependencies> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-hdds-common</artifactId> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-hdds-server-framework</artifactId> - </dependency> - - <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-core</artifactId> - <version>2.2.0</version> - <scope>test</scope> - </dependency> - - <dependency> - <groupId>org.yaml</groupId> - <artifactId>snakeyaml</artifactId> - <version>1.8</version> - </dependency> - - <dependency> - <groupId>io.dropwizard.metrics</groupId> - <artifactId>metrics-core</artifactId> - <scope>test</scope> - </dependency> - </dependencies> - - <build> - <plugins> - <plugin> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-maven-plugins</artifactId> - <executions> - <execution> - <id>compile-protoc</id> - <goals> - <goal>protoc</goal> - </goals> - <configuration> - <protocVersion>${protobuf.version}</protocVersion> - <protocCommand>${protoc.path}</protocCommand> - <imports> - <param> - ${basedir}/../../hadoop-common-project/hadoop-common/src/main/proto - </param> - <param> - ${basedir}/../../hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ - </param> - <param> - ${basedir}/../../hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ - </param> - <param> - ${basedir}/../../hadoop-hdds/common/src/main/proto/ - </param> - <param>${basedir}/src/main/proto</param> - </imports> - <source> - <directory>${basedir}/src/main/proto</directory> - <includes> - <include>StorageContainerDatanodeProtocol.proto</include> - </includes> - </source> - </configuration> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>findbugs-maven-plugin</artifactId> - <configuration> - <excludeFilterFile>${basedir}/dev-support/findbugsExcludeFile.xml</excludeFilterFile> - </configuration> - </plugin> - </plugins> - </build> -</project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java deleted file mode 100644 index d505be3..0000000 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java +++ /dev/null @@ -1,336 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.hadoop.hdds.scm; - -import com.google.common.base.Optional; -import com.google.common.base.Preconditions; -import com.google.common.base.Strings; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.ozone.OzoneConfigKeys; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.net.InetSocketAddress; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import static org.apache.hadoop.hdds.HddsConfigKeys - .HDDS_HEARTBEAT_INTERVAL; -import static org.apache.hadoop.hdds.HddsConfigKeys - .HDDS_HEARTBEAT_INTERVAL_DEFAULT; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys - .OZONE_SCM_DEADNODE_INTERVAL; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys - .OZONE_SCM_DEADNODE_INTERVAL_DEFAULT; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys - .OZONE_SCM_HEARTBEAT_LOG_WARN_DEFAULT; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys - .OZONE_SCM_HEARTBEAT_LOG_WARN_INTERVAL_COUNT; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys - .OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys - .OZONE_SCM_HEARTBEAT_RPC_TIMEOUT; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys - .OZONE_SCM_HEARTBEAT_RPC_TIMEOUT_DEFAULT; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys - .OZONE_SCM_STALENODE_INTERVAL; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys - .OZONE_SCM_STALENODE_INTERVAL_DEFAULT; -import static org.apache.hadoop.hdds.HddsUtils.*; -import static org.apache.hadoop.hdds.server.ServerUtils.sanitizeUserArgs; - -/** - * Hdds stateless helper functions for server side components. - */ -public final class HddsServerUtil { - - private HddsServerUtil() { - } - - private static final Logger LOG = LoggerFactory.getLogger( - HddsServerUtil.class); - - /** - * Retrieve the socket address that should be used by DataNodes to connect - * to the SCM. - * - * @param conf - * @return Target InetSocketAddress for the SCM service endpoint. - */ - public static InetSocketAddress getScmAddressForDataNodes( - Configuration conf) { - // We try the following settings in decreasing priority to retrieve the - // target host. - // - OZONE_SCM_DATANODE_ADDRESS_KEY - // - OZONE_SCM_CLIENT_ADDRESS_KEY - // - final Optional<String> host = getHostNameFromConfigKeys(conf, - ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, - ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY); - - if (!host.isPresent()) { - throw new IllegalArgumentException( - ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY + - " must be defined. See" + - " https://wiki.apache.org/hadoop/Ozone#Configuration " - + "for details on configuring Ozone."); - } - - // If no port number is specified then we'll just try the defaultBindPort. - final Optional<Integer> port = getPortNumberFromConfigKeys(conf, - ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY); - - InetSocketAddress addr = NetUtils.createSocketAddr(host.get() + ":" + - port.or(ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT)); - - return addr; - } - - /** - * Retrieve the socket address that should be used by clients to connect - * to the SCM. - * - * @param conf - * @return Target InetSocketAddress for the SCM client endpoint. - */ - public static InetSocketAddress getScmClientBindAddress( - Configuration conf) { - final Optional<String> host = getHostNameFromConfigKeys(conf, - ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_KEY); - - final Optional<Integer> port = getPortNumberFromConfigKeys(conf, - ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY); - - return NetUtils.createSocketAddr( - host.or(ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_DEFAULT) + ":" + - port.or(ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT)); - } - - /** - * Retrieve the socket address that should be used by clients to connect - * to the SCM Block service. - * - * @param conf - * @return Target InetSocketAddress for the SCM block client endpoint. - */ - public static InetSocketAddress getScmBlockClientBindAddress( - Configuration conf) { - final Optional<String> host = getHostNameFromConfigKeys(conf, - ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_BIND_HOST_KEY); - - final Optional<Integer> port = getPortNumberFromConfigKeys(conf, - ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY); - - return NetUtils.createSocketAddr( - host.or(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_BIND_HOST_DEFAULT) + - ":" + port.or(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT)); - } - - /** - * Retrieve the socket address that should be used by DataNodes to connect - * to the SCM. - * - * @param conf - * @return Target InetSocketAddress for the SCM service endpoint. - */ - public static InetSocketAddress getScmDataNodeBindAddress( - Configuration conf) { - final Optional<String> host = getHostNameFromConfigKeys(conf, - ScmConfigKeys.OZONE_SCM_DATANODE_BIND_HOST_KEY); - - // If no port number is specified then we'll just try the defaultBindPort. - final Optional<Integer> port = getPortNumberFromConfigKeys(conf, - ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY); - - return NetUtils.createSocketAddr( - host.or(ScmConfigKeys.OZONE_SCM_DATANODE_BIND_HOST_DEFAULT) + ":" + - port.or(ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT)); - } - - - /** - * Returns the interval in which the heartbeat processor thread runs. - * - * @param conf - Configuration - * @return long in Milliseconds. - */ - public static long getScmheartbeatCheckerInterval(Configuration conf) { - return conf.getTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, - ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_DEFAULT, - TimeUnit.MILLISECONDS); - } - - /** - * Heartbeat Interval - Defines the heartbeat frequency from a datanode to - * SCM. - * - * @param conf - Ozone Config - * @return - HB interval in milli seconds. - */ - public static long getScmHeartbeatInterval(Configuration conf) { - return conf.getTimeDuration(HDDS_HEARTBEAT_INTERVAL, - HDDS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS); - } - - /** - * Get the Stale Node interval, which is used by SCM to flag a datanode as - * stale, if the heartbeat from that node has been missing for this duration. - * - * @param conf - Configuration. - * @return - Long, Milliseconds to wait before flagging a node as stale. - */ - public static long getStaleNodeInterval(Configuration conf) { - - long staleNodeIntervalMs = - conf.getTimeDuration(OZONE_SCM_STALENODE_INTERVAL, - OZONE_SCM_STALENODE_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS); - - long heartbeatThreadFrequencyMs = getScmheartbeatCheckerInterval(conf); - - long heartbeatIntervalMs = getScmHeartbeatInterval(conf); - - - // Make sure that StaleNodeInterval is configured way above the frequency - // at which we run the heartbeat thread. - // - // Here we check that staleNodeInterval is at least five times more than the - // frequency at which the accounting thread is going to run. - try { - sanitizeUserArgs(staleNodeIntervalMs, heartbeatThreadFrequencyMs, - 5, 1000); - } catch (IllegalArgumentException ex) { - LOG.error("Stale Node Interval is cannot be honored due to " + - "mis-configured {}. ex: {}", - OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, ex); - throw ex; - } - - // Make sure that stale node value is greater than configured value that - // datanodes are going to send HBs. - try { - sanitizeUserArgs(staleNodeIntervalMs, heartbeatIntervalMs, 3, 1000); - } catch (IllegalArgumentException ex) { - LOG.error("Stale Node Interval MS is cannot be honored due to " + - "mis-configured {}. ex: {}", HDDS_HEARTBEAT_INTERVAL, ex); - throw ex; - } - return staleNodeIntervalMs; - } - - /** - * Gets the interval for dead node flagging. This has to be a value that is - * greater than stale node value, and by transitive relation we also know - * that this value is greater than heartbeat interval and heartbeatProcess - * Interval. - * - * @param conf - Configuration. - * @return - the interval for dead node flagging. - */ - public static long getDeadNodeInterval(Configuration conf) { - long staleNodeIntervalMs = getStaleNodeInterval(conf); - long deadNodeIntervalMs = conf.getTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, - OZONE_SCM_DEADNODE_INTERVAL_DEFAULT, - TimeUnit.MILLISECONDS); - - try { - // Make sure that dead nodes Ms is at least twice the time for staleNodes - // with a max of 1000 times the staleNodes. - sanitizeUserArgs(deadNodeIntervalMs, staleNodeIntervalMs, 2, 1000); - } catch (IllegalArgumentException ex) { - LOG.error("Dead Node Interval MS is cannot be honored due to " + - "mis-configured {}. ex: {}", OZONE_SCM_STALENODE_INTERVAL, ex); - throw ex; - } - return deadNodeIntervalMs; - } - - /** - * Timeout value for the RPC from Datanode to SCM, primarily used for - * Heartbeats and container reports. - * - * @param conf - Ozone Config - * @return - Rpc timeout in Milliseconds. - */ - public static long getScmRpcTimeOutInMilliseconds(Configuration conf) { - return conf.getTimeDuration(OZONE_SCM_HEARTBEAT_RPC_TIMEOUT, - OZONE_SCM_HEARTBEAT_RPC_TIMEOUT_DEFAULT, TimeUnit.MILLISECONDS); - } - - /** - * Log Warn interval. - * - * @param conf - Ozone Config - * @return - Log warn interval. - */ - public static int getLogWarnInterval(Configuration conf) { - return conf.getInt(OZONE_SCM_HEARTBEAT_LOG_WARN_INTERVAL_COUNT, - OZONE_SCM_HEARTBEAT_LOG_WARN_DEFAULT); - } - - /** - * returns the Container port. - * @param conf - Conf - * @return port number. - */ - public static int getContainerPort(Configuration conf) { - return conf.getInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT, - OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT); - } - - - /** - * Return the list of service addresses for the Ozone SCM. This method is used - * by the DataNodes to determine the service instances to connect to. - * - * @param conf - * @return list of SCM service addresses. - */ - public static Map<String, ? extends Map<String, InetSocketAddress>> - getScmServiceRpcAddresses(Configuration conf) { - - final Map<String, InetSocketAddress> serviceInstances = new HashMap<>(); - serviceInstances.put(OZONE_SCM_SERVICE_INSTANCE_ID, - getScmAddressForDataNodes(conf)); - - final Map<String, Map<String, InetSocketAddress>> services = - new HashMap<>(); - services.put(OZONE_SCM_SERVICE_ID, serviceInstances); - return services; - } - - public static String getOzoneDatanodeRatisDirectory(Configuration conf) { - final String ratisDir = File.separator + "ratis"; - String storageDir = conf.get( - OzoneConfigKeys.DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR); - - if (Strings.isNullOrEmpty(storageDir)) { - storageDir = conf.get(OzoneConfigKeys - .OZONE_METADATA_DIRS); - Preconditions.checkNotNull(storageDir, "ozone.metadata.dirs " + - "cannot be null, Please check your configs."); - storageDir = storageDir.concat(ratisDir); - LOG.warn("Storage directory for Ratis is not configured." + - "Mapping Ratis storage under {}. It is a good idea " + - "to map this to an SSD disk.", storageDir); - } - return storageDir; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/VersionInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/VersionInfo.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/VersionInfo.java deleted file mode 100644 index 4e52046..0000000 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/VersionInfo.java +++ /dev/null @@ -1,81 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdds.scm; - -/** - * This is a class that tracks versions of SCM. - */ -public final class VersionInfo { - - // We will just be normal and use positive counting numbers for versions. - private final static VersionInfo[] VERSION_INFOS = - {new VersionInfo("First version of SCM", 1)}; - - - public static final String DESCRIPTION_KEY = "Description"; - private final String description; - private final int version; - - /** - * Never created outside this class. - * - * @param description -- description - * @param version -- version number - */ - private VersionInfo(String description, int version) { - this.description = description; - this.version = version; - } - - /** - * Returns all versions. - * - * @return Version info array. - */ - public static VersionInfo[] getAllVersions() { - return VERSION_INFOS.clone(); - } - - /** - * Returns the latest version. - * - * @return versionInfo - */ - public static VersionInfo getLatestVersion() { - return VERSION_INFOS[VERSION_INFOS.length - 1]; - } - - /** - * Return description. - * - * @return String - */ - public String getDescription() { - return description; - } - - /** - * Return the version. - * - * @return int. - */ - public int getVersion() { - return version; - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/package-info.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/package-info.java deleted file mode 100644 index 5905468..0000000 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/package-info.java +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hdds.scm; http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java deleted file mode 100644 index 348196c..0000000 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java +++ /dev/null @@ -1,256 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdds.scm.ScmConfigKeys; -import org.apache.hadoop.hdds.HddsUtils; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdfs.DFSUtil; -import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; -import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils; -import org.apache.hadoop.ozone.container.common.statemachine - .DatanodeStateMachine; -import org.apache.hadoop.util.GenericOptionsParser; -import org.apache.hadoop.util.ServicePlugin; -import org.apache.hadoop.util.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.net.InetAddress; -import java.util.List; -import java.util.UUID; - -import static org.apache.hadoop.ozone.OzoneConfigKeys.HDDS_DATANODE_PLUGINS_KEY; -import static org.apache.hadoop.util.ExitUtil.terminate; - -/** - * Datanode service plugin to start the HDDS container services. - */ -public class HddsDatanodeService implements ServicePlugin { - - private static final Logger LOG = LoggerFactory.getLogger( - HddsDatanodeService.class); - - - private OzoneConfiguration conf; - private DatanodeDetails datanodeDetails; - private DatanodeStateMachine datanodeStateMachine; - private List<ServicePlugin> plugins; - - /** - * Default constructor. - */ - public HddsDatanodeService() { - this(null); - } - - /** - * Constructs {@link HddsDatanodeService} using the provided {@code conf} - * value. - * - * @param conf OzoneConfiguration - */ - public HddsDatanodeService(Configuration conf) { - if (conf == null) { - this.conf = new OzoneConfiguration(); - } else { - this.conf = new OzoneConfiguration(conf); - } - } - - /** - * Starts HddsDatanode services. - * - * @param service The service instance invoking this method - */ - @Override - public void start(Object service) { - OzoneConfiguration.activate(); - if (service instanceof Configurable) { - conf = new OzoneConfiguration(((Configurable) service).getConf()); - } - if (HddsUtils.isHddsEnabled(conf)) { - try { - String hostname = HddsUtils.getHostName(conf); - String ip = InetAddress.getByName(hostname).getHostAddress(); - datanodeDetails = initializeDatanodeDetails(); - datanodeDetails.setHostName(hostname); - datanodeDetails.setIpAddress(ip); - datanodeStateMachine = new DatanodeStateMachine(datanodeDetails, conf); - startPlugins(); - // Starting HDDS Daemons - datanodeStateMachine.startDaemon(); - } catch (IOException e) { - throw new RuntimeException("Can't start the HDDS datanode plugin", e); - } - } - } - - /** - * Returns DatanodeDetails or null in case of Error. - * - * @return DatanodeDetails - */ - private DatanodeDetails initializeDatanodeDetails() - throws IOException { - String idFilePath = HddsUtils.getDatanodeIdFilePath(conf); - if (idFilePath == null || idFilePath.isEmpty()) { - LOG.error("A valid file path is needed for config setting {}", - ScmConfigKeys.OZONE_SCM_DATANODE_ID); - throw new IllegalArgumentException(ScmConfigKeys.OZONE_SCM_DATANODE_ID + - " must be defined. See" + - " https://wiki.apache.org/hadoop/Ozone#Configuration"; + - " for details on configuring Ozone."); - } - - Preconditions.checkNotNull(idFilePath); - File idFile = new File(idFilePath); - if (idFile.exists()) { - return ContainerUtils.readDatanodeDetailsFrom(idFile); - } else { - // There is no datanode.id file, this might be the first time datanode - // is started. - String datanodeUuid = UUID.randomUUID().toString(); - return DatanodeDetails.newBuilder().setUuid(datanodeUuid).build(); - } - } - - /** - * Starts all the service plugins which are configured using - * OzoneConfigKeys.HDDS_DATANODE_PLUGINS_KEY. - */ - private void startPlugins() { - try { - plugins = conf.getInstances(HDDS_DATANODE_PLUGINS_KEY, - ServicePlugin.class); - } catch (RuntimeException e) { - String pluginsValue = conf.get(HDDS_DATANODE_PLUGINS_KEY); - LOG.error("Unable to load HDDS DataNode plugins. " + - "Specified list of plugins: {}", - pluginsValue, e); - throw e; - } - for (ServicePlugin plugin : plugins) { - try { - plugin.start(this); - LOG.info("Started plug-in {}", plugin); - } catch (Throwable t) { - LOG.warn("ServicePlugin {} could not be started", plugin, t); - } - } - } - - /** - * Returns the OzoneConfiguration used by this HddsDatanodeService. - * - * @return OzoneConfiguration - */ - public OzoneConfiguration getConf() { - return conf; - } - /** - * - * Return DatanodeDetails if set, return null otherwise. - * - * @return DatanodeDetails - */ - @VisibleForTesting - public DatanodeDetails getDatanodeDetails() { - return datanodeDetails; - } - - @VisibleForTesting - public DatanodeStateMachine getDatanodeStateMachine() { - return datanodeStateMachine; - } - - public void join() { - try { - datanodeStateMachine.join(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - LOG.info("Interrupted during StorageContainerManager join."); - } - } - - @Override - public void stop() { - if (plugins != null) { - for (ServicePlugin plugin : plugins) { - try { - plugin.stop(); - LOG.info("Stopped plug-in {}", plugin); - } catch (Throwable t) { - LOG.warn("ServicePlugin {} could not be stopped", plugin, t); - } - } - } - if (datanodeStateMachine != null) { - datanodeStateMachine.stopDaemon(); - } - } - - @Override - public void close() throws IOException { - if (plugins != null) { - for (ServicePlugin plugin : plugins) { - try { - plugin.close(); - } catch (Throwable t) { - LOG.warn("ServicePlugin {} could not be closed", plugin, t); - } - } - } - } - - public static HddsDatanodeService createHddsDatanodeService( - Configuration conf) { - return new HddsDatanodeService(conf); - } - - public static void main(String[] args) { - try { - if (DFSUtil.parseHelpArgument( - args, "Starts HDDS Datanode", System.out, false)) { - System.exit(0); - } - Configuration conf = new OzoneConfiguration(); - GenericOptionsParser hParser = new GenericOptionsParser(conf, args); - if (!hParser.isParseSuccessful()) { - GenericOptionsParser.printGenericCommandUsage(System.err); - System.exit(1); - } - StringUtils.startupShutdownMessage(HddsDatanodeService.class, args, LOG); - DefaultMetricsSystem.initialize("HddsDatanode"); - HddsDatanodeService hddsDatanodeService = - createHddsDatanodeService(conf); - hddsDatanodeService.start(null); - hddsDatanodeService.join(); - } catch (Throwable e) { - LOG.error("Exception in HddsDatanodeService.", e); - terminate(1, e); - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DataNodeLayoutVersion.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DataNodeLayoutVersion.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DataNodeLayoutVersion.java deleted file mode 100644 index 2d58c39..0000000 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DataNodeLayoutVersion.java +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone.container.common; - -/** - * Datanode layout version which describes information about the layout version - * on the datanode. - */ -public final class DataNodeLayoutVersion { - - // We will just be normal and use positive counting numbers for versions. - private final static DataNodeLayoutVersion[] VERSION_INFOS = - {new DataNodeLayoutVersion(1, "HDDS Datanode LayOut Version 1")}; - - private final String description; - private final int version; - - /** - * Never created outside this class. - * - * @param description -- description - * @param version -- version number - */ - private DataNodeLayoutVersion(int version, String description) { - this.description = description; - this.version = version; - } - - /** - * Returns all versions. - * - * @return Version info array. - */ - public static DataNodeLayoutVersion[] getAllVersions() { - return VERSION_INFOS.clone(); - } - - /** - * Returns the latest version. - * - * @return versionInfo - */ - public static DataNodeLayoutVersion getLatestVersion() { - return VERSION_INFOS[VERSION_INFOS.length - 1]; - } - - /** - * Return description. - * - * @return String - */ - public String getDescription() { - return description; - } - - /** - * Return the version. - * - * @return int. - */ - public int getVersion() { - return version; - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java deleted file mode 100644 index 2879001..0000000 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java +++ /dev/null @@ -1,121 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.container.common.helpers; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; -import org.apache.hadoop.metrics2.MetricsSystem; -import org.apache.hadoop.metrics2.annotation.Metric; -import org.apache.hadoop.metrics2.annotation.Metrics; -import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; -import org.apache.hadoop.metrics2.lib.MetricsRegistry; -import org.apache.hadoop.metrics2.lib.MutableCounterLong; -import org.apache.hadoop.metrics2.lib.MutableQuantiles; -import org.apache.hadoop.metrics2.lib.MutableRate; - -/** - * - * This class is for maintaining the various Storage Container - * DataNode statistics and publishing them through the metrics interfaces. - * This also registers the JMX MBean for RPC. - * <p> - * This class has a number of metrics variables that are publicly accessible; - * these variables (objects) have methods to update their values; - * for example: - * <p> {@link #numOps}.inc() - * - */ -@InterfaceAudience.Private -@Metrics(about="Storage Container DataNode Metrics", context="dfs") -public class ContainerMetrics { - @Metric private MutableCounterLong numOps; - private MutableCounterLong[] numOpsArray; - private MutableCounterLong[] opsBytesArray; - private MutableRate[] opsLatency; - private MutableQuantiles[][] opsLatQuantiles; - private MetricsRegistry registry = null; - - public ContainerMetrics(int[] intervals) { - int numEnumEntries = ContainerProtos.Type.values().length; - final int len = intervals.length; - this.numOpsArray = new MutableCounterLong[numEnumEntries]; - this.opsBytesArray = new MutableCounterLong[numEnumEntries]; - this.opsLatency = new MutableRate[numEnumEntries]; - this.opsLatQuantiles = new MutableQuantiles[numEnumEntries][len]; - this.registry = new MetricsRegistry("StorageContainerMetrics"); - for (int i = 0; i < numEnumEntries; i++) { - numOpsArray[i] = registry.newCounter( - "num" + ContainerProtos.Type.forNumber(i + 1), - "number of " + ContainerProtos.Type.forNumber(i + 1) + " ops", - (long) 0); - opsBytesArray[i] = registry.newCounter( - "bytes" + ContainerProtos.Type.forNumber(i + 1), - "bytes used by " + ContainerProtos.Type.forNumber(i + 1) + "op", - (long) 0); - opsLatency[i] = registry.newRate( - "latency" + ContainerProtos.Type.forNumber(i + 1), - ContainerProtos.Type.forNumber(i + 1) + " op"); - - for (int j = 0; j < len; j++) { - int interval = intervals[j]; - String quantileName = ContainerProtos.Type.forNumber(i + 1) + "Nanos" - + interval + "s"; - opsLatQuantiles[i][j] = registry.newQuantiles(quantileName, - "latency of Container ops", "ops", "latency", interval); - } - } - } - - public static ContainerMetrics create(Configuration conf) { - MetricsSystem ms = DefaultMetricsSystem.instance(); - // Percentile measurement is off by default, by watching no intervals - int[] intervals = - conf.getInts(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY); - return ms.register("StorageContainerMetrics", - "Storage Container Node Metrics", - new ContainerMetrics(intervals)); - } - - public void incContainerOpsMetrics(ContainerProtos.Type type) { - numOps.incr(); - numOpsArray[type.ordinal()].incr(); - } - - public long getContainerOpsMetrics(ContainerProtos.Type type){ - return numOpsArray[type.ordinal()].value(); - } - - public void incContainerOpsLatencies(ContainerProtos.Type type, - long latencyNanos) { - opsLatency[type.ordinal()].add(latencyNanos); - for (MutableQuantiles q: opsLatQuantiles[type.ordinal()]) { - q.add(latencyNanos); - } - } - - public void incContainerBytesStats(ContainerProtos.Type type, long bytes) { - opsBytesArray[type.ordinal()].incr(bytes); - } - - public long getContainerBytesMetrics(ContainerProtos.Type type){ - return opsBytesArray[type.ordinal()].value(); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerReport.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerReport.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerReport.java deleted file mode 100644 index a4c1f2f..0000000 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerReport.java +++ /dev/null @@ -1,205 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone.container.common.helpers; - -import com.google.common.base.Preconditions; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerInfo; - - -/** - * Container Report iterates the closed containers and sends a container report - * to SCM. - */ -public class ContainerReport { - private static final int UNKNOWN = -1; - private final String finalhash; - private long size; - private long keyCount; - private long bytesUsed; - private long readCount; - private long writeCount; - private long readBytes; - private long writeBytes; - private long containerID; - - public long getContainerID() { - return containerID; - } - - public void setContainerID(long containerID) { - this.containerID = containerID; - } - - /** - * Constructs the ContainerReport. - * - * @param containerID - Container ID. - * @param finalhash - Final Hash. - */ - public ContainerReport(long containerID, String finalhash) { - this.containerID = containerID; - this.finalhash = finalhash; - this.size = UNKNOWN; - this.keyCount = UNKNOWN; - this.bytesUsed = 0L; - this.readCount = 0L; - this.readBytes = 0L; - this.writeCount = 0L; - this.writeBytes = 0L; - } - - /** - * Gets a containerReport from protobuf class. - * - * @param info - ContainerInfo. - * @return - ContainerReport. - */ - public static ContainerReport getFromProtoBuf(ContainerInfo info) { - Preconditions.checkNotNull(info); - ContainerReport report = new ContainerReport(info.getContainerID(), - info.getFinalhash()); - if (info.hasSize()) { - report.setSize(info.getSize()); - } - if (info.hasKeyCount()) { - report.setKeyCount(info.getKeyCount()); - } - if (info.hasUsed()) { - report.setBytesUsed(info.getUsed()); - } - if (info.hasReadCount()) { - report.setReadCount(info.getReadCount()); - } - if (info.hasReadBytes()) { - report.setReadBytes(info.getReadBytes()); - } - if (info.hasWriteCount()) { - report.setWriteCount(info.getWriteCount()); - } - if (info.hasWriteBytes()) { - report.setWriteBytes(info.getWriteBytes()); - } - - report.setContainerID(info.getContainerID()); - return report; - } - - /** - * Returns the final signature for this container. - * - * @return - hash - */ - public String getFinalhash() { - return finalhash; - } - - /** - * Returns a positive number it is a valid number, -1 if not known. - * - * @return size or -1 - */ - public long getSize() { - return size; - } - - /** - * Sets the size of the container on disk. - * - * @param size - int - */ - public void setSize(long size) { - this.size = size; - } - - /** - * Gets number of keys in the container if known. - * - * @return - Number of keys or -1 for not known. - */ - public long getKeyCount() { - return keyCount; - } - - /** - * Sets the key count. - * - * @param keyCount - Key Count - */ - public void setKeyCount(long keyCount) { - this.keyCount = keyCount; - } - - public long getReadCount() { - return readCount; - } - - public void setReadCount(long readCount) { - this.readCount = readCount; - } - - public long getWriteCount() { - return writeCount; - } - - public void setWriteCount(long writeCount) { - this.writeCount = writeCount; - } - - public long getReadBytes() { - return readBytes; - } - - public void setReadBytes(long readBytes) { - this.readBytes = readBytes; - } - - public long getWriteBytes() { - return writeBytes; - } - - public void setWriteBytes(long writeBytes) { - this.writeBytes = writeBytes; - } - - public long getBytesUsed() { - return bytesUsed; - } - - public void setBytesUsed(long bytesUsed) { - this.bytesUsed = bytesUsed; - } - - /** - * Gets a containerInfo protobuf message from ContainerReports. - * - * @return ContainerInfo - */ - public ContainerInfo getProtoBufMessage() { - return ContainerInfo.newBuilder() - .setKeyCount(this.getKeyCount()) - .setSize(this.getSize()) - .setUsed(this.getBytesUsed()) - .setReadCount(this.getReadCount()) - .setReadBytes(this.getReadBytes()) - .setWriteCount(this.getWriteCount()) - .setWriteBytes(this.getWriteBytes()) - .setFinalhash(this.getFinalhash()) - .setContainerID(this.getContainerID()) - .build(); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java deleted file mode 100644 index d96849e..0000000 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java +++ /dev/null @@ -1,305 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.container.common.helpers; - -import com.google.common.base.Preconditions; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.hadoop.fs.FileAlreadyExistsException; -import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos - .ContainerCommandRequestProto; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos - .ContainerCommandResponseProto; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.scm.container.common.helpers - .StorageContainerException; -import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.container.common.impl.ContainerData; -import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml; -import org.apache.hadoop.ozone.container.common.impl.ContainerSet; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.nio.file.Paths; -import org.yaml.snakeyaml.Yaml; - -import static org.apache.commons.io.FilenameUtils.removeExtension; -import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos - .Result.CONTAINER_CHECKSUM_ERROR; -import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos - .Result.NO_SUCH_ALGORITHM; -import static org.apache.hadoop.ozone.container.common.impl.ContainerData - .CHARSET_ENCODING; - -/** - * A set of helper functions to create proper responses. - */ -public final class ContainerUtils { - - private ContainerUtils() { - //never constructed. - } - - /** - * Returns a Container Command Response Builder with the specified result - * and message. - * @param request requestProto message. - * @param result result of the command. - * @param message response message. - * @return ContainerCommand Response Builder. - */ - public static ContainerCommandResponseProto.Builder - getContainerCommandResponse( - ContainerCommandRequestProto request, Result result, String message) { - return ContainerCommandResponseProto.newBuilder() - .setCmdType(request.getCmdType()) - .setTraceID(request.getTraceID()) - .setResult(result) - .setMessage(message); - } - - /** - * Returns a Container Command Response Builder. This call is used to build - * success responses. Calling function can add other fields to the response - * as required. - * @param request requestProto message. - * @return ContainerCommand Response Builder with result as SUCCESS. - */ - public static ContainerCommandResponseProto.Builder getSuccessResponseBuilder( - ContainerCommandRequestProto request) { - return - ContainerCommandResponseProto.newBuilder() - .setCmdType(request.getCmdType()) - .setTraceID(request.getTraceID()) - .setResult(Result.SUCCESS); - } - - /** - * Returns a Container Command Response. This call is used for creating null - * success responses. - * @param request requestProto message. - * @return ContainerCommand Response with result as SUCCESS. - */ - public static ContainerCommandResponseProto getSuccessResponse( - ContainerCommandRequestProto request) { - ContainerCommandResponseProto.Builder builder = - getContainerCommandResponse(request, Result.SUCCESS, ""); - return builder.build(); - } - - /** - * We found a command type but no associated payload for the command. Hence - * return malformed Command as response. - * - * @param request - Protobuf message. - * @return ContainerCommandResponseProto - MALFORMED_REQUEST. - */ - public static ContainerCommandResponseProto malformedRequest( - ContainerCommandRequestProto request) { - return getContainerCommandResponse(request, Result.MALFORMED_REQUEST, - "Cmd type does not match the payload.").build(); - } - - /** - * We found a command type that is not supported yet. - * - * @param request - Protobuf message. - * @return ContainerCommandResponseProto - UNSUPPORTED_REQUEST. - */ - public static ContainerCommandResponseProto unsupportedRequest( - ContainerCommandRequestProto request) { - return getContainerCommandResponse(request, Result.UNSUPPORTED_REQUEST, - "Server does not support this command yet.").build(); - } - - /** - * Logs the error and returns a response to the caller. - * - * @param log - Logger - * @param ex - Exception - * @param request - Request Object - * @return Response - */ - public static ContainerCommandResponseProto logAndReturnError( - Logger log, StorageContainerException ex, - ContainerCommandRequestProto request) { - log.info("Operation: {} : Trace ID: {} : Message: {} : Result: {}", - request.getCmdType().name(), request.getTraceID(), - ex.getMessage(), ex.getResult().getValueDescriptor().getName()); - return getContainerCommandResponse(request, ex.getResult(), ex.getMessage()) - .build(); - } - - /** - * get containerName from a container file. - * - * @param containerFile - File - * @return Name of the container. - */ - public static String getContainerNameFromFile(File containerFile) { - Preconditions.checkNotNull(containerFile); - return Paths.get(containerFile.getParent()).resolve( - removeExtension(containerFile.getName())).toString(); - } - - public static long getContainerIDFromFile(File containerFile) { - Preconditions.checkNotNull(containerFile); - String containerID = getContainerNameFromFile(containerFile); - return Long.parseLong(containerID); - } - - /** - * Verifies that this is indeed a new container. - * - * @param containerFile - Container File to verify - * @throws IOException - */ - public static void verifyIsNewContainer(File containerFile) throws - FileAlreadyExistsException { - Logger log = LoggerFactory.getLogger(ContainerSet.class); - Preconditions.checkNotNull(containerFile, "containerFile Should not be " + - "null"); - if (containerFile.getParentFile().exists()) { - log.error("Container already exists on disk. File: {}", containerFile - .toPath()); - throw new FileAlreadyExistsException("container already exists on " + - "disk."); - } - } - - public static String getContainerDbFileName(String containerName) { - return containerName + OzoneConsts.DN_CONTAINER_DB; - } - - /** - * Persistent a {@link DatanodeDetails} to a local file. - * - * @throws IOException when read/write error occurs - */ - public synchronized static void writeDatanodeDetailsTo( - DatanodeDetails datanodeDetails, File path) throws IOException { - if (path.exists()) { - if (!path.delete() || !path.createNewFile()) { - throw new IOException("Unable to overwrite the datanode ID file."); - } - } else { - if(!path.getParentFile().exists() && - !path.getParentFile().mkdirs()) { - throw new IOException("Unable to create datanode ID directories."); - } - } - try (FileOutputStream out = new FileOutputStream(path)) { - HddsProtos.DatanodeDetailsProto proto = - datanodeDetails.getProtoBufMessage(); - proto.writeTo(out); - } - } - - /** - * Read {@link DatanodeDetails} from a local ID file. - * - * @param path ID file local path - * @return {@link DatanodeDetails} - * @throws IOException If the id file is malformed or other I/O exceptions - */ - public synchronized static DatanodeDetails readDatanodeDetailsFrom(File path) - throws IOException { - if (!path.exists()) { - throw new IOException("Datanode ID file not found."); - } - try(FileInputStream in = new FileInputStream(path)) { - return DatanodeDetails.getFromProtoBuf( - HddsProtos.DatanodeDetailsProto.parseFrom(in)); - } catch (IOException e) { - throw new IOException("Failed to parse DatanodeDetails from " - + path.getAbsolutePath(), e); - } - } - - /** - * Verify that the checksum stored in containerData is equal to the - * computed checksum. - * @param containerData - * @throws IOException - */ - public static void verifyChecksum(ContainerData containerData) - throws IOException { - String storedChecksum = containerData.getChecksum(); - - Yaml yaml = ContainerDataYaml.getYamlForContainerType( - containerData.getContainerType()); - containerData.computeAndSetChecksum(yaml); - String computedChecksum = containerData.getChecksum(); - - if (storedChecksum == null || !storedChecksum.equals(computedChecksum)) { - throw new StorageContainerException("Container checksum error for " + - "ContainerID: " + containerData.getContainerID() + ". " + - "\nStored Checksum: " + storedChecksum + - "\nExpected Checksum: " + computedChecksum, - CONTAINER_CHECKSUM_ERROR); - } - } - - /** - * Return the SHA-256 chesksum of the containerData. - * @param containerDataYamlStr ContainerData as a Yaml String - * @return Checksum of the container data - * @throws StorageContainerException - */ - public static String getChecksum(String containerDataYamlStr) - throws StorageContainerException { - MessageDigest sha; - try { - sha = MessageDigest.getInstance(OzoneConsts.FILE_HASH); - sha.update(containerDataYamlStr.getBytes(CHARSET_ENCODING)); - return DigestUtils.sha256Hex(sha.digest()); - } catch (NoSuchAlgorithmException e) { - throw new StorageContainerException("Unable to create Message Digest, " + - "usually this is a java configuration issue.", NO_SUCH_ALGORITHM); - } - } - - /** - * Get the .container file from the containerBaseDir. - * @param containerBaseDir container base directory. The name of this - * directory is same as the containerID - * @return the .container file - */ - public static File getContainerFile(File containerBaseDir) { - // Container file layout is - // .../<<containerID>>/metadata/<<containerID>>.container - String containerFilePath = OzoneConsts.CONTAINER_META_PATH + File.separator - + getContainerID(containerBaseDir) + OzoneConsts.CONTAINER_EXTENSION; - return new File(containerBaseDir, containerFilePath); - } - - /** - * ContainerID can be decoded from the container base directory name. - */ - public static long getContainerID(File containerBaseDir) { - return Long.parseLong(containerBaseDir.getName()); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DatanodeVersionFile.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DatanodeVersionFile.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DatanodeVersionFile.java deleted file mode 100644 index 4db6d31..0000000 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DatanodeVersionFile.java +++ /dev/null @@ -1,95 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.hadoop.ozone.container.common.helpers; - -import org.apache.hadoop.ozone.OzoneConsts; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.util.Properties; - -/** - * This is a utility class which helps to create the version file on datanode - * and also validate the content of the version file. - */ -public class DatanodeVersionFile { - - private final String storageId; - private final String clusterId; - private final String datanodeUuid; - private final long cTime; - private final int layOutVersion; - - public DatanodeVersionFile(String storageId, String clusterId, - String datanodeUuid, long cTime, int layOutVersion) { - this.storageId = storageId; - this.clusterId = clusterId; - this.datanodeUuid = datanodeUuid; - this.cTime = cTime; - this.layOutVersion = layOutVersion; - } - - private Properties createProperties() { - Properties properties = new Properties(); - properties.setProperty(OzoneConsts.STORAGE_ID, storageId); - properties.setProperty(OzoneConsts.CLUSTER_ID, clusterId); - properties.setProperty(OzoneConsts.DATANODE_UUID, datanodeUuid); - properties.setProperty(OzoneConsts.CTIME, String.valueOf(cTime)); - properties.setProperty(OzoneConsts.LAYOUTVERSION, String.valueOf( - layOutVersion)); - return properties; - } - - /** - * Creates a version File in specified path. - * @param path - * @throws IOException - */ - public void createVersionFile(File path) throws - IOException { - try (RandomAccessFile file = new RandomAccessFile(path, "rws"); - FileOutputStream out = new FileOutputStream(file.getFD())) { - file.getChannel().truncate(0); - Properties properties = createProperties(); - /* - * If server is interrupted before this line, - * the version file will remain unchanged. - */ - properties.store(out, null); - } - } - - - /** - * Creates a property object from the specified file content. - * @param versionFile - * @return Properties - * @throws IOException - */ - public static Properties readFrom(File versionFile) throws IOException { - try (RandomAccessFile file = new RandomAccessFile(versionFile, "rws"); - FileInputStream in = new FileInputStream(file.getFD())) { - Properties props = new Properties(); - props.load(in); - return props; - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DeletedContainerBlocksSummary.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DeletedContainerBlocksSummary.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DeletedContainerBlocksSummary.java deleted file mode 100644 index 9d0ec95..0000000 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DeletedContainerBlocksSummary.java +++ /dev/null @@ -1,104 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.hadoop.ozone.container.common.helpers; - -import com.google.common.collect.Maps; -import org.apache.hadoop.hdds.protocol.proto - .StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction; -import org.apache.hadoop.util.StringUtils; - -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * A helper class to wrap the info about under deletion container blocks. - */ -public final class DeletedContainerBlocksSummary { - - private final List<DeletedBlocksTransaction> blocks; - // key : txID - // value : times of this tx has been processed - private final Map<Long, Integer> txSummary; - // key : container name - // value : the number of blocks need to be deleted in this container - // if the message contains multiple entries for same block, - // blocks will be merged - private final Map<Long, Integer> blockSummary; - // total number of blocks in this message - private int numOfBlocks; - - private DeletedContainerBlocksSummary(List<DeletedBlocksTransaction> blocks) { - this.blocks = blocks; - txSummary = Maps.newHashMap(); - blockSummary = Maps.newHashMap(); - blocks.forEach(entry -> { - txSummary.put(entry.getTxID(), entry.getCount()); - if (blockSummary.containsKey(entry.getContainerID())) { - blockSummary.put(entry.getContainerID(), - blockSummary.get(entry.getContainerID()) - + entry.getLocalIDCount()); - } else { - blockSummary.put(entry.getContainerID(), entry.getLocalIDCount()); - } - numOfBlocks += entry.getLocalIDCount(); - }); - } - - public static DeletedContainerBlocksSummary getFrom( - List<DeletedBlocksTransaction> blocks) { - return new DeletedContainerBlocksSummary(blocks); - } - - public int getNumOfBlocks() { - return numOfBlocks; - } - - public int getNumOfContainers() { - return blockSummary.size(); - } - - public String getTXIDs() { - return String.join(",", txSummary.keySet() - .stream().map(String::valueOf).collect(Collectors.toList())); - } - - public String getTxIDSummary() { - List<String> txSummaryEntry = txSummary.entrySet().stream() - .map(entry -> entry.getKey() + "(" + entry.getValue() + ")") - .collect(Collectors.toList()); - return "[" + String.join(",", txSummaryEntry) + "]"; - } - - @Override public String toString() { - StringBuffer sb = new StringBuffer(); - for (DeletedBlocksTransaction blks : blocks) { - sb.append(" ") - .append("TXID=") - .append(blks.getTxID()) - .append(", ") - .append("TimesProceed=") - .append(blks.getCount()) - .append(", ") - .append(blks.getContainerID()) - .append(" : [") - .append(StringUtils.join(',', blks.getLocalIDList())).append("]") - .append("\n"); - } - return sb.toString(); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyValueContainerReport.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyValueContainerReport.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyValueContainerReport.java deleted file mode 100644 index b03487b..0000000 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyValueContainerReport.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone.container.common.helpers; - -import com.google.common.base.Preconditions; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerInfo; - -import static java.lang.Math.max; - -/** - * KeyValueContainer Report iterates the closed containers and sends a - * container report to SCM. - */ -public class KeyValueContainerReport extends ContainerReport{ - private long deleteTransactionId; - - /** - * Constructs the KeyValueContainerReport. - * - * @param containerID - Container ID. - * @param finalhash - Final Hash. - */ - public KeyValueContainerReport(long containerID, String finalhash) { - super(containerID, finalhash); - this.deleteTransactionId = 0; - } - - /** - * Sets the deleteTransactionId if it is greater than existing. - * @param transactionId - deleteTransactionId - */ - public void updateDeleteTransactionId(long transactionId) { - this.deleteTransactionId = max(transactionId, deleteTransactionId); - } - - /** - * Gets the deleteTransactionId. - * @return - deleteTransactionId. - */ - public long getDeleteTransactionId() { - return this.deleteTransactionId; - } - - /** - * Gets a containerReport from protobuf class. - * - * @param info - ContainerInfo. - * @return - ContainerReport. - */ - public static KeyValueContainerReport getFromProtoBuf(ContainerInfo info) { - Preconditions.checkNotNull(info); - KeyValueContainerReport report = new KeyValueContainerReport( - info.getContainerID(), info.getFinalhash()); - if (info.hasSize()) { - report.setSize(info.getSize()); - } - if (info.hasKeyCount()) { - report.setKeyCount(info.getKeyCount()); - } - if (info.hasUsed()) { - report.setBytesUsed(info.getUsed()); - } - if (info.hasReadCount()) { - report.setReadCount(info.getReadCount()); - } - if (info.hasReadBytes()) { - report.setReadBytes(info.getReadBytes()); - } - if (info.hasWriteCount()) { - report.setWriteCount(info.getWriteCount()); - } - if (info.hasWriteBytes()) { - report.setWriteBytes(info.getWriteBytes()); - } - if (info.hasDeleteTransactionId()) { - report.updateDeleteTransactionId(info.getDeleteTransactionId()); - } - report.setContainerID(info.getContainerID()); - return report; - } - - /** - * Gets a containerInfo protobuf message from ContainerReports. - * - * @return ContainerInfo - */ - @Override - public ContainerInfo getProtoBufMessage() { - return ContainerInfo.newBuilder() - .setKeyCount(this.getKeyCount()) - .setSize(this.getSize()) - .setUsed(this.getBytesUsed()) - .setReadCount(this.getReadCount()) - .setReadBytes(this.getReadBytes()) - .setWriteCount(this.getWriteCount()) - .setWriteBytes(this.getWriteBytes()) - .setFinalhash(this.getFinalhash()) - .setContainerID(this.getContainerID()) - .setDeleteTransactionId(this.getDeleteTransactionId()) - .build(); - } -} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
