This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 18f0fe1 HDDS-4733. Remove leveldb from codebase (#2274)
18f0fe1 is described below
commit 18f0fe14034c048a810365f130e3147001c27da8
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Wed Jun 2 15:13:13 2021 +0200
HDDS-4733. Remove leveldb from codebase (#2274)
Co-authored-by: lamberken <[email protected]>
---
hadoop-hdds/common/pom.xml | 5 -
.../apache/hadoop/hdds/utils/EntryConsumer.java | 38 --
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 1 -
.../container/keyvalue/KeyValueContainerCheck.java | 4 +-
.../container/keyvalue/helpers/BlockUtils.java | 2 +-
.../org/apache/hadoop/hdds/utils/LevelDBStore.java | 400 --------------
.../hadoop/hdds/utils/LevelDBStoreIterator.java | 64 ---
.../apache/hadoop/hdds/utils/MetadataStore.java | 233 --------
.../hadoop/hdds/utils/MetadataStoreBuilder.java | 151 ------
.../org/apache/hadoop/hdds/utils/RocksDBStore.java | 397 --------------
.../hadoop/hdds/utils/RocksDBStoreIterator.java | 66 ---
.../hadoop/hdds/utils/TestMetadataStore.java | 592 ---------------------
.../hadoop/hdds/utils/TestRocksDBStoreMBean.java | 230 --------
hadoop-hdds/hadoop-dependency-client/pom.xml | 4 -
hadoop-hdds/hadoop-dependency-server/pom.xml | 4 +
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 1 -
hadoop-ozone/dist/src/main/license/jar-report.txt | 1 -
.../apache/hadoop/ozone/om/BucketManagerImpl.java | 7 +-
hadoop-ozone/ozonefs-shaded/pom.xml | 1 -
.../ozone/genesis/BenchMarkMetadataStoreReads.java | 70 ---
.../genesis/BenchMarkMetadataStoreWrites.java | 62 ---
.../ozone/genesis/BenchMarkRocksDbStore.java | 119 -----
.../apache/hadoop/ozone/genesis/GenesisUtil.java | 32 --
.../org/apache/hadoop/ozone/scm/cli/SQLCLI.java | 562 -------------------
pom.xml | 6 -
25 files changed, 9 insertions(+), 3043 deletions(-)
diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml
index 137061a..c3f29df 100644
--- a/hadoop-hdds/common/pom.xml
+++ b/hadoop-hdds/common/pom.xml
@@ -81,11 +81,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd";>
</dependency>
<dependency>
- <groupId>org.fusesource.leveldbjni</groupId>
- <artifactId>leveldbjni-all</artifactId>
- </dependency>
-
- <dependency>
<groupId>io.dropwizard.metrics</groupId>
<artifactId>metrics-core</artifactId>
</dependency>
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/EntryConsumer.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/EntryConsumer.java
deleted file mode 100644
index dc08c2b..0000000
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/EntryConsumer.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.utils;
-
-import java.io.IOException;
-
-/**
- * A consumer for metadata store key-value entries.
- * Used by {@link MetadataStore} class.
- */
-@FunctionalInterface
-public interface EntryConsumer {
-
- /**
- * Consumes a key and value and produces a boolean result.
- * @param key key
- * @param value value
- * @return a boolean value produced by the consumer
- * @throws IOException
- */
- boolean consume(byte[] key, byte[] value) throws IOException;
-}
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index ee5912d..47bafe0 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -396,7 +396,6 @@ public final class OzoneConsts {
public static final String TRANSACTION_INFO_SPLIT_KEY = "#";
public static final String CONTAINER_DB_TYPE_ROCKSDB = "RocksDB";
- public static final String CONTAINER_DB_TYPE_LEVELDB = "LevelDB";
// SCM HA
public static final String SCM_SERVICE_ID_DEFAULT = "scmServiceIdDefault";
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerCheck.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerCheck.java
index 20c1455..ad18cb3 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerCheck.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerCheck.java
@@ -47,7 +47,6 @@ import
org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DB_TYPE_LEVELDB;
import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DB_TYPE_ROCKSDB;
/**
@@ -186,8 +185,7 @@ public class KeyValueContainerCheck {
}
dbType = onDiskContainerData.getContainerDBType();
- if (!dbType.equals(CONTAINER_DB_TYPE_ROCKSDB) &&
- !dbType.equals(CONTAINER_DB_TYPE_LEVELDB)) {
+ if (!dbType.equals(CONTAINER_DB_TYPE_ROCKSDB)) {
String errStr = "Unknown DBType [" + dbType
+ "] in Container File for [" + containerID + "]";
throw new IOException(errStr);
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java
index d989fd9..8749de3 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java
@@ -107,7 +107,7 @@ public final class BlockUtils {
*
* @param containerData containerData.
* @param conf configuration.
- * @return MetadataStore handle.
+ * @return DB handle.
* @throws StorageContainerException
*/
public static ReferenceCountedDB getDB(KeyValueContainerData containerData,
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/LevelDBStore.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/LevelDBStore.java
deleted file mode 100644
index 8998f6c..0000000
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/LevelDBStore.java
+++ /dev/null
@@ -1,400 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.utils;
-
-import org.apache.commons.lang3.tuple.ImmutablePair;
-import org.apache.hadoop.hdds.utils.MetadataKeyFilters.MetadataKeyFilter;
-import org.fusesource.leveldbjni.JniDBFactory;
-import org.iq80.leveldb.DB;
-import org.iq80.leveldb.DBIterator;
-import org.iq80.leveldb.Options;
-import org.iq80.leveldb.ReadOptions;
-import org.iq80.leveldb.Snapshot;
-import org.iq80.leveldb.WriteBatch;
-import org.iq80.leveldb.WriteOptions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-
-/**
- * LevelDB interface.
- */
-@Deprecated
-public class LevelDBStore implements MetadataStore {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(LevelDBStore.class);
-
- private DB db;
- private final File dbFile;
- private final Options dbOptions;
- private final WriteOptions writeOptions;
-
- public LevelDBStore(File dbPath, boolean createIfMissing)
- throws IOException {
- dbOptions = new Options();
- dbOptions.createIfMissing(createIfMissing);
- this.dbFile = dbPath;
- this.writeOptions = new WriteOptions().sync(true);
- openDB(dbPath, dbOptions);
- }
-
- /**
- * Opens a DB file.
- *
- * @param dbPath - DB File path
- * @throws IOException
- */
- public LevelDBStore(File dbPath, Options options)
- throws IOException {
- dbOptions = options;
- this.dbFile = dbPath;
- this.writeOptions = new WriteOptions().sync(true);
- openDB(dbPath, dbOptions);
- }
-
- private void openDB(File dbPath, Options options) throws IOException {
- if (dbPath.getParentFile().mkdirs()) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Db path {} created.", dbPath.getParentFile());
- }
- }
- db = JniDBFactory.factory.open(dbPath, options);
- if (LOG.isDebugEnabled()) {
- LOG.debug("LevelDB successfully opened");
- LOG.debug("[Option] cacheSize = {}", options.cacheSize());
- LOG.debug("[Option] createIfMissing = {}", options.createIfMissing());
- LOG.debug("[Option] blockSize = {}", options.blockSize());
- LOG.debug("[Option] compressionType= {}", options.compressionType());
- LOG.debug("[Option] maxOpenFiles= {}", options.maxOpenFiles());
- LOG.debug("[Option] writeBufferSize= {}", options.writeBufferSize());
- }
- }
-
- /**
- * Puts a Key into file.
- *
- * @param key - key
- * @param value - value
- */
- @Override
- public void put(byte[] key, byte[] value) {
- db.put(key, value, writeOptions);
- }
-
- /**
- * Get Key.
- *
- * @param key key
- * @return value
- */
- @Override
- public byte[] get(byte[] key) {
- return db.get(key);
- }
-
- /**
- * Delete Key.
- *
- * @param key - Key
- */
- @Override
- public void delete(byte[] key) {
- db.delete(key);
- }
-
- /**
- * Closes the DB.
- *
- * @throws IOException
- */
- @Override
- public void close() throws IOException {
- if (db != null){
- db.close();
- }
- }
-
- /**
- * Returns true if the DB is empty.
- *
- * @return boolean
- * @throws IOException
- */
- @Override
- public boolean isEmpty() throws IOException {
- try (DBIterator iter = db.iterator()) {
- iter.seekToFirst();
- boolean hasNext = !iter.hasNext();
- return hasNext;
- }
- }
-
- /**
- * Returns the actual levelDB object.
- * @return DB handle.
- */
- public DB getDB() {
- return db;
- }
-
- /**
- * Returns an iterator on all the key-value pairs in the DB.
- * @return an iterator on DB entries.
- */
- public DBIterator getIterator() {
- return db.iterator();
- }
-
-
- @Override
- public void destroy() throws IOException {
- close();
- JniDBFactory.factory.destroy(dbFile, dbOptions);
- }
-
- @Override
- public ImmutablePair<byte[], byte[]> peekAround(int offset,
- byte[] from) throws IOException, IllegalArgumentException {
- try (DBIterator it = db.iterator()) {
- if (from == null) {
- it.seekToFirst();
- } else {
- it.seek(from);
- }
- if (!it.hasNext()) {
- return null;
- }
- switch (offset) {
- case 0:
- Entry<byte[], byte[]> current = it.next();
- return new ImmutablePair<>(current.getKey(), current.getValue());
- case 1:
- if (it.next() != null && it.hasNext()) {
- Entry<byte[], byte[]> next = it.peekNext();
- return new ImmutablePair<>(next.getKey(), next.getValue());
- }
- break;
- case -1:
- if (it.hasPrev()) {
- Entry<byte[], byte[]> prev = it.peekPrev();
- return new ImmutablePair<>(prev.getKey(), prev.getValue());
- }
- break;
- default:
- throw new IllegalArgumentException(
- "Position can only be -1, 0 " + "or 1, but found " + offset);
- }
- }
- return null;
- }
-
- @Override
- public void iterate(byte[] from, EntryConsumer consumer)
- throws IOException {
- try (DBIterator iter = db.iterator()) {
- if (from != null) {
- iter.seek(from);
- } else {
- iter.seekToFirst();
- }
- while (iter.hasNext()) {
- Entry<byte[], byte[]> current = iter.next();
- if (!consumer.consume(current.getKey(),
- current.getValue())) {
- break;
- }
- }
- }
- }
-
- /**
- * Compacts the DB by removing deleted keys etc.
- * @throws IOException if there is an error.
- */
- @Override
- public void compactDB() throws IOException {
- if(db != null) {
- // From LevelDB docs : begin == null and end == null means the whole DB.
- db.compactRange(null, null);
- }
- }
-
- @Override
- public void flushDB(boolean sync) {
- // TODO: Implement flush for level db
- // do nothing
- }
-
- @Override
- public void writeBatch(BatchOperation operation) throws IOException {
- List<BatchOperation.SingleOperation> operations =
- operation.getOperations();
- if (!operations.isEmpty()) {
- try (WriteBatch writeBatch = db.createWriteBatch()) {
- for (BatchOperation.SingleOperation opt : operations) {
- switch (opt.getOpt()) {
- case DELETE:
- writeBatch.delete(opt.getKey());
- break;
- case PUT:
- writeBatch.put(opt.getKey(), opt.getValue());
- break;
- default:
- throw new IllegalArgumentException("Invalid operation "
- + opt.getOpt());
- }
- }
- db.write(writeBatch);
- }
- }
- }
-
- @Override
- public List<Map.Entry<byte[], byte[]>> getRangeKVs(byte[] startKey,
- int count, MetadataKeyFilters.MetadataKeyFilter... filters)
- throws IOException, IllegalArgumentException {
- return getRangeKVs(startKey, count, false, filters);
- }
-
- @Override
- public List<Map.Entry<byte[], byte[]>> getSequentialRangeKVs(byte[] startKey,
- int count, MetadataKeyFilters.MetadataKeyFilter... filters)
- throws IOException, IllegalArgumentException {
- return getRangeKVs(startKey, count, true, filters);
- }
-
- /**
- * Returns a certain range of key value pairs as a list based on a
- * startKey or count. Further a {@link MetadataKeyFilter} can be added to
- * filter keys if necessary. To prevent race conditions while listing
- * entries, this implementation takes a snapshot and lists the entries from
- * the snapshot. This may, on the other hand, cause the range result slight
- * different with actual data if data is updating concurrently.
- * <p>
- * If the startKey is specified and found in levelDB, this key and the keys
- * after this key will be included in the result. If the startKey is null
- * all entries will be included as long as other conditions are satisfied.
- * If the given startKey doesn't exist, an empty list will be returned.
- * <p>
- * The count argument is to limit number of total entries to return,
- * the value for count must be an integer greater than 0.
- * <p>
- * This method allows to specify one or more {@link MetadataKeyFilter}
- * to filter keys by certain condition. Once given, only the entries
- * whose key passes all the filters will be included in the result.
- *
- * @param startKey a start key.
- * @param count max number of entries to return.
- * @param filters customized one or more {@link MetadataKeyFilter}.
- * @return a list of entries found in the database or an empty list if the
- * startKey is invalid.
- * @throws IOException if there are I/O errors.
- * @throws IllegalArgumentException if count is less than 0.
- */
- private List<Entry<byte[], byte[]>> getRangeKVs(byte[] startKey,
- int count, boolean sequential, MetadataKeyFilter... filters)
- throws IOException {
- List<Entry<byte[], byte[]>> result = new ArrayList<>();
- long start = System.currentTimeMillis();
- if (count < 0) {
- throw new IllegalArgumentException(
- "Invalid count given " + count + ", count must be greater than 0");
- }
- Snapshot snapShot = null;
- DBIterator dbIter = null;
- try {
- snapShot = db.getSnapshot();
- ReadOptions readOptions = new ReadOptions().snapshot(snapShot);
- dbIter = db.iterator(readOptions);
- if (startKey == null) {
- dbIter.seekToFirst();
- } else {
- if (db.get(startKey) == null) {
- // Key not found, return empty list
- return result;
- }
- dbIter.seek(startKey);
- }
- while (dbIter.hasNext() && result.size() < count) {
- byte[] preKey = dbIter.hasPrev() ? dbIter.peekPrev().getKey() : null;
- byte[] nextKey = dbIter.hasNext() ? dbIter.peekNext().getKey() : null;
- Entry<byte[], byte[]> current = dbIter.next();
-
- if (filters == null) {
- result.add(current);
- } else {
- if (Arrays.asList(filters).stream().allMatch(
- entry -> entry.filterKey(preKey, current.getKey(), nextKey))) {
- result.add(current);
- } else {
- if (result.size() > 0 && sequential) {
- // if the caller asks for a sequential range of results,
- // and we met a dis-match, abort iteration from here.
- // if result is empty, we continue to look for the first match.
- break;
- }
- }
- }
- }
- } finally {
- if (snapShot != null) {
- snapShot.close();
- }
- if (dbIter != null) {
- dbIter.close();
- }
- if (LOG.isDebugEnabled()) {
- if (filters != null) {
- for (MetadataKeyFilters.MetadataKeyFilter filter : filters) {
- int scanned = filter.getKeysScannedNum();
- int hinted = filter.getKeysHintedNum();
- if (scanned > 0 || hinted > 0) {
- if (LOG.isDebugEnabled()) {
- LOG.debug(
- "getRangeKVs ({}) numOfKeysScanned={}, numOfKeysHinted={}",
- filter.getClass().getSimpleName(),
- filter.getKeysScannedNum(), filter.getKeysHintedNum());
- }
- }
- }
- }
- long end = System.currentTimeMillis();
- long timeConsumed = end - start;
- if (LOG.isDebugEnabled()) {
- LOG.debug("Time consumed for getRangeKVs() is {}ms,"
- + " result length is {}.", timeConsumed, result.size());
- }
- }
- }
- return result;
- }
-
- @Override
- public MetaStoreIterator<KeyValue> iterator() {
- return new LevelDBStoreIterator(db.iterator());
- }
-}
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/LevelDBStoreIterator.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/LevelDBStoreIterator.java
deleted file mode 100644
index 0a7abe3..0000000
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/LevelDBStoreIterator.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.utils;
-
-import org.iq80.leveldb.DBIterator;
-import java.util.Map;
-import java.util.NoSuchElementException;
-
-/**
- * LevelDB store iterator.
- */
-@Deprecated
-public class LevelDBStoreIterator
- implements MetaStoreIterator<MetadataStore.KeyValue> {
-
-
- private DBIterator levelDBIterator;
-
- public LevelDBStoreIterator(DBIterator iterator) {
- this.levelDBIterator = iterator;
- levelDBIterator.seekToFirst();
- }
-
- @Override
- public boolean hasNext() {
- return levelDBIterator.hasNext();
- }
-
- @Override
- public MetadataStore.KeyValue next() {
- if(levelDBIterator.hasNext()) {
- Map.Entry<byte[], byte[]> entry = levelDBIterator.next();
- return MetadataStore.KeyValue.create(entry.getKey(), entry.getValue());
- }
- throw new NoSuchElementException("LevelDB Store has no more elements");
- }
-
- @Override
- public void seekToFirst() {
- levelDBIterator.seekToFirst();
- }
-
- @Override
- public void seekToLast() {
- levelDBIterator.seekToLast();
- }
-
-}
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/MetadataStore.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/MetadataStore.java
deleted file mode 100644
index 5cb24d2..0000000
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/MetadataStore.java
+++ /dev/null
@@ -1,233 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.utils;
-
-import org.apache.commons.lang3.tuple.ImmutablePair;
-import org.apache.hadoop.hdds.annotation.InterfaceStability;
-import org.apache.hadoop.hdds.utils.MetadataKeyFilters.MetadataKeyFilter;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Interface for key-value store that stores ozone metadata.
- * Ozone metadata is stored as key value pairs, both key and value
- * are arbitrary byte arrays.
- */
[email protected]
-public interface MetadataStore extends Closeable{
-
- /**
- * Puts a key-value pair into the store.
- *
- * @param key metadata key
- * @param value metadata value
- */
- void put(byte[] key, byte[] value) throws IOException;
-
- /**
- * @return true if the metadata store is empty.
- *
- * @throws IOException
- */
- boolean isEmpty() throws IOException;
-
- /**
- * Returns the value mapped to the given key in byte array.
- *
- * @param key metadata key
- * @return value in byte array
- * @throws IOException
- */
- byte[] get(byte[] key) throws IOException;
-
- /**
- * Deletes a key from the metadata store.
- *
- * @param key metadata key
- * @throws IOException
- */
- void delete(byte[] key) throws IOException;
-
- /**
- * Returns a certain range of key value pairs as a list based on a
- * startKey or count. Further a {@link MetadataKeyFilter} can be added to
- * filter keys if necessary. To prevent race conditions while listing
- * entries, this implementation takes a snapshot and lists the entries from
- * the snapshot. This may, on the other hand, cause the range result slight
- * different with actual data if data is updating concurrently.
- * <p>
- * If the startKey is specified and found in levelDB, this key and the keys
- * after this key will be included in the result. If the startKey is null
- * all entries will be included as long as other conditions are satisfied.
- * If the given startKey doesn't exist and empty list will be returned.
- * <p>
- * The count argument is to limit number of total entries to return,
- * the value for count must be an integer greater than 0.
- * <p>
- * This method allows to specify one or more {@link MetadataKeyFilter}
- * to filter keys by certain condition. Once given, only the entries
- * whose key passes all the filters will be included in the result.
- *
- * @param startKey a start key.
- * @param count max number of entries to return.
- * @param filters customized one or more {@link MetadataKeyFilter}.
- * @return a list of entries found in the database or an empty list if the
- * startKey is invalid.
- * @throws IOException if there are I/O errors.
- * @throws IllegalArgumentException if count is less than 0.
- */
- List<Map.Entry<byte[], byte[]>> getRangeKVs(byte[] startKey,
- int count, MetadataKeyFilter... filters)
- throws IOException, IllegalArgumentException;
-
- /**
- * This method is very similar to {@link #getRangeKVs}, the only
- * different is this method is supposed to return a sequential range
- * of elements based on the filters. While iterating the elements,
- * if it met any entry that cannot pass the filter, the iterator will stop
- * from this point without looking for next match. If no filter is given,
- * this method behaves just like {@link #getRangeKVs}.
- *
- * @param startKey a start key.
- * @param count max number of entries to return.
- * @param filters customized one or more {@link MetadataKeyFilter}.
- * @return a list of entries found in the database.
- * @throws IOException
- * @throws IllegalArgumentException
- */
- List<Map.Entry<byte[], byte[]>> getSequentialRangeKVs(byte[] startKey,
- int count, MetadataKeyFilter... filters)
- throws IOException, IllegalArgumentException;
-
- /**
- * A batch of PUT, DELETE operations handled as a single atomic write.
- *
- * @throws IOException write fails
- */
- void writeBatch(BatchOperation operation) throws IOException;
-
- /**
- * Compact the entire database.
- * @throws IOException
- */
- void compactDB() throws IOException;
-
- /**
- * Flush the outstanding I/O operations of the DB.
- * @param sync if true will sync the outstanding I/Os to the disk.
- */
- void flushDB(boolean sync) throws IOException;
-
- /**
- * Destroy the content of the specified database,
- * a destroyed database will not be able to load again.
- * Be very careful with this method.
- *
- * @throws IOException if I/O error happens
- */
- void destroy() throws IOException;
-
- /**
- * Seek the database to a certain key, returns the key-value
- * pairs around this key based on the given offset. Note, this method
- * can only support offset -1 (left), 0 (current) and 1 (right),
- * any other offset given will cause a {@link IllegalArgumentException}.
- *
- * @param offset offset to the key
- * @param from from which key
- * @return a key-value pair
- * @throws IOException
- */
- ImmutablePair<byte[], byte[]> peekAround(int offset, byte[] from)
- throws IOException, IllegalArgumentException;
-
- /**
- * Iterates entries in the database from a certain key.
- * Applies the given {@link EntryConsumer} to the key and value of
- * each entry, the function produces a boolean result which is used
- * as the criteria to exit from iteration.
- *
- * @param from the start key
- * @param consumer
- * a {@link EntryConsumer} applied to each key and value. If the consumer
- * returns true, continues the iteration to next entry; otherwise exits
- * the iteration.
- * @throws IOException
- */
- void iterate(byte[] from, EntryConsumer consumer)
- throws IOException;
-
- /**
- * Returns the iterator for this metadata store.
- * @return MetaStoreIterator
- */
- MetaStoreIterator<KeyValue> iterator();
-
- /**
- * Class used to represent the key and value pair of a db entry.
- */
- class KeyValue {
-
- private final byte[] key;
- private final byte[] value;
-
- /**
- * KeyValue Constructor, used to represent a key and value of a db entry.
- * @param key
- * @param value
- */
- private KeyValue(byte[] key, byte[] value) {
- this.key = key;
- this.value = value;
- }
-
- /**
- * Return key.
- * @return byte[]
- */
- public byte[] getKey() {
- byte[] result = new byte[key.length];
- System.arraycopy(key, 0, result, 0, key.length);
- return result;
- }
-
- /**
- * Return value.
- * @return byte[]
- */
- public byte[] getValue() {
- byte[] result = new byte[value.length];
- System.arraycopy(value, 0, result, 0, value.length);
- return result;
- }
-
- /**
- * Create a KeyValue pair.
- * @param key
- * @param value
- * @return KeyValue object.
- */
- public static KeyValue create(byte[] key, byte[] value) {
- return new KeyValue(key, value);
- }
- }
-}
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/MetadataStoreBuilder.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/MetadataStoreBuilder.java
deleted file mode 100644
index 5e1a5a8..0000000
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/MetadataStoreBuilder.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.utils;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Map;
-import java.util.Optional;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.hadoop.hdds.conf.ConfigurationSource;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-
-import com.google.common.annotations.VisibleForTesting;
-import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_STORE_ROCKSDB_STATISTICS;
-import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_STORE_ROCKSDB_STATISTICS_DEFAULT;
-import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_STORE_ROCKSDB_STATISTICS_OFF;
-import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DB_TYPE_LEVELDB;
-import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DB_TYPE_ROCKSDB;
-
-import org.iq80.leveldb.Options;
-import org.rocksdb.BlockBasedTableConfig;
-import org.rocksdb.Statistics;
-import org.rocksdb.StatsLevel;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Builder for metadata store.
- */
-public class MetadataStoreBuilder {
-
- @VisibleForTesting
- static final Logger LOG =
LoggerFactory.getLogger(MetadataStoreBuilder.class);
- private File dbFile;
- private long cacheSize;
- private boolean createIfMissing = true;
- private Optional<ConfigurationSource> optionalConf = Optional.empty();
- private String dbType;
- @VisibleForTesting
- public static final Map<ConfigurationSource, org.rocksdb.Options>
- CACHED_OPTS = new ConcurrentHashMap<>();
- @VisibleForTesting
- public static final OzoneConfiguration DEFAULT_CONF =
- new OzoneConfiguration();
-
- public static MetadataStoreBuilder newBuilder() {
- return new MetadataStoreBuilder();
- }
-
- public MetadataStoreBuilder setDbFile(File dbPath) {
- this.dbFile = dbPath;
- return this;
- }
-
- public MetadataStoreBuilder setCacheSize(long cache) {
- this.cacheSize = cache;
- return this;
- }
-
- public MetadataStoreBuilder setCreateIfMissing(boolean doCreate) {
- this.createIfMissing = doCreate;
- return this;
- }
-
- public MetadataStoreBuilder setConf(ConfigurationSource configuration) {
- this.optionalConf = Optional.of(configuration);
- return this;
- }
-
- /**
- * Set the container DB Type.
- * @param type
- * @return MetadataStoreBuilder
- */
- public MetadataStoreBuilder setDBType(String type) {
- this.dbType = type;
- return this;
- }
-
- public MetadataStore build() throws IOException {
- if (dbFile == null) {
- throw new IllegalArgumentException("Failed to build metadata store, "
- + "dbFile is required but not found");
- }
-
- // Build db store based on configuration
- final ConfigurationSource conf = optionalConf.orElse(DEFAULT_CONF);
-
- if (dbType == null) {
- dbType = CONTAINER_DB_TYPE_ROCKSDB;
- LOG.debug("dbType is null, using dbType {}.", dbType);
- } else {
- LOG.debug("Using dbType {} for metastore", dbType);
- }
- if (CONTAINER_DB_TYPE_LEVELDB.equals(dbType)) {
- Options options = new Options();
- options.createIfMissing(createIfMissing);
- if (cacheSize > 0) {
- options.cacheSize(cacheSize);
- }
- return new LevelDBStore(dbFile, options);
- } else if (CONTAINER_DB_TYPE_ROCKSDB.equals(dbType)) {
- org.rocksdb.Options opts;
- // Used cached options if config object passed down is the same
- if (CACHED_OPTS.containsKey(conf)) {
- opts = CACHED_OPTS.get(conf);
- } else {
- opts = new org.rocksdb.Options();
- if (cacheSize > 0) {
- BlockBasedTableConfig tableConfig = new BlockBasedTableConfig();
- tableConfig.setBlockCacheSize(cacheSize);
- opts.setTableFormatConfig(tableConfig);
- }
-
- String rocksDbStat = conf.getTrimmed(
- OZONE_METADATA_STORE_ROCKSDB_STATISTICS,
- OZONE_METADATA_STORE_ROCKSDB_STATISTICS_DEFAULT);
-
- if (!rocksDbStat.equals(OZONE_METADATA_STORE_ROCKSDB_STATISTICS_OFF)) {
- Statistics statistics = new Statistics();
- statistics.setStatsLevel(StatsLevel.valueOf(rocksDbStat));
- opts = opts.setStatistics(statistics);
- }
- }
- opts.setCreateIfMissing(createIfMissing);
- CACHED_OPTS.put(conf, opts);
- return new RocksDBStore(dbFile, opts);
- }
-
- throw new IllegalArgumentException("Invalid Container DB type. Expecting "
- + CONTAINER_DB_TYPE_LEVELDB + " or "
- + CONTAINER_DB_TYPE_ROCKSDB + ", but met " + dbType);
- }
-}
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/RocksDBStore.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/RocksDBStore.java
deleted file mode 100644
index 97c267f..0000000
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/RocksDBStore.java
+++ /dev/null
@@ -1,397 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.utils;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.tuple.ImmutablePair;
-import org.apache.hadoop.hdds.HddsUtils;
-import org.apache.hadoop.metrics2.util.MBeans;
-import org.apache.ratis.thirdparty.com.google.common.annotations.
- VisibleForTesting;
-import org.rocksdb.DbPath;
-import org.rocksdb.Options;
-import org.rocksdb.RocksDB;
-import org.rocksdb.RocksDBException;
-import org.rocksdb.RocksIterator;
-import org.rocksdb.WriteBatch;
-import org.rocksdb.WriteOptions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.management.ObjectName;
-import java.io.File;
-import java.io.IOException;
-import java.util.AbstractMap;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.hadoop.hdds.utils.HddsServerUtil.toIOException;
-
-/**
- * RocksDB implementation of ozone metadata store.
- */
-public class RocksDBStore implements MetadataStore {
-
- private static final Logger LOG =
LoggerFactory.getLogger(RocksDBStore.class);
-
- private RocksDB db = null;
- private File dbLocation;
- private WriteOptions writeOptions;
- private Options dbOptions;
- private ObjectName statMBeanName;
-
- public RocksDBStore(File dbFile, Options options) throws IOException {
- Preconditions.checkNotNull(dbFile, "DB file location cannot be null");
- RocksDB.loadLibrary();
- dbOptions = options;
- dbLocation = dbFile;
- writeOptions = new WriteOptions();
- try {
- db = RocksDB.open(dbOptions, dbLocation.getAbsolutePath());
- if (dbOptions.statistics() != null) {
- Map<String, String> jmxProperties = new HashMap<String, String>();
- jmxProperties.put("dbName", dbFile.getName());
- statMBeanName = HddsUtils.registerWithJmxProperties(
- "Ozone", "RocksDbStore", jmxProperties,
- RocksDBStoreMBean.create(dbOptions.statistics(),
dbFile.getName()));
- if (statMBeanName == null) {
- LOG.warn("jmx registration failed during RocksDB init, db path :{}",
- dbFile.getAbsolutePath());
- }
- }
- } catch (RocksDBException e) {
- String msg = "Failed init RocksDB, db path : " + dbFile.getAbsolutePath()
- + ", " + "exception :" + (e.getCause() == null ?
- e.getClass().getCanonicalName() + " " + e.getMessage() :
- e.getCause().getClass().getCanonicalName() + " " +
- e.getCause().getMessage());
- throw new IOException(msg, e);
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("RocksDB successfully opened.");
- LOG.debug("[Option] dbLocation= {}", dbLocation.getAbsolutePath());
- LOG.debug("[Option] createIfMissing = {}", options.createIfMissing());
- LOG.debug("[Option] compactionPriority= {}", options.compactionStyle());
- LOG.debug("[Option] compressionType= {}", options.compressionType());
- LOG.debug("[Option] maxOpenFiles= {}", options.maxOpenFiles());
- LOG.debug("[Option] writeBufferSize= {}", options.writeBufferSize());
- }
- }
-
- @Override
- public void put(byte[] key, byte[] value) throws IOException {
- try {
- db.put(writeOptions, key, value);
- } catch (RocksDBException e) {
- throw toIOException("Failed to put key-value to metadata store", e);
- }
- }
-
- @Override
- public boolean isEmpty() throws IOException {
- RocksIterator it = null;
- try {
- it = db.newIterator();
- it.seekToFirst();
- return !it.isValid();
- } finally {
- if (it != null) {
- it.close();
- }
- }
- }
-
- @Override
- public byte[] get(byte[] key) throws IOException {
- try {
- return db.get(key);
- } catch (RocksDBException e) {
- throw toIOException("Failed to get the value for the given key", e);
- }
- }
-
- @Override
- public void delete(byte[] key) throws IOException {
- try {
- db.delete(key);
- } catch (RocksDBException e) {
- throw toIOException("Failed to delete the given key", e);
- }
- }
-
- @Override
- public List<Map.Entry<byte[], byte[]>> getRangeKVs(byte[] startKey,
- int count, MetadataKeyFilters.MetadataKeyFilter... filters)
- throws IOException, IllegalArgumentException {
- return getRangeKVs(startKey, count, false, filters);
- }
-
- @Override
- public List<Map.Entry<byte[], byte[]>> getSequentialRangeKVs(byte[] startKey,
- int count, MetadataKeyFilters.MetadataKeyFilter... filters)
- throws IOException, IllegalArgumentException {
- return getRangeKVs(startKey, count, true, filters);
- }
-
- private List<Map.Entry<byte[], byte[]>> getRangeKVs(byte[] startKey,
- int count, boolean sequential,
- MetadataKeyFilters.MetadataKeyFilter... filters)
- throws IOException, IllegalArgumentException {
- List<Map.Entry<byte[], byte[]>> result = new ArrayList<>();
- long start = System.currentTimeMillis();
- if (count < 0) {
- throw new IllegalArgumentException(
- "Invalid count given " + count + ", count must be greater than 0");
- }
- RocksIterator it = null;
- try {
- it = db.newIterator();
- if (startKey == null) {
- it.seekToFirst();
- } else {
- if(get(startKey) == null) {
- // Key not found, return empty list
- return result;
- }
- it.seek(startKey);
- }
- while(it.isValid() && result.size() < count) {
- byte[] currentKey = it.key();
- byte[] currentValue = it.value();
-
- it.prev();
- final byte[] prevKey = it.isValid() ? it.key() : null;
-
- it.seek(currentKey);
- it.next();
- final byte[] nextKey = it.isValid() ? it.key() : null;
-
- if (filters == null) {
- result.add(new AbstractMap.SimpleImmutableEntry<>(currentKey,
- currentValue));
- } else {
- if (Arrays.asList(filters).stream()
- .allMatch(entry -> entry.filterKey(prevKey,
- currentKey, nextKey))) {
- result.add(new AbstractMap.SimpleImmutableEntry<>(currentKey,
- currentValue));
- } else {
- if (result.size() > 0 && sequential) {
- // if the caller asks for a sequential range of results,
- // and we met a dis-match, abort iteration from here.
- // if result is empty, we continue to look for the first match.
- break;
- }
- }
- }
- }
- } finally {
- if (it != null) {
- it.close();
- }
- long end = System.currentTimeMillis();
- long timeConsumed = end - start;
- if (LOG.isDebugEnabled()) {
- if (filters != null) {
- for (MetadataKeyFilters.MetadataKeyFilter filter : filters) {
- int scanned = filter.getKeysScannedNum();
- int hinted = filter.getKeysHintedNum();
- if (scanned > 0 || hinted > 0) {
- LOG.debug(
- "getRangeKVs ({}) numOfKeysScanned={}, numOfKeysHinted={}",
- filter.getClass().getSimpleName(),
filter.getKeysScannedNum(),
- filter.getKeysHintedNum());
- }
- }
- }
- LOG.debug("Time consumed for getRangeKVs() is {}ms,"
- + " result length is {}.", timeConsumed, result.size());
- }
- }
- return result;
- }
-
- @Override
- public void writeBatch(BatchOperation operation)
- throws IOException {
- List<BatchOperation.SingleOperation> operations =
- operation.getOperations();
- if (!operations.isEmpty()) {
- try (WriteBatch writeBatch = new WriteBatch()) {
- for (BatchOperation.SingleOperation opt : operations) {
- switch (opt.getOpt()) {
- case DELETE:
- writeBatch.delete(opt.getKey());
- break;
- case PUT:
- writeBatch.put(opt.getKey(), opt.getValue());
- break;
- default:
- throw new IllegalArgumentException("Invalid operation "
- + opt.getOpt());
- }
- }
- db.write(writeOptions, writeBatch);
- } catch (RocksDBException e) {
- throw toIOException("Batch write operation failed", e);
- }
- }
- }
-
- @Override
- public void compactDB() throws IOException {
- if (db != null) {
- try {
- db.compactRange();
- } catch (RocksDBException e) {
- throw toIOException("Failed to compact db", e);
- }
- }
- }
-
- @Override
- public void flushDB(boolean sync) throws IOException {
- if (db != null) {
- try {
- // for RocksDB it is sufficient to flush the WAL as entire db can
- // be reconstructed using it.
- db.flushWal(sync);
- } catch (RocksDBException e) {
- throw toIOException("Failed to flush db", e);
- }
- }
- }
-
- private void deleteQuietly(File fileOrDir) {
- if (fileOrDir != null && fileOrDir.exists()) {
- try {
- FileUtils.forceDelete(fileOrDir);
- } catch (IOException e) {
- LOG.warn("Failed to delete dir {}", fileOrDir.getAbsolutePath(), e);
- }
- }
- }
-
- @Override
- public void destroy() throws IOException {
- // Make sure db is closed.
- close();
-
- // There is no destroydb java API available,
- // equivalently we can delete all db directories.
- deleteQuietly(dbLocation);
- deleteQuietly(new File(dbOptions.dbLogDir()));
- deleteQuietly(new File(dbOptions.walDir()));
- List<DbPath> dbPaths = dbOptions.dbPaths();
- if (dbPaths != null) {
- dbPaths.forEach(dbPath -> {
- deleteQuietly(new File(dbPath.toString()));
- });
- }
- }
-
- @Override
- public ImmutablePair<byte[], byte[]> peekAround(int offset,
- byte[] from) throws IOException, IllegalArgumentException {
- RocksIterator it = null;
- try {
- it = db.newIterator();
- if (from == null) {
- it.seekToFirst();
- } else {
- it.seek(from);
- }
- if (!it.isValid()) {
- return null;
- }
-
- switch (offset) {
- case 0:
- break;
- case 1:
- it.next();
- break;
- case -1:
- it.prev();
- break;
- default:
- throw new IllegalArgumentException(
- "Position can only be -1, 0 " + "or 1, but found " + offset);
- }
- return it.isValid() ? new ImmutablePair<>(it.key(), it.value()) : null;
- } finally {
- if (it != null) {
- it.close();
- }
- }
- }
-
- @Override
- public void iterate(byte[] from, EntryConsumer consumer)
- throws IOException {
- RocksIterator it = null;
- try {
- it = db.newIterator();
- if (from != null) {
- it.seek(from);
- } else {
- it.seekToFirst();
- }
- while (it.isValid()) {
- if (!consumer.consume(it.key(), it.value())) {
- break;
- }
- it.next();
- }
- } finally {
- if (it != null) {
- it.close();
- }
- }
- }
-
- @Override
- public void close() throws IOException {
- if (statMBeanName != null) {
- MBeans.unregister(statMBeanName);
- statMBeanName = null;
- }
- if (db != null) {
- db.close();
- db = null;
- }
-
- }
-
- @VisibleForTesting
- protected ObjectName getStatMBeanName() {
- return statMBeanName;
- }
-
- @Override
- public MetaStoreIterator<KeyValue> iterator() {
- return new RocksDBStoreIterator(db.newIterator());
- }
-
-}
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/RocksDBStoreIterator.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/RocksDBStoreIterator.java
deleted file mode 100644
index e39ec57..0000000
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/RocksDBStoreIterator.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.hadoop.hdds.utils;
-
-import org.rocksdb.RocksIterator;
-
-import java.util.NoSuchElementException;
-
-/**
- * RocksDB store iterator.
- */
-public class RocksDBStoreIterator
- implements MetaStoreIterator<MetadataStore.KeyValue> {
-
- private RocksIterator rocksDBIterator;
-
- public RocksDBStoreIterator(RocksIterator iterator) {
- this.rocksDBIterator = iterator;
- rocksDBIterator.seekToFirst();
- }
-
- @Override
- public boolean hasNext() {
- return rocksDBIterator.isValid();
- }
-
- @Override
- public MetadataStore.KeyValue next() {
- if (rocksDBIterator.isValid()) {
- MetadataStore.KeyValue value =
- MetadataStore.KeyValue.create(rocksDBIterator.key(), rocksDBIterator
- .value());
- rocksDBIterator.next();
- return value;
- }
- throw new NoSuchElementException("RocksDB Store has no more elements");
- }
-
- @Override
- public void seekToFirst() {
- rocksDBIterator.seekToFirst();
- }
-
- @Override
- public void seekToLast() {
- rocksDBIterator.seekToLast();
- }
-
-}
diff --git
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestMetadataStore.java
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestMetadataStore.java
deleted file mode 100644
index aa6f5ce..0000000
---
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestMetadataStore.java
+++ /dev/null
@@ -1,592 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
under
- * the License.
- */
-package org.apache.hadoop.hdds.utils;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.NoSuchElementException;
-import java.util.UUID;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.hadoop.hdds.StringUtils;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.utils.MetadataKeyFilters.KeyPrefixFilter;
-import org.apache.hadoop.hdds.utils.MetadataKeyFilters.MetadataKeyFilter;
-import org.apache.hadoop.test.GenericTestUtils;
-
-import com.google.common.collect.Lists;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.tuple.ImmutablePair;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.slf4j.event.Level;
-
-import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DB_TYPE_LEVELDB;
-import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DB_TYPE_ROCKSDB;
-import static org.apache.hadoop.test.PlatformAssumptions.assumeNotWindows;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.junit.runners.Parameterized.Parameters;
-import static java.nio.charset.StandardCharsets.UTF_8;
-/**
- * Test class for ozone metadata store.
- */
-@RunWith(Parameterized.class)
-public class TestMetadataStore {
-
- private static final int MAX_GETRANGE_LENGTH = 100;
- private final String storeImpl;
- @Rule
- public ExpectedException expectedException = ExpectedException.none();
- private MetadataStore store;
- private File testDir;
-
- public TestMetadataStore(String metadataImpl) {
- this.storeImpl = metadataImpl;
- }
-
- @Parameters
- public static Collection<Object[]> data() {
- return Arrays.asList(new Object[][] {
- {CONTAINER_DB_TYPE_LEVELDB},
- {CONTAINER_DB_TYPE_ROCKSDB}
- });
- }
-
- @Before
- public void init() throws IOException {
- if (CONTAINER_DB_TYPE_ROCKSDB.equals(storeImpl)) {
- // The initialization of RocksDB fails on Windows
- assumeNotWindows();
- }
-
- testDir = GenericTestUtils.getTestDir(getClass().getSimpleName()
- + "-" + storeImpl.toLowerCase());
-
- OzoneConfiguration conf = new OzoneConfiguration();
-
- store = MetadataStoreBuilder.newBuilder()
- .setConf(conf)
- .setCreateIfMissing(true)
- .setDbFile(testDir)
- .setDBType(storeImpl)
- .build();
-
- // Add 20 entries.
- // {a0 : a-value0} to {a9 : a-value9}
- // {b0 : b-value0} to {b9 : b-value9}
- for (int i = 0; i < 10; i++) {
- store.put(getBytes("a" + i), getBytes("a-value" + i));
- store.put(getBytes("b" + i), getBytes("b-value" + i));
- }
- }
-
- @Test
- public void testIterator() throws Exception {
- OzoneConfiguration conf = new OzoneConfiguration();
-
- File dbDir = GenericTestUtils.getRandomizedTestDir();
- MetadataStore dbStore = MetadataStoreBuilder.newBuilder()
- .setConf(conf)
- .setCreateIfMissing(true)
- .setDbFile(dbDir)
- .setDBType(storeImpl)
- .build();
-
- //As database is empty, check whether iterator is working as expected or
- // not.
- MetaStoreIterator<MetadataStore.KeyValue> metaStoreIterator =
- dbStore.iterator();
- assertFalse(metaStoreIterator.hasNext());
- try {
- metaStoreIterator.next();
- fail("testIterator failed");
- } catch (NoSuchElementException ex) {
- GenericTestUtils.assertExceptionContains("Store has no more elements",
- ex);
- }
-
- for (int i = 0; i < 10; i++) {
- store.put(getBytes("a" + i), getBytes("a-value" + i));
- }
-
- metaStoreIterator = dbStore.iterator();
-
- int i = 0;
- while (metaStoreIterator.hasNext()) {
- MetadataStore.KeyValue val = metaStoreIterator.next();
- assertEquals("a" + i, getString(val.getKey()));
- assertEquals("a-value" + i, getString(val.getValue()));
- i++;
- }
-
- // As we have iterated all the keys in database, hasNext should return
- // false and next() should throw NoSuchElement exception.
-
- assertFalse(metaStoreIterator.hasNext());
- try {
- metaStoreIterator.next();
- fail("testIterator failed");
- } catch (NoSuchElementException ex) {
- GenericTestUtils.assertExceptionContains("Store has no more elements",
- ex);
- }
- dbStore.close();
- dbStore.destroy();
- FileUtils.deleteDirectory(dbDir);
-
- }
-
- @Test
- public void testMetaStoreConfigDifferentFromType() throws IOException {
-
- OzoneConfiguration conf = new OzoneConfiguration();
-
- String dbType;
- GenericTestUtils.setLogLevel(MetadataStoreBuilder.LOG, Level.DEBUG);
- GenericTestUtils.LogCapturer logCapturer =
- GenericTestUtils.LogCapturer.captureLogs(MetadataStoreBuilder.LOG);
- if (storeImpl.equals(CONTAINER_DB_TYPE_LEVELDB)) {
- dbType = "LevelDB";
- } else {
- dbType = "RocksDB";
- }
-
- File dbDir = GenericTestUtils.getTestDir(getClass().getSimpleName()
- + "-" + dbType.toLowerCase() + "-test");
- MetadataStore dbStore = MetadataStoreBuilder.newBuilder().setConf(conf)
- .setCreateIfMissing(true).setDbFile(dbDir).setDBType(dbType).build();
- assertTrue(logCapturer.getOutput().contains("Using dbType " + dbType + "" +
- " for metastore"));
- dbStore.close();
- dbStore.destroy();
- FileUtils.deleteDirectory(dbDir);
-
- }
-
- @Test
- public void testdbTypeNotSet() throws IOException {
-
- OzoneConfiguration conf = new OzoneConfiguration();
-
- GenericTestUtils.setLogLevel(MetadataStoreBuilder.LOG, Level.DEBUG);
- GenericTestUtils.LogCapturer logCapturer =
- GenericTestUtils.LogCapturer.captureLogs(MetadataStoreBuilder.LOG);
-
- File dbDir = GenericTestUtils.getTestDir(getClass().getSimpleName()
- + "-" + storeImpl.toLowerCase() + "-test");
- MetadataStore dbStore = MetadataStoreBuilder.newBuilder().setConf(conf)
- .setCreateIfMissing(true).setDbFile(dbDir).build();
- assertTrue(logCapturer.getOutput().contains("dbType is null, using dbType"
+
- " " + CONTAINER_DB_TYPE_ROCKSDB));
- dbStore.close();
- dbStore.destroy();
- FileUtils.deleteDirectory(dbDir);
-
- }
-
- @After
- public void cleanup() throws IOException {
- if (store != null) {
- System.out.println("--- Closing Store: " + store.getClass());
- store.close();
- store.destroy();
- } else {
- System.out.println("--- Store already closed.");
- }
- if (testDir != null) {
- FileUtils.deleteDirectory(testDir);
- }
- }
-
- private byte[] getBytes(String str) {
- return str == null ? null :
- StringUtils.string2Bytes(str);
- }
-
- private String getString(byte[] bytes) {
- return bytes == null ? null :
- StringUtils.bytes2String(bytes);
- }
-
- @Test
- public void testGetDelete() throws IOException {
- for (int i = 0; i < 10; i++) {
- byte[] va = store.get(getBytes("a" + i));
- assertEquals("a-value" + i, getString(va));
-
- byte[] vb = store.get(getBytes("b" + i));
- assertEquals("b-value" + i, getString(vb));
- }
-
- String keyToDel = "del-" + UUID.randomUUID().toString();
- store.put(getBytes(keyToDel), getBytes(keyToDel));
- assertEquals(keyToDel, getString(store.get(getBytes(keyToDel))));
- store.delete(getBytes(keyToDel));
- assertEquals(null, store.get(getBytes(keyToDel)));
- }
-
- @Test
- public void testPeekFrom() throws IOException {
- // Test peek from an element that has prev as well as next
- testPeek("a3", "a2", "a4");
-
- // Test peek from an element that only has prev
- testPeek("b9", "b8", null);
-
- // Test peek from an element that only has next
- testPeek("a0", null, "a1");
- }
-
- private String getExpectedValue(String key) {
- if (key == null) {
- return null;
- }
- char[] arr = key.toCharArray();
- return new StringBuilder().append(arr[0]).append("-value")
- .append(arr[arr.length - 1]).toString();
- }
-
- private void testPeek(String peekKey, String prevKey, String nextKey)
- throws IOException {
- // Look for current
- String k = null;
- String v = null;
- ImmutablePair<byte[], byte[]> current =
- store.peekAround(0, getBytes(peekKey));
- if (current != null) {
- k = getString(current.getKey());
- v = getString(current.getValue());
- }
- assertEquals(peekKey, k);
- assertEquals(v, getExpectedValue(peekKey));
-
- // Look for prev
- k = null;
- v = null;
- ImmutablePair<byte[], byte[]> prev =
- store.peekAround(-1, getBytes(peekKey));
- if (prev != null) {
- k = getString(prev.getKey());
- v = getString(prev.getValue());
- }
- assertEquals(prevKey, k);
- assertEquals(v, getExpectedValue(prevKey));
-
- // Look for next
- k = null;
- v = null;
- ImmutablePair<byte[], byte[]> next =
- store.peekAround(1, getBytes(peekKey));
- if (next != null) {
- k = getString(next.getKey());
- v = getString(next.getValue());
- }
- assertEquals(nextKey, k);
- assertEquals(v, getExpectedValue(nextKey));
- }
-
- @Test
- public void testIterateKeys() throws IOException {
- // iterate keys from b0
- ArrayList<String> result = Lists.newArrayList();
- store.iterate(getBytes("b0"), (k, v) -> {
- // b-value{i}
- String value = getString(v);
- char num = value.charAt(value.length() - 1);
- // each value adds 1
- int i = Character.getNumericValue(num) + 1;
- value = value.substring(0, value.length() - 1) + i;
- result.add(value);
- return true;
- });
-
- assertFalse(result.isEmpty());
- for (int i = 0; i < result.size(); i++) {
- assertEquals("b-value" + (i + 1), result.get(i));
- }
-
- // iterate from a non exist key
- result.clear();
- store.iterate(getBytes("xyz"), (k, v) -> {
- result.add(getString(v));
- return true;
- });
- assertTrue(result.isEmpty());
-
- // iterate from the beginning
- result.clear();
- store.iterate(null, (k, v) -> {
- result.add(getString(v));
- return true;
- });
- assertEquals(20, result.size());
- }
-
- @Test
- public void testGetRangeKVs() throws IOException {
- List<Map.Entry<byte[], byte[]>> result = null;
-
- // Set empty startKey will return values from beginning.
- result = store.getRangeKVs(null, 5);
- assertEquals(5, result.size());
- assertEquals("a-value2", getString(result.get(2).getValue()));
-
- // Empty list if startKey doesn't exist.
- result = store.getRangeKVs(getBytes("a12"), 5);
- assertEquals(0, result.size());
-
- // Returns max available entries after a valid startKey.
- result = store.getRangeKVs(getBytes("b0"), MAX_GETRANGE_LENGTH);
- assertEquals(10, result.size());
- assertEquals("b0", getString(result.get(0).getKey()));
- assertEquals("b-value0", getString(result.get(0).getValue()));
- result = store.getRangeKVs(getBytes("b0"), 5);
- assertEquals(5, result.size());
-
- // Both startKey and count are honored.
- result = store.getRangeKVs(getBytes("a9"), 2);
- assertEquals(2, result.size());
- assertEquals("a9", getString(result.get(0).getKey()));
- assertEquals("a-value9", getString(result.get(0).getValue()));
- assertEquals("b0", getString(result.get(1).getKey()));
- assertEquals("b-value0", getString(result.get(1).getValue()));
-
- // Filter keys by prefix.
- // It should returns all "b*" entries.
- MetadataKeyFilter filter1 = new KeyPrefixFilter().addFilter("b");
- result = store.getRangeKVs(null, 100, filter1);
- assertEquals(10, result.size());
- assertTrue(result.stream().allMatch(entry ->
- new String(entry.getKey(), UTF_8).startsWith("b")
- ));
- assertEquals(20, filter1.getKeysScannedNum());
- assertEquals(10, filter1.getKeysHintedNum());
- result = store.getRangeKVs(null, 3, filter1);
- assertEquals(3, result.size());
- result = store.getRangeKVs(getBytes("b3"), 1, filter1);
- assertEquals("b-value3", getString(result.get(0).getValue()));
-
- // Define a customized filter that filters keys by suffix.
- // Returns all "*2" entries.
- MetadataKeyFilter filter2 = (preKey, currentKey, nextKey)
- -> getString(currentKey).endsWith("2");
- result = store.getRangeKVs(null, MAX_GETRANGE_LENGTH, filter2);
- assertEquals(2, result.size());
- assertEquals("a2", getString(result.get(0).getKey()));
- assertEquals("b2", getString(result.get(1).getKey()));
- result = store.getRangeKVs(null, 1, filter2);
- assertEquals(1, result.size());
- assertEquals("a2", getString(result.get(0).getKey()));
-
- // Apply multiple filters.
- result = store.getRangeKVs(null, MAX_GETRANGE_LENGTH, filter1, filter2);
- assertEquals(1, result.size());
- assertEquals("b2", getString(result.get(0).getKey()));
- assertEquals("b-value2", getString(result.get(0).getValue()));
-
- // If filter is null, no effect.
- result = store.getRangeKVs(null, 1, (MetadataKeyFilter[]) null);
- assertEquals(1, result.size());
- assertEquals("a0", getString(result.get(0).getKey()));
- }
-
- @Test
- public void testGetSequentialRangeKVs() throws IOException {
- MetadataKeyFilter suffixFilter = (preKey, currentKey, nextKey)
- -> StringUtils.bytes2String(currentKey).endsWith("2");
- // Suppose to return a2 and b2
- List<Map.Entry<byte[], byte[]>> result =
- store.getRangeKVs(null, MAX_GETRANGE_LENGTH, suffixFilter);
- assertEquals(2, result.size());
- assertEquals("a2", StringUtils.bytes2String(result.get(0).getKey()));
- assertEquals("b2", StringUtils.bytes2String(result.get(1).getKey()));
-
- // Suppose to return just a2, because when it iterates to a3,
- // the filter no long matches and it should stop from there.
- result = store.getSequentialRangeKVs(null,
- MAX_GETRANGE_LENGTH, suffixFilter);
- assertEquals(1, result.size());
- assertEquals("a2", StringUtils.bytes2String(result.get(0).getKey()));
- }
-
- @Test
- public void testGetRangeLength() throws IOException {
- List<Map.Entry<byte[], byte[]>> result = null;
-
- result = store.getRangeKVs(null, 0);
- assertEquals(0, result.size());
-
- result = store.getRangeKVs(null, 1);
- assertEquals(1, result.size());
-
- // Count less than zero is invalid.
- expectedException.expect(IllegalArgumentException.class);
- expectedException.expectMessage("Invalid count given");
- store.getRangeKVs(null, -1);
- }
-
- @Test
- public void testInvalidStartKey() throws IOException {
- // If startKey is invalid, the returned list should be empty.
- List<Map.Entry<byte[], byte[]>> kvs =
- store.getRangeKVs(getBytes("unknownKey"), MAX_GETRANGE_LENGTH);
- assertEquals(0, kvs.size());
- }
-
- @Test
- public void testDestroyDB() throws IOException {
- // create a new DB to test db destroy
- OzoneConfiguration conf = new OzoneConfiguration();
-
- File dbDir = GenericTestUtils.getTestDir(getClass().getSimpleName()
- + "-" + storeImpl.toLowerCase() + "-toDestroy");
- MetadataStore dbStore = MetadataStoreBuilder.newBuilder()
- .setConf(conf)
- .setCreateIfMissing(true)
- .setDbFile(dbDir)
- .setDBType(storeImpl)
- .build();
-
- dbStore.put(getBytes("key1"), getBytes("value1"));
- dbStore.put(getBytes("key2"), getBytes("value2"));
-
- assertFalse(dbStore.isEmpty());
- assertTrue(dbDir.exists());
- assertTrue(dbDir.listFiles().length > 0);
-
- dbStore.destroy();
-
- assertFalse(dbDir.exists());
- }
-
- @Test
- public void testBatchWrite() throws IOException {
- OzoneConfiguration conf = new OzoneConfiguration();
-
- File dbDir = GenericTestUtils.getTestDir(getClass().getSimpleName()
- + "-" + storeImpl.toLowerCase() + "-batchWrite");
- MetadataStore dbStore = MetadataStoreBuilder.newBuilder()
- .setConf(conf)
- .setCreateIfMissing(true)
- .setDbFile(dbDir)
- .setDBType(storeImpl)
- .build();
-
- List<String> expectedResult = Lists.newArrayList();
- for (int i = 0; i < 10; i++) {
- dbStore.put(getBytes("batch-" + i), getBytes("batch-value-" + i));
- expectedResult.add("batch-" + i);
- }
-
- BatchOperation batch = new BatchOperation();
- batch.delete(getBytes("batch-2"));
- batch.delete(getBytes("batch-3"));
- batch.delete(getBytes("batch-4"));
- batch.put(getBytes("batch-new-2"), getBytes("batch-new-value-2"));
-
- expectedResult.remove("batch-2");
- expectedResult.remove("batch-3");
- expectedResult.remove("batch-4");
- expectedResult.add("batch-new-2");
-
- dbStore.writeBatch(batch);
-
- Iterator<String> it = expectedResult.iterator();
- AtomicInteger count = new AtomicInteger(0);
- dbStore.iterate(null, (key, value) -> {
- count.incrementAndGet();
- return it.hasNext() && it.next().equals(getString(key));
- });
-
- assertEquals(8, count.get());
- }
-
- @Test
- public void testKeyPrefixFilter() throws IOException {
- List<Map.Entry<byte[], byte[]>> result = null;
- RuntimeException exception = null;
-
- try {
- new KeyPrefixFilter().addFilter("b0", true).addFilter("b");
- } catch (IllegalArgumentException e) {
- exception = e;
- assertTrue(exception.getMessage().contains("KeyPrefix: b already " +
- "rejected"));
- }
-
- try {
- new KeyPrefixFilter().addFilter("b0").addFilter("b", true);
- } catch (IllegalArgumentException e) {
- exception = e;
- assertTrue(exception.getMessage().contains("KeyPrefix: b already " +
- "accepted"));
- }
-
- try {
- new KeyPrefixFilter().addFilter("b", true).addFilter("b0");
- } catch (IllegalArgumentException e) {
- exception = e;
- assertTrue(exception.getMessage().contains("KeyPrefix: b0 already " +
- "rejected"));
- }
-
- try {
- new KeyPrefixFilter().addFilter("b").addFilter("b0", true);
- } catch (IllegalArgumentException e) {
- exception = e;
- assertTrue(exception.getMessage().contains("KeyPrefix: b0 already " +
- "accepted"));
- }
-
- MetadataKeyFilter filter1 = new KeyPrefixFilter(true)
- .addFilter("a0")
- .addFilter("a1")
- .addFilter("b", true);
- result = store.getRangeKVs(null, 100, filter1);
- assertEquals(2, result.size());
- assertTrue(result.stream().anyMatch(entry -> new String(entry.getKey(),
- UTF_8)
- .startsWith("a0")) && result.stream().anyMatch(entry -> new String(
- entry.getKey(), UTF_8).startsWith("a1")));
-
- filter1 = new KeyPrefixFilter(true).addFilter("b", true);
- result = store.getRangeKVs(null, 100, filter1);
- assertEquals(0, result.size());
-
- filter1 = new KeyPrefixFilter().addFilter("b", true);
- result = store.getRangeKVs(null, 100, filter1);
- assertEquals(10, result.size());
- assertTrue(result.stream().allMatch(entry -> new String(entry.getKey(),
- UTF_8)
- .startsWith("a")));
- }
-}
diff --git
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestRocksDBStoreMBean.java
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestRocksDBStoreMBean.java
deleted file mode 100644
index 4db70b8..0000000
---
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestRocksDBStoreMBean.java
+++ /dev/null
@@ -1,230 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.utils;
-
-import javax.management.MBeanServer;
-import java.io.File;
-import java.io.IOException;
-import java.lang.management.ManagementFactory;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.metrics2.AbstractMetric;
-import org.apache.hadoop.metrics2.MetricsCollector;
-import org.apache.hadoop.metrics2.MetricsInfo;
-import org.apache.hadoop.metrics2.MetricsRecordBuilder;
-import org.apache.hadoop.metrics2.MetricsSource;
-import org.apache.hadoop.metrics2.MetricsSystem;
-import org.apache.hadoop.metrics2.MetricsTag;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.test.GenericTestUtils;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-import org.junit.Assert;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Test the JMX interface for the rocksdb metastore implementation.
- */
-public class TestRocksDBStoreMBean {
-
- private OzoneConfiguration conf;
-
- @Before
- public void init() throws Exception {
- conf = new OzoneConfiguration();
- }
-
-
- @Test
- public void testJmxBeans() throws Exception {
-
- RocksDBStore metadataStore = getTestRocksDBStoreWithData();
-
- MBeanServer platformMBeanServer =
- ManagementFactory.getPlatformMBeanServer();
- Thread.sleep(2000);
-
- Object keysWritten = platformMBeanServer
- .getAttribute(metadataStore.getStatMBeanName(), "NUMBER_KEYS_WRITTEN");
-
- assertEquals(10L, keysWritten);
-
- Object dbWriteAverage = platformMBeanServer
- .getAttribute(metadataStore.getStatMBeanName(), "DB_WRITE_AVERAGE");
- assertTrue((double) dbWriteAverage > 0);
-
- metadataStore.close();
-
- }
-
- @Test()
- public void testDisabledStat() throws Exception {
- File testDir = GenericTestUtils
- .getTestDir(getClass().getSimpleName() + "-withoutstat");
-
- conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_ROCKSDB_STATISTICS,
- OzoneConfigKeys.OZONE_METADATA_STORE_ROCKSDB_STATISTICS_OFF);
-
- RocksDBStore metadataStore =
- (RocksDBStore) MetadataStoreBuilder.newBuilder().setConf(conf)
- .setCreateIfMissing(true).setDbFile(testDir).build();
-
- Assert.assertNull(metadataStore.getStatMBeanName());
- }
-
- @Test
- public void testMetricsSystemIntegration() throws Exception {
-
- RocksDBStore metadataStore = getTestRocksDBStoreWithData();
- Thread.sleep(2000);
-
- MetricsSystem ms = DefaultMetricsSystem.instance();
- MetricsSource rdbSource =
- ms.getSource("Rocksdb_TestRocksDBStoreMBean-withstat");
-
- BufferedMetricsCollector metricsCollector = new BufferedMetricsCollector();
- rdbSource.getMetrics(metricsCollector, true);
-
- Map<String, Double> metrics = metricsCollector.getMetricsRecordBuilder()
- .getMetrics();
- assertTrue(10.0 == metrics.get("NUMBER_KEYS_WRITTEN"));
- assertTrue(metrics.get("DB_WRITE_AVERAGE") > 0);
- metadataStore.close();
- }
-
- private RocksDBStore getTestRocksDBStoreWithData() throws IOException {
- File testDir =
- GenericTestUtils.getTestDir(getClass().getSimpleName() + "-withstat");
-
- conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_ROCKSDB_STATISTICS, "ALL");
-
- RocksDBStore metadataStore =
- (RocksDBStore) MetadataStoreBuilder.newBuilder().setConf(conf)
- .setCreateIfMissing(true).setDbFile(testDir).build();
-
- for (int i = 0; i < 10; i++) {
- metadataStore.put("key".getBytes(UTF_8), "value".getBytes(UTF_8));
- }
-
- return metadataStore;
- }
-}
-
-/**
- * Test class to buffer a single MetricsRecordBuilder instance.
- */
-class BufferedMetricsCollector implements MetricsCollector {
-
- private BufferedMetricsRecordBuilderImpl metricsRecordBuilder;
-
- BufferedMetricsCollector() {
- metricsRecordBuilder = new BufferedMetricsRecordBuilderImpl();
- }
-
- public BufferedMetricsRecordBuilderImpl getMetricsRecordBuilder() {
- return metricsRecordBuilder;
- }
-
- @Override
- public MetricsRecordBuilder addRecord(String s) {
- metricsRecordBuilder.setContext(s);
- return metricsRecordBuilder;
- }
-
- @Override
- public MetricsRecordBuilder addRecord(MetricsInfo metricsInfo) {
- return metricsRecordBuilder;
- }
-
- /**
- * Test class to buffer a single snapshot of metrics.
- */
- static class BufferedMetricsRecordBuilderImpl extends MetricsRecordBuilder {
-
- private Map<String, Double> metrics = new HashMap<>();
- private String contextName;
-
- public Map<String, Double> getMetrics() {
- return metrics;
- }
-
- @Override
- public MetricsRecordBuilder tag(MetricsInfo metricsInfo, String s) {
- return null;
- }
-
- @Override
- public MetricsRecordBuilder add(MetricsTag metricsTag) {
- return null;
- }
-
- @Override
- public MetricsRecordBuilder add(AbstractMetric abstractMetric) {
- return null;
- }
-
- @Override
- public MetricsRecordBuilder setContext(String s) {
- this.contextName = s;
- return this;
- }
-
- @Override
- public MetricsRecordBuilder addCounter(MetricsInfo metricsInfo, int i) {
- return null;
- }
-
- @Override
- public MetricsRecordBuilder addCounter(MetricsInfo metricsInfo, long l) {
- metrics.put(metricsInfo.name(), (double)l);
- return this;
- }
-
- @Override
- public MetricsRecordBuilder addGauge(MetricsInfo metricsInfo, int i) {
- return null;
- }
-
- @Override
- public MetricsRecordBuilder addGauge(MetricsInfo metricsInfo, long l) {
- return null;
- }
-
- @Override
- public MetricsRecordBuilder addGauge(MetricsInfo metricsInfo, float v) {
- return null;
- }
-
- @Override
- public MetricsRecordBuilder addGauge(MetricsInfo metricsInfo, double v) {
- metrics.put(metricsInfo.name(), v);
- return this;
- }
-
- @Override
- public MetricsCollector parent() {
- return null;
- }
- }
-}
\ No newline at end of file
diff --git a/hadoop-hdds/hadoop-dependency-client/pom.xml
b/hadoop-hdds/hadoop-dependency-client/pom.xml
index 9531725..a48d179 100644
--- a/hadoop-hdds/hadoop-dependency-client/pom.xml
+++ b/hadoop-hdds/hadoop-dependency-client/pom.xml
@@ -269,10 +269,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd";>
<artifactId>leveldbjni-all</artifactId>
</exclusion>
<exclusion>
- <groupId>org.fusesource.leveldbjni</groupId>
- <artifactId>leveldbjni-all</artifactId>
- </exclusion>
- <exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</exclusion>
diff --git a/hadoop-hdds/hadoop-dependency-server/pom.xml
b/hadoop-hdds/hadoop-dependency-server/pom.xml
index b9bd05e..e0e26dd 100644
--- a/hadoop-hdds/hadoop-dependency-server/pom.xml
+++ b/hadoop-hdds/hadoop-dependency-server/pom.xml
@@ -88,6 +88,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd";>
<groupId>io.netty</groupId>
<artifactId>*</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.fusesource.leveldbjni</groupId>
+ <artifactId>leveldbjni-all</artifactId>
+ </exclusion>
</exclusions>
</dependency>
</dependencies>
diff --git a/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt
b/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt
index 570fd7c..236aee9 100644
--- a/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt
+++ b/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt
@@ -385,7 +385,6 @@ BSD 3-Clause
com.google.re2j:re2j
com.jcraft:jsch
com.thoughtworks.paranamer:paranamer
- org.fusesource.leveldbjni:leveldbjni-all
org.ow2.asm:asm
diff --git a/hadoop-ozone/dist/src/main/license/jar-report.txt
b/hadoop-ozone/dist/src/main/license/jar-report.txt
index 7166d2b..06aeeca 100644
--- a/hadoop-ozone/dist/src/main/license/jar-report.txt
+++ b/hadoop-ozone/dist/src/main/license/jar-report.txt
@@ -169,7 +169,6 @@ share/ozone/lib/kerby-util.jar
share/ozone/lib/kerby-xdr.jar
share/ozone/lib/kotlin-stdlib-common.jar
share/ozone/lib/kotlin-stdlib.jar
-share/ozone/lib/leveldbjni-all.jar
share/ozone/lib/libthrift.jar
share/ozone/lib/listenablefuture-empty-to-avoid-conflict-with-guava.jar
share/ozone/lib/log4j-api.jar
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketManagerImpl.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketManagerImpl.java
index cdb18ea..9bffde9 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketManagerImpl.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketManagerImpl.java
@@ -39,7 +39,6 @@ import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Time;
import com.google.common.base.Preconditions;
-import org.iq80.leveldb.DBException;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -176,7 +175,7 @@ public class BucketManagerImpl implements BucketManager {
LOG.debug("created bucket: {} in volume: {}", bucketName,
volumeName);
}
- } catch (IOException | DBException ex) {
+ } catch (IOException ex) {
if (!(ex instanceof OMException)) {
LOG.error("Bucket creation failed for bucket:{} in volume:{}",
bucketName, volumeName, ex);
@@ -255,7 +254,7 @@ public class BucketManagerImpl implements BucketManager {
BUCKET_NOT_FOUND);
}
return value;
- } catch (IOException | DBException ex) {
+ } catch (IOException ex) {
if (!(ex instanceof OMException)) {
LOG.error("Exception while getting bucket info for bucket: {}",
bucketName, ex);
@@ -326,7 +325,7 @@ public class BucketManagerImpl implements BucketManager {
commitBucketInfoToDB(omBucketInfo);
- } catch (IOException | DBException ex) {
+ } catch (IOException ex) {
if (!(ex instanceof OMException)) {
LOG.error("Setting bucket property failed for bucket:{} in volume:{}",
bucketName, volumeName, ex);
diff --git a/hadoop-ozone/ozonefs-shaded/pom.xml
b/hadoop-ozone/ozonefs-shaded/pom.xml
index 8bf012c..8662e6a 100644
--- a/hadoop-ozone/ozonefs-shaded/pom.xml
+++ b/hadoop-ozone/ozonefs-shaded/pom.xml
@@ -106,7 +106,6 @@
<include>org.sqlite.**.*</include>
<include>org.tukaani.**.*</include>
<include>org.bouncycastle.**.*</include>
- <include>org.fusesource.leveldbjni.**.*</include>
<include>org.rocksdb.**.*</include>
<include>org.apache.commons.cli.**.*</include>
<include>org.apache.commons.compress.**.*</include>
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkMetadataStoreReads.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkMetadataStoreReads.java
deleted file mode 100644
index b810da2..0000000
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkMetadataStoreReads.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.genesis;
-
-import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.hadoop.hdds.utils.MetadataStore;
-import org.openjdk.jmh.annotations.Benchmark;
-import org.openjdk.jmh.annotations.Param;
-import org.openjdk.jmh.annotations.Scope;
-import org.openjdk.jmh.annotations.Setup;
-import org.openjdk.jmh.annotations.State;
-import org.openjdk.jmh.infra.Blackhole;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-
-import static org.apache.hadoop.ozone.genesis.GenesisUtil.CACHE_10MB_TYPE;
-import static org.apache.hadoop.ozone.genesis.GenesisUtil.CACHE_1GB_TYPE;
-import static org.apache.hadoop.ozone.genesis.GenesisUtil.CLOSED_TYPE;
-import static org.apache.hadoop.ozone.genesis.GenesisUtil.DEFAULT_TYPE;
-
-/**
- * Measure metadatastore read performance.
- */
-@State(Scope.Thread)
-public class BenchMarkMetadataStoreReads {
-
- private static final int DATA_LEN = 1024;
- private static final long MAX_KEYS = 1024 * 10;
-
- private MetadataStore store;
-
- @Param({DEFAULT_TYPE, CACHE_10MB_TYPE, CACHE_1GB_TYPE, CLOSED_TYPE})
- private String type;
-
- @Setup
- public void initialize() throws IOException {
- store = GenesisUtil.getMetadataStore(this.type);
- byte[] data = RandomStringUtils.randomAlphanumeric(DATA_LEN)
- .getBytes(StandardCharsets.UTF_8);
- for (int x = 0; x < MAX_KEYS; x++) {
- store.put(Long.toHexString(x).getBytes(StandardCharsets.UTF_8), data);
- }
- if (type.compareTo(CLOSED_TYPE) == 0) {
- store.compactDB();
- }
- }
-
- @Benchmark
- public void test(Blackhole bh) throws IOException {
- long x = org.apache.commons.lang3.RandomUtils.nextLong(0L, MAX_KEYS);
- bh.consume(
- store.get(Long.toHexString(x).getBytes(StandardCharsets.UTF_8)));
- }
-}
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkMetadataStoreWrites.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkMetadataStoreWrites.java
deleted file mode 100644
index 51010ec..0000000
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkMetadataStoreWrites.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.genesis;
-
-import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.hadoop.hdds.utils.MetadataStore;
-import org.openjdk.jmh.annotations.Benchmark;
-import org.openjdk.jmh.annotations.Param;
-import org.openjdk.jmh.annotations.Scope;
-import org.openjdk.jmh.annotations.Setup;
-import org.openjdk.jmh.annotations.State;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-
-import static org.apache.hadoop.ozone.genesis.GenesisUtil.CACHE_10MB_TYPE;
-import static org.apache.hadoop.ozone.genesis.GenesisUtil.CACHE_1GB_TYPE;
-import static org.apache.hadoop.ozone.genesis.GenesisUtil.DEFAULT_TYPE;
-
-/**
- * Measure default metadatastore put performance.
- */
-@State(Scope.Thread)
-public class BenchMarkMetadataStoreWrites {
-
- private static final int DATA_LEN = 1024;
- private static final long MAX_KEYS = 1024 * 10;
-
- private MetadataStore store;
- private byte[] data;
-
- @Param({DEFAULT_TYPE, CACHE_10MB_TYPE, CACHE_1GB_TYPE})
- private String type;
-
- @Setup
- public void initialize() throws IOException {
- data = RandomStringUtils.randomAlphanumeric(DATA_LEN)
- .getBytes(StandardCharsets.UTF_8);
- store = GenesisUtil.getMetadataStore(this.type);
- }
-
- @Benchmark
- public void test() throws IOException {
- long x = org.apache.commons.lang3.RandomUtils.nextLong(0L, MAX_KEYS);
- store.put(Long.toHexString(x).getBytes(StandardCharsets.UTF_8), data);
- }
-}
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkRocksDbStore.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkRocksDbStore.java
deleted file mode 100644
index 9f79b82..0000000
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkRocksDbStore.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
under
- * the License.
- *
- */
-package org.apache.hadoop.ozone.genesis;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.hadoop.conf.StorageUnit;
-import org.apache.hadoop.hdds.utils.MetadataStore;
-import org.apache.hadoop.hdds.utils.RocksDBStore;
-import org.openjdk.jmh.annotations.*;
-import org.openjdk.jmh.infra.Blackhole;
-import org.rocksdb.*;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Paths;
-
-/**
- * Benchmark rocksdb store.
- */
-@State(Scope.Thread)
-public class BenchMarkRocksDbStore {
- private static final int DATA_LEN = 1024;
- private static final long MAX_KEYS = 1024 * 10;
- private static final int DB_FILE_LEN = 7;
- private static final String TMP_DIR = "java.io.tmpdir";
-
- private MetadataStore store;
- private byte[] data;
-
- @Param(value = {"8"})
- private String blockSize; // 4KB default
-
- @Param(value = {"64"})
- private String writeBufferSize; //64 MB default
-
- @Param(value = {"16"})
- private String maxWriteBufferNumber; // 2 default
-
- @Param(value = {"4"})
- private String maxBackgroundFlushes; // 1 default
-
- @Param(value = {"512"})
- private String maxBytesForLevelBase;
-
- @Param(value = {"4"})
- private String backgroundThreads;
-
- @Param(value = {"5000"})
- private String maxOpenFiles;
-
- @Setup(Level.Trial)
- public void initialize() throws IOException {
- data = RandomStringUtils.randomAlphanumeric(DATA_LEN)
- .getBytes(StandardCharsets.UTF_8);
- org.rocksdb.Options opts = new org.rocksdb.Options();
- File dbFile = Paths.get(System.getProperty(TMP_DIR))
- .resolve(RandomStringUtils.randomNumeric(DB_FILE_LEN))
- .toFile();
- opts.setCreateIfMissing(true);
- opts.setWriteBufferSize(
- (long) StorageUnit.MB.toBytes(Long.parseLong(writeBufferSize)));
- opts.setMaxWriteBufferNumber(Integer.parseInt(maxWriteBufferNumber));
- opts.setMaxBackgroundFlushes(Integer.parseInt(maxBackgroundFlushes));
- BlockBasedTableConfig tableConfig = new BlockBasedTableConfig();
- tableConfig.setBlockSize(
- (long) StorageUnit.KB.toBytes(Long.parseLong(blockSize)));
- opts.setMaxOpenFiles(Integer.parseInt(maxOpenFiles));
- opts.setMaxBytesForLevelBase(
- (long) StorageUnit.MB.toBytes(Long.parseLong(maxBytesForLevelBase)));
- opts.setCompactionStyle(CompactionStyle.UNIVERSAL);
- opts.setLevel0FileNumCompactionTrigger(10);
- opts.setLevel0SlowdownWritesTrigger(20);
- opts.setLevel0StopWritesTrigger(40);
- opts.setTargetFileSizeBase(
- (long) StorageUnit.MB.toBytes(Long.parseLong(maxBytesForLevelBase))
- / 10);
- opts.setMaxBackgroundCompactions(8);
- opts.setUseFsync(false);
- opts.setBytesPerSync(8388608);
- org.rocksdb.Filter bloomFilter = new org.rocksdb.BloomFilter(20);
- tableConfig.setCacheIndexAndFilterBlocks(true);
- tableConfig.setIndexType(IndexType.kHashSearch);
- tableConfig.setFilter(bloomFilter);
- opts.setTableFormatConfig(tableConfig);
- opts.useCappedPrefixExtractor(4);
- store = new RocksDBStore(dbFile, opts);
- }
-
- @TearDown(Level.Trial)
- public void cleanup() throws IOException {
- store.destroy();
- FileUtils.deleteDirectory(new File(TMP_DIR));
- }
-
- @Benchmark
- public void test(Blackhole bh) throws IOException {
- long x = org.apache.commons.lang3.RandomUtils.nextLong(0L, MAX_KEYS);
- store.put(Long.toHexString(x).getBytes(StandardCharsets.UTF_8), data);
- bh.consume(
- store.get(Long.toHexString(x).getBytes(StandardCharsets.UTF_8)));
- }
-}
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java
index b511bee..cffb4c4 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java
@@ -25,7 +25,6 @@ import java.util.List;
import java.util.Random;
import java.util.UUID;
-import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.RatisReplicationConfig;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -39,8 +38,6 @@ import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.server.SCMConfigurator;
import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
-import org.apache.hadoop.hdds.utils.MetadataStore;
-import org.apache.hadoop.hdds.utils.MetadataStoreBuilder;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.ozone.common.Storage;
import org.apache.hadoop.ozone.om.OMConfigKeys;
@@ -48,9 +45,6 @@ import org.apache.hadoop.ozone.om.OMStorage;
import org.apache.hadoop.ozone.om.OzoneManager;
import
org.apache.hadoop.security.authentication.client.AuthenticationException;
-import org.apache.commons.lang3.RandomStringUtils;
-
-
/**
* Utility class for benchmark test cases.
*/
@@ -74,32 +68,6 @@ public final class GenesisUtil {
return Paths.get(System.getProperty(TMP_DIR));
}
- public static MetadataStore getMetadataStore(String dbType)
- throws IOException {
- OzoneConfiguration conf = new OzoneConfiguration();
- MetadataStoreBuilder builder = MetadataStoreBuilder.newBuilder();
- builder.setConf(conf);
- builder.setCreateIfMissing(true);
- builder.setDbFile(
- getTempPath().resolve(RandomStringUtils.randomNumeric(DB_FILE_LEN))
- .toFile());
- switch (dbType) {
- case DEFAULT_TYPE:
- break;
- case CLOSED_TYPE:
- break;
- case CACHE_10MB_TYPE:
- builder.setCacheSize((long) StorageUnit.MB.toBytes(10));
- break;
- case CACHE_1GB_TYPE:
- builder.setCacheSize((long) StorageUnit.GB.toBytes(1));
- break;
- default:
- throw new IllegalStateException("Unknown type: " + dbType);
- }
- return builder.build();
- }
-
public static DatanodeDetails createDatanodeDetails(UUID uuid) {
String ipAddress =
RANDOM.nextInt(256) + "." + RANDOM.nextInt(256) + "." + RANDOM
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
deleted file mode 100644
index 71039f4..0000000
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
+++ /dev/null
@@ -1,562 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.cli;
-
-import com.google.common.base.Preconditions;
-import com.google.common.primitives.Longs;
-import org.apache.commons.cli.BasicParser;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.ParseException;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.hdds.StringUtils;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneAclInfo;
-import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketInfo;
-import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyInfo;
-import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.VolumeInfo;
-import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.UserVolumeInfo;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.scm.container.ContainerInfo;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.hadoop.hdds.utils.MetadataStore;
-import org.apache.hadoop.hdds.utils.MetadataStoreBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.SQLException;
-import java.sql.Statement;
-
-import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DB_SUFFIX;
-import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME;
-import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
-import static org.apache.hadoop.ozone.OzoneConsts.OM_USER_PREFIX;
-
-/**
- * This is the CLI that can be use to convert an ozone metadata DB into
- * a sqlite DB file.
- *
- * NOTE: user should use this CLI in an offline fashion. Namely, this should
not
- * be used to convert a DB that is currently being used by Ozone. Instead,
- * this should be used to debug and diagnosis closed DB instances.
- *
- */
-public class SQLCLI extends Configured implements Tool {
-
- private Options options;
- private BasicParser parser;
- private final OzoneConfiguration conf;
-
- // for container.db
- private static final String CREATE_CONTAINER_INFO =
- "CREATE TABLE containerInfo (" +
- "containerID LONG PRIMARY KEY NOT NULL, " +
- "replicationType TEXT NOT NULL," +
- "replicationFactor TEXT NOT NULL," +
- "usedBytes LONG NOT NULL," +
- "owner TEXT," +
- "numberOfKeys LONG)";
- private static final String CREATE_DATANODE_INFO =
- "CREATE TABLE datanodeInfo (" +
- "hostName TEXT NOT NULL, " +
- "datanodeUUId TEXT PRIMARY KEY NOT NULL," +
- "ipAddress TEXT, " +
- "containerPort INTEGER NOT NULL);";
- private static final String INSERT_CONTAINER_INFO =
- "INSERT INTO containerInfo (containerID, replicationType, "
- + "replicationFactor, usedBytes, owner, "
- + "numberOfKeys) VALUES (\"%d\", \"%s\", \"%s\", \"%d\", "
- + "\"%s\", \"%d\")";
- private static final String INSERT_DATANODE_INFO =
- "INSERT INTO datanodeInfo (hostname, datanodeUUid, ipAddress, " +
- "containerPort) " +
- "VALUES (\"%s\", \"%s\", \"%s\", \"%d\")";
- private static final String INSERT_CONTAINER_MEMBERS =
- "INSERT INTO containerMembers (containerName, datanodeUUID) " +
- "VALUES (\"%s\", \"%s\")";
- // and reuse CREATE_DATANODE_INFO and INSERT_DATANODE_INFO
- // for openContainer.db
- private static final String CREATE_OPEN_CONTAINER =
- "CREATE TABLE openContainer (" +
- "containerName TEXT PRIMARY KEY NOT NULL, " +
- "containerUsed INTEGER NOT NULL)";
- private static final String INSERT_OPEN_CONTAINER =
- "INSERT INTO openContainer (containerName, containerUsed) " +
- "VALUES (\"%s\", \"%s\")";
-
- // for om.db
- private static final String CREATE_VOLUME_LIST =
- "CREATE TABLE volumeList (" +
- "userName TEXT NOT NULL," +
- "volumeName TEXT NOT NULL," +
- "PRIMARY KEY (userName, volumeName))";
- private static final String INSERT_VOLUME_LIST =
- "INSERT INTO volumeList (userName, volumeName) " +
- "VALUES (\"%s\", \"%s\")";
-
- private static final String CREATE_VOLUME_INFO =
- "CREATE TABLE volumeInfo (" +
- "adminName TEXT NOT NULL," +
- "ownerName TEXT NOT NULL," +
- "volumeName TEXT NOT NULL," +
- "PRIMARY KEY (adminName, ownerName, volumeName))";
- private static final String INSERT_VOLUME_INFO =
- "INSERT INTO volumeInfo (adminName, ownerName, volumeName) " +
- "VALUES (\"%s\", \"%s\", \"%s\")";
-
- private static final String CREATE_ACL_INFO =
- "CREATE TABLE aclInfo (" +
- "adminName TEXT NOT NULL," +
- "ownerName TEXT NOT NULL," +
- "volumeName TEXT NOT NULL," +
- "type TEXT NOT NULL," +
- "userName TEXT NOT NULL," +
- "rights TEXT NOT NULL," +
- "FOREIGN KEY (adminName, ownerName, volumeName, userName, type)" +
- "REFERENCES " +
- "volumeInfo(adminName, ownerName, volumeName, userName, type)" +
- "PRIMARY KEY (adminName, ownerName, volumeName, userName, type))";
- private static final String INSERT_ACL_INFO =
- "INSERT INTO aclInfo (adminName, ownerName, volumeName, type, " +
- "userName, rights) " +
- "VALUES (\"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\")";
-
- private static final String CREATE_BUCKET_INFO =
- "CREATE TABLE bucketInfo (" +
- "volumeName TEXT NOT NULL," +
- "bucketName TEXT NOT NULL," +
- "versionEnabled BOOLEAN NOT NULL," +
- "storageType TEXT," +
- "PRIMARY KEY (volumeName, bucketName))";
- private static final String INSERT_BUCKET_INFO =
- "INSERT INTO bucketInfo(volumeName, bucketName, " +
- "versionEnabled, storageType)" +
- "VALUES (\"%s\", \"%s\", \"%s\", \"%s\")";
-
- private static final String CREATE_KEY_INFO =
- "CREATE TABLE keyInfo (" +
- "volumeName TEXT NOT NULL," +
- "bucketName TEXT NOT NULL," +
- "keyName TEXT NOT NULL," +
- "dataSize INTEGER," +
- "blockKey TEXT NOT NULL," +
- "containerName TEXT NOT NULL," +
- "PRIMARY KEY (volumeName, bucketName, keyName))";
- private static final String INSERT_KEY_INFO =
- "INSERT INTO keyInfo (volumeName, bucketName, keyName, dataSize, " +
- "blockKey, containerName)" +
- "VALUES (\"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\")";
-
- private static final Logger LOG =
- LoggerFactory.getLogger(SQLCLI.class);
-
- public SQLCLI(OzoneConfiguration conf) {
- this.options = getOptions();
- this.parser = new BasicParser();
- this.conf = conf;
- }
-
- @SuppressWarnings("static-access")
- private Options getOptions() {
- Options allOptions = new Options();
- Option helpOpt = OptionBuilder
- .hasArg(false)
- .withLongOpt("help")
- .withDescription("display help message")
- .create("h");
- allOptions.addOption(helpOpt);
-
- Option dbPathOption = OptionBuilder
- .withArgName("DB path")
- .withLongOpt("dbPath")
- .hasArgs(1)
- .withDescription("specify DB path")
- .create("p");
- allOptions.addOption(dbPathOption);
-
- Option outPathOption = OptionBuilder
- .withArgName("output path")
- .withLongOpt("outPath")
- .hasArgs(1)
- .withDescription("specify output DB file path")
- .create("o");
- allOptions.addOption(outPathOption);
-
- return allOptions;
- }
-
- public void displayHelp() {
- HelpFormatter helpFormatter = new HelpFormatter();
- Options allOpts = getOptions();
- helpFormatter.printHelp("hdfs oz_debug -p <DB path>"
- + " -o <Output DB file path>", allOpts);
- }
-
- @Override
- public int run(String[] args) throws Exception {
- CommandLine commandLine = parseArgs(args);
- if (commandLine.hasOption("help")) {
- displayHelp();
- return 0;
- }
- if (!commandLine.hasOption("p") || !commandLine.hasOption("o")) {
- displayHelp();
- return -1;
- }
- String value = commandLine.getOptionValue("p");
- LOG.info("DB path {}", value);
- // the value is supposed to be an absolute path to a container file
- Path dbPath = Paths.get(value);
- if (!Files.exists(dbPath)) {
- LOG.error("DB path not exist:{}", dbPath);
- }
- Path parentPath = dbPath.getParent();
- Path dbName = dbPath.getFileName();
- if (parentPath == null || dbName == null) {
- LOG.error("Error processing db path {}", dbPath);
- return -1;
- }
-
- value = commandLine.getOptionValue("o");
- Path outPath = Paths.get(value);
- if (outPath == null || outPath.getParent() == null) {
- LOG.error("Error processing output path {}", outPath);
- return -1;
- }
-
- if (outPath.toFile().isDirectory()) {
- LOG.error("The db output path should be a file instead of a directory");
- return -1;
- }
-
- Path outParentPath = outPath.getParent();
- if (outParentPath != null) {
- if (!Files.exists(outParentPath)) {
- Files.createDirectories(outParentPath);
- }
- }
- LOG.info("Parent path [{}] db name [{}]", parentPath, dbName);
- if (dbName.toString().endsWith(CONTAINER_DB_SUFFIX)) {
- LOG.info("Converting container DB");
- convertContainerDB(dbPath, outPath);
- } else if (dbName.toString().equals(OM_DB_NAME)) {
- LOG.info("Converting om DB");
- convertOMDB(dbPath, outPath);
- } else {
- LOG.error("Unrecognized db name {}", dbName);
- }
- return 0;
- }
-
- private Connection connectDB(String dbPath) throws Exception {
- Class.forName("org.sqlite.JDBC");
- String connectPath =
- String.format("jdbc:sqlite:%s", dbPath);
- return DriverManager.getConnection(connectPath);
- }
-
- private void executeSQL(Connection conn, String sql) throws SQLException {
- try (Statement stmt = conn.createStatement()) {
- stmt.executeUpdate(sql);
- }
- }
-
- /**
- * Convert om.db to sqlite db file. With following schema.
- * (* for primary key)
- *
- * 1. for key type USER, it contains a username and a list volumes
- * volumeList
- * --------------------------------
- * userName* | volumeName*
- * --------------------------------
- *
- * 2. for key type VOLUME:
- *
- * volumeInfo
- * ----------------------------------------------
- * adminName | ownerName* | volumeName* | aclID
- * ----------------------------------------------
- *
- * aclInfo
- * ----------------------------------------------
- * aclEntryID* | type* | userName* | rights
- * ----------------------------------------------
- *
- * 3. for key type BUCKET
- * bucketInfo
- * --------------------------------------------------------
- * volumeName* | bucketName* | versionEnabled | storageType
- * --------------------------------------------------------
- *
- * TODO : the following table will be changed when key partition is added.
- * Only has the minimum entries for test purpose now.
- * 4. for key type KEY
- * -----------------------------------------------
- * volumeName* | bucketName* | keyName* | dataSize
- * -----------------------------------------------
- *
- *
- *
- * @param dbPath
- * @param outPath
- * @throws Exception
- */
- private void convertOMDB(Path dbPath, Path outPath) throws Exception {
- LOG.info("Create tables for sql om db.");
- File dbFile = dbPath.toFile();
- try (MetadataStore dbStore = MetadataStoreBuilder.newBuilder()
- .setConf(conf).setDbFile(dbFile).build();
- Connection conn = connectDB(outPath.toString())) {
- executeSQL(conn, CREATE_VOLUME_LIST);
- executeSQL(conn, CREATE_VOLUME_INFO);
- executeSQL(conn, CREATE_ACL_INFO);
- executeSQL(conn, CREATE_BUCKET_INFO);
- executeSQL(conn, CREATE_KEY_INFO);
-
- dbStore.iterate(null, (key, value) -> {
- String keyString = StringUtils.bytes2String(key);
- KeyType type = getKeyType(keyString);
- try {
- insertOMDB(conn, type, keyString, value);
- } catch (IOException | SQLException ex) {
- LOG.error("Exception inserting key {} type {}", keyString, type, ex);
- }
- return true;
- });
- }
- }
-
- private void insertOMDB(Connection conn, KeyType type, String keyName,
- byte[] value) throws IOException, SQLException {
- switch (type) {
- case USER:
- UserVolumeInfo volumeList = UserVolumeInfo.parseFrom(value);
- for (String volumeName : volumeList.getVolumeNamesList()) {
- String insertVolumeList =
- String.format(INSERT_VOLUME_LIST, keyName, volumeName);
- executeSQL(conn, insertVolumeList);
- }
- break;
- case VOLUME:
- VolumeInfo volumeInfo = VolumeInfo.parseFrom(value);
- String adminName = volumeInfo.getAdminName();
- String ownerName = volumeInfo.getOwnerName();
- String volumeName = volumeInfo.getVolume();
- String insertVolumeInfo =
- String.format(INSERT_VOLUME_INFO, adminName, ownerName, volumeName);
- executeSQL(conn, insertVolumeInfo);
- for (OzoneAclInfo aclInfo : volumeInfo.getVolumeAclsList()) {
- String insertAclInfo =
- String.format(INSERT_ACL_INFO, adminName, ownerName, volumeName,
- aclInfo.getType(), aclInfo.getName(), aclInfo.getRights());
- executeSQL(conn, insertAclInfo);
- }
- break;
- case BUCKET:
- BucketInfo bucketInfo = BucketInfo.parseFrom(value);
- String insertBucketInfo =
- String.format(INSERT_BUCKET_INFO, bucketInfo.getVolumeName(),
- bucketInfo.getBucketName(), bucketInfo.getIsVersionEnabled(),
- bucketInfo.getStorageType());
- executeSQL(conn, insertBucketInfo);
- break;
- case KEY:
- KeyInfo keyInfo = KeyInfo.parseFrom(value);
- // TODO : the two fields container name and block id are no longer used,
- // need to revisit this later.
- String insertKeyInfo =
- String.format(INSERT_KEY_INFO, keyInfo.getVolumeName(),
- keyInfo.getBucketName(), keyInfo.getKeyName(),
- keyInfo.getDataSize(), "EMPTY",
- "EMPTY");
- executeSQL(conn, insertKeyInfo);
- break;
- default:
- throw new IOException("Unknown key from om.db");
- }
- }
-
- // TODO: This has to be fixed.
- // we don't have prefix anymore. now each key is written into different
- // table. The logic has to be changed.
- private KeyType getKeyType(String key) {
- if (key.startsWith(OM_USER_PREFIX)) {
- return KeyType.USER;
- } else if (key.startsWith(OM_KEY_PREFIX)) {
- return key.replaceFirst(OM_KEY_PREFIX, "")
- .contains(OM_KEY_PREFIX) ? KeyType.BUCKET : KeyType.VOLUME;
- }else {
- return KeyType.KEY;
- }
- }
-
- private enum KeyType {
- USER,
- VOLUME,
- BUCKET,
- KEY,
- UNKNOWN
- }
-
- /**
- * Convert container.db to sqlite. The schema of sql db:
- * three tables, containerId, containerMachines, datanodeInfo
- * (* for primary key)
- *
- * containerInfo:
- * ----------------------------------------------
- * container name* | container lead datanode uuid
- * ----------------------------------------------
- *
- * containerMembers:
- * --------------------------------
- * container name* | datanodeUUid*
- * --------------------------------
- *
- * datanodeInfo:
- * ---------------------------------------------------------
- * hostname | datanodeUUid* | xferPort | ipcPort
- * ---------------------------------------------------------
- *
- * --------------------------------
- * | containerPort
- * --------------------------------
- *
- * @param dbPath path to container db.
- * @param outPath path to output sqlite
- * @throws IOException throws exception.
- */
- private void convertContainerDB(Path dbPath, Path outPath)
- throws Exception {
- LOG.info("Create tables for sql container db.");
- File dbFile = dbPath.toFile();
- try (MetadataStore dbStore = MetadataStoreBuilder.newBuilder()
- .setConf(conf).setDbFile(dbFile).build();
- Connection conn = connectDB(outPath.toString())) {
- executeSQL(conn, CREATE_CONTAINER_INFO);
-
- dbStore.iterate(null, (key, value) -> {
- long containerID = Longs.fromByteArray(key);
- ContainerInfo containerInfo = null;
- containerInfo = ContainerInfo.fromProtobuf(
- HddsProtos.ContainerInfoProto.PARSER.parseFrom(value));
- Preconditions.checkNotNull(containerInfo);
- try {
- //TODO: include container state to sqllite schema
- insertContainerDB(conn, containerInfo, containerID);
- return true;
- } catch (SQLException e) {
- throw new IOException(e);
- }
- });
- }
- }
-
- /**
- * Insert into the sqlite DB of container.db.
- * @param conn the connection to the sqlite DB.
- * @param containerInfo
- * @param containerID
- * @throws SQLException throws exception.
- */
- private void insertContainerDB(Connection conn, ContainerInfo containerInfo,
- long containerID) throws SQLException {
- LOG.info("Insert to sql container db, for container {}", containerID);
- String insertContainerInfo = String.format(
- INSERT_CONTAINER_INFO, containerID,
- containerInfo.getReplicationType(),
- containerInfo.getReplicationFactor(),
- containerInfo.getUsedBytes(),
- containerInfo.getOwner(),
- containerInfo.getNumberOfKeys());
-
- executeSQL(conn, insertContainerInfo);
- LOG.info("Insertion completed.");
- }
-
-
- /**
- * Convert openContainer.db to sqlite db file. This is rather simple db,
- * the schema has only one table:
- *
- * openContainer
- * -------------------------------
- * containerName* | containerUsed
- * -------------------------------
- *
- * @param dbPath path to container db.
- * @param outPath path to output sqlite
- * @throws IOException throws exception.
- */
- private void convertOpenContainerDB(Path dbPath, Path outPath)
- throws Exception {
- LOG.info("Create table for open container db.");
- File dbFile = dbPath.toFile();
- try (MetadataStore dbStore = MetadataStoreBuilder.newBuilder()
- .setConf(conf).setDbFile(dbFile).build();
- Connection conn = connectDB(outPath.toString())) {
- executeSQL(conn, CREATE_OPEN_CONTAINER);
-
- dbStore.iterate(null, (key, value) -> {
- String containerName = StringUtils.bytes2String(key);
- Long containerUsed =
- Long.parseLong(StringUtils.bytes2String(value));
- String insertOpenContainer = String
- .format(INSERT_OPEN_CONTAINER, containerName, containerUsed);
- try {
- executeSQL(conn, insertOpenContainer);
- return true;
- } catch (SQLException e) {
- throw new IOException(e);
- }
- });
- }
- }
-
- private CommandLine parseArgs(String[] argv)
- throws ParseException {
- return parser.parse(options, argv);
- }
-
- public static void main(String[] args) {
- Tool shell = new SQLCLI(new OzoneConfiguration());
- int res = 0;
- try {
- ToolRunner.run(shell, args);
- } catch (Exception ex) {
- LOG.error(ex.toString());
- if (LOG.isDebugEnabled()) {
- LOG.debug("Command execution failed", ex);
- }
- res = 1;
- }
- System.exit(res);
- }
-}
diff --git a/pom.xml b/pom.xml
index e535476..3a05cc0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1460,12 +1460,6 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xs
</dependency>
<dependency>
- <groupId>org.fusesource.leveldbjni</groupId>
- <artifactId>leveldbjni-all</artifactId>
- <version>1.8</version>
- </dependency>
-
- <dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>azure-storage</artifactId>
<version>7.0.0</version>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
