This is an automated email from the ASF dual-hosted git repository.
arp pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 82d88a8 HDDS-1705. Recon: Add estimatedTotalCount to the response of
containers and containers/{id} endpoints. Contributed by Vivek Ratnavel
Subramanian.
82d88a8 is described below
commit 82d88a8d30790c5841fc4f71ea39cc12b470c41f
Author: Vivek Ratnavel Subramanian <[email protected]>
AuthorDate: Mon Jul 8 21:06:50 2019 -0700
HDDS-1705. Recon: Add estimatedTotalCount to the response of containers and
containers/{id} endpoints. Contributed by Vivek Ratnavel Subramanian.
---
.../org/apache/hadoop/ozone/common/Storage.java | 6 +-
.../common/src/main/resources/ozone-default.xml | 6 +-
.../hadoop/ozone/om/OzoneManagerStarter.java | 2 +-
.../recon/codegen/ReconSchemaGenerationModule.java | 2 +
.../ozone/recon/schema/StatsSchemaDefinition.java | 61 ++++
.../apache/hadoop/ozone/recon/ReconConstants.java | 8 +-
.../org/apache/hadoop/ozone/recon/ReconServer.java | 22 ++
.../ozone/recon/api/ContainerKeyService.java | 22 +-
.../ozone/recon/api/types/ContainersResponse.java | 94 ++++++
.../hadoop/ozone/recon/api/types/KeysResponse.java | 93 ++++++
.../recon/spi/ContainerDBServiceProvider.java | 58 +++-
.../spi/impl/ContainerDBServiceProviderImpl.java | 137 ++++++++-
.../recon/spi/impl/ReconContainerDBProvider.java | 4 +
.../ozone/recon/tasks/ContainerKeyMapperTask.java | 57 +++-
.../recon/GuiceInjectorUtilsForTestsImpl.java} | 25 +-
.../ozone/recon/api/TestContainerKeyService.java | 186 +++++++-----
.../recon/persistence/AbstractSqlDatabaseTest.java | 12 +-
.../persistence/TestStatsSchemaDefinition.java | 147 ++++++++++
.../impl/TestContainerDBServiceProviderImpl.java | 326 +++++++++++++--------
.../impl/TestOzoneManagerServiceProviderImpl.java | 53 +---
.../recon/tasks/TestContainerKeyMapperTask.java | 127 ++++----
.../recon/types/GuiceInjectorUtilsForTests.java | 117 ++++++++
22 files changed, 1209 insertions(+), 356 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Storage.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Storage.java
index f393ed9..7992dad 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Storage.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Storage.java
@@ -81,7 +81,7 @@ public abstract class Storage {
/**
* Gets the path of the Storage dir.
- * @return Stoarge dir path
+ * @return Storage dir path
*/
public String getStorageDir() {
return storageDir.getAbsoluteFile().toString();
@@ -117,7 +117,7 @@ public abstract class Storage {
}
/**
- * Retreives the storageInfo instance to read/write the common
+ * Retrieves the storageInfo instance to read/write the common
* version file properties.
* @return the instance of the storageInfo class
*/
@@ -128,7 +128,7 @@ public abstract class Storage {
abstract protected Properties getNodeProperties();
/**
- * Sets the Node properties spaecific to OM/SCM.
+ * Sets the Node properties specific to OM/SCM.
*/
private void setNodeProperties() {
Properties nodeProperties = getNodeProperties();
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml
b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index c10aa33..219bd29 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -659,7 +659,7 @@
<value/>
<tag>OZONE, OM, SCM, CONTAINER, STORAGE, REQUIRED</tag>
<description>
- This setting is the fallback location for SCM, OM and DataNodes
+ This setting is the fallback location for SCM, OM, Recon and DataNodes
to store their metadata. This setting may be used only in test/PoC
clusters to simplify configuration.
@@ -2457,7 +2457,7 @@
<value/>
<tag>OZONE, RECON</tag>
<description>
- Ozone Recon datbase password.
+ Ozone Recon database password.
</description>
</property>
<property>
@@ -2484,7 +2484,7 @@
<description>
The max active connections to the SQL database. The default SQLite
database only allows single active connection, set this to a
- resonable value like 10, for external production database.
+ reasonable value like 10, for external production database.
</description>
</property>
<property>
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerStarter.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerStarter.java
index 8a0c317..fa229aa 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerStarter.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerStarter.java
@@ -60,7 +60,7 @@ public class OzoneManagerStarter extends GenericCli {
public Void call() throws Exception {
/**
* This method is invoked only when a sub-command is not called. Therefore
- * if someone runs "ozone om" with no parameters, this is the methood
+ * if someone runs "ozone om" with no parameters, this is the method
* which runs and starts the OM.
*/
commonInit();
diff --git
a/hadoop-ozone/ozone-recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/ReconSchemaGenerationModule.java
b/hadoop-ozone/ozone-recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/ReconSchemaGenerationModule.java
index 5e06b99..c393cc2 100644
---
a/hadoop-ozone/ozone-recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/ReconSchemaGenerationModule.java
+++
b/hadoop-ozone/ozone-recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/ReconSchemaGenerationModule.java
@@ -19,6 +19,7 @@ package org.hadoop.ozone.recon.codegen;
import org.hadoop.ozone.recon.schema.ReconInternalSchemaDefinition;
import org.hadoop.ozone.recon.schema.ReconSchemaDefinition;
+import org.hadoop.ozone.recon.schema.StatsSchemaDefinition;
import org.hadoop.ozone.recon.schema.UtilizationSchemaDefinition;
import com.google.inject.AbstractModule;
@@ -36,5 +37,6 @@ public class ReconSchemaGenerationModule extends
AbstractModule {
Multibinder.newSetBinder(binder(), ReconSchemaDefinition.class);
schemaBinder.addBinding().to(UtilizationSchemaDefinition.class);
schemaBinder.addBinding().to(ReconInternalSchemaDefinition.class);
+ schemaBinder.addBinding().to(StatsSchemaDefinition.class);
}
}
diff --git
a/hadoop-ozone/ozone-recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/StatsSchemaDefinition.java
b/hadoop-ozone/ozone-recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/StatsSchemaDefinition.java
new file mode 100644
index 0000000..6763bc8
--- /dev/null
+++
b/hadoop-ozone/ozone-recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/StatsSchemaDefinition.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.hadoop.ozone.recon.schema;
+
+import com.google.inject.Inject;
+import org.jooq.impl.DSL;
+import org.jooq.impl.SQLDataType;
+
+import javax.sql.DataSource;
+import java.sql.Connection;
+import java.sql.SQLException;
+
+/**
+ * Class used to create tables that are required for storing Ozone statistics.
+ */
+public class StatsSchemaDefinition implements ReconSchemaDefinition {
+
+ public static final String GLOBAL_STATS_TABLE_NAME = "global_stats";
+ private final DataSource dataSource;
+
+ @Inject
+ StatsSchemaDefinition(DataSource dataSource) {
+ this.dataSource = dataSource;
+ }
+
+ @Override
+ public void initializeSchema() throws SQLException {
+ Connection conn = dataSource.getConnection();
+ createGlobalStatsTable(conn);
+ }
+
+ /**
+ * Create the Ozone Global Stats table.
+ * @param conn connection
+ */
+ private void createGlobalStatsTable(Connection conn) {
+ DSL.using(conn).createTableIfNotExists(GLOBAL_STATS_TABLE_NAME)
+ .column("key", SQLDataType.VARCHAR(255))
+ .column("value", SQLDataType.BIGINT)
+ .column("last_updated_timestamp", SQLDataType.TIMESTAMP)
+ .constraint(DSL.constraint("pk_key")
+ .primaryKey("key"))
+ .execute();
+ }
+}
diff --git
a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java
b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java
index 6c768cf..86c8a32 100644
---
a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java
+++
b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java
@@ -32,14 +32,20 @@ public final class ReconConstants {
public static final String RECON_CONTAINER_DB = "recon-" +
CONTAINER_DB_SUFFIX;
+ public static final String CONTAINER_COUNT_KEY = "totalCount";
+
public static final String RECON_OM_SNAPSHOT_DB =
"om.snapshot.db";
public static final String CONTAINER_KEY_TABLE =
"containerKeyTable";
+ public static final String CONTAINER_KEY_COUNT_TABLE =
+ "containerKeyCountTable";
+
public static final String FETCH_ALL = "-1";
- public static final String RECON_QUERY_PREVKEY = "prev-key";
+ public static final String RECON_QUERY_PREVKEY = "prevKey";
+ public static final String PREV_CONTAINER_ID_DEFAULT_VALUE = "0";
public static final String RECON_QUERY_LIMIT = "limit";
}
diff --git
a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java
b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java
index fb2dcc3..d813622 100644
---
a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java
+++
b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java
@@ -33,6 +33,9 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider;
import org.apache.hadoop.ozone.recon.spi.OzoneManagerServiceProvider;
import org.apache.hadoop.ozone.recon.tasks.ContainerKeyMapperTask;
+import org.hadoop.ozone.recon.schema.ReconInternalSchemaDefinition;
+import org.hadoop.ozone.recon.schema.StatsSchemaDefinition;
+import org.hadoop.ozone.recon.schema.UtilizationSchemaDefinition;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -75,6 +78,25 @@ public class ReconServer extends GenericCli {
//Pass on injector to listener that does the Guice - Jersey HK2 bridging.
ReconGuiceServletContextListener.setInjector(injector);
+ LOG.info("Initializing Recon server...");
+ try {
+ StatsSchemaDefinition statsSchemaDefinition = injector.getInstance(
+ StatsSchemaDefinition.class);
+ statsSchemaDefinition.initializeSchema();
+
+ UtilizationSchemaDefinition utilizationSchemaDefinition =
+ injector.getInstance(UtilizationSchemaDefinition.class);
+ utilizationSchemaDefinition.initializeSchema();
+
+ ReconInternalSchemaDefinition reconInternalSchemaDefinition =
+ injector.getInstance(ReconInternalSchemaDefinition.class);
+ reconInternalSchemaDefinition.initializeSchema();
+
+ LOG.info("Recon server initialized successfully!");
+ } catch (Exception e) {
+ LOG.error("Error during initializing Recon server.", e);
+ }
+
httpServer = injector.getInstance(ReconHttpServer.class);
LOG.info("Starting Recon server");
httpServer.start();
diff --git
a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerKeyService.java
b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerKeyService.java
index 20992d2..8b8e8a7 100644
---
a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerKeyService.java
+++
b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerKeyService.java
@@ -44,14 +44,17 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix;
import org.apache.hadoop.ozone.recon.api.types.ContainerMetadata;
+import org.apache.hadoop.ozone.recon.api.types.ContainersResponse;
import org.apache.hadoop.ozone.recon.api.types.KeyMetadata;
import
org.apache.hadoop.ozone.recon.api.types.KeyMetadata.ContainerBlockMetadata;
+import org.apache.hadoop.ozone.recon.api.types.KeysResponse;
import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.hadoop.ozone.recon.ReconConstants.FETCH_ALL;
+import static
org.apache.hadoop.ozone.recon.ReconConstants.PREV_CONTAINER_ID_DEFAULT_VALUE;
import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_QUERY_LIMIT;
import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_QUERY_PREVKEY;
@@ -84,15 +87,20 @@ public class ContainerKeyService {
@GET
public Response getContainers(
@DefaultValue(FETCH_ALL) @QueryParam(RECON_QUERY_LIMIT) int limit,
- @DefaultValue("0") @QueryParam(RECON_QUERY_PREVKEY) long prevKey) {
+ @DefaultValue(PREV_CONTAINER_ID_DEFAULT_VALUE)
+ @QueryParam(RECON_QUERY_PREVKEY) long prevKey) {
Map<Long, ContainerMetadata> containersMap;
+ long containersCount;
try {
containersMap = containerDBServiceProvider.getContainers(limit, prevKey);
+ containersCount = containerDBServiceProvider.getCountForContainers();
} catch (IOException ioEx) {
throw new WebApplicationException(ioEx,
Response.Status.INTERNAL_SERVER_ERROR);
}
- return Response.ok(containersMap.values()).build();
+ ContainersResponse containersResponse =
+ new ContainersResponse(containersCount, containersMap.values());
+ return Response.ok(containersResponse).build();
}
/**
@@ -107,13 +115,14 @@ public class ContainerKeyService {
* @return {@link Response}
*/
@GET
- @Path("/{id}")
+ @Path("/{id}/keys")
public Response getKeysForContainer(
@PathParam("id") Long containerID,
@DefaultValue(FETCH_ALL) @QueryParam(RECON_QUERY_LIMIT) int limit,
@DefaultValue(StringUtils.EMPTY) @QueryParam(RECON_QUERY_PREVKEY)
String prevKeyPrefix) {
Map<String, KeyMetadata> keyMetadataMap = new LinkedHashMap<>();
+ long totalCount;
try {
Map<ContainerKeyPrefix, Integer> containerKeyPrefixMap =
containerDBServiceProvider.getKeyPrefixesForContainer(containerID,
@@ -187,10 +196,15 @@ public class ContainerKeyService {
}});
}
}
+
+ totalCount =
+ containerDBServiceProvider.getKeyCountForContainer(containerID);
} catch (IOException ioEx) {
throw new WebApplicationException(ioEx,
Response.Status.INTERNAL_SERVER_ERROR);
}
- return Response.ok(keyMetadataMap.values()).build();
+ KeysResponse keysResponse =
+ new KeysResponse(totalCount, keyMetadataMap.values());
+ return Response.ok(keysResponse).build();
}
}
diff --git
a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainersResponse.java
b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainersResponse.java
new file mode 100644
index 0000000..2bad498
--- /dev/null
+++
b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainersResponse.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.recon.api.types;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.hadoop.ozone.web.utils.JsonUtils;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+
+/**
+ * Class that represents the API Response structure of Containers.
+ */
+public class ContainersResponse {
+ /**
+ * Contains a map with total count of containers and list of containers.
+ */
+ @JsonProperty("data")
+ private ContainersResponseData containersResponseData;
+
+ public ContainersResponse() {
+ this(0, new ArrayList<>());
+ }
+
+ public ContainersResponse(long totalCount,
+ Collection<ContainerMetadata> containers) {
+ this.containersResponseData =
+ new ContainersResponseData(totalCount, containers);
+ }
+
+ public String toJsonString() {
+ try {
+ return JsonUtils.toJsonString(this);
+ } catch (IOException ignored) {
+ return null;
+ }
+ }
+
+ public ContainersResponseData getContainersResponseData() {
+ return containersResponseData;
+ }
+
+ public void setContainersResponseData(ContainersResponseData
+ containersResponseData) {
+ this.containersResponseData = containersResponseData;
+ }
+
+ /**
+ * Class that encapsulates the data presented in Containers API Response.
+ */
+ public static class ContainersResponseData {
+ /**
+ * Total count of the containers.
+ */
+ @JsonProperty("totalCount")
+ private long totalCount;
+
+ /**
+ * An array of containers.
+ */
+ @JsonProperty("containers")
+ private Collection<ContainerMetadata> containers;
+
+ ContainersResponseData(long totalCount,
+ Collection<ContainerMetadata> containers) {
+ this.totalCount = totalCount;
+ this.containers = containers;
+ }
+
+ public long getTotalCount() {
+ return totalCount;
+ }
+
+ public Collection<ContainerMetadata> getContainers() {
+ return containers;
+ }
+ }
+}
diff --git
a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/KeysResponse.java
b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/KeysResponse.java
new file mode 100644
index 0000000..f2704c5
--- /dev/null
+++
b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/KeysResponse.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.recon.api.types;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.hadoop.ozone.web.utils.JsonUtils;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+
+/**
+ * Class that represents the API Response structure of Keys within a container.
+ */
+public class KeysResponse {
+ /**
+ * Contains a map with total count of keys inside the given container and a
+ * list of keys with metadata.
+ */
+ @JsonProperty("data")
+ private KeysResponseData keysResponseData;
+
+ public KeysResponse() {
+ this(0, new ArrayList<>());
+ }
+
+ public KeysResponse(long totalCount,
+ Collection<KeyMetadata> keys) {
+ this.keysResponseData =
+ new KeysResponseData(totalCount, keys);
+ }
+
+ public String toJsonString() {
+ try {
+ return JsonUtils.toJsonString(this);
+ } catch (IOException ignored) {
+ return null;
+ }
+ }
+
+ public KeysResponseData getKeysResponseData() {
+ return keysResponseData;
+ }
+
+ public void setKeysResponseData(KeysResponseData keysResponseData) {
+ this.keysResponseData = keysResponseData;
+ }
+
+ /**
+ * Class that encapsulates the data presented in Keys API Response.
+ */
+ public static class KeysResponseData {
+ /**
+ * Total count of the keys.
+ */
+ @JsonProperty("totalCount")
+ private long totalCount;
+
+ /**
+ * An array of keys.
+ */
+ @JsonProperty("keys")
+ private Collection<KeyMetadata> keys;
+
+ KeysResponseData(long totalCount, Collection<KeyMetadata> keys) {
+ this.totalCount = totalCount;
+ this.keys = keys;
+ }
+
+ public long getTotalCount() {
+ return totalCount;
+ }
+
+ public Collection<KeyMetadata> getKeys() {
+ return keys;
+ }
+ }
+}
diff --git
a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ContainerDBServiceProvider.java
b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ContainerDBServiceProvider.java
index a17d67b..cdd6f50 100644
---
a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ContainerDBServiceProvider.java
+++
b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ContainerDBServiceProvider.java
@@ -52,15 +52,49 @@ public interface ContainerDBServiceProvider {
Integer count) throws IOException;
/**
- * Get the stored key prefix count for the given containerId, key prefix.
+ * Store the containerID -> no. of keys count into the container DB store.
*
- * @param containerKeyPrefix the containerId, key-prefix tuple.
+ * @param containerID the containerID.
+ * @param count count of the keys within the given containerID.
+ * @throws IOException
+ */
+ void storeContainerKeyCount(Long containerID, Long count) throws IOException;
+
+ /**
+ * Store the total count of containers into the container DB store.
+ *
+ * @param count count of the containers present in the system.
+ */
+ void storeContainerCount(Long count);
+
+ /**
+ * Get the stored key prefix count for the given containerID, key prefix.
+ *
+ * @param containerKeyPrefix the containerID, key-prefix tuple.
* @return count of keys with that prefix.
*/
- Integer getCountForForContainerKeyPrefix(
+ Integer getCountForContainerKeyPrefix(
ContainerKeyPrefix containerKeyPrefix) throws IOException;
/**
+ * Get the total count of keys within the given containerID.
+ *
+ * @param containerID the given containerId.
+ * @return count of keys within the given containerID.
+ * @throws IOException
+ */
+ long getKeyCountForContainer(Long containerID) throws IOException;
+
+ /**
+ * Get if a containerID exists or not.
+ *
+ * @param containerID the given containerID.
+ * @return if the given ContainerID exists or not.
+ * @throws IOException
+ */
+ boolean doesContainerExists(Long containerID) throws IOException;
+
+ /**
* Get the stored key prefixes for the given containerId.
*
* @param containerId the given containerId.
@@ -95,6 +129,7 @@ public interface ContainerDBServiceProvider {
/**
* Delete an entry in the container DB.
+ *
* @param containerKeyPrefix container key prefix to be deleted.
* @throws IOException exception.
*/
@@ -105,5 +140,20 @@ public interface ContainerDBServiceProvider {
* Get iterator to the entire container DB.
* @return TableIterator
*/
- TableIterator getContainerTableIterator();
+ TableIterator getContainerTableIterator() throws IOException;
+
+ /**
+ * Get the total count of containers present in the system.
+ *
+ * @return total count of containers.
+ * @throws IOException
+ */
+ long getCountForContainers() throws IOException;
+
+ /**
+ * Increment the total count for containers in the system by the given count.
+ *
+ * @param count no. of new containers to add to containers total count.
+ */
+ void incrementContainerCountBy(long count);
}
diff --git
a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ContainerDBServiceProviderImpl.java
b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ContainerDBServiceProviderImpl.java
index 9957307..42aab2e 100644
---
a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ContainerDBServiceProviderImpl.java
+++
b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ContainerDBServiceProviderImpl.java
@@ -18,10 +18,16 @@
package org.apache.hadoop.ozone.recon.spi.impl;
+import static org.apache.hadoop.ozone.recon.ReconConstants.CONTAINER_COUNT_KEY;
+import static
org.apache.hadoop.ozone.recon.ReconConstants.CONTAINER_KEY_COUNT_TABLE;
import static org.apache.hadoop.ozone.recon.ReconConstants.CONTAINER_KEY_TABLE;
+import static org.jooq.impl.DSL.currentTimestamp;
+import static org.jooq.impl.DSL.select;
+import static org.jooq.impl.DSL.using;
import java.io.File;
import java.io.IOException;
+import java.sql.Timestamp;
import java.util.LinkedHashMap;
import java.util.Map;
@@ -38,6 +44,9 @@ import org.apache.hadoop.utils.db.DBStore;
import org.apache.hadoop.utils.db.Table;
import org.apache.hadoop.utils.db.Table.KeyValue;
import org.apache.hadoop.utils.db.TableIterator;
+import org.hadoop.ozone.recon.schema.tables.daos.GlobalStatsDao;
+import org.hadoop.ozone.recon.schema.tables.pojos.GlobalStats;
+import org.jooq.Configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -52,6 +61,8 @@ public class ContainerDBServiceProviderImpl
LoggerFactory.getLogger(ContainerDBServiceProviderImpl.class);
private Table<ContainerKeyPrefix, Integer> containerKeyTable;
+ private Table<Long, Long> containerKeyCountTable;
+ private GlobalStatsDao globalStatsDao;
@Inject
private OzoneConfiguration configuration;
@@ -60,20 +71,28 @@ public class ContainerDBServiceProviderImpl
private DBStore containerDbStore;
@Inject
- public ContainerDBServiceProviderImpl(DBStore dbStore) {
+ private Configuration sqlConfiguration;
+
+ @Inject
+ public ContainerDBServiceProviderImpl(DBStore dbStore,
+ Configuration sqlConfiguration) {
+ globalStatsDao = new GlobalStatsDao(sqlConfiguration);
try {
this.containerKeyTable = dbStore.getTable(CONTAINER_KEY_TABLE,
ContainerKeyPrefix.class, Integer.class);
+ this.containerKeyCountTable = dbStore.getTable(CONTAINER_KEY_COUNT_TABLE,
+ Long.class, Long.class);
} catch (IOException e) {
- LOG.error("Unable to create Container Key Table. " + e);
+ LOG.error("Unable to create Container Key tables." + e);
}
}
/**
* Initialize a new container DB instance, getting rid of the old instance
* and then storing the passed in container prefix counts into the created
- * DB instance.
- * @param containerKeyPrefixCounts Map of containerId, key-prefix tuple to
+ * DB instance. Also, truncate or reset the SQL tables as required.
+ * @param containerKeyPrefixCounts Map of container key-prefix to
+ * number of keys with the prefix.
* @throws IOException
*/
@Override
@@ -91,17 +110,23 @@ public class ContainerDBServiceProviderImpl
oldDBLocation.getAbsolutePath());
FileUtils.deleteDirectory(oldDBLocation);
}
- for (Map.Entry<ContainerKeyPrefix, Integer> entry :
- containerKeyPrefixCounts.entrySet()) {
- containerKeyTable.put(entry.getKey(), entry.getValue());
+
+ if (containerKeyPrefixCounts != null) {
+ for (Map.Entry<ContainerKeyPrefix, Integer> entry :
+ containerKeyPrefixCounts.entrySet()) {
+ containerKeyTable.put(entry.getKey(), entry.getValue());
+ }
}
+
+ // reset total count of containers to zero
+ storeContainerCount(0L);
}
/**
- * Concatenate the containerId and Key Prefix using a delimiter and store the
+ * Concatenate the containerID and Key Prefix using a delimiter and store the
* count into the container DB store.
*
- * @param containerKeyPrefix the containerId, key-prefix tuple.
+ * @param containerKeyPrefix the containerID, key-prefix tuple.
* @param count Count of the keys matching that prefix.
* @throws IOException
*/
@@ -113,15 +138,53 @@ public class ContainerDBServiceProviderImpl
}
/**
+ * Store the containerID -> no. of keys count into the container DB store.
+ *
+ * @param containerID the containerID.
+ * @param count count of the keys within the given containerID.
+ * @throws IOException
+ */
+ @Override
+ public void storeContainerKeyCount(Long containerID, Long count)
+ throws IOException {
+ containerKeyCountTable.put(containerID, count);
+ }
+
+ /**
+ * Get the total count of keys within the given containerID.
+ *
+ * @param containerID the given containerID.
+ * @return count of keys within the given containerID.
+ * @throws IOException
+ */
+ @Override
+ public long getKeyCountForContainer(Long containerID) throws IOException {
+ Long keyCount = containerKeyCountTable.get(containerID);
+ return keyCount == null ? 0L : keyCount;
+ }
+
+ /**
+ * Get if a containerID exists or not.
+ *
+ * @param containerID the given containerID.
+ * @return if the given ContainerID exists or not.
+ * @throws IOException
+ */
+ @Override
+ public boolean doesContainerExists(Long containerID) throws IOException {
+ return containerKeyCountTable.get(containerID) != null;
+ }
+
+ /**
* Put together the key from the passed in object and get the count from
* the container DB store.
*
- * @param containerKeyPrefix the containerId, key-prefix tuple.
- * @return count of keys matching the containerId, key-prefix.
+ * @param containerKeyPrefix the containerID, key-prefix tuple.
+ * @return count of keys matching the containerID, key-prefix.
* @throws IOException
*/
@Override
- public Integer getCountForForContainerKeyPrefix(
+ public Integer getCountForContainerKeyPrefix(
ContainerKeyPrefix containerKeyPrefix) throws IOException {
Integer count = containerKeyTable.get(containerKeyPrefix);
return count == null ? Integer.valueOf(0) : count;
@@ -130,7 +193,7 @@ public class ContainerDBServiceProviderImpl
/**
* Get key prefixes for the given container ID.
*
- * @param containerId the given containerId.
+ * @param containerId the given containerID.
* @return Map of (Key-Prefix,Count of Keys).
*/
@Override
@@ -271,8 +334,56 @@ public class ContainerDBServiceProviderImpl
containerKeyTable.delete(containerKeyPrefix);
}
+ /**
+ * Get total count of containers.
+ *
+ * @return total count of containers.
+ */
+ @Override
+ public long getCountForContainers() {
+ GlobalStats containerCountRecord =
+ globalStatsDao.fetchOneByKey(CONTAINER_COUNT_KEY);
+
+ return (containerCountRecord == null) ? 0L :
+ containerCountRecord.getValue();
+ }
+
@Override
public TableIterator getContainerTableIterator() {
return containerKeyTable.iterator();
}
+
+ /**
+ * Store the total count of containers into the container DB store.
+ *
+ * @param count count of the containers present in the system.
+ */
+ @Override
+ public void storeContainerCount(Long count) {
+ // Get the current timestamp
+ Timestamp now =
+ using(sqlConfiguration).fetchValue(select(currentTimestamp()));
+ GlobalStats containerCountRecord =
+ globalStatsDao.fetchOneByKey(CONTAINER_COUNT_KEY);
+ GlobalStats globalStatsRecord =
+ new GlobalStats(CONTAINER_COUNT_KEY, count, now);
+
+ // Insert a new record for CONTAINER_COUNT_KEY if it does not exist
+ if (containerCountRecord == null) {
+ globalStatsDao.insert(globalStatsRecord);
+ } else {
+ globalStatsDao.update(globalStatsRecord);
+ }
+ }
+
+ /**
+ * Increment the total count for containers in the system by the given count.
+ *
+ * @param count no. of new containers to add to containers total count.
+ */
+ @Override
+ public void incrementContainerCountBy(long count) {
+ long containersCount = getCountForContainers();
+ storeContainerCount(containersCount + count);
+ }
}
\ No newline at end of file
diff --git
a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerDBProvider.java
b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerDBProvider.java
index fdb5913..de5c030 100644
---
a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerDBProvider.java
+++
b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerDBProvider.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.recon.spi.impl;
+import static
org.apache.hadoop.ozone.recon.ReconConstants.CONTAINER_KEY_COUNT_TABLE;
import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_CONTAINER_DB;
import static org.apache.hadoop.ozone.recon.ReconConstants.CONTAINER_KEY_TABLE;
import static
org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_DB_DIR;
@@ -30,6 +31,7 @@ import
org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix;
import org.apache.hadoop.utils.db.DBStore;
import org.apache.hadoop.utils.db.DBStoreBuilder;
import org.apache.hadoop.utils.db.IntegerCodec;
+import org.apache.hadoop.utils.db.LongCodec;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -69,7 +71,9 @@ public class ReconContainerDBProvider implements
Provider<DBStore> {
.setPath(metaDir)
.setName(dbName)
.addTable(CONTAINER_KEY_TABLE)
+ .addTable(CONTAINER_KEY_COUNT_TABLE)
.addCodec(ContainerKeyPrefix.class, new ContainerKeyPrefixCodec())
+ .addCodec(Long.class, new LongCodec())
.addCodec(Integer.class, new IntegerCodec())
.build();
} catch (Exception ex) {
diff --git
a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ContainerKeyMapperTask.java
b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ContainerKeyMapperTask.java
index 47dfff0..ec8695c 100644
---
a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ContainerKeyMapperTask.java
+++
b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ContainerKeyMapperTask.java
@@ -23,6 +23,7 @@ import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;
@@ -70,11 +71,14 @@ public class ContainerKeyMapperTask extends
ReconDBUpdateTask {
*/
@Override
public Pair<String, Boolean> reprocess(OMMetadataManager omMetadataManager) {
- int omKeyCount = 0;
+ long omKeyCount = 0;
try {
LOG.info("Starting a 'reprocess' run of ContainerKeyMapperTask.");
Instant start = Instant.now();
+ // initialize new container DB
+ containerDBServiceProvider.initNewContainerDB(new HashMap<>());
+
Table<String, OmKeyInfo> omKeyInfoTable =
omMetadataManager.getKeyTable();
try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
keyIter = omKeyInfoTable.iterator()) {
@@ -98,7 +102,6 @@ public class ContainerKeyMapperTask extends
ReconDBUpdateTask {
return new ImmutablePair<>(getTaskName(), true);
}
-
@Override
protected Collection<String> getTaskTables() {
return tables;
@@ -125,7 +128,8 @@ public class ContainerKeyMapperTask extends
ReconDBUpdateTask {
.getAction());
}
} catch (IOException e) {
- LOG.error("Unexpected exception while updating key data : {} ", e);
+ LOG.error("Unexpected exception while updating key data : {} ",
+ updatedKey, e);
return new ImmutablePair<>(getTaskName(), false);
}
}
@@ -133,7 +137,9 @@ public class ContainerKeyMapperTask extends
ReconDBUpdateTask {
}
/**
- * Delete an OM Key from Container DB.
+ * Delete an OM Key from Container DB and update containerID -> no. of keys
+ * count.
+ *
* @param key key String.
* @throws IOException If Unable to write to container DB.
*/
@@ -144,30 +150,42 @@ public class ContainerKeyMapperTask extends
ReconDBUpdateTask {
Table.KeyValue<ContainerKeyPrefix, Integer>> containerIterator =
containerDBServiceProvider.getContainerTableIterator();
- Set<ContainerKeyPrefix> keysToDeDeleted = new HashSet<>();
+ Set<ContainerKeyPrefix> keysToBeDeleted = new HashSet<>();
while (containerIterator.hasNext()) {
Table.KeyValue<ContainerKeyPrefix, Integer> keyValue =
containerIterator.next();
String keyPrefix = keyValue.getKey().getKeyPrefix();
if (keyPrefix.equals(key)) {
- keysToDeDeleted.add(keyValue.getKey());
+ keysToBeDeleted.add(keyValue.getKey());
}
}
- for (ContainerKeyPrefix containerKeyPrefix : keysToDeDeleted) {
+ for (ContainerKeyPrefix containerKeyPrefix : keysToBeDeleted) {
containerDBServiceProvider.deleteContainerMapping(containerKeyPrefix);
+
+ // decrement count and update containerKeyCount.
+ Long containerID = containerKeyPrefix.getContainerId();
+ long keyCount =
+ containerDBServiceProvider.getKeyCountForContainer(containerID);
+ if (keyCount > 0) {
+ containerDBServiceProvider.storeContainerKeyCount(containerID,
+ --keyCount);
+ }
}
}
/**
- * Write an OM key to container DB.
+ * Write an OM key to container DB and update containerID -> no. of keys
+ * count.
+ *
* @param key key String
* @param omKeyInfo omKeyInfo value
* @throws IOException if unable to write to recon DB.
*/
private void writeOMKeyToContainerDB(String key, OmKeyInfo omKeyInfo)
throws IOException {
+ long containerCountToIncrement = 0;
for (OmKeyLocationInfoGroup omKeyLocationInfoGroup : omKeyInfo
.getKeyLocationVersions()) {
long keyVersion = omKeyLocationInfoGroup.getVersion();
@@ -176,15 +194,36 @@ public class ContainerKeyMapperTask extends
ReconDBUpdateTask {
long containerId = omKeyLocationInfo.getContainerID();
ContainerKeyPrefix containerKeyPrefix = new ContainerKeyPrefix(
containerId, key, keyVersion);
- if (containerDBServiceProvider.getCountForForContainerKeyPrefix(
+ if (containerDBServiceProvider.getCountForContainerKeyPrefix(
containerKeyPrefix) == 0) {
// Save on writes. No need to save same container-key prefix
// mapping again.
containerDBServiceProvider.storeContainerKeyMapping(
containerKeyPrefix, 1);
+
+ // check if container already exists and
+ // increment the count of containers if it does not exist
+ if (!containerDBServiceProvider.doesContainerExists(containerId)) {
+ containerCountToIncrement++;
+ }
+
+ // update the count of keys for the given containerID
+ long keyCount =
+ containerDBServiceProvider.getKeyCountForContainer(containerId);
+
+ // increment the count and update containerKeyCount.
+ // keyCount will be 0 if containerID is not found. So, there is no
+ // need to initialize keyCount for the first time.
+ containerDBServiceProvider.storeContainerKeyCount(containerId,
+ ++keyCount);
}
}
}
+
+ if (containerCountToIncrement > 0) {
+ containerDBServiceProvider
+ .incrementContainerCountBy(containerCountToIncrement);
+ }
}
}
diff --git
a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java
b/hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/GuiceInjectorUtilsForTestsImpl.java
similarity index 58%
copy from
hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java
copy to
hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/GuiceInjectorUtilsForTestsImpl.java
index 6c768cf..6f16c1c 100644
---
a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java
+++
b/hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/GuiceInjectorUtilsForTestsImpl.java
@@ -18,28 +18,11 @@
package org.apache.hadoop.ozone.recon;
-import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DB_SUFFIX;
+import org.apache.hadoop.ozone.recon.types.GuiceInjectorUtilsForTests;
/**
- * Recon Server constants file.
+ * Implementation for GuiceInjectorUtilsForTests.
*/
-public final class ReconConstants {
-
- private ReconConstants() {
- // Never Constructed
- }
-
- public static final String RECON_CONTAINER_DB = "recon-" +
- CONTAINER_DB_SUFFIX;
-
- public static final String RECON_OM_SNAPSHOT_DB =
- "om.snapshot.db";
-
- public static final String CONTAINER_KEY_TABLE =
- "containerKeyTable";
-
- public static final String FETCH_ALL = "-1";
- public static final String RECON_QUERY_PREVKEY = "prev-key";
- public static final String RECON_QUERY_LIMIT = "limit";
-
+public class GuiceInjectorUtilsForTestsImpl implements
+ GuiceInjectorUtilsForTests {
}
diff --git
a/hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerKeyService.java
b/hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerKeyService.java
index 5146a6e..3ae39a6 100644
---
a/hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerKeyService.java
+++
b/hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerKeyService.java
@@ -18,14 +18,11 @@
package org.apache.hadoop.ozone.recon.api;
-import static
org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_DB_DIR;
-import static
org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_OM_SNAPSHOT_DB_DIR;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.FileInputStream;
-import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
@@ -34,6 +31,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import javax.sql.DataSource;
import javax.ws.rs.core.Response;
import org.apache.commons.lang3.StringUtils;
@@ -45,21 +43,23 @@ import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.recon.AbstractOMMetadataManagerTest;
+import org.apache.hadoop.ozone.recon.GuiceInjectorUtilsForTestsImpl;
import org.apache.hadoop.ozone.recon.ReconUtils;
import org.apache.hadoop.ozone.recon.api.types.ContainerMetadata;
+import org.apache.hadoop.ozone.recon.api.types.ContainersResponse;
import org.apache.hadoop.ozone.recon.api.types.KeyMetadata;
+import org.apache.hadoop.ozone.recon.api.types.KeysResponse;
import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider;
-import org.apache.hadoop.ozone.recon.spi.OzoneManagerServiceProvider;
-import org.apache.hadoop.ozone.recon.spi.impl.ContainerDBServiceProviderImpl;
import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl;
-import org.apache.hadoop.ozone.recon.spi.impl.ReconContainerDBProvider;
import org.apache.hadoop.ozone.recon.tasks.ContainerKeyMapperTask;
import org.apache.hadoop.utils.db.DBCheckpoint;
-import org.apache.hadoop.utils.db.DBStore;
import org.apache.http.impl.client.CloseableHttpClient;
-import org.junit.Assert;
+import org.hadoop.ozone.recon.schema.StatsSchemaDefinition;
+import org.jooq.impl.DSL;
+import org.jooq.impl.DefaultConfiguration;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito;
@@ -68,9 +68,9 @@ import
org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import com.google.inject.AbstractModule;
-import com.google.inject.Guice;
import com.google.inject.Injector;
-import com.google.inject.Singleton;
+
+import org.junit.rules.TemporaryFolder;
/**
* Test for container key service.
@@ -80,41 +80,57 @@ import com.google.inject.Singleton;
@PrepareForTest(ReconUtils.class)
public class TestContainerKeyService extends AbstractOMMetadataManagerTest {
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
private ContainerDBServiceProvider containerDbServiceProvider;
private OMMetadataManager omMetadataManager;
- private ReconOMMetadataManager reconOMMetadataManager;
private Injector injector;
private OzoneManagerServiceProviderImpl ozoneManagerServiceProvider;
private ContainerKeyService containerKeyService;
+ private GuiceInjectorUtilsForTestsImpl guiceInjectorTest =
+ new GuiceInjectorUtilsForTestsImpl();
+ private boolean isSetupDone = false;
- @Before
- public void setUp() throws Exception {
+ private void initializeInjector() throws Exception {
omMetadataManager = initializeNewOmMetadataManager();
- injector = Guice.createInjector(new AbstractModule() {
+ OzoneConfiguration configuration =
+ guiceInjectorTest.getTestOzoneConfiguration(temporaryFolder);
+
+ ozoneManagerServiceProvider = new OzoneManagerServiceProviderImpl(
+ configuration);
+ ReconOMMetadataManager reconOMMetadataManager =
+ getTestMetadataManager(omMetadataManager);
+
+ Injector parentInjector = guiceInjectorTest.getInjector(
+ ozoneManagerServiceProvider, reconOMMetadataManager, temporaryFolder);
+
+ injector = parentInjector.createChildInjector(new AbstractModule() {
@Override
protected void configure() {
- try {
- bind(OzoneConfiguration.class).toInstance(
- getTestOzoneConfiguration());
- reconOMMetadataManager = getTestMetadataManager(omMetadataManager);
-
bind(ReconOMMetadataManager.class).toInstance(reconOMMetadataManager);
- bind(DBStore.class).toProvider(ReconContainerDBProvider.class).
- in(Singleton.class);
- bind(ContainerDBServiceProvider.class).to(
- ContainerDBServiceProviderImpl.class).in(Singleton.class);
- ozoneManagerServiceProvider = new OzoneManagerServiceProviderImpl(
- getTestOzoneConfiguration());
- bind(OzoneManagerServiceProvider.class)
- .toInstance(ozoneManagerServiceProvider);
- containerKeyService = new ContainerKeyService();
- bind(ContainerKeyService.class).toInstance(containerKeyService);
- } catch (IOException e) {
- Assert.fail();
- }
+ containerKeyService = new ContainerKeyService();
+ bind(ContainerKeyService.class).toInstance(containerKeyService);
}
});
- containerDbServiceProvider = injector.getInstance(
- ContainerDBServiceProvider.class);
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ // The following setup runs only once
+ if (!isSetupDone) {
+ initializeInjector();
+
+ DSL.using(new DefaultConfiguration().set(
+ injector.getInstance(DataSource.class)));
+
+ containerDbServiceProvider = injector.getInstance(
+ ContainerDBServiceProvider.class);
+
+ StatsSchemaDefinition schemaDefinition = injector.getInstance(
+ StatsSchemaDefinition.class);
+ schemaDefinition.initializeSchema();
+
+ isSetupDone = true;
+ }
//Write Data to OM
Pipeline pipeline = getRandomPipeline();
@@ -204,8 +220,11 @@ public class TestContainerKeyService extends
AbstractOMMetadataManagerTest {
Response response = containerKeyService.getKeysForContainer(1L, -1, "");
- Collection<KeyMetadata> keyMetadataList =
- (Collection<KeyMetadata>) response.getEntity();
+ KeysResponse responseObject = (KeysResponse) response.getEntity();
+ KeysResponse.KeysResponseData data = responseObject.getKeysResponseData();
+ Collection<KeyMetadata> keyMetadataList = data.getKeys();
+
+ assertEquals(3, data.getTotalCount());
assertEquals(2, keyMetadataList.size());
Iterator<KeyMetadata> iterator = keyMetadataList.iterator();
@@ -229,13 +248,19 @@ public class TestContainerKeyService extends
AbstractOMMetadataManagerTest {
assertEquals(104, blockIds.get(1L).iterator().next().getLocalID());
response = containerKeyService.getKeysForContainer(3L, -1, "");
- keyMetadataList = (Collection<KeyMetadata>) response.getEntity();
+ responseObject = (KeysResponse) response.getEntity();
+ data = responseObject.getKeysResponseData();
+ keyMetadataList = data.getKeys();
assertTrue(keyMetadataList.isEmpty());
+ assertEquals(0, data.getTotalCount());
// test if limit works as expected
response = containerKeyService.getKeysForContainer(1L, 1, "");
- keyMetadataList = (Collection<KeyMetadata>) response.getEntity();
+ responseObject = (KeysResponse) response.getEntity();
+ data = responseObject.getKeysResponseData();
+ keyMetadataList = data.getKeys();
assertEquals(1, keyMetadataList.size());
+ assertEquals(3, data.getTotalCount());
}
@Test
@@ -244,8 +269,14 @@ public class TestContainerKeyService extends
AbstractOMMetadataManagerTest {
Response response = containerKeyService.getKeysForContainer(
1L, -1, "/sampleVol/bucketOne/key_one");
- Collection<KeyMetadata> keyMetadataList =
- (Collection<KeyMetadata>) response.getEntity();
+ KeysResponse responseObject =
+ (KeysResponse) response.getEntity();
+
+ KeysResponse.KeysResponseData data =
+ responseObject.getKeysResponseData();
+ assertEquals(3, data.getTotalCount());
+
+ Collection<KeyMetadata> keyMetadataList = data.getKeys();
assertEquals(1, keyMetadataList.size());
Iterator<KeyMetadata> iterator = keyMetadataList.iterator();
@@ -257,7 +288,11 @@ public class TestContainerKeyService extends
AbstractOMMetadataManagerTest {
response = containerKeyService.getKeysForContainer(
1L, -1, StringUtils.EMPTY);
- keyMetadataList = (Collection<KeyMetadata>) response.getEntity();
+ responseObject = (KeysResponse) response.getEntity();
+ data = responseObject.getKeysResponseData();
+ keyMetadataList = data.getKeys();
+
+ assertEquals(3, data.getTotalCount());
assertEquals(2, keyMetadataList.size());
iterator = keyMetadataList.iterator();
keyMetadata = iterator.next();
@@ -266,13 +301,19 @@ public class TestContainerKeyService extends
AbstractOMMetadataManagerTest {
// test for negative cases
response = containerKeyService.getKeysForContainer(
1L, -1, "/sampleVol/bucketOne/invalid_key");
- keyMetadataList = (Collection<KeyMetadata>) response.getEntity();
+ responseObject = (KeysResponse) response.getEntity();
+ data = responseObject.getKeysResponseData();
+ keyMetadataList = data.getKeys();
+ assertEquals(3, data.getTotalCount());
assertEquals(0, keyMetadataList.size());
response = containerKeyService.getKeysForContainer(
5L, -1, "");
- keyMetadataList = (Collection<KeyMetadata>) response.getEntity();
+ responseObject = (KeysResponse) response.getEntity();
+ data = responseObject.getKeysResponseData();
+ keyMetadataList = data.getKeys();
assertEquals(0, keyMetadataList.size());
+ assertEquals(0, data.getTotalCount());
}
@Test
@@ -280,8 +321,14 @@ public class TestContainerKeyService extends
AbstractOMMetadataManagerTest {
Response response = containerKeyService.getContainers(-1, 0L);
- List<ContainerMetadata> containers = new ArrayList<>(
- (Collection<ContainerMetadata>) response.getEntity());
+ ContainersResponse responseObject =
+ (ContainersResponse) response.getEntity();
+
+ ContainersResponse.ContainersResponseData data =
+ responseObject.getContainersResponseData();
+ assertEquals(2, data.getTotalCount());
+
+ List<ContainerMetadata> containers = new ArrayList<>(data.getContainers());
Iterator<ContainerMetadata> iterator = containers.iterator();
@@ -297,9 +344,11 @@ public class TestContainerKeyService extends
AbstractOMMetadataManagerTest {
// test if limit works as expected
response = containerKeyService.getContainers(1, 0L);
- containers = new ArrayList<>(
- (Collection<ContainerMetadata>) response.getEntity());
+ responseObject = (ContainersResponse) response.getEntity();
+ data = responseObject.getContainersResponseData();
+ containers = new ArrayList<>(data.getContainers());
assertEquals(1, containers.size());
+ assertEquals(2, data.getTotalCount());
}
@Test
@@ -307,8 +356,14 @@ public class TestContainerKeyService extends
AbstractOMMetadataManagerTest {
Response response = containerKeyService.getContainers(1, 1L);
- List<ContainerMetadata> containers = new ArrayList<>(
- (Collection<ContainerMetadata>) response.getEntity());
+ ContainersResponse responseObject =
+ (ContainersResponse) response.getEntity();
+
+ ContainersResponse.ContainersResponseData data =
+ responseObject.getContainersResponseData();
+ assertEquals(2, data.getTotalCount());
+
+ List<ContainerMetadata> containers = new ArrayList<>(data.getContainers());
Iterator<ContainerMetadata> iterator = containers.iterator();
@@ -318,37 +373,28 @@ public class TestContainerKeyService extends
AbstractOMMetadataManagerTest {
assertEquals(2L, containerMetadata.getContainerID());
response = containerKeyService.getContainers(-1, 0L);
- containers = new ArrayList<>(
- (Collection<ContainerMetadata>) response.getEntity());
+ responseObject = (ContainersResponse) response.getEntity();
+ data = responseObject.getContainersResponseData();
+ containers = new ArrayList<>(data.getContainers());
assertEquals(2, containers.size());
+ assertEquals(2, data.getTotalCount());
iterator = containers.iterator();
containerMetadata = iterator.next();
assertEquals(1L, containerMetadata.getContainerID());
// test for negative cases
response = containerKeyService.getContainers(-1, 5L);
- containers = new ArrayList<>(
- (Collection<ContainerMetadata>) response.getEntity());
+ responseObject = (ContainersResponse) response.getEntity();
+ data = responseObject.getContainersResponseData();
+ containers = new ArrayList<>(data.getContainers());
assertEquals(0, containers.size());
+ assertEquals(2, data.getTotalCount());
response = containerKeyService.getContainers(-1, -1L);
- containers = new ArrayList<>(
- (Collection<ContainerMetadata>) response.getEntity());
+ responseObject = (ContainersResponse) response.getEntity();
+ data = responseObject.getContainersResponseData();
+ containers = new ArrayList<>(data.getContainers());
assertEquals(2, containers.size());
- }
-
- /**
- * Get Test OzoneConfiguration instance.
- * @return OzoneConfiguration
- * @throws IOException ioEx.
- */
- private OzoneConfiguration getTestOzoneConfiguration()
- throws IOException {
- OzoneConfiguration configuration = new OzoneConfiguration();
- configuration.set(OZONE_RECON_OM_SNAPSHOT_DB_DIR,
- temporaryFolder.newFolder().getAbsolutePath());
- configuration.set(OZONE_RECON_DB_DIR, temporaryFolder.newFolder()
- .getAbsolutePath());
- return configuration;
+ assertEquals(2, data.getTotalCount());
}
}
\ No newline at end of file
diff --git
a/hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/AbstractSqlDatabaseTest.java
b/hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/AbstractSqlDatabaseTest.java
index c2fbce2..898dd19 100644
---
a/hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/AbstractSqlDatabaseTest.java
+++
b/hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/AbstractSqlDatabaseTest.java
@@ -43,17 +43,17 @@ public abstract class AbstractSqlDatabaseTest {
@ClassRule
public static TemporaryFolder temporaryFolder = new TemporaryFolder();
- private static File tempDir;
private static Injector injector;
private static DSLContext dslContext;
@BeforeClass
public static void setup() throws IOException {
- tempDir = temporaryFolder.newFolder();
+ File tempDir = temporaryFolder.newFolder();
DataSourceConfigurationProvider configurationProvider =
- new DataSourceConfigurationProvider();
+ new DataSourceConfigurationProvider(tempDir);
+
JooqPersistenceModule persistenceModule =
new JooqPersistenceModule(configurationProvider);
@@ -86,6 +86,12 @@ public abstract class AbstractSqlDatabaseTest {
public static class DataSourceConfigurationProvider implements
Provider<DataSourceConfiguration> {
+ private final File tempDir;
+
+ public DataSourceConfigurationProvider(File tempDir) {
+ this.tempDir = tempDir;
+ }
+
@Override
public DataSourceConfiguration get() {
return new DataSourceConfiguration() {
diff --git
a/hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestStatsSchemaDefinition.java
b/hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestStatsSchemaDefinition.java
new file mode 100644
index 0000000..864e59e
--- /dev/null
+++
b/hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestStatsSchemaDefinition.java
@@ -0,0 +1,147 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.recon.persistence;
+
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Pair;
+import org.hadoop.ozone.recon.schema.StatsSchemaDefinition;
+import org.hadoop.ozone.recon.schema.tables.daos.GlobalStatsDao;
+import org.hadoop.ozone.recon.schema.tables.pojos.GlobalStats;
+import org.jooq.Configuration;
+import org.junit.Assert;
+import org.junit.Test;
+
+import javax.sql.DataSource;
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.ResultSet;
+import java.sql.Timestamp;
+import java.sql.Types;
+import java.util.ArrayList;
+import java.util.List;
+
+import static
org.hadoop.ozone.recon.schema.StatsSchemaDefinition.GLOBAL_STATS_TABLE_NAME;
+
+/**
+ * Class used to test StatsSchemaDefinition.
+ */
+public class TestStatsSchemaDefinition extends AbstractSqlDatabaseTest {
+
+ @Test
+ public void testIfStatsSchemaCreated() throws Exception {
+ StatsSchemaDefinition schemaDefinition = getInjector().getInstance(
+ StatsSchemaDefinition.class);
+
+ schemaDefinition.initializeSchema();
+
+ Connection connection =
+ getInjector().getInstance(DataSource.class).getConnection();
+ // Verify table definition
+ DatabaseMetaData metaData = connection.getMetaData();
+ ResultSet resultSet = metaData.getColumns(null, null,
+ GLOBAL_STATS_TABLE_NAME, null);
+
+ List<Pair<String, Integer>> expectedPairs = new ArrayList<>();
+
+ expectedPairs.add(new ImmutablePair<>("key", Types.VARCHAR));
+ expectedPairs.add(new ImmutablePair<>("value", Types.INTEGER));
+ expectedPairs.add(new ImmutablePair<>("last_updated_timestamp",
+ Types.VARCHAR));
+
+ List<Pair<String, Integer>> actualPairs = new ArrayList<>();
+
+ while (resultSet.next()) {
+ actualPairs.add(new ImmutablePair<>(resultSet.getString("COLUMN_NAME"),
+ resultSet.getInt("DATA_TYPE")));
+ }
+
+ Assert.assertEquals(3, actualPairs.size());
+ Assert.assertEquals(expectedPairs, actualPairs);
+ }
+
+ @Test
+ public void testGlobalStatsCRUDOperations() throws Exception {
+ // Verify table exists
+ StatsSchemaDefinition schemaDefinition = getInjector().getInstance(
+ StatsSchemaDefinition.class);
+
+ schemaDefinition.initializeSchema();
+
+ DataSource ds = getInjector().getInstance(DataSource.class);
+ Connection connection = ds.getConnection();
+
+ DatabaseMetaData metaData = connection.getMetaData();
+ ResultSet resultSet = metaData.getTables(null, null,
+ GLOBAL_STATS_TABLE_NAME, null);
+
+ while (resultSet.next()) {
+ Assert.assertEquals(GLOBAL_STATS_TABLE_NAME,
+ resultSet.getString("TABLE_NAME"));
+ }
+
+ GlobalStatsDao dao = new GlobalStatsDao(
+ getInjector().getInstance(Configuration.class));
+
+ long now = System.currentTimeMillis();
+ GlobalStats newRecord = new GlobalStats();
+ newRecord.setLastUpdatedTimestamp(new Timestamp(now));
+ newRecord.setKey("key1");
+ newRecord.setValue(500L);
+
+ // Create
+ dao.insert(newRecord);
+ GlobalStats newRecord2 = new GlobalStats();
+ newRecord2.setLastUpdatedTimestamp(new Timestamp(now + 1000L));
+ newRecord2.setKey("key2");
+ newRecord2.setValue(10L);
+ dao.insert(newRecord2);
+
+ // Read
+ GlobalStats dbRecord = dao.findById("key1");
+
+ Assert.assertEquals("key1", dbRecord.getKey());
+ Assert.assertEquals(Long.valueOf(500), dbRecord.getValue());
+ Assert.assertEquals(new Timestamp(now),
dbRecord.getLastUpdatedTimestamp());
+
+ dbRecord = dao.findById("key2");
+ Assert.assertEquals("key2", dbRecord.getKey());
+ Assert.assertEquals(Long.valueOf(10), dbRecord.getValue());
+ Assert.assertEquals(new Timestamp(now + 1000L),
+ dbRecord.getLastUpdatedTimestamp());
+
+ // Update
+ dbRecord.setValue(100L);
+ dbRecord.setLastUpdatedTimestamp(new Timestamp(now + 2000L));
+ dao.update(dbRecord);
+
+ // Read updated
+ dbRecord = dao.findById("key2");
+
+ Assert.assertEquals(new Timestamp(now + 2000L),
+ dbRecord.getLastUpdatedTimestamp());
+ Assert.assertEquals(Long.valueOf(100L), dbRecord.getValue());
+
+ // Delete
+ dao.deleteById("key1");
+
+ // Verify
+ dbRecord = dao.findById("key1");
+
+ Assert.assertNull(dbRecord);
+ }
+}
diff --git
a/hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestContainerDBServiceProviderImpl.java
b/hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestContainerDBServiceProviderImpl.java
index eebcd5f..2392f8a 100644
---
a/hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestContainerDBServiceProviderImpl.java
+++
b/hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestContainerDBServiceProviderImpl.java
@@ -18,67 +18,93 @@
package org.apache.hadoop.ozone.recon.spi.impl;
-import static
org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_DB_DIR;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
-import java.io.File;
-import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.recon.GuiceInjectorUtilsForTestsImpl;
import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix;
import org.apache.hadoop.ozone.recon.api.types.ContainerMetadata;
import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider;
-import org.apache.hadoop.utils.db.DBStore;
-import org.junit.After;
+import org.hadoop.ozone.recon.schema.StatsSchemaDefinition;
+import org.jooq.impl.DSL;
+import org.jooq.impl.DefaultConfiguration;
import org.junit.Assert;
import org.junit.Before;
-import org.junit.Rule;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
-import com.google.inject.AbstractModule;
-import com.google.inject.Guice;
import com.google.inject.Injector;
-import com.google.inject.Singleton;
+
+import javax.sql.DataSource;
/**
* Unit Tests for ContainerDBServiceProviderImpl.
*/
public class TestContainerDBServiceProviderImpl {
- @Rule
- public TemporaryFolder tempFolder = new TemporaryFolder();
+ @ClassRule
+ public static TemporaryFolder tempFolder = new TemporaryFolder();
+ private static ContainerDBServiceProvider containerDbServiceProvider;
+ private static Injector injector;
+ private static GuiceInjectorUtilsForTestsImpl guiceInjectorTest =
+ new GuiceInjectorUtilsForTestsImpl();
- private ContainerDBServiceProvider containerDbServiceProvider;
- private Injector injector;
+ private String keyPrefix1 = "V3/B1/K1";
+ private String keyPrefix2 = "V3/B1/K2";
+ private String keyPrefix3 = "V3/B2/K1";
+
+ private void populateKeysInContainers(long containerId1, long containerId2)
+ throws Exception {
+
+ ContainerKeyPrefix containerKeyPrefix1 = new
+ ContainerKeyPrefix(containerId1, keyPrefix1, 0);
+ containerDbServiceProvider.storeContainerKeyMapping(containerKeyPrefix1,
+ 1);
+
+ ContainerKeyPrefix containerKeyPrefix2 = new ContainerKeyPrefix(
+ containerId1, keyPrefix2, 0);
+ containerDbServiceProvider.storeContainerKeyMapping(containerKeyPrefix2,
+ 2);
+
+ ContainerKeyPrefix containerKeyPrefix3 = new ContainerKeyPrefix(
+ containerId2, keyPrefix3, 0);
+
+ containerDbServiceProvider.storeContainerKeyMapping(containerKeyPrefix3,
+ 3);
+ }
+
+ private static void initializeInjector() throws Exception {
+ injector = guiceInjectorTest.getInjector(
+ null, null, tempFolder);
+ }
+
+ @BeforeClass
+ public static void setupOnce() throws Exception {
+
+ initializeInjector();
+
+ DSL.using(new DefaultConfiguration().set(
+ injector.getInstance(DataSource.class)));
- @Before
- public void setUp() throws IOException {
- tempFolder.create();
- injector = Guice.createInjector(new AbstractModule() {
- @Override
- protected void configure() {
- File dbDir = tempFolder.getRoot();
- OzoneConfiguration configuration = new OzoneConfiguration();
- configuration.set(OZONE_RECON_DB_DIR, dbDir.getAbsolutePath());
- bind(OzoneConfiguration.class).toInstance(configuration);
- bind(DBStore.class).toProvider(ReconContainerDBProvider.class).in(
- Singleton.class);
- bind(ContainerDBServiceProvider.class).to(
- ContainerDBServiceProviderImpl.class).in(Singleton.class);
- }
- });
containerDbServiceProvider = injector.getInstance(
ContainerDBServiceProvider.class);
+
+ StatsSchemaDefinition schemaDefinition = injector.getInstance(
+ StatsSchemaDefinition.class);
+ schemaDefinition.initializeSchema();
}
- @After
- public void tearDown() throws Exception {
- tempFolder.delete();
+ @Before
+ public void setUp() throws Exception {
+ // Reset containerDB before running each test
+ containerDbServiceProvider.initNewContainerDB(null);
}
@Test
@@ -104,7 +130,7 @@ public class TestContainerDBServiceProviderImpl {
}
assertEquals(1, containerDbServiceProvider
- .getCountForForContainerKeyPrefix(ckp1).intValue());
+ .getCountForContainerKeyPrefix(ckp1).intValue());
prefixCounts.clear();
prefixCounts.put(ckp2, 12);
@@ -127,7 +153,7 @@ public class TestContainerDBServiceProviderImpl {
assertEquals(15, keyPrefixesForContainer.get(ckp5).intValue());
assertEquals(0, containerDbServiceProvider
- .getCountForForContainerKeyPrefix(ckp1).intValue());
+ .getCountForContainerKeyPrefix(ckp1).intValue());
}
@Test
@@ -135,9 +161,9 @@ public class TestContainerDBServiceProviderImpl {
long containerId = System.currentTimeMillis();
Map<String, Integer> prefixCounts = new HashMap<>();
- prefixCounts.put("V1/B1/K1", 1);
- prefixCounts.put("V1/B1/K2", 2);
- prefixCounts.put("V1/B2/K3", 3);
+ prefixCounts.put(keyPrefix1, 1);
+ prefixCounts.put(keyPrefix2, 2);
+ prefixCounts.put(keyPrefix3, 3);
for (String prefix : prefixCounts.keySet()) {
ContainerKeyPrefix containerKeyPrefix = new ContainerKeyPrefix(
@@ -146,89 +172,119 @@ public class TestContainerDBServiceProviderImpl {
containerKeyPrefix, prefixCounts.get(prefix));
}
- Assert.assertTrue(
- containerDbServiceProvider.getCountForForContainerKeyPrefix(
- new ContainerKeyPrefix(containerId, "V1/B1/K1",
- 0)) == 1);
- Assert.assertTrue(
- containerDbServiceProvider.getCountForForContainerKeyPrefix(
- new ContainerKeyPrefix(containerId, "V1/B1/K2",
- 0)) == 2);
- Assert.assertTrue(
- containerDbServiceProvider.getCountForForContainerKeyPrefix(
- new ContainerKeyPrefix(containerId, "V1/B2/K3",
- 0)) == 3);
+ Assert.assertEquals(1,
+ containerDbServiceProvider.getCountForContainerKeyPrefix(
+ new ContainerKeyPrefix(containerId, keyPrefix1,
+ 0)).longValue());
+ Assert.assertEquals(2,
+ containerDbServiceProvider.getCountForContainerKeyPrefix(
+ new ContainerKeyPrefix(containerId, keyPrefix2,
+ 0)).longValue());
+ Assert.assertEquals(3,
+ containerDbServiceProvider.getCountForContainerKeyPrefix(
+ new ContainerKeyPrefix(containerId, keyPrefix3,
+ 0)).longValue());
+ }
+
+ @Test
+ public void testStoreContainerKeyCount() throws Exception {
+ long containerId = 1L;
+ long nextContainerId = 2L;
+ containerDbServiceProvider.storeContainerKeyCount(containerId, 2L);
+ containerDbServiceProvider.storeContainerKeyCount(nextContainerId, 3L);
+
+ assertEquals(2,
+ containerDbServiceProvider.getKeyCountForContainer(containerId));
+ assertEquals(3,
+ containerDbServiceProvider.getKeyCountForContainer(nextContainerId));
+
+ containerDbServiceProvider.storeContainerKeyCount(containerId, 20L);
+ assertEquals(20,
+ containerDbServiceProvider.getKeyCountForContainer(containerId));
}
@Test
- public void testGetCountForForContainerKeyPrefix() throws Exception {
+ public void testGetKeyCountForContainer() throws Exception {
+ long containerId = 1L;
+ long nextContainerId = 2L;
+ containerDbServiceProvider.storeContainerKeyCount(containerId, 2L);
+ containerDbServiceProvider.storeContainerKeyCount(nextContainerId, 3L);
+
+ assertEquals(2,
+ containerDbServiceProvider.getKeyCountForContainer(containerId));
+ assertEquals(3,
+ containerDbServiceProvider.getKeyCountForContainer(nextContainerId));
+
+ assertEquals(0,
+ containerDbServiceProvider.getKeyCountForContainer(5L));
+ }
+
+ @Test
+ public void testDoesContainerExists() throws Exception {
+ long containerId = 1L;
+ long nextContainerId = 2L;
+ containerDbServiceProvider.storeContainerKeyCount(containerId, 2L);
+ containerDbServiceProvider.storeContainerKeyCount(nextContainerId, 3L);
+
+ assertTrue(containerDbServiceProvider.doesContainerExists(containerId));
+
assertTrue(containerDbServiceProvider.doesContainerExists(nextContainerId));
+ assertFalse(containerDbServiceProvider.doesContainerExists(0L));
+ assertFalse(containerDbServiceProvider.doesContainerExists(3L));
+ }
+
+ @Test
+ public void testGetCountForContainerKeyPrefix() throws Exception {
long containerId = System.currentTimeMillis();
containerDbServiceProvider.storeContainerKeyMapping(new
- ContainerKeyPrefix(containerId, "V2/B1/K1"), 2);
+ ContainerKeyPrefix(containerId, keyPrefix1), 2);
Integer count = containerDbServiceProvider.
- getCountForForContainerKeyPrefix(new ContainerKeyPrefix(containerId,
- "V2/B1/K1"));
- assertTrue(count == 2);
+ getCountForContainerKeyPrefix(new ContainerKeyPrefix(containerId,
+ keyPrefix1));
+ assertEquals(2L, count.longValue());
+
+ count = containerDbServiceProvider.
+ getCountForContainerKeyPrefix(new ContainerKeyPrefix(containerId,
+ "invalid"));
+ assertEquals(0L, count.longValue());
}
@Test
public void testGetKeyPrefixesForContainer() throws Exception {
- long containerId = System.currentTimeMillis();
+ long containerId = 1L;
+ long nextContainerId = 2L;
+ populateKeysInContainers(containerId, nextContainerId);
ContainerKeyPrefix containerKeyPrefix1 = new
- ContainerKeyPrefix(containerId, "V3/B1/K1", 0);
- containerDbServiceProvider.storeContainerKeyMapping(containerKeyPrefix1,
- 1);
-
+ ContainerKeyPrefix(containerId, keyPrefix1, 0);
ContainerKeyPrefix containerKeyPrefix2 = new ContainerKeyPrefix(
- containerId, "V3/B1/K2", 0);
- containerDbServiceProvider.storeContainerKeyMapping(containerKeyPrefix2,
- 2);
-
- long nextContainerId = containerId + 1000L;
+ containerId, keyPrefix2, 0);
ContainerKeyPrefix containerKeyPrefix3 = new ContainerKeyPrefix(
- nextContainerId, "V3/B2/K1", 0);
- containerDbServiceProvider.storeContainerKeyMapping(containerKeyPrefix3,
- 3);
+ nextContainerId, keyPrefix3, 0);
+
Map<ContainerKeyPrefix, Integer> keyPrefixMap =
containerDbServiceProvider.getKeyPrefixesForContainer(containerId);
- assertTrue(keyPrefixMap.size() == 2);
+ assertEquals(2, keyPrefixMap.size());
- assertTrue(keyPrefixMap.get(containerKeyPrefix1) == 1);
- assertTrue(keyPrefixMap.get(containerKeyPrefix2) == 2);
+ assertEquals(1, keyPrefixMap.get(containerKeyPrefix1).longValue());
+ assertEquals(2, keyPrefixMap.get(containerKeyPrefix2).longValue());
keyPrefixMap = containerDbServiceProvider.getKeyPrefixesForContainer(
nextContainerId);
- assertTrue(keyPrefixMap.size() == 1);
- assertTrue(keyPrefixMap.get(containerKeyPrefix3) == 3);
+ assertEquals(1, keyPrefixMap.size());
+ assertEquals(3, keyPrefixMap.get(containerKeyPrefix3).longValue());
}
@Test
public void testGetKeyPrefixesForContainerWithKeyPrefix() throws Exception {
- long containerId = System.currentTimeMillis();
-
- String keyPrefix1 = "V3/B1/K1";
- String keyPrefix2 = "V3/B1/K2";
- String keyPrefix3 = "V3/B2/K1";
-
- ContainerKeyPrefix containerKeyPrefix1 = new
- ContainerKeyPrefix(containerId, keyPrefix1, 0);
- containerDbServiceProvider.storeContainerKeyMapping(containerKeyPrefix1,
- 1);
+ long containerId = 1L;
+ long nextContainerId = 2L;
+ populateKeysInContainers(containerId, nextContainerId);
ContainerKeyPrefix containerKeyPrefix2 = new ContainerKeyPrefix(
containerId, keyPrefix2, 0);
- containerDbServiceProvider.storeContainerKeyMapping(containerKeyPrefix2,
- 2);
-
- long nextContainerId = containerId + 1000L;
- ContainerKeyPrefix containerKeyPrefix3 = new ContainerKeyPrefix(
- nextContainerId, keyPrefix3, 0);
- containerDbServiceProvider.storeContainerKeyMapping(containerKeyPrefix3,
- 3);
Map<ContainerKeyPrefix, Integer> keyPrefixMap =
containerDbServiceProvider.getKeyPrefixesForContainer(containerId,
@@ -250,33 +306,15 @@ public class TestContainerDBServiceProviderImpl {
assertEquals(0, keyPrefixMap.size());
keyPrefixMap = containerDbServiceProvider.getKeyPrefixesForContainer(
- 1L, "");
+ 10L, "");
assertEquals(0, keyPrefixMap.size());
}
@Test
- public void testGetContainersWithPrevKey() throws Exception {
- long containerId = System.currentTimeMillis();
-
- String keyPrefix1 = "V3/B1/K1";
- String keyPrefix2 = "V3/B1/K2";
- String keyPrefix3 = "V3/B2/K1";
-
- ContainerKeyPrefix containerKeyPrefix1 = new
- ContainerKeyPrefix(containerId, keyPrefix1, 0);
- containerDbServiceProvider.storeContainerKeyMapping(containerKeyPrefix1,
- 1);
-
- ContainerKeyPrefix containerKeyPrefix2 = new ContainerKeyPrefix(
- containerId, keyPrefix2, 0);
- containerDbServiceProvider.storeContainerKeyMapping(containerKeyPrefix2,
- 2);
-
- long nextContainerId = containerId + 1000L;
- ContainerKeyPrefix containerKeyPrefix3 = new ContainerKeyPrefix(
- nextContainerId, keyPrefix3, 0);
- containerDbServiceProvider.storeContainerKeyMapping(containerKeyPrefix3,
- 3);
+ public void testGetContainersWithPrevContainer() throws Exception {
+ long containerId = 1L;
+ long nextContainerId = 2L;
+ populateKeysInContainers(containerId, nextContainerId);
Map<Long, ContainerMetadata> containerMap =
containerDbServiceProvider.getContainers(-1, 0L);
@@ -304,7 +342,7 @@ public class TestContainerDBServiceProviderImpl {
// test for negative cases
containerMap = containerDbServiceProvider.getContainers(
- -1, 1L);
+ -1, 10L);
assertEquals(0, containerMap.size());
containerMap = containerDbServiceProvider.getContainers(
@@ -313,27 +351,55 @@ public class TestContainerDBServiceProviderImpl {
}
@Test
- public void testDeleteContainerMapping() throws IOException {
- long containerId = System.currentTimeMillis();
-
- ContainerKeyPrefix containerKeyPrefix1 = new
- ContainerKeyPrefix(containerId, "V3/B1/K1", 0);
- containerDbServiceProvider.storeContainerKeyMapping(containerKeyPrefix1,
- 1);
-
- ContainerKeyPrefix containerKeyPrefix2 = new ContainerKeyPrefix(
- containerId, "V3/B1/K2", 0);
- containerDbServiceProvider.storeContainerKeyMapping(containerKeyPrefix2,
- 2);
+ public void testDeleteContainerMapping() throws Exception {
+ long containerId = 1L;
+ long nextContainerId = 2L;
+ populateKeysInContainers(containerId, nextContainerId);
Map<ContainerKeyPrefix, Integer> keyPrefixMap =
containerDbServiceProvider.getKeyPrefixesForContainer(containerId);
- assertTrue(keyPrefixMap.size() == 2);
+ assertEquals(2, keyPrefixMap.size());
containerDbServiceProvider.deleteContainerMapping(new ContainerKeyPrefix(
- containerId, "V3/B1/K2", 0));
+ containerId, keyPrefix2, 0));
keyPrefixMap =
containerDbServiceProvider.getKeyPrefixesForContainer(containerId);
- assertTrue(keyPrefixMap.size() == 1);
+ assertEquals(1, keyPrefixMap.size());
+ }
+
+ @Test
+ public void testGetCountForContainers() throws Exception {
+
+ assertEquals(0, containerDbServiceProvider.getCountForContainers());
+
+ containerDbServiceProvider.storeContainerCount(5L);
+
+ assertEquals(5L, containerDbServiceProvider.getCountForContainers());
+ containerDbServiceProvider.incrementContainerCountBy(1L);
+
+ assertEquals(6L, containerDbServiceProvider.getCountForContainers());
+
+ containerDbServiceProvider.storeContainerCount(10L);
+ assertEquals(10L, containerDbServiceProvider.getCountForContainers());
+ }
+
+ @Test
+ public void testStoreContainerCount() throws Exception {
+ containerDbServiceProvider.storeContainerCount(3L);
+ assertEquals(3L, containerDbServiceProvider.getCountForContainers());
+
+ containerDbServiceProvider.storeContainerCount(5L);
+ assertEquals(5L, containerDbServiceProvider.getCountForContainers());
+ }
+
+ @Test
+ public void testIncrementContainerCountBy() throws Exception {
+ assertEquals(0, containerDbServiceProvider.getCountForContainers());
+
+ containerDbServiceProvider.incrementContainerCountBy(1L);
+ assertEquals(1L, containerDbServiceProvider.getCountForContainers());
+
+ containerDbServiceProvider.incrementContainerCountBy(3L);
+ assertEquals(4L, containerDbServiceProvider.getCountForContainers());
}
}
diff --git
a/hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestOzoneManagerServiceProviderImpl.java
b/hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestOzoneManagerServiceProviderImpl.java
index e6a2405..fde8142 100644
---
a/hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestOzoneManagerServiceProviderImpl.java
+++
b/hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestOzoneManagerServiceProviderImpl.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.ozone.recon.spi.impl;
-import static
org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_OM_SNAPSHOT_DB_DIR;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
@@ -26,17 +25,15 @@ import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
-import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Paths;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.recon.AbstractOMMetadataManagerTest;
+import org.apache.hadoop.ozone.recon.GuiceInjectorUtilsForTestsImpl;
import org.apache.hadoop.ozone.recon.ReconUtils;
import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
-import org.apache.hadoop.ozone.recon.spi.OzoneManagerServiceProvider;
import org.apache.hadoop.utils.db.DBCheckpoint;
import org.apache.http.impl.client.CloseableHttpClient;
import org.junit.Assert;
@@ -50,8 +47,6 @@ import
org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
-import com.google.inject.AbstractModule;
-import com.google.inject.Guice;
import com.google.inject.Injector;
/**
@@ -66,7 +61,10 @@ public class TestOzoneManagerServiceProviderImpl extends
private OMMetadataManager omMetadataManager;
private ReconOMMetadataManager reconOMMetadataManager;
private Injector injector;
+ private GuiceInjectorUtilsForTestsImpl guiceInjectorTest =
+ new GuiceInjectorUtilsForTestsImpl();
private OzoneManagerServiceProviderImpl ozoneManagerServiceProvider;
+ private boolean isSetupDone = false;
@Rule
public TemporaryFolder temporaryFolder = new TemporaryFolder();
@@ -74,25 +72,17 @@ public class TestOzoneManagerServiceProviderImpl extends
@Before
public void setUp() throws Exception {
omMetadataManager = initializeNewOmMetadataManager();
- injector = Guice.createInjector(new AbstractModule() {
- @Override
- protected void configure() {
- try {
- initializeNewOmMetadataManager();
- writeDataToOm(omMetadataManager, "key_one");
- bind(OzoneConfiguration.class).toInstance(
- getTestOzoneConfiguration());
- reconOMMetadataManager = getTestMetadataManager(omMetadataManager);
-
bind(ReconOMMetadataManager.class).toInstance(reconOMMetadataManager);
- ozoneManagerServiceProvider = new OzoneManagerServiceProviderImpl(
- getTestOzoneConfiguration());
- bind(OzoneManagerServiceProvider.class)
- .toInstance(ozoneManagerServiceProvider);
- } catch (IOException e) {
- Assert.fail();
- }
- }
- });
+ writeDataToOm(omMetadataManager, "key_one");
+ reconOMMetadataManager = getTestMetadataManager(omMetadataManager);
+ ozoneManagerServiceProvider =
+ new OzoneManagerServiceProviderImpl(
+ guiceInjectorTest.getTestOzoneConfiguration(temporaryFolder));
+ if (!isSetupDone) {
+ injector = guiceInjectorTest.getInjector(ozoneManagerServiceProvider,
+ reconOMMetadataManager, temporaryFolder);
+
+ isSetupDone = true;
+ }
}
@Test
@@ -168,17 +158,4 @@ public class TestOzoneManagerServiceProviderImpl extends
assertTrue(checkpoint.getCheckpointLocation().toFile()
.listFiles().length == 2);
}
-
- /**
- * Get Test OzoneConfiguration instance.
- * @return OzoneConfiguration
- * @throws IOException ioEx.
- */
- private OzoneConfiguration getTestOzoneConfiguration() throws IOException {
- OzoneConfiguration configuration = new OzoneConfiguration();
- configuration.set(OZONE_RECON_OM_SNAPSHOT_DB_DIR,
- temporaryFolder.newFolder().getAbsolutePath());
- return configuration;
- }
-
}
\ No newline at end of file
diff --git
a/hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestContainerKeyMapperTask.java
b/hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestContainerKeyMapperTask.java
index c67d7fd..5cc9a48 100644
---
a/hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestContainerKeyMapperTask.java
+++
b/hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestContainerKeyMapperTask.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.ozone.recon.tasks;
-import static
org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_DB_DIR;
-import static
org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_OM_SNAPSHOT_DB_DIR;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -38,27 +36,25 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.recon.AbstractOMMetadataManagerTest;
+import org.apache.hadoop.ozone.recon.GuiceInjectorUtilsForTestsImpl;
import org.apache.hadoop.ozone.recon.ReconUtils;
import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix;
import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider;
-import org.apache.hadoop.ozone.recon.spi.OzoneManagerServiceProvider;
-import org.apache.hadoop.ozone.recon.spi.impl.ContainerDBServiceProviderImpl;
import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl;
-import org.apache.hadoop.ozone.recon.spi.impl.ReconContainerDBProvider;
-import org.apache.hadoop.utils.db.DBStore;
-import org.junit.Assert;
+import org.hadoop.ozone.recon.schema.StatsSchemaDefinition;
+import org.jooq.impl.DSL;
+import org.jooq.impl.DefaultConfiguration;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
-
-import com.google.inject.AbstractModule;
-import com.google.inject.Guice;
import com.google.inject.Injector;
-import com.google.inject.Singleton;
+import javax.sql.DataSource;
/**
* Unit test for Container Key mapper task.
@@ -73,33 +69,49 @@ public class TestContainerKeyMapperTask extends
AbstractOMMetadataManagerTest {
private ReconOMMetadataManager reconOMMetadataManager;
private Injector injector;
private OzoneManagerServiceProviderImpl ozoneManagerServiceProvider;
+ private boolean setUpIsDone = false;
+ private GuiceInjectorUtilsForTestsImpl guiceInjectorTest =
+ new GuiceInjectorUtilsForTestsImpl();
+
+ private Injector getInjector() {
+ return injector;
+ }
+
+ @Rule
+ TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ private void initializeInjector() throws Exception {
+ omMetadataManager = initializeNewOmMetadataManager();
+ OzoneConfiguration configuration =
+ guiceInjectorTest.getTestOzoneConfiguration(temporaryFolder);
+
+ ozoneManagerServiceProvider = new OzoneManagerServiceProviderImpl(
+ configuration);
+ reconOMMetadataManager = getTestMetadataManager(omMetadataManager);
+
+ injector = guiceInjectorTest.getInjector(
+ ozoneManagerServiceProvider, reconOMMetadataManager, temporaryFolder);
+ }
@Before
public void setUp() throws Exception {
- omMetadataManager = initializeNewOmMetadataManager();
- injector = Guice.createInjector(new AbstractModule() {
- @Override
- protected void configure() {
- try {
- bind(OzoneConfiguration.class).toInstance(
- getTestOzoneConfiguration());
-
- reconOMMetadataManager = getTestMetadataManager(omMetadataManager);
-
bind(ReconOMMetadataManager.class).toInstance(reconOMMetadataManager);
- ozoneManagerServiceProvider = new OzoneManagerServiceProviderImpl(
- getTestOzoneConfiguration());
- bind(OzoneManagerServiceProvider.class)
- .toInstance(ozoneManagerServiceProvider);
-
- bind(DBStore.class).toProvider(ReconContainerDBProvider.class).
- in(Singleton.class);
- bind(ContainerDBServiceProvider.class).to(
- ContainerDBServiceProviderImpl.class).in(Singleton.class);
- } catch (IOException e) {
- Assert.fail();
- }
- }
- });
+ // The following setup is run only once
+ if (!setUpIsDone) {
+ initializeInjector();
+
+ DSL.using(new DefaultConfiguration().set(
+ injector.getInstance(DataSource.class)));
+
+ containerDbServiceProvider = injector.getInstance(
+ ContainerDBServiceProvider.class);
+
+ StatsSchemaDefinition schemaDefinition = getInjector().getInstance(
+ StatsSchemaDefinition.class);
+ schemaDefinition.initializeSchema();
+
+ setUpIsDone = true;
+ }
+
containerDbServiceProvider = injector.getInstance(
ContainerDBServiceProvider.class);
}
@@ -146,7 +158,7 @@ public class TestContainerKeyMapperTask extends
AbstractOMMetadataManagerTest {
keyPrefixesForContainer =
containerDbServiceProvider.getKeyPrefixesForContainer(1);
- assertTrue(keyPrefixesForContainer.size() == 1);
+ assertEquals(1, keyPrefixesForContainer.size());
String omKey = omMetadataManager.getOzoneKey("sampleVol",
"bucketOne", "key_one");
ContainerKeyPrefix containerKeyPrefix = new ContainerKeyPrefix(1,
@@ -156,11 +168,19 @@ public class TestContainerKeyMapperTask extends
AbstractOMMetadataManagerTest {
keyPrefixesForContainer =
containerDbServiceProvider.getKeyPrefixesForContainer(2);
- assertTrue(keyPrefixesForContainer.size() == 1);
+ assertEquals(1, keyPrefixesForContainer.size());
containerKeyPrefix = new ContainerKeyPrefix(2, omKey,
0);
assertEquals(1,
keyPrefixesForContainer.get(containerKeyPrefix).intValue());
+
+ // Test if container key counts are updated
+ assertEquals(1, containerDbServiceProvider.getKeyCountForContainer(1L));
+ assertEquals(1, containerDbServiceProvider.getKeyCountForContainer(2L));
+ assertEquals(0, containerDbServiceProvider.getKeyCountForContainer(3L));
+
+ // Test if container count is updated
+ assertEquals(2, containerDbServiceProvider.getCountForContainers());
}
@Test
@@ -245,7 +265,7 @@ public class TestContainerKeyMapperTask extends
AbstractOMMetadataManagerTest {
keyPrefixesForContainer = containerDbServiceProvider
.getKeyPrefixesForContainer(1);
- assertTrue(keyPrefixesForContainer.size() == 1);
+ assertEquals(1, keyPrefixesForContainer.size());
keyPrefixesForContainer = containerDbServiceProvider
.getKeyPrefixesForContainer(2);
@@ -253,23 +273,33 @@ public class TestContainerKeyMapperTask extends
AbstractOMMetadataManagerTest {
keyPrefixesForContainer = containerDbServiceProvider
.getKeyPrefixesForContainer(3);
- assertTrue(keyPrefixesForContainer.size() == 1);
+ assertEquals(1, keyPrefixesForContainer.size());
+
+ assertEquals(1, containerDbServiceProvider.getKeyCountForContainer(1L));
+ assertEquals(0, containerDbServiceProvider.getKeyCountForContainer(2L));
+ assertEquals(1, containerDbServiceProvider.getKeyCountForContainer(3L));
// Process PUT & DELETE event.
containerKeyMapperTask.process(omUpdateEventBatch);
keyPrefixesForContainer = containerDbServiceProvider
.getKeyPrefixesForContainer(1);
- assertTrue(keyPrefixesForContainer.size() == 1);
+ assertEquals(1, keyPrefixesForContainer.size());
keyPrefixesForContainer = containerDbServiceProvider
.getKeyPrefixesForContainer(2);
- assertTrue(keyPrefixesForContainer.size() == 1);
+ assertEquals(1, keyPrefixesForContainer.size());
keyPrefixesForContainer = containerDbServiceProvider
.getKeyPrefixesForContainer(3);
assertTrue(keyPrefixesForContainer.isEmpty());
+ assertEquals(1, containerDbServiceProvider.getKeyCountForContainer(1L));
+ assertEquals(1, containerDbServiceProvider.getKeyCountForContainer(2L));
+ assertEquals(0, containerDbServiceProvider.getKeyCountForContainer(3L));
+
+ // Test if container count is updated
+ assertEquals(3, containerDbServiceProvider.getCountForContainers());
}
private OmKeyInfo buildOmKeyInfo(String volume,
@@ -287,19 +317,4 @@ public class TestContainerKeyMapperTask extends
AbstractOMMetadataManagerTest {
omKeyLocationInfoGroup))
.build();
}
- /**
- * Get Test OzoneConfiguration instance.
- * @return OzoneConfiguration
- * @throws IOException ioEx.
- */
- private OzoneConfiguration getTestOzoneConfiguration()
- throws IOException {
- OzoneConfiguration configuration = new OzoneConfiguration();
- configuration.set(OZONE_RECON_OM_SNAPSHOT_DB_DIR,
- temporaryFolder.newFolder().getAbsolutePath());
- configuration.set(OZONE_RECON_DB_DIR, temporaryFolder.newFolder()
- .getAbsolutePath());
- return configuration;
- }
-
}
\ No newline at end of file
diff --git
a/hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/types/GuiceInjectorUtilsForTests.java
b/hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/types/GuiceInjectorUtilsForTests.java
new file mode 100644
index 0000000..f6253bc
--- /dev/null
+++
b/hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/types/GuiceInjectorUtilsForTests.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon.types;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.Singleton;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.recon.persistence.AbstractSqlDatabaseTest;
+import org.apache.hadoop.ozone.recon.persistence.DataSourceConfiguration;
+import org.apache.hadoop.ozone.recon.persistence.JooqPersistenceModule;
+import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
+import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider;
+import org.apache.hadoop.ozone.recon.spi.OzoneManagerServiceProvider;
+import org.apache.hadoop.ozone.recon.spi.impl.ContainerDBServiceProviderImpl;
+import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl;
+import org.apache.hadoop.ozone.recon.spi.impl.ReconContainerDBProvider;
+import org.apache.hadoop.utils.db.DBStore;
+import org.junit.Assert;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.IOException;
+
+import static
org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_DB_DIR;
+import static
org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_OM_SNAPSHOT_DB_DIR;
+
+/**
+ * Utility methods to get guice injector and ozone configuration.
+ */
+public interface GuiceInjectorUtilsForTests {
+
+ /**
+ * Get Guice Injector with bindings.
+ * @param ozoneManagerServiceProvider
+ * @param reconOMMetadataManager
+ * @param temporaryFolder
+ * @return Injector
+ * @throws IOException ioEx.
+ */
+ default Injector getInjector(
+ OzoneManagerServiceProviderImpl ozoneManagerServiceProvider,
+ ReconOMMetadataManager reconOMMetadataManager,
+ TemporaryFolder temporaryFolder
+ ) throws IOException {
+
+ File tempDir = temporaryFolder.newFolder();
+ AbstractSqlDatabaseTest.DataSourceConfigurationProvider
+ configurationProvider =
+ new AbstractSqlDatabaseTest.DataSourceConfigurationProvider(tempDir);
+
+ JooqPersistenceModule jooqPersistenceModule =
+ new JooqPersistenceModule(configurationProvider);
+
+ return Guice.createInjector(jooqPersistenceModule,
+ new AbstractModule() {
+ @Override
+ protected void configure() {
+ try {
+ bind(DataSourceConfiguration.class)
+ .toProvider(configurationProvider);
+ bind(OzoneConfiguration.class).toInstance(
+ getTestOzoneConfiguration(temporaryFolder));
+
+ if (reconOMMetadataManager != null) {
+ bind(ReconOMMetadataManager.class)
+ .toInstance(reconOMMetadataManager);
+ }
+
+ if (ozoneManagerServiceProvider != null) {
+ bind(OzoneManagerServiceProvider.class)
+ .toInstance(ozoneManagerServiceProvider);
+ }
+
+ bind(DBStore.class).toProvider(ReconContainerDBProvider.class).
+ in(Singleton.class);
+ bind(ContainerDBServiceProvider.class).to(
+ ContainerDBServiceProviderImpl.class).in(Singleton.class);
+ } catch (IOException e) {
+ Assert.fail();
+ }
+ }
+ });
+ }
+
+ /**
+ * Get Test OzoneConfiguration instance.
+ * @return OzoneConfiguration
+ * @throws IOException ioEx.
+ */
+ default OzoneConfiguration getTestOzoneConfiguration(
+ TemporaryFolder temporaryFolder) throws IOException {
+ OzoneConfiguration configuration = new OzoneConfiguration();
+ configuration.set(OZONE_RECON_OM_SNAPSHOT_DB_DIR,
+ temporaryFolder.newFolder().getAbsolutePath());
+ configuration.set(OZONE_RECON_DB_DIR, temporaryFolder.newFolder()
+ .getAbsolutePath());
+ return configuration;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
