Repository: falcon Updated Branches: refs/heads/master 16d2b39b2 -> e84fca7ff
FALCON-1984 Provide proper hint and documentation if required titan storage backend is not configured in startup.properties Added validation of required configuration properties for graph db storage backend. Added error hints on the missing property in Falcon server log. Reorganized the order of the configuration properties in startup.properties so relevant properties for each backend are grouped in consecutive lines. Author: yzheng-hortonworks <[email protected]> Reviewers: "Balu Vellanki <[email protected]>, Venkat Ranganathan <[email protected]>" Closes #166 from yzheng-hortonworks/FALCON-1984 Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/e84fca7f Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/e84fca7f Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/e84fca7f Branch: refs/heads/master Commit: e84fca7ff2ce235a7d804b3c4806091a2a2c3037 Parents: 16d2b39 Author: yzheng-hortonworks <[email protected]> Authored: Fri Jun 3 07:53:16 2016 -0700 Committer: bvellanki <[email protected]> Committed: Fri Jun 3 07:53:16 2016 -0700 ---------------------------------------------------------------------- .../falcon/metadata/MetadataMappingService.java | 72 +++++++++++++++++++- common/src/main/resources/startup.properties | 17 ++++- .../metadata/MetadataMappingServiceTest.java | 6 +- docs/src/site/twiki/Configuration.twiki | 10 ++- .../resource/metadata/MetadataTestContext.java | 4 ++ src/conf/startup.properties | 18 ++--- 6 files changed, 106 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/e84fca7f/common/src/main/java/org/apache/falcon/metadata/MetadataMappingService.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/metadata/MetadataMappingService.java b/common/src/main/java/org/apache/falcon/metadata/MetadataMappingService.java index 66a3a58..727be56 100644 --- a/common/src/main/java/org/apache/falcon/metadata/MetadataMappingService.java +++ b/common/src/main/java/org/apache/falcon/metadata/MetadataMappingService.java @@ -35,6 +35,7 @@ import com.tinkerpop.blueprints.util.TransactionWork; import org.apache.commons.configuration.BaseConfiguration; import org.apache.commons.configuration.Configuration; import org.apache.falcon.FalconException; +import org.apache.falcon.FalconRuntimException; import org.apache.falcon.entity.store.ConfigurationStore; import org.apache.falcon.entity.v0.Entity; import org.apache.falcon.entity.v0.EntityType; @@ -48,6 +49,9 @@ import org.slf4j.LoggerFactory; import org.apache.falcon.workflow.WorkflowExecutionContext; import org.apache.falcon.workflow.WorkflowExecutionListener; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; import java.util.Map; import java.util.Properties; import java.util.Set; @@ -69,7 +73,26 @@ public class MetadataMappingService * Constant for the configuration property that indicates the prefix. */ private static final String FALCON_PREFIX = "falcon.graph."; - + /** + * Constant for the configuration property that indicates the storage backend. + */ + public static final String PROPERTY_KEY_STORAGE_BACKEND = "storage.backend"; + public static final String STORAGE_BACKEND_HBASE = "hbase"; + public static final String STORAGE_BACKEND_BDB = "berkeleyje"; + /** + * HBase configuration properties. + */ + public static final String PROPERTY_KEY_STORAGE_HOSTNAME = "storage.hostname"; + public static final String PROPERTY_KEY_STORAGE_TABLE = "storage.hbase.table"; + public static final Set<String> PROPERTY_KEYS_HBASE = Collections.unmodifiableSet(new HashSet<>(Arrays.asList( + PROPERTY_KEY_STORAGE_HOSTNAME, PROPERTY_KEY_STORAGE_TABLE))); + /** + * Berkeley DB configuration properties. + */ + public static final String PROPERTY_KEY_STORAGE_DIRECTORY = "storage.directory"; + public static final String PROPERTY_KEY_SERIALIZE_PATH = "serialize.path"; + public static final Set<String> PROPERTY_KEYS_BDB = Collections.unmodifiableSet(new HashSet<>(Arrays.asList( + PROPERTY_KEY_STORAGE_DIRECTORY, PROPERTY_KEY_SERIALIZE_PATH))); private Graph graph; private Set<String> vertexIndexedKeys; @@ -118,11 +141,56 @@ public class MetadataMappingService protected Graph initializeGraphDB() { LOG.info("Initializing graph db"); - Configuration graphConfig = getConfiguration(); + validateConfiguration(graphConfig); return GraphFactory.open(graphConfig); } + private void validateConfiguration(Configuration graphConfig) { + // check if storage backend if configured + if (!graphConfig.containsKey(PROPERTY_KEY_STORAGE_BACKEND)) { + throw new FalconRuntimException("Titan GraphDB storage backend is not configured. " + + "You need to choose either hbase or berkeleydb." + + "Please check Configuration twiki or " + + "the section Graph Database Properties in startup.properties " + + "on how to configure Titan GraphDB backend."); + } + + String backend = graphConfig.getString(PROPERTY_KEY_STORAGE_BACKEND); + switch (backend) { + case STORAGE_BACKEND_BDB: + // check required parameter for Berkeley DB backend + for (String key : PROPERTY_KEYS_BDB) { + if (!graphConfig.containsKey(key)) { + throw new FalconRuntimException("Required parameter " + FALCON_PREFIX + key + + " not found in startup.properties." + + "Please check Configuration twiki or " + + "the section Graph Database Properties in startup.properties " + + "on how to configure Berkeley DB storage backend."); + } + } + break; + case STORAGE_BACKEND_HBASE: + // check required parameter for HBase backend + for (String key : PROPERTY_KEYS_HBASE) { + if (!graphConfig.containsKey(key)) { + throw new FalconRuntimException("Required parameter " + FALCON_PREFIX + key + + " not found in startup.properties." + + "Please check Configuration twiki or " + + "the section Graph Database Properties in startup.properties " + + "on how to configure HBase storage backend."); + } + } + break; + default: + throw new FalconRuntimException("Invalid graph storage backend: " + backend + ". " + + "You need to choose either hbase or berkeleydb." + + "Please check Configuration twiki or " + + "the section Graph Database Properties in startup.properties " + + "on how to configure Titan GraphDB backend."); + } + } + public static Configuration getConfiguration() { Configuration graphConfig = new BaseConfiguration(); http://git-wip-us.apache.org/repos/asf/falcon/blob/e84fca7f/common/src/main/resources/startup.properties ---------------------------------------------------------------------- diff --git a/common/src/main/resources/startup.properties b/common/src/main/resources/startup.properties index 2229edf..8ce1174 100644 --- a/common/src/main/resources/startup.properties +++ b/common/src/main/resources/startup.properties @@ -156,9 +156,20 @@ it.workflow.execution.listeners=org.apache.falcon.catalog.CatalogPartitionHandle *.falcon.graph.blueprints.graph=com.thinkaurelius.titan.core.TitanFactory # Graph Storage -*.falcon.graph.storage.directory=${user.dir}/target/graphdb -*.falcon.graph.storage.backend=berkeleyje -*.falcon.graph.serialize.path=${user.dir}/target/graphdb +# IMPORTANT: Please enable one of the graph db backend: hbase or berkeleydb, per instructions below. + +# Enable the following for Berkeley DB. Make sure je-5.0.73.jar is downloaded and available +# under Falcon webapp directory or under falcon server classpath. +#*.falcon.graph.storage.backend=berkeleyje +#*.falcon.graph.storage.directory=/${falcon.home}/data/graphdb +#*.falcon.graph.serialize.path=${user.dir}/target/graphdb + +# Enable the following for HBase +#*.falcon.graph.storage.backend=hbase +# For standalone mode , set hostname to localhost; for distributed mode, set to the zookeeper quorum +# @see http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2 +#*.falcon.graph.storage.hostname=localhost +#*.falcon.graph.storage.hbase.table=falcon_titan # Avoid acquiring read lock when iterating over large graphs # See http://s3.thinkaurelius.com/docs/titan/0.5.4/bdb.html http://git-wip-us.apache.org/repos/asf/falcon/blob/e84fca7f/common/src/test/java/org/apache/falcon/metadata/MetadataMappingServiceTest.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/falcon/metadata/MetadataMappingServiceTest.java b/common/src/test/java/org/apache/falcon/metadata/MetadataMappingServiceTest.java index c0ae5fc..62db501 100644 --- a/common/src/test/java/org/apache/falcon/metadata/MetadataMappingServiceTest.java +++ b/common/src/test/java/org/apache/falcon/metadata/MetadataMappingServiceTest.java @@ -122,8 +122,10 @@ public class MetadataMappingServiceTest { configStore = ConfigurationStore.get(); Services.get().register(new WorkflowJobEndNotificationService()); - StartupProperties.get().setProperty("falcon.graph.storage.directory", - "target/graphdb-" + System.currentTimeMillis()); + StartupProperties.get().setProperty("falcon.graph.storage.backend", "berkeleyje"); + String graphDBDir = "target/graphdb-" + System.currentTimeMillis(); + StartupProperties.get().setProperty("falcon.graph.storage.directory", graphDBDir); + StartupProperties.get().setProperty("falcon.graph.serialize.path", graphDBDir); StartupProperties.get().setProperty("falcon.graph.preserve.history", "true"); service = new MetadataMappingService(); service.init(); http://git-wip-us.apache.org/repos/asf/falcon/blob/e84fca7f/docs/src/site/twiki/Configuration.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/Configuration.twiki b/docs/src/site/twiki/Configuration.twiki index bfca3d8..33eba20 100644 --- a/docs/src/site/twiki/Configuration.twiki +++ b/docs/src/site/twiki/Configuration.twiki @@ -325,10 +325,16 @@ to <verbatim>$FALCON_HOME/conf/startup.properties</verbatim> before starting the For details on the same, refer to [[FalconNativeScheduler][Falcon Native Scheduler]] ---+++Titan GraphDB backend -You can either choose to use 5.0.73 version of berkeleydb (the default for Falcon for the last few releases) or 1.1.x or later version HBase as the backend database. Falcon in its release distributions will have the titan storage plugins for both BerkeleyDB and HBase. +GraphDB backend needs to be configured to properly start Falcon server. +You can either choose to use 5.0.73 version of berkeleydb (the default for Falcon for the last few releases) or 1.1.x or later version HBase as the backend database. +Falcon in its release distributions will have the titan storage plugins for both BerkeleyDB and HBase. ----++++Using BerkeleyDB backend -Falcon distributions may not package berkeley db artifacts (je-5.0.73.jar) based on build profiles. If Berkeley DB is not packaged, you can download the Berkeley DB jar file from the URL: <verbatim>http://download.oracle.com/otn/berkeley-db/je-5.0.73.zip</verbatim>. The following properties describe an example berkeley db graph storage backend that can be specified in the configuration file <verbatim>$FALCON_HOME/conf/startup.properties</verbatim>. +Falcon distributions may not package berkeley db artifacts (je-5.0.73.jar) based on build profiles. +If Berkeley DB is not packaged, you can download the Berkeley DB jar file from the URL: +<verbatim>http://download.oracle.com/otn/berkeley-db/je-5.0.73.zip</verbatim>. +The following properties describe an example berkeley db graph storage backend that can be specified in the configuration file +<verbatim>$FALCON_HOME/conf/startup.properties</verbatim>. <verbatim> # Graph Storage http://git-wip-us.apache.org/repos/asf/falcon/blob/e84fca7f/prism/src/test/java/org/apache/falcon/resource/metadata/MetadataTestContext.java ---------------------------------------------------------------------- diff --git a/prism/src/test/java/org/apache/falcon/resource/metadata/MetadataTestContext.java b/prism/src/test/java/org/apache/falcon/resource/metadata/MetadataTestContext.java index 47d6ba1..0fc708d 100644 --- a/prism/src/test/java/org/apache/falcon/resource/metadata/MetadataTestContext.java +++ b/prism/src/test/java/org/apache/falcon/resource/metadata/MetadataTestContext.java @@ -95,6 +95,10 @@ public class MetadataTestContext { Services.get().register(new WorkflowJobEndNotificationService()); Assert.assertTrue(Services.get().isRegistered(WorkflowJobEndNotificationService.SERVICE_NAME)); + StartupProperties.get().setProperty("falcon.graph.storage.backend", "berkeleyje"); + String graphDBDir = "target/graphdb-" + System.currentTimeMillis(); + StartupProperties.get().setProperty("falcon.graph.storage.directory", graphDBDir); + StartupProperties.get().setProperty("falcon.graph.serialize.path", graphDBDir); StartupProperties.get().setProperty("falcon.graph.preserve.history", "true"); service = new MetadataMappingService(); service.init(); http://git-wip-us.apache.org/repos/asf/falcon/blob/e84fca7f/src/conf/startup.properties ---------------------------------------------------------------------- diff --git a/src/conf/startup.properties b/src/conf/startup.properties index d732013..00ac4cd 100644 --- a/src/conf/startup.properties +++ b/src/conf/startup.properties @@ -173,25 +173,19 @@ prism.configstore.listeners=org.apache.falcon.entity.v0.EntityGraph,\ *.falcon.graph.blueprints.graph=com.thinkaurelius.titan.core.TitanFactory # Graph Storage -# IMPORTANT: Please enable one of hbase or berkeleydb backends are enabled -# after the backend requirements are provisioned as needed. +# IMPORTANT: Please enable one of the graph db backend: hbase or berkeleydb, per instructions below. -# Enable the following for Berkeley DB. Make sure je-5.0.73.jar is -# downloaded and available under Falcon webapp directory or under falcon -# server classpath. - -#*.falcon.graph.storage.directory=/${falcon.home}/data/graphdb +# Enable the following for Berkeley DB. Make sure je-5.0.73.jar is downloaded and available +# under Falcon webapp directory or under falcon server classpath. #*.falcon.graph.storage.backend=berkeleyje - +#*.falcon.graph.storage.directory=/${falcon.home}/data/graphdb +#*.falcon.graph.serialize.path=${user.dir}/target/graphdb # Enable the following for HBase #*.falcon.graph.storage.backend=hbase -#For standalone mode , set hostname to localhost -#for distributed mode, set to the zookeeper quorum +# For standalone mode , set hostname to localhost; for distributed mode, set to the zookeeper quorum # @see http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2 - #*.falcon.graph.storage.hostname=localhost -#*.falcon.graph.serialize.path=${user.dir}/target/graphdb #*.falcon.graph.storage.hbase.table=falcon_titan # Avoid acquiring read lock when iterating over large graphs
