Repository: falcon Updated Branches: refs/heads/0.10 eddf2ee5e -> 805298520
FALCON-2104 Loss of data in GraphDB when upgrading Falcon 0.9 to 0.10 Author: bvellanki <[email protected]> Reviewers: "Venkat Ranganathan <[email protected]>, Sowmya Ramesh <[email protected]>" Closes #253 from bvellanki/FALCON-2104 (cherry picked from commit dda42366823e0fdb996ed70e34710fa8dfb4393c) Signed-off-by: bvellanki <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/80529852 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/80529852 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/80529852 Branch: refs/heads/0.10 Commit: 80529852009e8dfd3fd710c0c4ef062946136c3d Parents: eddf2ee Author: bvellanki <[email protected]> Authored: Tue Aug 2 14:44:30 2016 -0700 Committer: bvellanki <[email protected]> Committed: Tue Aug 2 14:44:44 2016 -0700 ---------------------------------------------------------------------- .../falcon/metadata/GraphUpdateUtils.java | 113 ++++++++++++++++++ .../falcon/metadata/MetadataMappingService.java | 4 +- docs/src/site/twiki/InstallationSteps.twiki | 15 ++- docs/src/site/twiki/MigrationInstructions.twiki | 29 ++++- release-docs/0.10/CHANGES.0.10.md | 1 + src/bin/graphdbutil.sh | 118 +++++++++++++++++++ 6 files changed, 264 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/80529852/common/src/main/java/org/apache/falcon/metadata/GraphUpdateUtils.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/metadata/GraphUpdateUtils.java b/common/src/main/java/org/apache/falcon/metadata/GraphUpdateUtils.java new file mode 100644 index 0000000..a256e46 --- /dev/null +++ b/common/src/main/java/org/apache/falcon/metadata/GraphUpdateUtils.java @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.falcon.metadata; + +import com.tinkerpop.blueprints.Graph; +import com.tinkerpop.blueprints.util.io.graphson.GraphSONReader; +import com.tinkerpop.blueprints.util.io.graphson.GraphSONWriter; +import org.apache.commons.configuration.Configuration; +import org.apache.commons.io.FileUtils; +import org.apache.falcon.FalconException; + +import java.io.File; + +/** + * Utility class for graph operations. + */ +public final class GraphUpdateUtils { + + private static final String BANNER_MSG = + "Before running this utility please make sure that Falcon startup properties " + + "has the right configuration settings for the graph database, " + + "Falcon server is stopped and no other access to the graph database is being performed."; + + private static final String IMPORT = "import"; + private static final String EXPORT = "export"; + private static final String INSTANCE_JSON_FILE = "instanceMetadata.json"; + + private GraphUpdateUtils() { + } + + public static void main(String[] args) { + if (args.length != 2) { + usage(); + System.exit(1); + } + System.out.println(BANNER_MSG); + String operation = args[0].toLowerCase(); + if (!(operation.equals(EXPORT) || operation.equals(IMPORT))) { + usage(); + System.exit(1); + } + String utilsDir = args[1]; + File utilsDirFile = new File(utilsDir); + if (!utilsDirFile.isDirectory()) { + System.err.println(utilsDir + " is not a valid directory"); + System.exit(1); + } + String jsonFile = new File(utilsDirFile, INSTANCE_JSON_FILE).getAbsolutePath(); + try { + Graph graph; + if (operation.equals(EXPORT)) { + graph = MetadataMappingService.initializeGraphDB(); + GraphSONWriter.outputGraph(graph, jsonFile); + System.out.println("Exported instance metadata to " + jsonFile); + } else { + // Backup existing graphDB dir + Configuration graphConfig = MetadataMappingService.getConfiguration(); + String graphStore = (String) graphConfig.getProperty("storage.directory"); + File graphStoreFile = new File(graphStore); + File graphDirBackup = new File(graphStore + "_backup"); + if (graphDirBackup.exists()) { + FileUtils.deleteDirectory(graphDirBackup); + } + FileUtils.copyDirectory(graphStoreFile, graphDirBackup); + + // delete graph dir first and then init graphDB to ensure IMPORT happens into empty DB. + FileUtils.deleteDirectory(graphStoreFile); + graph = MetadataMappingService.initializeGraphDB(); + + // Import, if there is an exception restore backup. + try { + GraphSONReader.inputGraph(graph, jsonFile); + System.out.println("Imported instance metadata to " + jsonFile); + } catch (Exception ex) { + String errorMsg = ex.getMessage(); + if (graphStoreFile.exists()) { + FileUtils.deleteDirectory(graphStoreFile); + } + FileUtils.copyDirectory(graphDirBackup, graphStoreFile); + throw new FalconException(errorMsg); + } + } + } catch (Exception e) { + System.err.println("Error " + operation + "ing JSON data to " + jsonFile + ", " + e.getMessage()); + e.printStackTrace(System.out); + System.exit(1); + } + System.exit(0); + } + + public static void usage() { + StringBuilder usageMessage = new StringBuilder(1024); + usageMessage.append("usage: java ").append(GraphUpdateUtils.class.getName()) + .append(" {").append(EXPORT).append('|').append(IMPORT).append("} <directory>"); + System.err.println(usageMessage); + } +} http://git-wip-us.apache.org/repos/asf/falcon/blob/80529852/common/src/main/java/org/apache/falcon/metadata/MetadataMappingService.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/metadata/MetadataMappingService.java b/common/src/main/java/org/apache/falcon/metadata/MetadataMappingService.java index 727be56..225e44a 100644 --- a/common/src/main/java/org/apache/falcon/metadata/MetadataMappingService.java +++ b/common/src/main/java/org/apache/falcon/metadata/MetadataMappingService.java @@ -139,14 +139,14 @@ public class MetadataMappingService } } - protected Graph initializeGraphDB() { + public static Graph initializeGraphDB() { LOG.info("Initializing graph db"); Configuration graphConfig = getConfiguration(); validateConfiguration(graphConfig); return GraphFactory.open(graphConfig); } - private void validateConfiguration(Configuration graphConfig) { + private static void validateConfiguration(Configuration graphConfig) { // check if storage backend if configured if (!graphConfig.containsKey(PROPERTY_KEY_STORAGE_BACKEND)) { throw new FalconRuntimException("Titan GraphDB storage backend is not configured. " http://git-wip-us.apache.org/repos/asf/falcon/blob/80529852/docs/src/site/twiki/InstallationSteps.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/InstallationSteps.twiki b/docs/src/site/twiki/InstallationSteps.twiki index 93b1eab..297d88e 100644 --- a/docs/src/site/twiki/InstallationSteps.twiki +++ b/docs/src/site/twiki/InstallationSteps.twiki @@ -27,16 +27,15 @@ $ mvn clean install </verbatim> It builds and installs the package into the local repository, for use as a dependency in other projects locally. -[optionally -Dhadoop.version=<<hadoop.version>> can be appended to build for a specific version of Hadoop] - -*NOTE:* Falcon drops support for Hadoop-1 and only supports Hadoop-2 from Falcon 0.6 onwards -[optionally -Doozie.version=<<oozie version>> can be appended to build with a specific version of Oozie. Oozie versions ->= 4 are supported] -NOTE: Falcon builds with JDK 1.7/1.8 using -noverify option - To compile Falcon with Hive Replication, optionally "-P hadoop-2,hivedr" can be appended. For this Hive >= 1.2.0 - and Oozie >= 4.2.0 should be available. +[optionally -Dhadoop.version=<<hadoop.version>> can be appended to build for a specific version of hadoop] +*Note 1:* Falcon drops support for Hadoop-1 and only supports Hadoop-2 from Falcon 0.6 onwards + Falcon build with JDK 1.7 using -noverify option +*Note 2:* To compile Falcon with addon extensions, append additional profiles to build command using syntax -P<<profile1,profile2>> + For Hive Mirroring extension, use profile"hivedr". Hive >= 1.2.0 and Oozie >= 4.2.0 is required + For HDFS Snapshot mirroring extension, use profile "hdfs-snapshot-mirroring". Hadoop >= 2.7.0 is required + For ADF integration, use profile "adf" ---+++Step 3 - Package and Deploy Falcon http://git-wip-us.apache.org/repos/asf/falcon/blob/80529852/docs/src/site/twiki/MigrationInstructions.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/MigrationInstructions.twiki b/docs/src/site/twiki/MigrationInstructions.twiki index 7c0e027..a11dbc4 100644 --- a/docs/src/site/twiki/MigrationInstructions.twiki +++ b/docs/src/site/twiki/MigrationInstructions.twiki @@ -1,15 +1,32 @@ ---+ Migration Instructions ----++ Migrate from 0.5-incubating to 0.6-incubating +---++ Migrate from 0.9 to 0.10 -This is a placeholder wiki for migration instructions from falcon 0.5-incubating to 0.6-incubating. +FALCON-1333 (Instance Search feature) requires Falcon to use titan-berkeleyje version 0.5.4 to support indexing. +Up until version 0.9 - Falcon used titan-berkeleyje-jre6 version 0.4.2. GraphDB created by version 0.4.2 cannot be +read by version 0.5.4. The solution is to migrate the GraphDB to be compatible with Falcon 0.10 release. Please make +sure that no falcon server is running while performing the migration. ----+++ Update Entities +---+++ 1. Install Falcon 0.10 +Install Falcon 0.10 by following the [[InstallationSteps][Installation Steps]]. Do not start the falcon server yet. +The tool to migrate graphDB is packaged with 0.10 Falcon server in falcon-common-0.10.jar. ----+++ Change cluster dir permissions +---+++ 2. Export GraphDB to JSON file using Falcon 0.9 +Please run the following command to generate the JSON file. ----+++ Enable/Disable TLS +<verbatim> + $FALCON_HOME/bin/graphdbutil.sh export <<java_home> <<hadoop_home>> <<falcon_0.9_home>> <<path_to_falcon-common-0.10.jar>> /jsonFile/dir/ +</verbatim> ----+++ Authorization +This command will create /jsonFile/dir/instanceMetadata.json +---+++ 3. Import GraphDB from JSON file using Falcon 0.10 +Please run the following command to import graphDB the JSON file. The location of graphDB will be based on property +"*.falcon.graph.storage.directory" set in startup.properties file. + +<verbatim> + $FALCON_HOME/bin/graphdbutil.sh export <<java_home> <<hadoop_home>> <<falcon_0.10_home>> <<path_to_falcon-common-0.10.jar>> /jsonFile/dir/ +</verbatim> + +This command will import from /jsonFile/dir/instanceMetadata.json, now start the Falcon 0.10 server. http://git-wip-us.apache.org/repos/asf/falcon/blob/80529852/release-docs/0.10/CHANGES.0.10.md ---------------------------------------------------------------------- diff --git a/release-docs/0.10/CHANGES.0.10.md b/release-docs/0.10/CHANGES.0.10.md index 999af01..668dc97 100644 --- a/release-docs/0.10/CHANGES.0.10.md +++ b/release-docs/0.10/CHANGES.0.10.md @@ -54,6 +54,7 @@ | JIRA | Summary | Priority | Component | Reporter | Contributor | |:---- |:---- | :--- |:---- |:---- |:---- | +| [FALCON-2104](https://issues.apache.org/jira/browse/FALCON-2104) | Loss of data in GraphDB when upgrading Falcon from 0.9 to 0.10. | Blocker | . | Balu Vellanki | Balu Vellanki | | [FALCON-2100](https://issues.apache.org/jira/browse/FALCON-2100) | Remove dependency on com.vividsolutions.jts | Major | . | Balu Vellanki | Balu Vellanki | | [FALCON-2090](https://issues.apache.org/jira/browse/FALCON-2090) | HDFS Snapshot failed with UnknownHostException when scheduling in HA Mode | Critical | replication | Murali Ramasami | Balu Vellanki | | [FALCON-2088](https://issues.apache.org/jira/browse/FALCON-2088) | Entity submission fails with EntityNotRegisteredException in distributed mode | Blocker | feed, prism, process | Pragya Mittal | Praveen Adlakha | http://git-wip-us.apache.org/repos/asf/falcon/blob/80529852/src/bin/graphdbutil.sh ---------------------------------------------------------------------- diff --git a/src/bin/graphdbutil.sh b/src/bin/graphdbutil.sh new file mode 100644 index 0000000..151ec2f --- /dev/null +++ b/src/bin/graphdbutil.sh @@ -0,0 +1,118 @@ +#!/bin/sh +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. +# + + +usage() { + echo "usage: $0 operation java-home hadoop-home falcon-home falcon-common-jar input/out-dir" + echo " where operation is either export OR import" + echo " java-home is the java installation location" + echo " hadoop-home is the hadoop installation location" + echo " falcon-home is the falcon home installation location" + echo " falcon-common-jar is the falcon-common-<version>.jar location with GraphUtils" + echo " input/output dir is the directory for the graph data" + exit 1 +} + +if [ $# != 6 ]; then + usage +fi + +operation=$1 +java_home=$2 +hadoop_home=$3 +falcon_home=$4 +falcon_common_jar=$5 +util_dir=$6 + +export=0 +import=0 +keep_temp=Y + +case $operation in + import) import=1 + ;; + export) export=1 + ;; + *) echo "Unknown operation $operation" + usage +esac + +if [ -d $java_home -a -f $java_home/bin/java -a -f $java_home/bin/jar ] ; then + : +else + echo "Invalid java home directory $java_home" + usage +fi + +if [ -d $hadoop_home -a -f $hadoop_home/bin/hadoop ] ; then + : +else + echo "Invalid hadoop home directory $hadoop_home" + usage +fi + +if [ -d $falcon_home -a -f $falcon_home/bin/falcon ] ; then + : +else + echo "Invalid falcon home directory $falcon_home" + usage +fi + +falcon_war=$falcon_home/server/webapp/falcon.war +if [ ! -f $falcon_war ]; then + echo "Falcon war file $falcon_war not available" + usage +fi + +if [ ! -f $falcon_common_jar ]; then + echo "Falcon commons jar file $falcon_common_jar not available" + usage +fi + + +util_tmpdir=/tmp/falcon-graphutil-tmp-$$ +echo "Using $util_tmpdir as temporary directory" +trap "rm -rf $util.tmpdir" 0 2 3 15 +rm -rf $util_tmpdir +mkdir -p $util_tmpdir + +if [ ! -d $util_dir ]; then + echo "Directory $util_dir does not exist" + usage +fi + +if [ x$import = x1 ]; then + if [ ! -f $metadata_file ]; then + echo "Directory $util_dir does not exist or $metadata_file not present" + usage + fi +fi + +cd $util_tmpdir +jar -xf $falcon_war +rm ./WEB-INF/lib/jackson* ./WEB-INF/lib/falcon-common*.jar ./WEB-INF/lib/slf4j* ./WEB-INF/lib/activemq* +cp $falcon_common_jar ./WEB-INF/lib/ + +JAVA_HOME=$java_home +export PATH=$JAVA_HOME/bin:$PATH +export CLASSPATH="$falcon_home/conf:./WEB-INF/lib/*:`$hadoop_home/bin/hadoop classpath`" +echo "Using classpath $CLASSPATH" +java -Dfalcon.log.dir=/tmp/ org.apache.falcon.metadata.GraphUpdateUtils $operation $util_dir + +if [ x$keep_temp = xY ]; then + : +else + rm -rf $util_tmpdir +fi \ No newline at end of file
