Repository: atlas Updated Branches: refs/heads/master 67d489530 -> 993180309
ATLAS-2461: Migration utility to export data from Atlas 0.8 version (cherry picked from commit 2a560ac1091bb4d4e2522f035fd4e0cd39fdaccb) (cherry picked from commit ef642b4cb612fb15c433932967c462e6705d005a) Signed-off-by: Madhan Neethiraj <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/99318030 Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/99318030 Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/99318030 Branch: refs/heads/master Commit: 9931803090d187cb2a92cf88e4d111329121c503 Parents: 67d4895 Author: Ashutosh Mestry <[email protected]> Authored: Sun Mar 4 09:21:04 2018 -0800 Committer: Madhan Neethiraj <[email protected]> Committed: Mon Mar 26 17:28:23 2018 -0700 ---------------------------------------------------------------------- distro/pom.xml | 1 + .../src/main/assemblies/migration-exporter.xml | 60 ++++++ pom.xml | 2 + tools/atlas-migration-exporter/pom.xml | 74 +++++++ .../org/apache/atlas/migration/Exporter.java | 193 +++++++++++++++++++ .../atlas/migration/NoOpNotification.java | 54 ++++++ .../NoOpNotificationChangeListener.java | 60 ++++++ .../src/main/resources/README | 37 ++++ .../src/main/resources/atlas-log4j.xml | 73 +++++++ .../main/resources/atlas_migration_export.py | 154 +++++++++++++++ .../src/main/resources/migrationContext.xml | 40 ++++ 11 files changed, 748 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/99318030/distro/pom.xml ---------------------------------------------------------------------- diff --git a/distro/pom.xml b/distro/pom.xml index 0103bef..f26d8a6 100644 --- a/distro/pom.xml +++ b/distro/pom.xml @@ -112,6 +112,7 @@ atlas.graph.index.search.solr.wait-searcher=true <descriptor>src/main/assemblies/atlas-storm-hook-package.xml</descriptor> <descriptor>src/main/assemblies/standalone-package.xml</descriptor> <descriptor>src/main/assemblies/src-package.xml</descriptor> + <descriptor>src/main/assemblies/migration-exporter.xml</descriptor> </descriptors> <finalName>apache-atlas-${project.version}</finalName> <tarLongFileMode>gnu</tarLongFileMode> http://git-wip-us.apache.org/repos/asf/atlas/blob/99318030/distro/src/main/assemblies/migration-exporter.xml ---------------------------------------------------------------------- diff --git a/distro/src/main/assemblies/migration-exporter.xml b/distro/src/main/assemblies/migration-exporter.xml new file mode 100644 index 0000000..be0a9d5 --- /dev/null +++ b/distro/src/main/assemblies/migration-exporter.xml @@ -0,0 +1,60 @@ +<!-- +** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* +--> +<assembly> + <id>migration-exporter</id> + <formats> + <format>dir</format> + <format>zip</format> + </formats> + + <baseDirectory>atlas-migration-exporter</baseDirectory> + + <fileSets> + <fileSet> + <includes> + <include>README*</include> + </includes> + </fileSet> + <fileSet> + <directory>../tools/atlas-migration-exporter/src/main/resources</directory> + <outputDirectory>.</outputDirectory> + <includes> + <include>*.py</include> + <include>migrationContext.xml</include> + <include>atlas-log4j.xml</include> + <include>README</include> + </includes> + <fileMode>0755</fileMode> + <directoryMode>0755</directoryMode> + </fileSet> + <fileSet> + <directory>../tools/atlas-migration-exporter/target</directory> + <outputDirectory>.</outputDirectory> + <includes> + <include>atlas-migration-*.jar</include> + </includes> + <excludes> + <exclude>*-test-sources.jar</exclude> + <exclude>*-sources.jar</exclude> + </excludes> + </fileSet> + </fileSets> +</assembly> http://git-wip-us.apache.org/repos/asf/atlas/blob/99318030/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index ee2746a..bfbb953 100644 --- a/pom.xml +++ b/pom.xml @@ -755,6 +755,8 @@ <module>addons/storm-bridge</module> <module>addons/hbase-bridge-shim</module> <module>addons/hbase-bridge</module> + <module>tools/atlas-migration-exporter</module> + <module>distro</module> </modules> http://git-wip-us.apache.org/repos/asf/atlas/blob/99318030/tools/atlas-migration-exporter/pom.xml ---------------------------------------------------------------------- diff --git a/tools/atlas-migration-exporter/pom.xml b/tools/atlas-migration-exporter/pom.xml new file mode 100644 index 0000000..6910254 --- /dev/null +++ b/tools/atlas-migration-exporter/pom.xml @@ -0,0 +1,74 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.atlas</groupId> + <artifactId>apache-atlas</artifactId> + <version>1.0.0-SNAPSHOT</version> + </parent> + <artifactId>atlas-migration-exporter</artifactId> + <description>Apache Atlas Migration Exporter</description> + <name>Apache Atlas Migration Exporter</name> + <packaging>jar</packaging> + + <properties> + <tinkerpop.version>2.6.0</tinkerpop.version> + <titan.version>0.5.4</titan.version> + <checkstyle.failOnViolation>false</checkstyle.failOnViolation> + </properties> + + <dependencies> + <dependency> + <groupId>commons-cli</groupId> + <artifactId>commons-cli</artifactId> + </dependency> + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-context</artifactId> + <version>${spring.version}</version> + </dependency> + <dependency> + <groupId>org.apache.atlas</groupId> + <artifactId>atlas-notification</artifactId> + </dependency> + <dependency> + <groupId>org.apache.atlas</groupId> + <artifactId>atlas-repository</artifactId> + </dependency> + <dependency> + <groupId>com.tinkerpop.blueprints</groupId> + <artifactId>blueprints-core</artifactId> + <version>${tinkerpop.version}</version> + </dependency> + <dependency> + <groupId>com.thinkaurelius.titan</groupId> + <artifactId>titan-core</artifactId> + <version>${titan.version}</version> + </dependency> + <dependency> + <groupId>org.apache.atlas</groupId> + <artifactId>atlas-graphdb-titan0</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/atlas/blob/99318030/tools/atlas-migration-exporter/src/main/java/org/apache/atlas/migration/Exporter.java ---------------------------------------------------------------------- diff --git a/tools/atlas-migration-exporter/src/main/java/org/apache/atlas/migration/Exporter.java b/tools/atlas-migration-exporter/src/main/java/org/apache/atlas/migration/Exporter.java new file mode 100644 index 0000000..2af8b7d --- /dev/null +++ b/tools/atlas-migration-exporter/src/main/java/org/apache/atlas/migration/Exporter.java @@ -0,0 +1,193 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.migration; + +import com.thinkaurelius.titan.core.TitanGraph; +import com.tinkerpop.blueprints.Graph; +import com.tinkerpop.blueprints.util.io.graphson.GraphSONMode; +import com.tinkerpop.blueprints.util.io.graphson.GraphSONWriter; +import org.apache.atlas.model.typedef.AtlasTypesDef; +import org.apache.atlas.repository.graphdb.titan0.Titan0GraphDatabase; +import org.apache.atlas.type.AtlasType; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.commons.cli.BasicParser; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Options; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.ApplicationContext; +import org.springframework.context.support.ClassPathXmlApplicationContext; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.OutputStream; +import java.util.ArrayList; + + +public class Exporter { + private static final Logger LOG = LoggerFactory.getLogger(Exporter.class); + + private static final String ATLAS_TYPE_REGISTRY = "atlasTypeRegistry"; + private static final String APPLICATION_CONTEXT = "migrationContext.xml"; + private static final String MIGRATION_TYPESDEF_FILENAME = "atlas-migration-typesdef.json"; + private static final String MIGRATION_DATA_FILENAME = "atlas-migration-data.json"; + private static final String LOG_MSG_PREFIX = "atlas-migration-export: "; + private static final int PROGRAM_ERROR_STATUS = -1; + private static final int PROGRAM_SUCCESS_STATUS = 0; + + private final String typesDefFileName; + private final String dataFileName; + private final AtlasTypeRegistry typeRegistry; + + public static void main(String args[]) { + int result; + + try { + String logFileName = System.getProperty("atlas.log.dir") + File.separatorChar + System.getProperty("atlas.log.file"); + + displayMessage("starting migration export. Log file location " + logFileName); + + Options options = new Options(); + options.addOption("d", "outputdir", true, "Output directory"); + + CommandLine cmd = (new BasicParser()).parse(options, args); + String outputDir = cmd.getOptionValue("d"); + + if (StringUtils.isEmpty(outputDir)) { + outputDir = System.getProperty("user.dir"); + } + + String typesDefFileName = outputDir + File.separatorChar + MIGRATION_TYPESDEF_FILENAME; + String dataFileName = outputDir + File.separatorChar + MIGRATION_DATA_FILENAME; + + Exporter exporter = new Exporter(typesDefFileName, dataFileName, APPLICATION_CONTEXT); + + exporter.perform(); + + result = PROGRAM_SUCCESS_STATUS; + + displayMessage("completed migration export!"); + } catch (Exception e) { + displayError("Failed", e); + + result = PROGRAM_ERROR_STATUS; + } + + System.exit(result); + } + + public Exporter(String typesDefFileName, String dataFileName, String contextXml) throws Exception { + validate(typesDefFileName, dataFileName); + + displayMessage("initializing"); + + ApplicationContext applicationContext = new ClassPathXmlApplicationContext(contextXml); + + this.typesDefFileName = typesDefFileName; + this.dataFileName = dataFileName; + this.typeRegistry = applicationContext.getBean(ATLAS_TYPE_REGISTRY, AtlasTypeRegistry.class);; + + displayMessage("initialized"); + } + + public void perform() throws Exception { + exportTypes(); + exportData(); + } + + private void validate(String typesDefFileName, String dataFileName) throws Exception { + File typesDefFile = new File(typesDefFileName); + File dataFile = new File(dataFileName); + + if (typesDefFile.exists()) { + throw new Exception("output file " + typesDefFileName + " already exists"); + } + + if (dataFile.exists()) { + throw new Exception("output file " + dataFileName + " already exists"); + } + } + + private void exportTypes() throws Exception { + displayMessage("exporting typesDef to file " + typesDefFileName); + + AtlasTypesDef typesDef = getTypesDef(typeRegistry); + + FileUtils.write(new File(typesDefFileName), AtlasType.toJson(typesDef)); + + displayMessage("exported typesDef to file " + typesDefFileName); + } + + private void exportData() throws Exception { + displayMessage("exporting data to file " + dataFileName); + + OutputStream os = null; + + try { + os = new FileOutputStream(dataFileName); + + Graph graph = getTitan0GraphDatabase(); + + GraphSONWriter.outputGraph(graph, os, GraphSONMode.EXTENDED); + } finally { + if (os != null) { + try { + os.close(); + } catch (Exception excp) { + // ignore + } + } + } + + displayMessage("exported data to file " + dataFileName); + } + + private AtlasTypesDef getTypesDef(AtlasTypeRegistry registry) { + return new AtlasTypesDef(new ArrayList<>(registry.getAllEnumDefs()), + new ArrayList<>(registry.getAllStructDefs()), + new ArrayList<>(registry.getAllClassificationDefs()), + new ArrayList<>(registry.getAllEntityDefs())); + } + + private TitanGraph getTitan0GraphDatabase() { + return Titan0GraphDatabase.getGraphInstance(); + } + + private static void displayMessage(String msg) { + LOG.info(LOG_MSG_PREFIX + msg); + + System.out.println(LOG_MSG_PREFIX + msg); + System.out.flush(); + } + + private static void displayError(String msg, Throwable t) { + LOG.error(LOG_MSG_PREFIX + msg, t); + + System.out.println(LOG_MSG_PREFIX + msg); + System.out.flush(); + + if (t != null) { + System.out.println("ERROR: " + t.getMessage()); + } + + System.out.flush(); + } +} http://git-wip-us.apache.org/repos/asf/atlas/blob/99318030/tools/atlas-migration-exporter/src/main/java/org/apache/atlas/migration/NoOpNotification.java ---------------------------------------------------------------------- diff --git a/tools/atlas-migration-exporter/src/main/java/org/apache/atlas/migration/NoOpNotification.java b/tools/atlas-migration-exporter/src/main/java/org/apache/atlas/migration/NoOpNotification.java new file mode 100644 index 0000000..a8301ae --- /dev/null +++ b/tools/atlas-migration-exporter/src/main/java/org/apache/atlas/migration/NoOpNotification.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.migration; + +import org.apache.atlas.notification.NotificationConsumer; +import org.apache.atlas.notification.NotificationException; +import org.apache.atlas.notification.NotificationInterface; +import org.springframework.stereotype.Component; + +import java.util.List; + +@Component +public class NoOpNotification implements NotificationInterface { + @Override + public void setCurrentUser(String user) { + + } + + @Override + public <T> List<NotificationConsumer<T>> createConsumers(NotificationType notificationType, int numConsumers) { + return null; + } + + @Override + public <T> void send(NotificationType type, T... messages) throws NotificationException { + + } + + @Override + public <T> void send(NotificationType type, List<T> messages) throws NotificationException { + + } + + @Override + public void close() { + + } +} http://git-wip-us.apache.org/repos/asf/atlas/blob/99318030/tools/atlas-migration-exporter/src/main/java/org/apache/atlas/migration/NoOpNotificationChangeListener.java ---------------------------------------------------------------------- diff --git a/tools/atlas-migration-exporter/src/main/java/org/apache/atlas/migration/NoOpNotificationChangeListener.java b/tools/atlas-migration-exporter/src/main/java/org/apache/atlas/migration/NoOpNotificationChangeListener.java new file mode 100644 index 0000000..6c2bf22 --- /dev/null +++ b/tools/atlas-migration-exporter/src/main/java/org/apache/atlas/migration/NoOpNotificationChangeListener.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.migration; + +import org.apache.atlas.AtlasException; +import org.apache.atlas.listener.EntityChangeListener; +import org.apache.atlas.v1.model.instance.Referenceable; +import org.apache.atlas.v1.model.instance.Struct; +import org.springframework.stereotype.Component; + +import java.util.Collection; + +@Component +public class NoOpNotificationChangeListener implements EntityChangeListener { + @Override + public void onEntitiesAdded(Collection<Referenceable> entities, boolean isImport) throws AtlasException { + + } + + @Override + public void onEntitiesUpdated(Collection<Referenceable> entities, boolean isImport) throws AtlasException { + + } + + @Override + public void onTraitsAdded(Referenceable entity, Collection<? extends Struct> traits) throws AtlasException { + + } + + @Override + public void onTraitsDeleted(Referenceable entity, Collection<String> traitNames) throws AtlasException { + + } + + @Override + public void onTraitsUpdated(Referenceable entity, Collection<? extends Struct> traits) throws AtlasException { + + } + + @Override + public void onEntitiesDeleted(Collection<Referenceable> entities, boolean isImport) throws AtlasException { + + } +} http://git-wip-us.apache.org/repos/asf/atlas/blob/99318030/tools/atlas-migration-exporter/src/main/resources/README ---------------------------------------------------------------------- diff --git a/tools/atlas-migration-exporter/src/main/resources/README b/tools/atlas-migration-exporter/src/main/resources/README new file mode 100644 index 0000000..30dd493 --- /dev/null +++ b/tools/atlas-migration-exporter/src/main/resources/README @@ -0,0 +1,37 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +Introduction + The purpose of this utility is to export type definitions and data from an Atlas repository. + +What is Exported? + All data and types are exported. + +How Much Time Will this Take? + The duration of the export process depends on the number of entities present in your database. While cluster configuration determines speed of operation, + on an average, for cluster with reasonable configuration, it takes 30 minutes to export 1 million entities. + +Steps to Start Export step of Migration + - Shutdown Atlas. This is critical to ensure that no updates are being made to Atlas database while the operation is in progress. + - Execute the following commands in the host where Atlas server runs: + - unzip atlas-migration-exporter.zip + - cd atlas-migration-exporter + - python atlas_migration_export.py + +Next Steps + Once done, please use the Atlas Migration Guide for next steps. http://git-wip-us.apache.org/repos/asf/atlas/blob/99318030/tools/atlas-migration-exporter/src/main/resources/atlas-log4j.xml ---------------------------------------------------------------------- diff --git a/tools/atlas-migration-exporter/src/main/resources/atlas-log4j.xml b/tools/atlas-migration-exporter/src/main/resources/atlas-log4j.xml new file mode 100755 index 0000000..ae0d27d --- /dev/null +++ b/tools/atlas-migration-exporter/src/main/resources/atlas-log4j.xml @@ -0,0 +1,73 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd"> + +<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/";> + <appender name="console" class="org.apache.log4j.ConsoleAppender"> + <param name="Target" value="System.out"/> + <layout class="org.apache.log4j.PatternLayout"> + <param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m (%C{1}:%L)%n"/> + </layout> + </appender> + + <appender name="FILE" class="org.apache.log4j.RollingFileAppender"> + <param name="File" value="${atlas.log.dir}/${atlas.log.file}"/> + <param name="Append" value="true"/> + <layout class="org.apache.log4j.PatternLayout"> + <param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m (%C{1}:%L)%n"/> + </layout> + </appender> + + <logger name="org.apache.atlas" additivity="false"> + <level value="info"/> + <appender-ref ref="FILE"/> + </logger> + + <logger name="com.thinkaurelius.titan" additivity="false"> + <level value="warn"/> + <appender-ref ref="FILE"/> + </logger> + + <logger name="org.springframework" additivity="false"> + <level value="warn"/> + <appender-ref ref="console"/> + </logger> + + <logger name="org.eclipse" additivity="false"> + <level value="warn"/> + <appender-ref ref="console"/> + </logger> + + <logger name="com.sun.jersey" additivity="false"> + <level value="warn"/> + <appender-ref ref="console"/> + </logger> + + <!-- to avoid logs - The configuration log.flush.interval.messages = 1 was supplied but isn't a known config --> + <logger name="org.apache.kafka.common.config.AbstractConfig" additivity="false"> + <level value="error"/> + <appender-ref ref="FILE"/> + </logger> + + <root> + <priority value="warn"/> + <appender-ref ref="FILE"/> + </root> +</log4j:configuration> http://git-wip-us.apache.org/repos/asf/atlas/blob/99318030/tools/atlas-migration-exporter/src/main/resources/atlas_migration_export.py ---------------------------------------------------------------------- diff --git a/tools/atlas-migration-exporter/src/main/resources/atlas_migration_export.py b/tools/atlas-migration-exporter/src/main/resources/atlas_migration_export.py new file mode 100755 index 0000000..ab7c8e5 --- /dev/null +++ b/tools/atlas-migration-exporter/src/main/resources/atlas_migration_export.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import sys +sys.path.insert(0, '/usr/hdp/current/atlas-server/bin/') + +import traceback +import subprocess +import atlas_config as mc + +ATLAS_LOG_OPTS="-Datlas.log.dir=%s -Datlas.log.file=atlas-migration-exporter.log" +ATLAS_COMMAND_OPTS="-Datlas.home=%s" +ATLAS_CONFIG_OPTS="-Datlas.conf=%s" +DEFAULT_JVM_HEAP_OPTS="-Xmx4096m -XX:MaxPermSize=512m" +DEFAULT_JVM_OPTS="-Dlog4j.configuration=atlas-log4j.xml -Djava.net.preferIPv4Stack=true -server" + +def main(): + is_setup = (len(sys.argv)>1) and sys.argv[1] is not None and sys.argv[1] == '-setup' + atlas_home = mc.atlasDir() + confdir = mc.dirMustExist(mc.confDir(atlas_home)) + mc.executeEnvSh(confdir) + logdir = mc.dirMustExist(mc.logDir(atlas_home)) + mc.dirMustExist(mc.dataDir(atlas_home)) + + if mc.isCygwin(): + # Pathnames that are passed to JVM must be converted to Windows format. + jvm_atlas_home = mc.convertCygwinPath(atlas_home) + jvm_confdir = mc.convertCygwinPath(confdir) + jvm_logdir = mc.convertCygwinPath(logdir) + else: + jvm_atlas_home = atlas_home + jvm_confdir = confdir + jvm_logdir = logdir + + #create sys property for conf dirs + jvm_opts_list = (ATLAS_LOG_OPTS % (jvm_logdir)).split() + + cmd_opts = (ATLAS_COMMAND_OPTS % jvm_atlas_home) + jvm_opts_list.extend(cmd_opts.split()) + + config_opts = (ATLAS_CONFIG_OPTS % jvm_confdir) + jvm_opts_list.extend(config_opts.split()) + + atlas_server_heap_opts = os.environ.get(mc.ATLAS_SERVER_HEAP, DEFAULT_JVM_HEAP_OPTS) + jvm_opts_list.extend(atlas_server_heap_opts.split()) + + atlas_server_jvm_opts = os.environ.get(mc.ATLAS_SERVER_OPTS) + if atlas_server_jvm_opts: + jvm_opts_list.extend(atlas_server_jvm_opts.split()) + + atlas_jvm_opts = os.environ.get(mc.ATLAS_OPTS, DEFAULT_JVM_OPTS) + jvm_opts_list.extend(atlas_jvm_opts.split()) + + #expand web app dir + web_app_dir = mc.webAppDir(atlas_home) + mc.expandWebApp(atlas_home) + + p = os.pathsep + atlas_classpath = os.path.join(os.getcwd(), ".", "*") + p \ + + confdir + p \ + + os.path.join(web_app_dir, "atlas", "WEB-INF", "classes" ) + p \ + + os.path.join(web_app_dir, "atlas", "WEB-INF", "lib", "*" ) + p \ + + os.path.join(atlas_home, "libext", "*") + + is_hbase = mc.is_hbase(confdir) + + if is_hbase: + #add hbase-site.xml to classpath + hbase_conf_dir = mc.hbaseConfDir(atlas_home) + + if os.path.exists(hbase_conf_dir): + atlas_classpath = atlas_classpath + p \ + + hbase_conf_dir + else: + if mc.is_hbase(confdir): + raise Exception("Could not find hbase-site.xml in %s. Please set env var HBASE_CONF_DIR to the hbase client conf dir", hbase_conf_dir) + + if mc.isCygwin(): + atlas_classpath = mc.convertCygwinPath(atlas_classpath, True) + + atlas_pid_file = mc.pidFile(atlas_home) + + if os.path.isfile(atlas_pid_file): + #Check if process listed in atlas.pid file is still running + pf = file(atlas_pid_file, 'r') + pid = pf.read().strip() + pf.close() + + if mc.exist_pid((int)(pid)): + mc.server_already_running(pid) + else: + mc.server_pid_not_running(pid) + + if is_hbase and mc.is_hbase_local(confdir): + print "configured for local hbase." + mc.configure_hbase(atlas_home) + mc.run_hbase_action(mc.hbaseBinDir(atlas_home), "start", hbase_conf_dir, logdir) + print "hbase started." + + web_app_path = os.path.join(web_app_dir, "atlas") + if (mc.isCygwin()): + web_app_path = mc.convertCygwinPath(web_app_path) + + start_migration_export(atlas_classpath, atlas_pid_file, jvm_logdir, jvm_opts_list, web_app_path) + +def start_migration_export(atlas_classpath, atlas_pid_file, jvm_logdir, jvm_opts_list, web_app_path): + args = [] + args.extend(sys.argv[1:]) + process = java("org.apache.atlas.migration.Exporter", args, atlas_classpath, jvm_opts_list) + +def java(classname, args, classpath, jvm_opts_list): + java_home = os.environ.get("JAVA_HOME", None) + if java_home: + prg = os.path.join(java_home, "bin", "java") + else: + prg = mc.which("java") + + if prg is None: + raise EnvironmentError('The java binary could not be found in your path or JAVA_HOME') + + commandline = [prg] + commandline.extend(jvm_opts_list) + commandline.append("-classpath") + commandline.append(classpath) + commandline.append(classname) + commandline.extend(args) + + p = subprocess.Popen(commandline) + p.communicate() + +if __name__ == '__main__': + try: + returncode = main() + except Exception as e: + print "Exception: %s " % str(e) + print traceback.format_exc() + returncode = -1 + + sys.exit(returncode) http://git-wip-us.apache.org/repos/asf/atlas/blob/99318030/tools/atlas-migration-exporter/src/main/resources/migrationContext.xml ---------------------------------------------------------------------- diff --git a/tools/atlas-migration-exporter/src/main/resources/migrationContext.xml b/tools/atlas-migration-exporter/src/main/resources/migrationContext.xml new file mode 100644 index 0000000..c0f9659 --- /dev/null +++ b/tools/atlas-migration-exporter/src/main/resources/migrationContext.xml @@ -0,0 +1,40 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor + license agreements. See the NOTICE file distributed with this work for additional + information regarding copyright ownership. The ASF licenses this file to + You under the Apache License, Version 2.0 (the "License"); you may not use + this file except in compliance with the License. You may obtain a copy of + the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required + by applicable law or agreed to in writing, software distributed under the + License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS + OF ANY KIND, either express or implied. See the License for the specific + language governing permissions and limitations under the License. --> + +<beans xmlns="http://www.springframework.org/schema/beans"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xmlns:context="http://www.springframework.org/schema/context"; + xmlns:aop="http://www.springframework.org/schema/aop"; + xsi:schemaLocation="http://www.springframework.org/schema/beans + http://www.springframework.org/schema/beans/spring-beans.xsd http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop.xsd";> + + <context:annotation-config/> + <aop:config proxy-target-class="true"/> + + <context:component-scan base-package="org.apache.atlas"> + <context:exclude-filter type="regex" expression="org\.apache\.atlas\.service\.ActiveInstanceElectorService.*"/> + <context:exclude-filter type="regex" expression="org\.apache\.atlas\.service\.EmbeddedKafkaServer.*"/> + <!--<context:exclude-filter type="regex" expression="org\.apache\.atlas\.service\.HBaseBasedAuditRepository.*"/>--> + <!-- for non-HBase setups comment the InMemoryEntityAuditRepository and comment the HBaseBasedAuditoryRepository --> + <context:exclude-filter type="regex" expression="org\.apache\.atlas\.service\.InMemoryEntityAuditRepository.*"/> + <context:exclude-filter type="regex" expression="org\.apache\.atlas\.service\.NoopEntityAuditRepository.*"/> + <context:exclude-filter type="regex" expression="org\.apache\.atlas\.service\.KafkaNotification.*"/> + <context:exclude-filter type="regex" expression="org\.apache\.atlas\.service\.NotificationHookConsumer.*"/> + <context:exclude-filter type="regex" expression="org\.apache\.atlas\.kafka.*"/> + <context:exclude-filter type="regex" expression="org\.apache\.atlas\.webapp.*"/> + <context:exclude-filter type="regex" expression="org\.apache\.atlas\.web.*"/> + <context:exclude-filter type="regex" expression="org\.apache\.atlas\.notification.hook.*"/> + <context:exclude-filter type="regex" expression="org\.apache\.atlas\.notification.entity.*"/> + <context:exclude-filter type="regex" expression="org\.apache\.atlas\.notification.NotificationHookConsumer.*"/> + <context:exclude-filter type="regex" expression="org\.apache\.atlas\.ha.*"/> + </context:component-scan> +</beans>
