Repository: atlas
Updated Branches:
  refs/heads/branch-0.8 752393106 -> 2a560ac10


ATLAS-2461: Migration utility.


Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/2a560ac1
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/2a560ac1
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/2a560ac1

Branch: refs/heads/branch-0.8
Commit: 2a560ac1091bb4d4e2522f035fd4e0cd39fdaccb
Parents: 7523931
Author: Ashutosh Mestry <ames...@hortonworks.com>
Authored: Sun Mar 4 09:21:04 2018 -0800
Committer: Ashutosh Mestry <ames...@hortonworks.com>
Committed: Wed Mar 7 10:13:24 2018 -0800

----------------------------------------------------------------------
 distro/pom.xml                                  |   1 +
 .../src/main/assemblies/migration-exporter.xml  |  58 +++++
 pom.xml                                         |   1 +
 tools/atlas-migration-exporter/pom.xml          |  59 +++++
 .../org/apache/atlas/migration/Exporter.java    | 213 +++++++++++++++++++
 .../atlas/migration/NoOpNotification.java       |  54 +++++
 .../NoOpNotificationChangeListener.java         |  61 ++++++
 .../src/main/resources/README                   |  41 ++++
 .../src/main/resources/atlas_migration.py       | 141 ++++++++++++
 .../resources/migration-export-request.json     |   4 +
 .../src/main/resources/migrationContext.xml     |  40 ++++
 11 files changed, 673 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/atlas/blob/2a560ac1/distro/pom.xml
----------------------------------------------------------------------
diff --git a/distro/pom.xml b/distro/pom.xml
index 68b967d..096181e 100644
--- a/distro/pom.xml
+++ b/distro/pom.xml
@@ -106,6 +106,7 @@ 
atlas.graph.index.search.solr.zookeeper-session-timeout=60000
                                     <descriptors>
                                         
<descriptor>src/main/assemblies/standalone-package.xml</descriptor>
                                         
<descriptor>src/main/assemblies/src-package.xml</descriptor>
+                                        
<descriptor>src/main/assemblies/migration-exporter.xml</descriptor>
                                     </descriptors>
                                     
<finalName>apache-atlas-${project.version}</finalName>
                                     <tarLongFileMode>gnu</tarLongFileMode>

http://git-wip-us.apache.org/repos/asf/atlas/blob/2a560ac1/distro/src/main/assemblies/migration-exporter.xml
----------------------------------------------------------------------
diff --git a/distro/src/main/assemblies/migration-exporter.xml 
b/distro/src/main/assemblies/migration-exporter.xml
new file mode 100644
index 0000000..8f751ff
--- /dev/null
+++ b/distro/src/main/assemblies/migration-exporter.xml
@@ -0,0 +1,58 @@
+<!--
+**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*
+*
+-->
+<assembly>
+    <id>migration-exporter</id>
+    <formats>
+        <format>dir</format>
+        <format>zip</format>
+    </formats>
+
+    <fileSets>
+        <fileSet>
+            <includes>
+                <include>README*</include>
+            </includes>
+        </fileSet>
+        <fileSet>
+            
<directory>../tools/atlas-migration-exporter/src/main/resources</directory>
+            <outputDirectory>.</outputDirectory>
+            <includes>
+                <include>*.py</include>
+                <include>migrationContext.xml</include>
+                <include>*.json</include>
+                <include>README</include>
+            </includes>
+            <fileMode>0755</fileMode>
+            <directoryMode>0755</directoryMode>
+        </fileSet>
+        <fileSet>
+            <directory>../tools/atlas-migration-exporter/target</directory>
+            <outputDirectory>.</outputDirectory>
+            <includes>
+                <include>atlas-migration-*.jar</include>
+            </includes>
+            <excludes>
+                <exclude>*-test-sources.jar</exclude>
+                <exclude>*-sources.jar</exclude>
+            </excludes>
+        </fileSet>
+    </fileSets>
+</assembly>

http://git-wip-us.apache.org/repos/asf/atlas/blob/2a560ac1/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 7db1be7..949167b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -661,6 +661,7 @@
         <module>addons/storm-bridge</module>
         <module>addons/hbase-bridge-shim</module>
         <module>addons/hbase-bridge</module>
+        <module>tools/atlas-migration-exporter</module>
        
         <module>distro</module>
     </modules>

http://git-wip-us.apache.org/repos/asf/atlas/blob/2a560ac1/tools/atlas-migration-exporter/pom.xml
----------------------------------------------------------------------
diff --git a/tools/atlas-migration-exporter/pom.xml 
b/tools/atlas-migration-exporter/pom.xml
new file mode 100644
index 0000000..5c6c61e
--- /dev/null
+++ b/tools/atlas-migration-exporter/pom.xml
@@ -0,0 +1,59 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/maven-v4_0_0.xsd";>
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.apache.atlas</groupId>
+        <artifactId>apache-atlas</artifactId>
+        <version>0.8.3-SNAPSHOT</version>
+    </parent>
+    <artifactId>atlas-migration</artifactId>
+    <description>Apache Atlas Migration Exporter</description>
+    <name>Apache Atlas Migration Exporter</name>
+    <packaging>jar</packaging>
+
+    <dependencies>
+
+        <dependency>
+            <groupId>commons-cli</groupId>
+            <artifactId>commons-cli</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-context</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-context</artifactId>
+            <version>4.3.8.RELEASE</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.atlas</groupId>
+            <artifactId>atlas-notification</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.atlas</groupId>
+            <artifactId>atlas-repository</artifactId>
+        </dependency>
+    </dependencies>
+
+</project>

http://git-wip-us.apache.org/repos/asf/atlas/blob/2a560ac1/tools/atlas-migration-exporter/src/main/java/org/apache/atlas/migration/Exporter.java
----------------------------------------------------------------------
diff --git 
a/tools/atlas-migration-exporter/src/main/java/org/apache/atlas/migration/Exporter.java
 
b/tools/atlas-migration-exporter/src/main/java/org/apache/atlas/migration/Exporter.java
new file mode 100644
index 0000000..a9873df
--- /dev/null
+++ 
b/tools/atlas-migration-exporter/src/main/java/org/apache/atlas/migration/Exporter.java
@@ -0,0 +1,213 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.migration;
+
+import org.apache.atlas.model.impexp.AtlasExportRequest;
+import org.apache.atlas.repository.impexp.ExportService;
+import org.apache.atlas.repository.impexp.ZipSink;
+import org.apache.atlas.type.AtlasType;
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
+public class Exporter {
+    private static final Logger LOG = LoggerFactory.getLogger(Exporter.class);
+
+    private static final String EXPORT_REQUEST_JSON_FILE = 
"migration-export-request.json";
+    private static final String ATLAS_EXPORT_SERVICE     = "exportService";
+    private static final String APPLICATION_CONTEXT      = 
"migrationContext.xml";
+    private static final int    PROGRAM_ERROR_STATUS     = -1;
+
+
+    private ApplicationContext applicationContext;
+
+
+    public static void main(String args[]) {
+        int result = PROGRAM_ERROR_STATUS;
+
+        try {
+            display("=== Atlas Migration: Export === >>");
+            String fileName = getExportToFileName(args);
+
+            Exporter exporter = new Exporter(APPLICATION_CONTEXT);
+
+            result = exporter.perform(fileName);
+
+            display("<< === Atlas Migration: Export: Done! ===");
+        } catch (Exception e) {
+            LOG.error("<=== Atlas Migration: Export: Failed! ===", e);
+
+            result = PROGRAM_ERROR_STATUS;
+        }
+
+        System.exit(result);
+    }
+
+    public Exporter(String contextXml) {
+        try {
+            applicationContext = new 
ClassPathXmlApplicationContext(contextXml);
+        } catch (Exception ex) {
+            LOG.error("Initialization failed!", ex);
+
+            throw ex;
+        }
+    }
+
+    public int perform(String fileName) {
+        LOG.info("Starting export to {}", fileName);
+
+        int                ret = 0;
+        AtlasExportRequest req = getRequest(getExportRequestFile(), 
getDefaultExportRequest());
+        OutputStream       os  = null;
+        ZipSink            zs  = null;
+
+        try {
+            os = new FileOutputStream(fileName);
+            zs = new ZipSink(os);
+
+            ExportService svc = getExportService();
+
+            svc.run(zs, req, getUserName(), getHostName(), getIPAddress());
+
+            ret = 0;
+        } catch (Exception ex) {
+            LOG.error("Export failed!", ex);
+
+            ret = PROGRAM_ERROR_STATUS;
+        } finally {
+            if (zs != null) {
+                try {
+                    zs.close();
+                } catch (Throwable t) {
+                    // ignore
+                }
+            }
+
+            if (os != null) {
+                try {
+                    os.close();
+                } catch (Throwable t) {
+                    // ignore
+                }
+            }
+        }
+
+        return ret;
+    }
+
+    private AtlasExportRequest getRequest(File requestFile, String 
defaultJson) {
+        String reqJson = null;
+
+        try {
+            if (requestFile.exists()) {
+                LOG.info("Using request from the file {}", 
requestFile.getPath());
+
+                reqJson = FileUtils.readFileToString(requestFile);
+            } else {
+                LOG.info("Using default request...");
+
+                reqJson = defaultJson;
+            }
+        } catch (IOException e) {
+            LOG.error("Error reading request from {}", requestFile.getPath());
+
+            reqJson = defaultJson;
+        }
+
+        LOG.info("Export request: {}", reqJson);
+
+        return AtlasType.fromJson(reqJson, AtlasExportRequest.class);
+    }
+
+    private ExportService getExportService() {
+        return applicationContext.getBean(ATLAS_EXPORT_SERVICE, 
ExportService.class);
+    }
+
+    private String getUserName() {
+        return System.getProperty("user.name");
+    }
+
+    private String getHostName() {
+        try {
+            return InetAddress.getLocalHost().getHostName();
+        } catch (UnknownHostException e) {
+            LOG.error("faild to get hostname; using localhost", e);
+
+            return "localhost";
+        }
+    }
+
+    private String getIPAddress() {
+        try {
+            return InetAddress.getLocalHost().toString();
+        } catch (UnknownHostException e) {
+            LOG.error("failed to get IP address; using 127.0.0.1", e);
+
+            return "127.0.0.1";
+        }
+    }
+
+    private File getExportRequestFile() {
+        return getFile(getCurrentDirectory(), EXPORT_REQUEST_JSON_FILE);
+    }
+
+    private File getFile(String currentDir, String fileName) {
+        LOG.info("Attempting to use request file: {}/{}", currentDir, 
fileName);
+        return new File(currentDir, fileName);
+    }
+
+    private String getCurrentDirectory() {
+        return System.getProperty("user.dir");
+    }
+
+    private String getDefaultExportRequest() {
+        return "{ \"itemsToExport\": [ { \"typeName\": \"hive_db\" } ], 
\"options\": {  \"fetchType\": \"FULL\", \"matchType\": \"forType\"} }";
+    }
+
+    private static void display(String s) {
+        LOG.info(s);
+    }
+
+    private static String getExportToFileName(String[] args) {
+        String fileName = (args.length > 0) ? args[0] : getDefaultFileName();
+
+        if (args.length == 0) {
+            printUsage(fileName);
+        }
+
+        return fileName;
+    }
+
+    private static void printUsage(String fileName) {
+        display("Exporting to file " + fileName + ". To export data to a 
different file, please specify the file path as argument");
+    }
+
+    private static String getDefaultFileName() {
+        return String.format("atlas-export-%s.zip", 
System.currentTimeMillis());
+    }
+}

http://git-wip-us.apache.org/repos/asf/atlas/blob/2a560ac1/tools/atlas-migration-exporter/src/main/java/org/apache/atlas/migration/NoOpNotification.java
----------------------------------------------------------------------
diff --git 
a/tools/atlas-migration-exporter/src/main/java/org/apache/atlas/migration/NoOpNotification.java
 
b/tools/atlas-migration-exporter/src/main/java/org/apache/atlas/migration/NoOpNotification.java
new file mode 100644
index 0000000..a8301ae
--- /dev/null
+++ 
b/tools/atlas-migration-exporter/src/main/java/org/apache/atlas/migration/NoOpNotification.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.migration;
+
+import org.apache.atlas.notification.NotificationConsumer;
+import org.apache.atlas.notification.NotificationException;
+import org.apache.atlas.notification.NotificationInterface;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+
+@Component
+public class NoOpNotification implements NotificationInterface {
+    @Override
+    public void setCurrentUser(String user) {
+
+    }
+
+    @Override
+    public <T> List<NotificationConsumer<T>> createConsumers(NotificationType 
notificationType, int numConsumers) {
+        return null;
+    }
+
+    @Override
+    public <T> void send(NotificationType type, T... messages) throws 
NotificationException {
+
+    }
+
+    @Override
+    public <T> void send(NotificationType type, List<T> messages) throws 
NotificationException {
+
+    }
+
+    @Override
+    public void close() {
+
+    }
+}

http://git-wip-us.apache.org/repos/asf/atlas/blob/2a560ac1/tools/atlas-migration-exporter/src/main/java/org/apache/atlas/migration/NoOpNotificationChangeListener.java
----------------------------------------------------------------------
diff --git 
a/tools/atlas-migration-exporter/src/main/java/org/apache/atlas/migration/NoOpNotificationChangeListener.java
 
b/tools/atlas-migration-exporter/src/main/java/org/apache/atlas/migration/NoOpNotificationChangeListener.java
new file mode 100644
index 0000000..7ee2048
--- /dev/null
+++ 
b/tools/atlas-migration-exporter/src/main/java/org/apache/atlas/migration/NoOpNotificationChangeListener.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.migration;
+
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.listener.EntityChangeListener;
+import org.apache.atlas.typesystem.IStruct;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.springframework.stereotype.Component;
+
+import java.util.Collection;
+import java.util.List;
+
+@Component
+public class NoOpNotificationChangeListener implements EntityChangeListener {
+    @Override
+    public void onEntitiesAdded(Collection<ITypedReferenceableInstance> 
entities, boolean isImport) throws AtlasException {
+
+    }
+
+    @Override
+    public void onEntitiesUpdated(Collection<ITypedReferenceableInstance> 
entities, boolean isImport) throws AtlasException {
+
+    }
+
+    @Override
+    public void onTraitsAdded(ITypedReferenceableInstance entity, Collection<? 
extends IStruct> traits) throws AtlasException {
+
+    }
+
+    @Override
+    public void onTraitsDeleted(ITypedReferenceableInstance entity, 
Collection<String> traitNames) throws AtlasException {
+
+    }
+
+    @Override
+    public void onTraitsUpdated(ITypedReferenceableInstance entity, 
Collection<? extends IStruct> traits) throws AtlasException {
+
+    }
+
+    @Override
+    public void onEntitiesDeleted(Collection<ITypedReferenceableInstance> 
entities, boolean isImport) throws AtlasException {
+
+    }
+}

http://git-wip-us.apache.org/repos/asf/atlas/blob/2a560ac1/tools/atlas-migration-exporter/src/main/resources/README
----------------------------------------------------------------------
diff --git a/tools/atlas-migration-exporter/src/main/resources/README 
b/tools/atlas-migration-exporter/src/main/resources/README
new file mode 100644
index 0000000..2f2bf3e
--- /dev/null
+++ b/tools/atlas-migration-exporter/src/main/resources/README
@@ -0,0 +1,41 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+Introduction
+  The purpose of this utility is to export data from Atlas to a ZIP file.
+
+What is Exported?
+  Metadata of all Hive databases and all entities connected to these databases 
(tables, columns, processes, paths) are
+  exported. To change the scope of export, modify the accompanying 
migration-export-request.json. More details of this
+  JSON file format can be found in http://atlas.apache.org/Export-API.html
+
+How Much Time Will this Take?
+  The duration of the export process depends on the number of entities present 
in your database. Approximately it takes
+  about 2.5ms for each entity in the database. For a database with 1 million 
entities, export will take about 40 minutes.
+
+Steps to Start Export step of Migration
+  - Shutdown Atlas. This is critical to ensure that no updates are being made 
to Atlas database while export is in progress.
+  - Execute the following commands in the host where Atlas server runs:
+    - unzip atlas-migration-kit.zip
+    - cd atlas-migration-kit
+    - python atlas_migration.py atlas-export.zip
+      if zip file path is no specified, exported data will be saved to a file 
named atlas-export-<timestamp>.zip
+  - To watch the progress: 'tail -f /var/log/atlas/application.log'.
+
+Next Steps
+  Once done, please use the Atlas Migration Guide for next steps.

http://git-wip-us.apache.org/repos/asf/atlas/blob/2a560ac1/tools/atlas-migration-exporter/src/main/resources/atlas_migration.py
----------------------------------------------------------------------
diff --git 
a/tools/atlas-migration-exporter/src/main/resources/atlas_migration.py 
b/tools/atlas-migration-exporter/src/main/resources/atlas_migration.py
new file mode 100755
index 0000000..199cde2
--- /dev/null
+++ b/tools/atlas-migration-exporter/src/main/resources/atlas_migration.py
@@ -0,0 +1,141 @@
+#!/usr/bin/env python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+import sys
+sys.path.insert(0, '/usr/hdp/current/atlas-server/bin/')
+
+import traceback
+import atlas_config as mc
+
+ATLAS_LOG_OPTS="-Datlas.log.dir=%s -Datlas.log.file=%s.log"
+ATLAS_COMMAND_OPTS="-Datlas.home=%s"
+ATLAS_CONFIG_OPTS="-Datlas.conf=%s"
+DEFAULT_JVM_HEAP_OPTS="-Xmx4096m -XX:MaxPermSize=512m"
+DEFAULT_JVM_OPTS="-Dlog4j.configuration=atlas-log4j.xml 
-Djava.net.preferIPv4Stack=true -server"
+
+def main():
+    is_setup = (len(sys.argv)>1) and sys.argv[1] is not None and sys.argv[1] 
== '-setup'
+    atlas_home = mc.atlasDir()
+    confdir = mc.dirMustExist(mc.confDir(atlas_home))
+    mc.executeEnvSh(confdir)
+    logdir = mc.dirMustExist(mc.logDir(atlas_home))
+    mc.dirMustExist(mc.dataDir(atlas_home))
+
+    if mc.isCygwin():
+        # Pathnames that are passed to JVM must be converted to Windows format.
+        jvm_atlas_home = mc.convertCygwinPath(atlas_home)
+        jvm_confdir = mc.convertCygwinPath(confdir)
+        jvm_logdir = mc.convertCygwinPath(logdir)
+    else:
+        jvm_atlas_home = atlas_home
+        jvm_confdir = confdir
+        jvm_logdir = logdir
+
+    #create sys property for conf dirs
+    if not is_setup:
+        jvm_opts_list = (ATLAS_LOG_OPTS % (jvm_logdir, "application")).split()
+    else:
+        jvm_opts_list = (ATLAS_LOG_OPTS % (jvm_logdir, "atlas_setup")).split()
+
+    cmd_opts = (ATLAS_COMMAND_OPTS % jvm_atlas_home)
+    jvm_opts_list.extend(cmd_opts.split())
+
+    config_opts = (ATLAS_CONFIG_OPTS % jvm_confdir)
+    jvm_opts_list.extend(config_opts.split())
+
+    atlas_server_heap_opts = os.environ.get(mc.ATLAS_SERVER_HEAP, 
DEFAULT_JVM_HEAP_OPTS)
+    jvm_opts_list.extend(atlas_server_heap_opts.split())
+
+    atlas_server_jvm_opts = os.environ.get(mc.ATLAS_SERVER_OPTS)
+    if atlas_server_jvm_opts:
+        jvm_opts_list.extend(atlas_server_jvm_opts.split())
+
+    atlas_jvm_opts = os.environ.get(mc.ATLAS_OPTS, DEFAULT_JVM_OPTS)
+    jvm_opts_list.extend(atlas_jvm_opts.split())
+
+    #expand web app dir
+    web_app_dir = mc.webAppDir(atlas_home)
+    mc.expandWebApp(atlas_home)
+
+    p = os.pathsep
+    atlas_classpath = confdir + p \
+                       + os.path.join(web_app_dir, "atlas", "WEB-INF", 
"classes" ) + p \
+                       + os.path.join(web_app_dir, "atlas", "WEB-INF", "lib", 
"*" )  + p \
+                       + os.path.join(atlas_home, "libext", "*") + p \
+                       + os.path.join(os.getcwd(), ".", "*")
+
+    is_hbase = mc.is_hbase(confdir)
+
+    if is_hbase:
+        #add hbase-site.xml to classpath
+        hbase_conf_dir = mc.hbaseConfDir(atlas_home)
+
+        if os.path.exists(hbase_conf_dir):
+            atlas_classpath = atlas_classpath + p \
+                            + hbase_conf_dir
+        else:
+            if mc.is_hbase(confdir):
+                raise Exception("Could not find hbase-site.xml in %s. Please 
set env var HBASE_CONF_DIR to the hbase client conf dir", hbase_conf_dir)
+
+    if mc.isCygwin():
+        atlas_classpath = mc.convertCygwinPath(atlas_classpath, True)
+
+    atlas_pid_file = mc.pidFile(atlas_home)
+
+    if os.path.isfile(atlas_pid_file):
+       #Check if process listed in atlas.pid file is still running
+       pf = file(atlas_pid_file, 'r')
+       pid = pf.read().strip()
+       pf.close()
+
+       if mc.exist_pid((int)(pid)):
+           mc.server_already_running(pid)
+       else:
+           mc.server_pid_not_running(pid)
+
+    if is_hbase and mc.is_hbase_local(confdir):
+        print "configured for local hbase."
+        mc.configure_hbase(atlas_home)
+        mc.run_hbase_action(mc.hbaseBinDir(atlas_home), "start", 
hbase_conf_dir, logdir)
+        print "hbase started."
+
+    web_app_path = os.path.join(web_app_dir, "atlas")
+    if (mc.isCygwin()):
+        web_app_path = mc.convertCygwinPath(web_app_path)
+
+    start_atlas_server(atlas_classpath, atlas_pid_file, jvm_logdir, 
jvm_opts_list, web_app_path)
+    print "Apache Atlas Migration started!\n"
+
+def start_atlas_server(atlas_classpath, atlas_pid_file, jvm_logdir, 
jvm_opts_list, web_app_path):
+    args = []
+    args.extend(sys.argv[1:])
+    print "Arguments: ".join(str(x) for x in args) 
+    process = mc.java("org.apache.atlas.migration.Exporter", args, 
atlas_classpath, jvm_opts_list, jvm_logdir)
+    print "To watch the progress use: tail -f /var/log/atlas/application.log"
+    mc.writePid(atlas_pid_file, process)
+
+
+if __name__ == '__main__':
+    try:
+        returncode = main()
+    except Exception as e:
+        print "Exception: %s " % str(e)
+        print traceback.format_exc()
+        returncode = -1
+
+    sys.exit(returncode)

http://git-wip-us.apache.org/repos/asf/atlas/blob/2a560ac1/tools/atlas-migration-exporter/src/main/resources/migration-export-request.json
----------------------------------------------------------------------
diff --git 
a/tools/atlas-migration-exporter/src/main/resources/migration-export-request.json
 
b/tools/atlas-migration-exporter/src/main/resources/migration-export-request.json
new file mode 100644
index 0000000..64002af
--- /dev/null
+++ 
b/tools/atlas-migration-exporter/src/main/resources/migration-export-request.json
@@ -0,0 +1,4 @@
+{
+  "itemsToExport": [ { "typeName": "hive_db" } ],
+  "options": { "fetchType": "FULL", "matchType": "forType" }
+}

http://git-wip-us.apache.org/repos/asf/atlas/blob/2a560ac1/tools/atlas-migration-exporter/src/main/resources/migrationContext.xml
----------------------------------------------------------------------
diff --git 
a/tools/atlas-migration-exporter/src/main/resources/migrationContext.xml 
b/tools/atlas-migration-exporter/src/main/resources/migrationContext.xml
new file mode 100644
index 0000000..c0f9659
--- /dev/null
+++ b/tools/atlas-migration-exporter/src/main/resources/migrationContext.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more 
contributor
+    license agreements. See the NOTICE file distributed with this work for 
additional
+    information regarding copyright ownership. The ASF licenses this file to
+    You under the Apache License, Version 2.0 (the "License"); you may not use
+    this file except in compliance with the License. You may obtain a copy of
+    the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+    by applicable law or agreed to in writing, software distributed under the
+    License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 
CONDITIONS
+    OF ANY KIND, either express or implied. See the License for the specific
+    language governing permissions and limitations under the License. -->
+
+<beans xmlns="http://www.springframework.org/schema/beans";
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+       xmlns:context="http://www.springframework.org/schema/context";
+       xmlns:aop="http://www.springframework.org/schema/aop";
+       xsi:schemaLocation="http://www.springframework.org/schema/beans
+       http://www.springframework.org/schema/beans/spring-beans.xsd 
http://www.springframework.org/schema/context 
http://www.springframework.org/schema/context/spring-context.xsd 
http://www.springframework.org/schema/aop 
http://www.springframework.org/schema/aop/spring-aop.xsd";>
+
+    <context:annotation-config/>
+    <aop:config proxy-target-class="true"/>
+
+    <context:component-scan base-package="org.apache.atlas">
+        <context:exclude-filter type="regex" 
expression="org\.apache\.atlas\.service\.ActiveInstanceElectorService.*"/>
+        <context:exclude-filter type="regex" 
expression="org\.apache\.atlas\.service\.EmbeddedKafkaServer.*"/>
+        <!--<context:exclude-filter type="regex" 
expression="org\.apache\.atlas\.service\.HBaseBasedAuditRepository.*"/>-->
+        <!-- for non-HBase setups comment the InMemoryEntityAuditRepository 
and comment the HBaseBasedAuditoryRepository -->
+        <context:exclude-filter type="regex" 
expression="org\.apache\.atlas\.service\.InMemoryEntityAuditRepository.*"/>
+        <context:exclude-filter type="regex" 
expression="org\.apache\.atlas\.service\.NoopEntityAuditRepository.*"/>
+        <context:exclude-filter type="regex" 
expression="org\.apache\.atlas\.service\.KafkaNotification.*"/>
+        <context:exclude-filter type="regex" 
expression="org\.apache\.atlas\.service\.NotificationHookConsumer.*"/>
+        <context:exclude-filter type="regex" 
expression="org\.apache\.atlas\.kafka.*"/>
+        <context:exclude-filter type="regex" 
expression="org\.apache\.atlas\.webapp.*"/>
+        <context:exclude-filter type="regex" 
expression="org\.apache\.atlas\.web.*"/>
+        <context:exclude-filter type="regex" 
expression="org\.apache\.atlas\.notification.hook.*"/>
+        <context:exclude-filter type="regex" 
expression="org\.apache\.atlas\.notification.entity.*"/>
+        <context:exclude-filter type="regex" 
expression="org\.apache\.atlas\.notification.NotificationHookConsumer.*"/>
+        <context:exclude-filter type="regex" 
expression="org\.apache\.atlas\.ha.*"/>
+    </context:component-scan>
+</beans>

Reply via email to