Repository: incubator-atlas Updated Branches: refs/heads/master b9ce7a111 -> 1ee2c1bc4
ATLAS-987 Atlas hooks should avoid adding dependent libraries to component CLASSPATH (madhan.neethiraj via shwethags) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/043d2aa6 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/043d2aa6 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/043d2aa6 Branch: refs/heads/master Commit: 043d2aa642e695c7c61c0b771d978fc352caf203 Parents: b9ce7a1 Author: Shwetha GS <[email protected]> Authored: Fri Jul 8 11:31:10 2016 +0530 Committer: Shwetha GS <[email protected]> Committed: Fri Jul 8 11:31:10 2016 +0530 ---------------------------------------------------------------------- addons/falcon-bridge-shim/pom.xml | 52 +++ .../atlas/falcon/service/AtlasService.java | 215 ++++++++++++ addons/falcon-bridge/pom.xml | 44 ++- addons/hive-bridge-shim/pom.xml | 53 +++ .../org/apache/atlas/hive/hook/HiveHook.java | 99 ++++++ addons/hive-bridge/pom.xml | 60 ++-- addons/sqoop-bridge-shim/pom.xml | 52 +++ .../org/apache/atlas/sqoop/hook/SqoopHook.java | 98 ++++++ addons/sqoop-bridge/pom.xml | 61 +++- addons/storm-bridge-shim/pom.xml | 58 ++++ .../apache/atlas/storm/hook/StormAtlasHook.java | 104 ++++++ addons/storm-bridge/pom.xml | 44 ++- plugin-classloader/pom.xml | 42 +++ .../classloader/AtlasPluginClassLoader.java | 327 +++++++++++++++++++ .../classloader/AtlasPluginClassLoaderUtil.java | 115 +++++++ .../classloader/AtlasPluginClassLoaderTest.java | 66 ++++ pom.xml | 40 ++- release-log.txt | 1 + 18 files changed, 1489 insertions(+), 42 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/043d2aa6/addons/falcon-bridge-shim/pom.xml ---------------------------------------------------------------------- diff --git a/addons/falcon-bridge-shim/pom.xml b/addons/falcon-bridge-shim/pom.xml new file mode 100755 index 0000000..ddb62ce --- /dev/null +++ b/addons/falcon-bridge-shim/pom.xml @@ -0,0 +1,52 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <artifactId>apache-atlas</artifactId> + <groupId>org.apache.atlas</groupId> + <version>0.8-incubating-SNAPSHOT</version> + <relativePath>../../</relativePath> + </parent> + <artifactId>falcon-bridge-shim</artifactId> + <description>Apache Atlas Falcon Bridge Shim Module</description> + <name>Apache Atlas Falcon Bridge Shim</name> + <packaging>jar</packaging> + + <properties> + <falcon.version>0.8</falcon.version> + <checkstyle.failOnViolation>true</checkstyle.failOnViolation> + </properties> + + <dependencies> + <!-- Logging --> + <dependency> + <groupId>org.apache.atlas</groupId> + <artifactId>atlas-plugin-classloader</artifactId> + </dependency> + + <dependency> + <groupId>org.apache.falcon</groupId> + <artifactId>falcon-common</artifactId> + <version>${falcon.version}</version> + <scope>provided</scope> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/043d2aa6/addons/falcon-bridge-shim/src/main/java/org/apache/atlas/falcon/service/AtlasService.java ---------------------------------------------------------------------- diff --git a/addons/falcon-bridge-shim/src/main/java/org/apache/atlas/falcon/service/AtlasService.java b/addons/falcon-bridge-shim/src/main/java/org/apache/atlas/falcon/service/AtlasService.java new file mode 100755 index 0000000..28607fc --- /dev/null +++ b/addons/falcon-bridge-shim/src/main/java/org/apache/atlas/falcon/service/AtlasService.java @@ -0,0 +1,215 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.falcon.service; + + +import org.apache.atlas.plugin.classloader.AtlasPluginClassLoader; +import org.apache.falcon.FalconException; +import org.apache.falcon.entity.v0.Entity; +import org.apache.falcon.service.ConfigurationChangeListener; +import org.apache.falcon.service.FalconService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Falcon hook used for atlas entity registration. + */ +public class AtlasService implements FalconService, ConfigurationChangeListener { + private static final Logger LOG = LoggerFactory.getLogger(AtlasService.class); + + private static final String ATLAS_PLUGIN_TYPE = "falcon"; + private static final String ATLAS_FALCON_HOOK_IMPL_CLASSNAME = "org.apache.atlas.falcon.hook.AtlasService"; + + private AtlasPluginClassLoader atlasPluginClassLoader = null; + private FalconService falconServiceImpl = null; + private ConfigurationChangeListener configChangeListenerImpl = null; + + public AtlasService() { + this.initialize(); + } + + @Override + public String getName() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasService.getName()"); + } + + String ret = null; + + try { + activatePluginClassLoader(); + ret = falconServiceImpl.getName(); + } finally { + deactivatePluginClassLoader(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasService.getName()"); + } + + return ret; + } + + @Override + public void init() throws FalconException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasService.init()"); + } + + try { + activatePluginClassLoader(); + falconServiceImpl.init(); + } finally { + deactivatePluginClassLoader(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasService.init()"); + } + } + + @Override + public void destroy() throws FalconException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasService.destroy()"); + } + + try { + activatePluginClassLoader(); + falconServiceImpl.destroy(); + } finally { + deactivatePluginClassLoader(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasService.destroy()"); + } + } + + @Override + public void onAdd(Entity entity) throws FalconException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasService.onAdd(" + entity + ")"); + } + + try { + activatePluginClassLoader(); + configChangeListenerImpl.onAdd(entity); + } finally { + deactivatePluginClassLoader(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasService.onAdd(" + entity + ")"); + } + } + + @Override + public void onRemove(Entity entity) throws FalconException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasService.onRemove(" + entity + ")"); + } + + try { + activatePluginClassLoader(); + configChangeListenerImpl.onRemove(entity); + } finally { + deactivatePluginClassLoader(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasService.onRemove(" + entity + ")"); + } + } + + @Override + public void onChange(Entity entity, Entity entity1) throws FalconException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasService.onChange(" + entity + ", " + entity1 + ")"); + } + + try { + activatePluginClassLoader(); + configChangeListenerImpl.onChange(entity, entity1); + } finally { + deactivatePluginClassLoader(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasService.onChange(" + entity + ", " + entity1 + ")"); + } + } + + @Override + public void onReload(Entity entity) throws FalconException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasService.onReload(" + entity + ")"); + } + + try { + activatePluginClassLoader(); + configChangeListenerImpl.onReload(entity); + } finally { + deactivatePluginClassLoader(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasService.onReload(" + entity + ")"); + } + } + + private void initialize() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasService.initialize()"); + } + + try { + atlasPluginClassLoader = AtlasPluginClassLoader.getInstance(ATLAS_PLUGIN_TYPE, this.getClass()); + + Class<?> cls = Class.forName(ATLAS_FALCON_HOOK_IMPL_CLASSNAME, true, atlasPluginClassLoader); + + activatePluginClassLoader(); + + Object atlasService = cls.newInstance(); + + falconServiceImpl = (FalconService) atlasService; + configChangeListenerImpl = (ConfigurationChangeListener) atlasService; + } catch (Exception excp) { + LOG.error("Error instantiating Atlas hook implementation", excp); + } finally { + deactivatePluginClassLoader(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasService.initialize()"); + } + } + + private void activatePluginClassLoader() { + if (atlasPluginClassLoader != null) { + atlasPluginClassLoader.activate(); + } + } + + private void deactivatePluginClassLoader() { + if (atlasPluginClassLoader != null) { + atlasPluginClassLoader.deactivate(); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/043d2aa6/addons/falcon-bridge/pom.xml ---------------------------------------------------------------------- diff --git a/addons/falcon-bridge/pom.xml b/addons/falcon-bridge/pom.xml index d79dda9..064fbf6 100644 --- a/addons/falcon-bridge/pom.xml +++ b/addons/falcon-bridge/pom.xml @@ -99,13 +99,13 @@ <artifactId>maven-dependency-plugin</artifactId> <executions> <execution> - <id>copy-hook-dependencies</id> + <id>copy-hook</id> <phase>package</phase> <goals> <goal>copy</goal> </goals> <configuration> - <outputDirectory>${project.build.directory}/dependency/hook/falcon</outputDirectory> + <outputDirectory>${project.build.directory}/dependency/hook/falcon/atlas-falcon-plugin-impl</outputDirectory> <overWriteReleases>false</overWriteReleases> <overWriteSnapshots>false</overWriteSnapshots> <overWriteIfNewer>true</overWriteIfNewer> @@ -195,6 +195,46 @@ <artifactId>kafka-clients</artifactId> <version>${kafka.version}</version> </artifactItem> + <artifactItem> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <version>${hadoop.version}</version> + </artifactItem> + <artifactItem> + <groupId>com.google.code.gson</groupId> + <artifactId>gson</artifactId> + <version>${gson.version}</version> + </artifactItem> + <artifactItem> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + <version>${log4j.version}</version> + </artifactItem> + </artifactItems> + </configuration> + </execution> + <execution> + <id>copy-hook-shim</id> + <phase>package</phase> + <goals> + <goal>copy</goal> + </goals> + <configuration> + <outputDirectory>${project.build.directory}/dependency/hook/falcon</outputDirectory> + <overWriteReleases>false</overWriteReleases> + <overWriteSnapshots>false</overWriteSnapshots> + <overWriteIfNewer>true</overWriteIfNewer> + <artifactItems> + <artifactItem> + <groupId>${project.groupId}</groupId> + <artifactId>falcon-bridge-shim</artifactId> + <version>${project.version}</version> + </artifactItem> + <artifactItem> + <groupId>${project.groupId}</groupId> + <artifactId>atlas-plugin-classloader</artifactId> + <version>${project.version}</version> + </artifactItem> </artifactItems> </configuration> </execution> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/043d2aa6/addons/hive-bridge-shim/pom.xml ---------------------------------------------------------------------- diff --git a/addons/hive-bridge-shim/pom.xml b/addons/hive-bridge-shim/pom.xml new file mode 100755 index 0000000..c699b29 --- /dev/null +++ b/addons/hive-bridge-shim/pom.xml @@ -0,0 +1,53 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <artifactId>apache-atlas</artifactId> + <groupId>org.apache.atlas</groupId> + <version>0.8-incubating-SNAPSHOT</version> + <relativePath>../../</relativePath> + </parent> + <artifactId>hive-bridge-shim</artifactId> + <description>Apache Atlas Hive Bridge Shim Module</description> + <name>Apache Atlas Hive Bridge Shim</name> + <packaging>jar</packaging> + + <properties> + <hive.version>1.2.1</hive.version> + <calcite.version>0.9.2-incubating</calcite.version> + <checkstyle.failOnViolation>true</checkstyle.failOnViolation> + </properties> + + <dependencies> + <!-- Logging --> + <dependency> + <groupId>org.apache.atlas</groupId> + <artifactId>atlas-plugin-classloader</artifactId> + </dependency> + + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-exec</artifactId> + <version>${hive.version}</version> + <scope>provided</scope> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/043d2aa6/addons/hive-bridge-shim/src/main/java/org/apache/atlas/hive/hook/HiveHook.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge-shim/src/main/java/org/apache/atlas/hive/hook/HiveHook.java b/addons/hive-bridge-shim/src/main/java/org/apache/atlas/hive/hook/HiveHook.java new file mode 100755 index 0000000..f4ca94f --- /dev/null +++ b/addons/hive-bridge-shim/src/main/java/org/apache/atlas/hive/hook/HiveHook.java @@ -0,0 +1,99 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.hive.hook; + + +import org.apache.atlas.plugin.classloader.AtlasPluginClassLoader; +import org.apache.hadoop.hive.ql.hooks.ExecuteWithHookContext; +import org.apache.hadoop.hive.ql.hooks.HookContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Hive hook used for atlas entity registration. + */ +public class HiveHook implements ExecuteWithHookContext { + private static final Logger LOG = LoggerFactory.getLogger(HiveHook.class); + + private static final String ATLAS_PLUGIN_TYPE = "hive"; + private static final String ATLAS_HIVE_HOOK_IMPL_CLASSNAME = "org.apache.atlas.hive.hook.HiveHook"; + + private AtlasPluginClassLoader atlasPluginClassLoader = null; + private ExecuteWithHookContext hiveHookImpl = null; + + public HiveHook() { + this.initialize(); + } + + @Override + public void run(final HookContext hookContext) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> HiveHook.run(" + hookContext + ")"); + } + + try { + activatePluginClassLoader(); + hiveHookImpl.run(hookContext); + } finally { + deactivatePluginClassLoader(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== HiveHook.run(" + hookContext + ")"); + } + } + + private void initialize() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> HiveHook.initialize()"); + } + + try { + atlasPluginClassLoader = AtlasPluginClassLoader.getInstance(ATLAS_PLUGIN_TYPE, this.getClass()); + + @SuppressWarnings("unchecked") + Class<ExecuteWithHookContext> cls = (Class<ExecuteWithHookContext>) Class + .forName(ATLAS_HIVE_HOOK_IMPL_CLASSNAME, true, atlasPluginClassLoader); + + activatePluginClassLoader(); + + hiveHookImpl = cls.newInstance(); + } catch (Exception excp) { + LOG.error("Error instantiating Atlas hook implementation", excp); + } finally { + deactivatePluginClassLoader(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== HiveHook.initialize()"); + } + } + + private void activatePluginClassLoader() { + if (atlasPluginClassLoader != null) { + atlasPluginClassLoader.activate(); + } + } + + private void deactivatePluginClassLoader() { + if (atlasPluginClassLoader != null) { + atlasPluginClassLoader.deactivate(); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/043d2aa6/addons/hive-bridge/pom.xml ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/pom.xml b/addons/hive-bridge/pom.xml index ddefdc2..2d0fb14 100755 --- a/addons/hive-bridge/pom.xml +++ b/addons/hive-bridge/pom.xml @@ -168,25 +168,7 @@ <goal>copy</goal> </goals> <configuration> - <artifactItems> - <artifactItem> - <groupId>${project.groupId}</groupId> - <artifactId>${project.artifactId}</artifactId> - <version>${project.version}</version> - <overWrite>true</overWrite> - <outputDirectory>${project.build.directory}/dependency/hook/hive</outputDirectory> - </artifactItem> - </artifactItems> - </configuration> - </execution> - <execution> - <id>copy-hook-dependencies</id> - <phase>package</phase> - <goals> - <goal>copy</goal> - </goals> - <configuration> - <outputDirectory>${project.build.directory}/dependency/hook/hive</outputDirectory> + <outputDirectory>${project.build.directory}/dependency/hook/hive/atlas-hive-plugin-impl</outputDirectory> <overWriteReleases>false</overWriteReleases> <overWriteSnapshots>false</overWriteSnapshots> <overWriteIfNewer>true</overWriteIfNewer> @@ -271,6 +253,46 @@ <artifactId>kafka-clients</artifactId> <version>${kafka.version}</version> </artifactItem> + <artifactItem> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <version>${hadoop.version}</version> + </artifactItem> + <artifactItem> + <groupId>com.google.code.gson</groupId> + <artifactId>gson</artifactId> + <version>${gson.version}</version> + </artifactItem> + <artifactItem> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + <version>${log4j.version}</version> + </artifactItem> + </artifactItems> + </configuration> + </execution> + <execution> + <id>copy-hook-shim</id> + <phase>package</phase> + <goals> + <goal>copy</goal> + </goals> + <configuration> + <outputDirectory>${project.build.directory}/dependency/hook/hive</outputDirectory> + <overWriteReleases>false</overWriteReleases> + <overWriteSnapshots>false</overWriteSnapshots> + <overWriteIfNewer>true</overWriteIfNewer> + <artifactItems> + <artifactItem> + <groupId>${project.groupId}</groupId> + <artifactId>hive-bridge-shim</artifactId> + <version>${project.version}</version> + </artifactItem> + <artifactItem> + <groupId>${project.groupId}</groupId> + <artifactId>atlas-plugin-classloader</artifactId> + <version>${project.version}</version> + </artifactItem> </artifactItems> </configuration> </execution> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/043d2aa6/addons/sqoop-bridge-shim/pom.xml ---------------------------------------------------------------------- diff --git a/addons/sqoop-bridge-shim/pom.xml b/addons/sqoop-bridge-shim/pom.xml new file mode 100755 index 0000000..efb5a11 --- /dev/null +++ b/addons/sqoop-bridge-shim/pom.xml @@ -0,0 +1,52 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <artifactId>apache-atlas</artifactId> + <groupId>org.apache.atlas</groupId> + <version>0.8-incubating-SNAPSHOT</version> + <relativePath>../../</relativePath> + </parent> + <artifactId>sqoop-bridge-shim</artifactId> + <description>Apache Atlas Sqoop Bridge Shim Module</description> + <name>Apache Atlas Sqoop Bridge Shim</name> + <packaging>jar</packaging> + + <properties> + <sqoop.version>1.4.6.2.3.99.0-195</sqoop.version> + <checkstyle.failOnViolation>true</checkstyle.failOnViolation> + </properties> + + <dependencies> + <!-- Logging --> + <dependency> + <groupId>org.apache.atlas</groupId> + <artifactId>atlas-plugin-classloader</artifactId> + </dependency> + + <dependency> + <groupId>org.apache.sqoop</groupId> + <artifactId>sqoop</artifactId> + <version>${sqoop.version}</version> + <scope>compile</scope> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/043d2aa6/addons/sqoop-bridge-shim/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java ---------------------------------------------------------------------- diff --git a/addons/sqoop-bridge-shim/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java b/addons/sqoop-bridge-shim/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java new file mode 100644 index 0000000..1170cb6 --- /dev/null +++ b/addons/sqoop-bridge-shim/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java @@ -0,0 +1,98 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.sqoop.hook; + + +import org.apache.atlas.plugin.classloader.AtlasPluginClassLoader; +import org.apache.sqoop.SqoopJobDataPublisher; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Sqoop hook used for atlas entity registration. + */ +public class SqoopHook extends SqoopJobDataPublisher { + private static final Logger LOG = LoggerFactory.getLogger(SqoopHook.class); + + private static final String ATLAS_PLUGIN_TYPE = "sqoop"; + private static final String ATLAS_SQOOP_HOOK_IMPL_CLASSNAME = "org.apache.atlas.sqoop.hook.SqoopHook"; + + private AtlasPluginClassLoader atlasPluginClassLoader = null; + private SqoopJobDataPublisher sqoopHookImpl = null; + + public SqoopHook() { + this.initialize(); + } + + @Override + public void publish(SqoopJobDataPublisher.Data data) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> SqoopHook.run(" + data + ")"); + } + + try { + activatePluginClassLoader(); + sqoopHookImpl.publish(data); + } finally { + deactivatePluginClassLoader(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== SqoopHook.run(" + data + ")"); + } + } + + private void initialize() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> HiveHook.initialize()"); + } + + try { + atlasPluginClassLoader = AtlasPluginClassLoader.getInstance(ATLAS_PLUGIN_TYPE, this.getClass()); + + @SuppressWarnings("unchecked") + Class<SqoopJobDataPublisher> cls = (Class<SqoopJobDataPublisher>) Class + .forName(ATLAS_SQOOP_HOOK_IMPL_CLASSNAME, true, atlasPluginClassLoader); + + activatePluginClassLoader(); + + sqoopHookImpl = cls.newInstance(); + } catch (Exception excp) { + LOG.error("Error instantiating Atlas hook implementation", excp); + } finally { + deactivatePluginClassLoader(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== HiveHook.initialize()"); + } + } + + private void activatePluginClassLoader() { + if (atlasPluginClassLoader != null) { + atlasPluginClassLoader.activate(); + } + } + + private void deactivatePluginClassLoader() { + if (atlasPluginClassLoader != null) { + atlasPluginClassLoader.deactivate(); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/043d2aa6/addons/sqoop-bridge/pom.xml ---------------------------------------------------------------------- diff --git a/addons/sqoop-bridge/pom.xml b/addons/sqoop-bridge/pom.xml index c792945..ceccf36 100644 --- a/addons/sqoop-bridge/pom.xml +++ b/addons/sqoop-bridge/pom.xml @@ -170,29 +170,16 @@ <goal>copy</goal> </goals> <configuration> + <outputDirectory>${project.build.directory}/dependency/hook/sqoop/atlas-sqoop-plugin-impl</outputDirectory> + <overWriteReleases>false</overWriteReleases> + <overWriteSnapshots>false</overWriteSnapshots> + <overWriteIfNewer>true</overWriteIfNewer> <artifactItems> <artifactItem> <groupId>${project.groupId}</groupId> <artifactId>${project.artifactId}</artifactId> <version>${project.version}</version> - <overWrite>true</overWrite> - <outputDirectory>${project.build.directory}/dependency/hook/sqoop</outputDirectory> </artifactItem> - </artifactItems> - </configuration> - </execution> - <execution> - <id>copy-hook-dependencies</id> - <phase>package</phase> - <goals> - <goal>copy</goal> - </goals> - <configuration> - <outputDirectory>${project.build.directory}/dependency/hook/sqoop</outputDirectory> - <overWriteReleases>false</overWriteReleases> - <overWriteSnapshots>false</overWriteSnapshots> - <overWriteIfNewer>true</overWriteIfNewer> - <artifactItems> <artifactItem> <groupId>${project.groupId}</groupId> <artifactId>${project.artifactId}</artifactId> @@ -273,6 +260,46 @@ <artifactId>kafka-clients</artifactId> <version>${kafka.version}</version> </artifactItem> + <artifactItem> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <version>${hadoop.version}</version> + </artifactItem> + <artifactItem> + <groupId>com.google.code.gson</groupId> + <artifactId>gson</artifactId> + <version>${gson.version}</version> + </artifactItem> + <artifactItem> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + <version>${log4j.version}</version> + </artifactItem> + </artifactItems> + </configuration> + </execution> + <execution> + <id>copy-hook-shim</id> + <phase>package</phase> + <goals> + <goal>copy</goal> + </goals> + <configuration> + <outputDirectory>${project.build.directory}/dependency/hook/sqoop</outputDirectory> + <overWriteReleases>false</overWriteReleases> + <overWriteSnapshots>false</overWriteSnapshots> + <overWriteIfNewer>true</overWriteIfNewer> + <artifactItems> + <artifactItem> + <groupId>${project.groupId}</groupId> + <artifactId>sqoop-bridge-shim</artifactId> + <version>${project.version}</version> + </artifactItem> + <artifactItem> + <groupId>${project.groupId}</groupId> + <artifactId>atlas-plugin-classloader</artifactId> + <version>${project.version}</version> + </artifactItem> </artifactItems> </configuration> </execution> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/043d2aa6/addons/storm-bridge-shim/pom.xml ---------------------------------------------------------------------- diff --git a/addons/storm-bridge-shim/pom.xml b/addons/storm-bridge-shim/pom.xml new file mode 100755 index 0000000..49c56e0 --- /dev/null +++ b/addons/storm-bridge-shim/pom.xml @@ -0,0 +1,58 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <artifactId>apache-atlas</artifactId> + <groupId>org.apache.atlas</groupId> + <version>0.8-incubating-SNAPSHOT</version> + <relativePath>../../</relativePath> + </parent> + <artifactId>storm-bridge-shim</artifactId> + <description>Apache Atlas Storm Bridge Shim Module</description> + <name>Apache Atlas Storm Bridge Shim</name> + <packaging>jar</packaging> + + <properties> + <storm.version>1.0.0</storm.version> + <checkstyle.failOnViolation>true</checkstyle.failOnViolation> + </properties> + + <dependencies> + <!-- Logging --> + <dependency> + <groupId>org.apache.atlas</groupId> + <artifactId>atlas-plugin-classloader</artifactId> + </dependency> + + <dependency> + <groupId>org.apache.storm</groupId> + <artifactId>storm-core</artifactId> + <version>${storm.version}</version> + <type>jar</type> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + </exclusions> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/043d2aa6/addons/storm-bridge-shim/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java ---------------------------------------------------------------------- diff --git a/addons/storm-bridge-shim/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java b/addons/storm-bridge-shim/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java new file mode 100644 index 0000000..2d304c5 --- /dev/null +++ b/addons/storm-bridge-shim/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.storm.hook; + + +import org.apache.atlas.plugin.classloader.AtlasPluginClassLoader; +import org.apache.storm.ISubmitterHook; +import org.apache.storm.generated.StormTopology; +import org.apache.storm.generated.TopologyInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; + +/** + * Storm hook used for atlas entity registration. + */ +public class StormAtlasHook implements ISubmitterHook { + private static final Logger LOG = LoggerFactory.getLogger(StormAtlasHook.class); + + private static final String ATLAS_PLUGIN_TYPE = "storm"; + private static final String ATLAS_STORM_HOOK_IMPL_CLASSNAME = "org.apache.atlas.storm.hook.StormAtlasHook"; + + private AtlasPluginClassLoader atlasPluginClassLoader = null; + private ISubmitterHook stormHook = null; + + + public StormAtlasHook() { + this.initialize(); + } + + @Override + public void notify(TopologyInfo topologyInfo, Map stormConf, StormTopology stormTopology) + throws IllegalAccessException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> StormAtlasHook.notify(" + topologyInfo + ", " + stormConf + ", " + stormTopology + ")"); + } + + try { + activatePluginClassLoader(); + stormHook.notify(topologyInfo, stormConf, stormTopology); + } finally { + deactivatePluginClassLoader(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== StormAtlasHook.notify(" + topologyInfo + ", " + stormConf + ", " + stormTopology + ")"); + } + } + + private void initialize() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> StormAtlasHook.initialize()"); + } + + try { + atlasPluginClassLoader = AtlasPluginClassLoader.getInstance(ATLAS_PLUGIN_TYPE, this.getClass()); + + @SuppressWarnings("unchecked") + Class<ISubmitterHook> cls = (Class<ISubmitterHook>) Class + .forName(ATLAS_STORM_HOOK_IMPL_CLASSNAME, true, atlasPluginClassLoader); + + activatePluginClassLoader(); + + stormHook = cls.newInstance(); + } catch (Exception excp) { + LOG.error("Error instantiating Atlas hook implementation", excp); + } finally { + deactivatePluginClassLoader(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== StormAtlasHook.initialize()"); + } + } + + private void activatePluginClassLoader() { + if (atlasPluginClassLoader != null) { + atlasPluginClassLoader.activate(); + } + } + + private void deactivatePluginClassLoader() { + if (atlasPluginClassLoader != null) { + atlasPluginClassLoader.deactivate(); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/043d2aa6/addons/storm-bridge/pom.xml ---------------------------------------------------------------------- diff --git a/addons/storm-bridge/pom.xml b/addons/storm-bridge/pom.xml index 9e8bf2f..12c1208 100644 --- a/addons/storm-bridge/pom.xml +++ b/addons/storm-bridge/pom.xml @@ -139,13 +139,13 @@ <artifactId>maven-dependency-plugin</artifactId> <executions> <execution> - <id>copy-hook-dependencies</id> + <id>copy-hook</id> <phase>package</phase> <goals> <goal>copy</goal> </goals> <configuration> - <outputDirectory>${project.build.directory}/dependency/hook/storm</outputDirectory> + <outputDirectory>${project.build.directory}/dependency/hook/storm/atlas-storm-plugin-impl</outputDirectory> <overWriteReleases>false</overWriteReleases> <overWriteSnapshots>false</overWriteSnapshots> <overWriteIfNewer>true</overWriteIfNewer> @@ -315,6 +315,46 @@ <artifactId>hbase-common</artifactId> <version>${hbase.version}</version> </artifactItem> + <artifactItem> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <version>${hadoop.version}</version> + </artifactItem> + <artifactItem> + <groupId>com.google.code.gson</groupId> + <artifactId>gson</artifactId> + <version>${gson.version}</version> + </artifactItem> + <artifactItem> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + <version>${log4j.version}</version> + </artifactItem> + </artifactItems> + </configuration> + </execution> + <execution> + <id>copy-hook-shim</id> + <phase>package</phase> + <goals> + <goal>copy</goal> + </goals> + <configuration> + <outputDirectory>${project.build.directory}/dependency/hook/storm</outputDirectory> + <overWriteReleases>false</overWriteReleases> + <overWriteSnapshots>false</overWriteSnapshots> + <overWriteIfNewer>true</overWriteIfNewer> + <artifactItems> + <artifactItem> + <groupId>${project.groupId}</groupId> + <artifactId>storm-bridge-shim</artifactId> + <version>${project.version}</version> + </artifactItem> + <artifactItem> + <groupId>${project.groupId}</groupId> + <artifactId>atlas-plugin-classloader</artifactId> + <version>${project.version}</version> + </artifactItem> </artifactItems> </configuration> </execution> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/043d2aa6/plugin-classloader/pom.xml ---------------------------------------------------------------------- diff --git a/plugin-classloader/pom.xml b/plugin-classloader/pom.xml new file mode 100644 index 0000000..c41b309 --- /dev/null +++ b/plugin-classloader/pom.xml @@ -0,0 +1,42 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <artifactId>apache-atlas</artifactId> + <groupId>org.apache.atlas</groupId> + <version>0.8-incubating-SNAPSHOT</version> + </parent> + <artifactId>atlas-plugin-classloader</artifactId> + <description>Apache Atlas Plugin Classloader</description> + <name>Apache Atlas Plugin Classloader</name> + <packaging>jar</packaging> + + <properties> + <checkstyle.failOnViolation>true</checkstyle.failOnViolation> + </properties> + + <dependencies> + <dependency> + <groupId>org.testng</groupId> + <artifactId>testng</artifactId> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/043d2aa6/plugin-classloader/src/main/java/org/apache/atlas/plugin/classloader/AtlasPluginClassLoader.java ---------------------------------------------------------------------- diff --git a/plugin-classloader/src/main/java/org/apache/atlas/plugin/classloader/AtlasPluginClassLoader.java b/plugin-classloader/src/main/java/org/apache/atlas/plugin/classloader/AtlasPluginClassLoader.java new file mode 100644 index 0000000..c5214bd --- /dev/null +++ b/plugin-classloader/src/main/java/org/apache/atlas/plugin/classloader/AtlasPluginClassLoader.java @@ -0,0 +1,327 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.plugin.classloader; + + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URL; +import java.net.URLClassLoader; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.security.PrivilegedExceptionAction; +import java.util.Enumeration; + +/** + * AtlasPluginClassLoader to use plugin classpath first, before component classpath. + */ +public final class AtlasPluginClassLoader extends URLClassLoader { + private static final Logger LOG = LoggerFactory.getLogger(AtlasPluginClassLoader.class); + + private final MyClassLoader componentClassLoader; + + private AtlasPluginClassLoader(String pluginType, Class<?> pluginClass) throws Exception { + this(AtlasPluginClassLoaderUtil.getPluginImplLibPath(pluginType, pluginClass)); + } + + //visible for testing + AtlasPluginClassLoader(String libraryPath) throws Exception { + super(AtlasPluginClassLoaderUtil.getFilesInDirectories(new String[]{libraryPath}), null); + + componentClassLoader = AccessController.doPrivileged(new PrivilegedAction<MyClassLoader>() { + public MyClassLoader run() { + return new MyClassLoader(Thread.currentThread().getContextClassLoader()); + } + }); + } + + public static AtlasPluginClassLoader getInstance(final String pluginType, final Class<?> pluginClass) + throws Exception { + AtlasPluginClassLoader ret = + AccessController.doPrivileged(new PrivilegedExceptionAction<AtlasPluginClassLoader>() { + public AtlasPluginClassLoader run() throws Exception { + return new AtlasPluginClassLoader(pluginType, pluginClass); + } + }); + + return ret; + } + + @Override + public Class<?> findClass(String name) throws ClassNotFoundException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasPluginClassLoader.findClass(" + name + ")"); + } + + Class<?> ret = null; + + try { + // first try to find the class in pluginClassloader + if (LOG.isDebugEnabled()) { + LOG.debug("AtlasPluginClassLoader.findClass(" + name + "): calling pluginClassLoader.findClass()"); + } + + ret = super.findClass(name); + } catch (Throwable e) { + // on failure to find in pluginClassLoader, try to find in componentClassLoader + MyClassLoader savedClassLoader = getComponentClassLoader(); + + if (savedClassLoader != null) { + if (LOG.isDebugEnabled()) { + LOG.debug( + "AtlasPluginClassLoader.findClass(" + name + "): calling componentClassLoader.findClass()"); + } + + ret = savedClassLoader.findClass(name); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasPluginClassLoader.findClass(" + name + "): " + ret); + } + + return ret; + } + + @Override + public Class<?> loadClass(String name) throws ClassNotFoundException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasPluginClassLoader.loadClass(" + name + ")"); + } + + Class<?> ret = null; + + try { + // first try to load the class from pluginClassloader + if (LOG.isDebugEnabled()) { + LOG.debug("AtlasPluginClassLoader.loadClass(" + name + "): calling pluginClassLoader.loadClass()"); + } + + ret = super.loadClass(name); + } catch (Throwable e) { + // on failure to load from pluginClassLoader, try to load from componentClassLoader + MyClassLoader savedClassLoader = getComponentClassLoader(); + + if (savedClassLoader != null) { + if (LOG.isDebugEnabled()) { + LOG.debug( + "AtlasPluginClassLoader.loadClass(" + name + "): calling componentClassLoader.loadClass()"); + } + + ret = savedClassLoader.loadClass(name); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasPluginClassLoader.loadClass(" + name + "): " + ret); + } + + return ret; + } + + @Override + public URL findResource(String name) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasPluginClassLoader.findResource(" + name + ") "); + } + + // first try to find the resource from pluginClassloader + if (LOG.isDebugEnabled()) { + LOG.debug("AtlasPluginClassLoader.findResource(" + name + "): calling pluginClassLoader.findResource()"); + } + + URL ret = super.findResource(name); + + if (ret == null) { + MyClassLoader savedClassLoader = getComponentClassLoader(); + + if (savedClassLoader != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("AtlasPluginClassLoader.findResource(" + name + + "): calling componentClassLoader.getResource()"); + } + + ret = savedClassLoader.getResource(name); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasPluginClassLoader.findResource(" + name + "): " + ret); + } + + return ret; + } + + @Override + public Enumeration<URL> findResources(String name) throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasPluginClassLoader.findResources(" + name + ")"); + } + + Enumeration<URL> ret = null; + + Enumeration<URL> resourcesInPluginClsLoader = findResourcesUsingPluginClassLoader(name); + Enumeration<URL> resourcesInComponentClsLoader = findResourcesUsingComponentClassLoader(name); + + if (resourcesInPluginClsLoader != null && resourcesInComponentClsLoader != null) { + ret = new MergeEnumeration(resourcesInPluginClsLoader, resourcesInComponentClsLoader); + } else if (resourcesInPluginClsLoader != null) { + ret = resourcesInPluginClsLoader; + } else { + ret = resourcesInComponentClsLoader; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasPluginClassLoader.findResources(" + name + "): " + ret); + } + + return ret; + } + + public void activate() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasPluginClassLoader.activate()"); + } + + Thread.currentThread().setContextClassLoader(this); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasPluginClassLoader.activate()"); + } + } + + public void deactivate() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasPluginClassLoader.deactivate()"); + } + + MyClassLoader savedClassLoader = getComponentClassLoader(); + + if (savedClassLoader != null && savedClassLoader.getParent() != null) { + Thread.currentThread().setContextClassLoader(savedClassLoader.getParent()); + } else { + LOG.warn("AtlasPluginClassLoader.deactivate() was not successful.Couldn't not get the saved " + + "componentClassLoader..."); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasPluginClassLoader.deactivate()"); + } + } + + private MyClassLoader getComponentClassLoader() { + return componentClassLoader; + } + + private Enumeration<URL> findResourcesUsingPluginClassLoader(String name) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasPluginClassLoader.findResourcesUsingPluginClassLoader(" + name + ")"); + } + + Enumeration<URL> ret = null; + + try { + ret = super.findResources(name); + } catch (Throwable excp) { + // Ignore exceptions + if (LOG.isDebugEnabled()) { + LOG.debug("AtlasPluginClassLoader.findResourcesUsingPluginClassLoader(" + name + + "): resource not found in plugin", excp); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasPluginClassLoader.findResourcesUsingPluginClassLoader(" + name + "): " + ret); + } + + return ret; + } + + private Enumeration<URL> findResourcesUsingComponentClassLoader(String name) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasPluginClassLoader.findResourcesUsingComponentClassLoader(" + name + ")"); + } + + Enumeration<URL> ret = null; + + try { + MyClassLoader savedClassLoader = getComponentClassLoader(); + + if (savedClassLoader != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("AtlasPluginClassLoader.findResourcesUsingComponentClassLoader(" + name + + "): calling componentClassLoader.getResources()"); + } + + ret = savedClassLoader.getResources(name); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasPluginClassLoader.findResourcesUsingComponentClassLoader(" + name + "): " + ret); + } + } catch (Throwable t) { + if (LOG.isDebugEnabled()) { + LOG.debug("AtlasPluginClassLoader.findResourcesUsingComponentClassLoader(" + name + + "): class not found in componentClassLoader.", t); + } + } + + return ret; + } + + static class MergeEnumeration implements Enumeration<URL> { //NOPMD + private Enumeration<URL> e1 = null; + private Enumeration<URL> e2 = null; + + public MergeEnumeration(Enumeration<URL> e1, Enumeration<URL> e2) { + this.e1 = e1; + this.e2 = e2; + } + + @Override + public boolean hasMoreElements() { + return ((e1 != null && e1.hasMoreElements()) || (e2 != null && e2.hasMoreElements())); + } + + @Override + public URL nextElement() { + URL ret = null; + + if (e1 != null && e1.hasMoreElements()) { + ret = e1.nextElement(); + } else if (e2 != null && e2.hasMoreElements()) { + ret = e2.nextElement(); + } + + return ret; + } + } + + static class MyClassLoader extends ClassLoader { + public MyClassLoader(ClassLoader realClassLoader) { + super(realClassLoader); + } + + @Override + public Class<?> findClass(String name) throws ClassNotFoundException { //NOPMD + return super.findClass(name); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/043d2aa6/plugin-classloader/src/main/java/org/apache/atlas/plugin/classloader/AtlasPluginClassLoaderUtil.java ---------------------------------------------------------------------- diff --git a/plugin-classloader/src/main/java/org/apache/atlas/plugin/classloader/AtlasPluginClassLoaderUtil.java b/plugin-classloader/src/main/java/org/apache/atlas/plugin/classloader/AtlasPluginClassLoaderUtil.java new file mode 100644 index 0000000..c3ec5e2 --- /dev/null +++ b/plugin-classloader/src/main/java/org/apache/atlas/plugin/classloader/AtlasPluginClassLoaderUtil.java @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.plugin.classloader; + + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.net.URI; +import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + + +/** + * AtlasPluginClassLoaderUtil used by AtlasPluginClassLoader. + */ +final class AtlasPluginClassLoaderUtil { + private static final Logger LOG = LoggerFactory.getLogger(AtlasPluginClassLoaderUtil.class); + + private static final String ATLAS_PLUGIN_LIBDIR = "atlas-%-plugin-impl"; + + private AtlasPluginClassLoaderUtil(){ } + + public static URL[] getFilesInDirectories(String[] libDirs) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasPluginClassLoaderUtil.getFilesInDirectories()"); + } + + List<URL> ret = new ArrayList<URL>(); + + for (String libDir : libDirs) { + getFilesInDirectory(libDir, ret); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasPluginClassLoaderUtil.getFilesInDirectories(): " + ret.size() + " files"); + } + + return ret.toArray(new URL[]{}); + } + + private static void getFilesInDirectory(String dirPath, List<URL> files) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasPluginClassLoaderUtil.getPluginFiles()"); + } + + if (dirPath != null) { + try { + File[] dirFiles = new File(dirPath).listFiles(); + + if (dirFiles != null) { + for (File dirFile : dirFiles) { + try { + URL jarPath = dirFile.toURI().toURL(); + + if (LOG.isDebugEnabled()) { + LOG.debug( + "getFilesInDirectory('" + dirPath + "'): adding " + dirFile.getAbsolutePath()); + } + + files.add(jarPath); + } catch (Exception excp) { + LOG.warn("getFilesInDirectory('" + dirPath + "'): failed to get URI for file " + dirFile + .getAbsolutePath(), excp); + } + } + } + } catch (Exception excp) { + LOG.warn("getFilesInDirectory('" + dirPath + "'): error", excp); + } + } else { + LOG.warn("getFilesInDirectory('" + dirPath + "'): could not find directory in path " + dirPath); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasPluginClassLoaderUtil.getFilesInDirectory(" + dirPath + ")"); + } + } + + public static String getPluginImplLibPath(String pluginType, Class<?> pluginClass) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasPluginClassLoaderUtil.getPluginImplLibPath for Class (" + pluginClass.getName() + ")"); + } + + URI uri = pluginClass.getProtectionDomain().getCodeSource().getLocation().toURI(); + Path path = Paths.get(URI.create(uri.toString())); + String ret = path.getParent().toString() + File.separatorChar + ATLAS_PLUGIN_LIBDIR.replaceAll("%", pluginType); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasPluginClassLoaderUtil.getPluginImplLibPath for Class " + pluginClass.getName() + "): " + + ret + ")"); + } + + return ret; + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/043d2aa6/plugin-classloader/src/test/java/org/apache/atlas/plugin/classloader/AtlasPluginClassLoaderTest.java ---------------------------------------------------------------------- diff --git a/plugin-classloader/src/test/java/org/apache/atlas/plugin/classloader/AtlasPluginClassLoaderTest.java b/plugin-classloader/src/test/java/org/apache/atlas/plugin/classloader/AtlasPluginClassLoaderTest.java new file mode 100644 index 0000000..6139a00 --- /dev/null +++ b/plugin-classloader/src/test/java/org/apache/atlas/plugin/classloader/AtlasPluginClassLoaderTest.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.plugin.classloader; + +import org.testng.Assert; +import org.testng.annotations.Test; + + +public class AtlasPluginClassLoaderTest { + + @Test + public void testClassLoader() throws Exception { + String cls = "org.apache.atlas.service.Services"; + + try { + loadClass(cls, null); + Assert.fail("Expected ClassNotFoundException"); + } catch (ClassNotFoundException e) { + //expected + } + + AtlasPluginClassLoader classLoader = new AtlasPluginClassLoader("../common/target"); + + classLoader.activate(); + + //org.apache.atlas.service.Services class should be loadable now + //should also load org.apache.atlas.service.Service + Class<?> servicesCls = loadClass(cls, null); + loadClass("org.apache.atlas.service.Service", servicesCls.getClassLoader()); + + //Fall back to current class loader should also work + loadClass(AtlasPluginClassLoaderUtil.class.getName(), null); + + classLoader.deactivate(); + + //After disable, class loading should fail again + try { + loadClass(cls, null); + Assert.fail("Expected ClassNotFoundException"); + } catch (ClassNotFoundException e) { + //expected + } + } + + private Class<?> loadClass(String cls, ClassLoader classLoader) throws ClassNotFoundException { + if (classLoader == null) { + classLoader = Thread.currentThread().getContextClassLoader(); + } + return Class.forName(cls, true, classLoader); + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/043d2aa6/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index f119525..2224450 100755 --- a/pom.xml +++ b/pom.xml @@ -378,6 +378,7 @@ <log4j.version>1.2.17</log4j.version> <akka.version>2.3.7</akka.version> <spray.version>1.3.1</spray.version> + <gson.version>2.5</gson.version> <guava.version>14.0</guava.version> <fastutil.version>6.5.16</fastutil.version> <guice.version>4.0</guice.version> @@ -470,12 +471,17 @@ <module>catalog</module> <module>dashboardv2</module> <module>webapp</module> - <module>docs</module> + <module>addons/hdfs-model</module> + <module>plugin-classloader</module> + <module>addons/hive-bridge-shim</module> <module>addons/hive-bridge</module> + <module>addons/falcon-bridge-shim</module> <module>addons/falcon-bridge</module> + <module>addons/sqoop-bridge-shim</module> <module>addons/sqoop-bridge</module> + <module>addons/storm-bridge-shim</module> <module>addons/storm-bridge</module> <module>distro</module> @@ -1111,6 +1117,30 @@ <version>${project.version}</version> </dependency> + <dependency> + <groupId>org.apache.atlas</groupId> + <artifactId>hive-bridge-shim</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>org.apache.atlas</groupId> + <artifactId>storm-bridge-shim</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>org.apache.atlas</groupId> + <artifactId>falcon-bridge-shim</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>org.apache.atlas</groupId> + <artifactId>sqoop-bridge-shim</artifactId> + <version>${project.version}</version> + </dependency> + <!--Scala dependencies--> <dependency> <groupId>org.scala-lang</groupId> @@ -1223,7 +1253,7 @@ <dependency> <groupId>com.google.code.gson</groupId> <artifactId>gson</artifactId> - <version>2.5</version> + <version>${gson.version}</version> </dependency> <dependency> @@ -1303,6 +1333,12 @@ </exclusion> </exclusions> </dependency> + + <dependency> + <groupId>org.apache.atlas</groupId> + <artifactId>atlas-plugin-classloader</artifactId> + <version>${project.version}</version> + </dependency> </dependencies> </dependencyManagement> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/043d2aa6/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index d700596..8c9c1eb 100644 --- a/release-log.txt +++ b/release-log.txt @@ -6,6 +6,7 @@ INCOMPATIBLE CHANGES: ALL CHANGES: +ATLAS-987 Atlas hooks should avoid adding dependent libraries to component CLASSPATH (madhan.neethiraj via shwethags) ATLAS-993 If condition in DSL order by clause is not defined then dsl query fails (guptaneeru via shwethags) ATLAS-968 Set group information from UGI for Ldap authentication (nixonrodrigues via shwethags) ATLAS-584 Integrate CSRF prevention filter (kevalbhatt18 via shwethags)
