This is an automated email from the ASF dual-hosted git repository. madhan pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 2c2d9197e68b809198d6e9f1232473d1c2d9e960 Author: Madhan Neethiraj <[email protected]> AuthorDate: Tue Nov 15 14:22:04 2022 -0800 ATLAS-3820: fix import-kafka.sh failures; updated docker setup for Kafka bridge --- addons/kafka-bridge/pom.xml | 72 +++++++++++++++++++++- addons/kafka-bridge/src/bin/import-kafka.sh | 18 +----- .../org/apache/atlas/kafka/bridge/KafkaBridge.java | 2 +- dev-support/atlas-docker/.dockerignore | 1 + dev-support/atlas-docker/Dockerfile.atlas-kafka | 21 ++++--- dev-support/atlas-docker/scripts/atlas-build.sh | 1 + .../atlas-kafka-application.properties} | 24 ++------ 7 files changed, 95 insertions(+), 44 deletions(-) diff --git a/addons/kafka-bridge/pom.xml b/addons/kafka-bridge/pom.xml index 9ce22a757..31f5d5b3a 100644 --- a/addons/kafka-bridge/pom.xml +++ b/addons/kafka-bridge/pom.xml @@ -125,7 +125,7 @@ <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> - <version>12.0.1</version> + <version>${guava.version}</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> @@ -223,6 +223,21 @@ <artifactId>atlas-common</artifactId> <version>${project.version}</version> </artifactItem> + <artifactItem> + <groupId>commons-collections</groupId> + <artifactId>commons-collections</artifactId> + <version>${commons-collections.version}</version> + </artifactItem> + <artifactItem> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + <version>${commons-logging.version}</version> + </artifactItem> + <artifactItem> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + <version>${commons-lang.version}</version> + </artifactItem> <artifactItem> <groupId>org.apache.kafka</groupId> <artifactId>kafka-clients</artifactId> @@ -233,11 +248,66 @@ <artifactId>jersey-multipart</artifactId> <version>${jersey.version}</version> </artifactItem> + <artifactItem> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <version>${jackson.databind.version}</version> + </artifactItem> + <artifactItem> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + <version>${jackson.version}</version> + </artifactItem> + <artifactItem> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-annotations</artifactId> + <version>${jackson.version}</version> + </artifactItem> + <artifactItem> + <groupId>com.fasterxml.jackson.jaxrs</groupId> + <artifactId>jackson-jaxrs-json-provider</artifactId> + <version>${jackson.version}</version> + </artifactItem> + <artifactItem> + <groupId>com.fasterxml.jackson.jaxrs</groupId> + <artifactId>jackson-jaxrs-base</artifactId> + <version>${jackson.version}</version> + </artifactItem> + <artifactItem> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-bundle</artifactId> + <version>${jersey.version}</version> + </artifactItem> + <artifactItem> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-auth</artifactId> + <version>${hadoop.version}</version> + </artifactItem> + <artifactItem> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + <version>${guava.version}</version> + </artifactItem> + <artifactItem> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + <version>${httpcomponents-httpclient.version}</version> + </artifactItem> + <artifactItem> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpcore</artifactId> + <version>${httpcomponents-httpcore.version}</version> + </artifactItem> <artifactItem> <groupId>commons-configuration</groupId> <artifactId>commons-configuration</artifactId> <version>${commons-conf.version}</version> </artifactItem> + <artifactItem> + <groupId>com.googlecode.json-simple</groupId> + <artifactId>json-simple</artifactId> + <version>${json-simple.version}</version> + </artifactItem> </artifactItems> </configuration> </execution> diff --git a/addons/kafka-bridge/src/bin/import-kafka.sh b/addons/kafka-bridge/src/bin/import-kafka.sh index ada3b4ae2..8fe27748a 100644 --- a/addons/kafka-bridge/src/bin/import-kafka.sh +++ b/addons/kafka-bridge/src/bin/import-kafka.sh @@ -99,25 +99,12 @@ fi KAFKA_CP="${KAFKA_CONF}" -for i in "${KAFKA_HOME}/libs/kafka-clients"*.jar "${KAFKA_HOME}/libs/slf4j-api"*.jar; do +for i in "${KAFKA_HOME}/libs/kafka-clients"*.jar "${KAFKA_HOME}/libs/slf4j-"*.jar "${KAFKA_HOME}/libs/log4j-"*.jar "${KAFKA_HOME}/libs/commons-"*.jar "${KAFKA_HOME}/libs/jackson-module-jaxb-annotations"*.jar; do KAFKA_CP="${KAFKA_CP}:$i" done -#Add hadoop conf in classpath -if [ ! -z "$HADOOP_CLASSPATH" ]; then - HADOOP_CP=$HADOOP_CLASSPATH -elif [ ! -z "$HADOOP_HOME" ]; then - HADOOP_CP=`$HADOOP_HOME/bin/hadoop classpath` -elif [ $(command -v hadoop) ]; then - HADOOP_CP=`hadoop classpath` - #echo $HADOOP_CP -else - echo "Environment variable HADOOP_CLASSPATH or HADOOP_HOME need to be set" - exit 1 -fi - -CP="${ATLASCPPATH}:${HADOOP_CP}:${KAFKA_CP}" +CP="${ATLASCPPATH}:${KAFKA_CP}" # If running in cygwin, convert pathnames and classpath to Windows format. if [ "${CYGWIN}" == "true" ] @@ -125,7 +112,6 @@ then ATLAS_LOG_DIR=`cygpath -w ${ATLAS_LOG_DIR}` LOGFILE=`cygpath -w ${LOGFILE}` KAFKA_CP=`cygpath -w ${KAFKA_CP}` - HADOOP_CP=`cygpath -w ${HADOOP_CP}` CP=`cygpath -w -p ${CP}` fi diff --git a/addons/kafka-bridge/src/main/java/org/apache/atlas/kafka/bridge/KafkaBridge.java b/addons/kafka-bridge/src/main/java/org/apache/atlas/kafka/bridge/KafkaBridge.java index 8be2fca3a..0878ec29f 100644 --- a/addons/kafka-bridge/src/main/java/org/apache/atlas/kafka/bridge/KafkaBridge.java +++ b/addons/kafka-bridge/src/main/java/org/apache/atlas/kafka/bridge/KafkaBridge.java @@ -134,7 +134,7 @@ public class KafkaBridge { KafkaBridge importer = new KafkaBridge(atlasConf, atlasClientV2, kafkaUtils); - if(!KAFKA_SCHEMA_REGISTRY_ENV_VARIABLE.isEmpty()){ + if(StringUtils.isNotEmpty(KAFKA_SCHEMA_REGISTRY_ENV_VARIABLE)) { KAFKA_SCHEMA_REGISTRY_HOSTNAME = KAFKA_SCHEMA_REGISTRY_ENV_VARIABLE; } diff --git a/dev-support/atlas-docker/.dockerignore b/dev-support/atlas-docker/.dockerignore index d4a3db26b..14ff6a9fc 100644 --- a/dev-support/atlas-docker/.dockerignore +++ b/dev-support/atlas-docker/.dockerignore @@ -2,6 +2,7 @@ !config !dist/apache-atlas-*-hbase-hook.tar.gz !dist/apache-atlas-*-hive-hook.tar.gz +!dist/apache-atlas-*-kafka-hook.tar.gz !dist/apache-atlas-*-server.tar.gz !downloads/* !scripts/* diff --git a/dev-support/atlas-docker/Dockerfile.atlas-kafka b/dev-support/atlas-docker/Dockerfile.atlas-kafka index e06d62845..d7a8e3f07 100644 --- a/dev-support/atlas-docker/Dockerfile.atlas-kafka +++ b/dev-support/atlas-docker/Dockerfile.atlas-kafka @@ -20,17 +20,24 @@ ARG ATLAS_VERSION ARG KAFKA_VERSION -COPY ./downloads/kafka_2.12-${KAFKA_VERSION}.tgz /home/atlas/dist/ +COPY ./dist/apache-atlas-${ATLAS_VERSION}-kafka-hook.tar.gz /home/atlas/dist/ +COPY ./downloads/kafka_2.12-${KAFKA_VERSION}.tgz /home/atlas/dist/ -COPY ./scripts/atlas-kafka-setup.sh /home/atlas/scripts/ -COPY ./scripts/atlas-kafka.sh /home/atlas/scripts/ +COPY ./scripts/atlas-kafka-setup.sh /home/atlas/scripts/ +COPY ./scripts/atlas-kafka.sh /home/atlas/scripts/ +COPY ./scripts/atlas-kafka-application.properties /home/atlas/scripts/ RUN tar xvfz /home/atlas/dist/kafka_2.12-${KAFKA_VERSION}.tgz --directory=/opt/ && \ ln -s /opt/kafka_2.12-${KAFKA_VERSION} /opt/kafka && \ - rm -f /home/atlas/dist/kafka_2.12-${KAFKA_VERSION}.tgz - -ENV KAFKA_HOME /opt/kafka -ENV PATH /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/kafka/bin + rm -f /home/atlas/dist/kafka_2.12-${KAFKA_VERSION}.tgz && \ + tar xvfz /home/atlas/dist/apache-atlas-${ATLAS_VERSION}-kafka-hook.tar.gz --directory=/opt/ && \ + ln -s /opt/apache-atlas-kafka-hook-${ATLAS_VERSION} /opt/apache-atlas-kafka-hook && \ + rm -f /home/atlas/dist/apache-atlas-${ATLAS_VERSION}-kafka-hook.tar.gz && \ + cp -f /home/atlas/scripts/atlas-kafka-application.properties /opt/kafka/config/atlas-application.properties + +ENV KAFKA_HOME /opt/kafka +ENV KAFKA_CONF_DIR /opt/kafka/config +ENV PATH /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/kafka/bin EXPOSE 9092 diff --git a/dev-support/atlas-docker/scripts/atlas-build.sh b/dev-support/atlas-docker/scripts/atlas-build.sh index e53ebbba9..592a055f7 100755 --- a/dev-support/atlas-docker/scripts/atlas-build.sh +++ b/dev-support/atlas-docker/scripts/atlas-build.sh @@ -89,3 +89,4 @@ mvn ${ARG_PROFILES} ${ARG_SKIPTESTS} -DskipDocs clean package mv -f distro/target/apache-atlas-${ATLAS_VERSION}-server.tar.gz /home/atlas/dist/ mv -f distro/target/apache-atlas-${ATLAS_VERSION}-hive-hook.tar.gz /home/atlas/dist/ mv -f distro/target/apache-atlas-${ATLAS_VERSION}-hbase-hook.tar.gz /home/atlas/dist/ +mv -f distro/target/apache-atlas-${ATLAS_VERSION}-kafka-hook.tar.gz /home/atlas/dist/ diff --git a/dev-support/atlas-docker/Dockerfile.atlas-kafka b/dev-support/atlas-docker/scripts/atlas-kafka-application.properties similarity index 53% copy from dev-support/atlas-docker/Dockerfile.atlas-kafka copy to dev-support/atlas-docker/scripts/atlas-kafka-application.properties index e06d62845..268e5c8c4 100644 --- a/dev-support/atlas-docker/Dockerfile.atlas-kafka +++ b/dev-support/atlas-docker/scripts/atlas-kafka-application.properties @@ -1,3 +1,4 @@ +# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -13,25 +14,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# -FROM atlas-base:latest - -ARG ATLAS_VERSION -ARG KAFKA_VERSION - - -COPY ./downloads/kafka_2.12-${KAFKA_VERSION}.tgz /home/atlas/dist/ - -COPY ./scripts/atlas-kafka-setup.sh /home/atlas/scripts/ -COPY ./scripts/atlas-kafka.sh /home/atlas/scripts/ - -RUN tar xvfz /home/atlas/dist/kafka_2.12-${KAFKA_VERSION}.tgz --directory=/opt/ && \ - ln -s /opt/kafka_2.12-${KAFKA_VERSION} /opt/kafka && \ - rm -f /home/atlas/dist/kafka_2.12-${KAFKA_VERSION}.tgz - -ENV KAFKA_HOME /opt/kafka -ENV PATH /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/kafka/bin -EXPOSE 9092 +atlas.rest.address=http://atlas.example.com:21000 -ENTRYPOINT [ "/home/atlas/scripts/atlas-kafka.sh" ] +atlas.kafka.zookeeper.connect=atlas-zk.example.com:2181 +atlas.kafka.bootstrap.servers=atlas-kafka.example.com:9092
