This is an automated email from the ASF dual-hosted git repository. rmerriman pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push: new e6ca3c7 METRON-2149 Shaded jar classifier is not consistent (merrimanr) closes apache/metron#1436 e6ca3c7 is described below commit e6ca3c71d8120516afcf816be2cda81093088b72 Author: merrimanr <merrim...@gmail.com> AuthorDate: Fri Aug 23 16:42:01 2019 -0500 METRON-2149 Shaded jar classifier is not consistent (merrimanr) closes apache/metron#1436 --- metron-analytics/metron-maas-common/pom.xml | 1 + metron-analytics/metron-maas-service/pom.xml | 6 ++ metron-analytics/metron-profiler-client/pom.xml | 8 ++ .../src/main/assembly/assembly.xml | 2 +- metron-analytics/metron-profiler-common/pom.xml | 6 ++ metron-analytics/metron-profiler-repl/pom.xml | 8 ++ .../src/main/assembly/assembly.xml | 2 +- metron-analytics/metron-profiler-spark/pom.xml | 3 + .../src/main/assembly/assembly.xml | 2 +- .../src/main/scripts/start_batch_profiler.sh | 4 +- metron-analytics/metron-profiler-storm/pom.xml | 8 ++ .../src/main/scripts/start_profiler_topology.sh | 3 +- metron-analytics/metron-statistics/pom.xml | 3 + metron-contrib/metron-performance/pom.xml | 8 ++ .../src/main/assembly/assembly.xml | 2 +- .../src/main/scripts/load_tool.sh | 2 +- .../common-services/METRON/CURRENT/metainfo.xml | 3 + .../packaging/docker/rpm-docker/SPECS/metron.spec | 35 +++++++-- .../packaging/docker/rpm-docker/pom.xml | 6 ++ metron-interface/metron-rest/pom.xml | 22 ++++++ .../metron-rest/src/main/assembly/assembly.xml | 2 +- .../metron-rest/src/main/scripts/metron-rest.sh | 2 + metron-platform/elasticsearch-shaded/pom.xml | 2 + .../metron-common-storm/pom.xml | 6 ++ metron-platform/metron-common/pom.xml | 51 +++++++------ .../metron-common/src/main/assembly/assembly.xml | 2 +- .../metron-common/src/main/scripts/stellar | 5 +- .../src/main/scripts/zk_load_configs.sh | 3 +- metron-platform/metron-data-management/pom.xml | 89 ++++++++++++++-------- .../src/main/assembly/assembly.xml | 2 +- .../src/main/scripts/flatfile_loader.sh | 7 +- .../src/main/scripts/flatfile_summarizer.sh | 7 +- .../src/main/scripts/maxmind_enrichment_load.sh | 10 +-- .../src/main/scripts/prune_hdfs_files.sh | 2 +- .../src/main/scripts/threatintel_bulk_prune.sh | 2 +- .../src/main/scripts/threatintel_taxii_load.sh | 2 +- .../metron-elasticsearch-common/pom.xml | 27 ++++++- .../metron-elasticsearch-storm/pom.xml | 19 +++++ .../main/scripts/start_elasticsearch_topology.sh | 5 +- .../metron-enrichment-common/pom.xml | 1 + .../metron-enrichment-storm/pom.xml | 6 ++ .../src/main/scripts/start_enrichment_topology.sh | 3 +- metron-platform/metron-hbase-client/pom.xml | 2 + metron-platform/metron-hbase-server/pom.xml | 6 ++ .../metron-indexing/metron-indexing-common/pom.xml | 11 +-- .../metron-indexing/metron-indexing-storm/pom.xml | 6 ++ .../src/main/scripts/start_hdfs_topology.sh | 3 +- metron-platform/metron-integration-test/pom.xml | 1 + metron-platform/metron-management/pom.xml | 3 + .../src/main/assembly/assembly.xml | 2 +- .../metron-parsing/metron-parsers-common/pom.xml | 6 ++ .../metron-parsing/metron-parsers/pom.xml | 6 ++ .../metron-parsing/metron-parsing-storm/pom.xml | 6 ++ .../src/main/scripts/start_parser_topology.sh | 8 +- metron-platform/metron-pcap-backend/pom.xml | 40 ++++++---- .../src/main/assembly/assembly.xml | 2 +- .../src/main/scripts/pcap_inspector.sh | 2 +- .../src/main/scripts/pcap_query.sh | 2 +- .../src/main/scripts/start_pcap_topology.sh | 6 +- metron-platform/metron-pcap/pom.xml | 14 ++++ .../metron-solr/metron-solr-common/pom.xml | 6 ++ .../metron-solr/metron-solr-storm/pom.xml | 6 ++ .../src/main/scripts/start_solr_topology.sh | 3 +- .../metron-storm-kafka-override/pom.xml | 2 + metron-platform/metron-storm-kafka/pom.xml | 1 + .../metron-writer/metron-writer-common/pom.xml | 45 ++++++----- .../metron-writer/metron-writer-storm/pom.xml | 6 ++ metron-stellar/stellar-common/pom.xml | 10 ++- .../stellar-common/src/main/assembly/assembly.xml | 12 --- .../src/main/scripts/deployed/stellar | 4 +- metron-stellar/stellar-zeppelin/pom.xml | 1 + 71 files changed, 444 insertions(+), 167 deletions(-) diff --git a/metron-analytics/metron-maas-common/pom.xml b/metron-analytics/metron-maas-common/pom.xml index 3d89800..b404111 100644 --- a/metron-analytics/metron-maas-common/pom.xml +++ b/metron-analytics/metron-maas-common/pom.xml @@ -43,6 +43,7 @@ <artifactId>hadoop-auth</artifactId> </exclusion> </exclusions> + <scope>provided</scope> </dependency> <dependency> <groupId>commons-cli</groupId> diff --git a/metron-analytics/metron-maas-service/pom.xml b/metron-analytics/metron-maas-service/pom.xml index 2bb06fe..a1671b5 100644 --- a/metron-analytics/metron-maas-service/pom.xml +++ b/metron-analytics/metron-maas-service/pom.xml @@ -44,6 +44,12 @@ <version>${project.parent.version}</version> </dependency> <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>stellar-common</artifactId> + <version>${project.parent.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>de.javakaffee</groupId> <artifactId>kryo-serializers</artifactId> <version>${global_kryo_serializers_version}</version> diff --git a/metron-analytics/metron-profiler-client/pom.xml b/metron-analytics/metron-profiler-client/pom.xml index 04d9740..876e1d6 100644 --- a/metron-analytics/metron-profiler-client/pom.xml +++ b/metron-analytics/metron-profiler-client/pom.xml @@ -59,6 +59,12 @@ </exclusions> </dependency> <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>stellar-common</artifactId> + <version>${project.parent.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> <version>${guava_version}</version> @@ -143,6 +149,8 @@ <artifactId>maven-shade-plugin</artifactId> <version>${global_shade_version}</version> <configuration> + <shadedArtifactAttached>true</shadedArtifactAttached> + <shadedClassifierName>uber</shadedClassifierName> <createDependencyReducedPom>false</createDependencyReducedPom> <artifactSet> <excludes> diff --git a/metron-analytics/metron-profiler-client/src/main/assembly/assembly.xml b/metron-analytics/metron-profiler-client/src/main/assembly/assembly.xml index b0ab070..86c27e5 100644 --- a/metron-analytics/metron-profiler-client/src/main/assembly/assembly.xml +++ b/metron-analytics/metron-profiler-client/src/main/assembly/assembly.xml @@ -28,7 +28,7 @@ <fileSet> <directory>${project.basedir}/target</directory> <includes> - <include>${project.artifactId}-${project.version}.jar</include> + <include>${project.artifactId}-${project.version}-uber.jar</include> </includes> <outputDirectory>lib</outputDirectory> <useDefaultExcludes>true</useDefaultExcludes> diff --git a/metron-analytics/metron-profiler-common/pom.xml b/metron-analytics/metron-profiler-common/pom.xml index bd2e5b6..13d5abe 100644 --- a/metron-analytics/metron-profiler-common/pom.xml +++ b/metron-analytics/metron-profiler-common/pom.xml @@ -48,6 +48,12 @@ </exclusions> </dependency> <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>stellar-common</artifactId> + <version>${project.parent.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>org.apache.hbase</groupId> <artifactId>hbase-client</artifactId> <version>${global_hbase_version}</version> diff --git a/metron-analytics/metron-profiler-repl/pom.xml b/metron-analytics/metron-profiler-repl/pom.xml index 86e917d..f3714b8 100644 --- a/metron-analytics/metron-profiler-repl/pom.xml +++ b/metron-analytics/metron-profiler-repl/pom.xml @@ -77,6 +77,12 @@ </exclusions> </dependency> <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>stellar-common</artifactId> + <version>${project.parent.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>org.apache.logging.log4j</groupId> <artifactId>log4j-api</artifactId> <version>${global_log4j_core_version}</version> @@ -96,6 +102,8 @@ <artifactId>maven-shade-plugin</artifactId> <version>${global_shade_version}</version> <configuration> + <shadedArtifactAttached>true</shadedArtifactAttached> + <shadedClassifierName>uber</shadedClassifierName> <createDependencyReducedPom>true</createDependencyReducedPom> </configuration> <executions> diff --git a/metron-analytics/metron-profiler-repl/src/main/assembly/assembly.xml b/metron-analytics/metron-profiler-repl/src/main/assembly/assembly.xml index fa6f0c1..6f883d3 100644 --- a/metron-analytics/metron-profiler-repl/src/main/assembly/assembly.xml +++ b/metron-analytics/metron-profiler-repl/src/main/assembly/assembly.xml @@ -27,7 +27,7 @@ <fileSet> <directory>${project.basedir}/target</directory> <includes> - <include>${project.artifactId}-${project.version}.jar</include> + <include>${project.artifactId}-${project.version}-uber.jar</include> </includes> <outputDirectory>lib</outputDirectory> <useDefaultExcludes>true</useDefaultExcludes> diff --git a/metron-analytics/metron-profiler-spark/pom.xml b/metron-analytics/metron-profiler-spark/pom.xml index b046cc4..a087663 100644 --- a/metron-analytics/metron-profiler-spark/pom.xml +++ b/metron-analytics/metron-profiler-spark/pom.xml @@ -63,6 +63,7 @@ <groupId>org.apache.metron</groupId> <artifactId>stellar-common</artifactId> <version>${project.parent.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.metron</groupId> @@ -158,6 +159,8 @@ <artifactId>maven-shade-plugin</artifactId> <version>${global_shade_version}</version> <configuration> + <shadedArtifactAttached>true</shadedArtifactAttached> + <shadedClassifierName>uber</shadedClassifierName> <createDependencyReducedPom>true</createDependencyReducedPom> </configuration> <executions> diff --git a/metron-analytics/metron-profiler-spark/src/main/assembly/assembly.xml b/metron-analytics/metron-profiler-spark/src/main/assembly/assembly.xml index 02f97eb..cb707be 100644 --- a/metron-analytics/metron-profiler-spark/src/main/assembly/assembly.xml +++ b/metron-analytics/metron-profiler-spark/src/main/assembly/assembly.xml @@ -51,7 +51,7 @@ <fileSet> <directory>${project.basedir}/target</directory> <includes> - <include>${project.artifactId}-${project.version}.jar</include> + <include>${project.artifactId}-${project.version}-uber.jar</include> </includes> <outputDirectory>lib</outputDirectory> <useDefaultExcludes>true</useDefaultExcludes> diff --git a/metron-analytics/metron-profiler-spark/src/main/scripts/start_batch_profiler.sh b/metron-analytics/metron-profiler-spark/src/main/scripts/start_batch_profiler.sh index 33d9aa4..134af6e 100644 --- a/metron-analytics/metron-profiler-spark/src/main/scripts/start_batch_profiler.sh +++ b/metron-analytics/metron-profiler-spark/src/main/scripts/start_batch_profiler.sh @@ -18,7 +18,8 @@ # METRON_VERSION=${project.version} METRON_HOME=/usr/metron/${METRON_VERSION} -PROFILER_JAR=${METRON_HOME}/lib/${project.artifactId}-${METRON_VERSION}.jar +PROFILER_JAR=${METRON_HOME}/lib/${project.artifactId}-${METRON_VERSION}-uber.jar +STELLAR_JAR=${METRON_HOME}/lib/stellar-common-$METRON_VERSION-uber.jar MAIN_CLASS=org.apache.metron.profiler.spark.cli.BatchProfilerCLI PROFILER_PROPS=${PROFILER_PROPS:-"${METRON_HOME}/config/batch-profiler.properties"} SPARK_HOME=${SPARK_HOME:-"/usr/hdp/current/spark2-client"} @@ -39,6 +40,7 @@ fi ${SPARK_HOME}/bin/spark-submit \ --class ${MAIN_CLASS} \ + --jars ${STELLAR_JAR} \ --properties-file ${PROFILER_PROPS} \ ${PROFILER_JAR} \ --config ${PROFILER_PROPS} \ diff --git a/metron-analytics/metron-profiler-storm/pom.xml b/metron-analytics/metron-profiler-storm/pom.xml index 09bd238..2ec108a 100644 --- a/metron-analytics/metron-profiler-storm/pom.xml +++ b/metron-analytics/metron-profiler-storm/pom.xml @@ -166,6 +166,12 @@ </exclusions> </dependency> <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>stellar-common</artifactId> + <version>${project.parent.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>com.esotericsoftware</groupId> <artifactId>kryo-shaded</artifactId> <version>${global_kryo_version}</version> @@ -325,6 +331,8 @@ <artifactId>maven-shade-plugin</artifactId> <version>${global_shade_version}</version> <configuration> + <shadedArtifactAttached>true</shadedArtifactAttached> + <shadedClassifierName>uber</shadedClassifierName> <createDependencyReducedPom>true</createDependencyReducedPom> </configuration> <executions> diff --git a/metron-analytics/metron-profiler-storm/src/main/scripts/start_profiler_topology.sh b/metron-analytics/metron-profiler-storm/src/main/scripts/start_profiler_topology.sh index 6ec78f5..0a0ea4e 100644 --- a/metron-analytics/metron-profiler-storm/src/main/scripts/start_profiler_topology.sh +++ b/metron-analytics/metron-profiler-storm/src/main/scripts/start_profiler_topology.sh @@ -19,4 +19,5 @@ METRON_VERSION=${project.version} METRON_HOME=/usr/metron/$METRON_VERSION TOPOLOGY_JAR=${project.artifactId}-$METRON_VERSION-uber.jar -storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/profiler/remote.yaml --filter $METRON_HOME/config/profiler.properties +STELLAR_JAR=stellar-common-$METRON_VERSION-uber.jar +storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/profiler/remote.yaml --filter $METRON_HOME/config/profiler.properties --jars "$METRON_HOME/lib/$STELLAR_JAR" diff --git a/metron-analytics/metron-statistics/pom.xml b/metron-analytics/metron-statistics/pom.xml index 2a4b9ce..4befad0 100644 --- a/metron-analytics/metron-statistics/pom.xml +++ b/metron-analytics/metron-statistics/pom.xml @@ -41,6 +41,7 @@ <groupId>org.apache.metron</groupId> <artifactId>stellar-common</artifactId> <version>${project.parent.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.commons</groupId> @@ -65,6 +66,8 @@ <artifactId>maven-shade-plugin</artifactId> <version>${global_shade_version}</version> <configuration> + <shadedArtifactAttached>true</shadedArtifactAttached> + <shadedClassifierName>uber</shadedClassifierName> <createDependencyReducedPom>true</createDependencyReducedPom> </configuration> <executions> diff --git a/metron-contrib/metron-performance/pom.xml b/metron-contrib/metron-performance/pom.xml index 0518fc8..40992ab 100644 --- a/metron-contrib/metron-performance/pom.xml +++ b/metron-contrib/metron-performance/pom.xml @@ -43,6 +43,12 @@ <scope>provided</scope> </dependency> <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>stellar-common</artifactId> + <version>${project.parent.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>org.apache.kafka</groupId> <artifactId>kafka_2.10</artifactId> <version>${global_kafka_version}</version> @@ -77,6 +83,8 @@ <goal>shade</goal> </goals> <configuration> + <shadedArtifactAttached>true</shadedArtifactAttached> + <shadedClassifierName>uber</shadedClassifierName> <filters> <filter> <artifact>*:*</artifact> diff --git a/metron-contrib/metron-performance/src/main/assembly/assembly.xml b/metron-contrib/metron-performance/src/main/assembly/assembly.xml index 3595284..0b5dfb8 100644 --- a/metron-contrib/metron-performance/src/main/assembly/assembly.xml +++ b/metron-contrib/metron-performance/src/main/assembly/assembly.xml @@ -33,7 +33,7 @@ <fileSet> <directory>${project.basedir}/target</directory> <includes> - <include>${project.artifactId}-${project.version}.jar</include> + <include>${project.artifactId}-${project.version}-uber.jar</include> </includes> <outputDirectory>lib</outputDirectory> <useDefaultExcludes>true</useDefaultExcludes> diff --git a/metron-contrib/metron-performance/src/main/scripts/load_tool.sh b/metron-contrib/metron-performance/src/main/scripts/load_tool.sh index 1b56a6e..da63197 100755 --- a/metron-contrib/metron-performance/src/main/scripts/load_tool.sh +++ b/metron-contrib/metron-performance/src/main/scripts/load_tool.sh @@ -30,7 +30,7 @@ fi export METRON_VERSION=${project.version} export METRON_HOME=/usr/metron/$METRON_VERSION export CLASSNAME="org.apache.metron.performance.load.LoadGenerator" -export GEN_JAR=${project.artifactId}-$METRON_VERSION.jar +export GEN_JAR=${project.artifactId}-$METRON_VERSION-uber.jar export PARSERS_JAR=metron-parsers-$METRON_VERSION-uber.jar java -cp $METRON_HOME/lib/$GEN_JAR:$METRON_HOME/lib/$PARSERS_JAR $CLASSNAME "$@" diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml index 417c225..473c860 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml @@ -508,6 +508,9 @@ <package> <name>metron-performance</name> </package> + <package> + <name>stellar-common</name> + </package> </packages> </osSpecific> <osSpecific> diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec index e12f6cd..673ace5 100644 --- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec +++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec @@ -66,6 +66,7 @@ Source19: metron-profiler-repl-%{full_version}-archive.tar.gz Source20: metron-parsing-storm-%{full_version}-archive.tar.gz Source21: metron-parsers-%{full_version}-archive.tar.gz Source22: metron-hbase-server-%{full_version}-archive.tar.gz +Source23: stellar-common-%{full_version}-archive.tar.gz %description Apache Metron provides a scalable advanced security analytics framework @@ -111,6 +112,7 @@ tar -xzf %{SOURCE19} -C %{buildroot}%{metron_home} tar -xzf %{SOURCE20} -C %{buildroot}%{metron_home} tar -xzf %{SOURCE21} -C %{buildroot}%{metron_home} tar -xzf %{SOURCE22} -C %{buildroot}%{metron_home} +tar -xzf %{SOURCE23} -C %{buildroot}%{metron_home} install %{buildroot}%{metron_home}/bin/metron-management-ui %{buildroot}/etc/init.d/ install %{buildroot}%{metron_home}/bin/metron-alerts-ui %{buildroot}/etc/init.d/ @@ -142,7 +144,7 @@ This package installs the Metron common files %{metron_home} %{metron_home}/bin/cluster_info.py %{metron_home}/bin/tgt_renew.py %{metron_home}/config/zookeeper/global.json -%attr(0644,root,root) %{metron_home}/lib/metron-common-%{full_version}.jar +%attr(0644,root,root) %{metron_home}/lib/metron-common-%{full_version}-uber.jar # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -262,7 +264,7 @@ This package installs performance tools useful for Metron %dir %{metron_home}/bin %dir %{metron_home}/lib %{metron_home}/bin/load_tool.sh -%attr(0644,root,root) %{metron_home}/lib/metron-performance-%{full_version}.jar +%attr(0644,root,root) %{metron_home}/lib/metron-performance-%{full_version}-uber.jar # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -288,7 +290,7 @@ This package installs the Metron Parser files %{metron_home}/bin/prune_hdfs_files.sh %{metron_home}/bin/threatintel_bulk_prune.sh %{metron_home}/bin/threatintel_taxii_load.sh -%attr(0644,root,root) %{metron_home}/lib/metron-data-management-%{full_version}.jar +%attr(0644,root,root) %{metron_home}/lib/metron-data-management-%{full_version}-uber.jar # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -455,7 +457,7 @@ This package installs the Metron Management Library %defattr(-,root,root,755) %dir %{metron_root} %dir %{metron_home}/lib -%attr(0644,root,root) %{metron_home}/lib/metron-management-%{full_version}.jar +%attr(0644,root,root) %{metron_home}/lib/metron-management-%{full_version}-uber.jar # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -484,7 +486,7 @@ This package installs the Metron PCAP files %{metron_home} %{metron_home}/bin/pcap_zeppelin_run.sh %{metron_home}/flux/pcap/remote.yaml %{metron_home}/config/zeppelin/metron/metron-pcap.json -%attr(0644,root,root) %{metron_home}/lib/metron-pcap-backend-%{full_version}.jar +%attr(0644,root,root) %{metron_home}/lib/metron-pcap-backend-%{full_version}-uber.jar # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -539,7 +541,7 @@ This package installs the Metron Rest %{metron_home} %{metron_home}/bin/metron-rest.sh %{metron_home}/bin/pcap_to_pdml.sh %{metron_home}/bin/install_metron_knox.sh -%attr(0644,root,root) %{metron_home}/lib/metron-rest-%{full_version}.jar +%attr(0644,root,root) %{metron_home}/lib/metron-rest-%{full_version}-uber.jar # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -676,7 +678,7 @@ This package installs the Metron Profiler for Spark %{metron_home} %dir %{metron_home}/bin %{metron_home}/bin/start_batch_profiler.sh %dir %{metron_home}/lib -%attr(0644,root,root) %{metron_home}/lib/metron-profiler-spark-%{full_version}.jar +%attr(0644,root,root) %{metron_home}/lib/metron-profiler-spark-%{full_version}-uber.jar # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -693,7 +695,7 @@ This package installs the Metron Profiler for the Stellar REPL %{metron_home} %dir %{metron_root} %dir %{metron_home} %dir %{metron_home}/lib -%attr(0644,root,root) %{metron_home}/lib/metron-profiler-repl-%{full_version}.jar +%attr(0644,root,root) %{metron_home}/lib/metron-profiler-repl-%{full_version}-uber.jar # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -716,6 +718,23 @@ This package installs the Metron HBase Server files # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +%package stellar-common +Summary: Stellar Common Files +Group: Applications/Internet +Provides: stellar-common = %{version} + +%description stellar-common +This package installs the Stellar Common files + +%files stellar-common +%defattr(-,root,root,755) +%dir %{metron_root} +%dir %{metron_home} +%dir %{metron_home}/lib +%attr(0644,root,root) %{metron_home}/lib/stellar-common-%{full_version}-uber.jar + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + %post config chkconfig --add metron-management-ui chkconfig --add metron-alerts-ui diff --git a/metron-deployment/packaging/docker/rpm-docker/pom.xml b/metron-deployment/packaging/docker/rpm-docker/pom.xml index 81e0c10..63c0402 100644 --- a/metron-deployment/packaging/docker/rpm-docker/pom.xml +++ b/metron-deployment/packaging/docker/rpm-docker/pom.xml @@ -245,6 +245,12 @@ <include>*.tar.gz</include> </includes> </resource> + <resource> + <directory>${metron_dir}/metron-stellar/stellar-common/target/</directory> + <includes> + <include>*.tar.gz</include> + </includes> + </resource> </resources> </configuration> </execution> diff --git a/metron-interface/metron-rest/pom.xml b/metron-interface/metron-rest/pom.xml index 84269c2..c67a5d5 100644 --- a/metron-interface/metron-rest/pom.xml +++ b/metron-interface/metron-rest/pom.xml @@ -191,6 +191,12 @@ </exclusion> </exclusions> </dependency> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>stellar-common</artifactId> + <version>${project.parent.version}</version> + <scope>provided</scope> + </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-client</artifactId> @@ -200,6 +206,10 @@ <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> </exclusion> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + </exclusion> </exclusions> </dependency> <dependency> @@ -211,12 +221,22 @@ <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> </exclusion> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + </exclusion> </exclusions> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-app</artifactId> <version>${global_hadoop_version}</version> + <exclusions> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> @@ -533,6 +553,8 @@ </dependency> </dependencies> <configuration> + <shadedArtifactAttached>true</shadedArtifactAttached> + <shadedClassifierName>uber</shadedClassifierName> <keepDependenciesWithProvidedScope>true</keepDependenciesWithProvidedScope> <createDependencyReducedPom>true</createDependencyReducedPom> <filters> diff --git a/metron-interface/metron-rest/src/main/assembly/assembly.xml b/metron-interface/metron-rest/src/main/assembly/assembly.xml index e0e4f14..ee0fe02 100644 --- a/metron-interface/metron-rest/src/main/assembly/assembly.xml +++ b/metron-interface/metron-rest/src/main/assembly/assembly.xml @@ -45,7 +45,7 @@ <fileSet> <directory>${project.basedir}/target</directory> <includes> - <include>${project.artifactId}-${project.version}.jar</include> + <include>${project.artifactId}-${project.version}-uber.jar</include> </includes> <outputDirectory>lib</outputDirectory> <useDefaultExcludes>true</useDefaultExcludes> diff --git a/metron-interface/metron-rest/src/main/scripts/metron-rest.sh b/metron-interface/metron-rest/src/main/scripts/metron-rest.sh index 19d0a33..87aee10 100644 --- a/metron-interface/metron-rest/src/main/scripts/metron-rest.sh +++ b/metron-interface/metron-rest/src/main/scripts/metron-rest.sh @@ -40,6 +40,7 @@ PARSER_CONTRIB=${PARSER_CONTRIB:-$METRON_HOME/parser_contrib} INDEXING_CONTRIB=${INDEXING_CONTRIB:-$METRON_HOME/indexing_contrib} METRON_PARSERS_PLATFORM="${METRON_PARSERS_PLATFORM:-storm}" PARSER_LIB=$(find $METRON_HOME/lib/ -name metron-parsing-${METRON_PARSERS_PLATFORM}*.jar) +STELLAR_LIB=$METRON_HOME/lib/stellar-common-$METRON_VERSION-uber.jar echo "METRON_VERSION=${METRON_VERSION}" echo "METRON_HOME=${METRON_HOME}" @@ -61,6 +62,7 @@ rest_jar_pattern="${METRON_HOME}/lib/metron-rest*.jar" rest_files=( ${rest_jar_pattern} ) echo "Default metron-rest jar is: ${rest_files[0]}" METRON_REST_CLASSPATH+=":${rest_files[0]}" +METRON_REST_CLASSPATH+=":$STELLAR_LIB" METRON_REST_CLASSPATH+=":$PARSER_LIB" if [ -d "$PARSER_CONTRIB" ]; then diff --git a/metron-platform/elasticsearch-shaded/pom.xml b/metron-platform/elasticsearch-shaded/pom.xml index a1ebe50..8651e05 100644 --- a/metron-platform/elasticsearch-shaded/pom.xml +++ b/metron-platform/elasticsearch-shaded/pom.xml @@ -103,6 +103,8 @@ <artifactId>maven-shade-plugin</artifactId> <version>${global_shade_version}</version> <configuration> + <shadedArtifactAttached>true</shadedArtifactAttached> + <shadedClassifierName>uber</shadedClassifierName> <createDependencyReducedPom>true</createDependencyReducedPom> </configuration> <executions> diff --git a/metron-platform/metron-common-streaming/metron-common-storm/pom.xml b/metron-platform/metron-common-streaming/metron-common-storm/pom.xml index 30371bd..27f3aa1 100644 --- a/metron-platform/metron-common-streaming/metron-common-storm/pom.xml +++ b/metron-platform/metron-common-streaming/metron-common-storm/pom.xml @@ -36,6 +36,12 @@ <version>${project.parent.version}</version> </dependency> <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>stellar-common</artifactId> + <version>${project.parent.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>org.apache.storm</groupId> <artifactId>storm-core</artifactId> <version>${global_storm_version}</version> diff --git a/metron-platform/metron-common/pom.xml b/metron-platform/metron-common/pom.xml index d215180..0bcf343 100644 --- a/metron-platform/metron-common/pom.xml +++ b/metron-platform/metron-common/pom.xml @@ -48,6 +48,7 @@ <groupId>org.apache.metron</groupId> <artifactId>stellar-common</artifactId> <version>${project.parent.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.commons</groupId> @@ -272,10 +273,10 @@ <artifactId>reflections</artifactId> <version>0.9.10</version> <exclusions> - <exclusion> - <groupId>com.google.code.findbugs</groupId> - <artifactId>annotations</artifactId> - </exclusion> + <exclusion> + <groupId>com.google.code.findbugs</groupId> + <artifactId>annotations</artifactId> + </exclusion> </exclusions> </dependency> <dependency> @@ -338,14 +339,14 @@ <artifactId>antlr4-maven-plugin</artifactId> <version>${global_antlr_version}</version> <configuration> - <outputDirectory>${basedir}/src/main/java</outputDirectory> + <outputDirectory>${basedir}/src/main/java</outputDirectory> </configuration> <executions> - <execution> - <goals> - <goal>antlr4</goal> - </goals> - </execution> + <execution> + <goals> + <goal>antlr4</goal> + </goals> + </execution> </executions> </plugin> @@ -368,16 +369,18 @@ <goal>shade</goal> </goals> <configuration> - <filters> - <filter> - <artifact>*:*</artifact> - <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> - </excludes> - </filter> - </filters> + <shadedArtifactAttached>true</shadedArtifactAttached> + <shadedClassifierName>uber</shadedClassifierName> + <filters> + <filter> + <artifact>*:*</artifact> + <excludes> + <exclude>META-INF/*.SF</exclude> + <exclude>META-INF/*.DSA</exclude> + <exclude>META-INF/*.RSA</exclude> + </excludes> + </filter> + </filters> <relocations> <relocation> <pattern>com.google.common</pattern> @@ -390,13 +393,13 @@ </relocations> <transformers> <transformer - implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer"> - <resources> + implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer"> + <resources> <resource>.yaml</resource> <resource>LICENSE.txt</resource> <resource>ASL2.0</resource> <resource>NOTICE.txt</resource> - </resources> + </resources> </transformer> <!-- UNCOMMENT THIS IF YOU NEED TO REGENERATE THE BEST GUESS NOTICES FILE WHICH REQUIRES PRUNING EVERY RELEASE --> <!--transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer"> @@ -404,7 +407,7 @@ <projectName>${project.name}</projectName> </transformer--> <transformer - implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" /> + implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/> <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> <mainClass></mainClass> diff --git a/metron-platform/metron-common/src/main/assembly/assembly.xml b/metron-platform/metron-common/src/main/assembly/assembly.xml index e0e4f14..ee0fe02 100644 --- a/metron-platform/metron-common/src/main/assembly/assembly.xml +++ b/metron-platform/metron-common/src/main/assembly/assembly.xml @@ -45,7 +45,7 @@ <fileSet> <directory>${project.basedir}/target</directory> <includes> - <include>${project.artifactId}-${project.version}.jar</include> + <include>${project.artifactId}-${project.version}-uber.jar</include> </includes> <outputDirectory>lib</outputDirectory> <useDefaultExcludes>true</useDefaultExcludes> diff --git a/metron-platform/metron-common/src/main/scripts/stellar b/metron-platform/metron-common/src/main/scripts/stellar index 142cc29..6f9418b 100644 --- a/metron-platform/metron-common/src/main/scripts/stellar +++ b/metron-platform/metron-common/src/main/scripts/stellar @@ -38,7 +38,8 @@ set -u export METRON_PARSERS_PLATFORM="${METRON_PARSERS_PLATFORM:-storm}" export HBASE_CONFIGS=$(hbase classpath) -export STELLAR_LIB=$(find $METRON_HOME/lib/ -name metron-parsing-${METRON_PARSERS_PLATFORM}*.jar) +export STELLAR_LIB=$(find $METRON_HOME/lib/ -name stellar-common*.jar) +export PARSING_LIB=$(find $METRON_HOME/lib/ -name metron-parsing-${METRON_PARSERS_PLATFORM}*.jar) export MANAGEMENT_LIB=$(find $METRON_HOME/lib/ -name metron-management*.jar) export PROFILER_LIB=$(find $METRON_HOME/lib/ -name metron-profiler-repl*.jar) -java $METRON_JVMFLAGS -cp "${CONTRIB:-$METRON_HOME/contrib/*}:$STELLAR_LIB:$MANAGEMENT_LIB:$PROFILER_LIB:$HBASE_CONFIGS" org.apache.metron.stellar.common.shell.cli.StellarShell "$@" +java $METRON_JVMFLAGS -cp "${CONTRIB:-$METRON_HOME/contrib/*}:$STELLAR_LIB:$PARSING_LIB:$MANAGEMENT_LIB:$PROFILER_LIB:$HBASE_CONFIGS" org.apache.metron.stellar.common.shell.cli.StellarShell "$@" diff --git a/metron-platform/metron-common/src/main/scripts/zk_load_configs.sh b/metron-platform/metron-common/src/main/scripts/zk_load_configs.sh index b22bb4b..3e8f787 100755 --- a/metron-platform/metron-common/src/main/scripts/zk_load_configs.sh +++ b/metron-platform/metron-common/src/main/scripts/zk_load_configs.sh @@ -30,9 +30,10 @@ export METRON_VERSION=${project.version} export METRON_HOME=/usr/metron/$METRON_VERSION export CLASSNAME="org.apache.metron.common.cli.ConfigurationManager" export JAR=metron-parsers-common-$METRON_VERSION-uber.jar +export STELLAR_JAR=stellar-common-$METRON_VERSION-uber.jar # TODO export the Storm jar? export HBASE_HOME=${HBASE_HOME:-/usr/hdp/current/hbase-client} -CP=$METRON_HOME/lib/$JAR:${HBASE_HOME}/lib/hbase-server.jar:`${HBASE_HOME}/bin/hbase classpath` +CP=$METRON_HOME/lib/$JAR:$METRON_HOME/lib/$STELLAR_JAR:${HBASE_HOME}/lib/hbase-server.jar:`${HBASE_HOME}/bin/hbase classpath` java $METRON_JVMFLAGS -cp $CP $CLASSNAME "$@" diff --git a/metron-platform/metron-data-management/pom.xml b/metron-platform/metron-data-management/pom.xml index 3e67c59..d4fae01 100644 --- a/metron-platform/metron-data-management/pom.xml +++ b/metron-platform/metron-data-management/pom.xml @@ -33,9 +33,16 @@ <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> - <version>${guava_version}</version> + <version>${global_hbase_guava_version}</version> </dependency> <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>stellar-common</artifactId> + <version>${project.parent.version}</version> + <scope>provided</scope> + </dependency> + + <dependency> <groupId>javax.xml.bind</groupId> <artifactId>jaxb-api</artifactId> <version>2.2.11</version> @@ -181,10 +188,10 @@ <artifactId>hbase-common</artifactId> <version>${global_hbase_version}</version> <exclusions> - <exclusion> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> - </exclusion> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + </exclusion> </exclusions> </dependency> <dependency> @@ -237,7 +244,7 @@ <version>2.1.14</version> <scope>test</scope> </dependency> - <dependency> + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-hdfs</artifactId> <version>${global_hadoop_version}</version> @@ -262,14 +269,14 @@ <version>${global_hbase_version}</version> <scope>test</scope> <exclusions> - <exclusion> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-hdfs</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-minicluster</artifactId> - </exclusion> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-minicluster</artifactId> + </exclusion> <exclusion> <groupId>org.slf4j</groupId> <artifactId>slf4j-log4j12</artifactId> @@ -367,17 +374,19 @@ <goal>shade</goal> </goals> <configuration> - <createDependencyReducedPom>false</createDependencyReducedPom> - <filters> - <filter> - <artifact>*:*</artifact> - <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> - </excludes> - </filter> - </filters> + <shadedArtifactAttached>true</shadedArtifactAttached> + <shadedClassifierName>uber</shadedClassifierName> + <createDependencyReducedPom>false</createDependencyReducedPom> + <filters> + <filter> + <artifact>*:*</artifact> + <excludes> + <exclude>META-INF/*.SF</exclude> + <exclude>META-INF/*.DSA</exclude> + <exclude>META-INF/*.RSA</exclude> + </excludes> + </filter> + </filters> <relocations> <relocation> <pattern>com.google.common</pattern> @@ -387,16 +396,16 @@ <pattern>org.apache.http</pattern> <shadedPattern>org.apache.metron.httpcore.dataload</shadedPattern> </relocation> - </relocations> - <transformers> + </relocations> + <transformers> <transformer - implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer"> - <resources> + implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer"> + <resources> <resource>.yaml</resource> <resource>LICENSE.txt</resource> <resource>ASL2.0</resource> <resource>NOTICE.txt</resource> - </resources> + </resources> </transformer> <!-- UNCOMMENT THIS IF YOU NEED TO REGENERATE THE BEST GUESS NOTICES FILE WHICH REQUIRES PRUNING EVERY RELEASE --> <!--transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer"> @@ -448,8 +457,24 @@ </execution> </executions> </plugin> - </plugins> - <resources> + + <!-- This plugin changes the integration tests in this module to use an external dependency jar for stellar-common. This is necessary because conflicting versions of + guava is required by code in both this module and the stellar-common module. --> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <version>${global_surefire_version}</version> + <configuration> + <additionalClasspathElements> + <additionalClasspathElement>${basedir}/../../metron-stellar/stellar-common/target/stellar-common-${project.parent.version}-uber.jar</additionalClasspathElement> + </additionalClasspathElements> + <classpathDependencyExcludes> + <classpathDependencyExclude>org.apache.metron:stellar-common</classpathDependencyExclude> + </classpathDependencyExcludes> + </configuration> + </plugin> + </plugins> + <resources> <resource> <directory>src/main/resources</directory> </resource> diff --git a/metron-platform/metron-data-management/src/main/assembly/assembly.xml b/metron-platform/metron-data-management/src/main/assembly/assembly.xml index 3595284..0b5dfb8 100644 --- a/metron-platform/metron-data-management/src/main/assembly/assembly.xml +++ b/metron-platform/metron-data-management/src/main/assembly/assembly.xml @@ -33,7 +33,7 @@ <fileSet> <directory>${project.basedir}/target</directory> <includes> - <include>${project.artifactId}-${project.version}.jar</include> + <include>${project.artifactId}-${project.version}-uber.jar</include> </includes> <outputDirectory>lib</outputDirectory> <useDefaultExcludes>true</useDefaultExcludes> diff --git a/metron-platform/metron-data-management/src/main/scripts/flatfile_loader.sh b/metron-platform/metron-data-management/src/main/scripts/flatfile_loader.sh index feccc33..5453bf4 100755 --- a/metron-platform/metron-data-management/src/main/scripts/flatfile_loader.sh +++ b/metron-platform/metron-data-management/src/main/scripts/flatfile_loader.sh @@ -30,15 +30,16 @@ fi export METRON_VERSION=${project.version} export METRON_HOME=/usr/metron/$METRON_VERSION export CLASSNAME="org.apache.metron.dataloads.nonbulk.flatfile.SimpleEnrichmentFlatFileLoader" -export DM_JAR=${project.artifactId}-$METRON_VERSION.jar +export DM_JAR=${project.artifactId}-$METRON_VERSION-uber.jar +export STELLAR_JAR=stellar-common-$METRON_VERSION-uber.jar export HBASE_CONF=${HBASE_CONF:-/etc/hbase/conf} export HADOOP_OPTS="$HADOOP_OPTS $METRON_JVMFLAGS" if [ $(which hadoop) ] then - export HADOOP_CLASSPATH="$HBASE_CONF" + export HADOOP_CLASSPATH="$METRON_HOME/lib/$DM_JAR:$HBASE_CONF:$METRON_HOME/lib/$STELLAR_JAR" hadoop jar $METRON_HOME/lib/$DM_JAR $CLASSNAME "$@" else echo "Warning: Metron cannot find the hadoop client on this node. This means that loading via Map Reduce will NOT function." - CP=$METRON_HOME/lib/$DM_JAR:$HBASE_CONF + CP=$METRON_HOME/lib/$DM_JAR:$HBASE_CONF:$METRON_HOME/lib/$STELLAR_JAR java $METRON_JVMFLAGS -cp $CP $CLASSNAME "$@" fi diff --git a/metron-platform/metron-data-management/src/main/scripts/flatfile_summarizer.sh b/metron-platform/metron-data-management/src/main/scripts/flatfile_summarizer.sh index 018d61a..1279ecc 100755 --- a/metron-platform/metron-data-management/src/main/scripts/flatfile_summarizer.sh +++ b/metron-platform/metron-data-management/src/main/scripts/flatfile_summarizer.sh @@ -30,12 +30,13 @@ fi export METRON_VERSION=${project.version} export METRON_HOME=/usr/metron/$METRON_VERSION export CLASSNAME="org.apache.metron.dataloads.nonbulk.flatfile.SimpleFlatFileSummarizer" -export DM_JAR=${project.artifactId}-$METRON_VERSION.jar +export DM_JAR=${project.artifactId}-$METRON_VERSION-uber.jar +export STELLAR_JAR=stellar-common-$METRON_VERSION-uber.jar export HBASE_HOME=${HBASE_HOME:-/usr/hdp/current/hbase-client} export HADOOP_OPTS="$HADOOP_OPTS $METRON_JVMFLAGS" if [ $(which hadoop) ] then - HADOOP_CLASSPATH=${HBASE_HOME}/lib/hbase-server.jar:`${HBASE_HOME}/bin/hbase classpath` + HADOOP_CLASSPATH=$METRON_HOME/lib/$DM_JAR:$METRON_HOME/lib/$STELLAR_JAR:${HBASE_HOME}/lib/hbase-server.jar:`${HBASE_HOME}/bin/hbase classpath` for jar in $(echo $HADOOP_CLASSPATH | sed 's/:/ /g');do if [ -f $jar ];then LIBJARS="$jar,$LIBJARS" @@ -45,7 +46,7 @@ then hadoop jar $METRON_HOME/lib/$DM_JAR $CLASSNAME -libjars ${LIBJARS} "$@" else echo "Warning: Metron cannot find the hadoop client on this node. This means that loading via Map Reduce will NOT function." - CP=$METRON_HOME/lib/$DM_JAR:/usr/metron/${METRON_VERSION}/lib/taxii-1.1.0.1.jar:`${HBASE_HOME}/bin/hbase classpath` + CP=$METRON_HOME/lib/$DM_JAR:$METRON_HOME/lib/$STELLAR_JAR:/usr/metron/${METRON_VERSION}/lib/taxii-1.1.0.1.jar:`${HBASE_HOME}/bin/hbase classpath` java $METRON_JVMFLAGS -cp $CP $CLASSNAME "$@" fi diff --git a/metron-platform/metron-data-management/src/main/scripts/maxmind_enrichment_load.sh b/metron-platform/metron-data-management/src/main/scripts/maxmind_enrichment_load.sh index cf43b14..0af400d 100644 --- a/metron-platform/metron-data-management/src/main/scripts/maxmind_enrichment_load.sh +++ b/metron-platform/metron-data-management/src/main/scripts/maxmind_enrichment_load.sh @@ -26,17 +26,17 @@ if [ -e /usr/libexec/bigtop-detect-javahome ]; then elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then . /usr/lib/bigtop-utils/bigtop-detect-javahome fi - +export METRON_VERSION=${project.version} +export METRON_HOME=/usr/metron/$METRON_VERSION +export DM_JAR=${project.artifactId}-$METRON_VERSION-uber.jar +export STELLAR_JAR=stellar-common-$METRON_VERSION-uber.jar export HBASE_HOME=${HBASE_HOME:-/usr/hdp/current/hbase-client} -HADOOP_CLASSPATH=${HBASE_HOME}/lib/hbase-server.jar:`${HBASE_HOME}/bin/hbase classpath` +HADOOP_CLASSPATH=${HBASE_HOME}/lib/hbase-server.jar:${METRON_HOME}/lib/${STELLAR_JAR}:`${HBASE_HOME}/bin/hbase classpath` for jar in $(echo $HADOOP_CLASSPATH | sed 's/:/ /g');do if [ -f $jar ];then LIBJARS="$jar,$LIBJARS" fi done export HADOOP_CLASSPATH -export METRON_VERSION=${project.version} -export METRON_HOME=/usr/metron/$METRON_VERSION -export DM_JAR=${project.artifactId}-$METRON_VERSION.jar export HADOOP_OPTS="$HADOOP_OPTS $METRON_JVMFLAGS" hadoop jar $METRON_HOME/lib/$DM_JAR org.apache.metron.dataloads.nonbulk.geo.MaxmindDbEnrichmentLoader "$@" diff --git a/metron-platform/metron-data-management/src/main/scripts/prune_hdfs_files.sh b/metron-platform/metron-data-management/src/main/scripts/prune_hdfs_files.sh index b8d9372..8670074 100644 --- a/metron-platform/metron-data-management/src/main/scripts/prune_hdfs_files.sh +++ b/metron-platform/metron-data-management/src/main/scripts/prune_hdfs_files.sh @@ -17,5 +17,5 @@ # limitations under the License. # -yarn jar /usr/metron/${project.version}/lib/${project.artifactId}-${project.version}.jar org.apache.metron.dataloads.bulk.HDFSDataPruner "$@" +yarn jar /usr/metron/${project.version}/lib/${project.artifactId}-${project.version}-uber.jar org.apache.metron.dataloads.bulk.HDFSDataPruner "$@" diff --git a/metron-platform/metron-data-management/src/main/scripts/threatintel_bulk_prune.sh b/metron-platform/metron-data-management/src/main/scripts/threatintel_bulk_prune.sh index 6156027..47daa2f 100755 --- a/metron-platform/metron-data-management/src/main/scripts/threatintel_bulk_prune.sh +++ b/metron-platform/metron-data-management/src/main/scripts/threatintel_bulk_prune.sh @@ -36,5 +36,5 @@ done export HADOOP_CLASSPATH export METRON_VERSION=${project.version} export METRON_HOME=/usr/metron/$METRON_VERSION -export DM_JAR=${project.artifactId}-$METRON_VERSION.jar +export DM_JAR=${project.artifactId}-$METRON_VERSION-uber.jar hadoop jar $METRON_HOME/lib/$DM_JAR org.apache.metron.dataloads.bulk.LeastRecentlyUsedPruner -libjars ${LIBJARS} "$@" diff --git a/metron-platform/metron-data-management/src/main/scripts/threatintel_taxii_load.sh b/metron-platform/metron-data-management/src/main/scripts/threatintel_taxii_load.sh index 953a86e..15ca27a 100755 --- a/metron-platform/metron-data-management/src/main/scripts/threatintel_taxii_load.sh +++ b/metron-platform/metron-data-management/src/main/scripts/threatintel_taxii_load.sh @@ -30,7 +30,7 @@ fi export HBASE_HOME=${HBASE_HOME:-/usr/hdp/current/hbase-client} export METRON_VERSION=${project.version} export METRON_HOME=/usr/metron/$METRON_VERSION -export DM_JAR=${project.artifactId}-$METRON_VERSION.jar +export DM_JAR=${project.artifactId}-$METRON_VERSION-uber.jar CP=$METRON_HOME/lib/$DM_JAR:/usr/metron/$METRON_VERSION/lib/taxii-1.1.0.1.jar:`${HBASE_HOME}/bin/hbase classpath` HADOOP_CLASSPATH=$(echo $CP ) diff --git a/metron-platform/metron-elasticsearch/metron-elasticsearch-common/pom.xml b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/pom.xml index 190bcb2..15df035 100644 --- a/metron-platform/metron-elasticsearch/metron-elasticsearch-common/pom.xml +++ b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/pom.xml @@ -38,6 +38,12 @@ <groupId>org.apache.metron</groupId> <artifactId>metron-common</artifactId> <version>${project.parent.version}</version> + <exclusions> + <exclusion> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.apache.metron</groupId> @@ -62,9 +68,16 @@ </exclusions> </dependency> <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - <version>${guava_version}</version> + <groupId>org.apache.metron</groupId> + <artifactId>stellar-common</artifactId> + <version>${project.parent.version}</version> + <scope>provided</scope> + <exclusions> + <exclusion> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.apache.hbase</groupId> @@ -117,6 +130,14 @@ <groupId>org.slf4j</groupId> <artifactId>slf4j-log4j12</artifactId> </exclusion> + <exclusion> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpcore</artifactId> + </exclusion> </exclusions> </dependency> <dependency> diff --git a/metron-platform/metron-elasticsearch/metron-elasticsearch-storm/pom.xml b/metron-platform/metron-elasticsearch/metron-elasticsearch-storm/pom.xml index 1804098..1ba1e27 100644 --- a/metron-platform/metron-elasticsearch/metron-elasticsearch-storm/pom.xml +++ b/metron-platform/metron-elasticsearch/metron-elasticsearch-storm/pom.xml @@ -40,6 +40,24 @@ <groupId>org.apache.metron</groupId> <artifactId>metron-storm-kafka</artifactId> <version>${project.parent.version}</version> + <exclusions> + <exclusion> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>stellar-common</artifactId> + <version>${project.parent.version}</version> + <scope>provided</scope> + <exclusions> + <exclusion> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.apache.storm</groupId> @@ -110,6 +128,7 @@ <type>test-jar</type> <scope>test</scope> </dependency> + </dependencies> <build> <plugins> diff --git a/metron-platform/metron-elasticsearch/metron-elasticsearch-storm/src/main/scripts/start_elasticsearch_topology.sh b/metron-platform/metron-elasticsearch/metron-elasticsearch-storm/src/main/scripts/start_elasticsearch_topology.sh index 20ce23f..d3a29c0 100755 --- a/metron-platform/metron-elasticsearch/metron-elasticsearch-storm/src/main/scripts/start_elasticsearch_topology.sh +++ b/metron-platform/metron-elasticsearch/metron-elasticsearch-storm/src/main/scripts/start_elasticsearch_topology.sh @@ -19,10 +19,11 @@ METRON_VERSION=${project.version} METRON_HOME=/usr/metron/$METRON_VERSION TOPOLOGY_JAR=${project.artifactId}-$METRON_VERSION-uber.jar +STELLAR_JAR=stellar-common-$METRON_VERSION-uber.jar INDEXING_CONTRIB=${INDEXING_CONTRIB:-$METRON_HOME/indexing_contrib} if [ -d "$INDEXING_CONTRIB" ]; then export EXTRA_JARS=$(ls -m $INDEXING_CONTRIB/*.jar | tr -d ' ' | tr -d '\n' | sed 's/\/\//\//g') - storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/indexing/random_access/remote.yaml --filter $METRON_HOME/config/elasticsearch.properties --jars "$EXTRA_JARS" + storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/indexing/random_access/remote.yaml --filter $METRON_HOME/config/elasticsearch.properties --jars "$METRON_HOME/lib/$STELLAR_JAR,$EXTRA_JARS" else - storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/indexing/random_access/remote.yaml --filter $METRON_HOME/config/elasticsearch.properties + storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/indexing/random_access/remote.yaml --filter $METRON_HOME/config/elasticsearch.properties --jars "$METRON_HOME/lib/$STELLAR_JAR" fi diff --git a/metron-platform/metron-enrichment/metron-enrichment-common/pom.xml b/metron-platform/metron-enrichment/metron-enrichment-common/pom.xml index 77ec1af..67938bc 100644 --- a/metron-platform/metron-enrichment/metron-enrichment-common/pom.xml +++ b/metron-platform/metron-enrichment/metron-enrichment-common/pom.xml @@ -41,6 +41,7 @@ <groupId>org.apache.metron</groupId> <artifactId>stellar-common</artifactId> <version>${project.parent.version}</version> + <scope>provided</scope> </dependency> <!-- Other --> diff --git a/metron-platform/metron-enrichment/metron-enrichment-storm/pom.xml b/metron-platform/metron-enrichment/metron-enrichment-storm/pom.xml index fe21399..6fe47e4 100644 --- a/metron-platform/metron-enrichment/metron-enrichment-storm/pom.xml +++ b/metron-platform/metron-enrichment/metron-enrichment-storm/pom.xml @@ -61,6 +61,12 @@ <artifactId>metron-statistics</artifactId> <version>${project.parent.version}</version> </dependency> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>stellar-common</artifactId> + <version>${project.parent.version}</version> + <scope>provided</scope> + </dependency> <!-- Hadoop --> diff --git a/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/scripts/start_enrichment_topology.sh b/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/scripts/start_enrichment_topology.sh index d9c2dc4..34886c9 100755 --- a/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/scripts/start_enrichment_topology.sh +++ b/metron-platform/metron-enrichment/metron-enrichment-storm/src/main/scripts/start_enrichment_topology.sh @@ -19,9 +19,10 @@ METRON_VERSION=${project.version} METRON_HOME=/usr/metron/$METRON_VERSION TOPOLOGY_JAR=${project.artifactId}-$METRON_VERSION-uber.jar +STELLAR_JAR=stellar-common-$METRON_VERSION-uber.jar DEFAULT_ARGS="--remote $METRON_HOME/flux/enrichment/remote.yaml --filter $METRON_HOME/config/enrichment.properties" # by passing in different args, the user can execute an alternative enrichment topology ARGS=${@:-$DEFAULT_ARGS} -storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux $ARGS +storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux $ARGS --jars $METRON_HOME/lib/$STELLAR_JAR diff --git a/metron-platform/metron-hbase-client/pom.xml b/metron-platform/metron-hbase-client/pom.xml index 37a7bf5..fe497df 100644 --- a/metron-platform/metron-hbase-client/pom.xml +++ b/metron-platform/metron-hbase-client/pom.xml @@ -66,6 +66,8 @@ <artifactId>maven-shade-plugin</artifactId> <version>${global_shade_version}</version> <configuration> + <shadedArtifactAttached>true</shadedArtifactAttached> + <shadedClassifierName>uber</shadedClassifierName> <createDependencyReducedPom>true</createDependencyReducedPom> <artifactSet> <excludes> diff --git a/metron-platform/metron-hbase-server/pom.xml b/metron-platform/metron-hbase-server/pom.xml index 0cbdeeb..171f800 100644 --- a/metron-platform/metron-hbase-server/pom.xml +++ b/metron-platform/metron-hbase-server/pom.xml @@ -58,6 +58,12 @@ <artifactId>metron-enrichment-common</artifactId> <version>${project.parent.version}</version> </dependency> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>stellar-common</artifactId> + <version>${project.parent.version}</version> + <classifier>uber</classifier> + </dependency> <!-- HBase --> diff --git a/metron-platform/metron-indexing/metron-indexing-common/pom.xml b/metron-platform/metron-indexing/metron-indexing-common/pom.xml index df81532..abbeb99 100644 --- a/metron-platform/metron-indexing/metron-indexing-common/pom.xml +++ b/metron-platform/metron-indexing/metron-indexing-common/pom.xml @@ -44,6 +44,12 @@ <artifactId>metron-writer-common</artifactId> <version>${project.parent.version}</version> </dependency> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>stellar-common</artifactId> + <version>${project.parent.version}</version> + <scope>provided</scope> + </dependency> <!-- Other --> @@ -91,11 +97,6 @@ <scope>provided</scope> </dependency> <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - <version>${guava_version}</version> - </dependency> - <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${global_hadoop_version}</version> diff --git a/metron-platform/metron-indexing/metron-indexing-storm/pom.xml b/metron-platform/metron-indexing/metron-indexing-storm/pom.xml index 581e819..36c0ef8 100644 --- a/metron-platform/metron-indexing/metron-indexing-storm/pom.xml +++ b/metron-platform/metron-indexing/metron-indexing-storm/pom.xml @@ -85,6 +85,12 @@ </exclusions> </dependency> <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>stellar-common</artifactId> + <version>${project.parent.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>org.apache.kafka</groupId> <artifactId>kafka-clients</artifactId> <version>${global_kafka_version}</version> diff --git a/metron-platform/metron-indexing/metron-indexing-storm/src/main/scripts/start_hdfs_topology.sh b/metron-platform/metron-indexing/metron-indexing-storm/src/main/scripts/start_hdfs_topology.sh index 77007be..c29e6c4 100755 --- a/metron-platform/metron-indexing/metron-indexing-storm/src/main/scripts/start_hdfs_topology.sh +++ b/metron-platform/metron-indexing/metron-indexing-storm/src/main/scripts/start_hdfs_topology.sh @@ -19,4 +19,5 @@ METRON_VERSION=${project.version} METRON_HOME=/usr/metron/$METRON_VERSION TOPOLOGY_JAR=metron-elasticsearch-storm-$METRON_VERSION-uber.jar -storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/indexing/batch/remote.yaml --filter $METRON_HOME/config/hdfs.properties +STELLAR_JAR=stellar-common-$METRON_VERSION-uber.jar +storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/indexing/batch/remote.yaml --filter $METRON_HOME/config/hdfs.properties --jars "$METRON_HOME/lib/$STELLAR_JAR" diff --git a/metron-platform/metron-integration-test/pom.xml b/metron-platform/metron-integration-test/pom.xml index 427290f..c89bce0 100644 --- a/metron-platform/metron-integration-test/pom.xml +++ b/metron-platform/metron-integration-test/pom.xml @@ -228,6 +228,7 @@ <groupId>org.apache.metron</groupId> <artifactId>stellar-common</artifactId> <version>${project.parent.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> diff --git a/metron-platform/metron-management/pom.xml b/metron-platform/metron-management/pom.xml index c5c319e..e308f8e 100644 --- a/metron-platform/metron-management/pom.xml +++ b/metron-platform/metron-management/pom.xml @@ -72,6 +72,7 @@ <groupId>org.apache.metron</groupId> <artifactId>stellar-common</artifactId> <version>${project.parent.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>com.jakewharton.fliptables</groupId> @@ -211,6 +212,8 @@ <artifactId>maven-shade-plugin</artifactId> <version>${global_shade_version}</version> <configuration> + <shadedArtifactAttached>true</shadedArtifactAttached> + <shadedClassifierName>uber</shadedClassifierName> <createDependencyReducedPom>false</createDependencyReducedPom> <artifactSet> <excludes> diff --git a/metron-platform/metron-management/src/main/assembly/assembly.xml b/metron-platform/metron-management/src/main/assembly/assembly.xml index a7e20ed..4cc9735 100644 --- a/metron-platform/metron-management/src/main/assembly/assembly.xml +++ b/metron-platform/metron-management/src/main/assembly/assembly.xml @@ -21,7 +21,7 @@ <fileSet> <directory>${project.basedir}/target</directory> <includes> - <include>${project.artifactId}-${project.version}.jar</include> + <include>${project.artifactId}-${project.version}-uber.jar</include> </includes> <outputDirectory>lib</outputDirectory> <useDefaultExcludes>true</useDefaultExcludes> diff --git a/metron-platform/metron-parsing/metron-parsers-common/pom.xml b/metron-platform/metron-parsing/metron-parsers-common/pom.xml index 631bdd8..407d311 100644 --- a/metron-platform/metron-parsing/metron-parsers-common/pom.xml +++ b/metron-platform/metron-parsing/metron-parsers-common/pom.xml @@ -55,6 +55,12 @@ <version>${project.parent.version}</version> </dependency> <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>stellar-common</artifactId> + <version>${project.parent.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${global_hadoop_version}</version> diff --git a/metron-platform/metron-parsing/metron-parsers/pom.xml b/metron-platform/metron-parsing/metron-parsers/pom.xml index d888e19..cbf9c2f 100644 --- a/metron-platform/metron-parsing/metron-parsers/pom.xml +++ b/metron-platform/metron-parsing/metron-parsers/pom.xml @@ -41,6 +41,12 @@ <artifactId>metron-common</artifactId> <version>${project.parent.version}</version> </dependency> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>stellar-common</artifactId> + <version>${project.parent.version}</version> + <scope>provided</scope> + </dependency> <!-- 3rd party dependencies --> <dependency> diff --git a/metron-platform/metron-parsing/metron-parsing-storm/pom.xml b/metron-platform/metron-parsing/metron-parsing-storm/pom.xml index 2c5fd37..98bfcff 100644 --- a/metron-platform/metron-parsing/metron-parsing-storm/pom.xml +++ b/metron-platform/metron-parsing/metron-parsing-storm/pom.xml @@ -56,6 +56,12 @@ <artifactId>metron-writer-storm</artifactId> <version>${project.parent.version}</version> </dependency> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>stellar-common</artifactId> + <version>${project.parent.version}</version> + <scope>provided</scope> + </dependency> <!-- Storm dependencies --> <dependency> diff --git a/metron-platform/metron-parsing/metron-parsing-storm/src/main/scripts/start_parser_topology.sh b/metron-platform/metron-parsing/metron-parsing-storm/src/main/scripts/start_parser_topology.sh index db777dc..0b21f35 100755 --- a/metron-platform/metron-parsing/metron-parsing-storm/src/main/scripts/start_parser_topology.sh +++ b/metron-platform/metron-parsing/metron-parsing-storm/src/main/scripts/start_parser_topology.sh @@ -19,13 +19,15 @@ METRON_VERSION=${project.version} METRON_HOME=/usr/metron/$METRON_VERSION TOPOLOGY_JAR=metron-parsing-storm-$METRON_VERSION-uber.jar +STELLAR_JAR=stellar-common-$METRON_VERSION-uber.jar PARSER_CONTRIB=${PARSER_CONTRIB:-$METRON_HOME/parser_contrib} +EXTRA_JARS="--jars $METRON_HOME/lib/$STELLAR_JAR" EXTRA_ARGS="$@" if [ -d "$PARSER_CONTRIB" ]; then export STORM_EXT_CLASSPATH=$METRON_HOME/lib/$TOPOLOGY_JAR - export EXTRA_JARS=$(ls -m $PARSER_CONTRIB/*.jar | tr -d ' ' | tr -d '\n' | sed 's/\/\//\//g') + export PARSER_CONTRIB_JARS=$(ls -m $PARSER_CONTRIB/*.jar | tr -d ' ' | tr -d '\n' | sed 's/\/\//\//g') STORM_VERSION=`storm version | grep Storm | awk '{print $2}' | cut -d. -f1,2,3` STORM_COMPAT_CUTOFF="1.1.0" @@ -33,7 +35,7 @@ if [ -d "$PARSER_CONTRIB" ]; then if [ "$STORM_COMPAT_VERSION" = "$STORM_COMPAT_CUTOFF" ]; then echo "Third party parser support for Storm >= 1.1.0; found Storm $STORM_VERSION" - EXTRA_ARGS+=" --jars $EXTRA_JARS" + EXTRA_JARS+=",$PARSER_CONTRIB_JARS" else echo "Legacy third party parser support for Storm < 1.1.0; found Storm $STORM_VERSION" EXTRA_ARGS+=" -c client.jartransformer.class=org.apache.metron.parsers.topology.MergeAndShadeTransformer" @@ -41,4 +43,4 @@ if [ -d "$PARSER_CONTRIB" ]; then fi echo "Submitting parser topology; args='$EXTRA_ARGS'" -storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.metron.parsers.topology.ParserTopologyCLI $EXTRA_ARGS +storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.metron.parsers.topology.ParserTopologyCLI $EXTRA_ARGS $EXTRA_JARS diff --git a/metron-platform/metron-pcap-backend/pom.xml b/metron-platform/metron-pcap-backend/pom.xml index ccc69b8..5042b70 100644 --- a/metron-platform/metron-pcap-backend/pom.xml +++ b/metron-platform/metron-pcap-backend/pom.xml @@ -58,6 +58,12 @@ </exclusions> </dependency> <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>stellar-common</artifactId> + <version>${project.parent.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <version>${global_junit_version}</version> @@ -248,16 +254,18 @@ <goal>shade</goal> </goals> <configuration> - <filters> - <filter> - <artifact>*:*</artifact> - <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> - </excludes> - </filter> - </filters> + <shadedArtifactAttached>true</shadedArtifactAttached> + <shadedClassifierName>uber</shadedClassifierName> + <filters> + <filter> + <artifact>*:*</artifact> + <excludes> + <exclude>META-INF/*.SF</exclude> + <exclude>META-INF/*.DSA</exclude> + <exclude>META-INF/*.RSA</exclude> + </excludes> + </filter> + </filters> <relocations> <relocation> <pattern>com.google.common</pattern> @@ -267,13 +275,13 @@ <transformers> <transformer implementation="org.atteo.classindex.ClassIndexTransformer"/> <transformer - implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer"> - <resources> + implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer"> + <resources> <resource>.yaml</resource> <resource>LICENSE.txt</resource> <resource>ASL2.0</resource> <resource>NOTICE.txt</resource> - </resources> + </resources> </transformer> <!-- UNCOMMENT THIS IF YOU NEED TO REGENERATE THE BEST GUESS NOTICES FILE WHICH REQUIRES PRUNING EVERY RELEASE --> <!--transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer"> @@ -281,7 +289,7 @@ <projectName>${project.name}</projectName> </transformer--> <transformer - implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" /> + implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/> <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> <mainClass></mainClass> @@ -297,8 +305,8 @@ <version>${global_classindex_version}</version> </dependency> </dependencies> - </plugin> - <plugin> + </plugin> + <plugin> <artifactId>maven-assembly-plugin</artifactId> <configuration> <descriptor>src/main/assembly/assembly.xml</descriptor> diff --git a/metron-platform/metron-pcap-backend/src/main/assembly/assembly.xml b/metron-platform/metron-pcap-backend/src/main/assembly/assembly.xml index f38582d..1df0f14 100644 --- a/metron-platform/metron-pcap-backend/src/main/assembly/assembly.xml +++ b/metron-platform/metron-pcap-backend/src/main/assembly/assembly.xml @@ -56,7 +56,7 @@ <fileSet> <directory>${project.basedir}/target</directory> <includes> - <include>${project.artifactId}-${project.version}.jar</include> + <include>${project.artifactId}-${project.version}-uber.jar</include> </includes> <outputDirectory>lib</outputDirectory> <useDefaultExcludes>true</useDefaultExcludes> diff --git a/metron-platform/metron-pcap-backend/src/main/scripts/pcap_inspector.sh b/metron-platform/metron-pcap-backend/src/main/scripts/pcap_inspector.sh index 438152c..12189cc 100755 --- a/metron-platform/metron-pcap-backend/src/main/scripts/pcap_inspector.sh +++ b/metron-platform/metron-pcap-backend/src/main/scripts/pcap_inspector.sh @@ -29,6 +29,6 @@ fi export METRON_VERSION=${project.version} export METRON_HOME=/usr/metron/$METRON_VERSION -export TOPOLOGIES_JAR=${project.artifactId}-$METRON_VERSION.jar +export TOPOLOGIES_JAR=${project.artifactId}-$METRON_VERSION-uber.jar export HADOOP_OPTS="$HADOOP_OPTS $METRON_JVMFLAGS" yarn jar $METRON_HOME/lib/$TOPOLOGIES_JAR org.apache.metron.utils.PcapInspector "$@" diff --git a/metron-platform/metron-pcap-backend/src/main/scripts/pcap_query.sh b/metron-platform/metron-pcap-backend/src/main/scripts/pcap_query.sh index 206e2b8..a3e3bb6 100755 --- a/metron-platform/metron-pcap-backend/src/main/scripts/pcap_query.sh +++ b/metron-platform/metron-pcap-backend/src/main/scripts/pcap_query.sh @@ -29,7 +29,7 @@ fi export METRON_VERSION=${project.version} export METRON_HOME=/usr/metron/$METRON_VERSION -export PCAP_BACKEND_JAR=${project.artifactId}-$METRON_VERSION.jar +export PCAP_BACKEND_JAR=${project.artifactId}-$METRON_VERSION-uber.jar export HADOOP_OPTS="$HADOOP_OPTS $METRON_JVMFLAGS" yarn jar $METRON_HOME/lib/$PCAP_BACKEND_JAR org.apache.metron.pcap.query.PcapCli "$@" diff --git a/metron-platform/metron-pcap-backend/src/main/scripts/start_pcap_topology.sh b/metron-platform/metron-pcap-backend/src/main/scripts/start_pcap_topology.sh index a38f905..f42ec6a 100644 --- a/metron-platform/metron-pcap-backend/src/main/scripts/start_pcap_topology.sh +++ b/metron-platform/metron-pcap-backend/src/main/scripts/start_pcap_topology.sh @@ -18,6 +18,6 @@ # export METRON_VERSION=${project.version} export METRON_HOME=/usr/metron/$METRON_VERSION -export TOPOLOGIES_JAR=${project.artifactId}-$METRON_VERSION.jar - -storm jar $METRON_HOME/lib/$TOPOLOGIES_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/pcap/remote.yaml --filter $METRON_HOME/config/pcap.properties \ No newline at end of file +export TOPOLOGIES_JAR=${project.artifactId}-$METRON_VERSION-uber.jar +export STELLAR_JAR=stellar-common-$METRON_VERSION-uber.jar +storm jar $METRON_HOME/lib/$TOPOLOGIES_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/pcap/remote.yaml --filter $METRON_HOME/config/pcap.properties --jars "$METRON_HOME/lib/$STELLAR_JAR" \ No newline at end of file diff --git a/metron-platform/metron-pcap/pom.xml b/metron-platform/metron-pcap/pom.xml index c839285..4fda810 100644 --- a/metron-platform/metron-pcap/pom.xml +++ b/metron-platform/metron-pcap/pom.xml @@ -46,6 +46,12 @@ </dependency> <dependency> <groupId>org.apache.metron</groupId> + <artifactId>stellar-common</artifactId> + <version>${project.parent.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.metron</groupId> <artifactId>metron-job</artifactId> <version>${project.parent.version}</version> </dependency> @@ -92,6 +98,14 @@ <artifactId>commons-beanutils</artifactId> <groupId>commons-beanutils</groupId> </exclusion> + <exclusion> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpcore</artifactId> + </exclusion> </exclusions> </dependency> <dependency> diff --git a/metron-platform/metron-solr/metron-solr-common/pom.xml b/metron-platform/metron-solr/metron-solr-common/pom.xml index ffaf6d9..bcf9366 100644 --- a/metron-platform/metron-solr/metron-solr-common/pom.xml +++ b/metron-platform/metron-solr/metron-solr-common/pom.xml @@ -30,6 +30,12 @@ </properties> <dependencies> <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>stellar-common</artifactId> + <version>${project.parent.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> <version>${guava_version}</version> diff --git a/metron-platform/metron-solr/metron-solr-storm/pom.xml b/metron-platform/metron-solr/metron-solr-storm/pom.xml index 22c8c50..b089986 100644 --- a/metron-platform/metron-solr/metron-solr-storm/pom.xml +++ b/metron-platform/metron-solr/metron-solr-storm/pom.xml @@ -55,6 +55,12 @@ <type>test-jar</type> </dependency> <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>stellar-common</artifactId> + <version>${project.parent.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>org.apache.solr</groupId> <artifactId>solr-test-framework</artifactId> <version>${global_solr_version}</version> diff --git a/metron-platform/metron-solr/metron-solr-storm/src/main/scripts/start_solr_topology.sh b/metron-platform/metron-solr/metron-solr-storm/src/main/scripts/start_solr_topology.sh index 614423e..e386fcb 100755 --- a/metron-platform/metron-solr/metron-solr-storm/src/main/scripts/start_solr_topology.sh +++ b/metron-platform/metron-solr/metron-solr-storm/src/main/scripts/start_solr_topology.sh @@ -19,4 +19,5 @@ METRON_VERSION=${project.version} METRON_HOME=/usr/metron/$METRON_VERSION TOPOLOGY_JAR=${project.artifactId}-$METRON_VERSION-uber.jar -storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/indexing/random_access/remote.yaml --filter $METRON_HOME/config/solr.properties +STELLAR_JAR=stellar-common-$METRON_VERSION-uber.jar +storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/indexing/random_access/remote.yaml --filter $METRON_HOME/config/solr.properties --jars "$METRON_HOME/lib/$STELLAR_JAR" diff --git a/metron-platform/metron-storm-kafka-override/pom.xml b/metron-platform/metron-storm-kafka-override/pom.xml index 3addee3..f557956 100644 --- a/metron-platform/metron-storm-kafka-override/pom.xml +++ b/metron-platform/metron-storm-kafka-override/pom.xml @@ -106,6 +106,8 @@ <artifactId>maven-shade-plugin</artifactId> <version>${global_shade_version}</version> <configuration> + <shadedArtifactAttached>true</shadedArtifactAttached> + <shadedClassifierName>uber</shadedClassifierName> <createDependencyReducedPom>true</createDependencyReducedPom> </configuration> <executions> diff --git a/metron-platform/metron-storm-kafka/pom.xml b/metron-platform/metron-storm-kafka/pom.xml index da762bf..0749a19 100644 --- a/metron-platform/metron-storm-kafka/pom.xml +++ b/metron-platform/metron-storm-kafka/pom.xml @@ -101,6 +101,7 @@ <groupId>org.apache.metron</groupId> <artifactId>stellar-common</artifactId> <version>${project.parent.version}</version> + <scope>provided</scope> </dependency> </dependencies> diff --git a/metron-platform/metron-writer/metron-writer-common/pom.xml b/metron-platform/metron-writer/metron-writer-common/pom.xml index 0d91b52..6b902ec 100644 --- a/metron-platform/metron-writer/metron-writer-common/pom.xml +++ b/metron-platform/metron-writer/metron-writer-common/pom.xml @@ -169,7 +169,7 @@ </exclusions> <scope>provided</scope> </dependency> - <dependency> + <dependency> <groupId>org.apache.metron</groupId> <artifactId>metron-common</artifactId> <version>${project.parent.version}</version> @@ -189,6 +189,7 @@ <groupId>org.apache.metron</groupId> <artifactId>stellar-common</artifactId> <version>${project.parent.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.metron</groupId> @@ -206,14 +207,14 @@ <artifactId>antlr4-maven-plugin</artifactId> <version>${antlr.version}</version> <configuration> - <outputDirectory>${basedir}/src/main/java</outputDirectory> + <outputDirectory>${basedir}/src/main/java</outputDirectory> </configuration> <executions> - <execution> - <goals> - <goal>antlr4</goal> - </goals> - </execution> + <execution> + <goals> + <goal>antlr4</goal> + </goals> + </execution> </executions> </plugin> @@ -236,16 +237,18 @@ <goal>shade</goal> </goals> <configuration> - <filters> - <filter> - <artifact>*:*</artifact> - <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> - </excludes> - </filter> - </filters> + <shadedArtifactAttached>true</shadedArtifactAttached> + <shadedClassifierName>uber</shadedClassifierName> + <filters> + <filter> + <artifact>*:*</artifact> + <excludes> + <exclude>META-INF/*.SF</exclude> + <exclude>META-INF/*.DSA</exclude> + <exclude>META-INF/*.RSA</exclude> + </excludes> + </filter> + </filters> <relocations> <relocation> <pattern>com.google.common</pattern> @@ -254,13 +257,13 @@ </relocations> <transformers> <transformer - implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer"> - <resources> + implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer"> + <resources> <resource>.yaml</resource> <resource>LICENSE.txt</resource> <resource>ASL2.0</resource> <resource>NOTICE.txt</resource> - </resources> + </resources> </transformer> <!-- UNCOMMENT THIS IF YOU NEED TO REGENERATE THE BEST GUESS NOTICES FILE WHICH REQUIRES PRUNING EVERY RELEASE --> <!--transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer"> @@ -268,7 +271,7 @@ <projectName>${project.name}</projectName> </transformer--> <transformer - implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" /> + implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/> <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> <mainClass></mainClass> diff --git a/metron-platform/metron-writer/metron-writer-storm/pom.xml b/metron-platform/metron-writer/metron-writer-storm/pom.xml index 7d2c829..b1e33a9 100644 --- a/metron-platform/metron-writer/metron-writer-storm/pom.xml +++ b/metron-platform/metron-writer/metron-writer-storm/pom.xml @@ -41,6 +41,12 @@ <version>${project.parent.version}</version> </dependency> <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>stellar-common</artifactId> + <version>${project.parent.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${global_hadoop_version}</version> diff --git a/metron-stellar/stellar-common/pom.xml b/metron-stellar/stellar-common/pom.xml index 7b79b84..c98cb32 100644 --- a/metron-stellar/stellar-common/pom.xml +++ b/metron-stellar/stellar-common/pom.xml @@ -353,6 +353,10 @@ <shadedPattern>org.apache.metron.commons.validator</shadedPattern> </relocation> --> + <relocation> + <pattern>com.jakewharton.fliptables</pattern> + <shadedPattern>org.apache.metron.fliptables</shadedPattern> + </relocation> </relocations> <artifactSet> <excludes> @@ -398,8 +402,10 @@ <plugin> <artifactId>maven-assembly-plugin</artifactId> <configuration> - <descriptor>src/main/assembly/assembly.xml</descriptor> - <descriptor>src/main/assembly/stand-alone-assembly.xml</descriptor> + <descriptors> + <descriptor>src/main/assembly/assembly.xml</descriptor> + <descriptor>src/main/assembly/stand-alone-assembly.xml</descriptor> + </descriptors> </configuration> <executions> <execution> diff --git a/metron-stellar/stellar-common/src/main/assembly/assembly.xml b/metron-stellar/stellar-common/src/main/assembly/assembly.xml index b50be00..4cc9735 100644 --- a/metron-stellar/stellar-common/src/main/assembly/assembly.xml +++ b/metron-stellar/stellar-common/src/main/assembly/assembly.xml @@ -19,18 +19,6 @@ <includeBaseDirectory>false</includeBaseDirectory> <fileSets> <fileSet> - <directory>${project.basedir}/src/main/scripts/deployed</directory> - <outputDirectory>bin</outputDirectory> - <useDefaultExcludes>true</useDefaultExcludes> - <excludes> - <exclude>**/*.formatted</exclude> - <exclude>**/*.filtered</exclude> - </excludes> - <fileMode>0755</fileMode> - <lineEnding>unix</lineEnding> - <filtered>true</filtered> - </fileSet> - <fileSet> <directory>${project.basedir}/target</directory> <includes> <include>${project.artifactId}-${project.version}-uber.jar</include> diff --git a/metron-stellar/stellar-common/src/main/scripts/deployed/stellar b/metron-stellar/stellar-common/src/main/scripts/deployed/stellar index 7b0f06d..b76f398 100644 --- a/metron-stellar/stellar-common/src/main/scripts/deployed/stellar +++ b/metron-stellar/stellar-common/src/main/scripts/deployed/stellar @@ -31,6 +31,6 @@ fi export HBASE_CONFIGS=$(hbase classpath) export METRON_VERSION=${project.version} export METRON_HOME=/usr/metron/$METRON_VERSION -export STELLAR_LIB=$(find $METRON_HOME/lib/ -name metron-parsers*.jar) +export STELLAR_LIB=stellar-common-$METRON_VERSION-uber.jar export MANAGEMENT_LIB=$(find $METRON_HOME/lib/ -name metron-management*.jar) -java $JVMFLAGS -cp "${CONTRIB:-$METRON_HOME/contrib}:$STELLAR_LIB:$MANAGEMENT_LIB:$HBASE_CONFIGS" org.apache.metron.stellar.common.shell.cli.StellarShell "$@" +java $JVMFLAGS -cp "${CONTRIB:-$METRON_HOME/contrib}:$METRON_HOME/lib/$STELLAR_LIB:$MANAGEMENT_LIB:$HBASE_CONFIGS" org.apache.metron.stellar.common.shell.cli.StellarShell "$@" diff --git a/metron-stellar/stellar-zeppelin/pom.xml b/metron-stellar/stellar-zeppelin/pom.xml index e9731b5..3984afe 100644 --- a/metron-stellar/stellar-zeppelin/pom.xml +++ b/metron-stellar/stellar-zeppelin/pom.xml @@ -33,6 +33,7 @@ <groupId>org.apache.metron</groupId> <artifactId>stellar-common</artifactId> <version>${project.parent.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.metron</groupId>