This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git
The following commit(s) were added to refs/heads/main by this push:
new 88ee1ec Analyze dependencies and fix spark/pom.xml
88ee1ec is described below
commit 88ee1ecf8695e57e8ca4d720d7917af6e279f456
Author: Christopher Tubbs <ctubb...@apache.org>
AuthorDate: Fri Apr 1 04:44:41 2022 -0400
Analyze dependencies and fix spark/pom.xml
* Add dependency analysis to POMs
* Update the spark/pom.xml (this fixes #93)
* Fix compilation issue with CopyPlus5K, which had missing imports
---
pom.xml | 39 +++++++--
spark/pom.xml | 98 ++++++++++++----------
.../java/org/apache/accumulo/spark/CopyPlus5K.java | 3 +
3 files changed, 88 insertions(+), 52 deletions(-)
diff --git a/pom.xml b/pom.xml
index 3fbe4a2..a5747c3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -77,31 +77,34 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
+ <artifactId>hadoop-client-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.htrace</groupId>
<artifactId>htrace-core</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core4</artifactId>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-1.2-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
+ <scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
+ <scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
+ <scope>runtime</scope>
</dependency>
<!-- Test dependencies -->
<dependency>
@@ -150,13 +153,33 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>3.3.0</version>
</plugin>
</plugins>
</pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>analyze</id>
+ <goals>
+ <goal>analyze-only</goal>
+ </goals>
+ <configuration>
+ <failOnWarning>true</failOnWarning>
+ <ignoredUsedUndeclaredDependencies />
+ <ignoredUnusedDeclaredDependencies>
+ <runtime>org.apache.logging.log4j:*</runtime>
+ </ignoredUnusedDeclaredDependencies>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<executions>
<execution>
diff --git a/spark/pom.xml b/spark/pom.xml
index 569aa58..e7b9918 100644
--- a/spark/pom.xml
+++ b/spark/pom.xml
@@ -20,29 +20,35 @@
<parent>
<groupId>org.apache</groupId>
<artifactId>apache</artifactId>
- <version>23</version>
+ <version>24</version>
<relativePath />
</parent>
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-spark</artifactId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.1.0-SNAPSHOT</version>
<name>Apache Accumulo Spark Example</name>
<description>Example Spark Application for Apache Accumulo</description>
<properties>
- <accumulo.version>2.0.0</accumulo.version>
+ <accumulo.version>2.1.0-SNAPSHOT</accumulo.version>
<eclipseFormatterStyle>contrib/Eclipse-Accumulo-Codestyle.xml</eclipseFormatterStyle>
- <hadoop.version>3.2.1</hadoop.version>
- <maven.compiler.release>8</maven.compiler.release>
- <maven.compiler.source>1.8</maven.compiler.source>
- <maven.compiler.target>1.8</maven.compiler.target>
- <zookeeper.version>3.4.14</zookeeper.version>
+ <hadoop.version>3.3.0</hadoop.version>
+ <maven.compiler.release>11</maven.compiler.release>
+ <maven.compiler.source>11</maven.compiler.source>
+ <maven.compiler.target>11</maven.compiler.target>
+ <minimalJavaBuildVersion>11</minimalJavaBuildVersion>
+ <minimalMavenBuildVersion>3.5.0</minimalMavenBuildVersion>
+ <!-- timestamp for reproducible outputs, updated on release by the release
plugin -->
+
<project.build.outputTimestamp>2020-12-17T22:06:50Z</project.build.outputTimestamp>
+ <zookeeper.version>3.7.0</zookeeper.version>
</properties>
<dependencyManagement>
<dependencies>
<dependency>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- <version>${zookeeper.version}</version>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-project</artifactId>
+ <version>${accumulo.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
@@ -50,23 +56,34 @@
<dependency>
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-core</artifactId>
- <version>${accumulo.version}</version>
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-hadoop-mapreduce</artifactId>
- <version>${accumulo.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-api</artifactId>
- <version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<version>2.4.5</version>
</dependency>
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
+ <version>2.11.12</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ <scope>runtime</scope>
+ </dependency>
</dependencies>
<build>
<pluginManagement>
@@ -74,7 +91,6 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
- <version>3.8.1</version>
<configuration>
<optimize>true</optimize>
</configuration>
@@ -82,46 +98,43 @@
<plugin>
<groupId>com.github.ekryd.sortpom</groupId>
<artifactId>sortpom-maven-plugin</artifactId>
- <version>2.10.0</version>
+ <version>3.0.0</version>
<configuration>
- <predefinedSortOrder>recommended_2008_06</predefinedSortOrder>
<createBackupFile>false</createBackupFile>
- <lineSeparator>\n</lineSeparator>
<expandEmptyElements>false</expandEmptyElements>
+ <keepBlankLines>false</keepBlankLines>
+ <lineSeparator>\n</lineSeparator>
<nrOfIndentSpace>2</nrOfIndentSpace>
+ <predefinedSortOrder>recommended_2008_06</predefinedSortOrder>
<sortDependencies>scope,groupId,artifactId</sortDependencies>
<sortProperties>true</sortProperties>
+ <spaceBeforeCloseEmptyElement>true</spaceBeforeCloseEmptyElement>
<verifyFail>Stop</verifyFail>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>3.2.1</version>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>3.3.0</version>
</plugin>
</plugins>
</pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-enforcer-plugin</artifactId>
+ <artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
- <!-- must be same id as in the apache parent pom, to override the
version -->
- <id>enforce-maven-version</id>
+ <id>analyze</id>
<goals>
- <goal>enforce</goal>
+ <goal>analyze-only</goal>
</goals>
- <phase>validate</phase>
<configuration>
- <rules>
- <requireMavenVersion>
- <version>[3.5.0,)</version>
- </requireMavenVersion>
- <requireJavaVersion>
- <version>[11,)</version>
- </requireJavaVersion>
- </rules>
+ <failOnWarning>true</failOnWarning>
+ <ignoredUsedUndeclaredDependencies />
+ <ignoredUnusedDeclaredDependencies>
+ <runtime>org.apache.logging.log4j:log4j-slf4j-impl</runtime>
+ </ignoredUnusedDeclaredDependencies>
</configuration>
</execution>
</executions>
@@ -130,7 +143,7 @@
<!-- This was added to ensure project only uses public API -->
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
- <version>3.1.0</version>
+ <version>3.1.2</version>
<configuration>
<configLocation>contrib/checkstyle.xml</configLocation>
</configuration>
@@ -138,7 +151,7 @@
<dependency>
<groupId>com.puppycrawl.tools</groupId>
<artifactId>checkstyle</artifactId>
- <version>8.29</version>
+ <version>9.3</version>
</dependency>
</dependencies>
<executions>
@@ -153,10 +166,10 @@
<plugin>
<groupId>net.revelc.code</groupId>
<artifactId>impsort-maven-plugin</artifactId>
- <version>1.3.0</version>
+ <version>1.6.2</version>
<configuration>
<removeUnused>true</removeUnused>
- <groups>java.,javax.,org.,com.</groups>
+ <groups>java.,javax.,jakarta.,org.,com.</groups>
</configuration>
<executions>
<execution>
@@ -191,19 +204,16 @@
<plugin>
<groupId>net.revelc.code.formatter</groupId>
<artifactId>formatter-maven-plugin</artifactId>
- <version>2.10.0</version>
+ <version>2.18.0</version>
<configuration>
<configFile>${eclipseFormatterStyle}</configFile>
- <compilerCompliance>${maven.compiler.source}</compilerCompliance>
- <compilerSource>${maven.compiler.source}</compilerSource>
-
<compilerTargetPlatform>${maven.compiler.target}</compilerTargetPlatform>
<lineEnding>LF</lineEnding>
<overrideConfigCompilerVersion>true</overrideConfigCompilerVersion>
- <skipJsFormatting>true</skipJsFormatting>
+ <skipCssFormatting>true</skipCssFormatting>
<skipHtmlFormatting>true</skipHtmlFormatting>
- <skipXmlFormatting>true</skipXmlFormatting>
+ <skipJsFormatting>true</skipJsFormatting>
<skipJsonFormatting>true</skipJsonFormatting>
- <skipCssFormatting>true</skipCssFormatting>
+ <skipXmlFormatting>true</skipXmlFormatting>
</configuration>
<executions>
<execution>
diff --git a/spark/src/main/java/org/apache/accumulo/spark/CopyPlus5K.java
b/spark/src/main/java/org/apache/accumulo/spark/CopyPlus5K.java
index cd58889..e68762a 100644
--- a/spark/src/main/java/org/apache/accumulo/spark/CopyPlus5K.java
+++ b/spark/src/main/java/org/apache/accumulo/spark/CopyPlus5K.java
@@ -25,6 +25,7 @@ import org.apache.accumulo.core.client.Accumulo;
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
@@ -38,6 +39,8 @@ import org.apache.spark.Partitioner;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaSparkContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class CopyPlus5K {
