richardantal commented on code in PR #133:
URL:
https://github.com/apache/phoenix-connectors/pull/133#discussion_r1689389067
##########
phoenix5-spark3/src/main/java/org/apache/phoenix/spark/sql/connector/PhoenixDSRelation.java:
##########
@@ -0,0 +1,90 @@
+/*
Review Comment:
This is not used anywhere, could you please remove it?
##########
phoenix5-spark/pom.xml:
##########
@@ -414,202 +419,160 @@
</dependency>
<dependency>
- <!-- Why is this not provided transitively via Phoenix ? -->
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-testing-util</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>${zookeeper.version}</version>
<scope>test</scope>
</dependency>
- <!-- Mark every Hadoop jar as provided -->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-annotations</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-auth</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-api</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-distcp</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-common</artifactId>
- <scope>provided</scope>
- </dependency>
<!-- We want to take the implementation from Spark -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- <version>2.5.0</version>
- <scope>provided</scope>
+ <scope>test</scope>
</dependency>
</dependencies>
<build>
- <plugins>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-test-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-test-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>src/it/java</source>
- <source>src/it/scala</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <artifactId>maven-dependency-plugin</artifactId>
- <configuration>
- <ignoreNonCompile>true</ignoreNonCompile>
- <ignoredUnusedDeclaredDependencies>
- <!-- These are all used -->
- <ignoredUnusedDeclaredDependency>
- org.apache.hadoop:hadoop-hdfs
- </ignoredUnusedDeclaredDependency>
- <ignoredUnusedDeclaredDependency>
- org.apache.hbase:hbase-it
- </ignoredUnusedDeclaredDependency>
- </ignoredUnusedDeclaredDependencies>
- </configuration>
- </plugin>
- <plugin>
- <groupId>net.alchim31.maven</groupId>
- <artifactId>scala-maven-plugin</artifactId>
- <configuration>
- <charset>${project.build.sourceEncoding}</charset>
- <jvmArgs>
- <jvmArg>-Xmx1024m</jvmArg>
- </jvmArgs>
- <scalaVersion>${scala.version}</scalaVersion>
- <scalaCompatVersion>${scala.binary.version}</scalaCompatVersion>
- </configuration>
- <executions>
- <execution>
- <id>scala-compile-first</id>
- <phase>process-resources</phase>
- <goals>
- <goal>add-source</goal>
- <goal>compile</goal>
- </goals>
- </execution>
- <execution>
- <id>scala-test-compile</id>
- <phase>process-test-resources</phase>
- <goals>
- <goal>testCompile</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <configuration>
+ <ignoredUnusedDeclaredDependencies>
+ <!-- These are test runtime dependencies that Maven has conecnt
Review Comment:
typo
##########
phoenix5-spark3/pom.xml:
##########
@@ -142,48 +211,96 @@
</dependencies>
<build>
<plugins>
- <plugin>
- <groupId>net.alchim31.maven</groupId>
- <artifactId>scala-maven-plugin</artifactId>
- <configuration>
- <charset>${project.build.sourceEncoding}</charset>
- <jvmArgs>
- <jvmArg>-Xmx1024m</jvmArg>
- </jvmArgs>
- <scalaVersion>${scala.version}</scalaVersion>
- <scalaCompatVersion>${scala.binary.version}</scalaCompatVersion>
- </configuration>
- <executions>
- <execution>
- <id>scala-compile-first</id>
- <phase>process-resources</phase>
- <goals>
- <goal>add-source</goal>
- <goal>compile</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
<plugin>
- <artifactId>maven-dependency-plugin</artifactId>
- <configuration>
- <ignoreNonCompile>true</ignoreNonCompile>
- <ignoredDependencies>
-
<ignoredDependency>org.apache.hadoop:hadoop-common</ignoredDependency>
-
<ignoredDependency>org.apache.hadoop:hadoop-mapreduce-client-core</ignoredDependency>
- </ignoredDependencies>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ <configuration>
+ <charset>${project.build.sourceEncoding}</charset>
+ <jvmArgs>
+ <jvmArg>-Xmx1024m</jvmArg>
+ </jvmArgs>
+ <scalaVersion>${scala.version}</scalaVersion>
+ <scalaCompatVersion>${scala.binary.version}</scalaCompatVersion>
+ </configuration>
+ <executions>
+ <execution>
+ <id>scala-compile-first</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>add-source</goal>
+ <goal>compile</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>scala-test-compile</id>
+ <goals>
+ <goal>testCompile</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest-maven-plugin</artifactId>
+ <version>1.0</version>
+ <configuration>
+
<reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
+ <junitxml>.</junitxml>
+ <filereports>WDF TestSuite.txt</filereports>
+ <skipTests>${skip.scalatest}</skipTests>
+ </configuration>
+ <executions>
+ <execution>
+ <id>integration-test</id>
+ <phase>integration-test</phase>
+ <goals>
+ <goal>test</goal>
+ </goals>
<configuration>
- <skip>${skip.spark.javadoc}</skip>
+ <!-- Need this false until we can switch to JUnit 4.13 due
+ to https://github.com/junit-team/junit4/issues/1223 -->
+ <parallel>false</parallel>
+ <tagsToExclude>Integration-Test</tagsToExclude>
+ <argLine>-XX:ReservedCodeCacheSize=512m ${argLine}</argLine>
</configuration>
- </plugin>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <configuration>
+ <!-- Test runtime dependencies -->
+ <ignoredUnusedDeclaredDependencies>
+
<ignoredUnsedDeclaredDependency>org.apache.phoenix:phoenix-hbase-compat-${hbase.compat.version}</ignoredUnsedDeclaredDependency>
+
<ignoredUnsedDeclaredDependency>org.apache.zookeeper:zookeeper</ignoredUnsedDeclaredDependency>
+
<ignoredUnsedDeclaredDependency>org.apache.hbase:hbase-testing-util</ignoredUnsedDeclaredDependency>
+
<ignoredUnsedDeclaredDependency>org.apache.hbase:hbase-it</ignoredUnsedDeclaredDependency>
+
<ignoredUnsedDeclaredDependency>org.apache.hadoop:hadoop-minicluster</ignoredUnsedDeclaredDependency>
+
<ignoredUnsedDeclaredDependency>org.apache.omid:omid-tso-server</ignoredUnsedDeclaredDependency>
+ </ignoredUnusedDeclaredDependencies>
+ <ignoredUsedUndeclaredDependencies>
+ <!-- No trace of these in the source. Perhaps used anonymously
+ in chained method calls ? -->
+
<ignoredUsedUndeclaredDependency>joda-time:joda-time</ignoredUsedUndeclaredDependency>
Review Comment:
It is used here:
https://github.com/stoty/phoenix-connectors/blob/PHOENIX-7366/phoenix5-spark3/src/main/scala/org/apache/phoenix/spark/PhoenixRecordWritable.scala#L20
##########
phoenix5-spark/src/it/java/org/apache/phoenix/spark/DataSourceApiIT.java:
##########
@@ -34,13 +34,16 @@
import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import org.slf4j.Logger;
Review Comment:
I don't think it is beeing used
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
