wangyum commented on a change in pull request #24371:
[SPARK-27176][FOLLOW-UP][SQL] Shade Parquet to sql/core
URL: https://github.com/apache/spark/pull/24371#discussion_r275835630
##########
File path:
sql/hive/src/test/scala/org/apache/spark/sql/sources/ParquetHadoopFsRelationSuite.scala
##########
@@ -124,7 +123,7 @@ class ParquetHadoopFsRelationSuite extends
HadoopFsRelationTest {
test("SPARK-8604: Parquet data source should write summary file while doing
appending") {
withSQLConf(
- ParquetOutputFormat.JOB_SUMMARY_LEVEL -> "ALL",
+ "parquet.summary.metadata.level" -> "ALL",
Review comment:
Still dependent on
`org.apache.parquet:parquet-hadoop-bundle:jar:1.8.1:compile` and
`org.apache.parquet:parquet-hadoop:jar:1.10.1:provided`:
```
[INFO] ------------------< org.apache.spark:spark-hive_2.12
>------------------
[INFO] Building Spark Project Hive 3.0.0-SNAPSHOT
[17/25]
[INFO] --------------------------------[ jar
]---------------------------------
[INFO]
[INFO] --- maven-dependency-plugin:3.1.1:tree (default-cli) @
spark-hive_2.12 ---
[INFO] org.apache.spark:spark-hive_2.12:jar:3.0.0-SNAPSHOT
[INFO] +- org.apache.parquet:parquet-hadoop-bundle:jar:1.8.1:compile
[INFO] +- org.apache.spark:spark-core_2.12:jar:3.0.0-SNAPSHOT:compile
[INFO] | +- com.thoughtworks.paranamer:paranamer:jar:2.8:runtime
[INFO] | +- com.twitter:chill_2.12:jar:0.9.3:compile
[INFO] | | \- com.esotericsoftware:kryo-shaded:jar:4.0.2:compile
[INFO] | | +- com.esotericsoftware:minlog:jar:1.3.0:compile
[INFO] | | \- org.objenesis:objenesis:jar:2.5.1:compile
[INFO] | +- com.twitter:chill-java:jar:0.9.3:compile
[INFO] | +- org.apache.xbean:xbean-asm7-shaded:jar:4.12:compile
[INFO] | +- org.apache.hadoop:hadoop-client:jar:3.2.0:compile
[INFO] | | +- org.apache.hadoop:hadoop-common:jar:3.2.0:compile
[INFO] | | | +- commons-collections:commons-collections:jar:3.2.2:compile
[INFO] | | | +- javax.servlet.jsp:jsp-api:jar:2.1:runtime
[INFO] | | | +- commons-beanutils:commons-beanutils:jar:1.9.3:compile
[INFO] | | | +-
org.apache.commons:commons-configuration2:jar:2.1.1:compile
[INFO] | | | +- org.apache.commons:commons-text:jar:1.6:compile
[INFO] | | | +- com.google.re2j:re2j:jar:1.1:compile
[INFO] | | | +- org.apache.hadoop:hadoop-auth:jar:3.2.0:compile
[INFO] | | | | +- com.nimbusds:nimbus-jose-jwt:jar:4.41.1:compile
[INFO] | | | | | \-
com.github.stephenc.jcip:jcip-annotations:jar:1.0-1:compile
[INFO] | | | | \- net.minidev:json-smart:jar:2.3:compile
[INFO] | | | | \- net.minidev:accessors-smart:jar:1.2:compile
[INFO] | | | +- org.apache.curator:curator-client:jar:2.13.0:compile
[INFO] | | | +-
org.apache.htrace:htrace-core4:jar:4.1.0-incubating:compile
[INFO] | | | +- org.apache.kerby:kerb-simplekdc:jar:1.0.1:compile
[INFO] | | | | +- org.apache.kerby:kerb-client:jar:1.0.1:compile
[INFO] | | | | | +- org.apache.kerby:kerby-config:jar:1.0.1:compile
[INFO] | | | | | +- org.apache.kerby:kerb-core:jar:1.0.1:compile
[INFO] | | | | | | \- org.apache.kerby:kerby-pkix:jar:1.0.1:compile
[INFO] | | | | | | +- org.apache.kerby:kerby-asn1:jar:1.0.1:compile
[INFO] | | | | | | \- org.apache.kerby:kerby-util:jar:1.0.1:compile
[INFO] | | | | | +- org.apache.kerby:kerb-common:jar:1.0.1:compile
[INFO] | | | | | | \- org.apache.kerby:kerb-crypto:jar:1.0.1:compile
[INFO] | | | | | +- org.apache.kerby:kerb-util:jar:1.0.1:compile
[INFO] | | | | | \- org.apache.kerby:token-provider:jar:1.0.1:compile
[INFO] | | | | \- org.apache.kerby:kerb-admin:jar:1.0.1:compile
[INFO] | | | | +- org.apache.kerby:kerb-server:jar:1.0.1:compile
[INFO] | | | | | \- org.apache.kerby:kerb-identity:jar:1.0.1:compile
[INFO] | | | | \- org.apache.kerby:kerby-xdr:jar:1.0.1:compile
[INFO] | | | +- org.codehaus.woodstox:stax2-api:jar:3.1.4:compile
[INFO] | | | +- com.fasterxml.woodstox:woodstox-core:jar:5.0.3:compile
[INFO] | | | \- dnsjava:dnsjava:jar:2.1.7:compile
[INFO] | | +- org.apache.hadoop:hadoop-hdfs-client:jar:3.2.0:compile
[INFO] | | | \- com.squareup.okhttp:okhttp:jar:2.7.5:compile
[INFO] | | | \- com.squareup.okio:okio:jar:1.6.0:compile
[INFO] | | +- org.apache.hadoop:hadoop-yarn-api:jar:3.2.0:compile
[INFO] | | | \- javax.xml.bind:jaxb-api:jar:2.2.11:compile
[INFO] | | +- org.apache.hadoop:hadoop-yarn-client:jar:3.2.0:compile
[INFO] | | +-
org.apache.hadoop:hadoop-mapreduce-client-core:jar:3.2.0:compile
[INFO] | | | \- org.apache.hadoop:hadoop-yarn-common:jar:3.2.0:compile
[INFO] | | | +-
com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar:2.9.8:compile
[INFO] | | | \-
com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar:2.9.5:compile
[INFO] | | | \-
com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:jar:2.9.5:compile
[INFO] | | +-
org.apache.hadoop:hadoop-mapreduce-client-jobclient:jar:3.2.0:compile
[INFO] | | | \-
org.apache.hadoop:hadoop-mapreduce-client-common:jar:3.2.0:compile
[INFO] | | \- org.apache.hadoop:hadoop-annotations:jar:3.2.0:compile
[INFO] | +- org.apache.spark:spark-launcher_2.12:jar:3.0.0-SNAPSHOT:compile
[INFO] | +- org.apache.spark:spark-kvstore_2.12:jar:3.0.0-SNAPSHOT:compile
[INFO] | | +- org.fusesource.leveldbjni:leveldbjni-all:jar:1.8:compile
[INFO] | | +- com.fasterxml.jackson.core:jackson-core:jar:2.9.8:compile
[INFO] | | \-
com.fasterxml.jackson.core:jackson-annotations:jar:2.9.8:compile
[INFO] | +-
org.apache.spark:spark-network-common_2.12:jar:3.0.0-SNAPSHOT:compile
[INFO] | | \- com.google.guava:guava:jar:14.0.1:provided
[INFO] | +-
org.apache.spark:spark-network-shuffle_2.12:jar:3.0.0-SNAPSHOT:compile
[INFO] | +- org.apache.spark:spark-unsafe_2.12:jar:3.0.0-SNAPSHOT:compile
[INFO] | +- javax.activation:activation:jar:1.1.1:compile
[INFO] | +- org.apache.curator:curator-recipes:jar:2.13.0:compile
[INFO] | | \- org.apache.curator:curator-framework:jar:2.13.0:compile
[INFO] | +- org.apache.zookeeper:zookeeper:jar:3.4.13:compile
[INFO] | | \- org.apache.yetus:audience-annotations:jar:0.5.0:compile
[INFO] | +- org.eclipse.jetty:jetty-plus:jar:9.4.12.v20180830:provided
[INFO] | | +- org.eclipse.jetty:jetty-webapp:jar:9.4.12.v20180830:compile
[INFO] | | | \- org.eclipse.jetty:jetty-xml:jar:9.4.12.v20180830:compile
[INFO] | | \- org.eclipse.jetty:jetty-jndi:jar:9.4.12.v20180830:provided
[INFO] | +- org.eclipse.jetty:jetty-security:jar:9.4.12.v20180830:provided
[INFO] | +- org.eclipse.jetty:jetty-util:jar:9.4.12.v20180830:provided
[INFO] | +- org.eclipse.jetty:jetty-server:jar:9.4.12.v20180830:provided
[INFO] | | \- org.eclipse.jetty:jetty-io:jar:9.4.12.v20180830:provided
[INFO] | +- org.eclipse.jetty:jetty-http:jar:9.4.12.v20180830:provided
[INFO] | +-
org.eclipse.jetty:jetty-continuation:jar:9.4.12.v20180830:provided
[INFO] | +- org.eclipse.jetty:jetty-servlet:jar:9.4.12.v20180830:provided
[INFO] | +- org.eclipse.jetty:jetty-proxy:jar:9.4.12.v20180830:provided
[INFO] | +- org.eclipse.jetty:jetty-client:jar:9.4.12.v20180830:provided
[INFO] | +- org.eclipse.jetty:jetty-servlets:jar:9.4.12.v20180830:provided
[INFO] | +- javax.servlet:javax.servlet-api:jar:3.1.0:compile
[INFO] | +- org.apache.commons:commons-lang3:jar:3.8.1:compile
[INFO] | +- org.apache.commons:commons-math3:jar:3.4.1:compile
[INFO] | +- org.slf4j:slf4j-api:jar:1.7.16:compile
[INFO] | +- org.slf4j:jul-to-slf4j:jar:1.7.16:compile
[INFO] | +- org.slf4j:jcl-over-slf4j:jar:1.7.16:compile
[INFO] | +- log4j:log4j:jar:1.2.17:compile
[INFO] | +- org.slf4j:slf4j-log4j12:jar:1.7.16:compile
[INFO] | +- com.ning:compress-lzf:jar:1.0.3:compile
[INFO] | +- org.xerial.snappy:snappy-java:jar:1.1.7.3:compile
[INFO] | +- org.lz4:lz4-java:jar:1.5.1:compile
[INFO] | +- com.github.luben:zstd-jni:jar:1.3.2-2:compile
[INFO] | +- org.roaringbitmap:RoaringBitmap:jar:0.7.45:compile
[INFO] | | \- org.roaringbitmap:shims:jar:0.7.45:compile
[INFO] | +- commons-net:commons-net:jar:3.1:compile
[INFO] | +- org.scala-lang:scala-library:jar:2.12.8:compile
[INFO] | +- org.scala-lang:scala-reflect:jar:2.12.8:compile
[INFO] | +- org.json4s:json4s-jackson_2.12:jar:3.5.3:compile
[INFO] | | \- org.json4s:json4s-core_2.12:jar:3.5.3:compile
[INFO] | | +- org.json4s:json4s-ast_2.12:jar:3.5.3:compile
[INFO] | | \- org.json4s:json4s-scalap_2.12:jar:3.5.3:compile
[INFO] | +- org.glassfish.jersey.core:jersey-client:jar:2.22.2:compile
[INFO] | | +- javax.ws.rs:javax.ws.rs-api:jar:2.0.1:compile
[INFO] | | +- org.glassfish.hk2:hk2-api:jar:2.4.0-b34:compile
[INFO] | | | +- org.glassfish.hk2:hk2-utils:jar:2.4.0-b34:compile
[INFO] | | | \-
org.glassfish.hk2.external:aopalliance-repackaged:jar:2.4.0-b34:compile
[INFO] | | +- org.glassfish.hk2.external:javax.inject:jar:2.4.0-b34:compile
[INFO] | | \- org.glassfish.hk2:hk2-locator:jar:2.4.0-b34:compile
[INFO] | | \- org.javassist:javassist:jar:3.18.1-GA:compile
[INFO] | +- org.glassfish.jersey.core:jersey-common:jar:2.22.2:compile
[INFO] | | +- javax.annotation:javax.annotation-api:jar:1.2:compile
[INFO] | | +-
org.glassfish.jersey.bundles.repackaged:jersey-guava:jar:2.22.2:compile
[INFO] | | \- org.glassfish.hk2:osgi-resource-locator:jar:1.0.1:compile
[INFO] | +- org.glassfish.jersey.core:jersey-server:jar:2.22.2:compile
[INFO] | | +-
org.glassfish.jersey.media:jersey-media-jaxb:jar:2.22.2:compile
[INFO] | | \- javax.validation:validation-api:jar:1.1.0.Final:compile
[INFO] | +-
org.glassfish.jersey.containers:jersey-container-servlet:jar:2.22.2:compile
[INFO] | +-
org.glassfish.jersey.containers:jersey-container-servlet-core:jar:2.22.2:compile
[INFO] | +- io.netty:netty-all:jar:4.1.30.Final:compile
[INFO] | +- io.netty:netty:jar:3.9.9.Final:compile
[INFO] | +- com.clearspring.analytics:stream:jar:2.7.0:compile
[INFO] | +- io.dropwizard.metrics:metrics-core:jar:3.1.5:compile
[INFO] | +- io.dropwizard.metrics:metrics-jvm:jar:3.1.5:compile
[INFO] | +- io.dropwizard.metrics:metrics-json:jar:3.1.5:compile
[INFO] | +- io.dropwizard.metrics:metrics-graphite:jar:3.1.5:compile
[INFO] | +- com.fasterxml.jackson.core:jackson-databind:jar:2.9.8:compile
[INFO] | +-
com.fasterxml.jackson.module:jackson-module-scala_2.12:jar:2.9.8:compile
[INFO] | | \-
com.fasterxml.jackson.module:jackson-module-paranamer:jar:2.9.8:compile
[INFO] | +- org.apache.ivy:ivy:jar:2.4.0:compile
[INFO] | +- oro:oro:jar:2.0.8:compile
[INFO] | +- net.razorvine:pyrolite:jar:4.13:compile
[INFO] | +- net.sf.py4j:py4j:jar:0.10.8.1:compile
[INFO] | +- org.apache.spark:spark-tags_2.12:jar:3.0.0-SNAPSHOT:compile
[INFO] | \- org.apache.commons:commons-crypto:jar:1.0.0:compile
[INFO] +- org.apache.spark:spark-core_2.12:test-jar:tests:3.0.0-SNAPSHOT:test
[INFO] +- org.apache.spark:spark-sql_2.12:jar:3.0.0-SNAPSHOT:compile
[INFO] | +- com.univocity:univocity-parsers:jar:2.7.3:compile
[INFO] | +- org.apache.spark:spark-sketch_2.12:jar:3.0.0-SNAPSHOT:compile
[INFO] | +- org.apache.spark:spark-catalyst_2.12:jar:3.0.0-SNAPSHOT:compile
[INFO] | +- org.apache.orc:orc-core:jar:1.5.5:compile
[INFO] | | +- org.apache.orc:orc-shims:jar:1.5.5:compile
[INFO] | | \- io.airlift:aircompressor:jar:0.10:compile
[INFO] | +- org.apache.orc:orc-mapreduce:jar:1.5.5:compile
[INFO] | +- org.apache.parquet:parquet-column:jar:1.10.1:provided
[INFO] | | +- org.apache.parquet:parquet-common:jar:1.10.1:provided
[INFO] | | \- org.apache.parquet:parquet-encoding:jar:1.10.1:provided
[INFO] | +- org.apache.parquet:parquet-hadoop:jar:1.10.1:provided
[INFO] | | +- org.apache.parquet:parquet-format:jar:2.4.0:provided
[INFO] | | \- org.apache.parquet:parquet-jackson:jar:1.10.1:provided
[INFO] | \- org.apache.arrow:arrow-vector:jar:0.12.0:compile
[INFO] | +- org.apache.arrow:arrow-format:jar:0.12.0:compile
[INFO] | +- org.apache.arrow:arrow-memory:jar:0.12.0:compile
[INFO] | +- com.carrotsearch:hppc:jar:0.7.2:compile
[INFO] | \- com.google.flatbuffers:flatbuffers-java:jar:1.9.0:compile
[INFO] +- org.apache.spark:spark-sql_2.12:test-jar:tests:3.0.0-SNAPSHOT:test
[INFO] +-
org.apache.spark:spark-catalyst_2.12:test-jar:tests:3.0.0-SNAPSHOT:test
[INFO] | +-
org.scala-lang.modules:scala-parser-combinators_2.12:jar:1.1.0:compile
[INFO] | +- org.codehaus.janino:janino:jar:3.0.11:compile
[INFO] | +- org.codehaus.janino:commons-compiler:jar:3.0.11:compile
[INFO] | \- org.antlr:antlr4-runtime:jar:4.7.1:compile
[INFO] +- org.apache.spark:spark-tags_2.12:test-jar:tests:3.0.0-SNAPSHOT:test
[INFO] +- org.apache.hive:hive-exec:jar:core:2.3.4:compile
[INFO] | +- org.apache.hive:hive-vector-code-gen:jar:2.3.4:compile
[INFO] | | \- org.apache.velocity:velocity:jar:1.5:compile
[INFO] | +- commons-io:commons-io:jar:2.4:compile
[INFO] | +- org.antlr:antlr-runtime:jar:3.5.2:compile
[INFO] | +- org.antlr:ST4:jar:4.0.4:compile
[INFO] | +- org.apache.commons:commons-compress:jar:1.9:compile
[INFO] | +- com.google.code.gson:gson:jar:2.2.4:compile
[INFO] | \- stax:stax-api:jar:1.0.1:compile
[INFO] +- org.apache.hive:hive-metastore:jar:2.3.4:compile
[INFO] | +- javolution:javolution:jar:5.5.1:compile
[INFO] | +- com.google.protobuf:protobuf-java:jar:2.5.0:compile
[INFO] | +- com.jolbox:bonecp:jar:0.8.0.RELEASE:compile
[INFO] | +- com.zaxxer:HikariCP:jar:2.5.1:compile
[INFO] | +- commons-cli:commons-cli:jar:1.2:compile
[INFO] | +- commons-lang:commons-lang:jar:2.6:compile
[INFO] | +- org.datanucleus:datanucleus-api-jdo:jar:4.2.4:compile
[INFO] | +- org.datanucleus:datanucleus-rdbms:jar:4.1.19:compile
[INFO] | +- commons-pool:commons-pool:jar:1.5.4:compile
[INFO] | +- commons-dbcp:commons-dbcp:jar:1.4:compile
[INFO] | +- javax.jdo:jdo-api:jar:3.0.1:compile
[INFO] | | \- javax.transaction:jta:jar:1.1:compile
[INFO] | \- org.datanucleus:javax.jdo:jar:3.2.0-m3:compile
[INFO] | \- javax.transaction:transaction-api:jar:1.1:compile
[INFO] +- org.apache.avro:avro:jar:1.8.2:compile
[INFO] | +- org.codehaus.jackson:jackson-core-asl:jar:1.9.13:compile
[INFO] | \- org.tukaani:xz:jar:1.5:compile
[INFO] +- org.apache.avro:avro-mapred:jar:hadoop2:1.8.2:compile
[INFO] | \- org.apache.avro:avro-ipc:jar:1.8.2:compile
[INFO] +- commons-httpclient:commons-httpclient:jar:3.1:compile
[INFO] | \- commons-logging:commons-logging:jar:1.0.4:compile
[INFO] +- org.apache.httpcomponents:httpclient:jar:4.5.6:compile
[INFO] | \- org.apache.httpcomponents:httpcore:jar:4.4.10:compile
[INFO] +- org.codehaus.jackson:jackson-mapper-asl:jar:1.9.13:compile
[INFO] +- commons-codec:commons-codec:jar:1.10:compile
[INFO] +- joda-time:joda-time:jar:2.9.3:compile
[INFO] +- org.jodd:jodd-core:jar:3.5.2:compile
[INFO] +- com.google.code.findbugs:jsr305:jar:3.0.0:compile
[INFO] +- org.datanucleus:datanucleus-core:jar:4.1.17:compile
[INFO] +- org.apache.thrift:libthrift:jar:0.12.0:compile
[INFO] +- org.apache.thrift:libfb303:jar:0.9.3:compile
[INFO] +- org.apache.derby:derby:jar:10.12.1.1:compile
[INFO] +- org.scala-lang:scala-compiler:jar:2.12.8:test
[INFO] | \- org.scala-lang.modules:scala-xml_2.12:jar:1.0.6:compile
[INFO] +- org.scalacheck:scalacheck_2.12:jar:1.13.5:test
[INFO] | \- org.scala-sbt:test-interface:jar:1.0:test
[INFO] +- org.apache.hive:hive-common:jar:2.3.4:compile
[INFO] | +- jline:jline:jar:2.14.6:compile
[INFO] | +- com.tdunning:json:jar:1.8:compile
[INFO] | \-
com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter:jar:0.1.2:compile
[INFO] +- org.apache.hive:hive-serde:jar:2.3.4:compile
[INFO] | +- org.apache.hive:hive-service-rpc:jar:2.3.4:compile
[INFO] | \- net.sf.opencsv:opencsv:jar:2.3:compile
[INFO] +- org.apache.hive:hive-shims:jar:2.3.4:compile
[INFO] | +- org.apache.hive.shims:hive-shims-common:jar:2.3.4:compile
[INFO] | +- org.apache.hive.shims:hive-shims-0.23:jar:2.3.4:runtime
[INFO] | \- org.apache.hive.shims:hive-shims-scheduler:jar:2.3.4:runtime
[INFO] +- org.apache.hive:hive-llap-common:jar:2.3.4:compile
[INFO] +- org.apache.hive:hive-llap-client:jar:2.3.4:test
[INFO] +- org.spark-project.spark:unused:jar:1.0.0:compile
[INFO] +- org.scalatest:scalatest_2.12:jar:3.0.5:test
[INFO] | \- org.scalactic:scalactic_2.12:jar:3.0.5:test
[INFO] +- junit:junit:jar:4.12:test
[INFO] | \- org.hamcrest:hamcrest-core:jar:1.3:test
[INFO] +- com.novocode:junit-interface:jar:0.11:test
[INFO] \- org.apache.hive:hive-storage-api:jar:2.6.0:compile
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
