This is an automated email from the ASF dual-hosted git repository.
andygrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 64b5ac3142 refactor: Move most of `comet-common` module into
`comet-spark` (#4325)
64b5ac3142 is described below
commit 64b5ac31421885b98e75aadfb4deedd90fd12fc1
Author: Andy Grove <[email protected]>
AuthorDate: Mon May 18 16:46:40 2026 -0600
refactor: Move most of `comet-common` module into `comet-spark` (#4325)
---
.github/actions/setup-spark-builder/action.yaml | 2 +-
.github/workflows/iceberg_spark_test.yml | 2 -
.github/workflows/pr_build_linux.yml | 2 +-
Makefile | 32 ++--
common/pom.xml | 194 ++-------------------
common/src/main/resources/log4j2.properties | 40 -----
common/src/test/resources/log4j.properties | 36 ----
common/src/test/resources/log4j2.properties | 40 -----
dev/release/build-release-comet.sh | 4 +-
native/jni-bridge/src/errors.rs | 3 +-
spark-integration/pom.xml | 18 +-
spark/pom.xml | 109 ++++++++++--
.../arrow/c/AbstractCometSchemaImporter.java | 0
.../java/org/apache/arrow/c/ArrowImporter.java | 0
.../java/org/apache/comet/CometSchemaImporter.java | 0
.../src/main/java/org/apache/comet/IcebergApi.java | 0
.../src/main/java/org/apache/comet/NativeBase.java | 0
.../apache/comet/parquet/AbstractColumnReader.java | 0
.../comet/parquet/ArrowConstantColumnReader.java | 0
.../comet/parquet/ArrowRowIndexColumnReader.java | 0
.../apache/comet/parquet/BloomFilterReader.java | 0
.../apache/comet/parquet/ColumnIndexReader.java | 0
.../org/apache/comet/parquet/ColumnPageReader.java | 0
.../org/apache/comet/parquet/ColumnReader.java | 0
.../comet/parquet/CometFileKeyUnwrapper.java | 0
.../org/apache/comet/parquet/CometInputFile.java | 0
.../apache/comet/parquet/DictionaryPageReader.java | 0
.../java/org/apache/comet/parquet/FileReader.java | 0
.../org/apache/comet/parquet/FooterReader.java | 0
.../parquet/IcebergCometNativeBatchReader.java | 0
.../java/org/apache/comet/parquet/IndexFilter.java | 0
.../org/apache/comet/parquet/LazyColumnReader.java | 0
.../main/java/org/apache/comet/parquet/Native.java | 0
.../apache/comet/parquet/NativeBatchReader.java | 0
.../apache/comet/parquet/NativeColumnReader.java | 0
.../apache/comet/parquet/ParquetColumnSpec.java | 0
.../comet/parquet/ParquetMetadataSerializer.java | 0
.../java/org/apache/comet/parquet/ReadOptions.java | 0
.../org/apache/comet/parquet/RowGroupFilter.java | 0
.../org/apache/comet/parquet/RowGroupReader.java | 0
.../java/org/apache/comet/parquet/TypeUtil.java | 0
.../main/java/org/apache/comet/parquet/Utils.java | 0
.../org/apache/comet/parquet/WrappedInputFile.java | 0
.../comet/parquet/WrappedSeekableInputStream.java | 0
.../java/org/apache/comet/udf/CometUdfBridge.java | 0
.../apache/comet/vector/CometDecodedVector.java | 0
.../apache/comet/vector/CometDelegateVector.java | 0
.../org/apache/comet/vector/CometDictionary.java | 0
.../apache/comet/vector/CometDictionaryVector.java | 0
.../org/apache/comet/vector/CometLazyVector.java | 0
.../org/apache/comet/vector/CometListVector.java | 0
.../org/apache/comet/vector/CometMapVector.java | 0
.../org/apache/comet/vector/CometPlainVector.java | 0
.../apache/comet/vector/CometSelectionVector.java | 0
.../org/apache/comet/vector/CometStructVector.java | 0
.../java/org/apache/comet/vector/CometVector.java | 0
.../main/scala/org/apache/comet/CometConf.scala | 0
.../main/scala/org/apache/comet/Constants.scala | 0
.../apache/comet/objectstore/NativeConfig.scala | 0
.../src/main/scala/org/apache/comet/package.scala | 0
.../apache/comet/parquet/CometParquetUtils.scala | 0
.../comet/parquet/CometReaderThreadPool.scala | 0
.../org/apache/comet/udf/CometLambdaRegistry.scala | 0
.../main/scala/org/apache/comet/udf/CometUDF.scala | 0
.../scala/org/apache/comet/vector/NativeUtil.scala | 0
.../org/apache/comet/vector/StreamReader.scala | 0
.../apache/spark/comet/CometTaskContextShim.scala | 0
.../spark/sql/comet/CastOverflowException.scala | 0
.../execution/arrow/ArrowReaderIterator.scala | 0
.../sql/comet/execution/arrow/ArrowWriters.scala | 0
.../execution/arrow/CometArrowConverters.scala | 0
.../comet/parquet/CometParquetReadSupport.scala | 0
.../CometSparkToParquetSchemaConverter.scala | 0
.../org/apache/spark/sql/comet/util/Utils.scala | 0
.../org/apache/comet/shims/ShimBatchReader.scala | 0
.../org/apache/comet/shims/ShimFileFormat.scala | 0
.../spark/sql/comet/shims/ShimTaskMetrics.scala | 0
.../org/apache/comet/shims/ShimBatchReader.scala | 0
.../org/apache/comet/shims/ShimFileFormat.scala | 0
.../spark/sql/comet/shims/ShimTaskMetrics.scala | 0
.../org/apache/comet/shims/CometTypeShim.scala | 0
.../org/apache/comet/shims/ShimCometConf.scala | 0
.../org/apache/comet/shims/CometTypeShim.scala | 0
.../org/apache/comet/shims/ShimBatchReader.scala | 0
.../org/apache/comet/shims/ShimCometConf.scala | 0
.../org/apache/comet/shims/ShimFileFormat.scala | 0
.../spark/sql/comet/shims/ShimTaskMetrics.scala | 0
.../org/apache/comet/parquet/TestColumnReader.java | 0
.../apache/comet/parquet/TestCometInputFile.java | 0
.../org/apache/comet/parquet/TestFileReader.java | 0
.../java/org/apache/comet/parquet/TestUtils.java | 0
91 files changed, 150 insertions(+), 332 deletions(-)
diff --git a/.github/actions/setup-spark-builder/action.yaml
b/.github/actions/setup-spark-builder/action.yaml
index 8f2882efe4..454a30fb57 100644
--- a/.github/actions/setup-spark-builder/action.yaml
+++ b/.github/actions/setup-spark-builder/action.yaml
@@ -51,7 +51,7 @@ runs:
path: |
~/.m2/repository
/root/.m2/repository
- key: ${{ runner.os }}-spark-sql-${{ hashFiles('spark/**/pom.xml',
'common/**/pom.xml') }}
+ key: ${{ runner.os }}-spark-sql-${{ hashFiles('common/**/pom.xml',
'spark/**/pom.xml') }}
restore-keys: |
${{ runner.os }}-spark-sql-
diff --git a/.github/workflows/iceberg_spark_test.yml
b/.github/workflows/iceberg_spark_test.yml
index 84abb1fb38..712c3d5178 100644
--- a/.github/workflows/iceberg_spark_test.yml
+++ b/.github/workflows/iceberg_spark_test.yml
@@ -33,7 +33,6 @@ on:
- "native/core/benches/**"
- "native/spark-expr/benches/**"
- "spark/src/test/**"
- - "common/src/test/**"
- "spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
- "spark-integration/**"
pull_request:
@@ -45,7 +44,6 @@ on:
- "native/core/benches/**"
- "native/spark-expr/benches/**"
- "spark/src/test/**"
- - "common/src/test/**"
- "spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
- "spark-integration/**"
# manual trigger
diff --git a/.github/workflows/pr_build_linux.yml
b/.github/workflows/pr_build_linux.yml
index bee0c38e8f..a370e8c2d8 100644
--- a/.github/workflows/pr_build_linux.yml
+++ b/.github/workflows/pr_build_linux.yml
@@ -189,7 +189,7 @@ jobs:
uses: ./.github/actions/setup-builder
with:
rust-version: ${{ env.RUST_VERSION }}
- jdk-version: 17 # JDK only needed for common module proto generation
+ jdk-version: 17 # JDK only needed for JVM module proto generation
- name: Restore Cargo cache
uses: actions/cache/restore@v5
diff --git a/Makefile b/Makefile
index 60ea51bf0c..8685d171be 100644
--- a/Makefile
+++ b/Makefile
@@ -68,28 +68,28 @@ endif
core-amd64:
rustup target add x86_64-apple-darwin
cd native && RUSTFLAGS="-Ctarget-cpu=skylake" CC=o64-clang
CXX=o64-clang++ cargo build --target x86_64-apple-darwin --release
$(FEATURES_ARG)
- mkdir -p common/target/classes/org/apache/comet/darwin/x86_64
- cp native/target/x86_64-apple-darwin/release/libcomet.dylib
common/target/classes/org/apache/comet/darwin/x86_64
+ mkdir -p spark/target/classes/org/apache/comet/darwin/x86_64
+ cp native/target/x86_64-apple-darwin/release/libcomet.dylib
spark/target/classes/org/apache/comet/darwin/x86_64
cd native && RUSTFLAGS="-Ctarget-cpu=x86-64-v3" cargo build --release
$(FEATURES_ARG)
- mkdir -p common/target/classes/org/apache/comet/linux/amd64
- cp native/target/release/libcomet.so
common/target/classes/org/apache/comet/linux/amd64
- jar -cf common/target/comet-native-x86_64.jar \
- -C common/target/classes/org/apache/comet darwin \
- -C common/target/classes/org/apache/comet linux
- ./dev/deploy-file common/target/comet-native-x86_64.jar
comet-native-x86_64${COMET_CLASSIFIER} jar
+ mkdir -p spark/target/classes/org/apache/comet/linux/amd64
+ cp native/target/release/libcomet.so
spark/target/classes/org/apache/comet/linux/amd64
+ jar -cf spark/target/comet-native-x86_64.jar \
+ -C spark/target/classes/org/apache/comet darwin \
+ -C spark/target/classes/org/apache/comet linux
+ ./dev/deploy-file spark/target/comet-native-x86_64.jar
comet-native-x86_64${COMET_CLASSIFIER} jar
core-arm64:
rustup target add aarch64-apple-darwin
cd native && RUSTFLAGS="-Ctarget-cpu=apple-m1"
CC=arm64-apple-darwin21.4-clang CXX=arm64-apple-darwin21.4-clang++
CARGO_FEATURE_NEON=1 cargo build --target aarch64-apple-darwin --release
$(FEATURES_ARG)
- mkdir -p common/target/classes/org/apache/comet/darwin/aarch64
- cp native/target/aarch64-apple-darwin/release/libcomet.dylib
common/target/classes/org/apache/comet/darwin/aarch64
+ mkdir -p spark/target/classes/org/apache/comet/darwin/aarch64
+ cp native/target/aarch64-apple-darwin/release/libcomet.dylib
spark/target/classes/org/apache/comet/darwin/aarch64
cd native && RUSTFLAGS="-Ctarget-cpu=neoverse-n1" cargo build --release
$(FEATURES_ARG)
- mkdir -p common/target/classes/org/apache/comet/linux/aarch64
- cp native/target/release/libcomet.so
common/target/classes/org/apache/comet/linux/aarch64
- jar -cf common/target/comet-native-aarch64.jar \
- -C common/target/classes/org/apache/comet darwin \
- -C common/target/classes/org/apache/comet linux
- ./dev/deploy-file common/target/comet-native-aarch64.jar
comet-native-aarch64${COMET_CLASSIFIER} jar
+ mkdir -p spark/target/classes/org/apache/comet/linux/aarch64
+ cp native/target/release/libcomet.so
spark/target/classes/org/apache/comet/linux/aarch64
+ jar -cf spark/target/comet-native-aarch64.jar \
+ -C spark/target/classes/org/apache/comet darwin \
+ -C spark/target/classes/org/apache/comet linux
+ ./dev/deploy-file spark/target/comet-native-aarch64.jar
comet-native-aarch64${COMET_CLASSIFIER} jar
release-linux: clean
rustup target add aarch64-apple-darwin x86_64-apple-darwin
diff --git a/common/pom.xml b/common/pom.xml
index 4556f6a334..43c5033dd1 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -33,196 +33,38 @@ under the License.
<artifactId>comet-common-spark${spark.version.short}_${scala.binary.version}</artifactId>
<name>comet-common</name>
+ <!--
+ Minimal bootstrap module. Contains only the exception classes that the
+ Rust JNI bridge needs to load via JNI (CometNativeException and friends).
+ Kept small on purpose so `mvn compile -pl common` is a cheap prerequisite
+ for `cargo test` in the Rust workspace. All other JVM code lives in
+ comet-spark.
+ -->
+
<properties>
<!-- Reverse default (skip installation), and then enable only for child
modules -->
<maven.deploy.skip>false</maven.deploy.skip>
</properties>
- <dependencies>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.parquet</groupId>
- <artifactId>parquet-column</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.parquet</groupId>
- <artifactId>parquet-hadoop</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.parquet</groupId>
- <artifactId>parquet-format-structures</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.arrow</groupId>
- <artifactId>arrow-vector</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.arrow</groupId>
- <artifactId>arrow-memory-unsafe</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.arrow</groupId>
- <artifactId>arrow-c-data</artifactId>
- </dependency>
- <dependency>
- <groupId>org.scala-lang.modules</groupId>
- <artifactId>scala-collection-compat_${scala.binary.version}</artifactId>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.assertj</groupId>
- <artifactId>assertj-core</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
<build>
<plugins>
- <plugin>
- <groupId>io.github.git-commit-id</groupId>
- <artifactId>git-commit-id-maven-plugin</artifactId>
- <version>${git-commit-id-maven-plugin.version}</version>
- <executions>
- <execution>
- <id>get-the-git-infos</id>
- <goals>
- <goal>revision</goal>
- </goals>
- <phase>initialize</phase>
- </execution>
- </executions>
- <configuration>
- <generateGitPropertiesFile>true</generateGitPropertiesFile>
-
<generateGitPropertiesFilename>${project.build.outputDirectory}/comet-git-info.properties</generateGitPropertiesFilename>
- <commitIdGenerationMode>full</commitIdGenerationMode>
- <includeOnlyProperties>
- <includeOnlyProperty>^git.branch$</includeOnlyProperty>
- <includeOnlyProperty>^git.build.*$</includeOnlyProperty>
-
<includeOnlyProperty>^git.commit.id.(abbrev|full)$</includeOnlyProperty>
- <includeOnlyProperty>^git.remote.*$</includeOnlyProperty>
- </includeOnlyProperties>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <createSourcesJar>true</createSourcesJar>
- <shadeSourcesContent>true</shadeSourcesContent>
- <shadedArtifactAttached>false</shadedArtifactAttached>
- <createDependencyReducedPom>true</createDependencyReducedPom>
- <artifactSet>
- <includes>
- <!-- We shade & relocation most of the Arrow classes, to
prevent them from
- conflicting with those in Spark -->
- <include>org.apache.arrow:*</include>
- </includes>
- </artifactSet>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>**/*.thrift</exclude>
- <exclude>git.properties</exclude>
- <exclude>log4j.properties</exclude>
- <exclude>log4j2.properties</exclude>
- <exclude>arrow-git.properties</exclude>
- </excludes>
- </filter>
- <filter>
- <artifact>org.apache.arrow:arrow-vector</artifact>
- <excludes>
- <!-- Comet doesn't need codegen templates on Arrow -->
- <exclude>codegen/**</exclude>
- </excludes>
- </filter>
- </filters>
- <relocations>
- <relocation>
- <pattern>org.apache.arrow</pattern>
-
<shadedPattern>${comet.shade.packageName}.arrow</shadedPattern>
- <excludes>
- <!-- We can't allocate Jni classes. These classes has no
extra dependencies
- so it should be OK to exclude -->
- <exclude>org/apache/arrow/c/jni/JniWrapper</exclude>
- <exclude>org/apache/arrow/c/jni/PrivateData</exclude>
- <exclude>org/apache/arrow/c/jni/CDataJniException</exclude>
- <!-- Also used by JNI:
https://github.com/apache/arrow/blob/apache-arrow-11.0.0/java/c/src/main/cpp/jni_wrapper.cc#L341
- Note this class is not used by us, but required when
loading the native lib -->
-
<exclude>org/apache/arrow/c/ArrayStreamExporter$ExportedArrayStreamPrivateData
- </exclude>
- </excludes>
- </relocation>
- </relocations>
- </configuration>
- </execution>
- </executions>
- </plugin>
+ <!-- scala-maven-plugin handles javac too: the parent pom's
+ maven-compiler-plugin is configured with skipMain=true. -->
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
</plugin>
+ <!-- This module has no Scala sources; skip scalafix so the lint job
+ does not fail with "No files to fix". -->
<plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-shim-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>src/main/${shims.majorVerSrc}</source>
- <source>src/main/${shims.minorVerSrc}</source>
- </sources>
- </configuration>
- </execution>
- </executions>
+ <groupId>io.github.evis</groupId>
+ <artifactId>scalafix-maven-plugin_${scala.binary.version}</artifactId>
+ <version>${scalafix-maven-plugin.version}</version>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
</plugin>
</plugins>
- <resources>
- <resource>
- <directory>${project.basedir}/src/main/resources</directory>
- </resource>
- <resource>
-
<directory>${project.basedir}/../native/target/x86_64-apple-darwin/release</directory>
- <includes>
- <include>libcomet.dylib</include>
- </includes>
- <targetPath>org/apache/comet/darwin/x86_64</targetPath>
- </resource>
- <resource>
-
<directory>${project.basedir}/../native/target/aarch64-apple-darwin/release</directory>
- <includes>
- <include>libcomet.dylib</include>
- </includes>
- <targetPath>org/apache/comet/darwin/aarch64</targetPath>
- </resource>
- <resource>
- <directory>${jni.dir}</directory>
- <includes>
- <include>libcomet.dylib</include>
- <include>libcomet.so</include>
- <include>comet.dll</include>
- </includes>
- <targetPath>org/apache/comet/${platform}/${arch}</targetPath>
- </resource>
- </resources>
</build>
</project>
diff --git a/common/src/main/resources/log4j2.properties
b/common/src/main/resources/log4j2.properties
deleted file mode 100644
index 04cdf85330..0000000000
--- a/common/src/main/resources/log4j2.properties
+++ /dev/null
@@ -1,40 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Set everything to be logged to the file target/unit-tests.log
-rootLogger.level = info
-rootLogger.appenderRef.file.ref = ${sys:test.appender:-File}
-
-appender.file.type = File
-appender.file.name = File
-appender.file.fileName = target/unit-tests.log
-appender.file.layout.type = PatternLayout
-appender.file.layout.pattern = %d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n
-
-# Tests that launch java subprocesses can set the "test.appender" system
property to
-# "console" to avoid having the child process's logs overwrite the unit test's
-# log file.
-appender.console.type = Console
-appender.console.name = console
-appender.console.target = SYSTEM_ERR
-appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = %t: %m%n
-
-# Ignore messages below warning level from Jetty, because it's a bit verbose
-logger.jetty.name = org.sparkproject.jetty
-logger.jetty.level = warn
-
diff --git a/common/src/test/resources/log4j.properties
b/common/src/test/resources/log4j.properties
deleted file mode 100644
index 2f46ce1553..0000000000
--- a/common/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,36 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Set everything to be logged to the file target/unit-tests.log
-test.appender=file
-log4j.rootCategory=INFO, ${test.appender}
-log4j.appender.file=org.apache.log4j.FileAppender
-log4j.appender.file.append=true
-log4j.appender.file.file=target/unit-tests.log
-log4j.appender.file.layout=org.apache.log4j.PatternLayout
-log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p
%c{1}: %m%n
-
-# Tests that launch java subprocesses can set the "test.appender" system
property to
-# "console" to avoid having the child process's logs overwrite the unit test's
-# log file.
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%t: %m%n
-
-# Ignore messages below warning level from Jetty, because it's a bit verbose
-log4j.logger.org.sparkproject.jetty=WARN
diff --git a/common/src/test/resources/log4j2.properties
b/common/src/test/resources/log4j2.properties
deleted file mode 100644
index 04cdf85330..0000000000
--- a/common/src/test/resources/log4j2.properties
+++ /dev/null
@@ -1,40 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Set everything to be logged to the file target/unit-tests.log
-rootLogger.level = info
-rootLogger.appenderRef.file.ref = ${sys:test.appender:-File}
-
-appender.file.type = File
-appender.file.name = File
-appender.file.fileName = target/unit-tests.log
-appender.file.layout.type = PatternLayout
-appender.file.layout.pattern = %d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n
-
-# Tests that launch java subprocesses can set the "test.appender" system
property to
-# "console" to avoid having the child process's logs overwrite the unit test's
-# log file.
-appender.console.type = Console
-appender.console.name = console
-appender.console.target = SYSTEM_ERR
-appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = %t: %m%n
-
-# Ignore messages below warning level from Jetty, because it's a bit verbose
-logger.jetty.name = org.sparkproject.jetty
-logger.jetty.level = warn
-
diff --git a/dev/release/build-release-comet.sh
b/dev/release/build-release-comet.sh
index 23363c8761..159394e0f5 100755
--- a/dev/release/build-release-comet.sh
+++ b/dev/release/build-release-comet.sh
@@ -125,7 +125,7 @@ docker build --no-cache \
# Clean previous Java build
pushd $COMET_HOME_DIR && ./mvnw clean && popd
-# Clean previous native build. This is required because common/pom.xml has
+# Clean previous native build. This is required because spark/pom.xml has
# unconditional resource entries that bundle libcomet.dylib from
# native/target/{x86_64,aarch64}-apple-darwin/release. If a release manager
# previously cross-compiled those targets locally, stale dylibs would leak
@@ -169,7 +169,7 @@ fi
echo "Building binaries completed"
echo "Copying to java build directories"
-JVM_TARGET_DIR=$COMET_HOME_DIR/common/target/classes/org/apache/comet
+JVM_TARGET_DIR=$COMET_HOME_DIR/spark/target/classes/org/apache/comet
mkdir -p $JVM_TARGET_DIR
mkdir -p $JVM_TARGET_DIR/linux/amd64
diff --git a/native/jni-bridge/src/errors.rs b/native/jni-bridge/src/errors.rs
index 4fbcb28e2a..7bf4073c8d 100644
--- a/native/jni-bridge/src/errors.rs
+++ b/native/jni-bridge/src/errors.rs
@@ -718,7 +718,8 @@ mod tests {
init();
INIT.call_once(|| {
- // Add common classes to the classpath in so that we can find
CometException
+ // Add comet-common classes to the classpath so we can find the
Comet exception
+ // classes (CometNativeException, CometQueryExecutionException,
etc.).
let mut common_classes = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
common_classes.push("../../common/target/classes");
let mut class_path = common_classes
diff --git a/spark-integration/pom.xml b/spark-integration/pom.xml
index 356f91adfd..9aacf8db23 100644
--- a/spark-integration/pom.xml
+++ b/spark-integration/pom.xml
@@ -45,11 +45,19 @@ under the License.
<artifactId>comet-spark-spark${spark.version.short}_${scala.binary.version}</artifactId>
<version>${project.version}</version>
<exclusions>
- <!-- This is shaded into the jar -->
- <exclusion>
- <groupId>org.apache.datafusion</groupId>
-
<artifactId>comet-common-spark${spark.version.short}_${scala.binary.version}</artifactId>
- </exclusion>
+ <!-- comet-common is bundled into the shaded comet-spark jar
-->
+ <exclusion>
+ <groupId>org.apache.datafusion</groupId>
+
<artifactId>comet-common-spark${spark.version.short}_${scala.binary.version}</artifactId>
+ </exclusion>
+ <!-- Arrow classes are shaded into the comet-spark jar
(relocated to
+ org.apache.comet.shaded.arrow.*); a small set of JNI
classes intentionally
+ keep their original org.apache.arrow.* package paths.
Pulling Arrow
+ transitively here would duplicate those JNI classes on
the classpath. -->
+ <exclusion>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
</exclusions>
</dependency>
</dependencies>
diff --git a/spark/pom.xml b/spark/pom.xml
index d3c18ccf87..6d97ea831f 100644
--- a/spark/pom.xml
+++ b/spark/pom.xml
@@ -43,17 +43,27 @@ under the License.
<groupId>org.apache.datafusion</groupId>
<artifactId>comet-common-spark${spark.version.short}_${scala.binary.version}</artifactId>
<version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.arrow</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.binary.version}</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.parquet</groupId>
+ <artifactId>parquet-column</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.parquet</groupId>
+ <artifactId>parquet-format-structures</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>arrow-vector</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.scala-lang.modules</groupId>
+ <artifactId>scala-collection-compat_${scala.binary.version}</artifactId>
+ </dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
@@ -128,18 +138,16 @@ under the License.
<!-- Note we don't use test scope for this artifact. This is because
it's only needed
to provide InMemoryKMS class that is shaded below, to make Spark test
happy. -->
</dependency>
- <!-- We shade & relocate Arrow dependencies in comet-common, so
comet-spark module no longer
- depends on Arrow. However, when running `mvn test` we still need
Arrow classes in the
- classpath, since the Maven shading happens in 'package' phase which
is after 'test' -->
+ <!-- Arrow memory + C-data deps required at compile scope. Shading
relocates these into
+ org.apache.comet.shaded.arrow during the package phase to avoid
colliding with Spark's
+ own Arrow on the executor classpath. -->
<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-memory-unsafe</artifactId>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-c-data</artifactId>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -355,6 +363,31 @@ under the License.
<build>
<plugins>
+ <plugin>
+ <groupId>io.github.git-commit-id</groupId>
+ <artifactId>git-commit-id-maven-plugin</artifactId>
+ <version>${git-commit-id-maven-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>get-the-git-infos</id>
+ <goals>
+ <goal>revision</goal>
+ </goals>
+ <phase>initialize</phase>
+ </execution>
+ </executions>
+ <configuration>
+ <generateGitPropertiesFile>true</generateGitPropertiesFile>
+
<generateGitPropertiesFilename>${project.build.outputDirectory}/comet-git-info.properties</generateGitPropertiesFilename>
+ <commitIdGenerationMode>full</commitIdGenerationMode>
+ <includeOnlyProperties>
+ <includeOnlyProperty>^git.branch$</includeOnlyProperty>
+ <includeOnlyProperty>^git.build.*$</includeOnlyProperty>
+
<includeOnlyProperty>^git.commit.id.(abbrev|full)$</includeOnlyProperty>
+ <includeOnlyProperty>^git.remote.*$</includeOnlyProperty>
+ </includeOnlyProperties>
+ </configuration>
+ </plugin>
<plugin>
<groupId>com.github.os72</groupId>
<artifactId>protoc-jar-maven-plugin</artifactId>
@@ -394,11 +427,13 @@ under the License.
<createDependencyReducedPom>true</createDependencyReducedPom>
<artifactSet>
<includes>
+ <!-- Bundle comet-common so the shaded uber-jar is
self-contained -->
<include>org.apache.datafusion:comet-common-spark${spark.version.short}_${scala.binary.version}</include>
+ <!-- Shade & relocate Arrow to prevent conflicts with
Spark's own Arrow -->
+ <include>org.apache.arrow:*</include>
<!-- Relocate Protobuf since Spark uses 2.5.0 while Comet
uses 3.x -->
<include>com.google.protobuf:protobuf-java</include>
<include>com.google.guava:guava</include>
-
<include>org.scala-lang.modules:scala-collection-compat_${scala.binary.version}</include>
</includes>
</artifactSet>
<filters>
@@ -410,9 +445,17 @@ under the License.
<exclude>git.properties</exclude>
<exclude>log4j.properties</exclude>
<exclude>log4j2.properties</exclude>
+ <exclude>arrow-git.properties</exclude>
<exclude>**/SparkFilterApi.*</exclude>
</excludes>
</filter>
+ <filter>
+ <artifact>org.apache.arrow:arrow-vector</artifact>
+ <excludes>
+ <!-- Comet doesn't need codegen templates on Arrow -->
+ <exclude>codegen/**</exclude>
+ </excludes>
+ </filter>
<filter>
<artifact>org.apache.parquet:parquet-hadoop:tests</artifact>
<includes>
@@ -422,6 +465,20 @@ under the License.
</filter>
</filters>
<relocations>
+ <relocation>
+ <pattern>org.apache.arrow</pattern>
+
<shadedPattern>${comet.shade.packageName}.arrow</shadedPattern>
+ <excludes>
+ <!-- These classes are referenced from Arrow JNI native
code and must stay
+ at their original package paths. They have no extra
dependencies so the
+ exclusion is safe. -->
+ <exclude>org/apache/arrow/c/jni/JniWrapper</exclude>
+ <exclude>org/apache/arrow/c/jni/PrivateData</exclude>
+ <exclude>org/apache/arrow/c/jni/CDataJniException</exclude>
+ <!-- Also required by JNI per upstream Arrow C++
jni_wrapper.cc -->
+
<exclude>org/apache/arrow/c/ArrayStreamExporter$ExportedArrayStreamPrivateData</exclude>
+ </excludes>
+ </relocation>
<relocation>
<pattern>com.google.protobuf</pattern>
<shadedPattern>${comet.shade.packageName}.protobuf</shadedPattern>
@@ -492,6 +549,34 @@ under the License.
</executions>
</plugin>
</plugins>
+ <resources>
+ <resource>
+ <directory>${project.basedir}/src/main/resources</directory>
+ </resource>
+ <resource>
+
<directory>${project.basedir}/../native/target/x86_64-apple-darwin/release</directory>
+ <includes>
+ <include>libcomet.dylib</include>
+ </includes>
+ <targetPath>org/apache/comet/darwin/x86_64</targetPath>
+ </resource>
+ <resource>
+
<directory>${project.basedir}/../native/target/aarch64-apple-darwin/release</directory>
+ <includes>
+ <include>libcomet.dylib</include>
+ </includes>
+ <targetPath>org/apache/comet/darwin/aarch64</targetPath>
+ </resource>
+ <resource>
+ <directory>${jni.dir}</directory>
+ <includes>
+ <include>libcomet.dylib</include>
+ <include>libcomet.so</include>
+ <include>comet.dll</include>
+ </includes>
+ <targetPath>org/apache/comet/${platform}/${arch}</targetPath>
+ </resource>
+ </resources>
</build>
</project>
diff --git
a/common/src/main/java/org/apache/arrow/c/AbstractCometSchemaImporter.java
b/spark/src/main/java/org/apache/arrow/c/AbstractCometSchemaImporter.java
similarity index 100%
rename from
common/src/main/java/org/apache/arrow/c/AbstractCometSchemaImporter.java
rename to
spark/src/main/java/org/apache/arrow/c/AbstractCometSchemaImporter.java
diff --git a/common/src/main/java/org/apache/arrow/c/ArrowImporter.java
b/spark/src/main/java/org/apache/arrow/c/ArrowImporter.java
similarity index 100%
rename from common/src/main/java/org/apache/arrow/c/ArrowImporter.java
rename to spark/src/main/java/org/apache/arrow/c/ArrowImporter.java
diff --git a/common/src/main/java/org/apache/comet/CometSchemaImporter.java
b/spark/src/main/java/org/apache/comet/CometSchemaImporter.java
similarity index 100%
rename from common/src/main/java/org/apache/comet/CometSchemaImporter.java
rename to spark/src/main/java/org/apache/comet/CometSchemaImporter.java
diff --git a/common/src/main/java/org/apache/comet/IcebergApi.java
b/spark/src/main/java/org/apache/comet/IcebergApi.java
similarity index 100%
rename from common/src/main/java/org/apache/comet/IcebergApi.java
rename to spark/src/main/java/org/apache/comet/IcebergApi.java
diff --git a/common/src/main/java/org/apache/comet/NativeBase.java
b/spark/src/main/java/org/apache/comet/NativeBase.java
similarity index 100%
rename from common/src/main/java/org/apache/comet/NativeBase.java
rename to spark/src/main/java/org/apache/comet/NativeBase.java
diff --git
a/common/src/main/java/org/apache/comet/parquet/AbstractColumnReader.java
b/spark/src/main/java/org/apache/comet/parquet/AbstractColumnReader.java
similarity index 100%
rename from
common/src/main/java/org/apache/comet/parquet/AbstractColumnReader.java
rename to spark/src/main/java/org/apache/comet/parquet/AbstractColumnReader.java
diff --git
a/common/src/main/java/org/apache/comet/parquet/ArrowConstantColumnReader.java
b/spark/src/main/java/org/apache/comet/parquet/ArrowConstantColumnReader.java
similarity index 100%
rename from
common/src/main/java/org/apache/comet/parquet/ArrowConstantColumnReader.java
rename to
spark/src/main/java/org/apache/comet/parquet/ArrowConstantColumnReader.java
diff --git
a/common/src/main/java/org/apache/comet/parquet/ArrowRowIndexColumnReader.java
b/spark/src/main/java/org/apache/comet/parquet/ArrowRowIndexColumnReader.java
similarity index 100%
rename from
common/src/main/java/org/apache/comet/parquet/ArrowRowIndexColumnReader.java
rename to
spark/src/main/java/org/apache/comet/parquet/ArrowRowIndexColumnReader.java
diff --git
a/common/src/main/java/org/apache/comet/parquet/BloomFilterReader.java
b/spark/src/main/java/org/apache/comet/parquet/BloomFilterReader.java
similarity index 100%
rename from common/src/main/java/org/apache/comet/parquet/BloomFilterReader.java
rename to spark/src/main/java/org/apache/comet/parquet/BloomFilterReader.java
diff --git
a/common/src/main/java/org/apache/comet/parquet/ColumnIndexReader.java
b/spark/src/main/java/org/apache/comet/parquet/ColumnIndexReader.java
similarity index 100%
rename from common/src/main/java/org/apache/comet/parquet/ColumnIndexReader.java
rename to spark/src/main/java/org/apache/comet/parquet/ColumnIndexReader.java
diff --git
a/common/src/main/java/org/apache/comet/parquet/ColumnPageReader.java
b/spark/src/main/java/org/apache/comet/parquet/ColumnPageReader.java
similarity index 100%
rename from common/src/main/java/org/apache/comet/parquet/ColumnPageReader.java
rename to spark/src/main/java/org/apache/comet/parquet/ColumnPageReader.java
diff --git a/common/src/main/java/org/apache/comet/parquet/ColumnReader.java
b/spark/src/main/java/org/apache/comet/parquet/ColumnReader.java
similarity index 100%
rename from common/src/main/java/org/apache/comet/parquet/ColumnReader.java
rename to spark/src/main/java/org/apache/comet/parquet/ColumnReader.java
diff --git
a/common/src/main/java/org/apache/comet/parquet/CometFileKeyUnwrapper.java
b/spark/src/main/java/org/apache/comet/parquet/CometFileKeyUnwrapper.java
similarity index 100%
rename from
common/src/main/java/org/apache/comet/parquet/CometFileKeyUnwrapper.java
rename to
spark/src/main/java/org/apache/comet/parquet/CometFileKeyUnwrapper.java
diff --git a/common/src/main/java/org/apache/comet/parquet/CometInputFile.java
b/spark/src/main/java/org/apache/comet/parquet/CometInputFile.java
similarity index 100%
rename from common/src/main/java/org/apache/comet/parquet/CometInputFile.java
rename to spark/src/main/java/org/apache/comet/parquet/CometInputFile.java
diff --git
a/common/src/main/java/org/apache/comet/parquet/DictionaryPageReader.java
b/spark/src/main/java/org/apache/comet/parquet/DictionaryPageReader.java
similarity index 100%
rename from
common/src/main/java/org/apache/comet/parquet/DictionaryPageReader.java
rename to spark/src/main/java/org/apache/comet/parquet/DictionaryPageReader.java
diff --git a/common/src/main/java/org/apache/comet/parquet/FileReader.java
b/spark/src/main/java/org/apache/comet/parquet/FileReader.java
similarity index 100%
rename from common/src/main/java/org/apache/comet/parquet/FileReader.java
rename to spark/src/main/java/org/apache/comet/parquet/FileReader.java
diff --git a/common/src/main/java/org/apache/comet/parquet/FooterReader.java
b/spark/src/main/java/org/apache/comet/parquet/FooterReader.java
similarity index 100%
rename from common/src/main/java/org/apache/comet/parquet/FooterReader.java
rename to spark/src/main/java/org/apache/comet/parquet/FooterReader.java
diff --git
a/common/src/main/java/org/apache/comet/parquet/IcebergCometNativeBatchReader.java
b/spark/src/main/java/org/apache/comet/parquet/IcebergCometNativeBatchReader.java
similarity index 100%
rename from
common/src/main/java/org/apache/comet/parquet/IcebergCometNativeBatchReader.java
rename to
spark/src/main/java/org/apache/comet/parquet/IcebergCometNativeBatchReader.java
diff --git a/common/src/main/java/org/apache/comet/parquet/IndexFilter.java
b/spark/src/main/java/org/apache/comet/parquet/IndexFilter.java
similarity index 100%
rename from common/src/main/java/org/apache/comet/parquet/IndexFilter.java
rename to spark/src/main/java/org/apache/comet/parquet/IndexFilter.java
diff --git
a/common/src/main/java/org/apache/comet/parquet/LazyColumnReader.java
b/spark/src/main/java/org/apache/comet/parquet/LazyColumnReader.java
similarity index 100%
rename from common/src/main/java/org/apache/comet/parquet/LazyColumnReader.java
rename to spark/src/main/java/org/apache/comet/parquet/LazyColumnReader.java
diff --git a/common/src/main/java/org/apache/comet/parquet/Native.java
b/spark/src/main/java/org/apache/comet/parquet/Native.java
similarity index 100%
rename from common/src/main/java/org/apache/comet/parquet/Native.java
rename to spark/src/main/java/org/apache/comet/parquet/Native.java
diff --git
a/common/src/main/java/org/apache/comet/parquet/NativeBatchReader.java
b/spark/src/main/java/org/apache/comet/parquet/NativeBatchReader.java
similarity index 100%
rename from common/src/main/java/org/apache/comet/parquet/NativeBatchReader.java
rename to spark/src/main/java/org/apache/comet/parquet/NativeBatchReader.java
diff --git
a/common/src/main/java/org/apache/comet/parquet/NativeColumnReader.java
b/spark/src/main/java/org/apache/comet/parquet/NativeColumnReader.java
similarity index 100%
rename from
common/src/main/java/org/apache/comet/parquet/NativeColumnReader.java
rename to spark/src/main/java/org/apache/comet/parquet/NativeColumnReader.java
diff --git
a/common/src/main/java/org/apache/comet/parquet/ParquetColumnSpec.java
b/spark/src/main/java/org/apache/comet/parquet/ParquetColumnSpec.java
similarity index 100%
rename from common/src/main/java/org/apache/comet/parquet/ParquetColumnSpec.java
rename to spark/src/main/java/org/apache/comet/parquet/ParquetColumnSpec.java
diff --git
a/common/src/main/java/org/apache/comet/parquet/ParquetMetadataSerializer.java
b/spark/src/main/java/org/apache/comet/parquet/ParquetMetadataSerializer.java
similarity index 100%
rename from
common/src/main/java/org/apache/comet/parquet/ParquetMetadataSerializer.java
rename to
spark/src/main/java/org/apache/comet/parquet/ParquetMetadataSerializer.java
diff --git a/common/src/main/java/org/apache/comet/parquet/ReadOptions.java
b/spark/src/main/java/org/apache/comet/parquet/ReadOptions.java
similarity index 100%
rename from common/src/main/java/org/apache/comet/parquet/ReadOptions.java
rename to spark/src/main/java/org/apache/comet/parquet/ReadOptions.java
diff --git a/common/src/main/java/org/apache/comet/parquet/RowGroupFilter.java
b/spark/src/main/java/org/apache/comet/parquet/RowGroupFilter.java
similarity index 100%
rename from common/src/main/java/org/apache/comet/parquet/RowGroupFilter.java
rename to spark/src/main/java/org/apache/comet/parquet/RowGroupFilter.java
diff --git a/common/src/main/java/org/apache/comet/parquet/RowGroupReader.java
b/spark/src/main/java/org/apache/comet/parquet/RowGroupReader.java
similarity index 100%
rename from common/src/main/java/org/apache/comet/parquet/RowGroupReader.java
rename to spark/src/main/java/org/apache/comet/parquet/RowGroupReader.java
diff --git a/common/src/main/java/org/apache/comet/parquet/TypeUtil.java
b/spark/src/main/java/org/apache/comet/parquet/TypeUtil.java
similarity index 100%
rename from common/src/main/java/org/apache/comet/parquet/TypeUtil.java
rename to spark/src/main/java/org/apache/comet/parquet/TypeUtil.java
diff --git a/common/src/main/java/org/apache/comet/parquet/Utils.java
b/spark/src/main/java/org/apache/comet/parquet/Utils.java
similarity index 100%
rename from common/src/main/java/org/apache/comet/parquet/Utils.java
rename to spark/src/main/java/org/apache/comet/parquet/Utils.java
diff --git
a/common/src/main/java/org/apache/comet/parquet/WrappedInputFile.java
b/spark/src/main/java/org/apache/comet/parquet/WrappedInputFile.java
similarity index 100%
rename from common/src/main/java/org/apache/comet/parquet/WrappedInputFile.java
rename to spark/src/main/java/org/apache/comet/parquet/WrappedInputFile.java
diff --git
a/common/src/main/java/org/apache/comet/parquet/WrappedSeekableInputStream.java
b/spark/src/main/java/org/apache/comet/parquet/WrappedSeekableInputStream.java
similarity index 100%
rename from
common/src/main/java/org/apache/comet/parquet/WrappedSeekableInputStream.java
rename to
spark/src/main/java/org/apache/comet/parquet/WrappedSeekableInputStream.java
diff --git a/common/src/main/java/org/apache/comet/udf/CometUdfBridge.java
b/spark/src/main/java/org/apache/comet/udf/CometUdfBridge.java
similarity index 100%
rename from common/src/main/java/org/apache/comet/udf/CometUdfBridge.java
rename to spark/src/main/java/org/apache/comet/udf/CometUdfBridge.java
diff --git
a/common/src/main/java/org/apache/comet/vector/CometDecodedVector.java
b/spark/src/main/java/org/apache/comet/vector/CometDecodedVector.java
similarity index 100%
rename from common/src/main/java/org/apache/comet/vector/CometDecodedVector.java
rename to spark/src/main/java/org/apache/comet/vector/CometDecodedVector.java
diff --git
a/common/src/main/java/org/apache/comet/vector/CometDelegateVector.java
b/spark/src/main/java/org/apache/comet/vector/CometDelegateVector.java
similarity index 100%
rename from
common/src/main/java/org/apache/comet/vector/CometDelegateVector.java
rename to spark/src/main/java/org/apache/comet/vector/CometDelegateVector.java
diff --git a/common/src/main/java/org/apache/comet/vector/CometDictionary.java
b/spark/src/main/java/org/apache/comet/vector/CometDictionary.java
similarity index 100%
rename from common/src/main/java/org/apache/comet/vector/CometDictionary.java
rename to spark/src/main/java/org/apache/comet/vector/CometDictionary.java
diff --git
a/common/src/main/java/org/apache/comet/vector/CometDictionaryVector.java
b/spark/src/main/java/org/apache/comet/vector/CometDictionaryVector.java
similarity index 100%
rename from
common/src/main/java/org/apache/comet/vector/CometDictionaryVector.java
rename to spark/src/main/java/org/apache/comet/vector/CometDictionaryVector.java
diff --git a/common/src/main/java/org/apache/comet/vector/CometLazyVector.java
b/spark/src/main/java/org/apache/comet/vector/CometLazyVector.java
similarity index 100%
rename from common/src/main/java/org/apache/comet/vector/CometLazyVector.java
rename to spark/src/main/java/org/apache/comet/vector/CometLazyVector.java
diff --git a/common/src/main/java/org/apache/comet/vector/CometListVector.java
b/spark/src/main/java/org/apache/comet/vector/CometListVector.java
similarity index 100%
rename from common/src/main/java/org/apache/comet/vector/CometListVector.java
rename to spark/src/main/java/org/apache/comet/vector/CometListVector.java
diff --git a/common/src/main/java/org/apache/comet/vector/CometMapVector.java
b/spark/src/main/java/org/apache/comet/vector/CometMapVector.java
similarity index 100%
rename from common/src/main/java/org/apache/comet/vector/CometMapVector.java
rename to spark/src/main/java/org/apache/comet/vector/CometMapVector.java
diff --git a/common/src/main/java/org/apache/comet/vector/CometPlainVector.java
b/spark/src/main/java/org/apache/comet/vector/CometPlainVector.java
similarity index 100%
rename from common/src/main/java/org/apache/comet/vector/CometPlainVector.java
rename to spark/src/main/java/org/apache/comet/vector/CometPlainVector.java
diff --git
a/common/src/main/java/org/apache/comet/vector/CometSelectionVector.java
b/spark/src/main/java/org/apache/comet/vector/CometSelectionVector.java
similarity index 100%
rename from
common/src/main/java/org/apache/comet/vector/CometSelectionVector.java
rename to spark/src/main/java/org/apache/comet/vector/CometSelectionVector.java
diff --git
a/common/src/main/java/org/apache/comet/vector/CometStructVector.java
b/spark/src/main/java/org/apache/comet/vector/CometStructVector.java
similarity index 100%
rename from common/src/main/java/org/apache/comet/vector/CometStructVector.java
rename to spark/src/main/java/org/apache/comet/vector/CometStructVector.java
diff --git a/common/src/main/java/org/apache/comet/vector/CometVector.java
b/spark/src/main/java/org/apache/comet/vector/CometVector.java
similarity index 100%
rename from common/src/main/java/org/apache/comet/vector/CometVector.java
rename to spark/src/main/java/org/apache/comet/vector/CometVector.java
diff --git a/common/src/main/scala/org/apache/comet/CometConf.scala
b/spark/src/main/scala/org/apache/comet/CometConf.scala
similarity index 100%
rename from common/src/main/scala/org/apache/comet/CometConf.scala
rename to spark/src/main/scala/org/apache/comet/CometConf.scala
diff --git a/common/src/main/scala/org/apache/comet/Constants.scala
b/spark/src/main/scala/org/apache/comet/Constants.scala
similarity index 100%
rename from common/src/main/scala/org/apache/comet/Constants.scala
rename to spark/src/main/scala/org/apache/comet/Constants.scala
diff --git
a/common/src/main/scala/org/apache/comet/objectstore/NativeConfig.scala
b/spark/src/main/scala/org/apache/comet/objectstore/NativeConfig.scala
similarity index 100%
rename from
common/src/main/scala/org/apache/comet/objectstore/NativeConfig.scala
rename to spark/src/main/scala/org/apache/comet/objectstore/NativeConfig.scala
diff --git a/common/src/main/scala/org/apache/comet/package.scala
b/spark/src/main/scala/org/apache/comet/package.scala
similarity index 100%
rename from common/src/main/scala/org/apache/comet/package.scala
rename to spark/src/main/scala/org/apache/comet/package.scala
diff --git
a/common/src/main/scala/org/apache/comet/parquet/CometParquetUtils.scala
b/spark/src/main/scala/org/apache/comet/parquet/CometParquetUtils.scala
similarity index 100%
rename from
common/src/main/scala/org/apache/comet/parquet/CometParquetUtils.scala
rename to spark/src/main/scala/org/apache/comet/parquet/CometParquetUtils.scala
diff --git
a/common/src/main/scala/org/apache/comet/parquet/CometReaderThreadPool.scala
b/spark/src/main/scala/org/apache/comet/parquet/CometReaderThreadPool.scala
similarity index 100%
rename from
common/src/main/scala/org/apache/comet/parquet/CometReaderThreadPool.scala
rename to
spark/src/main/scala/org/apache/comet/parquet/CometReaderThreadPool.scala
diff --git
a/common/src/main/scala/org/apache/comet/udf/CometLambdaRegistry.scala
b/spark/src/main/scala/org/apache/comet/udf/CometLambdaRegistry.scala
similarity index 100%
rename from common/src/main/scala/org/apache/comet/udf/CometLambdaRegistry.scala
rename to spark/src/main/scala/org/apache/comet/udf/CometLambdaRegistry.scala
diff --git a/common/src/main/scala/org/apache/comet/udf/CometUDF.scala
b/spark/src/main/scala/org/apache/comet/udf/CometUDF.scala
similarity index 100%
rename from common/src/main/scala/org/apache/comet/udf/CometUDF.scala
rename to spark/src/main/scala/org/apache/comet/udf/CometUDF.scala
diff --git a/common/src/main/scala/org/apache/comet/vector/NativeUtil.scala
b/spark/src/main/scala/org/apache/comet/vector/NativeUtil.scala
similarity index 100%
rename from common/src/main/scala/org/apache/comet/vector/NativeUtil.scala
rename to spark/src/main/scala/org/apache/comet/vector/NativeUtil.scala
diff --git a/common/src/main/scala/org/apache/comet/vector/StreamReader.scala
b/spark/src/main/scala/org/apache/comet/vector/StreamReader.scala
similarity index 100%
rename from common/src/main/scala/org/apache/comet/vector/StreamReader.scala
rename to spark/src/main/scala/org/apache/comet/vector/StreamReader.scala
diff --git
a/common/src/main/scala/org/apache/spark/comet/CometTaskContextShim.scala
b/spark/src/main/scala/org/apache/spark/comet/CometTaskContextShim.scala
similarity index 100%
rename from
common/src/main/scala/org/apache/spark/comet/CometTaskContextShim.scala
rename to spark/src/main/scala/org/apache/spark/comet/CometTaskContextShim.scala
diff --git
a/common/src/main/scala/org/apache/spark/sql/comet/CastOverflowException.scala
b/spark/src/main/scala/org/apache/spark/sql/comet/CastOverflowException.scala
similarity index 100%
rename from
common/src/main/scala/org/apache/spark/sql/comet/CastOverflowException.scala
rename to
spark/src/main/scala/org/apache/spark/sql/comet/CastOverflowException.scala
diff --git
a/common/src/main/scala/org/apache/spark/sql/comet/execution/arrow/ArrowReaderIterator.scala
b/spark/src/main/scala/org/apache/spark/sql/comet/execution/arrow/ArrowReaderIterator.scala
similarity index 100%
rename from
common/src/main/scala/org/apache/spark/sql/comet/execution/arrow/ArrowReaderIterator.scala
rename to
spark/src/main/scala/org/apache/spark/sql/comet/execution/arrow/ArrowReaderIterator.scala
diff --git
a/common/src/main/scala/org/apache/spark/sql/comet/execution/arrow/ArrowWriters.scala
b/spark/src/main/scala/org/apache/spark/sql/comet/execution/arrow/ArrowWriters.scala
similarity index 100%
rename from
common/src/main/scala/org/apache/spark/sql/comet/execution/arrow/ArrowWriters.scala
rename to
spark/src/main/scala/org/apache/spark/sql/comet/execution/arrow/ArrowWriters.scala
diff --git
a/common/src/main/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowConverters.scala
b/spark/src/main/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowConverters.scala
similarity index 100%
rename from
common/src/main/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowConverters.scala
rename to
spark/src/main/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowConverters.scala
diff --git
a/common/src/main/scala/org/apache/spark/sql/comet/parquet/CometParquetReadSupport.scala
b/spark/src/main/scala/org/apache/spark/sql/comet/parquet/CometParquetReadSupport.scala
similarity index 100%
rename from
common/src/main/scala/org/apache/spark/sql/comet/parquet/CometParquetReadSupport.scala
rename to
spark/src/main/scala/org/apache/spark/sql/comet/parquet/CometParquetReadSupport.scala
diff --git
a/common/src/main/scala/org/apache/spark/sql/comet/parquet/CometSparkToParquetSchemaConverter.scala
b/spark/src/main/scala/org/apache/spark/sql/comet/parquet/CometSparkToParquetSchemaConverter.scala
similarity index 100%
rename from
common/src/main/scala/org/apache/spark/sql/comet/parquet/CometSparkToParquetSchemaConverter.scala
rename to
spark/src/main/scala/org/apache/spark/sql/comet/parquet/CometSparkToParquetSchemaConverter.scala
diff --git a/common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala
b/spark/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala
similarity index 100%
rename from common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala
rename to spark/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala
diff --git
a/common/src/main/spark-3.4/org/apache/comet/shims/ShimBatchReader.scala
b/spark/src/main/spark-3.4/org/apache/comet/shims/ShimBatchReader.scala
similarity index 100%
rename from
common/src/main/spark-3.4/org/apache/comet/shims/ShimBatchReader.scala
rename to spark/src/main/spark-3.4/org/apache/comet/shims/ShimBatchReader.scala
diff --git
a/common/src/main/spark-3.4/org/apache/comet/shims/ShimFileFormat.scala
b/spark/src/main/spark-3.4/org/apache/comet/shims/ShimFileFormat.scala
similarity index 100%
rename from
common/src/main/spark-3.4/org/apache/comet/shims/ShimFileFormat.scala
rename to spark/src/main/spark-3.4/org/apache/comet/shims/ShimFileFormat.scala
diff --git
a/common/src/main/spark-3.4/org/apache/spark/sql/comet/shims/ShimTaskMetrics.scala
b/spark/src/main/spark-3.4/org/apache/spark/sql/comet/shims/ShimTaskMetrics.scala
similarity index 100%
rename from
common/src/main/spark-3.4/org/apache/spark/sql/comet/shims/ShimTaskMetrics.scala
rename to
spark/src/main/spark-3.4/org/apache/spark/sql/comet/shims/ShimTaskMetrics.scala
diff --git
a/common/src/main/spark-3.5/org/apache/comet/shims/ShimBatchReader.scala
b/spark/src/main/spark-3.5/org/apache/comet/shims/ShimBatchReader.scala
similarity index 100%
rename from
common/src/main/spark-3.5/org/apache/comet/shims/ShimBatchReader.scala
rename to spark/src/main/spark-3.5/org/apache/comet/shims/ShimBatchReader.scala
diff --git
a/common/src/main/spark-3.5/org/apache/comet/shims/ShimFileFormat.scala
b/spark/src/main/spark-3.5/org/apache/comet/shims/ShimFileFormat.scala
similarity index 100%
rename from
common/src/main/spark-3.5/org/apache/comet/shims/ShimFileFormat.scala
rename to spark/src/main/spark-3.5/org/apache/comet/shims/ShimFileFormat.scala
diff --git
a/common/src/main/spark-3.5/org/apache/spark/sql/comet/shims/ShimTaskMetrics.scala
b/spark/src/main/spark-3.5/org/apache/spark/sql/comet/shims/ShimTaskMetrics.scala
similarity index 100%
rename from
common/src/main/spark-3.5/org/apache/spark/sql/comet/shims/ShimTaskMetrics.scala
rename to
spark/src/main/spark-3.5/org/apache/spark/sql/comet/shims/ShimTaskMetrics.scala
diff --git
a/common/src/main/spark-3.x/org/apache/comet/shims/CometTypeShim.scala
b/spark/src/main/spark-3.x/org/apache/comet/shims/CometTypeShim.scala
similarity index 100%
rename from common/src/main/spark-3.x/org/apache/comet/shims/CometTypeShim.scala
rename to spark/src/main/spark-3.x/org/apache/comet/shims/CometTypeShim.scala
diff --git
a/common/src/main/spark-3.x/org/apache/comet/shims/ShimCometConf.scala
b/spark/src/main/spark-3.x/org/apache/comet/shims/ShimCometConf.scala
similarity index 100%
rename from common/src/main/spark-3.x/org/apache/comet/shims/ShimCometConf.scala
rename to spark/src/main/spark-3.x/org/apache/comet/shims/ShimCometConf.scala
diff --git
a/common/src/main/spark-4.x/org/apache/comet/shims/CometTypeShim.scala
b/spark/src/main/spark-4.x/org/apache/comet/shims/CometTypeShim.scala
similarity index 100%
rename from common/src/main/spark-4.x/org/apache/comet/shims/CometTypeShim.scala
rename to spark/src/main/spark-4.x/org/apache/comet/shims/CometTypeShim.scala
diff --git
a/common/src/main/spark-4.x/org/apache/comet/shims/ShimBatchReader.scala
b/spark/src/main/spark-4.x/org/apache/comet/shims/ShimBatchReader.scala
similarity index 100%
rename from
common/src/main/spark-4.x/org/apache/comet/shims/ShimBatchReader.scala
rename to spark/src/main/spark-4.x/org/apache/comet/shims/ShimBatchReader.scala
diff --git
a/common/src/main/spark-4.x/org/apache/comet/shims/ShimCometConf.scala
b/spark/src/main/spark-4.x/org/apache/comet/shims/ShimCometConf.scala
similarity index 100%
rename from common/src/main/spark-4.x/org/apache/comet/shims/ShimCometConf.scala
rename to spark/src/main/spark-4.x/org/apache/comet/shims/ShimCometConf.scala
diff --git
a/common/src/main/spark-4.x/org/apache/comet/shims/ShimFileFormat.scala
b/spark/src/main/spark-4.x/org/apache/comet/shims/ShimFileFormat.scala
similarity index 100%
rename from
common/src/main/spark-4.x/org/apache/comet/shims/ShimFileFormat.scala
rename to spark/src/main/spark-4.x/org/apache/comet/shims/ShimFileFormat.scala
diff --git
a/common/src/main/spark-4.x/org/apache/spark/sql/comet/shims/ShimTaskMetrics.scala
b/spark/src/main/spark-4.x/org/apache/spark/sql/comet/shims/ShimTaskMetrics.scala
similarity index 100%
rename from
common/src/main/spark-4.x/org/apache/spark/sql/comet/shims/ShimTaskMetrics.scala
rename to
spark/src/main/spark-4.x/org/apache/spark/sql/comet/shims/ShimTaskMetrics.scala
diff --git
a/common/src/test/java/org/apache/comet/parquet/TestColumnReader.java
b/spark/src/test/java/org/apache/comet/parquet/TestColumnReader.java
similarity index 100%
rename from common/src/test/java/org/apache/comet/parquet/TestColumnReader.java
rename to spark/src/test/java/org/apache/comet/parquet/TestColumnReader.java
diff --git
a/common/src/test/java/org/apache/comet/parquet/TestCometInputFile.java
b/spark/src/test/java/org/apache/comet/parquet/TestCometInputFile.java
similarity index 100%
rename from
common/src/test/java/org/apache/comet/parquet/TestCometInputFile.java
rename to spark/src/test/java/org/apache/comet/parquet/TestCometInputFile.java
diff --git a/common/src/test/java/org/apache/comet/parquet/TestFileReader.java
b/spark/src/test/java/org/apache/comet/parquet/TestFileReader.java
similarity index 100%
rename from common/src/test/java/org/apache/comet/parquet/TestFileReader.java
rename to spark/src/test/java/org/apache/comet/parquet/TestFileReader.java
diff --git a/common/src/test/java/org/apache/comet/parquet/TestUtils.java
b/spark/src/test/java/org/apache/comet/parquet/TestUtils.java
similarity index 100%
rename from common/src/test/java/org/apache/comet/parquet/TestUtils.java
rename to spark/src/test/java/org/apache/comet/parquet/TestUtils.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
