This is an automated email from the ASF dual-hosted git repository.
ming pushed a commit to branch master
in repository
https://gitbox.apache.org/repos/asf/incubator-hugegraph-toolchain.git
The following commit(s) were added to refs/heads/master by this push:
new 0ea28fdf fix: Spark with loader has dependency conflicts (#480)
0ea28fdf is described below
commit 0ea28fdf27f7a4e57668a48e1c7c0c884fa15314
Author: alan.zhao <[email protected]>
AuthorDate: Mon Aug 7 19:38:49 2023 +0800
fix: Spark with loader has dependency conflicts (#480)
Co-authored-by: alanzhao <[email protected]>
Co-authored-by: Simon Cheung <[email protected]>
---
hugegraph-loader/assembly/descriptor/assembly.xml | 3 ++
.../assembly/static/bin/hugegraph-spark-loader.sh | 2 +-
hugegraph-loader/pom.xml | 60 ++++++++++++++++++++++
.../loader/spark/HugeGraphSparkLoader.java | 11 ++--
.../shaded.jakarta.ws.rs.client.ClientBuilder | 16 ++++++
...glassfish.hk2.extension.ServiceLocatorGenerator | 16 ++++++
6 files changed, 104 insertions(+), 4 deletions(-)
diff --git a/hugegraph-loader/assembly/descriptor/assembly.xml
b/hugegraph-loader/assembly/descriptor/assembly.xml
index 62fe6793..efac0b5c 100644
--- a/hugegraph-loader/assembly/descriptor/assembly.xml
+++ b/hugegraph-loader/assembly/descriptor/assembly.xml
@@ -48,6 +48,9 @@
<fileSet>
<directory>${project.build.directory}</directory>
<outputDirectory>lib</outputDirectory>
+ <excludes>
+ <exclude>hugegraph-loader-1.0.0.jar</exclude>
+ </excludes>
<includes>
<include>*.jar</include>
</includes>
diff --git a/hugegraph-loader/assembly/static/bin/hugegraph-spark-loader.sh
b/hugegraph-loader/assembly/static/bin/hugegraph-spark-loader.sh
index c6bb08b8..ee93c63f 100755
--- a/hugegraph-loader/assembly/static/bin/hugegraph-spark-loader.sh
+++ b/hugegraph-loader/assembly/static/bin/hugegraph-spark-loader.sh
@@ -25,7 +25,7 @@ get_params "$@"
echo "engine_params: $ENGINE_PARAMS"
echo "hugegraph_params: $HUGEGRAPH_PARAMS"
-ASSEMBLY_JAR_NAME=$(find "${LIB_DIR}" -name 'hugegraph-loader*.jar')
+ASSEMBLY_JAR_NAME=$(find "${LIB_DIR}" -name 'apache-hugegraph-loader*.jar')
DEFAULT_APP_NAME="hugegraph-spark-loader"
APP_NAME=${APP_NAME:-$DEFAULT_APP_NAME}
diff --git a/hugegraph-loader/pom.xml b/hugegraph-loader/pom.xml
index 6ff5e69c..c62b8b28 100644
--- a/hugegraph-loader/pom.xml
+++ b/hugegraph-loader/pom.xml
@@ -90,8 +90,17 @@
<artifactId>jersey-server</artifactId>
<groupId>org.glassfish.jersey.core</groupId>
</exclusion>
+ <exclusion>
+ <artifactId>gson</artifactId>
+ <groupId>com.google.code.gson</groupId>
+ </exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <artifactId>gson</artifactId>
+ <groupId>com.google.code.gson</groupId>
+ <version>${gson.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.version}</artifactId>
@@ -614,6 +623,57 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>3.4.1</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+
<createDependencyReducedPom>false</createDependencyReducedPom>
+ <finalName>${final.name}-shaded</finalName>
+ <relocations>
+ <relocation>
+ <pattern>org.glassfish.jersey</pattern>
+
<shadedPattern>shaded.org.glassfish.jersey</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>jakarta.ws.rs</pattern>
+
<shadedPattern>shaded.jakarta.ws.rs</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.google.common.base</pattern>
+
<shadedPattern>shaded.com.google.common.base</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.glassfish.hk2</pattern>
+
<shadedPattern>shaded.org.glassfish.hk2</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.jvnet.hk2</pattern>
+
<shadedPattern>shaded.org.jvnet.hk2</shadedPattern>
+ </relocation>
+ </relocations>
+ <transformers>
+ <transformer
implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer">
+ </transformer>
+ </transformers>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>module-info.class</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
diff --git
a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/spark/HugeGraphSparkLoader.java
b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/spark/HugeGraphSparkLoader.java
index 228f2db0..5ee06072 100644
---
a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/spark/HugeGraphSparkLoader.java
+++
b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/spark/HugeGraphSparkLoader.java
@@ -60,6 +60,7 @@ import org.apache.spark.util.LongAccumulator;
import org.slf4j.Logger;
import java.io.Serializable;
+import java.util.Optional;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
@@ -292,9 +293,13 @@ public class HugeGraphSparkLoader implements Serializable {
case FILE:
case HDFS:
FileSource fileSource = struct.input().asFileSource();
- String delimiter = fileSource.delimiter();
- elements = builder.build(fileSource.header(),
-
row.mkString(delimiter).split(delimiter));
+ String delimiter = fileSource.delimiter() ;
+ if (Optional.ofNullable(delimiter).isPresent()) {
+ elements = builder.build(fileSource.header(),
+ row.mkString(delimiter).split(delimiter));
+ }else {
+ elements = builder.build(row);
+ }
break;
case JDBC:
Object[] structFields =
JavaConverters.asJavaCollection(row.schema().toList())
diff --git
a/hugegraph-loader/src/main/resources/META-INF/services/shaded.jakarta.ws.rs.client.ClientBuilder
b/hugegraph-loader/src/main/resources/META-INF/services/shaded.jakarta.ws.rs.client.ClientBuilder
new file mode 100644
index 00000000..7205ceb4
--- /dev/null
+++
b/hugegraph-loader/src/main/resources/META-INF/services/shaded.jakarta.ws.rs.client.ClientBuilder
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+shaded.org.glassfish.jersey.client.JerseyClientBuilder
diff --git
a/hugegraph-loader/src/main/resources/META-INF/services/shaded.org.glassfish.hk2.extension.ServiceLocatorGenerator
b/hugegraph-loader/src/main/resources/META-INF/services/shaded.org.glassfish.hk2.extension.ServiceLocatorGenerator
new file mode 100644
index 00000000..bad820aa
--- /dev/null
+++
b/hugegraph-loader/src/main/resources/META-INF/services/shaded.org.glassfish.hk2.extension.ServiceLocatorGenerator
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+org.jvnet.hk2.external.generator.ServiceLocatorGeneratorImpl
