This is an automated email from the ASF dual-hosted git repository.
weibin pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-graphar.git
The following commit(s) were added to refs/heads/main by this push:
new c0f0665e chore(spark): Add reference document about how to integrate
cloud storage and remove related dependencies (#572)
c0f0665e is described below
commit c0f0665ee1067debc3cf723ed4e069385c7d0ad3
Author: Weibin Zeng <[email protected]>
AuthorDate: Thu Aug 1 16:50:57 2024 +0800
chore(spark): Add reference document about how to integrate cloud storage
and remove related dependencies (#572)
Signed-off-by: acezen <[email protected]>
---
docs/libraries/spark/spark.md | 7 +++++++
maven-projects/spark/graphar/pom.xml | 28 ----------------------------
maven-projects/spark/pom.xml | 2 --
3 files changed, 7 insertions(+), 30 deletions(-)
diff --git a/docs/libraries/spark/spark.md b/docs/libraries/spark/spark.md
index 7043005d..eaf79b8b 100644
--- a/docs/libraries/spark/spark.md
+++ b/docs/libraries/spark/spark.md
@@ -225,6 +225,13 @@ For more information on usage, please refer to the
examples:
- [TransformExample.scala][transform-example] shows an example for graph data
conversion between different file types or different adjList types.
- [Neo4j2GraphAr.scala][neo4j2graphar] and
[GraphAr2Neo4j.scala][graphar2neo4j] are examples to conduct data
importing/exporting for Neo4j.
+### Working with Cloud Storage (AWS S3, aliyun OSS)
+
+The Spark library for GraphAr supports reading and writing data from/to cloud
storage services such as AWS S3, to do so, you need to include the Hadoop AWS
dependency in your project. See the reference documentation for more details.
+
+- [AWS S3](https://spark.apache.org/docs/latest/cloud-integration.html)
+- [Aliyun
OSS](https://hadoop.apache.org/docs/stable/hadoop-aliyun/tools/hadoop-aliyun/index.html)
+
[test-graph-info]:
https://github.com/apache/incubator-graphar/blob/main/maven-projects/spark/graphar/src/test/scala/org/apache/graphar/TestGraphInfo.scala
[test-index-generator]:
https://github.com/apache/incubator-graphar/blob/main/maven-projects/spark/graphar/src/test/scala/org/apache/graphar/TestIndexGenerator.scala
diff --git a/maven-projects/spark/graphar/pom.xml
b/maven-projects/spark/graphar/pom.xml
index 74626a62..9dc46eb4 100644
--- a/maven-projects/spark/graphar/pom.xml
+++ b/maven-projects/spark/graphar/pom.xml
@@ -88,34 +88,6 @@
<artifactId>snakeyaml</artifactId>
<version>2.0</version>
</dependency>
- <dependency>
- <groupId>com.aliyun.odps</groupId>
- <artifactId>hadoop-fs-oss</artifactId>
- <version>${cupid.sdk.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>com.aliyun.odps</groupId>
- <artifactId>odps-spark-datasource_2.11</artifactId>
- <version>${cupid.sdk.version}</version>
- <exclusions>
- <exclusion>
- <groupId>net.jpountz.lz4</groupId>
- <artifactId>lz4</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>com.aliyun.odps</groupId>
- <artifactId>cupid-sdk</artifactId>
- <version>${cupid.sdk.version}</version>
- <scope>provided</scope>
- </dependency>
<dependency>
<groupId>org.neo4j</groupId>
<artifactId>neo4j-connector-apache-spark_2.12</artifactId>
diff --git a/maven-projects/spark/pom.xml b/maven-projects/spark/pom.xml
index e04ed4ae..455fb175 100644
--- a/maven-projects/spark/pom.xml
+++ b/maven-projects/spark/pom.xml
@@ -49,7 +49,6 @@
<spark.version>3.2.2</spark.version>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
- <cupid.sdk.version>3.3.8-public</cupid.sdk.version>
</properties>
<modules>
<module>graphar</module>
@@ -72,7 +71,6 @@
<spark.version>3.3.4</spark.version>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
- <cupid.sdk.version>3.3.8-public</cupid.sdk.version>
</properties>
<modules>
<module>graphar</module>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
