This is an automated email from the ASF dual-hosted git repository.
weibin pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-graphar.git
The following commit(s) were added to refs/heads/main by this push:
new e265e0e3 feat(java-info): Refactor the java-info module base on the
format protocol files (#595)
e265e0e3 is described below
commit e265e0e38e69dc57c94be04d8482d6f7fa1630e2
Author: John <sp...@apache.org>
AuthorDate: Thu Sep 19 10:04:50 2024 +0800
feat(java-info): Refactor the java-info module base on the format protocol
files (#595)
---
maven-projects/info/pom.xml | 6 +
.../java/org/apache/graphar/info/AdjacentList.java | 40 +--
.../java/org/apache/graphar/info/EdgeInfo.java | 278 +++++++--------------
.../java/org/apache/graphar/info/GraphInfo.java | 221 ++++++----------
.../java/org/apache/graphar/info/Property.java | 42 ++--
.../org/apache/graphar/info/PropertyGroup.java | 117 ++++++---
.../java/org/apache/graphar/info/VertexInfo.java | 150 ++++-------
.../FileType.java => loader/GraphLoader.java} | 26 +-
.../graphar/info/loader/LocalYamlGraphLoader.java | 81 ++++++
.../{type/FileType.java => saver/GraphSaver.java} | 26 +-
.../graphar/info/saver/LocalYamlGraphSaver.java | 69 +++++
.../org/apache/graphar/info/type/AdjListType.java | 50 ----
.../org/apache/graphar/info/type/DataType.java | 69 -----
.../apache/graphar/info/yaml/AdjacentListYaml.java | 23 +-
.../org/apache/graphar/info/yaml/EdgeYaml.java | 23 +-
.../apache/graphar/info/yaml/EnumTransferUtil.java | 130 ++++++++++
.../org/apache/graphar/info/yaml/GraphYaml.java | 3 +-
.../graphar/info/yaml/PropertyGroupYaml.java | 9 +-
.../org/apache/graphar/info/yaml/PropertyYaml.java | 10 +-
.../org/apache/graphar/info/yaml/VertexYaml.java | 13 +-
.../org/apache/graphar/info/GraphLoaderTest.java | 61 +++++
.../org/apache/graphar/info/GraphSaverTest.java | 54 ++++
.../java/org/apache/graphar/info/TestUtil.java | 143 +++++++++++
23 files changed, 951 insertions(+), 693 deletions(-)
diff --git a/maven-projects/info/pom.xml b/maven-projects/info/pom.xml
index e9b0a607..bcb89d55 100644
--- a/maven-projects/info/pom.xml
+++ b/maven-projects/info/pom.xml
@@ -42,9 +42,15 @@
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <proto-info.version>0.13.0.dev-SNAPSHOT</proto-info.version>
</properties>
<dependencies>
+ <dependency>
+ <groupId>org.apache.graphar</groupId>
+ <artifactId>proto</artifactId>
+ <version>${proto-info.version}</version>
+ </dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
diff --git
a/maven-projects/info/src/main/java/org/apache/graphar/info/AdjacentList.java
b/maven-projects/info/src/main/java/org/apache/graphar/info/AdjacentList.java
index 091cc80c..161eac27 100644
---
a/maven-projects/info/src/main/java/org/apache/graphar/info/AdjacentList.java
+++
b/maven-projects/info/src/main/java/org/apache/graphar/info/AdjacentList.java
@@ -19,38 +19,42 @@
package org.apache.graphar.info;
-import org.apache.graphar.info.type.AdjListType;
-import org.apache.graphar.info.type.FileType;
-import org.apache.graphar.info.yaml.AdjacentListYaml;
+import org.apache.graphar.proto.AdjListType;
+import org.apache.graphar.proto.FileType;
public class AdjacentList {
- private final AdjListType type;
- private final FileType fileType;
- private final String prefix;
+ private final org.apache.graphar.proto.AdjacentList protoAdjacentList;
public AdjacentList(AdjListType type, FileType fileType, String prefix) {
- this.type = type;
- this.fileType = fileType;
- this.prefix = prefix;
+ protoAdjacentList =
+ org.apache.graphar.proto.AdjacentList.newBuilder()
+ .setType(type)
+ .setFileType(fileType)
+ .setPrefix(prefix)
+ .build();
}
- AdjacentList(AdjacentListYaml yamlParser) {
- this.type =
- AdjListType.fromOrderedAndAlignedBy(
- yamlParser.isOrdered(), yamlParser.isAligned_by());
- this.fileType = FileType.valueOf(yamlParser.getFile_type());
- this.prefix = yamlParser.getPrefix();
+ private AdjacentList(org.apache.graphar.proto.AdjacentList
protoAdjacentList) {
+ this.protoAdjacentList = protoAdjacentList;
+ }
+
+ public static AdjacentList ofProto(org.apache.graphar.proto.AdjacentList
protoAdjacentList) {
+ return new AdjacentList(protoAdjacentList);
}
public AdjListType getType() {
- return type;
+ return protoAdjacentList.getType();
}
public FileType getFileType() {
- return fileType;
+ return protoAdjacentList.getFileType();
}
public String getPrefix() {
- return prefix;
+ return protoAdjacentList.getPrefix();
+ }
+
+ org.apache.graphar.proto.AdjacentList getProto() {
+ return protoAdjacentList;
}
}
diff --git
a/maven-projects/info/src/main/java/org/apache/graphar/info/EdgeInfo.java
b/maven-projects/info/src/main/java/org/apache/graphar/info/EdgeInfo.java
index ff974c63..556f2672 100644
--- a/maven-projects/info/src/main/java/org/apache/graphar/info/EdgeInfo.java
+++ b/maven-projects/info/src/main/java/org/apache/graphar/info/EdgeInfo.java
@@ -19,37 +19,23 @@
package org.apache.graphar.info;
-import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import org.apache.graphar.info.type.AdjListType;
-import org.apache.graphar.info.type.DataType;
import org.apache.graphar.info.yaml.EdgeYaml;
import org.apache.graphar.info.yaml.GraphYaml;
+import org.apache.graphar.proto.AdjListType;
+import org.apache.graphar.proto.DataType;
import org.apache.graphar.util.GeneralParams;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.yaml.snakeyaml.LoaderOptions;
import org.yaml.snakeyaml.Yaml;
-import org.yaml.snakeyaml.constructor.Constructor;
public class EdgeInfo {
- private final EdgeTriplet edgeTriplet;
- private final long chunkSize;
- private final long srcChunkSize;
- private final long dstChunkSize;
- private final boolean directed;
- private final String prefix;
- private final Map<AdjListType, AdjacentList> adjacentLists;
- private final PropertyGroups propertyGroups;
- private final String version;
+ private final org.apache.graphar.proto.EdgeInfo protoEdgeInfo;
+ private final Map<AdjListType, AdjacentList> cachedAdjacentLists;
+ private final PropertyGroups cachedPropertyGroups;
public EdgeInfo(
String srcLabel,
@@ -61,135 +47,100 @@ public class EdgeInfo {
boolean directed,
String prefix,
List<AdjacentList> adjacentListsAsList,
- List<PropertyGroup> propertyGroupsAsList,
- String version) {
- this.edgeTriplet = new EdgeTriplet(srcLabel, edgeLabel, dstLabel);
- this.chunkSize = chunkSize;
- this.srcChunkSize = srcChunkSize;
- this.dstChunkSize = dstChunkSize;
- this.directed = directed;
- this.prefix = prefix;
- this.adjacentLists =
+ List<PropertyGroup> propertyGroupsAsList) {
+ this.cachedAdjacentLists =
adjacentListsAsList.stream()
.collect(
Collectors.toUnmodifiableMap(
AdjacentList::getType,
Function.identity()));
- this.propertyGroups = new PropertyGroups(propertyGroupsAsList);
- this.version = version;
- }
-
- private EdgeInfo(EdgeYaml yamlParser) {
- this(
- yamlParser.getSrc_type(),
- yamlParser.getEdge_type(),
- yamlParser.getDst_type(),
- yamlParser.getChunk_size(),
- yamlParser.getSrc_chunk_size(),
- yamlParser.getDst_chunk_size(),
- yamlParser.isDirected(),
- yamlParser.getPrefix(),
- yamlParser.getAdj_lists().stream()
- .map(AdjacentList::new)
- .collect(Collectors.toUnmodifiableList()),
- yamlParser.getProperty_groups().stream()
- .map(PropertyGroup::new)
- .collect(Collectors.toUnmodifiableList()),
- yamlParser.getVersion());
+ this.cachedPropertyGroups = new PropertyGroups(propertyGroupsAsList);
+ this.protoEdgeInfo =
+ org.apache.graphar.proto.EdgeInfo.newBuilder()
+ .setSourceVertexType(srcLabel)
+ .setType(edgeLabel)
+ .setDestinationVertexChunkSize(dstChunkSize)
+ .setChunkSize(chunkSize)
+ .setSourceVertexChunkSize(srcChunkSize)
+ .setDestinationVertexType(dstLabel)
+ .setIsDirected(directed)
+ .setPrefix(prefix)
+ .addAllAdjacentList(
+ adjacentListsAsList.stream()
+ .map(AdjacentList::getProto)
+
.collect(Collectors.toUnmodifiableList()))
+ .addAllProperties(
+ propertyGroupsAsList.stream()
+ .map(PropertyGroup::getProto)
+
.collect(Collectors.toUnmodifiableList()))
+ .build();
+ }
+
+ public EdgeInfo(org.apache.graphar.proto.EdgeInfo protoEdgeInfo) {
+ this.protoEdgeInfo = protoEdgeInfo;
+ this.cachedAdjacentLists =
+ protoEdgeInfo.getAdjacentListList().stream()
+ .map(AdjacentList::ofProto)
+ .collect(
+ Collectors.toUnmodifiableMap(
+ AdjacentList::getType,
Function.identity()));
+ this.cachedPropertyGroups =
PropertyGroups.ofProto(protoEdgeInfo.getPropertiesList());
}
private EdgeInfo(
- EdgeTriplet edgeTriplet,
- long chunkSize,
- long srcChunkSize,
- long dstChunkSize,
- boolean directed,
- String prefix,
- Map<AdjListType, AdjacentList> adjacentLists,
- PropertyGroups propertyGroups,
- String version) {
- this.edgeTriplet = edgeTriplet;
- this.chunkSize = chunkSize;
- this.srcChunkSize = srcChunkSize;
- this.dstChunkSize = dstChunkSize;
- this.directed = directed;
- this.prefix = prefix;
- this.adjacentLists = adjacentLists;
- this.propertyGroups = propertyGroups;
- this.version = version;
- }
-
- public static EdgeInfo load(String edgeInfoPath) throws IOException {
- return load(edgeInfoPath, new Configuration());
- }
-
- public static EdgeInfo load(String edgeInfoPath, Configuration conf)
throws IOException {
- if (conf == null) {
- throw new IllegalArgumentException("Configuration is null");
- }
- return load(edgeInfoPath, FileSystem.get(conf));
+ org.apache.graphar.proto.EdgeInfo protoEdgeInfo,
+ Map<AdjListType, AdjacentList> cachedAdjacentLists,
+ PropertyGroups cachedPropertyGroups) {
+ this.protoEdgeInfo = protoEdgeInfo;
+ this.cachedAdjacentLists = cachedAdjacentLists;
+ this.cachedPropertyGroups = cachedPropertyGroups;
}
- public static EdgeInfo load(String edgeInfoPath, FileSystem fileSystem)
throws IOException {
- if (fileSystem == null) {
- throw new IllegalArgumentException("FileSystem is null");
- }
- FSDataInputStream inputStream = fileSystem.open(new
Path(edgeInfoPath));
- Yaml edgeInfoYamlLoader = new Yaml(new Constructor(EdgeYaml.class, new
LoaderOptions()));
- EdgeYaml edgeInfoYaml = edgeInfoYamlLoader.load(inputStream);
- return new EdgeInfo(edgeInfoYaml);
+ org.apache.graphar.proto.EdgeInfo getProto() {
+ return protoEdgeInfo;
+ }
+
+ public static String concat(String srcLabel, String edgeLabel, String
dstLabel) {
+ return srcLabel
+ + GeneralParams.regularSeparator
+ + edgeLabel
+ + GeneralParams.regularSeparator
+ + dstLabel;
}
public Optional<EdgeInfo> addAdjacentListAsNew(AdjacentList adjacentList) {
- if (adjacentList == null ||
adjacentLists.containsKey(adjacentList.getType())) {
+ if (adjacentList == null ||
cachedAdjacentLists.containsKey(adjacentList.getType())) {
return Optional.empty();
}
Map<AdjListType, AdjacentList> newAdjacentLists =
Stream.concat(
- adjacentLists.entrySet().stream(),
+ cachedAdjacentLists.entrySet().stream(),
Map.of(adjacentList.getType(),
adjacentList).entrySet().stream())
.collect(
Collectors.toUnmodifiableMap(
Map.Entry::getKey,
Map.Entry::getValue));
- return Optional.of(
- new EdgeInfo(
- edgeTriplet,
- chunkSize,
- srcChunkSize,
- dstChunkSize,
- directed,
- prefix,
- newAdjacentLists,
- propertyGroups,
- version));
+ return Optional.of(new EdgeInfo(protoEdgeInfo, newAdjacentLists,
cachedPropertyGroups));
}
public Optional<EdgeInfo> addPropertyGroupAsNew(PropertyGroup
propertyGroup) {
- return propertyGroups
+ // do not need check property group exist, because PropertyGroups will
check it
+ return cachedPropertyGroups
.addPropertyGroupAsNew(propertyGroup)
.map(
newPropertyGroups ->
new EdgeInfo(
- edgeTriplet,
- chunkSize,
- srcChunkSize,
- dstChunkSize,
- directed,
- prefix,
- adjacentLists,
- newPropertyGroups,
- version));
+ protoEdgeInfo, cachedAdjacentLists,
newPropertyGroups));
}
public boolean hasAdjListType(AdjListType adjListType) {
- return adjacentLists.containsKey(adjListType);
+ return cachedAdjacentLists.containsKey(adjListType);
}
public boolean hasProperty(String propertyName) {
- return propertyGroups.hasProperty(propertyName);
+ return cachedPropertyGroups.hasProperty(propertyName);
}
public boolean hasPropertyGroup(PropertyGroup propertyGroup) {
- return propertyGroups.hasPropertyGroup(propertyGroup);
+ return cachedPropertyGroups.hasPropertyGroup(propertyGroup);
}
public AdjacentList getAdjacentList(AdjListType adjListType) {
@@ -197,15 +148,15 @@ public class EdgeInfo {
// other methods which get adjacent list in this class should call
this method first,
// so we don't check AdjListType in other methods.
checkAdjListTypeExist(adjListType);
- return adjacentLists.get(adjListType);
+ return cachedAdjacentLists.get(adjListType);
}
public int getPropertyGroupNum() {
- return propertyGroups.getPropertyGroupNum();
+ return cachedPropertyGroups.getPropertyGroupNum();
}
public PropertyGroup getPropertyGroup(String property) {
- return propertyGroups.getPropertyGroup(property);
+ return cachedPropertyGroups.getPropertyGroup(property);
}
public String getPropertyGroupPrefix(PropertyGroup propertyGroup) {
@@ -243,35 +194,15 @@ public class EdgeInfo {
}
public DataType getPropertyType(String propertyName) {
- return propertyGroups.getPropertyType(propertyName);
+ return cachedPropertyGroups.getPropertyType(propertyName);
}
public boolean isPrimaryKey(String propertyName) {
- return propertyGroups.isPrimaryKey(propertyName);
+ return cachedPropertyGroups.isPrimaryKey(propertyName);
}
public boolean isNullableKey(String propertyName) {
- return propertyGroups.isNullableKey(propertyName);
- }
-
- public void save(String filePath) throws IOException {
- save(filePath, new Configuration());
- }
-
- public void save(String filePath, Configuration conf) throws IOException {
- if (conf == null) {
- throw new IllegalArgumentException("Configuration is null");
- }
- save(filePath, FileSystem.get(conf));
- }
-
- public void save(String fileName, FileSystem fileSystem) throws
IOException {
- if (fileSystem == null) {
- throw new IllegalArgumentException("FileSystem is null");
- }
- FSDataOutputStream outputStream = fileSystem.create(new
Path(fileName));
- outputStream.writeBytes(dump());
- outputStream.close();
+ return cachedPropertyGroups.isNullableKey(propertyName);
}
public String dump() {
@@ -281,63 +212,63 @@ public class EdgeInfo {
}
public String getConcat() {
- return edgeTriplet.getConcat();
+ return concat(getSrcLabel(), getEdgeLabel(), getDstLabel());
}
public String getSrcLabel() {
- return edgeTriplet.getSrcLabel();
+ return protoEdgeInfo.getSourceVertexType();
}
public String getEdgeLabel() {
- return edgeTriplet.getEdgeLabel();
+ return protoEdgeInfo.getType();
}
public String getDstLabel() {
- return edgeTriplet.getDstLabel();
+ return protoEdgeInfo.getDestinationVertexType();
}
public long getChunkSize() {
- return chunkSize;
+ return protoEdgeInfo.getChunkSize();
}
public long getSrcChunkSize() {
- return srcChunkSize;
+ return protoEdgeInfo.getSourceVertexChunkSize();
}
public long getDstChunkSize() {
- return dstChunkSize;
+ return protoEdgeInfo.getDestinationVertexChunkSize();
}
public boolean isDirected() {
- return directed;
+ return protoEdgeInfo.getIsDirected();
}
public String getPrefix() {
- return prefix;
+ return protoEdgeInfo.getPrefix();
}
- public Map<AdjListType, AdjacentList> getAdjacentLists() {
- return adjacentLists;
+ public String getEdgePath() {
+ return getPrefix() + "/" + getConcat() + ".edge.yaml";
}
- public List<PropertyGroup> getPropertyGroups() {
- return propertyGroups.getPropertyGroupList();
+ public Map<AdjListType, AdjacentList> getAdjacentLists() {
+ return cachedAdjacentLists;
}
- public String getVersion() {
- return version;
+ public List<PropertyGroup> getPropertyGroups() {
+ return cachedPropertyGroups.getPropertyGroupList();
}
private void checkAdjListTypeExist(AdjListType adjListType) {
if (adjListType == null) {
throw new IllegalArgumentException("The adjacency list type is
null");
}
- if (!adjacentLists.containsKey(adjListType)) {
+ if (!cachedAdjacentLists.containsKey(adjListType)) {
throw new IllegalArgumentException(
"The adjacency list type "
+ adjListType
+ " does not exist in the edge info "
- + this.edgeTriplet.getConcat());
+ + getConcat());
}
}
@@ -353,41 +284,4 @@ public class EdgeInfo {
+ getConcat());
}
}
-
- private static class EdgeTriplet {
- private final String srcLabel;
- private final String edgeLabel;
- private final String dstLabel;
-
- public EdgeTriplet(String srcLabel, String edgeLabel, String dstLabel)
{
- this.srcLabel = srcLabel;
- this.edgeLabel = edgeLabel;
- this.dstLabel = dstLabel;
- }
-
- public String getConcat() {
- return srcLabel
- + GeneralParams.regularSeparator
- + edgeLabel
- + GeneralParams.regularSeparator
- + dstLabel;
- }
-
- @Override
- public String toString() {
- return getConcat();
- }
-
- public String getSrcLabel() {
- return srcLabel;
- }
-
- public String getEdgeLabel() {
- return edgeLabel;
- }
-
- public String getDstLabel() {
- return dstLabel;
- }
- }
}
diff --git
a/maven-projects/info/src/main/java/org/apache/graphar/info/GraphInfo.java
b/maven-projects/info/src/main/java/org/apache/graphar/info/GraphInfo.java
index c8d00d61..b2177080 100644
--- a/maven-projects/info/src/main/java/org/apache/graphar/info/GraphInfo.java
+++ b/maven-projects/info/src/main/java/org/apache/graphar/info/GraphInfo.java
@@ -19,8 +19,6 @@
package org.apache.graphar.info;
-import java.io.IOException;
-import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -29,114 +27,56 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.graphar.info.yaml.GraphYaml;
import org.apache.graphar.util.GeneralParams;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.yaml.snakeyaml.LoaderOptions;
import org.yaml.snakeyaml.Yaml;
-import org.yaml.snakeyaml.constructor.Constructor;
public class GraphInfo {
- private final String name;
- private final List<VertexInfo> vertexInfos;
- private final List<EdgeInfo> edgeInfos;
- private final String prefix;
- private final Map<String, VertexInfo> vertexLabel2VertexInfo;
- private final Map<String, EdgeInfo> edgeConcat2EdgeInfo;
- private final String version;
+ private final org.apache.graphar.proto.GraphInfo protoGraphInfo;
+ private final List<VertexInfo> cachedVertexInfoList;
+ private final List<EdgeInfo> cachedEdgeInfoList;
+ private final Map<String, VertexInfo> cachedVertexInfoMap;
+ private final Map<String, EdgeInfo> cachedEdgeInfoMap;
public GraphInfo(
- String name,
- List<VertexInfo> vertexInfos,
- List<EdgeInfo> edgeInfos,
- String prefix,
- String version) {
- this.name = name;
- this.vertexInfos = List.copyOf(vertexInfos);
- this.edgeInfos = List.copyOf(edgeInfos);
- this.prefix = prefix;
- this.version = version;
- this.vertexLabel2VertexInfo =
+ String name, List<VertexInfo> vertexInfos, List<EdgeInfo>
edgeInfos, String prefix) {
+ this.cachedVertexInfoList = List.copyOf(vertexInfos);
+ this.cachedEdgeInfoList = List.copyOf(edgeInfos);
+ this.cachedVertexInfoMap =
vertexInfos.stream()
.collect(
Collectors.toUnmodifiableMap(
- VertexInfo::getLabel,
Function.identity()));
- this.edgeConcat2EdgeInfo =
+ VertexInfo::getType,
Function.identity()));
+ this.cachedEdgeInfoMap =
edgeInfos.stream()
.collect(
Collectors.toUnmodifiableMap(
EdgeInfo::getConcat,
Function.identity()));
- }
-
- private GraphInfo(GraphYaml graphYaml, Configuration conf) throws
IOException {
- this(
- graphYaml.getName(),
- vertexFileNames2VertexInfos(graphYaml.getVertices(), conf),
- edgeFileNames2EdgeInfos(graphYaml.getEdges(), conf),
- graphYaml.getPrefix(),
- graphYaml.getVersion());
+ this.protoGraphInfo =
+ org.apache.graphar.proto.GraphInfo.newBuilder()
+ .setName(name)
+ .addAllVertices(
+ vertexInfos.stream()
+ .map(VertexInfo::getVertexPath)
+ .collect(Collectors.toList()))
+ .addAllEdges(
+ edgeInfos.stream()
+ .map(EdgeInfo::getEdgePath)
+ .collect(Collectors.toList()))
+ .setPrefix(prefix)
+ .build();
}
private GraphInfo(
- String name,
- List<VertexInfo> vertexInfos,
- List<EdgeInfo> edgeInfos,
- String prefix,
- Map<String, VertexInfo> vertexLabel2VertexInfo,
- Map<String, EdgeInfo> edgeConcat2EdgeInfo,
- String version) {
- this.name = name;
- this.vertexInfos = vertexInfos;
- this.edgeInfos = edgeInfos;
- this.prefix = prefix;
- this.vertexLabel2VertexInfo = vertexLabel2VertexInfo;
- this.edgeConcat2EdgeInfo = edgeConcat2EdgeInfo;
- this.version = version;
- }
-
- public static GraphInfo load(String graphPath) throws IOException {
- return load(graphPath, new Configuration());
- }
-
- public static GraphInfo load(String graphPath, FileSystem fileSystem)
throws IOException {
- if (fileSystem == null) {
- throw new IllegalArgumentException("FileSystem is null");
- }
- return load(graphPath, fileSystem.getConf());
- }
-
- public static GraphInfo load(String graphPath, Configuration conf) throws
IOException {
- if (conf == null) {
- throw new IllegalArgumentException("Configuration is null");
- }
- Path path = new Path(graphPath);
- FileSystem fileSystem = path.getFileSystem(conf);
- FSDataInputStream inputStream = fileSystem.open(path);
- Yaml graphYamlLoader = new Yaml(new Constructor(GraphYaml.class, new
LoaderOptions()));
- GraphYaml graphYaml = graphYamlLoader.load(inputStream);
- return new GraphInfo(graphYaml, conf);
- }
-
- public void save(String filePath) throws IOException {
- save(filePath, new Configuration());
- }
+ org.apache.graphar.proto.GraphInfo protoGraphInfo,
+ List<VertexInfo> cachedVertexInfoList,
+ List<EdgeInfo> cachedEdgeInfoList,
+ Map<String, VertexInfo> cachedVertexInfoMap,
+ Map<String, EdgeInfo> cachedEdgeInfoMap) {
- public void save(String filePath, Configuration conf) throws IOException {
- if (conf == null) {
- throw new IllegalArgumentException("Configuration is null");
- }
- save(filePath, FileSystem.get(conf));
- }
-
- public void save(String fileName, FileSystem fileSystem) throws
IOException {
- if (fileSystem == null) {
- throw new IllegalArgumentException("FileSystem is null");
- }
- FSDataOutputStream outputStream = fileSystem.create(new
Path(fileName));
- outputStream.writeBytes(dump());
- outputStream.close();
+ this.protoGraphInfo = protoGraphInfo;
+ this.cachedVertexInfoList = cachedVertexInfoList;
+ this.cachedEdgeInfoList = cachedEdgeInfoList;
+ this.cachedVertexInfoMap = cachedVertexInfoMap;
+ this.cachedEdgeInfoMap = cachedEdgeInfoMap;
}
public String dump() {
@@ -146,28 +86,30 @@ public class GraphInfo {
}
public Optional<GraphInfo> addVertexAsNew(VertexInfo vertexInfo) {
- if (vertexInfo == null || hasVertexInfo(vertexInfo.getLabel())) {
+ if (vertexInfo == null || hasVertexInfo(vertexInfo.getType())) {
return Optional.empty();
}
- List<VertexInfo> newVertexInfos =
- Stream.concat(vertexInfos.stream(), Stream.of(vertexInfo))
+ final org.apache.graphar.proto.GraphInfo newProtoGraphInfo =
+ org.apache.graphar.proto.GraphInfo.newBuilder(protoGraphInfo)
+ .addVertices(vertexInfo.getVertexPath())
+ .build();
+ final List<VertexInfo> newVertexInfoList =
+ Stream.concat(cachedVertexInfoList.stream(),
Stream.of(vertexInfo))
.collect(Collectors.toList());
- Map<String, VertexInfo> newVertexLabel2VertexInfo =
+ final Map<String, VertexInfo> newVertexInfoMap =
Stream.concat(
- vertexLabel2VertexInfo.entrySet().stream(),
- Stream.of(Map.entry(vertexInfo.getLabel(),
vertexInfo)))
+ cachedVertexInfoMap.entrySet().stream(),
+ Stream.of(Map.entry(vertexInfo.getType(),
vertexInfo)))
.collect(
Collectors.toUnmodifiableMap(
Map.Entry::getKey,
Map.Entry::getValue));
return Optional.of(
new GraphInfo(
- name,
- newVertexInfos,
- edgeInfos,
- prefix,
- newVertexLabel2VertexInfo,
- edgeConcat2EdgeInfo,
- version));
+ newProtoGraphInfo,
+ newVertexInfoList,
+ cachedEdgeInfoList,
+ newVertexInfoMap,
+ cachedEdgeInfoMap));
}
public Optional<GraphInfo> addEdgeAsNew(EdgeInfo edgeInfo) {
@@ -176,88 +118,69 @@ public class GraphInfo {
edgeInfo.getSrcLabel(), edgeInfo.getEdgeLabel(),
edgeInfo.getDstLabel())) {
return Optional.empty();
}
- List<EdgeInfo> newEdgeInfos =
- Stream.concat(edgeInfos.stream(),
Stream.of(edgeInfo)).collect(Collectors.toList());
- Map<String, EdgeInfo> newEdgeConcat2EdgeInfo =
+ final org.apache.graphar.proto.GraphInfo newProtoGraphInfo =
+ org.apache.graphar.proto.GraphInfo.newBuilder(protoGraphInfo)
+ .addEdges(edgeInfo.getEdgePath())
+ .build();
+ final List<EdgeInfo> newEdgeInfos =
+ Stream.concat(cachedEdgeInfoList.stream(), Stream.of(edgeInfo))
+ .collect(Collectors.toList());
+ final Map<String, EdgeInfo> newEdgeConcat2EdgeInfo =
Stream.concat(
- edgeConcat2EdgeInfo.entrySet().stream(),
+ cachedEdgeInfoMap.entrySet().stream(),
Stream.of(Map.entry(edgeInfo.getConcat(),
edgeInfo)))
.collect(
Collectors.toUnmodifiableMap(
Map.Entry::getKey,
Map.Entry::getValue));
return Optional.of(
new GraphInfo(
- name,
- vertexInfos,
+ newProtoGraphInfo,
+ cachedVertexInfoList,
newEdgeInfos,
- prefix,
- vertexLabel2VertexInfo,
- newEdgeConcat2EdgeInfo,
- version));
+ cachedVertexInfoMap,
+ newEdgeConcat2EdgeInfo));
}
public boolean hasVertexInfo(String label) {
- return vertexLabel2VertexInfo.containsKey(label);
+ return cachedVertexInfoMap.containsKey(label);
}
public boolean hasEdgeInfo(String srcLabel, String edgeLabel, String
dstLabel) {
- return edgeConcat2EdgeInfo.containsKey(srcLabel + dstLabel +
edgeLabel);
+ return cachedEdgeInfoMap.containsKey(EdgeInfo.concat(srcLabel,
edgeLabel, dstLabel));
}
public VertexInfo getVertexInfo(String label) {
checkVertexExist(label);
- return vertexLabel2VertexInfo.get(label);
+ return cachedVertexInfoMap.get(label);
}
public EdgeInfo getEdgeInfo(String srcLabel, String edgeLabel, String
dstLabel) {
checkEdgeExist(srcLabel, edgeLabel, dstLabel);
- return edgeConcat2EdgeInfo.get(srcLabel + edgeLabel + dstLabel);
+ return cachedEdgeInfoMap.get(EdgeInfo.concat(srcLabel, edgeLabel,
dstLabel));
}
public int getVertexInfoNum() {
- return vertexInfos.size();
+ return cachedVertexInfoList.size();
}
public int getEdgeInfoNum() {
- return edgeInfos.size();
+ return cachedEdgeInfoList.size();
}
public String getName() {
- return name;
+ return protoGraphInfo.getName();
}
public List<VertexInfo> getVertexInfos() {
- return vertexInfos;
+ return cachedVertexInfoList;
}
public List<EdgeInfo> getEdgeInfos() {
- return edgeInfos;
+ return cachedEdgeInfoList;
}
public String getPrefix() {
- return prefix;
- }
-
- public String getVersion() {
- return version;
- }
-
- private static List<VertexInfo> vertexFileNames2VertexInfos(
- List<String> vertexFileNames, Configuration conf) throws
IOException {
- ArrayList<VertexInfo> tempVertices = new
ArrayList<>(vertexFileNames.size());
- for (String vertexFileName : vertexFileNames) {
- tempVertices.add(VertexInfo.load(vertexFileName, conf));
- }
- return List.copyOf(tempVertices);
- }
-
- private static List<EdgeInfo> edgeFileNames2EdgeInfos(
- List<String> edgeFileNames, Configuration conf) throws IOException
{
- ArrayList<EdgeInfo> tempEdges = new ArrayList<>(edgeFileNames.size());
- for (String edgeFileName : edgeFileNames) {
- tempEdges.add(EdgeInfo.load(edgeFileName, conf));
- }
- return List.copyOf(tempEdges);
+ return protoGraphInfo.getPrefix();
}
private void checkVertexExist(String label) {
diff --git
a/maven-projects/info/src/main/java/org/apache/graphar/info/Property.java
b/maven-projects/info/src/main/java/org/apache/graphar/info/Property.java
index 1221ebc1..09ce2361 100644
--- a/maven-projects/info/src/main/java/org/apache/graphar/info/Property.java
+++ b/maven-projects/info/src/main/java/org/apache/graphar/info/Property.java
@@ -19,42 +19,46 @@
package org.apache.graphar.info;
-import org.apache.graphar.info.type.DataType;
-import org.apache.graphar.info.yaml.PropertyYaml;
+import org.apache.graphar.proto.DataType;
public class Property {
- private final String name;
- private final DataType dataType;
- private final boolean primary;
- private final boolean nullable;
+ private final org.apache.graphar.proto.Property protoProperty;
public Property(String name, DataType dataType, boolean primary, boolean
nullable) {
- this.name = name;
- this.dataType = dataType;
- this.primary = primary;
- this.nullable = nullable;
+ protoProperty =
+ org.apache.graphar.proto.Property.newBuilder()
+ .setName(name)
+ .setType(dataType)
+ .setIsPrimaryKey(primary)
+ .setIsNullable(nullable)
+ .build();
}
- Property(PropertyYaml yamlParser) {
- this.name = yamlParser.getName();
- this.dataType = DataType.fromString(yamlParser.getData_type());
- this.primary = yamlParser.getIs_primary();
- this.nullable = yamlParser.getIs_nullable();
+ private Property(org.apache.graphar.proto.Property protoProperty) {
+ this.protoProperty = protoProperty;
+ }
+
+ public static Property ofProto(org.apache.graphar.proto.Property
protoProperty) {
+ return new Property(protoProperty);
}
public String getName() {
- return name;
+ return protoProperty.getName();
}
public DataType getDataType() {
- return dataType;
+ return protoProperty.getType();
}
public boolean isPrimary() {
- return primary;
+ return protoProperty.getIsPrimaryKey();
}
public boolean isNullable() {
- return nullable;
+ return protoProperty.getIsNullable();
+ }
+
+ org.apache.graphar.proto.Property getProto() {
+ return protoProperty;
}
}
diff --git
a/maven-projects/info/src/main/java/org/apache/graphar/info/PropertyGroup.java
b/maven-projects/info/src/main/java/org/apache/graphar/info/PropertyGroup.java
index 1d922c99..87b8d5bd 100644
---
a/maven-projects/info/src/main/java/org/apache/graphar/info/PropertyGroup.java
+++
b/maven-projects/info/src/main/java/org/apache/graphar/info/PropertyGroup.java
@@ -27,80 +27,109 @@ import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import org.apache.graphar.info.type.DataType;
-import org.apache.graphar.info.type.FileType;
-import org.apache.graphar.info.yaml.PropertyGroupYaml;
+import org.apache.graphar.proto.DataType;
+import org.apache.graphar.proto.FileType;
import org.apache.graphar.util.GeneralParams;
public class PropertyGroup implements Iterable<Property> {
- private final List<Property> propertyList;
- private final Map<String, Property> propertyMap;
- private final FileType fileType;
- private final String prefix;
-
- public PropertyGroup(List<Property> propertyMap, FileType fileType, String
prefix) {
- this.propertyList = List.copyOf(propertyMap);
- this.propertyMap =
- propertyMap.stream()
+ private final org.apache.graphar.proto.PropertyGroup protoPropertyGroup;
+ private final List<Property> cachedPropertyList;
+ private final Map<String, Property> cachedPropertyMap;
+
+ public PropertyGroup(List<Property> propertyList, FileType fileType,
String prefix) {
+ this.cachedPropertyList = List.copyOf(propertyList);
+ this.cachedPropertyMap =
+ cachedPropertyList.stream()
+ .collect(
+ Collectors.toUnmodifiableMap(
+ Property::getName,
Function.identity()));
+ this.protoPropertyGroup =
+ org.apache.graphar.proto.PropertyGroup.newBuilder()
+ .addAllProperties(
+ cachedPropertyList.stream()
+ .map(Property::getProto)
+ .collect(Collectors.toList()))
+ .setFileType(fileType)
+ .setPrefix(prefix)
+ .build();
+ }
+
+ private PropertyGroup(org.apache.graphar.proto.PropertyGroup
protoPropertyGroup) {
+ this.protoPropertyGroup = protoPropertyGroup;
+ this.cachedPropertyList =
+ protoPropertyGroup.getPropertiesList().stream()
+ .map(Property::ofProto)
+ .collect(Collectors.toUnmodifiableList());
+ this.cachedPropertyMap =
+ cachedPropertyList.stream()
.collect(
Collectors.toUnmodifiableMap(
Property::getName,
Function.identity()));
- this.fileType = fileType;
- this.prefix = prefix;
}
- PropertyGroup(PropertyGroupYaml yamlParser) {
- this(
- yamlParser.getProperties().stream()
- .map(Property::new)
- .collect(Collectors.toUnmodifiableList()),
- FileType.fromString(yamlParser.getFile_type()),
- yamlParser.getPrefix());
+ public static PropertyGroup ofProto(org.apache.graphar.proto.PropertyGroup
protoPropertyGroup) {
+ return new PropertyGroup(protoPropertyGroup);
}
public Optional<PropertyGroup> addPropertyAsNew(Property property) {
- if (property == null || !propertyMap.containsKey(property.getName())) {
+ if (property == null ||
cachedPropertyMap.containsKey(property.getName())) {
return Optional.empty();
}
- List<Property> newPropertyMap =
- Stream.concat(propertyMap.values().stream(),
Stream.of(property))
+ List<Property> newPropertyList =
+ Stream.concat(
+ protoPropertyGroup.getPropertiesList().stream()
+ .map(Property::ofProto),
+ Stream.of(property))
.collect(Collectors.toUnmodifiableList());
- return Optional.of(new PropertyGroup(newPropertyMap, fileType,
prefix));
+ return Optional.of(
+ new PropertyGroup(
+ newPropertyList,
+ protoPropertyGroup.getFileType(),
+ protoPropertyGroup.getPrefix()));
}
@Override
public Iterator<Property> iterator() {
- return propertyList.iterator();
+ return cachedPropertyList.iterator();
}
@Override
public String toString() {
- return propertyList.stream()
+ return cachedPropertyList.stream()
.map(Property::getName)
.collect(Collectors.joining(GeneralParams.regularSeparator));
}
public int size() {
- return propertyList.size();
+ return cachedPropertyList.size();
}
public List<Property> getPropertyList() {
- return propertyList;
+ return cachedPropertyList;
}
- public Map<String, Property> getPropertyMap() {
- return propertyMap;
+ public Map<String, Property> getCachedPropertyMap() {
+ return cachedPropertyMap;
}
public FileType getFileType() {
- return fileType;
+ return protoPropertyGroup.getFileType();
}
public String getPrefix() {
- return prefix;
+ return protoPropertyGroup.getPrefix();
+ }
+
+ org.apache.graphar.proto.PropertyGroup getProto() {
+ return protoPropertyGroup;
}
}
+/**
+ * PropertyGroups is a helper class to manage a list of PropertyGroup objects,
which can be used by
+ * {@link org.apache.graphar.info.VertexInfo} and {@link
org.apache.graphar.info.EdgeInfo}
+ * conveniently. but should not be exposed to outer side of the package.
+ */
class PropertyGroups {
private final List<PropertyGroup> propertyGroupList;
private final Map<String, PropertyGroup> propertyGroupMap;
@@ -110,7 +139,9 @@ class PropertyGroups {
this.propertyGroupList = List.copyOf(propertyGroupList);
this.properties =
propertyGroupList.stream()
- .flatMap(propertyGroup ->
propertyGroup.getPropertyMap().values().stream())
+ .flatMap(
+ propertyGroup ->
+
propertyGroup.getCachedPropertyMap().values().stream())
.collect(
Collectors.toUnmodifiableMap(
Property::getName,
Function.identity()));
@@ -132,6 +163,14 @@ class PropertyGroups {
this.properties = properties;
}
+ static PropertyGroups ofProto(
+ List<org.apache.graphar.proto.PropertyGroup> protoPropertyGroups) {
+ return new PropertyGroups(
+ protoPropertyGroups.stream()
+ .map(PropertyGroup::ofProto)
+ .collect(Collectors.toList()));
+ }
+
Optional<PropertyGroups> addPropertyGroupAsNew(PropertyGroup
propertyGroup) {
if (propertyGroup == null || propertyGroup.size() == 0 ||
hasPropertyGroup(propertyGroup)) {
return Optional.empty();
@@ -144,22 +183,20 @@ class PropertyGroups {
List<PropertyGroup> newPropertyGroupsAsList =
Stream.concat(propertyGroupList.stream(),
Stream.of(propertyGroup))
.collect(Collectors.toUnmodifiableList());
- Map<String, PropertyGroup> tempPropertyGroupAsMap = new
HashMap<>(propertyGroupMap);
+ Map<String, PropertyGroup> tempPropertyGroupMap = new
HashMap<>(propertyGroupMap);
for (Property property : propertyGroup) {
- tempPropertyGroupAsMap.put(property.getName(), propertyGroup);
+ tempPropertyGroupMap.put(property.getName(), propertyGroup);
}
Map<String, Property> newProperties =
Stream.concat(
properties.values().stream(),
-
propertyGroup.getPropertyMap().values().stream())
+
propertyGroup.getCachedPropertyMap().values().stream())
.collect(
Collectors.toUnmodifiableMap(
Property::getName,
Function.identity()));
return Optional.of(
new PropertyGroups(
- newPropertyGroupsAsList,
- Map.copyOf(tempPropertyGroupAsMap),
- newProperties));
+ newPropertyGroupsAsList,
Map.copyOf(tempPropertyGroupMap), newProperties));
}
boolean hasProperty(String propertyName) {
diff --git
a/maven-projects/info/src/main/java/org/apache/graphar/info/VertexInfo.java
b/maven-projects/info/src/main/java/org/apache/graphar/info/VertexInfo.java
index c03bab5d..d3247562 100644
--- a/maven-projects/info/src/main/java/org/apache/graphar/info/VertexInfo.java
+++ b/maven-projects/info/src/main/java/org/apache/graphar/info/VertexInfo.java
@@ -19,119 +19,81 @@
package org.apache.graphar.info;
-import java.io.IOException;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
-import org.apache.graphar.info.type.DataType;
import org.apache.graphar.info.yaml.GraphYaml;
import org.apache.graphar.info.yaml.VertexYaml;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.yaml.snakeyaml.LoaderOptions;
+import org.apache.graphar.proto.DataType;
import org.yaml.snakeyaml.Yaml;
-import org.yaml.snakeyaml.constructor.Constructor;
public class VertexInfo {
- private final String label;
- private final long chunkSize;
- private final PropertyGroups propertyGroups;
- private final String prefix;
- private final String version;
+ private final org.apache.graphar.proto.VertexInfo protoVertexInfo;
+ private final PropertyGroups cachedPropertyGroups;
public VertexInfo(
- String label,
- long chunkSize,
- List<PropertyGroup> propertyGroups,
- String prefix,
- String version) {
- this.label = label;
- this.chunkSize = chunkSize;
- this.propertyGroups = new PropertyGroups(propertyGroups);
- this.prefix = prefix;
- this.version = version;
- }
-
- private VertexInfo(VertexYaml parser) {
- this(
- parser.getType(),
- parser.getChunk_size(),
- parser.getProperty_groups().stream()
- .map(PropertyGroup::new)
- .collect(Collectors.toUnmodifiableList()),
- parser.getPrefix(),
- parser.getVersion());
- }
-
- private VertexInfo(
- String label,
- long chunkSize,
- PropertyGroups propertyGroups,
- String prefix,
- String version) {
- this.label = label;
- this.chunkSize = chunkSize;
- this.propertyGroups = propertyGroups;
- this.prefix = prefix;
- this.version = version;
- }
-
- public static VertexInfo load(String vertexInfoPath) throws IOException {
- return load(vertexInfoPath, new Configuration());
- }
-
- public static VertexInfo load(String vertexInfoPath, Configuration conf)
throws IOException {
- if (conf == null) {
- throw new IllegalArgumentException("Configuration is null");
- }
- return load(vertexInfoPath, FileSystem.get(conf));
+ String type, long chunkSize, List<PropertyGroup> propertyGroups,
String prefix) {
+ this.cachedPropertyGroups = new PropertyGroups(propertyGroups);
+ this.protoVertexInfo =
+ org.apache.graphar.proto.VertexInfo.newBuilder()
+ .setType(type)
+ .setChunkSize(chunkSize)
+ .addAllProperties(
+ propertyGroups.stream()
+ .map(PropertyGroup::getProto)
+ .collect(Collectors.toList()))
+ .setPrefix(prefix)
+ .build();
}
- public static VertexInfo load(String vertexInfoPath, FileSystem
fileSystem) throws IOException {
- if (fileSystem == null) {
- throw new IllegalArgumentException("FileSystem is null");
- }
- FSDataInputStream inputStream = fileSystem.open(new
Path(vertexInfoPath));
- Yaml vertexInfoYamlLoader =
- new Yaml(new Constructor(VertexYaml.class, new
LoaderOptions()));
- VertexYaml vertexInfoYaml = vertexInfoYamlLoader.load(inputStream);
- return new VertexInfo(vertexInfoYaml);
+ private VertexInfo(org.apache.graphar.proto.VertexInfo protoVertexInfo) {
+ this.protoVertexInfo = protoVertexInfo;
+ this.cachedPropertyGroups =
PropertyGroups.ofProto(protoVertexInfo.getPropertiesList());
+ }
+
+ public static VertexInfo ofProto(org.apache.graphar.proto.VertexInfo
protoVertexInfo) {
+ return new VertexInfo(protoVertexInfo);
+ }
+
+ org.apache.graphar.proto.VertexInfo getProto() {
+ return protoVertexInfo;
}
public Optional<VertexInfo> addPropertyGroupAsNew(PropertyGroup
propertyGroup) {
- return propertyGroups
+ // do not need check property group exist, because PropertyGroups will
check it
+ return cachedPropertyGroups
.addPropertyGroupAsNew(propertyGroup)
.map(
newPropertyGroups ->
new VertexInfo(
- label, chunkSize, newPropertyGroups,
prefix, version));
+ protoVertexInfo.getType(),
+ protoVertexInfo.getChunkSize(),
+
newPropertyGroups.getPropertyGroupList(),
+ protoVertexInfo.getPrefix()));
}
public int propertyGroupNum() {
- return propertyGroups.getPropertyGroupNum();
+ return cachedPropertyGroups.getPropertyGroupNum();
}
public DataType getPropertyType(String propertyName) {
- return propertyGroups.getPropertyType(propertyName);
+ return cachedPropertyGroups.getPropertyType(propertyName);
}
public boolean hasProperty(String propertyName) {
- return propertyGroups.hasProperty(propertyName);
+ return cachedPropertyGroups.hasProperty(propertyName);
}
public boolean isPrimaryKey(String propertyName) {
- return propertyGroups.isPrimaryKey(propertyName);
+ return cachedPropertyGroups.isPrimaryKey(propertyName);
}
public boolean isNullableKey(String propertyName) {
- return propertyGroups.isNullableKey(propertyName);
+ return cachedPropertyGroups.isNullableKey(propertyName);
}
public boolean hasPropertyGroup(PropertyGroup propertyGroup) {
- return propertyGroups.hasPropertyGroup(propertyGroup);
+ return cachedPropertyGroups.hasPropertyGroup(propertyGroup);
}
public String getPropertyGroupPrefix(PropertyGroup propertyGroup) {
@@ -148,50 +110,30 @@ public class VertexInfo {
return getPrefix() + "/vertex_count";
}
- public void save(String filePath) throws IOException {
- save(filePath, new Configuration());
- }
-
- public void save(String filePath, Configuration conf) throws IOException {
- if (conf == null) {
- throw new IllegalArgumentException("Configuration is null");
- }
- save(filePath, FileSystem.get(conf));
- }
-
- public void save(String fileName, FileSystem fileSystem) throws
IOException {
- if (fileSystem == null) {
- throw new IllegalArgumentException("FileSystem is null");
- }
- FSDataOutputStream outputStream = fileSystem.create(new
Path(fileName));
- outputStream.writeBytes(dump());
- outputStream.close();
- }
-
public String dump() {
Yaml yaml = new Yaml(GraphYaml.getDumperOptions());
VertexYaml vertexYaml = new VertexYaml(this);
return yaml.dump(vertexYaml);
}
- public String getLabel() {
- return label;
+ public String getType() {
+ return protoVertexInfo.getType();
}
public long getChunkSize() {
- return chunkSize;
+ return protoVertexInfo.getChunkSize();
}
public List<PropertyGroup> getPropertyGroups() {
- return propertyGroups.getPropertyGroupList();
+ return cachedPropertyGroups.getPropertyGroupList();
}
public String getPrefix() {
- return prefix;
+ return protoVertexInfo.getPrefix();
}
- public String getVersion() {
- return version;
+ public String getVertexPath() {
+ return getPrefix() + "/" + getType() + ".vertex.yaml";
}
private void checkPropertyGroupExist(PropertyGroup propertyGroup) {
@@ -203,7 +145,7 @@ public class VertexInfo {
"Property group "
+ propertyGroup
+ " does not exist in the vertex "
- + getLabel());
+ + getType());
}
}
}
diff --git
a/maven-projects/info/src/main/java/org/apache/graphar/info/type/FileType.java
b/maven-projects/info/src/main/java/org/apache/graphar/info/loader/GraphLoader.java
similarity index 60%
copy from
maven-projects/info/src/main/java/org/apache/graphar/info/type/FileType.java
copy to
maven-projects/info/src/main/java/org/apache/graphar/info/loader/GraphLoader.java
index b45038b8..b21238c1 100644
---
a/maven-projects/info/src/main/java/org/apache/graphar/info/type/FileType.java
+++
b/maven-projects/info/src/main/java/org/apache/graphar/info/loader/GraphLoader.java
@@ -17,26 +17,12 @@
* under the License.
*/
-package org.apache.graphar.info.type;
+package org.apache.graphar.info.loader;
-public enum FileType {
- CSV,
- PARQUET,
- ORC;
+import java.io.IOException;
+import org.apache.graphar.info.GraphInfo;
- public static FileType fromString(String fileType) {
- if (fileType == null) {
- return null;
- }
- switch (fileType) {
- case "csv":
- return CSV;
- case "parquet":
- return PARQUET;
- case "orc":
- return ORC;
- default:
- throw new IllegalArgumentException("Unknown file type: " +
fileType);
- }
- }
+@FunctionalInterface
+public interface GraphLoader {
+ public GraphInfo load(String graphYamlPath) throws IOException;
}
diff --git
a/maven-projects/info/src/main/java/org/apache/graphar/info/loader/LocalYamlGraphLoader.java
b/maven-projects/info/src/main/java/org/apache/graphar/info/loader/LocalYamlGraphLoader.java
new file mode 100644
index 00000000..f2936fb4
--- /dev/null
+++
b/maven-projects/info/src/main/java/org/apache/graphar/info/loader/LocalYamlGraphLoader.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.graphar.info.loader;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.graphar.info.EdgeInfo;
+import org.apache.graphar.info.GraphInfo;
+import org.apache.graphar.info.VertexInfo;
+import org.apache.graphar.info.yaml.EdgeYaml;
+import org.apache.graphar.info.yaml.GraphYaml;
+import org.apache.graphar.info.yaml.VertexYaml;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.yaml.snakeyaml.LoaderOptions;
+import org.yaml.snakeyaml.Yaml;
+import org.yaml.snakeyaml.constructor.Constructor;
+
+public class LocalYamlGraphLoader implements GraphLoader {
+ private static FileSystem fileSystem = null;
+
+ public LocalYamlGraphLoader() {}
+
+ @Override
+ public GraphInfo load(String graphYamlPath) throws IOException {
+ if (fileSystem == null) {
+ fileSystem = FileSystem.get(new Configuration());
+ }
+ // load graph itself
+ final Path path = new Path(graphYamlPath);
+ final FSDataInputStream inputStream = fileSystem.open(path);
+ final Yaml yamlLoader = new Yaml(new Constructor(GraphYaml.class, new
LoaderOptions()));
+ final GraphYaml graphYaml = yamlLoader.load(inputStream);
+ // load vertices
+ final String ABSOLUTE_PREFIX = path.getParent().toString();
+ List<VertexInfo> vertexInfos = new
ArrayList<>(graphYaml.getVertices().size());
+ for (String vertexYamlName : graphYaml.getVertices()) {
+ vertexInfos.add(loadVertex(ABSOLUTE_PREFIX + "/" +
vertexYamlName));
+ }
+ // load edges
+ List<EdgeInfo> edgeInfos = new
ArrayList<>(graphYaml.getEdges().size());
+ for (String edgeYamlName : graphYaml.getEdges()) {
+ edgeInfos.add(loadEdge(ABSOLUTE_PREFIX + "/" + edgeYamlName));
+ }
+ return new GraphInfo(graphYaml.getName(), vertexInfos, edgeInfos,
graphYaml.getPrefix());
+ }
+
+ private VertexInfo loadVertex(String path) throws IOException {
+ FSDataInputStream inputStream = fileSystem.open(new Path(path));
+ Yaml vertexYamlLoader = new Yaml(new Constructor(VertexYaml.class, new
LoaderOptions()));
+ VertexYaml vertexYaml = vertexYamlLoader.load(inputStream);
+ return vertexYaml.toVertexInfo();
+ }
+
+ private EdgeInfo loadEdge(String path) throws IOException {
+ FSDataInputStream inputStream = fileSystem.open(new Path(path));
+ Yaml edgeYamlLoader = new Yaml(new Constructor(EdgeYaml.class, new
LoaderOptions()));
+ EdgeYaml edgeYaml = edgeYamlLoader.load(inputStream);
+ return edgeYaml.toEdgeInfo();
+ }
+}
diff --git
a/maven-projects/info/src/main/java/org/apache/graphar/info/type/FileType.java
b/maven-projects/info/src/main/java/org/apache/graphar/info/saver/GraphSaver.java
similarity index 60%
rename from
maven-projects/info/src/main/java/org/apache/graphar/info/type/FileType.java
rename to
maven-projects/info/src/main/java/org/apache/graphar/info/saver/GraphSaver.java
index b45038b8..5ee1d916 100644
---
a/maven-projects/info/src/main/java/org/apache/graphar/info/type/FileType.java
+++
b/maven-projects/info/src/main/java/org/apache/graphar/info/saver/GraphSaver.java
@@ -17,26 +17,12 @@
* under the License.
*/
-package org.apache.graphar.info.type;
+package org.apache.graphar.info.saver;
-public enum FileType {
- CSV,
- PARQUET,
- ORC;
+import java.io.IOException;
+import org.apache.graphar.info.GraphInfo;
- public static FileType fromString(String fileType) {
- if (fileType == null) {
- return null;
- }
- switch (fileType) {
- case "csv":
- return CSV;
- case "parquet":
- return PARQUET;
- case "orc":
- return ORC;
- default:
- throw new IllegalArgumentException("Unknown file type: " +
fileType);
- }
- }
+@FunctionalInterface
+public interface GraphSaver {
+ void save(String path, GraphInfo graphInfo) throws IOException;
}
diff --git
a/maven-projects/info/src/main/java/org/apache/graphar/info/saver/LocalYamlGraphSaver.java
b/maven-projects/info/src/main/java/org/apache/graphar/info/saver/LocalYamlGraphSaver.java
new file mode 100644
index 00000000..500268d1
--- /dev/null
+++
b/maven-projects/info/src/main/java/org/apache/graphar/info/saver/LocalYamlGraphSaver.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.graphar.info.saver;
+
+import java.io.IOException;
+import org.apache.graphar.info.EdgeInfo;
+import org.apache.graphar.info.GraphInfo;
+import org.apache.graphar.info.VertexInfo;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+public class LocalYamlGraphSaver implements GraphSaver {
+ private static final FileSystem fileSystem;
+
+ static {
+ try {
+ fileSystem = FileSystem.get(new Configuration());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void save(String path, GraphInfo graphInfo) throws IOException {
+ try (FSDataOutputStream outputStream =
+ fileSystem.create(new Path(path + "/" + graphInfo.getName() +
".graph.yaml"))) {
+ outputStream.writeBytes(graphInfo.dump());
+ for (VertexInfo vertexInfo : graphInfo.getVertexInfos()) {
+ saveVertex(path, vertexInfo);
+ }
+ for (EdgeInfo edgeInfo : graphInfo.getEdgeInfos()) {
+ saveEdge(path, edgeInfo);
+ }
+ }
+ }
+
+ private void saveVertex(String path, VertexInfo vertexInfo) throws
IOException {
+ try (FSDataOutputStream outputStream =
+ fileSystem.create(new Path(path + "/" + vertexInfo.getType() +
".vertex.yaml"))) {
+ outputStream.writeBytes(vertexInfo.dump());
+ }
+ }
+
+ private void saveEdge(String path, EdgeInfo edgeInfo) throws IOException {
+ try (FSDataOutputStream outputStream =
+ fileSystem.create(new Path(path + "/" + edgeInfo.getConcat() +
".edge.yaml"))) {
+ outputStream.writeBytes(edgeInfo.dump());
+ }
+ }
+}
diff --git
a/maven-projects/info/src/main/java/org/apache/graphar/info/type/AdjListType.java
b/maven-projects/info/src/main/java/org/apache/graphar/info/type/AdjListType.java
deleted file mode 100644
index a077d870..00000000
---
a/maven-projects/info/src/main/java/org/apache/graphar/info/type/AdjListType.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.graphar.info.type;
-
-public enum AdjListType {
- /** collection of edges by source, but unordered, can represent COO format
*/
- unordered_by_source,
- /** collection of edges by destination, but unordered, can represent COO */
- unordered_by_dest,
- /** collection of edges by source, ordered by source, can represent CSR
format */
- ordered_by_source,
- /** collection of edges by destination, ordered by destination, can
represent CSC format */
- ordered_by_dest;
-
- public static AdjListType fromOrderedAndAlignedBy(boolean ordered, String
alignedBy) {
- switch (alignedBy) {
- case "src":
- return ordered ? ordered_by_source : unordered_by_source;
- case "dst":
- return ordered ? ordered_by_dest : unordered_by_dest;
- default:
- throw new IllegalArgumentException("Invalid alignedBy: " +
alignedBy);
- }
- }
-
- public boolean isOrdered() {
- return this == ordered_by_source || this == ordered_by_dest;
- }
-
- public String getAlignedBy() {
- return this == ordered_by_source || this == unordered_by_source ?
"src" : "dst";
- }
-}
diff --git
a/maven-projects/info/src/main/java/org/apache/graphar/info/type/DataType.java
b/maven-projects/info/src/main/java/org/apache/graphar/info/type/DataType.java
deleted file mode 100644
index 83c31aec..00000000
---
a/maven-projects/info/src/main/java/org/apache/graphar/info/type/DataType.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.graphar.info.type;
-
-public enum DataType {
- /** Boolean */
- BOOL,
-
- /** Signed 32-bit integer */
- INT32,
-
- /** Signed 64-bit integer */
- INT64,
-
- /** 4-byte floating point value */
- FLOAT,
-
- /** 8-byte floating point value */
- DOUBLE,
-
- /** UTF8 variable-length string */
- STRING,
-
- /** List of same type */
- LIST;
-
- public static DataType fromString(String s) {
- switch (s) {
- case "bool":
- return BOOL;
- case "int32":
- return INT32;
- case "int64":
- return INT64;
- case "float":
- return FLOAT;
- case "double":
- return DOUBLE;
- case "string":
- return STRING;
- case "list":
- return LIST;
- default:
- throw new IllegalArgumentException("Unknown data type: " + s);
- }
- }
-
- @Override
- public String toString() {
- return name().toLowerCase();
- }
-}
diff --git
a/maven-projects/info/src/main/java/org/apache/graphar/info/yaml/AdjacentListYaml.java
b/maven-projects/info/src/main/java/org/apache/graphar/info/yaml/AdjacentListYaml.java
index 90d3a1cb..b65da68d 100644
---
a/maven-projects/info/src/main/java/org/apache/graphar/info/yaml/AdjacentListYaml.java
+++
b/maven-projects/info/src/main/java/org/apache/graphar/info/yaml/AdjacentListYaml.java
@@ -19,6 +19,9 @@
package org.apache.graphar.info.yaml;
+import org.apache.graphar.info.AdjacentList;
+import org.apache.graphar.proto.AdjListType;
+
public class AdjacentListYaml {
private boolean ordered;
private String aligned_by;
@@ -33,12 +36,26 @@ public class AdjacentListYaml {
}
public AdjacentListYaml(org.apache.graphar.info.AdjacentList adjacentList)
{
- this.ordered = adjacentList.getType().isOrdered();
- this.aligned_by = adjacentList.getType().getAlignedBy();
- this.file_type = adjacentList.getFileType().toString();
+ final var adjListType = adjacentList.getType();
+ this.ordered =
+ adjListType == AdjListType.ORDERED_BY_SOURCE
+ || adjListType == AdjListType.ORDERED_BY_DESTINATION;
+ this.aligned_by =
+ adjListType == AdjListType.ORDERED_BY_SOURCE
+ || adjListType ==
AdjListType.UNORDERED_BY_SOURCE
+ ? "src"
+ : "dst";
+ this.file_type =
EnumTransferUtil.fileType2String(adjacentList.getFileType());
this.prefix = adjacentList.getPrefix();
}
+ public AdjacentList toAdjacentList() {
+ return new AdjacentList(
+ EnumTransferUtil.orderedAndAlignedBy2AdjListType(ordered,
aligned_by),
+ EnumTransferUtil.string2FileType(file_type),
+ prefix);
+ }
+
public boolean isOrdered() {
return ordered;
}
diff --git
a/maven-projects/info/src/main/java/org/apache/graphar/info/yaml/EdgeYaml.java
b/maven-projects/info/src/main/java/org/apache/graphar/info/yaml/EdgeYaml.java
index e955306b..ff31aae4 100644
---
a/maven-projects/info/src/main/java/org/apache/graphar/info/yaml/EdgeYaml.java
+++
b/maven-projects/info/src/main/java/org/apache/graphar/info/yaml/EdgeYaml.java
@@ -48,12 +48,12 @@ public class EdgeYaml {
this.prefix = "";
this.adj_lists = new ArrayList<>();
this.property_groups = new ArrayList<>();
- this.version = "";
+ this.version = "v1";
}
public EdgeYaml(EdgeInfo edgeInfo) {
this.src_type = edgeInfo.getSrcLabel();
- this.edge_type = edgeInfo.getEdgeLabel();
+ this.edge_type = edgeInfo.getSrcLabel();
this.dst_type = edgeInfo.getDstLabel();
this.chunk_size = edgeInfo.getChunkSize();
this.src_chunk_size = edgeInfo.getSrcChunkSize();
@@ -68,7 +68,24 @@ public class EdgeYaml {
edgeInfo.getPropertyGroups().stream()
.map(PropertyGroupYaml::new)
.collect(Collectors.toList());
- this.version = edgeInfo.getVersion();
+ }
+
+ public EdgeInfo toEdgeInfo() {
+ return new EdgeInfo(
+ src_type,
+ edge_type,
+ dst_type,
+ chunk_size,
+ src_chunk_size,
+ dst_chunk_size,
+ directed,
+ prefix,
+ adj_lists.stream()
+ .map(AdjacentListYaml::toAdjacentList)
+ .collect(Collectors.toUnmodifiableList()),
+ property_groups.stream()
+ .map(PropertyGroupYaml::toPropertyGroup)
+ .collect(Collectors.toList()));
}
public String getSrc_type() {
diff --git
a/maven-projects/info/src/main/java/org/apache/graphar/info/yaml/EnumTransferUtil.java
b/maven-projects/info/src/main/java/org/apache/graphar/info/yaml/EnumTransferUtil.java
new file mode 100644
index 00000000..2c484da8
--- /dev/null
+++
b/maven-projects/info/src/main/java/org/apache/graphar/info/yaml/EnumTransferUtil.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.graphar.info.yaml;
+
+import org.apache.graphar.proto.AdjListType;
+import org.apache.graphar.proto.FileType;
+
+public class EnumTransferUtil {
+ static String fileType2String(FileType fileType) {
+ switch (fileType) {
+ case CSV:
+ return "csv";
+ case PARQUET:
+ return "parquet";
+ case ORC:
+ return "orc";
+ case JSON:
+ return "json";
+ case AVRO:
+ return "avro";
+ case HDF5:
+ return "hdf5";
+ default:
+ throw new IllegalArgumentException("Invalid fileType: " +
fileType);
+ }
+ }
+
+ static FileType string2FileType(String fileType) {
+ switch (fileType) {
+ case "csv":
+ return FileType.CSV;
+ case "parquet":
+ return FileType.PARQUET;
+ case "orc":
+ return FileType.ORC;
+ case "json":
+ return FileType.JSON;
+ case "avro":
+ return FileType.AVRO;
+ case "hdf5":
+ return FileType.HDF5;
+ default:
+ throw new IllegalArgumentException("Invalid fileType: " +
fileType);
+ }
+ }
+
+ static String dataType2String(org.apache.graphar.proto.DataType dataType) {
+ switch (dataType) {
+ case BOOL:
+ return "bool";
+ case INT32:
+ return "int32";
+ case INT64:
+ return "int64";
+ case FLOAT:
+ return "float";
+ case DOUBLE:
+ return "double";
+ case STRING:
+ return "string";
+ case LIST:
+ return "list";
+ case DATE:
+ return "date";
+ case TIMESTAMP:
+ return "timestamp";
+ case TIME:
+ return "time";
+ default:
+ throw new IllegalArgumentException("Invalid dataType: " +
dataType);
+ }
+ }
+
+ static org.apache.graphar.proto.DataType string2DataType(String dataType) {
+ switch (dataType) {
+ case "bool":
+ return org.apache.graphar.proto.DataType.BOOL;
+ case "int32":
+ return org.apache.graphar.proto.DataType.INT32;
+ case "int64":
+ return org.apache.graphar.proto.DataType.INT64;
+ case "float":
+ return org.apache.graphar.proto.DataType.FLOAT;
+ case "double":
+ return org.apache.graphar.proto.DataType.DOUBLE;
+ case "string":
+ return org.apache.graphar.proto.DataType.STRING;
+ case "list":
+ return org.apache.graphar.proto.DataType.LIST;
+ case "date":
+ return org.apache.graphar.proto.DataType.DATE;
+ case "timestamp":
+ return org.apache.graphar.proto.DataType.TIMESTAMP;
+ case "time":
+ return org.apache.graphar.proto.DataType.TIME;
+ default:
+ throw new IllegalArgumentException("Invalid dataType: " +
dataType);
+ }
+ }
+
+ public static AdjListType orderedAndAlignedBy2AdjListType(boolean ordered,
String alignedBy) {
+ switch (alignedBy) {
+ case "src":
+ return ordered ? AdjListType.ORDERED_BY_SOURCE :
AdjListType.UNORDERED_BY_SOURCE;
+ case "dst":
+ return ordered
+ ? AdjListType.ORDERED_BY_DESTINATION
+ : AdjListType.UNORDERED_BY_DESTINATION;
+ default:
+ throw new IllegalArgumentException("Invalid alignedBy: " +
alignedBy);
+ }
+ }
+}
diff --git
a/maven-projects/info/src/main/java/org/apache/graphar/info/yaml/GraphYaml.java
b/maven-projects/info/src/main/java/org/apache/graphar/info/yaml/GraphYaml.java
index faec9bc4..4b52ea4e 100644
---
a/maven-projects/info/src/main/java/org/apache/graphar/info/yaml/GraphYaml.java
+++
b/maven-projects/info/src/main/java/org/apache/graphar/info/yaml/GraphYaml.java
@@ -54,13 +54,12 @@ public class GraphYaml {
this.prefix = graphInfo.getPrefix();
this.vertices =
graphInfo.getVertexInfos().stream()
- .map(vertexInfo -> vertexInfo.getLabel() +
".vertex.yaml")
+ .map(vertexInfo -> vertexInfo.getType() +
".vertex.yaml")
.collect(Collectors.toList());
this.edges =
graphInfo.getEdgeInfos().stream()
.map(edgeInfo -> edgeInfo.getConcat() + ".edge.yaml")
.collect(Collectors.toList());
- this.version = graphInfo.getVersion();
}
public static DumperOptions getDumperOptions() {
diff --git
a/maven-projects/info/src/main/java/org/apache/graphar/info/yaml/PropertyGroupYaml.java
b/maven-projects/info/src/main/java/org/apache/graphar/info/yaml/PropertyGroupYaml.java
index 202d6ed0..1e992539 100644
---
a/maven-projects/info/src/main/java/org/apache/graphar/info/yaml/PropertyGroupYaml.java
+++
b/maven-projects/info/src/main/java/org/apache/graphar/info/yaml/PropertyGroupYaml.java
@@ -40,10 +40,17 @@ public class PropertyGroupYaml {
propertyGroup.getPropertyList().stream()
.map(PropertyYaml::new)
.collect(Collectors.toList());
- this.file_type = propertyGroup.getFileType().toString();
+ this.file_type =
EnumTransferUtil.fileType2String(propertyGroup.getFileType());
this.prefix = propertyGroup.getPrefix();
}
+ PropertyGroup toPropertyGroup() {
+ return new PropertyGroup(
+
properties.stream().map(PropertyYaml::toProperty).collect(Collectors.toList()),
+ EnumTransferUtil.string2FileType(file_type),
+ prefix);
+ }
+
public List<PropertyYaml> getProperties() {
return properties;
}
diff --git
a/maven-projects/info/src/main/java/org/apache/graphar/info/yaml/PropertyYaml.java
b/maven-projects/info/src/main/java/org/apache/graphar/info/yaml/PropertyYaml.java
index bd06b577..89fa87a0 100644
---
a/maven-projects/info/src/main/java/org/apache/graphar/info/yaml/PropertyYaml.java
+++
b/maven-projects/info/src/main/java/org/apache/graphar/info/yaml/PropertyYaml.java
@@ -37,11 +37,19 @@ public class PropertyYaml {
public PropertyYaml(Property property) {
this.name = property.getName();
- this.data_type = property.getDataType().toString();
+ this.data_type =
EnumTransferUtil.dataType2String(property.getDataType());
this.is_primary = property.isPrimary();
this.is_nullable = Optional.of(property.isNullable());
}
+ Property toProperty() {
+ return new Property(
+ name,
+ EnumTransferUtil.string2DataType(data_type),
+ is_primary,
+ is_nullable.orElseGet(() -> !is_primary));
+ }
+
public String getName() {
return name;
}
diff --git
a/maven-projects/info/src/main/java/org/apache/graphar/info/yaml/VertexYaml.java
b/maven-projects/info/src/main/java/org/apache/graphar/info/yaml/VertexYaml.java
index a759724f..6449c716 100644
---
a/maven-projects/info/src/main/java/org/apache/graphar/info/yaml/VertexYaml.java
+++
b/maven-projects/info/src/main/java/org/apache/graphar/info/yaml/VertexYaml.java
@@ -40,14 +40,23 @@ public class VertexYaml {
}
public VertexYaml(VertexInfo vertexInfo) {
- this.type = vertexInfo.getLabel();
+ this.type = vertexInfo.getType();
this.chunk_size = vertexInfo.getChunkSize();
this.property_groups =
vertexInfo.getPropertyGroups().stream()
.map(PropertyGroupYaml::new)
.collect(Collectors.toList());
this.prefix = vertexInfo.getPrefix();
- this.version = vertexInfo.getVersion();
+ }
+
+ public VertexInfo toVertexInfo() {
+ return new VertexInfo(
+ type,
+ chunk_size,
+ property_groups.stream()
+ .map(PropertyGroupYaml::toPropertyGroup)
+ .collect(Collectors.toList()),
+ prefix);
}
public String getType() {
diff --git
a/maven-projects/info/src/test/java/org/apache/graphar/info/GraphLoaderTest.java
b/maven-projects/info/src/test/java/org/apache/graphar/info/GraphLoaderTest.java
new file mode 100644
index 00000000..00272042
--- /dev/null
+++
b/maven-projects/info/src/test/java/org/apache/graphar/info/GraphLoaderTest.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.graphar.info;
+
+import java.io.IOException;
+import org.apache.graphar.info.loader.GraphLoader;
+import org.apache.graphar.info.loader.LocalYamlGraphLoader;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class GraphLoaderTest {
+
+ @BeforeClass
+ public static void init() {
+ TestUtil.checkTestData();
+ }
+
+ @AfterClass
+ public static void clean() {}
+
+ @Test
+ public void testLoad() {
+ final GraphLoader graphLoader = new LocalYamlGraphLoader();
+ final String GRAPH_PATH = TestUtil.getLdbcSampleGraphPath();
+ try {
+ final GraphInfo graphInfo = graphLoader.load(GRAPH_PATH);
+ Assert.assertNotNull(graphInfo);
+ Assert.assertNotNull(graphInfo.getEdgeInfos());
+ Assert.assertEquals(1, graphInfo.getEdgeInfos().size());
+ for (EdgeInfo edgeInfo : graphInfo.getEdgeInfos()) {
+ Assert.assertNotNull(edgeInfo.getConcat());
+ }
+ Assert.assertNotNull(graphInfo.getVertexInfos());
+ Assert.assertEquals(1, graphInfo.getVertexInfos().size());
+ for (VertexInfo vertexInfo : graphInfo.getVertexInfos()) {
+ Assert.assertNotNull(vertexInfo.getType());
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git
a/maven-projects/info/src/test/java/org/apache/graphar/info/GraphSaverTest.java
b/maven-projects/info/src/test/java/org/apache/graphar/info/GraphSaverTest.java
new file mode 100644
index 00000000..3fe219e2
--- /dev/null
+++
b/maven-projects/info/src/test/java/org/apache/graphar/info/GraphSaverTest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.graphar.info;
+
+import java.io.File;
+import org.apache.graphar.info.saver.GraphSaver;
+import org.apache.graphar.info.saver.LocalYamlGraphSaver;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class GraphSaverTest {
+
+ @Test
+ public void testSave() {
+ final String LDBC_SAMPLE_SAVE_DIR = TestUtil.SAVE_DIR +
"/ldbc_sample/";
+ final GraphSaver graphSaver = new LocalYamlGraphSaver();
+ final GraphInfo graphInfo = TestUtil.getLdbcSampleDataSet();
+ try {
+ graphSaver.save(LDBC_SAMPLE_SAVE_DIR, graphInfo);
+ Assert.assertTrue(
+ new File(LDBC_SAMPLE_SAVE_DIR + "/" + graphInfo.getName()
+ ".graph.yaml")
+ .exists());
+ for (VertexInfo vertexInfo : graphInfo.getVertexInfos()) {
+ Assert.assertTrue(
+ new File(LDBC_SAMPLE_SAVE_DIR + "/" +
vertexInfo.getType() + ".vertex.yaml")
+ .exists());
+ }
+ for (EdgeInfo edgeInfo : graphInfo.getEdgeInfos()) {
+ Assert.assertTrue(
+ new File(LDBC_SAMPLE_SAVE_DIR + "/" +
edgeInfo.getConcat() + ".edge.yaml")
+ .exists());
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git
a/maven-projects/info/src/test/java/org/apache/graphar/info/TestUtil.java
b/maven-projects/info/src/test/java/org/apache/graphar/info/TestUtil.java
new file mode 100644
index 00000000..3660b905
--- /dev/null
+++ b/maven-projects/info/src/test/java/org/apache/graphar/info/TestUtil.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.graphar.info;
+
+import java.util.List;
+import org.apache.graphar.proto.AdjListType;
+import org.apache.graphar.proto.DataType;
+import org.apache.graphar.proto.FileType;
+import org.junit.Assume;
+
+public class TestUtil {
+ private static String GAR_TEST_DATA = null;
+
+ static final String SAVE_DIR = "/tmp/graphar/test/";
+
+ private static String LDBC_SAMPLE_GRAPH_PATH =
"/ldbc_sample/csv/ldbc_sample.graph.yml";
+
+ public static String getTestData() {
+ return GAR_TEST_DATA;
+ }
+
+ public static String getLdbcSampleGraphPath() {
+ return getTestData() + "/" + LDBC_SAMPLE_GRAPH_PATH;
+ }
+
+ public static GraphInfo getLdbcSampleDataSet() {
+ // create vertex info of yaml:
+ // type: person
+ // chunk_size: 100
+ // prefix: vertex/person/
+ // property_groups:
+ // - properties:
+ // - name: id
+ // data_type: int64
+ // is_primary: true
+ // is_nullable: false
+ // prefix: id/
+ // file_type: csv
+ // - properties:
+ // - name: firstName
+ // data_type: string
+ // is_primary: false
+ // - name: lastName
+ // data_type: string
+ // is_primary: false
+ // - name: gender
+ // data_type: string
+ // is_primary: false
+ // is_nullable: true
+ // prefix: firstName_lastName_gender/
+ // file_type: csv
+ // version: gar/v1
+
+ Property id = new Property("id", DataType.INT64, true, false);
+ Property firstName = new Property("firstName", DataType.STRING, false,
false);
+ Property lastName = new Property("lastName", DataType.STRING, false,
false);
+ Property gender = new Property("gender", DataType.STRING, false, true);
+ PropertyGroup pg1 = new PropertyGroup(List.of(id), FileType.CSV,
"id/");
+ PropertyGroup pg2 =
+ new PropertyGroup(
+ List.of(firstName, lastName, gender), FileType.CSV,
"firstName_lastName");
+ VertexInfo person = new VertexInfo("person", 100, List.of(pg1, pg2),
"vertex/person/");
+
+ // create edge info of yaml:
+ // src_type: person
+ // edge_type: knows
+ // dst_type: person
+ // chunk_size: 1024
+ // src_chunk_size: 100
+ // dst_chunk_size: 100
+ // directed: false
+ // prefix: edge/person_knows_person/
+ // adj_lists:
+ // - ordered: true
+ // aligned_by: src
+ // prefix: ordered_by_source/
+ // file_type: csv
+ // - ordered: true
+ // aligned_by: dst
+ // prefix: ordered_by_dest/
+ // file_type: csv
+ // property_groups:
+ // - prefix: creationDate/
+ // file_type: csv
+ // properties:
+ // - name: creationDate
+ // data_type: string
+ // is_primary: false
+ // version: gar/v1
+ AdjacentList orderedBySource =
+ new AdjacentList(AdjListType.ORDERED_BY_SOURCE, FileType.CSV,
"ordered_by_source/");
+ AdjacentList orderedByDest =
+ new AdjacentList(
+ AdjListType.ORDERED_BY_DESTINATION, FileType.CSV,
"ordered_by_dest/");
+ Property creationDate = new Property("creationDate", DataType.STRING,
false, false);
+ PropertyGroup pg3 = new PropertyGroup(List.of(creationDate),
FileType.CSV, "creationDate/");
+ EdgeInfo knows =
+ new EdgeInfo(
+ "person",
+ "knows",
+ "person",
+ 1024,
+ 100,
+ 100,
+ false,
+ "edge/person_knows_person/",
+ List.of(orderedBySource, orderedByDest),
+ List.of(pg3));
+
+ // create graph info of yaml:
+ // name: ldbc_sample
+ // vertices:
+ // - person.vertex.yml
+ // edges:
+ // - person_knows_person.edge.yml
+ // version: gar/v1
+ return new GraphInfo("ldbc_sample", List.of(person), List.of(knows),
"");
+ }
+
+ public static void checkTestData() {
+ if (GAR_TEST_DATA == null) {
+ GAR_TEST_DATA = System.getenv("GAR_TEST_DATA");
+ }
+ Assume.assumeTrue("GAR_TEST_DATA is not set", GAR_TEST_DATA != null);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@graphar.apache.org
For additional commands, e-mail: commits-h...@graphar.apache.org
