javeme commented on code in PR #358:
URL:
https://github.com/apache/incubator-hugegraph-toolchain/pull/358#discussion_r1015432205
##########
hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/EdgeBuilder.java:
##########
@@ -68,7 +69,7 @@ public EdgeBuilder(LoadContext context, InputStruct struct,
public EdgeMapping mapping() {
return this.mapping;
}
-
+
Review Comment:
also remove the spaces " "?
##########
hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/EdgeBuilder.java:
##########
@@ -109,6 +110,53 @@ public List<Edge> build(String[] names, Object[] values) {
}
return edges;
}
+
+ @Override
+ public List<Edge> build( Row row) {
+ String[] names = row.schema().fieldNames();
+ Object[] values = new Object[row.size()];
+ for (int i = 0; i < row.size(); i++) {
+ values[i] = row.get(i);
+ }
+ if (this.vertexIdsIndex == null ||
+ !Arrays.equals(this.lastNames, names)) {
Review Comment:
let "!Arrays" align with "this"
##########
hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/VertexBuilder.java:
##########
@@ -50,7 +51,6 @@ public VertexBuilder(LoadContext context, InputStruct struct,
public VertexMapping mapping() {
return this.mapping;
}
-
Review Comment:
keep the blank line
##########
hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/EdgeBuilder.java:
##########
@@ -109,6 +110,53 @@ public List<Edge> build(String[] names, Object[] values) {
}
return edges;
}
+
+ @Override
+ public List<Edge> build( Row row) {
+ String[] names = row.schema().fieldNames();
+ Object[] values = new Object[row.size()];
+ for (int i = 0; i < row.size(); i++) {
+ values[i] = row.get(i);
+ }
+ if (this.vertexIdsIndex == null ||
+ !Arrays.equals(this.lastNames, names)) {
+ this.vertexIdsIndex = this.extractVertexIdsIndex(names);
+ }
+
+ this.lastNames = names;
+ EdgeKVPairs kvPairs = this.newEdgeKVPairs();
+ kvPairs.source.extractFromEdge(names, values,
+ this.vertexIdsIndex.sourceIndexes);
+ kvPairs.target.extractFromEdge(names, values,
+ this.vertexIdsIndex.targetIndexes);
+ kvPairs.extractProperties(names, values);
+
+ List<Vertex> sources = kvPairs.source.buildVertices(false);
+ List<Vertex> targets = kvPairs.target.buildVertices(false);
+ if (sources.isEmpty() || targets.isEmpty()) {
+ return ImmutableList.of();
+ }
+ E.checkArgument(sources.size() == 1 || targets.size() == 1 ||
+ sources.size() == targets.size(),
+ "The elements number of source and target must be: " +
Review Comment:
align with sources
##########
hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/DirectLoader.java:
##########
@@ -0,0 +1,84 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.loader.direct.loader;
+
+import com.baidu.hugegraph.loader.builder.EdgeBuilder;
+import com.baidu.hugegraph.loader.builder.ElementBuilder;
+import com.baidu.hugegraph.loader.builder.VertexBuilder;
+import com.baidu.hugegraph.loader.executor.LoadContext;
+import com.baidu.hugegraph.loader.executor.LoadOptions;
+import com.baidu.hugegraph.loader.mapping.EdgeMapping;
+import com.baidu.hugegraph.loader.mapping.InputStruct;
+import com.baidu.hugegraph.loader.mapping.VertexMapping;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+
+import java.io.Serializable;
+import java.util.LinkedList;
+import java.util.List;
+
+
Review Comment:
one blank line is ok
##########
hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/HBaseDirectLoader.java:
##########
@@ -0,0 +1,264 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.loader.direct.loader;
+
+import com.baidu.hugegraph.loader.builder.ElementBuilder;
+import com.baidu.hugegraph.loader.constant.Constants;
+import com.baidu.hugegraph.loader.direct.util.SinkToHBase;
+import com.baidu.hugegraph.loader.executor.LoadOptions;
+import com.baidu.hugegraph.loader.mapping.ElementMapping;
+import com.baidu.hugegraph.loader.mapping.InputStruct;
+import com.baidu.hugegraph.loader.metrics.LoadDistributeMetrics;
+import com.baidu.hugegraph.loader.util.HugeClientHolder;
+import com.baidu.hugegraph.serializer.direct.HBaseSerializer;
+import com.baidu.hugegraph.structure.GraphElement;
+import com.baidu.hugegraph.structure.graph.Edge;
+import com.baidu.hugegraph.structure.graph.Vertex;
+import com.baidu.hugegraph.util.Log;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsShell;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.spark.Partitioner;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.function.PairFlatMapFunction;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.slf4j.Logger;
+import scala.Tuple2;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+
+public class HBaseDirectLoader extends DirectLoader<ImmutableBytesWritable,
KeyValue> {
+
+ private SinkToHBase sinkToHBase ;
+ private LoadDistributeMetrics loadDistributeMetrics;;
+
+ public static final Logger LOG = Log.logger(HBaseDirectLoader.class);
+
+ public HBaseDirectLoader(LoadOptions loadOptions, InputStruct struct,
LoadDistributeMetrics loadDistributeMetrics) {
+ super(loadOptions,struct);
+ this.loadDistributeMetrics=loadDistributeMetrics;
+ this.sinkToHBase=new SinkToHBase(loadOptions);
+
+ }
+
+ public String getTableName(){
+
+ String tableName = null;
+ if (struct.edges().size() > 0) {
+ tableName = this.loadOptions.edgeTablename;
+
+ } else if (struct.vertices().size() > 0) {
+ tableName = this.loadOptions.vertexTablename;
+
+ }
+ return tableName;
+ }
+
+ public Integer getTablePartitions(){
+ return struct.edges().size() > 0 ? loadOptions.edgePartitions
Review Comment:
keep one space in "return struct"
##########
hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/VertexBuilder.java:
##########
@@ -59,6 +59,21 @@ public List<Vertex> build(String[] names, Object[] values) {
return kvPairs.buildVertices(true);
}
+ @Override
+ public List<Vertex> build(Row row) {
+
Review Comment:
remove the unused blank line
##########
hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/VertexBuilder.java:
##########
@@ -59,6 +59,21 @@ public List<Vertex> build(String[] names, Object[] values) {
return kvPairs.buildVertices(true);
}
+ @Override
+ public List<Vertex> build(Row row) {
+
+ VertexKVPairs kvPairs = this.newKVPairs(this.vertexLabel,
+ this.mapping.unfold());
+ String[] names = row.schema().fieldNames();
+ Object[] values = new Object[row.size()];
+ for (int i = 0; i < row.size(); i++) {
+ values[i] = row.get(i);
+ }
+ kvPairs.extractFromVertex(names, values);
+ return kvPairs.buildVertices(true);
+
Review Comment:
remove the unused blank line
##########
hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/HBaseDirectLoader.java:
##########
@@ -0,0 +1,264 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.loader.direct.loader;
+
+import com.baidu.hugegraph.loader.builder.ElementBuilder;
+import com.baidu.hugegraph.loader.constant.Constants;
+import com.baidu.hugegraph.loader.direct.util.SinkToHBase;
+import com.baidu.hugegraph.loader.executor.LoadOptions;
+import com.baidu.hugegraph.loader.mapping.ElementMapping;
+import com.baidu.hugegraph.loader.mapping.InputStruct;
+import com.baidu.hugegraph.loader.metrics.LoadDistributeMetrics;
+import com.baidu.hugegraph.loader.util.HugeClientHolder;
+import com.baidu.hugegraph.serializer.direct.HBaseSerializer;
+import com.baidu.hugegraph.structure.GraphElement;
+import com.baidu.hugegraph.structure.graph.Edge;
+import com.baidu.hugegraph.structure.graph.Vertex;
+import com.baidu.hugegraph.util.Log;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsShell;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.spark.Partitioner;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.function.PairFlatMapFunction;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.slf4j.Logger;
+import scala.Tuple2;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+
+public class HBaseDirectLoader extends DirectLoader<ImmutableBytesWritable,
KeyValue> {
+
+ private SinkToHBase sinkToHBase ;
+ private LoadDistributeMetrics loadDistributeMetrics;;
+
+ public static final Logger LOG = Log.logger(HBaseDirectLoader.class);
+
+ public HBaseDirectLoader(LoadOptions loadOptions, InputStruct struct,
LoadDistributeMetrics loadDistributeMetrics) {
+ super(loadOptions,struct);
+ this.loadDistributeMetrics=loadDistributeMetrics;
+ this.sinkToHBase=new SinkToHBase(loadOptions);
+
+ }
+
+ public String getTableName(){
+
+ String tableName = null;
+ if (struct.edges().size() > 0) {
+ tableName = this.loadOptions.edgeTablename;
+
+ } else if (struct.vertices().size() > 0) {
+ tableName = this.loadOptions.vertexTablename;
+
+ }
+ return tableName;
+ }
+
+ public Integer getTablePartitions(){
+ return struct.edges().size() > 0 ? loadOptions.edgePartitions
+ : loadOptions.vertexPartitions;
Review Comment:
align with "?"
##########
hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/DirectLoader.java:
##########
@@ -0,0 +1,84 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.loader.direct.loader;
+
+import com.baidu.hugegraph.loader.builder.EdgeBuilder;
+import com.baidu.hugegraph.loader.builder.ElementBuilder;
+import com.baidu.hugegraph.loader.builder.VertexBuilder;
+import com.baidu.hugegraph.loader.executor.LoadContext;
+import com.baidu.hugegraph.loader.executor.LoadOptions;
+import com.baidu.hugegraph.loader.mapping.EdgeMapping;
+import com.baidu.hugegraph.loader.mapping.InputStruct;
+import com.baidu.hugegraph.loader.mapping.VertexMapping;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+
+import java.io.Serializable;
+import java.util.LinkedList;
+import java.util.List;
+
+
+public abstract class DirectLoader<T,R> implements Serializable {
+ LoadOptions loadOptions ;
+ InputStruct struct;
+
+
Review Comment:
one blank line is ok
##########
hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/DirectLoader.java:
##########
@@ -0,0 +1,84 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.loader.direct.loader;
+
+import com.baidu.hugegraph.loader.builder.EdgeBuilder;
+import com.baidu.hugegraph.loader.builder.ElementBuilder;
+import com.baidu.hugegraph.loader.builder.VertexBuilder;
+import com.baidu.hugegraph.loader.executor.LoadContext;
+import com.baidu.hugegraph.loader.executor.LoadOptions;
+import com.baidu.hugegraph.loader.mapping.EdgeMapping;
+import com.baidu.hugegraph.loader.mapping.InputStruct;
+import com.baidu.hugegraph.loader.mapping.VertexMapping;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+
+import java.io.Serializable;
+import java.util.LinkedList;
+import java.util.List;
+
+
+public abstract class DirectLoader<T,R> implements Serializable {
+ LoadOptions loadOptions ;
+ InputStruct struct;
+
+
+ public DirectLoader(LoadOptions loadOptions,
+ InputStruct struct) {
+ this.loadOptions = loadOptions;
+ this.struct=struct;
+ }
+
+
+
+
Review Comment:
one blank line is ok
##########
hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/direct/loader/DirectLoader.java:
##########
@@ -0,0 +1,84 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.loader.direct.loader;
+
+import com.baidu.hugegraph.loader.builder.EdgeBuilder;
+import com.baidu.hugegraph.loader.builder.ElementBuilder;
+import com.baidu.hugegraph.loader.builder.VertexBuilder;
+import com.baidu.hugegraph.loader.executor.LoadContext;
+import com.baidu.hugegraph.loader.executor.LoadOptions;
+import com.baidu.hugegraph.loader.mapping.EdgeMapping;
+import com.baidu.hugegraph.loader.mapping.InputStruct;
+import com.baidu.hugegraph.loader.mapping.VertexMapping;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+
+import java.io.Serializable;
+import java.util.LinkedList;
+import java.util.List;
+
+
+public abstract class DirectLoader<T,R> implements Serializable {
+ LoadOptions loadOptions ;
+ InputStruct struct;
+
+
+ public DirectLoader(LoadOptions loadOptions,
+ InputStruct struct) {
+ this.loadOptions = loadOptions;
+ this.struct=struct;
+ }
+
+
+
+
+ public final void bulkload(Dataset<Row> ds){
+ JavaPairRDD<T, R> javaPairRDD = buildVertexAndEdge(ds);
+ String path = generateFiles(javaPairRDD);
+ loadFiles(path);
+ };
+
+
+ protected List<ElementBuilder> getElementBuilders(){
+ LoadContext context = new LoadContext(loadOptions);
+ context.schemaCache().updateAll();
+ List<ElementBuilder> buildersForGraphElement = new LinkedList<>();
+ for (VertexMapping vertexMapping : struct.vertices()) {
+ buildersForGraphElement.add(
+ new VertexBuilder(context, struct, vertexMapping)
+ );
+ }
+ for (EdgeMapping edgeMapping : struct.edges()) {
+ buildersForGraphElement.add(new EdgeBuilder(context, struct,
edgeMapping));
+ }
+ context.close();
+ return buildersForGraphElement;
+ }
+
+
+ abstract JavaPairRDD<T, R> buildVertexAndEdge(Dataset<Row> ds);
+
+
+
Review Comment:
one blank line is ok
##########
hugegraph-loader/src/main/java/com/baidu/hugegraph/loader/builder/VertexBuilder.java:
##########
@@ -59,6 +59,21 @@ public List<Vertex> build(String[] names, Object[] values) {
return kvPairs.buildVertices(true);
}
+ @Override
+ public List<Vertex> build(Row row) {
+
+ VertexKVPairs kvPairs = this.newKVPairs(this.vertexLabel,
+ this.mapping.unfold());
Review Comment:
align with this.vertexLabel
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
