imbajin commented on code in PR #683:
URL:
https://github.com/apache/incubator-hugegraph-toolchain/pull/683#discussion_r2464760164
##########
hugegraph-loader/src/main/java/org/apache/hugegraph/loader/HugeGraphLoader.java:
##########
@@ -153,11 +271,288 @@ private void createSchema() {
throw new LoadException("Failed to read schema file '%s'", e,
options.schema);
}
- groovyExecutor.execute(script, client);
+
+ if (!options.shorterIDConfigs.isEmpty()) {
+ for (ShortIdConfig config : options.shorterIDConfigs) {
+ PropertyKey propertyKey =
client.schema().propertyKey(config.getIdFieldName())
+ .ifNotExist()
+
.dataType(config.getIdFieldType())
+ .build();
+ client.schema().addPropertyKey(propertyKey);
+ }
+ groovyExecutor.execute(script, client);
+ List<VertexLabel> vertexLabels =
client.schema().getVertexLabels();
+ for (VertexLabel vertexLabel : vertexLabels) {
+ ShortIdConfig config;
+ if ((config =
options.getShortIdConfig(vertexLabel.name())) != null) {
+ config.setLabelID(vertexLabel.id());
+ IndexLabel indexLabel = client.schema()
+
.indexLabel(config.getVertexLabel() + "By" +
+
config.getIdFieldName())
+
.onV(config.getVertexLabel())
+
.by(config.getIdFieldName())
+ .secondary()
+ .ifNotExist()
+ .build();
+ client.schema().addIndexLabel(indexLabel);
+ }
+ }
+ } else {
+ groovyExecutor.execute(script, client);
+ }
}
+
+ // create schema for Graph Source
+ List<InputStruct> structs = this.mapping.structs();
+ for (InputStruct struct : structs) {
+ if (SourceType.GRAPH.equals(struct.input().type())) {
+ GraphSource graphSouce = (GraphSource) struct.input();
+ if (StringUtils.isEmpty(graphSouce.getPdPeers())) {
+ graphSouce.setPdPeers(this.options.pdPeers);
+ }
+ if (StringUtils.isEmpty(graphSouce.getMetaEndPoints())) {
+ graphSouce.setMetaEndPoints(this.options.metaEndPoints);
+ }
+ if (StringUtils.isEmpty(graphSouce.getCluster())) {
+ graphSouce.setCluster(this.options.cluster);
+ }
+ if (StringUtils.isEmpty(graphSouce.getUsername())) {
+ graphSouce.setUsername(this.options.username);
+ }
+ if (StringUtils.isEmpty(graphSouce.getPassword())) {
+ graphSouce.setPassword(this.options.password);
+ }
+
+ GraphSource graphSource = (GraphSource) struct.input();
+ createGraphSourceSchema(graphSource);
+ }
+ }
+
this.context.updateSchemaCache();
}
+ /**
+ * create schema like graphdb when source is graphdb;
+ *
+ * @param graphSource
+ */
+ private void createGraphSourceSchema(GraphSource graphSource) {
+ try (HugeClient sourceClient = graphSource.createHugeClient();
+ HugeClient client = HugeClientHolder.create(this.options, false))
{
+ createGraphSourceVertexLabel(sourceClient, client, graphSource);
+ createGraphSourceEdgeLabel(sourceClient, client, graphSource);
+ createGraphSourceIndexLabel(sourceClient, client, graphSource);
+ } catch (Exception e) {
+ LOG.error("Failed to create graph source schema for {}: {}",
+ graphSource.getGraph(), e.getMessage(), e);
+ throw new LoadException("Schema creation failed", e);
+ }
+ }
+
+ // handles labels (can be used for both VertexLabel and EdgeLabel)
+ private void createGraphSourceLabels(
+ HugeClient sourceClient,
+ HugeClient targetClient,
+ List<? extends SchemaLabel> labels, // VertexLabel or EdgeLabel
+ Map<String, GraphSource.SeletedLabelDes> selectedMap,
+ Map<String, GraphSource.IgnoredLabelDes> ignoredMap,
+ boolean isVertex) {
+
+ for (SchemaLabel label : labels) {
+ if (ignoredMap.containsKey(label.name())) {
+ GraphSource.IgnoredLabelDes des
+ = ignoredMap.get(label.name());
+
+ if (des.getProperties() != null) {
+ des.getProperties()
+ .forEach((p) -> label.properties().remove(p));
+ }
+ }
+
+ Set<String> existedPKs =
+ targetClient.schema().getPropertyKeys().stream()
+ .map(pk ->
pk.name()).collect(Collectors.toSet());
+
+ for (String pkName : label.properties()) {
+ PropertyKey pk = sourceClient.schema()
+ .getPropertyKey(pkName);
+ if (!existedPKs.contains(pk.name())) {
+ targetClient.schema().addPropertyKey(pk);
+ }
+ }
+
+ if (isVertex) {
+ if (!(label instanceof VertexLabel)) {
+ throw new IllegalArgumentException("Expected VertexLabel
but got " + label.getClass());
+ }
+ targetClient.schema().addVertexLabel((VertexLabel) label);
+ } else {
+ if (!(label instanceof EdgeLabel)) {
+ throw new IllegalArgumentException("Expected EdgeLabel but
got " + label.getClass());
+ }
+ targetClient.schema().addEdgeLabel((EdgeLabel) label);
+ }
+ }
+ }
+
+ private void createGraphSourceVertexLabel(HugeClient sourceClient,
+ HugeClient targetClient,
+ GraphSource graphSource) {
+
+ sourceClient.assignGraph(graphSource.getGraphSpace(),
+ graphSource.getGraph());
+
+ // Create Vertex Schema
+ List<VertexLabel> vertexLabels = new ArrayList<>();
+ if (graphSource.getSelectedVertices() != null) {
+ List<String> selectedVertexLabels =
+ graphSource.getSelectedVertices()
+ .stream().map((des) -> des.getLabel())
+ .collect(Collectors.toList());
+
+ if (!CollectionUtils.isEmpty(selectedVertexLabels)) {
+ vertexLabels =
Review Comment:
⚠️ **Important: Graph-to-Graph schema 同步缺少完整性验证**
在 `createGraphSourceLabels()` 中:
```java
Set<String> existedPKs =
targetClient.schema().getPropertyKeys().stream()
.map(pk -> pk.name()).collect(Collectors.toSet());
for (String pkName : label.properties()) {
PropertyKey pk = sourceClient.schema().getPropertyKey(pkName);
if (!existedPKs.contains(pk.name())) {
targetClient.schema().addPropertyKey(pk);
}
}
```
问题:
1. 只检查了 PropertyKey 的名称,没有验证数据类型是否一致
2. 如果目标图中已存在同名但类型不同的 PropertyKey,会导致数据不兼容
3. 没有处理 PropertyKey 的其他属性(如 Cardinality)
**建议:**
- 比对 PropertyKey 的完整定义,包括 dataType、cardinality 等
- 如果存在不兼容的 schema,给出明确的错误提示
- 考虑添加强制覆盖选项
##########
hugegraph-loader/src/main/java/org/apache/hugegraph/loader/HugeGraphLoader.java:
##########
@@ -153,11 +271,288 @@ private void createSchema() {
throw new LoadException("Failed to read schema file '%s'", e,
options.schema);
}
- groovyExecutor.execute(script, client);
+
+ if (!options.shorterIDConfigs.isEmpty()) {
+ for (ShortIdConfig config : options.shorterIDConfigs) {
+ PropertyKey propertyKey =
client.schema().propertyKey(config.getIdFieldName())
+ .ifNotExist()
+
.dataType(config.getIdFieldType())
+ .build();
+ client.schema().addPropertyKey(propertyKey);
+ }
+ groovyExecutor.execute(script, client);
+ List<VertexLabel> vertexLabels =
client.schema().getVertexLabels();
+ for (VertexLabel vertexLabel : vertexLabels) {
+ ShortIdConfig config;
+ if ((config =
options.getShortIdConfig(vertexLabel.name())) != null) {
+ config.setLabelID(vertexLabel.id());
+ IndexLabel indexLabel = client.schema()
+
.indexLabel(config.getVertexLabel() + "By" +
+
config.getIdFieldName())
+
.onV(config.getVertexLabel())
+
.by(config.getIdFieldName())
+ .secondary()
+ .ifNotExist()
+ .build();
+ client.schema().addIndexLabel(indexLabel);
+ }
+ }
+ } else {
+ groovyExecutor.execute(script, client);
+ }
}
+
+ // create schema for Graph Source
+ List<InputStruct> structs = this.mapping.structs();
+ for (InputStruct struct : structs) {
+ if (SourceType.GRAPH.equals(struct.input().type())) {
+ GraphSource graphSouce = (GraphSource) struct.input();
+ if (StringUtils.isEmpty(graphSouce.getPdPeers())) {
+ graphSouce.setPdPeers(this.options.pdPeers);
+ }
+ if (StringUtils.isEmpty(graphSouce.getMetaEndPoints())) {
+ graphSouce.setMetaEndPoints(this.options.metaEndPoints);
+ }
+ if (StringUtils.isEmpty(graphSouce.getCluster())) {
+ graphSouce.setCluster(this.options.cluster);
+ }
+ if (StringUtils.isEmpty(graphSouce.getUsername())) {
+ graphSouce.setUsername(this.options.username);
+ }
+ if (StringUtils.isEmpty(graphSouce.getPassword())) {
+ graphSouce.setPassword(this.options.password);
+ }
+
+ GraphSource graphSource = (GraphSource) struct.input();
+ createGraphSourceSchema(graphSource);
+ }
+ }
+
this.context.updateSchemaCache();
}
+ /**
+ * create schema like graphdb when source is graphdb;
+ *
+ * @param graphSource
+ */
+ private void createGraphSourceSchema(GraphSource graphSource) {
+ try (HugeClient sourceClient = graphSource.createHugeClient();
+ HugeClient client = HugeClientHolder.create(this.options, false))
{
+ createGraphSourceVertexLabel(sourceClient, client, graphSource);
+ createGraphSourceEdgeLabel(sourceClient, client, graphSource);
+ createGraphSourceIndexLabel(sourceClient, client, graphSource);
+ } catch (Exception e) {
+ LOG.error("Failed to create graph source schema for {}: {}",
+ graphSource.getGraph(), e.getMessage(), e);
+ throw new LoadException("Schema creation failed", e);
+ }
+ }
+
+ // handles labels (can be used for both VertexLabel and EdgeLabel)
+ private void createGraphSourceLabels(
+ HugeClient sourceClient,
+ HugeClient targetClient,
+ List<? extends SchemaLabel> labels, // VertexLabel or EdgeLabel
+ Map<String, GraphSource.SeletedLabelDes> selectedMap,
+ Map<String, GraphSource.IgnoredLabelDes> ignoredMap,
+ boolean isVertex) {
+
+ for (SchemaLabel label : labels) {
+ if (ignoredMap.containsKey(label.name())) {
+ GraphSource.IgnoredLabelDes des
+ = ignoredMap.get(label.name());
+
+ if (des.getProperties() != null) {
+ des.getProperties()
+ .forEach((p) -> label.properties().remove(p));
+ }
+ }
+
+ Set<String> existedPKs =
+ targetClient.schema().getPropertyKeys().stream()
+ .map(pk ->
pk.name()).collect(Collectors.toSet());
+
+ for (String pkName : label.properties()) {
+ PropertyKey pk = sourceClient.schema()
+ .getPropertyKey(pkName);
+ if (!existedPKs.contains(pk.name())) {
+ targetClient.schema().addPropertyKey(pk);
+ }
+ }
+
+ if (isVertex) {
+ if (!(label instanceof VertexLabel)) {
+ throw new IllegalArgumentException("Expected VertexLabel
but got " + label.getClass());
+ }
+ targetClient.schema().addVertexLabel((VertexLabel) label);
+ } else {
+ if (!(label instanceof EdgeLabel)) {
+ throw new IllegalArgumentException("Expected EdgeLabel but
got " + label.getClass());
+ }
+ targetClient.schema().addEdgeLabel((EdgeLabel) label);
+ }
+ }
+ }
+
+ private void createGraphSourceVertexLabel(HugeClient sourceClient,
+ HugeClient targetClient,
+ GraphSource graphSource) {
+
+ sourceClient.assignGraph(graphSource.getGraphSpace(),
+ graphSource.getGraph());
+
+ // Create Vertex Schema
+ List<VertexLabel> vertexLabels = new ArrayList<>();
+ if (graphSource.getSelectedVertices() != null) {
+ List<String> selectedVertexLabels =
+ graphSource.getSelectedVertices()
+ .stream().map((des) -> des.getLabel())
+ .collect(Collectors.toList());
+
+ if (!CollectionUtils.isEmpty(selectedVertexLabels)) {
+ vertexLabels =
+ sourceClient.schema()
+ .getVertexLabels(selectedVertexLabels);
+ }
+ } else {
+ vertexLabels = sourceClient.schema().getVertexLabels();
+ }
+
+ Map<String, GraphSource.SeletedLabelDes> mapSelectedVertices
+ = new HashMap<>();
+ if (graphSource.getSelectedVertices() != null) {
+ for (GraphSource.SeletedLabelDes des :
+ graphSource.getSelectedVertices()) {
+ mapSelectedVertices.put(des.getLabel(), des);
+ }
+ }
+
+ for (VertexLabel label : vertexLabels) {
+ if (mapSelectedVertices.getOrDefault(label.name(),
+ null) != null) {
+ List<String> selectedProperties = mapSelectedVertices.get(
+ label.name()).getProperties();
+
+ if (selectedProperties != null) {
+ label.properties().clear();
Review Comment:
🧹 **Minor: 代码重复 - createGraphSourceLabels 可以进一步简化**
在 `createGraphSourceVertexLabel` 和 `createGraphSourceEdgeLabel` 中存在大量重复代码:
```java
Map<String, GraphSource.SeletedLabelDes> mapSelectedVertices = new
HashMap<>();
if (graphSource.getSelectedVertices() != null) {
for (GraphSource.SeletedLabelDes des :
graphSource.getSelectedVertices()) {
mapSelectedVertices.put(des.getLabel(), des);
}
}
```
**建议:**
提取公共方法:
```java
private <T extends GraphSource.LabelDes> Map<String, T> toLabelMap(List<T>
labels) {
if (labels == null) return Collections.emptyMap();
return labels.stream().collect(Collectors.toMap(
GraphSource.LabelDes::getLabel,
Function.identity()
));
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
