Repository: metamodel Updated Branches: refs/heads/master fe02a59b8 -> 71b57403b
Neo4j: column types support. Project: http://git-wip-us.apache.org/repos/asf/metamodel/repo Commit: http://git-wip-us.apache.org/repos/asf/metamodel/commit/fdaaed05 Tree: http://git-wip-us.apache.org/repos/asf/metamodel/tree/fdaaed05 Diff: http://git-wip-us.apache.org/repos/asf/metamodel/diff/fdaaed05 Branch: refs/heads/master Commit: fdaaed05d7ed3aadfe79b3ce71dd49bc45b6b9f4 Parents: fe02a59 Author: jakub <[email protected]> Authored: Fri May 18 09:49:48 2018 +0200 Committer: jakub <[email protected]> Committed: Fri May 18 09:49:48 2018 +0200 ---------------------------------------------------------------------- .../metamodel/neo4j/Neo4jDataContext.java | 210 +++++++++++++++---- .../apache/metamodel/neo4j/Neo4jDataSet.java | 26 ++- 2 files changed, 181 insertions(+), 55 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metamodel/blob/fdaaed05/neo4j/src/main/java/org/apache/metamodel/neo4j/Neo4jDataContext.java ---------------------------------------------------------------------- diff --git a/neo4j/src/main/java/org/apache/metamodel/neo4j/Neo4jDataContext.java b/neo4j/src/main/java/org/apache/metamodel/neo4j/Neo4jDataContext.java index 00d0b5a..70686ef 100644 --- a/neo4j/src/main/java/org/apache/metamodel/neo4j/Neo4jDataContext.java +++ b/neo4j/src/main/java/org/apache/metamodel/neo4j/Neo4jDataContext.java @@ -19,6 +19,8 @@ package org.apache.metamodel.neo4j; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; import java.util.LinkedHashSet; import java.util.List; import java.util.Set; @@ -36,6 +38,7 @@ import org.apache.metamodel.data.DocumentSource; import org.apache.metamodel.query.FilterItem; import org.apache.metamodel.query.SelectItem; import org.apache.metamodel.schema.Column; +import org.apache.metamodel.schema.ColumnType; import org.apache.metamodel.schema.MutableSchema; import org.apache.metamodel.schema.MutableTable; import org.apache.metamodel.schema.Schema; @@ -158,59 +161,178 @@ public class Neo4jDataContext extends QueryPostprocessDataContext implements Dat } public SimpleTableDef[] detectTableDefs() { - List<SimpleTableDef> tableDefs = new ArrayList<SimpleTableDef>(); - - String labelsJsonString = _requestWrapper.executeRestRequest(new HttpGet(_serviceRoot + "/labels")); - - JSONArray labelsJsonArray; + final List<SimpleTableDef> tableDefs = new ArrayList<>(); + final String labelsJsonString = _requestWrapper.executeRestRequest(new HttpGet(_serviceRoot + "/labels")); + final JSONArray labelsJsonArray; + try { labelsJsonArray = new JSONArray(labelsJsonString); + for (int i = 0; i < labelsJsonArray.length(); i++) { - String label = labelsJsonArray.getString(i); + fillTableDefsFromLabel(labelsJsonArray.getString(i), tableDefs); + } + + return tableDefs.toArray(new SimpleTableDef[tableDefs.size()]); + } catch (final JSONException e) { + logger.error("Error occured in parsing JSON while detecting the schema: ", e); + throw new IllegalStateException(e); + } + } + + private void fillTableDefsFromLabel(final String label, final List<SimpleTableDef> tableDefs) throws JSONException { + final List<JSONObject> nodesPerLabel = getAllNodesPerLabel(label); + final List<String> propertiesPerLabel = getPropertiesFromLabelNodes(nodesPerLabel); + final Set<String> relationshipPropertiesPerLabel = new LinkedHashSet<>(); + + for (final JSONObject node : nodesPerLabel) { + final Long nodeId = (Long) node.getJSONObject("metadata").get("id"); + fillRelationshipPropertiesPerLabel(nodeId, relationshipPropertiesPerLabel); + } + + propertiesPerLabel.addAll(relationshipPropertiesPerLabel); + + // Do not add a table if label has no nodes (empty tables are considered non-existent) + if (!nodesPerLabel.isEmpty()) { + final String[] columnNames = propertiesPerLabel.toArray(new String[propertiesPerLabel.size()]); + final ColumnType[] columnTypes = guessColumnTypesFromValues(nodesPerLabel.get(0), + new ArrayList<>(Arrays.asList(columnNames))); + final SimpleTableDef tableDef = new SimpleTableDef(label, columnNames, columnTypes); + tableDefs.add(tableDef); + } + } - List<JSONObject> nodesPerLabel = getAllNodesPerLabel(label); + private void fillRelationshipPropertiesPerLabel(final Long nodeId, final Set<String> relationshipPropertiesPerLabel) + throws JSONException { + final List<JSONObject> relationshipsPerNode = getOutgoingRelationshipsPerNode(nodeId); + + for (final JSONObject relationship : relationshipsPerNode) { + // Add the relationship as a column in the table + final String relationshipName = relationship.getString("type"); + final String relationshipNameProperty = RELATIONSHIP_PREFIX + relationshipName; + + if (!relationshipPropertiesPerLabel.contains(relationshipNameProperty)) { + relationshipPropertiesPerLabel.add(relationshipNameProperty); + } - List<String> propertiesPerLabel = new ArrayList<String>(); - for (JSONObject node : nodesPerLabel) { - List<String> propertiesPerNode = getAllPropertiesPerNode(node); - for (String property : propertiesPerNode) { - if (!propertiesPerLabel.contains(property)) { - propertiesPerLabel.add(property); - } - } + // Add all the relationship properties as table columns + final List<String> propertiesPerRelationship = getAllPropertiesPerRelationship(relationship); + relationshipPropertiesPerLabel.addAll(propertiesPerRelationship); + } + } + + private List<String> getPropertiesFromLabelNodes(final List<JSONObject> nodesPerLabel) { + final List<String> propertiesPerLabel = new ArrayList<>(); + + for (final JSONObject node : nodesPerLabel) { + final List<String> propertiesPerNode = getAllPropertiesPerNode(node); + + for (final String property : propertiesPerNode) { + if (!propertiesPerLabel.contains(property)) { + propertiesPerLabel.add(property); } + } + } + + return propertiesPerLabel; + } - Set<String> relationshipPropertiesPerLabel = new LinkedHashSet<String>(); - for (JSONObject node : nodesPerLabel) { - Integer nodeId = (Integer) node.getJSONObject("metadata").get("id"); - List<JSONObject> relationshipsPerNode = getOutgoingRelationshipsPerNode(nodeId); - for (JSONObject relationship : relationshipsPerNode) { - // Add the relationship as a column in the table - String relationshipName = relationship.getString("type"); - String relationshipNameProperty = RELATIONSHIP_PREFIX + relationshipName; - if (!relationshipPropertiesPerLabel.contains(relationshipNameProperty)) { - relationshipPropertiesPerLabel.add(relationshipNameProperty); - } + private ColumnType[] guessColumnTypesFromValues(final JSONObject jsonObject, final List<String> columnNames) { + final List<ColumnType> columnTypes = new ArrayList<>(); + + try { + fillColumnTypesFromMetadata(jsonObject, columnNames, columnTypes); + fillColumnTypesFromData(jsonObject, columnNames, columnTypes); + } catch (final JSONException e) { + // ignore missing data + } + + fillColumnTypesFromRemainingColumns(columnNames, columnTypes); + return columnTypes.toArray(new ColumnType[columnTypes.size()]); + } - // Add all the relationship properties as table columns - List<String> propertiesPerRelationship = getAllPropertiesPerRelationship(relationship); - relationshipPropertiesPerLabel.addAll(propertiesPerRelationship); - } + private void fillColumnTypesFromData(final JSONObject jsonObject, final List<String> columnNames, + final List<ColumnType> columnTypes) throws JSONException { + final String dataKey = "data"; + + if (jsonObject.has(dataKey)) { + final JSONObject data = jsonObject.getJSONObject(dataKey); + final Iterator keysIterator = data.keys(); + + while (keysIterator.hasNext()) { + final String key = (String) keysIterator.next(); + final ColumnType type = getTypeFromValue(data, key); + columnTypes.add(type); + removeIfAvailable(columnNames, key); + } + } + } + + private void fillColumnTypesFromMetadata(final JSONObject jsonObject, final List<String> columnNames, + final List<ColumnType> columnTypes) throws JSONException { + final String metadataKey = "metadata"; + + if (jsonObject.has(metadataKey)) { + final JSONObject metadata = jsonObject.getJSONObject(metadataKey); + + if (metadata.has("id")) { + columnTypes.add(ColumnType.BIGINT); + removeIfAvailable(columnNames, "_id"); + } + } + } + + private void fillColumnTypesFromRemainingColumns(final List<String> columnNames, + final List<ColumnType> columnTypes) { + for (final String remainingColumnName : columnNames) { + if (remainingColumnName.contains("rel_")) { + if (remainingColumnName.contains("#")) { + columnTypes.add(ColumnType.ARRAY); + } else { + columnTypes.add(ColumnType.BIGINT); } - propertiesPerLabel.addAll(relationshipPropertiesPerLabel); - - // Do not add a table if label has no nodes (empty tables are - // considered non-existent) - if (!nodesPerLabel.isEmpty()) { - SimpleTableDef tableDef = new SimpleTableDef(label, - propertiesPerLabel.toArray(new String[propertiesPerLabel.size()])); - tableDefs.add(tableDef); + } else { + columnTypes.add(ColumnType.STRING); + } + } + } + + private void removeIfAvailable(final List<String> list, final String key) { + if (list.contains(key)) { + list.remove(key); + } + } + + private ColumnType getTypeFromValue(final JSONObject data, final String key) { + try { + data.getBoolean(key); + return ColumnType.BOOLEAN; + } catch (final JSONException e1) { + try { + data.getInt(key); + return ColumnType.INTEGER; + } catch (final JSONException e2) { + try { + data.getLong(key); + return ColumnType.BIGINT; + } catch (final JSONException e3) { + try { + data.getDouble(key); + return ColumnType.DOUBLE; + } catch (final JSONException e4) { + try { + data.getJSONArray(key); + return ColumnType.ARRAY; + } catch (final JSONException e5) { + try { + data.getJSONObject(key); + return ColumnType.MAP; + } catch (final JSONException e6) { + return ColumnType.STRING; + } + } + } } } - return tableDefs.toArray(new SimpleTableDef[tableDefs.size()]); - } catch (JSONException e) { - logger.error("Error occured in parsing JSON while detecting the schema: ", e); - throw new IllegalStateException(e); } } @@ -236,8 +358,8 @@ public class Neo4jDataContext extends QueryPostprocessDataContext implements Dat } } - private List<JSONObject> getOutgoingRelationshipsPerNode(Integer nodeId) { - List<JSONObject> outgoingRelationshipsPerNode = new ArrayList<JSONObject>(); + private List<JSONObject> getOutgoingRelationshipsPerNode(final Long nodeId) { + List<JSONObject> outgoingRelationshipsPerNode = new ArrayList<>(); String outgoingRelationshipsPerNodeJsonString = _requestWrapper.executeRestRequest(new HttpGet(_serviceRoot + "/node/" + nodeId + "/relationships/out")); http://git-wip-us.apache.org/repos/asf/metamodel/blob/fdaaed05/neo4j/src/main/java/org/apache/metamodel/neo4j/Neo4jDataSet.java ---------------------------------------------------------------------- diff --git a/neo4j/src/main/java/org/apache/metamodel/neo4j/Neo4jDataSet.java b/neo4j/src/main/java/org/apache/metamodel/neo4j/Neo4jDataSet.java index 537a068..6c1af50 100644 --- a/neo4j/src/main/java/org/apache/metamodel/neo4j/Neo4jDataSet.java +++ b/neo4j/src/main/java/org/apache/metamodel/neo4j/Neo4jDataSet.java @@ -44,28 +44,32 @@ final class Neo4jDataSet extends AbstractDataSet { @Override public boolean next() { try { - JSONArray resultsArray = _resultJSONObject.getJSONArray("results"); + final JSONArray resultsArray = _resultJSONObject.getJSONArray("results"); + if (resultsArray.length() > 0) { - JSONObject results = resultsArray.getJSONObject(0); - JSONArray data = results.getJSONArray("data"); + final JSONObject results = resultsArray.getJSONObject(0); + final JSONArray data = results.getJSONArray("data"); + if (_currentRowIndex < data.length()) { - JSONObject row = data.getJSONObject(_currentRowIndex); - JSONArray jsonValues = row.getJSONArray("row"); - - Object[] objectValues = new Object[jsonValues.length()]; + final JSONObject row = data.getJSONObject(_currentRowIndex); + final JSONArray jsonValues = row.getJSONArray("row"); + final Object[] objectValues = new Object[jsonValues.length()]; + for (int i = 0; i < jsonValues.length(); i++) { - objectValues[i] = jsonValues.getString(i); + objectValues[i] = jsonValues.get(i); } + _row = new DefaultRow(new SimpleDataSetHeader(getSelectItems()), objectValues); _currentRowIndex++; + return true; } } else { - JSONArray errorArray = _resultJSONObject.getJSONArray("errors"); - JSONObject error = errorArray.getJSONObject(0); + final JSONArray errorArray = _resultJSONObject.getJSONArray("errors"); + final JSONObject error = errorArray.getJSONObject(0); throw new IllegalStateException(error.toString()); } - } catch (JSONException e) { + } catch (final JSONException e) { throw new IllegalStateException(e); } return false;
