This is an automated email from the ASF dual-hosted git repository.
wyk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 67a249221f [ASTERIXDB-3430] Log schema as a one-liner JSON
67a249221f is described below
commit 67a249221f88c66ed0cf1f6faa9bca1cf67c1b39
Author: preetham0202 <[email protected]>
AuthorDate: Mon Jun 17 23:27:57 2024 +0530
[ASTERIXDB-3430] Log schema as a one-liner JSON
Change-Id: I9c8f6d1739a3e5aafb1ead1ae6c19417c6f0a060
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18374
Reviewed-by: Wail Alkowaileet <[email protected]>
Integration-Tests: Jenkins <[email protected]>
Tested-by: Jenkins <[email protected]>
---
.../operation/lsm/flush/FlushColumnMetadata.java | 8 +-
.../operation/query/QueryColumnMetadata.java | 6 +-
.../column/util/SchemaJSONBuilderVisitor.java | 138 +++++++++++++++++++++
3 files changed, 145 insertions(+), 7 deletions(-)
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnMetadata.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnMetadata.java
index 268620468e..f5146386d0 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnMetadata.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnMetadata.java
@@ -47,7 +47,7 @@ import
org.apache.asterix.column.metadata.schema.primitive.PrimitiveSchemaNode;
import
org.apache.asterix.column.metadata.schema.visitor.SchemaBuilderFromIATypeVisitor;
import org.apache.asterix.column.util.ColumnValuesUtil;
import org.apache.asterix.column.util.RunLengthIntArray;
-import org.apache.asterix.column.util.SchemaStringBuilderVisitor;
+import org.apache.asterix.column.util.SchemaJSONBuilderVisitor;
import org.apache.asterix.column.values.IColumnValuesWriter;
import org.apache.asterix.column.values.IColumnValuesWriterFactory;
import org.apache.asterix.column.values.writer.AbstractColumnValuesWriter;
@@ -582,12 +582,12 @@ public final class FlushColumnMetadata extends
AbstractColumnMetadata {
return;
}
// This should be a low frequency object creation
- SchemaStringBuilderVisitor schemaBuilder = new
SchemaStringBuilderVisitor(fieldNamesDictionary);
+ SchemaJSONBuilderVisitor schemaBuilder = new
SchemaJSONBuilderVisitor(fieldNamesDictionary);
String recordSchema =
LogRedactionUtil.userData(schemaBuilder.build(root));
- LOGGER.debug("Schema for {} has changed: \n {}", RECORD_SCHEMA,
recordSchema);
+ LOGGER.debug("Schema for {} has changed: {}", RECORD_SCHEMA,
recordSchema);
if (metaRoot != null) {
String metaRecordSchema =
LogRedactionUtil.userData(schemaBuilder.build(metaRoot));
- LOGGER.debug("Schema for {} has changed: \n {}",
META_RECORD_SCHEMA, metaRecordSchema);
+ LOGGER.debug("Schema for {} has changed: {}", META_RECORD_SCHEMA,
metaRecordSchema);
}
}
}
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnMetadata.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnMetadata.java
index 50287065a8..e507d530a1 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnMetadata.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnMetadata.java
@@ -46,7 +46,7 @@ import
org.apache.asterix.column.metadata.dictionary.AbstractFieldNamesDictionar
import org.apache.asterix.column.metadata.schema.AbstractSchemaNode;
import org.apache.asterix.column.metadata.schema.ObjectSchemaNode;
import org.apache.asterix.column.metadata.schema.visitor.SchemaClipperVisitor;
-import org.apache.asterix.column.util.SchemaStringBuilderVisitor;
+import org.apache.asterix.column.util.SchemaJSONBuilderVisitor;
import org.apache.asterix.column.values.IColumnValuesReader;
import org.apache.asterix.column.values.IColumnValuesReaderFactory;
import org.apache.asterix.column.values.reader.PrimitiveColumnValuesReader;
@@ -273,9 +273,9 @@ public class QueryColumnMetadata extends
AbstractColumnImmutableReadMetadata {
protected static void logSchema(String jobId, ObjectSchemaNode root,
String schemaSource,
IFieldNamesDictionary fieldNamesDictionary) throws
HyracksDataException {
if (jobId != null && LOGGER.isDebugEnabled() && System.nanoTime() %
LOG_PROBABILITY == 0) {
- SchemaStringBuilderVisitor schemaBuilder = new
SchemaStringBuilderVisitor(fieldNamesDictionary);
+ SchemaJSONBuilderVisitor schemaBuilder = new
SchemaJSONBuilderVisitor(fieldNamesDictionary);
String schema =
LogRedactionUtil.userData(schemaBuilder.build(root));
- LOGGER.debug("Queried {} schema [{}]: \n {}", schemaSource, jobId,
schema);
+ LOGGER.debug("Queried {} schema [{}]: {}", schemaSource, jobId,
schema);
}
}
}
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/SchemaJSONBuilderVisitor.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/SchemaJSONBuilderVisitor.java
new file mode 100644
index 0000000000..b425a26453
--- /dev/null
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/SchemaJSONBuilderVisitor.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.util;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.asterix.column.metadata.IFieldNamesDictionary;
+import org.apache.asterix.column.metadata.schema.AbstractSchemaNode;
+import org.apache.asterix.column.metadata.schema.ISchemaNodeVisitor;
+import org.apache.asterix.column.metadata.schema.ObjectSchemaNode;
+import org.apache.asterix.column.metadata.schema.UnionSchemaNode;
+import
org.apache.asterix.column.metadata.schema.collection.AbstractCollectionSchemaNode;
+import org.apache.asterix.column.metadata.schema.primitive.PrimitiveSchemaNode;
+import
org.apache.asterix.dataflow.data.nontagged.serde.AStringSerializerDeserializer;
+import org.apache.asterix.om.base.AString;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.util.ByteArrayAccessibleDataInputStream;
+import org.apache.hyracks.data.std.util.ByteArrayAccessibleInputStream;
+import org.apache.hyracks.util.string.UTF8StringReader;
+import org.apache.hyracks.util.string.UTF8StringWriter;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.IntNode;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.databind.node.TextNode;
+
+import it.unimi.dsi.fastutil.ints.IntList;
+
+public class SchemaJSONBuilderVisitor implements ISchemaNodeVisitor<JsonNode,
Void> {
+ private static final String TYPE_FIELD = "type";
+ private static final String ITEM_FIELD = "item";
+ private static final String ITEMS_FIELD = "items";
+ private static final String FIELDS_FIELD = "fields";
+ private static final String INDEX_FIELD = "index";
+ private static final String LEVEL_FIELD = "level";
+ private static final String OBJECT_TYPE = "object";
+ private static final String ARRAY_TYPE = "array";
+ private static final String UNION_TYPE = "union";
+ private final List<String> fieldNames;
+ private int level;
+
+ public SchemaJSONBuilderVisitor(IFieldNamesDictionary dictionary) throws
HyracksDataException {
+ this.fieldNames = new ArrayList<>();
+ AStringSerializerDeserializer stringSerDer =
+ new AStringSerializerDeserializer(new UTF8StringWriter(), new
UTF8StringReader());
+ List<IValueReference> extractedFieldNames = dictionary.getFieldNames();
+ //Deserialize field names
+ ByteArrayAccessibleInputStream in = new
ByteArrayAccessibleInputStream(new byte[0], 0, 0);
+ ByteArrayAccessibleDataInputStream dataIn = new
ByteArrayAccessibleDataInputStream(in);
+ for (IValueReference serFieldName : extractedFieldNames) {
+ in.setContent(serFieldName.getByteArray(), 0,
serFieldName.getLength());
+ AString fieldName = stringSerDer.deserialize(dataIn);
+ this.fieldNames.add(fieldName.getStringValue());
+ }
+ level = 0;
+ }
+
+ public String build(ObjectSchemaNode root) throws HyracksDataException {
+ JsonNode jsonNode = visit(root, null);
+ return jsonNode.toString();
+ }
+
+ @Override
+ public JsonNode visit(ObjectSchemaNode objectNode, Void arg) throws
HyracksDataException {
+ List<AbstractSchemaNode> children = objectNode.getChildren();
+ IntList fieldNameIndexes = objectNode.getChildrenFieldNameIndexes();
+ ObjectNode jsonNode = JsonNodeFactory.instance.objectNode();
+ jsonNode.putIfAbsent(TYPE_FIELD, TextNode.valueOf(OBJECT_TYPE));
+ jsonNode.putIfAbsent(LEVEL_FIELD, IntNode.valueOf(level));
+ ObjectNode fieldsNode = JsonNodeFactory.instance.objectNode();
+ level++;
+ for (int i = 0; i < children.size(); i++) {
+ int index = fieldNameIndexes.getInt(i);
+ String fieldName = index < 0 ? "<empty>" : fieldNames.get(index);
+ AbstractSchemaNode child = children.get(i);
+ JsonNode childNode = child.accept(this, null);
+ fieldsNode.putIfAbsent(fieldName, childNode);
+ }
+ level--;
+ jsonNode.putIfAbsent(FIELDS_FIELD, fieldsNode);
+ return jsonNode;
+ }
+
+ @Override
+ public JsonNode visit(AbstractCollectionSchemaNode collectionNode, Void
arg) throws HyracksDataException {
+ ObjectNode jsonNode = JsonNodeFactory.instance.objectNode();
+ jsonNode.putIfAbsent(TYPE_FIELD, TextNode.valueOf(ARRAY_TYPE));
+ jsonNode.putIfAbsent(LEVEL_FIELD, IntNode.valueOf(level));
+ AbstractSchemaNode itemNode = collectionNode.getItemNode();
+ level++;
+ jsonNode.putIfAbsent(ITEM_FIELD, itemNode.accept(this, null));
+ level--;
+ return jsonNode;
+ }
+
+ @Override
+ public JsonNode visit(UnionSchemaNode unionNode, Void arg) throws
HyracksDataException {
+ ObjectNode jsonNode = JsonNodeFactory.instance.objectNode();
+ jsonNode.putIfAbsent(TYPE_FIELD, TextNode.valueOf(UNION_TYPE));
+ jsonNode.putIfAbsent(LEVEL_FIELD, IntNode.valueOf(level));
+ ArrayNode itemsNode = JsonNodeFactory.instance.arrayNode();
+ for (AbstractSchemaNode child : unionNode.getChildren().values()) {
+ itemsNode.add(child.accept(this, null));
+ }
+ jsonNode.putIfAbsent(ITEMS_FIELD, itemsNode);
+ return jsonNode;
+ }
+
+ @Override
+ public JsonNode visit(PrimitiveSchemaNode primitiveNode, Void arg) throws
HyracksDataException {
+ ObjectNode jsonNode = JsonNodeFactory.instance.objectNode();
+ jsonNode.putIfAbsent(TYPE_FIELD,
TextNode.valueOf(primitiveNode.getTypeTag().toString()));
+ jsonNode.putIfAbsent(LEVEL_FIELD, IntNode.valueOf(level));
+ jsonNode.putIfAbsent(INDEX_FIELD,
IntNode.valueOf(primitiveNode.getColumnIndex()));
+ return jsonNode;
+ }
+
+}
