This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit d0c631087bf70cf6f50a935f6946182f76f8c015 Author: Ali Alsuliman <[email protected]> AuthorDate: Thu Jan 4 15:01:35 2024 -0800 [NO ISSUE][MTD] Add metadata API to get metadata entities as JSON - user model changes: no - storage format changes: no - interface changes: yes Details: Change-Id: I4aeb520d49d1b2548e83d9afc16ff02eb0d8246d Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18028 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Ali Alsuliman <[email protected]> Reviewed-by: Murtadha Hubail <[email protected]> --- .../apache/asterix/metadata/MetadataManager.java | 12 +++++ .../org/apache/asterix/metadata/MetadataNode.java | 58 ++++++++++++++++++++++ .../asterix/metadata/api/IMetadataManager.java | 16 ++++++ .../apache/asterix/metadata/api/IMetadataNode.java | 17 +++++++ .../DatasetTupleTranslator.java | 2 +- .../DataverseTupleTranslator.java | 2 +- .../FunctionTupleTranslator.java | 2 +- .../IndexTupleTranslator.java | 2 +- .../LibraryTupleTranslator.java | 2 +- .../SynonymTupleTranslator.java | 2 +- .../java/org/apache/hyracks/util/JSONUtil.java | 4 ++ 11 files changed, 113 insertions(+), 6 deletions(-) diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java index 0ab5c7b64e..254c372471 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java @@ -39,6 +39,7 @@ import org.apache.asterix.external.indexing.ExternalFile; import org.apache.asterix.metadata.api.IAsterixStateProxy; import org.apache.asterix.metadata.api.IExtensionMetadataEntity; import org.apache.asterix.metadata.api.IExtensionMetadataSearchKey; +import org.apache.asterix.metadata.api.IMetadataIndex; import org.apache.asterix.metadata.api.IMetadataManager; import org.apache.asterix.metadata.api.IMetadataNode; import org.apache.asterix.metadata.entities.CompactionPolicy; @@ -64,6 +65,7 @@ import org.apache.hyracks.util.ExitUtil; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.base.Strings; /** @@ -1191,6 +1193,16 @@ public abstract class MetadataManager implements IMetadataManager { } } + @Override + public JsonNode getEntitiesAsJson(MetadataTransactionContext mdTxnCtx, IMetadataIndex metadataIndex, + int payloadPosition) throws AlgebricksException { + try { + return metadataNode.getEntitiesAsJson(mdTxnCtx.getTxnId(), metadataIndex, payloadPosition); + } catch (RemoteException e) { + throw new MetadataException(ErrorCode.REMOTE_EXCEPTION_WHEN_CALLING_METADATA_NODE, e); + } + } + @Override public void rebindMetadataNode() { rebindMetadataNode = true; diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java index 87f5129ef4..7a6de19d9d 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java @@ -22,6 +22,7 @@ package org.apache.asterix.metadata; import static org.apache.asterix.common.api.IIdentifierMapper.Modifier.PLURAL; import static org.apache.asterix.common.utils.IdentifierUtil.dataset; +import java.io.PrintStream; import java.rmi.RemoteException; import java.util.ArrayList; import java.util.Arrays; @@ -48,6 +49,7 @@ import org.apache.asterix.common.transactions.TransactionOptions; import org.apache.asterix.common.transactions.TxnId; import org.apache.asterix.common.utils.StoragePathUtil; import org.apache.asterix.external.indexing.ExternalFile; +import org.apache.asterix.formats.nontagged.CleanJSONPrinterFactoryProvider; import org.apache.asterix.formats.nontagged.NullIntrospector; import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider; import org.apache.asterix.formats.nontagged.TypeTraitProvider; @@ -119,11 +121,14 @@ import org.apache.asterix.transaction.management.opcallbacks.UpsertOperationCall import org.apache.asterix.transaction.management.service.transaction.DatasetIdFactory; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.common.utils.Triple; +import org.apache.hyracks.algebricks.data.IPrinter; +import org.apache.hyracks.algebricks.data.IPrinterFactory; import org.apache.hyracks.api.dataflow.value.IBinaryComparator; import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer; import org.apache.hyracks.api.exceptions.ErrorCode; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.data.std.util.ByteArrayAccessibleOutputStream; import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder; import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference; import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; @@ -142,9 +147,12 @@ import org.apache.hyracks.storage.common.IIndexCursor; import org.apache.hyracks.storage.common.IModificationOperationCallback; import org.apache.hyracks.storage.common.MultiComparator; import org.apache.hyracks.util.ExitUtil; +import org.apache.hyracks.util.JSONUtil; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.google.common.base.Strings; public class MetadataNode implements IMetadataNode { @@ -327,6 +335,16 @@ public class MetadataNode implements IMetadataNode { return getEntities(txnId, searchKey.getSearchKey(), tupleTranslator, index); } + @Override + public JsonNode getEntitiesAsJson(TxnId txnId, IMetadataIndex metadataIndex, int payloadPosition) + throws AlgebricksException, RemoteException { + try { + return getJsonNodes(txnId, metadataIndex, payloadPosition); + } catch (HyracksDataException e) { + throw new AlgebricksException(e); + } + } + private <T extends IExtensionMetadataEntity> ExtensionMetadataDataset<T> getExtensionMetadataDataset( ExtensionMetadataDatasetId datasetId) throws AlgebricksException { ExtensionMetadataDataset<T> index = (ExtensionMetadataDataset<T>) extensionDatasets.get(datasetId); @@ -2485,4 +2503,44 @@ public class MetadataNode implements IMetadataNode { public ITxnIdFactory getTxnIdFactory() { return txnIdFactory; } + + private ArrayNode getJsonNodes(TxnId txnId, IMetadataIndex mdIndex, int payloadPosition) + throws AlgebricksException, HyracksDataException { + IValueExtractor<JsonNode> valueExtractor = createValueExtractor(mdIndex, payloadPosition); + List<JsonNode> results = new ArrayList<>(); + searchIndex(txnId, mdIndex, null, valueExtractor, results); + ArrayNode array = JSONUtil.createArray(); + results.forEach(array::add); + return array; + } + + private static IValueExtractor<JsonNode> createValueExtractor(IMetadataIndex mdIndex, int payloadFieldIndex) { + return new IValueExtractor<>() { + + final ARecordType payloadRecordType = mdIndex.getPayloadRecordType(); + final IPrinterFactory printerFactory = + CleanJSONPrinterFactoryProvider.INSTANCE.getPrinterFactory(payloadRecordType); + final IPrinter printer = printerFactory.createPrinter(); + final ByteArrayAccessibleOutputStream outputStream = new ByteArrayAccessibleOutputStream(); + final PrintStream printStream = new PrintStream(outputStream); + + @Override + public JsonNode getValue(TxnId txnId, ITupleReference tuple) { + try { + byte[] serRecord = tuple.getFieldData(payloadFieldIndex); + int recordStartOffset = tuple.getFieldStart(payloadFieldIndex); + int recordLength = tuple.getFieldLength(payloadFieldIndex); + + printer.init(); + outputStream.reset(); + + printer.print(serRecord, recordStartOffset, recordLength, printStream); + printStream.flush(); + return JSONUtil.readTree(outputStream.getByteArray(), 0, outputStream.getLength()); + } catch (Throwable th) { + return JSONUtil.createObject(); + } + } + }; + } } diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java index 5ff8a036de..d08578af95 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java @@ -46,6 +46,8 @@ import org.apache.asterix.metadata.entities.NodeGroup; import org.apache.asterix.metadata.entities.Synonym; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; +import com.fasterxml.jackson.databind.JsonNode; + /** * A metadata manager provides user access to Asterix metadata (e.g., types, * datasets, indexes, etc.). A metadata manager satisfies requests by contacting @@ -864,6 +866,20 @@ public interface IMetadataManager extends IMetadataBootstrap { <T extends IExtensionMetadataEntity> List<T> getEntities(MetadataTransactionContext mdTxnCtx, IExtensionMetadataSearchKey searchKey) throws AlgebricksException; + /** + * Gets all the records of a metadata dataset as JSON. + * + * @param mdTxnCtx metadata transaction context + * @param metadataIndex the metadata dataset + * @param payloadPosition the position of the record in the tuple + * + * @return the metadata records as JSON + * + * @throws AlgebricksException AlgebricksException + */ + JsonNode getEntitiesAsJson(MetadataTransactionContext mdTxnCtx, IMetadataIndex metadataIndex, int payloadPosition) + throws AlgebricksException; + /** * Indicate when the metadata node has left or rejoined the cluster, and the * MetadataManager should rebind it diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java index 561a4fa413..378cf47ed1 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java @@ -47,6 +47,8 @@ import org.apache.asterix.metadata.entities.Synonym; import org.apache.asterix.transaction.management.opcallbacks.AbstractIndexModificationOperationCallback; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; +import com.fasterxml.jackson.databind.JsonNode; + /** * A metadata node stores metadata in its local storage structures (currently * BTrees). A metadata node services requests on behalf of the (possibly remote) @@ -969,6 +971,21 @@ public interface IMetadataNode extends Remote, Serializable { <T extends IExtensionMetadataEntity> List<T> getEntities(TxnId txnId, IExtensionMetadataSearchKey searchKey) throws AlgebricksException, RemoteException; + /** + * Gets all the records of a metadata dataset as JSON. + * + * @param txnId transaction id + * @param metadataIndex the metadata dataset + * @param payloadPosition the position of the record in the tuple + * + * @return the metadata records as JSON + * + * @throws AlgebricksException AlgebricksException + * @throws RemoteException RemoteException + */ + JsonNode getEntitiesAsJson(TxnId txnId, IMetadataIndex metadataIndex, int payloadPosition) + throws AlgebricksException, RemoteException; + void addFeedConnection(TxnId txnId, FeedConnection feedConnection) throws AlgebricksException, RemoteException; FeedConnection getFeedConnection(TxnId txnId, DataverseName dataverseName, String feedName, String datasetName) diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java index 0d05d4624b..d502322e4d 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java @@ -84,7 +84,7 @@ import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; public class DatasetTupleTranslator extends AbstractTupleTranslator<Dataset> { // Payload field containing serialized Dataset. - private static final int DATASET_PAYLOAD_TUPLE_FIELD_INDEX = 2; + public static final int DATASET_PAYLOAD_TUPLE_FIELD_INDEX = 2; protected AMutableInt32 aInt32; protected AMutableInt64 aInt64; diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DataverseTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DataverseTupleTranslator.java index 77c838ac46..700beb99fd 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DataverseTupleTranslator.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DataverseTupleTranslator.java @@ -39,7 +39,7 @@ import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; public class DataverseTupleTranslator extends AbstractTupleTranslator<Dataverse> { // Payload field containing serialized Dataverse. - private static final int DATAVERSE_PAYLOAD_TUPLE_FIELD_INDEX = 1; + public static final int DATAVERSE_PAYLOAD_TUPLE_FIELD_INDEX = 1; protected AMutableInt32 aInt32; diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java index ef9c143853..9ad83c1f88 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java @@ -81,7 +81,7 @@ import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; public class FunctionTupleTranslator extends AbstractDatatypeTupleTranslator<Function> { // Payload field containing serialized Function. - private static final int FUNCTION_PAYLOAD_TUPLE_FIELD_INDEX = 3; + public static final int FUNCTION_PAYLOAD_TUPLE_FIELD_INDEX = 3; protected OrderedListBuilder dependenciesListBuilder; protected OrderedListBuilder dependencyListBuilder; diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java index b00a706b03..08f947ff1b 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java @@ -83,7 +83,7 @@ import com.google.common.base.Strings; public class IndexTupleTranslator extends AbstractTupleTranslator<Index> { // Payload field containing serialized Index. - private static final int INDEX_PAYLOAD_TUPLE_FIELD_INDEX = 3; + public static final int INDEX_PAYLOAD_TUPLE_FIELD_INDEX = 3; // Field name of open field. public static final String GRAM_LENGTH_FIELD_NAME = "GramLength"; diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/LibraryTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/LibraryTupleTranslator.java index e3e879b7e8..f255fff48a 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/LibraryTupleTranslator.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/LibraryTupleTranslator.java @@ -45,7 +45,7 @@ import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; public class LibraryTupleTranslator extends AbstractTupleTranslator<Library> { // Payload field containing serialized Library. - private static final int LIBRARY_PAYLOAD_TUPLE_FIELD_INDEX = 2; + public static final int LIBRARY_PAYLOAD_TUPLE_FIELD_INDEX = 2; protected LibraryTupleTranslator(boolean getTuple) { super(getTuple, MetadataPrimaryIndexes.LIBRARY_DATASET, LIBRARY_PAYLOAD_TUPLE_FIELD_INDEX); diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/SynonymTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/SynonymTupleTranslator.java index 6bad2d47bf..12b410ccd2 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/SynonymTupleTranslator.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/SynonymTupleTranslator.java @@ -36,7 +36,7 @@ public final class SynonymTupleTranslator extends AbstractTupleTranslator<Synony // Payload field containing serialized Synonym. - private static final int SYNONYM_PAYLOAD_TUPLE_FIELD_INDEX = 2; + public static final int SYNONYM_PAYLOAD_TUPLE_FIELD_INDEX = 2; protected SynonymTupleTranslator(boolean getTuple) { super(getTuple, MetadataPrimaryIndexes.SYNONYM_DATASET, SYNONYM_PAYLOAD_TUPLE_FIELD_INDEX); diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/JSONUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/JSONUtil.java index f937e979d1..191df1f98c 100644 --- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/JSONUtil.java +++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/JSONUtil.java @@ -50,6 +50,10 @@ public class JSONUtil { SORTED_MAPPER.configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true); } + public static JsonNode readTree(byte[] bytes, int start, int size) throws IOException { + return OBJECT_MAPPER.readTree(bytes, start, size); + } + public static String convertNode(final JsonNode node) throws JsonProcessingException { return SORTED_MAPPER.writeValueAsString(SORTED_MAPPER.treeToValue(node, Object.class)); }
