This is an automated email from the ASF dual-hosted git repository. alsuliman pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit b59f647bff727e4495f50c93ccd396c6d45ce38f Author: Ali Alsuliman <[email protected]> AuthorDate: Tue Feb 28 20:16:26 2023 -0800 [ASTERIXDB-3122][FUN] escape string values in DUMP_INDEX - user model changes: no - storage format changes: no - interface changes: no Change-Id: I9127d2346541f2cb2bc9d64016f4608ec4ec695c Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17405 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Ali Alsuliman <[email protected]> Reviewed-by: Murtadha Hubail <[email protected]> --- .../apache/asterix/app/function/DumpIndexDatasource.java | 7 +++++++ .../org/apache/asterix/app/function/DumpIndexReader.java | 4 ++++ .../misc/dump_index/dump_index.2.update.sqlpp | 3 ++- .../runtimets/results/misc/dump_index/dump_index.10.adm | 2 +- .../external/adapter/factory/GenericAdapterFactory.java | 16 +++++++--------- .../asterix/metadata/declared/FunctionDataSource.java | 11 ++++++++++- 6 files changed, 31 insertions(+), 12 deletions(-) diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java index 3351075143..691be4706c 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java @@ -19,6 +19,8 @@ package org.apache.asterix.app.function; import org.apache.asterix.common.cluster.IClusterStateManager; +import org.apache.asterix.external.api.IDataParserFactory; +import org.apache.asterix.external.parser.factory.JSONDataParserFactory; import org.apache.asterix.metadata.api.IDatasourceFunction; import org.apache.asterix.metadata.declared.DataSourceId; import org.apache.asterix.metadata.declared.FunctionDataSource; @@ -59,4 +61,9 @@ public class DumpIndexDatasource extends FunctionDataSource { AlgebricksAbsolutePartitionConstraint locations) { return new DumpIndexFunction(locations, indexDataflowHelperFactory, recDesc, comparatorFactories); } + + @Override + protected IDataParserFactory createDataParserFactory() { + return new JSONDataParserFactory(); + } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexReader.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexReader.java index e60b015b06..8ef094e923 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexReader.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexReader.java @@ -25,6 +25,7 @@ import java.nio.ByteBuffer; import org.apache.asterix.external.api.IRawRecord; import org.apache.asterix.external.input.record.CharArrayRecord; import org.apache.asterix.om.base.ARecord; +import org.apache.asterix.om.base.AString; import org.apache.asterix.om.base.IACollection; import org.apache.asterix.om.base.IACursor; import org.apache.asterix.om.base.IAObject; @@ -132,6 +133,9 @@ public class DumpIndexReader extends FunctionReader { case DATETIME: JSONUtil.quoteAndEscape(recordBuilder, field.toString()); break; + case STRING: + JSONUtil.quoteAndEscape(recordBuilder, ((AString) field).getStringValue()); + break; case MISSING: break; default: diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dump_index/dump_index.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dump_index/dump_index.2.update.sqlpp index 10378f2d82..3469c9a0ac 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dump_index/dump_index.2.update.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dump_index/dump_index.2.update.sqlpp @@ -18,5 +18,6 @@ */ USE test; INSERT INTO ds ([{"id":1, "name": "name1"}, {"id":2, "name": "name2"}]); -UPSERT INTO ds2 ([{"id": 1,"age":30, "a1": {"b": [{"x": [1,2]}, {"x": [1,2]}]}, "a2": [{"x": [1,2]}, {"x": [1,2]}] }]); +UPSERT INTO ds2 ([{"id": 1,"age":"t\\'s", "a1": {"b": [{"x": [1,2]}, {"x": [1,2]}]}, "a2": [{"x": [1,2]}, {"x": [1,2]}] +}]); ANALYZE DATASET ds2; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dump_index/dump_index.10.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dump_index/dump_index.10.adm index a52ceaeb97..d13e4524a7 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dump_index/dump_index.10.adm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dump_index/dump_index.10.adm @@ -1 +1 @@ -[ { "values": [ 1, { "id": 1, "age": 30, "a1": { "b": [ { "x": [ 1, 2 ] }, { "x": [ 1, 2 ] } ] }, "a2": [ { "x": [ 1, 2 ] }, { "x": [ 1, 2 ] } ] } ] } ] \ No newline at end of file +[ { "values": [ 1, { "id": 1, "age": "t\\'s", "a1": { "b": [ { "x": [ 1, 2 ] }, { "x": [ 1, 2 ] } ] }, "a2": [ { "x": [ 1, 2 ] }, { "x": [ 1, 2 ] } ] } ] } ] \ No newline at end of file diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java index ac45f7a72b..8ec5af037e 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java @@ -37,7 +37,6 @@ import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController; import org.apache.asterix.external.dataset.adapter.FeedAdapter; import org.apache.asterix.external.dataset.adapter.GenericAdapter; import org.apache.asterix.external.indexing.ExternalFile; -import org.apache.asterix.external.parser.factory.ADMDataParserFactory; import org.apache.asterix.external.provider.DataflowControllerProvider; import org.apache.asterix.external.provider.DatasourceFactoryProvider; import org.apache.asterix.external.provider.ParserFactoryProvider; @@ -47,7 +46,6 @@ import org.apache.asterix.external.util.ExternalDataUtils; import org.apache.asterix.external.util.FeedLogManager; import org.apache.asterix.external.util.FeedUtils; import org.apache.asterix.om.types.ARecordType; -import org.apache.asterix.om.utils.RecordUtil; import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.api.application.ICCServiceContext; @@ -209,18 +207,18 @@ public class GenericAdapterFactory implements IIndexingAdapterFactory, ITypedAda } /** - * Use pre-configured datasource factory - * For function datasources + * Use pre-configured datasource factory For function datasources * * @param dataSourceFactory - * the function datasource factory + * the function datasource factory + * @param dataParserFactory + * the function data parser factory * @throws AlgebricksException */ - public void configure(IExternalDataSourceFactory dataSourceFactory) throws AlgebricksException { + public void configure(IExternalDataSourceFactory dataSourceFactory, IDataParserFactory dataParserFactory) + throws AlgebricksException { this.dataSourceFactory = dataSourceFactory; - dataParserFactory = new ADMDataParserFactory(); - dataParserFactory.setRecordType(RecordUtil.FULLY_OPEN_RECORD_TYPE); - dataParserFactory.configure(Collections.emptyMap()); + this.dataParserFactory = dataParserFactory; configuration = Collections.emptyMap(); } diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java index a7d560c70c..5c874d7feb 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java @@ -27,6 +27,8 @@ import java.util.Set; import org.apache.asterix.common.cluster.IClusterStateManager; import org.apache.asterix.common.functions.FunctionSignature; import org.apache.asterix.external.adapter.factory.GenericAdapterFactory; +import org.apache.asterix.external.api.IDataParserFactory; +import org.apache.asterix.external.parser.factory.ADMDataParserFactory; import org.apache.asterix.metadata.api.IDatasourceFunction; import org.apache.asterix.om.types.IAType; import org.apache.asterix.om.utils.RecordUtil; @@ -98,7 +100,10 @@ public abstract class FunctionDataSource extends DataSource { IClusterStateManager csm = metadataProvider.getApplicationContext().getClusterStateManager(); FunctionDataSourceFactory factory = new FunctionDataSourceFactory(createFunction(metadataProvider, getLocations(csm))); - adapterFactory.configure(factory); + IDataParserFactory dataParserFactory = createDataParserFactory(); + dataParserFactory.setRecordType(RecordUtil.FULLY_OPEN_RECORD_TYPE); + dataParserFactory.configure(Collections.emptyMap()); + adapterFactory.configure(factory, dataParserFactory); return metadataProvider.buildExternalDatasetDataScannerRuntime(jobSpec, itemType, adapterFactory, tupleFilterFactory, outputLimit); } @@ -112,6 +117,10 @@ public abstract class FunctionDataSource extends DataSource { return new AlgebricksAbsolutePartitionConstraint(ncs.toArray(new String[ncs.size()])); } + protected IDataParserFactory createDataParserFactory() { + return new ADMDataParserFactory(); + } + protected static DataSourceId createDataSourceId(FunctionIdentifier fid, String... parameters) { return new DataSourceId(FunctionSignature.getDataverseName(fid), fid.getName(), parameters); }
