This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 4915df6b1856a0e1788a77be2500042dd8ad2b68 Merge: 9bafae25a6 7bd62f396c Author: Michael Blow <[email protected]> AuthorDate: Tue Nov 11 20:19:26 2025 -0500 Merge branch 'gerrit/phoenix' into 'master' * [ASTERIXDB-3670][COMP] Fix ArrayIndexOutOfBoundsException with array index (MB-69201) * [NO ISSUR][EXT]: fix reading parquet with assume role auth (MB-69108) * [ASTERIXDB-3669][CLOUD] Introduced S3 sync client for parallel downloader (MB-69226) * [ASTERIXDB-3669][CLOUD] Retry downloadDirectories in parallel downloader (MB-69226) * [ASTERIXDB-3671][IDX] Incorrect result with array index and index nested loop join (MB-69202) * [NO ISSUE][CLOUD] Fix premature buffer release caused by flatMap cancel (MB-69283) * [ASTERIXDB-3668][EXT]: support disableSslVerify + AD auth for parquet (MB-68900) * [ASTERIXDB-3672][EXT] Fix reading array null elements (MB-66766) Change-Id: Id8177dceab07593e92a01cb8ba30c8ac78d5eafa .../optimizer/rules/am/AccessMethodUtils.java | 3 + .../optimizer/rules/am/BTreeAccessMethod.java | 3 + .../rules/am/IntroduceJoinAccessMethodRule.java | 15 +- .../optimizer/rules/am/RTreeAccessMethod.java | 4 +- .../data/parquet/01/array_of_primitives.parquet | Bin 0 -> 549 bytes .../01/array_of_single_field_struct.parquet | Bin 0 -> 570 bytes .../parquet/01/array_of_structs_with_nulls.parquet | Bin 0 -> 897 bytes .../data/parquet/01/repeated_field.parquet | Bin 0 -> 511 bytes .../data/parquet/01/repeated_struct.parquet | Bin 0 -> 705 bytes .../apache/asterix/app/config/ConfigValidator.java | 5 + .../asterix/hyracks/bootstrap/CCApplication.java | 2 + .../asterix/hyracks/bootstrap/NCApplication.java | 2 + .../apache/asterix/test/common/TestConstants.java | 29 +- .../apache/asterix/test/common/TestExecutor.java | 20 +- .../external_dataset/ExternalDatasetTestUtils.java | 13 + .../AzureBlobStorageExternalDatasetTest.java | 60 +- .../parquet/BinaryFileConverterUtil.java | 1 + .../use-case-5/query1.sqlpp} | 26 +- .../use-case-5/query2.sqlpp} | 26 +- .../use-case-5/query1.sqlpp} | 29 +- .../use-case-5/query2.sqlpp} | 30 +- .../atomic-and-array-queries/query8.plan | 104 +- .../atomic-and-array-queries/query9.plan | 122 +- .../join-quantified-queries/use-case-5/query1.plan | 75 + .../join-quantified-queries/use-case-5/query2.plan | 75 + .../use-case-5/query1.plan | 63 + .../use-case-5/query2.plan | 63 + .../atomic-and-array-queries/query8.plan | 104 +- .../atomic-and-array-queries/query9.plan | 122 +- .../use-case-5/use-case-5.1.ddl.sqlpp} | 16 +- .../use-case-5/use-case-5.2.update.sqlpp} | 33 +- .../use-case-5/use-case-5.3.query.sqlpp} | 20 +- .../use-case-5/use-case-5.4.query.sqlpp} | 20 +- .../copy-to/csv/datetime/test.040.ddl.sqlpp | 14 +- .../copy-to/csv/delimiter/delimiter.04.ddl.sqlpp | 2 +- .../copy-to/csv/delimiter/delimiter.06.ddl.sqlpp | 2 +- .../copy-to/csv/header/header.04.ddl.sqlpp | 2 +- .../copy-to/csv/header/header.11.ddl.sqlpp | 2 +- .../copy-to/csv/header/header.21.ddl.sqlpp | 2 +- .../copy-to/csv/header/header.31.ddl.sqlpp | 2 +- .../copy-to/csv/null/null.04.ddl.sqlpp | 2 +- .../csv/quote-escape/quote-escape.03.ddl.sqlpp | 2 +- .../csv/quote-escape/quote-escape.12.ddl.sqlpp | 2 +- .../copy-to/csv/simple-csv/simple-csv.04.ddl.sqlpp | 2 +- .../copy-to/csv/simple-csv/simple-csv.12.ddl.sqlpp | 2 +- .../csv/type-mismatch/type-mismatch.03.ddl.sqlpp | 2 +- .../default-namespace.01.ddl.sqlpp | 2 +- .../default-namespace.05.ddl.sqlpp | 6 +- .../copy-to/empty-path/empty-path.03.ddl.sqlpp | 3 +- .../copy-to/empty-path/empty-path.04.update.sqlpp | 2 + .../copy-to/empty-path/empty-path.05.ddl.sqlpp | 2 + .../empty-over.01.ddl.sqlpp | 2 +- .../early-missing/early-missing.01.ddl.sqlpp | 2 +- .../negative/empty-over/empty-over.01.ddl.sqlpp | 2 +- .../negative/long-path/long-path.01.ddl.sqlpp | 2 +- .../non-empty-folder/non-empty-folder.01.ddl.sqlpp | 2 +- .../runtime-missing/runtime-missing.01.ddl.sqlpp | 2 +- .../runtime-missing/runtime-missing.04.ddl.sqlpp | 2 +- .../supported-adapters.01.ddl.sqlpp | 2 +- .../copy-to/order-by/order-by.01.ddl.sqlpp | 2 +- .../copy-to/order-by/order-by.03.ddl.sqlpp | 2 +- .../copy-to/order-by/order-by.21.ddl.sqlpp | 2 +- .../copy-to/order-by/order-by.31.ddl.sqlpp | 2 +- .../parquet-cover-data-types.04.ddl.sqlpp | 2 +- .../parquet-empty-array.04.ddl.sqlpp | 2 +- .../parquet-field-names.04.ddl.sqlpp | 2 +- .../parquet-field-names.07.ddl.sqlpp | 2 +- .../parquet-file-writers.04.ddl.sqlpp | 2 +- .../parquet-heterogeneous.04.ddl.sqlpp | 2 +- .../parquet-null-type.04.ddl.sqlpp | 2 +- .../parquet-null-type.07.ddl.sqlpp | 2 +- .../parquet-null1/parquet-null1.03.ddl.sqlpp | 2 +- .../parquet-null2/parquet-null2.03.ddl.sqlpp | 2 +- .../parquet-null3/parquet-null3.01.ddl.sqlpp} | 10 +- .../parquet-null3/parquet-null3.02.update.sqlpp | 1735 ++++++++++++++++++++ .../parquet-null3.03.ddl.sqlpp} | 2 +- .../parquet-null3/parquet-null3.04.query.sqlpp} | 7 +- .../parquet-partition-heterogeneous.04.ddl.sqlpp | 2 +- .../parquet-simple/parquet-simple.03.ddl.sqlpp | 2 +- .../parquet-tweet/parquet-tweet.04.ddl.sqlpp | 2 +- .../parquet-type-hierarchy.04.update.sqlpp | 2 +- .../copy-to/parquet-utf8/parquet-utf8.04.ddl.sqlpp | 2 +- .../copy-to/partition/partition.01.ddl.sqlpp | 2 +- .../copy-to/partition/partition.03.ddl.sqlpp | 2 +- .../queries_sqlpp/copy-to/query/query.01.ddl.sqlpp | 2 +- .../queries_sqlpp/copy-to/query/query.03.ddl.sqlpp | 2 +- .../copy-to/simple-write/simple-write.01.ddl.sqlpp | 2 +- .../copy-to/simple-write/simple-write.03.ddl.sqlpp | 2 +- .../copy-to/union-all/union-all.01.ddl.sqlpp | 2 +- .../test.000.ddl.sqlpp | 4 +- .../test.000.ddl.sqlpp | 4 +- .../test.000.ddl.sqlpp | 4 +- .../account-key-missing/test.000.ddl.sqlpp | 4 +- .../account-name-missing/test.000.ddl.sqlpp | 4 +- .../test.000.ddl.sqlpp | 4 +- .../client-certificate-present/test.000.ddl.sqlpp | 5 +- .../client-id-present/test.000.ddl.sqlpp | 4 +- .../client-secret-present/test.000.ddl.sqlpp | 4 +- .../managed-identity-id-present/test.000.ddl.sqlpp | 6 +- .../test.000.ddl.sqlpp | 4 +- .../tenant-id-present/test.000.ddl.sqlpp | 4 +- .../client-certificate-present/test.000.ddl.sqlpp | 35 - .../client-secret-present/test.000.ddl.sqlpp | 4 +- .../tenant-id-present/test.000.ddl.sqlpp | 4 +- .../test.000.ddl.sqlpp | 36 - .../test.000.ddl.sqlpp | 38 - .../test.000.ddl.sqlpp | 38 - .../client-id/tenant-id-missing/test.000.ddl.sqlpp | 4 +- .../test.000.ddl.sqlpp | 36 - .../test.999.ddl.sqlpp | 20 - .../client-certificate-present/test.000.ddl.sqlpp | 36 - .../client-certificate-present/test.999.ddl.sqlpp | 20 - .../client-id-present/test.000.ddl.sqlpp | 36 - .../client-id-present/test.999.ddl.sqlpp | 20 - .../client-secret-present/test.000.ddl.sqlpp | 6 +- .../tenant-id-present/test.000.ddl.sqlpp | 6 +- .../no-endpoint/test.000.ddl.sqlpp | 4 +- .../test.000.ddl.sqlpp | 36 - .../test.999.ddl.sqlpp | 20 - .../client-certificate-present/test.000.ddl.sqlpp | 36 - .../client-certificate-present/test.999.ddl.sqlpp | 20 - .../client-id-present/test.000.ddl.sqlpp | 4 +- .../client-secret-present/test.000.ddl.sqlpp | 4 +- .../managed-identity-id-present/test.000.ddl.sqlpp | 6 +- .../tenant-id-present/test.000.ddl.sqlpp | 4 +- .../test.000.ddl.sqlpp | 4 +- .../shared-access-signature/test.000.ddl.sqlpp | 4 +- .../avro/array-access/array-access.01.ddl.sqlpp | 2 +- .../avro/avro-types/avro-map/avro-map.01.ddl.sqlpp | 2 +- .../avro-nested-records.01.ddl.sqlpp | 2 +- .../avro-primitives/avro-primitives.01.ddl.sqlpp | 2 +- .../avro-types/avro-union/avro-union.01.ddl.sqlpp | 4 +- .../avro/field-access/field-access.01.ddl.sqlpp | 8 +- .../heterogeneous-access.1.ddl.sqlpp | 2 +- .../avro/invalid-avro-files/test.000.ddl.sqlpp | 2 +- .../avro/invalid-type/invalid-type.1.ddl.sqlpp | 2 +- .../avro/missing-fields/missing-fields.1.ddl.sqlpp | 2 +- .../multi-file-multi-schema.1.ddl.sqlpp | 4 +- .../common/avro/no-files/no-files.1.ddl.sqlpp | 2 +- .../avro/object-concat/object-concat.1.ddl.sqlpp | 2 +- .../select-all-fields.1.ddl.sqlpp | 2 +- .../select-count-one-field.1.ddl.sqlpp | 2 +- .../string-standard-utf8.1.ddl.sqlpp | 2 +- .../avro/type-mismatch/type-mismatch.1.ddl.sqlpp | 2 +- .../bucket-does-not-exist/test.000.ddl.sqlpp | 1 + .../common/byte_order_mark/csv/test.000.ddl.sqlpp | 1 + .../common/byte_order_mark/json/test.000.ddl.sqlpp | 1 + .../common/byte_order_mark/tsv/test.000.ddl.sqlpp | 1 + .../common/csv-header/query-dataset.001.ddl.sqlpp | 1 + .../csv-no-header/query-dataset.001.ddl.sqlpp | 1 + .../csv-warnings/query-dataset.002.ddl.sqlpp | 1 + .../csv-warnings/query-dataset.007.ddl.sqlpp | 1 + .../csv-warnings/query-dataset.010.ddl.sqlpp | 1 + .../csv-warnings/query-dataset.012.ddl.sqlpp | 1 + .../csv-warnings/query-dataset.015.ddl.sqlpp | 1 + .../common/csv/csv/query-dataset.000.ddl.sqlpp | 1 + .../common/csv/gz/query-dataset.000.ddl.sqlpp | 1 + .../common/csv/mixed/query-dataset.000.ddl.sqlpp | 1 + .../definition-does-not-exist/test.000.ddl.sqlpp | 1 + .../avro/embed-flat/embed-flat.000.ddl.sqlpp | 2 +- .../embed-multiple-values.000.ddl.sqlpp | 2 +- .../embed-one-value/embed-one-value.000.ddl.sqlpp | 4 +- .../embed-flat.000.ddl.sqlpp | 2 +- .../avro/one-field/one-field.000.ddl.sqlpp | 4 +- .../dynamic-prefixes/avro/query/test.000.ddl.sqlpp | 6 +- .../avro/using-limit/test.000.ddl.sqlpp | 2 +- .../dynamic-prefixes/avro/views/test.000.ddl.sqlpp | 4 +- .../computed-field-at-start/test.000.ddl.sqlpp | 1 + .../test.000.ddl.sqlpp | 2 + .../test.000.ddl.sqlpp | 2 + .../test.000.ddl.sqlpp | 1 + .../test.001.ddl.sqlpp | 1 + .../conflicted-fields/test.001.ddl.sqlpp | 1 + .../conflicted-fields/test.002.ddl.sqlpp | 1 + .../embed-multiple-missing-values.000.ddl.sqlpp | 1 + .../embed-multiple-values.000.ddl.sqlpp | 1 + .../embed-one-value/embed-one-value.000.ddl.sqlpp | 1 + .../duplicate-field/test.001.ddl.sqlpp | 1 + .../duplicate-field/test.002.ddl.sqlpp | 1 + .../duplicate-field/test.003.ddl.sqlpp | 1 + .../duplicate-field/test.004.ddl.sqlpp | 1 + .../duplicate-field/test.005.ddl.sqlpp | 1 + .../embed-flat/embed-flat.000.ddl.sqlpp | 1 + .../embed-multiple-values.000.ddl.sqlpp | 1 + .../embed-one-value/embed-one-value.000.ddl.sqlpp | 2 + .../embed-with-closed-type.000.ddl.sqlpp | 1 + .../embed-flat.000.ddl.sqlpp | 1 + .../dynamic-prefixes/not-in/not-in.000.ddl.sqlpp | 2 + .../one-field/one-field.000.ddl.sqlpp | 2 + .../computed-field-at-start/test.000.ddl.sqlpp | 1 + .../test.000.ddl.sqlpp | 2 + .../test.000.ddl.sqlpp | 2 + .../test.000.ddl.sqlpp | 1 + .../test.001.ddl.sqlpp | 1 + .../parquet/duplicate-field/test.001.ddl.sqlpp | 1 + .../parquet/duplicate-field/test.002.ddl.sqlpp | 1 + .../parquet/duplicate-field/test.003.ddl.sqlpp | 1 + .../parquet/duplicate-field/test.004.ddl.sqlpp | 1 + .../parquet/duplicate-field/test.005.ddl.sqlpp | 1 + .../parquet/embed-flat/embed-flat.000.ddl.sqlpp | 1 + .../embed-multiple-values.000.ddl.sqlpp | 1 + .../embed-one-value/embed-one-value.000.ddl.sqlpp | 2 + .../embed-flat.000.ddl.sqlpp | 1 + .../parquet/one-field/one-field.000.ddl.sqlpp | 2 + .../parquet/query/test.000.ddl.sqlpp | 3 + .../parquet/type-mismatch/test.000.ddl.sqlpp | 1 + .../parquet/using-limit/test.000.ddl.sqlpp | 1 + .../parquet/views/test.000.ddl.sqlpp | 2 + .../dynamic-prefixes/query/test.000.ddl.sqlpp | 3 + .../type-mismatch/test.000.ddl.sqlpp | 1 + .../using-limit/test.000.ddl.sqlpp | 1 + .../dynamic-prefixes/views/test.000.ddl.sqlpp | 2 + .../external_dataset.000.ddl.sqlpp | 1 + .../include-exclude/bad-name-1/test.000.ddl.sqlpp | 1 + .../include-exclude/bad-name-2/test.000.ddl.sqlpp | 1 + .../include-exclude/bad-name-3/test.000.ddl.sqlpp | 1 + .../common/include-exclude/both/test.000.ddl.sqlpp | 1 + .../include-exclude/exclude-1/test.000.ddl.sqlpp | 1 + .../include-exclude/exclude-2/test.000.ddl.sqlpp | 1 + .../include-exclude/exclude-3/test.000.ddl.sqlpp | 1 + .../include-exclude/exclude-4/test.000.ddl.sqlpp | 1 + .../include-exclude/exclude-5/test.000.ddl.sqlpp | 1 + .../include-exclude/exclude-6/test.000.ddl.sqlpp | 1 + .../include-exclude/exclude-all/test.000.ddl.sqlpp | 1 + .../include-exclude/include-1/test.000.ddl.sqlpp | 1 + .../include-exclude/include-10/test.000.ddl.sqlpp | 1 + .../include-exclude/include-11/test.000.ddl.sqlpp | 1 + .../include-exclude/include-12/test.000.ddl.sqlpp | 1 + .../include-exclude/include-2/test.000.ddl.sqlpp | 1 + .../include-exclude/include-3/test.000.ddl.sqlpp | 1 + .../include-exclude/include-4/test.000.ddl.sqlpp | 1 + .../include-exclude/include-5/test.000.ddl.sqlpp | 1 + .../include-exclude/include-6/test.000.ddl.sqlpp | 1 + .../include-exclude/include-7/test.000.ddl.sqlpp | 1 + .../include-exclude/include-8/test.000.ddl.sqlpp | 1 + .../include-exclude/include-9/test.000.ddl.sqlpp | 1 + .../include-exclude/include-all/test.000.ddl.sqlpp | 1 + .../common/invalid-endpoint/test.000.ddl.sqlpp | 1 + .../json-warnings/json-warnings.002.ddl.sqlpp | 1 + .../common/json/gz/external_dataset.000.ddl.sqlpp | 4 + .../json/json/external_dataset.000.ddl.sqlpp | 8 + .../json/mixed/external_dataset.000.ddl.sqlpp | 4 + .../common/jsonl/query-dataset.001.ddl.sqlpp | 1 + .../malformed-json.000.container.sqlpp | 2 +- .../malformed-json/malformed-json.001.ddl.sqlpp | 3 +- .../malformed-json.003.container.sqlpp | 2 +- .../malformed-json.005.container.sqlpp | 2 +- .../malformed-json.007.container.sqlpp | 2 +- .../malformed-json.009.container.sqlpp | 2 +- .../common/negative/negative.000.ddl.sqlpp | 1 + .../test.000.ddl.sqlpp | 1 + .../exclude-all-files/test.000.ddl.sqlpp | 1 + .../include-no-files/test.000.ddl.sqlpp | 1 + .../over-1000-objects.000.ddl.sqlpp | 1 + .../ASTERIXDB-3540/ASTERIXDB-3540.01.ddl.sqlpp | 1 + .../array-access-pushdown.01.ddl.sqlpp | 1 + .../field-access-pushdown.01.ddl.sqlpp | 4 + .../heterogeneous-access-pushdown.1.ddl.sqlpp | 1 + .../invalid-parquet-files/test.000.ddl.sqlpp | 1 + .../parquet/invalid-type/invalid-type.1.ddl.sqlpp | 1 + .../missing-fields/missing-fields.1.ddl.sqlpp | 1 + .../multi-file-multi-schema.1.ddl.sqlpp | 2 + .../common/parquet/no-files/no-files.1.ddl.sqlpp | 1 + .../null-in-array.01.ddl.sqlpp} | 5 +- .../null-in-array/null-in-array.02.query.sqlpp} | 4 +- .../null-in-array/null-in-array.99.ddl.sqlpp} | 2 +- .../object-concat/object-concat.1.ddl.sqlpp | 1 + .../parquet-types/decimal/decimal.1.ddl.sqlpp | 2 + .../disable-json-parsing.1.ddl.sqlpp | 1 + .../invalid-timezone/temporal.1.ddl.sqlpp | 1 + .../parquet-types/temporal/temporal.1.ddl.sqlpp | 1 + .../unset-flags/unset-flags.1.ddl.sqlpp | 1 + .../pushdown-plans/pushdown-plans.01.ddl.sqlpp | 2 + .../select-all-fields.1.ddl.sqlpp | 1 + .../select-count-one-field.1.ddl.sqlpp | 1 + .../string-standard-utf8.1.ddl.sqlpp | 1 + .../type-mismatch/type-mismatch.1.ddl.sqlpp | 1 + .../query-with-limit-plan/test.000.ddl.sqlpp | 1 + .../common/query-with-limit/test.001.ddl.sqlpp | 1 + .../external_dataset.000.ddl.sqlpp | 1 + .../common/tsv-header/query-dataset.001.ddl.sqlpp | 1 + .../tsv-no-header/query-dataset.001.ddl.sqlpp | 1 + .../tsv-warnings/query-dataset.002.ddl.sqlpp | 1 + .../common/tsv/gz/query-dataset.000.ddl.sqlpp | 1 + .../common/tsv/mixed/query-dataset.000.ddl.sqlpp | 1 + .../common/tsv/tsv/query-dataset.000.ddl.sqlpp | 1 + .../use-case-5/use-case-5.3.adm | 0 .../use-case-5/use-case-5.4.adm | 2 + .../parquet-null-type/parquet-null-type.05.adm | 10 +- .../parquet-null-type/parquet-null-type.08.adm | 10 +- .../copy-to/parquet-null2/parquet-null2.04.adm | 2 +- .../copy-to/parquet-null3/parquet-null3.04.adm | 50 + .../parquet-type-hierarchy.05.adm | 2 +- .../parquet/embed-one-value/one-field.110.adm | 2 +- .../array-access-pushdown.02.adm | 2 +- .../array-access-pushdown.04.adm | 2 +- .../array-access-pushdown.06.adm | 1 + .../array-access-pushdown.08.adm | 1 + .../parquet/missing-fields/missing-fields.3.adm | 4 +- .../parquet/null-in-array/null-in-array.02.adm | 10 + .../parquet-types/unset-flags/unset-flags.02.adm | 2 +- .../select-all-fields/select-all-fields.2.adm | 2 +- .../select-all-fields/select-all-fields.3.adm | 2 +- .../parquet/type-mismatch/type-mismatch.02.adm | 4 +- .../parquet/type-mismatch/type-mismatch.04.adm | 4 +- ...stsuite_external_dataset_azure_blob_storage.xml | 452 ++--- .../runtimets/testsuite_external_dataset_s3.xml | 441 ++--- .../resources/runtimets/testsuite_sqlpp_hdfs.xml | 22 +- .../asterix/cloud/AbstractCloudIOManager.java | 4 +- .../apache/asterix/cloud/EagerCloudIOManager.java | 2 +- .../apache/asterix/cloud/LazyCloudIOManager.java | 2 +- .../cloud/clients/AbstractParallelDownloader.java | 84 + .../asterix/cloud/clients/IParallelDownloader.java | 3 +- .../cloud/clients/aws/s3/S3ClientConfig.java | 45 +- .../cloud/clients/aws/s3/S3CloudClient.java | 12 +- .../cloud/clients/aws/s3/S3ParallelDownloader.java | 25 +- .../cloud/clients/aws/s3/S3SyncDownloader.java | 201 +++ .../blobstorage/AzBlobStorageClientConfig.java | 2 +- .../blobstorage/AzBlobStorageCloudClient.java | 16 +- .../azure/blobstorage/AzureParallelDownloader.java | 22 +- .../ReactiveExponentialRetryPolicy.java | 91 + .../cloud/clients/google/gcs/GCSClientConfig.java | 4 +- .../cloud/clients/google/gcs/GCSCloudClient.java | 2 +- .../clients/google/gcs/GCSParallelDownloader.java | 9 +- .../cloud/writer/AzureExternalFileWriter.java | 2 +- .../writer/AzureExternalFileWriterFactory.java | 8 +- .../cloud/writer/GCSExternalFileWriterFactory.java | 6 +- .../asterix/cloud/azure/LSMAzBlobStorageTest.java | 4 +- .../org/apache/asterix/cloud/s3/LSMS3Test.java | 4 +- .../asterix/common/config/CloudProperties.java | 24 +- asterixdb/asterix-external-data/pom.xml | 4 + .../reader/azure/blob/AzureBlobInputStream.java | 4 +- .../azure/blob/AzureBlobInputStreamFactory.java | 2 +- .../azure/datalake/AzureDataLakeInputStream.java | 4 +- .../datalake/AzureDataLakeInputStreamFactory.java | 17 +- .../parquet/AzureBlobParquetReaderFactory.java | 18 +- .../parquet/AzureDataLakeParquetReaderFactory.java | 57 +- .../input/record/reader/gcs/GCSInputStream.java | 4 +- .../record/reader/gcs/GCSInputStreamFactory.java | 2 +- .../reader/gcs/delta/GCSDeltaReaderFactory.java | 5 +- .../gcs/parquet/GCSParquetReaderFactory.java | 7 +- .../parquet/converter/ParquetConverterContext.java | 12 + .../parquet/converter/nested/ObjectConverter.java | 32 + .../converter/nested/ObjectRepeatedConverter.java | 32 +- .../converter/nested/RepeatedConverter.java | 11 +- .../primitve/PrimitiveRepeatedConverter.java | 2 +- .../external/util/ExternalDataConstants.java | 1 + .../asterix/external/util/ExternalDataUtils.java | 36 +- .../apache/asterix/external/util/aws/AwsUtils.java | 42 +- .../asterix/external/util/aws/s3/S3Constants.java | 10 +- .../asterix/external/util/aws/s3/S3Utils.java | 83 +- .../azure/{blob_storage => }/AzureConstants.java | 58 +- .../asterix/external/util/azure/AzureUtils.java | 116 ++ .../external/util/azure/blob/BlobConstants.java} | 7 +- .../external/util/azure/blob/BlobUtils.java | 344 ++++ .../util/azure/blob_storage/AzureUtils.java | 670 -------- .../util/azure/datalake/DatalakeConstants.java} | 32 +- .../util/azure/datalake/DatalakeUtils.java | 312 ++++ .../util/google/{gcs => }/GCSConstants.java | 2 +- .../{gcs/GCSAuthUtils.java => GCSUtils.java} | 166 +- .../asterix/external/util/google/gcs/GCSUtils.java | 150 -- .../org/apache/http/client/utils/URIBuilder.java | 49 + .../org/apache/hyracks/api/util/CleanupUtils.java | 8 + .../cloud/util/CloudRetryableRequestUtil.java | 4 +- .../hyracks/util/ExponentialRetryPolicy.java | 12 + 365 files changed, 5268 insertions(+), 2391 deletions(-) diff --cc asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java index 224ba34d8e,de31531de4..37946103e5 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java @@@ -18,8 -18,7 +18,9 @@@ */ package org.apache.asterix.external.util; + import static org.apache.asterix.common.exceptions.ErrorCode.INVALID_PARAM_VALUE_ALLOWED_VALUE; +import static org.apache.asterix.common.exceptions.ErrorCode.PARSE_ERROR; +import static org.apache.asterix.common.exceptions.ErrorCode.PROPERTY_INVALID_VALUE_TYPE; import static org.apache.asterix.common.metadata.MetadataConstants.DEFAULT_DATABASE; import static org.apache.asterix.common.utils.CSVConstants.KEY_DELIMITER; import static org.apache.asterix.common.utils.CSVConstants.KEY_EMPTY_STRING_AS_NULL; @@@ -88,14 -86,9 +90,14 @@@ import org.apache.asterix.external.util import org.apache.asterix.external.util.aws.AwsConstants; import org.apache.asterix.external.util.aws.s3.S3Constants; import org.apache.asterix.external.util.aws.s3.S3Utils; - import org.apache.asterix.external.util.azure.blob_storage.AzureConstants; - import org.apache.asterix.external.util.google.gcs.GCSConstants; - import org.apache.asterix.external.util.google.gcs.GCSUtils; + import org.apache.asterix.external.util.azure.AzureConstants; + import org.apache.asterix.external.util.google.GCSConstants; + import org.apache.asterix.external.util.google.GCSUtils; +import org.apache.asterix.object.base.AdmArrayNode; +import org.apache.asterix.object.base.AdmBooleanNode; +import org.apache.asterix.object.base.AdmObjectNode; +import org.apache.asterix.object.base.AdmStringNode; +import org.apache.asterix.object.base.IAdmNode; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.ATypeTag; import org.apache.asterix.om.types.AUnionType; @@@ -1202,38 -1193,19 +1202,54 @@@ public class ExternalDataUtils return ExternalDataConstants.KEY_COMPRESSION_GZIP.equalsIgnoreCase(compression); } + public static boolean getDisableSslVerify(Map<String, String> configuration) throws CompilationException { + String disableSslVerifyString = configuration.get(DISABLE_SSL_VERIFY_FIELD_NAME); + return validateAndGetBooleanProperty(DISABLE_SSL_VERIFY_FIELD_NAME, disableSslVerifyString); + } + + public static boolean validateAndGetBooleanProperty(String propertyKey, String propertyValue) + throws CompilationException { + if (propertyValue == null) { + return false; + } + if (!"true".equalsIgnoreCase(propertyValue) && !"false".equalsIgnoreCase(propertyValue)) { + throw new CompilationException(INVALID_PARAM_VALUE_ALLOWED_VALUE, propertyKey, "true, false"); + } + return "true".equalsIgnoreCase(propertyValue); + } ++ + public static Map<String, String> convertStringArrayParamIntoNumberedParameters(AdmObjectNode withObjectNode, + SourceLocation sourceLocation) throws CompilationException { + Map<String, String> properties = new HashMap<>(); + for (Map.Entry<String, IAdmNode> me : withObjectNode.getFields()) { + String fieldName = me.getKey(); + IAdmNode fieldValue = me.getValue(); + String fieldValueStr; + switch (fieldValue.getType()) { + case STRING: + fieldValueStr = ((AdmStringNode) fieldValue).get(); + break; + case BOOLEAN: + fieldValueStr = Boolean.toString(((AdmBooleanNode) fieldValue).get()); + break; + case ARRAY: + AdmArrayNode arrayNode = (AdmArrayNode) fieldValue; + Iterator<IAdmNode> iterator = arrayNode.iterator(); + int i = 0; + while (iterator.hasNext()) { + IAdmNode node = iterator.next(); + if (node.getType() != ATypeTag.STRING) { + throw new CompilationException(PROPERTY_INVALID_VALUE_TYPE, fieldName, + ATypeTag.STRING.toString()); + } + properties.put(fieldName + "#" + i++, ((AdmStringNode) node).get()); + } + continue; + default: + throw new CompilationException(PARSE_ERROR, sourceLocation, fieldName); + } + properties.put(fieldName, fieldValueStr); + } + return properties; + } }
