This is an automated email from the ASF dual-hosted git repository. wyk pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push: new 087eaf5 [ASTERIXDB-2955] Upgrade to Hadoop3 087eaf5 is described below commit 087eaf55aef22aef414c22f1f9108c58c781437f Author: Wail Alkowaileet <wael....@gmail.com> AuthorDate: Tue Aug 31 10:36:27 2021 -0700 [ASTERIXDB-2955] Upgrade to Hadoop3 - user model changes: no - storage format changes: no - interface changes: no Details: Upgrade to Hadoop v3 Change-Id: Ia212ddbe5eed114042118fecc4cc08db93cc4e09 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12623 Reviewed-by: Wael Alkowaileet <wael....@gmail.com> Reviewed-by: Michael Blow <mb...@apache.org> Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu> --- .../external/library/java/base/JInterval.java | 2 +- .../asterix/external/util/ExternalDataUtils.java | 6 +-- asterixdb/asterix-server/pom.xml | 24 ++++----- asterixdb/pom.xml | 39 +++++++++++++- .../appended-resources/supplemental-models.xml | 63 +++------------------- .../main/licenses/content/golang.org_LICENSE.txt | 27 ++++++++++ hyracks-fullstack/pom.xml | 2 +- 7 files changed, 85 insertions(+), 78 deletions(-) diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JInterval.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JInterval.java index 14caa1e..39a8909 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JInterval.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JInterval.java @@ -27,7 +27,7 @@ import org.apache.asterix.om.base.AMutableInterval; import org.apache.asterix.om.types.ATypeTag; import org.apache.asterix.om.types.BuiltinType; import org.apache.asterix.om.types.IAType; -import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.lang3.ArrayUtils; import org.apache.hyracks.api.exceptions.HyracksDataException; public final class JInterval extends JObject<List<Long>> { diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java index dacf464..08d803e 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java @@ -41,7 +41,6 @@ import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.C import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CLIENT_SECRET_FIELD_NAME; import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CONNECTION_STRING_FIELD_NAME; import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.TENANT_ID_FIELD_NAME; -import static org.apache.asterix.external.util.ExternalDataConstants.CONTAINER_NAME_FIELD_NAME; import static org.apache.asterix.external.util.ExternalDataConstants.GCS.JSON_CREDENTIALS_FIELD_NAME; import static org.apache.asterix.external.util.ExternalDataConstants.KEY_ADAPTER_NAME_GCS; import static org.apache.asterix.external.util.ExternalDataConstants.KEY_DELIMITER; @@ -92,6 +91,7 @@ import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.ATypeTag; import org.apache.asterix.om.types.AUnionType; import org.apache.asterix.runtime.evaluators.common.NumberUtils; +import org.apache.hadoop.fs.s3a.Constants; import org.apache.hadoop.mapred.JobConf; import org.apache.hyracks.algebricks.common.exceptions.NotImplementedException; import org.apache.hyracks.api.exceptions.HyracksDataException; @@ -878,7 +878,6 @@ public class ExternalDataUtils { String accessKeyId = configuration.get(ExternalDataConstants.AwsS3.ACCESS_KEY_ID_FIELD_NAME); String secretAccessKey = configuration.get(ExternalDataConstants.AwsS3.SECRET_ACCESS_KEY_FIELD_NAME); String sessionToken = configuration.get(ExternalDataConstants.AwsS3.SESSION_TOKEN_FIELD_NAME); - String regionId = configuration.get(ExternalDataConstants.AwsS3.REGION_FIELD_NAME); String serviceEndpoint = configuration.get(ExternalDataConstants.AwsS3.SERVICE_END_POINT_FIELD_NAME); /* @@ -915,7 +914,8 @@ public class ExternalDataUtils { // Validation of the URL should be done at hadoop-aws level conf.set(ExternalDataConstants.AwsS3.HADOOP_SERVICE_END_POINT, serviceEndpoint); } else { - conf.set(ExternalDataConstants.AwsS3.HADOOP_REGION, regionId); + //Region is ignored and buckets could be found by the central endpoint + conf.set(ExternalDataConstants.AwsS3.HADOOP_SERVICE_END_POINT, Constants.CENTRAL_ENDPOINT); } } diff --git a/asterixdb/asterix-server/pom.xml b/asterixdb/asterix-server/pom.xml index e76814c..96007c1 100644 --- a/asterixdb/asterix-server/pom.xml +++ b/asterixdb/asterix-server/pom.xml @@ -199,25 +199,11 @@ </gavs> <noticeUrl>https://raw.githubusercontent.com/aws/aws-sdk-java-v2/2.10.83/NOTICE.txt</noticeUrl> </override> - <!-- Hadoop AWS SDK --> <override> - <gavs> - <gav>com.amazonaws:aws-java-sdk-core:1.12.1</gav> - <gav>com.amazonaws:jmespath-java:1.12.1</gav> - <gav>com.amazonaws:aws-java-sdk-s3:1.12.1</gav> - <gav>com.amazonaws:aws-java-sdk-kms:1.12.1</gav> - </gavs> + <gav>com.amazonaws:aws-java-sdk-bundle:1.12.1</gav> <noticeUrl>https://raw.githubusercontent.com/aws/aws-sdk-java/1.12.1/NOTICE.txt</noticeUrl> </override> <override> - <gav>software.amazon.eventstream:eventstream:1.0.1</gav> - <noticeUrl>https://raw.githubusercontent.com/awslabs/aws-eventstream-java/7be2dd80e12f8835674c8ffb0f4a2efb64c7b585/NOTICE</noticeUrl> - </override> - <override> - <gav>software.amazon.ion:ion-java:1.0.2</gav> - <noticeUrl>https://raw.githubusercontent.com/amzn/ion-java/v1.0.2/NOTICE</noticeUrl> - </override> - <override> <gavs> <gav>com.azure:azure-core:1.17.0</gav> <gav>com.azure:azure-core-http-netty:1.10.0</gav> @@ -456,6 +442,7 @@ <aliasUrl>https://raw.githubusercontent.com/googleapis/java-common-protos/v2.1.0/proto-google-common-protos/LICENSE</aliasUrl> <aliasUrl>https://raw.githubusercontent.com/googleapis/java-iam/v1.0.13/proto-google-iam-v1/LICENSE</aliasUrl> <aliasUrl>https://raw.githubusercontent.com/googleapis/java-storage/v1.114.0/LICENSE</aliasUrl> + <aliasUrl>http://repository.jboss.org/licenses/apache-2.0.txt</aliasUrl> </aliasUrls> <metric>1</metric> </license> @@ -528,6 +515,13 @@ <aliasUrl>https://raw.githubusercontent.com/luben/zstd-jni/v1.4.9-1/LICENSE</aliasUrl> </aliasUrls> </license> + <license> + <displayName>The Go license</displayName> + <url>http://golang.org/LICENSE</url> + <aliasUrls> + <aliasUrl>https://golang.org/LICENSE</aliasUrl> + </aliasUrls> + </license> </licenses> <templateProperties> <packageName>Apache AsterixDB Server Install</packageName> diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml index 7fc4e26..71228df 100644 --- a/asterixdb/pom.xml +++ b/asterixdb/pom.xml @@ -84,7 +84,7 @@ <!-- Versions under dependencymanagement or used in many projects via properties --> <algebricks.version>0.3.7-SNAPSHOT</algebricks.version> <hyracks.version>0.3.7-SNAPSHOT</hyracks.version> - <hadoop.version>2.8.5</hadoop.version> + <hadoop.version>3.3.1</hadoop.version> <jacoco.version>0.7.6.201602180812</jacoco.version> <log4j.version>2.14.1</log4j.version> <awsjavasdk.version>2.10.83</awsjavasdk.version> @@ -989,7 +989,7 @@ <artifactId>commons-logging</artifactId> </exclusion> <exclusion> - <groupId>stax</groupId> + <groupId>org.codehaus.woodstox</groupId> <artifactId>stax-api</artifactId> </exclusion> <exclusion> @@ -1008,6 +1008,26 @@ <groupId>log4j</groupId> <artifactId>log4j</artifactId> </exclusion> + <exclusion> + <groupId>com.fasterxml</groupId> + <artifactId>woodstox-core</artifactId> + </exclusion> + <exclusion> + <groupId>dnsjava</groupId> + <artifactId>dnsjava</artifactId> + </exclusion> + <exclusion> + <groupId>org.wildfly.openssl</groupId> + <artifactId>wildfly-openssl</artifactId> + </exclusion> + <exclusion> + <groupId>org.jline</groupId> + <artifactId>jline</artifactId> + </exclusion> + <exclusion> + <groupId>javax.ws.rs</groupId> + <artifactId>jsr311-api</artifactId> + </exclusion> </exclusions> </dependency> <dependency> @@ -1046,6 +1066,10 @@ <groupId>log4j</groupId> <artifactId>log4j</artifactId> </exclusion> + <exclusion> + <groupId>org.jline</groupId> + <artifactId>jline</artifactId> + </exclusion> </exclusions> </dependency> <dependency> @@ -1767,6 +1791,17 @@ </exclusions> </dependency> <dependency> + <groupId>com.amazonaws</groupId> + <artifactId>aws-java-sdk-bundle</artifactId> + <version>${hadoop-awsjavasdk.version}</version> + <exclusions> + <exclusion> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-aws</artifactId> <version>${hadoop.version}</version> diff --git a/asterixdb/src/main/appended-resources/supplemental-models.xml b/asterixdb/src/main/appended-resources/supplemental-models.xml index 305176b..f99dc60 100644 --- a/asterixdb/src/main/appended-resources/supplemental-models.xml +++ b/asterixdb/src/main/appended-resources/supplemental-models.xml @@ -485,68 +485,19 @@ <!-- AWS SDK end --> <!-- AWS Hadoop SDK start --> - <!-- software.amazon.awssdk is ALv2, and does not contain any embedded LICENSE or NOTICE file --> - <!-- license override not needed, ALv2 is specified in its pom.xml --> - <!-- see https://github.com/aws/aws-sdk-java-v2/blob/master/LICENSE.txt --> - <supplement> - <project> - <groupId>com.amazonaws</groupId> - <artifactId>aws-java-sdk-core</artifactId> - <properties> - <license.ignoreMissingEmbeddedLicense>1.12.1</license.ignoreMissingEmbeddedLicense> - <license.ignoreMissingEmbeddedNotice>1.12.1</license.ignoreMissingEmbeddedNotice> - <license.ignoreNoticeOverride>1.12.1</license.ignoreNoticeOverride> - </properties> - </project> - </supplement> - - <supplement> - <project> - <groupId>com.amazonaws</groupId> - <artifactId>jmespath-java</artifactId> - <properties> - <license.ignoreMissingEmbeddedLicense>1.12.1</license.ignoreMissingEmbeddedLicense> - <license.ignoreMissingEmbeddedNotice>1.12.1</license.ignoreMissingEmbeddedNotice> - <license.ignoreNoticeOverride>1.12.1</license.ignoreNoticeOverride> - </properties> - </project> - </supplement> - - <supplement> - <project> - <groupId>com.amazonaws</groupId> - <artifactId>aws-java-sdk-s3</artifactId> - <properties> - <license.ignoreMissingEmbeddedLicense>1.12.1</license.ignoreMissingEmbeddedLicense> - <license.ignoreMissingEmbeddedNotice>1.12.1</license.ignoreMissingEmbeddedNotice> - <license.ignoreNoticeOverride>1.12.1</license.ignoreNoticeOverride> - </properties> - </project> - </supplement> - + <!-- Contains (LICENSE and LICENSE.txt) and (NOTICE and NOTICE.txt) --> + <!-- Overridden to ignore the warning of having multiple licenses and notices --> <supplement> <project> <groupId>com.amazonaws</groupId> - <artifactId>aws-java-sdk-kms</artifactId> + <artifactId>aws-java-sdk-bundle</artifactId> <properties> <license.ignoreMissingEmbeddedLicense>1.12.1</license.ignoreMissingEmbeddedLicense> <license.ignoreMissingEmbeddedNotice>1.12.1</license.ignoreMissingEmbeddedNotice> + <license.ignoreLicenseOverride>1.12.1</license.ignoreLicenseOverride> <license.ignoreNoticeOverride>1.12.1</license.ignoreNoticeOverride> - </properties> - </project> - </supplement> - - <!-- software.amazon.ion is ALv2, and does not contain any embedded LICENSE or NOTICE file --> - <!-- license override not needed, ALv2 is specified in its pom.xml --> - <!-- see https://github.com/amzn/ion-java --> - <supplement> - <project> - <groupId>software.amazon.ion</groupId> - <artifactId>ion-java</artifactId> - <properties> - <license.ignoreMissingEmbeddedLicense>1.0.2</license.ignoreMissingEmbeddedLicense> - <license.ignoreMissingEmbeddedNotice>1.0.2</license.ignoreMissingEmbeddedNotice> - <license.ignoreNoticeOverride>1.0.2</license.ignoreNoticeOverride> + <license.alternateLicenseFile>1.12.1:www.apache.org_licenses_LICENSE-2.0.txt</license.alternateLicenseFile> + <license.alternateNoticeFile>1.12.1:raw.githubusercontent.com_aws_aws-sdk-java_1.12.1_NOTICE.txt</license.alternateNoticeFile> </properties> </project> </supplement> @@ -561,7 +512,7 @@ <artifactId>netty-reactive-streams</artifactId> <properties> <license.ignoreMissingEmbeddedLicense>2.0.4</license.ignoreMissingEmbeddedLicense> - <license.ignoreMissingEmbeddedNotice>2.0.4</license.ignoreMissingEmbeddedNotice> + <license.ignoreMissingEmbeddedNotice>2.0.4</license.ignoreMissingEmbeddedNotice>doesn </properties> </project> </supplement> diff --git a/asterixdb/src/main/licenses/content/golang.org_LICENSE.txt b/asterixdb/src/main/licenses/content/golang.org_LICENSE.txt new file mode 100644 index 0000000..ea5ea89 --- /dev/null +++ b/asterixdb/src/main/licenses/content/golang.org_LICENSE.txt @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/hyracks-fullstack/pom.xml b/hyracks-fullstack/pom.xml index 15c1c15..5f83aab 100644 --- a/hyracks-fullstack/pom.xml +++ b/hyracks-fullstack/pom.xml @@ -70,7 +70,7 @@ <test.includes>${global.test.includes}</test.includes> <test.excludes>${global.test.excludes}</test.excludes> <!-- Versions under dependencymanagement or used in many projects via properties --> - <hadoop.version>2.8.5</hadoop.version> + <hadoop.version>3.3.1</hadoop.version> <jacoco.version>0.7.6.201602180812</jacoco.version> <log4j.version>2.14.1</log4j.version> <snappy.version>1.1.8.4</snappy.version>