[
https://issues.apache.org/jira/browse/HADOOP-14837?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17828513#comment-17828513
]
ASF GitHub Bot commented on HADOOP-14837:
-----------------------------------------
steveloughran commented on code in PR #6407:
URL: https://github.com/apache/hadoop/pull/6407#discussion_r1530919362
##########
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/S3ObjectStorageClassFilter.java:
##########
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.api;
+
+import java.util.Set;
+import java.util.function.Function;
+
+import software.amazon.awssdk.services.s3.model.ObjectStorageClass;
+import software.amazon.awssdk.services.s3.model.S3Object;
+
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
+
+
+/**
+ * <pre>
+ * {@link S3ObjectStorageClassFilter} will filter the S3 files based on the
+ * {@code fs.s3a.glacier.read.restored.objects} configuration set in {@link
S3AFileSystem}
+ * The config can have 3 values:
+ * {@code READ_ALL}: Retrieval of Glacier files will fail with
InvalidObjectStateException:
+ * The operation is not valid for the object's storage class.
+ * {@code SKIP_ALL_GLACIER}: If this value is set then this will ignore any S3
Objects which are
+ * tagged with Glacier storage classes and retrieve the others.
+ * {@code READ_RESTORED_GLACIER_OBJECTS}: If this value is set then restored
status of the Glacier
+ * object will be checked, if restored the objects would be read like normal
S3 objects
+ * else they will be ignored as the objects would not have been retrieved from
the S3 Glacier.
+ * </pre>
+ */
+public enum S3ObjectStorageClassFilter {
+ READ_ALL(o -> true),
+ SKIP_ALL_GLACIER(S3ObjectStorageClassFilter::isNotGlacierObject),
+
READ_RESTORED_GLACIER_OBJECTS(S3ObjectStorageClassFilter::isCompletedRestoredObject);
+
+ private static final Set<ObjectStorageClass> GLACIER_STORAGE_CLASSES =
Sets.newHashSet(
+ ObjectStorageClass.GLACIER, ObjectStorageClass.DEEP_ARCHIVE);
+
+ private final Function<S3Object, Boolean> filter;
+
+ S3ObjectStorageClassFilter(Function<S3Object, Boolean> filter) {
Review Comment:
should this be private, or it implicit with enums?
##########
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/S3ObjectStorageClassFilter.java:
##########
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.api;
+
+import java.util.Set;
+import java.util.function.Function;
+
+import software.amazon.awssdk.services.s3.model.ObjectStorageClass;
+import software.amazon.awssdk.services.s3.model.S3Object;
+
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
Review Comment:
could you use org.apache.hadoop.util.Sets here. its part of our attempt to
isolate ourselves better from guava changes and the pain that causes downstream
##########
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/list/ITestS3AReadRestoredGlacierObjects.java:
##########
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.list;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Assume;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.RestoreObjectRequest;
+import software.amazon.awssdk.services.s3.model.S3Object;
+import software.amazon.awssdk.services.s3.model.Tier;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.contract.s3a.S3AContract;
+import org.apache.hadoop.fs.s3a.AbstractS3ATestBase;
+import org.apache.hadoop.fs.s3a.S3ListRequest;
+import org.apache.hadoop.fs.s3a.api.S3ObjectStorageClassFilter;
+import org.apache.hadoop.fs.store.audit.AuditSpan;
+import org.apache.hadoop.test.LambdaTestUtils;
+
+import static org.apache.hadoop.fs.s3a.Constants.READ_RESTORED_GLACIER_OBJECTS;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS_DEEP_ARCHIVE;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS_GLACIER;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching;
+import static
org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
+import static
org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfStorageClassTestsDisabled;
+import static
org.apache.hadoop.fs.statistics.StoreStatisticNames.OBJECT_LIST_REQUEST;
+
+
+/**
+ * Tests of various cases related to Glacier/Deep Archive Storage class.
+ */
+@RunWith(Parameterized.class)
+public class ITestS3AReadRestoredGlacierObjects extends AbstractS3ATestBase {
+
+ enum Type { GLACIER_AND_DEEP_ARCHIVE, GLACIER }
+
+ @Parameterized.Parameters(name = "storage-class-{1}")
+ public static Collection<Object[]> data(){
+ return Arrays.asList(new Object[][] {
+ {Type.GLACIER_AND_DEEP_ARCHIVE, STORAGE_CLASS_GLACIER},
+ {Type.GLACIER_AND_DEEP_ARCHIVE, STORAGE_CLASS_DEEP_ARCHIVE},
+ {Type.GLACIER, STORAGE_CLASS_GLACIER}
+ });
+ }
+
+ private final int maxRetries = 100;
+ private final int retryDelayMs = 5000;
+
+ private final Type type;
+ private final String glacierClass;
+
+ public ITestS3AReadRestoredGlacierObjects(Type type, String glacierClass) {
+ this.type = type;
+ this.glacierClass = glacierClass;
+ }
+
+ private FileSystem createFiles(String s3ObjectStorageClassFilter) throws
Throwable {
+ Configuration conf = this.createConfiguration();
+ conf.set(READ_RESTORED_GLACIER_OBJECTS, s3ObjectStorageClassFilter);
+ // Create Glacier objects:Storage Class:DEEP_ARCHIVE/GLACIER
+ conf.set(STORAGE_CLASS, glacierClass);
+ S3AContract contract = (S3AContract) createContract(conf);
+ contract.init();
+
+ FileSystem fs = contract.getTestFileSystem();
+ Path path = new Path(methodPath(), "glaciated");
+ ContractTestUtils.touch(fs, path);
+ return fs;
+ }
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration newConf = super.createConfiguration();
+ skipIfStorageClassTestsDisabled(newConf);
+ disableFilesystemCaching(newConf);
+ removeBaseAndBucketOverrides(newConf, STORAGE_CLASS,
READ_RESTORED_GLACIER_OBJECTS);
+ return newConf;
+ }
+
+ @Test
+ public void testConfigWithInvalidValue() throws Throwable {
+ Assume.assumeTrue(type == Type.GLACIER_AND_DEEP_ARCHIVE);
+ String invalidValue = "ABCDE";
+ try (FileSystem fs = createFiles(invalidValue)) {
+ Assertions.assertThat(
+ fs.listStatus(methodPath()))
+ .describedAs("FileStatus List of %s", methodPath()).isNotEmpty();
+ }
+ }
+
+ @Test
+ public void testIgnoreGlacierObject() throws Throwable {
+ Assume.assumeTrue(type == Type.GLACIER_AND_DEEP_ARCHIVE);
+ try (FileSystem fs =
createFiles(S3ObjectStorageClassFilter.SKIP_ALL_GLACIER.name())) {
+ Assertions.assertThat(
+ fs.listStatus(methodPath()))
+ .describedAs("FileStatus List of %s", methodPath()).isEmpty();
+ }
+ }
+
+ @Test
+ public void testIgnoreRestoringGlacierObject() throws Throwable {
+ Assume.assumeTrue(type == Type.GLACIER_AND_DEEP_ARCHIVE);
+ try (FileSystem fs =
createFiles(S3ObjectStorageClassFilter.READ_RESTORED_GLACIER_OBJECTS.name())) {
+ Assertions.assertThat(
+ fs.listStatus(
+ methodPath()))
+ .describedAs("FileStatus List of %s", methodPath()).isEmpty();
+ }
+ }
+
+ @Test
+ public void testRestoredGlacierObject() throws Throwable {
+ // Skipping this test for Deep Archive as expedited retrieval is not
supported
+ Assume.assumeTrue(type == Type.GLACIER);
+ try (FileSystem fs =
createFiles(S3ObjectStorageClassFilter.READ_RESTORED_GLACIER_OBJECTS.name())) {
+ restoreGlacierObject(getFilePrefixForListObjects(), 2);
+ Assertions.assertThat(
+ fs.listStatus(
+ methodPath()))
+ .describedAs("FileStatus List of %s", methodPath()).isNotEmpty();
+ }
+ }
+
+ @Test
+ public void testReadAllObjects() throws Throwable {
+ Assume.assumeTrue(type == Type.GLACIER_AND_DEEP_ARCHIVE);
+ try (FileSystem fs =
createFiles(S3ObjectStorageClassFilter.READ_ALL.name())) {
+ Assertions.assertThat(
+ fs.listStatus(methodPath()))
+ .describedAs("FileStatus List of %s", methodPath()).isNotEmpty();
+ }
+ }
+
+
+ private void restoreGlacierObject(String glacierObjectKey, int
expirationDays) throws Exception {
+ try (AuditSpan auditSpan = getSpanSource().createSpan(OBJECT_LIST_REQUEST,
"", "").activate()) {
+
Review Comment:
so there's actually a span() method in the test superclass, an need to worry
about closing it, but you've learned more about audit spans...
##########
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/list/ITestS3AReadRestoredGlacierObjects.java:
##########
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.list;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Assume;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.RestoreObjectRequest;
+import software.amazon.awssdk.services.s3.model.S3Object;
+import software.amazon.awssdk.services.s3.model.Tier;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.contract.s3a.S3AContract;
+import org.apache.hadoop.fs.s3a.AbstractS3ATestBase;
+import org.apache.hadoop.fs.s3a.S3ListRequest;
+import org.apache.hadoop.fs.s3a.api.S3ObjectStorageClassFilter;
+import org.apache.hadoop.fs.store.audit.AuditSpan;
+import org.apache.hadoop.test.LambdaTestUtils;
+
+import static org.apache.hadoop.fs.s3a.Constants.READ_RESTORED_GLACIER_OBJECTS;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS_DEEP_ARCHIVE;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS_GLACIER;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching;
+import static
org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
+import static
org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfStorageClassTestsDisabled;
+import static
org.apache.hadoop.fs.statistics.StoreStatisticNames.OBJECT_LIST_REQUEST;
+
+
+/**
+ * Tests of various cases related to Glacier/Deep Archive Storage class.
+ */
+@RunWith(Parameterized.class)
+public class ITestS3AReadRestoredGlacierObjects extends AbstractS3ATestBase {
+
+ enum Type { GLACIER_AND_DEEP_ARCHIVE, GLACIER }
+
+ @Parameterized.Parameters(name = "storage-class-{1}")
+ public static Collection<Object[]> data(){
+ return Arrays.asList(new Object[][] {
+ {Type.GLACIER_AND_DEEP_ARCHIVE, STORAGE_CLASS_GLACIER},
+ {Type.GLACIER_AND_DEEP_ARCHIVE, STORAGE_CLASS_DEEP_ARCHIVE},
+ {Type.GLACIER, STORAGE_CLASS_GLACIER}
+ });
+ }
+
+ private final int maxRetries = 100;
+ private final int retryDelayMs = 5000;
+
+ private final Type type;
+ private final String glacierClass;
+
+ public ITestS3AReadRestoredGlacierObjects(Type type, String glacierClass) {
+ this.type = type;
+ this.glacierClass = glacierClass;
+ }
+
+ private FileSystem createFiles(String s3ObjectStorageClassFilter) throws
Throwable {
+ Configuration conf = this.createConfiguration();
+ conf.set(READ_RESTORED_GLACIER_OBJECTS, s3ObjectStorageClassFilter);
+ // Create Glacier objects:Storage Class:DEEP_ARCHIVE/GLACIER
+ conf.set(STORAGE_CLASS, glacierClass);
+ S3AContract contract = (S3AContract) createContract(conf);
Review Comment:
you don't need to do this. just do
```
fs = new S3AFileSystem()
fs.init(conf)
```
##########
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/list/ITestS3AReadRestoredGlacierObjects.java:
##########
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.list;
+
+import static org.apache.hadoop.fs.s3a.Constants.READ_RESTORED_GLACIER_OBJECTS;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS_DEEP_ARCHIVE;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS_GLACIER;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching;
+import static
org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
+import static
org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfStorageClassTestsDisabled;
+import static
org.apache.hadoop.fs.statistics.StoreStatisticNames.OBJECT_LIST_REQUEST;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.contract.s3a.S3AContract;
+import org.apache.hadoop.fs.s3a.AbstractS3ATestBase;
+import org.apache.hadoop.fs.s3a.S3ListRequest;
+import org.apache.hadoop.fs.s3a.S3ObjectStorageClassFilter;
+import org.apache.hadoop.fs.store.audit.AuditSpan;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.assertj.core.api.Assertions;
+import org.junit.Assume;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.RestoreObjectRequest;
+import software.amazon.awssdk.services.s3.model.S3Object;
+import software.amazon.awssdk.services.s3.model.Tier;
+
+/**
+ * Tests of various cases related to Glacier/Deep Archive Storage class.
+ */
+@RunWith(Parameterized.class)
+public class ITestS3AReadRestoredGlacierObjects extends AbstractS3ATestBase {
+
+ enum Type { GLACIER_AND_DEEP_ARCHIVE, GLACIER }
+
+ @Parameterized.Parameters(name = "storage-class-{1}")
+ public static Collection<Object[]> data(){
+ return Arrays.asList(new Object[][] {
+ {Type.GLACIER_AND_DEEP_ARCHIVE, STORAGE_CLASS_GLACIER},
+ {Type.GLACIER_AND_DEEP_ARCHIVE, STORAGE_CLASS_DEEP_ARCHIVE},
+ {Type.GLACIER, STORAGE_CLASS_GLACIER}
+ });
+ }
+
+ private final int maxRetries = 100;
+ private final int retryDelayMs = 5000;
+
+ private final Type type;
+ private final String glacierClass;
+
+ public ITestS3AReadRestoredGlacierObjects(Type type, String glacierClass) {
+ this.type = type;
+ this.glacierClass = glacierClass;
+ }
+
+ private FileSystem createFiles(String s3ObjectStorageClassFilter) throws
Throwable {
+ Configuration conf = this.createConfiguration();
Review Comment:
sorry I meant "remove the `this.` prefix"; just use createConfiguration()
directly
##########
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/list/ITestS3AReadRestoredGlacierObjects.java:
##########
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.list;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Assume;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.RestoreObjectRequest;
+import software.amazon.awssdk.services.s3.model.S3Object;
+import software.amazon.awssdk.services.s3.model.Tier;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.contract.s3a.S3AContract;
+import org.apache.hadoop.fs.s3a.AbstractS3ATestBase;
+import org.apache.hadoop.fs.s3a.S3ListRequest;
+import org.apache.hadoop.fs.s3a.api.S3ObjectStorageClassFilter;
+import org.apache.hadoop.fs.store.audit.AuditSpan;
+import org.apache.hadoop.test.LambdaTestUtils;
+
+import static org.apache.hadoop.fs.s3a.Constants.READ_RESTORED_GLACIER_OBJECTS;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS_DEEP_ARCHIVE;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS_GLACIER;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching;
+import static
org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
+import static
org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfStorageClassTestsDisabled;
+import static
org.apache.hadoop.fs.statistics.StoreStatisticNames.OBJECT_LIST_REQUEST;
+
+
+/**
+ * Tests of various cases related to Glacier/Deep Archive Storage class.
+ */
+@RunWith(Parameterized.class)
+public class ITestS3AReadRestoredGlacierObjects extends AbstractS3ATestBase {
+
+ enum Type { GLACIER_AND_DEEP_ARCHIVE, GLACIER }
+
+ @Parameterized.Parameters(name = "storage-class-{1}")
+ public static Collection<Object[]> data(){
+ return Arrays.asList(new Object[][] {
+ {Type.GLACIER_AND_DEEP_ARCHIVE, STORAGE_CLASS_GLACIER},
+ {Type.GLACIER_AND_DEEP_ARCHIVE, STORAGE_CLASS_DEEP_ARCHIVE},
+ {Type.GLACIER, STORAGE_CLASS_GLACIER}
+ });
+ }
+
+ private final int maxRetries = 100;
+ private final int retryDelayMs = 5000;
+
+ private final Type type;
+ private final String glacierClass;
+
+ public ITestS3AReadRestoredGlacierObjects(Type type, String glacierClass) {
+ this.type = type;
+ this.glacierClass = glacierClass;
+ }
+
+ private FileSystem createFiles(String s3ObjectStorageClassFilter) throws
Throwable {
+ Configuration conf = this.createConfiguration();
+ conf.set(READ_RESTORED_GLACIER_OBJECTS, s3ObjectStorageClassFilter);
+ // Create Glacier objects:Storage Class:DEEP_ARCHIVE/GLACIER
+ conf.set(STORAGE_CLASS, glacierClass);
+ S3AContract contract = (S3AContract) createContract(conf);
+ contract.init();
+
+ FileSystem fs = contract.getTestFileSystem();
+ Path path = new Path(methodPath(), "glaciated");
+ ContractTestUtils.touch(fs, path);
+ return fs;
+ }
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration newConf = super.createConfiguration();
+ skipIfStorageClassTestsDisabled(newConf);
+ disableFilesystemCaching(newConf);
+ removeBaseAndBucketOverrides(newConf, STORAGE_CLASS,
READ_RESTORED_GLACIER_OBJECTS);
+ return newConf;
+ }
+
+ @Test
+ public void testConfigWithInvalidValue() throws Throwable {
+ Assume.assumeTrue(type == Type.GLACIER_AND_DEEP_ARCHIVE);
+ String invalidValue = "ABCDE";
+ try (FileSystem fs = createFiles(invalidValue)) {
+ Assertions.assertThat(
+ fs.listStatus(methodPath()))
+ .describedAs("FileStatus List of %s", methodPath()).isNotEmpty();
+ }
+ }
+
+ @Test
+ public void testIgnoreGlacierObject() throws Throwable {
+ Assume.assumeTrue(type == Type.GLACIER_AND_DEEP_ARCHIVE);
+ try (FileSystem fs =
createFiles(S3ObjectStorageClassFilter.SKIP_ALL_GLACIER.name())) {
+ Assertions.assertThat(
+ fs.listStatus(methodPath()))
+ .describedAs("FileStatus List of %s", methodPath()).isEmpty();
Review Comment:
can you put the .isEmpty() on the line below; we like to try and split them
up as they're very complex to read.
##########
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/S3ObjectStorageClassFilter.java:
##########
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.api;
+
+import java.util.Set;
+import java.util.function.Function;
+
+import software.amazon.awssdk.services.s3.model.ObjectStorageClass;
+import software.amazon.awssdk.services.s3.model.S3Object;
+
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
+
+
+/**
+ * <pre>
+ * {@link S3ObjectStorageClassFilter} will filter the S3 files based on the
+ * {@code fs.s3a.glacier.read.restored.objects} configuration set in {@link
S3AFileSystem}
+ * The config can have 3 values:
+ * {@code READ_ALL}: Retrieval of Glacier files will fail with
InvalidObjectStateException:
+ * The operation is not valid for the object's storage class.
+ * {@code SKIP_ALL_GLACIER}: If this value is set then this will ignore any S3
Objects which are
+ * tagged with Glacier storage classes and retrieve the others.
+ * {@code READ_RESTORED_GLACIER_OBJECTS}: If this value is set then restored
status of the Glacier
+ * object will be checked, if restored the objects would be read like normal
S3 objects
+ * else they will be ignored as the objects would not have been retrieved from
the S3 Glacier.
+ * </pre>
+ */
+public enum S3ObjectStorageClassFilter {
+ READ_ALL(o -> true),
+ SKIP_ALL_GLACIER(S3ObjectStorageClassFilter::isNotGlacierObject),
+
READ_RESTORED_GLACIER_OBJECTS(S3ObjectStorageClassFilter::isCompletedRestoredObject);
+
+ private static final Set<ObjectStorageClass> GLACIER_STORAGE_CLASSES =
Sets.newHashSet(
+ ObjectStorageClass.GLACIER, ObjectStorageClass.DEEP_ARCHIVE);
+
+ private final Function<S3Object, Boolean> filter;
Review Comment:
nit: add a javadoc here and in the getter
##########
hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md:
##########
@@ -1308,8 +1308,34 @@ The switch to turn S3A auditing on or off.
Should auditing of S3A requests be enabled?
</description>
</property>
+```
+## <a name="glacier"></a> Glacier Object Support
+
+[Amazon S3 Glacier (S3
Glacier)](https://docs.aws.amazon.com/amazonglacier/latest/dev/introduction.html)
is a secure and durable service for low-cost data archiving and
+long-term backup.
+With S3 Glacier, you can store your data cost effectively for months, years,
or even decades.
Review Comment:
replace "you" with something like "it is possible to store data more
cost-effectively"; this is the docs for the connector, not marketing...
##########
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/list/ITestS3AReadRestoredGlacierObjects.java:
##########
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.list;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Assume;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.RestoreObjectRequest;
+import software.amazon.awssdk.services.s3.model.S3Object;
+import software.amazon.awssdk.services.s3.model.Tier;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.contract.s3a.S3AContract;
+import org.apache.hadoop.fs.s3a.AbstractS3ATestBase;
+import org.apache.hadoop.fs.s3a.S3ListRequest;
+import org.apache.hadoop.fs.s3a.api.S3ObjectStorageClassFilter;
+import org.apache.hadoop.fs.store.audit.AuditSpan;
+import org.apache.hadoop.test.LambdaTestUtils;
+
+import static org.apache.hadoop.fs.s3a.Constants.READ_RESTORED_GLACIER_OBJECTS;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS_DEEP_ARCHIVE;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS_GLACIER;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching;
+import static
org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
+import static
org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfStorageClassTestsDisabled;
+import static
org.apache.hadoop.fs.statistics.StoreStatisticNames.OBJECT_LIST_REQUEST;
+
+
+/**
+ * Tests of various cases related to Glacier/Deep Archive Storage class.
+ */
+@RunWith(Parameterized.class)
+public class ITestS3AReadRestoredGlacierObjects extends AbstractS3ATestBase {
+
+ enum Type { GLACIER_AND_DEEP_ARCHIVE, GLACIER }
+
+ @Parameterized.Parameters(name = "storage-class-{1}")
+ public static Collection<Object[]> data(){
+ return Arrays.asList(new Object[][] {
+ {Type.GLACIER_AND_DEEP_ARCHIVE, STORAGE_CLASS_GLACIER},
+ {Type.GLACIER_AND_DEEP_ARCHIVE, STORAGE_CLASS_DEEP_ARCHIVE},
+ {Type.GLACIER, STORAGE_CLASS_GLACIER}
+ });
+ }
+
+ private final int maxRetries = 100;
Review Comment:
these are static/final, so make constants and upper case.
##########
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/list/ITestS3AReadRestoredGlacierObjects.java:
##########
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.list;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Assume;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.RestoreObjectRequest;
+import software.amazon.awssdk.services.s3.model.S3Object;
+import software.amazon.awssdk.services.s3.model.Tier;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.contract.s3a.S3AContract;
+import org.apache.hadoop.fs.s3a.AbstractS3ATestBase;
+import org.apache.hadoop.fs.s3a.S3ListRequest;
+import org.apache.hadoop.fs.s3a.api.S3ObjectStorageClassFilter;
+import org.apache.hadoop.fs.store.audit.AuditSpan;
+import org.apache.hadoop.test.LambdaTestUtils;
+
+import static org.apache.hadoop.fs.s3a.Constants.READ_RESTORED_GLACIER_OBJECTS;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS_DEEP_ARCHIVE;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS_GLACIER;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching;
+import static
org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
+import static
org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfStorageClassTestsDisabled;
+import static
org.apache.hadoop.fs.statistics.StoreStatisticNames.OBJECT_LIST_REQUEST;
+
+
+/**
+ * Tests of various cases related to Glacier/Deep Archive Storage class.
+ */
+@RunWith(Parameterized.class)
+public class ITestS3AReadRestoredGlacierObjects extends AbstractS3ATestBase {
+
+ enum Type { GLACIER_AND_DEEP_ARCHIVE, GLACIER }
+
+ @Parameterized.Parameters(name = "storage-class-{1}")
+ public static Collection<Object[]> data(){
+ return Arrays.asList(new Object[][] {
+ {Type.GLACIER_AND_DEEP_ARCHIVE, STORAGE_CLASS_GLACIER},
+ {Type.GLACIER_AND_DEEP_ARCHIVE, STORAGE_CLASS_DEEP_ARCHIVE},
+ {Type.GLACIER, STORAGE_CLASS_GLACIER}
+ });
+ }
+
+ private final int maxRetries = 100;
+ private final int retryDelayMs = 5000;
+
+ private final Type type;
+ private final String glacierClass;
+
+ public ITestS3AReadRestoredGlacierObjects(Type type, String glacierClass) {
+ this.type = type;
+ this.glacierClass = glacierClass;
+ }
+
+ private FileSystem createFiles(String s3ObjectStorageClassFilter) throws
Throwable {
+ Configuration conf = this.createConfiguration();
+ conf.set(READ_RESTORED_GLACIER_OBJECTS, s3ObjectStorageClassFilter);
+ // Create Glacier objects:Storage Class:DEEP_ARCHIVE/GLACIER
+ conf.set(STORAGE_CLASS, glacierClass);
+ S3AContract contract = (S3AContract) createContract(conf);
+ contract.init();
+
+ FileSystem fs = contract.getTestFileSystem();
+ Path path = new Path(methodPath(), "glaciated");
+ ContractTestUtils.touch(fs, path);
+ return fs;
+ }
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration newConf = super.createConfiguration();
+ skipIfStorageClassTestsDisabled(newConf);
+ disableFilesystemCaching(newConf);
+ removeBaseAndBucketOverrides(newConf, STORAGE_CLASS,
READ_RESTORED_GLACIER_OBJECTS);
+ return newConf;
+ }
+
+ @Test
+ public void testConfigWithInvalidValue() throws Throwable {
+ Assume.assumeTrue(type == Type.GLACIER_AND_DEEP_ARCHIVE);
+ String invalidValue = "ABCDE";
+ try (FileSystem fs = createFiles(invalidValue)) {
+ Assertions.assertThat(
+ fs.listStatus(methodPath()))
+ .describedAs("FileStatus List of %s", methodPath()).isNotEmpty();
+ }
+ }
+
+ @Test
+ public void testIgnoreGlacierObject() throws Throwable {
+ Assume.assumeTrue(type == Type.GLACIER_AND_DEEP_ARCHIVE);
+ try (FileSystem fs =
createFiles(S3ObjectStorageClassFilter.SKIP_ALL_GLACIER.name())) {
+ Assertions.assertThat(
+ fs.listStatus(methodPath()))
+ .describedAs("FileStatus List of %s", methodPath()).isEmpty();
+ }
+ }
+
+ @Test
+ public void testIgnoreRestoringGlacierObject() throws Throwable {
+ Assume.assumeTrue(type == Type.GLACIER_AND_DEEP_ARCHIVE);
+ try (FileSystem fs =
createFiles(S3ObjectStorageClassFilter.READ_RESTORED_GLACIER_OBJECTS.name())) {
+ Assertions.assertThat(
+ fs.listStatus(
+ methodPath()))
+ .describedAs("FileStatus List of %s", methodPath()).isEmpty();
+ }
+ }
+
+ @Test
+ public void testRestoredGlacierObject() throws Throwable {
+ // Skipping this test for Deep Archive as expedited retrieval is not
supported
+ Assume.assumeTrue(type == Type.GLACIER);
+ try (FileSystem fs =
createFiles(S3ObjectStorageClassFilter.READ_RESTORED_GLACIER_OBJECTS.name())) {
+ restoreGlacierObject(getFilePrefixForListObjects(), 2);
+ Assertions.assertThat(
+ fs.listStatus(
+ methodPath()))
+ .describedAs("FileStatus List of %s", methodPath()).isNotEmpty();
+ }
+ }
+
+ @Test
+ public void testReadAllObjects() throws Throwable {
+ Assume.assumeTrue(type == Type.GLACIER_AND_DEEP_ARCHIVE);
+ try (FileSystem fs =
createFiles(S3ObjectStorageClassFilter.READ_ALL.name())) {
+ Assertions.assertThat(
+ fs.listStatus(methodPath()))
+ .describedAs("FileStatus List of %s", methodPath()).isNotEmpty();
+ }
+ }
+
+
+ private void restoreGlacierObject(String glacierObjectKey, int
expirationDays) throws Exception {
+ try (AuditSpan auditSpan = getSpanSource().createSpan(OBJECT_LIST_REQUEST,
"", "").activate()) {
Review Comment:
and use a describe() call to print out what is about to happen; useful for
IDE runs
> Handle S3A "glacier" data
> -------------------------
>
> Key: HADOOP-14837
> URL: https://issues.apache.org/jira/browse/HADOOP-14837
> Project: Hadoop Common
> Issue Type: Sub-task
> Components: fs/s3
> Affects Versions: 3.0.0-beta1
> Reporter: Steve Loughran
> Assignee: Bhavay Pahuja
> Priority: Minor
> Labels: pull-request-available
>
> SPARK-21797 covers how if you have AWS S3 set to copy some files to glacier,
> they appear in the listing but GETs fail, and so does everything else
> We should think about how best to handle this.
> # report better
> # if listings can identify files which are glaciated then maybe we could have
> an option to filter them out
> # test & see what happens
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
