rich7420 commented on code in PR #11461:
URL: https://github.com/apache/ozone/pull/11461#discussion_r4257186172
##########
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/PutBucketLifecycleConfigurationUnmarshaller.java:
##########
@@ -17,14 +17,67 @@
package org.apache.hadoop.ozone.s3.endpoint;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.S3_XML_NAMESPACE;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import java.util.ArrayDeque;
+import java.util.Deque;
+import org.xml.sax.Attributes;
+import org.xml.sax.SAXException;
+
/**
* Custom unmarshaller to read Lifecycle configuration.
*/
public class PutBucketLifecycleConfigurationUnmarshaller
extends MessageUnmarshaller<S3LifecycleConfiguration> {
+ private static final ImmutableMap<String, ImmutableSet<String>>
ALLOWED_CHILDREN =
+ ImmutableMap.<String, ImmutableSet<String>>builder()
+ .put("", ImmutableSet.of("LifecycleConfiguration"))
+ .put("LifecycleConfiguration", ImmutableSet.of("Rule"))
+ .put("Rule", ImmutableSet.of("ID", "Status", "Prefix", "Expiration",
"AbortIncompleteMultipartUpload",
+ "Filter", "Transition"))
+ .put("Expiration", ImmutableSet.of("Days", "Date"))
+ .put("AbortIncompleteMultipartUpload",
ImmutableSet.of("DaysAfterInitiation"))
+ .put("Filter", ImmutableSet.of("Prefix", "Tag", "And"))
+ .put("And", ImmutableSet.of("Prefix", "Tag"))
+ .put("Tag", ImmutableSet.of("Key", "Value"))
+ .put("Transition", ImmutableSet.of("Days", "Date", "StorageClass"))
+ .build();
public PutBucketLifecycleConfigurationUnmarshaller() {
super(S3LifecycleConfiguration.class);
}
+ @Override
+ protected XmlNamespaceFilter createNamespaceFilter() {
+ return new LifecycleNamespaceFilter();
+ }
+
+ private static class LifecycleNamespaceFilter extends XmlNamespaceFilter {
+ private final Deque<String> parents = new ArrayDeque<>();
+
+ LifecycleNamespaceFilter() {
+ super(S3_XML_NAMESPACE);
+ }
+
+ @Override
+ public void startElement(String uri, String localName, String qName,
Attributes attributes) throws SAXException {
+ // Match JAXB's name selection when the SAX parser is not
namespace-aware.
+ String name = localName == null || localName.isEmpty() ? qName :
localName;
+ String parent = parents.isEmpty() ? "" : parents.peek();
+ if (!ALLOWED_CHILDREN.getOrDefault(parent,
ImmutableSet.of()).contains(name)) {
+ throw new SAXException("Unsupported lifecycle element: " + name);
Review Comment:
> Can we change this to "Unsupported or invalid lifecycle element"?
The local patch uses this wording; push is pending.
##########
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/PutBucketLifecycleConfigurationUnmarshaller.java:
##########
@@ -17,14 +17,67 @@
package org.apache.hadoop.ozone.s3.endpoint;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.S3_XML_NAMESPACE;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import java.util.ArrayDeque;
+import java.util.Deque;
+import org.xml.sax.Attributes;
+import org.xml.sax.SAXException;
+
/**
* Custom unmarshaller to read Lifecycle configuration.
*/
public class PutBucketLifecycleConfigurationUnmarshaller
extends MessageUnmarshaller<S3LifecycleConfiguration> {
+ private static final ImmutableMap<String, ImmutableSet<String>>
ALLOWED_CHILDREN =
Review Comment:
> any new supported actions/fields should be added into this map.
The local patch adds this maintenance note; push is pending.
##########
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/PutBucketLifecycleConfigurationUnmarshaller.java:
##########
@@ -17,14 +17,67 @@
package org.apache.hadoop.ozone.s3.endpoint;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.S3_XML_NAMESPACE;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import java.util.ArrayDeque;
+import java.util.Deque;
+import org.xml.sax.Attributes;
+import org.xml.sax.SAXException;
+
/**
* Custom unmarshaller to read Lifecycle configuration.
*/
public class PutBucketLifecycleConfigurationUnmarshaller
extends MessageUnmarshaller<S3LifecycleConfiguration> {
+ private static final ImmutableMap<String, ImmutableSet<String>>
ALLOWED_CHILDREN =
+ ImmutableMap.<String, ImmutableSet<String>>builder()
+ .put("", ImmutableSet.of("LifecycleConfiguration"))
+ .put("LifecycleConfiguration", ImmutableSet.of("Rule"))
+ .put("Rule", ImmutableSet.of("ID", "Status", "Prefix", "Expiration",
"AbortIncompleteMultipartUpload",
+ "Filter", "Transition"))
+ .put("Expiration", ImmutableSet.of("Days", "Date"))
+ .put("AbortIncompleteMultipartUpload",
ImmutableSet.of("DaysAfterInitiation"))
+ .put("Filter", ImmutableSet.of("Prefix", "Tag", "And"))
+ .put("And", ImmutableSet.of("Prefix", "Tag"))
+ .put("Tag", ImmutableSet.of("Key", "Value"))
+ .put("Transition", ImmutableSet.of("Days", "Date", "StorageClass"))
+ .build();
public PutBucketLifecycleConfigurationUnmarshaller() {
Review Comment:
> We should return the detail message to caller
The local patch preserves the parser message, falling back to the default
when empty; push is pending.
##########
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3LifecycleConfigurationPut.java:
##########
@@ -917,6 +922,183 @@ public void
testPutLifecycleMatchingOwnerStillValidatesXml() throws Exception {
verify(proxy, never()).setLifecycleConfiguration(any());
}
+ @ParameterizedTest
+ @MethodSource("versioningActions")
+ public void testRejectVersioningActions(String action, String status,
boolean namespace) throws Exception {
+ ClientProtocol proxy = mock(ClientProtocol.class);
+ BucketEndpoint endpoint = newProtocolEndpoint(proxy, headers,
BucketLayout.OBJECT_STORE);
+ String xml = "<LifecycleConfiguration" + (namespace ? " xmlns=\"" +
S3Consts.S3_XML_NAMESPACE + "\"" : "")
+ + "><Rule><ID>versioning</ID><Prefix>prefix/</Prefix><Status>" +
status + "</Status>"
+ +
"<AbortIncompleteMultipartUpload><DaysAfterInitiation>1</DaysAfterInitiation>"
+ + "</AbortIncompleteMultipartUpload>" + action +
"</Rule></LifecycleConfiguration>";
+ assertErrorResponse(MALFORMED_XML,
Review Comment:
> assert the returned error message has the expected action/field name
The local patch asserts the rejected element name in these tests; push is
pending.
##########
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3LifecycleConfigurationPut.java:
##########
@@ -917,6 +922,183 @@ public void
testPutLifecycleMatchingOwnerStillValidatesXml() throws Exception {
verify(proxy, never()).setLifecycleConfiguration(any());
}
+ @ParameterizedTest
+ @MethodSource("versioningActions")
+ public void testRejectVersioningActions(String action, String status,
boolean namespace) throws Exception {
+ ClientProtocol proxy = mock(ClientProtocol.class);
+ BucketEndpoint endpoint = newProtocolEndpoint(proxy, headers,
BucketLayout.OBJECT_STORE);
+ String xml = "<LifecycleConfiguration" + (namespace ? " xmlns=\"" +
S3Consts.S3_XML_NAMESPACE + "\"" : "")
+ + "><Rule><ID>versioning</ID><Prefix>prefix/</Prefix><Status>" +
status + "</Status>"
+ +
"<AbortIncompleteMultipartUpload><DaysAfterInitiation>1</DaysAfterInitiation>"
+ + "</AbortIncompleteMultipartUpload>" + action +
"</Rule></LifecycleConfiguration>";
+ assertErrorResponse(MALFORMED_XML,
+ () -> endpoint.put("bucket1", new
ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))));
+ verify(proxy, never()).setLifecycleConfiguration(any());
+ }
+
+ private static Stream<Arguments> versioningActions() {
+ String transition =
"<NoncurrentVersionTransition><NoncurrentDays>1</NoncurrentDays>"
+ + "<StorageClass>GLACIER</StorageClass></NoncurrentVersionTransition>";
+ return Stream.of(
+
"<NoncurrentVersionExpiration><NoncurrentDays>1</NoncurrentDays></NoncurrentVersionExpiration>",
+ "<NoncurrentVersionExpiration/>",
+ "<NoncurrentVersionExpiration
xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"; xsi:nil=\"true\"/>",
+ "<NoncurrentVersionTransition/>",
+ "<NoncurrentVersionTransition
xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"; xsi:nil=\"true\"/>",
+ transition, transition + transition,
+
"<Expiration><Days>30</Days><ExpiredObjectDeleteMarker>true</ExpiredObjectDeleteMarker></Expiration>",
+
"<Expiration><Days>30</Days><ExpiredObjectDeleteMarker>false</ExpiredObjectDeleteMarker></Expiration>",
+ "<Expiration><Days>30</Days><ExpiredObjectDeleteMarker/></Expiration>",
+ "<Expiration><Days>30</Days><ExpiredObjectDeleteMarker "
+ + "xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\";
xsi:nil=\"true\"/></Expiration>",
+
"<Expiration><ExpiredObjectDeleteMarker>true</ExpiredObjectDeleteMarker></Expiration>")
+ .flatMap(action -> Stream.of("Enabled", "Disabled")
+ .flatMap(status -> Stream.of(true, false).map(namespace ->
Arguments.of(action, status, namespace))));
+ }
+
+ @Test
+ public void testRejectedVersioningConfigurationPreservesExistingRules()
throws Exception {
+ bucketEndpoint.put("bucket1", onePrefix());
+ String xml =
"<LifecycleConfiguration><Rule><ID>replacement</ID><Prefix>replacement/</Prefix>"
+ + "<Status>Enabled</Status>"
+ +
"<Expiration><Days>1</Days></Expiration></Rule><Rule><ID>unsupported</ID><Prefix>prefix/</Prefix>"
+ + "<Status>Enabled</Status>"
+ +
"<NoncurrentVersionExpiration><NoncurrentDays>1</NoncurrentDays></NoncurrentVersionExpiration>"
+ + "</Rule></LifecycleConfiguration>";
+ assertErrorResponse(MALFORMED_XML,
+ () -> bucketEndpoint.put("bucket1", new
ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))));
+ try (Response response = bucketEndpoint.get("bucket1")) {
+ S3LifecycleConfiguration configuration = (S3LifecycleConfiguration)
response.getEntity();
+ assertThat(configuration.getRules()).hasSize(1);
+ S3LifecycleConfiguration.Rule rule = configuration.getRules().get(0);
+ assertThat(rule.getId()).isEqualTo("remove logs after 30 days");
+ assertThat(rule.getPrefix()).isEqualTo("prefix/");
+ assertThat(rule.getExpiration().getDays()).isEqualTo(30);
+ StringWriter xmlOutput = new StringWriter();
+
JAXBContext.newInstance(S3LifecycleConfiguration.class).createMarshaller().marshal(configuration,
xmlOutput);
+ assertThat(xmlOutput.toString()).doesNotContain("NoncurrentVersion",
"ExpiredObjectDeleteMarker");
+ }
+ }
+
+ @Test
+ public void testVersioningNamesInTextRemainSupported() throws Exception {
+ String xml =
"<LifecycleConfiguration><Rule><ID>NoncurrentVersionExpiration</ID>"
+ +
"<Prefix>NoncurrentVersionTransition/ExpiredObjectDeleteMarker/</Prefix><Status>Enabled</Status>"
+ + "<Expiration><Days>30</Days></Expiration>"
+ + "</Rule></LifecycleConfiguration>";
+ try (Response response = bucketEndpoint.put("bucket1",
+ new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8)))) {
+ assertThat(response.getStatus()).isEqualTo(HTTP_OK);
+ }
+ }
+
+ @Test
+ public void testVersioningRuleStillChecksStatusAndOwner() throws Exception {
+ ClientProtocol proxy = mock(ClientProtocol.class);
+ BucketEndpoint endpoint = newProtocolEndpoint(proxy, headers,
BucketLayout.OBJECT_STORE);
+ String xml = "<LifecycleConfiguration><Rule><Status>invalid</Status>"
+ + "<NoncurrentVersionExpiration/></Rule></LifecycleConfiguration>";
+ assertErrorResponse(MALFORMED_XML,
+ () -> endpoint.put("bucket1", new
ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))));
+
when(headers.getHeaderString(EXPECTED_BUCKET_OWNER_HEADER)).thenReturn("other-owner");
+ assertErrorResponse(ACCESS_DENIED,
+ () -> endpoint.put("bucket1", new
ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))));
+ verify(proxy, never()).setLifecycleConfiguration(any());
+ }
+
+ @ParameterizedTest
+ @MethodSource("unsupportedLifecycleElements")
+ public void testRejectUnsupportedLifecycleElements(String elements, boolean
namespace) throws Exception {
+ ClientProtocol proxy = mock(ClientProtocol.class);
+ BucketEndpoint endpoint = newProtocolEndpoint(proxy, headers,
BucketLayout.OBJECT_STORE);
+ String xml = "<LifecycleConfiguration" + (namespace ? " xmlns=\"" +
S3Consts.S3_XML_NAMESPACE + "\"" : "")
+ + "><Rule><ID>unsupported</ID><Status>Enabled</Status>"
+ + elements + "</Rule></LifecycleConfiguration>";
+ assertErrorResponse(MALFORMED_XML,
+ () -> endpoint.put("bucket1", new
ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))));
+ verify(proxy, never()).setLifecycleConfiguration(any());
+ }
+
+ private static Stream<Arguments> unsupportedLifecycleElements() {
+ String prefix = "<Prefix>prefix/</Prefix>";
+ String expiration = "<Expiration><Days>30</Days></Expiration>";
+ return Stream.of(
+ prefix + expiration + "<UnknownElement/>",
+ prefix + "<Expiration><Days>30</Days><UnknownElement/></Expiration>",
+ prefix + "<Expiration><Days>30<UnknownElement/></Days></Expiration>",
+ prefix +
"<AbortIncompleteMultipartUpload><DaysAfterInitiation>1</DaysAfterInitiation>"
+ + "<UnknownElement/></AbortIncompleteMultipartUpload>",
+ expiration +
"<Filter><Prefix>prefix/</Prefix><ObjectSizeGreaterThan>1</ObjectSizeGreaterThan></Filter>",
+ expiration +
"<Filter><Prefix>prefix/</Prefix><ObjectSizeLessThan>1000</ObjectSizeLessThan></Filter>",
+ expiration +
"<Filter><And><Prefix>prefix/</Prefix><Tag><Key>key</Key><Value>value</Value></Tag>"
+ +
"<ObjectSizeGreaterThan>1</ObjectSizeGreaterThan></And></Filter>",
+ expiration +
"<Filter><Tag><Key>key</Key><Value>value</Value><UnknownElement/></Tag></Filter>",
+ prefix +
"<Expiration><Days>30</Days><ExpiredObjectDeleteMarker>true</ExpiredObjectDeleteMarker>"
+ + "</Expiration>" + expiration,
+ prefix +
"<Expiration><Days>30</Days><Tag><Key>key</Key><Value>value</Value></Tag></Expiration>",
+ prefix +
"<Transition><Days>30</Days><StorageClass>GLACIER</StorageClass><UnknownElement/></Transition>")
+ .flatMap(elements -> Stream.of(Arguments.of(elements, false),
Arguments.of(elements, true)));
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"<UnknownElement/>", "<Prefix>prefix/</Prefix>"})
+ public void testRejectUnsupportedRootElements(String element) throws
Exception {
+ ClientProtocol proxy = mock(ClientProtocol.class);
+ BucketEndpoint endpoint = newProtocolEndpoint(proxy, headers,
BucketLayout.OBJECT_STORE);
+ String xml = "<LifecycleConfiguration>" + element
+ +
"<Rule><Prefix>prefix/</Prefix><Status>Enabled</Status><Expiration><Days>30</Days></Expiration>"
+ + "</Rule></LifecycleConfiguration>";
+ assertErrorResponse(MALFORMED_XML,
+ () -> endpoint.put("bucket1", new
ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))));
+ verify(proxy, never()).setLifecycleConfiguration(any());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {
+ "<Rule xsi:nil=\"true\"><Prefix>prefix/</Prefix><Status>Enabled</Status>"
+ + "<Expiration><Days>30</Days></Expiration><UnknownElement/></Rule>",
+ "<Rule><Prefix>prefix/</Prefix><Status>Enabled</Status>"
+ + "<Expiration
xsi:nil=\"true\"><Days>30</Days><UnknownElement/></Expiration></Rule>",
+ "<Rule><Prefix>prefix/</Prefix><Status>Enabled</Status>"
+ + "<Expiration
xsi:type=\"xs:string\"><Days>30</Days><UnknownElement/></Expiration></Rule>"
+ })
+ public void testRejectUnsupportedElementsWithXmlTypeOverrides(String rule)
throws Exception {
+ ClientProtocol proxy = mock(ClientProtocol.class);
+ BucketEndpoint endpoint = newProtocolEndpoint(proxy, headers,
BucketLayout.OBJECT_STORE);
+ String xml = "<LifecycleConfiguration
xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"";
+ + " xmlns:xs=\"http://www.w3.org/2001/XMLSchema\";>" + rule +
"</LifecycleConfiguration>";
+ assertErrorResponse(MALFORMED_XML,
+ () -> endpoint.put("bucket1", new
ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))));
+ verify(proxy, never()).setLifecycleConfiguration(any());
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ public void testSupportedLifecycleElementPaths(boolean namespace) throws
Exception {
+ String xml = "<LifecycleConfiguration" + (namespace ? " xmlns=\"" +
S3Consts.S3_XML_NAMESPACE + "\"" : "")
Review Comment:
> the configuration doesn't start with "LifecycleConfiguration"
The local patch tests unexpected roots and incorrect casing, with and
without the S3 namespace; push is pending.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
