rich7420 commented on code in PR #11461:
URL: https://github.com/apache/ozone/pull/11461#discussion_r4257186172
##########
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/PutBucketLifecycleConfigurationUnmarshaller.java:
##########
@@ -17,14 +17,67 @@
package org.apache.hadoop.ozone.s3.endpoint;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.S3_XML_NAMESPACE;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import java.util.ArrayDeque;
+import java.util.Deque;
+import org.xml.sax.Attributes;
+import org.xml.sax.SAXException;
+
/**
* Custom unmarshaller to read Lifecycle configuration.
*/
public class PutBucketLifecycleConfigurationUnmarshaller
extends MessageUnmarshaller<S3LifecycleConfiguration> {
+ private static final ImmutableMap<String, ImmutableSet<String>>
ALLOWED_CHILDREN =
+ ImmutableMap.<String, ImmutableSet<String>>builder()
+ .put("", ImmutableSet.of("LifecycleConfiguration"))
+ .put("LifecycleConfiguration", ImmutableSet.of("Rule"))
+ .put("Rule", ImmutableSet.of("ID", "Status", "Prefix", "Expiration",
"AbortIncompleteMultipartUpload",
+ "Filter", "Transition"))
+ .put("Expiration", ImmutableSet.of("Days", "Date"))
+ .put("AbortIncompleteMultipartUpload",
ImmutableSet.of("DaysAfterInitiation"))
+ .put("Filter", ImmutableSet.of("Prefix", "Tag", "And"))
+ .put("And", ImmutableSet.of("Prefix", "Tag"))
+ .put("Tag", ImmutableSet.of("Key", "Value"))
+ .put("Transition", ImmutableSet.of("Days", "Date", "StorageClass"))
+ .build();
public PutBucketLifecycleConfigurationUnmarshaller() {
super(S3LifecycleConfiguration.class);
}
+ @Override
+ protected XmlNamespaceFilter createNamespaceFilter() {
+ return new LifecycleNamespaceFilter();
+ }
+
+ private static class LifecycleNamespaceFilter extends XmlNamespaceFilter {
+ private final Deque<String> parents = new ArrayDeque<>();
+
+ LifecycleNamespaceFilter() {
+ super(S3_XML_NAMESPACE);
+ }
+
+ @Override
+ public void startElement(String uri, String localName, String qName,
Attributes attributes) throws SAXException {
+ // Match JAXB's name selection when the SAX parser is not
namespace-aware.
+ String name = localName == null || localName.isEmpty() ? qName :
localName;
+ String parent = parents.isEmpty() ? "" : parents.peek();
+ if (!ALLOWED_CHILDREN.getOrDefault(parent,
ImmutableSet.of()).contains(name)) {
+ throw new SAXException("Unsupported lifecycle element: " + name);
Review Comment:
> Can we change this to "Unsupported or invalid lifecycle element"?
Updated to "Unsupported or invalid lifecycle element".
([696e0153aa](https://github.com/rich7420/ozone/commit/696e0153aa6ec2ba4ba94b9f851f3112ccb5ab45))
##########
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/PutBucketLifecycleConfigurationUnmarshaller.java:
##########
@@ -17,14 +17,67 @@
package org.apache.hadoop.ozone.s3.endpoint;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.S3_XML_NAMESPACE;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import java.util.ArrayDeque;
+import java.util.Deque;
+import org.xml.sax.Attributes;
+import org.xml.sax.SAXException;
+
/**
* Custom unmarshaller to read Lifecycle configuration.
*/
public class PutBucketLifecycleConfigurationUnmarshaller
extends MessageUnmarshaller<S3LifecycleConfiguration> {
+ private static final ImmutableMap<String, ImmutableSet<String>>
ALLOWED_CHILDREN =
Review Comment:
> any new supported actions/fields should be added into this map.
Added a note that newly supported actions and fields must be added to the
map.
([696e0153aa](https://github.com/rich7420/ozone/commit/696e0153aa6ec2ba4ba94b9f851f3112ccb5ab45))
##########
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/PutBucketLifecycleConfigurationUnmarshaller.java:
##########
@@ -17,14 +17,67 @@
package org.apache.hadoop.ozone.s3.endpoint;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.S3_XML_NAMESPACE;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import java.util.ArrayDeque;
+import java.util.Deque;
+import org.xml.sax.Attributes;
+import org.xml.sax.SAXException;
+
/**
* Custom unmarshaller to read Lifecycle configuration.
*/
public class PutBucketLifecycleConfigurationUnmarshaller
extends MessageUnmarshaller<S3LifecycleConfiguration> {
+ private static final ImmutableMap<String, ImmutableSet<String>>
ALLOWED_CHILDREN =
+ ImmutableMap.<String, ImmutableSet<String>>builder()
+ .put("", ImmutableSet.of("LifecycleConfiguration"))
+ .put("LifecycleConfiguration", ImmutableSet.of("Rule"))
+ .put("Rule", ImmutableSet.of("ID", "Status", "Prefix", "Expiration",
"AbortIncompleteMultipartUpload",
+ "Filter", "Transition"))
+ .put("Expiration", ImmutableSet.of("Days", "Date"))
+ .put("AbortIncompleteMultipartUpload",
ImmutableSet.of("DaysAfterInitiation"))
+ .put("Filter", ImmutableSet.of("Prefix", "Tag", "And"))
+ .put("And", ImmutableSet.of("Prefix", "Tag"))
+ .put("Tag", ImmutableSet.of("Key", "Value"))
+ .put("Transition", ImmutableSet.of("Days", "Date", "StorageClass"))
+ .build();
public PutBucketLifecycleConfigurationUnmarshaller() {
Review Comment:
> We should return the detail message to caller
Preserved the parser message when mapping to MalformedXML, with the default
message retained when no detail is available.
([696e0153aa](https://github.com/rich7420/ozone/commit/696e0153aa6ec2ba4ba94b9f851f3112ccb5ab45))
##########
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3LifecycleConfigurationPut.java:
##########
@@ -917,6 +922,183 @@ public void
testPutLifecycleMatchingOwnerStillValidatesXml() throws Exception {
verify(proxy, never()).setLifecycleConfiguration(any());
}
+ @ParameterizedTest
+ @MethodSource("versioningActions")
+ public void testRejectVersioningActions(String action, String status,
boolean namespace) throws Exception {
+ ClientProtocol proxy = mock(ClientProtocol.class);
+ BucketEndpoint endpoint = newProtocolEndpoint(proxy, headers,
BucketLayout.OBJECT_STORE);
+ String xml = "<LifecycleConfiguration" + (namespace ? " xmlns=\"" +
S3Consts.S3_XML_NAMESPACE + "\"" : "")
+ + "><Rule><ID>versioning</ID><Prefix>prefix/</Prefix><Status>" +
status + "</Status>"
+ +
"<AbortIncompleteMultipartUpload><DaysAfterInitiation>1</DaysAfterInitiation>"
+ + "</AbortIncompleteMultipartUpload>" + action +
"</Rule></LifecycleConfiguration>";
+ assertErrorResponse(MALFORMED_XML,
Review Comment:
> assert the returned error message has the expected action/field name
Added assertions for the rejected element name to the unsupported-action and
invalid-element tests.
([696e0153aa](https://github.com/rich7420/ozone/commit/696e0153aa6ec2ba4ba94b9f851f3112ccb5ab45))
##########
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3LifecycleConfigurationPut.java:
##########
@@ -917,6 +922,183 @@ public void
testPutLifecycleMatchingOwnerStillValidatesXml() throws Exception {
verify(proxy, never()).setLifecycleConfiguration(any());
}
+ @ParameterizedTest
+ @MethodSource("versioningActions")
+ public void testRejectVersioningActions(String action, String status,
boolean namespace) throws Exception {
+ ClientProtocol proxy = mock(ClientProtocol.class);
+ BucketEndpoint endpoint = newProtocolEndpoint(proxy, headers,
BucketLayout.OBJECT_STORE);
+ String xml = "<LifecycleConfiguration" + (namespace ? " xmlns=\"" +
S3Consts.S3_XML_NAMESPACE + "\"" : "")
+ + "><Rule><ID>versioning</ID><Prefix>prefix/</Prefix><Status>" +
status + "</Status>"
+ +
"<AbortIncompleteMultipartUpload><DaysAfterInitiation>1</DaysAfterInitiation>"
+ + "</AbortIncompleteMultipartUpload>" + action +
"</Rule></LifecycleConfiguration>";
+ assertErrorResponse(MALFORMED_XML,
+ () -> endpoint.put("bucket1", new
ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))));
+ verify(proxy, never()).setLifecycleConfiguration(any());
+ }
+
+ private static Stream<Arguments> versioningActions() {
+ String transition =
"<NoncurrentVersionTransition><NoncurrentDays>1</NoncurrentDays>"
+ + "<StorageClass>GLACIER</StorageClass></NoncurrentVersionTransition>";
+ return Stream.of(
+
"<NoncurrentVersionExpiration><NoncurrentDays>1</NoncurrentDays></NoncurrentVersionExpiration>",
+ "<NoncurrentVersionExpiration/>",
+ "<NoncurrentVersionExpiration
xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"; xsi:nil=\"true\"/>",
+ "<NoncurrentVersionTransition/>",
+ "<NoncurrentVersionTransition
xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"; xsi:nil=\"true\"/>",
+ transition, transition + transition,
+
"<Expiration><Days>30</Days><ExpiredObjectDeleteMarker>true</ExpiredObjectDeleteMarker></Expiration>",
+
"<Expiration><Days>30</Days><ExpiredObjectDeleteMarker>false</ExpiredObjectDeleteMarker></Expiration>",
+ "<Expiration><Days>30</Days><ExpiredObjectDeleteMarker/></Expiration>",
+ "<Expiration><Days>30</Days><ExpiredObjectDeleteMarker "
+ + "xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\";
xsi:nil=\"true\"/></Expiration>",
+
"<Expiration><ExpiredObjectDeleteMarker>true</ExpiredObjectDeleteMarker></Expiration>")
+ .flatMap(action -> Stream.of("Enabled", "Disabled")
+ .flatMap(status -> Stream.of(true, false).map(namespace ->
Arguments.of(action, status, namespace))));
+ }
+
+ @Test
+ public void testRejectedVersioningConfigurationPreservesExistingRules()
throws Exception {
+ bucketEndpoint.put("bucket1", onePrefix());
+ String xml =
"<LifecycleConfiguration><Rule><ID>replacement</ID><Prefix>replacement/</Prefix>"
+ + "<Status>Enabled</Status>"
+ +
"<Expiration><Days>1</Days></Expiration></Rule><Rule><ID>unsupported</ID><Prefix>prefix/</Prefix>"
+ + "<Status>Enabled</Status>"
+ +
"<NoncurrentVersionExpiration><NoncurrentDays>1</NoncurrentDays></NoncurrentVersionExpiration>"
+ + "</Rule></LifecycleConfiguration>";
+ assertErrorResponse(MALFORMED_XML,
+ () -> bucketEndpoint.put("bucket1", new
ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))));
+ try (Response response = bucketEndpoint.get("bucket1")) {
+ S3LifecycleConfiguration configuration = (S3LifecycleConfiguration)
response.getEntity();
+ assertThat(configuration.getRules()).hasSize(1);
+ S3LifecycleConfiguration.Rule rule = configuration.getRules().get(0);
+ assertThat(rule.getId()).isEqualTo("remove logs after 30 days");
+ assertThat(rule.getPrefix()).isEqualTo("prefix/");
+ assertThat(rule.getExpiration().getDays()).isEqualTo(30);
+ StringWriter xmlOutput = new StringWriter();
+
JAXBContext.newInstance(S3LifecycleConfiguration.class).createMarshaller().marshal(configuration,
xmlOutput);
+ assertThat(xmlOutput.toString()).doesNotContain("NoncurrentVersion",
"ExpiredObjectDeleteMarker");
+ }
+ }
+
+ @Test
+ public void testVersioningNamesInTextRemainSupported() throws Exception {
+ String xml =
"<LifecycleConfiguration><Rule><ID>NoncurrentVersionExpiration</ID>"
+ +
"<Prefix>NoncurrentVersionTransition/ExpiredObjectDeleteMarker/</Prefix><Status>Enabled</Status>"
+ + "<Expiration><Days>30</Days></Expiration>"
+ + "</Rule></LifecycleConfiguration>";
+ try (Response response = bucketEndpoint.put("bucket1",
+ new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8)))) {
+ assertThat(response.getStatus()).isEqualTo(HTTP_OK);
+ }
+ }
+
+ @Test
+ public void testVersioningRuleStillChecksStatusAndOwner() throws Exception {
+ ClientProtocol proxy = mock(ClientProtocol.class);
+ BucketEndpoint endpoint = newProtocolEndpoint(proxy, headers,
BucketLayout.OBJECT_STORE);
+ String xml = "<LifecycleConfiguration><Rule><Status>invalid</Status>"
+ + "<NoncurrentVersionExpiration/></Rule></LifecycleConfiguration>";
+ assertErrorResponse(MALFORMED_XML,
+ () -> endpoint.put("bucket1", new
ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))));
+
when(headers.getHeaderString(EXPECTED_BUCKET_OWNER_HEADER)).thenReturn("other-owner");
+ assertErrorResponse(ACCESS_DENIED,
+ () -> endpoint.put("bucket1", new
ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))));
+ verify(proxy, never()).setLifecycleConfiguration(any());
+ }
+
+ @ParameterizedTest
+ @MethodSource("unsupportedLifecycleElements")
+ public void testRejectUnsupportedLifecycleElements(String elements, boolean
namespace) throws Exception {
+ ClientProtocol proxy = mock(ClientProtocol.class);
+ BucketEndpoint endpoint = newProtocolEndpoint(proxy, headers,
BucketLayout.OBJECT_STORE);
+ String xml = "<LifecycleConfiguration" + (namespace ? " xmlns=\"" +
S3Consts.S3_XML_NAMESPACE + "\"" : "")
+ + "><Rule><ID>unsupported</ID><Status>Enabled</Status>"
+ + elements + "</Rule></LifecycleConfiguration>";
+ assertErrorResponse(MALFORMED_XML,
+ () -> endpoint.put("bucket1", new
ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))));
+ verify(proxy, never()).setLifecycleConfiguration(any());
+ }
+
+ private static Stream<Arguments> unsupportedLifecycleElements() {
+ String prefix = "<Prefix>prefix/</Prefix>";
+ String expiration = "<Expiration><Days>30</Days></Expiration>";
+ return Stream.of(
+ prefix + expiration + "<UnknownElement/>",
+ prefix + "<Expiration><Days>30</Days><UnknownElement/></Expiration>",
+ prefix + "<Expiration><Days>30<UnknownElement/></Days></Expiration>",
+ prefix +
"<AbortIncompleteMultipartUpload><DaysAfterInitiation>1</DaysAfterInitiation>"
+ + "<UnknownElement/></AbortIncompleteMultipartUpload>",
+ expiration +
"<Filter><Prefix>prefix/</Prefix><ObjectSizeGreaterThan>1</ObjectSizeGreaterThan></Filter>",
+ expiration +
"<Filter><Prefix>prefix/</Prefix><ObjectSizeLessThan>1000</ObjectSizeLessThan></Filter>",
+ expiration +
"<Filter><And><Prefix>prefix/</Prefix><Tag><Key>key</Key><Value>value</Value></Tag>"
+ +
"<ObjectSizeGreaterThan>1</ObjectSizeGreaterThan></And></Filter>",
+ expiration +
"<Filter><Tag><Key>key</Key><Value>value</Value><UnknownElement/></Tag></Filter>",
+ prefix +
"<Expiration><Days>30</Days><ExpiredObjectDeleteMarker>true</ExpiredObjectDeleteMarker>"
+ + "</Expiration>" + expiration,
+ prefix +
"<Expiration><Days>30</Days><Tag><Key>key</Key><Value>value</Value></Tag></Expiration>",
+ prefix +
"<Transition><Days>30</Days><StorageClass>GLACIER</StorageClass><UnknownElement/></Transition>")
+ .flatMap(elements -> Stream.of(Arguments.of(elements, false),
Arguments.of(elements, true)));
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"<UnknownElement/>", "<Prefix>prefix/</Prefix>"})
+ public void testRejectUnsupportedRootElements(String element) throws
Exception {
+ ClientProtocol proxy = mock(ClientProtocol.class);
+ BucketEndpoint endpoint = newProtocolEndpoint(proxy, headers,
BucketLayout.OBJECT_STORE);
+ String xml = "<LifecycleConfiguration>" + element
+ +
"<Rule><Prefix>prefix/</Prefix><Status>Enabled</Status><Expiration><Days>30</Days></Expiration>"
+ + "</Rule></LifecycleConfiguration>";
+ assertErrorResponse(MALFORMED_XML,
+ () -> endpoint.put("bucket1", new
ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))));
+ verify(proxy, never()).setLifecycleConfiguration(any());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {
+ "<Rule xsi:nil=\"true\"><Prefix>prefix/</Prefix><Status>Enabled</Status>"
+ + "<Expiration><Days>30</Days></Expiration><UnknownElement/></Rule>",
+ "<Rule><Prefix>prefix/</Prefix><Status>Enabled</Status>"
+ + "<Expiration
xsi:nil=\"true\"><Days>30</Days><UnknownElement/></Expiration></Rule>",
+ "<Rule><Prefix>prefix/</Prefix><Status>Enabled</Status>"
+ + "<Expiration
xsi:type=\"xs:string\"><Days>30</Days><UnknownElement/></Expiration></Rule>"
+ })
+ public void testRejectUnsupportedElementsWithXmlTypeOverrides(String rule)
throws Exception {
+ ClientProtocol proxy = mock(ClientProtocol.class);
+ BucketEndpoint endpoint = newProtocolEndpoint(proxy, headers,
BucketLayout.OBJECT_STORE);
+ String xml = "<LifecycleConfiguration
xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"";
+ + " xmlns:xs=\"http://www.w3.org/2001/XMLSchema\";>" + rule +
"</LifecycleConfiguration>";
+ assertErrorResponse(MALFORMED_XML,
+ () -> endpoint.put("bucket1", new
ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))));
+ verify(proxy, never()).setLifecycleConfiguration(any());
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ public void testSupportedLifecycleElementPaths(boolean namespace) throws
Exception {
+ String xml = "<LifecycleConfiguration" + (namespace ? " xmlns=\"" +
S3Consts.S3_XML_NAMESPACE + "\"" : "")
Review Comment:
> the configuration doesn't start with "LifecycleConfiguration"
Added tests for an unexpected root and incorrect root casing, with and
without the S3 XML namespace.
([696e0153aa](https://github.com/rich7420/ozone/commit/696e0153aa6ec2ba4ba94b9f851f3112ccb5ab45))
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
