Repository: beam Updated Branches: refs/heads/master 7d6f28633 -> 2e9b728a8
Ignore HTTP 404 when deleting GCS files You can get a 404 if a previous delete RPC actually succeeded but returned a failed response due to some network issues, so the next attempt gets a 404. Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/17e346ee Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/17e346ee Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/17e346ee Branch: refs/heads/master Commit: 17e346ee342df11c12db5ba856c2004880cfa40e Parents: 7d6f286 Author: Eugene Kirpichov <[email protected]> Authored: Tue Sep 5 15:55:08 2017 -0700 Committer: Eugene Kirpichov <[email protected]> Committed: Tue Sep 5 19:50:20 2017 -0700 ---------------------------------------------------------------------- .../java/org/apache/beam/sdk/util/GcsUtil.java | 6 ++- .../org/apache/beam/sdk/util/GcsUtilTest.java | 45 ++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/17e346ee/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsUtil.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsUtil.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsUtil.java index d7205bf..da4a1e0 100644 --- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsUtil.java +++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsUtil.java @@ -767,7 +767,11 @@ public class GcsUtil { @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) throws IOException { - throw new IOException(String.format("Error trying to delete %s: %s", file, e)); + if (e.getCode() == 404) { + LOG.info("Ignoring failed deletion of file {} which already does not exist: {}", file, e); + } else { + throw new IOException(String.format("Error trying to delete %s: %s", file, e)); + } } }); } http://git-wip-us.apache.org/repos/asf/beam/blob/17e346ee/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsUtilTest.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsUtilTest.java b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsUtilTest.java index 2164e75..897cd53 100644 --- a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsUtilTest.java +++ b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsUtilTest.java @@ -524,6 +524,51 @@ public class GcsUtilTest { } @Test + public void testRemoveWhenFileNotFound() throws Exception { + JsonFactory jsonFactory = new JacksonFactory(); + + String contentBoundary = "batch_foobarbaz"; + String contentBoundaryLine = "--" + contentBoundary; + String endOfContentBoundaryLine = "--" + contentBoundary + "--"; + + GenericJson error = new GenericJson() + .set("error", new GenericJson().set("code", 404)); + error.setFactory(jsonFactory); + + String content = contentBoundaryLine + "\n" + + "Content-Type: application/http\n" + + "\n" + + "HTTP/1.1 404 Not Found\n" + + "Content-Length: -1\n" + + "\n" + + error.toString() + + "\n" + + "\n" + + endOfContentBoundaryLine + + "\n"; + + final LowLevelHttpResponse mockResponse = Mockito.mock(LowLevelHttpResponse.class); + when(mockResponse.getContentType()).thenReturn("multipart/mixed; boundary=" + contentBoundary); + when(mockResponse.getStatusCode()).thenReturn(200); + when(mockResponse.getContent()).thenReturn(toStream(content)); + + // A mock transport that lets us mock the API responses. + MockLowLevelHttpRequest request = new MockLowLevelHttpRequest() { + @Override + public LowLevelHttpResponse execute() throws IOException { + return mockResponse; + } + }; + MockHttpTransport mockTransport = + new MockHttpTransport.Builder().setLowLevelHttpRequest(request).build(); + + GcsUtil gcsUtil = gcsOptionsWithTestCredential().getGcsUtil(); + gcsUtil.setStorageClient( + new Storage(mockTransport, Transport.getJsonFactory(), new RetryHttpRequestInitializer())); + gcsUtil.remove(Arrays.asList("gs://some-bucket/already-deleted")); + } + + @Test public void testCreateBucket() throws IOException { GcsOptions pipelineOptions = gcsOptionsWithTestCredential(); GcsUtil gcsUtil = pipelineOptions.getGcsUtil();
