Updated Branches: refs/heads/master 0982e0005 -> 6fbc1932e
JCLOUDS-73. Add bulk operations to swift Project: http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs-openstack/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs-openstack/commit/6fbc1932 Tree: http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs-openstack/tree/6fbc1932 Diff: http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs-openstack/diff/6fbc1932 Branch: refs/heads/master Commit: 6fbc1932e9c163f6894c108d7d216d06f8dd82e7 Parents: 0982e00 Author: Adrian Cole <[email protected]> Authored: Sun Sep 29 10:55:32 2013 -0700 Committer: Adrian Cole <[email protected]> Committed: Sun Sep 29 11:26:00 2013 -0700 ---------------------------------------------------------------------- openstack-swift/pom.xml | 7 + .../jclouds/openstack/swift/v1/SwiftApi.java | 4 + .../openstack/swift/v1/SwiftApiMetadata.java | 7 +- .../swift/v1/config/SwiftHttpApiModule.java | 12 -- .../swift/v1/config/SwiftTypeAdapters.java | 119 ++++++++++++++++ .../openstack/swift/v1/domain/Account.java | 3 +- .../swift/v1/domain/BulkDeleteResponse.java | 89 ++++++++++++ .../swift/v1/domain/ExtractArchiveResponse.java | 80 +++++++++++ .../openstack/swift/v1/features/AccountApi.java | 1 - .../openstack/swift/v1/features/BulkApi.java | 110 ++++++++++++++ .../swift/v1/config/SwiftTypeAdaptersTest.java | 90 ++++++++++++ .../swift/v1/features/BulkApiLiveTest.java | 142 +++++++++++++++++++ .../swift/v1/features/BulkApiMockTest.java | 70 +++++++++ .../features/UrlEncodeAndJoinOnNewlineTest.java | 45 ++++++ 14 files changed, 764 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs-openstack/blob/6fbc1932/openstack-swift/pom.xml ---------------------------------------------------------------------- diff --git a/openstack-swift/pom.xml b/openstack-swift/pom.xml index 6159d6a..9520153 100644 --- a/openstack-swift/pom.xml +++ b/openstack-swift/pom.xml @@ -100,6 +100,13 @@ <scope>test</scope> </dependency> <dependency> + <groupId>org.jboss.shrinkwrap</groupId> + <artifactId>shrinkwrap-depchain</artifactId> + <version>1.2.0</version> + <type>pom</type> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.apache.jclouds.driver</groupId> <artifactId>jclouds-slf4j</artifactId> <version>${project.parent.version}</version> http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs-openstack/blob/6fbc1932/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/SwiftApi.java ---------------------------------------------------------------------- diff --git a/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/SwiftApi.java b/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/SwiftApi.java index f803c47..b18a521 100644 --- a/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/SwiftApi.java +++ b/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/SwiftApi.java @@ -26,6 +26,7 @@ import org.jclouds.javax.annotation.Nullable; import org.jclouds.location.Region; import org.jclouds.location.functions.RegionToEndpoint; import org.jclouds.openstack.swift.v1.features.AccountApi; +import org.jclouds.openstack.swift.v1.features.BulkApi; import org.jclouds.openstack.swift.v1.features.ContainerApi; import org.jclouds.openstack.swift.v1.features.ObjectApi; import org.jclouds.rest.annotations.Delegate; @@ -50,6 +51,9 @@ public interface SwiftApi extends Closeable { AccountApi accountApiInRegion(@EndpointParam(parser = RegionToEndpoint.class) @Nullable String region); @Delegate + BulkApi bulkApiInRegion(@EndpointParam(parser = RegionToEndpoint.class) @Nullable String region); + + @Delegate ContainerApi containerApiInRegion(@EndpointParam(parser = RegionToEndpoint.class) @Nullable String region); @Delegate http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs-openstack/blob/6fbc1932/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/SwiftApiMetadata.java ---------------------------------------------------------------------- diff --git a/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/SwiftApiMetadata.java b/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/SwiftApiMetadata.java index 0cde00b..1c08c78 100644 --- a/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/SwiftApiMetadata.java +++ b/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/SwiftApiMetadata.java @@ -18,16 +18,20 @@ package org.jclouds.openstack.swift.v1; import static org.jclouds.openstack.keystone.v2_0.config.KeystoneProperties.CREDENTIAL_TYPE; import static org.jclouds.openstack.keystone.v2_0.config.KeystoneProperties.SERVICE_TYPE; + import java.net.URI; import java.util.Properties; + import org.jclouds.apis.ApiMetadata; import org.jclouds.openstack.keystone.v2_0.config.AuthenticationApiModule; import org.jclouds.openstack.keystone.v2_0.config.CredentialTypes; import org.jclouds.openstack.keystone.v2_0.config.KeystoneAuthenticationModule; import org.jclouds.openstack.keystone.v2_0.config.KeystoneAuthenticationModule.RegionModule; import org.jclouds.openstack.swift.v1.config.SwiftHttpApiModule; +import org.jclouds.openstack.swift.v1.config.SwiftTypeAdapters; import org.jclouds.openstack.v2_0.ServiceType; import org.jclouds.rest.internal.BaseHttpApiMetadata; + import com.google.common.collect.ImmutableSet; import com.google.inject.Module; @@ -63,7 +67,7 @@ public class SwiftApiMetadata extends BaseHttpApiMetadata<SwiftApi> { protected Builder() { id("openstack-swift") - .name("OpenStack Swift Diablo+ API") + .name("OpenStack Swift Grizzly+ API") .identityName("${tenantName}:${userName} or ${userName}, if your keystone supports a default tenant") .credentialName("${password}") .documentation(URI.create("http://docs.openstack.org/api/openstack-object-storage/1.0/content/ch_object-storage-dev-overview.html";)) @@ -75,6 +79,7 @@ public class SwiftApiMetadata extends BaseHttpApiMetadata<SwiftApi> { .add(AuthenticationApiModule.class) .add(KeystoneAuthenticationModule.class) .add(RegionModule.class) + .add(SwiftTypeAdapters.class) .add(SwiftHttpApiModule.class).build()); } http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs-openstack/blob/6fbc1932/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/config/SwiftHttpApiModule.java ---------------------------------------------------------------------- diff --git a/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/config/SwiftHttpApiModule.java b/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/config/SwiftHttpApiModule.java index c0c68c7..37e4b2d 100644 --- a/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/config/SwiftHttpApiModule.java +++ b/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/config/SwiftHttpApiModule.java @@ -19,28 +19,16 @@ import org.jclouds.http.HttpErrorHandler; import org.jclouds.http.annotation.ClientError; import org.jclouds.http.annotation.Redirection; import org.jclouds.http.annotation.ServerError; -import org.jclouds.json.config.GsonModule.DateAdapter; -import org.jclouds.json.config.GsonModule.Iso8601DateAdapter; import org.jclouds.openstack.swift.v1.SwiftApi; import org.jclouds.openstack.swift.v1.handlers.SwiftErrorHandler; import org.jclouds.rest.ConfiguresHttpApi; import org.jclouds.rest.config.HttpApiModule; -/** - * @author Adrian Cole - * @author Zack Shoylev - */ @ConfiguresHttpApi public class SwiftHttpApiModule extends HttpApiModule<SwiftApi> { public SwiftHttpApiModule() { } - - @Override - protected void configure() { - bind(DateAdapter.class).to(Iso8601DateAdapter.class); - super.configure(); - } @Override protected void bindErrorHandlers() { http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs-openstack/blob/6fbc1932/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/config/SwiftTypeAdapters.java ---------------------------------------------------------------------- diff --git a/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/config/SwiftTypeAdapters.java b/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/config/SwiftTypeAdapters.java new file mode 100644 index 0000000..f556142 --- /dev/null +++ b/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/config/SwiftTypeAdapters.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jclouds.openstack.swift.v1.config; + +import java.io.IOException; +import java.lang.reflect.Type; +import java.net.URI; +import java.util.Map; + +import org.jclouds.json.config.GsonModule.DateAdapter; +import org.jclouds.json.config.GsonModule.Iso8601DateAdapter; +import org.jclouds.openstack.swift.v1.domain.BulkDeleteResponse; +import org.jclouds.openstack.swift.v1.domain.ExtractArchiveResponse; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMap.Builder; +import com.google.gson.TypeAdapter; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonWriter; +import com.google.inject.AbstractModule; +import com.google.inject.Provides; + +public class SwiftTypeAdapters extends AbstractModule { + + @Override + protected void configure() { + bind(DateAdapter.class).to(Iso8601DateAdapter.class); + } + + @Provides + public Map<Type, Object> provideCustomAdapterBindings() { + return ImmutableMap.<Type, Object> builder() // + .put(ExtractArchiveResponse.class, new ExtractArchiveResponseAdapter()) // + .put(BulkDeleteResponse.class, new BulkDeleteResponseAdapter()).build(); + } + + static class ExtractArchiveResponseAdapter extends TypeAdapter<ExtractArchiveResponse> { + + @Override + public ExtractArchiveResponse read(JsonReader reader) throws IOException { + int created = 0; + Builder<String, String> errors = ImmutableMap.<String, String> builder(); + reader.beginObject(); + while (reader.hasNext()) { + String key = reader.nextName(); + if (key.equals("Number Files Created")) { + created = reader.nextInt(); + } else if (key.equals("Errors")) { + readErrors(reader, errors); + } else { + reader.skipValue(); + } + } + reader.endObject(); + return ExtractArchiveResponse.create(created, errors.build()); + } + + @Override + public void write(JsonWriter arg0, ExtractArchiveResponse arg1) throws IOException { + throw new UnsupportedOperationException(); + } + } + + static class BulkDeleteResponseAdapter extends TypeAdapter<BulkDeleteResponse> { + + @Override + public BulkDeleteResponse read(JsonReader reader) throws IOException { + int deleted = 0; + int notFound = 0; + Builder<String, String> errors = ImmutableMap.<String, String> builder(); + reader.beginObject(); + while (reader.hasNext()) { + String key = reader.nextName(); + if (key.equals("Number Deleted")) { + deleted = reader.nextInt(); + } else if (key.equals("Number Not Found")) { + notFound = reader.nextInt(); + } else if (key.equals("Errors")) { + readErrors(reader, errors); + } else { + reader.skipValue(); + } + } + reader.endObject(); + return BulkDeleteResponse.create(deleted, notFound, errors.build()); + } + + @Override + public void write(JsonWriter arg0, BulkDeleteResponse arg1) throws IOException { + throw new UnsupportedOperationException(); + } + } + + static void readErrors(JsonReader reader, Builder<String, String> errors) throws IOException { + reader.beginArray(); + while (reader.hasNext()) { + reader.beginArray(); + String decodedPath = URI.create(reader.nextString()).getPath(); + errors.put(decodedPath, reader.nextString()); + reader.endArray(); + } + reader.endArray(); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs-openstack/blob/6fbc1932/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/domain/Account.java ---------------------------------------------------------------------- diff --git a/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/domain/Account.java b/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/domain/Account.java index 02f13aa..304f329 100644 --- a/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/domain/Account.java +++ b/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/domain/Account.java @@ -161,7 +161,8 @@ public class Account { public Builder fromContainer(Account from) { return containerCount(from.containerCount())// .objectCount(from.objectCount())// - .bytesUsed(from.bytesUsed()); + .bytesUsed(from.bytesUsed()) // + .metadata(from.metadata()); } } } http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs-openstack/blob/6fbc1932/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/domain/BulkDeleteResponse.java ---------------------------------------------------------------------- diff --git a/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/domain/BulkDeleteResponse.java b/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/domain/BulkDeleteResponse.java new file mode 100644 index 0000000..c96a66f --- /dev/null +++ b/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/domain/BulkDeleteResponse.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jclouds.openstack.swift.v1.domain; + +import static com.google.common.base.Objects.equal; +import static com.google.common.base.Objects.toStringHelper; +import static com.google.common.base.Preconditions.checkNotNull; + +import java.util.Map; + +import com.google.common.base.Objects; + +/** + * @see <a + * href="http://docs.openstack.org/developer/swift/misc.html#module-swift.common.middleware.bulk";> + * Swift Bulk Middleware</a> + */ +public class BulkDeleteResponse { + public static BulkDeleteResponse create(int deleted, int notFound, Map<String, String> errors) { + return new BulkDeleteResponse(deleted, notFound, errors); + } + + private final int deleted; + private final int notFound; + private final Map<String, String> errors; + + private BulkDeleteResponse(int deleted, int notFound, Map<String, String> errors) { + this.deleted = deleted; + this.notFound = notFound; + this.errors = checkNotNull(errors, "errors"); + } + + /** number of files deleted. */ + public int deleted() { + return deleted; + } + + /** number of files not found. */ + public int notFound() { + return notFound; + } + + /** For each path that failed to delete, a corresponding error response. */ + public Map<String, String> errors() { + return errors; + } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (object instanceof BulkDeleteResponse) { + BulkDeleteResponse that = BulkDeleteResponse.class.cast(object); + return equal(deleted(), that.deleted()) // + && equal(notFound(), that.notFound()) // + && equal(errors(), that.errors()); + } else { + return false; + } + } + + @Override + public int hashCode() { + return Objects.hashCode(deleted(), notFound(), errors()); + } + + @Override + public String toString() { + return toStringHelper("") // + .add("deleted", deleted()) // + .add("notFound", notFound()) // + .add("errors", errors()).toString(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs-openstack/blob/6fbc1932/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/domain/ExtractArchiveResponse.java ---------------------------------------------------------------------- diff --git a/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/domain/ExtractArchiveResponse.java b/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/domain/ExtractArchiveResponse.java new file mode 100644 index 0000000..5644377 --- /dev/null +++ b/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/domain/ExtractArchiveResponse.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jclouds.openstack.swift.v1.domain; + +import static com.google.common.base.Objects.equal; +import static com.google.common.base.Objects.toStringHelper; +import static com.google.common.base.Preconditions.checkNotNull; + +import java.util.Map; + +import com.google.common.base.Objects; + +/** + * @see <a + * href="http://docs.openstack.org/developer/swift/misc.html#module-swift.common.middleware.bulk";> + * Swift Bulk Middleware</a> + */ +public class ExtractArchiveResponse { + public static ExtractArchiveResponse create(int created, Map<String, String> errors) { + return new ExtractArchiveResponse(created, errors); + } + + private final int created; + private final Map<String, String> errors; + + private ExtractArchiveResponse(int created, Map<String, String> errors) { + this.created = created; + this.errors = checkNotNull(errors, "errors"); + } + + /** number of files created. */ + public int created() { + return created; + } + + /** For each path that failed to create, a corresponding error response. */ + public Map<String, String> errors() { + return errors; + } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (object instanceof ExtractArchiveResponse) { + ExtractArchiveResponse that = ExtractArchiveResponse.class.cast(object); + return equal(created(), that.created()) // + && equal(errors(), that.errors()); + } else { + return false; + } + } + + @Override + public int hashCode() { + return Objects.hashCode(created(), errors()); + } + + @Override + public String toString() { + return toStringHelper("") // + .add("created", created()) // + .add("errors", errors()).toString(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs-openstack/blob/6fbc1932/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/AccountApi.java ---------------------------------------------------------------------- diff --git a/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/AccountApi.java b/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/AccountApi.java index 0b0c65c..5e0bc6d 100644 --- a/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/AccountApi.java +++ b/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/AccountApi.java @@ -45,7 +45,6 @@ import org.jclouds.rest.annotations.ResponseParser; * appropriately using a binder/parser. * * @see {@link Account} - * @see metadata * @see <a * href="http://docs.openstack.org/api/openstack-object-storage/1.0/content/storage-account-services.html";> * Storage Account Services API</a> http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs-openstack/blob/6fbc1932/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/BulkApi.java ---------------------------------------------------------------------- diff --git a/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/BulkApi.java b/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/BulkApi.java new file mode 100644 index 0000000..0cc4546 --- /dev/null +++ b/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/BulkApi.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jclouds.openstack.swift.v1.features; + +import static com.google.common.collect.Iterables.transform; +import static com.google.common.net.UrlEscapers.urlFragmentEscaper; +import static javax.ws.rs.core.MediaType.APPLICATION_JSON; +import static javax.ws.rs.core.MediaType.TEXT_PLAIN; + +import javax.inject.Named; +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.QueryParam; + +import org.jclouds.http.HttpRequest; +import org.jclouds.io.Payload; +import org.jclouds.io.Payloads; +import org.jclouds.openstack.keystone.v2_0.filters.AuthenticateRequest; +import org.jclouds.openstack.swift.v1.domain.BulkDeleteResponse; +import org.jclouds.openstack.swift.v1.domain.ExtractArchiveResponse; +import org.jclouds.openstack.swift.v1.features.ObjectApi.SetPayload; +import org.jclouds.rest.Binder; +import org.jclouds.rest.annotations.BinderParam; +import org.jclouds.rest.annotations.QueryParams; +import org.jclouds.rest.annotations.RequestFilters; + +import com.google.common.base.Joiner; + +/** + * Provides access to the Swift Bulk API. + * + * <h3>Note</h3> + * + * As of the Grizzly release, these operations occur <a + * href="https://blueprints.launchpad.net/swift/+spec/concurrent-bulk";>serially + * on the backend</a>. + * + * @see <a + * href="http://docs.openstack.org/developer/swift/misc.html#module-swift.common.middleware.bulk";> + * Swift Bulk Middleware</a> + */ +@RequestFilters(AuthenticateRequest.class) +@Consumes(APPLICATION_JSON) +public interface BulkApi { + + /** + * Extracts a tar archive at the path specified as {@code path}. + * + * @param path + * path to extract under, if not empty string. + * @param tar + * valid tar archive + * @param format + * one of {@code tar}, {@code tar.gz}, or {@code tar.bz2} + * + * @return {@link BulkDeleteResponse#errors()} are empty on success. + */ + @Named("ExtractArchive") + @PUT + @Path("/{path}") + ExtractArchiveResponse extractArchive(@PathParam("path") String path, + @BinderParam(SetPayload.class) Payload payload, @QueryParam("extract-archive") String format); + + /** + * Deletes multiple objects or containers, if present. + * + * @param paths + * format of {@code container}, for an empty container, or + * {@code container/object} for an object. + * + * @return {@link BulkDeleteResponse#errors()} are empty on success. + */ + @Named("BulkDelete") + @DELETE + @Path("/") + @QueryParams(keys = "bulk-delete") + BulkDeleteResponse bulkDelete(@BinderParam(UrlEncodeAndJoinOnNewline.class) Iterable<String> paths); + + // NOTE: this cannot be tested on MWS and is also brittle, as it relies on + // sending a body on DELETE. + // https://bugs.launchpad.net/swift/+bug/1232787 + static class UrlEncodeAndJoinOnNewline implements Binder { + @SuppressWarnings("unchecked") + @Override + public <R extends HttpRequest> R bindToRequest(R request, Object input) { + String encodedAndNewlineDelimited = Joiner.on('\n').join( + transform(Iterable.class.cast(input), urlFragmentEscaper().asFunction())); + Payload payload = Payloads.newStringPayload(encodedAndNewlineDelimited); + payload.getContentMetadata().setContentType(TEXT_PLAIN); + return (R) request.toBuilder().payload(payload).build(); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs-openstack/blob/6fbc1932/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/config/SwiftTypeAdaptersTest.java ---------------------------------------------------------------------- diff --git a/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/config/SwiftTypeAdaptersTest.java b/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/config/SwiftTypeAdaptersTest.java new file mode 100644 index 0000000..30edc8f --- /dev/null +++ b/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/config/SwiftTypeAdaptersTest.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jclouds.openstack.swift.v1.config; + +import static org.testng.Assert.assertEquals; + +import org.jclouds.openstack.swift.v1.config.SwiftTypeAdapters.BulkDeleteResponseAdapter; +import org.jclouds.openstack.swift.v1.config.SwiftTypeAdapters.ExtractArchiveResponseAdapter; +import org.jclouds.openstack.swift.v1.domain.BulkDeleteResponse; +import org.jclouds.openstack.swift.v1.domain.ExtractArchiveResponse; +import org.testng.annotations.Test; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +@Test +public class SwiftTypeAdaptersTest { + Gson gson = new GsonBuilder() // + .registerTypeAdapter(ExtractArchiveResponse.class, new ExtractArchiveResponseAdapter()) // + .registerTypeAdapter(BulkDeleteResponse.class, new BulkDeleteResponseAdapter()) // + .create(); + + public void extractArchiveWithoutErrors() { + assertEquals(gson.fromJson("" // + + "{\n" // + + " \"Response Status\": \"201 Created\",\n" // + + " \"Response Body\": \"\",\n" // + + " \"Errors\": [],\n" // + + " \"Number Files Created\": 10\n" // + + "}", ExtractArchiveResponse.class), ExtractArchiveResponse.create(10, ImmutableMap.<String, String> of())); + } + + public void extractArchiveWithErrorsAndDecodesPaths() { + assertEquals( + gson.fromJson("" // + + "{\n" // + + " \"Response Status\": \"201 Created\",\n" // + + " \"Response Body\": \"\",\n" // + + " \"Errors\": [\n" // + + " [\"/v1/12345678912345/mycontainer/home/xx%3Cyy\", \"400 Bad Request\"],\n" // + + " [\"/v1/12345678912345/mycontainer/../image.gif\", \"400 Bad Request\"]\n" // + + " ],\n" // + + " \"Number Files Created\": 8\n" // + + "}", ExtractArchiveResponse.class), + ExtractArchiveResponse.create( + 8, + ImmutableMap.<String, String> builder() + .put("/v1/12345678912345/mycontainer/home/xx<yy", "400 Bad Request") + .put("/v1/12345678912345/mycontainer/../image.gif", "400 Bad Request").build())); + } + + public void bulkDeleteWithoutErrors() { + assertEquals(gson.fromJson("" // + + "{\n" // + + " \"Response Status\": \"200 OK\",\n" // + + " \"Response Body\": \"\",\n" // + + " \"Errors\": [],\n" // + + " \"Number Not Found\": 1,\n" // + + " \"Number Deleted\": 9\n" // + + "}", BulkDeleteResponse.class), BulkDeleteResponse.create(9, 1, ImmutableMap.<String, String> of())); + } + + public void bulkDeleteWithErrorsAndDecodesPaths() { + assertEquals(gson.fromJson("" // + + "{\n" // + + " \"Response Status\": \"400 Bad Request\",\n" // + + " \"Response Body\": \"\",\n" // + + " \"Errors\": [\n" // + + " [\"/v1/12345678912345/Not%20Empty\", \"409 Conflict\"]" // + + " ],\n" // + + " \"Number Deleted\": 0\n" // + + "}", BulkDeleteResponse.class), + BulkDeleteResponse.create(0, 0, ImmutableMap.of("/v1/12345678912345/Not Empty", "409 Conflict"))); + } +} http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs-openstack/blob/6fbc1932/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/BulkApiLiveTest.java ---------------------------------------------------------------------- diff --git a/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/BulkApiLiveTest.java b/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/BulkApiLiveTest.java new file mode 100644 index 0000000..e2b17ec --- /dev/null +++ b/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/BulkApiLiveTest.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jclouds.openstack.swift.v1.features; + +import static com.google.common.base.Preconditions.checkState; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +import java.io.IOException; +import java.util.List; +import java.util.UUID; + +import org.jboss.shrinkwrap.api.GenericArchive; +import org.jboss.shrinkwrap.api.ShrinkWrap; +import org.jboss.shrinkwrap.api.asset.StringAsset; +import org.jboss.shrinkwrap.api.exporter.TarGzExporter; +import org.jclouds.io.Payloads; +import org.jclouds.openstack.swift.v1.domain.BulkDeleteResponse; +import org.jclouds.openstack.swift.v1.domain.ExtractArchiveResponse; +import org.jclouds.openstack.swift.v1.domain.SwiftObject; +import org.jclouds.openstack.swift.v1.internal.BaseSwiftApiLiveTest; +import org.jclouds.openstack.swift.v1.options.CreateContainerOptions; +import org.jclouds.openstack.swift.v1.options.ListContainerOptions; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import com.google.common.base.Function; +import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.io.ByteStreams; + +@Test(groups = "live", testName = "BulkApiLiveTest") +public class BulkApiLiveTest extends BaseSwiftApiLiveTest { + + static final int OBJECT_COUNT = 10; + + private String containerName = getClass().getSimpleName(); + + public void notPresentWhenDeleting() throws Exception { + for (String regionId : api.configuredRegions()) { + BulkDeleteResponse deleteResponse = api.bulkApiInRegion(regionId).bulkDelete( + ImmutableList.of(UUID.randomUUID().toString())); + assertEquals(deleteResponse.deleted(), 0); + assertEquals(deleteResponse.notFound(), 1); + assertTrue(deleteResponse.errors().isEmpty()); + } + } + + public void extractArchive() throws Exception { + for (String regionId : api.configuredRegions()) { + ExtractArchiveResponse extractResponse = api.bulkApiInRegion(regionId).extractArchive(containerName, + Payloads.newPayload(tarGz), "tar.gz"); + assertEquals(extractResponse.created(), OBJECT_COUNT); + assertTrue(extractResponse.errors().isEmpty()); + assertEquals(api.containerApiInRegion(regionId).get(containerName).objectCount(), OBJECT_COUNT); + + // repeat the command + api.bulkApiInRegion(regionId).extractArchive(containerName, Payloads.newPayload(tarGz), "tar.gz"); + assertEquals(extractResponse.created(), OBJECT_COUNT); + assertTrue(extractResponse.errors().isEmpty()); + } + } + + @Test(dependsOnMethods = "extractArchive") + public void bulkDelete() throws Exception { + for (String regionId : api.configuredRegions()) { + BulkDeleteResponse deleteResponse = api.bulkApiInRegion(regionId).bulkDelete(paths); + assertEquals(deleteResponse.deleted(), OBJECT_COUNT); + assertEquals(deleteResponse.notFound(), 0); + assertTrue(deleteResponse.errors().isEmpty()); + assertEquals(api.containerApiInRegion(regionId).get(containerName).objectCount(), 0); + } + } + + List<String> paths = Lists.newArrayList(); + byte[] tarGz; + + @Override + @BeforeClass(groups = "live") + public void setup() { + super.setup(); + for (String regionId : api.configuredRegions()) { + boolean created = api.containerApiInRegion(regionId).createIfAbsent(containerName, + new CreateContainerOptions()); + if (!created) { + deleteAllObjectsInContainer(regionId); + } + } + GenericArchive files = ShrinkWrap.create(GenericArchive.class, "files.tar.gz"); + StringAsset content = new StringAsset("foo"); + for (int i = 0; i < OBJECT_COUNT; i++) { + paths.add(containerName + "/file" + i); + files.add(content, "/file" + i); + } + try { + tarGz = ByteStreams.toByteArray(files.as(TarGzExporter.class).exportAsInputStream()); + } catch (IOException e) { + throw Throwables.propagate(e); + } + } + + @Override + @AfterClass(groups = "live") + public void tearDown() { + for (String regionId : api.configuredRegions()) { + deleteAllObjectsInContainer(regionId); + api.containerApiInRegion(regionId).deleteIfEmpty(containerName); + } + super.tearDown(); + } + + void deleteAllObjectsInContainer(String regionId) { + ImmutableList<String> pathsToDelete = api.objectApiInRegionForContainer(regionId, containerName) + .list(new ListContainerOptions()).transform(new Function<SwiftObject, String>() { + + public String apply(SwiftObject input) { + return containerName + "/" + input.name(); + } + + }).toList(); + if (!pathsToDelete.isEmpty()) { + BulkDeleteResponse response = api.bulkApiInRegion(regionId).bulkDelete(pathsToDelete); + checkState(response.errors().isEmpty(), "Errors deleting paths %s: %s", pathsToDelete, response); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs-openstack/blob/6fbc1932/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/BulkApiMockTest.java ---------------------------------------------------------------------- diff --git a/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/BulkApiMockTest.java b/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/BulkApiMockTest.java new file mode 100644 index 0000000..86832b4 --- /dev/null +++ b/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/BulkApiMockTest.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jclouds.openstack.swift.v1.features; + +import static org.jclouds.io.Payloads.newByteArrayPayload; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +import org.jboss.shrinkwrap.api.GenericArchive; +import org.jboss.shrinkwrap.api.ShrinkWrap; +import org.jboss.shrinkwrap.api.asset.StringAsset; +import org.jboss.shrinkwrap.api.exporter.TarGzExporter; +import org.jclouds.openstack.swift.v1.SwiftApi; +import org.jclouds.openstack.swift.v1.domain.ExtractArchiveResponse; +import org.jclouds.openstack.swift.v1.internal.BaseSwiftMockTest; +import org.testng.annotations.Test; + +import com.google.common.io.ByteStreams; +import com.squareup.okhttp.mockwebserver.MockResponse; +import com.squareup.okhttp.mockwebserver.MockWebServer; +import com.squareup.okhttp.mockwebserver.RecordedRequest; + +// TODO: cannot yet test bulk delete offline +@Test +public class BulkApiMockTest extends BaseSwiftMockTest { + + public void extractArchive() throws Exception { + GenericArchive files = ShrinkWrap.create(GenericArchive.class, "files.tar.gz"); + StringAsset content = new StringAsset("foo"); + for (int i = 0; i < 10; i++) { + files.add(content, "/file" + i); + } + byte[] tarGz = ByteStreams.toByteArray(files.as(TarGzExporter.class).exportAsInputStream()); + + MockWebServer server = mockSwiftServer(); + server.enqueue(new MockResponse().setBody(access)); + server.enqueue(new MockResponse().setResponseCode(201).setBody("{\"Number Files Created\": 10, \"Errors\": []}")); + + try { + SwiftApi api = swiftApi(server.getUrl("/").toString()); + ExtractArchiveResponse response = api.bulkApiInRegion("DFW").extractArchive("myContainer", + newByteArrayPayload(tarGz), "tar.gz"); + assertEquals(response.created(), 10); + assertTrue(response.errors().isEmpty()); + + assertEquals(server.getRequestCount(), 2); + assertEquals(server.takeRequest().getRequestLine(), "POST /tokens HTTP/1.1"); + RecordedRequest extractRequest = server.takeRequest(); + assertEquals(extractRequest.getRequestLine(), + "PUT /v1/MossoCloudFS_5bcf396e-39dd-45ff-93a1-712b9aba90a9/myContainer?extract-archive=tar.gz HTTP/1.1"); + assertEquals(extractRequest.getBody(), tarGz); + } finally { + server.shutdown(); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs-openstack/blob/6fbc1932/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/UrlEncodeAndJoinOnNewlineTest.java ---------------------------------------------------------------------- diff --git a/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/UrlEncodeAndJoinOnNewlineTest.java b/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/UrlEncodeAndJoinOnNewlineTest.java new file mode 100644 index 0000000..a45386b --- /dev/null +++ b/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/UrlEncodeAndJoinOnNewlineTest.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jclouds.openstack.swift.v1.features; + +import static org.testng.Assert.assertEquals; + +import org.jclouds.http.HttpRequest; +import org.jclouds.openstack.swift.v1.features.BulkApi.UrlEncodeAndJoinOnNewline; +import org.testng.annotations.Test; + +import com.google.common.collect.ImmutableList; + +@Test +public class UrlEncodeAndJoinOnNewlineTest { + UrlEncodeAndJoinOnNewline binder = new UrlEncodeAndJoinOnNewline(); + + public void urlEncodesPaths() { + HttpRequest request = HttpRequest.builder() + .method("DELETE") + .endpoint("https://storage101.dfw1.clouddrive.com/v1/MossoCloudFS_XXXXXX/";) + .addQueryParam("bulk-delete").build(); + + request = binder.bindToRequest(request, ImmutableList.<String> builder() + .add("/v1/12345678912345/mycontainer/home/xx<yy") + .add("/v1/12345678912345/mycontainer/../image.gif").build()); + + assertEquals(request.getPayload().getRawContent(), "" // + + "/v1/12345678912345/mycontainer/home/xx%3Cyy\n" // + + "/v1/12345678912345/mycontainer/../image.gif"); + } +}
