Repository: nifi Updated Branches: refs/heads/master 7a4990e7f -> f742a3a6a
NIFI-5156: Updated GCP SDK to latest version Signed-off-by: Pierre Villard <[email protected]> This closes #2680. Project: http://git-wip-us.apache.org/repos/asf/nifi/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/f742a3a6 Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/f742a3a6 Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/f742a3a6 Branch: refs/heads/master Commit: f742a3a6accd7695a32b1e4c42c2ec93619a7b9a Parents: 7a4990e Author: zenfenan <[email protected]> Authored: Sat May 5 14:45:40 2018 +0530 Committer: Pierre Villard <[email protected]> Committed: Wed May 9 18:19:17 2018 +0200 ---------------------------------------------------------------------- .../nifi-gcp-bundle/nifi-gcp-processors/pom.xml | 16 +------------ .../processors/gcp/AbstractGCPProcessor.java | 5 ++-- .../gcp/storage/AbstractGCSProcessor.java | 24 ++++++++------------ .../processors/gcp/storage/ListGCSBucket.java | 2 +- .../MockCredentialsFactoryProcessor.java | 6 ++--- .../MockCredentialsServiceProcessor.java | 6 ++--- .../processors/gcp/storage/AbstractGCSIT.java | 6 ++--- .../processors/gcp/storage/AbstractGCSTest.java | 5 +--- .../gcp/storage/FetchGCSObjectTest.java | 5 ---- .../gcp/storage/ListGCSBucketTest.java | 3 ++- nifi-nar-bundles/nifi-gcp-bundle/pom.xml | 16 +++++++++++++ 11 files changed, 40 insertions(+), 54 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi/blob/f742a3a6/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/pom.xml b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/pom.xml index c9119b8..eeb757b 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/pom.xml +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/pom.xml @@ -63,21 +63,12 @@ </dependency> <dependency> <groupId>com.google.cloud</groupId> - <artifactId>google-cloud</artifactId> - <version>0.8.0</version> + <artifactId>google-cloud-storage</artifactId> <exclusions> - <exclusion> <!-- NIFI-3089 --> - <groupId>org.json</groupId> - <artifactId>json</artifactId> - </exclusion> <exclusion> <groupId>com.google.code.findbugs</groupId> <artifactId>jsr305</artifactId> </exclusion> - <exclusion> <!-- NIFI-5124 --> - <groupId>commons-fileupload</groupId> - <artifactId>commons-fileupload</artifactId> - </exclusion> </exclusions> </dependency> <dependency> @@ -85,11 +76,6 @@ <artifactId>json</artifactId> <version>1.8</version> </dependency> - <dependency> - <groupId>commons-fileupload</groupId> - <artifactId>commons-fileupload</artifactId> - <version>1.3.3</version> - </dependency> </dependencies> <build> http://git-wip-us.apache.org/repos/asf/nifi/blob/f742a3a6/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/AbstractGCPProcessor.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/AbstractGCPProcessor.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/AbstractGCPProcessor.java index 5f51194..0505d63 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/AbstractGCPProcessor.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/AbstractGCPProcessor.java @@ -17,8 +17,8 @@ package org.apache.nifi.processors.gcp; import com.google.auth.oauth2.GoogleCredentials; -import com.google.cloud.HttpServiceOptions; import com.google.cloud.Service; +import com.google.cloud.ServiceOptions; import com.google.common.collect.ImmutableList; import org.apache.nifi.annotation.lifecycle.OnScheduled; import org.apache.nifi.components.PropertyDescriptor; @@ -36,8 +36,7 @@ import java.util.List; */ public abstract class AbstractGCPProcessor< CloudService extends Service<CloudServiceOptions>, - CloudServiceRpc, - CloudServiceOptions extends HttpServiceOptions<CloudService, CloudServiceRpc, CloudServiceOptions>> extends AbstractProcessor { + CloudServiceOptions extends ServiceOptions<CloudService, CloudServiceOptions>> extends AbstractProcessor { public static final PropertyDescriptor PROJECT_ID = new PropertyDescriptor .Builder().name("gcp-project-id") http://git-wip-us.apache.org/repos/asf/nifi/blob/f742a3a6/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/AbstractGCSProcessor.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/AbstractGCSProcessor.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/AbstractGCSProcessor.java index 7cc854d..bed596b 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/AbstractGCSProcessor.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/AbstractGCSProcessor.java @@ -18,12 +18,12 @@ package org.apache.nifi.processors.gcp.storage; import com.google.api.client.http.HttpTransport; import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.gax.retrying.RetrySettings; import com.google.auth.http.HttpTransportFactory; import com.google.auth.oauth2.GoogleCredentials; -import com.google.cloud.RetryParams; +import com.google.cloud.http.HttpTransportOptions; import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageOptions; -import com.google.cloud.storage.spi.StorageRpc; import com.google.common.collect.ImmutableList; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.processor.ProcessContext; @@ -44,7 +44,7 @@ import java.util.Set; * * Every GCS processor operation requires a bucket, whether it's reading or writing from said bucket. */ -public abstract class AbstractGCSProcessor extends AbstractGCPProcessor<Storage, StorageRpc, StorageOptions> { +public abstract class AbstractGCSProcessor extends AbstractGCPProcessor<Storage, StorageOptions> { public static final Relationship REL_SUCCESS = new Relationship.Builder().name("success") .description("FlowFiles are routed to this relationship after a successful Google Cloud Storage operation.") @@ -80,25 +80,19 @@ public abstract class AbstractGCSProcessor extends AbstractGCPProcessor<Storage, StorageOptions.Builder storageOptionsBuilder = StorageOptions.newBuilder() .setCredentials(credentials) .setProjectId(projectId) - .setRetryParams(RetryParams.newBuilder() - .setRetryMaxAttempts(retryCount) - .setRetryMinAttempts(retryCount) + .setRetrySettings(RetrySettings.newBuilder() + .setMaxAttempts(retryCount) .build()); if (!StringUtils.isBlank(proxyHost) && proxyPort > 0) { - storageOptionsBuilder.setHttpTransportFactory(new HttpTransportFactory() { + storageOptionsBuilder.setTransportOptions(HttpTransportOptions.newBuilder().setHttpTransportFactory(new HttpTransportFactory() { @Override public HttpTransport create() { - final HttpTransport transport = new NetHttpTransport.Builder() - .setProxy( - new Proxy( - Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort) - ) - ) + return new NetHttpTransport.Builder() + .setProxy(new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort))) .build(); - return transport; } - }); + }).build()); } return storageOptionsBuilder.build(); } http://git-wip-us.apache.org/repos/asf/nifi/blob/f742a3a6/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/ListGCSBucket.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/ListGCSBucket.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/ListGCSBucket.java index 6bb3812..46b48fb 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/ListGCSBucket.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/ListGCSBucket.java @@ -16,7 +16,7 @@ */ package org.apache.nifi.processors.gcp.storage; -import com.google.cloud.Page; +import com.google.api.gax.paging.Page; import com.google.cloud.storage.Acl; import com.google.cloud.storage.Blob; import com.google.cloud.storage.BlobInfo; http://git-wip-us.apache.org/repos/asf/nifi/blob/f742a3a6/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/credentials/factory/MockCredentialsFactoryProcessor.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/credentials/factory/MockCredentialsFactoryProcessor.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/credentials/factory/MockCredentialsFactoryProcessor.java index 0734c40..84614e9 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/credentials/factory/MockCredentialsFactoryProcessor.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/credentials/factory/MockCredentialsFactoryProcessor.java @@ -17,8 +17,8 @@ package org.apache.nifi.processors.gcp.credentials.factory; import com.google.auth.oauth2.GoogleCredentials; -import com.google.cloud.HttpServiceOptions; import com.google.cloud.Service; +import com.google.cloud.ServiceOptions; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.components.ValidationContext; import org.apache.nifi.components.ValidationResult; @@ -53,8 +53,8 @@ public class MockCredentialsFactoryProcessor extends AbstractGCPProcessor { } @Override - protected HttpServiceOptions getServiceOptions(ProcessContext context, GoogleCredentials credentials) { - HttpServiceOptions mockOptions = mock(HttpServiceOptions.class); + protected ServiceOptions getServiceOptions(ProcessContext context, GoogleCredentials credentials) { + ServiceOptions mockOptions = mock(ServiceOptions.class); Service mockService = mock(Service.class); when(mockOptions.getService()).thenReturn(mockService); http://git-wip-us.apache.org/repos/asf/nifi/blob/f742a3a6/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/credentials/service/MockCredentialsServiceProcessor.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/credentials/service/MockCredentialsServiceProcessor.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/credentials/service/MockCredentialsServiceProcessor.java index 36c71d6..e351e4e 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/credentials/service/MockCredentialsServiceProcessor.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/credentials/service/MockCredentialsServiceProcessor.java @@ -17,8 +17,8 @@ package org.apache.nifi.processors.gcp.credentials.service; import com.google.auth.oauth2.GoogleCredentials; -import com.google.cloud.HttpServiceOptions; import com.google.cloud.Service; +import com.google.cloud.ServiceOptions; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.ProcessSession; @@ -43,8 +43,8 @@ public class MockCredentialsServiceProcessor extends AbstractGCPProcessor { } @Override - protected HttpServiceOptions getServiceOptions(ProcessContext context, GoogleCredentials credentials) { - HttpServiceOptions mockOptions = mock(HttpServiceOptions.class); + protected ServiceOptions getServiceOptions(ProcessContext context, GoogleCredentials credentials) { + ServiceOptions mockOptions = mock(ServiceOptions.class); Service mockService = mock(Service.class); when(mockOptions.getService()).thenReturn(mockService); http://git-wip-us.apache.org/repos/asf/nifi/blob/f742a3a6/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/AbstractGCSIT.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/AbstractGCSIT.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/AbstractGCSIT.java index 725f824..d66df78 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/AbstractGCSIT.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/AbstractGCSIT.java @@ -33,7 +33,6 @@ import org.junit.BeforeClass; import org.junit.experimental.categories.Category; import java.util.Arrays; -import java.util.Iterator; import static org.junit.Assert.fail; @@ -78,10 +77,9 @@ public abstract class AbstractGCSIT { public static void tearDown() { try { // Empty the bucket before deleting it. - Iterator<Blob> blobIterator = storage.list(BUCKET, Storage.BlobListOption.versions(true)).iterateAll(); + Iterable<Blob> blobIterable = storage.list(BUCKET, Storage.BlobListOption.versions(true)).iterateAll(); - while(blobIterator.hasNext()) { - Blob blob = blobIterator.next(); + for (final Blob blob : blobIterable) { storage.delete(blob.getBlobId()); } http://git-wip-us.apache.org/repos/asf/nifi/blob/f742a3a6/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/AbstractGCSTest.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/AbstractGCSTest.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/AbstractGCSTest.java index 6b35cde..8b94c47 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/AbstractGCSTest.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/AbstractGCSTest.java @@ -88,10 +88,7 @@ public abstract class AbstractGCSTest { PROJECT_ID, options.getProjectId()); assertEquals("Retry counts should match", - RETRIES.intValue(), options.getRetryParams().getRetryMinAttempts()); - - assertEquals("Retry counts should match", - RETRIES.intValue(), options.getRetryParams().getRetryMaxAttempts()); + RETRIES.intValue(), options.getRetrySettings().getMaxAttempts()); assertSame("Credentials should be configured correctly", mockCredentials, options.getCredentials()); http://git-wip-us.apache.org/repos/asf/nifi/blob/f742a3a6/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/FetchGCSObjectTest.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/FetchGCSObjectTest.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/FetchGCSObjectTest.java index 0d8b209..5e51270 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/FetchGCSObjectTest.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/FetchGCSObjectTest.java @@ -142,11 +142,6 @@ public class FetchGCSObjectTest extends AbstractGCSTest { } @Override - public void chunkSize(int i) { - - } - - @Override public void setChunkSize(int i) { } http://git-wip-us.apache.org/repos/asf/nifi/blob/f742a3a6/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/ListGCSBucketTest.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/ListGCSBucketTest.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/ListGCSBucketTest.java index 6c21d91..eca81bb 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/ListGCSBucketTest.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/ListGCSBucketTest.java @@ -16,7 +16,8 @@ */ package org.apache.nifi.processors.gcp.storage; -import com.google.cloud.Page; + +import com.google.api.gax.paging.Page; import com.google.cloud.storage.Acl; import com.google.cloud.storage.Blob; import com.google.cloud.storage.BlobInfo; http://git-wip-us.apache.org/repos/asf/nifi/blob/f742a3a6/nifi-nar-bundles/nifi-gcp-bundle/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-gcp-bundle/pom.xml b/nifi-nar-bundles/nifi-gcp-bundle/pom.xml index 36a33be..341633c 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-gcp-bundle/pom.xml @@ -26,6 +26,22 @@ <version>1.7.0-SNAPSHOT</version> <packaging>pom</packaging> + <properties> + <google.cloud.sdk.version>0.47.0-alpha</google.cloud.sdk.version> + </properties> + + <dependencyManagement> + <dependencies> + <dependency> + <groupId>com.google.cloud</groupId> + <artifactId>google-cloud</artifactId> + <version>${google.cloud.sdk.version}</version> + <type>pom</type> + <scope>import</scope> + </dependency> + </dependencies> + </dependencyManagement> + <modules> <module>nifi-gcp-services-api</module> <module>nifi-gcp-services-api-nar</module>
