This is an automated email from the ASF dual-hosted git repository. ilyak pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push: new e5d266e IGNITE-14346 Implement Azure Blob Storage based IP Finder - Fixes #8897. e5d266e is described below commit e5d266ede810c39b4ad8a3376b50ae689d5c8b86 Author: Atri Sharma <atri.j...@gmail.com> AuthorDate: Fri Apr 16 18:37:32 2021 +0300 IGNITE-14346 Implement Azure Blob Storage based IP Finder - Fixes #8897. Signed-off-by: Ilya Kasnacheev <ilya.kasnach...@gmail.com> --- assembly/dependencies-apache-ignite-slim.xml | 1 + assembly/libs/README.txt | 1 + docs/_docs/clustering/discovery-in-the-cloud.adoc | 42 +++ docs/_docs/code-snippets/java/pom.xml | 5 + .../ignite/snippets/DiscoveryInTheCloud.java | 24 ++ docs/_docs/setup.adoc | 1 + modules/azure/README.txt | 32 ++ modules/azure/pom.xml | 342 ++++++++++++++++++ .../azure/TcpDiscoveryAzureBlobStoreIpFinder.java | 382 +++++++++++++++++++++ .../spi/discovery/tcp/ipfinder/package-info.java | 23 ++ ...TcpDiscoveryAzureBlobStoreIpFinderSelfTest.java | 91 +++++ .../discovery/tcp/ipfinder/azure/package-info.java | 21 ++ .../ignite/testsuites/IgniteAzureTestSuite.java | 61 ++++ parent/pom.xml | 2 + pom.xml | 1 + 15 files changed, 1029 insertions(+) diff --git a/assembly/dependencies-apache-ignite-slim.xml b/assembly/dependencies-apache-ignite-slim.xml index 6d222c7..43f0fdf 100644 --- a/assembly/dependencies-apache-ignite-slim.xml +++ b/assembly/dependencies-apache-ignite-slim.xml @@ -145,6 +145,7 @@ <!-- Removed from slim packaging are: --> <exclude>org.apache.ignite:ignite-aop</exclude> <exclude>org.apache.ignite:ignite-aws</exclude> + <exclude>org.apache.ignite:ignite-azure</exclude> <exclude>org.apache.ignite:ignite-cassandra-serializers</exclude> <exclude>org.apache.ignite:ignite-cassandra-store</exclude> <exclude>org.apache.ignite:ignite-cloud</exclude> diff --git a/assembly/libs/README.txt b/assembly/libs/README.txt index 39f342b..12d1256 100644 --- a/assembly/libs/README.txt +++ b/assembly/libs/README.txt @@ -73,6 +73,7 @@ All optional modules can be imported just like the core module, but with differe The following modules are available: - ignite-aop (for AOP-based grid-enabling) - ignite-aws (for seemless cluster discovery on AWS S3) +- ignite-azure (for automatic cluster discovery on Azure Blob Storage) - ignite-cassandra (for Apache Cassandra integration) - ignite-cloud (for Apache JClouds integration) - ignite-gce (for automatic cluster discovery on Google Compute Engine) diff --git a/docs/_docs/clustering/discovery-in-the-cloud.adoc b/docs/_docs/clustering/discovery-in-the-cloud.adoc index 0d74b81..4c73f7825 100644 --- a/docs/_docs/clustering/discovery-in-the-cloud.adoc +++ b/docs/_docs/clustering/discovery-in-the-cloud.adoc @@ -34,6 +34,7 @@ To mitigate the constantly changing IP addresses problem, Ignite supports a numb * Amazon S3 IP Finder * Amazon ELB IP Finder * Google Cloud Storage IP Finder +* Azure Blob Storage IP Finder TIP: Cloud-based IP Finders allow you to create your configuration once and reuse it for all instances. @@ -268,3 +269,44 @@ include::{javaFile}[tag=google,indent=0] tab:C#/.NET[unsupported] tab:C++[unsupported] -- + +== Azure Blob Storage + +Ignite supports automatic node discovery by utilizing Azure Blob Storage. +This mechanism is implemented in `TcpDiscoveryAzureBlobStorageIpFinder`. +On start-up, each node registers its IP address in the storage and discovers other nodes by reading the storage. + +IMPORTANT: To use `TcpDiscoveryAzureBlobStorageIpFinder`, enable the `ignite-azure` link:setup#enabling-modules[module] in your application. + +Here is an example of how to configure Azure Blob Storage based IP finder: + +[tabs] +-- +tab:XML[] +[source,xml] +---- +<bean class="org.apache.ignite.configuration.IgniteConfiguration"> + + <property name="discoverySpi"> + <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi"> + <property name="ipFinder"> + <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.azure.TcpDiscoveryAzureBlobStoreIpFinder"> + <property name="accountName" value="YOUR_AZURE_BLOB_STORAGE_ACCOUNT_NAME"/> + <property name="accountKey" value="YOUR_AZURE_BLOB_STORAGE_ACCOUNT_KEY"/> + <property name="accountEndpoint" value="YOUR_END_POINT"/> + <property name="containerName" value="YOUR_CONTAINER_NAME"/> + </bean> + </property> + </bean> + </property> +</bean> +---- + +tab:Java[] +[source,java] +---- +include::{javaFile}[tag=google,indent=0] +---- +tab:C#/.NET[unsupported] +tab:C++[unsupported] +-- diff --git a/docs/_docs/code-snippets/java/pom.xml b/docs/_docs/code-snippets/java/pom.xml index de5623d..4589866 100644 --- a/docs/_docs/code-snippets/java/pom.xml +++ b/docs/_docs/code-snippets/java/pom.xml @@ -91,6 +91,11 @@ </dependency> <dependency> <groupId>org.apache.ignite</groupId> + <artifactId>ignite-azure</artifactId> + <version>${ignite.version}</version> + </dependency> + <dependency> + <groupId>org.apache.ignite</groupId> <artifactId>ignite-compress</artifactId> <version>${ignite.version}</version> </dependency> diff --git a/docs/_docs/code-snippets/java/src/main/java/org/apache/ignite/snippets/DiscoveryInTheCloud.java b/docs/_docs/code-snippets/java/src/main/java/org/apache/ignite/snippets/DiscoveryInTheCloud.java index 576b36d..607eb5a 100644 --- a/docs/_docs/code-snippets/java/src/main/java/org/apache/ignite/snippets/DiscoveryInTheCloud.java +++ b/docs/_docs/code-snippets/java/src/main/java/org/apache/ignite/snippets/DiscoveryInTheCloud.java @@ -148,4 +148,28 @@ public class DiscoveryInTheCloud { Ignition.start(cfg); //end::google[] } + + public static void azureBlobStorageExample() { + //tag::azureBlobStorage[] + TcpDiscoverySpi spi = new TcpDiscoverySpi(); + + TcpDiscoveryAzureBlobStorageIpFinder ipFinder = new TcpDiscoveryGoogleStorageIpFinder(); + + finder.setAccountName("yourAccountName"); + finder.setAccountKey("yourAccountKey"); + finder.setAccountEndpoint("yourEndpoint"); + + finder.setContainerName("yourContainerName"); + + spi.setIpFinder(ipFinder); + + IgniteConfiguration cfg = new IgniteConfiguration(); + + // Override default discovery SPI. + cfg.setDiscoverySpi(spi); + + // Start the node. + Ignition.start(cfg); + //end::azureBlobStorage[] + } } diff --git a/docs/_docs/setup.adoc b/docs/_docs/setup.adoc index 603934b..754f4e7 100644 --- a/docs/_docs/setup.adoc +++ b/docs/_docs/setup.adoc @@ -203,6 +203,7 @@ adding @Gridify annotation to it. |ignite-aws |Cluster discovery on AWS S3. Refer to link:clustering/discovery-in-the-cloud#amazon-s3-ip-finder[Amazon S3 IP Finder] for details. +|ignite-azure| Ignite Azure provides Azure Blob Storage-based implementation of IP finder for TCP discovery. |ignite-cassandra-serializers | The Ignite Cassandra Serializers module provides additional serializers to store objects as BLOBs in Cassandra. The module could be used as in conjunction with the Ignite Cassandra Store module. diff --git a/modules/azure/README.txt b/modules/azure/README.txt new file mode 100644 index 0000000..33501f0 --- /dev/null +++ b/modules/azure/README.txt @@ -0,0 +1,32 @@ +Apache Ignite Azure Module +------------------------ + +Apache Ignite Azure module provides Azure Blob Storage based implementation of IP finder for TCP discovery. + +To enable Azure module when starting a standalone node, move 'optional/ignite-azure' folder to +'libs' folder before running 'ignite.{sh|bat}' script. The content of the module folder will +be added to classpath in this case. + +Importing Azure Module In Maven Project +------------------------------------- + +If you are using Maven to manage dependencies of your project, you can add Azure module +dependency like this (replace '${ignite.version}' with actual Ignite version you are +interested in): + +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 + http://maven.apache.org/xsd/maven-4.0.0.xsd"> + ... + <dependencies> + ... + <dependency> + <groupId>org.apache.ignite</groupId> + <artifactId>ignite-azure</artifactId> + <version>${ignite.version}</version> + </dependency> + ... + </dependencies> + ... +</project> diff --git a/modules/azure/pom.xml b/modules/azure/pom.xml new file mode 100644 index 0000000..c3951b3 --- /dev/null +++ b/modules/azure/pom.xml @@ -0,0 +1,342 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <parent> + <artifactId>ignite-parent</artifactId> + <groupId>org.apache.ignite</groupId> + <version>1</version> + <relativePath>../../parent</relativePath> + </parent> + <modelVersion>4.0.0</modelVersion> + + <artifactId>ignite-azure</artifactId> + <version>2.11.0-SNAPSHOT</version> + <url>http://ignite.apache.org</url> + + <dependencyManagement> + <dependencies> + <dependency> + <groupId>com.azure</groupId> + <artifactId>azure-sdk-bom</artifactId> + <version>1.0.2</version> + <type>pom</type> + <scope>import</scope> + </dependency> + </dependencies> + </dependencyManagement> + + <dependencies> + <dependency> + <groupId>org.apache.ignite</groupId> + <artifactId>ignite-core</artifactId> + <version>${project.version}</version> + <scope>compile</scope> + </dependency> + + <dependency> + <groupId>org.apache.ignite</groupId> + <artifactId>ignite-core</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.apache.ignite</groupId> + <artifactId>ignite-tools</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + + <!-- https://mvnrepository.com/artifact/org.jetbrains/annotations --> + <dependency> + <groupId>org.jetbrains</groupId> + <artifactId>annotations</artifactId> + <version>${jetbrains.annotations.version}</version> + </dependency> + + <dependency> + <groupId>com.azure</groupId> + <artifactId>azure-core-http-netty</artifactId> + <version>1.0.0</version> + </dependency> + + <dependency> + <groupId>com.azure</groupId> + <artifactId>azure-core</artifactId> + <version>1.0.0</version> + </dependency> + + <dependency> + <groupId>com.azure</groupId> + <artifactId>azure-storage-blob</artifactId> + <version>${azure.sdk.version}</version> + </dependency> + + <dependency> + <groupId>com.azure</groupId> + <artifactId>azure-storage-common</artifactId> + <version>${azure.sdk.version}</version> + </dependency> + + <!-- https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-annotations --> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-annotations</artifactId> + <version>${jackson.version}</version> + </dependency> + + <dependency> + <groupId>com.azure</groupId> + <artifactId>azure-security-keyvault-secrets</artifactId> + </dependency> + <dependency> + <groupId>com.azure</groupId> + <artifactId>azure-identity</artifactId> + </dependency> + + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <version>${jackson.version}</version> + </dependency> + + <!-- https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml --> + <dependency> + <groupId>com.fasterxml.jackson.dataformat</groupId> + <artifactId>jackson-dataformat-xml</artifactId> + <version>${jackson.version}</version> + </dependency> + + <!-- https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-jsr310 --> + <dependency> + <groupId>com.fasterxml.jackson.datatype</groupId> + <artifactId>jackson-datatype-jsr310</artifactId> + <version>${jackson.version}</version> + </dependency> + + <!-- https://mvnrepository.com/artifact/com.fasterxml.jackson.module/jackson-module-jaxb-annotations --> + <dependency> + <groupId>com.fasterxml.jackson.module</groupId> + <artifactId>jackson-module-jaxb-annotations</artifactId> + <version>${jackson.version}</version> + </dependency> + + <!-- https://mvnrepository.com/artifact/org.apache.logging.log4j/log4j-api --> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-api</artifactId> + <version>${log4j.version}</version> + </dependency> + + <!-- https://mvnrepository.com/artifact/org.apache.logging.log4j/log4j-core --> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-core</artifactId> + <version>${log4j.version}</version> + </dependency> + + <!-- https://mvnrepository.com/artifact/org.apache.logging.log4j/log4j-slf4j-impl --> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-slf4j-impl</artifactId> + <version>${log4j.version}</version> + </dependency> + + <!-- https://mvnrepository.com/artifact/io.netty/netty-buffer --> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-buffer</artifactId> + <version>${azure.netty.version}</version> + </dependency> + + <!-- https://mvnrepository.com/artifact/io.netty/netty-codec --> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-codec</artifactId> + <version>${azure.netty.version}</version> + </dependency> + + <!-- https://mvnrepository.com/artifact/io.netty/netty-codec-http2 --> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-codec-http2</artifactId> + <version>${azure.netty.version}</version> + </dependency> + + <!-- https://mvnrepository.com/artifact/io.netty/netty-codec-http --> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-codec-http</artifactId> + <version>${azure.netty.version}</version> + </dependency> + + <!-- https://mvnrepository.com/artifact/io.netty/netty-codec-socks --> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-codec-socks</artifactId> + <version>${azure.netty.version}</version> + </dependency> + + <!-- https://mvnrepository.com/artifact/io.netty/netty-common --> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-common</artifactId> + <version>${azure.netty.version}</version> + </dependency> + + <!-- https://mvnrepository.com/artifact/io.netty/netty-handler --> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-handler</artifactId> + <version>${azure.netty.version}</version> + </dependency> + + <!-- https://mvnrepository.com/artifact/io.netty/netty-handler-proxy --> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-handler-proxy</artifactId> + <version>${azure.netty.version}</version> + </dependency> + + <!-- https://mvnrepository.com/artifact/io.netty/netty-resolver --> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-resolver</artifactId> + <version>${azure.netty.version}</version> + </dependency> + + <!-- https://mvnrepository.com/artifact/io.netty/netty-transport --> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-transport</artifactId> + <version>${azure.netty.version}</version> + </dependency> + + <!-- https://mvnrepository.com/artifact/io.netty/netty-transport-native-epoll --> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-transport-native-epoll</artifactId> + <version>${azure.netty.version}</version> + </dependency> + + <!-- https://mvnrepository.com/artifact/io.netty/netty-transport-native-kqueue --> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-transport-native-kqueue</artifactId> + <version>${azure.netty.version}</version> + </dependency> + + <!-- https://mvnrepository.com/artifact/io.netty/netty-transport-native-kqueue --> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-transport-native-kqueue</artifactId> + <version>${azure.netty.version}</version> + </dependency> + + <!-- https://mvnrepository.com/artifact/io.netty/netty-transport-native-unix-common --> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-transport-native-unix-common</artifactId> + <version>${azure.netty.version}</version> + </dependency> + + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + <version>${slf4j.version}</version> + </dependency> + + <dependency> + <groupId>org.reactivestreams</groupId> + <artifactId>reactive-streams</artifactId> + <version>1.0.3</version> + </dependency> + + <!-- https://mvnrepository.com/artifact/io.projectreactor/reactor-core --> + <dependency> + <groupId>io.projectreactor</groupId> + <artifactId>reactor-core</artifactId> + <version>3.3.0.RELEASE</version> + </dependency> + + <!-- https://mvnrepository.com/artifact/io.projectreactor.netty/reactor-netty --> + <dependency> + <groupId>io.projectreactor.netty</groupId> + <artifactId>reactor-netty</artifactId> + <version>0.9.0.RELEASE</version> + </dependency> + + <!-- https://mvnrepository.com/artifact/org.codehaus.woodstox/stax2-api --> + <dependency> + <groupId>org.codehaus.woodstox</groupId> + <artifactId>stax2-api</artifactId> + <version>4.2.1</version> + </dependency> + + <!-- https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-core --> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + <version>${jackson.version}</version> + </dependency> + + <!-- https://mvnrepository.com/artifact/org.reflections/reflections --> + <dependency> + <groupId>org.reflections</groupId> + <artifactId>reflections</artifactId> + <version>0.9.12</version> + </dependency> + + <!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-simple --> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-simple</artifactId> + <version>${slf4j.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-beans</artifactId> + <version>${spring.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-context</artifactId> + <version>${spring.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-core</artifactId> + <version>${spring.version}</version> + <scope>test</scope> + </dependency> + </dependencies> + +</project> diff --git a/modules/azure/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/azure/TcpDiscoveryAzureBlobStoreIpFinder.java b/modules/azure/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/azure/TcpDiscoveryAzureBlobStoreIpFinder.java new file mode 100644 index 0000000..031554b --- /dev/null +++ b/modules/azure/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/azure/TcpDiscoveryAzureBlobStoreIpFinder.java @@ -0,0 +1,382 @@ +package org.apache.ignite.spi.discovery.tcp.ipfinder.azure; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.net.InetSocketAddress; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; + +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.BlobServiceClient; +import com.azure.storage.blob.BlobServiceClientBuilder; +import com.azure.storage.blob.models.BlobItem; +import com.azure.storage.blob.models.BlobStorageException; +import com.azure.storage.blob.specialized.BlockBlobClient; +import com.azure.storage.common.StorageSharedKeyCredential; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.internal.IgniteInterruptedCheckedException; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.resources.LoggerResource; +import org.apache.ignite.spi.IgniteSpiConfiguration; +import org.apache.ignite.spi.IgniteSpiException; +import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinderAdapter; + +import static com.nimbusds.oauth2.sdk.util.URLUtils.CHARSET; + +/** + * Azure Blob Storage based IP Finder + * <p> + * For information about Blob Storage visit <a href="https://azure.microsoft.com/en-in/services/storage/blobs/">azure.microsoft.com</a>. + * <h1 class="header">Configuration</h1> + * <h2 class="header">Mandatory</h2> + * <ul> + * <li>AccountName (see {@link #setAccountName(String)})</li> + * <li>AccountKey (see {@link #setAccountKey(String)})</li> + * <li>Account Endpoint (see {@link #setAccountEndpoint(String)})</li> + * <li>Container Name (see {@link #setContainerName(String)})</li> + * </ul> + * <h2 class="header">Optional</h2> + * <ul> + * <li>Shared flag (see {@link #setShared(boolean)})</li> + * </ul> + * <p> + * The finder will create a container with the provided name. The container will contain entries named + * like the following: {@code 192.168.1.136#1001}. + * <p> + * Note that storing data in Azure Blob Storage service will result in charges to your Azure account. + * Choose another implementation of {@link org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder} for local + * or home network tests. + * <p> + * Note that this finder is shared by default (see {@link org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder#isShared()}. + */ +public class TcpDiscoveryAzureBlobStoreIpFinder extends TcpDiscoveryIpFinderAdapter { + /** Default object's content. */ + private static final byte[] OBJECT_CONTENT = new byte[0]; + + /** Grid logger. */ + @LoggerResource + private IgniteLogger log; + + /** Azure Blob Storage's account name*/ + private String accountName; + + /** Azure Blob Storage's account key */ + private String accountKey; + + /** End point URL */ + private String endPoint; + + /** Container name */ + private String containerName; + + /** Storage credential */ + StorageSharedKeyCredential credential; + + /** Blob service client */ + private BlobServiceClient blobServiceClient; + + /** Blob container client */ + private BlobContainerClient blobContainerClient; + + /** Init routine guard. */ + private final AtomicBoolean initGuard = new AtomicBoolean(); + + /** Init routine latch. */ + private final CountDownLatch initLatch = new CountDownLatch(1); + + /** + * Default constructor + */ + public TcpDiscoveryAzureBlobStoreIpFinder() { + setShared(true); + } + + /** {@inheritDoc} */ + @Override public Collection<InetSocketAddress> getRegisteredAddresses() throws IgniteSpiException { + init(); + + Collection<InetSocketAddress> addrs = new ArrayList<>(); + Set<String> seenBlobNames = new HashSet<>(); + + Iterator<BlobItem> blobItemIterator = blobContainerClient.listBlobs().iterator(); + + while (blobItemIterator.hasNext()) { + BlobItem blobItem = blobItemIterator.next(); + + // https://github.com/Azure/azure-sdk-for-java/issues/20515 + if (seenBlobNames.contains(blobItem.getName())) { + break; + } + + try { + if (!blobItem.isDeleted()) { + addrs.add(addrFromString(blobItem.getName())); + seenBlobNames.add(blobItem.getName()); + } + } + catch (Exception e) { + throw new IgniteSpiException("Failed to get content from the container: " + containerName, e); + } + } + + return addrs; + } + + /** {@inheritDoc} */ + @Override public void registerAddresses(Collection<InetSocketAddress> addrs) throws IgniteSpiException { + assert !F.isEmpty(addrs); + + init(); + + for (InetSocketAddress addr : addrs) { + String key = keyFromAddr(addr); + try { + key = URLEncoder.encode(key, CHARSET); + } catch (UnsupportedEncodingException e) { + throw new IgniteSpiException("Unable to encode URL due to error " + + e.getMessage()); + } + + BlockBlobClient blobClient = blobContainerClient.getBlobClient(key).getBlockBlobClient(); + InputStream dataStream = new ByteArrayInputStream(OBJECT_CONTENT); + + try { + blobClient.upload(dataStream, OBJECT_CONTENT.length); + } + catch (BlobStorageException e) { + // If the blob already exists, ignore + if (!(e.getStatusCode() == 409)) { + throw new IgniteSpiException("Failed to upload blob with exception " + + e.getMessage()); + } + } + + try { + dataStream.close(); + } + catch (IOException e) { + throw new IgniteSpiException(e.getMessage()); + } + } + } + + /** {@inheritDoc} */ + @Override public void unregisterAddresses(Collection<InetSocketAddress> addrs) throws IgniteSpiException { + assert !F.isEmpty(addrs); + + init(); + + for (InetSocketAddress addr : addrs) { + String key = keyFromAddr(addr); + + try { + blobContainerClient.getBlobClient(key).delete(); + } catch (Exception e) { + // https://github.com/Azure/azure-sdk-for-java/issues/20551 + if ((!(e.getMessage().contains("InterruptedException"))) || (e instanceof BlobStorageException + && (!((BlobStorageException) e).getErrorCode().equals(404)))) { + throw new IgniteSpiException("Failed to delete entry [containerName=" + containerName + + ", entry=" + key + ']', e); + } + } + } + } + + /** + * Sets Azure Blob Storage Account Name. + * <p> + * For details refer to Azure Blob Storage API reference. + * + * @param accountName Account Name + * @return {@code this} for chaining. + */ + @IgniteSpiConfiguration(optional = false) + public TcpDiscoveryAzureBlobStoreIpFinder setAccountName(String accountName) { + this.accountName = accountName; + + return this; + } + + /** + * Sets Azure Blob Storage Account Key + * <p> + * For details refer to Azure Blob Storage API reference. + * + * @param accountKey Account Key + * @return {@code this} for chaining. + */ + @IgniteSpiConfiguration(optional = false) + public TcpDiscoveryAzureBlobStoreIpFinder setAccountKey(String accountKey) { + this.accountKey = accountKey; + + return this; + } + + /** + * Sets Azure Blob Storage endpoint + * <p> + * For details refer to Azure Blob Storage API reference. + * + * @param endPoint Endpoint for storage + * @return {@code this} for chaining. + */ + @IgniteSpiConfiguration(optional = false) + public TcpDiscoveryAzureBlobStoreIpFinder setAccountEndpoint(String endPoint) { + this.endPoint = endPoint; + + return this; + } + + /** + * Sets container name for using in the context + * If the container name doesn't exist Ignite will automatically create itß. + * + * @param containerName Container Name. + * @return {@code this} for chaining. + */ + @IgniteSpiConfiguration(optional = false) + public TcpDiscoveryAzureBlobStoreIpFinder setContainerName(String containerName) { + this.containerName = containerName; + + return this; + } + + /** + * Initialize the IP finder + * @throws IgniteSpiException + */ + private void init() throws IgniteSpiException { + if (initGuard.compareAndSet(false, true)) { + if (accountKey == null || accountName == null || containerName == null || endPoint == null) { + throw new IgniteSpiException( + "One or more of the required parameters is not set [accountName=" + + accountName + ", accountKey=" + accountKey + ", containerName=" + + containerName + ", endPoint=" + endPoint + "]"); + } + + try { + credential = new StorageSharedKeyCredential(accountName, accountKey); + blobServiceClient = new BlobServiceClientBuilder().endpoint(endPoint).credential(credential).buildClient(); + blobContainerClient = blobServiceClient.getBlobContainerClient(containerName); + + if (!blobContainerClient.exists()) { + U.warn(log, "Container doesn't exist, will create it [containerName=" + containerName + "]"); + + blobContainerClient.create(); + } + } + finally { + initLatch.countDown(); + } + } + else { + try { + U.await(initLatch); + } + catch (IgniteInterruptedCheckedException e) { + throw new IgniteSpiException("Thread has been interrupted.", e); + } + + try { + if (!blobContainerClient.exists()) + throw new IgniteSpiException("IpFinder has not been initialized properly"); + } catch (Exception e) { + // Check if this is a nested exception wrapping an InterruptedException + // https://github.com/Azure/azure-sdk-for-java/issues/20551 + if (!(e.getCause() instanceof InterruptedException)) { + throw e; + } + } + } + } + + /** + * Constructs a node address from bucket's key. + * + * @param key Bucket key. + * @return Node address. + * @throws IgniteSpiException In case of error. + */ + private InetSocketAddress addrFromString(String key) throws IgniteSpiException { + //TODO: This needs to move out to a generic helper class + String[] res = key.split("#"); + + if (res.length != 2) + throw new IgniteSpiException("Invalid address string: " + key); + + int port; + + try { + port = Integer.parseInt(res[1]); + } + catch (NumberFormatException ignored) { + throw new IgniteSpiException("Invalid port number: " + res[1]); + } + + return new InetSocketAddress(res[0], port); + } + + /** + * Constructs bucket's key from an address. + * + * @param addr Node address. + * @return Bucket key. + */ + private String keyFromAddr(InetSocketAddress addr) { + // TODO: This needs to move out to a generic helper class + return addr.getAddress().getHostAddress() + "#" + addr.getPort(); + } + + /** + * Used by TEST SUITES only. Called through reflection. + * + * @param containerName Container to delete + */ + private void removeContainer(String containerName) { + init(); + + try { + blobServiceClient.getBlobContainerClient(containerName).delete(); + } + catch (Exception e) { + throw new IgniteSpiException("Failed to remove the container: " + containerName, e); + } + } + + /** {@inheritDoc} */ + @Override public TcpDiscoveryAzureBlobStoreIpFinder setShared(boolean shared) { + super.setShared(shared); + + return this; + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(TcpDiscoveryAzureBlobStoreIpFinder.class, this); + } +} diff --git a/modules/azure/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/package-info.java b/modules/azure/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/package-info.java new file mode 100644 index 0000000..b8410b3 --- /dev/null +++ b/modules/azure/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/package-info.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * <!-- Package description. --> + * Contains Azure Blob Storage IP finder. + */ + +package org.apache.ignite.spi.discovery.tcp.ipfinder; diff --git a/modules/azure/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/azure/TcpDiscoveryAzureBlobStoreIpFinderSelfTest.java b/modules/azure/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/azure/TcpDiscoveryAzureBlobStoreIpFinderSelfTest.java new file mode 100644 index 0000000..3c172c2 --- /dev/null +++ b/modules/azure/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/azure/TcpDiscoveryAzureBlobStoreIpFinderSelfTest.java @@ -0,0 +1,91 @@ +package org.apache.ignite.spi.discovery.tcp.ipfinder.azure; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.lang.reflect.Method; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.util.Collection; + +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinderAbstractSelfTest; +import org.apache.ignite.testsuites.IgniteAzureTestSuite; + +public class TcpDiscoveryAzureBlobStoreIpFinderSelfTest + extends TcpDiscoveryIpFinderAbstractSelfTest<TcpDiscoveryAzureBlobStoreIpFinder> { + private static String containerName; + + /** + * Constructor. + * + * @throws Exception If any error occurs. + */ + public TcpDiscoveryAzureBlobStoreIpFinderSelfTest() throws Exception { + // No-op. + } + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + containerName = "ip-finder-test-container-" + InetAddress.getLocalHost().getAddress()[3]; + + super.beforeTestsStarted(); + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() { + try { + Method method = TcpDiscoveryAzureBlobStoreIpFinder.class.getDeclaredMethod("removeContainer", String.class); + + method.setAccessible(true); + + method.invoke(finder, containerName); + } + catch (Exception e) { + log.warning("Failed to remove bucket on Azure [containerName=" + containerName + ", mes=" + e.getMessage() + ']'); + } + } + + /** {@inheritDoc} */ + @Override protected TcpDiscoveryAzureBlobStoreIpFinder ipFinder() throws Exception { + TcpDiscoveryAzureBlobStoreIpFinder finder = new TcpDiscoveryAzureBlobStoreIpFinder(); + + injectLogger(finder); + + assert finder.isShared() : "Ip finder must be shared by default."; + + finder.setAccountName(IgniteAzureTestSuite.getAccountName()); + finder.setAccountKey(IgniteAzureTestSuite.getAccountKey()); + finder.setAccountEndpoint(IgniteAzureTestSuite.getEndpoint()); + + finder.setContainerName(containerName); + + for (int i = 0; i < 5; i++) { + Collection<InetSocketAddress> addrs = finder.getRegisteredAddresses(); + + if (!addrs.isEmpty()) + finder.unregisterAddresses(addrs); + else + return finder; + + U.sleep(1000); + } + + if (!finder.getRegisteredAddresses().isEmpty()) + throw new Exception("Failed to initialize IP finder."); + + return finder; + } +} diff --git a/modules/azure/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/azure/package-info.java b/modules/azure/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/azure/package-info.java new file mode 100644 index 0000000..c3f6711 --- /dev/null +++ b/modules/azure/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/azure/package-info.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * <!-- Package description. --> + * Contains Azure Blob Storage IP finder internal tests. + */ +package org.apache.ignite.spi.discovery.tcp.ipfinder.azure; diff --git a/modules/azure/src/test/java/org/apache/ignite/testsuites/IgniteAzureTestSuite.java b/modules/azure/src/test/java/org/apache/ignite/testsuites/IgniteAzureTestSuite.java new file mode 100644 index 0000000..0549568 --- /dev/null +++ b/modules/azure/src/test/java/org/apache/ignite/testsuites/IgniteAzureTestSuite.java @@ -0,0 +1,61 @@ +package org.apache.ignite.testsuites; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.ignite.spi.discovery.tcp.ipfinder.azure.TcpDiscoveryAzureBlobStoreIpFinderSelfTest; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +/** + * Azure integration tests + */ +@RunWith(Suite.class) +@Suite.SuiteClasses({TcpDiscoveryAzureBlobStoreIpFinderSelfTest.class}) +public class IgniteAzureTestSuite { + /** + * @return Account Name + */ + public static String getAccountName() { + String id = System.getenv("test.azure.account.name"); + + assert id != null : "Environment variable 'test.azure.account.name' is not set"; + + return id; + } + + /** + * @return Account Key + */ + public static String getAccountKey() { + String path = System.getenv("test.azure.account.key"); + + assert path != null : "Environment variable 'test.azure.account.key' is not set"; + + return path; + } + + /** + * @return Endpoint + */ + public static String getEndpoint() { + String name = System.getenv("test.azure.endpoint"); + + assert name != null : "Environment variable 'test.azure.endpoint' is not set"; + + return name; + } +} diff --git a/parent/pom.xml b/parent/pom.xml index b227c30..05203a1 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -54,6 +54,8 @@ <aspectj.version>1.8.13</aspectj.version> <aws.sdk.bundle.version>1.10.12_1</aws.sdk.bundle.version> <aws.sdk.version>1.11.75</aws.sdk.version> + <azure.sdk.version>12.0.0</azure.sdk.version> + <azure.netty.version>4.1.54.Final</azure.netty.version> <camel.version>2.22.0</camel.version> <aws.encryption.sdk.version>1.3.2</aws.encryption.sdk.version> <bouncycastle.version>1.60</bouncycastle.version> diff --git a/pom.xml b/pom.xml index a054b86..ed2c607 100644 --- a/pom.xml +++ b/pom.xml @@ -67,6 +67,7 @@ <module>modules/jcl</module> <module>modules/codegen</module> <module>modules/gce</module> + <module>modules/azure</module> <module>modules/cloud</module> <module>modules/mesos</module> <module>modules/yarn</module>