This is an automated email from the ASF dual-hosted git repository.
ppalaga pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/camel-quarkus.git
The following commit(s) were added to refs/heads/master by this push:
new 01f3001 Azure Storage Data Lake JVM support #2289
01f3001 is described below
commit 01f3001ff577423015fa00ef8862dc46f66e4792
Author: Peter Palaga <[email protected]>
AuthorDate: Tue Mar 2 12:58:05 2021 +0100
Azure Storage Data Lake JVM support #2289
---
catalog/pom.xml | 13 ++
.../extensions/azure-storage-datalake.adoc | 33 ++++
.../components/azure-storage-datalake.adoc | 14 +-
.../azure-storage-datalake/deployment/pom.xml | 67 ++++++++
.../deployment/AzureStorageDatalakeProcessor.java | 46 ++++++
.../integration-test/pom.xml | 93 +++++++++++
.../datalake/it/AzureStorageDatalakeResource.java | 179 +++++++++++++++++++++
.../src/main/resources/application.properties | 26 +++
.../datalake/it/AzureStorageDatalakeTest.java | 115 +++++++++++++
extensions-jvm/azure-storage-datalake/pom.xml | 40 +++++
.../azure-storage-datalake/runtime/pom.xml | 116 +++++++++++++
.../main/resources/META-INF/quarkus-extension.yaml | 32 ++++
extensions-jvm/pom.xml | 1 +
.../support/azure/AzureStorageTestResource.java | 65 ++++++--
poms/bom/pom.xml | 15 ++
15 files changed, 841 insertions(+), 14 deletions(-)
diff --git a/catalog/pom.xml b/catalog/pom.xml
index bfe3963..792adc3 100644
--- a/catalog/pom.xml
+++ b/catalog/pom.xml
@@ -738,6 +738,19 @@
</dependency>
<dependency>
<groupId>org.apache.camel.quarkus</groupId>
+ <artifactId>camel-quarkus-azure-storage-datalake</artifactId>
+ <version>${project.version}</version>
+ <type>pom</type>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel.quarkus</groupId>
<artifactId>camel-quarkus-azure-storage-queue</artifactId>
<version>${project.version}</version>
<type>pom</type>
diff --git
a/docs/modules/ROOT/pages/reference/extensions/azure-storage-datalake.adoc
b/docs/modules/ROOT/pages/reference/extensions/azure-storage-datalake.adoc
new file mode 100644
index 0000000..787af79
--- /dev/null
+++ b/docs/modules/ROOT/pages/reference/extensions/azure-storage-datalake.adoc
@@ -0,0 +1,33 @@
+// Do not edit directly!
+// This file was generated by
camel-quarkus-maven-plugin:update-extension-doc-page
+= Azure storage datalake service
+:cq-artifact-id: camel-quarkus-azure-storage-datalake
+:cq-native-supported: false
+:cq-status: Preview
+:cq-description: Camel Azure Datalake Gen2 Component
+:cq-deprecated: false
+:cq-jvm-since: 1.8.0
+:cq-native-since: n/a
+
+[.badges]
+[.badge-key]##JVM since##[.badge-supported]##1.8.0##
[.badge-key]##Native##[.badge-unsupported]##unsupported##
+
+Camel Azure Datalake Gen2 Component
+
+== What's inside
+
+* xref:{cq-camel-components}::azure-storage-datalake-component.adoc[Azure
storage datalake service component], URI syntax:
`azure-storage-datalake:accountName/fileSystemName`
+
+Please refer to the above link for usage and configuration details.
+
+== Maven coordinates
+
+[source,xml]
+----
+<dependency>
+ <groupId>org.apache.camel.quarkus</groupId>
+ <artifactId>camel-quarkus-azure-storage-datalake</artifactId>
+</dependency>
+----
+
+Check the xref:user-guide/index.adoc[User guide] for more information about
writing Camel Quarkus applications.
diff --git
a/docs/modules/ROOT/partials/reference/components/azure-storage-datalake.adoc
b/docs/modules/ROOT/partials/reference/components/azure-storage-datalake.adoc
index a509c1d..ff44960 100644
---
a/docs/modules/ROOT/partials/reference/components/azure-storage-datalake.adoc
+++
b/docs/modules/ROOT/partials/reference/components/azure-storage-datalake.adoc
@@ -1 +1,13 @@
-// Empty partial for a Camel bit unsupported by Camel Quarkus to avoid
warnings when this file is included from a Camel page
+// Do not edit directly!
+// This file was generated by
camel-quarkus-maven-plugin:update-extension-doc-page
+:cq-artifact-id: camel-quarkus-azure-storage-datalake
+:cq-artifact-id-base: azure-storage-datalake
+:cq-native-supported: false
+:cq-status: Preview
+:cq-deprecated: false
+:cq-jvm-since: 1.8.0
+:cq-native-since: n/a
+:cq-camel-part-name: azure-storage-datalake
+:cq-camel-part-title: Azure storage datalake service
+:cq-camel-part-description: Camel Azure Datalake Gen2 Component
+:cq-extension-page-title: Azure storage datalake service
diff --git a/extensions-jvm/azure-storage-datalake/deployment/pom.xml
b/extensions-jvm/azure-storage-datalake/deployment/pom.xml
new file mode 100644
index 0000000..f7ec4c2
--- /dev/null
+++ b/extensions-jvm/azure-storage-datalake/deployment/pom.xml
@@ -0,0 +1,67 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.camel.quarkus</groupId>
+ <artifactId>camel-quarkus-azure-storage-datalake-parent</artifactId>
+ <version>1.8.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>camel-quarkus-azure-storage-datalake-deployment</artifactId>
+ <name>Camel Quarkus :: Azure storage datalake service :: Deployment</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.camel.quarkus</groupId>
+ <artifactId>camel-quarkus-core-deployment</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel.quarkus</groupId>
+
<artifactId>camel-quarkus-support-azure-core-deployment</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel.quarkus</groupId>
+ <artifactId>camel-quarkus-azure-storage-datalake</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <annotationProcessorPaths>
+ <path>
+ <groupId>io.quarkus</groupId>
+
<artifactId>quarkus-extension-processor</artifactId>
+ <version>${quarkus.version}</version>
+ </path>
+ </annotationProcessorPaths>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
diff --git
a/extensions-jvm/azure-storage-datalake/deployment/src/main/java/org/apache/camel/quarkus/component/azure/storage/datalake/deployment/AzureStorageDatalakeProcessor.java
b/extensions-jvm/azure-storage-datalake/deployment/src/main/java/org/apache/camel/quarkus/component/azure/storage/datalake/deployment/AzureStorageDatalakeProcessor.java
new file mode 100644
index 0000000..ba05640
--- /dev/null
+++
b/extensions-jvm/azure-storage-datalake/deployment/src/main/java/org/apache/camel/quarkus/component/azure/storage/datalake/deployment/AzureStorageDatalakeProcessor.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.quarkus.component.azure.storage.datalake.deployment;
+
+import io.quarkus.deployment.annotations.BuildStep;
+import io.quarkus.deployment.annotations.ExecutionTime;
+import io.quarkus.deployment.annotations.Record;
+import io.quarkus.deployment.builditem.FeatureBuildItem;
+import io.quarkus.deployment.pkg.steps.NativeBuild;
+import org.apache.camel.quarkus.core.JvmOnlyRecorder;
+import org.jboss.logging.Logger;
+
+class AzureStorageDatalakeProcessor {
+
+ private static final Logger LOG =
Logger.getLogger(AzureStorageDatalakeProcessor.class);
+ private static final String FEATURE = "camel-azure-storage-datalake";
+
+ @BuildStep
+ FeatureBuildItem feature() {
+ return new FeatureBuildItem(FEATURE);
+ }
+
+ /**
+ * Remove this once this extension starts supporting the native mode.
+ */
+ @BuildStep(onlyIf = NativeBuild.class)
+ @Record(value = ExecutionTime.RUNTIME_INIT)
+ void warnJvmInNative(JvmOnlyRecorder recorder) {
+ JvmOnlyRecorder.warnJvmInNative(LOG, FEATURE); // warn at build time
+ recorder.warnJvmInNative(FEATURE); // warn at runtime
+ }
+}
diff --git a/extensions-jvm/azure-storage-datalake/integration-test/pom.xml
b/extensions-jvm/azure-storage-datalake/integration-test/pom.xml
new file mode 100644
index 0000000..88d39bb
--- /dev/null
+++ b/extensions-jvm/azure-storage-datalake/integration-test/pom.xml
@@ -0,0 +1,93 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.camel.quarkus</groupId>
+ <artifactId>camel-quarkus-build-parent-it</artifactId>
+ <version>1.8.0-SNAPSHOT</version>
+ <relativePath>../../../poms/build-parent-it/pom.xml</relativePath>
+ </parent>
+
+
<artifactId>camel-quarkus-azure-storage-datalake-integration-test</artifactId>
+ <name>Camel Quarkus :: Azure storage datalake service :: Integration
Test</name>
+ <description>Integration tests for Camel Quarkus Azure storage datalake
service extension</description>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.camel.quarkus</groupId>
+ <artifactId>camel-quarkus-bom-test</artifactId>
+ <version>${project.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.camel.quarkus</groupId>
+ <artifactId>camel-quarkus-azure-storage-datalake</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>io.quarkus</groupId>
+ <artifactId>quarkus-resteasy</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>io.quarkus</groupId>
+ <artifactId>quarkus-resteasy-jackson</artifactId>
+ </dependency>
+
+ <!-- test dependencies -->
+ <dependency>
+ <groupId>io.quarkus</groupId>
+ <artifactId>quarkus-junit5</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>io.rest-assured</groupId>
+ <artifactId>rest-assured</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel.quarkus</groupId>
+
<artifactId>camel-quarkus-integration-tests-support-azure</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- The following dependencies guarantee that this module is built
after them. You can update them by running `mvn process-resources -Pformat -N`
from the source tree root directory -->
+ <dependency>
+ <groupId>org.apache.camel.quarkus</groupId>
+
<artifactId>camel-quarkus-azure-storage-datalake-deployment</artifactId>
+ <version>${project.version}</version>
+ <type>pom</type>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+</project>
diff --git
a/extensions-jvm/azure-storage-datalake/integration-test/src/main/java/org/apache/camel/quarkus/component/azure/storage/datalake/it/AzureStorageDatalakeResource.java
b/extensions-jvm/azure-storage-datalake/integration-test/src/main/java/org/apache/camel/quarkus/component/azure/storage/datalake/it/AzureStorageDatalakeResource.java
new file mode 100644
index 0000000..f8c7ab5
--- /dev/null
+++
b/extensions-jvm/azure-storage-datalake/integration-test/src/main/java/org/apache/camel/quarkus/component/azure/storage/datalake/it/AzureStorageDatalakeResource.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.quarkus.component.azure.storage.datalake.it;
+
+import java.net.URI;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import javax.enterprise.context.ApplicationScoped;
+import javax.inject.Inject;
+import javax.inject.Named;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DELETE;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
+import com.azure.core.http.policy.HttpLogDetailLevel;
+import com.azure.core.http.policy.HttpLogOptions;
+import com.azure.storage.common.StorageSharedKeyCredential;
+import com.azure.storage.file.datalake.DataLakeServiceClient;
+import com.azure.storage.file.datalake.DataLakeServiceClientBuilder;
+import com.azure.storage.file.datalake.models.FileSystemItem;
+import com.azure.storage.file.datalake.models.PathItem;
+import org.apache.camel.ConsumerTemplate;
+import org.apache.camel.ProducerTemplate;
+import org.apache.camel.component.azure.storage.datalake.DataLakeConstants;
+import
org.apache.camel.component.azure.storage.datalake.DataLakeOperationsDefinition;
+import org.eclipse.microprofile.config.inject.ConfigProperty;
+
+@Path("/azure-storage-datalake")
+@ApplicationScoped
+public class AzureStorageDatalakeResource {
+
+ @Inject
+ ProducerTemplate producerTemplate;
+
+ @Inject
+ ConsumerTemplate consumerTemplate;
+
+ @ConfigProperty(name = "azure.storage.account-name")
+ String azureStorageAccountName;
+
+ @ConfigProperty(name = "azure.storage.account-key")
+ String azureStorageAccountKey;
+
+ @ConfigProperty(name = "azure.datalake.service.url")
+ String serviceUrl;
+
+ @javax.enterprise.inject.Produces
+ @Named("azureDatalakeServiceClient")
+ public DataLakeServiceClient createDatalakeServiceClient() throws
Exception {
+ StorageSharedKeyCredential credentials = new
StorageSharedKeyCredential(azureStorageAccountName,
+ azureStorageAccountKey);
+ return new DataLakeServiceClientBuilder()
+ .endpoint(serviceUrl)
+ .credential(credentials)
+ .httpLogOptions(new
HttpLogOptions().setLogLevel(HttpLogDetailLevel.BODY_AND_HEADERS).setPrettyPrintBody(true))
+ .buildClient();
+ }
+
+ @Path("/filesystem/{filesystem}")
+ @POST
+ @Produces(MediaType.TEXT_PLAIN)
+ public Response createFileSystem(@PathParam("filesystem") String
filesystem) throws Exception {
+ producerTemplate.sendBody(componentUri(filesystem,
DataLakeOperationsDefinition.createFileSystem), null);
+ return Response.created(new URI("https://camel.apache.org/";)).build();
+ }
+
+ @Path("/filesystem/{filesystem}/path/{filename}")
+ @POST
+ @Consumes(MediaType.TEXT_PLAIN)
+ public Response upload(@PathParam("filesystem") String filesystem,
+ @PathParam("filename") String filename,
+ byte[] body) throws Exception {
+ producerTemplate.sendBodyAndHeader(
+ componentUri(filesystem, DataLakeOperationsDefinition.upload),
+ body,
+ DataLakeConstants.FILE_NAME,
+ filename);
+ return Response.created(new URI("https://camel.apache.org/";)).build();
+ }
+
+ @Path("/filesystem/{filesystem}/path/{filename}")
+ @GET
+ @Produces(MediaType.TEXT_PLAIN)
+ public String getFile(@PathParam("filesystem") String filesystem,
+ @PathParam("filename") String filename) throws Exception {
+ return producerTemplate.requestBodyAndHeader(
+ componentUri(filesystem, DataLakeOperationsDefinition.getFile),
+ null,
+ DataLakeConstants.FILE_NAME,
+ filename,
+ String.class);
+ }
+
+ @Path("/filesystem/{filesystem}/path/{filename}")
+ @DELETE
+ public void deleteFile(@PathParam("filesystem") String filesystem,
+ @PathParam("filename") String filename) throws Exception {
+ producerTemplate.sendBodyAndHeader(
+ componentUri(filesystem,
DataLakeOperationsDefinition.deleteFile),
+ null,
+ DataLakeConstants.FILE_NAME,
+ filename);
+ }
+
+ @Path("/filesystem/{filesystem}")
+ @DELETE
+ @Produces(MediaType.TEXT_PLAIN)
+ public void deleteFileSystem(@PathParam("filesystem") String filesystem)
throws Exception {
+ producerTemplate.sendBody(componentUri(filesystem,
DataLakeOperationsDefinition.deleteFileSystem), null);
+ }
+
+ @Path("/filesystem/{filesystem}")
+ @GET
+ @Produces(MediaType.APPLICATION_JSON)
+ public List<String> listFileSystem(@PathParam("filesystem") String
filesystem) throws Exception {
+ @SuppressWarnings("unchecked")
+ List<FileSystemItem> filesystems = producerTemplate.requestBody(
+ componentUri(filesystem,
DataLakeOperationsDefinition.listFileSystem),
+ null,
+ List.class);
+ return filesystems.stream()
+ .map(FileSystemItem::getName)
+ .collect(Collectors.toList());
+ }
+
+ @Path("/filesystem/{filesystem}/paths")
+ @GET
+ @Produces(MediaType.APPLICATION_JSON)
+ public List<String> listPaths(@PathParam("filesystem") String filesystem)
throws Exception {
+ @SuppressWarnings("unchecked")
+ List<PathItem> filesystems = producerTemplate.requestBody(
+ componentUri(filesystem,
DataLakeOperationsDefinition.listPaths),
+ null,
+ List.class);
+ return filesystems.stream()
+ .map(PathItem::getName)
+ .collect(Collectors.toList());
+ }
+
+ @Path("/consumer/{filesystem}/path/{filename}")
+ @GET
+ @Produces(MediaType.TEXT_PLAIN)
+ public String consumer(@PathParam("filesystem") String filesystem,
+ @PathParam("filename") String filename) throws Exception {
+ return consumerTemplate.receiveBody(
+ "azure-storage-datalake://" + azureStorageAccountName + "/" +
filesystem
+ +
"?serviceClient=#azureDatalakeServiceClient&fileName=" + filename,
+ 10000, String.class);
+ }
+
+ private String componentUri(final String filesystem, final
DataLakeOperationsDefinition operation) {
+ return
String.format("azure-storage-datalake://%s%s?serviceClient=#azureDatalakeServiceClient&operation=%s",
+ azureStorageAccountName,
+ filesystem == null ? "" : ("/" + filesystem),
+ operation.name());
+ }
+
+}
diff --git
a/extensions-jvm/azure-storage-datalake/integration-test/src/main/resources/application.properties
b/extensions-jvm/azure-storage-datalake/integration-test/src/main/resources/application.properties
new file mode 100644
index 0000000..b51fdf8
--- /dev/null
+++
b/extensions-jvm/azure-storage-datalake/integration-test/src/main/resources/application.properties
@@ -0,0 +1,26 @@
+## ---------------------------------------------------------------------------
+## Licensed to the Apache Software Foundation (ASF) under one or more
+## contributor license agreements. See the NOTICE file distributed with
+## this work for additional information regarding copyright ownership.
+## The ASF licenses this file to You under the Apache License, Version 2.0
+## (the "License"); you may not use this file except in compliance with
+## the License. You may obtain a copy of the License at
+##
+## http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+## ---------------------------------------------------------------------------
+#
+
+
+# Azurite test account credentials although Data Lake is not supported
+# by Azurite yet https://github.com/Azure/Azurite/issues/553
+azurite.storage.account.name = devstoreaccount1
+azurite.storage.account.key =
Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+
+azure.storage.account-name =
${AZURE_STORAGE_ACCOUNT_NAME:${azurite.storage.account.name}}
+azure.storage.account-key =
${AZURE_STORAGE_ACCOUNT_KEY:${azurite.storage.account.key}}
diff --git
a/extensions-jvm/azure-storage-datalake/integration-test/src/test/java/org/apache/camel/quarkus/component/azure/storage/datalake/it/AzureStorageDatalakeTest.java
b/extensions-jvm/azure-storage-datalake/integration-test/src/test/java/org/apache/camel/quarkus/component/azure/storage/datalake/it/AzureStorageDatalakeTest.java
new file mode 100644
index 0000000..2f7f4d3
--- /dev/null
+++
b/extensions-jvm/azure-storage-datalake/integration-test/src/test/java/org/apache/camel/quarkus/component/azure/storage/datalake/it/AzureStorageDatalakeTest.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.quarkus.component.azure.storage.datalake.it;
+
+import io.quarkus.test.common.QuarkusTestResource;
+import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.RestAssured;
+import org.apache.camel.quarkus.test.support.azure.AzureStorageTestResource;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.hamcrest.Matchers;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
+
+// Datalake not supported by Azurite
https://github.com/Azure/Azurite/issues/553
+@EnabledIfEnvironmentVariable(named = "AZURE_STORAGE_ACCOUNT_NAME", matches =
".+")
+@EnabledIfEnvironmentVariable(named = "AZURE_STORAGE_ACCOUNT_KEY", matches =
".+")
+@QuarkusTest
+@QuarkusTestResource(AzureStorageTestResource.class)
+class AzureStorageDatalakeTest {
+
+ @Test
+ public void crud() {
+ final String filesystem = "cqfs" + RandomStringUtils.randomNumeric(16);
+ final String filename = "file" + RandomStringUtils.randomNumeric(16);
+
+ /* The filesystem does not exist initially */
+ // TODO this causes an infinite loop of requests see
+ // https://github.com/apache/camel-quarkus/issues/2304
+ // RestAssured.get("/azure-storage-datalake/filesystems")
+ // .then()
+ // .statusCode(200)
+ // .body(Matchers.not(Matchers.hasItem(filesystem)));
+
+ try {
+ /* Create the filesystem */
+ RestAssured.given()
+ .post("/azure-storage-datalake/filesystem/" + filesystem)
+ .then()
+ .statusCode(201);
+
+ /* Now it should exist */
+ // TODO this causes an infinite loop of requests see
+ // https://github.com/apache/camel-quarkus/issues/2304
+ //
RestAssured.get("/azure-storage-datalake/filesystem/" + filesystem)
+ // .then()
+ // .statusCode(200)
+ // .body(Matchers.hasItem(filesystem));
+
+ /* No paths yet */
+ RestAssured.get("/azure-storage-datalake/filesystem/" + filesystem
+ "/paths")
+ .then()
+ .statusCode(200)
+ .body("", Matchers.hasSize(0));
+
+ String content = "Hello " + RandomStringUtils.randomNumeric(16);
+ /* Upload */
+ RestAssured.given()
+ .body(content)
+ .post("/azure-storage-datalake/filesystem/" + filesystem +
"/path/" + filename)
+ .then()
+ .statusCode(201);
+
+ /* The path occurs in the list */
+ RestAssured.get("/azure-storage-datalake/filesystem/" + filesystem
+ "/paths")
+ .then()
+ .statusCode(200)
+ .body("", Matchers.hasItem(filename));
+
+ /* Get the file */
+ RestAssured.given()
+ .get("/azure-storage-datalake/filesystem/" + filesystem +
"/path/" + filename)
+ .then()
+ .statusCode(200)
+ // TODO should be Matches.is(content)
https://github.com/apache/camel-quarkus/issues/2302
+ .body(Matchers.startsWith(content));
+
+ /* Consumer */
+ RestAssured.given()
+ .get("/azure-storage-datalake/consumer/" + filesystem +
"/path/" + filename)
+ .then()
+ .statusCode(200)
+ // TODO should be Matches.is(content)
https://github.com/apache/camel-quarkus/issues/2302
+ .body(Matchers.startsWith(content));
+
+ } finally {
+ /* Clean up */
+
+ RestAssured.given()
+ .delete("/azure-storage-datalake/filesystem/" + filesystem
+ "/path/" + filename)
+ .then()
+ .statusCode(204);
+
+ RestAssured.given()
+ .delete("/azure-storage-datalake/filesystem/" + filesystem)
+ .then()
+ .statusCode(204);
+ }
+
+ }
+
+}
diff --git a/extensions-jvm/azure-storage-datalake/pom.xml
b/extensions-jvm/azure-storage-datalake/pom.xml
new file mode 100644
index 0000000..f309ae2
--- /dev/null
+++ b/extensions-jvm/azure-storage-datalake/pom.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.camel.quarkus</groupId>
+ <artifactId>camel-quarkus-build-parent</artifactId>
+ <version>1.8.0-SNAPSHOT</version>
+ <relativePath>../../poms/build-parent/pom.xml</relativePath>
+ </parent>
+
+ <artifactId>camel-quarkus-azure-storage-datalake-parent</artifactId>
+ <name>Camel Quarkus :: Azure storage datalake service</name>
+ <packaging>pom</packaging>
+
+ <modules>
+ <module>deployment</module>
+ <module>runtime</module>
+ <module>integration-test</module>
+ </modules>
+</project>
diff --git a/extensions-jvm/azure-storage-datalake/runtime/pom.xml
b/extensions-jvm/azure-storage-datalake/runtime/pom.xml
new file mode 100644
index 0000000..f7d408e
--- /dev/null
+++ b/extensions-jvm/azure-storage-datalake/runtime/pom.xml
@@ -0,0 +1,116 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.camel.quarkus</groupId>
+ <artifactId>camel-quarkus-azure-storage-datalake-parent</artifactId>
+ <version>1.8.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>camel-quarkus-azure-storage-datalake</artifactId>
+ <name>Camel Quarkus :: Azure storage datalake service :: Runtime</name>
+ <description>Camel Azure Datalake Gen2 Component</description>
+
+ <properties>
+ <camel.quarkus.jvmSince>1.8.0</camel.quarkus.jvmSince>
+ </properties>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.camel.quarkus</groupId>
+ <artifactId>camel-quarkus-bom</artifactId>
+ <version>${project.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.camel.quarkus</groupId>
+ <artifactId>camel-quarkus-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel.quarkus</groupId>
+ <artifactId>camel-quarkus-support-azure-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-azure-storage-datalake</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>io.quarkus</groupId>
+ <artifactId>quarkus-bootstrap-maven-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <annotationProcessorPaths>
+ <path>
+ <groupId>io.quarkus</groupId>
+
<artifactId>quarkus-extension-processor</artifactId>
+ <version>${quarkus.version}</version>
+ </path>
+ </annotationProcessorPaths>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+
+ <profiles>
+ <profile>
+ <id>full</id>
+ <activation>
+ <property>
+ <name>!quickly</name>
+ </property>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.camel.quarkus</groupId>
+ <artifactId>camel-quarkus-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>update-extension-doc-page</id>
+ <goals>
+ <goal>update-extension-doc-page</goal>
+ </goals>
+ <phase>process-classes</phase>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+</project>
diff --git
a/extensions-jvm/azure-storage-datalake/runtime/src/main/resources/META-INF/quarkus-extension.yaml
b/extensions-jvm/azure-storage-datalake/runtime/src/main/resources/META-INF/quarkus-extension.yaml
new file mode 100644
index 0000000..5a38449
--- /dev/null
+++
b/extensions-jvm/azure-storage-datalake/runtime/src/main/resources/META-INF/quarkus-extension.yaml
@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is a generated file. Do not edit directly!
+# To re-generate, run the following command from the top level directory:
+#
+# mvn -N cq:update-quarkus-metadata
+#
+---
+name: "Camel Azure storage datalake service"
+description: "Camel Azure Datalake Gen2 Component"
+metadata:
+ unlisted: true
+ guide:
"https://camel.apache.org/camel-quarkus/latest/reference/extensions/azure-storage-datalake.html";
+ categories:
+ - "integration"
+ status:
+ - "preview"
diff --git a/extensions-jvm/pom.xml b/extensions-jvm/pom.xml
index 683fa04..8fdd1a5 100644
--- a/extensions-jvm/pom.xml
+++ b/extensions-jvm/pom.xml
@@ -40,6 +40,7 @@
<module>atmos</module>
<module>atomix</module>
<module>aws-xray</module>
+ <module>azure-storage-datalake</module>
<module>barcode</module>
<module>beanio</module>
<module>beanstalk</module>
diff --git
a/integration-tests-support/azure/src/main/java/org/apache/camel/quarkus/test/support/azure/AzureStorageTestResource.java
b/integration-tests-support/azure/src/main/java/org/apache/camel/quarkus/test/support/azure/AzureStorageTestResource.java
index 75b76ca..83c93d7 100644
---
a/integration-tests-support/azure/src/main/java/org/apache/camel/quarkus/test/support/azure/AzureStorageTestResource.java
+++
b/integration-tests-support/azure/src/main/java/org/apache/camel/quarkus/test/support/azure/AzureStorageTestResource.java
@@ -20,6 +20,7 @@ package org.apache.camel.quarkus.test.support.azure;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.UUID;
+import java.util.stream.Stream;
import io.quarkus.runtime.configuration.ConfigUtils;
import io.quarkus.test.common.QuarkusTestResourceLifecycleManager;
@@ -34,8 +35,41 @@ import org.testcontainers.containers.wait.strategy.Wait;
public class AzureStorageTestResource implements
QuarkusTestResourceLifecycleManager {
private static final Logger LOGGER =
LoggerFactory.getLogger(AzureStorageTestResource.class);
private static final String AZURITE_IMAGE =
"mcr.microsoft.com/azure-storage/azurite:3.10.0";
- private static final int BLOB_SERVICE_PORT = 10000;
- private static final int QUEUE_SERVICE_PORT = 10001;
+
+ public enum Service {
+ blob(10000),
+ queue(10001),
+ datalake(-1, "dfs") // Datalake not supported by Azurite
https://github.com/Azure/Azurite/issues/553
+ ;
+
+ private final int azuritePort;
+ private final String azureServiceCode;
+
+ Service(int port) {
+ this(port, null);
+ }
+
+ Service(int port, String azureServiceCode) {
+ this.azuritePort = port;
+ this.azureServiceCode = azureServiceCode;
+ }
+
+ public static Integer[] getAzuritePorts() {
+ return Stream.of(values())
+ .mapToInt(Service::getAzuritePort)
+ .filter(p -> p >= 0)
+ .mapToObj(p -> Integer.valueOf(p))
+ .toArray(Integer[]::new);
+ }
+
+ public int getAzuritePort() {
+ return azuritePort;
+ }
+
+ public String getAzureServiceCode() {
+ return azureServiceCode == null ? name() : azureServiceCode;
+ }
+ }
private GenericContainer<?> container;
@@ -58,20 +92,21 @@ public class AzureStorageTestResource implements
QuarkusTestResourceLifecycleMan
MockBackendUtils.logMockBackendUsed();
try {
container = new GenericContainer<>(AZURITE_IMAGE)
- .withExposedPorts(BLOB_SERVICE_PORT,
QUEUE_SERVICE_PORT)
+ .withExposedPorts(Service.getAzuritePorts())
.withLogConsumer(new Slf4jLogConsumer(LOGGER))
.waitingFor(Wait.forListeningPort());
container.start();
- final String blobServiceUrl = "http://"; +
container.getContainerIpAddress() + ":"
- + container.getMappedPort(BLOB_SERVICE_PORT) + "/" +
azureStorageAccountName + "/"
- + azureBlobContainername;
- final String queueServiceUrl = "http://"; +
container.getContainerIpAddress() + ":"
- + container.getMappedPort(QUEUE_SERVICE_PORT) + "/" +
azureStorageAccountName;
-
result.put("azure.blob.container.name",
azureBlobContainername);
- result.put("azure.blob.service.url", blobServiceUrl);
- result.put("azure.queue.service.url", queueServiceUrl);
+ Stream.of(Service.values())
+ .forEach(s -> {
+ result.put(
+ "azure." + s.name() + ".service.url",
+ "http://"; +
container.getContainerIpAddress() + ":"
+ + (s.azuritePort >= 0 ?
container.getMappedPort(s.azuritePort) : s.azuritePort)
+ + "/"
+ + azureStorageAccountName);
+ });
} catch (Exception e) {
throw new RuntimeException(e);
}
@@ -82,8 +117,12 @@ public class AzureStorageTestResource implements
QuarkusTestResourceLifecycleMan
}
MockBackendUtils.logRealBackendUsed();
result.put("azure.blob.container.name", azureBlobContainername);
- result.put("azure.blob.service.url", "https://"; +
realAzureStorageAccountName + ".blob.core.windows.net");
- result.put("azure.queue.service.url", "https://"; +
realAzureStorageAccountName + ".queue.core.windows.net");
+ Stream.of(Service.values())
+ .forEach(s -> {
+ result.put(
+ "azure." + s.name() + ".service.url",
+ "https://"; + realAzureStorageAccountName + "."
+ s.getAzureServiceCode() + ".core.windows.net");
+ });
}
return result;
}
diff --git a/poms/bom/pom.xml b/poms/bom/pom.xml
index 0556c3f..b3d0193 100644
--- a/poms/bom/pom.xml
+++ b/poms/bom/pom.xml
@@ -521,6 +521,11 @@
</dependency>
<dependency>
<groupId>org.apache.camel</groupId>
+ <artifactId>camel-azure-storage-datalake</artifactId>
+ <version>${camel.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
<artifactId>camel-azure-storage-queue</artifactId>
<version>${camel.version}</version>
<exclusions>
@@ -2706,6 +2711,16 @@
</dependency>
<dependency>
<groupId>org.apache.camel.quarkus</groupId>
+ <artifactId>camel-quarkus-azure-storage-datalake</artifactId>
+ <version>${camel-quarkus.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel.quarkus</groupId>
+
<artifactId>camel-quarkus-azure-storage-datalake-deployment</artifactId>
+ <version>${camel-quarkus.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.camel.quarkus</groupId>
<artifactId>camel-quarkus-azure-storage-queue</artifactId>
<version>${camel-quarkus.version}</version>
</dependency>
