This is an automated email from the ASF dual-hosted git repository.
karan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new c7d864d3bc Update container creation in AzureTestUtil.java (#13911)
c7d864d3bc is described below
commit c7d864d3bccf82601c757f44c859eaeae34cfc1d
Author: abhagraw <[email protected]>
AuthorDate: Thu Mar 16 11:04:43 2023 +0530
Update container creation in AzureTestUtil.java (#13911)
*
1. Handling deletion/creation of container created during the previously
run test in AzureTestUtil.java.
2. Adding/updating log messages and comments in Azure and GCS deep storage
tests.
---
.github/workflows/reusable-revised-its.yml | 2 +-
.github/workflows/revised-its.yml | 2 +-
.../AbstractAzureInputSourceParallelIndexTest.java | 1 +
.../indexer/ITGcsToGcsParallelIndexTest.java | 14 ++++++--------
.../testsEx/msq/ITAzureSQLBasedIngestionTest.java | 4 ++--
.../testsEx/msq/ITGcsSQLBasedIngestionTest.java | 9 +++++++++
.../apache/druid/testsEx/utils/AzureTestUtil.java | 21 ++++++++++++++++++++-
7 files changed, 40 insertions(+), 13 deletions(-)
diff --git a/.github/workflows/reusable-revised-its.yml
b/.github/workflows/reusable-revised-its.yml
index 91ff249b2c..5a220d5755 100644
--- a/.github/workflows/reusable-revised-its.yml
+++ b/.github/workflows/reusable-revised-its.yml
@@ -48,7 +48,7 @@ env:
SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
jobs:
- test: # Github job that runs a given revised/new IT against retrieved cached
druid docker image
+ test: # GitHub job that runs a given revised/new IT against retrieved cached
druid docker image
name: ${{ inputs.it }} integration test (Compile=jdk${{ inputs.build_jdk
}}, Run=jdk${{ inputs.runtime_jdk }}, Indexer=${{ inputs.use_indexer }})
runs-on: ubuntu-22.04
steps:
diff --git a/.github/workflows/revised-its.yml
b/.github/workflows/revised-its.yml
index e4cf409ee6..83cad23f42 100644
--- a/.github/workflows/revised-its.yml
+++ b/.github/workflows/revised-its.yml
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# Github workflow that runs revised/new ITs
+# GitHub workflow that runs revised/new ITs
on:
workflow_call:
workflow_dispatch:
diff --git
a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/AbstractAzureInputSourceParallelIndexTest.java
b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/AbstractAzureInputSourceParallelIndexTest.java
index 3098956a8c..81852df11c 100644
---
a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/AbstractAzureInputSourceParallelIndexTest.java
+++
b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/AbstractAzureInputSourceParallelIndexTest.java
@@ -42,6 +42,7 @@ public class AbstractAzureInputSourceParallelIndexTest
extends AbstractCloudInpu
public static void uploadDataFilesToAzure()
{
try {
+ LOG.info("Uploading files to Azure");
azure = new AzureTestUtil();
// Creating a container with name set in AZURE_CONTAINER env variable.
azure.createStorageContainer();
diff --git
a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/ITGcsToGcsParallelIndexTest.java
b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/ITGcsToGcsParallelIndexTest.java
index 0eaef4e64c..ca202797bd 100644
---
a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/ITGcsToGcsParallelIndexTest.java
+++
b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/ITGcsToGcsParallelIndexTest.java
@@ -31,15 +31,13 @@ import java.util.List;
/**
* IMPORTANT:
- * To run this test, you must:
- * 1) Set the bucket and path for your data. This can be done by setting
-Ddruid.test.config.cloudBucket and
- * -Ddruid.test.config.cloudPath or setting "cloud_bucket" and "cloud_path"
in the config file.
- * 2) Copy wikipedia_index_data1.json, wikipedia_index_data2.json, and
wikipedia_index_data3.json
- * located in integration-tests/src/test/resources/data/batch_index/json to
your GCS at the location set in step 1.
- * 3) Provide -Doverride.config.path=<PATH_TO_FILE> with gcs configs set. See
- * integration-tests/docker/environment-configs/override-examples/gcs for
env vars to provide.
- * 4) Provide -Dresource.file.dir.path=<PATH_TO_FOLDER> with folder that
contains GOOGLE_APPLICATION_CREDENTIALS file
+ * To run this test, you must set the following env variables in the build
environment -
+ * GOOGLE_PREFIX - path inside the bucket where the test data files will be
uploaded
+ * GOOGLE_BUCKET - Google cloud bucket name
+ * GOOGLE_APPLICATION_CREDENTIALS - path to the json file containing google
cloud credentials
+ * <a
href="https://druid.apache.org/docs/latest/development/extensions-core/google.html";>Google
Cloud Storage setup in druid</a>
*/
+
@RunWith(DruidTestRunner.class)
@Category(GcsDeepStorage.class)
public class ITGcsToGcsParallelIndexTest extends
AbstractGcsInputSourceParallelIndexTest
diff --git
a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITAzureSQLBasedIngestionTest.java
b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITAzureSQLBasedIngestionTest.java
index c54e891696..54f207704a 100644
---
a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITAzureSQLBasedIngestionTest.java
+++
b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITAzureSQLBasedIngestionTest.java
@@ -22,7 +22,7 @@ package org.apache.druid.testsEx.msq;
import junitparams.Parameters;
import junitparams.naming.TestCaseName;
import org.apache.druid.java.util.common.Pair;
-import org.apache.druid.testsEx.categories.S3DeepStorage;
+import org.apache.druid.testsEx.categories.AzureDeepStorage;
import org.apache.druid.testsEx.config.DruidTestRunner;
import
org.apache.druid.testsEx.indexer.AbstractAzureInputSourceParallelIndexTest;
import org.junit.Test;
@@ -41,7 +41,7 @@ import java.util.List;
*/
@RunWith(DruidTestRunner.class)
-@Category(S3DeepStorage.class)
+@Category(AzureDeepStorage.class)
public class ITAzureSQLBasedIngestionTest extends
AbstractAzureInputSourceParallelIndexTest
{
private static final String CLOUD_INGEST_SQL =
"/multi-stage-query/wikipedia_cloud_index_msq.sql";
diff --git
a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITGcsSQLBasedIngestionTest.java
b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITGcsSQLBasedIngestionTest.java
index a6031c6e1f..125b30ef81 100644
---
a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITGcsSQLBasedIngestionTest.java
+++
b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITGcsSQLBasedIngestionTest.java
@@ -31,6 +31,15 @@ import org.junit.runner.RunWith;
import java.util.List;
+/**
+ * IMPORTANT:
+ * To run this test, you must set the following env variables in the build
environment -
+ * GOOGLE_PREFIX - path inside the bucket where the test data files will be
uploaded
+ * GOOGLE_BUCKET - Google cloud bucket name
+ * GOOGLE_APPLICATION_CREDENTIALS - path to the json file containing google
cloud credentials
+ * <a
href="https://druid.apache.org/docs/latest/development/extensions-core/google.html";>Google
Cloud Storage setup in druid</a>
+ */
+
@RunWith(DruidTestRunner.class)
@Category(GcsDeepStorage.class)
public class ITGcsSQLBasedIngestionTest extends
AbstractGcsInputSourceParallelIndexTest
diff --git
a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/utils/AzureTestUtil.java
b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/utils/AzureTestUtil.java
index 4b31206a1d..90be0cbd46 100644
---
a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/utils/AzureTestUtil.java
+++
b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/utils/AzureTestUtil.java
@@ -25,6 +25,7 @@ import com.microsoft.azure.storage.blob.CloudBlobClient;
import com.microsoft.azure.storage.blob.CloudBlobContainer;
import com.microsoft.azure.storage.blob.CloudBlockBlob;
import org.apache.druid.java.util.common.logger.Logger;
+import org.apache.druid.testing.utils.ITRetryUtil;
import java.io.File;
import java.io.IOException;
@@ -83,9 +84,26 @@ public class AzureTestUtil
public void createStorageContainer() throws URISyntaxException,
StorageException
{
+ LOG.info("Creating azure container " + AZURE_CONTAINER);
CloudBlobContainer container =
azureStorageClient.getContainerReference(AZURE_CONTAINER);
// Create the container if it does not exist.
- container.createIfNotExists();
+
+ // From the azure documentation -
+ // When a container is deleted, a container with the same name can't be
created for at least 30 seconds.
+ // The container might not be available for more than 30 seconds if the
service is still processing the request.
+ // While the container is being deleted, attempts to create a container of
the same name fail with status
+ // code 409 (Conflict). The service indicates that the container is being
deleted.
+ // All other operations, including operations on any blobs under the
container,
+ // fail with status code 404 (Not Found) while the container is being
deleted.
+ ITRetryUtil.retryUntil(
+ () -> container.createIfNotExists(),
+ true,
+ 10000,
+ 13,
+ "Create Azure container : " + AZURE_CONTAINER + " "
+ );
+
+ LOG.info("Azure container " + AZURE_CONTAINER + " created");
}
public void deleteStorageContainer() throws URISyntaxException,
StorageException
@@ -109,6 +127,7 @@ public class AzureTestUtil
// Create or overwrite the "myimage.jpg" blob with contents from a local
file.
File source = new File(filePath);
CloudBlockBlob blob = container.getBlockBlobReference(DRUID_CLOUD_PATH +
'/' + source.getName());
+ LOG.info("Uploading file " + DRUID_CLOUD_PATH + '/' + source.getName() + "
in azure container " + AZURE_CONTAINER);
blob.upload(Files.newInputStream(source.toPath()), source.length());
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
