This is an automated email from the ASF dual-hosted git repository. jamesnetherton pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/camel-quarkus-examples.git
commit cc7baac170583761821a26b4edb929b180088401 Author: shravani <[email protected]> AuthorDate: Thu Nov 23 15:20:11 2023 +0530 Add multiple datasources to Jdbc example (#180) * Add multiple datasources to Jdbc example * excluded .gitignore and added license * Changed README.adoc * Updated examples.json * skipping test containers-tests --- docs/modules/ROOT/attachments/examples.json | 10 +- jdbc-datasource/README.adoc | 115 +++++++++++++++------ jdbc-datasource/pom.xml | 109 +++++++++++-------- .../src/main/java/org/acme/jdbc/JdbcRoutes.java | 36 +++++-- .../jdbc/{JdbcResource.java => JdbcService.java} | 36 ++++--- .../java/org/acme/jdbc/JdbcTestHarness.java} | 11 +- .../src/main/resources/application.properties | 28 +++-- .../jdbc/{JdbcDataSourceIT.java => JdbcIT.java} | 3 +- .../{JdbcDataSourceTest.java => JdbcTest.java} | 31 +++--- .../jdbc/PostgresSourceDatabaseTestResource.java | 78 ++++++++++++++ .../jdbc/PostgresTargetDatabaseTestResource.java | 78 ++++++++++++++ .../src/test/resources/init-source-db.sql | 23 +++++ .../src/test/resources/init-target-db.sql | 18 ++++ 13 files changed, 431 insertions(+), 145 deletions(-) diff --git a/docs/modules/ROOT/attachments/examples.json b/docs/modules/ROOT/attachments/examples.json index 89dbc38..d2e7c04 100644 --- a/docs/modules/ROOT/attachments/examples.json +++ b/docs/modules/ROOT/attachments/examples.json @@ -4,11 +4,6 @@ "description": "Shows how to use Camel CXF SOAP component.", "link": "https://github.com/apache/camel-quarkus-examples/tree/main/cxf-soap"; }, - { - "title": "Connecting to a JDBC DataSource", - "description": "Shows how to connect to a Database using Datastores.", - "link": "https://github.com/apache/camel-quarkus-examples/tree/main/jdbc-datasource"; - }, { "title": "Custom `main()`", "description": "Shows how to start Camel from a custom `main()` method", @@ -19,6 +14,11 @@ "description": "Shows how to deploy a Camel Quarkus route as an AWS Lambda function", "link": "https://github.com/apache/camel-quarkus-examples/tree/main/aws-lambda"; }, + { + "title": "Extract, Transform and Load between two databases", + "description": "Shows how to extract, transform and load between two databases", + "link": "https://github.com/apache/camel-quarkus-examples/tree/main/jdbc-datasource"; + }, { "title": "File consumer with Bindy \u0026 FTP", "description": "Shows how to consume CSV files, marshal \u0026 unmarshal the data and send it onwards via FTP", diff --git a/jdbc-datasource/README.adoc b/jdbc-datasource/README.adoc index d44c5a0..b32eb02 100644 --- a/jdbc-datasource/README.adoc +++ b/jdbc-datasource/README.adoc @@ -1,69 +1,118 @@ -= Connecting to a JDBC DataSource: A Camel Quarkus example -:cq-example-description: An example that shows how to connect to a Database using Datastores. += Extract, Transform and Load between two databases: A Camel Quarkus example +:cq-example-description: An example that shows how to extract, transform and load between two databases {cq-description} -In particular, it demonstrates the following: +TIP: Check the https://camel.apache.org/camel-quarkus/latest/first-steps.html[Camel Quarkus User guide] for prerequisites +and other general information. -1. Defining a DataSource -2. Querying the Database defined in the previous `DataSource` -3. Usage of properties defined in `application.properties` -4. No Java code required or used, the route defined in XML can still be compiled to native code. +== Start the source and target databases -This example will connect to an H2 database with the connection details defined in `application.properties`. -If the example is run on Development mode and no database exists, Quarkus will create a matching database -https://quarkus.io/guides/datasource#dev-services[as described here]. +All the commands in this example are expected to be run from the example directory, at the same level than the `pom.xml` file. -TIP: Check the https://camel.apache.org/camel-quarkus/latest/first-steps.html[Camel Quarkus User guide] for prerequisites -and other general information. +In a first terminal, let's start the source database by executing the command below: -== Start in the Development mode +[source,shell] +---- +docker run -p 5432:5432 \ +-e POSTGRES_USER=ETL_source_user \ +-e POSTGRES_PASSWORD=1234567@8_source \ +-e POSTGRES_DB=source_db \ +-v ${PWD}/src/test/resources/init-source-db.sql:/docker-entrypoint-initdb.d/init-source-db.sql \ +docker.io/postgres:15.0 +---- + +In a second terminal, let's start the target database: [source,shell] ---- -$ mvn clean compile quarkus:dev +docker run -p 5433:5432 \ +-e POSTGRES_USER=ETL_target_user \ +-e POSTGRES_PASSWORD=1234567@8_target \ +-e POSTGRES_DB=target_db \ +-v ${PWD}/src/test/resources/init-target-db.sql:/docker-entrypoint-initdb.d/init-target-db.sql \ +docker.io/postgres:15.0 ---- -The above command compiles the project, starts the application and lets the Quarkus tooling watch for changes in your -workspace. Any modifications in your project will automatically take effect in the running application. +== Running the application in dev mode + +You can run your application in dev mode that enables live coding using: + +[source,shell] +---- +mvn compile quarkus:dev +---- + +[NOTE] +==== +Quarkus now ships with a Dev UI, which is available in dev mode only at http://localhost:8080/q/dev/. +==== TIP: Please refer to the Development mode section of https://camel.apache.org/camel-quarkus/latest/first-steps.html#_development_mode[Camel Quarkus User guide] for more details. -=== Package and run the application +Extract, Transform and Load related logs should be output as below: + +[source,shell] +---- +2023-11-14 15:12:55,878 INFO [route17] (Camel (camel-9) thread #9 - timer://insertCamel) Extracting data from source database +2023-11-14 15:12:55,881 INFO [route17] (Camel (camel-9) thread #9 - timer://insertCamel) -> Transforming review for hotel 'Grand Hotel' +2023-11-14 15:12:55,886 INFO [route17] (Camel (camel-9) thread #9 - timer://insertCamel) -> Loading transformed data in target database +2023-11-14 15:12:55,893 INFO [route17] (Camel (camel-9) thread #9 - timer://insertCamel) -> Transforming review for hotel 'Middle Hotel' +2023-11-14 15:12:55,897 INFO [route17] (Camel (camel-9) thread #9 - timer://insertCamel) -> Loading transformed data in target database +2023-11-14 15:12:55,904 INFO [route17] (Camel (camel-9) thread #9 - timer://insertCamel) -> Transforming review for hotel 'Small Hotel' +2023-11-14 15:12:55,909 INFO [route17] (Camel (camel-9) thread #9 - timer://insertCamel) -> Loading transformed data in target database +---- + +=== Packaging and running the application Once you are done with developing you may want to package and run the application. TIP: Find more details about the JVM mode and Native mode in the Package and run section of https://camel.apache.org/camel-quarkus/latest/first-steps.html#_package_and_run_the_application[Camel Quarkus User guide] -==== JVM mode +The application can be packaged using: [source,shell] ---- -$ mvn clean package -$ java -jar target/quarkus-app/quarkus-run.jar -... +mvn package +---- + +It produces the `quarkus-run.jar` file in the `target/quarkus-app/` directory. +Be aware that it’s not an _über-jar_ as the dependencies are copied into the `target/quarkus-app/lib/` directory. + +The application is now runnable executing the command below: -[io.quarkus] (main) camel-quarkus-examples-... started in 0.570s. +[source,shell] +---- +java -jar target/quarkus-app/quarkus-run.jar ---- -==== Native mode +The application should output the same logs than in previous section. + +==== Creating a native executable + +Finally, the application can be compiled to native with the following command: -IMPORTANT: Native mode requires having GraalVM and other tools installed. Please check the Prerequisites section -of https://camel.apache.org/camel-quarkus/latest/first-steps.html#_prerequisites[Camel Quarkus User guide]. +[source,shell] +---- +mvn package -Dnative +---- + +Or, if you don't have GraalVM installed, you can run the native executable build in a container using: + +[source,shell] +---- +mvn package -Dnative -Dquarkus.native.container-build=true +---- -To prepare a native executable using GraalVM, run the following command: +Either way, the resulting native executable could be start as below: [source,shell] ---- -$ mvn clean package -Pnative -$ ./target/*-runner -... -[io.quarkus] (main) camel-quarkus-examples-... started in 0.011s. -... +./target/*-runner ---- -== Feedback +The application should output the same logs than in previous section. -Please report bugs and propose improvements via https://github.com/apache/camel-quarkus/issues[GitHub issues of Camel Quarkus] project. +If you want to learn more about building native executables, please consult https://quarkus.io/guides/maven-tooling. \ No newline at end of file diff --git a/jdbc-datasource/pom.xml b/jdbc-datasource/pom.xml index 3eaa040..2491243 100644 --- a/jdbc-datasource/pom.xml +++ b/jdbc-datasource/pom.xml @@ -66,50 +66,59 @@ </dependencies> </dependencyManagement> <dependencies> - <dependency> - <groupId>org.apache.camel.quarkus</groupId> - <artifactId>camel-quarkus-microprofile-health</artifactId> - </dependency> - <dependency> - <groupId>org.apache.camel.quarkus</groupId> - <artifactId>camel-quarkus-xml-io-dsl</artifactId> - </dependency> - <dependency> - <groupId>org.apache.camel.quarkus</groupId> - <artifactId>camel-quarkus-log</artifactId> - </dependency> - <dependency> - <groupId>org.apache.camel.quarkus</groupId> - <artifactId>camel-quarkus-timer</artifactId> - </dependency> - <dependency> - <groupId>org.apache.camel.quarkus</groupId> - <artifactId>camel-quarkus-jdbc</artifactId> - </dependency> - <dependency> - <groupId>io.quarkus</groupId> - <artifactId>quarkus-jdbc-h2</artifactId> - </dependency> - <dependency> - <groupId>io.quarkus</groupId> - <artifactId>quarkus-agroal</artifactId> - </dependency> - <dependency> - <groupId>io.quarkus</groupId> - <artifactId>quarkus-junit5</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>io.quarkus</groupId> - <artifactId>quarkus-test-h2</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.awaitility</groupId> - <artifactId>awaitility</artifactId> - <scope>test</scope> - </dependency> - </dependencies> + <dependency> + <groupId>org.apache.camel.quarkus</groupId> + <artifactId>camel-quarkus-bean</artifactId> + </dependency> + <dependency> + <groupId>org.apache.camel.quarkus</groupId> + <artifactId>camel-quarkus-core</artifactId> + </dependency> + <dependency> + <groupId>org.apache.camel.quarkus</groupId> + <artifactId>camel-quarkus-jdbc</artifactId> + </dependency> + <dependency> + <groupId>org.apache.camel.quarkus</groupId> + <artifactId>camel-quarkus-timer</artifactId> + </dependency> + <dependency> + <groupId>org.apache.camel.quarkus</groupId> + <artifactId>camel-quarkus-platform-http</artifactId> + </dependency> + <dependency> + <groupId>io.quarkus</groupId> + <artifactId>quarkus-agroal</artifactId> + </dependency> + <dependency> + <groupId>io.quarkus</groupId> + <artifactId>quarkus-jdbc-postgresql</artifactId> + </dependency> + <dependency> + <groupId>io.quarkus</groupId> + <artifactId>quarkus-arc</artifactId> + </dependency> + <dependency> + <groupId>io.quarkus</groupId> + <artifactId>quarkus-junit5</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.testcontainers</groupId> + <artifactId>testcontainers</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>io.rest-assured</groupId> + <artifactId>rest-assured</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.awaitility</groupId> + <artifactId>awaitility</artifactId> + <scope>test</scope> + </dependency> + </dependencies> <build> <pluginManagement> <plugins> @@ -184,6 +193,7 @@ <exclude>**/NOTICE</exclude> <exclude>**/README</exclude> <exclude>**/pom.xml.versionsBackup</exclude> + <exclude>**/quarkus.log*</exclude> </excludes> <mapping> <java>SLASHSTAR_STYLE</java> @@ -273,5 +283,16 @@ <quarkus.package.type>native</quarkus.package.type> </properties> </profile> + <profile> + <id>skip-testcontainers-tests</id> + <activation> + <property> + <name>skip-testcontainers-tests</name> + </property> + </activation> + <properties> + <skipTests>true</skipTests> + </properties> + </profile> </profiles> </project> diff --git a/jdbc-datasource/src/main/java/org/acme/jdbc/JdbcRoutes.java b/jdbc-datasource/src/main/java/org/acme/jdbc/JdbcRoutes.java index 3298db0..3afa331 100644 --- a/jdbc-datasource/src/main/java/org/acme/jdbc/JdbcRoutes.java +++ b/jdbc-datasource/src/main/java/org/acme/jdbc/JdbcRoutes.java @@ -16,19 +16,37 @@ */ package org.acme.jdbc; +import java.util.HashMap; +import java.util.Map; + import org.apache.camel.builder.RouteBuilder; public class JdbcRoutes extends RouteBuilder { @Override public void configure() throws Exception { - from("timer://insertCamel?period=1000") - .log("Inserting Camel ${messageTimestamp}") - .setBody().simple("INSERT INTO Camel (timestamp) VALUES (${messageTimestamp})") - .to("jdbc:camel-ds") - .log("Inserted Camel ${messageTimestamp}") - .setBody().simple("SELECT * FROM Camel") - .to("jdbc:camel-ds") - .log("We have ${header[CamelJdbcRowCount]} camels in the database.") - .log("Camels found: ${body}"); + // Define a mapping for the review values + HashMap<String, Integer> reviewMapping = new HashMap<>(); + reviewMapping.put("best", 1); + reviewMapping.put("good", 0); + reviewMapping.put("worst", -1); + + from("timer://insertCamel?period=1000&repeatCount={{etl.timer.repeatcount}}") + .setBody().simple("DELETE FROM Target") + .to("jdbc:target_db") + .setBody().simple("SELECT * FROM Source") + .to("jdbc:source_db") + .log("Extracting data from source database") + .split(body()) + .process(exchange -> { + Map<String, Object> sourceData = exchange.getIn().getBody(Map.class); + String review = (String) sourceData.get("review"); + int mappedReview = reviewMapping.getOrDefault(review, 0); + sourceData.put("review", mappedReview); + }) + .log("-> Transforming review for hotel '${body[hotel_name]}'") + .setBody() + .simple("INSERT INTO Target (id, hotel_name, price, review) VALUES(${body[id]}, '${body[hotel_name]}', ${body[price]}, ${body[review]})") + .to("jdbc:target_db") + .log("-> Loading transformed data in target database"); } } diff --git a/jdbc-datasource/src/main/java/org/acme/jdbc/JdbcResource.java b/jdbc-datasource/src/main/java/org/acme/jdbc/JdbcService.java similarity index 59% rename from jdbc-datasource/src/main/java/org/acme/jdbc/JdbcResource.java rename to jdbc-datasource/src/main/java/org/acme/jdbc/JdbcService.java index 5e6f9fa..9aa2cae 100644 --- a/jdbc-datasource/src/main/java/org/acme/jdbc/JdbcResource.java +++ b/jdbc-datasource/src/main/java/org/acme/jdbc/JdbcService.java @@ -16,31 +16,35 @@ */ package org.acme.jdbc; -import java.sql.Connection; -import java.sql.Statement; +import java.sql.ResultSet; +import java.sql.SQLException; import io.agroal.api.AgroalDataSource; import io.quarkus.agroal.DataSource; -import io.quarkus.runtime.StartupEvent; +import io.quarkus.runtime.annotations.RegisterForReflection; import jakarta.enterprise.context.ApplicationScoped; -import jakarta.enterprise.event.Observes; import jakarta.inject.Inject; -import org.apache.camel.CamelContext; +import jakarta.inject.Named; +@Named("reviewService") @ApplicationScoped -public class JdbcResource { +@RegisterForReflection +public class JdbcService { @Inject - @DataSource("camel-ds") - AgroalDataSource dataSource; - - void startup(@Observes StartupEvent event, CamelContext context) throws Exception { - try (Connection con = dataSource.getConnection()) { - try (Statement statement = con.createStatement()) { - con.setAutoCommit(true); - statement.execute("DROP TABLE IF EXISTS camel"); - statement.execute("CREATE TABLE camel (id SERIAL PRIMARY KEY, timestamp VARCHAR(255))"); - } + @DataSource("target_db") + AgroalDataSource targetDb; + + String getHotelReviews() throws SQLException { + + StringBuilder sb = new StringBuilder(); + + ResultSet rs = targetDb.getConnection().createStatement().executeQuery("SELECT (hotel_name, review) FROM Target"); + + while (rs.next()) { + sb.append(rs.getString(1)); } + + return sb.toString(); } } diff --git a/jdbc-datasource/src/test/java/org/acme/jdbc/JdbcDataSourceIT.java b/jdbc-datasource/src/main/java/org/acme/jdbc/JdbcTestHarness.java similarity index 75% copy from jdbc-datasource/src/test/java/org/acme/jdbc/JdbcDataSourceIT.java copy to jdbc-datasource/src/main/java/org/acme/jdbc/JdbcTestHarness.java index 41c899e..17b2256 100644 --- a/jdbc-datasource/src/test/java/org/acme/jdbc/JdbcDataSourceIT.java +++ b/jdbc-datasource/src/main/java/org/acme/jdbc/JdbcTestHarness.java @@ -16,9 +16,12 @@ */ package org.acme.jdbc; -import io.quarkus.test.junit.QuarkusIntegrationTest; - -@QuarkusIntegrationTest -class JdbcDataSourceIT extends JdbcDataSourceTest { +import org.apache.camel.builder.RouteBuilder; +public class JdbcTestHarness extends RouteBuilder { + @Override + public void configure() throws Exception { + from("platform-http:/getHotelReviews") + .bean("reviewService", "getHotelReviews"); + } } diff --git a/jdbc-datasource/src/main/resources/application.properties b/jdbc-datasource/src/main/resources/application.properties index b178e02..706c31d 100644 --- a/jdbc-datasource/src/main/resources/application.properties +++ b/jdbc-datasource/src/main/resources/application.properties @@ -20,20 +20,18 @@ quarkus.banner.enabled = false quarkus.log.file.enable = true -#Default DataSource -quarkus.datasource.camel-ds.db-kind=h2 +# Set how many time the route should be applied +etl.timer.repeatcount = 0 +%test.etl.timer.repeatCount = 1 -#If you want to have more than one DataSource, you can use an identifier as this: -#quarkus.datasource.$identifier.db-kind=h2 -#Then use it on the route by name -#.to("jdbc:$identifier") +# Source Database Configuration +quarkus.datasource.source_db.db-kind = postgresql +quarkus.datasource.source_db.jdbc.url = jdbc:postgresql://localhost:5432/source_db +quarkus.datasource.source_db.username = ETL_source_user +quarkus.datasource.source_db.password = 1234567@8_source -#Configure the following section to use a maven profile (called prod) -#configured database (using postgresql on this case) -#Remember to edit the pom.xml to add the database driver needed -#<artifactId>quarkus-jdbc-postgresql on this case</artifactId> -#%prod.quarkus.datasource.db-kind=postgresql -#%prod.quarkus.datasource.username=${POSTGRESQL_USER} -#%prod.quarkus.datasource.password=${POSTGRESQL_PASSWORD} -#%prod.quarkus.datasource.jdbc.url=${POSTGRESQL_JDBC_URL} -#%prod.quarkus.datasource.jdbc.max-size=16 \ No newline at end of file +# Target Database Configuration +quarkus.datasource.target_db.db-kind = postgresql +quarkus.datasource.target_db.jdbc.url = jdbc:postgresql://localhost:5433/target_db +quarkus.datasource.target_db.username = ETL_target_user +quarkus.datasource.target_db.password = 1234567@8_target diff --git a/jdbc-datasource/src/test/java/org/acme/jdbc/JdbcDataSourceIT.java b/jdbc-datasource/src/test/java/org/acme/jdbc/JdbcIT.java similarity index 94% rename from jdbc-datasource/src/test/java/org/acme/jdbc/JdbcDataSourceIT.java rename to jdbc-datasource/src/test/java/org/acme/jdbc/JdbcIT.java index 41c899e..c6595e1 100644 --- a/jdbc-datasource/src/test/java/org/acme/jdbc/JdbcDataSourceIT.java +++ b/jdbc-datasource/src/test/java/org/acme/jdbc/JdbcIT.java @@ -19,6 +19,5 @@ package org.acme.jdbc; import io.quarkus.test.junit.QuarkusIntegrationTest; @QuarkusIntegrationTest -class JdbcDataSourceIT extends JdbcDataSourceTest { - +public class JdbcIT extends JdbcTest { } diff --git a/jdbc-datasource/src/test/java/org/acme/jdbc/JdbcDataSourceTest.java b/jdbc-datasource/src/test/java/org/acme/jdbc/JdbcTest.java similarity index 58% rename from jdbc-datasource/src/test/java/org/acme/jdbc/JdbcDataSourceTest.java rename to jdbc-datasource/src/test/java/org/acme/jdbc/JdbcTest.java index e38dbd5..18b3de7 100644 --- a/jdbc-datasource/src/test/java/org/acme/jdbc/JdbcDataSourceTest.java +++ b/jdbc-datasource/src/test/java/org/acme/jdbc/JdbcTest.java @@ -16,32 +16,29 @@ */ package org.acme.jdbc; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; import java.util.concurrent.TimeUnit; import io.quarkus.test.common.QuarkusTestResource; -import io.quarkus.test.h2.H2DatabaseTestResource; import io.quarkus.test.junit.QuarkusTest; +import io.restassured.RestAssured; import org.junit.jupiter.api.Test; import static org.awaitility.Awaitility.await; @QuarkusTest -@QuarkusTestResource(H2DatabaseTestResource.class) -public class JdbcDataSourceTest { +@QuarkusTestResource(PostgresSourceDatabaseTestResource.class) +@QuarkusTestResource(PostgresTargetDatabaseTestResource.class) +public class JdbcTest { + @Test - public void testCamelsInDatabase() throws Exception { - // Verify that camels are being inserted in the database: - await() - .atMost(10L, TimeUnit.SECONDS) - .pollDelay(1, TimeUnit.SECONDS) - .until(() -> { - String log = new String(Files.readAllBytes( - Paths.get("target/quarkus.log")), - StandardCharsets.UTF_8); - return log.contains("We have 2 camels in the database."); - }); + public void etlBridgeShouldTransferValuesBetweenDatebases() { + await().atMost(30L, TimeUnit.SECONDS).pollDelay(500, TimeUnit.MILLISECONDS).until(() -> { + String hotelReviews = RestAssured + .get("/getHotelReviews") + .then() + .extract().asString(); + + return "(\"Grand Hotel\",1)(\"Middle Hotel\",0)(\"Small Hotel\",-1)".equals(hotelReviews); + }); } } diff --git a/jdbc-datasource/src/test/java/org/acme/jdbc/PostgresSourceDatabaseTestResource.java b/jdbc-datasource/src/test/java/org/acme/jdbc/PostgresSourceDatabaseTestResource.java new file mode 100644 index 0000000..b0f9a14 --- /dev/null +++ b/jdbc-datasource/src/test/java/org/acme/jdbc/PostgresSourceDatabaseTestResource.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.acme.jdbc; + +import java.util.Map; + +import io.quarkus.test.common.QuarkusTestResourceLifecycleManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.BindMode; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.TestcontainersConfiguration; + +import static org.apache.camel.util.CollectionHelper.mapOf; + +public class PostgresSourceDatabaseTestResource<T extends GenericContainer> implements QuarkusTestResourceLifecycleManager { + + private static final Logger LOG = LoggerFactory.getLogger(PostgresSourceDatabaseTestResource.class); + + private static final int POSTGRES_PORT = 5432; + private static final String POSTGRES_IMAGE = "docker.io/postgres:15.0"; + + private static final String POSTGRES_SOURCE_DB_NAME = "source_db"; + private static final String POSTGRES_SOURCE_PASSWORD = "1234567@8_source"; + private static final String POSTGRES_SOURCE_USER = "ETL_source_user"; + + private GenericContainer<?> sourceDbContainer; + + @Override + public Map<String, String> start() { + LOG.info(TestcontainersConfiguration.getInstance().toString()); + + sourceDbContainer = new GenericContainer<>(POSTGRES_IMAGE) + .withExposedPorts(POSTGRES_PORT) + .withEnv("POSTGRES_USER", POSTGRES_SOURCE_USER) + .withEnv("POSTGRES_PASSWORD", POSTGRES_SOURCE_PASSWORD) + .withEnv("POSTGRES_DB", POSTGRES_SOURCE_DB_NAME) + .withClasspathResourceMapping("init-source-db.sql", "/docker-entrypoint-initdb.d/init-source-db.sql", + BindMode.READ_ONLY) + .withLogConsumer(new Slf4jLogConsumer(LOG)).waitingFor(Wait.forListeningPort()); + sourceDbContainer.start(); + + // Print Postgres server connectivity information + String sourceJbdcUrl = String.format("jdbc:postgresql://%s:%s/%s", sourceDbContainer.getHost(), + sourceDbContainer.getMappedPort(POSTGRES_PORT), POSTGRES_SOURCE_DB_NAME); + LOG.info("The test source_db could be accessed through the following JDBC url: " + sourceJbdcUrl); + + return mapOf("quarkus.datasource.source_db.jdbc.url", sourceJbdcUrl); + } + + @Override + public void stop() { + try { + if (sourceDbContainer != null) { + sourceDbContainer.stop(); + } + } catch (Exception ex) { + LOG.error("An issue occured while stopping the sourceDbContainer", ex); + } + } + +} diff --git a/jdbc-datasource/src/test/java/org/acme/jdbc/PostgresTargetDatabaseTestResource.java b/jdbc-datasource/src/test/java/org/acme/jdbc/PostgresTargetDatabaseTestResource.java new file mode 100644 index 0000000..3f67ff1 --- /dev/null +++ b/jdbc-datasource/src/test/java/org/acme/jdbc/PostgresTargetDatabaseTestResource.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.acme.jdbc; + +import java.util.Map; + +import io.quarkus.test.common.QuarkusTestResourceLifecycleManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.BindMode; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.TestcontainersConfiguration; + +import static org.apache.camel.util.CollectionHelper.mapOf; + +public class PostgresTargetDatabaseTestResource<T extends GenericContainer> implements QuarkusTestResourceLifecycleManager { + + private static final Logger LOG = LoggerFactory.getLogger(PostgresTargetDatabaseTestResource.class); + + private static final int POSTGRES_PORT = 5432; + private static final String POSTGRES_IMAGE = "docker.io/postgres:15.0"; + + private static final String POSTGRES_TARGET_DB_NAME = "target_db"; + private static final String POSTGRES_TARGET_PASSWORD = "1234567@8_target"; + private static final String POSTGRES_TARGET_USER = "ETL_target_user"; + + private GenericContainer<?> targetDbContainer; + + @Override + public Map<String, String> start() { + LOG.info(TestcontainersConfiguration.getInstance().toString()); + + targetDbContainer = new GenericContainer<>(POSTGRES_IMAGE) + .withExposedPorts(POSTGRES_PORT) + .withEnv("POSTGRES_USER", POSTGRES_TARGET_USER) + .withEnv("POSTGRES_PASSWORD", POSTGRES_TARGET_PASSWORD) + .withEnv("POSTGRES_DB", POSTGRES_TARGET_DB_NAME) + .withClasspathResourceMapping("init-target-db.sql", "/docker-entrypoint-initdb.d/init-target-db.sql", + BindMode.READ_ONLY) + .withLogConsumer(new Slf4jLogConsumer(LOG)).waitingFor(Wait.forListeningPort()); + targetDbContainer.start(); + + // Print Postgres server connectivity information + String targetJbdcUrl = String.format("jdbc:postgresql://%s:%s/%s", targetDbContainer.getHost(), + targetDbContainer.getMappedPort(POSTGRES_PORT), POSTGRES_TARGET_DB_NAME); + LOG.info("The test target_db could be accessed through the following JDBC url: " + targetJbdcUrl); + + return mapOf("quarkus.datasource.target_db.jdbc.url", targetJbdcUrl); + } + + @Override + public void stop() { + try { + if (targetDbContainer != null) { + targetDbContainer.stop(); + } + } catch (Exception ex) { + LOG.error("An issue occured while stopping the targetDbContainer", ex); + } + } + +} diff --git a/jdbc-datasource/src/test/resources/init-source-db.sql b/jdbc-datasource/src/test/resources/init-source-db.sql new file mode 100644 index 0000000..0b0c7cb --- /dev/null +++ b/jdbc-datasource/src/test/resources/init-source-db.sql @@ -0,0 +1,23 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +CREATE TABLE IF NOT EXISTS Source (id SERIAL PRIMARY KEY, hotel_name VARCHAR(255), price DECIMAL, review VARCHAR(255)); + +INSERT INTO Source (id, hotel_name, price, review) VALUES +(1, 'Grand Hotel', 100, 'best'), +(2, 'Middle Hotel', 20, 'good'), +(3, 'Small Hotel', 17, 'worst'); \ No newline at end of file diff --git a/jdbc-datasource/src/test/resources/init-target-db.sql b/jdbc-datasource/src/test/resources/init-target-db.sql new file mode 100644 index 0000000..15eef38 --- /dev/null +++ b/jdbc-datasource/src/test/resources/init-target-db.sql @@ -0,0 +1,18 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +CREATE TABLE IF NOT EXISTS Target (id SERIAL PRIMARY KEY, hotel_name VARCHAR(255), price DECIMAL, review DECIMAL); \ No newline at end of file
