This is an automated email from the ASF dual-hosted git repository. mgreber pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/kudu.git
commit c0b6da9ba4f79eccd9ad15d136118b343e9d73a4 Author: Marton Greber <[email protected]> AuthorDate: Wed Oct 8 11:07:57 2025 +0200 KUDU-3478 Add Java client examples test script This patch adds test-java-examples.sh to verify that Java client examples run correctly against a local Kudu cluster. The script: - Publishes locally built Kudu Java artifacts to Maven local repository - Builds each Maven-based example project - Starts a local Kudu cluster using shared test-cluster-common.sh - Runs the example JARs against the test cluster - Supports testing all examples or individual examples Examples tested: - java-example: Basic CRUD operations and table alteration - insert-loadgen: Random data insertion (auto-creates table) - collectl: Build verification only (requires socket input at runtime) The script is integrated into build-and-test.sh to run during CI. Changes to examples: - examples/java/java-example/pom.xml: Added Maven profile to skip kudu-binary dependency when using -DuseLocalKuduBin=true, enabling builds against locally built Kudu without downloading the large kudu-binary JAR from Maven Central. - examples/java/java-example/README.adoc: Documented two build modes: client application development (using Maven Central artifacts) vs Kudu development (using local build). Explained -DuseLocalKuduBin flag and how tests auto-discover binaries via PATH. - examples/java/insert-loadgen/src/main/java/.../InsertLoadgen.java: Added createTableIfNotExists() method to auto-create test table if it doesn't exist, making the example more self-contained and easier to run in testing scenarios. Change-Id: I7f45295d4dfa32e28f9a6369612bb5bc25a2a038 Reviewed-on: http://gerrit.cloudera.org:8080/23510 Tested-by: Marton Greber <[email protected]> Reviewed-by: Alexey Serbin <[email protected]> --- build-support/jenkins/build-and-test.sh | 12 ++ .../org/apache/kudu/examples/InsertLoadgen.java | 42 ++++- examples/java/java-example/README.adoc | 53 ++++++- examples/java/java-example/pom.xml | 30 +++- java/test-java-examples.sh | 174 +++++++++++++++++++++ 5 files changed, 296 insertions(+), 15 deletions(-) diff --git a/build-support/jenkins/build-and-test.sh b/build-support/jenkins/build-and-test.sh index 3c572ef1e..975993e98 100755 --- a/build-support/jenkins/build-and-test.sh +++ b/build-support/jenkins/build-and-test.sh @@ -602,6 +602,12 @@ if [ "$BUILD_JAVA" == "1" ]; then EXIT_STATUS=1 FAILURES="$FAILURES"$'Could not submit Java distributed test job\n' fi + + # Run the Java examples tests + if ! ./test-java-examples.sh ; then + TESTS_FAILED=1 + FAILURES="$FAILURES"$'Java examples tests failed\n' + fi else if [ "$DO_COVERAGE" == "1" ]; then # Clean previous report results @@ -623,6 +629,12 @@ if [ "$BUILD_JAVA" == "1" ]; then TESTS_FAILED=1 FAILURES="$FAILURES"$'Java Gradle build/test failed\n' fi + + # Run the Java examples tests + if ! ./test-java-examples.sh ; then + TESTS_FAILED=1 + FAILURES="$FAILURES"$'Java examples tests failed\n' + fi fi fi set +x diff --git a/examples/java/insert-loadgen/src/main/java/org/apache/kudu/examples/InsertLoadgen.java b/examples/java/insert-loadgen/src/main/java/org/apache/kudu/examples/InsertLoadgen.java index 923d5bcac..1eeba1483 100644 --- a/examples/java/insert-loadgen/src/main/java/org/apache/kudu/examples/InsertLoadgen.java +++ b/examples/java/insert-loadgen/src/main/java/org/apache/kudu/examples/InsertLoadgen.java @@ -23,10 +23,13 @@ import java.util.List; import java.util.Random; import java.util.UUID; +import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.Type; -import org.apache.kudu.client.Insert; +import org.apache.kudu.client.CreateTableOptions; +import org.apache.kudu.client.Upsert; import org.apache.kudu.client.KuduClient; +import org.apache.kudu.client.KuduException; import org.apache.kudu.client.KuduSession; import org.apache.kudu.client.KuduTable; import org.apache.kudu.client.PartialRow; @@ -95,6 +98,32 @@ public class InsertLoadgen { } } + private static void createTableIfNotExists(KuduClient client, String tableName) + throws KuduException { + if (client.tableExists(tableName)) { + System.out.println("Table '" + tableName + "' already exists"); + return; + } + + System.out.println("Table '" + tableName + "' does not exist, creating it..."); + List<ColumnSchema> columns = new ArrayList<>(3); + columns.add(new ColumnSchema.ColumnSchemaBuilder("key", Type.INT32).key(true).build()); + columns.add(new ColumnSchema.ColumnSchemaBuilder("string_val", Type.STRING) + .nullable(true).build()); + columns.add(new ColumnSchema.ColumnSchemaBuilder("int_val", Type.INT32) + .nullable(true).build()); + Schema schema = new Schema(columns); + + CreateTableOptions cto = new CreateTableOptions(); + List<String> hashKeys = new ArrayList<>(1); + hashKeys.add("key"); + cto.addHashPartitions(hashKeys, 4); + cto.setNumReplicas(1); + + client.createTable(tableName, schema, cto); + System.out.println("Table '" + tableName + "' created successfully"); + } + public static void main(String[] args) throws Exception { if (args.length != 2) { System.err.println("Usage: InsertLoadgen master_addresses table_name"); @@ -105,6 +134,7 @@ public class InsertLoadgen { String tableName = args[1]; try (KuduClient client = new KuduClient.KuduClientBuilder(masterAddrs).build()) { + createTableIfNotExists(client, tableName); KuduTable table = client.openTable(tableName); Schema schema = table.getSchema(); List<RandomDataGenerator> generators = new ArrayList<>(schema.getColumnCount()); @@ -114,16 +144,16 @@ public class InsertLoadgen { KuduSession session = client.newSession(); session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND); - for (int insertCount = 0; ; insertCount++) { - Insert insert = table.newInsert(); - PartialRow row = insert.getRow(); + for (int upsertCount = 0; ; upsertCount++) { + Upsert upsert= table.newUpsert(); + PartialRow row = upsert.getRow(); for (int i = 0; i < schema.getColumnCount(); i++) { generators.get(i).generateColumnData(row); } - session.apply(insert); + session.apply(upsert); // Check for errors. This is done periodically since inserts are batched. - if (insertCount % 1000 == 0 && session.countPendingErrors() > 0) { + if (upsertCount % 1000 == 0 && session.countPendingErrors() > 0) { throw new RuntimeException(session.getPendingErrors().getRowErrors()[0].toString()); } } diff --git a/examples/java/java-example/README.adoc b/examples/java/java-example/README.adoc index 950c4e51c..52014df5e 100644 --- a/examples/java/java-example/README.adoc +++ b/examples/java/java-example/README.adoc @@ -27,7 +27,11 @@ This is an example program that uses the synchronous Kudu Java client APIs to - Scan some rows - Delete the table -To build and run, ensure maven is installed and from the java-example directory run: +== Building for client application development + +If you are developing a Kudu client application and want to use published Kudu +artifacts from Maven Central, ensure maven is installed and from the java-example +directory run: [source,bash] ---- @@ -35,6 +39,13 @@ $ mvn package $ java -jar target/kudu-java-example-1.0-SNAPSHOT.jar ---- +This will: + +- Download `kudu-client` and all dependencies from Maven Central +- Download `kudu-binary` (mini-cluster binaries) for running unit tests +- Run unit tests using the packaged mini-cluster +- Build the executable JAR + By default, the example assumes the Kudu cluster has a single master running on localhost with the default port 7051. To specify a different set of masters for Kudu cluster, set the property `kuduMasters` to a CSV of the master addresses in @@ -44,3 +55,43 @@ the form `host:port`, as shown: ---- $ java -DkuduMasters=master-0:7051,master-1:7051,master-2:7051 -jar target/kudu-java-example-1.0-SNAPSHOT.jar ---- + +== Building for Kudu development (using locally built binaries) + +If you are developing Kudu itself and have built it from source, you should use your +locally built Kudu client and binaries. This ensures you're testing against your +local changes rather than the published versions from Maven Central. + +First, publish your local build to your local Maven repository: + +[source,bash] +---- +$ cd $KUDU_HOME/java +$ ./gradlew publishToMavenLocal +---- + +Then build the example with the `-DuseLocalKuduBin=true` property. You must also +specify the Kudu version via `-Dkudu-version` to match your local build (found in +`$KUDU_HOME/version.txt`, e.g., `1.19.0-SNAPSHOT`): + +[source,bash] +---- +$ cd $KUDU_HOME/examples/java/java-example +$ mvn package -DuseLocalKuduBin=true -Dkudu-version=$(cat "$KUDU_HOME/version.txt") +---- + +This will: + +- Use the locally published `kudu-client` from `~/.m2/repository` +- Skip downloading the `kudu-binary` artifact (large, only in releases) +- Run unit tests using your local Kudu binaries (auto-discovered via PATH or `which kudu`) +- Build the executable JAR + +If `kudu` is not in your PATH, you can explicitly specify the binary directory: + +[source,bash] +---- +$ mvn package -DuseLocalKuduBin=true \ + -Dkudu-version=1.19.0-SNAPSHOT \ + -DkuduBinDir=$KUDU_HOME/build/latest/bin +---- diff --git a/examples/java/java-example/pom.xml b/examples/java/java-example/pom.xml index 72d373c76..24df294a4 100644 --- a/examples/java/java-example/pom.xml +++ b/examples/java/java-example/pom.xml @@ -94,14 +94,6 @@ <scope>test</scope> </dependency> - <dependency> - <groupId>org.apache.kudu</groupId> - <artifactId>kudu-binary</artifactId> - <version>${kudu-version}</version> - <scope>test</scope> - <classifier>${os.detected.classifier}</classifier> - </dependency> - <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> @@ -109,4 +101,26 @@ <scope>test</scope> </dependency> </dependencies> + + <profiles> + <!-- Profile to include kudu-binary JAR for running tests with packaged mini-cluster binaries. + Activated by default. Disable with -DuseLocalKuduBin=true when using locally built Kudu. --> + <profile> + <id>kudu-binary</id> + <activation> + <property> + <name>!useLocalKuduBin</name> + </property> + </activation> + <dependencies> + <dependency> + <groupId>org.apache.kudu</groupId> + <artifactId>kudu-binary</artifactId> + <version>${kudu-version}</version> + <scope>test</scope> + <classifier>${os.detected.classifier}</classifier> + </dependency> + </dependencies> + </profile> + </profiles> </project> diff --git a/java/test-java-examples.sh b/java/test-java-examples.sh new file mode 100755 index 000000000..c1f9d2c26 --- /dev/null +++ b/java/test-java-examples.sh @@ -0,0 +1,174 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# This script verifies that the Java client examples can run correctly +# against a local Kudu cluster. +# +# Usage: +# test-java-examples.sh [example-name] +# +# If no argument is provided, tests all Java examples. +# +# To add a new example: +# 1. Add the example directory name to the VALID_EXAMPLES array below +# 2. Add a case in the test_java_example() function with the command to run it + +# List of valid example names (single source of truth) +VALID_EXAMPLES=( + "java-example" + "insert-loadgen" + "collectl" +) + +JAVA_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")"; pwd) + +# KUDU_HOME should point to the root of the Kudu repository +if [[ -z "$KUDU_HOME" ]]; then + # Try to infer it if not set + KUDU_HOME=$(cd "$JAVA_DIR/.."; pwd) + echo "KUDU_HOME not set, inferring from script location: $KUDU_HOME" +fi + +if [[ ! -d "$KUDU_HOME" ]]; then + exit_error "KUDU_HOME directory does not exist: $KUDU_HOME" +fi + +EXAMPLES_DIR="$KUDU_HOME/examples/java" + +# Source the common cluster management functions +source "$KUDU_HOME/build-support/test-cluster-common.sh" + +BUILD_DIR="$KUDU_HOME/build/latest" +BIN_DIR="$BUILD_DIR/bin" + +# Clean up after the test. Must be idempotent. +cleanup() { + cleanup_cluster +} +trap cleanup EXIT + +set -e +set -o pipefail +set -x + +if ! command -v mvn &> /dev/null; then + exit_error "Maven (mvn) not found. Please install Maven to run Java examples tests." +fi + +KUDU_VERSION=$(cat "$KUDU_HOME/version.txt") +echo "Using Kudu version: $KUDU_VERSION" + +echo "Publishing Kudu Java artifacts to local Maven repository..." +pushd "$JAVA_DIR" +./gradlew publishToMavenLocal +popd + +test_java_example() { + local example_name=$1 + local example_dir="$EXAMPLES_DIR/$example_name" + + echo "Testing Java example: $example_name" + + if [[ ! -d "$example_dir" ]]; then + echo "WARNING: Example directory not found: $example_dir, skipping" + return 0 + fi + + pushd "$example_dir" + + echo "Building $example_name with Maven..." + # Build the example using locally built Kudu + if ! mvn clean package \ + -Dkudu-version=$KUDU_VERSION \ + -DkuduBinDir=$KUDU_HOME/build/latest/bin \ + -DuseLocalKuduBin=true ; then + exit_error "Failed to build $example_name" + fi + + echo "Running $example_name..." + case "$example_name" in + java-example) + if ! java -DkuduMasters=$LOCALHOST_IP:$MASTER_RPC_PORT \ + -jar target/kudu-java-example-*.jar ; then + exit_error "$example_name failed" + fi + ;; + insert-loadgen) + # Run insert-loadgen for a limited time + local test_table="loadgen_test_table" + + # Run in background and kill after 5 seconds + java -DkuduMasters=$LOCALHOST_IP:$MASTER_RPC_PORT \ + -jar target/kudu-insert-loadgen-*.jar \ + $LOCALHOST_IP:$MASTER_RPC_PORT "$test_table" & + local loadgen_pid=$! + sleep 5 + kill $loadgen_pid 2>/dev/null || true + wait $loadgen_pid 2>/dev/null || true + + local row_count=$("$BIN_DIR/kudu" table scan $LOCALHOST_IP:$MASTER_RPC_PORT "$test_table" 2>/dev/null | wc -l) + if [[ $row_count -lt 1 ]]; then + exit_error "insert-loadgen did not insert enough data (row_count=$row_count)" + fi + echo "insert-loadgen inserted data successfully (rows: $row_count)" + ;; + collectl) + # Collectl requires external socket input, so just verify it builds + echo "collectl requires external input, verifying build only" + ;; + *) + echo "Unknown example: $example_name" + ;; + esac + + echo "$example_name completed successfully!" + popd +} + +# Parse command-line arguments +EXAMPLE_NAME="$1" + +# Validate example name (optional) +if [[ -n "$EXAMPLE_NAME" ]]; then + local valid=false + for example in "${VALID_EXAMPLES[@]}"; do + if [[ "$EXAMPLE_NAME" == "$example" ]]; then + valid=true + break + fi + done + if [[ "$valid" != "true" ]]; then + exit_error "Invalid example name: $EXAMPLE_NAME. Must be one of: ${VALID_EXAMPLES[*]}" + fi +fi + +start_test_cluster "$BIN_DIR" "java_examples-test" + +if [[ -z "$EXAMPLE_NAME" ]]; then + echo "No example specified, testing all Java examples" + for example in "${VALID_EXAMPLES[@]}"; do + test_java_example "$example" + done +else + test_java_example "$EXAMPLE_NAME" +fi + +echo "All Java examples tests completed successfully!" +
