This is an automated email from the ASF dual-hosted git repository.
gortiz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new add121e6a2 Add javadoc and improve usability of FluentQueryTest
(#13817)
add121e6a2 is described below
commit add121e6a2ec35370ead5c6dd94d32fa8d15e7ba
Author: Gonzalo Ortiz Jaureguizar <[email protected]>
AuthorDate: Mon Sep 2 15:55:19 2024 +0200
Add javadoc and improve usability of FluentQueryTest (#13817)
This adds:
* FluentQueryTest.open(), which returns a FluentQueryTest.Closeable that
can be used in try-with-resources manner.
* FluentQueryTest.test(Consumer<FluentQueryTest>) that can be used with a
consumer (usually a lambda).
---
.../org/apache/pinot/queries/FluentQueryTest.java | 241 +++++++++++++++++++--
.../AbstractAggregationQueryBenchmark.java | 4 +-
2 files changed, 227 insertions(+), 18 deletions(-)
diff --git
a/pinot-core/src/test/java/org/apache/pinot/queries/FluentQueryTest.java
b/pinot-core/src/test/java/org/apache/pinot/queries/FluentQueryTest.java
index f3982e65e5..76fcc97125 100644
--- a/pinot-core/src/test/java/org/apache/pinot/queries/FluentQueryTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/queries/FluentQueryTest.java
@@ -19,6 +19,7 @@
package org.apache.pinot.queries;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import java.io.File;
import java.io.FileWriter;
@@ -32,6 +33,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
+import java.util.function.Consumer;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.commons.csv.CSVFormat;
@@ -56,36 +58,125 @@ import org.intellij.lang.annotations.Language;
import org.testng.Assert;
+/**
+ * A fluent API for testing single-stage queries.
+ *
+ * Use {@link #withBaseDir(File)} to start a new test.
+ *
+ * This test framework is intended to be used in a way to write semantically
rich tests that are easy to read.
+ * They are more useful when small amount of data is needed to test a specific
query.
+ *
+ * By default, this framework creates a single broker and a single server.
+ * This should be enough for most tests.
+ * But some tests may need to create more than one server.
+ * The framework is able to create up to two servers, internally called
<em>instances</em>.
+ * The DSL force the user to create the first instance before executing a
query, but the second instance can be created
+ * by calling {@link OnFirstInstance#andOnSecondInstance(Object[]...)}.
+ *
+ * @see
org.apache.pinot.core.query.aggregation.function.CountAggregationFunctionTest
+ */
public class FluentQueryTest {
private final FluentBaseQueriesTest _baseQueriesTest;
- private final File _baseDir;
+ final File _baseDir;
private final Map<String, String> _extraQueryOptions = new HashMap<>();
- private FluentQueryTest(FluentBaseQueriesTest baseQueriesTest, File baseDir)
{
+ FluentQueryTest(FluentBaseQueriesTest baseQueriesTest, File baseDir) {
_baseQueriesTest = baseQueriesTest;
_baseDir = baseDir;
}
+ /**
+ * Start a new test with the given base directory.
+ *
+ * Usually the base directory will be created before every test and
destroyed after that using lifecycle testing
+ * hooks like {@link org.testng.annotations.BeforeClass} and {@link
org.testng.annotations.AfterClass}.
+ *
+ * Each test will create its own subdirectory in the base directory, so
multiple tests may use the same base
+ * directory.
+ *
+ * @param baseDir the base directory for the test. It must exist, be a
directory and be writable.
+ * @return The fluent API for testing queries, where eventually {@link
#givenTable(Schema, TableConfig)} will be
+ * called.
+ */
public static FluentQueryTest withBaseDir(File baseDir) {
+ Preconditions.checkArgument(baseDir.exists(), "Base directory must exist");
+ Preconditions.checkArgument(baseDir.isDirectory(), "Base directory must be
a directory");
+ Preconditions.checkArgument(baseDir.canWrite(), "Base directory must be
writable");
return new FluentQueryTest(new FluentBaseQueriesTest(), baseDir);
}
+ /**
+ * Creates a new test with a temporary directory.
+ *
+ * @param consumer the test to run. The received FluentQueryTest will use a
temporary directory that will be removed
+ * after the consumer is executed, even if a throwable is
thrown.
+ */
+ public static void test(Consumer<FluentQueryTest> consumer) {
+ StackWalker walker =
StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE);
+ try (Closeable test = new
Closeable(walker.getCallerClass().getSimpleName())) {
+ consumer.accept(test);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ /**
+ * Creates a new test with a temporary directory.
+ *
+ * The returned object is intended to be used in a try-with-resources manner.
+ * Its close method will remove the temporary directory.
+ */
+ public static FluentQueryTest.Closeable open() {
+ StackWalker walker =
StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE);
+ try {
+ return new
FluentQueryTest.Closeable(walker.getCallerClass().getSimpleName());
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ /**
+ * Sets the given extra query options to the queries that will be executed
on this test.
+ *
+ * Older properties (including null handling) will be removed.
+ */
public FluentQueryTest withExtraQueryOptions(Map<String, String>
extraQueryOptions) {
_extraQueryOptions.clear();
_extraQueryOptions.putAll(extraQueryOptions);
return this;
}
+ /**
+ * Sets the null handling to the queries that will be executed on this test.
+ */
public FluentQueryTest withNullHandling(boolean enabled) {
_extraQueryOptions.put("enableNullHandling", Boolean.toString(enabled));
return this;
}
+ /**
+ * Declares a table with the given schema and table configuration.
+ *
+ * @return a {@link DeclaringTable} object to declare the segments of the
table.
+ */
public DeclaringTable givenTable(Schema schema, TableConfig tableConfig) {
return new DeclaringTable(_baseQueriesTest, tableConfig, schema, _baseDir,
_extraQueryOptions);
}
+ public static class Closeable extends FluentQueryTest implements
AutoCloseable {
+
+ private Closeable(String testBaseName)
+ throws IOException {
+ super(new FluentBaseQueriesTest(),
Files.createTempDirectory(testBaseName).toFile());
+ }
+
+ @Override
+ public void close() {
+ _baseDir.delete();
+ }
+ }
+
public static class DeclaringTable {
private final FluentBaseQueriesTest _baseQueriesTest;
private final TableConfig _tableConfig;
@@ -100,17 +191,33 @@ public class FluentQueryTest {
_schema = schema;
_baseDir = baseDir;
_extraQueryOptions = extraQueryOptions;
+ Preconditions.checkArgument(_schema.getSchemaName() != null, "Schema
must have a name");
}
- public OnFirstInstance getFirstInstance() {
+ /**
+ * Moves the fluent DSL to the first instance (aka server).
+ * @return
+ */
+ public OnFirstInstance onFirstInstance() {
return new OnFirstInstance(_tableConfig, _schema, _baseDir, false,
_baseQueriesTest, _extraQueryOptions);
}
+ /**
+ * Creates one segment on the first instance (aka server) with the given
content.
+ *
+ * @param content the content of the segment.
+ * @see OnFirstInstance#andSegment(String...) to learn more about the
content syntax
+ */
public OnFirstInstance onFirstInstance(String... content) {
return new OnFirstInstance(_tableConfig, _schema, _baseDir, false,
_baseQueriesTest, _extraQueryOptions)
.andSegment(content);
}
+ /**
+ * Creates one segment on the first instance (aka server) with the given
content.
+ * @param content the content of the segment. Each element of the array is
a row. Each row is an array of objects
+ * that should be compatible with the table definition.
+ */
public OnFirstInstance onFirstInstance(Object[]... content) {
return new OnFirstInstance(_tableConfig, _schema, _baseDir, false,
_baseQueriesTest, _extraQueryOptions)
.andSegment(content);
@@ -197,15 +304,33 @@ public class FluentQueryTest {
_segmentContents.clear();
}
+ /**
+ * Executes the given query and returns an object that can be used to
assert the results.
+ */
public QueryExecuted whenQuery(@Language("sql") String query) {
processSegments();
BrokerResponseNative brokerResponse =
_baseQueriesTest.getBrokerResponse(query, _extraQueryOptions);
return new QueryExecuted(_baseQueriesTest, brokerResponse,
_extraQueryOptions);
}
+ /**
+ * Creates another table.
+ *
+ * The older tables can still be used.
+ */
public DeclaringTable givenTable(Schema schema, TableConfig tableConfig) {
+ processSegments();
return new DeclaringTable(_baseQueriesTest, tableConfig, schema,
_indexDir.getParentFile(), _extraQueryOptions);
}
+
+ public TableWithSegments prepareToQuery() {
+ processSegments();
+ return this;
+ }
+
+ public void tearDown() {
+ _baseQueriesTest.shutdownExecutor();
+ }
}
public static class OnFirstInstance extends TableWithSegments {
@@ -214,23 +339,59 @@ public class FluentQueryTest {
super(tableConfig, schema, baseDir, onSecondInstance, baseQueriesTest,
extraQueryOptions);
}
+ /**
+ * Adds a new segment to the table in this instance.
+ * @param content the content of the segment. Each element of the array is
a row. Each row is an array of objects
+ * that should be compatible with the table definition.
+ */
public OnFirstInstance andSegment(Object[]... content) {
_segmentContents.add(new FakeSegmentContent(content));
return this;
}
+ /**
+ * Adds a new segment to the table in this instance.
+ *
+ * The content is a table in text format. The first row is the header, and
the rest of the rows are the data.
+ * Each column must be separated by pipes ({@code |}).
+ * The header must be the name of the column (as declared in the schema).
+ * The order of the columns doesn't have to match the order of the columns
in the schema.
+ *
+ * After the header, each row must have the same number of columns as the
header and will contain the data.
+ * Each entry in the row must be a valid value for the column type.
+ * The rules to parse these values are:
+ * <ol>
+ * <li>First, the value will be trimmed</li>
+ * <li>{@code null} will always be treated as null</li>
+ * <li>{@code "null"} will be parsed as
+ * {@link PinotDataType#convert(Object, PinotDataType)
PinotDataType.convert("null", type)}</li>
+ * <li>Any other value will be parsed as
+ * {@link PinotDataType#convert(Object, PinotDataType)
PinotDataType.convert(value, type)}</li>
+ * </ol>
+ *
+ * @param tableText the content of the segment, as explained above.
+ */
public OnFirstInstance andSegment(String... tableText) {
super.andSegment(tableText);
return this;
}
- public OnSecondInstance getSecondInstance() {
+ /**
+ * Moves the fluent DSL to the second instance (aka server).
+ */
+ public OnSecondInstance andOnSecondInstance() {
processSegments();
return new OnSecondInstance(
_tableConfig, _schema, _indexDir.getParentFile(),
!_onSecondInstance, _baseQueriesTest, _extraQueryOptions
);
}
+ /**
+ * Moves the fluent DSL to the second instance (aka server), adding the
content as the first segment.
+ *
+ * @param content the content of the segment. Each element of the array is
a row. Each row is an array of objects
+ * that should be compatible with the table definition.
+ */
public OnSecondInstance andOnSecondInstance(Object[]... content) {
processSegments();
return new OnSecondInstance(
@@ -238,6 +399,12 @@ public class FluentQueryTest {
.andSegment(content);
}
+ /**
+ * Creates one segment on a second instance (aka server).
+ *
+ * @param content the content of the segment.
+ * @see OnFirstInstance#andSegment(String...) to learn more about the
content syntax
+ */
public OnSecondInstance andOnSecondInstance(String... content) {
processSegments();
return new OnSecondInstance(
@@ -246,13 +413,9 @@ public class FluentQueryTest {
}
public OnFirstInstance prepareToQuery() {
- processSegments();
+ super.prepareToQuery();
return this;
}
-
- public void tearDown() {
- _baseQueriesTest.shutdownExecutor();
- }
}
public static class OnSecondInstance extends TableWithSegments {
@@ -261,24 +424,30 @@ public class FluentQueryTest {
super(tableConfig, schema, baseDir, onSecondInstance, baseQueriesTest,
extraQueryOptions);
}
+ /**
+ * Adds a new segment to the table in this instance.
+ * @param content the content of the segment. Each element of the array is
a row. Each row is an array of objects
+ * that should be compatible with the table definition.
+ */
public OnSecondInstance andSegment(Object[]... content) {
_segmentContents.add(new FakeSegmentContent(content));
return this;
}
+ /**
+ * Adds a new segment to the table in this instance.
+ * @param tableText the content of the segment.
+ * @see OnFirstInstance#andSegment(String...) to learn more about the
+ */
public OnSecondInstance andSegment(String... tableText) {
super.andSegment(tableText);
return this;
}
public OnSecondInstance prepareToQuery() {
- processSegments();
+ super.prepareToQuery();
return this;
}
-
- public void tearDown() {
- _baseQueriesTest.shutdownExecutor();
- }
}
public static class QueryExecuted {
@@ -293,6 +462,26 @@ public class FluentQueryTest {
_extraQueryOptions = extraQueryOptions;
}
+ /**
+ * Asserts that the result of the query is the given table.
+ *
+ * The table is a text table. The first row is the header, and the rest of
the rows are the data.
+ * Each column must be separated by pipes ({@code |}).
+ * The header must be a valid column type (as defined by {@link
PinotDataType}, although it will be trimmed and
+ * uppercased).
+ *
+ * After the header, each row must have the same number of columns as the
header and will contain the data.
+ * Each entry in the row must be a valid value for the column type.
+ * The rules to parse these values are:
+ * <ol>
+ * <li>First, the value will be trimmed</li>
+ * <li>{@code null} will always be treated as null</li>
+ * <li>{@code "null"} will be parsed as
+ * {@link PinotDataType#convert(Object, PinotDataType)
PinotDataType.convert("null", type)}</li>
+ * <li>Any other value will be parsed as
+ * {@link PinotDataType#convert(Object, PinotDataType)
PinotDataType.convert(value, type)}</li>
+ * </ol>
+ */
public QueryExecuted thenResultIs(String... tableText) {
Object[][] rows = tableAsRows(
headerCells -> Arrays.stream(headerCells)
@@ -307,6 +496,9 @@ public class FluentQueryTest {
return this;
}
+ /**
+ * Asserts that the result of the query is the given table.
+ */
public QueryExecuted thenResultIs(Object[]... expectedResult) {
if (_brokerResponse.getExceptionsSize() > 0) {
Assert.fail("Query failed with " + _brokerResponse.getExceptions());
@@ -317,7 +509,8 @@ public class FluentQueryTest {
for (int i = 0; i < rowsToAnalyze; i++) {
Object[] actualRow = actualRows.get(i);
Object[] expectedRow = expectedResult[i];
- for (int j = 0; j < actualRow.length; j++) {
+ int colsToAnalyze = Math.min(actualRow.length, expectedRow.length);
+ for (int j = 0; j < colsToAnalyze; j++) {
Object actualCell = actualRow[j];
Object expectedCell = expectedRow[j];
if (actualCell != null && expectedCell != null) {
@@ -333,24 +526,40 @@ public class FluentQueryTest {
Assert.assertEquals(actualCell, expectedCell, "On row " + i + "
and column " + j);
}
}
+ Assert.assertEquals(actualRow.length, expectedRow.length, "Unexpected
number of columns on row " + i);
}
Assert.assertEquals(actualRows.size(), expectedResult.length,
"Unexpected number of rows");
return this;
}
+ /**
+ * Sets the given extra query options to the queries that will be executed
on this test.
+ *
+ * Older properties (including null handling) will be removed.
+ */
public QueryExecuted withExtraQueryOptions(Map<String, String>
extraQueryOptions) {
_extraQueryOptions.clear();
_extraQueryOptions.putAll(extraQueryOptions);
return this;
}
+ /**
+ * Sets the null handling to the queries that will be executed on this
test.
+ *
+ * <strong>Important:</strong> This change will only affect new queries.
+ */
public QueryExecuted withNullHandling(boolean enabled) {
_extraQueryOptions.put("enableNullHandling", Boolean.toString(enabled));
return this;
}
+ /**
+ * Executes the given query and returns an object that can be used to
assert the results.
+ *
+ * The tables and segments already created can still be used.
+ */
public QueryExecuted whenQuery(@Language("sql") String query) {
- BrokerResponseNative brokerResponse =
_baseQueriesTest.getBrokerResponse(query);
+ BrokerResponseNative brokerResponse =
_baseQueriesTest.getBrokerResponse(query, _extraQueryOptions);
return new QueryExecuted(_baseQueriesTest, brokerResponse,
_extraQueryOptions);
}
}
diff --git
a/pinot-perf/src/main/java/org/apache/pinot/perf/aggregation/AbstractAggregationQueryBenchmark.java
b/pinot-perf/src/main/java/org/apache/pinot/perf/aggregation/AbstractAggregationQueryBenchmark.java
index 7e8a3ded16..7a3c21c51f 100644
---
a/pinot-perf/src/main/java/org/apache/pinot/perf/aggregation/AbstractAggregationQueryBenchmark.java
+++
b/pinot-perf/src/main/java/org/apache/pinot/perf/aggregation/AbstractAggregationQueryBenchmark.java
@@ -48,14 +48,14 @@ public abstract class AbstractAggregationQueryBenchmark {
FluentQueryTest.withBaseDir(_baseDir)
.withNullHandling(nullHandlingEnabled)
.givenTable(schema, tableConfig)
- .getFirstInstance();
+ .onFirstInstance();
List<Object[][]> segmentsOnFirstServer = segmentsPerServer.get(0);
for (Object[][] segment : segmentsOnFirstServer) {
onFirstInstance.andSegment(segment);
}
- FluentQueryTest.OnSecondInstance onSecondInstance =
onFirstInstance.getSecondInstance();
+ FluentQueryTest.OnSecondInstance onSecondInstance =
onFirstInstance.andOnSecondInstance();
List<Object[][]> segmentsOnSecondServer = segmentsPerServer.get(1);
for (Object[][] segment : segmentsOnSecondServer) {
onSecondInstance.andSegment(segment);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
