xiangfu0 commented on code in PR #19090:
URL: https://github.com/apache/pinot/pull/19090#discussion_r3659928099
##########
pinot-tools/src/main/java/org/apache/pinot/tools/RealtimeQuickStart.java:
##########
@@ -48,71 +67,126 @@ public static void main(String[] args)
@Override
protected Map<String, Object> getConfigOverrides() {
- Map<String, Object> configOverrides = new HashMap<>();
+ Map<String, Object> configOverrides = new
HashMap<>(super.getConfigOverrides());
configOverrides.put(CommonConstants.Server.CONFIG_OF_ENABLE_THREAD_CPU_TIME_MEASUREMENT,
true);
return configOverrides;
}
@Override
public void runSampleQueries(QuickstartRunner runner)
throws Exception {
- String q1 = "select count(*) from meetupRsvp limit 1";
- printStatus(Color.YELLOW, "Total number of documents in the table");
- printStatus(Color.CYAN, "Query : " + q1);
- printStatus(Color.YELLOW, prettyPrintResponse(runner.runQuery(q1)));
- printStatus(Color.GREEN,
"***************************************************");
-
- String q2 =
- "select group_city, sum(rsvp_count) from meetupRsvp group by
group_city order by sum(rsvp_count) desc limit 10";
- printStatus(Color.YELLOW, "Top 10 cities with the most rsvp");
- printStatus(Color.CYAN, "Query : " + q2);
- printStatus(Color.YELLOW, prettyPrintResponse(runner.runQuery(q2)));
- printStatus(Color.GREEN,
"***************************************************");
-
- String q3 = "select * from meetupRsvp order by mtime limit 10";
- printStatus(Color.YELLOW, "Show 10 most recent rsvps");
- printStatus(Color.CYAN, "Query : " + q3);
- printStatus(Color.YELLOW, prettyPrintResponse(runner.runQuery(q3)));
- printStatus(Color.GREEN,
"***************************************************");
-
- String q4 =
- "select event_name, sum(rsvp_count) from meetupRsvp group by
event_name order by sum(rsvp_count) desc limit 10";
- printStatus(Color.YELLOW, "Show top 10 rsvp'ed events");
- printStatus(Color.CYAN, "Query : " + q4);
- printStatus(Color.YELLOW, prettyPrintResponse(runner.runQuery(q4)));
- printStatus(Color.GREEN,
"***************************************************");
-
- String q5 = "select count(*) from meetupRsvp limit 1";
- printStatus(Color.YELLOW, "Total number of documents in the table");
- printStatus(Color.CYAN, "Query : " + q5);
- printStatus(Color.YELLOW, prettyPrintResponse(runner.runQuery(q5)));
- printStatus(Color.GREEN,
"***************************************************");
-
- String q6 = "select count(*) from fineFoodReviews";
- printStatus(Color.YELLOW, "Total number of documents in fineFoodReviews");
- printStatus(Color.CYAN, "Query : " + q6);
- printStatus(Color.YELLOW, prettyPrintResponse(runner.runQuery(q6)));
- printStatus(Color.GREEN,
"***************************************************");
-
- String q7 = "select count(*) from \"fineFoodReviews-federated\"";
- printStatus(Color.YELLOW, "Total number of documents in
fineFoodReviews-federated");
- printStatus(Color.CYAN, "Query : " + q7);
- printStatus(Color.YELLOW, prettyPrintResponse(runner.runQuery(q7)));
- printStatus(Color.GREEN,
"***************************************************");
-
- String q8 = "select count(*) from \"fineFoodReviews_part_0\"";
- printStatus(Color.YELLOW, "Total number of documents in
fineFoodReviews_part_0");
- printStatus(Color.CYAN, "Query : " + q8);
- printStatus(Color.YELLOW, prettyPrintResponse(runner.runQuery(q8)));
- printStatus(Color.GREEN,
"***************************************************");
-
- String q9 = "select count(*) from \"fineFoodReviews_part_1\"";
- printStatus(Color.YELLOW, "Total number of documents in
fineFoodReviews_part_1");
- printStatus(Color.CYAN, "Query : " + q9);
- printStatus(Color.YELLOW, prettyPrintResponse(runner.runQuery(q9)));
- printStatus(Color.GREEN,
"***************************************************");
-
- runVectorQueryExamples(runner);
+ runMeetupRsvpQueries(runner);
+ runJsonIndexQueries(runner);
+ runComplexTypeQueries(runner);
+ runUpsertQueries(runner);
+ runUpsertJsonIndexQueries(runner);
+ runPartialUpsertQueries(runner);
+ runFineFoodReviewsQueries(runner);
+ if (hasTables("fineFoodReviews")) {
+ runVectorQueryExamples(runner);
+ }
+ }
+
+ private void runMeetupRsvpQueries(QuickstartRunner runner)
+ throws Exception {
+ if (!hasTables("meetupRsvp")) {
+ return;
+ }
+ printStatus(Color.YELLOW, "***** Meetup RSVPs *****");
+ runAndPrintQuery(runner, "Total number of documents in the table", "select
count(*) from meetupRsvp limit 1");
+ runAndPrintQuery(runner, "Top 10 cities with the most rsvp", "select
group_city, sum(rsvp_count) from meetupRsvp "
+ + "group by group_city order by sum(rsvp_count) desc limit 10");
+ runAndPrintQuery(runner, "Show 10 most recent rsvps", "select * from
meetupRsvp order by mtime limit 10");
+ runAndPrintQuery(runner, "Show top 10 rsvp'ed events", "select event_name,
sum(rsvp_count) from meetupRsvp "
+ + "group by event_name order by sum(rsvp_count) desc limit 10");
+ }
+
+ private void runJsonIndexQueries(QuickstartRunner runner)
+ throws Exception {
+ if (!hasTables("meetupRsvpJson")) {
+ return;
+ }
+ printStatus(Color.YELLOW, "***** JSON index *****");
+ runAndPrintQuery(runner, "Events related to topic_name0",
+ "select json_extract_scalar(event_json, '$.event_name', 'STRING') from
meetupRsvpJson where json_match"
+ + "(group_json,
'\"$.group_topics[*].topic_name\"=''topic_name0''') limit 10");
+ }
+
+ private void runComplexTypeQueries(QuickstartRunner runner)
+ throws Exception {
+ if (!hasTables("meetupRsvpComplexType")) {
+ return;
+ }
+ printStatus(Color.YELLOW, "***** Complex type handling *****");
+ runAndPrintQuery(runner, "Flattened group topics",
+ "select \"group.group_topics.urlkey\",
\"group.group_topics.topic_name\", \"group.group_id\" from "
+ + "meetupRsvpComplexType limit 10");
+ }
+
+ private void runUpsertQueries(QuickstartRunner runner)
+ throws Exception {
+ if (!hasTables("upsertMeetupRsvp")) {
+ return;
+ }
+ printStatus(Color.YELLOW, "***** Upsert *****");
+ printStatus(Color.YELLOW, "***** The expected number of documents per
event_id is 1 *****");
+ runAndPrintQuery(runner, "Total number of documents per event_id in the
table",
+ "select event_id, count(*) from upsertMeetupRsvp group by event_id
limit 10");
+ }
+
+ private void runUpsertJsonIndexQueries(QuickstartRunner runner)
+ throws Exception {
+ if (!hasTables("upsertJsonMeetupRsvp")) {
+ return;
+ }
+ printStatus(Color.YELLOW, "***** Upsert with JSON index *****");
+ runAndPrintQuery(runner, "Events related to topic_name0",
+ "select json_extract_scalar(event_json, '$.event_name', 'STRING') from
upsertJsonMeetupRsvp where json_match"
+ + "(group_json,
'\"$.group_topics[*].topic_name\"=''topic_name0''') limit 10");
+ }
+
+ private void runPartialUpsertQueries(QuickstartRunner runner)
+ throws Exception {
+ if (!hasTables("upsertPartialMeetupRsvp")) {
+ return;
+ }
+ // The expected behavior for total number of documents per PK should be 1.
+ // The expected behavior for total number of rsvp_counts per PK should >=1
since it's incremented and updated.
+ // The expected behavior for nums of values in group_name fields should
equals to rsvp_counts.
+ printStatus(Color.YELLOW, "***** Partial upsert *****");
+ printStatus(Color.YELLOW, "***** The expected behavior for total number of
documents per PK should be 1 *****");
+ printStatus(Color.YELLOW,
+ "***** The expected behavior for total number of rsvp_counts per PK
should >=1 since it's incremented and "
+ + "updated. *****");
+ printStatus(Color.YELLOW,
+ "***** The expected behavior for nums of values in group_name fields
should equals to rsvp_counts. *****");
+ runAndPrintQuery(runner, "Total number of documents, total number of
rsvp_counts per event_id in the table",
+ "select event_id, count(*), sum(rsvp_count) from
upsertPartialMeetupRsvp group by event_id order by sum"
+ + "(rsvp_count) desc limit 10");
+
+ printStatus(Color.YELLOW,
+ "***** Nums of values in group_name fields should less than or equals
to rsvp_count. Duplicate records are "
+ + "not allowed. *****");
+ printStatus(Color.YELLOW,
+ "***** Nums of values in renue_name fields should equals to
rsvp_count. Duplicates are allowed. *****");
Review Comment:
Fixed both in `efb2d68` — `renue_name` → `venue_name`, and the duplicated
"per per" on the next line.
##########
pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java:
##########
@@ -110,48 +146,120 @@ true, getAuthProvider(),
QuickstartRunner.DEFAULT_CONTROLLER_PORT));
}
+ /**
+ * Three servers so that the multi-stage sample queries below actually
exercise a cross-server exchange, which is
+ * what the merged MULTI_STAGE and JOIN quickstarts used to set up.
+ */
protected int getNumQuickstartRunnerServers() {
- return 1;
+ return 3;
}
Review Comment:
Correct, and that is intended — but the wording was wrong, so I've fixed the
doc in `efb2d68`.
Every quickstart extending `Quickstart` now gets 3 servers. `MULTI_STAGE`
and `JOIN` already used 3, and since both are merged into `BATCH`, keeping the
default at 1 would have dropped all cross-server multi-stage exchange coverage
from CI — the quickstart job only runs `BATCH`, `streaming` and `hybrid`.
Subclasses that don't need distributed execution can still override;
`TPCHQuickStart` does.
This is called out in the PR description under Backward incompatibility.
##########
pinot-tools/src/test/java/org/apache/pinot/tools/TestQuickStartBase.java:
##########
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.tools;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.commons.io.FileUtils;
+import org.testng.Assert;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+
+/**
+ * Tests for the sample-query guard that the merged quickstarts rely on. No
cluster is started here; only the
+ * bootstrap bookkeeping is exercised.
+ */
+public class TestQuickStartBase {
Review Comment:
Renamed to `BaseQuickStartTest` in `efb2d68`.
##########
pinot-tools/src/test/java/org/apache/pinot/tools/TestQuickStartBase.java:
##########
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.tools;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.commons.io.FileUtils;
+import org.testng.Assert;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+
+/**
Review Comment:
Done in `efb2d68` — converted to `///` markdown javadoc across the comments
this PR adds, in both the test and main classes.
##########
pinot-tools/src/test/java/org/apache/pinot/tools/admin/command/TestQuickStartCommand.java:
##########
Review Comment:
Renamed to `QuickStartTest` in `efb2d68`.
Heads up for anyone on a case-insensitive checkout: `.gitignore` line 47 has
`quickstart*`, which matches `QuickStartTest.java` when `core.ignorecase=true`.
`git mv` stages it fine, but a plain `git add` of the new path would be
silently ignored.
##########
pinot-tools/src/test/java/org/apache/pinot/tools/admin/command/TestQuickStartCommand.java:
##########
@@ -71,53 +63,61 @@ public void testMatchStringToCommand()
Assert.assertEquals(quickStartClassFor("REALTIME-MINION"),
RealtimeQuickStartWithMinion.class);
Assert.assertEquals(quickStartClassFor("REALTIME"),
RealtimeQuickStart.class);
+ Assert.assertEquals(quickStartClassFor("STREAM"),
RealtimeQuickStart.class);
Assert.assertEquals(quickStartClassFor("HYBRID"), HybridQuickstart.class);
- Assert.assertEquals(quickStartClassFor("JOIN"), JoinQuickStart.class);
-
- Assert.assertEquals(quickStartClassFor("UPSERT"), UpsertQuickStart.class);
-
- Assert.assertEquals(quickStartClassFor("OFFLINE_JSON_INDEX"),
JsonIndexQuickStart.class);
- Assert.assertEquals(quickStartClassFor("OFFLINE-JSON-INDEX"),
JsonIndexQuickStart.class);
- Assert.assertEquals(quickStartClassFor("BATCH_JSON_INDEX"),
JsonIndexQuickStart.class);
- Assert.assertEquals(quickStartClassFor("BATCH-JSON-INDEX"),
JsonIndexQuickStart.class);
-
- Assert.assertEquals(quickStartClassFor("REALTIME_JSON_INDEX"),
RealtimeJsonIndexQuickStart.class);
- Assert.assertEquals(quickStartClassFor("REALTIME-JSON-INDEX"),
RealtimeJsonIndexQuickStart.class);
- Assert.assertEquals(quickStartClassFor("STREAM_JSON_INDEX"),
RealtimeJsonIndexQuickStart.class);
- Assert.assertEquals(quickStartClassFor("STREAM-JSON-INDEX"),
RealtimeJsonIndexQuickStart.class);
-
- Assert.assertEquals(quickStartClassFor("UPSERT_JSON_INDEX"),
UpsertJsonQuickStart.class);
- Assert.assertEquals(quickStartClassFor("UPSERT-JSON-INDEX"),
UpsertJsonQuickStart.class);
-
- Assert.assertEquals(quickStartClassFor("OFFLINE_COMPLEX_TYPE"),
- OfflineComplexTypeHandlingQuickStart.class);
- Assert.assertEquals(quickStartClassFor("OFFLINE-COMPLEX-TYPE"),
- OfflineComplexTypeHandlingQuickStart.class);
- Assert.assertEquals(quickStartClassFor("BATCH_COMPLEX_TYPE"),
- OfflineComplexTypeHandlingQuickStart.class);
- Assert.assertEquals(quickStartClassFor("BATCH-COMPLEX-TYPE"),
- OfflineComplexTypeHandlingQuickStart.class);
-
- Assert.assertEquals(quickStartClassFor("REALTIME_COMPLEX_TYPE"),
- RealtimeComplexTypeHandlingQuickStart.class);
- Assert.assertEquals(quickStartClassFor("REALTIME-COMPLEX-TYPE"),
- RealtimeComplexTypeHandlingQuickStart.class);
- Assert.assertEquals(quickStartClassFor("STREAM_COMPLEX_TYPE"),
- RealtimeComplexTypeHandlingQuickStart.class);
- Assert.assertEquals(quickStartClassFor("STREAM-COMPLEX-TYPE"),
- RealtimeComplexTypeHandlingQuickStart.class);
-
- Assert.assertEquals(quickStartClassFor("TIMESTAMP"),
- TimestampIndexQuickstart.class);
-
Assert.assertEquals(quickStartClassFor("MULTI_CLUSTER"),
MultiClusterQuickstart.class);
Assert.assertEquals(quickStartClassFor("MULTICLUSTER"),
MultiClusterQuickstart.class);
}
+ /**
Review Comment:
Done in `efb2d68`.
##########
pinot-tools/src/test/java/org/apache/pinot/tools/admin/command/TestQuickStartCommand.java:
##########
@@ -71,53 +63,61 @@ public void testMatchStringToCommand()
Assert.assertEquals(quickStartClassFor("REALTIME-MINION"),
RealtimeQuickStartWithMinion.class);
Assert.assertEquals(quickStartClassFor("REALTIME"),
RealtimeQuickStart.class);
+ Assert.assertEquals(quickStartClassFor("STREAM"),
RealtimeQuickStart.class);
Assert.assertEquals(quickStartClassFor("HYBRID"), HybridQuickstart.class);
- Assert.assertEquals(quickStartClassFor("JOIN"), JoinQuickStart.class);
-
- Assert.assertEquals(quickStartClassFor("UPSERT"), UpsertQuickStart.class);
-
- Assert.assertEquals(quickStartClassFor("OFFLINE_JSON_INDEX"),
JsonIndexQuickStart.class);
- Assert.assertEquals(quickStartClassFor("OFFLINE-JSON-INDEX"),
JsonIndexQuickStart.class);
- Assert.assertEquals(quickStartClassFor("BATCH_JSON_INDEX"),
JsonIndexQuickStart.class);
- Assert.assertEquals(quickStartClassFor("BATCH-JSON-INDEX"),
JsonIndexQuickStart.class);
-
- Assert.assertEquals(quickStartClassFor("REALTIME_JSON_INDEX"),
RealtimeJsonIndexQuickStart.class);
- Assert.assertEquals(quickStartClassFor("REALTIME-JSON-INDEX"),
RealtimeJsonIndexQuickStart.class);
- Assert.assertEquals(quickStartClassFor("STREAM_JSON_INDEX"),
RealtimeJsonIndexQuickStart.class);
- Assert.assertEquals(quickStartClassFor("STREAM-JSON-INDEX"),
RealtimeJsonIndexQuickStart.class);
-
- Assert.assertEquals(quickStartClassFor("UPSERT_JSON_INDEX"),
UpsertJsonQuickStart.class);
- Assert.assertEquals(quickStartClassFor("UPSERT-JSON-INDEX"),
UpsertJsonQuickStart.class);
-
- Assert.assertEquals(quickStartClassFor("OFFLINE_COMPLEX_TYPE"),
- OfflineComplexTypeHandlingQuickStart.class);
- Assert.assertEquals(quickStartClassFor("OFFLINE-COMPLEX-TYPE"),
- OfflineComplexTypeHandlingQuickStart.class);
- Assert.assertEquals(quickStartClassFor("BATCH_COMPLEX_TYPE"),
- OfflineComplexTypeHandlingQuickStart.class);
- Assert.assertEquals(quickStartClassFor("BATCH-COMPLEX-TYPE"),
- OfflineComplexTypeHandlingQuickStart.class);
-
- Assert.assertEquals(quickStartClassFor("REALTIME_COMPLEX_TYPE"),
- RealtimeComplexTypeHandlingQuickStart.class);
- Assert.assertEquals(quickStartClassFor("REALTIME-COMPLEX-TYPE"),
- RealtimeComplexTypeHandlingQuickStart.class);
- Assert.assertEquals(quickStartClassFor("STREAM_COMPLEX_TYPE"),
- RealtimeComplexTypeHandlingQuickStart.class);
- Assert.assertEquals(quickStartClassFor("STREAM-COMPLEX-TYPE"),
- RealtimeComplexTypeHandlingQuickStart.class);
-
- Assert.assertEquals(quickStartClassFor("TIMESTAMP"),
- TimestampIndexQuickstart.class);
-
Assert.assertEquals(quickStartClassFor("MULTI_CLUSTER"),
MultiClusterQuickstart.class);
Assert.assertEquals(quickStartClassFor("MULTICLUSTER"),
MultiClusterQuickstart.class);
}
+ /**
+ * The batch-side quickstarts that only differed by their sample queries
were merged into {@link Quickstart}; their
+ * types must keep resolving so that existing docs and scripts do not break.
+ */
+ @Test
+ public void testDeprecatedBatchTypesResolveToQuickstart()
+ throws InvocationTargetException, NoSuchMethodException,
InstantiationException, IllegalAccessException {
+ for (String type : new String[]{
+ "MULTI_STAGE", "JOIN", "TIMESTAMP", "OFFLINE_JSON_INDEX",
"OFFLINE-JSON-INDEX", "BATCH_JSON_INDEX",
+ "BATCH-JSON-INDEX", "OFFLINE_COMPLEX_TYPE", "OFFLINE-COMPLEX-TYPE",
"BATCH_COMPLEX_TYPE", "BATCH-COMPLEX-TYPE"
+ }) {
+ Assert.assertEquals(quickStartClassFor(type), Quickstart.class, type);
Review Comment:
Done in `efb2d68` — both test classes now static-import the TestNG
assertions.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
