This is an automated email from the ASF dual-hosted git repository. xiangfu pushed a commit to branch release-0.3.0 in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 23b5c4fe200fb1d44812184f84881884642341d5 Author: Xiang Fu <[email protected]> AuthorDate: Fri Mar 6 20:26:19 2020 -0800 Always copy test data out from jar file (#5119) * Always copy test data out from jar file * Address comments --- .travis_quickstart.sh | 49 ++++++++++++- .../org/apache/pinot/tools/HybridQuickstart.java | 3 + .../tools/admin/command/QuickstartRunner.java | 10 ++- .../org/apache/pinot/tools/utils/JarUtils.java | 83 ++++++++++++++++++++++ 4 files changed, 142 insertions(+), 3 deletions(-) diff --git a/.travis_quickstart.sh b/.travis_quickstart.sh index 2158b61..fcfb1b0 100755 --- a/.travis_quickstart.sh +++ b/.travis_quickstart.sh @@ -74,7 +74,52 @@ if [ "${COUNT_STAR_RES_3}" -le "${COUNT_STAR_RES_2}" ] || [ "${COUNT_STAR_RES_2} echo 'Hybrid Quickstart: Not getting incremental counts for 3 consecutive count star queries with 15 seconds interval.' exit 1 fi -# kill $PID -# sleep 30 +kill $PID +sleep 30 + +cd bin +# Test quick-start-batch +./quick-start-batch.sh & +PID=$! +sleep 60 +COUNT_STAR_RES=`curl -X POST --header 'Accept: application/json' -d '{"sql":"select count(*) from baseballStats limit 1","trace":false}' http://localhost:8000/query/sql | jq '.resultTable.rows[0][0]'` +if [ "${COUNT_STAR_RES}" -ne 97889 ]; then + echo 'Batch Quickstart: Incorrect result for count star query.' + exit 1 +fi +kill $PID +sleep 30 + +# Test quick-start-streaming +./quick-start-streaming.sh & +PID=$! +sleep 60 +COUNT_STAR_RES_1=`curl -X POST --header 'Accept: application/json' -d '{"sql":"select count(*) from meetupRsvp limit 1","trace":false}' http://localhost:8000/query/sql | jq '.resultTable.rows[0][0]'` +sleep 15 +COUNT_STAR_RES_2=`curl -X POST --header 'Accept: application/json' -d '{"sql":"select count(*) from meetupRsvp limit 1","trace":false}' http://localhost:8000/query/sql | jq '.resultTable.rows[0][0]'` +sleep 15 +COUNT_STAR_RES_3=`curl -X POST --header 'Accept: application/json' -d '{"sql":"select count(*) from meetupRsvp limit 1","trace":false}' http://localhost:8000/query/sql | jq '.resultTable.rows[0][0]'` +if [ "${COUNT_STAR_RES_3}" -le "${COUNT_STAR_RES_2}" ] || [ "${COUNT_STAR_RES_2}" -le "${COUNT_STAR_RES_1}" ]; then + echo 'Streaming Quickstart: Not getting incremental counts for 3 consecutive count star queries with 15 seconds interval.' + exit 1 +fi +kill $PID +sleep 30 + +# Test quick-start-hybrid +./quick-start-hybrid.sh & +PID=$! +sleep 60 +COUNT_STAR_RES_1=`curl -X POST --header 'Accept: application/json' -d '{"sql":"select count(*) from airlineStats limit 1","trace":false}' http://localhost:8000/query/sql | jq '.resultTable.rows[0][0]'` +sleep 15 +COUNT_STAR_RES_2=`curl -X POST --header 'Accept: application/json' -d '{"sql":"select count(*) from airlineStats limit 1","trace":false}' http://localhost:8000/query/sql | jq '.resultTable.rows[0][0]'` +sleep 15 +COUNT_STAR_RES_3=`curl -X POST --header 'Accept: application/json' -d '{"sql":"select count(*) from airlineStats limit 1","trace":false}' http://localhost:8000/query/sql | jq '.resultTable.rows[0][0]'` +if [ "${COUNT_STAR_RES_3}" -le "${COUNT_STAR_RES_2}" ] || [ "${COUNT_STAR_RES_2}" -le "${COUNT_STAR_RES_1}" ]; then + echo 'Hybrid Quickstart: Not getting incremental counts for 3 consecutive count star queries with 15 seconds interval.' + exit 1 +fi +kill $PID +sleep 30 exit 0 diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java b/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java index 7951a3f..4ab2aeb 100644 --- a/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java +++ b/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java @@ -157,6 +157,9 @@ public class HybridQuickstart { printStatus(Color.YELLOW, "***** 7. Built and pushed an offline segment *****"); printStatus(Color.YELLOW, "***** 8. Started publishing a Kafka stream for the realtime instance to start consuming *****"); + printStatus(Color.YELLOW, + "***** 9. Sleep 5 Seconds to wait for all components brought up *****"); + Thread.sleep(5000); String q1 = "select count(*) from airlineStats limit 10"; printStatus(Color.YELLOW, "Total number of documents in the table"); diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java index bdbbaa6..0858db3 100644 --- a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java +++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java @@ -35,6 +35,7 @@ import org.apache.pinot.spi.ingestion.batch.IngestionJobLauncher; import org.apache.pinot.spi.ingestion.batch.spec.SegmentGenerationJobSpec; import org.apache.pinot.spi.utils.JsonUtils; import org.apache.pinot.tools.QuickstartTableRequest; +import org.apache.pinot.tools.utils.JarUtils; import org.yaml.snakeyaml.Yaml; @@ -178,7 +179,14 @@ public class QuickstartRunner { String inputDirURI = spec.getInputDirURI(); if (!new File(inputDirURI).exists()) { URL resolvedInputDirURI = QuickstartRunner.class.getClassLoader().getResource(inputDirURI); - spec.setInputDirURI(resolvedInputDirURI.toURI().toString()); + if (resolvedInputDirURI.getProtocol().equals("jar")) { + String[] splits = resolvedInputDirURI.getFile().split("!"); + String inputDir = new File(_tempDir, "inputData").toString(); + JarUtils.copyResourcesToDirectory(splits[0], splits[1].substring(1), inputDir); + spec.setInputDirURI(inputDir); + } else { + spec.setInputDirURI(resolvedInputDirURI.toString()); + } } IngestionJobLauncher.runIngestionJob(spec); } diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/utils/JarUtils.java b/pinot-tools/src/main/java/org/apache/pinot/tools/utils/JarUtils.java new file mode 100644 index 0000000..b850e1a --- /dev/null +++ b/pinot-tools/src/main/java/org/apache/pinot/tools/utils/JarUtils.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.tools.utils; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Enumeration; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; + + +public class JarUtils { + private static final String JAR_PREFIX = "jar:"; + private static final String FILE_PREFIX = "file:"; + + private JarUtils() { + } // non-instantiable + + /** + * Copies a directory from a jar file to an external directory. + */ + public static void copyResourcesToDirectory(String fromJarFilePath, String jarDir, String destDir) + throws IOException { + if (fromJarFilePath.startsWith(JAR_PREFIX)) { + fromJarFilePath = fromJarFilePath.substring(JAR_PREFIX.length()); + } + if (fromJarFilePath.startsWith(FILE_PREFIX)) { + fromJarFilePath = fromJarFilePath.substring(FILE_PREFIX.length()); + } + JarFile fromJar = new JarFile(fromJarFilePath); + for (Enumeration<JarEntry> entries = fromJar.entries(); entries.hasMoreElements(); ) { + JarEntry entry = entries.nextElement(); + if (entry.getName().startsWith(jarDir + "/") && !entry.isDirectory()) { + File dest = new File(destDir + "/" + entry.getName().substring(jarDir.length() + 1)); + File parent = dest.getParentFile(); + if (parent != null) { + parent.mkdirs(); + } + + FileOutputStream out = new FileOutputStream(dest); + InputStream in = fromJar.getInputStream(entry); + + try { + byte[] buffer = new byte[8 * 1024]; + + int s = 0; + while ((s = in.read(buffer)) > 0) { + out.write(buffer, 0, s); + } + } catch (IOException e) { + throw new IOException("Could not copy asset from jar file", e); + } finally { + try { + in.close(); + } catch (IOException ignored) { + } + try { + out.close(); + } catch (IOException ignored) { + } + } + } + } + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
