Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1713#discussion_r164929882
--- Diff:
examples/spark2/src/main/scala/org/apache/carbondata/benchmark/ConcurrencyQueryBenchmark.scala
---
@@ -15,47 +15,63 @@
* limitations under the License.
*/
-package org.apache.carbondata.examples
+package org.apache.carbondata.benchmark
import java.io.File
+import java.text.SimpleDateFormat
import java.util
+import java.util.Date
import java.util.concurrent.{Callable, Executors, Future, TimeUnit}
import scala.util.Random
import org.apache.spark.sql.{DataFrame, Row, SaveMode, SparkSession}
import org.apache.spark.sql.types._
-import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.constants.{CarbonCommonConstants,
CarbonVersionConstants}
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
// scalastyle:off println
-object ConcurrencyTest {
-
- var totalNum = 100 * 1000 * 1000
- var ThreadNum = 16
- var TaskNum = 100
- var ResultIsEmpty = true
- val cardinalityId = 10000 * 10000
+object ConcurrencyQueryBenchmark {
+
+ // generate number of data
+ var totalNum = 1 * 1000 * 1000
--- End diff --
Because user can change the variable in shell script, like:
xubo # cat query3.sh
#!/bin/bash
for j in 100000000
do
for k in {1..32}
do
for l in true
do
for i in {1..8}
do
echo 'num,executors,resultIsEmpty,times'
echo $j
echo $k
echo $l
echo $i
spark-submit \
--class org.apache.carbondata.examples.ConcurrencyTest \
--master yarn \
--deploy-mode client \
--driver-memory 16g \
--executor-cores $k \
--executor-memory 24g \
--num-executors 3 \
concurrencyTest.jar \
$j 16 100 $l '/tmp/carbondata' false false false
done
done
done
done
---
