Repository: ignite
Updated Branches:
  refs/heads/master 1142a3681 -> ac3d54504


IGNITE-7531: Data load benchmarks. This closes #3571.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ac3d5450
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ac3d5450
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ac3d5450

Branch: refs/heads/master
Commit: ac3d545045037446014b66108364bbef39362847
Parents: 1142a36
Author: Pavel Kuznetsov <palmi...@gmail.com>
Authored: Tue Mar 13 15:46:36 2018 +0300
Committer: devozerov <voze...@gridgain.com>
Committed: Tue Mar 13 15:46:36 2018 +0300

----------------------------------------------------------------------
 .../ignite-localhost-persistence-config.xml     |  71 +++++++
 .../benchmark-jdbc-thin-inmemory.properties     | 108 +++++++++++
 .../benchmark-jdbc-thin-persistence.properties  | 113 +++++++++++
 .../yardstick/IgniteBenchmarkArguments.java     |   9 +-
 .../yardstick/jdbc/AbstractJdbcBenchmark.java   |  23 ++-
 .../upload/AbstractNativeBenchmark.java         | 114 +++++++++++
 .../upload/AbstractUploadBenchmark.java         | 187 +++++++++++++++++++
 .../upload/BatchedInsertBenchmark.java          |  65 +++++++
 .../ignite/yardstick/upload/CopyBenchmark.java  | 125 +++++++++++++
 .../yardstick/upload/InsertBenchmark.java       |  52 ++++++
 .../yardstick/upload/NativePutBenchmark.java    |  39 ++++
 .../upload/NativeStreamerBenchmark.java         |  66 +++++++
 .../upload/UploadBenchmarkArguments.java        | 158 ++++++++++++++++
 .../yardstick/upload/model/QueryFactory.java    | 181 ++++++++++++++++++
 .../ignite/yardstick/upload/model/Values10.java |  76 ++++++++
 .../org/yardstickframework/package-info.java    |  25 +++
 .../probes/TotalTimeProbe.java                  | 137 ++++++++++++++
 17 files changed, 1543 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/config/ignite-localhost-persistence-config.xml
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/ignite-localhost-persistence-config.xml 
b/modules/yardstick/config/ignite-localhost-persistence-config.xml
new file mode 100644
index 0000000..2e9d9ff
--- /dev/null
+++ b/modules/yardstick/config/ignite-localhost-persistence-config.xml
@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~      http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<!--
+    Ignite Spring configuration file to startup grid.
+-->
+<beans xmlns="http://www.springframework.org/schema/beans";
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+       xsi:schemaLocation="
+        http://www.springframework.org/schema/beans 
http://www.springframework.org/schema/beans/spring-beans-2.5.xsd";>
+    <import resource="ignite-base-config.xml"/>
+
+    <bean id="grid.cfg" 
class="org.apache.ignite.configuration.IgniteConfiguration" 
parent="base-ignite.cfg">
+        <property name="localHost" value="127.0.0.1"/>
+
+        <property name="discoverySpi">
+            <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
+                <property name="ipFinder">
+                    <bean 
class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder">
+                        <property name="addresses">
+                            <list>
+                                <value>127.0.0.1:47500</value>
+                                <value>127.0.0.1:47501</value>
+                                <value>127.0.0.1:47502</value>
+                                <value>127.0.0.1:47503</value>
+                                <value>127.0.0.1:47504</value>
+                                <value>127.0.0.1:47505</value>
+                                <value>127.0.0.1:47506</value>
+                                <value>127.0.0.1:47507</value>
+                                <value>127.0.0.1:47508</value>
+                                <value>127.0.0.1:47509</value>
+                            </list>
+                        </property>
+                    </bean>
+                </property>
+            </bean>
+        </property>
+
+        <property name="dataStorageConfiguration" >
+            <bean 
class="org.apache.ignite.configuration.DataStorageConfiguration">
+                <property name="defaultDataRegionConfiguration">
+                    <bean 
class="org.apache.ignite.configuration.DataRegionConfiguration">
+                        <property name="persistenceEnabled" value="true"/>
+                    </bean>
+                </property>
+            </bean>
+        </property>
+
+        <property name="communicationSpi">
+            <bean 
class="org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi">
+                <property name="sharedMemoryPort" value="-1"/>
+            </bean>
+        </property>
+    </bean>
+</beans>

http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/config/upload/benchmark-jdbc-thin-inmemory.properties
----------------------------------------------------------------------
diff --git 
a/modules/yardstick/config/upload/benchmark-jdbc-thin-inmemory.properties 
b/modules/yardstick/config/upload/benchmark-jdbc-thin-inmemory.properties
new file mode 100644
index 0000000..cacf726
--- /dev/null
+++ b/modules/yardstick/config/upload/benchmark-jdbc-thin-inmemory.properties
@@ -0,0 +1,108 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# Benchmarks for data upload in inmemory mode (persistence disabled).
+#
+
+now0=`date +'%H%M%S'`
+
+# JVM options.
+JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
+
+# Uncomment to enable concurrent garbage collection (GC) if you encounter long 
GC pauses.
+JVM_OPTS=${JVM_OPTS}" \
+-Xms8g \
+-Xmx8g \
+-Xloggc:./gc${now0}.log \
+-XX:+PrintGCDetails \
+-verbose:gc \
+-XX:+UseParNewGC \
+-XX:+UseConcMarkSweepGC \
+-XX:+PrintGCDateStamps \
+"
+
+#Ignite version
+ver="RELEASE-"
+
+# List of default probes.
+# Add DStatProbe or VmStatProbe if your OS supports it (e.g. if running on 
Linux).
+BENCHMARK_DEFAULT_PROBES=TotalTimeProbe
+
+# Packages where the specified benchmark is searched by reflection mechanism.
+BENCHMARK_PACKAGES=org.yardstickframework,org.apache.ignite.yardstick
+
+# Flag which indicates to restart the servers before every benchmark execution.
+RESTART_SERVERS=true
+
+# Probe point writer class name.
+# BENCHMARK_WRITER=
+
+# The benchmark is applicable only for 2 servers (the second server is started 
in client mode) and 1 driver.
+SERVER_HOSTS=localhost,localhost
+DRIVER_HOSTS=localhost
+
+# Remote username.
+# REMOTE_USER=
+
+# Number of nodes, used to wait for the specified number of nodes to start.
+nodesNum=$((`echo ${SERVER_HOSTS} | tr ',' '\n' | wc -l` + `echo 
${DRIVER_HOSTS} | tr ',' '\n' | wc -l`))
+
+# Backups count.
+b=1
+
+# Warmup.
+w=0
+
+# Threads count.
+t=1
+
+# Sync mode.
+sm=FULL_SYNC
+
+
+# Run configuration which contains all benchmarks.
+# Note that each benchmark is set to run only one time, warmup parameter is 
set to 0 due to custom warmup operation.
+CONFIGS="\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 
${b} --warmup ${w} --operations 1 \
+  -jdbc jdbc:ignite:thin://auto.find/ \
+  --threads ${t} \
+  --syncMode ${sm} -dn NativePutBenchmark -sn IgniteNode -ds 
${ver}sql-upload-native-inmemory-jdbc-thin-r1-${b}-backup \
+  --upload-rows 1000000 -cl --clientNodesAfterId 0, \
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 
${b} --warmup ${w} --operations 1 \
+  -jdbc jdbc:ignite:thin://auto.find/ \
+  --threads ${t} \
+  --syncMode ${sm} -dn NativeStreamerBenchmark -sn IgniteNode -ds 
${ver}sql-upload-native-inmemory-jdbc-thin-r1-${b}-backup \
+  --upload-rows 1000000 -cl --clientNodesAfterId 0 \
+  --streamer-local-batch-size 1000, \
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 
${b} --warmup ${w} --operations 1 \
+  -jdbc jdbc:ignite:thin://auto.find/ \
+  --threads ${t} \
+  --syncMode ${sm} -dn CopyBenchmark -sn IgniteNode -ds 
${ver}sql-upload-inmemory-jdbc-thin-r1-${b}-backup \
+  --upload-rows 1000000 -cl --clientNodesAfterId 0, \
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 
${b} --warmup ${w} --operations 1 \
+  -jdbc jdbc:ignite:thin://auto.find/ \
+  --threads ${t} \
+  --syncMode ${sm} -dn InsertBenchmark -sn IgniteNode -ds 
${ver}sql-upload-inmemory-jdbc-thin-r1-${b}-backup \
+  --upload-rows 1000000 -cl --clientNodesAfterId 0, \
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 
${b} --warmup ${w} --operations 1 \
+  -jdbc jdbc:ignite:thin://auto.find/ \
+  --threads ${t} \
+  --syncMode ${sm} -dn BatchedInsertBenchmark -sn IgniteNode -ds 
${ver}sql-upload-inmemory-jdbc-thin-r1-${b}-backup \
+  --upload-rows 1000000 -cl --clientNodesAfterId 0 \
+  --upload-jdbc-batch-size 1000 \
+"

http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/config/upload/benchmark-jdbc-thin-persistence.properties
----------------------------------------------------------------------
diff --git 
a/modules/yardstick/config/upload/benchmark-jdbc-thin-persistence.properties 
b/modules/yardstick/config/upload/benchmark-jdbc-thin-persistence.properties
new file mode 100644
index 0000000..b72e9b7
--- /dev/null
+++ b/modules/yardstick/config/upload/benchmark-jdbc-thin-persistence.properties
@@ -0,0 +1,113 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# Benchmarks for data upload with enabled persistence.
+#
+
+now0=`date +'%H%M%S'`
+
+# JVM options.
+JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
+
+# Uncomment to enable concurrent garbage collection (GC) if you encounter long 
GC pauses.
+JVM_OPTS=${JVM_OPTS}" \
+-Xms8g \
+-Xmx8g \
+-Xloggc:./gc${now0}.log \
+-XX:+PrintGCDetails \
+-verbose:gc \
+-XX:+UseParNewGC \
+-XX:+UseConcMarkSweepGC \
+-XX:+PrintGCDateStamps \
+"
+
+#Ignite version
+ver="RELEASE-"
+
+# List of default probes.
+# Add DStatProbe or VmStatProbe if your OS supports it (e.g. if running on 
Linux).
+BENCHMARK_DEFAULT_PROBES=TotalTimeProbe
+
+# Packages where the specified benchmark is searched by reflection mechanism.
+BENCHMARK_PACKAGES=org.yardstickframework,org.apache.ignite.yardstick
+
+# Flag which indicates to restart the servers before every benchmark execution.
+RESTART_SERVERS=true
+
+# Probe point writer class name.
+# BENCHMARK_WRITER=
+
+# The benchmark is applicable only for 2 servers (the second server is started 
in client mode) and 1 driver.
+SERVER_HOSTS=localhost,localhost
+DRIVER_HOSTS=localhost
+
+# Remote username.
+# REMOTE_USER=
+
+# Number of nodes, used to wait for the specified number of nodes to start.
+nodesNum=$((`echo ${SERVER_HOSTS} | tr ',' '\n' | wc -l` + `echo 
${DRIVER_HOSTS} | tr ',' '\n' | wc -l`))
+
+# Backups count.
+b=1
+
+# Warmup.
+w=0
+
+# Threads count.
+t=1
+
+# Sync mode.
+sm=FULL_SYNC
+
+# Whither or not turn off WAL just for upload period.
+switchWal=false
+
+# Run configuration which contains all benchmarks.
+# Note that each benchmark is set to run only one time, warmup parameter is 
set to 0 due to custom warmup operation.
+CONFIGS="\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-persistence-config.xml -nn 
${nodesNum} -b ${b} --warmup ${w} --operations 1 \
+  -jdbc jdbc:ignite:thin://auto.find/ \
+  --threads ${t} \
+  --syncMode ${sm} -dn NativePutBenchmark -sn IgniteNode -ds 
${ver}sql-upload-native-persistence-jdbc-thin-r1-${b}-backup \
+  --upload-rows 1000000 -cl --clientNodesAfterId 0, \
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-persistence-config.xml -nn 
${nodesNum} -b ${b} --warmup ${w} --operations 1 \
+  -jdbc jdbc:ignite:thin://auto.find/ \
+  --threads ${t} \
+  --syncMode ${sm} -dn NativeStreamerBenchmark -sn IgniteNode -ds 
${ver}sql-upload-native-persistence-jdbc-thin-r1-${b}-backup \
+  --upload-rows 1000000 -cl --clientNodesAfterId 0 \
+  --streamer-local-batch-size 1000, \
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-persistence-config.xml -nn 
${nodesNum} -b ${b} --warmup ${w} --operations 1 \
+  -jdbc jdbc:ignite:thin://auto.find/ \
+  --threads ${t} \
+  --syncMode ${sm} -dn CopyBenchmark -sn IgniteNode -ds 
${ver}sql-upload-persistence-jdbc-thin-r1-${b}-backup \
+  --upload-rows 1000000 -cl --clientNodesAfterId 0 \
+  --disable-wal ${switchWal}, \
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-persistence-config.xml -nn 
${nodesNum} -b ${b} --warmup ${w} --operations 1 \
+  -jdbc jdbc:ignite:thin://auto.find/ \
+  --threads ${t} \
+  --syncMode ${sm} -dn InsertBenchmark -sn IgniteNode -ds 
${ver}sql-upload-persistence-jdbc-thin-r1-${b}-backup \
+  --upload-rows 1000000 -cl --clientNodesAfterId 0 \
+  --disable-wal ${switchWal}, \
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-persistence-config.xml -nn 
${nodesNum} -b ${b} --warmup ${w} --operations 1 \
+  -jdbc jdbc:ignite:thin://auto.find/ \
+  --threads ${t} \
+  --syncMode ${sm} -dn BatchedInsertBenchmark -sn IgniteNode -ds 
${ver}sql-upload-persistence-jdbc-thin-r1-${b}-backup \
+  --upload-rows 1000000 -cl --clientNodesAfterId 0 \
+  --upload-jdbc-batch-size 1000 \
+  --disable-wal ${switchWal} \
+"

http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
----------------------------------------------------------------------
diff --git 
a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
 
b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
index 8c453ad..9789aea 100644
--- 
a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
+++ 
b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
@@ -18,6 +18,8 @@
 package org.apache.ignite.yardstick;
 
 import com.beust.jcommander.Parameter;
+import com.beust.jcommander.ParametersDelegate;
+import java.util.Collections;
 import org.apache.ignite.IgniteDataStreamer;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
 import org.apache.ignite.configuration.DataStorageConfiguration;
@@ -29,6 +31,7 @@ import org.apache.ignite.transactions.TransactionIsolation;
 import java.util.ArrayList;
 import java.util.List;
 import org.apache.ignite.yardstick.cache.IgniteStreamerBenchmark;
+import org.apache.ignite.yardstick.upload.UploadBenchmarkArguments;
 import org.jetbrains.annotations.Nullable;
 
 /**
@@ -262,6 +265,10 @@ public class IgniteBenchmarkArguments {
     @GridToStringInclude
     private int clientNodesAfterId = -1;
 
+    @ParametersDelegate
+    @GridToStringInclude
+    public UploadBenchmarkArguments upload = new UploadBenchmarkArguments();
+
     /**
      * @return {@code True} if need set {@link DataStorageConfiguration}.
      */
@@ -384,7 +391,7 @@ public class IgniteBenchmarkArguments {
     /**
      * @return {@code True} if flag for native benchmarking is set.
      */
-    public boolean isNative(){
+    public boolean isNative() {
         return ntv;
     }
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/AbstractJdbcBenchmark.java
----------------------------------------------------------------------
diff --git 
a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/AbstractJdbcBenchmark.java
 
b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/AbstractJdbcBenchmark.java
index 9ef8a77..6ab024f 100644
--- 
a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/AbstractJdbcBenchmark.java
+++ 
b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/AbstractJdbcBenchmark.java
@@ -37,18 +37,18 @@ import static org.yardstickframework.BenchmarkUtils.println;
 /**
  * JDBC benchmark that performs query operations.
  */
-abstract public class AbstractJdbcBenchmark extends IgniteAbstractBenchmark {
+public abstract class AbstractJdbcBenchmark extends IgniteAbstractBenchmark {
     /** All {@link Connection}s associated with threads. */
     private final List<Connection> threadConnections = new ArrayList<>();
 
     /** JDBC URL. */
-    private String url;
+    protected String url;
 
     /** Each connection is also a transaction, so we better pin them to 
threads. */
     protected ThreadLocal<Connection> conn = new ThreadLocal<Connection>() {
         @Override protected Connection initialValue() {
             try {
-                Connection conn = connection();
+                Connection conn = connection(url);
 
                 synchronized (threadConnections) {
                     threadConnections.add(conn);
@@ -66,6 +66,9 @@ abstract public class AbstractJdbcBenchmark extends 
IgniteAbstractBenchmark {
     @Override public void setUp(BenchmarkConfiguration cfg) throws Exception {
         super.setUp(cfg);
 
+        // activate cluster if it is not auto activated
+        ignite().cluster().active(true);
+
         if (url == null) {
             if (args.jdbcUrl().startsWith(JdbcThinUtils.URL_PREFIX)) {
                 String addr = findThinAddress();
@@ -77,12 +80,22 @@ abstract public class AbstractJdbcBenchmark extends 
IgniteAbstractBenchmark {
 
         println("Using jdbc url:" + url);
 
-        fillData(cfg, (IgniteEx)ignite(), args.range());
+        setupData();
 
         ignite().close();
     }
 
     /**
+     * Sets up test data
+     *
+     * Gets executed before local Ignite node is closed
+     * @throws Exception On error.
+     */
+    protected void setupData() throws Exception {
+        fillData(cfg, (IgniteEx)ignite(), args.range());
+    }
+
+    /**
      * Find address of client node, that thin driver should use.
      *
      * @return Address for thin driver.
@@ -128,7 +141,7 @@ abstract public class AbstractJdbcBenchmark extends 
IgniteAbstractBenchmark {
      * @return JDBC connection.
      * @throws SQLException On error.
      */
-    private Connection connection() throws SQLException {
+    protected final Connection connection(String url) throws SQLException {
         println("JDBC connect to: " + url);
 
         Connection conn = DriverManager.getConnection(url);

http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/AbstractNativeBenchmark.java
----------------------------------------------------------------------
diff --git 
a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/AbstractNativeBenchmark.java
 
b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/AbstractNativeBenchmark.java
new file mode 100644
index 0000000..de1ad28
--- /dev/null
+++ 
b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/AbstractNativeBenchmark.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload;
+
+import java.util.Map;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.yardstick.IgniteAbstractBenchmark;
+import org.apache.ignite.yardstick.upload.model.Values10;
+import org.yardstickframework.BenchmarkConfiguration;
+import org.yardstickframework.BenchmarkUtils;
+
+/**
+ * Base class for benchmarks that perform upload using java api.
+ */
+public abstract class AbstractNativeBenchmark extends IgniteAbstractBenchmark {
+    /** Number of entries to be uploaded during warmup. */
+    private long insertRowsCnt;
+
+    /** Name of the {@link #cache} */
+    private String cacheName;
+
+    /** Cache method {@link test(Map)} uploads data to */
+    private IgniteCache<Long, Values10> cache;
+
+    /**
+     * Sets up benchmark: performs warmup on one cache and creates another for 
{@link #test(Map)} method.
+     *
+     * @param cfg Benchmark configuration.
+     * @throws Exception - on error.
+     */
+    @Override public void setUp(BenchmarkConfiguration cfg) throws Exception {
+        super.setUp(cfg);
+
+        cacheName = this.getClass().getSimpleName();
+
+        insertRowsCnt = args.upload.uploadRowsCnt();
+
+        // Number of entries to be uploaded during test().
+        long warmupRowsCnt = args.upload.warmupRowsCnt();
+
+        // warmup
+        BenchmarkUtils.println(cfg, "Starting custom warmup.");
+        String warmupCacheName = cacheName + "Warmup";
+
+        try (IgniteCache<Long, Values10> warmupCache = 
ignite().createCache(warmupCacheName)) {
+            upload(warmupCacheName, warmupRowsCnt);
+        }
+        finally {
+            ignite().destroyCache(warmupCacheName);
+        }
+
+        BenchmarkUtils.println(cfg, "Custom warmup finished.");
+
+        // cache for benchmarked action
+        cache = ignite().createCache(cacheName);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void tearDown() throws Exception {
+        try {
+            if (cache == null)
+                throw new IllegalStateException("Cache is null, probably an 
error during setUp or warmup");
+
+            long size = cache.sizeLong();
+
+            if (size != insertRowsCnt) {
+                String msg = "Incorrect cache size: [actual=" + size + ", 
expected=" + insertRowsCnt +"].";
+
+                BenchmarkUtils.println(cfg, "TearDown: " + msg);
+
+                throw new RuntimeException(msg);
+            }
+
+            cache.close();
+
+            ignite().destroyCache(cacheName);
+
+        }
+        catch (IgniteException ex) {
+            BenchmarkUtils.println(cfg, "Could not close or destroy cache: " + 
ex);
+
+            throw ex;
+        }
+        finally {
+            super.tearDown();
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean test(Map<Object, Object> ctx) throws Exception {
+        upload(cacheName, insertRowsCnt);
+
+        return true;
+    }
+
+    /** Uploads {@param insertsCnt} to cache with name {@param cacheName} 
using java api. */
+    protected abstract void upload(String cacheName, long insertsCnt);
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/AbstractUploadBenchmark.java
----------------------------------------------------------------------
diff --git 
a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/AbstractUploadBenchmark.java
 
b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/AbstractUploadBenchmark.java
new file mode 100644
index 0000000..81a93e2
--- /dev/null
+++ 
b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/AbstractUploadBenchmark.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.Map;
+import org.apache.ignite.yardstick.jdbc.AbstractJdbcBenchmark;
+import org.apache.ignite.yardstick.upload.model.QueryFactory;
+import org.yardstickframework.BenchmarkConfiguration;
+import org.yardstickframework.BenchmarkUtils;
+
+/**
+ * Base class for upload benchmarks.
+ * Designed to run test method one single time.
+ * Introduces custom warmup operation.
+ */
+public abstract class AbstractUploadBenchmark extends AbstractJdbcBenchmark {
+    /** Total inserts size. */
+    long insertRowsCnt;
+
+    /** Rows count to be inserted and deleted during warmup */
+    long warmupRowsCnt;
+
+    /** Factory that hides all the test data details. */
+    protected QueryFactory queries = new QueryFactory();
+
+    /** {@inheritDoc} */
+    @Override public final void setUp(BenchmarkConfiguration cfg) throws 
Exception {
+        super.setUp(cfg);
+
+        insertRowsCnt = args.upload.uploadRowsCnt();
+        warmupRowsCnt = args.upload.warmupRowsCnt();
+
+        init();
+
+        // Perform warmup keeping in mind wal optimization.
+        BenchmarkUtils.println(this.cfg, "Starting custom warmup");
+
+        if (args.upload.disableWal())
+            executeUpdate(QueryFactory.TURN_OFF_WAL);
+
+        try (Connection warmupConn = uploadConnection()) {
+            warmup(warmupConn);
+        }
+
+        if (args.upload.disableWal())
+            executeUpdate(QueryFactory.TURN_ON_WAL);
+
+        BenchmarkUtils.println(this.cfg, "Custom warmup finished");
+
+        dropAndCreate();
+    }
+
+    /**
+     * Inits benchmark fields.
+     */
+    protected void init() {
+        // No-op.
+    }
+
+    /**
+     * Method to warm up Benchmark server. <br/>
+     * In upload benchmarks we need warmup action
+     * and real test action to be separated.
+     */
+    protected abstract void warmup(Connection warmupConn) throws Exception;
+
+    /**
+     * Creates empty table.
+     */
+    @Override protected void setupData() throws Exception{
+        dropAndCreate();
+    }
+
+    /**
+     * Uploads data using this special connection, that may have additional
+     * url parameters, such as {@code streaming=true}.
+     */
+    protected abstract void upload(Connection uploadConn) throws Exception;
+
+    /** {@inheritDoc} */
+    @Override public final boolean test(Map<Object, Object> ctx) throws 
Exception {
+        if (args.upload.disableWal())
+            executeUpdate(QueryFactory.TURN_OFF_WAL);
+
+        try (Connection uploadConn = uploadConnection()) {
+            upload(uploadConn);
+        }
+
+        if (args.upload.disableWal())
+            executeUpdate(QueryFactory.TURN_ON_WAL);
+
+        return true;
+    }
+
+    /**
+     * Drops and re-creates test table.
+     */
+    private final void dropAndCreate() throws SQLException {
+        executeUpdate(QueryFactory.DROP_TABLE_IF_EXISTS);
+        executeUpdate(queries.createTable());
+    }
+
+    /**
+     * Retrieves records count in the test table.
+     */
+    public long count() throws SQLException {
+        try(PreparedStatement cnt = 
conn.get().prepareStatement(QueryFactory.COUNT)){
+            try (ResultSet rs = cnt.executeQuery()) {
+                rs.next();
+
+                return rs.getLong(1);
+            }
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public void tearDown() throws Exception {
+        BenchmarkUtils.println(cfg, "Tearing down");
+
+        try {
+            long cnt = count();
+
+            if (cnt != insertRowsCnt) {
+                String msg = "Rows count is incorrect: [actual=" + cnt + ", 
expected=" + insertRowsCnt + "]";
+
+                BenchmarkUtils.println(cfg, "TearDown: " + msg);
+
+                throw new RuntimeException(msg);
+            }
+
+            BenchmarkUtils.println(cfg, "Test table contains " + cnt + " 
rows.");
+        }
+        finally {
+            super.tearDown();
+        }
+
+        BenchmarkUtils.println(cfg, "TearDown successfully finished.");
+    }
+
+    /**
+     * Facility method for executing update queries.
+     */
+    private int executeUpdate(String updQry) throws SQLException {
+        try(PreparedStatement update = conn.get().prepareStatement(updQry)){
+            return update.executeUpdate();
+        }
+    }
+
+    /**
+     *  Creates new connection only for upload purpose.
+     *  This connection is special, since it may have additional jdbc url 
parameters.
+     */
+    private Connection uploadConnection() throws SQLException {
+        String urlParams = "";
+
+        // We can't just pass entire params string, due to yardstick, which 
relies on bash,
+        // has some troubles with escaping ampersand character.
+        List<String> rawParams = args.upload.uploadJdbcParams();
+
+        if (!rawParams.isEmpty()) {
+            String kvList = String.join("&", rawParams);
+            urlParams = "?" + kvList;
+        }
+
+        return connection(url + urlParams);
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/BatchedInsertBenchmark.java
----------------------------------------------------------------------
diff --git 
a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/BatchedInsertBenchmark.java
 
b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/BatchedInsertBenchmark.java
new file mode 100644
index 0000000..a460e20
--- /dev/null
+++ 
b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/BatchedInsertBenchmark.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+
+/**
+ * Measures total time of data upload using batched insert.
+ * Supports streaming.
+ */
+public class BatchedInsertBenchmark extends AbstractUploadBenchmark {
+    /** Number of inserts in batch */
+    private long batchSize;
+
+    /** {@inheritDoc} */
+    @Override protected void init() {
+        batchSize = args.upload.jdbcBatchSize();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void warmup(Connection warmupConn) throws SQLException 
{
+        performBatchUpdate(warmupConn, warmupRowsCnt);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void upload(Connection uploadConn) throws Exception {
+        performBatchUpdate(uploadConn, insertRowsCnt);
+    }
+
+    /**
+     * Actually performs batched inserts, using specified connection.
+     *
+     * @param uploadConn Special connection for upload purposes.
+     * @param rowsCnt Number of rows to insert.
+     */
+    private void performBatchUpdate(Connection uploadConn, long rowsCnt) 
throws SQLException {
+        try (PreparedStatement insert = 
uploadConn.prepareStatement(queries.insert())) {
+            for (long id = 1; id <= rowsCnt; id++) {
+                queries.setRandomInsertArgs(insert, id);
+
+                insert.addBatch();
+
+                if (id % batchSize == 0 || id == rowsCnt)
+                    insert.executeBatch();
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/CopyBenchmark.java
----------------------------------------------------------------------
diff --git 
a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/CopyBenchmark.java
 
b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/CopyBenchmark.java
new file mode 100644
index 0000000..f0dcf20
--- /dev/null
+++ 
b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/CopyBenchmark.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload;
+
+import java.io.File;
+import java.io.PrintStream;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import org.yardstickframework.BenchmarkUtils;
+
+/**
+ * Single shot benchmark for the COPY FROM... sql query.
+ * Measures total time of load number of rows from csv file using thin driver.
+ *
+ * Enabling streaming is useless for this benchmark, since copy uses streamer 
under the hood.
+ */
+public class CopyBenchmark extends AbstractUploadBenchmark {
+    /** csv file for warmup. */
+    private String warmupCsvPath;
+
+    /** csv file for benchmared action. */
+    private String realCsvPath;
+
+    /** packet_size parameter for sql COPY command.*/
+    private Long packetSize;
+
+    /** {@inheritDoc} */
+    @Override protected void init() {
+        super.init();
+
+        packetSize = args.upload.copyPacketSize();
+
+        warmupCsvPath = generateWarmupCsv();
+        realCsvPath = generateRealCsv();
+
+    }
+
+    /**
+     * Generate csv file for copy operation being benchmarked.
+     */
+    private String generateRealCsv() {
+        String prefix = "data-" + insertRowsCnt + "-rows-";
+
+        return generate(prefix, insertRowsCnt);
+    }
+
+    /**
+     * Generate csv file for copy operation being performed during warmup.
+     */
+    private String generateWarmupCsv() {
+        String prefix = "warmup-" + warmupRowsCnt + "-rows-" ;
+
+        return generate(prefix, warmupRowsCnt);
+    }
+
+    /**
+     * Generate csv file with specified prefix and containing specified number
+     * of records (lines).
+     * <br/>
+     * Each record contain incremental (starting 1) id and set of random 
values.
+     * <br/>
+     * File is created in current directory and is deleted on JVM exits.
+     *
+     * @param filePrefix prefix for csv to be generated.
+     * @param records how many csv lines generate.
+     *
+     * @return absolute path for generated csv file.
+     */
+    private String generate(String filePrefix, long records){
+        try {
+            File workDir = new File(System.getProperty("user.dir"));
+
+            File f = File.createTempFile(filePrefix, ".csv", workDir);
+
+            f.deleteOnExit();
+
+            BenchmarkUtils.println("Generating file: " + f.getAbsolutePath());
+
+            try (PrintStream out = new PrintStream(f)) {
+                for (long id = 1; id <= records; id++) {
+                    String csvLine = queries.randomCsvLine(id);
+                    out.println(csvLine);
+                }
+            }
+
+            long sizeMb = f.length() / (1024L * 1024L);
+
+            BenchmarkUtils.println("File have been generated (" + sizeMb + 
"MiB). It will be removed on exit");
+
+            return f.getAbsolutePath();
+        }
+        catch (Exception ex){
+            throw new RuntimeException("Couldn't generate CSV file, 
terminating", ex);
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void warmup(Connection warmupConn) throws Exception {
+        try (PreparedStatement fromCsv = 
warmupConn.prepareStatement(queries.copyFrom(warmupCsvPath, packetSize))) {
+            fromCsv.executeUpdate();
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public void upload(Connection uploadConn) throws Exception {
+        try (PreparedStatement fromCsv = 
uploadConn.prepareStatement(queries.copyFrom(realCsvPath, packetSize))) {
+            fromCsv.executeUpdate();
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/InsertBenchmark.java
----------------------------------------------------------------------
diff --git 
a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/InsertBenchmark.java
 
b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/InsertBenchmark.java
new file mode 100644
index 0000000..ecf5412
--- /dev/null
+++ 
b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/InsertBenchmark.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+
+/**
+ * Measures total time of upload data using sequence of single inserts;
+ * Supports streaming.
+ */
+public class InsertBenchmark extends AbstractUploadBenchmark {
+    /** {@inheritDoc} */
+    @Override public void warmup(Connection warmupConn) throws SQLException {
+        try (PreparedStatement insert = 
warmupConn.prepareStatement(queries.insert())) {
+            for (long id = 1; id <= warmupRowsCnt; id++) {
+                queries.setRandomInsertArgs(insert, id);
+
+                insert.executeUpdate();
+            }
+        }
+    }
+
+    /**
+     * Sequence of single inserts.
+     */
+    @Override public void upload (Connection uploadConn) throws Exception {
+        try (PreparedStatement insert = 
uploadConn.prepareStatement(queries.insert())) {
+            for (long id = 1; id <= insertRowsCnt; id++) {
+                queries.setRandomInsertArgs(insert, id);
+
+                insert.executeUpdate();
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/NativePutBenchmark.java
----------------------------------------------------------------------
diff --git 
a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/NativePutBenchmark.java
 
b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/NativePutBenchmark.java
new file mode 100644
index 0000000..60fbbda
--- /dev/null
+++ 
b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/NativePutBenchmark.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.yardstick.upload.model.Values10;
+
+/**
+ * Benchmark that inserts single upload of number of entries using {@link 
IgniteCache#put(Object, Object)}.
+ */
+public class NativePutBenchmark extends AbstractNativeBenchmark {
+    /**
+     * Uploads randomly generated data using simple put.
+     *
+     * @param cacheName - name of the cache.
+     * @param insertsCnt - how many entries should be uploaded.
+     */
+    @Override protected void upload(String cacheName, long insertsCnt) {
+        IgniteCache<Object, Object> c = ignite().cache(cacheName);
+
+        for (long id = 1; id <= insertsCnt; id++)
+            c.put(id, new Values10() );
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/NativeStreamerBenchmark.java
----------------------------------------------------------------------
diff --git 
a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/NativeStreamerBenchmark.java
 
b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/NativeStreamerBenchmark.java
new file mode 100644
index 0000000..e5117bc
--- /dev/null
+++ 
b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/NativeStreamerBenchmark.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.yardstick.upload.model.Values10;
+
+/**
+ * Benchmark that performs single upload of number of entries using {@link 
IgniteDataStreamer}.
+ */
+public class NativeStreamerBenchmark extends AbstractNativeBenchmark {
+    /**
+     * Uploads randomly generated entries to specified cache.
+     *
+     * @param cacheName - name of the cache.
+     * @param insertsCnt - how many entries should be uploaded.
+     */
+    @Override protected void upload(String cacheName, long insertsCnt) {
+        try (IgniteDataStreamer<Long, Values10> streamer = 
ignite().dataStreamer(cacheName)) {
+            if (args.upload.streamerBufSize() != null)
+                streamer.perNodeBufferSize(args.upload.streamerBufSize());
+
+            if (args.upload.streamerNodeParOps() != null)
+                
streamer.perNodeParallelOperations(args.upload.streamerNodeParOps());
+
+            int batchSize = args.upload.streamerLocBatchSize();
+
+            // IgniteDataStreamer.addData(Object, Object) has known 
performance issue,
+            // so we have an option to work it around.
+            if (batchSize == 1) {
+                for (long i = 1; i <= insertsCnt; i++)
+                    streamer.addData(i, new Values10());
+            }
+            else {
+                Map<Long, Values10> buf = new HashMap<>(batchSize);
+
+                for (long i = 1; i <= insertsCnt; i++) {
+                    buf.put(i, new Values10());
+
+                    if (i % batchSize == 0 || i == insertsCnt) {
+                        streamer.addData(buf);
+
+                        buf.clear();
+                    }
+                }
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/UploadBenchmarkArguments.java
----------------------------------------------------------------------
diff --git 
a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/UploadBenchmarkArguments.java
 
b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/UploadBenchmarkArguments.java
new file mode 100644
index 0000000..27a2164
--- /dev/null
+++ 
b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/UploadBenchmarkArguments.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload;
+
+import com.beust.jcommander.Parameter;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.internal.util.tostring.GridToStringBuilder;
+import org.apache.ignite.yardstick.IgniteBenchmarkArguments;
+
+/**
+ * Represents command line arguments that are specific to upload benchmarks.
+ *
+ * @see IgniteBenchmarkArguments
+ */
+@SuppressWarnings({"UnusedDeclaration", "FieldCanBeLocal"})
+public class UploadBenchmarkArguments {
+    /** Whither or not temporary disable Write Ahead Log during upload. */
+    @Parameter(names = {"--disable-wal"},
+        arity = 1,
+        description = "Upload benchmark only: " +
+            "turn off Write Ahead Log before data uploading " +
+            "and turn it on again when upload is done.")
+    private boolean disableWal = false;
+
+    /**
+     * Parameters for JDBC connection, that only uploads data.
+     *
+     * We can't just pass entire params string, due to yardstick, which relies 
on bash,
+     * has some troubles with escaping ampersand character.
+     */
+    @Parameter(names = {"--sql-jdbc-params"},
+        variableArity = true,
+        description = "Upload benchmark only: " +
+            "Additional url parameters (space separated key=value) for special 
JDBC connection that only uploads data. ")
+    private List<String> uploadJdbcParams = Collections.emptyList();
+
+    @Parameter(names = {"--sql-copy-packet-size"},
+        description = "Upload benchmark only: use custom packet_size (in 
bytes) for copy command.")
+    private Long copyPacketSize = null;
+
+    @Parameter(names = {"--streamer-node-buf-size"},
+        description = "Native Streamer benchmark only: Set streamer's 
perNodeBufferSize property")
+    private Integer streamerBufSize = null;
+
+    @Parameter(names = {"--streamer-node-par-ops"},
+        description = "Native Streamer benchmark only: Set streamer's 
perNodeParallelOperations property")
+    private Integer streamerNodeParOps = null;
+
+    @Parameter(names = {"--streamer-local-batch-size"},
+        description = "Native Streamer benchmark only: collect entries to 
local map before passing to streamer." +
+            "If set to 1, than entries will be passed directly.")
+    private int streamerLocBatchSize = 1;
+
+    /** How many rows to upload during warmup. */
+    @Parameter(names = {"--upload-warmup-rows"})
+    private long warmupRowsCnt = 3_000_000;
+
+    /** How many rows to upload during real test. */
+    @Parameter(names = {"--upload-rows"})
+    private long uploadRowsCnt = -1;
+
+    /** How many rows to include in each batch ({@link BatchedInsertBenchmark} 
only). */
+    @Parameter(names = {"--upload-jdbc-batch-size"})
+    private long jdbcBatchSize = -1;
+
+    /**
+     * @return Switch wal.
+     */
+    public boolean disableWal() {
+        return disableWal;
+    }
+
+    /** @return parameters for JDBC url. */
+    public List<String> uploadJdbcParams() {
+        return uploadJdbcParams;
+    }
+
+    /** @return packet_size value for copy command or {@code null} for default 
value. */
+    @Nullable
+    public Long copyPacketSize() {
+        return copyPacketSize;
+    }
+
+    /**
+     * @return Value for {@link IgniteDataStreamer#perNodeBufferSize(int)}.
+     */
+    @Nullable
+    public Integer streamerBufSize() {
+        return streamerBufSize;
+    }
+
+    /**
+     * @return Value for {@link 
IgniteDataStreamer#perNodeParallelOperations(int)}.
+     */
+    @Nullable
+    public Integer streamerNodeParOps() {
+        return streamerNodeParOps;
+    }
+
+    /**
+     * How many entries to collect before passing map to {@link 
IgniteDataStreamer#addData(Map)}.
+     * If set to 1, {@link IgniteDataStreamer#addData(Object, Object)} method 
will be used.
+     */
+    public int streamerLocBatchSize() {
+        return streamerLocBatchSize;
+    }
+
+    /**
+     * See {@link #warmupRowsCnt}.
+     */
+    public long warmupRowsCnt() {
+        return warmupRowsCnt;
+    }
+
+    /**
+     * See {@link #uploadRowsCnt}.
+     */
+    public long uploadRowsCnt() {
+        if (uploadRowsCnt < 0)
+            throw new IllegalStateException("Upload rows count is not 
specified. Check arguments.");
+
+        return uploadRowsCnt;
+    }
+
+    /**
+     * See {@link #jdbcBatchSize}.
+     */
+    public long jdbcBatchSize() {
+        if (jdbcBatchSize < 0)
+            throw new IllegalStateException("JDBC batch size is not specified. 
Check arguments.");
+
+        return jdbcBatchSize;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return GridToStringBuilder.toString(UploadBenchmarkArguments.class, 
this);
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/model/QueryFactory.java
----------------------------------------------------------------------
diff --git 
a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/model/QueryFactory.java
 
b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/model/QueryFactory.java
new file mode 100644
index 0000000..e14a091
--- /dev/null
+++ 
b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/model/QueryFactory.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload.model;
+
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.concurrent.ThreadLocalRandom;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Factory that hides all test data details:
+ * what query to use to create table
+ * or what random arguments to set in prepared statement.
+ */
+public class QueryFactory {
+    /** Query to drop table if it exists. */
+    public static final String DROP_TABLE_IF_EXISTS = "DROP TABLE IF EXISTS 
test_upload;";
+
+    /** Query to count table size. */
+    public static final String COUNT = "SELECT COUNT(id) FROM test_upload;";
+
+    /** Turns off Write Ahead Log. */
+    public static final String TURN_OFF_WAL = "ALTER TABLE test_upload 
NOLOGGING";
+
+    /** Turns on Write Ahead Log. */
+    public static final String TURN_ON_WAL = "ALTER TABLE test_upload LOGGING";
+
+    /** Number of "values" fields in the test table (any field except primary 
key). */
+    private int valFieldsCnt = 10;
+
+    /** Create table with long primary key and number of long and varchar 
fields */
+    private String createTable = newCreateTableQuery();
+
+    /** Parametrised query to insert new row. */
+    private String insert = newInsertQuery();
+
+    /**
+     * See {@link #createTable}.
+     */
+    private String newCreateTableQuery() {
+        StringBuilder create = new StringBuilder("CREATE TABLE test_upload (id 
LONG PRIMARY KEY");
+
+        for (int vi = 1; vi <= valFieldsCnt; vi++) {
+            create.append(", val_").append(vi);
+
+            if (vi % 2 == 1)
+                create.append(" VARCHAR(255)");
+            else
+                create.append(" LONG");
+
+        }
+
+        create.append(");");
+
+        return create.toString();
+    }
+
+    /**
+     * See {@link #insert}.
+     */
+    private String newInsertQuery() {
+        StringBuilder insert = new StringBuilder("INSERT INTO test_upload 
VALUES (?");
+        for (int vi = 1; vi <= valFieldsCnt; vi++)
+            insert.append(", ?");
+
+        insert.append(");");
+        return insert.toString();
+    }
+
+    /**
+     * See {@link #createTable}.
+     */
+    public String createTable() {
+        return createTable;
+    }
+
+    /**
+     * See {@link #insert}.
+     */
+    public String insert() {
+        return insert;
+    }
+
+    /**
+     * @param csvFilePath path to csv file.
+     * @param packetSize if not null, add packet_size query option.
+     * @return sql query that inserts data from specified csv file.
+     */
+    public String copyFrom(String csvFilePath, @Nullable Long packetSize) {
+        String pSizeExpr = "";
+
+        if (packetSize != null)
+            pSizeExpr = " packet_size " + packetSize;
+
+        return "COPY FROM \"" + csvFilePath + "\" " +
+            "INTO test_upload " + attributes() + " " +
+            "FORMAT CSV" + pSizeExpr + ";";
+    }
+
+    /**
+     * Creates string - comma-separated attributes of test table, surrounded 
with braces
+     * Is used as a part of sql statement.
+     *
+     * @return attributes list of test table as part of sql statement.
+     */
+    private String attributes() {
+        StringBuilder attrs = new StringBuilder("(id");
+
+        for (int vi = 1; vi <= valFieldsCnt; vi++)
+            attrs.append(", val_").append(vi);
+
+        attrs.append(')');
+
+        return attrs.toString();
+    }
+
+    /**
+     * Fills specified prepared statement with random values and specified id 
(primary key).
+     *
+     * @param stmt prepared statement, built from {@link #insert} query.
+     * @param id id in the test table.
+     * @throws SQLException if statement is not correct.
+     */
+    public void setRandomInsertArgs(PreparedStatement stmt, long id) throws 
SQLException {
+        stmt.setLong(1, id);
+        ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+        for (int vi = 1; vi <= valFieldsCnt; vi++) {
+            // vi is value index (among all values), but we also have "id" 
which is primary key
+            // so index in query is value index shifted by 1
+            int qryIdx = vi + 1;
+
+            long nextVal = rnd.nextLong();
+
+            if (vi % 2 == 1)
+                stmt.setLong(qryIdx, nextVal);
+            else
+                stmt.setString(qryIdx, String.valueOf(nextVal));
+        }
+    }
+
+    /**
+     * Generates CSV line containing specified id and random values.
+     * This line corresponds 1 row of the test table,
+     * which will be inserted in the end.
+     *
+     * @param id key in the test table.
+     * @return generated comma-separated line.
+     */
+    public String randomCsvLine(long id) {
+        ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+        StringBuilder line = new StringBuilder().append(id);
+
+        for (int vi = 1; vi <= valFieldsCnt; vi++) {
+            line.append(',');
+
+            if (vi % 2 == 1)
+                line.append(rnd.nextLong());
+            else
+                line.append('"').append(rnd.nextLong()).append('"');
+        }
+
+        return line.toString();
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/model/Values10.java
----------------------------------------------------------------------
diff --git 
a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/model/Values10.java
 
b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/model/Values10.java
new file mode 100644
index 0000000..f0bdee0
--- /dev/null
+++ 
b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/model/Values10.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload.model;
+
+import java.util.concurrent.ThreadLocalRandom;
+
+/**
+ * Describes data model.
+ * Matches data model, defined in {@link QueryFactory#createTable()}
+ */
+public class Values10 {
+    /** */
+    final String val1;
+
+    /** */
+    final long val2;
+
+    /** */
+    final String val3;
+
+    /** */
+    final long val4;
+
+    /** */
+    final String val5;
+
+    /** */
+    final long val6;
+
+    /** */
+    final String val7;
+
+    /** */
+    final long val8;
+
+    /** */
+    final String val9;
+
+    /** */
+    final long val10;
+
+    /** Creates new object with randomly initialized fields */
+    public Values10(){
+        ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+        val1 = String.valueOf(rnd.nextLong());
+        val2 = rnd.nextLong();
+
+        val3 = String.valueOf(rnd.nextLong());
+        val4 = rnd.nextLong();
+
+        val5 = String.valueOf(rnd.nextLong());
+        val6 = rnd.nextLong();
+
+        val7 = String.valueOf(rnd.nextLong());
+        val8 = rnd.nextLong();
+
+        val9 = String.valueOf(rnd.nextLong());
+        val10 = rnd.nextLong();
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/yardstickframework/package-info.java
----------------------------------------------------------------------
diff --git 
a/modules/yardstick/src/main/java/org/yardstickframework/package-info.java 
b/modules/yardstick/src/main/java/org/yardstickframework/package-info.java
new file mode 100644
index 0000000..989a667
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/yardstickframework/package-info.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Additional classes for yardstick framework
+ * This package name chosen so, according to default package, yardstick
+ * searches for classes to load
+ */
+
+package org.yardstickframework;
+

http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/yardstickframework/probes/TotalTimeProbe.java
----------------------------------------------------------------------
diff --git 
a/modules/yardstick/src/main/java/org/yardstickframework/probes/TotalTimeProbe.java
 
b/modules/yardstick/src/main/java/org/yardstickframework/probes/TotalTimeProbe.java
new file mode 100644
index 0000000..e4153ab
--- /dev/null
+++ 
b/modules/yardstick/src/main/java/org/yardstickframework/probes/TotalTimeProbe.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.yardstickframework.probes;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import org.yardstickframework.BenchmarkConfiguration;
+import org.yardstickframework.BenchmarkDriver;
+import org.yardstickframework.BenchmarkExecutionAwareProbe;
+import org.yardstickframework.BenchmarkProbePoint;
+import org.yardstickframework.BenchmarkTotalsOnlyProbe;
+
+import static org.yardstickframework.BenchmarkUtils.println;
+
+/**
+ * Probe that measures total execution time of action for each thread.
+ */
+public class TotalTimeProbe implements BenchmarkExecutionAwareProbe, 
BenchmarkTotalsOnlyProbe {
+    /** Thread local agents. */
+    private volatile ThreadAgent[] agents;
+
+    /** Benchmark configuration. */
+    private BenchmarkConfiguration cfg;
+
+    /** {@inheritDoc} */
+    @Override public void beforeExecute(int threadIdx) {
+        agents[threadIdx].before();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void afterExecute(int threadIdx) {
+        agents[threadIdx].after();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void start(BenchmarkDriver drv, BenchmarkConfiguration 
cfg) throws Exception {
+        this.cfg = cfg;
+
+        agents = new ThreadAgent[cfg.threads()];
+
+        for (int thId = 0; thId < agents.length; thId++)
+            agents[thId] = new ThreadAgent();
+
+        println(cfg, "TotalTime probe started");
+    }
+
+    /** {@inheritDoc} */
+    @Override public void stop() throws Exception {
+        println(cfg, "TotalTime probe stopped");
+    }
+
+    /**
+     * This probe producing two-dimension points:
+     * Number of thread that executed {@code test} method and
+     * time it took to finish.
+     *
+     * @return meta information about each point.
+     */
+    @Override public Collection<String> metaInfo() {
+        return Arrays.asList("Thread ID", "Duration, ms");
+    }
+
+    /** {@inheritDoc} */
+    @Override public Collection<BenchmarkProbePoint> points() {
+        Collection<BenchmarkProbePoint> points = new ArrayList<>();
+
+        for (int thId = 0; thId < agents.length; thId++) {
+
+            double durationMs = agents[thId].getDurationNs() / 1000_000d;
+
+            BenchmarkProbePoint pnt = new BenchmarkProbePoint(thId, new 
double[] {durationMs});
+
+            points.add(pnt);
+        }
+
+        return points;
+    }
+
+    /**
+     * Useless for {@link BenchmarkTotalsOnlyProbe}.
+     */
+    @Override public void buildPoint(long time) {
+        // No-op
+    }
+
+    /**
+     * Measures execution time. Used in thread local context.
+     */
+    static class ThreadAgent {
+        /** Start time of {@link 
org.yardstickframework.BenchmarkDriver#test(java.util.Map)} method. */
+        private volatile long startTs;
+
+        /** Finish time. */
+        private volatile long finishTs;
+
+        /**
+         * Start measuring.
+         */
+        public void before(){
+            startTs = System.nanoTime();
+        }
+
+        /**
+         * Stop measuring.
+         */
+        public void after(){
+            finishTs = System.nanoTime();
+        }
+
+        /**
+         * @return measured duration in nanoseconds.
+         */
+        public long getDurationNs(){
+            if (finishTs == 0L && startTs != 0)
+                throw new IllegalStateException("Cannot provide results of 
TotalTime probe; " +
+                    "after() method haven't been called");
+
+            return (finishTs - startTs);
+        }
+    }
+}

Reply via email to