Changeset: fe97b880b235 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=fe97b880b235 Removed Files: monetdb5/extras/pyapi/Benchmarks/db_test.sh Modified Files: monetdb5/extras/pyapi/Benchmarks/monetdb_testing.py monetdb5/extras/pyapi/Benchmarks/pyapi_test.sh monetdb5/extras/pyapi/Benchmarks/randomstrings.c Branch: pyapi Log Message:
Added Postgres benchmark. diffs (truncated from 1189 to 300 lines): diff --git a/monetdb5/extras/pyapi/Benchmarks/db_test.sh b/monetdb5/extras/pyapi/Benchmarks/db_test.sh deleted file mode 100644 --- a/monetdb5/extras/pyapi/Benchmarks/db_test.sh +++ /dev/null @@ -1,26 +0,0 @@ - - -export PYAPI_BASE_DIR=/home/mytherin/ - -export PYAPI_TEST_DIR=$PYAPI_BASE_DIR/monetdb_pyapi_test -export PYAPI_MONETDB_DIR=$PYAPI_TEST_DIR/MonetDB-pyapi -export PYAPI_SRC_DIR=$PYAPI_MONETDB_DIR/monetdb5/extras/pyapi -export PYAPI_BUILD_DIR=$PYAPI_TEST_DIR/build -export PYAPI_OUTPUT_DIR=$PYAPI_TEST_DIR/output - -export POSTGRES_BASEDIR=$PYAPI_TEST_DIR/postgres -export POSTGRES_TEST_DIR=$POSTGRES_BASEDIR/postgres_test -export POSTGRES_BUILD_DIR=$POSTGRES_BASEDIR/build -export PGDATA=$POSTGRES_BASEDIR/postgres_data - -export POSTGRES_VERSION=9.4.4 -export POSTGRES_BASE=postgresql-$POSTGRES_VERSION -export POSTGRES_TAR_FILE=$POSTGRES_BASE.tar.gz -export POSTGRES_TAR_URL=https://ftp.postgresql.org/pub/source/v$POSTGRES_VERSION/$POSTGRES_TAR_FILE - - -function postgres_build() { - cd $PYAPI_TEST_DIR - wget $POSTGRES_TAR_URL && tar xvzf $POSTGRES_TAR_FILE && cd $POSTGRES_BASE && ./configure --prefix=$POSTGRES_BUILD_DIR --with-python && make && make install && $POSTGRES_BUILD_DIR/bin/initdb && $POSTGRES_BUILD_DIR/bin/createdb python_test -} - diff --git a/monetdb5/extras/pyapi/Benchmarks/monetdb_testing.py b/monetdb5/extras/pyapi/Benchmarks/monetdb_testing.py --- a/monetdb5/extras/pyapi/Benchmarks/monetdb_testing.py +++ b/monetdb5/extras/pyapi/Benchmarks/monetdb_testing.py @@ -92,62 +92,398 @@ import os import sys import time +c_compiler = "gcc" + # The arguments are -# [1] => Type of test ['INPUT', 'OUTPUT'] -# [2] => Output file name -# [3] => Number of tests for each value -# [4] => Mapi Port -# [5+] => List of input values +# [1] => Database to test on ("MonetDB", "Postgres") +# [2] => Type of test ['INPUT', 'OUTPUT'] +# [3] => Output file name +# [4] => Number of tests for each value +# [5] => Mapi Port +# [6+] => List of input values arguments = sys.argv -if (len(arguments) <= 5): +if (len(arguments) <= 6): print("Too few arguments provided.") quit() -output_file = os.path.join(os.getcwd(), arguments[2]) +args_input_database = arguments[1] +args_test_type = arguments[2] +args_output_file = arguments[3] +args_ntests = arguments[4] +args_port = arguments[5] +parameters_start = 6 + +output_file = os.path.join(os.getcwd(), args_output_file) temp_file = os.path.join(os.getcwd(), 'temp_output.tsv') -test_count = int(arguments[3]) -port = int(arguments[4]) -parameters_start = 5 +test_count = int(args_ntests) +port = int(args_port) max_retries = 15 max_size = 1000 random_seed = 33 -import monetdb.sql -# Try to connect to the database -# We try a couple of times because starting up the database takes some time, so it might fail the first few times -for i in range(0, max_retries): - try: - connection = monetdb.sql.connect(username="monetdb", password="monetdb", hostname="localhost",port=port,database="demo") - break - except: - time.sleep(3) - connection = None +if str(args_input_database).lower() == "monetdb": + import monetdb.sql + # Try to connect to the database + # We try a couple of times because starting up the database takes some time, so it might fail the first few times + for i in range(0, max_retries): + try: + connection = monetdb.sql.connect(username="monetdb", password="monetdb", hostname="localhost",port=port,database="demo") + break + except: + time.sleep(3) + connection = None -if connection is None: - print("Failed to connect to MonetDB Server (mserver5) in " + str(max_retries) + " attempts.") - sys.exit(1) + if connection is None: + print("Failed to connect to MonetDB Server (mserver5) in " + str(max_retries) + " attempts.") + sys.exit(1) + cursor = connection.cursor() -cursor = connection.cursor() + if str(args_test_type).lower() == "input" or str(args_test_type).lower() == "input-map" or str(args_test_type).lower() == "input-null": + # Input testing + # First create a function that generates the desired input size (in MB) and pass it to the database + if str(args_test_type).lower() == "input-null": + #if the type is input-null, we simply set all negative numbers to NULL + def generate_integers(mb, random_seed): + import random + import math + numpy.random.seed(random_seed) + byte_size = mb * 1000 * 1000 + integer_size_byte = 4 + max_int = math.pow(2,31) - 1 + min_int = -max_int + integer_count = int(byte_size / integer_size_byte) + integers = numpy.random.random_integers(min_int, max_int, integer_count).astype(numpy.int32) + return numpy.ma.masked_array(integers, numpy.less(integers, 0)) + else: + def generate_integers(mb, random_seed): + import random + import math + numpy.random.seed(random_seed) + byte_size = mb * 1000 * 1000 + integer_size_byte = 4 + max_int = math.pow(2,31) - 1 + min_int = -max_int + integer_count = int(byte_size / integer_size_byte) + return numpy.random.random_integers(min_int, max_int, integer_count).astype(numpy.int32) -if str(arguments[1]).lower() == "input" or str(arguments[1]).lower() == "input-map" or str(arguments[1]).lower() == "input-null": - # Input testing + cursor.execute(export_function(generate_integers, ['float', 'integer'], ['i integer'], table=True, test=False)) - # First create a function that generates the desired input size (in MB) and pass it to the database - if str(arguments[1]).lower() == "input-null": - #if the type is input-null, we simply set all negative numbers to NULL + # Our import test function returns a single boolean value and doesn't do anything with the actual input + # This way the input loading is the only relevant factor in running time, because the time taken for function execution/output handling is constant + def import_test(inp): + return(True) + + cursor.execute(export_function(import_test, ['integer'], ['boolean'], multithreading=str(args_test_type).lower() == "input-map")) + + f = open(output_file + '.tsv', "w+") + f.write(format_headers('[AXIS]:Data Size (MB)', '[MEASUREMENT]:Total Time (s)', '[MEASUREMENT]:PyAPI Memory (MB)', '[MEASUREMENT]:PyAPI Time (s)')) + mb = [] + for i in range(parameters_start, len(arguments)): + mb.append(float(arguments[i])) + + for size in mb: + cursor.execute('CREATE TABLE integers (i integer);') + temp_size = size + for increment in range(0, int(math.ceil(float(size) / float(max_size)))): + current_size = temp_size if temp_size < max_size else max_size + cursor.execute('INSERT INTO integers SELECT * FROM generate_integers(' + str(current_size) + ',' + str(random_seed + increment) + ');') + temp_size -= max_size + + if (str(args_test_type).lower() == "input"): + results = [] + result_file = open(temp_file, 'w+') + result_file.write("Peak Memory Usage (Bytes)\tExecution Time (s)\n") + result_file.close(); + for i in range(0,test_count): + start = time.time() + cursor.execute('select import_test(i) from integers;'); + cursor.fetchall(); + end = time.time() + list.append(results, end - start) + result_file = open(temp_file, 'r') + result_file.readline() + for result in results: + pyapi_results = result_file.readline().translate(None, '\n').split('\t') + f.write(format_output(size, result, float(pyapi_results[0]) / 1000**2, pyapi_results[1])) + f.flush() + else: + # for input-map we need to do some special analysis of the PyAPI output + # this is because every thread writes memory usage and execution time to the temp_file + # rather than just having one entry for per query + # so we have to analyse the result file for every query we perform + results = [[], [], []] + for i in range(0,test_count): + # clear the result file + result_file = open(temp_file, 'w+') + result_file.write("") + result_file.close(); + # execute the query, measure the total time + start = time.time() + cursor.execute('select import_test(i) from integers;'); + cursor.fetchall(); + end = time.time() + list.append(results[0], end - start) + # now we need to analyze the result file + # we use the total memory usage of all threads (sum) and the highest of all the execution times of the threads (max) + memory_usage = 0 + peak_execution_time = 0 + with open(temp_file, 'r') as result_file: + for line in result_file: + pyapi_results = line.translate(None, '\n').split('\t') + memory_usage = memory_usage + float(pyapi_results[0]) / 1000 ** 2 + if float(pyapi_results[1]) > peak_execution_time: peak_execution_time = float(pyapi_results[1]) + list.append(results[1], memory_usage) + list.append(results[2], peak_execution_time) + for i in range(0, len(results[0])): + f.write(format_output(size, results[0][i], results[1][i], results[2][i])) + f.flush() + cursor.execute('drop table integers;') + f.close() + + #cursor.execute('drop function generate_integers'); + #cursor.execute('drop function import_test'); + cursor.execute('rollback') + elif str(args_test_type).lower() == "output": + # output testing + + # we use a single scalar as input (the amount of MB to generate) so the input handling is fast + # we do some computation (namely creating the output array) but that should only be a single malloc call, and should be negligible compared to the copying + # that malloc call is also the same for both zero copy and copy, so it shouldn't make any difference in the comparison + def generate_output(mb): + byte_size = mb * 1000 * 1000 + integer_size_byte = 4 + integer_count = int(byte_size / integer_size_byte) + integers = numpy.zeros(integer_count, dtype=numpy.int32) + return integers + + cursor.execute(export_function(generate_output, ['float'], ['i integer'], table=True)) + + f = open(output_file + '.tsv', "w+") + f.write(format_headers('[AXIS]:Data Size (MB)', '[MEASUREMENT]:Total Time (s)', '[MEASUREMENT]:PyAPI Memory (MB)', '[MEASUREMENT]:PyAPI Time (s)')) + mb = [] + for i in range(parameters_start, len(arguments)): + mb.append(float(arguments[i])) + + for size in mb: + results = [] + result_file = open(temp_file, 'w+') + result_file.write("Peak Memory Usage (Bytes)\tExecution Time (s)\n") + result_file.close(); + for i in range(0,test_count): + start = time.time() + cursor.execute('select count(*) from generate_output(' + str(size) + ');'); + cursor.fetchall(); + end = time.time() + list.append(results, end - start) + result_file = open(temp_file, 'r') + result_file.readline() + for result in results: + pyapi_results = result_file.readline().translate(None, '\n').split('\t') + f.write(format_output(size, result, float(pyapi_results[0]) / 1000**2, pyapi_results[1])) + f.flush() + f.close() + + #cursor.execute('drop function generate_output'); + cursor.execute('rollback') + + elif str(args_test_type).lower() == "string_samelength" or str(args_test_type).lower() == "string_extremeunicode": + benchmark_dir = os.environ["PYAPI_BENCHMARKS_DIR"] + os.system("%s " % c_compiler + benchmark_dir + "/randomstrings.c -o randomstrings") + result_path = os.path.join(os.getcwd(), 'result.txt') + + if str(args_test_type).lower() == "string_samelength": + def generate_strings_samelength(length): + return 'A' * length + cursor.execute(export_function(generate_strings_samelength, ['integer'], ['i string'], table=True, test=False)) + else: + def generate_strings_samelength(length): + return unichr(0x100) * length + cursor.execute(export_function(generate_strings_samelength, ['integer'], ['i string'], table=True, test=False)) + + mb = [] + lens = [] + for i in range(parameters_start, len(arguments)): + tple = arguments[i].translate(None, '()').split(',') + mb.append(float(tple[0])) + lens.append(int(tple[1])) + + def import_test(inp): + return(True) + + cursor.execute(export_function(import_test, ['string'], ['boolean'])) + + f = open(output_file + '.tsv', "w+") + f.write(format_headers('[AXIS]:Data Size (MB)', '[AXIS]:String Length (Characters)', '[MEASUREMENT]:Total Time (s)', '[MEASUREMENT]:PyAPI Memory (MB)', '[MEASUREMENT]:PyAPI Time (s)')) + for j in range(0,len(mb)): + size = mb[j] + length = lens[j] + os.system("%s %s %s %s" % ("./randomstrings", str(size), str(length), result_path)) + cursor.execute('CREATE TABLE strings(i string);') + cursor.execute("COPY INTO strings FROM '%s';" % result_path) + cursor.execute('INSERT INTO strings SELECT * FROM generate_strings_samelength(' + str(length) + ');') + #cursor.execute('create table strings as SELECT * FROM generate_strings_samelength(\'' + result_path + '\',' + str(length) + ') with data;') + results = [] + result_file = open(temp_file, 'w+') _______________________________________________ checkin-list mailing list [email protected] https://www.monetdb.org/mailman/listinfo/checkin-list
