Getting MRQL scripts ready for prime time.
Project: http://git-wip-us.apache.org/repos/asf/vxquery/repo Commit: http://git-wip-us.apache.org/repos/asf/vxquery/commit/eefadb25 Tree: http://git-wip-us.apache.org/repos/asf/vxquery/tree/eefadb25 Diff: http://git-wip-us.apache.org/repos/asf/vxquery/diff/eefadb25 Branch: refs/heads/site Commit: eefadb254a1a30a191d01778fa62c2d7d9582593 Parents: 17bedfa Author: Preston Carman <[email protected]> Authored: Wed Oct 8 15:11:40 2014 -0700 Committer: Preston Carman <[email protected]> Committed: Wed Oct 8 15:11:40 2014 -0700 ---------------------------------------------------------------------- .../noaa-ghcn-daily/other_systems/mrql/q06.mrql | 2 +- .../other_systems/mrql_scripts/clear.sh | 9 ++++ .../mrql_scripts/run_group_test.sh | 55 ++++++++++++++++++++ .../mrql_scripts/run_mrql_tests.sh | 5 +- .../noaa-ghcn-daily/scripts/run_mrql_tests.sh | 42 --------------- 5 files changed, 69 insertions(+), 44 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/vxquery/blob/eefadb25/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql/q06.mrql ---------------------------------------------------------------------- diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql/q06.mrql b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql/q06.mrql index 583a5b9..a50dfe2 100644 --- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql/q06.mrql +++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql/q06.mrql @@ -19,7 +19,7 @@ from sensors in source(xml, args[0], {"data"}), d in sensors.date, v in sensors.value, stations in source(xml, args[1], {"station"}), - n in stations.displayName, + n in stations.displayName where text(stations.id) = text(sensors.station) and toInt(substring(text(d), 0, 4)) = 2000 and text(sensors.dataType) = "TMAX" http://git-wip-us.apache.org/repos/asf/vxquery/blob/eefadb25/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear.sh ---------------------------------------------------------------------- diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear.sh index da7cabe..b775de2 100755 --- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear.sh +++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear.sh @@ -16,4 +16,13 @@ # See the License for the specific language governing permissions and # limitations under the License. # + +# Hadoop data reset hadoop namenode -format + +# Remove data +rm -rf disk1/hadoop/data +rm -rf disk2/hadoop/data +rm -rf disk1/hadoop/tmp +rm -rf disk1/hadoop/logs + http://git-wip-us.apache.org/repos/asf/vxquery/blob/eefadb25/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh ---------------------------------------------------------------------- diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh new file mode 100755 index 0000000..c34ec95 --- /dev/null +++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +NODES=2 +REPEAT=1 + +# Start Hadoop +sh saved/hadoop/hadoop-1.2.1/bin/start-all.sh + +# Prepare hadoop file system +hadoop fs -mkdir all +hadoop fs -mkdir all/sensors +hadoop fs -mkdir all/stations + + +# Upload test data +n=0 +while [ ${n} -lt ${NODES} ]; +do + # Add each sensor block + cp saved/backups/mr/all_sensors_${n}.xml.gz disk1/hadoop/upload/ + gunzip disk1/hadoop/upload/all_sensors_${n}.xml.gz + hadoop fs -copyFromLocal disk1/hadoop/upload/all_sensors_${n}.xml all/sensors + rm -f disk1/hadoop/upload/all_sensors_${n}.xml + + # Add each station block + cp saved/backups/mr/all_stations_${n}.xml.gz disk1/hadoop/upload/ + gunzip disk1/hadoop/upload/all_stations_${n}.xml.gz + hadoop fs -copyFromLocal disk1/hadoop/upload/all_stations_${n}.xml all/stations + rm -f disk1/hadoop/upload/all_stations_${n}.xml +done + + +# Start test +sh vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_mrql_tests.sh vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql/ ${NODES} ${REPEAT} + + +# Stop Hadoop +sh saved/hadoop/hadoop-1.2.1/bin/stop-all.sh \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/eefadb25/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_mrql_tests.sh ---------------------------------------------------------------------- diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_mrql_tests.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_mrql_tests.sh index 10ab4d9..1e512e1 100755 --- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_mrql_tests.sh +++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_mrql_tests.sh @@ -25,11 +25,14 @@ REPEAT=${3} DATASET="all" +# Make log folder +mkdir -p ~/disk1/weather_data/mrql/query_logs/${NODES}nodes/ + for j in $(find ${1} -name '*q??.mrql') do date echo "Running MRQL query: ${j}" - time for i in {1..${REPEAT}}; do ~/mrql/incubator-mrql/bin/mrql -dist -nodes ${NODES} ${j} ${DATASET}/sensors.xml ${DATASET}/stations.xml >> ~/disk1/weather_data/mrql/query_logs/$(basename "${j}").log 2>&1; done; + time for i in {1..${REPEAT}}; do ~/mrql/incubator-mrql/bin/mrql -dist -nodes ${NODES} ${j} ${DATASET}/sensors/ ${DATASET}/stations/ >> ~/disk1/weather_data/mrql/query_logs/${NODES}nodes/$(basename "${j}").log 2>&1; done; done http://git-wip-us.apache.org/repos/asf/vxquery/blob/eefadb25/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/run_mrql_tests.sh ---------------------------------------------------------------------- diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/run_mrql_tests.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/run_mrql_tests.sh deleted file mode 100755 index a6788be..0000000 --- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/run_mrql_tests.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -export JAVA_HOME=/home/ecarm002/java/jdk1.6.0_45 -REPEAT=${1} -DATASET="hcn" - -for n in `seq 0 7` -#for n in 0 -do - date - echo "Running q0${n} on ${DATASET} for MRQL." - time for i in {1..${REPEAT}}; do ~/mrql/incubator-mrql/bin/mrql -dist -nodes 5 ~/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_${DATASET}/q0${n}.mrql >> weather_data/mrql/query_logs/${DATASET}/q0${n}.mrql.log 2>&1; done; -done - -if which programname >/dev/null; -then - echo "Sending out e-mail notification." - SUBJECT="MRQL Tests Finished (${DATASET})" - EMAIL="[email protected]" - /bin/mail -s "${SUBJECT}" "${EMAIL}" <<EOM - Completed all MRQL tests on ${DATASET}. - EOM -else - echo "No mail command to use." -fi; \ No newline at end of file
