Added a background processes to help with parallelizing the data loading.
Project: http://git-wip-us.apache.org/repos/asf/vxquery/repo Commit: http://git-wip-us.apache.org/repos/asf/vxquery/commit/31b3f4d9 Tree: http://git-wip-us.apache.org/repos/asf/vxquery/tree/31b3f4d9 Diff: http://git-wip-us.apache.org/repos/asf/vxquery/diff/31b3f4d9 Branch: refs/heads/site Commit: 31b3f4d985c6344d1ccc15ce57fb57a1cb2e3011 Parents: eefadb2 Author: Preston Carman <[email protected]> Authored: Thu Oct 9 12:19:27 2014 -0700 Committer: Preston Carman <[email protected]> Committed: Thu Oct 9 12:19:27 2014 -0700 ---------------------------------------------------------------------- .../mrql_scripts/load_node_file.sh | 37 ++++++++++++++++++++ .../mrql_scripts/run_group_test.sh | 15 +++----- 2 files changed, 41 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/vxquery/blob/31b3f4d9/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/load_node_file.sh ---------------------------------------------------------------------- diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/load_node_file.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/load_node_file.sh new file mode 100755 index 0000000..048274f --- /dev/null +++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/load_node_file.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +if [ -z "${1}" ] +then + echo "Please enter the node number." + exit +fi + + +# Add each sensor block +cp saved/backups/mr/all_sensors_${1}.xml.gz disk1/hadoop/upload/ +gunzip disk1/hadoop/upload/all_sensors_${1}.xml.gz +hadoop fs -copyFromLocal disk1/hadoop/upload/all_sensors_${1}.xml all/sensors +rm -f disk1/hadoop/upload/all_sensors_${1}.xml + +# Add each station block +cp saved/backups/mr/all_stations_${1}.xml.gz disk1/hadoop/upload/ +gunzip disk1/hadoop/upload/all_stations_${1}.xml.gz +hadoop fs -copyFromLocal disk1/hadoop/upload/all_stations_${1}.xml all/stations +rm -f disk1/hadoop/upload/all_stations_${1}.xml http://git-wip-us.apache.org/repos/asf/vxquery/blob/31b3f4d9/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh ---------------------------------------------------------------------- diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh index c34ec95..60dc255 100755 --- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh +++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh @@ -33,19 +33,12 @@ hadoop fs -mkdir all/stations n=0 while [ ${n} -lt ${NODES} ]; do - # Add each sensor block - cp saved/backups/mr/all_sensors_${n}.xml.gz disk1/hadoop/upload/ - gunzip disk1/hadoop/upload/all_sensors_${n}.xml.gz - hadoop fs -copyFromLocal disk1/hadoop/upload/all_sensors_${n}.xml all/sensors - rm -f disk1/hadoop/upload/all_sensors_${n}.xml - - # Add each station block - cp saved/backups/mr/all_stations_${n}.xml.gz disk1/hadoop/upload/ - gunzip disk1/hadoop/upload/all_stations_${n}.xml.gz - hadoop fs -copyFromLocal disk1/hadoop/upload/all_stations_${n}.xml all/stations - rm -f disk1/hadoop/upload/all_stations_${n}.xml + sh vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/load_node_file.sh ${n} & done +# After all files have been uploaded, continue. +wait + # Start test sh vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_mrql_tests.sh vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql/ ${NODES} ${REPEAT}
