More tweaking to get the MRQL benchmark to work efficiently.
Project: http://git-wip-us.apache.org/repos/asf/vxquery/repo Commit: http://git-wip-us.apache.org/repos/asf/vxquery/commit/7f06298f Tree: http://git-wip-us.apache.org/repos/asf/vxquery/tree/7f06298f Diff: http://git-wip-us.apache.org/repos/asf/vxquery/diff/7f06298f Branch: refs/heads/site Commit: 7f06298fa350ca8b0a81bb9ffb78aa29b26368c1 Parents: 31b3f4d Author: Preston Carman <[email protected]> Authored: Thu Oct 9 15:29:16 2014 -0700 Committer: Preston Carman <[email protected]> Committed: Thu Oct 9 15:29:16 2014 -0700 ---------------------------------------------------------------------- .../other_systems/mrql_scripts/README.md | 23 ++++++++++++++++++++ .../other_systems/mrql_scripts/clear.sh | 4 ---- .../mrql_scripts/load_node_file.sh | 17 ++++++++------- .../mrql_scripts/run_group_test.sh | 22 +++++++++++++------ .../other_systems/mrql_scripts/start.sh | 20 ----------------- .../other_systems/mrql_scripts/stop.sh | 20 ----------------- 6 files changed, 47 insertions(+), 59 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/vxquery/blob/7f06298f/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/README.md ---------------------------------------------------------------------- diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/README.md b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/README.md new file mode 100644 index 0000000..53a7ecf --- /dev/null +++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/README.md @@ -0,0 +1,23 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + + +clear.sh + +hadoop namenode -format + +run_group_test.sh \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/7f06298f/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear.sh ---------------------------------------------------------------------- diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear.sh index b775de2..9a912b2 100755 --- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear.sh +++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear.sh @@ -17,12 +17,8 @@ # limitations under the License. # -# Hadoop data reset -hadoop namenode -format - # Remove data rm -rf disk1/hadoop/data rm -rf disk2/hadoop/data rm -rf disk1/hadoop/tmp rm -rf disk1/hadoop/logs - http://git-wip-us.apache.org/repos/asf/vxquery/blob/7f06298f/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/load_node_file.sh ---------------------------------------------------------------------- diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/load_node_file.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/load_node_file.sh index 048274f..ead0902 100755 --- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/load_node_file.sh +++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/load_node_file.sh @@ -23,15 +23,16 @@ then exit fi +echo "Loading node ${1} data file in to cluster." # Add each sensor block -cp saved/backups/mr/all_sensors_${1}.xml.gz disk1/hadoop/upload/ -gunzip disk1/hadoop/upload/all_sensors_${1}.xml.gz -hadoop fs -copyFromLocal disk1/hadoop/upload/all_sensors_${1}.xml all/sensors -rm -f disk1/hadoop/upload/all_sensors_${1}.xml +cp saved/backups/mr/all_sensors_${1}.xml.gz disk1/hadoop/ +gunzip disk1/hadoop/all_sensors_${1}.xml.gz +hadoop fs -copyFromLocal disk1/hadoop/all_sensors_${1}.xml all/sensors +rm -f disk1/hadoop/all_sensors_${1}.xml # Add each station block -cp saved/backups/mr/all_stations_${1}.xml.gz disk1/hadoop/upload/ -gunzip disk1/hadoop/upload/all_stations_${1}.xml.gz -hadoop fs -copyFromLocal disk1/hadoop/upload/all_stations_${1}.xml all/stations -rm -f disk1/hadoop/upload/all_stations_${1}.xml +cp saved/backups/mr/all_stations_${1}.xml.gz disk1/hadoop/ +gunzip disk1/hadoop/all_stations_${1}.xml.gz +hadoop fs -copyFromLocal disk1/hadoop/all_stations_${1}.xml all/stations +rm -f disk1/hadoop/all_stations_${1}.xml http://git-wip-us.apache.org/repos/asf/vxquery/blob/7f06298f/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh ---------------------------------------------------------------------- diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh index 60dc255..f42a451 100755 --- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh +++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh @@ -17,28 +17,36 @@ # limitations under the License. # -NODES=2 +if [ -z "${1}" ] +then + echo "Please enter the number of nodes." + exit +fi + +NODES=${1} REPEAT=1 # Start Hadoop sh saved/hadoop/hadoop-1.2.1/bin/start-all.sh +sleep 10 + # Prepare hadoop file system hadoop fs -mkdir all +hadoop fs -ls hadoop fs -mkdir all/sensors hadoop fs -mkdir all/stations +hadoop fs -ls all # Upload test data -n=0 -while [ ${n} -lt ${NODES} ]; +COUNTER=0 +while [ ${COUNTER} -lt ${NODES} ]; do - sh vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/load_node_file.sh ${n} & + sh vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/load_node_file.sh ${COUNTER} + let COUNTER=COUNTER+1 done -# After all files have been uploaded, continue. -wait - # Start test sh vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_mrql_tests.sh vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql/ ${NODES} ${REPEAT} http://git-wip-us.apache.org/repos/asf/vxquery/blob/7f06298f/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/start.sh ---------------------------------------------------------------------- diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/start.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/start.sh deleted file mode 100755 index a1766c9..0000000 --- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/start.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -bin/start-all.sh http://git-wip-us.apache.org/repos/asf/vxquery/blob/7f06298f/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/stop.sh ---------------------------------------------------------------------- diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/stop.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/stop.sh deleted file mode 100755 index e49d818..0000000 --- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/stop.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -bin/stop-all.sh
