incubator-hawq git commit: HAWQ-1476. Augment enable-ranger-plugin.sh to support kerberos. (close #1248)
Repository: incubator-hawq
Updated Branches:
refs/heads/master abd74ed1c -> ed5029c76
HAWQ-1476. Augment enable-ranger-plugin.sh to support kerberos.
(close #1248)
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/ed5029c7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/ed5029c7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/ed5029c7
Branch: refs/heads/master
Commit: ed5029c76515d36cae0c41e96d19eee0b4710561
Parents: abd74ed
Author: stanlyxiang
Authored: Thu Jun 1 15:41:55 2017 +0800
Committer: Lili Ma
Committed: Thu Jun 1 16:34:03 2017 +0800
--
ranger-plugin/conf/ranger-servicedef-hawq.json | 2 +-
ranger-plugin/scripts/enable-ranger-plugin.sh | 28 -
2 files changed, 28 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ed5029c7/ranger-plugin/conf/ranger-servicedef-hawq.json
--
diff --git a/ranger-plugin/conf/ranger-servicedef-hawq.json
b/ranger-plugin/conf/ranger-servicedef-hawq.json
index 172eb3a..b8ce53e 100644
--- a/ranger-plugin/conf/ranger-servicedef-hawq.json
+++ b/ranger-plugin/conf/ranger-servicedef-hawq.json
@@ -244,7 +244,7 @@
"name": "authentication",
"type": "enum",
"subType": "authType",
- "mandatory": false,
+ "mandatory": true,
"validationRegEx": "",
"validationMessage": "",
"uiHint": "",
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ed5029c7/ranger-plugin/scripts/enable-ranger-plugin.sh
--
diff --git a/ranger-plugin/scripts/enable-ranger-plugin.sh
b/ranger-plugin/scripts/enable-ranger-plugin.sh
index 5b702c9..0e839b7 100755
--- a/ranger-plugin/scripts/enable-ranger-plugin.sh
+++ b/ranger-plugin/scripts/enable-ranger-plugin.sh
@@ -20,7 +20,7 @@
#
function usage() {
- echo "USAGE: enable-ranger-plugin.sh -r ranger_host:ranger_port -u
ranger_user -p ranger_password [-h hawq_host:hawq_port] -w hawq_user -q
hawq_password"
+ echo "USAGE: enable-ranger-plugin.sh -r ranger_host:ranger_port -u
ranger_user -p ranger_password [-h hawq_host:hawq_port -t
lookup_authentication_type -s hawq_kerberos_service_name] -w hawq_user -q
hawq_password"
exit 1
}
@@ -131,6 +131,18 @@ function get_hawq_password() {
done
}
+function get_hawq_kerberos_service_name() {
+ if [[ -z "$HAWQ_KERBEROS_SERVICE_NAME" ]]; then
+HAWQ_KERBEROS_SERVICE_NAME="postgres"
+ fi
+}
+
+function get_lookup_authentication_type() {
+ if [[ -z "$LOOKUP_AUTHENTICATION_TYPE" ]]; then
+LOOKUP_AUTHENTICATION_TYPE="simple"
+ fi
+}
+
function parse_params() {
while [[ $# -gt 0 ]]
do
@@ -160,6 +172,14 @@ function parse_params() {
HAWQ_PASSWORD="$2"
shift
;;
+ -s)
+HAWQ_KERBEROS_SERVICE_NAME="$2"
+shift
+;;
+ -t)
+LOOKUP_AUTHENTICATION_TYPE="$2"
+shift
+;;
*)
usage
;;
@@ -175,6 +195,8 @@ function validate_params() {
get_hawq_url
get_hawq_user
get_hawq_password
+ get_hawq_kerberos_service_name
+ get_lookup_authentication_type
echo "RANGER URL = ${RANGER_URL}"
echo "RANGER User = ${RANGER_USER}"
echo "RANGER Password = $(mask ${RANGER_PASSWORD})"
@@ -182,6 +204,8 @@ function validate_params() {
echo "HAWQ PORT = ${HAWQ_PORT}"
echo "HAWQ User = ${HAWQ_USER}"
echo "HAWQ Password = $(mask ${HAWQ_PASSWORD})"
+ echo "HAWQ Lookup Authentication Type = ${LOOKUP_AUTHENTICATION_TYPE}"
+ echo "HAWQ Kerberos Service Name = ${HAWQ_KERBEROS_SERVICE_NAME}"
}
function check_hawq_service_definition() {
@@ -217,6 +241,8 @@ function create_hawq_service_instance() {
\"isEnabled\":true,
\"configs\":{\"username\":\"${HAWQ_USER}\",
\"password\":\"${HAWQ_PASSWORD}\",
+
\"authentication\":\"${LOOKUP_AUTHENTICATION_TYPE}\",
+ \"principal\":\"${HAWQ_KERBEROS_SERVICE_NAME}\",
\"hostname\":\"${HAWQ_HOST}\",
\"port\":\"${HAWQ_PORT}\"}}"
incubator-hawq git commit: HAWQ-1426. Fix bug that hawq extract meets error after the table was reorganized
Repository: incubator-hawq
Updated Branches:
refs/heads/master 1a835a945 -> f04c9673f
HAWQ-1426. Fix bug that hawq extract meets error after the table was reorganized
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/f04c9673
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/f04c9673
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/f04c9673
Branch: refs/heads/master
Commit: f04c9673ff094a7b794931b008d72ddd5bc4868a
Parents: 1a835a9
Author: Chunling Wang
Authored: Fri Apr 7 16:28:22 2017 +0800
Committer: Lili Ma
Committed: Mon Apr 10 10:32:16 2017 +0800
--
.../ManagementTool/test_hawq_extract.cpp| 103 +++
.../feature/ManagementTool/test_hawq_extract.h | 46 +
tools/bin/hawqextract | 27 -
3 files changed, 172 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f04c9673/src/test/feature/ManagementTool/test_hawq_extract.cpp
--
diff --git a/src/test/feature/ManagementTool/test_hawq_extract.cpp
b/src/test/feature/ManagementTool/test_hawq_extract.cpp
new file mode 100644
index 000..5b6e5ee
--- /dev/null
+++ b/src/test/feature/ManagementTool/test_hawq_extract.cpp
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include
+
+#include "lib/command.h"
+#include "lib/sql_util.h"
+#include "lib/string_util.h"
+#include "lib/hdfs_config.h"
+#include "lib/file_replace.h"
+#include "test_hawq_extract.h"
+
+#include "gtest/gtest.h"
+
+using std::string;
+using hawq::test::SQLUtility;
+using hawq::test::Command;
+using hawq::test::HdfsConfig;
+
+TEST_F(TestHawqExtract, TestExtractAfterReorganize) {
+SQLUtility util;
+util.execute("drop table if exists table_extract_ao;");
+util.execute("drop table if exists table_extract_parquet;");
+util.execute("drop table if exists table_extract_ao_new;");
+util.execute("drop table if exists table_extract_parquet_new;");
+// create an ao and a parquet table and insert data
+util.execute("CREATE TABLE table_extract_ao(id int);");
+util.execute("CREATE TABLE table_extract_parquet(id int) WITH
(APPENDONLY=true, ORIENTATION=parquet);");
+util.execute("insert into table_extract_ao values(1),(2),(3);");
+util.execute("insert into table_extract_parquet values(1),(2),(3);");
+
+// reorganize table
+util.execute("alter table table_extract_ao set with (reorganize=true);");
+util.execute("alter table table_extract_parquet set with
(reorganize=true);");
+
+// extract table to .yml
+EXPECT_EQ(0, Command::getCommandStatus("hawq extract -d " + (string)
HAWQ_DB + " -o table_extract_ao.yml
testhawqextract_testextractafterreorganize.table_extract_ao"));
+EXPECT_EQ(0, Command::getCommandStatus("hawq extract -d " + (string)
HAWQ_DB + " -o table_extract_parquet.yml
testhawqextract_testextractafterreorganize.table_extract_parquet"));
+
+// register .yml to new table
+EXPECT_EQ(0, Command::getCommandStatus("hawq register -d " + (string)
HAWQ_DB + " -c table_extract_ao.yml
testhawqextract_testextractafterreorganize.table_extract_ao_new"));
+EXPECT_EQ(0, Command::getCommandStatus("hawq register -d " + (string)
HAWQ_DB + " -c table_extract_parquet.yml
testhawqextract_testextractafterreorganize.table_extract_parquet_new"));
+util.query("select * from table_extract_ao_new;", 3);
+util.query("select * from table_extract_parquet_new;", 3);
+
+EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("rm -rf
table_extract_ao.yml")));
+EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("rm -rf
table_extract_parquet.yml")));
+util.execute("drop table table_extract_ao;");
+util.execute("drop table table_extract_parquet;");
+util.execute("drop table table_extract_ao_new;");
+util.execute("drop table table_extract_parquet_new;");
+}
+
+TEST_F(TestHawqExtract, TestExtractA
incubator-hawq git commit: HAWQ-1418. Move print executing command after setup logging
Repository: incubator-hawq
Updated Branches:
refs/heads/master 46a9621bb -> 1e9ea9945
HAWQ-1418. Move print executing command after setup logging
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/1e9ea994
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/1e9ea994
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/1e9ea994
Branch: refs/heads/master
Commit: 1e9ea99458a251c3084e69147bdde2c19e702049
Parents: 46a9621
Author: Chunling Wang
Authored: Tue Mar 28 18:15:00 2017 +0800
Committer: Chunling Wang
Committed: Tue Mar 28 18:17:07 2017 +0800
--
tools/bin/hawqregister | 10 +-
1 file changed, 5 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1e9ea994/tools/bin/hawqregister
--
diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister
index e5c3f6a..ff2bb96 100755
--- a/tools/bin/hawqregister
+++ b/tools/bin/hawqregister
@@ -34,17 +34,17 @@ except ImportError, e:
sys.stderr.write('Cannot import module, please check that you have source
greenplum_path.sh\n')
sys.exit(2)
+# setup logging
+logger = get_default_logger()
+EXECNAME = os.path.split(__file__)[-1]
+setup_tool_logging(EXECNAME, getLocalHostname(), getUserName())
+
# print executing command
cmd = "Executing Command: ";
for arg in sys.argv:
cmd += arg + " "
logger.info(cmd);
-# setup logging
-logger = get_default_logger()
-EXECNAME = os.path.split(__file__)[-1]
-setup_tool_logging(EXECNAME, getLocalHostname(), getUserName())
-
def option_parser():
'''option parser'''
parser = OptParser(option_class=OptChecker,
incubator-hawq git commit: HAWQ-1140. Rename yml file name and table name in TestUsage2Case1ErrorHashTableRegistry (close #1200)
Repository: incubator-hawq
Updated Branches:
refs/heads/master bbdcf95a0 -> 46a9621bb
HAWQ-1140. Rename yml file name and table name in
TestUsage2Case1ErrorHashTableRegistry
(close #1200)
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/46a9621b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/46a9621b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/46a9621b
Branch: refs/heads/master
Commit: 46a9621bb4b02e6932feb5bb0c07193e2812e66b
Parents: bbdcf95
Author: Chunling Wang
Authored: Tue Mar 28 17:31:55 2017 +0800
Committer: Lili Ma
Committed: Tue Mar 28 17:42:05 2017 +0800
--
.../test_hawq_register_usage2_case1.cpp | 90 ++--
1 file changed, 45 insertions(+), 45 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/46a9621b/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp
--
diff --git
a/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp
b/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp
index 3623528..997a549 100644
--- a/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp
+++ b/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp
@@ -214,51 +214,51 @@ TEST_F(TestHawqRegister,
TestUsage2Case1HDFSFilePathContainErrorSymbol) {
TEST_F(TestHawqRegister, TestUsage2Case1ErrorHashTableRegistry) {
SQLUtility util;
-util.execute("drop table if exists t_1_1;");
-util.execute("drop table if exists t_1_2;");
-util.execute("drop table if exists t_1_3;");
-util.execute("drop table if exists t_2;");
-util.execute("drop table if exists nt_1;");
-util.execute("drop table if exists nt_2;");
-
-util.execute("create table t_1_1(i int, j int, k int) with
(appendonly=true, orientation=row, bucketnum=12) distributed by (i, j);");
-util.execute("insert into t_1_1 select generate_series(1, 100);");
-util.query("select * from t_1_1;", 100);
-util.execute("create table t_1_2(i int, j int, k int) with
(appendonly=true, orientation=row) distributed by (i);");
-util.execute("insert into t_1_2 select generate_series(1, 100);");
-util.query("select * from t_1_2;", 100);
-util.execute("create table t_1_3(i int, j int, k int) with
(appendonly=true, orientation=row) distributed randomly;");
-util.execute("insert into t_1_3 select generate_series(1, 100);");
-util.query("select * from t_1_3;", 100);
-util.execute("create table t_2(i int, j int, k int) with (appendonly=true,
orientation=row) distributed by (i);");
-util.execute("insert into t_2 select generate_series(1, 100);");
-util.query("select * from t_2;", 100);
-util.execute("create table nt_1(i int, j int, k int) with
(appendonly=true, orientation=row) distributed by (i, j);");
-util.execute("insert into nt_1 select generate_series(1, 100);");
-util.query("select * from nt_1;", 100);
-util.execute("create table nt_2(i int, j int, k int) with
(appendonly=true, orientation=row) distributed by (j);");
-util.execute("insert into nt_2 select generate_series(1, 100);");
-util.query("select * from nt_2;", 100);
-
-EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hawq
extract -d %s -o t_1_1.yml
testhawqregister_testusage2case1errorhashtableregistry.t_1_1", HAWQ_DB)));
-EXPECT_EQ(1, Command::getCommandStatus(hawq::test::stringFormat("hawq
register -d %s -c t_1_1.yml
testhawqregister_testusage2case1errorhashtableregistry.nt_1", HAWQ_DB)));
-EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hawq
extract -d %s -o t_1_2.yml
testhawqregister_testusage2case1errorhashtableregistry.t_1_2", HAWQ_DB)));
-EXPECT_EQ(1, Command::getCommandStatus(hawq::test::stringFormat("hawq
register -d %s -c t_1_2.yml
testhawqregister_testusage2case1errorhashtableregistry.nt_1", HAWQ_DB)));
-EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hawq
extract -d %s -o t_1_3.yml
testhawqregister_testusage2case1errorhashtableregistry.t_1_3", HAWQ_DB)));
-EXPECT_EQ(1, Command::getCommandStatus(hawq::test::stringFormat("hawq
register -d %s -c t_1_3.yml
testhawqregister_testusage2case1errorhashtableregistry.nt_1", HAWQ_DB)));
-EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hawq
extract -d %s -o t_2.yml
testhawqregister_testusage2case1errorhashtableregistry.t_2", HAWQ_DB)));
-EXPECT_EQ(1, Command::getCommandStatus(hawq::test::stringFormat("hawq
register -d %s -c t_2.yml
testhawqregister_testusage2case1errorhashtableregistry.nt_2", HAWQ_DB)));
-
-EXPECT_EQ(0, Command::getCommandStatus("rm -rf t_1_1.yml"));
-EXPECT_EQ(0, Command::getCommandStatus(
incubator-hawq git commit: HAWQ-1418. Print executing command for hawq register (close #1199)
Repository: incubator-hawq
Updated Branches:
refs/heads/master c2ba42026 -> bbdcf95a0
HAWQ-1418. Print executing command for hawq register
(close #1199)
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/bbdcf95a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/bbdcf95a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/bbdcf95a
Branch: refs/heads/master
Commit: bbdcf95a02a93c12ef1e786437cdad77c515ffa7
Parents: c2ba420
Author: Chunling Wang
Authored: Tue Mar 28 17:08:35 2017 +0800
Committer: Lili Ma
Committed: Tue Mar 28 17:20:29 2017 +0800
--
tools/bin/hawqregister | 6 ++
1 file changed, 6 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bbdcf95a/tools/bin/hawqregister
--
diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister
index 39d0c23..e5c3f6a 100755
--- a/tools/bin/hawqregister
+++ b/tools/bin/hawqregister
@@ -34,6 +34,12 @@ except ImportError, e:
sys.stderr.write('Cannot import module, please check that you have source
greenplum_path.sh\n')
sys.exit(2)
+# print executing command
+cmd = "Executing Command: ";
+for arg in sys.argv:
+cmd += arg + " "
+logger.info(cmd);
+
# setup logging
logger = get_default_logger()
EXECNAME = os.path.split(__file__)[-1]
incubator-hawq git commit: Revert "HAWQ-1415. Set the default_value of JAVA_HOME for running RPS"
Repository: incubator-hawq
Updated Branches:
refs/heads/master 22306b09a -> c2ba42026
Revert "HAWQ-1415. Set the default_value of JAVA_HOME for running RPS"
This reverts commit a803aab4eec4a71e83d2bd18bb72b93ba4be31a2.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/c2ba4202
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/c2ba4202
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/c2ba4202
Branch: refs/heads/master
Commit: c2ba42026d4d462540e50c400f025d32c1ce735f
Parents: 22306b0
Author: Lili Ma
Authored: Tue Mar 28 14:30:35 2017 +0800
Committer: Lili Ma
Committed: Tue Mar 28 14:34:25 2017 +0800
--
ranger-plugin/conf/rps.properties | 3 ---
ranger-plugin/scripts/enable-ranger-plugin.sh | 21 +
ranger-plugin/scripts/rps.sh | 1 -
3 files changed, 1 insertion(+), 24 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c2ba4202/ranger-plugin/conf/rps.properties
--
diff --git a/ranger-plugin/conf/rps.properties
b/ranger-plugin/conf/rps.properties
index 7565885..81a6986 100644
--- a/ranger-plugin/conf/rps.properties
+++ b/ranger-plugin/conf/rps.properties
@@ -36,6 +36,3 @@ RPS_HEAP_SIZE=128m
# use CATALINA_HOME in default and use /usr/lib/bigtop-tomcat if CATALINA_HOME
not set
CATALINA_HOME=${CATALINA_HOME:-/usr/lib/bigtop-tomcat}
-
-# use JAVA_HOME in default and use /usr/java/default if JAVA_HOME not set
-JAVA_HOME=${JAVA_HOME:-/usr/java/default}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c2ba4202/ranger-plugin/scripts/enable-ranger-plugin.sh
--
diff --git a/ranger-plugin/scripts/enable-ranger-plugin.sh
b/ranger-plugin/scripts/enable-ranger-plugin.sh
index febaf81..05a0b1d 100755
--- a/ranger-plugin/scripts/enable-ranger-plugin.sh
+++ b/ranger-plugin/scripts/enable-ranger-plugin.sh
@@ -150,7 +150,7 @@ function validate_params() {
get_hawq_password
echo "RANGER URL = ${RANGER_URL}"
echo "RANGER User = ${RANGER_USER}"
- echo "RANGER Password = $(mask ${RANGER_PASSWORD})"
+ echo "RANGER Password = $(mask ${RANGER_PASSWORD})"
echo "HAWQ HOST = ${HAWQ_HOST}"
echo "HAWQ PORT = ${HAWQ_PORT}"
echo "HAWQ User = ${HAWQ_USER}"
@@ -211,24 +211,6 @@ function update_ranger_url() {
echo "Updated POLICY_MGR_URL to ${policy_mgr_url} in ${prop_file}"
}
-function update_java_home() {
- local java_base="/usr/jdk64"
- if [[ -d ${java_base} ]]; then
-local DIR_NAME=$(ls ${java_base} | sort -r | head -1)
-
-if [[ ${DIR_NAME} ]]; then
- JAVA_HOME_DIR="${java_base}/${DIR_NAME}"
- local prop_file=$(dirname ${SCRIPT_DIR})/etc/rps.properties
- sed -i -e "s|/usr/java/default|${JAVA_HOME_DIR}|g" ${prop_file}
- echo "Updated default value of JAVA_HOME to ${JAVA_HOME_DIR} in
${prop_file}"
-fi
- fi
-
- if [[ ! ${JAVA_HOME_DIR} && ! ${JAVA_HOME} ]]; then
-echo "Unable to locate JAVA_HOME on this machine. Please modify the
default value of JAVA_HOME in ${prop_file}."
- fi
-}
-
main() {
if [[ $# -lt 1 ]]; then
usage
@@ -239,6 +221,5 @@ main() {
create_hawq_service_definition
create_hawq_service_instance
update_ranger_url
- update_java_home
}
main "$@"
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c2ba4202/ranger-plugin/scripts/rps.sh
--
diff --git a/ranger-plugin/scripts/rps.sh b/ranger-plugin/scripts/rps.sh
index 70aa547..fa268c9 100755
--- a/ranger-plugin/scripts/rps.sh
+++ b/ranger-plugin/scripts/rps.sh
@@ -32,7 +32,6 @@ BASEDIR=$( dirname ${CWDIR} )
# read properties from the file
source ${BASEDIR}/etc/rps.properties
-export JAVA_HOME
export CATALINA_HOME
export CATALINA_BASE=${BASEDIR}/plugin-service
export CATALINA_PID=${CATALINA_BASE}/work/rps.pid
incubator-hawq git commit: HAWQ-1415. Set the default_value of JAVA_HOME for running RPS (close #1197)
Repository: incubator-hawq
Updated Branches:
refs/heads/master 34cea88d1 -> a803aab4e
HAWQ-1415. Set the default_value of JAVA_HOME for running RPS
(close #1197)
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/a803aab4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/a803aab4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/a803aab4
Branch: refs/heads/master
Commit: a803aab4eec4a71e83d2bd18bb72b93ba4be31a2
Parents: 34cea88
Author: ljainpivotalio
Authored: Mon Mar 27 16:28:42 2017 -0700
Committer: Lili Ma
Committed: Tue Mar 28 10:40:04 2017 +0800
--
ranger-plugin/conf/rps.properties | 3 +++
ranger-plugin/scripts/enable-ranger-plugin.sh | 21 -
ranger-plugin/scripts/rps.sh | 1 +
3 files changed, 24 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a803aab4/ranger-plugin/conf/rps.properties
--
diff --git a/ranger-plugin/conf/rps.properties
b/ranger-plugin/conf/rps.properties
index 81a6986..7565885 100644
--- a/ranger-plugin/conf/rps.properties
+++ b/ranger-plugin/conf/rps.properties
@@ -36,3 +36,6 @@ RPS_HEAP_SIZE=128m
# use CATALINA_HOME in default and use /usr/lib/bigtop-tomcat if CATALINA_HOME
not set
CATALINA_HOME=${CATALINA_HOME:-/usr/lib/bigtop-tomcat}
+
+# use JAVA_HOME in default and use /usr/java/default if JAVA_HOME not set
+JAVA_HOME=${JAVA_HOME:-/usr/java/default}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a803aab4/ranger-plugin/scripts/enable-ranger-plugin.sh
--
diff --git a/ranger-plugin/scripts/enable-ranger-plugin.sh
b/ranger-plugin/scripts/enable-ranger-plugin.sh
index 05a0b1d..febaf81 100755
--- a/ranger-plugin/scripts/enable-ranger-plugin.sh
+++ b/ranger-plugin/scripts/enable-ranger-plugin.sh
@@ -150,7 +150,7 @@ function validate_params() {
get_hawq_password
echo "RANGER URL = ${RANGER_URL}"
echo "RANGER User = ${RANGER_USER}"
- echo "RANGER Password = $(mask ${RANGER_PASSWORD})"
+ echo "RANGER Password = $(mask ${RANGER_PASSWORD})"
echo "HAWQ HOST = ${HAWQ_HOST}"
echo "HAWQ PORT = ${HAWQ_PORT}"
echo "HAWQ User = ${HAWQ_USER}"
@@ -211,6 +211,24 @@ function update_ranger_url() {
echo "Updated POLICY_MGR_URL to ${policy_mgr_url} in ${prop_file}"
}
+function update_java_home() {
+ local java_base="/usr/jdk64"
+ if [[ -d ${java_base} ]]; then
+local DIR_NAME=$(ls ${java_base} | sort -r | head -1)
+
+if [[ ${DIR_NAME} ]]; then
+ JAVA_HOME_DIR="${java_base}/${DIR_NAME}"
+ local prop_file=$(dirname ${SCRIPT_DIR})/etc/rps.properties
+ sed -i -e "s|/usr/java/default|${JAVA_HOME_DIR}|g" ${prop_file}
+ echo "Updated default value of JAVA_HOME to ${JAVA_HOME_DIR} in
${prop_file}"
+fi
+ fi
+
+ if [[ ! ${JAVA_HOME_DIR} && ! ${JAVA_HOME} ]]; then
+echo "Unable to locate JAVA_HOME on this machine. Please modify the
default value of JAVA_HOME in ${prop_file}."
+ fi
+}
+
main() {
if [[ $# -lt 1 ]]; then
usage
@@ -221,5 +239,6 @@ main() {
create_hawq_service_definition
create_hawq_service_instance
update_ranger_url
+ update_java_home
}
main "$@"
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a803aab4/ranger-plugin/scripts/rps.sh
--
diff --git a/ranger-plugin/scripts/rps.sh b/ranger-plugin/scripts/rps.sh
index fa268c9..70aa547 100755
--- a/ranger-plugin/scripts/rps.sh
+++ b/ranger-plugin/scripts/rps.sh
@@ -32,6 +32,7 @@ BASEDIR=$( dirname ${CWDIR} )
# read properties from the file
source ${BASEDIR}/etc/rps.properties
+export JAVA_HOME
export CATALINA_HOME
export CATALINA_BASE=${BASEDIR}/plugin-service
export CATALINA_PID=${CATALINA_BASE}/work/rps.pid
incubator-hawq git commit: HAWQ-1366. Throw unsupported error out for dictionary page in Parquet storage
Repository: incubator-hawq
Updated Branches:
refs/heads/master 4820193c9 -> 6d5792c40
HAWQ-1366. Throw unsupported error out for dictionary page in Parquet storage
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/6d5792c4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/6d5792c4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/6d5792c4
Branch: refs/heads/master
Commit: 6d5792c408461d7d58fa7633d939dd8900138681
Parents: 4820193
Author: Lili Ma
Authored: Tue Feb 28 17:57:21 2017 +0800
Committer: Lili Ma
Committed: Tue Feb 28 18:02:16 2017 +0800
--
src/backend/cdb/cdbparquetcolumn.c | 5 +
1 file changed, 5 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6d5792c4/src/backend/cdb/cdbparquetcolumn.c
--
diff --git a/src/backend/cdb/cdbparquetcolumn.c
b/src/backend/cdb/cdbparquetcolumn.c
index 89ce684..083f669 100644
--- a/src/backend/cdb/cdbparquetcolumn.c
+++ b/src/backend/cdb/cdbparquetcolumn.c
@@ -146,6 +146,11 @@ ParquetExecutorReadColumn(ParquetColumnReader
*columnReader, File file)
/*just process data page now*/
if(pageHeader->page_type != DATA_PAGE){
+ if(pageHeader->page_type == DICTIONARY_PAGE) {
+ ereport(ERROR,
(errcode(ERRCODE_GP_INTERNAL_ERROR),
+ errmsg("HAWQ
does not support dictionary page type resolver for Parquet format in column
\'%s\' ",
+
columnChunkMetadata->colName)));
+ }
buffer += pageHeader->compressed_page_size;
continue;
}
incubator-hawq git commit: HAWQ-1091. Fix HAWQ InputFormat Bugs.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 19f3fa81b -> 28d192d23
HAWQ-1091. Fix HAWQ InputFormat Bugs.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/28d192d2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/28d192d2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/28d192d2
Branch: refs/heads/master
Commit: 28d192d2329dc68373616fe65530c590d501a7b1
Parents: 19f3fa8
Author: xunzhang
Authored: Sun Oct 9 20:35:31 2016 +0800
Committer: Lili Ma
Committed: Mon Oct 10 13:57:24 2016 +0800
--
.../com/pivotal/hawq/mapreduce/metadata/MetadataAccessor.java | 2 ++
.../pivotal/hawq/mapreduce/metadata/MetadataYAMLAccessor.java | 6 ++
.../test/java/com/pivotal/hawq/mapreduce/TPCHLocalTester.java | 2 +-
.../hawq/mapreduce/pt/HAWQInputFormatPerformanceTest_TPCH.java | 2 +-
.../hawq/mapreduce/ut/HAWQInputFormatUnitTest_TPCH.java| 5 +
5 files changed, 15 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/28d192d2/contrib/hawq-hadoop/hawq-mapreduce-common/src/main/java/com/pivotal/hawq/mapreduce/metadata/MetadataAccessor.java
--
diff --git
a/contrib/hawq-hadoop/hawq-mapreduce-common/src/main/java/com/pivotal/hawq/mapreduce/metadata/MetadataAccessor.java
b/contrib/hawq-hadoop/hawq-mapreduce-common/src/main/java/com/pivotal/hawq/mapreduce/metadata/MetadataAccessor.java
index 250eb86..5a46489 100644
---
a/contrib/hawq-hadoop/hawq-mapreduce-common/src/main/java/com/pivotal/hawq/mapreduce/metadata/MetadataAccessor.java
+++
b/contrib/hawq-hadoop/hawq-mapreduce-common/src/main/java/com/pivotal/hawq/mapreduce/metadata/MetadataAccessor.java
@@ -111,6 +111,8 @@ public abstract class MetadataAccessor {
try {
return new MetadataYAMLAccessor(file);
+ } catch (IllegalStateException e) {
+ throw new IllegalStateException(e.getMessage());
} catch (Exception e) {
throw new MetadataAccessException("error occurred when
reading " + file, e);
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/28d192d2/contrib/hawq-hadoop/hawq-mapreduce-common/src/main/java/com/pivotal/hawq/mapreduce/metadata/MetadataYAMLAccessor.java
--
diff --git
a/contrib/hawq-hadoop/hawq-mapreduce-common/src/main/java/com/pivotal/hawq/mapreduce/metadata/MetadataYAMLAccessor.java
b/contrib/hawq-hadoop/hawq-mapreduce-common/src/main/java/com/pivotal/hawq/mapreduce/metadata/MetadataYAMLAccessor.java
index 3e193b5..4570bc2 100644
---
a/contrib/hawq-hadoop/hawq-mapreduce-common/src/main/java/com/pivotal/hawq/mapreduce/metadata/MetadataYAMLAccessor.java
+++
b/contrib/hawq-hadoop/hawq-mapreduce-common/src/main/java/com/pivotal/hawq/mapreduce/metadata/MetadataYAMLAccessor.java
@@ -158,6 +158,9 @@ class MetadataYAMLAccessor extends MetadataAccessor {
Map file = (Map) files.get(i);
String filePath = file.get("path").toString();
long fileLen =
Long.parseLong(file.get("size").toString());
+ if (fileLen == 0) {
+ throw new IllegalStateException("Empty file can not
be processed by HAWQAOInputFormat since it is an empty file.");
+ }
fileStatuses.add(new HAWQAOFileStatus(filePath,
fileLen, checksum, compressType, blockSize));
}
@@ -198,6 +201,9 @@ class MetadataYAMLAccessor extends MetadataAccessor {
Map file = (Map) files.get(i);
final String filePath = file.get("path").toString();
final long fileLen =
Long.parseLong(file.get("size").toString());
+ if (fileLen == 0) {
+ throw new IllegalStateException("Empty file can not
be processed by HAWQParquetInputFormat since it is an empty file.");
+ }
fileStatuses.add(new HAWQFileStatus(filePath, fileLen));
}
return fileStatuses;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/28d192d2/contrib/hawq-hadoop/hawq-mapreduce-tool/src/test/java/com/pivotal/hawq/mapreduce/TPCHLocalTester.java
--
diff --git
a/contrib/hawq-hadoop/hawq-mapreduce-tool/src/test/java/com/pivotal/hawq/mapreduce/TPCHLocalTester.java
b/contrib/hawq-hadoop/hawq-mapreduce-tool/src/test/java/com/pivotal/hawq/mapreduce/TPCHLocalTester.java
index 7368b61..19f34f1 100644
---
a/contrib/hawq-hadoop/hawq-
incubator-hawq git commit: HAWQ-991. Fix upper/lower case of tablename, use hadoop fs to avoid hdfs dfs bug in 2.7.1 hadoop.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 4ddbc249e -> 2ec18bdf8
HAWQ-991. Fix upper/lower case of tablename, use hadoop fs to avoid hdfs dfs
bug in 2.7.1 hadoop.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/2ec18bdf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/2ec18bdf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/2ec18bdf
Branch: refs/heads/master
Commit: 2ec18bdf8de8015a0379d53f2e0e43a970739466
Parents: 4ddbc24
Author: xunzhang
Authored: Wed Sep 28 14:06:58 2016 +0800
Committer: xunzhang
Committed: Wed Sep 28 14:06:58 2016 +0800
--
tools/bin/hawqregister | 22 +++---
1 file changed, 11 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2ec18bdf/tools/bin/hawqregister
--
diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister
index 29cd56a..69809f7 100755
--- a/tools/bin/hawqregister
+++ b/tools/bin/hawqregister
@@ -299,7 +299,7 @@ class HawqRegister(object):
self.yml = options.yml_config
self.filepath = options.filepath
self.database = options.database
-self.tablename = table
+self.tablename = table.lower()
self.filesize = options.filesize
self.accessor = GpRegisterAccessor(conn)
self.utility_accessor = GpRegisterAccessor(utility_conn)
@@ -488,13 +488,13 @@ class HawqRegister(object):
def check_file_not_folder():
for fn in self.files:
-hdfscmd = 'hdfs dfs -test -f %s' % fn
+hdfscmd = 'hadoop fs -test -f %s' % fn
if local_ssh(hdfscmd, logger):
logger.info('%s is not a file in hdfs, please check the
yaml configuration file.' % fn)
sys.exit(1)
def is_folder(filepath):
-hdfscmd = 'hdfs dfs -test -d %s' % filepath
+hdfscmd = 'hadoop fs -test -d %s' % filepath
if local_ssh(hdfscmd, logger):
return False
else:
@@ -511,7 +511,7 @@ class HawqRegister(object):
self.failure_handler.rollback()
sys.exit(1)
for k, fn in enumerate(self.files):
-hdfscmd = 'hdfs dfs -du %s' % fn
+hdfscmd = 'hadoop fs -du %s' % fn
_, out, _ = local_ssh_output(hdfscmd)
if self.sizes[k] > int(out.strip().split()[0]):
logger.error('File size(%s) in yaml configuration file
should not exceed actual length(%s) of file %s.' % (self.sizes[k],
out.strip().split()[0], fn))
@@ -673,13 +673,13 @@ class HawqRegister(object):
def _get_files_in_hdfs(self, filepath):
'''Get all the files refered by 'filepath', which could be a file or a
directory containing all the files'''
files, sizes = [], []
-hdfscmd = "hdfs dfs -test -e %s" % filepath
+hdfscmd = "hadoop fs -test -e %s" % filepath
result = local_ssh(hdfscmd, logger)
if result != 0:
logger.error("Path '%s' does not exist in hdfs" % filepath)
self.failure_handler.rollback()
sys.exit(1)
-hdfscmd = "hdfs dfs -ls -R %s" % filepath
+hdfscmd = "hadoop fs -ls -R %s" % filepath
result, out, err = local_ssh_output(hdfscmd)
outlines = out.splitlines()
# recursively search all the files under path 'filepath'
@@ -701,13 +701,13 @@ class HawqRegister(object):
def _check_parquet_format(self, files):
'''Check whether the file to be registered is parquet format'''
for f in files:
-hdfscmd = 'hdfs dfs -du -h %s | head -c 1' % f
+hdfscmd = 'hadoop fs -du -h %s | head -c 1' % f
rc, out, err = local_ssh_output(hdfscmd)
if out == '0':
continue
-hdfscmd = 'hdfs dfs -cat %s | head -c 4 | grep PAR1' % f
+hdfscmd = 'hadoop fs -cat %s | head -c 4 | grep PAR1' % f
result1 = local_ssh(hdfscmd)
-hdfscmd = 'hdfs dfs -cat %s | tail -c 4 | grep PAR1' % f
+hdfscmd = 'hadoop fs -cat %s | tail -c 4 | grep PAR1' % f
result2 = local_ssh(hdfscmd)
if result1 or result2:
logger.error('File %s is not parquet format' % f)
@@ -722,7 +722,7 @@ class HawqRegister(object):
dstfile = self.tabledir + str(segno)
segno += 1
if srcfile != dstfile:
-hdfscmd = 'hdfs dfs -mv %s %s' % (srcfile, dstfile)
+hdfscmd = 'hadoop fs -mv %s %s' % (srcfile, dstfile)
sys.stdout.write('hdfscmd: "%s"\n' % hdfscmd)
incubator-hawq git commit: HAWQ-1061. Check folder and eof not conflict under usage1.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 98d48e78d -> bc873239e
HAWQ-1061. Check folder and eof not conflict under usage1.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/bc873239
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/bc873239
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/bc873239
Branch: refs/heads/master
Commit: bc873239e4e7b9cf00e63b2e2dc9982ed7273f66
Parents: 98d48e7
Author: xunzhang
Authored: Tue Sep 27 16:34:54 2016 +0800
Committer: Lili Ma
Committed: Wed Sep 28 10:22:58 2016 +0800
--
tools/bin/hawqregister | 10 ++
1 file changed, 10 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bc873239/tools/bin/hawqregister
--
diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister
index cc65491..4b7655c 100755
--- a/tools/bin/hawqregister
+++ b/tools/bin/hawqregister
@@ -487,6 +487,13 @@ class HawqRegister(object):
logger.info('%s is not a file in hdfs, please check the
yaml configuration file.' % fn)
sys.exit(1)
+def is_folder(filepath):
+hdfscmd = 'hdfs dfs -test -d %s' % filepath
+if local_ssh(hdfscmd, logger):
+return False
+else:
+return True
+
def check_sizes_valid():
for sz in self.sizes:
if type(sz) != type(1):
@@ -532,6 +539,9 @@ class HawqRegister(object):
check_policy_consistency()
check_no_regex_filepath(self.files)
else:
+if is_folder(self.filepath) and self.filesize:
+logger.error('-e option is only supported with single file
case.')
+sys.exit(1)
self.file_format = 'Parquet'
check_hash_type() # Usage1 only support randomly distributed table
if not self.filepath:
incubator-hawq git commit: HAWQ-1029. Update hawqregister_help info for 2.0.1.0 hawq release.
Repository: incubator-hawq Updated Branches: refs/heads/master 1aa0fbf5a -> b661d3a0e HAWQ-1029. Update hawqregister_help info for 2.0.1.0 hawq release. Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/b661d3a0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/b661d3a0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/b661d3a0 Branch: refs/heads/master Commit: b661d3a0ed40aec90f3c23f9badd8898fbec53f5 Parents: 1aa0fbf Author: xunzhang Authored: Thu Sep 1 21:08:34 2016 +0800 Committer: Lili Ma Committed: Fri Sep 23 20:08:13 2016 +0800 -- tools/doc/hawqregister_help | 67 +--- 1 file changed, 42 insertions(+), 25 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b661d3a0/tools/doc/hawqregister_help -- diff --git a/tools/doc/hawqregister_help b/tools/doc/hawqregister_help index a664127..b8577d7 100644 --- a/tools/doc/hawqregister_help +++ b/tools/doc/hawqregister_help @@ -1,14 +1,13 @@ COMMAND NAME: hawq register -Usage1: Register parquet files generated by other system into the corrsponding table in HAWQ -Usage2: Register parquet/ao table from laterst-sync-metadata in yaml format +Usage1: Register parquet files generated by other system into corrsponding table in HAWQ. +Usage2: Register parquet/ao table from yaml configuration file. * SYNOPSIS * - -Usage1: hawq register [-h hostname] [-p port] [-U username] [-d databasename] [-f filepath] -Usage2: hawq register [-h hostname] [-p port] [-U username] [-d databasename] [-c config] +Usage1: hawq register [-h hostname] [-p port] [-U username] [-d databasename] [-f filepath] [-e eof] +Usage2: hawq register [-h hostname] [-p port] [-U username] [-d databasename] [-c config] [-F, --force] hawq register help hawq register -? @@ -18,10 +17,9 @@ hawq register --version * DESCRIPTION * - Use Case1: -"hawq register" is a utility to register file(s) on HDFS into -the table in HAWQ. It moves the file in the path(if path +"hawq register" is an utility to register file(s) on HDFS into +the table in HAWQ. It moves file(s) in the path(if path refers to a file) or files under the path(if path refers to a directory) into the table directory corresponding to the table, and then update the table meta data to include the files. @@ -37,10 +35,23 @@ The file(s) to be registered and the table in HAWQ must be in the same HDFS cluster. Use Case2: -User should be able to use hawq register to register table files into a new HAWQ cluster. -It is some kind of protecting against corruption from users' perspective. -Users use the last-known-good metadata to update the portion of catalog managing HDFS blocks. -The table files or dictionary should be backuped(such as using distcp) into the same path in the new HDFS setting. +Hawq register can register both AO and parquet format table, and the files to be registered are listed in the .yml configuration file. +This configuration file can be generated by hawq extract. Register through .yml configuration doesnât require the table already exist, +since .yml file contains table schema already. +HAWQ register behaviors differently with different options: + * If the table does not exist, hawq register will create table and do register. + * If table already exist, hawq register will append the files to the existing table. + * If --force option specified, hawq register will erase existing catalog + table pg_aoseg.pg_aoseg_$relid/pg_aoseg.pg_paqseg_$relid data for the table and + re-register according to .yml configuration file definition. Note. If there are + files under table directory which are not specified in .yml configuration file, it will throw error out. +Note. Without --force specified, if some file specified in .yml configuration file lie under the table directory, + hawq register will throw error out. +Note. With --force option specified, if there are files under table directory which are not specified in .yml configuration file, + hawq register will throw error out. +Note. In usage2, if the table is hash distributed, hawq register just check the file number to be registered + has to be multiple times of this tableâs bucket number, and check whether the distribution key specified + in .yml configuration file is same as that of table. It does not check whether files are actually distributed by the key. To use "hawq register", HAWQ must have been started. Currently "ha
incubator-hawq git commit: HAWQ-991. Support registering hash distribution table into random distribution table.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 921b908ef -> a4f57020c
HAWQ-991. Support registering hash distribution table into random distribution
table.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/a4f57020
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/a4f57020
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/a4f57020
Branch: refs/heads/master
Commit: a4f57020c0b88a860fa5cea6060c3c17c7434c08
Parents: 921b908
Author: xunzhang
Authored: Fri Sep 23 15:49:20 2016 +0800
Committer: Lili Ma
Committed: Fri Sep 23 16:41:46 2016 +0800
--
tools/bin/hawqregister | 37 +
1 file changed, 17 insertions(+), 20 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a4f57020/tools/bin/hawqregister
--
diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister
index 3661f3e..b2c85ea 100755
--- a/tools/bin/hawqregister
+++ b/tools/bin/hawqregister
@@ -117,7 +117,7 @@ def register_yaml_dict_check(D, table_column_num,
src_tablename):
yml_column_num = len(D['AO_Schema'])
if table_column_num != yml_column_num and table_column_num > 0:
logger.error('Column number of table in yaml file is not equals to the
column number of table %s.' % src_tablename)
-sys.exit(1)
+sys.exit(1)
class FailureHandler(object):
@@ -136,7 +136,8 @@ class FailureHandler(object):
return ' '.join(lst[:-2] + [lst[-1], lst[-2]])
def rollback(self):
-logger.info('Error found, Hawqregister starts to rollback...')
+if len(self.operations) != 0:
+logger.info('Error found, Hawqregister starts to rollback...')
for (typ, cmd) in reversed(self.operations):
if typ == 'SQL':
sql = self.assemble_SQL(cmd)
@@ -153,7 +154,8 @@ class FailureHandler(object):
if result != 0:
logger.error('Fail to rollback: %s.' % hdfscmd)
sys.exit(1)
-logger.info('Hawq Register Rollback Succeed.')
+if len(self.operations) != 0:
+logger.info('Hawq Register Rollback Finished.')
class GpRegisterAccessor(object):
@@ -221,11 +223,9 @@ class GpRegisterAccessor(object):
rows = self.exec_query(qry)
if len(rows) == 0:
logger.error('Table %s is not an append-only table. There is no
record in gp_distribution_policy table.' % tablename)
-self.failure_handler.rollback()
sys.exit(1)
if rows[0]['attrnums']:
logger.error('Cannot register file(s) to a table which is hash
distributed.')
-self.failure_handler.rollback()
sys.exit(1)
# pg_paqseg_#
@@ -236,15 +236,13 @@ class GpRegisterAccessor(object):
rows = self.exec_query(query)
if len(rows) == 0:
logger.error('table "%s" not found in db "%s"' % (tablename,
database))
-self.failure_handler.rollback()
-sys.exit(1)
+return ('', False)
relname = rows[0]['relname']
if fmt == 'Parquet':
if relname.find('paq') == -1:
logger.error("table '%s' is not parquet format" % tablename)
-self.failure_handler.rollback()
-sys.exit(1)
-return relname
+return ('', False)
+return (relname, True)
def get_distribution_policy_info(self, tablename):
query = "select oid from pg_class where relname = '%s';" %
tablename.split('.')[-1].lower()
@@ -353,7 +351,8 @@ class HawqRegister(object):
except pg.DatabaseError as e:
print e
sys.exit(1)
-self.failure_handler.commit(('SQL', query))
+if ret:
+self.failure_handler.commit(('SQL', query))
return ret
def get_seg_name():
@@ -368,12 +367,7 @@ class HawqRegister(object):
def check_policy_consistency():
policy = get_distribution_policy() # "" or "{1,3}"
if policy is None:
-if ' '.join(self.distribution_policy.strip().split()).lower()
== 'distributed randomly':
-return
-else:
-logger.error('Distribution policy of %s from yaml file is
not consistent with the policy of existing table.' % self.tablename)
-self.failure_handler.rollback()
-sys.exit(1)
+return
tmp_dict = {}
for i, d in enumerate(self.schema):
tmp_dict[d['name']] = i + 1
@@ -535,7 +529,10 @@ class HawqRegister(object):
[1/2] incubator-hawq git commit: HAWQ-1024. Add rollback before all necessary exit.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 21b867a10 -> 921b908ef
HAWQ-1024. Add rollback before all necessary exit.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/31363940
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/31363940
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/31363940
Branch: refs/heads/master
Commit: 313639400b5a5bdf8afcb8c3123f141b580f238b
Parents: 21b867a
Author: xunzhang
Authored: Fri Sep 23 11:23:58 2016 +0800
Committer: xunzhang
Committed: Fri Sep 23 11:23:58 2016 +0800
--
tools/bin/hawqregister | 27 +++
1 file changed, 27 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/31363940/tools/bin/hawqregister
--
diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister
index 08041b4..570253f 100755
--- a/tools/bin/hawqregister
+++ b/tools/bin/hawqregister
@@ -136,6 +136,7 @@ class FailureHandler(object):
return ' '.join(lst[:-2] + [lst[-1], lst[-2]])
def rollback(self):
+logger.info('Error found, Hawqregister starts to rollback...')
for (typ, cmd) in reversed(self.operations):
if typ == 'SQL':
sql = self.assemble_SQL(cmd)
@@ -220,9 +221,11 @@ class GpRegisterAccessor(object):
rows = self.exec_query(qry)
if len(rows) == 0:
logger.error('Table %s is not an append-only table. There is no
record in gp_distribution_policy table.' % tablename)
+self.failure_handler.rollback()
sys.exit(1)
if rows[0]['attrnums']:
logger.error('Cannot register file(s) to a table which is hash
distributed.')
+self.failure_handler.rollback()
sys.exit(1)
# pg_paqseg_#
@@ -233,11 +236,13 @@ class GpRegisterAccessor(object):
rows = self.exec_query(query)
if len(rows) == 0:
logger.error('table "%s" not found in db "%s"' % (tablename,
database))
+self.failure_handler.rollback()
sys.exit(1)
relname = rows[0]['relname']
if fmt == 'Parquet':
if relname.find('paq') == -1:
logger.error("table '%s' is not parquet format" % tablename)
+self.failure_handler.rollback()
sys.exit(1)
return relname
@@ -331,6 +336,7 @@ class HawqRegister(object):
if self.distribution_policy.startswith('DISTRIBUTED BY'):
if len(self.files) % self.bucket_number != 0:
logger.error('Files to be registered must be multiple
times to the bucket number of hash table.')
+self.failure_handler.rollback()
sys.exit(1)
def check_database_encoding():
@@ -366,6 +372,7 @@ class HawqRegister(object):
return
else:
logger.error('Distribution policy of %s from yaml file is
not consistent with the policy of existing table.' % self.tablename)
+self.failure_handler.rollback()
sys.exit(1)
tmp_dict = {}
for i, d in enumerate(self.schema):
@@ -375,6 +382,7 @@ class HawqRegister(object):
original_policy = ','.join([str(tmp_dict[col]) for col in cols])
if policy.strip('{').strip('}') != original_policy:
logger.error('Distribution policy of %s from yaml file is not
consistent with the policy of existing table.' % self.tablename)
+self.failure_handler.rollback()
sys.exit(1)
def check_bucket_number():
@@ -383,6 +391,7 @@ class HawqRegister(object):
if self.bucket_number != get_bucket_number():
logger.error('Bucket number of %s is not consistent with
previous bucket number.' % self.tablename)
+self.failure_handler.rollback()
sys.exit(1)
def set_yml_dataa(file_format, files, sizes, tablename, schema,
distribution_policy, file_locations,\
@@ -476,12 +485,14 @@ class HawqRegister(object):
for sz in self.sizes:
if sz < 0:
logger.error('File size(%s) in yaml configuration file
should not be less than 0.' % sz)
+self.failure_handler.rollback()
sys.exit(1)
for k, fn in enumerate(self.files):
hdfscmd = 'hdfs dfs -du %s' % fn
_, out, _ = local_ssh_output(hdfscmd)
if self.sizes[k] > int(out.strip().split()[0]):
logger.error('File size(%s) in yaml configuration file
should not exceed actual length(%s) of
[2/2] incubator-hawq git commit: HAWQ-1061. Check filesize type.
HAWQ-1061. Check filesize type.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/921b908e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/921b908e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/921b908e
Branch: refs/heads/master
Commit: 921b908ef30e4ee676e114136c712b80b187f36c
Parents: 3136394
Author: xunzhang
Authored: Fri Sep 23 11:40:00 2016 +0800
Committer: xunzhang
Committed: Fri Sep 23 11:40:00 2016 +0800
--
tools/bin/hawqregister | 4
1 file changed, 4 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/921b908e/tools/bin/hawqregister
--
diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister
index 570253f..3661f3e 100755
--- a/tools/bin/hawqregister
+++ b/tools/bin/hawqregister
@@ -483,6 +483,10 @@ class HawqRegister(object):
def check_sizes_valid():
for sz in self.sizes:
+if type(sz) != type(1):
+logger.error('File size(%s) in yaml configuration file
should be int type.' % sz)
+self.failure_handler.rollback()
+sys.exit(1)
if sz < 0:
logger.error('File size(%s) in yaml configuration file
should not be less than 0.' % sz)
self.failure_handler.rollback()
[1/2] incubator-hawq git commit: HAWQ-1061. Ensure filepath is not regex-like.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 5a152dcdd -> 21b867a10
HAWQ-1061. Ensure filepath is not regex-like.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/21b867a1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/21b867a1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/21b867a1
Branch: refs/heads/master
Commit: 21b867a10322c75e3a8b18a5f502a9e16c2e58d4
Parents: 06412f3
Author: xunzhang
Authored: Thu Sep 22 22:13:34 2016 +0800
Committer: Lili Ma
Committed: Fri Sep 23 10:36:39 2016 +0800
--
tools/bin/hawqregister | 20 +++-
1 file changed, 19 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/21b867a1/tools/bin/hawqregister
--
diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister
index 65a5e44..08041b4 100755
--- a/tools/bin/hawqregister
+++ b/tools/bin/hawqregister
@@ -316,7 +316,11 @@ class HawqRegister(object):
else:
return 'usage2_table_not_exist'
else:
-return 'usage1'
+if not table_existed():
+logger.error('You should create table before registering the
data.')
+sys.exit(1)
+else:
+return 'usage1'
def _init(self):
def check_hash_type():
@@ -480,6 +484,18 @@ class HawqRegister(object):
logger.error('File size(%s) in yaml configuration file
should not exceed actual length(%s) of file %s.' % (self.sizes[k],
out.strip().split()[0], fn))
sys.exit(1)
+def check_no_regex_filepath(files):
+for fn in files:
+tmp_lst = fn.split('/')
+for v in tmp_lst:
+if v == '.':
+logger.error('Hawq register does not support file path
with regex: %s.' % fn)
+sys.exit(1)
+for ch in ['..', '*']:
+if fn.find(ch) != -1:
+logger.error('Hawq register does not support file path
with regex: %s.' % fn)
+sys.exit(1)
+
if self.yml:
option_parser_yml(options.yml_config)
self.filepath = self.files[0][:self.files[0].rfind('/')] if
self.files else ''
@@ -491,6 +507,7 @@ class HawqRegister(object):
check_bucket_number()
check_distribution_policy()
check_policy_consistency()
+check_no_regex_filepath(self.files)
else:
self.file_format = 'Parquet'
check_hash_type() # Usage1 only support randomly distributed table
@@ -562,6 +579,7 @@ class HawqRegister(object):
self._check_files_and_table_in_same_hdfs_cluster(self.filepath,
self.tabledir)
if not self.yml:
+check_no_regex_filepath([self.filepath])
self.files, self.sizes = self._get_files_in_hdfs(self.filepath)
print 'New file(s) to be registered: ', self.files
if self.files_update:
[2/2] incubator-hawq git commit: HAWQ-1061. Bugfix for specific eof is zeor and check greater than zeor(thanks @wcl14).
HAWQ-1061. Bugfix for specific eof is zeor and check greater than zeor(thanks
@wcl14).
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/06412f3e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/06412f3e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/06412f3e
Branch: refs/heads/master
Commit: 06412f3eb8e6c2cb4a6a8411e0126b70e90014ca
Parents: 5a152dc
Author: xunzhang
Authored: Thu Sep 22 23:31:21 2016 +0800
Committer: Lili Ma
Committed: Fri Sep 23 10:36:39 2016 +0800
--
tools/bin/hawqregister | 8 ++--
1 file changed, 6 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/06412f3e/tools/bin/hawqregister
--
diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister
index 2e945a6..65a5e44 100755
--- a/tools/bin/hawqregister
+++ b/tools/bin/hawqregister
@@ -51,7 +51,7 @@ def option_parser():
parser.add_option('-U', '--user', help='username of the target DB')
parser.add_option('-d', '--database', default='postgres', dest='database',
help='database name')
parser.add_option('-f', '--filepath', dest='filepath', help='file name in
HDFS')
-parser.add_option('-e', '--eof', dest='filesize', type='int', default=0,
help='eof of the file to be registered')
+parser.add_option('-e', '--eof', dest='filesize', type='int',
default=None, help='eof of the file to be registered')
parser.add_option('-c', '--config', dest='yml_config', default='',
help='configuration file in YAML format')
parser.add_option('-F', '--force', dest='force', action='store_true',
default=False)
parser.add_option('-R', '--repair', dest='repair', action='store_true',
default=False)
@@ -469,6 +469,10 @@ class HawqRegister(object):
sys.exit(1)
def check_sizes_valid():
+for sz in self.sizes:
+if sz < 0:
+logger.error('File size(%s) in yaml configuration file
should not be less than 0.' % sz)
+sys.exit(1)
for k, fn in enumerate(self.files):
hdfscmd = 'hdfs dfs -du %s' % fn
_, out, _ = local_ssh_output(hdfscmd)
@@ -563,7 +567,7 @@ class HawqRegister(object):
if self.files_update:
print 'Catalog info need to be updated for these files: ',
self.files_update
-if self.filesize:
+if self.filesize is not None:
if len(self.files) != 1:
logger.error('-e option is only supported with single file
case.')
sys.exit(1)
[2/2] incubator-hawq git commit: HAWQ-991. Disable --repair mode and partiton table support of hawq register for 2.0.1.0 release.
HAWQ-991. Disable --repair mode and partiton table support of hawq register for
2.0.1.0 release.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/dc12e94b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/dc12e94b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/dc12e94b
Branch: refs/heads/master
Commit: dc12e94b43bddb2a9c71189efe284bb81d81afd3
Parents: 25e082a
Author: xunzhang
Authored: Thu Sep 22 14:39:21 2016 +0800
Committer: xunzhang
Committed: Thu Sep 22 15:57:41 2016 +0800
--
.../ManagementTool/test_hawq_register_usage2_case3.cpp | 2 +-
tools/bin/hawqregister | 8
2 files changed, 9 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/dc12e94b/src/test/feature/ManagementTool/test_hawq_register_usage2_case3.cpp
--
diff --git
a/src/test/feature/ManagementTool/test_hawq_register_usage2_case3.cpp
b/src/test/feature/ManagementTool/test_hawq_register_usage2_case3.cpp
index 1d021d3..8e8fb6d 100644
--- a/src/test/feature/ManagementTool/test_hawq_register_usage2_case3.cpp
+++ b/src/test/feature/ManagementTool/test_hawq_register_usage2_case3.cpp
@@ -14,7 +14,7 @@ using hawq::test::SQLUtility;
using hawq::test::Command;
using hawq::test::HdfsConfig;
-TEST_F(TestHawqRegister, TestUsage2Case3Expected) {
+TEST_F(TestHawqRegister, DISABLED_TestUsage2Case3Expected) {
SQLUtility util;
string test_root(util.getTestRootPath());
std::vector create_table_matrix = {"distributed by (i)",
"distributed randomly"};
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/dc12e94b/tools/bin/hawqregister
--
diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister
index 1a6b5d3..09b8532 100755
--- a/tools/bin/hawqregister
+++ b/tools/bin/hawqregister
@@ -296,6 +296,8 @@ class HawqRegister(object):
if force:
return 'force'
elif repair:
+logger.info('--repair mode is not supported in current release
of hawq register.')
+sys.exit(0)
if not table_existed():
logger.error('--repair mode asserts the table has been
already created.')
sys.exit(1)
@@ -405,6 +407,9 @@ class HawqRegister(object):
if params['FileFormat'].lower() == 'parquet':
partitionby =
params.get('Parquet_FileLocations').get('PartitionBy')
+if partitionby:
+logger.info('Partition table is not supported in current
release of hawq register.')
+sys.exit(0)
if params.get('Parquet_FileLocations').get('Partitions') and
len(params['Parquet_FileLocations']['Partitions']):
partitions_checksum = [d['Checksum'] for d in
params['Parquet_FileLocations']['Partitions']]
partitions_compression_level = [d['CompressionLevel'] for
d in params['Parquet_FileLocations']['Partitions']]
@@ -424,6 +429,9 @@ class HawqRegister(object):
else: #AO format
partitionby = params.get('AO_FileLocations').get('PartitionBy')
+if partitionby:
+logger.info('Partition table is not supported in current
release of hawq register.')
+sys.exit(0)
if params.get('AO_FileLocations').get('Partitions') and
len(params['AO_FileLocations']['Partitions']):
partitions_checksum = [d['Checksum'] for d in
params['AO_FileLocations']['Partitions']]
partitions_compressionLevel = [d['CompressionLevel'] for d
in params['AO_FileLocations']['Partitions']]
[1/2] incubator-hawq git commit: HAWQ-1061. Bugfix for --force mode of hawq register.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 107b79dfe -> dc12e94b4
HAWQ-1061. Bugfix for --force mode of hawq register.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/25e082aa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/25e082aa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/25e082aa
Branch: refs/heads/master
Commit: 25e082aaf24df7a8b54fae930d633cb20c0373d5
Parents: 107b79d
Author: xunzhang
Authored: Thu Sep 22 14:38:48 2016 +0800
Committer: xunzhang
Committed: Thu Sep 22 14:38:48 2016 +0800
--
tools/bin/hawqregister | 17 -
1 file changed, 12 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/25e082aa/tools/bin/hawqregister
--
diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister
index bef1460..1a6b5d3 100755
--- a/tools/bin/hawqregister
+++ b/tools/bin/hawqregister
@@ -301,9 +301,9 @@ class HawqRegister(object):
sys.exit(1)
return 'repair'
else:
-return 'second_normal'
+return 'usage2_table_not_exist'
else:
-return 'first'
+return 'usage1'
def _init(self):
def check_hash_type():
@@ -463,7 +463,7 @@ class HawqRegister(object):
check_database_encoding()
if self.mode != 'force' and self.mode != 'repair':
if not create_table():
-self.mode = 'second_exist'
+self.mode = 'usage2_table_exist'
check_bucket_number()
check_distribution_policy()
check_policy_consistency()
@@ -471,7 +471,7 @@ class HawqRegister(object):
self.file_format = 'Parquet'
check_hash_type() # Usage1 only support randomly distributed table
if not self.filepath:
-if self.mode == 'first':
+if self.mode == 'usage1':
logger.info('Please specify filepath with -f option.')
else:
logger.info('Hawq Register Succeed.')
@@ -496,7 +496,7 @@ class HawqRegister(object):
logger.error('Can not register in repair mode since giving
larger file size: %s' % self.sizes[k])
sys.exit(1)
-if self.mode == 'second_exist':
+if self.mode == 'usage2_table_exist':
if self.tabledir.strip('/') == self.filepath.strip('/'):
logger.error('Files to be registered should not be the same
with table path.')
sys.exit(1)
@@ -511,6 +511,9 @@ class HawqRegister(object):
else:
self.do_not_move, self.files_update, self.sizes_update =
True, self.files, self.sizes
self.files, self.sizes = [], []
+elif len(self.files) < len(existed_files):
+logger.error('In force mode, you should include existing table
files in yaml configuration file. Otherwise you should drop the previous table
before register --force.')
+sys.exit(1)
else:
files_old, sizes_old = [f for f in self.files], [sz for sz in
self.sizes]
for k, f in enumerate(files_old):
@@ -519,6 +522,10 @@ class HawqRegister(object):
self.sizes_update.append(sizes_old[k])
self.files.remove(files_old[k])
self.sizes.remove(sizes_old[k])
+if sorted(self.files_update) != sorted(existed_files):
+logger.error('In force mode, you should include existing
table files in yaml configuration file. Otherwise you should drop the previous
table before register --force.')
+sys.exit(1)
+
elif self.mode == 'repair':
self.do_not_move = True
self.files_update, self.sizes_update = [fn for fn in self.files],
[sz for sz in self.sizes]
incubator-hawq git commit: HAWQ-1024. Rollback if hawq register failed in process.
Repository: incubator-hawq
Updated Branches:
refs/heads/master b5c77addd -> 107b79dfe
HAWQ-1024. Rollback if hawq register failed in process.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/107b79df
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/107b79df
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/107b79df
Branch: refs/heads/master
Commit: 107b79dfe1deaf3098be37be4b10581a9743a0e9
Parents: b5c77ad
Author: xunzhang
Authored: Wed Sep 21 20:10:00 2016 +0800
Committer: Lili Ma
Committed: Thu Sep 22 09:54:19 2016 +0800
--
tools/bin/hawqregister | 101 +---
1 file changed, 76 insertions(+), 25 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/107b79df/tools/bin/hawqregister
--
diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister
index fa23a1a..bef1460 100755
--- a/tools/bin/hawqregister
+++ b/tools/bin/hawqregister
@@ -112,6 +112,42 @@ def register_yaml_dict_check(D):
logger.error('Wrong configuration yaml file format: "%s"
attribute does not exist.\n See example in "hawq register --help".' %
'AO_FileLocations.%s' % attr)
sys.exit(1)
+
+class FailureHandler(object):
+def __init__(self, conn):
+self.operations = []
+self.conn = conn
+
+def commit(self, cmd):
+self.operations.append(cmd)
+
+def assemble_SQL(self, cmd):
+return 'DROP TABLE %s' % cmd[cmd.find('table')+6:cmd.find('(')]
+
+def assemble_hdfscmd(self, cmd):
+lst = cmd.strip().split()
+return ' '.join(lst[:-2] + [lst[-1], lst[-2]])
+
+def rollback(self):
+for (typ, cmd) in reversed(self.operations):
+if typ == 'SQL':
+sql = self.assemble_SQL(cmd)
+try:
+self.conn.query(sql)
+except pg.DatabaseError as e:
+logger.error('Rollback failure: %s.' % sql)
+print e
+sys.exit(1)
+if typ == 'HDFSCMD':
+hdfscmd = self.assemble_hdfscmd(cmd)
+sys.stdout.write('Rollback hdfscmd: "%s"\n' % hdfscmd)
+result = local_ssh(hdfscmd, logger)
+if result != 0:
+logger.error('Fail to rollback: %s.' % hdfscmd)
+sys.exit(1)
+logger.info('Hawq Register Rollback Succeed.')
+
+
class GpRegisterAccessor(object):
def __init__(self, conn):
self.conn = conn
@@ -135,21 +171,21 @@ class GpRegisterAccessor(object):
def do_create_table(self, src_table_name, tablename, schema_info, fmt,
distrbution_policy, file_locations, bucket_number, partitionby,
partitions_constraint, partitions_name):
if self.get_table_existed(tablename):
-return False
+return False, ''
schema = ','.join([k['name'] + ' ' + k['type'] for k in schema_info])
partlist = ""
for index in range(len(partitions_constraint)):
- if index > 0:
- partlist += ", "
- partition_refine_name = partitions_name[index]
- splitter = src_table_name.split(".")[-1] + '_1_prt_'
- partition_refine_name = partition_refine_name.split(splitter)[-1]
- #in some case, constraint contains "partition XXX" but in other
case, it doesn't contain. we need to treat them separately.
- if partitions_constraint[index].strip().startswith("DEFAULT
PARTITION") or partitions_constraint[index].strip().startswith("PARTITION") or
(len(partition_refine_name) > 0 and partition_refine_name[0].isdigit()):
- partlist = partlist + " " + partitions_constraint[index]
- else:
- partlist = partlist + "PARTITION " + partition_refine_name + " "
+ partitions_constraint[index]
-
+if index > 0:
+partlist += ", "
+partition_refine_name = partitions_name[index]
+splitter = src_table_name.split(".")[-1] + '_1_prt_'
+partition_refine_name = partition_refine_name.split(splitter)[-1]
+#in some case, constraint contains "partition XXX" but in other
case, it doesn't contain. we need to treat them separately.
+if partitions_constraint[index].strip().startswith("DEFAULT
PARTITION") or partitions_constraint[index].strip().startswith("PARTITION") or
(len(partition_refine_name) > 0 and partition_refine_name[0].isdigit()):
+partlist = partlist + " " + partitions_constraint[index]
+else:
+partlist = partlist + "PARTITION " + partition_refine_name + "
" + partitions_constraint[index]
+
[2/2] incubator-hawq git commit: HAWQ-1044. Add some error path test cases for hawq register (TestUsage2Case1ErrorEncoding, TestUsage2Case1Bucket0, TestUsage2Case1IncludeDirectory)
HAWQ-1044. Add some error path test cases for hawq register
(TestUsage2Case1ErrorEncoding, TestUsage2Case1Bucket0,
TestUsage2Case1IncludeDirectory)
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/01c740d9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/01c740d9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/01c740d9
Branch: refs/heads/master
Commit: 01c740d960389b1317022c1706712d03ecd2eb18
Parents: c8319de
Author: Chunling Wang
Authored: Wed Sep 21 10:30:47 2016 +0800
Committer: Chunling Wang
Committed: Wed Sep 21 16:32:54 2016 +0800
--
.../test_hawq_register_usage2_case1.cpp | 55 ++
.../test_hawq_register_usage2_case2.cpp | 80 ++--
.../test_hawq_register_usage2_case3.cpp | 78 ++-
.../ManagementTool/usage2case1/bucket0_tpl.yml | 31
.../usage2case1/error_encoding_tpl.yml | 21 +
.../usage2case1/includedirectory.yml| 21 +
6 files changed, 211 insertions(+), 75 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/01c740d9/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp
--
diff --git
a/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp
b/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp
index c967bb0..44d2d76 100644
--- a/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp
+++ b/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp
@@ -4,6 +4,7 @@
#include "lib/sql_util.h"
#include "lib/string_util.h"
#include "lib/hdfs_config.h"
+#include "lib/file_replace.h"
#include "test_hawq_register.h"
#include "gtest/gtest.h"
@@ -50,3 +51,57 @@ TEST_F(TestHawqRegister, TestUsage2Case1Expected) {
}
}
+TEST_F(TestHawqRegister, TestUsage2Case1ErrorEncoding) {
+SQLUtility util;
+string test_root(util.getTestRootPath());
+util.execute("create table t(i int) with (appendonly=true,
orientation=row) distributed randomly;");
+util.execute("insert into t select generate_series(1, 100);");
+util.query("select * from t;", 100);
+util.execute("create table nt(i int) with (appendonly=true,
orientation=row) distributed randomly;");
+util.execute("insert into nt select generate_series(1, 100);");
+util.query("select * from nt;", 100);
+string
t_yml(hawq::test::stringFormat("%s/ManagementTool/usage2case1/error_encoding.yml",
test_root.c_str()));
+string
t_yml_tpl(hawq::test::stringFormat("%s/ManagementTool/usage2case1/error_encoding_tpl.yml",
test_root.c_str()));
+hawq::test::FileReplace frep;
+std::unordered_map strs_src_dst;
+strs_src_dst["@DATABASE_OID@"]= getDatabaseOid();
+strs_src_dst["@TABLE_OID@"]= getTableOid("t");
+frep.replace(t_yml_tpl, t_yml, strs_src_dst);
+EXPECT_EQ(1, Command::getCommandStatus(hawq::test::stringFormat("hawq
register -d %s -c %s testhawqregister_testusage2case1errorencoding.nt",
HAWQ_DB, t_yml.c_str(;
+
+EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("rm -rf
%s", t_yml.c_str(;
+util.execute("drop table t;");
+util.execute("drop table nt;");
+}
+
+TEST_F(TestHawqRegister, TestUsage2Case1Bucket0) {
+SQLUtility util;
+string test_root(util.getTestRootPath());
+util.execute("create table t(i int) with (appendonly=true,
orientation=row) distributed by (i);");
+util.execute("insert into t select generate_series(1, 100);");
+util.query("select * from t;", 100);
+util.execute("create table nt(i int) with (appendonly=true,
orientation=row) distributed by (i);");
+util.execute("insert into nt select generate_series(1, 100);");
+util.query("select * from nt;", 100);
+string
t_yml(hawq::test::stringFormat("%s/ManagementTool/usage2case1/bucket0.yml",
test_root.c_str()));
+string
t_yml_tpl(hawq::test::stringFormat("%s/ManagementTool/usage2case1/bucket0_tpl.yml",
test_root.c_str()));
+hawq::test::FileReplace frep;
+std::unordered_map strs_src_dst;
+strs_src_dst["@DATABASE_OID@"]= getDatabaseOid();
+strs_src_dst["@TABLE_OID@"]= getTableOid("t");
+frep.replace(t_yml_tpl, t_yml, strs_src_dst);
+EXPECT_EQ(1, Command::getCommandStatus(hawq::test::stringFormat("hawq
register -d %s -c %s testhawqregister_testusage2case1bucket0.nt", HAWQ_DB,
t_yml.c_str(;
+
+EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("rm -rf
%s", t_yml.c_str(;
+util.execute("drop table t;");
+util.execute("drop table nt;");
+}
+
+TEST_F(TestHawqRegister, TestUsage2Case1IncludeDirectory) {
+SQLUtility util;
+string test_root(util.getTestRootPath());
+EXPECT_E
[1/2] incubator-hawq git commit: HAWQ-1044. Add some error path test case for hawq register(TestUsage2Case1ErrorHashTableRegistry, TestUsage2Case1LargerEof, TestUsage2Case1WrongDistributionPolicy)
Repository: incubator-hawq
Updated Branches:
refs/heads/master c8319dedb -> b5c77addd
HAWQ-1044. Add some error path test case for hawq
register(TestUsage2Case1ErrorHashTableRegistry, TestUsage2Case1LargerEof,
TestUsage2Case1WrongDistributionPolicy)
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/b5c77add
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/b5c77add
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/b5c77add
Branch: refs/heads/master
Commit: b5c77addd51236242de1561fbf519be7d1c1b05b
Parents: 01c740d
Author: Chunling Wang
Authored: Wed Sep 21 16:32:38 2016 +0800
Committer: Chunling Wang
Committed: Wed Sep 21 16:32:54 2016 +0800
--
.../feature/ManagementTool/test_hawq_register.h | 31 ++
.../test_hawq_register_usage2_case1.cpp | 100 +++
.../test_hawq_register_usage2_case2.cpp | 5 +
.../test_hawq_register_usage2_case3.cpp | 5 +
.../usage2case1/larger_eof_tpl.yml | 21
.../wrong_distributed_policy_tpl.yml| 21
6 files changed, 183 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b5c77add/src/test/feature/ManagementTool/test_hawq_register.h
--
diff --git a/src/test/feature/ManagementTool/test_hawq_register.h
b/src/test/feature/ManagementTool/test_hawq_register.h
index 70a2998..f033f6d 100644
--- a/src/test/feature/ManagementTool/test_hawq_register.h
+++ b/src/test/feature/ManagementTool/test_hawq_register.h
@@ -49,6 +49,37 @@ class TestHawqRegister : public ::testing::Test {
return "";
}
+void checkPgAOSegValue(std::string relname, std::string value,
std::string fmt) {
+std::string reloid = getTableOid(relname);
+
+const hawq::test::PSQLQueryResult &result1 = conn->getQueryResult(
+hawq::test::stringFormat("SELECT tupcount from
pg_aoseg.pg_%s_%s;", fmt.c_str(), reloid.c_str()));
+std::vector> table = result1.getRows();
+if (table.size() > 0) {
+for (int i = 0; i < table.size(); i++) {
+EXPECT_EQ(table[i][0], value);
+}
+}
+
+const hawq::test::PSQLQueryResult &result2 = conn->getQueryResult(
+hawq::test::stringFormat("SELECT varblockcount from
pg_aoseg.pg_%s_%s;", fmt.c_str(), reloid.c_str()));
+table = result2.getRows();
+if (table.size() > 0) {
+for (int i = 0; i < table.size(); i++) {
+EXPECT_EQ(table[i][0], value);
+}
+}
+
+const hawq::test::PSQLQueryResult &result3 = conn->getQueryResult(
+hawq::test::stringFormat("SELECT eofuncompressed from
pg_aoseg.pg_%s_%s;", fmt.c_str(), reloid.c_str()));
+table = result3.getRows();
+if (table.size() > 0) {
+for (int i = 0; i < table.size(); i++) {
+EXPECT_EQ(table[i][0], value);
+}
+}
+}
+
private:
std::unique_ptr conn;
};
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b5c77add/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp
--
diff --git
a/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp
b/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp
index 44d2d76..6640775 100644
--- a/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp
+++ b/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp
@@ -54,6 +54,9 @@ TEST_F(TestHawqRegister, TestUsage2Case1Expected) {
TEST_F(TestHawqRegister, TestUsage2Case1ErrorEncoding) {
SQLUtility util;
string test_root(util.getTestRootPath());
+util.execute("drop table if exists t;");
+util.execute("drop table if exists nt;");
+
util.execute("create table t(i int) with (appendonly=true,
orientation=row) distributed randomly;");
util.execute("insert into t select generate_series(1, 100);");
util.query("select * from t;", 100);
@@ -77,6 +80,9 @@ TEST_F(TestHawqRegister, TestUsage2Case1ErrorEncoding) {
TEST_F(TestHawqRegister, TestUsage2Case1Bucket0) {
SQLUtility util;
string test_root(util.getTestRootPath());
+util.execute("drop table if exists t;");
+util.execute("drop table if exists nt;");
+
util.execute("create table t(i int) with (appendonly=true,
orientation=row) distributed by (i);");
util.execute("insert into t select generate_series(1, 100);");
util.query("select * from t;", 100);
@@ -105,3 +111,97 @@ TEST_F(TestHawqRegister,
[2/2] incubator-hawq git commit: HAWQ-1061. Fix data loss when file locations include directories, check policy and bucketnum in all mode.
HAWQ-1061. Fix data loss when file locations include directories, check policy
and bucketnum in all mode.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/8954090c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/8954090c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/8954090c
Branch: refs/heads/master
Commit: 8954090c29a770889c3e3269e14bd4bdaa6926aa
Parents: a683b5c
Author: xunzhang
Authored: Mon Sep 19 17:42:22 2016 +0800
Committer: Lili Ma
Committed: Tue Sep 20 15:56:49 2016 +0800
--
tools/bin/hawqregister | 14 +++---
1 file changed, 11 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8954090c/tools/bin/hawqregister
--
diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister
index 153ea9d..2b9b343 100755
--- a/tools/bin/hawqregister
+++ b/tools/bin/hawqregister
@@ -375,14 +375,24 @@ class HawqRegister(object):
set_yml_dataa('AO', files, sizes, params['AO_Schema'],
params['Distribution_Policy'], params['AO_FileLocations'], params['Bucketnum'],
partitionby, partitions_constraint,\
partitions_name, partitions_compression_level,
partitions_compression_type, partitions_checksum, partitions_filepaths,
partitions_filesizes, encoding)
+def check_file_not_folder():
+for fn in self.files:
+hdfscmd = 'hdfs dfs -test -f %s' % fn
+if local_ssh(hdfscmd, logger):
+logger.info('%s is not a file in hdfs, please check the
yaml configuration file.' % fn)
+sys.exit(1)
+
if self.yml:
option_parser_yml(options.yml_config)
self.filepath = self.files[0][:self.files[0].rfind('/')] if
self.files else ''
-check_distribution_policy()
+check_file_not_folder()
check_database_encoding()
if self.mode != 'force' and self.mode != 'repair':
if not create_table():
self.mode = 'second_exist'
+check_bucket_number()
+check_distribution_policy()
+check_policy_consistency()
else:
self.file_format = 'Parquet'
check_hash_type() # Usage1 only support randomly distributed table
@@ -400,8 +410,6 @@ class HawqRegister(object):
if self.tabledir.strip('/') != self.filepath.strip('/'):
logger.error("In repair mode, file path from yaml file should
be the same with table's path.")
sys.exit(1)
-check_policy_consistency()
-check_bucket_number()
existed_files, existed_sizes =
self._get_files_in_hdfs(self.filepath)
existed_info = {}
for k, fn in enumerate(existed_files):
[1/2] incubator-hawq git commit: HAWQ-1061. Fix update catelog inconsistency issue.
Repository: incubator-hawq
Updated Branches:
refs/heads/master a683b5c31 -> 50c1aa9ea
HAWQ-1061. Fix update catelog inconsistency issue.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/50c1aa9e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/50c1aa9e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/50c1aa9e
Branch: refs/heads/master
Commit: 50c1aa9eab39868f11f8946d9564aa832f5eb96c
Parents: 8954090
Author: xunzhang
Authored: Tue Sep 20 12:37:56 2016 +0800
Committer: Lili Ma
Committed: Tue Sep 20 15:56:49 2016 +0800
--
tools/bin/hawqregister | 16
1 file changed, 12 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/50c1aa9e/tools/bin/hawqregister
--
diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister
index 2b9b343..bdd6947 100755
--- a/tools/bin/hawqregister
+++ b/tools/bin/hawqregister
@@ -572,8 +572,12 @@ class HawqRegister(object):
query = "set allow_system_table_mods='dml';"
query += "begin transaction;"
segno_lst = [f.split('/')[-1] for f in self.files_update]
-for i, eof in enumerate(eofs):
-query += "update pg_aoseg.%s set eof = '%s' where segno =
'%s';" % (self.seg_name, eof, segno_lst[i])
+if self.file_format == 'Parquet':
+for i, eof in enumerate(eofs):
+query += "update pg_aoseg.%s set eof = '%s', tupcount =
'%s', eofuncompressed = '%s' where segno = '%s';" % (self.seg_name, eof, -1,
-1, segno_lst[i])
+else:
+for i, eof in enumerate(eofs):
+query += "update pg_aoseg.%s set eof = '%s', tupcount =
'%s', varblockcount = '%s', eofuncompressed = '%s' where segno = '%s';" %
(self.seg_name, eof, -1, -1, -1, segno_lst[i])
query += "end transaction;"
else: # update_and_insert
eofs = self.sizes
@@ -590,8 +594,12 @@ class HawqRegister(object):
query += ';'
segno_lst = [f.split('/')[-1] for f in self.files_update]
-for i, eof in enumerate(self.sizes_update):
-query += "update pg_aoseg.%s set eof = '%s' where segno =
'%s';" % (self.seg_name, eof, segno_lst[i])
+if self.file_format == 'Parquet':
+for i, eof in enumerate(self.sizes_update):
+query += "update pg_aoseg.%s set eof = '%s', tupcount =
'%s', eofuncompressed = '%s' where segno = '%s';" % (self.seg_name, eof, -1,
-1, segno_lst[i])
+else:
+for i, eof in enumerate(self.sizes_update):
+query += "update pg_aoseg.%s set eof = '%s', tupcount =
'%s', varblockcount = '%s', eofuncompressed = '%s' where segno = '%s';" %
(self.seg_name, eof, -1, -1, -1, segno_lst[i])
query += "end transaction;"
return self.utility_accessor.update_catalog(query)
incubator-hawq git commit: HAWQ-991. Check bucketnum and encoding consistency in hawq register.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 77e245dde -> 410f35a7c
HAWQ-991. Check bucketnum and encoding consistency in hawq register.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/410f35a7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/410f35a7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/410f35a7
Branch: refs/heads/master
Commit: 410f35a7c6105713afc90a349b626650267a21dd
Parents: 77e245d
Author: xunzhang
Authored: Sun Sep 18 17:52:23 2016 +0800
Committer: Lili Ma
Committed: Mon Sep 19 11:08:15 2016 +0800
--
tools/bin/hawqregister | 31 ++-
1 file changed, 26 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/410f35a7/tools/bin/hawqregister
--
diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister
index 0c75662..1366bce 100755
--- a/tools/bin/hawqregister
+++ b/tools/bin/hawqregister
@@ -65,6 +65,9 @@ def register_yaml_dict_check(D):
if D.get(attr) is None:
logger.error('Wrong configuration yaml file format: "%s" attribute
does not exist.\n See example in "hawq register --help".' % attr)
sys.exit(1)
+if D['Bucketnum'] <= 0:
+logger.error('Bucketnum should not be zero, please check your yaml
configuration file.')
+sys.exit(1)
if D['FileFormat'] in ['Parquet', 'AO']:
prefix = D['FileFormat']
local_check_list = ['%s_FileLocations' % prefix, '%s_Schema' % prefix]
@@ -198,6 +201,14 @@ class GpRegisterAccessor(object):
tabledir = '/'.join([D['location'].strip(), str(D['tablespace_oid']),
str(D['database_oid']), str(D['relfilenode']), ''])
return firstsegno, tabledir
+def get_database_encoding_indx(self, database):
+query = "select encoding from pg_database where datname = '%s';" %
database
+return self.exec_query(query)[0]['encoding']
+
+def get_database_encoding(self, encoding_indx):
+query = "select pg_encoding_to_char(%s);" % encoding_indx
+return self.exec_query(query)[0]['pg_encoding_to_char']
+
def update_catalog(self, query):
self.conn.query(query)
@@ -242,6 +253,13 @@ class HawqRegister(object):
logger.error('Files to be registered must be multiple
times to the bucket number of hash table.')
sys.exit(1)
+def check_database_encoding():
+encoding_indx =
self.accessor.get_database_encoding_indx(self.database)
+encoding = self.accessor.get_database_encoding(encoding_indx)
+if self.encoding.strip() != encoding:
+logger.error('Database encoding from yaml configuration
file(%s) is not consistent with encoding from input args(%s).' %
(self.encoding, encoding))
+sys.exit(1)
+
def create_table():
return self.accessor.do_create_table(self.tablename, self.schema,
self.file_format, self.distribution_policy, self.file_locations,
self.bucket_number)
@@ -282,7 +300,7 @@ class HawqRegister(object):
def set_yml_dataa(file_format, files, sizes, schema,
distribution_policy, file_locations,\
bucket_number, partitionby, partitions_constraint,
partitions_name, partitions_compression_level,\
- partitions_compression_type, partitions_checksum,
partitions_filepaths, partitions_filesizes):
+ partitions_compression_type, partitions_checksum,
partitions_filepaths, partitions_filesizes, encoding):
self.file_format = file_format
self.files = files
self.sizes = sizes
@@ -298,6 +316,7 @@ class HawqRegister(object):
self.partitions_checksum = partitions_checksum
self.partitions_filepaths = partitions_filepaths
self.partitions_filesizes = partitions_filesizes
+self.encoding = encoding
def option_parser_yml(yml_file):
import yaml
@@ -311,8 +330,7 @@ class HawqRegister(object):
partitions_checksum = []
partitions_compression_level = []
partitions_compression_type = []
-files = []
-sizes = []
+files, sizes = [], []
if params['FileFormat'].lower() == 'parquet':
partitionby =
params.get('Parquet_FileLocations').get('PartitionBy')
@@ -329,8 +347,9 @@ class HawqRegister(object):
partitions_name = [d['Name'] for d in
params['Parquet_FileLocations']['Partitions']]
if len(params['Parquet_FileLocations']['Files']):
file
[2/2] incubator-hawq git commit: HAWQ-1060. Refactor hawq register with better readability and quality.
HAWQ-1060. Refactor hawq register with better readability and quality.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/637f9d57
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/637f9d57
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/637f9d57
Branch: refs/heads/master
Commit: 637f9d5787aec24d72a3159b356d388ba116991e
Parents: 981c0a9
Author: xunzhang
Authored: Fri Sep 16 18:48:39 2016 +0800
Committer: Lili Ma
Committed: Sun Sep 18 14:39:54 2016 +0800
--
tools/bin/hawqregister | 680 ++--
1 file changed, 335 insertions(+), 345 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/637f9d57/tools/bin/hawqregister
--
diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister
index bbdc946..ee5275b 100755
--- a/tools/bin/hawqregister
+++ b/tools/bin/hawqregister
@@ -20,80 +20,81 @@
# Usage1: hawq register [-h hostname] [-p port] [-U username] [-d database]
[-f filepath] [-e eof]
# Usage2: hawq register [-h hostname] [-p port] [-U username] [-d database]
[-c config] [--force] [--repair]
-import os, sys, optparse, getpass, re, urlparse
+import os
+import sys
try:
from gppylib.commands.unix import getLocalHostname, getUserName
from gppylib.db import dbconn
from gppylib.gplog import get_default_logger, setup_tool_logging
from gppylib.gpparseopts import OptParser, OptChecker
from pygresql import pg
-from pygresql.pgdb import DatabaseError
from hawqpylib.hawqlib import local_ssh, local_ssh_output
except ImportError, e:
print e
-sys.stderr.write('cannot import module, please check that you have source
greenplum_path.sh\n')
+sys.stderr.write('Cannot import module, please check that you have source
greenplum_path.sh\n')
sys.exit(2)
# setup logging
logger = get_default_logger()
EXECNAME = os.path.split(__file__)[-1]
-setup_tool_logging(EXECNAME,getLocalHostname(),getUserName())
+setup_tool_logging(EXECNAME, getLocalHostname(), getUserName())
def option_parser():
+'''option parser'''
parser = OptParser(option_class=OptChecker,
usage='usage: %prog [options] table_name',
version='%prog version $Revision: #1 $')
parser.remove_option('-h')
-parser.add_option('-?', '--help', action = 'help')
-parser.add_option('-h', '--host', help = 'host of the target DB')
-parser.add_option('-p', '--port', help = 'port of the target DB', type =
'int', default = 0)
-parser.add_option('-U', '--user', help = 'username of the target DB')
-parser.add_option('-d', '--database', default = 'postgres', dest =
'database', help='database name')
-parser.add_option('-f', '--filepath', dest = 'filepath', help = 'file name
in HDFS')
-parser.add_option('-e', '--eof', dest = 'filesize', type = 'int', default
= 0, help = 'eof of the file to be registered')
-parser.add_option('-c', '--config', dest = 'yml_config', default = '',
help = 'configuration file in YAML format')
-parser.add_option('--force', action = 'store_true', default = False)
-parser.add_option('--repair', action = 'store_true', default = False)
+parser.add_option('-?', '--help', action='help')
+parser.add_option('-h', '--host', help='host of the target DB')
+parser.add_option('-p', '--port', help='port of the target DB',
type='int', default=0)
+parser.add_option('-U', '--user', help='username of the target DB')
+parser.add_option('-d', '--database', default='postgres', dest='database',
help='database name')
+parser.add_option('-f', '--filepath', dest='filepath', help='file name in
HDFS')
+parser.add_option('-e', '--eof', dest='filesize', type='int', default=0,
help='eof of the file to be registered')
+parser.add_option('-c', '--config', dest='yml_config', default='',
help='configuration file in YAML format')
+parser.add_option('--force', action='store_true', default=False)
+parser.add_option('--repair', action='store_true', default=False)
return parser
def register_yaml_dict_check(D):
-# check exists
+'''check exists'''
check_list = ['DFS_URL', 'Distribution_Policy', 'FileFormat', 'TableName',
'Bucketnum']
for attr in check_list:
-if D.get(attr) == None:
+if D.get(attr) is None:
logger.error('Wrong configuration yaml file format: "%s" attribute
does not exist.\n See example in "hawq register --help".' % attr)
sys.exit(1)
if D['FileFormat'] in ['Parquet', 'AO']:
prefix = D['FileFormat']
local_check_list = ['%s_FileLocations' % prefix, '%s_Schema' % prefix]
for attr in local_check_lis
[1/2] incubator-hawq git commit: HAWQ-1034. Implement --repair option for hawq register.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 981c0a939 -> ef2aef879
HAWQ-1034. Implement --repair option for hawq register.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/ef2aef87
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/ef2aef87
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/ef2aef87
Branch: refs/heads/master
Commit: ef2aef87958941082a016afeea45b7bbcccb9779
Parents: 637f9d5
Author: xunzhang
Authored: Sat Sep 17 20:20:43 2016 +0800
Committer: Lili Ma
Committed: Sun Sep 18 14:39:54 2016 +0800
--
tools/bin/hawqregister | 115 ++--
1 file changed, 99 insertions(+), 16 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ef2aef87/tools/bin/hawqregister
--
diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister
index ee5275b..ffae437 100755
--- a/tools/bin/hawqregister
+++ b/tools/bin/hawqregister
@@ -54,8 +54,8 @@ def option_parser():
parser.add_option('-f', '--filepath', dest='filepath', help='file name in
HDFS')
parser.add_option('-e', '--eof', dest='filesize', type='int', default=0,
help='eof of the file to be registered')
parser.add_option('-c', '--config', dest='yml_config', default='',
help='configuration file in YAML format')
-parser.add_option('--force', action='store_true', default=False)
-parser.add_option('--repair', action='store_true', default=False)
+parser.add_option('-F', '--force', dest='force', action='store_true',
default=False)
+parser.add_option('-R', '--repair', dest='repair', action='store_true',
default=False)
return parser
@@ -166,7 +166,7 @@ class GpRegisterAccessor(object):
qry = """select attrnums from gp_distribution_policy, pg_class where
pg_class.relname = '%s' and pg_class.oid = gp_distribution_policy.localoid;"""
% tablename
rows = self.exec_query(qry)
if len(rows) == 0:
-logger.error('Table %s not found in table gp_distribution_policy.'
% tablename)
+logger.error('Table %s is not an append-only table. There is no
record in gp_distribution_policy table.' % tablename)
sys.exit(1)
if rows[0]['attrnums']:
logger.error('Cannot register file(s) to a table which is hash
distribuetd.')
@@ -196,6 +196,14 @@ class GpRegisterAccessor(object):
rows = self.exec_query(query)
return rows[0]['attrnums']
+def get_bucket_number(self, tablename):
+query = "select oid from pg_class where relname = '%s';" %
tablename.split('.')[-1].lower()
+rows = self.exec_query(query)
+oid = rows[0]['oid']
+query = "select * from gp_distribution_policy where localoid = '%s';"
% oid
+rows = self.exec_query(query)
+return rows[0]['bucketnum']
+
def get_metadata_from_database(self, tablename, seg_name):
query = 'select segno from pg_aoseg.%s;' % seg_name
firstsegno = len(self.exec_query(query)) + 1
@@ -233,7 +241,7 @@ class HawqRegister(object):
return 'force'
elif repair:
if not table_existed():
-logger.error('--repair mode asserts the table is already
create.')
+logger.error('--repair mode asserts the table has been
already created.')
sys.exit(1)
return 'repair'
else:
@@ -261,32 +269,76 @@ class HawqRegister(object):
def get_metadata():
return self.accessor.get_metadata_from_database(self.tablename,
self.seg_name)
+def get_distribution_policy():
+return self.accessor.get_distribution_policy_info(self.tablename)
+
+def check_policy_consistency():
+policy = get_distribution_policy() # "" or "{1,3}"
+if policy is None:
+if ' '.join(self.distribution_policy.strip().split()).lower()
== 'distributed randomly':
+return
+else:
+logger.error('Distribution policy of %s from yaml file is
not consistent with the policy of existing table.' % self.tablename)
+sys.exit(1)
+tmp_dict = {}
+for i, d in enumerate(self.schema):
+tmp_dict[d['name']] = i + 1
+# 'DISTRIBUETD BY (1,3)' -> {1,3}
+cols =
self.distribution_policy.strip().split()[-1].strip('(').strip(')').split(',')
+original_policy = ','.join([str(tmp_dict[col]) for col in cols])
+if policy.strip('{').strip('}') != original_policy:
+logger.error('Distribution policy of %s from yaml file is not
consistent with
incubator-hawq git commit: HAWQ-1050. Support help without dash for register.
Repository: incubator-hawq Updated Branches: refs/heads/master be201dcc6 -> cd81289bc HAWQ-1050. Support help without dash for register. Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/cd81289b Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/cd81289b Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/cd81289b Branch: refs/heads/master Commit: cd81289bcbf646c415f97f3f3ee95e2304e8246b Parents: be201dc Author: xunzhang Authored: Wed Sep 14 14:35:44 2016 +0800 Committer: xunzhang Committed: Wed Sep 14 14:35:44 2016 +0800 -- tools/bin/hawq | 10 -- 1 file changed, 8 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/cd81289b/tools/bin/hawq -- diff --git a/tools/bin/hawq b/tools/bin/hawq index cb06298..82dfd51 100755 --- a/tools/bin/hawq +++ b/tools/bin/hawq @@ -6,9 +6,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -176,6 +176,12 @@ def main(): cmd = "%s; gpscp %s" % (source_hawq_env, sub_args) result = local_run(cmd) elif hawq_command == "register": +if second_arg in ['', 'help', '--help', '-h']: +doc_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../docs/cli_help/','hawqregister_help') +with open(doc_path, 'r') as f: +REGISTER_HELP = f.read() +print REGISTER_HELP +sys.exit(1) cmd = "%s; hawqregister %s" % (source_hawq_env, sub_args) result = local_run(cmd) elif hawq_command == "version" or hawq_command == "--version":
[3/3] incubator-hawq git commit: HAWQ-778. Refine hawq register, add sanity checks.
HAWQ-778. Refine hawq register, add sanity checks.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/3e3f93dd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/3e3f93dd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/3e3f93dd
Branch: refs/heads/master
Commit: 3e3f93dd1c0ddb92d4971ba0c88264c739973c31
Parents: 355c437
Author: Yancheng Luo
Authored: Fri Jun 3 14:48:33 2016 +0800
Committer: Lili Ma
Committed: Tue Jun 7 11:03:56 2016 +0800
--
src/test/feature/ManagementTool/test_hawq.paq | Bin 657 -> 0 bytes
.../ManagementTool/test_hawq_register.cpp | 78 ++
.../ManagementTool/test_hawq_register_hawq.paq | Bin 0 -> 657 bytes
.../ManagementTool/test_hawq_register_hive.paq | Bin 0 -> 212 bytes
.../ManagementTool/test_hawq_register_not_paq | Bin 0 -> 48 bytes
src/test/feature/ManagementTool/test_hive.paq | Bin 212 -> 0 bytes
src/test/feature/ManagementTool/test_not_paq| Bin 48 -> 0 bytes
tools/bin/hawqpylib/HAWQ_HELP.py| 1 +
tools/bin/hawqregister | 144 ---
tools/doc/hawqregister_help | 15 +-
10 files changed, 148 insertions(+), 90 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/3e3f93dd/src/test/feature/ManagementTool/test_hawq.paq
--
diff --git a/src/test/feature/ManagementTool/test_hawq.paq
b/src/test/feature/ManagementTool/test_hawq.paq
deleted file mode 100644
index f2adb4b..000
Binary files a/src/test/feature/ManagementTool/test_hawq.paq and /dev/null
differ
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/3e3f93dd/src/test/feature/ManagementTool/test_hawq_register.cpp
--
diff --git a/src/test/feature/ManagementTool/test_hawq_register.cpp
b/src/test/feature/ManagementTool/test_hawq_register.cpp
index 328c19d..2efbd84 100644
--- a/src/test/feature/ManagementTool/test_hawq_register.cpp
+++ b/src/test/feature/ManagementTool/test_hawq_register.cpp
@@ -19,15 +19,15 @@ class TestHawqRegister : public ::testing::Test {
TEST_F(TestHawqRegister, TestSingleHawqFile) {
SQLUtility util;
string rootPath(util.getTestRootPath());
- string relativePath("/testhawqregister/test_hawq.paq");
+ string relativePath("/ManagementTool/test_hawq_register_hawq.paq");
string filePath = rootPath + relativePath;
- EXPECT_EQ(0, Command::getCommandStatus("hadoop fs -put " + filePath + "
/hawq_register_hawq.paq"));
+ EXPECT_EQ(0, Command::getCommandStatus("hadoop fs -put " + filePath + "
hdfs://localhost:8020/hawq_register_hawq.paq"));
util.execute("create table hawqregister(i int) with (appendonly=true,
orientation=parquet);");
util.query("select * from hawqregister;", 0);
- EXPECT_EQ(0, Command::getCommandStatus("hawq register postgres
hawqregister /hawq_register_hawq.paq"));
+ EXPECT_EQ(0, Command::getCommandStatus("hawq register postgres
hawqregister hdfs://localhost:8020/hawq_register_hawq.paq"));
util.query("select * from hawqregister;", 3);
util.execute("insert into hawqregister values(1);");
@@ -38,15 +38,15 @@ TEST_F(TestHawqRegister, TestSingleHawqFile) {
TEST_F(TestHawqRegister, TestSingleHiveFile) {
SQLUtility util;
string rootPath(util.getTestRootPath());
- string relativePath("/testhawqregister/test_hive.paq");
+ string relativePath("/ManagementTool/test_hawq_register_hive.paq");
string filePath = rootPath + relativePath;
- EXPECT_EQ(0, Command::getCommandStatus("hadoop fs -put " + filePath + "
/hawq_register_hive.paq"));
+ EXPECT_EQ(0, Command::getCommandStatus("hadoop fs -put " + filePath + "
hdfs://localhost:8020/hawq_register_hive.paq"));
util.execute("create table hawqregister(i int) with (appendonly=true,
orientation=parquet);");
util.query("select * from hawqregister;", 0);
- EXPECT_EQ(0, Command::getCommandStatus("hawq register postgres
hawqregister /hawq_register_hive.paq"));
+ EXPECT_EQ(0, Command::getCommandStatus("hawq register postgres
hawqregister hdfs://localhost:8020/hawq_register_hive.paq"));
util.query("select * from hawqregister;", 1);
util.execute("insert into hawqregister values(1);");
@@ -57,83 +57,83 @@ TEST_F(TestHawqRegister, TestSingleHiveFile) {
TEST_F(TestHawqRegister, TestFiles) {
SQLUtility util;
string rootPath(util.getTestRootPath());
- string relativePath("/testhawqregister/test_hawq.paq");
+ string relativePath("/ManagementTool/test_hawq_register_hawq.paq");
string filePath1 = rootPath + relativePat
[2/3] incubator-hawq git commit: HAWQ-778. Add detail error information for hawq register
HAWQ-778. Add detail error information for hawq register
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/035b48b2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/035b48b2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/035b48b2
Branch: refs/heads/master
Commit: 035b48b2c1d055e0c38019d7065c7c5877077375
Parents: 3e3f93d
Author: Yancheng Luo
Authored: Tue Jun 7 08:58:40 2016 +0800
Committer: Lili Ma
Committed: Tue Jun 7 11:03:56 2016 +0800
--
tools/bin/hawqregister | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/035b48b2/tools/bin/hawqregister
--
diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister
index 8e0e621..fe979a9 100755
--- a/tools/bin/hawqregister
+++ b/tools/bin/hawqregister
@@ -160,11 +160,9 @@ def check_files_and_table_in_same_hdfs_cluster(filepath,
tabledir):
sys.exit(1)
fileroot = filepath.split('/')
tableroot = tabledir.split('/')
-print fileroot
-print tableroot
# check the root url of them. eg: for
'hdfs://localhost:8020/temp/tempfile', we check 'hdfs://localohst:8020'
if fileroot[0] != tableroot[0] or fileroot[1] != tableroot[1] or
fileroot[2] != tableroot[2]:
-logger.error("Files to be registered and the table are not in the same
hdfs cluster.")
+logger.error("Files to be registered and the table are not in the same
hdfs cluster.\n Files to be registered are '%s'\n Table path in HDFS is '%s'" %
(filepath, tabledir))
sys.exit(1)
[1/3] incubator-hawq git commit: HAWQ-778. Fix typos in hawq register
Repository: incubator-hawq
Updated Branches:
refs/heads/master 355c43704 -> 195909207
HAWQ-778. Fix typos in hawq register
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/19590920
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/19590920
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/19590920
Branch: refs/heads/master
Commit: 195909207313d802a110666c76187cefab96d09c
Parents: 035b48b
Author: Yancheng Luo
Authored: Tue Jun 7 10:59:33 2016 +0800
Committer: Lili Ma
Committed: Tue Jun 7 11:03:56 2016 +0800
--
tools/bin/hawqregister | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/19590920/tools/bin/hawqregister
--
diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister
index fe979a9..380a548 100755
--- a/tools/bin/hawqregister
+++ b/tools/bin/hawqregister
@@ -162,7 +162,7 @@ def check_files_and_table_in_same_hdfs_cluster(filepath,
tabledir):
tableroot = tabledir.split('/')
# check the root url of them. eg: for
'hdfs://localhost:8020/temp/tempfile', we check 'hdfs://localohst:8020'
if fileroot[0] != tableroot[0] or fileroot[1] != tableroot[1] or
fileroot[2] != tableroot[2]:
-logger.error("Files to be registered and the table are not in the same
hdfs cluster.\n Files to be registered are '%s'\n Table path in HDFS is '%s'" %
(filepath, tabledir))
+logger.error("Files to be registered and the table are not in the same
hdfs cluster.\nFile(s) to be registered: '%s'\nTable path in HDFS: '%s'" %
(filepath, tabledir))
sys.exit(1)
@@ -216,7 +216,7 @@ def move_files_in_hdfs(options, databasename, tablename,
files, firstsegno, tabl
segno += 1
if srcfile != dstfile:
hdfscmd = "hadoop fs -mv %s %s" % (srcfile, dstfile)
-sys.stdout.write("hafscmd: '%s'\n" % hdfscmd)
+sys.stdout.write("hdfscmd: '%s'\n" % hdfscmd)
result = local_ssh(hdfscmd)
if result != 0:
logger.error("Fail to move '%s' to '%s'" % (srcfile,
dstfile))
@@ -229,7 +229,7 @@ def move_files_in_hdfs(options, databasename, tablename,
files, firstsegno, tabl
segno += 1
if srcfile != dstfile:
hdfscmd = "hadoop fs -mv %s %s" % (srcfile, dstfile)
-sys.stdout.write("hafscmd: '%s'\n" % hdfscmd)
+sys.stdout.write("hdfscmd: '%s'\n" % hdfscmd)
result = local_ssh(hdfscmd)
if result != 0:
logger.error("Fail to move '%s' to '%s'" % (srcfile,
dstfile))
incubator-hawq git commit: HAWQ-774. Add snappy compression support to row oriented storage
Repository: incubator-hawq
Updated Branches:
refs/heads/master c589334d6 -> 355c43704
HAWQ-774. Add snappy compression support to row oriented storage
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/355c4370
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/355c4370
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/355c4370
Branch: refs/heads/master
Commit: 355c43704cf8fd74ed46da86f7e45ac9a0d235c0
Parents: c589334
Author: Paul Guo
Authored: Mon Jun 6 10:16:15 2016 +0800
Committer: Paul Guo
Committed: Mon Jun 6 18:18:06 2016 +0800
--
src/backend/access/common/reloptions.c | 3 +-
src/backend/catalog/pg_compression.c | 124 +++-
src/include/catalog/pg_appendonly.h| 2 +-
src/include/catalog/pg_compression.h | 2 +
src/include/catalog/pg_proc.h | 20 +
src/include/catalog/pg_proc.sql| 10 +++
src/include/utils/builtins.h | 6 ++
7 files changed, 161 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/355c4370/src/backend/access/common/reloptions.c
--
diff --git a/src/backend/access/common/reloptions.c
b/src/backend/access/common/reloptions.c
index 38b404a..8822d01 100644
--- a/src/backend/access/common/reloptions.c
+++ b/src/backend/access/common/reloptions.c
@@ -543,8 +543,7 @@ default_reloptions(Datum reloptions, bool validate, char
relkind,
errOmitLocation(true)));
}
- if (!(columnstore == RELSTORAGE_PARQUET) &&
((strcmp(compresstype, "snappy") == 0)
- || (strcmp(compresstype, "gzip") == 0)))
+ if (!(columnstore == RELSTORAGE_PARQUET) &&
(strcmp(compresstype, "gzip") == 0))
{
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/355c4370/src/backend/catalog/pg_compression.c
--
diff --git a/src/backend/catalog/pg_compression.c
b/src/backend/catalog/pg_compression.c
index 9d0ff4a..8eb51d5 100644
--- a/src/backend/catalog/pg_compression.c
+++ b/src/backend/catalog/pg_compression.c
@@ -46,6 +46,8 @@
#include "utils/relcache.h"
#include "utils/syscache.h"
+#include "snappy-c.h"
+
/* names we expect to see in ENCODING clauses */
char *storage_directive_names[] = {"compresstype", "compresslevel",
"blocksize",
NULL};
@@ -124,7 +126,7 @@ GetCompressionImplementation(char *comptype)
comptype)));
funcs = palloc0(sizeof(PGFunction) * NUM_COMPRESS_FUNCS);
-
+
ctup = (Form_pg_compression)GETSTRUCT(tuple);
Insist(OidIsValid(ctup->compconstructor));
@@ -350,6 +352,121 @@ zlib_validator(PG_FUNCTION_ARGS)
}
Datum
+snappy_constructor(PG_FUNCTION_ARGS)
+{
+ TupleDesc td = PG_GETARG_POINTER(0);
+ StorageAttributes *sa = PG_GETARG_POINTER(1);
+ CompressionState*cs = palloc0(sizeof(CompressionState));
+
+ cs->opaque = NULL;
+ cs->desired_sz = snappy_max_compressed_length;
+
+ Insist(PointerIsValid(td));
+ Insist(PointerIsValid(sa->comptype));
+
+ PG_RETURN_POINTER(cs);
+}
+
+Datum
+snappy_destructor(PG_FUNCTION_ARGS)
+{
+ CompressionState*cs = PG_GETARG_POINTER(0);
+
+ if (cs->opaque)
+ {
+ Insist(PointerIsValid(cs->opaque));
+ pfree(cs->opaque);
+ }
+
+ PG_RETURN_VOID();
+}
+
+static void
+elog_snappy_error(snappy_status retval, char *func_name,
+ int src_sz, int dst_sz, int dst_used)
+{
+ switch (retval)
+ {
+ case SNAPPY_INVALID_INPUT:
+ elog(ERROR, "invalid input for %s(): "
+"src_sz=%d dst_sz=%d dst_used=%d",
+func_name, src_sz, dst_sz, dst_used);
+ break;
+ case SNAPPY_BUFFER_TOO_SMALL:
+ elog(ERROR, "buffer is too small in %s(): "
+"src_sz=%d dst_sz=%d dst_used=%d",
+func_name, src_sz, dst_sz, dst_used);
+ break;
+ default:
+ elog(ERROR, "unknown failure (return value %d) for
%s(): "
+"src_sz=%d dst_sz=%d dst_used=%d", retval,
func_name,
+src_sz,
incubator-hawq git commit: HAWQ-760. Add hawq register function and tests
Repository: incubator-hawq
Updated Branches:
refs/heads/master ec098032e -> 2b50f7fee
HAWQ-760. Add hawq register function and tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/2b50f7fe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/2b50f7fe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/2b50f7fe
Branch: refs/heads/master
Commit: 2b50f7fee9a71ade3ca0e569e50562dde9a64bbc
Parents: ec09803
Author: Yancheng Luo
Authored: Thu Jun 2 10:13:30 2016 +0800
Committer: Lili Ma
Committed: Thu Jun 2 10:28:56 2016 +0800
--
src/backend/cdb/cdbparquetfooterserializer.c| 129 ++---
src/backend/cdb/cdbparquetstoragewrite.c| 5 +-
src/backend/executor/execMain.c | 5 +-
src/include/cdb/cdbparquetstoragewrite.h| 5 +
.../testhawqregister/test-hawq-register.cpp | 149 ++
src/test/feature/testhawqregister/test_hawq.paq | Bin 0 -> 657 bytes
src/test/feature/testhawqregister/test_hive.paq | Bin 0 -> 212 bytes
src/test/feature/testhawqregister/test_not_paq | Bin 0 -> 48 bytes
tools/bin/hawq | 3 +
tools/bin/hawqregister | 272 +++
tools/doc/hawqregister_help | 135 +
11 files changed, 669 insertions(+), 34 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2b50f7fe/src/backend/cdb/cdbparquetfooterserializer.c
--
diff --git a/src/backend/cdb/cdbparquetfooterserializer.c
b/src/backend/cdb/cdbparquetfooterserializer.c
index 802793f..b142600 100644
--- a/src/backend/cdb/cdbparquetfooterserializer.c
+++ b/src/backend/cdb/cdbparquetfooterserializer.c
@@ -25,6 +25,7 @@
*/
#include "cdb/cdbparquetfooterserializer.h"
+#include "cdb/cdbparquetstoragewrite.h"
#include "access/parquetmetadata_c++/MetadataInterface.h"
#include "lib/stringinfo.h"
#include "postgres.h"
@@ -245,7 +246,21 @@ readParquetFileMetadata(
break;
}
break;
+ case 5:
+ /* Skip this optional field now */
+ if (ftype == T_LIST) {
+ xfer += skipType(prot, ftype);
+ }
+ break;
+ case 6:
+ /* Skip this optional field now */
+ if (ftype == T_STRING) {
+ xfer += skipType(prot, ftype);
+ }
+ break;
default:
+ ereport(ERROR,
+ (errcode(ERRCODE_GP_INTERNAL_ERROR),
errmsg("file metadata field not recognized with fid: %d", fid)));
break;
}
@@ -491,7 +506,27 @@ readSchemaElement_Single(
xfer += skipType(prot, ftype);
}
break;
+ case 7:
+ /* Skip this optional field now */
+ if (ftype == T_I32) {
+ xfer += skipType(prot, ftype);
+ }
+ break;
+ case 8:
+ /* Skip this optional field now */
+ if (ftype == T_I32) {
+ xfer += skipType(prot, ftype);
+ }
+ break;
+ case 9:
+ /* Skip this optional field now */
+ if (ftype == T_I32) {
+ xfer += skipType(prot, ftype);
+ }
+ break;
default:
+ ereport(ERROR,
+ (errcode(ERRCODE_GP_INTERNAL_ERROR),
errmsg("file metadata: schema element field not recognizd with fid: %d", fid)));
break;
}
}
@@ -567,7 +602,15 @@ readRowGroupInfo(
isset_num_rows = true;
}
break;
+ case 4:
+ /* Skip this optional field now */
+ if (ftype == T_LIST) {
+ xfer += skipType(prot, ftype);
+ }
+ break;
default:
+ ereport(ERROR,
+ (errcode(ERRCODE_GP_INTERNAL_ERROR),
errmsg("file metadata: row group field not recognized with fid: %d", fid)));
break;
}
}
@@ -633,6 +676,8 @@ readColumnChunk(
incubator-hawq git commit: HAWQ-731. Implement Data Generator
Repository: incubator-hawq
Updated Branches:
refs/heads/master c3d8546e7 -> c0dcb5d60
HAWQ-731. Implement Data Generator
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/c0dcb5d6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/c0dcb5d6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/c0dcb5d6
Branch: refs/heads/master
Commit: c0dcb5d6049bdd621d6750127a900c0f6dd4fb9d
Parents: c3d8546
Author: Lili Ma
Authored: Wed May 11 18:09:52 2016 +0800
Committer: Lili Ma
Committed: Thu May 12 08:10:42 2016 +0800
--
src/test/feature/lib/Makefile | 2 +-
src/test/feature/lib/data-gen.cpp | 138 +
src/test/feature/lib/data-gen.h | 58
src/test/feature/lib/psql.cpp | 2 +-
src/test/feature/testlib/test-lib.cpp | 12 +++
5 files changed, 210 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c0dcb5d6/src/test/feature/lib/Makefile
--
diff --git a/src/test/feature/lib/Makefile b/src/test/feature/lib/Makefile
index 8c7d74d..411613a 100644
--- a/src/test/feature/lib/Makefile
+++ b/src/test/feature/lib/Makefile
@@ -8,7 +8,7 @@ override CPPFLAGS := -I/usr/include -I/usr/local/include
-I/usr/include/libxml2
override LIBS := $(LIBS) -lpq -lxml2
override LDFLAGS += -L/usr/local/lib -L/usr/lib
-PROG = string-util.cpp psql.cpp command.cpp xml-parser.cpp hawq-config.cpp
+PROG = string-util.cpp psql.cpp command.cpp xml-parser.cpp hawq-config.cpp
data-gen.cpp
all:
g++ $(CPPFLAGS) $(CXXFLAGS) $(PROG)
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c0dcb5d6/src/test/feature/lib/data-gen.cpp
--
diff --git a/src/test/feature/lib/data-gen.cpp
b/src/test/feature/lib/data-gen.cpp
new file mode 100644
index 000..da58b32
--- /dev/null
+++ b/src/test/feature/lib/data-gen.cpp
@@ -0,0 +1,138 @@
+#include
+#include "data-gen.h"
+
+void
+DataGenerator::genSimpleTable(std::string tableName,
+ bool appendonly,
+ std::string orientation,
+ std::string compresstype,
+ int compresslevel) {
+ std::string desc =
+ genTableDesc(appendonly, orientation, compresstype, compresslevel);
+ std::string createSql = "create table " + tableName
+ + "(a int, b int) " + desc;
+ sqlUtil.execute(createSql);
+
+ std::string insertSql = "insert into " + tableName
+ + " values(51,62), (14,15), (1,3);";
+ sqlUtil.execute(insertSql);
+}
+
+void
+DataGenerator::genTableWithSeries(std::string tableName,
+ bool appendonly,
+ std::string orientation,
+ std::string compresstype,
+ int compresslevel) {
+ std::string desc =
+genTableDesc(appendonly, orientation, compresstype, compresslevel);
+ std::string createSql = "create table " + tableName
+ + "(a int, b varchar(20)) " + desc;
+ sqlUtil.execute(createSql);
+
+ std::string insertSql = "insert into " + tableName
+ + " values(generate_series(1,1), 'abc')";
+ sqlUtil.execute(insertSql);
+}
+
+
+void
+DataGenerator::genTableWithFullTypes(std::string tableName,
+ bool appendonly,
+ std::string orientation,
+ std::string compresstype,
+ int compresslevel) {
+ std::string desc =
+ genTableDesc(appendonly, orientation, compresstype, compresslevel);
+ std::string createSql = "create table " + tableName + "(c0 int4, c1 polygon,
"
+ "c2 text, c3 time, c4 timetz, c5 macaddr, c6 timestamptz, c7 char(10), "
+ "c8 int2, c9 bool, c10 cidr, c11 circle, c12 lseg, c13 interval, "
+ "c14 bit, c15 money, c16 box, c17 bytea, c18 xml, c19 bit(5), "
+ "c20 varchar(10), c21 inet, c22 int8, c23 varbit, c24 serial, "
+ "c25 float4, c26 point, c27 date, c28 float8) " + desc;
+ sqlUtil.execute(createSql);
+
+ std::string insertSql = "insert into " + tableName +
+ " values (2147483647, null, null, '00:00:00', null, 'FF:89:71:45:AE:01',"
+ " '2000-01-01 08:00:00+09', null, 32767, 'true', '192.168.1.255/32', "
+ "'<(1,2),3>', '[(0,0),(6,6)]', '-17800 years', '0', '-21474836.48', "
+ "'((100,200),(200,400))', null, 'bb', null, '123456789a', "
+ "'2001:db8:85a3:8d3:1319:8a2e:370:7344/64', null, null, 1, 0,
POINT(1,2),"
+ " '4277-12-31 AD', 128);";
+ sqlUtil.execute(insertSql);
+
+ std::string insertSql2 = "insert into
incubator-hawq git commit: HAWQ-642. Make sure the number of records generated in AppendOnlyWriter equal to the number of QEs in function SetSegnoForWrite when we do insert/copy
Repository: incubator-hawq
Updated Branches:
refs/heads/master a313c5472 -> 300d9149f
HAWQ-642. Make sure the number of records generated in AppendOnlyWriter equal
to the number of QEs in function SetSegnoForWrite when we do insert/copy
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/300d9149
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/300d9149
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/300d9149
Branch: refs/heads/master
Commit: 300d9149fd1772ccd88f3855751d34117218de22
Parents: a313c54
Author: Lili Ma
Authored: Fri Apr 8 14:24:00 2016 +0800
Committer: Lili Ma
Committed: Fri Apr 8 16:07:36 2016 +0800
--
src/backend/access/appendonly/appendonlywriter.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/300d9149/src/backend/access/appendonly/appendonlywriter.c
--
diff --git a/src/backend/access/appendonly/appendonlywriter.c
b/src/backend/access/appendonly/appendonlywriter.c
index 40d097c..3c7f9a2 100644
--- a/src/backend/access/appendonly/appendonlywriter.c
+++ b/src/backend/access/appendonly/appendonlywriter.c
@@ -1139,7 +1139,8 @@ List *SetSegnoForWrite(List *existing_segnos, Oid relid,
int segment_num,
if (remaining_num > 0)
{
//generate new segment_num to make sure that in keepHash mode,
all segment node has at least one segfile is writable
-for(int i= 1; i<= segment_num; i++)
+int newAllocatedNum = remaining_num;
+for(int i= 1; i<= newAllocatedNum; i++)
{
int new_status = AORelGetSegfileStatus(aoentry);
if (new_status == NEXT_END_OF_LIST)
[1/2] incubator-hawq git commit: HAWQ-619. Add comments for inputformat test cases.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 92d5acd34 -> ec8e79184
HAWQ-619. Add comments for inputformat test cases.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/ec8e7918
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/ec8e7918
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/ec8e7918
Branch: refs/heads/master
Commit: ec8e791849e6d549d452f5211a7e9a7774b1a783
Parents: 53a9f76
Author: Chunling Wang
Authored: Tue Apr 5 12:03:19 2016 +0800
Committer: Lili Ma
Committed: Tue Apr 5 13:46:19 2016 +0800
--
.../java/com/pivotal/hawq/mapreduce/SimpleTableLocalTester.java | 4
1 file changed, 4 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ec8e7918/contrib/hawq-hadoop/hawq-mapreduce-tool/src/test/java/com/pivotal/hawq/mapreduce/SimpleTableLocalTester.java
--
diff --git
a/contrib/hawq-hadoop/hawq-mapreduce-tool/src/test/java/com/pivotal/hawq/mapreduce/SimpleTableLocalTester.java
b/contrib/hawq-hadoop/hawq-mapreduce-tool/src/test/java/com/pivotal/hawq/mapreduce/SimpleTableLocalTester.java
index bcec9f1..9c064a2 100644
---
a/contrib/hawq-hadoop/hawq-mapreduce-tool/src/test/java/com/pivotal/hawq/mapreduce/SimpleTableLocalTester.java
+++
b/contrib/hawq-hadoop/hawq-mapreduce-tool/src/test/java/com/pivotal/hawq/mapreduce/SimpleTableLocalTester.java
@@ -51,6 +51,8 @@ public class SimpleTableLocalTester extends SimpleTableTester
{
final File answerFile = new File(caseFolder, tableName +
".ans");
final File metadataFile = new File(caseFolder, tableName +
".yaml");
final File outputFile = new File(caseFolder,
"output/part-r-0");
+ final String caseName = tableName.replaceAll("\\.", "_");
+ System.out.println("Executing test case: " + caseName);
List answers;
@@ -106,5 +108,7 @@ public class SimpleTableLocalTester extends
SimpleTableTester {
// compare result
List outputs = Files.readLines(outputFile,
Charsets.UTF_8);
checkOutput(answers, outputs, table);
+
+ System.out.println("Successfully finish test case: " +
caseName);
}
}
[2/2] incubator-hawq git commit: HAWQ-619. Change gpextract to hawqextract for InputFormat unit test.
HAWQ-619. Change gpextract to hawqextract for InputFormat unit test.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/53a9f76f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/53a9f76f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/53a9f76f
Branch: refs/heads/master
Commit: 53a9f76f04d3f56684f3c0e3cb3dd17ba1ae1997
Parents: 92d5acd
Author: Chunling Wang
Authored: Fri Apr 1 17:00:07 2016 +0800
Committer: Lili Ma
Committed: Tue Apr 5 13:46:19 2016 +0800
--
.../java/com/pivotal/hawq/mapreduce/SimpleTableLocalTester.java| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/53a9f76f/contrib/hawq-hadoop/hawq-mapreduce-tool/src/test/java/com/pivotal/hawq/mapreduce/SimpleTableLocalTester.java
--
diff --git
a/contrib/hawq-hadoop/hawq-mapreduce-tool/src/test/java/com/pivotal/hawq/mapreduce/SimpleTableLocalTester.java
b/contrib/hawq-hadoop/hawq-mapreduce-tool/src/test/java/com/pivotal/hawq/mapreduce/SimpleTableLocalTester.java
index ca226c8..bcec9f1 100644
---
a/contrib/hawq-hadoop/hawq-mapreduce-tool/src/test/java/com/pivotal/hawq/mapreduce/SimpleTableLocalTester.java
+++
b/contrib/hawq-hadoop/hawq-mapreduce-tool/src/test/java/com/pivotal/hawq/mapreduce/SimpleTableLocalTester.java
@@ -78,7 +78,7 @@ public class SimpleTableLocalTester extends SimpleTableTester
{
// extract metadata
MRFormatTestUtils.runShellCommand(
- String.format("gpextract -d %s
-o %s %s",
+ String.format("hawqextract -d
%s -o %s %s",
TEST_DB_NAME, metadataFile.getPath(), tableName));
// copy data files to local in order to run
local mapreduce job
incubator-hawq git commit: HAWQ-625. Fix the build failure on MAC: function referenced before declaration
Repository: incubator-hawq
Updated Branches:
refs/heads/master cf2744dc5 -> 92d5acd34
HAWQ-625. Fix the build failure on MAC: function referenced before declaration
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/92d5acd3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/92d5acd3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/92d5acd3
Branch: refs/heads/master
Commit: 92d5acd341ee67c1d926882ee091ec7bde8e9ad2
Parents: cf2744d
Author: Lili Ma
Authored: Tue Apr 5 11:17:41 2016 +0800
Committer: Lili Ma
Committed: Tue Apr 5 11:21:19 2016 +0800
--
src/backend/cdb/cdbquerycontextdispatching.c | 5 -
src/bin/gpfusion/gpbridgeapi.c | 1 +
2 files changed, 5 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/92d5acd3/src/backend/cdb/cdbquerycontextdispatching.c
--
diff --git a/src/backend/cdb/cdbquerycontextdispatching.c
b/src/backend/cdb/cdbquerycontextdispatching.c
index b3cf083..c6647cd 100644
--- a/src/backend/cdb/cdbquerycontextdispatching.c
+++ b/src/backend/cdb/cdbquerycontextdispatching.c
@@ -172,6 +172,9 @@ static void
prepareDispatchedCatalogExternalTable(QueryContextInfo *cxt, Oid relid);
static void
+prepareDfsAddressForDispatch(QueryContextInfo* cxt);
+
+static void
addOperatorToDispatchContext(QueryContextInfo *ctx, Oid opOid);
static void
@@ -3022,7 +3025,7 @@ GetResultRelSegFileInfos(Oid relid, List *segnomaps, List
*existing_seginfomaps)
* a secure filesystem.
*/
-void
+static void
prepareDfsAddressForDispatch(QueryContextInfo* cxt)
{
if (!enable_secure_filesystem)
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/92d5acd3/src/bin/gpfusion/gpbridgeapi.c
--
diff --git a/src/bin/gpfusion/gpbridgeapi.c b/src/bin/gpfusion/gpbridgeapi.c
index 63fc565..bd304ac 100644
--- a/src/bin/gpfusion/gpbridgeapi.c
+++ b/src/bin/gpfusion/gpbridgeapi.c
@@ -62,6 +62,7 @@ void build_uri_for_write(gphadoop_context* context,
PxfServer* rest_server);
size_t fill_buffer(gphadoop_context* context, char* start, size_t size);
void add_delegation_token(PxfInputData *inputData);
void free_token_resources(PxfInputData *inputData);
+void free_dfs_address();
/* Custom protocol entry point for read
*/
[1/2] incubator-hawq git commit: HAWQ-592. QD shouldn't try to reconnect to segment when meeting invalid QE for queries which only need to dispatch to all cached QEs
Repository: incubator-hawq
Updated Branches:
refs/heads/master 6107a7ce3 -> ae8aff2ea
HAWQ-592. QD shouldn't try to reconnect to segment when meeting invalid QE for
queries which only need to dispatch to all cached QEs
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/84d40fa6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/84d40fa6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/84d40fa6
Branch: refs/heads/master
Commit: 84d40fa606b8e3d90faab68988bde64c2ddb8d1a
Parents: 6107a7c
Author: Lili Ma
Authored: Wed Mar 30 15:33:37 2016 +0800
Committer: Lili Ma
Committed: Thu Mar 31 12:10:53 2016 +0800
--
src/backend/cdb/dispatcher.c | 41 +--
src/backend/cdb/dispatcher_mgt.c | 14 +---
src/backend/cdb/executormgr.c| 5 +
src/include/cdb/dispatcher_mgt.h | 3 ++-
src/include/cdb/executormgr.h| 1 +
5 files changed, 44 insertions(+), 20 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/84d40fa6/src/backend/cdb/dispatcher.c
--
diff --git a/src/backend/cdb/dispatcher.c b/src/backend/cdb/dispatcher.c
index d2ee4b7..67530fd 100644
--- a/src/backend/cdb/dispatcher.c
+++ b/src/backend/cdb/dispatcher.c
@@ -946,22 +946,25 @@ dispatcher_bind_executor(DispatchData *data)
DispatchTask*task = &slice->tasks[j];
struct SegmentDatabaseDescriptor *desc;
- executor =
dispmgt_get_query_executor_iterator(&iterator);
+ executor =
dispmgt_get_query_executor_iterator(&iterator, false);
desc = executormgr_allocate_executor(task->segment,
task->id.is_writer, task->entrydb);
if (!desc)
{
- struct ConcurrentConnectExecutorInfo *info;
-
- info =
dispmgt_build_preconnect_info(task->segment,
-
task->id.is_writer,
-
executor,
-
data,
-
slice,
-
task);
- concurrent_connect_executors =
lappend(concurrent_connect_executors, info);
- data->num_of_new_connected_executors++;
- continue;
+ // may have problem for tasks assigned to entrydb for
dispatch_statement_node
+ // and dispatch_statement_string
+ if (task->segment != NULL)
+ {
+ struct ConcurrentConnectExecutorInfo *info;
+
+ info = dispmgt_build_preconnect_info(task->segment,
+ task->id.is_writer, executor,
+ data, slice, task);
+ concurrent_connect_executors =
lappend(concurrent_connect_executors,
+ info);
+ data->num_of_new_connected_executors++;
+ }
+ continue;
}
#ifdef FAULT_INJECTOR
@@ -1021,7 +1024,9 @@ dispatcher_serialize_state(DispatchData *data)
{
DispatchTask*task = &slice->tasks[j];
- executor =
dispmgt_get_query_executor_iterator(&iterator);
+ executor =
dispmgt_get_query_executor_iterator(&iterator, true);
+ if (executor == NULL)
+ continue;
executormgr_serialize_executor_state(data, executor,
task, slice);
if (sliceTable)
@@ -1094,7 +1099,9 @@ dispatcher_unbind_executor(DispatchData *data)
{
DispatchTask*task = &slice->tasks[j];
- executor =
dispmgt_get_query_executor_iterator(&iterator);
+ executor =
dispmgt_get_query_executor_iterator(&iterator, true);
+ if (executor == NULL)
+ continue;
executormgr_unbind_executor_task(data, executor, task,
slice);
}
}
@@ -12
[2/2] incubator-hawq git commit: HAWQ-607. Fix bug: Copy large table to file, and kill QE process, copy doesn't fail.
HAWQ-607. Fix bug: Copy large table to file, and kill QE process, copy doesn't
fail.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/ae8aff2e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/ae8aff2e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/ae8aff2e
Branch: refs/heads/master
Commit: ae8aff2ea3b93abaa17a2cde3cdc9a5be3689466
Parents: 84d40fa
Author: Lili Ma
Authored: Wed Mar 30 17:57:13 2016 +0800
Committer: Lili Ma
Committed: Thu Mar 31 12:11:16 2016 +0800
--
src/backend/cdb/cdbcopy.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae8aff2e/src/backend/cdb/cdbcopy.c
--
diff --git a/src/backend/cdb/cdbcopy.c b/src/backend/cdb/cdbcopy.c
index 98f0510..6474bcc 100644
--- a/src/backend/cdb/cdbcopy.c
+++ b/src/backend/cdb/cdbcopy.c
@@ -445,6 +445,8 @@ bool cdbCopyGetData(CdbCopy *c, bool copy_cancel, uint64
*rows_processed)
*/
else
{
+ c->io_errors = true;
+
/* should never happen since we are blocking.
Don't bother to try again, exit with error. */
if (nbytes == 0)
appendStringInfo(&(c->err_msg),
@@ -467,8 +469,6 @@ bool cdbCopyGetData(CdbCopy *c, bool copy_cancel, uint64
*rows_processed)
/* start over from first seg as we just
changes the seg list */
break;
}
-
- c->io_errors = true;
}
}
incubator-hawq git commit: HAWQ-572. Add fault injection in parquet_getnext().
Repository: incubator-hawq
Updated Branches:
refs/heads/master aea5e6046 -> adbf7c549
HAWQ-572. Add fault injection in parquet_getnext().
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/adbf7c54
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/adbf7c54
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/adbf7c54
Branch: refs/heads/master
Commit: adbf7c549927dbeba22501c139c7819b3b94d833
Parents: aea5e60
Author: Chunling Wang
Authored: Fri Mar 25 18:23:44 2016 +0800
Committer: Lili Ma
Committed: Wed Mar 30 10:08:44 2016 +0800
--
src/backend/access/parquet/parquetam.c | 8
src/backend/cdb/dispatcher.c | 2 +-
src/backend/utils/misc/faultinjector.c | 5 -
src/include/utils/faultinjector.h | 3 ++-
tools/bin/gppylib/programs/clsInjectFault.py | 1 +
tools/bin/hawqpylib/programs/clsInjectFault.py | 1 +
6 files changed, 17 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/adbf7c54/src/backend/access/parquet/parquetam.c
--
diff --git a/src/backend/access/parquet/parquetam.c
b/src/backend/access/parquet/parquetam.c
index a01f89b..bec8422 100644
--- a/src/backend/access/parquet/parquetam.c
+++ b/src/backend/access/parquet/parquetam.c
@@ -266,6 +266,14 @@ void parquet_getnext(ParquetScanDesc scan, ScanDirection
direction,
AOTupleId aoTupleId;
Assert(ScanDirectionIsForward(direction));
+#ifdef FAULT_INJECTOR
+ FaultInjector_InjectFaultIfSet(
+
FailQeWhenParquetGetNext,
+
DDLNotSpecified,
+
"", // databaseName
+
""); // tableName
+#endif
+
for(;;)
{
if(scan->bufferDone)
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/adbf7c54/src/backend/cdb/dispatcher.c
--
diff --git a/src/backend/cdb/dispatcher.c b/src/backend/cdb/dispatcher.c
index ad826d3..d2ee4b7 100644
--- a/src/backend/cdb/dispatcher.c
+++ b/src/backend/cdb/dispatcher.c
@@ -1239,7 +1239,7 @@ dispatch_run(DispatchData *data)
#ifdef FAULT_INJECTOR
FaultInjector_InjectFaultIfSet(
-
FaillQeAfterConnection,
+
FailQeAfterConnection,
DDLNotSpecified,
"", // databaseName
""); // tableName
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/adbf7c54/src/backend/utils/misc/faultinjector.c
--
diff --git a/src/backend/utils/misc/faultinjector.c
b/src/backend/utils/misc/faultinjector.c
index a760b4d..b1dea54 100644
--- a/src/backend/utils/misc/faultinjector.c
+++ b/src/backend/utils/misc/faultinjector.c
@@ -293,6 +293,8 @@ FaultInjectorIdentifierEnumToString[] = {
/* inject fault when QE actually working, set error */
_("fail_qe_when_begin_parquet_scan"),
/* inject fault when begin scan parquet table, set error */
+ _("fail_qe_when_parquet_get_next"),
+ /* inject fault when get next, set error */
/* _("make_dispatch_thread"),
inject fault when initialing memory structure for dispatcher
thread*/
_("before_dispatch"),
@@ -1083,7 +1085,7 @@ FaultInjector_NewHashEntry(
case ConnectionFailAfterGangCreation:
case CreateCdbDispathResultObject:
case WorkerManagerSubmitJob:
- case FaillQeAfterConnection:
+ case FailQeAfterConnection:
/* These faults are designed for master. */
if(!AmIMaster())
@@ -,6 +1113,7 @@ FaultInjector_NewHashEntry(
case MultiExecHashLargeVmem:
case FailQeWhenDoQuery:
case FailQeWhenBeginParquetScan:
+ case FailQeWhenParquetGetNext:
/* SEGMENT */
incubator-hawq git commit: HAWQ-568. Check validation the QD to QE connection by checking whether the recv character is 'E' which means there's error on the other side
Repository: incubator-hawq
Updated Branches:
refs/heads/master 7c9009f0c -> 61598138b
HAWQ-568. Check validation the QD to QE connection by checking whether the recv
character is 'E' which means there's error on the other side
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/61598138
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/61598138
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/61598138
Branch: refs/heads/master
Commit: 61598138b02e2336ad4a8e8e3116e4e5b8245dbd
Parents: 7c9009f
Author: Lili Ma
Authored: Tue Mar 29 10:53:49 2016 +0800
Committer: Lili Ma
Committed: Tue Mar 29 10:55:49 2016 +0800
--
src/backend/cdb/dispatcher.c | 10 +++---
1 file changed, 7 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/61598138/src/backend/cdb/dispatcher.c
--
diff --git a/src/backend/cdb/dispatcher.c b/src/backend/cdb/dispatcher.c
index f355b05..ad826d3 100644
--- a/src/backend/cdb/dispatcher.c
+++ b/src/backend/cdb/dispatcher.c
@@ -1851,9 +1851,13 @@ bool dispatch_validate_conn(pgsocket sock)
if (ret == 0) /* socket has been closed. EOF */
return false;
- if (ret > 0) /* data waiting on socket, it must be OK. */
-return true;
-
+ if (ret > 0) /* data waiting on socket */
+ {
+if (buf == 'E') /* waiting data indicates error */
+ return false;
+else
+ return true;
+ }
if (ret == -1) /* error, or would be block. */
{
if (errno == EAGAIN || errno == EINPROGRESS || errno == EWOULDBLOCK)
incubator-hawq git commit: HAWQ-598. Initialize dispatchResult during makeCdbCopy
Repository: incubator-hawq Updated Branches: refs/heads/master 6947d1989 -> 7c9009f0c HAWQ-598. Initialize dispatchResult during makeCdbCopy Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/7c9009f0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/7c9009f0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/7c9009f0 Branch: refs/heads/master Commit: 7c9009f0c1202405e5c2712866e88112ce0aeb9c Parents: 6947d19 Author: Lili Ma Authored: Mon Mar 28 17:16:39 2016 +0800 Committer: Lili Ma Committed: Tue Mar 29 09:46:43 2016 +0800 -- src/backend/cdb/cdbcopy.c | 2 ++ 1 file changed, 2 insertions(+) -- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7c9009f0/src/backend/cdb/cdbcopy.c -- diff --git a/src/backend/cdb/cdbcopy.c b/src/backend/cdb/cdbcopy.c index ee2ff80..98f0510 100644 --- a/src/backend/cdb/cdbcopy.c +++ b/src/backend/cdb/cdbcopy.c @@ -80,6 +80,8 @@ makeCdbCopy(bool is_copy_in, QueryResource *resource) c->executors.segment_conns = NIL; c->executors.errbuf.data = NULL; + c->executors.result = NULL; + c->executors.numresults = 0; /* Initialize the state of each segment database */ c->segdb_state = (SegDbState **) palloc((c->partition_num) * sizeof(SegDbState *));
incubator-hawq git commit: HAWQ-591. Explain analyze doesn't enter function cdbexplain_recvExecStats after QD finding QE error
Repository: incubator-hawq
Updated Branches:
refs/heads/master f86b7a03f -> e224fdb59
HAWQ-591. Explain analyze doesn't enter function cdbexplain_recvExecStats after
QD finding QE error
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/e224fdb5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/e224fdb5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/e224fdb5
Branch: refs/heads/master
Commit: e224fdb59b3e11e983f55ae0289ce271d50b9457
Parents: f86b7a0
Author: Lili Ma
Authored: Fri Mar 25 17:08:24 2016 +0800
Committer: Lili Ma
Committed: Mon Mar 28 09:49:45 2016 +0800
--
src/backend/commands/explain.c | 2 +-
src/backend/executor/nodeSubplan.c | 8 ++--
2 files changed, 7 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/e224fdb5/src/backend/commands/explain.c
--
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 0e7bff5..3b64d22 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -584,7 +584,7 @@ ExplainOnePlan_internal(PlannedStmt *plannedstmt,
cdbexplain_localExecStats(queryDesc->planstate, es->showstatctx);
/* Fill in the plan's Instrumentation with stats from qExecs. */
-if (estate->dispatch_data)
+if (estate->dispatch_data &&
!dispatcher_has_error(estate->dispatch_data))
cdbexplain_recvExecStats(queryDesc->planstate,
dispatch_get_results(estate->dispatch_data),
LocallyExecutingSliceIndex(estate),
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/e224fdb5/src/backend/executor/nodeSubplan.c
--
diff --git a/src/backend/executor/nodeSubplan.c
b/src/backend/executor/nodeSubplan.c
index 1a9b31b..999ffb4 100644
--- a/src/backend/executor/nodeSubplan.c
+++ b/src/backend/executor/nodeSubplan.c
@@ -1305,13 +1305,17 @@ ExecSetParamPlan(SubPlanState *node, ExprContext
*econtext, QueryDesc *gbl_query
/* Jam stats into subplan's Instrumentation nodes. */
explainRecvStats = true;
-cdbexplain_recvExecStats(planstate,
+if (queryDesc->estate->dispatch_data &&
+!dispatcher_has_error(queryDesc->estate->dispatch_data))
+{
+ cdbexplain_recvExecStats(planstate,
dispatch_get_results(queryDesc->estate->dispatch_data),
LocallyExecutingSliceIndex(queryDesc->estate),
econtext->ecxt_estate->showstatctx,
dispatch_get_segment_num(queryDesc->estate->dispatch_data));
-}
+}
+}
/*
* Wait for all gangs to finish. Check and free the results.
* If the dispatcher or any QE had an error, report it and
incubator-hawq git commit: HAWQ-574. Modify the wrong judge during dispatch_free_result
Repository: incubator-hawq
Updated Branches:
refs/heads/master 4d0b99f3b -> e4cee64a4
HAWQ-574. Modify the wrong judge during dispatch_free_result
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/e4cee64a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/e4cee64a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/e4cee64a
Branch: refs/heads/master
Commit: e4cee64a46b9fae5758f7fdd4917185c74f86810
Parents: 4d0b99f
Author: Lili Ma
Authored: Wed Mar 23 13:28:52 2016 +0800
Committer: Lili Ma
Committed: Wed Mar 23 13:29:31 2016 +0800
--
src/backend/cdb/dispatcher.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/e4cee64a/src/backend/cdb/dispatcher.c
--
diff --git a/src/backend/cdb/dispatcher.c b/src/backend/cdb/dispatcher.c
index e3c5e94..f355b05 100644
--- a/src/backend/cdb/dispatcher.c
+++ b/src/backend/cdb/dispatcher.c
@@ -1570,7 +1570,7 @@ dispatch_free_result(DispatchDataResult *result)
result->errbuf.data = NULL;
}
- if (!result->result)
+ if (result->result)
{
for (int i = 0; i <= result->numresults; i++){
PQclear(result->result[i]);
incubator-hawq git commit: HAWQ-574. Fix memory leak in dispatcher
Repository: incubator-hawq
Updated Branches:
refs/heads/master 31fdef71c -> 26a5b2029
HAWQ-574. Fix memory leak in dispatcher
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/26a5b202
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/26a5b202
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/26a5b202
Branch: refs/heads/master
Commit: 26a5b202906bb62016bba0b1c865d9478bb6f53e
Parents: 31fdef7
Author: Lili Ma
Authored: Wed Mar 23 09:48:59 2016 +0800
Committer: Lili Ma
Committed: Wed Mar 23 09:48:59 2016 +0800
--
src/backend/cdb/dispatcher.c | 10 ++
src/backend/gp_libpq_fe/fe-exec.c | 13 -
2 files changed, 18 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/26a5b202/src/backend/cdb/dispatcher.c
--
diff --git a/src/backend/cdb/dispatcher.c b/src/backend/cdb/dispatcher.c
index d9d0771..e3c5e94 100644
--- a/src/backend/cdb/dispatcher.c
+++ b/src/backend/cdb/dispatcher.c
@@ -1569,6 +1569,16 @@ dispatch_free_result(DispatchDataResult *result)
pfree(result->errbuf.data);
result->errbuf.data = NULL;
}
+
+ if (!result->result)
+ {
+ for (int i = 0; i <= result->numresults; i++){
+ PQclear(result->result[i]);
+ result->result[i] = NULL;
+ }
+ free(result->result);
+ }
+
dispmgt_free_takeoved_segment_conns(result->segment_conns);
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/26a5b202/src/backend/gp_libpq_fe/fe-exec.c
--
diff --git a/src/backend/gp_libpq_fe/fe-exec.c
b/src/backend/gp_libpq_fe/fe-exec.c
index 0285430..8382683 100644
--- a/src/backend/gp_libpq_fe/fe-exec.c
+++ b/src/backend/gp_libpq_fe/fe-exec.c
@@ -681,11 +681,14 @@ PQclear(PGresult *res)
if (res->sendback)
{
- if (res->sendback->eof)
- free(res->sendback->eof);
- if (res->sendback->uncompressed_eof)
- free(res->sendback->uncompressed_eof);
- free(res->sendback);
+ for (int i = 0; i < res->numSendback; i++)
+ {
+ if (res->sendback[i].eof)
+ free(res->sendback[i].eof);
+ if (res->sendback[i].uncompressed_eof)
+ free(res->sendback[i].uncompressed_eof);
+ }
+ free(res->sendback);
}
/* Free the PGresult structure itself */
incubator-hawq git commit: HAWQ-572. Add fault injection test cases for QE side
Repository: incubator-hawq Updated Branches: refs/heads/master a84d55c34 -> 31fdef71c HAWQ-572. Add fault injection test cases for QE side Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/31fdef71 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/31fdef71 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/31fdef71 Branch: refs/heads/master Commit: 31fdef71c1d77070b1db84d37c460127487456cd Parents: a84d55c Author: Chunling Wang Authored: Tue Mar 22 16:49:07 2016 +0800 Committer: Chunling Wang Committed: Tue Mar 22 16:49:07 2016 +0800 -- src/backend/access/parquet/parquetam.c | 9 + src/backend/cdb/dispatcher.c | 9 + src/backend/tcop/pquery.c | 9 + src/backend/utils/misc/faultinjector.c | 9 + src/include/utils/faultinjector.h | 3 +++ tools/bin/gppylib/programs/clsInjectFault.py | 3 +++ tools/bin/hawqpylib/programs/clsInjectFault.py | 3 +++ 7 files changed, 45 insertions(+) -- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/31fdef71/src/backend/access/parquet/parquetam.c -- diff --git a/src/backend/access/parquet/parquetam.c b/src/backend/access/parquet/parquetam.c index 3c2b7df..a01f89b 100644 --- a/src/backend/access/parquet/parquetam.c +++ b/src/backend/access/parquet/parquetam.c @@ -33,6 +33,7 @@ #include "utils/relcache.h" #include "utils/tqual.h" #include "utils/rel.h" +#include "utils/faultinjector.h" #include "access/aomd.h" #include "cdb/cdbvars.h" #include "cdb/cdbparquetam.h" @@ -149,6 +150,14 @@ parquet_beginscan( scan->aoEntry = aoEntry; Assert(aoEntry->majorversion == 1 && aoEntry->minorversion == 0); +#ifdef FAULT_INJECTOR + FaultInjector_InjectFaultIfSet( + FailQeWhenBeginParquetScan, + DDLNotSpecified, + "", // databaseName + ""); // tableName +#endif + /* * initialize the scan descriptor */ http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/31fdef71/src/backend/cdb/dispatcher.c -- diff --git a/src/backend/cdb/dispatcher.c b/src/backend/cdb/dispatcher.c index 55a7823..d9d0771 100644 --- a/src/backend/cdb/dispatcher.c +++ b/src/backend/cdb/dispatcher.c @@ -1236,6 +1236,15 @@ dispatch_run(DispatchData *data) INSTR_TIME_SET_CURRENT(data->time_begin); if (!dispatcher_bind_executor(data)) goto error; + +#ifdef FAULT_INJECTOR + FaultInjector_InjectFaultIfSet( + FaillQeAfterConnection, + DDLNotSpecified, + "", // databaseName + ""); // tableName +#endif + /* * Only after we have the executors, we can serialize the state. Or we * don't know the executor listening address. http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/31fdef71/src/backend/tcop/pquery.c -- diff --git a/src/backend/tcop/pquery.c b/src/backend/tcop/pquery.c index f097f01..99b6a1d 100644 --- a/src/backend/tcop/pquery.c +++ b/src/backend/tcop/pquery.c @@ -43,6 +43,7 @@ #include "tcop/pquery.h" #include "tcop/tcopprot.h" #include "tcop/utility.h" +#include "utils/faultinjector.h" #include "utils/memutils.h" #include "utils/resscheduler.h" #include "commands/vacuum.h" @@ -1410,6 +1411,14 @@ PortalRun(Portal portal, int64 count, bool isTopLevel, AssertArg(PortalIsValid(portal)); +#ifdef FAULT_INJECTOR + FaultInjector_InjectFaultIfSet( + FailQeWhenDoQuery, + DDLNotSpecified, + "", // databaseName +
incubator-hawq git commit: HAWQ-559. Fix QD hang when QE is immediately killed after created
Repository: incubator-hawq
Updated Branches:
refs/heads/master 6e88cbabf -> 0bb22dce1
HAWQ-559. Fix QD hang when QE is immediately killed after created
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/0bb22dce
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/0bb22dce
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/0bb22dce
Branch: refs/heads/master
Commit: 0bb22dce146cbd34507293b32f5603a6dc706aec
Parents: 6e88cba
Author: Lili Ma
Authored: Fri Mar 18 16:49:06 2016 +0800
Committer: Lili Ma
Committed: Mon Mar 21 09:49:06 2016 +0800
--
src/backend/cdb/dispatcher.c | 2 ++
src/backend/cdb/executormgr.c | 20 +++-
2 files changed, 17 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/0bb22dce/src/backend/cdb/dispatcher.c
--
diff --git a/src/backend/cdb/dispatcher.c b/src/backend/cdb/dispatcher.c
index 5d7105c..55a7823 100644
--- a/src/backend/cdb/dispatcher.c
+++ b/src/backend/cdb/dispatcher.c
@@ -1240,6 +1240,8 @@ dispatch_run(DispatchData *data)
* Only after we have the executors, we can serialize the state. Or we
* don't know the executor listening address.
*/
+ CHECK_FOR_INTERRUPTS();
+
dispatcher_serialize_state(data);
dispatcher_serialize_query_resource(data);
dispatcher_set_state_run(data);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/0bb22dce/src/backend/cdb/executormgr.c
--
diff --git a/src/backend/cdb/executormgr.c b/src/backend/cdb/executormgr.c
index 47f5bbe..2bc5059 100644
--- a/src/backend/cdb/executormgr.c
+++ b/src/backend/cdb/executormgr.c
@@ -364,7 +364,19 @@ executormgr_validate_conn(PGconn *conn)
{
if (conn == NULL)
return false;
- return dispatch_validate_conn(conn->sock);
+ if (!dispatch_validate_conn(conn->sock))
+ {
+printfPQExpBuffer(&conn->errorMessage,
+ libpq_gettext(
+ "server closed the connection unexpectedly\n"
+ "\tThis probably means the server terminated abnormally\n"
+ "\tbefore or while processing the request.\n"));
+conn->status = CONNECTION_BAD;
+closesocket(conn->sock);
+conn->sock = -1;
+return false;
+ }
+ return true;
}
/*
@@ -408,7 +420,7 @@ executormgr_dispatch_and_run(struct DispatchData *data,
QueryExecutor *executor)
DispatchCommandQueryParms *parms =
dispatcher_get_QueryParms(data);
if (!executormgr_is_dispatchable(executor))
- return false;
+ goto error;
TIMING_BEGIN(executor->time_dispatch_begin);
query = PQbuildGpQueryString(parms->strCommand, parms->strCommandlen,
@@ -573,10 +585,8 @@ executormgr_catch_error(QueryExecutor *executor)
msg = PQerrorMessage(conn);
- if (msg && (strcmp("", msg) != 0) && (executor->refResult->errcode ==
0)){
+ if (msg && (strcmp("", msg) != 0) && (executor->refResult->errcode ==
0)) {
errCode = ERRCODE_GP_INTERCONNECTION_ERROR;
- if (errCode == ERRCODE_GP_INTERCONNECTION_ERROR)
- errCode = ERRCODE_GP_INTERCONNECTION_ERROR;
}
PQExpBufferData selfDesc;
incubator-hawq git commit: HAWQ-548. Fix wrong unit test case for ip address parsing of interconnect
Repository: incubator-hawq Updated Branches: refs/heads/master 1a75a2a7c -> d259efffb HAWQ-548. Fix wrong unit test case for ip address parsing of interconnect Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/d259efff Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/d259efff Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/d259efff Branch: refs/heads/master Commit: d259efffbce3d9b1eb4e68646632d6419cf2c745 Parents: 1a75a2a Author: Lili Ma Authored: Wed Mar 16 13:40:05 2016 +0800 Committer: Lili Ma Committed: Wed Mar 16 14:18:49 2016 +0800 -- src/test/regress/expected/function.out | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d259efff/src/test/regress/expected/function.out -- diff --git a/src/test/regress/expected/function.out b/src/test/regress/expected/function.out index d493ed5..bda1ded 100644 --- a/src/test/regress/expected/function.out +++ b/src/test/regress/expected/function.out @@ -881,7 +881,6 @@ DROP FUNCTION outer(int); DROP FUNCTION inner(int); -- TEARDOWN DROP TABLE foo; - -- HAWQ-510 drop table if exists testEntryDB; NOTICE: table "testentrydb" does not exist, skipping @@ -895,3 +894,6 @@ where t1.value=t2.value; -+--- 1 | 0 2 | 0 +(2 rows) + +drop table testEntryDB;
[2/2] incubator-hawq git commit: HAWQ-528. Reset gp_connections_per_thread for dispatcher guc range from 1 to 512, 0 marks as invalid.
HAWQ-528. Reset gp_connections_per_thread for dispatcher guc range from 1 to
512, 0 marks as invalid.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/4b53168b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/4b53168b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/4b53168b
Branch: refs/heads/master
Commit: 4b53168b537832f03aa6b4160a78afb63f34fbec
Parents: d5e7b92
Author: Lili Ma
Authored: Mon Mar 14 17:39:31 2016 +0800
Committer: Lili Ma
Committed: Tue Mar 15 09:38:40 2016 +0800
--
src/backend/cdb/dispatcher.c | 7 +--
src/backend/cdb/dispatcher_mgt.c | 9 +
src/backend/utils/misc/guc.c | 2 +-
3 files changed, 3 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4b53168b/src/backend/cdb/dispatcher.c
--
diff --git a/src/backend/cdb/dispatcher.c b/src/backend/cdb/dispatcher.c
index 57ccdf4..20e97ab 100644
--- a/src/backend/cdb/dispatcher.c
+++ b/src/backend/cdb/dispatcher.c
@@ -1656,12 +1656,7 @@ dispatcher_compute_threads_num(DispatchData *data)
/* TODO: decide the groups(threads) number. */
if (query_executors_num == 0)
threads_num = 1;
- if (executors_num_per_thread == 0)
- {
- threads_num = query_executors_num;
- executors_num_per_thread = 1;
- }
- else if (executors_num_per_thread > query_executors_num)
+ if (executors_num_per_thread > query_executors_num)
threads_num = 1;
else
threads_num = (query_executors_num / executors_num_per_thread)
+ ((query_executors_num % executors_num_per_thread == 0) ? 0 : 1);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4b53168b/src/backend/cdb/dispatcher_mgt.c
--
diff --git a/src/backend/cdb/dispatcher_mgt.c b/src/backend/cdb/dispatcher_mgt.c
index 2920249..d0fd7e7 100644
--- a/src/backend/cdb/dispatcher_mgt.c
+++ b/src/backend/cdb/dispatcher_mgt.c
@@ -449,14 +449,7 @@ dispmgt_create_concurrent_connect_state(List *executors,
int executors_num_per_t
int i, j;
/* Compute threads_num */
- if (executors_num == 0)
- return NIL;
- else if (executors_num_per_thread == 0)
- {
- threads_num = executors_num;
- executors_num_per_thread = 1;
- }
- else if (executors_num_per_thread > executors_num)
+ if (executors_num_per_thread > executors_num)
threads_num = 1;
else
threads_num = (executors_num / executors_num_per_thread) +
((executors_num % executors_num_per_thread == 0) ? 0 : 1);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4b53168b/src/backend/utils/misc/guc.c
--
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 5b04054..aac7192 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -5647,7 +5647,7 @@ static struct config_int ConfigureNamesInt[] =
GUC_NOT_IN_SAMPLE | GUC_GPDB_ADDOPT
},
&gp_connections_per_thread,
- 512, 0, INT_MAX, assign_gp_connections_per_thread,
show_gp_connections_per_thread
+ 512, 1, INT_MAX, assign_gp_connections_per_thread,
show_gp_connections_per_thread
},
{
[1/2] incubator-hawq git commit: HAWQ-524. Modify processing for cdbDispatchResult->resultBuf when creation and free
Repository: incubator-hawq
Updated Branches:
refs/heads/master d5e7b9270 -> ac26f48fd
HAWQ-524. Modify processing for cdbDispatchResult->resultBuf when creation and
free
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/ac26f48f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/ac26f48f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/ac26f48f
Branch: refs/heads/master
Commit: ac26f48fd44af17c26cee0ce48a44bf50ddeb9ca
Parents: 4b53168
Author: Lili Ma
Authored: Mon Mar 14 17:47:42 2016 +0800
Committer: Lili Ma
Committed: Tue Mar 15 09:38:40 2016 +0800
--
src/backend/cdb/cdbdispatchresult.c | 19 +++
src/backend/cdb/executormgr.c | 4 +++-
2 files changed, 14 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ac26f48f/src/backend/cdb/cdbdispatchresult.c
--
diff --git a/src/backend/cdb/cdbdispatchresult.c
b/src/backend/cdb/cdbdispatchresult.c
index b036b26..9ba90c0 100644
--- a/src/backend/cdb/cdbdispatchresult.c
+++ b/src/backend/cdb/cdbdispatchresult.c
@@ -183,15 +183,18 @@ void
cdbdisp_resetResult(CdbDispatchResult *dispatchResult)
{
PQExpBuffer buf = dispatchResult->resultbuf;
-PGresult **begp = (PGresult **)buf->data;
-PGresult **endp = (PGresult **)(buf->data + buf->len);
-PGresult **p;
-
-/* Free the PGresult objects. */
-for (p = begp; p < endp; ++p)
+if (buf != NULL)
{
-Assert(*p != NULL);
-PQclear(*p);
+PGresult **begp = (PGresult **)buf->data;
+PGresult **endp = (PGresult **)(buf->data + buf->len);
+PGresult **p;
+
+/* Free the PGresult objects. */
+for (p = begp; p < endp; ++p)
+{
+Assert(*p != NULL);
+PQclear(*p);
+}
}
/* Reset summary indicators. */
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ac26f48f/src/backend/cdb/executormgr.c
--
diff --git a/src/backend/cdb/executormgr.c b/src/backend/cdb/executormgr.c
index 88dd32f..47f5bbe 100644
--- a/src/backend/cdb/executormgr.c
+++ b/src/backend/cdb/executormgr.c
@@ -220,7 +220,9 @@ executormgr_bind_executor_task(struct DispatchData *data,
executor->refResult = cdbdisp_makeResult(dispatch_get_results(data),
executor->desc,
dispatch_get_task_identity(task)->slice_id);
- Assert(executor->refResult != NULL);
+
+ if (executor->refResult == NULL)
+ return false;
/* Transfer any connection errors from segdbDesc. */
executormgr_merge_error(executor);
incubator-hawq git commit: HAWQ-523. Remove dead code in executormgr
Repository: incubator-hawq
Updated Branches:
refs/heads/master b3e2add6d -> 29a9beb1d
HAWQ-523. Remove dead code in executormgr
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/29a9beb1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/29a9beb1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/29a9beb1
Branch: refs/heads/master
Commit: 29a9beb1df109815deebf3750c23e2386fa5377e
Parents: b3e2add
Author: Lili Ma
Authored: Mon Mar 14 11:36:19 2016 +0800
Committer: Lili Ma
Committed: Mon Mar 14 14:00:14 2016 +0800
--
src/backend/cdb/executormgr.c | 8 ++--
1 file changed, 2 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/29a9beb1/src/backend/cdb/executormgr.c
--
diff --git a/src/backend/cdb/executormgr.c b/src/backend/cdb/executormgr.c
index 4d1351d..88dd32f 100644
--- a/src/backend/cdb/executormgr.c
+++ b/src/backend/cdb/executormgr.c
@@ -202,13 +202,9 @@ executormgr_bind_executor_task(struct DispatchData *data,
struct DispatchTask
*task,
struct DispatchSlice
*slice)
{
- executor->state = QES_UNINIT;
- if (desc == NULL)
- {
- executor->health = QEH_ERROR;
- return false;
- }
+ Assert(desc != NULL);
+ executor->state = QES_UNINIT;
executor->desc = desc;
executor->state = QES_DISPATCHABLE;
executor->health = QEH_NA;
incubator-hawq git commit: HAWQ-510. Modify proc->listenerAddr to 127.0.0.1 in entryDB for connection to QD
Repository: incubator-hawq
Updated Branches:
refs/heads/master 3debd597e -> 2eb936a42
HAWQ-510. Modify proc->listenerAddr to 127.0.0.1 in entryDB for connection to QD
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/2eb936a4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/2eb936a4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/2eb936a4
Branch: refs/heads/master
Commit: 2eb936a42a1becde5f46b357f3e11444e97cbc13
Parents: 3debd59
Author: Lili Ma
Authored: Fri Mar 11 13:10:03 2016 +0800
Committer: Lili Ma
Committed: Fri Mar 11 13:16:38 2016 +0800
--
src/backend/cdb/motion/ic_common.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2eb936a4/src/backend/cdb/motion/ic_common.c
--
diff --git a/src/backend/cdb/motion/ic_common.c
b/src/backend/cdb/motion/ic_common.c
index 8c1ab48..6484a11 100644
--- a/src/backend/cdb/motion/ic_common.c
+++ b/src/backend/cdb/motion/ic_common.c
@@ -1014,7 +1014,7 @@ void adjustMasterRouting(Slice *recvSlice)
if (cdbProc->listenerAddr == NULL)
{
if (strcmp(MyProcPort->remote_host, "[local]") == 0)
- cdbProc->listenerAddr = pstrdup("localhost");
+ cdbProc->listenerAddr = pstrdup("127.0.0.1");
else
cdbProc->listenerAddr =
pstrdup(MyProcPort->remote_host);
}
[1/2] incubator-hawq git commit: HAWQ-510. Change cdbProc->listenerAddr to localhost on entryDB when it tries to parse connection from QD, instead of direct using [local]
Repository: incubator-hawq
Updated Branches:
refs/heads/master 88bf600a0 -> b204a4a6f
HAWQ-510. Change cdbProc->listenerAddr to localhost on entryDB when it tries to
parse connection from QD, instead of direct using [local]
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/b204a4a6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/b204a4a6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/b204a4a6
Branch: refs/heads/master
Commit: b204a4a6f0aa1e9b928aebc2383f5325e4066ea3
Parents: cda5c66
Author: Lili Ma
Authored: Thu Mar 10 17:53:31 2016 +0800
Committer: Lili Ma
Committed: Fri Mar 11 09:46:32 2016 +0800
--
src/backend/cdb/motion/ic_common.c | 7 ++-
src/test/regress/expected/function.out | 14 ++
src/test/regress/expected/function_optimizer.out | 16
src/test/regress/sql/function.sql| 12
4 files changed, 48 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b204a4a6/src/backend/cdb/motion/ic_common.c
--
diff --git a/src/backend/cdb/motion/ic_common.c
b/src/backend/cdb/motion/ic_common.c
index 7eba340..8c1ab48 100644
--- a/src/backend/cdb/motion/ic_common.c
+++ b/src/backend/cdb/motion/ic_common.c
@@ -1012,6 +1012,11 @@ void adjustMasterRouting(Slice *recvSlice)
CdbProcess *cdbProc = (CdbProcess *)lfirst(lc);
if (cdbProc->listenerAddr == NULL)
- cdbProc->listenerAddr =
pstrdup(MyProcPort->remote_host);
+ {
+ if (strcmp(MyProcPort->remote_host, "[local]") == 0)
+ cdbProc->listenerAddr = pstrdup("localhost");
+ else
+ cdbProc->listenerAddr =
pstrdup(MyProcPort->remote_host);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b204a4a6/src/test/regress/expected/function.out
--
diff --git a/src/test/regress/expected/function.out
b/src/test/regress/expected/function.out
index b4fb083..d493ed5 100644
--- a/src/test/regress/expected/function.out
+++ b/src/test/regress/expected/function.out
@@ -881,3 +881,17 @@ DROP FUNCTION outer(int);
DROP FUNCTION inner(int);
-- TEARDOWN
DROP TABLE foo;
+
+-- HAWQ-510
+drop table if exists testEntryDB;
+NOTICE: table "testentrydb" does not exist, skipping
+create table testEntryDB(key int, value int) distributed randomly;
+insert into testEntryDB values(1, 0);
+select t2.key, t2.value
+from (select key, value from testEntryDB where value = 0) as t1,
+ (select generate_series(1,2)::int as key, 0::int as value) as t2
+where t1.value=t2.value;
+ key | value
+-+---
+ 1 | 0
+ 2 | 0
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b204a4a6/src/test/regress/expected/function_optimizer.out
--
diff --git a/src/test/regress/expected/function_optimizer.out
b/src/test/regress/expected/function_optimizer.out
index 2225f85..0b0babc 100644
--- a/src/test/regress/expected/function_optimizer.out
+++ b/src/test/regress/expected/function_optimizer.out
@@ -893,3 +893,19 @@ DROP FUNCTION outer(int);
DROP FUNCTION inner(int);
-- TEARDOWN
DROP TABLE foo;
+-- HAWQ-510
+drop table if exists testEntryDB;
+NOTICE: table "testentrydb" does not exist, skipping
+create table testEntryDB(key int, value int) distributed randomly;
+insert into testEntryDB values(1, 0);
+select t2.key, t2.value
+from (select key, value from testEntryDB where value = 0) as t1,
+ (select generate_series(1,2)::int as key, 0::int as value) as t2
+where t1.value=t2.value;
+ key | value
+-+---
+ 1 | 0
+ 2 | 0
+(2 rows)
+
+drop table testEntryDB;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b204a4a6/src/test/regress/sql/function.sql
--
diff --git a/src/test/regress/sql/function.sql
b/src/test/regress/sql/function.sql
index 6f18840..a050b22 100644
--- a/src/test/regress/sql/function.sql
+++ b/src/test/regress/sql/function.sql
@@ -425,3 +425,15 @@ DROP FUNCTION inner(int);
-- TEARDOWN
DROP TABLE foo;
+
+
+
+-- HAWQ-510
+drop table if exists testEntryDB;
+create table testEntryDB(key int, value int) distributed randomly;
+insert into testEntryDB values(1, 0);
+select t2.key, t2.value
+from (select key, value from testEntryDB where value = 0) as t1,
+ (select generate_series(1,2)::int as key, 0::int as value) as t2
+where t1.value=t2.value;
+drop table testEntryDB;
[2/2] incubator-hawq git commit: HAWQ-509. Change pg_stat_activity definition, only returns one record for one query
HAWQ-509. Change pg_stat_activity definition, only returns one record for one query Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/cda5c66c Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/cda5c66c Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/cda5c66c Branch: refs/heads/master Commit: cda5c66ce488e0b212cabdbefb19137f872fcce3 Parents: 88bf600 Author: Lili Ma Authored: Thu Mar 10 14:16:44 2016 +0800 Committer: Lili Ma Committed: Fri Mar 11 09:46:32 2016 +0800 -- src/backend/catalog/system_views.sql | 14 -- 1 file changed, 12 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/cda5c66c/src/backend/catalog/system_views.sql -- diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 8ba0a51..4895672 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -355,9 +355,19 @@ CREATE VIEW pg_stat_activity AS S.application_name, S.xact_start, S.waiting_resource -FROM pg_database D, pg_stat_get_activity(NULL) AS S, pg_authid U +FROM pg_database D, pg_stat_get_activity(NULL) AS S, pg_authid U, + ( + SELECT + sess_id, + min(backend_start) AS min_backend_start + FROM + pg_stat_get_activity(NULL) + GROUP BY + sess_id)S2 WHERE S.datid = D.oid AND -S.usesysid = U.oid; + S.usesysid = U.oid AND + S.sess_id = S2.sess_id AND + S.backend_start = S2.min_backend_start; CREATE VIEW pg_stat_database AS SELECT
[incubator-hawq] Git Push Summary
Repository: incubator-hawq Updated Branches: refs/heads/HAWQ-275 [deleted] e427c9cdd
[1/2] incubator-hawq git commit: HAWQ-275. Add connection check when dispatcher wants to reuse the QE forked by former queries, and when dispather dispatches the detailed query command to QE
Repository: incubator-hawq
Updated Branches:
refs/heads/master 27b12112f -> e6cabe821
HAWQ-275. Add connection check when dispatcher wants to reuse the QE forked by
former queries, and when dispather dispatches the detailed query command to QE
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/ca7afdce
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/ca7afdce
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/ca7afdce
Branch: refs/heads/master
Commit: ca7afdce5481fb1aaee90bd759eaffb022d9a703
Parents: 629ecd7
Author: Lili Ma
Authored: Mon Feb 22 09:10:55 2016 +0800
Committer: Lili Ma
Committed: Mon Feb 22 09:10:55 2016 +0800
--
src/backend/cdb/dispatcher.c| 37 ++
src/backend/cdb/executormgr.c | 44
src/backend/cdb/motion/ic_udp.c | 43 +--
src/include/cdb/dispatcher.h| 2 ++
4 files changed, 69 insertions(+), 57 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ca7afdce/src/backend/cdb/dispatcher.c
--
diff --git a/src/backend/cdb/dispatcher.c b/src/backend/cdb/dispatcher.c
index f72aa35..57ccdf4 100644
--- a/src/backend/cdb/dispatcher.c
+++ b/src/backend/cdb/dispatcher.c
@@ -1803,3 +1803,40 @@ dispatcher_print_statistics(StringInfo buf, DispatchData
*data)
INSTR_TIME_GET_MILLISEC(data->time_total_free) /
data->num_of_dispatched);
}
+
+
+/*
+ * Check the connection from the dispatcher to verify that it is still there.
+ * Return true if the dispatcher connection is still alive.
+ */
+bool dispatch_validate_conn(pgsocket sock)
+{
+ ssize_t ret;
+ charbuf;
+
+ if (sock < 0)
+return false;
+
+#ifndef WIN32
+ret = recv(sock, &buf, 1, MSG_PEEK|MSG_DONTWAIT);
+#else
+ret = recv(sock, &buf, 1, MSG_PEEK|MSG_PARTIAL);
+#endif
+
+ if (ret == 0) /* socket has been closed. EOF */
+return false;
+
+ if (ret > 0) /* data waiting on socket, it must be OK. */
+return true;
+
+ if (ret == -1) /* error, or would be block. */
+ {
+if (errno == EAGAIN || errno == EINPROGRESS || errno == EWOULDBLOCK)
+ return true; /* connection intact, no data available */
+else
+ return false;
+ }
+ /* not reached */
+
+ return true;
+}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ca7afdce/src/backend/cdb/executormgr.c
--
diff --git a/src/backend/cdb/executormgr.c b/src/backend/cdb/executormgr.c
index 6216d23..4d1351d 100644
--- a/src/backend/cdb/executormgr.c
+++ b/src/backend/cdb/executormgr.c
@@ -361,6 +361,14 @@ executormgr_cancel(QueryExecutor *executor)
return success;
}
+static bool
+executormgr_validate_conn(PGconn *conn)
+{
+ if (conn == NULL)
+return false;
+ return dispatch_validate_conn(conn->sock);
+}
+
/*
* executormgr_is_dispatchable
* Return the true iff executor can receive query.
@@ -379,7 +387,7 @@ executormgr_is_dispatchable(QueryExecutor *executor)
return false;
}
- if (PQstatus(conn) == CONNECTION_BAD)
+ if (!executormgr_validate_conn(conn) || PQstatus(conn) ==
CONNECTION_BAD)
{
write_log("function executormgr_is_dispatchable meets error,
connection is bad.");
executormgr_catch_error(executor);
@@ -389,9 +397,6 @@ executormgr_is_dispatchable(QueryExecutor *executor)
return true;
}
-
-
-
/*
* executormgr_dispatch_and_run
* Dispatch data and run the query.
@@ -670,21 +675,30 @@
executormgr_free_takeovered_segment_conn(SegmentDatabaseDescriptor *desc)
}
static SegmentDatabaseDescriptor *
-executormgr_allocate_any_executor(bool is_writer)
+executormgr_allocate_any_executor(bool is_writer, bool is_entrydb)
{
- return poolmgr_get_random_item(executor_cache.pool);
-}
-
-static SegmentDatabaseDescriptor *
-executormgr_allocate_executor_on_entrydb(bool is_writer)
-{
- return poolmgr_get_random_item(executor_cache.entrydb_pool);
+ // get executor from pool and check whether the connection is valid, keep
+ // running until finding a valid one or the pool becomes NULL
+ struct PoolMgrState *executor_pool =
+ is_entrydb ? executor_cache.entrydb_pool : executor_cache.pool;
+ SegmentDatabaseDescriptor *desc = poolmgr_get_random_item(executor_pool);
+ while (desc != NULL && !executormgr_validate_conn(desc->conn)) {
+desc = poolmgr_get_random_item(executor_pool);
+ }
+ return desc;
}
static SegmentDatabaseDescriptor *
executormgr_allocate_executor_by_name(const char *name, bool is_writer)
{
- return poolmgr_get_item_by_name(executor_cache.
[2/2] incubator-hawq git commit: Merge branch 'HAWQ-275'
Merge branch 'HAWQ-275' Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/e6cabe82 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/e6cabe82 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/e6cabe82 Branch: refs/heads/master Commit: e6cabe821794d0d6470292b0e6749ab0fc02a3e9 Parents: 27b1211 ca7afdc Author: Lili Ma Authored: Mon Feb 22 09:14:52 2016 +0800 Committer: Lili Ma Committed: Mon Feb 22 09:14:52 2016 +0800 -- src/backend/cdb/dispatcher.c| 37 ++ src/backend/cdb/executormgr.c | 44 src/backend/cdb/motion/ic_udp.c | 43 +-- src/include/cdb/dispatcher.h| 2 ++ 4 files changed, 69 insertions(+), 57 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/e6cabe82/src/backend/cdb/motion/ic_udp.c --
incubator-hawq git commit: Add connection check when dispatcher wants to reuse the QE forked
Repository: incubator-hawq
Updated Branches:
refs/heads/HAWQ-275 [created] e427c9cdd
Add connection check when dispatcher wants to reuse the QE forked
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/e427c9cd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/e427c9cd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/e427c9cd
Branch: refs/heads/HAWQ-275
Commit: e427c9cddb9790f9755446c5e8105d5eb4c48b48
Parents: 629ecd7
Author: Lili Ma
Authored: Fri Feb 19 17:33:43 2016 +0800
Committer: Lili Ma
Committed: Fri Feb 19 17:33:43 2016 +0800
--
src/backend/cdb/dispatcher.c| 37 ++
src/backend/cdb/executormgr.c | 44
src/backend/cdb/motion/ic_udp.c | 43 +--
src/include/cdb/dispatcher.h| 2 ++
4 files changed, 69 insertions(+), 57 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/e427c9cd/src/backend/cdb/dispatcher.c
--
diff --git a/src/backend/cdb/dispatcher.c b/src/backend/cdb/dispatcher.c
index f72aa35..57ccdf4 100644
--- a/src/backend/cdb/dispatcher.c
+++ b/src/backend/cdb/dispatcher.c
@@ -1803,3 +1803,40 @@ dispatcher_print_statistics(StringInfo buf, DispatchData
*data)
INSTR_TIME_GET_MILLISEC(data->time_total_free) /
data->num_of_dispatched);
}
+
+
+/*
+ * Check the connection from the dispatcher to verify that it is still there.
+ * Return true if the dispatcher connection is still alive.
+ */
+bool dispatch_validate_conn(pgsocket sock)
+{
+ ssize_t ret;
+ charbuf;
+
+ if (sock < 0)
+return false;
+
+#ifndef WIN32
+ret = recv(sock, &buf, 1, MSG_PEEK|MSG_DONTWAIT);
+#else
+ret = recv(sock, &buf, 1, MSG_PEEK|MSG_PARTIAL);
+#endif
+
+ if (ret == 0) /* socket has been closed. EOF */
+return false;
+
+ if (ret > 0) /* data waiting on socket, it must be OK. */
+return true;
+
+ if (ret == -1) /* error, or would be block. */
+ {
+if (errno == EAGAIN || errno == EINPROGRESS || errno == EWOULDBLOCK)
+ return true; /* connection intact, no data available */
+else
+ return false;
+ }
+ /* not reached */
+
+ return true;
+}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/e427c9cd/src/backend/cdb/executormgr.c
--
diff --git a/src/backend/cdb/executormgr.c b/src/backend/cdb/executormgr.c
index 6216d23..4d1351d 100644
--- a/src/backend/cdb/executormgr.c
+++ b/src/backend/cdb/executormgr.c
@@ -361,6 +361,14 @@ executormgr_cancel(QueryExecutor *executor)
return success;
}
+static bool
+executormgr_validate_conn(PGconn *conn)
+{
+ if (conn == NULL)
+return false;
+ return dispatch_validate_conn(conn->sock);
+}
+
/*
* executormgr_is_dispatchable
* Return the true iff executor can receive query.
@@ -379,7 +387,7 @@ executormgr_is_dispatchable(QueryExecutor *executor)
return false;
}
- if (PQstatus(conn) == CONNECTION_BAD)
+ if (!executormgr_validate_conn(conn) || PQstatus(conn) ==
CONNECTION_BAD)
{
write_log("function executormgr_is_dispatchable meets error,
connection is bad.");
executormgr_catch_error(executor);
@@ -389,9 +397,6 @@ executormgr_is_dispatchable(QueryExecutor *executor)
return true;
}
-
-
-
/*
* executormgr_dispatch_and_run
* Dispatch data and run the query.
@@ -670,21 +675,30 @@
executormgr_free_takeovered_segment_conn(SegmentDatabaseDescriptor *desc)
}
static SegmentDatabaseDescriptor *
-executormgr_allocate_any_executor(bool is_writer)
+executormgr_allocate_any_executor(bool is_writer, bool is_entrydb)
{
- return poolmgr_get_random_item(executor_cache.pool);
-}
-
-static SegmentDatabaseDescriptor *
-executormgr_allocate_executor_on_entrydb(bool is_writer)
-{
- return poolmgr_get_random_item(executor_cache.entrydb_pool);
+ // get executor from pool and check whether the connection is valid, keep
+ // running until finding a valid one or the pool becomes NULL
+ struct PoolMgrState *executor_pool =
+ is_entrydb ? executor_cache.entrydb_pool : executor_cache.pool;
+ SegmentDatabaseDescriptor *desc = poolmgr_get_random_item(executor_pool);
+ while (desc != NULL && !executormgr_validate_conn(desc->conn)) {
+desc = poolmgr_get_random_item(executor_pool);
+ }
+ return desc;
}
static SegmentDatabaseDescriptor *
executormgr_allocate_executor_by_name(const char *name, bool is_writer)
{
- return poolmgr_get_item_by_name(executor_cache.pool, name);
+ // get executor from pool and check whether the connection is valid, keep
+
[incubator-hawq] Git Push Summary
Repository: incubator-hawq Updated Branches: refs/heads/HAWQ-275 [deleted] f8dfd675e
incubator-hawq git commit: HAWQ-275. Add connection check when dispatcher wants to reuse the QE forked by former queries, and when dispather dispatches the detailed query command to QE
Repository: incubator-hawq
Updated Branches:
refs/heads/HAWQ-275 [created] f8dfd675e
HAWQ-275. Add connection check when dispatcher wants to reuse the QE forked by
former queries, and when dispather dispatches the detailed query command to QE
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/f8dfd675
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/f8dfd675
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/f8dfd675
Branch: refs/heads/HAWQ-275
Commit: f8dfd675e8aa11ac5a03a38f1d455f4090115794
Parents: 629ecd7
Author: Lili Ma
Authored: Fri Feb 19 09:10:58 2016 +0800
Committer: Lili Ma
Committed: Fri Feb 19 09:10:58 2016 +0800
--
src/backend/cdb/executormgr.c | 69 +-
1 file changed, 54 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f8dfd675/src/backend/cdb/executormgr.c
--
diff --git a/src/backend/cdb/executormgr.c b/src/backend/cdb/executormgr.c
index 6216d23..107e860 100644
--- a/src/backend/cdb/executormgr.c
+++ b/src/backend/cdb/executormgr.c
@@ -361,6 +361,39 @@ executormgr_cancel(QueryExecutor *executor)
return success;
}
+static bool
+executormgr_validate_conn(PGconn *conn)
+{
+ ssize_t ret;
+ charbuf;
+
+ if (conn->sock < 0)
+return false;
+
+#ifndef WIN32
+ret = recv(conn->sock, &buf, 1, MSG_PEEK|MSG_DONTWAIT);
+#else
+ret = recv(conn->sock, &buf, 1, MSG_PEEK|MSG_PARTIAL);
+#endif
+
+ if (ret == 0) /* socket has been closed. EOF */
+return false;
+
+ if (ret > 0) /* data waiting on socket, it must be OK. */
+return true;
+
+ if (ret == -1) /* error, or would be block. */
+ {
+if (errno == EAGAIN || errno == EINPROGRESS)
+ return true; /* connection intact, no data available */
+else
+ return false;
+ }
+ /* not reached */
+
+ return true;
+}
+
/*
* executormgr_is_dispatchable
* Return the true iff executor can receive query.
@@ -379,7 +412,7 @@ executormgr_is_dispatchable(QueryExecutor *executor)
return false;
}
- if (PQstatus(conn) == CONNECTION_BAD)
+ if (!executormgr_validate_conn(conn)|| PQstatus(conn) == CONNECTION_BAD)
{
write_log("function executormgr_is_dispatchable meets error,
connection is bad.");
executormgr_catch_error(executor);
@@ -389,9 +422,6 @@ executormgr_is_dispatchable(QueryExecutor *executor)
return true;
}
-
-
-
/*
* executormgr_dispatch_and_run
* Dispatch data and run the query.
@@ -670,21 +700,30 @@
executormgr_free_takeovered_segment_conn(SegmentDatabaseDescriptor *desc)
}
static SegmentDatabaseDescriptor *
-executormgr_allocate_any_executor(bool is_writer)
+executormgr_allocate_any_executor(bool is_writer, bool is_entrydb)
{
- return poolmgr_get_random_item(executor_cache.pool);
-}
-
-static SegmentDatabaseDescriptor *
-executormgr_allocate_executor_on_entrydb(bool is_writer)
-{
- return poolmgr_get_random_item(executor_cache.entrydb_pool);
+ // get executor from pool and check whether the connection is valid, keep
+ // running until finding a valid one or the pool becomes NULL
+ struct PoolMgrState *executor_pool =
+ is_entrydb ? executor_cache.entrydb_pool : executor_cache.pool;
+ SegmentDatabaseDescriptor *desc = poolmgr_get_random_item(executor_pool);
+ while (desc != NULL && !executormgr_validate_conn(desc->conn)) {
+desc = poolmgr_get_random_item(executor_pool);
+ }
+ return desc;
}
static SegmentDatabaseDescriptor *
executormgr_allocate_executor_by_name(const char *name, bool is_writer)
{
- return poolmgr_get_item_by_name(executor_cache.pool, name);
+ // get executor from pool and check whether the connection is valid, keep
+ // running until finding a valid one or the pool becomes NULL
+ SegmentDatabaseDescriptor *desc =
+ poolmgr_get_item_by_name(executor_cache.pool, name);
+ while (desc != NULL && !executormgr_validate_conn(desc->conn)) {
+desc = poolmgr_get_item_by_name(executor_cache.pool, name);
+ }
+ return desc;
}
/*
@@ -697,9 +736,9 @@ executormgr_allocate_executor(Segment *segment, bool
is_writer, bool is_entrydb)
SegmentDatabaseDescriptor *ret;
if (is_entrydb || (segment != NULL && segment->master))
- ret = executormgr_allocate_executor_on_entrydb(is_writer);
+ ret = executormgr_allocate_any_executor(is_writer, true);
else if (segment == NULL)
- ret = executormgr_allocate_any_executor(is_writer);
+ ret = executormgr_allocate_any_executor(is_writer, false);
else
ret =
executormgr_allocate_executor_by_name
[incubator-hawq] Git Push Summary
Repository: incubator-hawq Updated Branches: refs/heads/HAWQ-407 [deleted] cf59b7b66
[1/3] incubator-hawq git commit: HAWQ-407. Fix memory leak in parquet metadata processing for reading table with data
Repository: incubator-hawq Updated Branches: refs/heads/master 2968e5154 -> 4c3a09180 HAWQ-407. Fix memory leak in parquet metadata processing for reading table with data Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/9defbc12 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/9defbc12 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/9defbc12 Branch: refs/heads/master Commit: 9defbc12a818971b31ab402ea0935b1ff266589f Parents: 95a5da1 Author: Lili Ma Authored: Mon Feb 15 14:59:32 2016 +0800 Committer: Lili Ma Committed: Mon Feb 15 14:59:32 2016 +0800 -- src/backend/access/parquet/parquetam.c | 4 ++-- src/backend/cdb/cdbparquetfooterprocessor.c | 3 +++ src/backend/cdb/cdbparquetfooterserializer.c | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/9defbc12/src/backend/access/parquet/parquetam.c -- diff --git a/src/backend/access/parquet/parquetam.c b/src/backend/access/parquet/parquetam.c index 67359ca..6438e32 100644 --- a/src/backend/access/parquet/parquetam.c +++ b/src/backend/access/parquet/parquetam.c @@ -583,8 +583,8 @@ ParquetInsertDesc parquet_insert_init(Relation rel, ResultRelSegFileInfo *segfil parquetInsertDesc->aoEntry = aoentry; parquetInsertDesc->insertCount = 0; - parquetInsertDesc->parquetMetadata = (struct ParquetMetadata_4C *) - palloc0(sizeof(struct ParquetMetadata_4C)); +// parquetInsertDesc->parquetMetadata = (struct ParquetMetadata_4C *) +// palloc0(sizeof(struct ParquetMetadata_4C)); /*SHOULD CALL OPEN METADATA FILE HERE, AND GET parquetMetadata INFO*/ initStringInfo(&titleBuf); http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/9defbc12/src/backend/cdb/cdbparquetfooterprocessor.c -- diff --git a/src/backend/cdb/cdbparquetfooterprocessor.c b/src/backend/cdb/cdbparquetfooterprocessor.c index 510a65f..691534a 100644 --- a/src/backend/cdb/cdbparquetfooterprocessor.c +++ b/src/backend/cdb/cdbparquetfooterprocessor.c @@ -111,6 +111,9 @@ bool readParquetFooter(File fileHandler, ParquetMetadata *parquetMetadata, DetectHostEndian(); + *parquetMetadata = (struct ParquetMetadata_4C *) + palloc0(sizeof(struct ParquetMetadata_4C)); + /* if file size is 0, means there's no data in file, return false*/ if (eof == 0) return false; http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/9defbc12/src/backend/cdb/cdbparquetfooterserializer.c -- diff --git a/src/backend/cdb/cdbparquetfooterserializer.c b/src/backend/cdb/cdbparquetfooterserializer.c index 8677078..247bbb1 100644 --- a/src/backend/cdb/cdbparquetfooterserializer.c +++ b/src/backend/cdb/cdbparquetfooterserializer.c @@ -167,8 +167,8 @@ void initDeserializeFooter( initCompactProtocol(*footerProtocol, file, fileName, footerLength, PARQUET_FOOTER_BUFFERMODE_READ); - *parquetMetadata = - (struct ParquetMetadata_4C*) palloc0(sizeof(struct ParquetMetadata_4C)); +// *parquetMetadata = +// (struct ParquetMetadata_4C*) palloc0(sizeof(struct ParquetMetadata_4C)); readParquetFileMetadata(parquetMetadata, *footerProtocol); }
[3/3] incubator-hawq git commit: HAWQ-407. Fix memory leak in parquet metadata processing for reading table with data
HAWQ-407. Fix memory leak in parquet metadata processing for reading table with data Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/4c3a0918 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/4c3a0918 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/4c3a0918 Branch: refs/heads/master Commit: 4c3a09180e6465567e569706ff4a71117ce6a0b3 Parents: 2968e51 cf59b7b Author: Lili Ma Authored: Wed Feb 17 09:49:55 2016 +0800 Committer: Lili Ma Committed: Wed Feb 17 09:49:55 2016 +0800 -- src/backend/access/parquet/parquetam.c | 4 src/backend/cdb/cdbparquetfooterprocessor.c | 3 +++ src/backend/cdb/cdbparquetfooterserializer.c | 3 --- 3 files changed, 3 insertions(+), 7 deletions(-) --
[2/3] incubator-hawq git commit: HAWQ-407. Remove unnecessary comments
HAWQ-407. Remove unnecessary comments Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/cf59b7b6 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/cf59b7b6 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/cf59b7b6 Branch: refs/heads/master Commit: cf59b7b66beef57eda08f39472a98c43d6b76f18 Parents: 9defbc1 Author: Lili Ma Authored: Wed Feb 17 09:28:32 2016 +0800 Committer: Lili Ma Committed: Wed Feb 17 09:28:32 2016 +0800 -- src/backend/access/parquet/parquetam.c | 4 src/backend/cdb/cdbparquetfooterserializer.c | 3 --- 2 files changed, 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/cf59b7b6/src/backend/access/parquet/parquetam.c -- diff --git a/src/backend/access/parquet/parquetam.c b/src/backend/access/parquet/parquetam.c index 6438e32..3c2b7df 100644 --- a/src/backend/access/parquet/parquetam.c +++ b/src/backend/access/parquet/parquetam.c @@ -583,10 +583,6 @@ ParquetInsertDesc parquet_insert_init(Relation rel, ResultRelSegFileInfo *segfil parquetInsertDesc->aoEntry = aoentry; parquetInsertDesc->insertCount = 0; -// parquetInsertDesc->parquetMetadata = (struct ParquetMetadata_4C *) -// palloc0(sizeof(struct ParquetMetadata_4C)); - /*SHOULD CALL OPEN METADATA FILE HERE, AND GET parquetMetadata INFO*/ - initStringInfo(&titleBuf); appendStringInfo(&titleBuf, "Write of Parquet relation '%s'", RelationGetRelationName(parquetInsertDesc->parquet_rel)); http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/cf59b7b6/src/backend/cdb/cdbparquetfooterserializer.c -- diff --git a/src/backend/cdb/cdbparquetfooterserializer.c b/src/backend/cdb/cdbparquetfooterserializer.c index 247bbb1..802793f 100644 --- a/src/backend/cdb/cdbparquetfooterserializer.c +++ b/src/backend/cdb/cdbparquetfooterserializer.c @@ -167,9 +167,6 @@ void initDeserializeFooter( initCompactProtocol(*footerProtocol, file, fileName, footerLength, PARQUET_FOOTER_BUFFERMODE_READ); -// *parquetMetadata = -// (struct ParquetMetadata_4C*) palloc0(sizeof(struct ParquetMetadata_4C)); - readParquetFileMetadata(parquetMetadata, *footerProtocol); }
incubator-hawq git commit: HAWQ-407. Remove unnecessary comments
Repository: incubator-hawq Updated Branches: refs/heads/HAWQ-407 9defbc12a -> cf59b7b66 HAWQ-407. Remove unnecessary comments Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/cf59b7b6 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/cf59b7b6 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/cf59b7b6 Branch: refs/heads/HAWQ-407 Commit: cf59b7b66beef57eda08f39472a98c43d6b76f18 Parents: 9defbc1 Author: Lili Ma Authored: Wed Feb 17 09:28:32 2016 +0800 Committer: Lili Ma Committed: Wed Feb 17 09:28:32 2016 +0800 -- src/backend/access/parquet/parquetam.c | 4 src/backend/cdb/cdbparquetfooterserializer.c | 3 --- 2 files changed, 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/cf59b7b6/src/backend/access/parquet/parquetam.c -- diff --git a/src/backend/access/parquet/parquetam.c b/src/backend/access/parquet/parquetam.c index 6438e32..3c2b7df 100644 --- a/src/backend/access/parquet/parquetam.c +++ b/src/backend/access/parquet/parquetam.c @@ -583,10 +583,6 @@ ParquetInsertDesc parquet_insert_init(Relation rel, ResultRelSegFileInfo *segfil parquetInsertDesc->aoEntry = aoentry; parquetInsertDesc->insertCount = 0; -// parquetInsertDesc->parquetMetadata = (struct ParquetMetadata_4C *) -// palloc0(sizeof(struct ParquetMetadata_4C)); - /*SHOULD CALL OPEN METADATA FILE HERE, AND GET parquetMetadata INFO*/ - initStringInfo(&titleBuf); appendStringInfo(&titleBuf, "Write of Parquet relation '%s'", RelationGetRelationName(parquetInsertDesc->parquet_rel)); http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/cf59b7b6/src/backend/cdb/cdbparquetfooterserializer.c -- diff --git a/src/backend/cdb/cdbparquetfooterserializer.c b/src/backend/cdb/cdbparquetfooterserializer.c index 247bbb1..802793f 100644 --- a/src/backend/cdb/cdbparquetfooterserializer.c +++ b/src/backend/cdb/cdbparquetfooterserializer.c @@ -167,9 +167,6 @@ void initDeserializeFooter( initCompactProtocol(*footerProtocol, file, fileName, footerLength, PARQUET_FOOTER_BUFFERMODE_READ); -// *parquetMetadata = -// (struct ParquetMetadata_4C*) palloc0(sizeof(struct ParquetMetadata_4C)); - readParquetFileMetadata(parquetMetadata, *footerProtocol); }
incubator-hawq git commit: HAWQ-407. Fix memory leak in parquet metadata processing for reading table with data
Repository: incubator-hawq Updated Branches: refs/heads/HAWQ-407 [created] 9defbc12a HAWQ-407. Fix memory leak in parquet metadata processing for reading table with data Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/9defbc12 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/9defbc12 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/9defbc12 Branch: refs/heads/HAWQ-407 Commit: 9defbc12a818971b31ab402ea0935b1ff266589f Parents: 95a5da1 Author: Lili Ma Authored: Mon Feb 15 14:59:32 2016 +0800 Committer: Lili Ma Committed: Mon Feb 15 14:59:32 2016 +0800 -- src/backend/access/parquet/parquetam.c | 4 ++-- src/backend/cdb/cdbparquetfooterprocessor.c | 3 +++ src/backend/cdb/cdbparquetfooterserializer.c | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/9defbc12/src/backend/access/parquet/parquetam.c -- diff --git a/src/backend/access/parquet/parquetam.c b/src/backend/access/parquet/parquetam.c index 67359ca..6438e32 100644 --- a/src/backend/access/parquet/parquetam.c +++ b/src/backend/access/parquet/parquetam.c @@ -583,8 +583,8 @@ ParquetInsertDesc parquet_insert_init(Relation rel, ResultRelSegFileInfo *segfil parquetInsertDesc->aoEntry = aoentry; parquetInsertDesc->insertCount = 0; - parquetInsertDesc->parquetMetadata = (struct ParquetMetadata_4C *) - palloc0(sizeof(struct ParquetMetadata_4C)); +// parquetInsertDesc->parquetMetadata = (struct ParquetMetadata_4C *) +// palloc0(sizeof(struct ParquetMetadata_4C)); /*SHOULD CALL OPEN METADATA FILE HERE, AND GET parquetMetadata INFO*/ initStringInfo(&titleBuf); http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/9defbc12/src/backend/cdb/cdbparquetfooterprocessor.c -- diff --git a/src/backend/cdb/cdbparquetfooterprocessor.c b/src/backend/cdb/cdbparquetfooterprocessor.c index 510a65f..691534a 100644 --- a/src/backend/cdb/cdbparquetfooterprocessor.c +++ b/src/backend/cdb/cdbparquetfooterprocessor.c @@ -111,6 +111,9 @@ bool readParquetFooter(File fileHandler, ParquetMetadata *parquetMetadata, DetectHostEndian(); + *parquetMetadata = (struct ParquetMetadata_4C *) + palloc0(sizeof(struct ParquetMetadata_4C)); + /* if file size is 0, means there's no data in file, return false*/ if (eof == 0) return false; http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/9defbc12/src/backend/cdb/cdbparquetfooterserializer.c -- diff --git a/src/backend/cdb/cdbparquetfooterserializer.c b/src/backend/cdb/cdbparquetfooterserializer.c index 8677078..247bbb1 100644 --- a/src/backend/cdb/cdbparquetfooterserializer.c +++ b/src/backend/cdb/cdbparquetfooterserializer.c @@ -167,8 +167,8 @@ void initDeserializeFooter( initCompactProtocol(*footerProtocol, file, fileName, footerLength, PARQUET_FOOTER_BUFFERMODE_READ); - *parquetMetadata = - (struct ParquetMetadata_4C*) palloc0(sizeof(struct ParquetMetadata_4C)); +// *parquetMetadata = +// (struct ParquetMetadata_4C*) palloc0(sizeof(struct ParquetMetadata_4C)); readParquetFileMetadata(parquetMetadata, *footerProtocol); }
