incubator-hawq git commit: HAWQ-582. Handle ApplicationMasterNotRegisteredException in libyarn
Repository: incubator-hawq
Updated Branches:
refs/heads/master 536d99a6d -> 598ec5892
HAWQ-582. Handle ApplicationMasterNotRegisteredException in libyarn
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/598ec589
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/598ec589
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/598ec589
Branch: refs/heads/master
Commit: 598ec589272efcda10c625edf85e377611164014
Parents: 536d99a
Author: Wen Lin
Authored: Thu Mar 24 14:19:43 2016 +0800
Committer: Wen Lin
Committed: Thu Mar 24 14:19:43 2016 +0800
--
depends/libyarn/src/CMakeLists.txt | 2 +-
depends/libyarn/src/common/Exception.cpp| 3 +
depends/libyarn/src/common/Exception.h | 15 +-
.../libyarn/src/libyarnclient/LibYarnClient.cpp | 218 +++
.../libyarnserver/ApplicationMasterProtocol.cpp | 12 +-
5 files changed, 145 insertions(+), 105 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/598ec589/depends/libyarn/src/CMakeLists.txt
--
diff --git a/depends/libyarn/src/CMakeLists.txt
b/depends/libyarn/src/CMakeLists.txt
index 4278d6f..ba70739 100644
--- a/depends/libyarn/src/CMakeLists.txt
+++ b/depends/libyarn/src/CMakeLists.txt
@@ -2,7 +2,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
SET(libyarn_VERSION_MAJOR 0)
SET(libyarn_VERSION_MINOR 1)
-SET(libyarn_VERSION_PATCH 13)
+SET(libyarn_VERSION_PATCH 14)
SET(libyarn_VERSION_STRING
"${libyarn_VERSION_MAJOR}.${libyarn_VERSION_MINOR}.${libyarn_VERSION_PATCH}")
SET(libyarn_VERSION_API 1)
SET(libyarn_ROOT_SOURCES_DIR ${CMAKE_SOURCE_DIR}/src)
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/598ec589/depends/libyarn/src/common/Exception.cpp
--
diff --git a/depends/libyarn/src/common/Exception.cpp
b/depends/libyarn/src/common/Exception.cpp
index 76699e8..09d1312 100644
--- a/depends/libyarn/src/common/Exception.cpp
+++ b/depends/libyarn/src/common/Exception.cpp
@@ -69,6 +69,9 @@ const char * YarnInvalidBlockToken::ReflexName =
const char * SaslException::ReflexName = "javax.security.sasl.SaslException";
+const char * ApplicationMasterNotRegisteredException::ReflexName =
+
"org.apache.hadoop.yarn.exceptions.ApplicationMasterNotRegisteredException";
+
YarnException::YarnException(const std::string & arg, const char * file,
int line, const char * stack) :
std::runtime_error(arg) {
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/598ec589/depends/libyarn/src/common/Exception.h
--
diff --git a/depends/libyarn/src/common/Exception.h
b/depends/libyarn/src/common/Exception.h
index 2f0cca9..46af910 100644
--- a/depends/libyarn/src/common/Exception.h
+++ b/depends/libyarn/src/common/Exception.h
@@ -482,7 +482,7 @@ public:
class ResourceManagerStandbyException: public YarnException {
public:
- ResourceManagerStandbyException(const std::string & arg, const char *
file,
+ResourceManagerStandbyException(const std::string & arg, const char * file,
int line, const char * stack) :
YarnException(arg, file, line, stack) {
}
@@ -494,7 +494,18 @@ public:
static const char * ReflexName;
};
+class ApplicationMasterNotRegisteredException: public YarnException {
+public:
+ApplicationMasterNotRegisteredException(const std::string & arg, const
char * file,
+int line, const char * stack) :
+YarnException(arg, file, line, stack) {
+}
-}
+~ApplicationMasterNotRegisteredException() throw () {
+}
+public:
+static const char * ReflexName;
+};
+}
#endif /* _YARN_LIBYARN_COMMON_EXCEPTION_H_ */
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/598ec589/depends/libyarn/src/libyarnclient/LibYarnClient.cpp
--
diff --git a/depends/libyarn/src/libyarnclient/LibYarnClient.cpp
b/depends/libyarn/src/libyarnclient/LibYarnClient.cpp
index 87876e4..3c76f92 100644
--- a/depends/libyarn/src/libyarnclient/LibYarnClient.cpp
+++ b/depends/libyarn/src/libyarnclient/LibYarnClient.cpp
@@ -118,8 +118,17 @@ void* heartbeatFunc(void* args) {
try {
client->dummyAllocate();
failcounter = 0;
- }
- catch(const YarnException &e) {
+ } catch (const ApplicationMasterNotRegisteredException &e) {
+ /*
+* In case catch this exception,
+* heartbeat
incubator-hawq git commit: HAWQ-544. Fixed assertion failure in AdvanceMemoryAccountingGeneration when we have a generation overflow
Repository: incubator-hawq
Updated Branches:
refs/heads/master b754d92f4 -> 536d99a6d
HAWQ-544. Fixed assertion failure in AdvanceMemoryAccountingGeneration when we
have a generation overflow
Signed-off-by: George Caragea
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/536d99a6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/536d99a6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/536d99a6
Branch: refs/heads/master
Commit: 536d99a6d1211ded20aaf502c0d70c2e8b5f9b52
Parents: b754d92
Author: Foyzur Rahman
Authored: Tue Mar 15 12:19:44 2016 -0700
Committer: Ming LI
Committed: Thu Mar 24 10:52:22 2016 +0800
--
src/backend/utils/mmgr/memaccounting.c | 2 ++
1 file changed, 2 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/536d99a6/src/backend/utils/mmgr/memaccounting.c
--
diff --git a/src/backend/utils/mmgr/memaccounting.c
b/src/backend/utils/mmgr/memaccounting.c
index 8b2e4c2..1c60235 100644
--- a/src/backend/utils/mmgr/memaccounting.c
+++ b/src/backend/utils/mmgr/memaccounting.c
@@ -1170,6 +1170,8 @@ AdvanceMemoryAccountingGeneration()
{
/* Overflow happened, we need to adjust for generation */
elog(LOG, "Migrating all allocated memory chunks to generation
%u due to generation counter exhaustion and QE recycling",
MemoryAccountingCurrentGeneration);
+ /* elog might have allocated more memory and we need to record
that in current RolloverMemoryAccount->peak */
+ RolloverMemoryAccount->peak = Max(RolloverMemoryAccount->peak,
MemoryAccountingPeakBalance);
HandleMemoryAccountingGenerationOverflow(TopMemoryContext);
}
incubator-hawq git commit: HAWQ-579. As we hasn't supported the guc value get_tmpdir_from_rm to be true, so remove them and add error information when users set get_tmpdir_from_rm=true;
Repository: incubator-hawq
Updated Branches:
refs/heads/master 3114af839 -> b754d92f4
HAWQ-579. As we hasn't supported the guc value get_tmpdir_from_rm to be true,
so remove them and add error information when users set get_tmpdir_from_rm=true;
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/b754d92f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/b754d92f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/b754d92f
Branch: refs/heads/master
Commit: b754d92f4cb93514f73f7f39f4431d0e25faa554
Parents: 3114af8
Author: doli
Authored: Wed Mar 23 14:37:56 2016 +0800
Committer: Ming LI
Committed: Thu Mar 24 10:08:46 2016 +0800
--
src/backend/postmaster/identity.c | 10 +-
src/backend/resourcemanager/communication/rmcomm_QD2RM.c | 3 ++-
.../resourcemanager/include/communication/rmcomm_QD2RM.h | 2 +-
src/backend/utils/init/postinit.c | 4
4 files changed, 16 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b754d92f/src/backend/postmaster/identity.c
--
diff --git a/src/backend/postmaster/identity.c
b/src/backend/postmaster/identity.c
index f4dd1ed..1bb558d 100644
--- a/src/backend/postmaster/identity.c
+++ b/src/backend/postmaster/identity.c
@@ -100,8 +100,9 @@ static SegmentIdentity SegmentId = { SEGMENT_ROLE_INVALID };
static void DebugSegmentIdentity(struct SegmentIdentity *id);
static void DebugProcessIdentity(struct ProcessIdentity *id);
static boolDeserializeProcessIdentity(struct ProcessIdentity *id, const
char *str);
-static void GetLocalTmpDirFromRM(char *host, uint16_t port, int session_id,
int command_id, int qeidx);
+//static void GetLocalTmpDirFromRM(char *host, uint16_t port, int session_id,
int command_id, int qeidx);
+/*
static void
GetLocalTmpDirFromRM(char *host,
uint16_t port,
@@ -156,6 +157,7 @@ GetLocalTmpDirFromRM(char *host,
session_id, command_id, qeidx);
}
}
+*/
static void
SetSegmentRole(const char *name, SegmentIdentity *segment)
@@ -384,7 +386,10 @@ SetupProcessIdentity(const char *str)
if (get_tmpdir_from_rm)
{
+elog(ERROR, "The GUC value get_tmpdir_from_rm to be true hasn't
been supported yet. "
+ "Please set get_tmpdir_from_rm=false ");
/* If QE is under one segment. */
+/*
if ( GetQEIndex() != -1 ) {
GetLocalTmpDirFromRM("127.0.0.1",//DRMGlobalInstance->SocketLocalHostName.Str,
rm_segment_port,
@@ -392,7 +397,9 @@ SetupProcessIdentity(const char *str)
gp_command_count,
GetQEIndex());
}
+*/
/* QE is under master. */
+/*
else {
GetLocalTmpDirFromRM("127.0.0.1",//DRMGlobalInstance->SocketLocalHostName.Str,
rm_master_port,
@@ -403,6 +410,7 @@ SetupProcessIdentity(const char *str)
elog(DEBUG1, "Get temporary directory from segment resource
manager, %s",
LocalTempPath);
+*/
}
else
{
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b754d92f/src/backend/resourcemanager/communication/rmcomm_QD2RM.c
--
diff --git a/src/backend/resourcemanager/communication/rmcomm_QD2RM.c
b/src/backend/resourcemanager/communication/rmcomm_QD2RM.c
index 59494a0..f08f9d5 100644
--- a/src/backend/resourcemanager/communication/rmcomm_QD2RM.c
+++ b/src/backend/resourcemanager/communication/rmcomm_QD2RM.c
@@ -1221,6 +1221,7 @@ exit:
destroySelfMaintainBuffer(&recvbuffer);
}
+/*
int getLocalTmpDirFromMasterRM(char *errorbuf, int errorbufsize)
{
static char errorbuf2[ERRORMESSAGE_SIZE];
@@ -1273,7 +1274,7 @@ exit:
destroySelfMaintainBuffer(&recvbuffer);
return res;
}
-
+*/
int acquireResourceQuotaFromRM(int64_t user_oid,
uint32_t
max_seg_count_fix,
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b754d92f/src/backend/resourcemanager/include/communication/rmcomm_QD2RM.h
--
diff --git a/src/backend/resourcemanager/include/communication/rmcomm_QD2RM.h
b/src/backend/resourcemanager/include/communication/rmcomm_QD2RM.h
index 98a51b7..1fdc02c 100644
--- a/src/backend/resourcemanager/include/communi
incubator-hawq git commit: HAWQ-459. Return default empty response for patterns with no tables
Repository: incubator-hawq
Updated Branches:
refs/heads/HAWQ-459 287229645 -> bb7773478
HAWQ-459. Return default empty response for patterns with no tables
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/bb777347
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/bb777347
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/bb777347
Branch: refs/heads/HAWQ-459
Commit: bb7773478bd2ab0f1de8f5e18f8ac7a9f390a56c
Parents: 2872296
Author: Shivram Mani
Authored: Wed Mar 23 15:01:02 2016 -0700
Committer: Shivram Mani
Committed: Wed Mar 23 15:01:02 2016 -0700
--
.../pxf/plugins/hive/HiveMetadataFetcher.java | 6 +++
.../plugins/hive/utilities/HiveUtilities.java | 3 +-
.../pxf/service/MetadataResponseFormatter.java | 3 +-
.../service/MetadataResponseFormatterTest.java | 42 +---
4 files changed, 29 insertions(+), 25 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bb777347/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java
--
diff --git
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java
index c035a55..d228ec5 100644
---
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java
+++
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java
@@ -54,6 +54,12 @@ public class HiveMetadataFetcher extends MetadataFetcher {
public List getMetadata(String pattern) throws Exception {
List tblsDesc =
HiveUtilities.extractTablesFromPattern(client, pattern);
+
+if(tblsDesc == null || tblsDesc.isEmpty()) {
+LOG.warn("No tables found for the given pattern: " + pattern);
+return null;
+}
+
List metadataList = new ArrayList();
for(Metadata.Item tblDesc: tblsDesc) {
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bb777347/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilities.java
--
diff --git
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilities.java
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilities.java
index 70574a4..7dfe410 100644
---
a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilities.java
+++
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilities.java
@@ -272,7 +272,8 @@ public class HiveUtilities {
try {
databases = client.getDatabases(dbPattern);
if(databases.isEmpty()) {
-throw new IllegalArgumentException("No database found for the
given pattern");
+LOG.warn("No database found for the given pattern: " +
dbPattern);
+return null;
}
for(String dbName: databases) {
for(String tableName: client.getTables(dbName, tablePattern)) {
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bb777347/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/MetadataResponseFormatter.java
--
diff --git
a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/MetadataResponseFormatter.java
b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/MetadataResponseFormatter.java
index 04d3d79..a420ca7 100644
---
a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/MetadataResponseFormatter.java
+++
b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/MetadataResponseFormatter.java
@@ -37,6 +37,7 @@ import org.apache.hawq.pxf.api.Metadata;
public class MetadataResponseFormatter {
private static final Log LOG =
LogFactory.getLog(MetadataResponseFormatter.class);
+private static final String METADATA_DEFAULT_RESPONSE =
"{\"PXFMetadata\":[]}";
/**
* Converts list of {@link Metadata} to JSON String format.
@@ -59,7 +60,7 @@ public class MetadataResponseFormatter {
private static String metadataToJSON(List metadataList) throws
IOException {
if (metadataList == null || metadataList.isEmpty()) {
-throw new IllegalArgumentException("no metadata objects found -
cannot serialize");
+ return METADATA_DEFAULT_RESPONSE;
}
StringBuilder result = null;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bb777347/pxf/pxf-service/src/test/java/org/apache/hawq/pxf/service/MetadataResponseFormatterTest.java
-
[incubator-hawq] Git Push Summary
Repository: incubator-hawq Updated Branches: refs/heads/HAWQ-514 [deleted] 4828bb6b1
incubator-hawq git commit: HAWQ-514. Updated pxf rpm summary/description
Repository: incubator-hawq
Updated Branches:
refs/heads/master e4cee64a4 -> 3114af839
HAWQ-514. Updated pxf rpm summary/description
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/3114af83
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/3114af83
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/3114af83
Branch: refs/heads/master
Commit: 3114af8395c3a051a435e9e32a8fd2d31d65939d
Parents: e4cee64
Author: Shivram Mani
Authored: Wed Mar 23 10:40:24 2016 -0700
Committer: Shivram Mani
Committed: Wed Mar 23 10:40:24 2016 -0700
--
pxf/build.gradle | 13 -
1 file changed, 12 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/3114af83/pxf/build.gradle
--
diff --git a/pxf/build.gradle b/pxf/build.gradle
index db5deea..ed7b9f0 100644
--- a/pxf/build.gradle
+++ b/pxf/build.gradle
@@ -111,7 +111,6 @@ subprojects { subProject ->
//buildRpm
ospackage {
-summary = 'The PXF extensions library for HAWQ'
vendor = project.vendor
release = buildNumber()
version = subProject.version.split('-')[0];
@@ -191,6 +190,8 @@ project('pxf-service') {
}
ospackage {
+summary = 'HAWQ Extension Framework (PXF), service REST API'
+description = 'Rest API for the HAWQ Extenstion framework'
requires('apache-tomcat', "$tomcatVersion", GREATER | EQUAL)
requires('hadoop', "$hadoopVersion", GREATER | EQUAL)
@@ -287,6 +288,9 @@ project('pxf-hdfs') {
}
ospackage {
+summary = 'HAWQ Extension Framework (PXF), HDFS plugin'
+description = 'Querying external data stored in HDFS'
+
requires('pxf-service', project.version, GREATER | EQUAL)
requires('hadoop', "$hadoopVersion", GREATER | EQUAL)
requires('hadoop-mapreduce', "$hadoopVersion", GREATER | EQUAL)
@@ -317,6 +321,10 @@ project('pxf-hive') {
}
ospackage {
+
+summary = 'HAWQ Extension Framework (PXF), Hive plugin'
+description = 'Querying external data stored in Hive'
+
requires('pxf-hdfs', project.version, GREATER | EQUAL)
requires('hive', "$hiveVersion", GREATER | EQUAL)
@@ -336,6 +344,9 @@ project('pxf-hbase') {
}
ospackage {
+summary = 'HAWQ Extension Framework (PXF), HBase plugin'
+description = 'Querying external data stored in HBase'
+
requires('pxf-service', project.version, GREATER | EQUAL)
requires('hbase', "$hbaseVersionRPM", GREATER | EQUAL)
incubator-hawq git commit: HAWQ-514. Updated pxf rpm summary/description
Repository: incubator-hawq
Updated Branches:
refs/heads/HAWQ-514 [created] 4828bb6b1
HAWQ-514. Updated pxf rpm summary/description
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/4828bb6b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/4828bb6b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/4828bb6b
Branch: refs/heads/HAWQ-514
Commit: 4828bb6b120c919655c23a314631c14f1664d7cb
Parents: e4cee64
Author: Shivram Mani
Authored: Wed Mar 23 10:38:59 2016 -0700
Committer: Shivram Mani
Committed: Wed Mar 23 10:38:59 2016 -0700
--
pxf/build.gradle | 13 -
1 file changed, 12 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4828bb6b/pxf/build.gradle
--
diff --git a/pxf/build.gradle b/pxf/build.gradle
index db5deea..ed7b9f0 100644
--- a/pxf/build.gradle
+++ b/pxf/build.gradle
@@ -111,7 +111,6 @@ subprojects { subProject ->
//buildRpm
ospackage {
-summary = 'The PXF extensions library for HAWQ'
vendor = project.vendor
release = buildNumber()
version = subProject.version.split('-')[0];
@@ -191,6 +190,8 @@ project('pxf-service') {
}
ospackage {
+summary = 'HAWQ Extension Framework (PXF), service REST API'
+description = 'Rest API for the HAWQ Extenstion framework'
requires('apache-tomcat', "$tomcatVersion", GREATER | EQUAL)
requires('hadoop', "$hadoopVersion", GREATER | EQUAL)
@@ -287,6 +288,9 @@ project('pxf-hdfs') {
}
ospackage {
+summary = 'HAWQ Extension Framework (PXF), HDFS plugin'
+description = 'Querying external data stored in HDFS'
+
requires('pxf-service', project.version, GREATER | EQUAL)
requires('hadoop', "$hadoopVersion", GREATER | EQUAL)
requires('hadoop-mapreduce', "$hadoopVersion", GREATER | EQUAL)
@@ -317,6 +321,10 @@ project('pxf-hive') {
}
ospackage {
+
+summary = 'HAWQ Extension Framework (PXF), Hive plugin'
+description = 'Querying external data stored in Hive'
+
requires('pxf-hdfs', project.version, GREATER | EQUAL)
requires('hive', "$hiveVersion", GREATER | EQUAL)
@@ -336,6 +344,9 @@ project('pxf-hbase') {
}
ospackage {
+summary = 'HAWQ Extension Framework (PXF), HBase plugin'
+description = 'Querying external data stored in HBase'
+
requires('pxf-service', project.version, GREATER | EQUAL)
requires('hbase', "$hbaseVersionRPM", GREATER | EQUAL)
