Repository: incubator-hawq Updated Branches: refs/heads/HAWQ-546 91e5da112 -> 5e67e48dd (forced update)
HAWQ-546. Implemented call of pxf_get_object_fields for Hive on psql. Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/5e67e48d Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/5e67e48d Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/5e67e48d Branch: refs/heads/HAWQ-546 Commit: 5e67e48dde588fd6c2133a92bcea52b8e667a84e Parents: cac5e17 Author: Oleksandr Diachenko <odiache...@pivotal.io> Authored: Tue Mar 29 14:44:05 2016 -0700 Committer: Oleksandr Diachenko <odiache...@pivotal.io> Committed: Wed Mar 30 16:01:40 2016 -0700 ---------------------------------------------------------------------- src/bin/psql/describe.c | 121 +++++++++++++++++--- src/test/regress/input/hcatalog_lookup.source | 4 + src/test/regress/output/hcatalog_lookup.source | 7 ++ 3 files changed, 114 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/5e67e48d/src/bin/psql/describe.c ---------------------------------------------------------------------- diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index 1672e64..0b7d23d 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -42,12 +42,16 @@ static bool describeOneTSConfig(const char *oid, const char *nspname, static void printACLColumn(PQExpBuffer buf, const char *colname); static bool isGPDB(void); static bool isGPDB4200OrLater(void); -static bool describeHcatalogTable(const char *pattern, bool verbose); +static bool describePxfTable(const char *profile, const char *pattern, bool verbose); +static void parsePxfPattern(const char *user_pattern, char **pattern); /* GPDB 3.2 used PG version 8.2.10, and we've moved the minor number up since then for each release, 4.1 = 8.2.15 */ /* Allow for a couple of future releases. If the version isn't in this range, we are talking to PostgreSQL, not GPDB */ #define mightBeGPDB() (pset.sversion >= 80210 && pset.sversion < 80222) +#define HiveProfileName "Hive" +#define HcatalogSourceName "hcatalog" + static bool isGPDB(void) { static enum { gpdb_maybe, gpdb_yes, gpdb_no } talking_to_gpdb; @@ -1152,12 +1156,24 @@ describeTableDetails(const char *pattern, bool verbose, bool showSystem) PQExpBufferData buf; PGresult *res; int i; - char const *hcatalog = "hcatalog"; + //Hive hook in this method + if(pattern && strncmp(pattern, HcatalogSourceName, strlen(HcatalogSourceName)) == 0) + { + char *pxf_pattern = NULL; + char *pattern_dup = strdup(pattern); + parsePxfPattern(pattern_dup, &pxf_pattern); + if (!pxf_pattern) + { + fprintf(stderr, _("Invalid pattern provided.\n")); + return false; + } + + bool success = describePxfTable(HiveProfileName, pxf_pattern, verbose); + free(pattern_dup); + return success; + } - //Hcatalog hook in this method - if(strncmp(pattern, hcatalog, strlen(hcatalog)) == 0) - return describeHcatalogTable(pattern, verbose); initPQExpBuffer(&buf); @@ -4221,42 +4237,111 @@ printACLColumn(PQExpBuffer buf, const char *colname) colname, gettext_noop("Access privileges")); } - +/* + * parsePxfPattern + * + * Splits user_pattern by "." and writes second part to pattern. + */ static void -parseExternalPattern(const char *user_pattern, char **profile, char **pattern) +parsePxfPattern(const char *user_pattern, char **pattern) { - - *profile = strtok(user_pattern, "."); + strtok(user_pattern, "."); *pattern = strtok(NULL, "/0"); - } +/* + * describePxfTable + * + * Describes external PXF table. + */ static bool -describeHcatalogTable(const char *pattern, bool verbose) +describePxfTable(const char *profile, const char *pattern, bool verbose) { PQExpBufferData buf; + PQExpBufferData title; PGresult *res; printQueryOpt myopt = pset.popt; - char *pxf_profile = "*"; - char *pxf_pattern = "*"; - - parseExternalPattern(pattern, &pxf_profile, &pattern); + printTableContent cont; + int cols = 0; + int total_numrows = 0; + char *headers[2]; + bool printTableInitialized = false; initPQExpBuffer(&buf); printfPQExpBuffer(&buf, "SELECT * FROM\n" - "pxf_get_object_fields('%s', '%s') \n", pxf_profile, pxf_pattern); + "pxf_get_item_fields('%s', '%s') \n", profile, pattern); res = PSQLexec(buf.data, false); + total_numrows = PQntuples(res); termPQExpBuffer(&buf); if (!res) return false; myopt.nullPrint = NULL; - myopt.title = _("List of Hcatalog tables"); + myopt.title = _("List of Hive tables"); myopt.translate_header = true; - printQuery(res, &myopt, pset.queryFout, pset.logfile); + + /* Header */ + headers[0] = gettext_noop("Column"); + headers[1] = gettext_noop("Type"); + cols = 2; + + char *previous_path = NULL; + char *previous_itemname = NULL; + + + for (int i = 0; i < total_numrows; i++) + { + + /* First row for current table*/ + if (previous_itemname == NULL + || strncmp(previous_itemname, PQgetvalue(res, i, 1), + strlen(previous_itemname)) != 0 + || strncmp(previous_path, PQgetvalue(res, i, 0), + strlen(previous_path)) != 0) + { + + if (previous_itemname != NULL) + printTable(&cont, pset.queryFout, pset.logfile); + + /* Do clean-up for previous tables if any*/ + if (printTableInitialized) + { + printTableCleanup(&cont); + termPQExpBuffer(&title); + printTableInitialized = false; + } + + /* Initialize */ + initPQExpBuffer(&title); + printfPQExpBuffer(&title, _("PXF %s Table \"%s.%s\""), profile, PQgetvalue(res, i, 0), PQgetvalue(res, i, 1)); + printTableInit(&cont, &myopt, title.data, cols, total_numrows); + printTableInitialized = true; + + for (int j = 0; j < cols; j++) + printTableAddHeader(&cont, headers[j], true, 'l'); + } + + /* Column */ + printTableAddCell(&cont, PQgetvalue(res, i, 2), false, false); + + /* Type */ + printTableAddCell(&cont, PQgetvalue(res, i, 3), false, false); + + previous_path = PQgetvalue(res, i, 0); + previous_itemname = PQgetvalue(res, i, 1); + + } + + printTable(&cont, pset.queryFout, pset.logfile); + + if (printTableInitialized) + { + printTableCleanup(&cont); + termPQExpBuffer(&title); + } PQclear(res); return true; http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/5e67e48d/src/test/regress/input/hcatalog_lookup.source ---------------------------------------------------------------------- diff --git a/src/test/regress/input/hcatalog_lookup.source b/src/test/regress/input/hcatalog_lookup.source index 3d7279f..239494a 100644 --- a/src/test/regress/input/hcatalog_lookup.source +++ b/src/test/regress/input/hcatalog_lookup.source @@ -7,11 +7,15 @@ SET hcatalog_enable = false; SELECT * from hcatalog.db.t; SELECT * FROM pxf_get_item_fields('Hive', '*'); + \d hcatalog.*.* -- enable GUC SET hcatalog_enable = true; SELECT * FROM pxf_get_item_fields('Hive', '*abc*abc*'); +\d hcatalog.*abc*.*abc* +\d hcatalog +\d hcatalog. -- Create function to insert and scan in-memory data to pg_class CREATE OR REPLACE FUNCTION convert_to_hcatalog_schema(schemaName text) RETURNS text http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/5e67e48d/src/test/regress/output/hcatalog_lookup.source ---------------------------------------------------------------------- diff --git a/src/test/regress/output/hcatalog_lookup.source b/src/test/regress/output/hcatalog_lookup.source index ed4a7ef..533603a 100644 --- a/src/test/regress/output/hcatalog_lookup.source +++ b/src/test/regress/output/hcatalog_lookup.source @@ -9,6 +9,8 @@ LINE 1: SELECT * from hcatalog.db.t; ^ SELECT * FROM pxf_get_item_fields('Hive', '*'); ERROR: HCatalog querying is not enabled +\d hcatalog.*.* +ERROR: HCatalog querying is not enabled -- enable GUC SET hcatalog_enable = true; SELECT * FROM pxf_get_item_fields('Hive', '*abc*abc*'); @@ -16,6 +18,11 @@ SELECT * FROM pxf_get_item_fields('Hive', '*abc*abc*'); ------+----------+-----------+----------- (0 rows) +\d hcatalog.*abc*.*abc* +\d hcatalog +Invalid pattern provided. +\d hcatalog. +Invalid pattern provided. -- Create function to insert and scan in-memory data to pg_class CREATE OR REPLACE FUNCTION convert_to_hcatalog_schema(schemaName text) RETURNS text AS '@abs_builddir@/regress@DLSUFFIX@', 'convert_to_hcatalog_schema'