Repository: incubator-hawq Updated Branches: refs/heads/HAWQ-546 5e67e48dd -> 6a6b8a461 (forced update)
HAWQ-546. Implemented call of pxf_get_object_fields for Hive on psql. Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/6a6b8a46 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/6a6b8a46 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/6a6b8a46 Branch: refs/heads/HAWQ-546 Commit: 6a6b8a461e4c3cd42e8083a8d5603abade3be391 Parents: cac5e17 Author: Oleksandr Diachenko <odiache...@pivotal.io> Authored: Tue Mar 29 14:44:05 2016 -0700 Committer: Oleksandr Diachenko <odiache...@pivotal.io> Committed: Wed Mar 30 16:56:15 2016 -0700 ---------------------------------------------------------------------- src/bin/psql/describe.c | 132 +++++++++++++++++--- src/test/regress/input/hcatalog_lookup.source | 4 + src/test/regress/output/hcatalog_lookup.source | 7 ++ 3 files changed, 125 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6a6b8a46/src/bin/psql/describe.c ---------------------------------------------------------------------- diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index 1672e64..ec299a7 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -42,12 +42,16 @@ static bool describeOneTSConfig(const char *oid, const char *nspname, static void printACLColumn(PQExpBuffer buf, const char *colname); static bool isGPDB(void); static bool isGPDB4200OrLater(void); -static bool describeHcatalogTable(const char *pattern, bool verbose); +static bool describePxfTable(const char *profile, const char *pattern, bool verbose); +static void parsePxfPattern(const char *user_pattern, char **pattern); /* GPDB 3.2 used PG version 8.2.10, and we've moved the minor number up since then for each release, 4.1 = 8.2.15 */ /* Allow for a couple of future releases. If the version isn't in this range, we are talking to PostgreSQL, not GPDB */ #define mightBeGPDB() (pset.sversion >= 80210 && pset.sversion < 80222) +#define HiveProfileName "Hive" +#define HcatalogSourceName "hcatalog" + static bool isGPDB(void) { static enum { gpdb_maybe, gpdb_yes, gpdb_no } talking_to_gpdb; @@ -1152,12 +1156,25 @@ describeTableDetails(const char *pattern, bool verbose, bool showSystem) PQExpBufferData buf; PGresult *res; int i; - char const *hcatalog = "hcatalog"; + //Hive hook in this method + if(pattern && strncmp(pattern, HcatalogSourceName, strlen(HcatalogSourceName)) == 0) + { + char *pxf_pattern = NULL; + char *pattern_dup = strdup(pattern); + parsePxfPattern(pattern_dup, &pxf_pattern); + if (!pxf_pattern) + { + fprintf(stderr, _("Invalid pattern provided.\n")); + free(pattern_dup); + return false; + } + + bool success = describePxfTable(HiveProfileName, pxf_pattern, verbose); + free(pattern_dup); + return success; + } - //Hcatalog hook in this method - if(strncmp(pattern, hcatalog, strlen(hcatalog)) == 0) - return describeHcatalogTable(pattern, verbose); initPQExpBuffer(&buf); @@ -4221,42 +4238,121 @@ printACLColumn(PQExpBuffer buf, const char *colname) colname, gettext_noop("Access privileges")); } - +/* + * parsePxfPattern + * + * Splits user_pattern by "." and writes second part to pattern. + */ static void -parseExternalPattern(const char *user_pattern, char **profile, char **pattern) +parsePxfPattern(const char *user_pattern, char **pattern) { - - *profile = strtok(user_pattern, "."); + strtok(user_pattern, "."); *pattern = strtok(NULL, "/0"); - } +/* + * describePxfTable + * + * Describes external PXF table. + */ static bool -describeHcatalogTable(const char *pattern, bool verbose) +describePxfTable(const char *profile, const char *pattern, bool verbose) { PQExpBufferData buf; + PQExpBufferData title; PGresult *res; printQueryOpt myopt = pset.popt; - char *pxf_profile = "*"; - char *pxf_pattern = "*"; + printTableContent cont; + int cols = 0; + int total_numrows = 0; + char *headers[2]; + bool printTableInitialized = false; + + char *previous_path = NULL; + char *previous_itemname = NULL; - parseExternalPattern(pattern, &pxf_profile, &pattern); + char *path; + char *itemname; + char *fieldname; + char *fieldtype; + int total_fields = 0; //needed to know how much memory allocate for current table initPQExpBuffer(&buf); - printfPQExpBuffer(&buf, "SELECT * FROM\n" - "pxf_get_object_fields('%s', '%s') \n", pxf_profile, pxf_pattern); + printfPQExpBuffer(&buf, "SELECT t.*, COUNT() OVER(PARTITION BY path, itemname) as total_fields FROM\n" + "pxf_get_item_fields('%s', '%s') t\n", profile, pattern); res = PSQLexec(buf.data, false); + total_numrows = PQntuples(res); termPQExpBuffer(&buf); if (!res) return false; myopt.nullPrint = NULL; - myopt.title = _("List of Hcatalog tables"); + myopt.title = _("List of Hive tables"); myopt.translate_header = true; - printQuery(res, &myopt, pset.queryFout, pset.logfile); + /* Header */ + headers[0] = gettext_noop("Column"); + headers[1] = gettext_noop("Type"); + cols = 2; + + for (int i = 0; i < total_numrows; i++) + { + + path = PQgetvalue(res, i, 0); + itemname = PQgetvalue(res, i, 1); + fieldname = PQgetvalue(res, i, 2); + fieldtype = PQgetvalue(res, i, 3); + total_fields = PQgetvalue(res, i, 4); + + /* First row for current table */ + if (previous_itemname == NULL + || strncmp(previous_itemname, itemname, + strlen(previous_itemname)) != 0 + || strncmp(previous_path, path, + strlen(previous_path)) != 0) + { + + if (previous_itemname != NULL) + printTable(&cont, pset.queryFout, pset.logfile); + + /* Do clean-up for previous tables if any */ + if (printTableInitialized) + { + printTableCleanup(&cont); + termPQExpBuffer(&title); + printTableInitialized = false; + } + + /* Initialize */ + initPQExpBuffer(&title); + printfPQExpBuffer(&title, _("PXF %s Table \"%s.%s\""), profile, path, itemname); + printTableInit(&cont, &myopt, title.data, cols, total_fields); + printTableInitialized = true; + + for (int j = 0; j < cols; j++) + printTableAddHeader(&cont, headers[j], true, 'l'); + } + + /* Column */ + printTableAddCell(&cont, fieldname, false, false); + + /* Type */ + printTableAddCell(&cont, fieldtype, false, false); + + previous_path = path; + previous_itemname = itemname; + + } + + printTable(&cont, pset.queryFout, pset.logfile); + + if (printTableInitialized) + { + printTableCleanup(&cont); + termPQExpBuffer(&title); + } PQclear(res); return true; http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6a6b8a46/src/test/regress/input/hcatalog_lookup.source ---------------------------------------------------------------------- diff --git a/src/test/regress/input/hcatalog_lookup.source b/src/test/regress/input/hcatalog_lookup.source index 3d7279f..239494a 100644 --- a/src/test/regress/input/hcatalog_lookup.source +++ b/src/test/regress/input/hcatalog_lookup.source @@ -7,11 +7,15 @@ SET hcatalog_enable = false; SELECT * from hcatalog.db.t; SELECT * FROM pxf_get_item_fields('Hive', '*'); + \d hcatalog.*.* -- enable GUC SET hcatalog_enable = true; SELECT * FROM pxf_get_item_fields('Hive', '*abc*abc*'); +\d hcatalog.*abc*.*abc* +\d hcatalog +\d hcatalog. -- Create function to insert and scan in-memory data to pg_class CREATE OR REPLACE FUNCTION convert_to_hcatalog_schema(schemaName text) RETURNS text http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6a6b8a46/src/test/regress/output/hcatalog_lookup.source ---------------------------------------------------------------------- diff --git a/src/test/regress/output/hcatalog_lookup.source b/src/test/regress/output/hcatalog_lookup.source index ed4a7ef..533603a 100644 --- a/src/test/regress/output/hcatalog_lookup.source +++ b/src/test/regress/output/hcatalog_lookup.source @@ -9,6 +9,8 @@ LINE 1: SELECT * from hcatalog.db.t; ^ SELECT * FROM pxf_get_item_fields('Hive', '*'); ERROR: HCatalog querying is not enabled +\d hcatalog.*.* +ERROR: HCatalog querying is not enabled -- enable GUC SET hcatalog_enable = true; SELECT * FROM pxf_get_item_fields('Hive', '*abc*abc*'); @@ -16,6 +18,11 @@ SELECT * FROM pxf_get_item_fields('Hive', '*abc*abc*'); ------+----------+-----------+----------- (0 rows) +\d hcatalog.*abc*.*abc* +\d hcatalog +Invalid pattern provided. +\d hcatalog. +Invalid pattern provided. -- Create function to insert and scan in-memory data to pg_class CREATE OR REPLACE FUNCTION convert_to_hcatalog_schema(schemaName text) RETURNS text AS '@abs_builddir@/regress@DLSUFFIX@', 'convert_to_hcatalog_schema'
