Repository: incubator-hawq Updated Branches: refs/heads/master 1e3cdd448 -> 95f337ce4
HAWQ-932. Added localhsot to curl resolution option. Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/95f337ce Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/95f337ce Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/95f337ce Branch: refs/heads/master Commit: 95f337ce4a7a88e97d85e4139e70009b57e1824d Parents: 1e3cdd4 Author: Oleksandr Diachenko <[email protected]> Authored: Fri Jul 22 14:19:17 2016 -0700 Committer: Oleksandr Diachenko <[email protected]> Committed: Fri Jul 22 14:19:31 2016 -0700 ---------------------------------------------------------------------- src/backend/access/external/libchurl.c | 55 ++++++++++++++----------- src/backend/access/external/pxfmasterapi.c | 2 +- src/backend/catalog/external/externalmd.c | 2 +- src/backend/utils/adt/pxf_functions.c | 3 +- src/include/access/libchurl.h | 3 ++ 5 files changed, 37 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/95f337ce/src/backend/access/external/libchurl.c ---------------------------------------------------------------------- diff --git a/src/backend/access/external/libchurl.c b/src/backend/access/external/libchurl.c index 5af0ea9..b0a76bb 100644 --- a/src/backend/access/external/libchurl.c +++ b/src/backend/access/external/libchurl.c @@ -304,28 +304,31 @@ void churl_headers_cleanup(CHURL_HEADERS headers) pfree(settings); } - -CHURL_HANDLE churl_init_upload(const char* url, CHURL_HEADERS headers) +CHURL_HANDLE churl_init(const char* url, CHURL_HEADERS headers) { churl_context* context = churl_new_context(); create_curl_handle(context); - context->upload = true; clear_error_buffer(context); + /* needed to resolve localhost */ + if (strstr(url, LocalhostIpV4) != NULL) { + struct curl_slist *resolve_hosts = NULL; + char *pxf_host_entry = (char *) palloc0(strlen(pxf_service_address) + strlen(LocalhostIpV4Entry) + 1); + strcat(pxf_host_entry, pxf_service_address); + strcat(pxf_host_entry, LocalhostIpV4Entry); + resolve_hosts = curl_slist_append(NULL, pxf_host_entry); + set_curl_option(context, CURLOPT_RESOLVE, resolve_hosts); + pfree(pxf_host_entry); + } + set_curl_option(context, CURLOPT_URL, url); set_curl_option(context, CURLOPT_VERBOSE, (const void*)FALSE); set_curl_option(context, CURLOPT_ERRORBUFFER, context->curl_error_buffer); set_curl_option(context, CURLOPT_IPRESOLVE, (const void*)CURL_IPRESOLVE_V4); - set_curl_option(context, CURLOPT_POST, (const void*)TRUE); - set_curl_option(context, CURLOPT_READFUNCTION, read_callback); - set_curl_option(context, CURLOPT_READDATA, context); set_curl_option(context, CURLOPT_WRITEFUNCTION, write_callback); set_curl_option(context, CURLOPT_WRITEDATA, context); set_curl_option(context, CURLOPT_HEADERFUNCTION, header_callback); set_curl_option(context, CURLOPT_HEADERDATA, context); - churl_headers_append(headers, "Content-Type", "application/octet-stream"); - churl_headers_append(headers, "Transfer-Encoding", "chunked"); - churl_headers_append(headers, "Expect", "100-continue"); churl_headers_set(context, headers); print_http_headers(headers); @@ -334,25 +337,27 @@ CHURL_HANDLE churl_init_upload(const char* url, CHURL_HEADERS headers) return (CHURL_HANDLE)context; } -CHURL_HANDLE churl_init_download(const char* url, CHURL_HEADERS headers) +CHURL_HANDLE churl_init_upload(const char* url, CHURL_HEADERS headers) { - churl_context* context = churl_new_context(); - create_curl_handle(context); - context->upload = false; - clear_error_buffer(context); + churl_context* context = churl_init(url, headers); - set_curl_option(context, CURLOPT_URL, url); - set_curl_option(context, CURLOPT_VERBOSE, (const void*)FALSE); - set_curl_option(context, CURLOPT_ERRORBUFFER, context->curl_error_buffer); - set_curl_option(context, CURLOPT_IPRESOLVE, (const void*)CURL_IPRESOLVE_V4); - set_curl_option(context, CURLOPT_WRITEFUNCTION, write_callback); - set_curl_option(context, CURLOPT_WRITEDATA, context); - set_curl_option(context, CURLOPT_HEADERFUNCTION, header_callback); - set_curl_option(context, CURLOPT_HEADERDATA, context); - churl_headers_set(context, headers); + context->upload = true; - print_http_headers(headers); - setup_multi_handle(context); + set_curl_option(context, CURLOPT_POST, (const void*) TRUE); + set_curl_option(context, CURLOPT_READFUNCTION, read_callback); + set_curl_option(context, CURLOPT_READDATA, context); + churl_headers_append(headers, "Content-Type", "application/octet-stream"); + churl_headers_append(headers, "Transfer-Encoding", "chunked"); + churl_headers_append(headers, "Expect", "100-continue"); + + return (CHURL_HANDLE)context; +} + +CHURL_HANDLE churl_init_download(const char* url, CHURL_HEADERS headers) +{ + churl_context* context = churl_init(url, headers); + + context->upload = false; return (CHURL_HANDLE)context; } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/95f337ce/src/backend/access/external/pxfmasterapi.c ---------------------------------------------------------------------- diff --git a/src/backend/access/external/pxfmasterapi.c b/src/backend/access/external/pxfmasterapi.c index e07277e..833218a 100644 --- a/src/backend/access/external/pxfmasterapi.c +++ b/src/backend/access/external/pxfmasterapi.c @@ -434,5 +434,5 @@ List* get_and_cache_external_metadata(GPHDUri* hadoop_uri, char *profile, char * List* get_no_cache_external_metadata(GPHDUri* hadoop_uri, char *profile, char *pattern, ClientContext *client_context) { - return get_external_metadata(hadoop_uri, profile, pattern, client_context, NULL); + return get_external_metadata(hadoop_uri, profile, pattern, client_context, InvalidOid); } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/95f337ce/src/backend/catalog/external/externalmd.c ---------------------------------------------------------------------- diff --git a/src/backend/catalog/external/externalmd.c b/src/backend/catalog/external/externalmd.c index aca07c4..0e39d25 100644 --- a/src/backend/catalog/external/externalmd.c +++ b/src/backend/catalog/external/externalmd.c @@ -84,7 +84,7 @@ List *ParsePxfEntries(StringInfo json, char *profile, Oid dboid) { struct json_object *jsonItem = json_object_array_get_idx(jsonItems, i); PxfItem *pxfItem = ParsePxfItem(jsonItem, profile); - if (dboid != NULL) + if (dboid != InvalidOid) LoadPxfItem(pxfItem, dboid); tables = lappend(tables, pxfItem); } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/95f337ce/src/backend/utils/adt/pxf_functions.c ---------------------------------------------------------------------- diff --git a/src/backend/utils/adt/pxf_functions.c b/src/backend/utils/adt/pxf_functions.c index 806565a..f17c9e0 100644 --- a/src/backend/utils/adt/pxf_functions.c +++ b/src/backend/utils/adt/pxf_functions.c @@ -19,6 +19,7 @@ #include "postgres.h" +#include "access/hd_work_mgr.h" #include "catalog/external/externalmd.h" #include "fmgr.h" #include "funcapi.h" @@ -43,7 +44,7 @@ pxf_item_fields_enum_start(text *profile, text *pattern) char *profile_cstr = text_to_cstring(profile); char *pattern_cstr = text_to_cstring(pattern); - items = get_pxf_item_metadata(profile_cstr, pattern_cstr, NULL); + items = get_pxf_item_metadata(profile_cstr, pattern_cstr, InvalidOid); if (items == NIL) return NULL; http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/95f337ce/src/include/access/libchurl.h ---------------------------------------------------------------------- diff --git a/src/include/access/libchurl.h b/src/include/access/libchurl.h index 5db2a06..c3ad17d 100644 --- a/src/include/access/libchurl.h +++ b/src/include/access/libchurl.h @@ -143,3 +143,6 @@ void print_http_headers(CHURL_HEADERS headers); #endif + +#define LocalhostIpV4Entry ":127.0.0.1" +#define LocalhostIpV4 "localhost"
