Repository: incubator-hawq Updated Branches: refs/heads/master 6f3337c65 -> ae2af74b2
HAWQ-1629. Add ORC format using pluggable storage framework. Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/ae2af74b Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/ae2af74b Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/ae2af74b Branch: refs/heads/master Commit: ae2af74b2cf6f6679deca6f26be76bdbf7d02fa6 Parents: 6f3337c Author: oushu1wangziming1 <[email protected]> Authored: Fri Aug 24 10:40:02 2018 +0800 Committer: Ruilong Huo <[email protected]> Committed: Thu Aug 30 14:27:36 2018 +0800 ---------------------------------------------------------------------- configure | 36 ++++ configure.in | 9 + contrib/Makefile | 6 +- contrib/exthdfs/exthdfs.c | 4 +- contrib/orc/Makefile | 33 ++++ contrib/orc/README | 5 + contrib/orc/orc.c | 237 +++++++++++++++++++++++++ contrib/orc/orc_init.sql | 60 +++++++ contrib/orc/orc_install.sql | 70 ++++++++ contrib/orc/orc_uninstall.sql | 36 ++++ src/Makefile.global.in | 1 + src/backend/access/external/plugstorage.c | 44 +++-- src/backend/catalog/Makefile | 10 ++ src/backend/utils/Makefile | 2 +- src/backend/utils/cache/lsyscache.c | 12 ++ src/include/access/plugstorage_utils.h | 18 +- src/include/catalog/pg_exttable.h | 2 + src/include/utils/lsyscache.h | 1 + 18 files changed, 560 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae2af74b/configure ---------------------------------------------------------------------- diff --git a/configure b/configure index b0063d3..de64b1b 100755 --- a/configure +++ b/configure @@ -731,6 +731,7 @@ krb_srvtab with_krb5 with_gssapi with_pgcrypto +with_orc with_r with_java with_python @@ -860,6 +861,7 @@ with_python with_java with_r with_pgcrypto +with_orc with_gssapi with_krb5 with_krb_srvnam @@ -5835,6 +5837,40 @@ $as_echo "$with_pgcrypto" >&6; } # +# Optionally build with orc in pluggable storage framework. +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build orc modules" >&5 +$as_echo_n "checking whether to build orc modules... " >&6; } + +pgac_args="$pgac_args with_orc" + + +# Check whether --with-orc was given. +if test "${with_orc+set}" = set; then : + withval=$with_orc; + case $withval in + yes) + : + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-orc option" "$LINENO" 5 + ;; + esac + +else + with_orc=yes + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_orc" >&5 +$as_echo "$with_orc" >&6; } + + +# # GSSAPI # { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with GSSAPI support" >&5 http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae2af74b/configure.in ---------------------------------------------------------------------- diff --git a/configure.in b/configure.in index b2b6af4..7d1825c 100644 --- a/configure.in +++ b/configure.in @@ -723,6 +723,15 @@ AC_MSG_RESULT([$with_pgcrypto]) AC_SUBST(with_pgcrypto) # +# Optionally build with orc format in pluggable storage framework. +# +AC_MSG_CHECKING([whether to build orc modules]) +PGAC_ARG_BOOL(with, orc, yes, [ --with-orc build with orc format]) +AC_MSG_RESULT([$with_orc]) +AC_SUBST(with_orc) +# + +# # GSSAPI # AC_MSG_CHECKING([whether to build with GSSAPI support]) http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae2af74b/contrib/Makefile ---------------------------------------------------------------------- diff --git a/contrib/Makefile b/contrib/Makefile index e5daff9..a5c87d4 100644 --- a/contrib/Makefile +++ b/contrib/Makefile @@ -10,12 +10,16 @@ WANTED_DIRS = \ gp_cancel_query \ formatter_fixedwidth \ exthdfs\ - hawq-hadoop + hawq-hadoop\ ifeq ($(with_pgcrypto), yes) WANTED_DIRS += pgcrypto endif +ifeq ($(with_orc), yes) +WANTED_DIRS += orc +endif + # Missing: # start-scripts \ (does not have a makefile) # xml2 \ (requires libxml installed) http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae2af74b/contrib/exthdfs/exthdfs.c ---------------------------------------------------------------------- diff --git a/contrib/exthdfs/exthdfs.c b/contrib/exthdfs/exthdfs.c index 1378734..548d784 100644 --- a/contrib/exthdfs/exthdfs.c +++ b/contrib/exthdfs/exthdfs.c @@ -251,13 +251,13 @@ Datum hdfsprotocol_validate(PG_FUNCTION_ARGS) } } } - /*if(1) + if(1) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("hdfsprotocol_validate : " "no formatter is supported for external hdfs"))); - }*/ + } if (!isCsv && !isText && !isOrc) { ereport(ERROR, http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae2af74b/contrib/orc/Makefile ---------------------------------------------------------------------- diff --git a/contrib/orc/Makefile b/contrib/orc/Makefile new file mode 100644 index 0000000..f793f48 --- /dev/null +++ b/contrib/orc/Makefile @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +MODULE_big = orc +OBJS = orc.o + +PG_CPPFLAGS = -I$(libpq_srcdir) +PG_LIBS = $(libpq_pgport) + + +ifdef USE_PGXS +PGXS := $(shell pg_config --pgxs) +include $(PGXS) +else +subdir = contrib/orc +top_builddir = ../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae2af74b/contrib/orc/README ---------------------------------------------------------------------- diff --git a/contrib/orc/README b/contrib/orc/README new file mode 100644 index 0000000..0d669d3 --- /dev/null +++ b/contrib/orc/README @@ -0,0 +1,5 @@ +1. Compile ORC format in pluggable storage framework + $ ./configure --with-orc; make -j8; make install + +2. Configure and initialize cluster + $ hawq init cluster -a http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae2af74b/contrib/orc/orc.c ---------------------------------------------------------------------- diff --git a/contrib/orc/orc.c b/contrib/orc/orc.c new file mode 100644 index 0000000..729c729 --- /dev/null +++ b/contrib/orc/orc.c @@ -0,0 +1,237 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +#include <json-c/json.h> + +#include "c.h" +#include "port.h" +#include "postgres.h" +#include "fmgr.h" +#include "funcapi.h" +#include "nodes/pg_list.h" +#include "utils/memutils.h" +#include "utils/relcache.h" +#include "utils/uri.h" +#include "utils/formatting.h" +#include "utils/lsyscache.h" +#include "utils/datetime.h" +#include "mb/pg_wchar.h" +#include "commands/defrem.h" +#include "commands/copy.h" +#include "access/tupdesc.h" +#include "access/filesplit.h" +#include "access/plugstorage.h" +#include "cdb/cdbvars.h" +#include "catalog/pg_exttable.h" +#include "catalog/namespace.h" +#include "postmaster/identity.h" +#include "nodes/makefuncs.h" +#include "nodes/plannodes.h" +#include "utils/uri.h" + + +#define ORC_TIMESTAMP_EPOCH_JDATE 2457024 /* == date2j(2015, 1, 1) */ +#define MAX_ORC_ARRAY_DIMS 10000 + +/* Do the module magic dance */ +PG_MODULE_MAGIC; + +/* Validators for pluggable storage format ORC */ +PG_FUNCTION_INFO_V1(orc_validate_interfaces); +PG_FUNCTION_INFO_V1(orc_validate_options); +PG_FUNCTION_INFO_V1(orc_validate_encodings); +PG_FUNCTION_INFO_V1(orc_validate_datatypes); + +/* Accessors for pluggable storage format ORC */ +PG_FUNCTION_INFO_V1(orc_beginscan); +PG_FUNCTION_INFO_V1(orc_getnext_init); +PG_FUNCTION_INFO_V1(orc_getnext); +PG_FUNCTION_INFO_V1(orc_rescan); +PG_FUNCTION_INFO_V1(orc_endscan); +PG_FUNCTION_INFO_V1(orc_stopscan); +PG_FUNCTION_INFO_V1(orc_insert_init); +PG_FUNCTION_INFO_V1(orc_insert); +PG_FUNCTION_INFO_V1(orc_insert_finish); + +/* Definitions of validators for pluggable storage format ORC */ +Datum orc_validate_interfaces(PG_FUNCTION_ARGS); +Datum orc_validate_options(PG_FUNCTION_ARGS); +Datum orc_validate_encodings(PG_FUNCTION_ARGS); +Datum orc_validate_datatypes(PG_FUNCTION_ARGS); + +/* Definitions of accessors for pluggable storage format ORC */ +Datum orc_beginscan(PG_FUNCTION_ARGS); +Datum orc_getnext_init(PG_FUNCTION_ARGS); +Datum orc_getnext(PG_FUNCTION_ARGS); +Datum orc_rescan(PG_FUNCTION_ARGS); +Datum orc_endscan(PG_FUNCTION_ARGS); +Datum orc_stopscan(PG_FUNCTION_ARGS); +Datum orc_insert_init(PG_FUNCTION_ARGS); +Datum orc_insert(PG_FUNCTION_ARGS); +Datum orc_insert_finish(PG_FUNCTION_ARGS); + + +Datum orc_validate_interfaces(PG_FUNCTION_ARGS) +{ + elog(ERROR, "Funtion orc_validate_interfaces has not be completed, please fill it"); + PG_RETURN_VOID(); +} + +/* + * void + * orc_validate_options(List *formatOptions, + * char *formatStr, + * bool isWritable) + */ +Datum orc_validate_options(PG_FUNCTION_ARGS) +{ + + elog(ERROR, "Funtion orc_validate_options has not be completed, please fill it"); + PG_RETURN_VOID(); +} + +/* + * void + * orc_validate_encodings(char *encodingName) + */ +Datum orc_validate_encodings(PG_FUNCTION_ARGS) +{ + elog(ERROR, "Funtion orc_validate_encodings has not be completed, please fill it"); + PG_RETURN_VOID(); +} + +/* + * void + * orc_validate_datatypes(TupleDesc tupDesc) + */ +Datum orc_validate_datatypes(PG_FUNCTION_ARGS) +{ + + elog(ERROR, "Funtion orc_validate_datatypes has not be completed, please fill it"); + PG_RETURN_VOID(); +} + +/* + * FileScanDesc + * orc_beginscan(ExternalScan *extScan, + * ScanState *scanState, + * Relation relation, + * int formatterType, + * char *formatterName) + */ +Datum orc_beginscan(PG_FUNCTION_ARGS) +{ + + elog(ERROR, "Funtion orc_beginscan has not be completed, please fill it"); + PG_RETURN_POINTER(NULL); +} + +/* + * ExternalSelectDesc + * orc_getnext_init(PlanState *planState, + * ExternalScanState *extScanState) + */ +Datum orc_getnext_init(PG_FUNCTION_ARGS) +{ + + elog(ERROR, "Funtion orc_getnext_init has not be completed, please fill it"); + PG_RETURN_POINTER(NULL); +} + +/* + * bool + * orc_getnext(FileScanDesc fileScanDesc, + * ScanDirection direction, + * ExternalSelectDesc extSelectDesc, + * ScanState *scanState, + * TupleTableSlot *tupTableSlot) + */ +Datum orc_getnext(PG_FUNCTION_ARGS) +{ + elog(ERROR, "Funtion orc_getnext has not be completed, please fill it"); + PG_RETURN_VOID(); +} + +/* + * void + * orc_rescan(FileScanDesc scan) + */ +Datum orc_rescan(PG_FUNCTION_ARGS) +{ + elog(ERROR, "Funtion orc_rescan has not be completed, please fill it"); + PG_RETURN_VOID(); +} + +/* + * void + * orc_endscan(FileScanDesc scan) + */ +Datum orc_endscan(PG_FUNCTION_ARGS) +{ + + elog(ERROR, "Funtion orc_endscan has not be completed, please fill it"); + PG_RETURN_VOID(); +} + +/* + * void + * orc_stopscan(FileScanDesc scan) + */ +Datum orc_stopscan(PG_FUNCTION_ARGS) +{ + elog(ERROR, "Funtion orc_stopscan has not be completed, please fill it"); + PG_RETURN_VOID(); +} + +/* + * ExternalInsertDesc + * orc_insert_init(Relation relation, + * int formatterType, + * char *formatterName) + */ +Datum orc_insert_init(PG_FUNCTION_ARGS) +{ + + elog(ERROR, "Funtion orc_insert_init has not be completed, please fill it"); + PG_RETURN_POINTER(NULL); +} + +/* + * Oid + * orc_insert(ExternalInsertDesc extInsertDesc, + * TupleTableSlot *tupTableSlot) + */ +Datum orc_insert(PG_FUNCTION_ARGS) +{ + + elog(ERROR, "Funtion orc_insert has not be completed, please fill it"); + PG_RETURN_OID(InvalidOid); +} + +/* + * void + * orc_insert_finish(ExternalInsertDesc extInsertDesc) + */ +Datum orc_insert_finish(PG_FUNCTION_ARGS) +{ + elog(ERROR, "Funtion orc_insert_finish has not be completed, please fill it"); + PG_RETURN_VOID(); +} + http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae2af74b/contrib/orc/orc_init.sql ---------------------------------------------------------------------- diff --git a/contrib/orc/orc_init.sql b/contrib/orc/orc_init.sql new file mode 100644 index 0000000..fa720b7 --- /dev/null +++ b/contrib/orc/orc_init.sql @@ -0,0 +1,60 @@ +-- -------------------------------------------------------------------- +-- +-- orc_init.sql +-- +-- Support ORC format in pluggable storage framework at initialization +-- +-- -------------------------------------------------------------------- + +CREATE OR REPLACE FUNCTION pg_catalog.orc_validate_interfaces() RETURNS void +AS '$libdir/orc.so', 'orc_validate_interfaces' +LANGUAGE C STABLE; + +CREATE OR REPLACE FUNCTION pg_catalog.orc_validate_options() RETURNS void +AS '$libdir/orc.so', 'orc_validate_options' +LANGUAGE C STABLE; + +CREATE OR REPLACE FUNCTION pg_catalog.orc_validate_encodings() RETURNS void +AS '$libdir/orc.so', 'orc_validate_encodings' +LANGUAGE C STABLE; + +CREATE OR REPLACE FUNCTION pg_catalog.orc_validate_datatypes() RETURNS void +AS '$libdir/orc.so', 'orc_validate_datatypes' +LANGUAGE C STABLE; + +CREATE OR REPLACE FUNCTION pg_catalog.orc_beginscan() RETURNS bytea +AS '$libdir/orc.so', 'orc_beginscan' +LANGUAGE C STABLE; + +CREATE OR REPLACE FUNCTION pg_catalog.orc_getnext_init() RETURNS bytea +AS '$libdir/orc.so', 'orc_getnext_init' +LANGUAGE C STABLE; + +CREATE OR REPLACE FUNCTION pg_catalog.orc_getnext() RETURNS bytea +AS '$libdir/orc.so', 'orc_getnext' +LANGUAGE C STABLE; + +CREATE OR REPLACE FUNCTION pg_catalog.orc_rescan() RETURNS void +AS '$libdir/orc.so', 'orc_rescan' +LANGUAGE C STABLE; + +CREATE OR REPLACE FUNCTION pg_catalog.orc_endscan() RETURNS void +AS '$libdir/orc.so', 'orc_endscan' +LANGUAGE C STABLE; + +CREATE OR REPLACE FUNCTION pg_catalog.orc_stopscan() RETURNS void +AS '$libdir/orc.so', 'orc_stopscan' +LANGUAGE C STABLE; + +CREATE OR REPLACE FUNCTION pg_catalog.orc_insert_init() RETURNS bytea +AS '$libdir/orc.so', 'orc_insert_init' +LANGUAGE C STABLE; + +CREATE OR REPLACE FUNCTION pg_catalog.orc_insert() RETURNS bytea +AS '$libdir/orc.so', 'orc_insert' +LANGUAGE C STABLE; + +CREATE OR REPLACE FUNCTION pg_catalog.orc_insert_finish() RETURNS void +AS '$libdir/orc.so', 'orc_insert_finish' +LANGUAGE C STABLE; + http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae2af74b/contrib/orc/orc_install.sql ---------------------------------------------------------------------- diff --git a/contrib/orc/orc_install.sql b/contrib/orc/orc_install.sql new file mode 100644 index 0000000..424019e --- /dev/null +++ b/contrib/orc/orc_install.sql @@ -0,0 +1,70 @@ +-- -------------------------------------------------------------------- +-- +-- orc_install.sql +-- +-- Support ORC format in pluggable storage framework +-- +-- -------------------------------------------------------------------- + +SET allow_system_table_mods=ddl; + +CREATE OR REPLACE FUNCTION pg_catalog.orc_validate_interfaces() RETURNS void +AS '$libdir/orc.so', 'orc_validate_interfaces' +LANGUAGE C STABLE; + +CREATE OR REPLACE FUNCTION pg_catalog.orc_validate_options() RETURNS void +AS '$libdir/orc.so', 'orc_validate_options' +LANGUAGE C STABLE; + +CREATE OR REPLACE FUNCTION pg_catalog.orc_validate_encodings() RETURNS void +AS '$libdir/orc.so', 'orc_validate_encodings' +LANGUAGE C STABLE; + +CREATE OR REPLACE FUNCTION pg_catalog.orc_validate_datatypes() RETURNS void +AS '$libdir/orc.so', 'orc_validate_datatypes' +LANGUAGE C STABLE; + +CREATE OR REPLACE FUNCTION pg_catalog.orc_beginscan() RETURNS bytea +AS '$libdir/orc.so', 'orc_beginscan' +LANGUAGE C STABLE; + +CREATE OR REPLACE FUNCTION pg_catalog.orc_getnext_init() RETURNS bytea +AS '$libdir/orc.so', 'orc_getnext_init' +LANGUAGE C STABLE; + +CREATE OR REPLACE FUNCTION pg_catalog.orc_getnext() RETURNS bytea +AS '$libdir/orc.so', 'orc_getnext' +LANGUAGE C STABLE; + +CREATE OR REPLACE FUNCTION pg_catalog.orc_rescan() RETURNS void +AS '$libdir/orc.so', 'orc_rescan' +LANGUAGE C STABLE; + +CREATE OR REPLACE FUNCTION pg_catalog.orc_endscan() RETURNS void +AS '$libdir/orc.so', 'orc_endscan' +LANGUAGE C STABLE; + +CREATE OR REPLACE FUNCTION pg_catalog.orc_stopscan() RETURNS void +AS '$libdir/orc.so', 'orc_stopscan' +LANGUAGE C STABLE; + +CREATE OR REPLACE FUNCTION pg_catalog.orc_insert_init() RETURNS bytea +AS '$libdir/orc.so', 'orc_insert_init' +LANGUAGE C STABLE; + +CREATE OR REPLACE FUNCTION pg_catalog.orc_insert() RETURNS bytea +AS '$libdir/orc.so', 'orc_insert' +LANGUAGE C STABLE; + +CREATE OR REPLACE FUNCTION pg_catalog.orc_insert_finish() RETURNS void +AS '$libdir/orc.so', 'orc_insert_finish' +LANGUAGE C STABLE; + +CREATE OR REPLACE FUNCTION hdfs_validate() RETURNS void +AS '$libdir/exthdfs.so', 'hdfsprotocol_validate' +LANGUAGE C STABLE; + +CREATE OR REPLACE FUNCTION hdfs_blocklocation() RETURNS void +AS '$libdir/exthdfs.so', 'hdfsprotocol_blocklocation' +LANGUAGE C STABLE; + http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae2af74b/contrib/orc/orc_uninstall.sql ---------------------------------------------------------------------- diff --git a/contrib/orc/orc_uninstall.sql b/contrib/orc/orc_uninstall.sql new file mode 100644 index 0000000..1ad03b4 --- /dev/null +++ b/contrib/orc/orc_uninstall.sql @@ -0,0 +1,36 @@ +-- -------------------------------------------------------------------- +-- +-- orc_uninstall.sql +-- +-- Remove ORC format in pluggable storage framework +-- +-- -------------------------------------------------------------------- + +SET allow_system_table_mods=ddl; + +DROP FUNCTION IF EXISTS pg_catalog.orc_validate_interfaces(); + +DROP FUNCTION IF EXISTS pg_catalog.orc_validate_options(); + +DROP FUNCTION IF EXISTS pg_catalog.orc_validate_encodings(); + +DROP FUNCTION IF EXISTS pg_catalog.orc_validate_datatypes(); + +DROP FUNCTION IF EXISTS pg_catalog.orc_beginscan(); + +DROP FUNCTION IF EXISTS pg_catalog.orc_getnext_init(); + +DROP FUNCTION IF EXISTS pg_catalog.orc_getnext(); + +DROP FUNCTION IF EXISTS pg_catalog.orc_rescan(); + +DROP FUNCTION IF EXISTS pg_catalog.orc_endscan(); + +DROP FUNCTION IF EXISTS pg_catalog.orc_stopscan(); + +DROP FUNCTION IF EXISTS pg_catalog.orc_insert_init(); + +DROP FUNCTION IF EXISTS pg_catalog.orc_insert(); + +DROP FUNCTION IF EXISTS pg_catalog.orc_insert_finish(); + http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae2af74b/src/Makefile.global.in ---------------------------------------------------------------------- diff --git a/src/Makefile.global.in b/src/Makefile.global.in index 9dae4bb..3691d79 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -169,6 +169,7 @@ with_python = @with_python@ with_tcl = @with_tcl@ with_r = @with_r@ with_pgcrypto = @with_pgcrypto@ +with_orc = @with_orc@ with_java = @with_java@ with_openssl = @with_openssl@ with_system_tzdata = @with_system_tzdata@ http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae2af74b/src/backend/access/external/plugstorage.c ---------------------------------------------------------------------- diff --git a/src/backend/access/external/plugstorage.c b/src/backend/access/external/plugstorage.c index fe7c82a..91cb11c 100644 --- a/src/backend/access/external/plugstorage.c +++ b/src/backend/access/external/plugstorage.c @@ -33,7 +33,7 @@ #include "parser/parse_func.h" /* - * getExternalTableTypeIn(List/Str) + * getExternalTableTypeList * * Return the table type for a given external table * @@ -59,16 +59,22 @@ void getExternalTableTypeInList(const char formatType, } else if (fmttype_is_custom(formatType)) { - *formatterType = ExternalTableType_CUSTOM; - *formatterName = NULL; - } - else if (fmttype_is_custom(formatType)) - { Assert(formatOptions); *formatterName = getExtTblFormatterTypeInFmtOptsList(formatOptions); - *formatterType = ExternalTableType_PLUG; + if (pg_strncasecmp(*formatterName, "text", strlen("text")) == 0) + { + *formatterType = ExternalTableType_TEXT_CUSTOM; + } + else if (pg_strncasecmp(*formatterName, "csv", strlen("csv")) == 0) + { + *formatterType = ExternalTableType_CSV_CUSTOM; + } + else + { + *formatterType = ExternalTableType_PLUG; + } } else { @@ -78,9 +84,13 @@ void getExternalTableTypeInList(const char formatType, } } +/* + * getExternalTableTypeStr + * + */ void getExternalTableTypeInStr(const char formatType, char *formatOptions, - int *formatterType, + int *formatterType, char **formatterName) { if (fmttype_is_text(formatType)) @@ -95,17 +105,23 @@ void getExternalTableTypeInStr(const char formatType, } else if (fmttype_is_custom(formatType)) { - *formatterType = ExternalTableType_CUSTOM; - *formatterName = NULL; - } - else if (fmttype_is_custom(formatType)) - { Assert(formatOptions); *formatterName = getExtTblFormatterTypeInFmtOptsStr(formatOptions); Assert(*formatterName); - *formatterType = ExternalTableType_PLUG; + if (pg_strncasecmp(*formatterName, "text", strlen("text")) == 0) + { + *formatterType = ExternalTableType_TEXT_CUSTOM; + } + else if (pg_strncasecmp(*formatterName, "csv", strlen("csv")) == 0) + { + *formatterType = ExternalTableType_CSV_CUSTOM; + } + else + { + *formatterType = ExternalTableType_PLUG; + } } else { http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae2af74b/src/backend/catalog/Makefile ---------------------------------------------------------------------- diff --git a/src/backend/catalog/Makefile b/src/backend/catalog/Makefile index 68bfa2b..f50074a 100644 --- a/src/backend/catalog/Makefile +++ b/src/backend/catalog/Makefile @@ -30,6 +30,10 @@ include $(top_srcdir)/src/backend/common.mk cdb_init.sql: cdb_schema.sql filesystem.sql cat $(call vpathsearch,cdb_schema.sql) $(call vpathsearch,cdb_external_extensions.sql) $(call vpathsearch,filesystem.sql) > cdb_init.sql +ifeq ($(with_orc), yes) + cat $(srcdir)/../../../contrib/orc/orc_init.sql >> cdb_init.sql +endif + cdb_schema.sql: cdb_schema.in sed -e 's,@pkglibdir@,$(pkglibdir),g' \ -e 's,@dlsuffix@,$(DLSUFFIX),g' \ @@ -139,6 +143,12 @@ install-data: $(BKIFILES) installdirs $(INSTALL_DATA) $(call vpathsearch,cdb_init.sql) '$(DESTDIR)$(datadir)/cdb_init.sql' $(INSTALL_DATA) $(call vpathsearch,cdb_external_extensions.sql) '$(DESTDIR)$(datadir)/cdb_external_extensions.sql' $(INSTALL_DATA) $(call vpathsearch,filesystem.sql) '$(DESTDIR)$(datadir)/filesystem.sql' + +ifeq ($(with_orc), yes) + $(INSTALL_DATA) $(srcdir)/../../../contrib/orc/orc_install.sql '$(DESTDIR)$(datadir)/orc_install.sql' + $(INSTALL_DATA) $(srcdir)/../../../contrib/orc/orc_uninstall.sql '$(DESTDIR)$(datadir)/orc_uninstall.sql' +endif + $(INSTALL_DATA) $(srcdir)/sql_features.txt '$(DESTDIR)$(datadir)/sql_features.txt' $(INSTALL_DATA) $(call vpathsearch,gp_toolkit.sql) '$(DESTDIR)$(datadir)/gp_toolkit.sql' http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae2af74b/src/backend/utils/Makefile ---------------------------------------------------------------------- diff --git a/src/backend/utils/Makefile b/src/backend/utils/Makefile index da8a82e..05b02f0 100644 --- a/src/backend/utils/Makefile +++ b/src/backend/utils/Makefile @@ -8,7 +8,7 @@ subdir = src/backend/utils top_builddir = ../../.. include $(top_builddir)/src/Makefile.global -OBJS = fmgrtab.o session_state.o +OBJS = fmgrtab.o session_state.o SUBDIRS = adt cache error fmgr hash init mb misc mmgr resowner \ resscheduler sort time gpmon gp workfile_manager http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae2af74b/src/backend/utils/cache/lsyscache.c ---------------------------------------------------------------------- diff --git a/src/backend/utils/cache/lsyscache.c b/src/backend/utils/cache/lsyscache.c index 3ccf847..2bd929f 100644 --- a/src/backend/utils/cache/lsyscache.c +++ b/src/backend/utils/cache/lsyscache.c @@ -2553,6 +2553,18 @@ type_is_rowtype(Oid typid) } /* + * type_is_basetype + * + * Convenience function to determine whether a type OID represents + * a "rowtype" type --- neither invalid & a named base type. + */ +bool +type_is_basetype(Oid typid) +{ + return (typid != InvalidOid && get_typtype(typid) == TYPTYPE_BASE); +} + +/* * get_typ_typrelid * * Given the type OID, get the typrelid (InvalidOid if not a complex http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae2af74b/src/include/access/plugstorage_utils.h ---------------------------------------------------------------------- diff --git a/src/include/access/plugstorage_utils.h b/src/include/access/plugstorage_utils.h index e0a397d..77686a0 100644 --- a/src/include/access/plugstorage_utils.h +++ b/src/include/access/plugstorage_utils.h @@ -80,14 +80,16 @@ typedef struct PlugStorageInsertFuncs * method for them for each tuple. * */ -typedef enum -{ - ExternalTableType_GENERIC, /* GENERIC external table format and protocol */ - ExternalTableType_TEXT, /* TEXT format with gpfdist(s), http, command protocol */ - ExternalTableType_CSV, /* CSV format with gpfdist(s), http, command protocol */ - ExternalTableType_CUSTOM, /* Temporary workaround to support the old framework */ - ExternalTableType_PLUG, /* Pluggable format with hdfs protocol, i.e., ORC */ - ExternalTableType_Invalid +typedef enum { + ExternalTableType_GENERIC, /* GENERIC external table format and protocol */ + ExternalTableType_TEXT, /* TEXT format with gpfdist(s), http, command protocol + */ + ExternalTableType_CSV, /* CSV format with gpfdist(s), http, command protocol + */ + ExternalTableType_TEXT_CUSTOM, /* TEXT format with hdfs protocol */ + ExternalTableType_CSV_CUSTOM, /* CSV format with hdfs protocol */ + ExternalTableType_PLUG, /* Pluggable format with hdfs protocol, i.e., ORC */ + ExternalTableType_Invalid } ExternalTableType; #endif /* PLUGSTORAGE_UTILS_H */ http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae2af74b/src/include/catalog/pg_exttable.h ---------------------------------------------------------------------- diff --git a/src/include/catalog/pg_exttable.h b/src/include/catalog/pg_exttable.h index 11f053c..d5dd8dc 100644 --- a/src/include/catalog/pg_exttable.h +++ b/src/include/catalog/pg_exttable.h @@ -167,6 +167,7 @@ RemoveExtTableEntry(Oid relid); #define CustomFormatType 'b' #define TextFormatType 't' #define CsvFormatType 'c' +#define OrcFormatType 'o' #define NonCustomFormatType 'n' /* PXF formats*/ @@ -176,5 +177,6 @@ RemoveExtTableEntry(Oid relid); #define fmttype_is_custom(c) (c == CustomFormatType) #define fmttype_is_text(c) (c == TextFormatType) #define fmttype_is_csv(c) (c == CsvFormatType) +#define fmttype_is_orc(c) (c == OrcFormatType) #endif /* PG_EXTTABLE_H */ http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae2af74b/src/include/utils/lsyscache.h ---------------------------------------------------------------------- diff --git a/src/include/utils/lsyscache.h b/src/include/utils/lsyscache.h index 6ee99be..b3c9828 100644 --- a/src/include/utils/lsyscache.h +++ b/src/include/utils/lsyscache.h @@ -125,6 +125,7 @@ extern char get_typstorage(Oid typid); extern Node *get_typdefault(Oid typid); extern char get_typtype(Oid typid); extern bool type_is_rowtype(Oid typid); +extern bool type_is_basetype(Oid typid); extern Oid get_typ_typrelid(Oid typid); extern Oid get_element_type(Oid typid); extern Oid get_array_type(Oid typid);
