On 10/10/2012 05:34 PM, Tom Lane wrote:
Hannu Krosing <ha...@krosing.net> writes:
One way would be to check that we are in an any --> cstring
function - perhaps just by setting some static flag et entry and resetting
it at exit - and pass the original byte representation as "bytes" (or
string for py2.x)
Totally aside from the ugliness of driving that off the *other* end
being cstring, it seems quite insufficient to me.
Please find attached a patch, which solves the typeio function recursion
problem by simply testing if the function we are currently in is a type-io
function (fn_oid == argTypeStruct->typoutput ... )
This is definitely WIP, put here just to verify the approach is mostly sane.
Also there are not integrated tests in the patch or docs yet.
See attached test-pytypeio.sql for sample usage.
It is usable for simple cases, like non-toastable fixed length types
- both pass by value and pass py reference - and
non-toastable varlen types. It has no expicit support yet for any
more fancy things like toasting or new short varlen headers.
It also has the beginnings of support for type "internal" so that also
send and receive functions can be written in plpython.
Some of the work also went into accepting shell types so that you
actually can define typeio functions,
For example, if the
data type in question is toastable, you don't really want to leave the
Python code with the problem of detoasting a toasted value.
The next think I'll do is to fashion my raw input/output functions for
toastable cases after bytea. Currently they are just tested for simple
"old postgresql varlen type".
Even if
it's just an int, your proposal saddles the Python code with enddianness
problems.
This can also be seen as a feature, that is you _can_ encode the binary
exactly as you like. For example you have 4-byte strings encoded in
int4-sized pass-by value chunks. or 16 digit decimals encoded as 16
4-bit nibbles.
And endianness dead simple to do in pythons struct module, as
it's just one char prefix in format string.
I think my suggestion of a way to pretend the argument or result is of
some specified other type for conversion purposes is quite a lot superior.
In the toastable-type case, referencing bytea would be enough to get the
Python code out from under detoasting and length-word management.
Will look into it.
----
Hannu Krosing
diff --git a/src/pl/plpython/plpy_procedure.c b/src/pl/plpython/plpy_procedure.c
index a28cb9a..17e9dd1 100644
--- a/src/pl/plpython/plpy_procedure.c
+++ b/src/pl/plpython/plpy_procedure.c
@@ -193,6 +193,16 @@ PLy_procedure_create(HeapTuple procTup, Oid fn_oid, bool is_trigger)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("trigger functions can only be called as triggers")));
+ else if (procStruct->prorettype == CSTRINGOID){
+ /* do the real work */
+ PLy_output_datum_func(&proc->result, rvTypeTup);
+ }
+ else if (rvTypeStruct->typisdefined == false){
+ /* leave shell type as uninitialised */
+ proc->result.out.d.typoid = procStruct->prorettype;
+ proc->result.out.d.typmod = -1;
+ proc->result.is_rowtype = -1;
+ }
else if (procStruct->prorettype != VOIDOID &&
procStruct->prorettype != RECORDOID)
ereport(ERROR,
@@ -212,6 +222,10 @@ PLy_procedure_create(HeapTuple procTup, Oid fn_oid, bool is_trigger)
proc->result.out.d.typmod = -1;
proc->result.is_rowtype = 2;
}
+ else if ((fn_oid == rvTypeStruct->typinput)||
+ (fn_oid == rvTypeStruct->typreceive)) {
+ PLy_output_datum_func_direct(&proc->result, rvTypeTup);
+ }
else
{
/* do the real work */
@@ -276,7 +290,22 @@ PLy_procedure_create(HeapTuple procTup, Oid fn_oid, bool is_trigger)
switch (argTypeStruct->typtype)
{
case TYPTYPE_PSEUDO:
- /* Disallow pseudotype argument */
+ if ((types[i] == CSTRINGOID) ||
+ (types[i] == INTERNALOID)){
+ PLy_input_datum_func(&(proc->args[pos]),
+ types[i],
+ argTypeTup);
+ break;
+ }
+ if (argTypeStruct->typisdefined == false) {
+ /*
+ * should be safe to leave shell type uninitialised
+ * by the time the function is called, it will be a full type
+ */
+ proc->args[pos].is_rowtype = -1;
+ break;
+ }
+ /* Disallow other pseudotype arguments */
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("PL/Python functions cannot accept type %s",
@@ -287,6 +316,14 @@ PLy_procedure_create(HeapTuple procTup, Oid fn_oid, bool is_trigger)
proc->args[pos].is_rowtype = 2;
break;
default:
+ /* type io functions do direct conversion */
+ if ((fn_oid == argTypeStruct->typoutput) ||
+ (fn_oid == argTypeStruct->typsend)) {
+ PLy_input_datum_func_direct(&(proc->args[pos]),
+ types[i],
+ argTypeTup);
+ break;
+ }
PLy_input_datum_func(&(proc->args[pos]),
types[i],
argTypeTup);
diff --git a/src/pl/plpython/plpy_typeio.c b/src/pl/plpython/plpy_typeio.c
index 8f2367d..bb09472 100644
--- a/src/pl/plpython/plpy_typeio.c
+++ b/src/pl/plpython/plpy_typeio.c
@@ -41,8 +41,11 @@ static PyObject *PLyInt_FromInt32(PLyDatumToOb *arg, Datum d);
static PyObject *PLyLong_FromInt64(PLyDatumToOb *arg, Datum d);
static PyObject *PLyLong_FromOid(PLyDatumToOb *arg, Datum d);
static PyObject *PLyBytes_FromBytea(PLyDatumToOb *arg, Datum d);
+static PyObject *PLyString_FromCString(PLyDatumToOb *arg, Datum d);
+static PyObject *PLyLong_FromInternal(PLyDatumToOb *arg, Datum d);
static PyObject *PLyString_FromDatum(PLyDatumToOb *arg, Datum d);
static PyObject *PLyList_FromArray(PLyDatumToOb *arg, Datum d);
+static PyObject *PLyBytes_FromRawType(PLyDatumToOb *arg, Datum d);
/* conversion from Python objects to Datums */
static Datum PLyObject_ToBool(PLyObToDatum *arg, int32 typmod, PyObject *plrv);
@@ -51,6 +54,9 @@ static Datum PLyObject_ToComposite(PLyObToDatum *arg, int32 typmod, PyObject *pl
static Datum PLyObject_ToDatum(PLyObToDatum *arg, int32 typmod, PyObject *plrv);
static Datum PLySequence_ToArray(PLyObToDatum *arg, int32 typmod, PyObject *plrv);
+static Datum PLyBytes_ToRawType(PLyObToDatum *arg, int32 typmod, PyObject *plrv);
+
+
/* conversion from Python objects to composite Datums (used by triggers and SRFs) */
static Datum PLyString_ToComposite(PLyTypeInfo *info, TupleDesc desc, PyObject *string);
static Datum PLyMapping_ToComposite(PLyTypeInfo *info, TupleDesc desc, PyObject *mapping);
@@ -360,6 +366,107 @@ PLyObject_ToCompositeDatum(PLyTypeInfo *info, TupleDesc desc, PyObject *plrv)
return datum;
}
+void
+PLy_output_datum_func_direct(PLyTypeInfo *ti, HeapTuple typeTup)
+{
+ PLyObToDatum *arg = &(ti->out.d);
+ Form_pg_type typeStruct = (Form_pg_type) GETSTRUCT(typeTup);
+
+ perm_fmgr_info(typeStruct->typinput, &arg->typfunc);
+ arg->typoid = HeapTupleGetOid(typeTup);
+ arg->typmod = -1;
+ arg->typioparam = getTypeIOParam(typeTup);
+ arg->typbyval = typeStruct->typbyval;
+ arg->typlen = typeStruct->typlen;
+ arg->typalign = typeStruct->typalign;
+
+ arg->func = PLyBytes_ToRawType;
+
+}
+
+/*
+ * Convert a Python bytes to a PostgreSQL bytea datum. This doesn't
+ * go through the generic conversion function to circumvent problems
+ * with embedded nulls. And it's faster this way.
+ */
+static Datum
+PLyBytes_ToRawType(PLyObToDatum *arg, int32 typmod, PyObject *plrv)
+{
+ // copy of PLyObjecttoBytea now, will change if basic functioning is checked ok
+ PyObject *volatile plrv_so = NULL;
+ Datum rv;
+
+ Assert(plrv != Py_None);
+
+ plrv_so = PyObject_Bytes(plrv);
+ if (!plrv_so)
+ PLy_elog(ERROR, "could not create bytes representation of Python object");
+
+ PG_TRY();
+ {
+ char *plrv_sc = PyBytes_AsString(plrv_so);
+ size_t len = PyBytes_Size(plrv_so);
+ size_t size;
+ void *result;
+
+ /* return varlen types as bytea */
+ if (arg->typlen == -1) {
+ size = len + VARHDRSZ;
+ result = palloc(size);
+ SET_VARSIZE(result, size);
+ memcpy(VARDATA(result), plrv_sc, len);
+ rv = PointerGetDatum(result);
+ }
+ else {
+ size = arg->typlen;
+ if (len != size)
+ ereport(ERROR,
+ (errcode(ERRCODE_DATA_EXCEPTION),
+ errmsg("PL/Python type output conversion data length mismatch"),
+ errdetail("Python length %d, PostgreSQL type length %d", (int)len, (int)size)));
+ if (arg->typbyval){
+ switch (size) {
+ case 1:
+ rv = ((Datum) SET_1_BYTE(*(uint8*)plrv_sc));
+ break;
+ case 2:
+ rv = ((Datum) SET_2_BYTES(*(int16*)plrv_sc));
+ break;
+ case 4:
+ rv = ((Datum) SET_4_BYTES(*(int32*)plrv_sc));
+ break;
+ case 8:
+ rv = ((Datum) SET_8_BYTES(*(int64*)plrv_sc));
+ break;
+ default:
+ ereport(ERROR,
+ (errcode(ERRCODE_DATA_EXCEPTION),
+ errmsg("PL/Python type output conversion not supported for pass-by-value length: %d", (int)size)
+ ));
+ }
+ }
+ else {
+ result = palloc(size);
+ memcpy(result, plrv_sc, len);
+ rv = PointerGetDatum(result);
+ }
+ }
+ }
+ PG_CATCH();
+ {
+ Py_XDECREF(plrv_so);
+ PG_RE_THROW();
+ }
+ PG_END_TRY();
+
+ Py_XDECREF(plrv_so);
+
+ if (get_typtype(arg->typoid) == TYPTYPE_DOMAIN) // ToDo: this should be probably cached as arg->typtype
+ domain_check(rv, false, arg->typoid, &arg->typfunc.fn_extra, arg->typfunc.fn_mcxt);
+
+ return rv;
+}
+
static void
PLy_output_datum_func2(PLyObToDatum *arg, HeapTuple typeTup)
{
@@ -422,6 +529,49 @@ PLy_output_datum_func2(PLyObToDatum *arg, HeapTuple typeTup)
}
}
+/*
+ * convert PostgreSQL binary representation directly to python.
+ * This is used by types output() and send() functions which
+ * must manage the actual conversion
+ */
+
+void
+PLy_input_datum_func_direct(PLyTypeInfo *ti, Oid typeOid, HeapTuple typeTup)
+{
+ PLyDatumToOb *arg = &(ti->in.d);
+ Form_pg_type typeStruct = (Form_pg_type) GETSTRUCT(typeTup);
+
+ /* Get the type's conversion information */
+ perm_fmgr_info(typeStruct->typoutput, &arg->typfunc);
+ arg->typoid = HeapTupleGetOid(typeTup);
+ arg->typmod = -1;
+ arg->typioparam = getTypeIOParam(typeTup);
+ arg->typbyval = typeStruct->typbyval;
+ arg->typlen = typeStruct->typlen;
+ arg->typalign = typeStruct->typalign;
+
+ arg->func = PLyBytes_FromRawType;
+
+}
+
+/*
+ * convert postgresql datum directly to python bytes()
+ * used in type-io functions where real conversion is done in python
+ */
+static PyObject *
+PLyBytes_FromRawType(PLyDatumToOb *arg, Datum d)
+{
+ if (arg->typbyval) {
+ return PyBytes_FromStringAndSize((char*) &d, arg->typlen);
+ }
+ else {
+ bytea *var = DatumGetByteaP(d);
+ char *bytes = VARDATA(var);
+ size_t size = (arg->typlen > 0)? arg->typlen: VARSIZE(var) - VARHDRSZ;
+ return PyBytes_FromStringAndSize(bytes, size);
+ }
+}
+
static void
PLy_input_datum_func2(PLyDatumToOb *arg, Oid typeOid, HeapTuple typeTup)
{
@@ -467,6 +617,12 @@ PLy_input_datum_func2(PLyDatumToOb *arg, Oid typeOid, HeapTuple typeTup)
case BYTEAOID:
arg->func = PLyBytes_FromBytea;
break;
+ case CSTRINGOID:
+ arg->func = PLyString_FromCString;
+ break;
+ case INTERNALOID:
+ arg->func = PLyLong_FromInternal;
+ break;
default:
arg->func = PLyString_FromDatum;
break;
@@ -567,6 +723,23 @@ PLyBytes_FromBytea(PLyDatumToOb *arg, Datum d)
}
static PyObject *
+PLyString_FromCString(PLyDatumToOb *arg, Datum d)
+{
+ const char *txt = DatumGetCString(d);
+
+ return PyBytes_FromString(txt);
+}
+
+static PyObject *
+PLyLong_FromInternal(PLyDatumToOb *arg, Datum d)
+{
+ /* convert pgsql type 'internal' to integer so it can be used by ctypes */
+ void* p = DatumGetPointer(d);
+
+ return PyLong_FromVoidPtr(p);
+}
+
+static PyObject *
PLyString_FromDatum(PLyDatumToOb *arg, Datum d)
{
char *x = OutputFunctionCall(&arg->typfunc, d);
diff --git a/src/pl/plpython/plpy_typeio.h b/src/pl/plpython/plpy_typeio.h
index 82e472a..4648ddf 100644
--- a/src/pl/plpython/plpy_typeio.h
+++ b/src/pl/plpython/plpy_typeio.h
@@ -94,6 +94,9 @@ extern void PLy_typeinfo_dealloc(PLyTypeInfo *arg);
extern void PLy_input_datum_func(PLyTypeInfo *arg, Oid typeOid, HeapTuple typeTup);
extern void PLy_output_datum_func(PLyTypeInfo *arg, HeapTuple typeTup);
+extern void PLy_input_datum_func_direct(PLyTypeInfo *arg, Oid typeOid, HeapTuple typeTup);
+extern void PLy_output_datum_func_direct(PLyTypeInfo *arg, HeapTuple typeTup);
+
extern void PLy_input_tuple_funcs(PLyTypeInfo *arg, TupleDesc desc);
extern void PLy_output_tuple_funcs(PLyTypeInfo *arg, TupleDesc desc);
CREATE EXTENSION plpythonu;
CREATE OR REPLACE FUNCTION get_cstring(OUT cs cstring) LANGUAGE plpythonu AS $$
return 'this_is_a_cstring'
$$;
SELECT get_cstring();
SELECT pg_typeof(get_cstring());
CREATE OR REPLACE FUNCTION cstring_to_text(IN cs cstring, OUT tx text) LANGUAGE plpythonu AS $$
return cs
$$;
SELECT cstring_to_text('a cstring argument'::cstring);
SELECT pg_typeof(cstring_to_text('a cstring argument'::cstring));
/*
* CREATE TYPE pydata , the python equivalent of text
*/
DROP TYPE pydata CASCADE;
CREATE TYPE pydata;
CREATE OR REPLACE FUNCTION pydata_input(IN cs_pydata cstring, OUT raw_pydata pydata) LANGUAGE plpythonu AS $$
#plpy.notice('pydata_input: %s, type %s' % (repr(cs_pydata), type(cs_pydata)))
return cs_pydata
$$;
CREATE OR REPLACE FUNCTION pydata_output(IN raw_pydata pydata, OUT cs_pydata cstring) LANGUAGE plpythonu AS $$
#plpy.notice('pydata_output: %s, type %s' % (repr(raw_pydata)), type(raw_pydata)))
return raw_pydata
$$;
CREATE TYPE pydata (
INPUT = pydata_input,
OUTPUT = pydata_output,
INTERNALLENGTH = VARIABLE
);
select 'pydatasample'::pydata;
/*
* CREATE TYPE pyint4 , the python equivalent of int4
*/
DROP TYPE IF EXISTS pyint4 CASCADE;
CREATE TYPE pyint4;
CREATE OR REPLACE FUNCTION pyint4_input(IN cs_pydata cstring, OUT raw_pydata pyint4) LANGUAGE plpythonu AS $$
#plpy.notice('pydata_input: %s, type %s' % (repr(cs_pydata), type(cs_pydata)))
import struct
return struct.pack('i',int(cs_pydata));
$$;
CREATE OR REPLACE FUNCTION pyint4_output(IN raw_pydata pyint4, OUT cs_pydata cstring) LANGUAGE plpythonu AS $$
#plpy.notice('pydata_output: %s, type %s' % (repr(raw_pydata), type(raw_pydata)))
import struct
return struct.unpack('i',raw_pydata)[0]
$$;
CREATE TYPE pyint4 (
INPUT = pyint4_input,
OUTPUT = pyint4_output,
INTERNALLENGTH = 4,
PASSEDBYVALUE
);
SELECT '42'::pyint4;
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
