Hello again list,
so, now the diff to current svn for the rest of the implementation:
Index: setup.py
===================================================================
--- setup.py (revisión: 4263)
+++ setup.py (copia de trabajo)
@@ -377,6 +377,7 @@
'utilsExtension',
'hdf5Extension',
'tableExtension',
+ 'reference',
'_comp_lzo',
'_comp_bzip2' ]
if VERSION.endswith('pro'):
@@ -494,6 +495,7 @@
hdf5Extension_libs = LIBS + [hdf5_package.library_name]
tableExtension_libs = LIBS + [hdf5_package.library_name]
indexesExtension_libs = LIBS + [hdf5_package.library_name]
+reference_libs = LIBS + [hdf5_package.library_name]
lrucacheExtension_libs = [] # Doesn't need external libraries
# Compressor modules only need other libraries if they are enabled.
@@ -547,6 +549,15 @@
extra_link_args=LFLAGS,
extra_compile_args=CFLAGS ),
+ Extension( "tables.reference",
+ include_dirs=inc_dirs,
+ define_macros=def_macros,
+ sources=[ pyrex_extfiles['reference'] ],
+ library_dirs=lib_dirs,
+ libraries=reference_libs,
+ extra_link_args=LFLAGS,
+ extra_compile_args=CFLAGS ),
+
Extension( "tables._comp_lzo",
include_dirs=inc_dirs,
define_macros=def_macros,
Index: tables/__init__.py
===================================================================
--- tables/__init__.py (revisión: 4263)
+++ tables/__init__.py (copia de trabajo)
@@ -64,6 +64,7 @@
from tables.utilsExtension import (
isHDF5File, isPyTablesFile, whichLibVersion, lrange )
+from tables.reference import reference
from tables.misc.enum import Enum
from tables.atom import *
Index: tables/hdf5Extension.pyx
===================================================================
--- tables/hdf5Extension.pyx (revisión: 4263)
+++ tables/hdf5Extension.pyx (copia de trabajo)
@@ -75,11 +75,13 @@
H5Gcreate, H5Gopen, H5Gclose, H5Glink, H5Gunlink, H5Gmove, \
H5Gmove2, H5Gget_objinfo, \
H5Dopen, H5Dclose, H5Dread, H5Dwrite, H5Dget_type, \
- H5Dget_space, H5Dvlen_reclaim, \
- H5Tget_native_type, H5Tget_super, H5Tget_class, H5Tcopy, \
- H5Tclose, H5Tis_variable_str, H5Tget_sign, \
+ H5Dget_space, H5Dvlen_reclaim, H5Tcreate, H5Tvlen_create,\
+ H5Tget_native_type, H5Tget_super, H5Tget_class, H5Tcopy, H5Tinsert, \
+ H5Tclose, H5Tis_variable_str, H5Tget_sign, H5Tget_member_name, \
+ H5Tget_member_offset, H5Tget_member_type, H5Tget_nmembers, H5Tget_size, \
+ H5Acreate, H5Aget_space, H5Aopen_name, \
H5Adelete, H5Aget_num_attrs, H5Aget_name, H5Aopen_idx, \
- H5Aread, H5Aclose, H5Pcreate, H5Pclose, \
+ H5Aread, H5Awrite, H5Aclose, H5Pcreate, H5Pclose, \
H5Pset_cache, H5Pset_sieve_buf_size, H5Pset_fapl_log, \
H5Sselect_all, H5Sselect_elements, H5Sselect_hyperslab, \
H5Screate_simple, H5Sget_simple_extent_ndims, \
@@ -90,9 +92,12 @@
H5ARRAYget_ndims, H5ARRAYget_info, \
set_cache_size, get_objinfo, Giterate, Aiterate, H5UIget_info, \
get_len_of_range, get_order, set_order, is_complex, \
- conv_float64_timeval32, truncate_dset
+ conv_float64_timeval32, truncate_dset, H5R_OBJECT, H5T_STD_REF_OBJ, \
+ hobj_ref_t, H5E_auto_t, H5Eget_auto, H5Eset_auto
+from reference cimport reference
+
# Include conversion tables
include "convtypetables.pxi"
@@ -248,17 +253,30 @@
ntype = None
return ntype
+def islist(l):
+ try:
+ dtype = l[0].dtype
+ for e in l:
+ if e.dtype != dtype:
+ return None
+ return dtype
+ except (TypeError, IndexError):
+ return None
+cdef void deleteIfExisting(hid_t node, char *name):
+ cdef H5E_auto_t func
+ cdef void *data
+ H5Eget_auto(&func, &data)
+ H5Eset_auto(NULL, NULL)
+ H5Adelete(node, name)
+ H5Eset_auto(func, data)
# Type extensions declarations (these are subclassed by PyTables
# Python classes)
cdef class File:
- cdef hid_t file_id
- cdef hid_t access_plist
- cdef object name
+ # Instance variables declared in .pxd
-
def _g_new(self, name, pymode, **params):
# Create a new file using default properties
self.name = name
@@ -404,11 +422,12 @@
type.
"""
- cdef int ret
- cdef hid_t dset_id, type_id
+ cdef int ret, i
+ cdef hid_t dset_id, type_id, vlen_id, space_id, attr_id
cdef hsize_t *dims
+ cdef hvl_t *vl_buf
cdef ndarray ndv
- cdef object byteorder, rabyteorder, baseatom
+ cdef object byteorder, rabyteorder, baseatom, basetype
# The dataset id of the node
dset_id = node._v_objectID
@@ -417,9 +436,11 @@
if isinstance(value, numpy.generic):
value = numpy.array(value)
+ basetype = islist(value)
+
# Check if value is a NumPy ndarray and of a supported type
if (isinstance(value, numpy.ndarray) and
- value.dtype.kind in ('V', 'S', 'b', 'i', 'u', 'f', 'c')):
+ value.dtype.kind in 'VSbiufcr'):
if value.dtype.kind == 'V':
description, rabyteorder = descr_from_dtype(value.dtype)
byteorder = byteorders[rabyteorder]
@@ -441,6 +462,37 @@
# Release resources
free(<void *>dims)
H5Tclose(type_id)
+ elif basetype is not None and basetype.kind in 'VSbiufcr':
+ if basetype.kind == 'V':
+ description, rabyteorder = descr_from_dtype(basetype)
+ byteorder = byteorders[rabyteorder]
+ type_id = createNestedType(description, byteorder)
+ else:
+ # Get the associated native HDF5 type of the scalar type
+ baseatom = Atom.from_dtype(basetype.base)
+ byteorder = byteorders[basetype.byteorder]
+ type_id = AtomToHDF5Type(baseatom, byteorder)
+ dims = <hsize_t *> malloc(sizeof(hsize_t))
+ dims[0] = len(value)
+ deleteIfExisting(dset_id, name)
+ space_id = H5Screate_simple(1, dims, NULL)
+ vlen_id = H5Tvlen_create(type_id)
+ attr_id = H5Acreate(dset_id, name, vlen_id, space_id, H5P_DEFAULT)
+ vl_buf = <hvl_t *> malloc(len(value) * sizeof(hvl_t))
+ i = 0
+ for v in value:
+ ndv = v
+ vl_buf[i].len = len(v)
+ vl_buf[i].p = ndv.data
+ i = i + 1
+ H5Awrite(attr_id, vlen_id, vl_buf)
+ H5Dvlen_reclaim(type_id, space_id, H5P_DEFAULT, vl_buf)
+ free(vl_buf)
+ H5Sclose(space_id)
+ free(dims)
+ H5Tclose(vlen_id)
+ H5Tclose(type_id)
+ H5Aclose(attr_id)
else:
# Object cannot be natively represented in HDF5.
# Unicode attributes has to be pickled until we can definitely switch
@@ -469,8 +521,10 @@
cdef H5T_class_t class_id
cdef size_t type_size
cdef hid_t mem_type, dset_id, type_id, native_type
- cdef int rank, ret, enumtype
+ cdef hid_t vlen_id, native_type, attr_id, space_id
+ cdef int rank, ret, enumtype, i
cdef void *rbuf
+ cdef hvl_t *vl_buf
cdef char *str_value
cdef ndarray ndvalue
cdef object shape, stype_atom, shape_atom, retvalue
@@ -502,6 +556,38 @@
self._v_unimplemented.append(attrname)
return None
shape = ()
+ elif (rank == 1 and class_id == H5T_VLEN):
+ attr_id = H5Aopen_name(dset_id, attrname)
+ space_id = H5Aget_space(attr_id)
+ H5Sget_simple_extent_dims(space_id, &nelements, NULL)
+ vl_buf = <hvl_t *> malloc(nelements * sizeof(hvl_t))
+ vlen_id = H5Tget_super(type_id)
+ try:
+ stype_, shape_ = HDF5ToNPExtType(vlen_id, pure_numpy_types=True)
+ dtype = numpy.dtype(stype_, shape_)
+ except TypeError:
+ # This class is not supported. Instead of raising a TypeError, issue a
+ # warning explaining the problem. This will allow to continue browsing
+ # native HDF5 files, while informing the user about the problem.
+ warnings.warn("""\
+Unsupported type for attribute '%s' in node '%s'. Offending HDF5 class: %d"""
+ % (attrname, self.name, class_id), DataTypeWarning)
+ self._v_unimplemented.append(attrname)
+ return None
+ H5Tclose(vlen_id)
+ H5Aread(attr_id, type_id, vl_buf)
+ H5Aclose(attr_id)
+ retvalue = [ ]
+ for 0 <= i < nelements:
+ # Get the container for data
+ ndvalue = numpy.empty(vl_buf[i].len, dtype=dtype)
+ # Get the pointer to the buffer data area
+ memcpy(ndvalue.data, vl_buf[i].p, vl_buf[i].len * dtype.itemsize)
+ retvalue.append(ndvalue)
+ H5Dvlen_reclaim(type_id, space_id, H5P_DEFAULT, vl_buf)
+ H5Tclose(type_id)
+ H5Sclose(space_id)
+ return retvalue
else:
# General case
Index: tables/definitions.pxd
===================================================================
--- tables/definitions.pxd (revisión: 4263)
+++ tables/definitions.pxd (copia de trabajo)
@@ -173,6 +173,7 @@
# such an unsigned long long type.
ctypedef long long hsize_t
ctypedef signed long long hssize_t
+ ctypedef long long hobj_ref_t
ctypedef struct hvl_t:
size_t len # Length of VL data (in base type units)
@@ -203,6 +204,9 @@
H5G_DATASET, # Object is a dataset
H5G_TYPE, # Object is a named data type
+ ctypedef enum H5R_type_t:
+ H5R_OBJECT
+
# Values for fill value status
cdef enum H5D_fill_value_t:
H5D_FILL_VALUE_ERROR = -1,
@@ -219,6 +223,11 @@
size_t linklen
#H5O_stat_t ohdr # Object header information. New in HDF5 1.6
+ ctypedef struct H5E_auto_t_t:
+ pass
+
+ ctypedef H5E_auto_t_t *H5E_auto_t
+
# HDF5 layouts
cdef enum H5D_layout_t:
H5D_LAYOUT_ERROR = -1,
@@ -315,6 +324,10 @@
H5T_UNIX_D32BE
H5T_UNIX_D64BE
+ # reference types
+ cdef enum:
+ H5T_STD_REF_OBJ
+
# The order to retrieve atomic native datatype
cdef enum H5T_direction_t:
H5T_DIR_DEFAULT = 0, #default direction is inscendent
@@ -404,6 +417,7 @@
hid_t H5Tcreate(H5T_class_t type, size_t size)
hid_t H5Tcopy(hid_t type_id)
herr_t H5Tclose(hid_t type_id)
+ hid_t H5Tvlen_create(hid_t base_type_id)
# Operations defined on string data types
htri_t H5Tis_variable_str(hid_t dtype_id)
@@ -412,6 +426,7 @@
int H5Tget_nmembers(hid_t type_id)
char *H5Tget_member_name(hid_t type_id, unsigned membno)
hid_t H5Tget_member_type(hid_t type_id, unsigned membno)
+ size_t H5Tget_member_offset(hid_t type_id, unsigned membno)
hid_t H5Tget_native_type(hid_t type_id, H5T_direction_t direction)
herr_t H5Tget_member_value(hid_t type_id, int membno, void *value)
int H5Tget_offset(hid_t type_id)
@@ -434,7 +449,12 @@
size_t H5Aget_name(hid_t attr_id, size_t buf_size, char *buf)
hid_t H5Aopen_idx(hid_t loc_id, unsigned int idx)
herr_t H5Aread(hid_t attr_id, hid_t mem_type_id, void *buf)
+ herr_t H5Awrite(hid_t attr_id, hid_t mem_type_id, void *buf)
herr_t H5Aclose(hid_t attr_id)
+ hid_t H5Acreate(hid_t loc_id, char *attr_name, hid_t type_id,
+ hid_t space_id, hid_t acpl_id)
+ herr_t H5Aopen_name(hid_t loc_id, char *name)
+ hid_t H5Aget_space(hid_t attr_id)
# Operations with properties
hid_t H5Pcreate(hid_t plist_id)
@@ -447,7 +467,14 @@
H5D_layout_t H5Pget_layout(hid_t plist)
int H5Pget_chunk(hid_t plist, int max_ndims, hsize_t *dims)
+ # error handling
+ herr_t H5Eget_auto(H5E_auto_t *func, void **client_data)
+ herr_t H5Eset_auto(H5E_auto_t func, void *client_data)
+ # Operations with references
+ herr_t H5Rcreate(void *ref, hid_t loc_id, char *name, H5R_type_t ref_type,
+ hid_t space_id)
+
# Specific HDF5 functions for PyTables
cdef extern from "H5ATTR.h":
herr_t H5ATTRget_attribute(hid_t loc_id, char *attr_name,
Index: tables/utilsExtension.pyx
===================================================================
--- tables/utilsExtension.pyx (revisión: 4263)
+++ tables/utilsExtension.pyx (copia de trabajo)
@@ -665,6 +665,8 @@
tid = H5Tcopy(H5T_STD_B8);
elif atom.kind == 'enum':
tid = enumToHDF5(atom, byteorder)
+ elif atom.kind == 'reference':
+ tid = H5Tcopy(H5T_STD_REF_OBJ)
else:
raise TypeError("Invalid type for atom %s" % (atom,))
# Create an H5T_ARRAY in case of non-scalar atoms
@@ -774,6 +776,8 @@
stype = "u%s" % (itemsize)
elif class_id == H5T_FLOAT:
stype = "f%s" % (itemsize)
+ elif class_id == H5T_REFERENCE:
+ stype = "r8"
elif class_id == H5T_COMPOUND:
if is_complex(type_id):
stype = "c%s" % (itemsize)
Index: tables/hdf5Extension.pxd
===================================================================
--- tables/hdf5Extension.pxd (revisión: 4263)
+++ tables/hdf5Extension.pxd (copia de trabajo)
@@ -2,6 +2,11 @@
# Declaration of instance variables for shared classes
+cdef class File:
+ cdef hid_t file_id
+ cdef hid_t access_plist
+ cdef object name
+
cdef class Node:
cdef char *name
cdef hid_t parent_id
Index: tables/atom.py
===================================================================
--- tables/atom.py (revisión: 4263)
+++ tables/atom.py (copia de trabajo)
@@ -619,7 +619,16 @@
def __init__(self, shape=(), dflt=_defvalue):
Atom.__init__(self, self.type, shape, dflt)
+class ReferenceAtom(Atom):
+ kind = 'reference'
+ itemsize = 8
+ type = 'reference'
+ _deftype = 'reference64'
+ _defvalue = None
+ def __init__(self, shape=(), dflt=_defvalue):
+ Atom.__init__(self, self.type, shape, dflt)
+
class IntAtom(Atom):
"""Defines an atom of a signed integral type (``int`` kind)."""
kind = 'int'
Index: tables/convtypetables.pxi
===================================================================
--- tables/convtypetables.pxi (revisión: 4263)
+++ tables/convtypetables.pxi (copia de trabajo)
@@ -35,7 +35,7 @@
H5T_STD_B8BE, H5T_UNIX_D32BE, H5T_UNIX_D64BE, \
H5T_STD_I8BE, H5T_STD_I16BE, H5T_STD_I32BE, H5T_STD_I64BE, \
H5T_STD_U8BE, H5T_STD_U16BE, H5T_STD_U32BE, H5T_STD_U64BE, \
- H5T_IEEE_F32BE, H5T_IEEE_F64BE
+ H5T_IEEE_F32BE, H5T_IEEE_F64BE, H5T_STD_REF_OBJ
# Platform-dependent types
if sys.byteorder == "little":
@@ -87,7 +87,7 @@
}
# Special cases whose byteorder cannot be directly changed
-PTSpecialKinds = ['complex', 'string', 'enum', 'bool']
+PTSpecialKinds = ['complex', 'string', 'enum', 'bool', 'reference']
# Conversion table from NumPy extended codes prefixes to PyTables kinds
NPExtPrefixesToPTKinds = {
@@ -99,6 +99,7 @@
"c": "complex",
"t": "time",
"e": "enum",
+ "r": "reference"
}
# Names of HDF5 classes
Index: tables/description.py
===================================================================
--- tables/description.py (revisión: 4263)
+++ tables/description.py (copia de trabajo)
@@ -732,7 +732,7 @@
"are not supported yet, sorry" )
fbyteorder = byteorder
# Non-nested column
- if kind in 'biufSc':
+ if kind in 'biufScOr':
col = Col.from_dtype(dtype, pos=pos)
# Nested column
elif kind == 'V' and dtype.shape in [(), (1,)]:
------------------------------------------------------------------------------
Come build with us! The BlackBerry(R) Developer Conference in SF, CA
is the only developer event you need to attend this year. Jumpstart your
developing skills, take BlackBerry mobile applications to market and stay
ahead of the curve. Join us from November 9 - 12, 2009. Register now!
http://p.sf.net/sfu/devconference
_______________________________________________
Pytables-users mailing list
Pytables-users@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/pytables-users
