Hi Francesc,
Sorry for the delay. I do not have time to work this last few days. Now,
I attached a example and a possible fixation.
LittleBigBrain
Francesc Alted:
By the way, after I change shape to (24,3e6), the pytables.Expr
returns an Error:
The error was --> <type 'exceptions.AttributeError'>: 'Expr' object
has no attribute 'BUFFERTIMES'.
Hmm, that should be a bug. Could you send a small self-contained
example so that I can fix that?
I think this is because you have not updated the expression.py for
new ‘BUFFER_TIMES’ parameter? So I add:
from tables.parameters import BUFFER_TIMES
change self.BUFFERTIMES to BUFFER_TIMES
I hope this is correct.
Hope that helps,
>>>
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
PyTables version: 2.2
HDF5 version: 1.8.5
NumPy version: 1.4.1
Numexpr version: 1.3.1 (using VML/MKL 10.2.5)
Zlib version: 1.2.3 (in Python interpreter)
Blosc version: 1.0 (2010-07-01)
Python version: 2.6.5 (r265:79096, Mar 19 2010, 18:02:59) [MSC v.1500 64 bit
(AMD64)]
Byte-ordering: little
Detected cores: 4
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=
pytable hdf5 file info:G:/PyTables_vs_Memmap/Exprbug-src-None-noncomp.h5 (File)
''
Last modif.: 'Wed Oct 20 00:34:14 2010'
Object Tree:
/ (RootGroup) ''
/TestArray (CArray(2, 1000000)) ''
pytable Array info:/TestArray (CArray(2, 1000000)) ''
#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=
#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=
pytable hdf5 file info:G:/PyTables_vs_Memmap/Exprbug-dst-None-noncomp.h5 (File)
''
Last modif.: 'Wed Oct 20 00:34:14 2010'
Object Tree:
/ (RootGroup) ''
/TestArray (CArray(2, 1000000)) ''
pytable Array info:/TestArray (CArray(2, 1000000)) ''
#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=#=
(1, 16384)
Problems compute data from G:/PyTables_vs_Memmap/Exprbug-src-None-noncomp.h5 to
G:/PyTables_vs_Memmap/Exprbug-dst-None-noncomp.h5
The error was --> <type 'exceptions.AttributeError'>: 'Expr' object has no
attribute 'BUFFERTIMES'
*** print exception traceback:
Traceback (most recent call last):
File "G:\pyTablesTests\pytableExprBug1.py", line 113, in SerialRWTables
exprtbs.eval()
File "C:\Python26\lib\site-packages\tables\expression.py", line 534, in eval
self._get_info(shape, maindim)
File "C:\Python26\lib\site-packages\tables\expression.py", line 485, in
_get_info
nrows = self._calc_nrowsinbuf(val)
File "C:\Python26\lib\site-packages\tables\expression.py", line 364, in
_calc_nrowsinbuf
maxrowsize = self.BUFFERTIMES * buffersize
AttributeError: 'Expr' object has no attribute 'BUFFERTIMES'
>>>
#######################################################################
# This script compares I/O speed and the speed of the computation of
#a polynomial for different (numpy.memmap+numexpr and tables.Expr)
#
# Author: Little Big Brain
# Date: 2010-10-20
#######################################################################
import numpy as npy
import tables as pytbs
import numexpr as nep
import time
import os.path
import sys,traceback
expr = ".25*a**3 + .75*a**2 - 1.5*a - 2"
def
CreateH5file(h5srcname,shape=None,dtype=None,chunkshape=None,srcArrayName=None):
atom=pytbs.Atom.from_dtype(npy.dtype(dtype))
h5file = pytbs.openFile(h5srcname, mode = "w")
ca = h5file.createCArray(\
h5file.root,srcArrayName,\
atom,shape,\
chunkshape=chunkshape)
print '#='*20
print 'pytable hdf5 file info:\r\n',h5file
print 'pytable Array info:\r\n',ca
print '#='*20
for i in range(0,10):
#output random floating number into array
ca[:,i*shape[1]/10:(i+1)*shape[1]/10]=\
npy.random.randn(shape[0],shape[1]/10)
h5file.close()
return os.path.getsize(h5srcname)
def SerialRWTables(h5srcname,h5dstname=None,\
srcArrayName=None,\
dstArrayName=None,\
filters=None):
if not os.path.isfile(h5srcname):
raise RuntimeError, '\''+\
h5srcname+'\'' +\
' does not exist. Please Check Destination File Name'
try:
h5src=pytbs.openFile(h5srcname,mode='r')
except:
(exc_type, exc_value, exc_traceback) = sys.exc_info()
print "Problems open %s" % \
h5srcname
print "The error was --> %s: %s" % (exc_type, exc_value)
raise RuntimeError, "Please check Destination File Name"
try:
srcArray=h5src.getNode('/'+srcArrayName)
except:
(exc_type, exc_value, exc_traceback) = sys.exc_info()
print "Problems open %s/%s" % \
(h5srcname, srcArrayName)
print "The error was --> %s: %s" % (exc_type, exc_value)
print "The source file looks like:\n", h5src
h5src.close()
raise RuntimeError, "Please check Source Array Name"
shape=srcArray.shape
atom=srcArray.atom
filters=filters
dstchunkshape=srcArray.chunkshape
if not dstArrayName:
dstArrayName=srcArrayName
try:
h5dst = pytbs.openFile(\
h5dstname,\
mode = "w")
except:
(exc_type, exc_value, exc_traceback) = sys.exc_info()
print "Problems open %s" % \
h5dstname
print "The error was --> %s: %s" % (exc_type, exc_value)
h5src.close()
h5dst.close()
raise RuntimeError, "Please check Destination File Name"
try:
dstArray = h5dst.createCArray(\
h5dst.root,dstArrayName,\
atom,shape,filters=filters,\
chunkshape=dstchunkshape)
except:
(exc_type, exc_value, exc_traceback) = sys.exc_info()
print "Problems create %s/%s" % \
(h5dstname, dstArrayName)
print "The error was --> %s: %s" % (exc_type, exc_value)
print "The destination file looks like:\n", h5dst
h5src.close()
h5dst.close()
raise RuntimeError, "Please check Destination Array parameters"
print '#='*20
print 'pytable hdf5 file info:\r\n',h5dst
print 'pytable Array info:\r\n',dstArray
print '#='*20
try:
print srcArray.chunkshape
a=srcArray
exprtbs = pytbs.Expr(expr)
exprtbs.setOutput(dstArray)
exprtbs.eval()
except:
(exc_type, exc_value, exc_traceback) = sys.exc_info()
print "Problems compute data from %s to %s" % \
(h5srcname,h5dstname)
print "The error was --> %s: %s" % (exc_type, exc_value)
print "*** print exception traceback:"
traceback.print_exception(exc_type, exc_value\
, exc_traceback\
,file=sys.stdout)
h5src.close()
h5dst.close()
#print 'close all files'
else:
h5src.close()
h5dst.close()
if __name__ == '__main__':
version='Exprbug-'
pytbs.print_versions()
shape=(2,1e6)
chunkshape=None
srcArrayName='TestArray'
testFolder='G:/PyTables_vs_Memmap'
h5srcname=testFolder+'/'+version+'src-'+str(chunkshape)+'-noncomp'+'.h5'
h5dstname1=testFolder+'/'+version+'dst-'+str(chunkshape)+'-noncomp'+'.h5'
fileSize=CreateH5file(\
h5srcname,shape=shape,chunkshape=chunkshape\
,srcArrayName=srcArrayName,dtype='float64')
SerialRWTables(h5srcname,h5dstname=h5dstname1,\
srcArrayName=srcArrayName)
########################################################################
#
# License: BSD
# Created: December 17, 2004
# Author: Francesc Alted - fal...@pytables.com
#
# $Id: exceptions.py 4358 2010-02-17 18:38:15Z faltet $
#
########################################################################
"""
Declare exceptions and warnings that are specific to PyTables.
Classes:
`HDF5ExtError`
A low level HDF5 operation failed.
`ClosedNodeError`
The operation can not be completed because the node is closed.
`ClosedFileError`
The operation can not be completed because the hosting file is
closed.
`FileModeError`
The operation can not be carried out because the mode in which the
hosting file is opened is not adequate.
`NodeError`
Invalid hierarchy manipulation operation requested.
`NoSuchNodeError`
An operation was requested on a node that does not exist.
`UndoRedoError`
Problems with doing/redoing actions with Undo/Redo feature.
`UndoRedoWarning`
Issued when an action not supporting undo/redo is run.
`NaturalNameWarning`
Issued when a non-pythonic name is given for a node.
`PerformanceWarning`
Warning for operations which may cause a performance drop.
`FlavorError`
Unsupported or unavailable flavor or flavor conversion.
`FlavorWarning`
Unsupported or unavailable flavor conversion.
`FiltersWarning`
Unavailable filters.
`NoIndexingError`
Indexing is not supported.
`NoIndexingWarning`
Indexing is not supported.
`OldIndexWarning`
Unsupported index format.
`DataTypeWarning`
Unsupported data type.
`Incompat16Warning`
Format incompatible with HDF5 1.6.x series.
"""
__docformat__ = 'reStructuredText'
"""The format of documentation strings in this module."""
__version__ = '$Revision: 4358 $'
"""Repository version of this file."""
class HDF5ExtError(RuntimeError):
"""
A low level HDF5 operation failed.
This exception is raised by ``hdf5Extension`` (the low level
PyTables component used for accessing HDF5 files). It usually
signals that something is not going well in the HDF5 library or even
at the Input/Output level, and uses to be accompanied by an
extensive HDF5 back trace on standard error.
"""
pass
# The following exceptions are concretions of the ``ValueError`` exceptions
# raised by ``file`` objects on certain operations.
class ClosedNodeError(ValueError):
"""
The operation can not be completed because the node is closed.
For instance, listing the children of a closed group is not allowed.
"""
pass
class ClosedFileError(ValueError):
"""
The operation can not be completed because the hosting file is
closed.
For instance, getting an existing node from a closed file is not
allowed.
"""
pass
class FileModeError(ValueError):
"""
The operation can not be carried out because the mode in which the
hosting file is opened is not adequate.
For instance, removing an existing leaf from a read-only file is not
allowed.
"""
pass
class NodeError(AttributeError, LookupError):
"""
Invalid hierarchy manipulation operation requested.
This exception is raised when the user requests an operation on the
hierarchy which can not be run because of the current layout of the
tree. This includes accessing nonexistent nodes, moving or copying
or creating over an existing node, non-recursively removing groups
with children, and other similarly invalid operations.
A node in a PyTables database cannot be simply overwritten by
replacing it. Instead, the old node must be removed explicitely
before another one can take its place. This is done to protect
interactive users from inadvertedly deleting whole trees of data by
a single erroneous command.
"""
pass
class NoSuchNodeError(NodeError):
"""
An operation was requested on a node that does not exist.
This exception is raised when an operation gets a path name or a
``(where, name)`` pair leading to a nonexistent node.
"""
pass
class UndoRedoError(Exception):
"""
Problems with doing/redoing actions with Undo/Redo feature.
This exception indicates a problem related to the Undo/Redo
mechanism, such as trying to undo or redo actions with this
mechanism disabled, or going to a nonexistent mark.
"""
pass
class UndoRedoWarning(Warning):
"""
Issued when an action not supporting Undo/Redo is run.
This warning is only shown when the Undo/Redo mechanism is enabled.
"""
pass
class NaturalNameWarning(Warning):
"""
Issued when a non-pythonic name is given for a node.
This is not an error and may even be very useful in certain
contexts, but one should be aware that such nodes cannot be
accessed using natural naming (instead, ``getattr()`` must be
used explicitly).
"""
pass
class PerformanceWarning(Warning):
"""
Warning for operations which may cause a performance drop.
This warning is issued when an operation is made on the database
which may cause it to slow down on future operations (i.e. making
the node tree grow too much).
"""
pass
class FlavorError(ValueError):
"""
Unsupported or unavailable flavor or flavor conversion.
This exception is raised when an unsupported or unavailable flavor
is given to a dataset, or when a conversion of data between two
given flavors is not supported nor available.
A supported flavor may be unavailable if the package which
implements it is not installed locally, e.g. you may specify the
``numeric`` flavor, which is supported by PyTables, but if Numeric
is not installed on your machine, you will get this error.
"""
pass
class FlavorWarning(Warning):
"""
Unsupported or unavailable flavor conversion.
This warning is issued when a conversion of data between two given
flavors is not supported nor available, and raising an error would
render the data inaccessible (e.g. on a dataset of an unavailable
flavor in a read-only file).
See the `FlavorError` class for more information.
"""
pass
class FiltersWarning(Warning):
"""
Unavailable filters.
This warning is issued when a valid filter is specified but it is
not available in the system. It may mean that an available default
filter is to be used instead.
"""
pass
_no_indexing_message = (
"This version of PyTables does not support indexing. "
"Please consider using the PyTables Pro edition "
"(http://www.pytables.org/moin/PyTablesPro)." )
class NoIndexingError(NotImplementedError):
"""
Indexing is not supported.
This exception is raised when an indexing-related operation is
requested under a version of PyTables which does not support
indexing. Please consider using the PyTables Pro edition
(http://www.pytables.org/moin/PyTablesPro).
"""
def __init__(self):
NotImplementedError.__init__(self, _no_indexing_message)
class NoIndexingWarning(Warning):
"""
Indexing is not supported.
This warning is issued when opening an indexed table under a version
of PyTables which does not support indexing. Please consider using
the PyTables Pro edition
(http://www.pytables.org/moin/PyTablesPro).
"""
def __init__(self, message):
message = '%s. %s' % (message, _no_indexing_message)
Warning.__init__(self, message)
class OldIndexWarning(Warning):
"""
Unsupported index format.
This warning is issued when an index in an unsupported format is
found. The index will be marked as invalid and will behave as if
doesn't exist.
"""
pass
class DataTypeWarning(Warning):
"""
Unsupported data type.
This warning is issued when an unsupported HDF5 data type is found
(normally in a file created with other tool than PyTables).
"""
pass
class Incompat16Warning(Warning):
"""
Format incompatible with HDF5 1.6.x format.
This warning is issued when using a functionality that is
incompatible with the HDF5 1.6.x format and that may create issues
for reading the files with PyTables compiled against HDF5 1.6.x.
"""
pass
class ExperimentalFeatureWarning(Warning):
"""
Generic warning for experimental features.
This warning is issued when using a functionality that is still
experimental and that users have to use with care.
"""
pass
## Local Variables:
## mode: python
## py-indent-offset: 4
## tab-width: 4
## fill-column: 72
## End:
------------------------------------------------------------------------------
Download new Adobe(R) Flash(R) Builder(TM) 4
The new Adobe(R) Flex(R) 4 and Flash(R) Builder(TM) 4 (formerly
Flex(R) Builder(TM)) enable the development of rich applications that run
across multiple browsers and platforms. Download your free trials today!
http://p.sf.net/sfu/adobe-dev2dev
_______________________________________________
Pytables-users mailing list
Pytables-users@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/pytables-users
