This is an automated email from the git hooks/post-receive script. sebastic pushed a commit to branch master in repository netcdf4-python.
commit f27a4f58bc3a6cf358f00f5c9351453f2ee189b4 Author: Bas Couwenberg <[email protected]> Date: Sun Jun 18 10:18:25 2017 +0200 Imported Upstream version 1.2.9 --- .travis.yml | 7 ++-- Changelog | 9 ++++ MANIFEST.in | 5 +-- README.md | 5 +++ docs/netCDF4/index.html | 13 +++--- netCDF4/_netCDF4.pyx | 109 +++++++++++++++++++++++++----------------------- setup.py | 56 ++++++++++--------------- test/issue671.nc | Bin 0 -> 123416 bytes test/issue672.nc | Bin 0 -> 224227 bytes test/tst_Unsigned.py | 8 ++++ test/tst_masked4.py | 13 ++++++ utils/nc3tonc4 | 4 -- utils/nc4tonc3 | 4 -- utils/ncinfo | 4 -- 14 files changed, 123 insertions(+), 114 deletions(-) diff --git a/.travis.yml b/.travis.yml index a6cb3aa..e064866 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,21 +11,22 @@ addons: env: global: - - DEPENDS="numpy cython" + - DEPENDS="numpy cython setuptools==18.0.1" - NO_NET=1 python: - "2.7" - "3.5" - "3.6" - - "3.7-dev" matrix: + allow_failures: + - python: "3.7-dev" include: # Absolute minimum dependencies. - python: 2.7 env: - - DEPENDS="numpy==1.7.0 cython==0.19 ordereddict==1.1" + - DEPENDS="numpy==1.7.0 cython==0.19 ordereddict==1.1 setuptools==18.0" notifications: email: false diff --git a/Changelog b/Changelog index 7e2b48d..846c012 100644 --- a/Changelog +++ b/Changelog @@ -1,3 +1,12 @@ + version 1.2.9 (tag v1.2.9rel) +============================== + * Fix for auto scaling and masking when _Unsigned attribute set (create + view as unsigned type after scaling and masking). Issue #671. + * Always mask values outside valid_min, valid_max (not just when + missing_value attribue present). Issue #672. + * Fix setup.py so pip install doesn't fail if cython not installed. + setuptools >= 18.0 now required for installation (Issue #666). + version 1.2.8 (tag v1.2.8rel) ============================== * recognize _Unsigned attribute used by netcdf-java to designate unsigned diff --git a/MANIFEST.in b/MANIFEST.in index f35c12c..b699e50 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,6 @@ recursive-include docs * recursive-include man * -recursive-include conda.recipe +recursive-include conda.recipe * include MANIFEST.in include README.md include COPYING @@ -14,9 +14,6 @@ include examples/*ipynb include examples/README.md include test/*py include test/*nc -include utils/nc3tonc4 -include utils/nc4tonc3 -include utils/ncinfo include netcdftime/__init__.py include netcdftime/_netcdftime.pyx include netCDF4/__init__.py diff --git a/README.md b/README.md index e1b7506..63bfce3 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,11 @@ ## News For the latest updates, see the [Changelog](https://github.com/Unidata/netcdf4-python/blob/master/Changelog). +6/10/2017: Version [1.2.9](https://pypi.python.org/pypi/netCDF4/1.2.9) released. Fixes for auto-scaling +and masking when `_Unsigned` and/or `valid_min`, `valid_max` attributes present. setup.py updated +so that `pip install` works if cython not installed. Now requires [setuptools](https://pypi.python.org/pypi/setuptools) +version 18.0 or greater. + 6/1/2017: Version [1.2.8](https://pypi.python.org/pypi/netCDF4/1.2.8) released. From Changelog: * recognize `_Unsigned` attribute used by [netcdf-java](http://www.unidata.ucar.edu/software/thredds/current/netcdf-java/) to designate unsigned integer data stored with a signed integer type in netcdf-3 diff --git a/docs/netCDF4/index.html b/docs/netCDF4/index.html index 5977c7c..a931a7e 100644 --- a/docs/netCDF4/index.html +++ b/docs/netCDF4/index.html @@ -4,7 +4,7 @@ <meta name="viewport" content="width=device-width, initial-scale=1, minimum-scale=1" /> <title>netCDF4 API documentation</title> - <meta name="description" content="Version 1.2.8 + <meta name="description" content="Version 1.2.9 ------------- - - - @@ -1268,7 +1268,7 @@ table { <header id="section-intro"> <h1 class="title"><span class="name">netCDF4</span> module</h1> - <h2>Version 1.2.8</h2> + <h2>Version 1.2.9</h2> <hr /> <h1>Introduction</h1> <p>netcdf4-python is a Python interface to the netCDF C library. </p> @@ -1298,12 +1298,9 @@ types) are not supported.</p> <ul> <li>Python 2.7 or later (python 3 works too).</li> <li><a href="http://numpy.scipy.org";>numpy array module</a>, version 1.7.0 or later.</li> -<li><a href="http://cython.org";>Cython</a>, version 0.19 or later, is optional - if it is installed setup.py will - use it to recompile the Cython source code into C, using conditional compilation - to enable features in the netCDF API that have been added since version 4.1.1. If - Cython is not installed, these features (such as the ability to rename Group objects) - will be disabled to preserve backward compatibility with older versions of the netCDF - library.</li> +<li><a href="http://cython.org";>Cython</a>, version 0.19 or later.</li> +<li><a href="https://pypi.python.org/pypi/setuptools";>setuptools</a>, version 18.0 or + later.</li> <li>The HDF5 C library version 1.8.4-patch1 or higher (1.8.x recommended) from <a href="ftp://ftp.hdfgroup.org/HDF5/current/src";></a>. <strong><em>netCDF version 4.4.1 or higher is recommended if using HDF5 1.10.x - diff --git a/netCDF4/_netCDF4.pyx b/netCDF4/_netCDF4.pyx index eabe485..8a5818e 100644 --- a/netCDF4/_netCDF4.pyx +++ b/netCDF4/_netCDF4.pyx @@ -1,5 +1,5 @@ """ -Version 1.2.8 +Version 1.2.9 ------------- - - - @@ -39,6 +39,8 @@ Requires - Python 2.7 or later (python 3 works too). - [numpy array module](http://numpy.scipy.org), version 1.7.0 or later. - [Cython](http://cython.org), version 0.19 or later. + - [setuptools](https://pypi.python.org/pypi/setuptools), version 18.0 or + later. - The HDF5 C library version 1.8.4-patch1 or higher (1.8.x recommended) from [](ftp://ftp.hdfgroup.org/HDF5/current/src). ***netCDF version 4.4.1 or higher is recommended if using HDF5 1.10.x - @@ -931,7 +933,7 @@ except ImportError: # python3: zip is already python2's itertools.izip pass -__version__ = "1.2.8" +__version__ = "1.2.9" # Initialize numpy import posixpath @@ -3837,13 +3839,6 @@ rename a `netCDF4.Variable` attribute named `oldname` to `newname`.""" # length 1. if data.ndim != 0: data = numpy.asarray(data[0]) - # if attribute _Unsigned is True, and variable has signed integer - # dtype, return view with corresponding unsigned dtype (issue #656) - if self.scale: # only do this if autoscale option is on. - is_unsigned = getattr(self, '_Unsigned', False) - if is_unsigned and data.dtype.kind == 'i': - data = data.view('u%s' % data.dtype.itemsize) - # if auto_scale mode set to True, (through # a call to set_auto_scale or set_auto_maskandscale), # perform automatic unpacking using scale_factor/add_offset. @@ -3861,8 +3856,17 @@ rename a `netCDF4.Variable` attribute named `oldname` to `newname`.""" if self.scale: msg = 'invalid scale_factor or add_offset attribute, no unpacking done...' warnings.warn(msg) + if self.mask and (self._isprimitive or self._isenum): data = self._toma(data) + + # if attribute _Unsigned is True, and variable has signed integer + # dtype, return view with corresponding unsigned dtype (issue #656) + if self.scale: # only do this if autoscale option is on. + is_unsigned = getattr(self, '_Unsigned', False) + if is_unsigned and data.dtype.kind == 'i': + data = data.view('u%s' % data.dtype.itemsize) + if self.scale and self._isprimitive and valid_scaleoffset: # if variable has scale_factor and add_offset attributes, rescale. if hasattr(self, 'scale_factor') and hasattr(self, 'add_offset') and\ @@ -3907,43 +3911,6 @@ rename a `netCDF4.Variable` attribute named `oldname` to `newname`.""" mval = numpy.array(self.missing_value, self.dtype) # create mask from missing values. mvalmask = numpy.zeros(data.shape, numpy.bool) - # set mask=True for data outside valid_min,valid_max. - # (issue #576) - validmin = None; validmax = None - # if valid_range exists use that, otherwise - # look for valid_min, valid_max. No special - # treatment of byte data as described at - # http://www.unidata.ucar.edu/software/netcdf/docs/attribute_conventions.html). - if hasattr(self, 'valid_range') and len(self.valid_range) == 2: - validmin = numpy.array(self.valid_range[0], self.dtype) - validmax = numpy.array(self.valid_range[1], self.dtype) - else: - if hasattr(self, 'valid_min'): - validmin = numpy.array(self.valid_min, self.dtype) - if hasattr(self, 'valid_max'): - validmax = numpy.array(self.valid_max, self.dtype) - # http://www.unidata.ucar.edu/software/netcdf/docs/attribute_conventions.html). - # "If the data type is byte and _FillValue - # is not explicitly defined, - # then the valid range should include all possible values. - # Otherwise, the valid range should exclude the _FillValue - # (whether defined explicitly or by default) as follows. - # If the _FillValue is positive then it defines a valid maximum, - # otherwise it defines a valid minimum." - byte_type = self.dtype.str[1:] in ['u1','i1'] - if hasattr(self, '_FillValue'): - fval = numpy.array(self._FillValue, self.dtype) - else: - fval = numpy.array(default_fillvals[self.dtype.str[1:]],self.dtype) - if byte_type: fval = None - if validmin is None and (fval is not None and fval <= 0): - validmin = fval - elif validmax is None and (fval is not None and fval > 0): - validmax = fval - if validmin is not None: - mvalmask += data < validmin - if validmax is not None: - mvalmask += data > validmax if mval.shape == (): # mval a scalar. mval = [mval] # make into iterable. for m in mval: @@ -3962,7 +3929,7 @@ rename a `netCDF4.Variable` attribute named `oldname` to `newname`.""" # if missing_value is a vector). fill_value = mval[0] totalmask += mvalmask - # set mask=True for missing data + # set mask=True for data == fill value if hasattr(self, '_FillValue'): fval = numpy.array(self._FillValue, self.dtype) # is _FillValue a NaN? @@ -4006,14 +3973,52 @@ rename a `netCDF4.Variable` attribute named `oldname` to `newname`.""" # in that case convert to an array. if type(has_fillval) == bool: has_fillval=numpy.asarray(has_fillval) if has_fillval.any(): - mask=data==fillval if fill_value is None: fill_value = fillval + mask=data==fillval totalmask += mask - # all values where data == missing_value or _FillValue are - # masked. fill_value set to missing_value if it exists, - # otherwise _FillValue. - if fill_value is not None: + # set mask=True for data outside valid_min,valid_max. + # (issue #576) + validmin = None; validmax = None + # if valid_range exists use that, otherwise + # look for valid_min, valid_max. No special + # treatment of byte data as described at + # http://www.unidata.ucar.edu/software/netcdf/docs/attribute_conventions.html). + if hasattr(self, 'valid_range') and len(self.valid_range) == 2: + validmin = numpy.array(self.valid_range[0], self.dtype) + validmax = numpy.array(self.valid_range[1], self.dtype) + else: + if hasattr(self, 'valid_min'): + validmin = numpy.array(self.valid_min, self.dtype) + if hasattr(self, 'valid_max'): + validmax = numpy.array(self.valid_max, self.dtype) + # http://www.unidata.ucar.edu/software/netcdf/docs/attribute_conventions.html). + # "If the data type is byte and _FillValue + # is not explicitly defined, + # then the valid range should include all possible values. + # Otherwise, the valid range should exclude the _FillValue + # (whether defined explicitly or by default) as follows. + # If the _FillValue is positive then it defines a valid maximum, + # otherwise it defines a valid minimum." + byte_type = self.dtype.str[1:] in ['u1','i1'] + if hasattr(self, '_FillValue'): + fval = numpy.array(self._FillValue, self.dtype) + else: + fval = numpy.array(default_fillvals[self.dtype.str[1:]],self.dtype) + if byte_type: fval = None + if self.dtype.kind != 'S': # don't set mask for character data + if validmin is None and (fval is not None and fval <= 0): + validmin = fval + if validmax is None and (fval is not None and fval > 0): + validmax = fval + if validmin is not None: + totalmask += data < validmin + if validmax is not None: + totalmask += data > validmax + if fill_value is None and fval is not None: + fill_value = fval + # create masked array with computed mask + if totalmask.any() and fill_value is not None: data = ma.masked_array(data,mask=totalmask,fill_value=fill_value) # issue 515 scalar array with mask=True should be converted # to numpy.ma.MaskedConstant to be consistent with slicing diff --git a/setup.py b/setup.py index 22d88ce..4de3b0e 100644 --- a/setup.py +++ b/setup.py @@ -1,31 +1,18 @@ import os, sys, subprocess import os.path as osp - -try: - from setuptools import setup, Extension - setuptools_extra_kwargs = { - "install_requires": ["numpy>=1.7","cython>=0.19"], - "entry_points": { - 'console_scripts': [ - 'ncinfo = netCDF4.utils:ncinfo', - 'nc4tonc3 = netCDF4.utils:nc4tonc3', - 'nc3tonc4 = netCDF4.utils:nc3tonc4', - ] - }, - } -except ImportError: - from distutils.core import setup, Extension - setuptools_extra_kwargs = { - "scripts": ['utils/nc3tonc4','utils/nc4tonc3','utils/ncinfo'] - } +from setuptools import setup, Extension from distutils.dist import Distribution - -try: - from Cython.Build import cythonize - from Cython import __version__ as cython_version - if cython_version < '0.19': raise ImportError -except ImportError: - raise ImportError('cython >= 0.19 required') +setuptools_extra_kwargs = { + "install_requires": ["numpy>=1.7"], + "setup_requires": ['setuptools>=18.0',"cython>=0.19"], + "entry_points": { + 'console_scripts': [ + 'ncinfo = netCDF4.utils:ncinfo', + 'nc4tonc3 = netCDF4.utils:nc4tonc3', + 'nc3tonc4 = netCDF4.utils:nc3tonc4', + ] + }, +} if sys.version_info[0] < 3: import ConfigParser as configparser @@ -384,7 +371,7 @@ else: # get netcdf library version. netcdf_lib_version = getnetcdfvers(lib_dirs) if netcdf_lib_version is None: - sys.stdout.write('unable to detect netcdf library version') + sys.stdout.write('unable to detect netcdf library version\n') else: sys.stdout.write('using netcdf library version %s\n' % netcdf_lib_version) @@ -393,13 +380,6 @@ netcdf4_src_root = osp.join('netCDF4', '_netCDF4') netcdf4_src_c = netcdf4_src_root + '.c' if 'sdist' not in sys.argv[1:] and 'clean' not in sys.argv[1:]: sys.stdout.write('using Cython to compile netCDF4.pyx...\n') - extensions = [Extension("netCDF4._netCDF4", - [netcdf4_src_root + '.pyx'], - libraries=libs, - library_dirs=lib_dirs, - include_dirs=inc_dirs, - runtime_library_dirs=runtime_lib_dirs), - Extension('netcdftime._netcdftime', ['netcdftime/_netcdftime.pyx'])] # remove netCDF4.c file if it exists, so cython will recompile netCDF4.pyx. # run for build *and* install (issue #263). Otherwise 'pip install' will # not regenerate netCDF4.c, even if the C lib supports the new features. @@ -445,13 +425,19 @@ if 'sdist' not in sys.argv[1:] and 'clean' not in sys.argv[1:]: f.write('DEF HAS_CDF5_FORMAT = 0\n') f.close() - ext_modules = cythonize(extensions, include_path=['include']) + ext_modules = [Extension("netCDF4._netCDF4", + [netcdf4_src_root + '.pyx'], + libraries=libs, + library_dirs=lib_dirs, + include_dirs=inc_dirs+['include'], + runtime_library_dirs=runtime_lib_dirs), + Extension('netcdftime._netcdftime', ['netcdftime/_netcdftime.pyx'])] else: ext_modules = None setup(name = "netCDF4", cmdclass = cmdclass, - version = "1.2.8", + version = "1.2.9", long_description = "netCDF version 4 has many features not found in earlier versions of the library, such as hierarchical groups, zlib compression, multiple unlimited dimensions, and new data types. It is implemented on top of HDF5. This module implements most of the new features, and can read and write netCDF files compatible with older versions of the library. The API is modelled after Scientific.IO.NetCDF, and should be familiar to users of that module.\n\nThis project has a `Sub [...] author = "Jeff Whitaker", author_email = "[email protected]", diff --git a/test/issue671.nc b/test/issue671.nc new file mode 100644 index 0000000..465dd6a Binary files /dev/null and b/test/issue671.nc differ diff --git a/test/issue672.nc b/test/issue672.nc new file mode 100644 index 0000000..d49b653 Binary files /dev/null and b/test/issue672.nc differ diff --git a/test/tst_Unsigned.py b/test/tst_Unsigned.py index ecd9e4d..260ca2c 100644 --- a/test/tst_Unsigned.py +++ b/test/tst_Unsigned.py @@ -22,6 +22,14 @@ class Test_Unsigned(unittest.TestCase): assert data2.dtype.str[1:] == 'i1' assert_array_equal(data2,np.array([0,-1],np.int8)) f.close() + # issue 671 + f = netCDF4.Dataset('issue671.nc') + data1 = f['soil_moisture'][:] + assert(np.ma.isMA(data1)) + f.set_auto_scale(False) + data2 = f['soil_moisture'][:] + assert(data1.mask.sum() == data2.mask.sum()) + f.close() if __name__ == '__main__': unittest.main() diff --git a/test/tst_masked4.py b/test/tst_masked4.py index 3e013d5..9d10ae6 100755 --- a/test/tst_masked4.py +++ b/test/tst_masked4.py @@ -100,7 +100,20 @@ class SetValidMinMax(unittest.TestCase): v.set_auto_scale(False) v = v[:] self.assertTrue(np.all(self.v == v.data)) + f.close() + # issue 672 + f = Dataset('issue672.nc') + field = 'azi_angle_trip' + v = f.variables[field] + data1 = v[:] + v.set_auto_scale(False) + data2 = v[:] + v.set_auto_maskandscale(False) + data3 = v[:] + assert(data1[(data3 < v.valid_min)].mask.sum() == 12) + assert(data2[(data3 < v.valid_min)].mask.sum() == + data1[(data3 < v.valid_min)].mask.sum()) f.close() diff --git a/utils/nc3tonc4 b/utils/nc3tonc4 deleted file mode 100755 index ed4964e..0000000 --- a/utils/nc3tonc4 +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env python - -from netCDF4.utils import nc3tonc4 -nc3tonc4() diff --git a/utils/nc4tonc3 b/utils/nc4tonc3 deleted file mode 100755 index efa12f6..0000000 --- a/utils/nc4tonc3 +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env python - -from netCDF4.utils import nc4tonc3 -nc4tonc3() diff --git a/utils/ncinfo b/utils/ncinfo deleted file mode 100755 index 3afff8f..0000000 --- a/utils/ncinfo +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env python - -from netCDF4.utils import ncinfo -ncinfo() -- Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-grass/netcdf4-python.git _______________________________________________ Pkg-grass-devel mailing list [email protected] http://lists.alioth.debian.org/cgi-bin/mailman/listinfo/pkg-grass-devel
