diff --git a/.gitignore b/.gitignore index 11d620216..a808b3552 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ dist/ netCDF4/_netCDF4.c netCDF4/*.so include/constants.pyx +include/parallel_support_imports.pxi netcdftime/_netcdftime.c venv/ .eggs/ diff --git a/Changelog b/Changelog index 8970167b9..8abe91883 100644 --- a/Changelog +++ b/Changelog @@ -1,4 +1,9 @@ - version 1.6.5 (not yet released) + version 1.7.0 (not yet released) +=============================== + * fix for deprecated Cython `DEF` and `IF` statements using compatibility header + with shims for unavailable functionality + + version 1.6.5 (tag v1.6.5rel) =============================== * fix for issue #1271 (mask ignored if bool MA assinged to uint8 var) * include information on specific object when reporting errors from netcdf-c diff --git a/include/mpi-compat.h b/include/mpi-compat.h index 9a604f851..5563ae933 100644 --- a/include/mpi-compat.h +++ b/include/mpi-compat.h @@ -4,6 +4,10 @@ #ifndef MPI_COMPAT_H #define MPI_COMPAT_H +#include "netcdf-compat.h" + +#if HAS_PARALLEL_SUPPORT + #include #if (MPI_VERSION < 3) && !defined(PyMPI_HAVE_MPI_Message) @@ -16,4 +20,6 @@ typedef void *PyMPI_MPI_Session; #define MPI_Session PyMPI_MPI_Session #endif +#endif /* HAS_PARALLEL_SUPPORT */ + #endif/*MPI_COMPAT_H*/ diff --git a/include/netCDF4.pxi b/include/netCDF4.pxi index 72b876585..7bd220274 100644 --- a/include/netCDF4.pxi +++ b/include/netCDF4.pxi @@ -358,94 +358,17 @@ cdef extern from "netcdf.h": int nc_set_chunk_cache(size_t size, size_t nelems, float preemption) nogil int nc_set_var_chunk_cache(int ncid, int varid, size_t size, size_t nelems, float preemption) nogil int nc_get_var_chunk_cache(int ncid, int varid, size_t *sizep, size_t *nelemsp, float *preemptionp) nogil - int nc_rename_grp(int grpid, char *name) nogil int nc_def_enum(int ncid, nc_type base_typeid, char *name, nc_type *typeidp) nogil int nc_insert_enum(int ncid, nc_type xtype, char *name, void *value) nogil int nc_inq_enum(int ncid, nc_type xtype, char *name, nc_type *base_nc_typep,\ size_t *base_sizep, size_t *num_membersp) nogil int nc_inq_enum_member(int ncid, nc_type xtype, int idx, char *name, void *value) nogil - int nc_inq_enum_ident(int ncid, nc_type xtype, long long value, char *identifier) nogil - int nc_rc_set(char* key, char* value) nogil - -IF HAS_QUANTIZATION_SUPPORT: - cdef extern from "netcdf.h": - cdef enum: - NC_NOQUANTIZE - NC_QUANTIZE_BITGROOM - NC_QUANTIZE_GRANULARBR - NC_QUANTIZE_BITROUND - int nc_def_var_quantize(int ncid, int varid, int quantize_mode, int nsd) nogil - int nc_inq_var_quantize(int ncid, int varid, int *quantize_modep, int *nsdp) nogil - -IF HAS_NCFILTER: - cdef extern from "netcdf_filter.h": - int nc_inq_filter_avail(int ncid, unsigned filterid) nogil - -IF HAS_SZIP_SUPPORT: - cdef extern from "netcdf.h": - cdef enum: - H5Z_FILTER_SZIP - int nc_def_var_quantize(int ncid, int varid, int quantize_mode, int nsd) nogil - int nc_inq_var_quantize(int ncid, int varid, int *quantize_modep, int *nsdp) nogil - int nc_def_var_szip(int ncid, int varid, int options_mask, int pixels_per_bloc) nogil - int nc_inq_var_szip(int ncid, int varid, int *options_maskp, int *pixels_per_blockp) nogil - -IF HAS_ZSTANDARD_SUPPORT: - cdef extern from "netcdf_filter.h": - cdef enum: - H5Z_FILTER_ZSTD - int nc_def_var_zstandard(int ncid, int varid, int level) nogil - int nc_inq_var_zstandard(int ncid, int varid, int* hasfilterp, int *levelp) nogil - -IF HAS_BZIP2_SUPPORT: - cdef extern from "netcdf_filter.h": - cdef enum: - H5Z_FILTER_BZIP2 - int nc_def_var_bzip2(int ncid, int varid, int level) nogil - int nc_inq_var_bzip2(int ncid, int varid, int* hasfilterp, int *levelp) nogil -IF HAS_BLOSC_SUPPORT: - cdef extern from "netcdf_filter.h": - cdef enum: - H5Z_FILTER_BLOSC - int nc_def_var_blosc(int ncid, int varid, unsigned subcompressor, unsigned level, unsigned blocksize, unsigned addshuffle) nogil - int nc_inq_var_blosc(int ncid, int varid, int* hasfilterp, unsigned* subcompressorp, unsigned* levelp, unsigned* blocksizep, unsigned* addshufflep) nogil -IF HAS_NC_OPEN_MEM: - cdef extern from "netcdf_mem.h": - int nc_open_mem(const char *path, int mode, size_t size, void* memory, int *ncidp) nogil +cdef extern from "mpi-compat.h": + pass -IF HAS_NC_CREATE_MEM: - cdef extern from "netcdf_mem.h": - int nc_create_mem(const char *path, int mode, size_t initialize, int *ncidp) nogil - ctypedef struct NC_memio: - size_t size - void* memory - int flags - int nc_close_memio(int ncid, NC_memio* info) nogil - -IF HAS_PARALLEL4_SUPPORT or HAS_PNETCDF_SUPPORT: - cdef extern from "mpi-compat.h": pass - cdef extern from "netcdf_par.h": - ctypedef int MPI_Comm - ctypedef int MPI_Info - int nc_create_par(char *path, int cmode, MPI_Comm comm, MPI_Info info, int *ncidp) nogil - int nc_open_par(char *path, int mode, MPI_Comm comm, MPI_Info info, int *ncidp) nogil - int nc_var_par_access(int ncid, int varid, int par_access) nogil - cdef enum: - NC_COLLECTIVE - NC_INDEPENDENT - cdef extern from "netcdf.h": - cdef enum: - NC_MPIIO - NC_MPIPOSIX - NC_PNETCDF - -IF HAS_SET_ALIGNMENT: - cdef extern from "netcdf.h": - int nc_set_alignment(int threshold, int alignment) - int nc_get_alignment(int *threshold, int *alignment) # taken from numpy.pxi in numpy 1.0rc2. cdef extern from "numpy/arrayobject.h": @@ -459,3 +382,86 @@ cdef extern from "numpy/arrayobject.h": char* PyArray_BYTES(ndarray) nogil npy_intp* PyArray_STRIDES(ndarray) nogil void import_array() + + +include "parallel_support_imports.pxi" + +# Compatibility shims +cdef extern from "netcdf-compat.h": + int nc_rename_grp(int grpid, char *name) nogil + int nc_set_alignment(int threshold, int alignment) + int nc_get_alignment(int *threshold, int *alignment) + int nc_rc_set(char* key, char* value) nogil + + int nc_open_mem(const char *path, int mode, size_t size, void* memory, int *ncidp) nogil + int nc_create_mem(const char *path, int mode, size_t initialize, int *ncidp) nogil + ctypedef struct NC_memio: + size_t size + void* memory + int flags + int nc_close_memio(int ncid, NC_memio* info) nogil + + # Quantize shims + int nc_def_var_quantize(int ncid, int varid, int quantize_mode, int nsd) nogil + int nc_inq_var_quantize(int ncid, int varid, int *quantize_modep, int *nsdp) nogil + + # Filter shims + int nc_inq_filter_avail(int ncid, unsigned filterid) nogil + + int nc_def_var_szip(int ncid, int varid, int options_mask, + int pixels_per_bloc) nogil + int nc_inq_var_szip(int ncid, int varid, int *options_maskp, + int *pixels_per_blockp) nogil + + int nc_def_var_zstandard(int ncid, int varid, int level) nogil + int nc_inq_var_zstandard(int ncid, int varid, int* hasfilterp, int *levelp) nogil + + int nc_def_var_bzip2(int ncid, int varid, int level) nogil + int nc_inq_var_bzip2(int ncid, int varid, int* hasfilterp, int *levelp) nogil + + int nc_def_var_blosc(int ncid, int varid, unsigned subcompressor, unsigned level, + unsigned blocksize, unsigned addshuffle) nogil + int nc_inq_var_blosc(int ncid, int varid, int* hasfilterp, unsigned* subcompressorp, + unsigned* levelp, unsigned* blocksizep, + unsigned* addshufflep) nogil + + # Parallel shims + int nc_create_par(char *path, int cmode, MPI_Comm comm, MPI_Info info, int *ncidp) nogil + int nc_open_par(char *path, int mode, MPI_Comm comm, MPI_Info info, int *ncidp) nogil + int nc_var_par_access(int ncid, int varid, int par_access) nogil + + cdef enum: + HAS_RENAME_GRP + HAS_NC_INQ_PATH + HAS_NC_INQ_FORMAT_EXTENDED + HAS_NC_OPEN_MEM + HAS_NC_CREATE_MEM + HAS_CDF5_FORMAT + HAS_PARALLEL_SUPPORT + HAS_PARALLEL4_SUPPORT + HAS_PNETCDF_SUPPORT + HAS_SZIP_SUPPORT + HAS_QUANTIZATION_SUPPORT + HAS_ZSTANDARD_SUPPORT + HAS_BZIP2_SUPPORT + HAS_BLOSC_SUPPORT + HAS_SET_ALIGNMENT + HAS_NCFILTER + HAS_NCRCSET + + NC_NOQUANTIZE + NC_QUANTIZE_BITGROOM + NC_QUANTIZE_GRANULARBR + NC_QUANTIZE_BITROUND + + H5Z_FILTER_SZIP + H5Z_FILTER_ZSTD + H5Z_FILTER_BZIP2 + H5Z_FILTER_BLOSC + + NC_COLLECTIVE + NC_INDEPENDENT + + NC_MPIIO + NC_MPIPOSIX + NC_PNETCDF diff --git a/include/netcdf-compat.h b/include/netcdf-compat.h new file mode 100644 index 000000000..89e7b20fe --- /dev/null +++ b/include/netcdf-compat.h @@ -0,0 +1,205 @@ +#ifndef NETCDF_COMPAT_H +#define NETCDF_COMPAT_H + +#include +#include + +#define NC_VERSION_EQ(MAJOR, MINOR, PATCH) \ + ((NC_VERSION_MAJOR == (MAJOR)) && \ + (NC_VERSION_MINOR == (MINOR)) && \ + (NC_VERSION_PATCH == (PATCH))) + +#define NC_VERSION_GT(MAJOR, MINOR, PATCH) \ + (NC_VERSION_MAJOR > (MAJOR) || \ + (NC_VERSION_MAJOR == (MAJOR) && \ + (NC_VERSION_MINOR > (MINOR) || \ + (NC_VERSION_MINOR == (MINOR) && \ + (NC_VERSION_PATCH > (PATCH)))))) + +#define NC_VERSION_GE(MAJOR, MINOR, PATCH) \ + (NC_VERSION_GT(MAJOR, MINOR, PATCH) || \ + NC_VERSION_EQ(MAJOR, MINOR, PATCH)) + +#if NC_VERSION_GE(4, 3, 0) +#define HAS_RENAME_GRP 1 +#else +#define HAS_RENAME_GRP 0 +static inline int nc_rename_grp(int grpid, const char* name) { return NC_EINVAL; } +#endif + +#if NC_VERSION_GE(4, 1, 2) +#define HAS_NC_INQ_PATH 1 +#else +#define HAS_NC_INQ_PATH 0 +static inline int nc_inq_path(int ncid, size_t *pathlen, char *path) { + *pathlen = 0; *path = "\0"; return NC_EINVAL; +} +#endif + +#if NC_VERSION_GE(4, 3, 1) +#define HAS_NC_INQ_FORMAT_EXTENDED 1 +#else +#define HAS_NC_INQ_FORMAT_EXTENDED 0 +static inline int nc_inq_format_extended(int ncid, int *formatp, int* modep) { + *formatp = 0; *modep = 0; return NC_EINVAL; +} +#endif + +#if NC_VERSION_GE(4, 9, 0) +#define HAS_SET_ALIGNMENT 1 +#else +#define HAS_SET_ALIGNMENT 0 +static inline int nc_set_alignment(int threshold, int alignment) { return NC_EINVAL; } +static inline int nc_get_alignment(int* thresholdp, int* alignmentp) { + *thresholdp = 0; *alignmentp = 0; return NC_EINVAL; +} +#endif + +#if NC_VERSION_GE(4, 9, 0) +#define HAS_NCRCSET 1 +#else +#define HAS_NCRCSET 0 +static inline int nc_rc_set(const char* key, const char* value) { return NC_EINVAL; } +#endif + +#if NC_VERSION_GE(4, 4, 0) +#include +#define HAS_NC_OPEN_MEM 1 +#else +#define HAS_NC_OPEN_MEM 0 +static inline int nc_open_mem(const char *path, int mode, size_t size, void* memory, int *ncidp) { return NC_EINVAL; } +#endif + +#if NC_VERSION_GE(4, 6, 2) +#define HAS_NC_CREATE_MEM 1 +#else +#define HAS_NC_CREATE_MEM 0 +static inline int nc_create_mem(const char *path, int mode, size_t initialize, int *ncidp) { return NC_EINVAL; } +typedef struct NC_memio { + size_t size; + void* memory; + int flags; +} NC_memio; +static inline int nc_close_memio(int ncid, NC_memio* info) { return NC_EINVAL; } +#endif + +#if defined(NC_HAS_CDF5) && NC_HAS_CDF5 +#define HAS_CDF5_FORMAT 1 +#else +# ifndef NC_HAS_CDF5 +# define NC_64BIT_DATA 0x0020 +# define NC_CDF5 NC_64BIT_DATA +# define NC_FORMAT_64BIT_OFFSET (2) +# define NC_FORMAT_64BIT_DATA (5) +# endif +#define HAS_CDF5_FORMAT 0 +#endif + +#if defined(NC_HAS_PARALLEL) && NC_HAS_PARALLEL +#include +#define HAS_PARALLEL_SUPPORT 1 +#else +#define HAS_PARALLEL_SUPPORT 0 +typedef int MPI_Comm; +typedef int MPI_Info; +static inline int nc_create_par(const char *path, int cmode, MPI_Comm comm, MPI_Info info, int *ncidp) { return NC_EINVAL; } +static inline int nc_open_par(const char *path, int mode, MPI_Comm comm, MPI_Info info, int *ncidp) { return NC_EINVAL; } +static inline int nc_var_par_access(int ncid, int varid, int par_access) { return NC_EINVAL; } +# ifndef NC_INDEPENDENT +# define NC_INDEPENDENT 0 +# define NC_COLLECTIVE 1 +# endif +# ifndef NC_MPIIO +# define NC_MPIIO 0x2000 +# define NC_MPIPOSIX NC_MPIIO +# define NC_PNETCDF (NC_MPIIO) +# endif +#endif + +#if defined(NC_HAS_PARALLEL4) && NC_HAS_PARALLEL4 +#define HAS_PARALLEL4_SUPPORT 1 +#else +#define HAS_PARALLEL4_SUPPORT 0 +#endif + +#if defined(NC_HAS_PNETCDF) && NC_HAS_PNETCDF +#define HAS_PNETCDF_SUPPORT 1 +#else +#define HAS_PNETCDF_SUPPORT 0 +#endif + +#if NC_VERSION_GE(4, 7, 0) +#include +#endif + +#if NC_VERSION_GE(4, 9, 0) +#define HAS_NCFILTER 1 +#else +#define HAS_NCFILTER 0 +static inline int nc_inq_filter_avail(int ncid, unsigned filterid) { return -136; } +#endif + +#if defined(NC_HAS_SZIP) && NC_HAS_SZIP +#define HAS_SZIP_SUPPORT 1 +#else +#define HAS_SZIP_SUPPORT 0 +# ifndef NC_HAS_SZIP +static inline int nc_def_var_szip(int ncid, int varid, int options_mask, int pixels_per_bloc) { return NC_EINVAL; } +# endif +# ifndef H5Z_FILTER_SZIP +# define H5Z_FILTER_SZIP 4 +# endif +#endif + +#if defined(NC_HAS_QUANTIZE) && NC_HAS_QUANTIZE +#define HAS_QUANTIZATION_SUPPORT 1 +#else +#define HAS_QUANTIZATION_SUPPORT 0 +# ifndef NC_HAS_QUANTIZE +static inline int nc_def_var_quantize(int ncid, int varid, int quantize_mode, int nsd) { return NC_EINVAL; } +static inline int nc_inq_var_quantize(int ncid, int varid, int *quantize_modep, int *nsdp) { return NC_EINVAL; } +# define NC_NOQUANTIZE 0 +# define NC_QUANTIZE_BITGROOM 1 +# define NC_QUANTIZE_GRANULARBR 2 +# define NC_QUANTIZE_BITROUND 3 +# endif +#endif + +#if defined(NC_HAS_ZSTD) && NC_HAS_ZSTD +#define HAS_ZSTANDARD_SUPPORT 1 +#else +# ifndef NC_HAS_ZSTD +static inline int nc_def_var_zstandard(int ncid, int varid, int level) { return NC_EINVAL; } +static inline int nc_inq_var_zstandard(int ncid, int varid, int* hasfilterp, int *levelp) { return NC_EINVAL; } +# define H5Z_FILTER_ZSTD 32015 +# endif +#define HAS_ZSTANDARD_SUPPORT 0 +#endif + +#if defined(NC_HAS_BZ2) && NC_HAS_BZ2 +#define HAS_BZIP2_SUPPORT 1 +#else +# ifndef NC_HAS_BZ2 +static inline int nc_def_var_bzip2(int ncid, int varid, int level) { return NC_EINVAL; } +static inline int nc_inq_var_bzip2(int ncid, int varid, int* hasfilterp, int *levelp) { return NC_EINVAL; } +# define H5Z_FILTER_BZIP2 307 +# endif +#define HAS_BZIP2_SUPPORT 0 +#endif + +#if defined(NC_HAS_BLOSC) && NC_HAS_BLOSC +#define HAS_BLOSC_SUPPORT 1 +#else +# ifndef NC_HAS_BLOSC +static inline int nc_def_var_blosc(int ncid, int varid, unsigned subcompressor, unsigned level, unsigned blocksize, unsigned addshuffle) { + return NC_EINVAL; +} +static inline int nc_inq_var_blosc(int ncid, int varid, int* hasfilterp, unsigned* subcompressorp, unsigned* levelp, unsigned* blocksizep, unsigned* addshufflep) { + return NC_EINVAL; +} +# define H5Z_FILTER_BLOSC 32001 +# endif +#define HAS_BLOSC_SUPPORT 0 +#endif + +#endif /* NETCDF_COMPAT_H */ diff --git a/include/no_parallel_support_imports.pxi.in b/include/no_parallel_support_imports.pxi.in new file mode 100644 index 000000000..dd22f1968 --- /dev/null +++ b/include/no_parallel_support_imports.pxi.in @@ -0,0 +1,10 @@ +# Stubs for when parallel support is not enabled + +ctypedef int MPI_Comm +ctypedef int MPI_Info +ctypedef int Comm +ctypedef int Info +cdef MPI_Comm MPI_COMM_WORLD +cdef MPI_Info MPI_INFO_NULL +MPI_COMM_WORLD = 0 +MPI_INFO_NULL = 0 diff --git a/include/parallel_support_imports.pxi.in b/include/parallel_support_imports.pxi.in new file mode 100644 index 000000000..5379bedc4 --- /dev/null +++ b/include/parallel_support_imports.pxi.in @@ -0,0 +1,16 @@ +# Imports and typedefs required at compile time for enabling parallel support + +cimport mpi4py.MPI as MPI +from mpi4py.libmpi cimport ( + MPI_Comm, + MPI_Info, + MPI_Comm_dup, + MPI_Info_dup, + MPI_Comm_free, + MPI_Info_free, + MPI_INFO_NULL, + MPI_COMM_WORLD, +) + +ctypedef MPI.Comm Comm +ctypedef MPI.Info Info diff --git a/setup.py b/setup.py index 5bf61e16a..0cb0c3543 100644 --- a/setup.py +++ b/setup.py @@ -43,118 +43,24 @@ def check_ifnetcdf4(netcdf4_includedir): isnetcdf4 = True return isnetcdf4 -def check_api(inc_dirs,netcdf_lib_version): - has_rename_grp = False - has_nc_inq_path = False - has_nc_inq_format_extended = False - has_cdf5_format = False - has_nc_open_mem = False - has_nc_create_mem = False + +def check_has_parallel_support(inc_dirs: list) -> bool: has_parallel_support = False - has_parallel4_support = False - has_pnetcdf_support = False - has_szip_support = False - has_quantize = False - has_zstandard = False - has_bzip2 = False - has_blosc = False - has_ncfilter = False - has_set_alignment = False - has_nc_rc_set = False for d in inc_dirs: - try: - f = open(os.path.join(d, 'netcdf.h'), **open_kwargs) - except OSError: + ncmetapath = os.path.join(d,'netcdf_meta.h') + if not os.path.exists(ncmetapath): continue - has_nc_open_mem = os.path.exists(os.path.join(d, 'netcdf_mem.h')) - has_nc_filter = os.path.exists(os.path.join(d, 'netcdf_filter.h')) - - for line in f: - if line.startswith('nc_rename_grp'): - has_rename_grp = True - if line.startswith('nc_inq_path'): - has_nc_inq_path = True - if line.startswith('nc_inq_format_extended'): - has_nc_inq_format_extended = True - if line.startswith('#define NC_FORMAT_64BIT_DATA'): - has_cdf5_format = True - if line.startswith('nc_def_var_quantize'): - has_quantize = True - if line.startswith('nc_set_alignment'): - has_set_alignment = True - if line.startswith('EXTERNL int nc_rc_set'): - has_nc_rc_set = True - - if has_nc_open_mem: - try: - f = open(os.path.join(d, 'netcdf_mem.h'), **open_kwargs) - except OSError: - continue - for line in f: - if line.startswith('EXTERNL int nc_create_mem'): - has_nc_create_mem = True - - if has_nc_filter: - try: - f = open(os.path.join(d, 'netcdf_filter.h'), **open_kwargs) - except OSError: - continue + with open(ncmetapath) as f: for line in f: - if line.startswith('EXTERNL int nc_def_var_zstandard'): - has_zstandard = True - if line.startswith('EXTERNL int nc_def_var_bzip2'): - has_bzip2 = True - if line.startswith('EXTERNL int nc_def_var_blosc'): - has_blosc = True - if line.startswith('EXTERNL int nc_inq_filter_avail'): - has_ncfilter = True - - ncmetapath = os.path.join(d,'netcdf_meta.h') - if os.path.exists(ncmetapath): - for line in open(ncmetapath): - if line.startswith('#define NC_HAS_CDF5'): - try: - has_cdf5_format = bool(int(line.split()[2])) - except ValueError: - pass # keep default False if value cannot be parsed if line.startswith('#define NC_HAS_PARALLEL'): try: has_parallel_support = bool(int(line.split()[2])) except ValueError: pass - if line.startswith('#define NC_HAS_PARALLEL4'): - try: - has_parallel4_support = bool(int(line.split()[2])) - except ValueError: - pass - if line.startswith('#define NC_HAS_PNETCDF'): - try: - has_pnetcdf_support = bool(int(line.split()[2])) - except ValueError: - pass - if line.startswith('#define NC_HAS_SZIP_WRITE'): - try: - has_szip_support = bool(int(line.split()[2])) - except ValueError: - pass - - # NC_HAS_PARALLEL4 missing in 4.6.1 (issue #964) - if not has_parallel4_support and has_parallel_support and not has_pnetcdf_support: - has_parallel4_support = True - # for 4.6.1, if NC_HAS_PARALLEL=NC_HAS_PNETCDF=1, guess that - # parallel HDF5 is enabled (must guess since there is no - # NC_HAS_PARALLEL4) - elif netcdf_lib_version == "4.6.1" and not has_parallel4_support and has_parallel_support: - has_parallel4_support = True - break - return has_rename_grp, has_nc_inq_path, has_nc_inq_format_extended, \ - has_cdf5_format, has_nc_open_mem, has_nc_create_mem, \ - has_parallel4_support, has_pnetcdf_support, has_szip_support, has_quantize, \ - has_zstandard, has_bzip2, has_blosc, has_set_alignment, has_ncfilter, \ - has_nc_rc_set + return has_parallel_support def getnetcdfvers(libdirs): @@ -564,147 +470,35 @@ def _populate_hdf5_info(dirstosearch, inc_dirs, libs, lib_dirs): if len(sys.argv) >= 2: if os.path.exists(netcdf4_src_c): os.remove(netcdf4_src_c) - # this determines whether renameGroup and filepath methods will work. - has_rename_grp, has_nc_inq_path, has_nc_inq_format_extended, \ - has_cdf5_format, has_nc_open_mem, has_nc_create_mem, \ - has_parallel4_support, has_pnetcdf_support, has_szip_support, has_quantize, \ - has_zstandard, has_bzip2, has_blosc, has_set_alignment, has_ncfilter, has_nc_rc_set = \ - check_api(inc_dirs,netcdf_lib_version) + # for netcdf 4.4.x CDF5 format is always enabled. if netcdf_lib_version is not None and\ (netcdf_lib_version > "4.4" and netcdf_lib_version < "4.5"): has_cdf5_format = True - # disable parallel support if mpi4py not available. - #try: - # import mpi4py - #except ImportError: - # f.write('disabling mpi parallel support because mpi4py not found\n') - # has_parallel4_support = False - # has_pnetcdf_support = False - - f = open(osp.join('include', 'constants.pyx'), 'w') - if has_rename_grp: - sys.stdout.write('netcdf lib has group rename capability\n') - f.write('DEF HAS_RENAME_GRP = 1\n') - else: - sys.stdout.write('netcdf lib does not have group rename capability\n') - f.write('DEF HAS_RENAME_GRP = 0\n') - - if has_nc_inq_path: - sys.stdout.write('netcdf lib has nc_inq_path function\n') - f.write('DEF HAS_NC_INQ_PATH = 1\n') - else: - sys.stdout.write('netcdf lib does not have nc_inq_path function\n') - f.write('DEF HAS_NC_INQ_PATH = 0\n') - - if has_nc_inq_format_extended: - sys.stdout.write('netcdf lib has nc_inq_format_extended function\n') - f.write('DEF HAS_NC_INQ_FORMAT_EXTENDED = 1\n') - else: - sys.stdout.write( - 'netcdf lib does not have nc_inq_format_extended function\n') - f.write('DEF HAS_NC_INQ_FORMAT_EXTENDED = 0\n') - - if has_nc_open_mem: - sys.stdout.write('netcdf lib has nc_open_mem function\n') - f.write('DEF HAS_NC_OPEN_MEM = 1\n') - else: - sys.stdout.write('netcdf lib does not have nc_open_mem function\n') - f.write('DEF HAS_NC_OPEN_MEM = 0\n') - - if has_nc_create_mem: - sys.stdout.write('netcdf lib has nc_create_mem function\n') - f.write('DEF HAS_NC_CREATE_MEM = 1\n') - else: - sys.stdout.write('netcdf lib does not have nc_create_mem function\n') - f.write('DEF HAS_NC_CREATE_MEM = 0\n') - - if has_cdf5_format: - sys.stdout.write('netcdf lib has cdf-5 format capability\n') - f.write('DEF HAS_CDF5_FORMAT = 1\n') - else: - sys.stdout.write('netcdf lib does not have cdf-5 format capability\n') - f.write('DEF HAS_CDF5_FORMAT = 0\n') - - if has_parallel4_support: - sys.stdout.write('netcdf lib has netcdf4 parallel functions\n') - f.write('DEF HAS_PARALLEL4_SUPPORT = 1\n') - else: - sys.stdout.write('netcdf lib does not have netcdf4 parallel functions\n') - f.write('DEF HAS_PARALLEL4_SUPPORT = 0\n') - - if has_pnetcdf_support: - sys.stdout.write('netcdf lib has pnetcdf parallel functions\n') - f.write('DEF HAS_PNETCDF_SUPPORT = 1\n') - else: - sys.stdout.write('netcdf lib does not have pnetcdf parallel functions\n') - f.write('DEF HAS_PNETCDF_SUPPORT = 0\n') - - if has_quantize: - sys.stdout.write('netcdf lib has bit-grooming/quantization functions\n') - f.write('DEF HAS_QUANTIZATION_SUPPORT = 1\n') - else: - sys.stdout.write('netcdf lib does not have bit-grooming/quantization functions\n') - f.write('DEF HAS_QUANTIZATION_SUPPORT = 0\n') - - if has_zstandard: - sys.stdout.write('netcdf lib has zstandard compression functions\n') - f.write('DEF HAS_ZSTANDARD_SUPPORT = 1\n') - else: - sys.stdout.write('netcdf lib does not have zstandard compression functions\n') - f.write('DEF HAS_ZSTANDARD_SUPPORT = 0\n') - - if has_bzip2: - sys.stdout.write('netcdf lib has bzip2 compression functions\n') - f.write('DEF HAS_BZIP2_SUPPORT = 1\n') - else: - sys.stdout.write('netcdf lib does not have bzip2 compression functions\n') - f.write('DEF HAS_BZIP2_SUPPORT = 0\n') - - if has_blosc: - sys.stdout.write('netcdf lib has blosc compression functions\n') - f.write('DEF HAS_BLOSC_SUPPORT = 1\n') - else: - sys.stdout.write('netcdf lib does not have blosc compression functions\n') - f.write('DEF HAS_BLOSC_SUPPORT = 0\n') - - if has_szip_support: - sys.stdout.write('netcdf lib has szip compression functions\n') - f.write('DEF HAS_SZIP_SUPPORT = 1\n') - else: - sys.stdout.write('netcdf lib does not have szip compression functions\n') - f.write('DEF HAS_SZIP_SUPPORT = 0\n') - - if has_set_alignment: - sys.stdout.write('netcdf lib has nc_set_alignment function\n') - f.write('DEF HAS_SET_ALIGNMENT = 1\n') - else: - sys.stdout.write('netcdf lib does not have nc_set_alignment function\n') - f.write('DEF HAS_SET_ALIGNMENT = 0\n') + has_parallel_support = check_has_parallel_support(inc_dirs) + has_has_not = "has" if has_parallel_support else "does not have" + sys.stdout.write(f"netcdf lib {has_has_not} parallel functions\n") - if has_ncfilter: - sys.stdout.write('netcdf lib has nc_inq_filter_avail function\n') - f.write('DEF HAS_NCFILTER = 1\n') - else: - sys.stdout.write('netcdf lib does not have nc_inq_filter_avail function\n') - f.write('DEF HAS_NCFILTER = 0\n') - - if has_nc_rc_set: - sys.stdout.write('netcdf lib has nc_rc_set function\n') - f.write('DEF HAS_NCRCSET = 1\n') - else: - sys.stdout.write('netcdf lib does not have nc_rc_set function\n') - f.write('DEF HAS_NCRCSET = 0\n') - - f.close() - - if has_parallel4_support or has_pnetcdf_support: + if has_parallel_support: import mpi4py inc_dirs.append(mpi4py.get_include()) # mpi_incdir should not be needed if using nc-config # (should be included in nc-config --cflags) - if mpi_incdir is not None: inc_dirs.append(mpi_incdir) + if mpi_incdir is not None: + inc_dirs.append(mpi_incdir) + + # Name of file containing imports required for parallel support + parallel_support_imports = "parallel_support_imports.pxi.in" + else: + parallel_support_imports = "no_parallel_support_imports.pxi.in" + + # Copy the specific version of the file containing parallel + # support imports + shutil.copyfile( + osp.join("include", parallel_support_imports), + osp.join("include", "parallel_support_imports.pxi") + ) ext_modules = [Extension("netCDF4._netCDF4", [netcdf4_src_pyx], diff --git a/src/netCDF4/_netCDF4.pyx b/src/netCDF4/_netCDF4.pyx index 3edbc6f34..9f62643ff 100644 --- a/src/netCDF4/_netCDF4.pyx +++ b/src/netCDF4/_netCDF4.pyx @@ -1,5 +1,5 @@ """ -Version 1.6.5 +Version 1.7.0 ------------- # Introduction @@ -1227,7 +1227,7 @@ from .utils import (_StartCountStride, _quantize, _find_dim, _walk_grps, import sys import functools -__version__ = "1.6.5" +__version__ = "1.7.0" # Initialize numpy import posixpath @@ -1244,36 +1244,36 @@ from numpy import ma from libc.string cimport memcpy, memset from libc.stdlib cimport malloc, free numpy.import_array() -include "constants.pyx" include "membuf.pyx" include "netCDF4.pxi" -IF HAS_PARALLEL4_SUPPORT or HAS_PNETCDF_SUPPORT: - cimport mpi4py.MPI as MPI - from mpi4py.libmpi cimport MPI_Comm, MPI_Info, MPI_Comm_dup, MPI_Info_dup, \ - MPI_Comm_free, MPI_Info_free, MPI_INFO_NULL,\ - MPI_COMM_WORLD - ctypedef MPI.Comm Comm - ctypedef MPI.Info Info -ELSE: - ctypedef object Comm - ctypedef object Info + +__has_rename_grp__ = HAS_RENAME_GRP +__has_nc_inq_path__ = HAS_NC_INQ_PATH +__has_nc_inq_format_extended__ = HAS_NC_INQ_FORMAT_EXTENDED +__has_cdf5_format__ = HAS_CDF5_FORMAT +__has_nc_open_mem__ = HAS_NC_OPEN_MEM +__has_nc_create_mem__ = HAS_NC_CREATE_MEM +__has_parallel4_support__ = HAS_PARALLEL4_SUPPORT +__has_pnetcdf_support__ = HAS_PNETCDF_SUPPORT +__has_parallel_support__ = HAS_PARALLEL_SUPPORT +__has_quantization_support__ = HAS_QUANTIZATION_SUPPORT +__has_zstandard_support__ = HAS_ZSTANDARD_SUPPORT +__has_bzip2_support__ = HAS_BZIP2_SUPPORT +__has_blosc_support__ = HAS_BLOSC_SUPPORT +__has_szip_support__ = HAS_SZIP_SUPPORT +__has_set_alignment__ = HAS_SET_ALIGNMENT +__has_ncfilter__ = HAS_NCFILTER + # set path to SSL certificates (issue #1246) -IF HAS_NCRCSET: # available starting in version 4.9.1 +# available starting in version 4.9.1 +if HAS_NCRCSET: import certifi - cdef _set_curl_certpath(certpath): - cdef char *cert_path - cdef char *key - cdef int ierr - bytestr = _strencode(certpath) - cert_path = bytestr - ierr = nc_rc_set("HTTP.SSL.CAINFO",cert_path) - if ierr != 0: - raise RuntimeError('error setting path to SSL certificates') - _set_curl_certpath(certifi.where()) + if nc_rc_set("HTTP.SSL.CAINFO", _strencode(certifi.where())) != 0: + raise RuntimeError('error setting path to SSL certificates') -# check for required version of netcdf-4 and hdf5. +# check for required version of netcdf-4 and hdf5. def _gethdf5libversion(): cdef unsigned int majorvers, minorvers, releasevers cdef herr_t ierr @@ -1336,10 +1336,9 @@ details.""" ierr = nc_set_chunk_cache(sizep,nelemsp, preemptionp) _ensure_nc_success(ierr) -IF HAS_SET_ALIGNMENT: - def get_alignment(): - """ - **`get_alignment()`** + +def get_alignment(): + """**`get_alignment()`** return current netCDF alignment within HDF5 files in a tuple (threshold,alignment). See netcdf C library documentation for @@ -1347,57 +1346,46 @@ IF HAS_SET_ALIGNMENT: `set_alignment`. This function was added in netcdf 4.9.0.""" - cdef int ierr - cdef int thresholdp, alignmentp - ierr = nc_get_alignment(&thresholdp, &alignmentp) - _ensure_nc_success(ierr) - threshold = thresholdp - alignment = alignmentp - return (threshold,alignment) - def set_alignment(threshold, alignment): - """ - **`set_alignment(threshold,alignment)`** + if not __has_set_alignment__: + raise RuntimeError( + "This function requires netcdf4 4.9.0+ to be used at compile time" + ) + + cdef int ierr + cdef int thresholdp, alignmentp + ierr = nc_get_alignment(&thresholdp, &alignmentp) + _ensure_nc_success(ierr) + threshold = thresholdp + alignment = alignmentp + return (threshold, alignment) + + +def set_alignment(threshold, alignment): + """**`set_alignment(threshold,alignment)`** Change the HDF5 file alignment. See netcdf C library documentation for `nc_set_alignment` for details. This function was added in netcdf 4.9.0.""" - cdef int ierr - cdef int thresholdp, alignmentp - thresholdp = threshold - alignmentp = alignment - ierr = nc_set_alignment(thresholdp, alignmentp) - _ensure_nc_success(ierr) -ELSE: - def get_alignment(): + if not __has_set_alignment__: raise RuntimeError( "This function requires netcdf4 4.9.0+ to be used at compile time" ) - def set_alignment(threshold, alignment): - raise RuntimeError( - "This function requires netcdf4 4.9.0+ to be used at compile time" - ) + cdef int ierr + cdef int thresholdp, alignmentp + thresholdp = threshold + alignmentp = alignment + + ierr = nc_set_alignment(thresholdp, alignmentp) + _ensure_nc_success(ierr) + __netcdf4libversion__ = getlibversion().split()[0] __hdf5libversion__ = _gethdf5libversion() -__has_rename_grp__ = HAS_RENAME_GRP -__has_nc_inq_path__ = HAS_NC_INQ_PATH -__has_nc_inq_format_extended__ = HAS_NC_INQ_FORMAT_EXTENDED -__has_cdf5_format__ = HAS_CDF5_FORMAT -__has_nc_open_mem__ = HAS_NC_OPEN_MEM -__has_nc_create_mem__ = HAS_NC_CREATE_MEM -__has_parallel4_support__ = HAS_PARALLEL4_SUPPORT -__has_pnetcdf_support__ = HAS_PNETCDF_SUPPORT -__has_quantization_support__ = HAS_QUANTIZATION_SUPPORT -__has_zstandard_support__ = HAS_ZSTANDARD_SUPPORT -__has_bzip2_support__ = HAS_BZIP2_SUPPORT -__has_blosc_support__ = HAS_BLOSC_SUPPORT -__has_szip_support__ = HAS_SZIP_SUPPORT -__has_set_alignment__ = HAS_SET_ALIGNMENT _needsworkaround_issue485 = __netcdf4libversion__ < "4.4.0" or \ (__netcdf4libversion__.startswith("4.4.0") and \ "-development" in __netcdf4libversion__) @@ -1412,6 +1400,17 @@ to HDF5 version 1.8.x is highly recommended (see https://github.com/Unidata/netcdf-c/issues/250).""" warnings.warn(msg) + +class NetCDF4MissingFeatureException(Exception): + """Custom exception when trying to use features missing from the linked netCDF library""" + def __init__(self, feature: str, version: str): + super().__init__( + f"{feature} requires netCDF lib >= {version} (using {__netcdf4libversion__}). " + f"To enable, rebuild netcdf4-python using netCDF {version} or higher " + f"(and possibly enable {feature})" + ) + + # numpy data type <--> netCDF 4 data type mapping. _nptonctype = {'S1' : NC_CHAR, 'i1' : NC_BYTE, @@ -1448,23 +1447,23 @@ _blosc_dict={'blosc_lz':0,'blosc_lz4':1,'blosc_lz4hc':2,'blosc_snappy':3,'blosc_ _blosc_dict_inv = {v: k for k, v in _blosc_dict.items()} _szip_dict = {'ec': 4, 'nn': 32} _szip_dict_inv = {v: k for k, v in _szip_dict.items()} -IF HAS_CDF5_FORMAT: +if __has_cdf5_format__: # NETCDF3_64BIT deprecated, saved for compatibility. # use NETCDF3_64BIT_OFFSET instead. _format_dict['NETCDF3_64BIT_OFFSET'] = NC_FORMAT_64BIT_OFFSET _format_dict['NETCDF3_64BIT_DATA'] = NC_FORMAT_64BIT_DATA _cmode_dict['NETCDF3_64BIT_OFFSET'] = NC_64BIT_OFFSET _cmode_dict['NETCDF3_64BIT_DATA'] = NC_64BIT_DATA -ELSE: +else: _format_dict['NETCDF3_64BIT'] = NC_FORMAT_64BIT _cmode_dict['NETCDF3_64BIT'] = NC_64BIT_OFFSET # invert dictionary mapping _reverse_format_dict = dict((v, k) for k, v in _format_dict.iteritems()) # add duplicate entry (NETCDF3_64BIT == NETCDF3_64BIT_OFFSET) -IF HAS_CDF5_FORMAT: +if __has_cdf5_format__: _format_dict['NETCDF3_64BIT'] = NC_FORMAT_64BIT_OFFSET _cmode_dict['NETCDF3_64BIT'] = NC_64BIT_OFFSET -ELSE: +else: _format_dict['NETCDF3_64BIT_OFFSET'] = NC_FORMAT_64BIT _cmode_dict['NETCDF3_64BIT_OFFSET'] = NC_64BIT_OFFSET @@ -1629,30 +1628,33 @@ cdef _get_format(int grpid): raise ValueError('format not supported by python interface') return _reverse_format_dict[formatp] + cdef _get_full_format(int grpid): - # Private function to get the underlying disk format + """Private function to get the underlying disk format""" + + if not __has_nc_inq_format_extended__: + return "UNDEFINED" + cdef int ierr, formatp, modep - IF HAS_NC_INQ_FORMAT_EXTENDED: - with nogil: - ierr = nc_inq_format_extended(grpid, &formatp, &modep) - _ensure_nc_success(ierr) - if formatp == NC_FORMAT_NC3: - return 'NETCDF3' - elif formatp == NC_FORMAT_NC_HDF5: - return 'HDF5' - elif formatp == NC_FORMAT_NC_HDF4: - return 'HDF4' - elif formatp == NC_FORMAT_PNETCDF: - return 'PNETCDF' - elif formatp == NC_FORMAT_DAP2: - return 'DAP2' - elif formatp == NC_FORMAT_DAP4: - return 'DAP4' - elif formatp == NC_FORMAT_UNDEFINED: - return 'UNDEFINED' - ELSE: + with nogil: + ierr = nc_inq_format_extended(grpid, &formatp, &modep) + _ensure_nc_success(ierr) + if formatp == NC_FORMAT_NC3: + return 'NETCDF3' + if formatp == NC_FORMAT_NC_HDF5: + return 'HDF5' + if formatp == NC_FORMAT_NC_HDF4: + return 'HDF4' + if formatp == NC_FORMAT_PNETCDF: + return 'PNETCDF' + if formatp == NC_FORMAT_DAP2: + return 'DAP2' + if formatp == NC_FORMAT_DAP4: + return 'DAP4' + if formatp == NC_FORMAT_UNDEFINED: return 'UNDEFINED' + cdef issue485_workaround(int grpid, int varid, char* attname): # check to see if attribute already exists # and is NC_CHAR, if so delete it and re-create it @@ -2124,7 +2126,7 @@ strings. def __init__(self, filename, mode='r', clobber=True, format='NETCDF4', diskless=False, persist=False, keepweakref=False, memory=None, encoding=None, parallel=False, - Comm comm=None, Info info=None, **kwargs): + comm=None, info=None, **kwargs): """ **`__init__(self, filename, mode="r", clobber=True, diskless=False, persist=False, keepweakref=False, memory=None, encoding=None, @@ -2225,9 +2227,8 @@ strings. cdef char *path cdef char namstring[NC_MAX_NAME+1] cdef int cmode, parmode - IF HAS_PARALLEL4_SUPPORT or HAS_PNETCDF_SUPPORT: - cdef MPI_Comm mpicomm - cdef MPI_Info mpiinfo + cdef MPI_Comm mpicomm + cdef MPI_Info mpiinfo memset(&self._buffer, 0, sizeof(self._buffer)) @@ -2248,30 +2249,29 @@ strings. raise ValueError(msg) if parallel: - IF HAS_PARALLEL4_SUPPORT != 1 and HAS_PNETCDF_SUPPORT != 1: - msg='parallel mode requires MPI enabled netcdf-c' + if not __has_parallel_support__: + raise ValueError("parallel mode requires MPI enabled netcdf-c") + + parallel_formats = [] + if __has_parallel4_support__: + parallel_formats += ['NETCDF4','NETCDF4_CLASSIC'] + if __has_pnetcdf_support__: + parallel_formats += ['NETCDF3_CLASSIC', + 'NETCDF3_64BIT_OFFSET', + 'NETCDF3_64BIT_DATA', + 'NETCDF3_64BIT'] + if format not in parallel_formats: + msg='parallel mode only works with the following formats: ' + ' '.join(parallel_formats) raise ValueError(msg) - ELSE: - parallel_formats = [] - IF HAS_PARALLEL4_SUPPORT: - parallel_formats += ['NETCDF4','NETCDF4_CLASSIC'] - IF HAS_PNETCDF_SUPPORT: - parallel_formats += ['NETCDF3_CLASSIC', - 'NETCDF3_64BIT_OFFSET', - 'NETCDF3_64BIT_DATA', - 'NETCDF3_64BIT'] - if format not in parallel_formats: - msg='parallel mode only works with the following formats: ' + ' '.join(parallel_formats) - raise ValueError(msg) - if comm is not None: - mpicomm = comm.ob_mpi - else: - mpicomm = MPI_COMM_WORLD - if info is not None: - mpiinfo = info.ob_mpi - else: - mpiinfo = MPI_INFO_NULL - parmode = NC_MPIIO | _cmode_dict[format] + if comm is not None: + mpicomm = (comm).ob_mpi + else: + mpicomm = MPI_COMM_WORLD + if info is not None: + mpiinfo = (info).ob_mpi + else: + mpiinfo = MPI_INFO_NULL + parmode = NC_MPIIO | _cmode_dict[format] self._inmemory = False @@ -2282,61 +2282,32 @@ strings. if mode == 'w' or (mode in ['a','r+'] and not os.path.exists(filename)): _set_default_format(format=format) if memory is not None: + if not __has_nc_create_mem__: + raise NetCDF4MissingFeatureException("nc_create_mem", "4.6.2") + # if memory is not None and mode='w', memory # kwarg is interpreted as advisory size. - IF HAS_NC_CREATE_MEM: - initialsize = memory - with nogil: - ierr = nc_create_mem(path, 0, initialsize, &grpid) - self._inmemory = True # checked in close method - ELSE: - msg = """ - nc_create_mem functionality not enabled. To enable, install Cython, make sure you have - version 4.6.2 or higher of the netcdf C lib, and rebuild netcdf4-python.""" - raise ValueError(msg) + initialsize = memory + with nogil: + ierr = nc_create_mem(path, 0, initialsize, &grpid) + self._inmemory = True # checked in close method + else: - if clobber: - if parallel: - IF HAS_PARALLEL4_SUPPORT or HAS_PNETCDF_SUPPORT: - cmode = NC_CLOBBER | parmode - with nogil: - ierr = nc_create_par(path, cmode, \ - mpicomm, mpiinfo, &grpid) - ELSE: - pass - elif diskless: - if persist: - cmode = NC_WRITE | NC_CLOBBER | NC_DISKLESS | NC_PERSIST - with nogil: - ierr = nc_create(path, cmode, &grpid) - else: - cmode = NC_CLOBBER | NC_DISKLESS - with nogil: - ierr = nc_create(path, cmode , &grpid) - else: - with nogil: - ierr = nc_create(path, NC_CLOBBER, &grpid) + cmode = NC_CLOBBER if clobber else NC_NOCLOBBER + + if parallel: + cmode |= parmode + with nogil: + ierr = nc_create_par(path, cmode, mpicomm, mpiinfo, &grpid) else: - if parallel: - IF HAS_PARALLEL4_SUPPORT or HAS_PNETCDF_SUPPORT: - cmode = NC_NOCLOBBER | parmode - with nogil: - ierr = nc_create_par(path, cmode, \ - mpicomm, mpiinfo, &grpid) - ELSE: - pass - elif diskless: + if diskless: + cmode |= NC_DISKLESS if persist: - cmode = NC_WRITE | NC_NOCLOBBER | NC_DISKLESS | NC_PERSIST - with nogil: - ierr = nc_create(path, cmode, &grpid) - else: - cmode = NC_NOCLOBBER | NC_DISKLESS - with nogil: - ierr = nc_create(path, cmode , &grpid) - else: - with nogil: - ierr = nc_create(path, NC_NOCLOBBER, &grpid) + cmode |= NC_WRITE | NC_PERSIST + + with nogil: + ierr = nc_create(path, cmode, &grpid) + # reset default format to netcdf3 - this is a workaround # for issue 170 (nc_open'ing a DAP dataset after switching # format to NETCDF4). This bug should be fixed in version @@ -2346,27 +2317,21 @@ strings. #_set_default_format(format='NETCDF3_64BIT_OFFSET') elif mode in ('r', 'rs'): if memory is not None: - IF HAS_NC_OPEN_MEM: - # Store reference to memory - result = PyObject_GetBuffer(memory, &self._buffer, PyBUF_SIMPLE | PyBUF_ANY_CONTIGUOUS) - if result != 0: - raise ValueError("Unable to retrieve Buffer from %s" % (memory,)) + if not __has_nc_open_mem__: + raise NetCDF4MissingFeatureException("nc_open_mem", "4.4.1") + + # Store reference to memory + result = PyObject_GetBuffer(memory, &self._buffer, PyBUF_SIMPLE | PyBUF_ANY_CONTIGUOUS) + if result != 0: + raise ValueError("Unable to retrieve Buffer from %s" % (memory,)) + + with nogil: + ierr = nc_open_mem(path, 0, self._buffer.len, self._buffer.buf, &grpid) - with nogil: - ierr = nc_open_mem(path, 0, self._buffer.len, self._buffer.buf, &grpid) - ELSE: - msg = """ - nc_open_mem functionality not enabled. To enable, install Cython, make sure you have - version 4.4.1 or higher of the netcdf C lib, and rebuild netcdf4-python.""" - raise ValueError(msg) elif parallel: - IF HAS_PARALLEL4_SUPPORT or HAS_PNETCDF_SUPPORT: - cmode = NC_NOWRITE | NC_MPIIO - with nogil: - ierr = nc_open_par(path, cmode, \ - mpicomm, mpiinfo, &grpid) - ELSE: - pass + cmode = NC_NOWRITE | NC_MPIIO + with nogil: + ierr = nc_open_par(path, cmode, mpicomm, mpiinfo, &grpid) elif diskless: cmode = NC_NOWRITE | NC_DISKLESS with nogil: @@ -2384,13 +2349,9 @@ strings. ierr = nc_open(path, NC_NOWRITE, &grpid) elif mode in ['a','r+'] and os.path.exists(filename): if parallel: - IF HAS_PARALLEL4_SUPPORT or HAS_PNETCDF_SUPPORT: - cmode = NC_WRITE | NC_MPIIO - with nogil: - ierr = nc_open_par(path, cmode, \ - mpicomm, mpiinfo, &grpid) - ELSE: - pass + cmode = NC_WRITE | NC_MPIIO + with nogil: + ierr = nc_open_par(path, cmode, mpicomm, mpiinfo, &grpid) elif diskless: cmode = NC_WRITE | NC_DISKLESS with nogil: @@ -2401,13 +2362,9 @@ strings. elif mode in ['as','r+s'] and os.path.exists(filename): if parallel: # NC_SHARE ignored - IF HAS_PARALLEL4_SUPPORT or HAS_PNETCDF_SUPPORT: - cmode = NC_WRITE | NC_MPIIO - with nogil: - ierr = nc_open_par(path, cmode, \ - mpicomm, mpiinfo, &grpid) - ELSE: - pass + cmode = NC_WRITE | NC_MPIIO + with nogil: + ierr = nc_open_par(path, cmode, mpicomm, mpiinfo, &grpid) elif diskless: cmode = NC_SHARE | NC_DISKLESS with nogil: @@ -2420,13 +2377,9 @@ strings. if clobber: if parallel: # NC_SHARE ignored - IF HAS_PARALLEL4_SUPPORT or HAS_PNETCDF_SUPPORT: - cmode = NC_CLOBBER | parmode - with nogil: - ierr = nc_create_par(path, NC_CLOBBER | cmode, \ - mpicomm, mpiinfo, &grpid) - ELSE: - pass + cmode = NC_CLOBBER | parmode + with nogil: + ierr = nc_create_par(path, NC_CLOBBER | cmode, mpicomm, mpiinfo, &grpid) elif diskless: if persist: cmode = NC_WRITE | NC_SHARE | NC_CLOBBER | NC_DISKLESS @@ -2443,13 +2396,9 @@ strings. else: if parallel: # NC_SHARE ignored - IF HAS_PARALLEL4_SUPPORT or HAS_PNETCDF_SUPPORT: - cmode = NC_NOCLOBBER | parmode - with nogil: - ierr = nc_create_par(path, cmode, \ - mpicomm, mpiinfo, &grpid) - ELSE: - pass + cmode = NC_NOCLOBBER | parmode + with nogil: + ierr = nc_create_par(path, cmode, mpicomm, mpiinfo, &grpid) elif diskless: if persist: cmode = NC_WRITE | NC_SHARE | NC_NOCLOBBER | NC_DISKLESS @@ -2522,40 +2471,38 @@ strings. raise IndexError('%s not found in %s' % (lastname,group.path)) def filepath(self,encoding=None): + """**`filepath(self,encoding=None)`** + + Get the file system path (or the opendap URL) which was used to + open/create the Dataset. Requires netcdf >= 4.1.2. The path + is decoded into a string using `sys.getfilesystemencoding()` by default, this can be + changed using the `encoding` kwarg. """ -**`filepath(self,encoding=None)`** + if not __has_nc_inq_path__: + raise NetCDF4MissingFeatureException("filepath method", "4.1.2") -Get the file system path (or the opendap URL) which was used to -open/create the Dataset. Requires netcdf >= 4.1.2. The path -is decoded into a string using `sys.getfilesystemencoding()` by default, this can be -changed using the `encoding` kwarg.""" cdef int ierr cdef size_t pathlen cdef char *c_path if encoding is None: encoding = sys.getfilesystemencoding() - IF HAS_NC_INQ_PATH: + + with nogil: + ierr = nc_inq_path(self._grpid, &pathlen, NULL) + _ensure_nc_success(ierr) + + c_path = malloc(sizeof(char) * (pathlen + 1)) + if not c_path: + raise MemoryError() + try: with nogil: - ierr = nc_inq_path(self._grpid, &pathlen, NULL) + ierr = nc_inq_path(self._grpid, &pathlen, c_path) _ensure_nc_success(ierr) - c_path = malloc(sizeof(char) * (pathlen + 1)) - if not c_path: - raise MemoryError() - try: - with nogil: - ierr = nc_inq_path(self._grpid, &pathlen, c_path) - _ensure_nc_success(ierr) - - py_path = c_path[:pathlen] # makes a copy of pathlen bytes from c_string - finally: - free(c_path) - return py_path.decode(encoding) - ELSE: - msg = """ -filepath method not enabled. To enable, install Cython, make sure you have -version 4.1.2 or higher of the netcdf C lib, and rebuild netcdf4-python.""" - raise ValueError(msg) + py_path = c_path[:pathlen] # makes a copy of pathlen bytes from c_string + finally: + free(c_path) + return py_path.decode(encoding) def __repr__(self): return self.__str__() @@ -2596,37 +2543,31 @@ version 4.1.2 or higher of the netcdf C lib, and rebuild netcdf4-python.""" # view.obj is checked, ref on obj is decremented and obj will be null'd out PyBuffer_Release(&self._buffer) - IF HAS_NC_CREATE_MEM: - def _close_mem(self, check_err): - cdef int ierr - cdef NC_memio memio - with nogil: - ierr = nc_close_memio(self._grpid, &memio) - - if check_err: - _ensure_nc_success(ierr) + def _close_mem(self, check_err): + cdef int ierr + cdef NC_memio memio + with nogil: + ierr = nc_close_memio(self._grpid, &memio) - self._isopen = 0 - PyBuffer_Release(&self._buffer) + if check_err: + _ensure_nc_success(ierr) - # membuf_fromptr from membuf.pyx - creates a python memoryview - # from a raw pointer without making a copy. - return memview_fromptr(memio.memory, memio.size) + self._isopen = 0 + PyBuffer_Release(&self._buffer) + # membuf_fromptr from membuf.pyx - creates a python memoryview + # from a raw pointer without making a copy. + return memview_fromptr(memio.memory, memio.size) def close(self): - """ -**`close(self)`** + """**`close(self)`** -Close the Dataset. + Close the Dataset. """ - IF HAS_NC_CREATE_MEM: - if self._inmemory: - return self._close_mem(True) - else: - self._close(True) - ELSE: - self._close(True) + if __has_nc_create_mem__ and self._inmemory: + return self._close_mem(True) + + self._close(True) def isopen(self): """ @@ -3164,27 +3105,27 @@ rename a `Dataset` or `Group` attribute named `oldname` to `newname`.""" rename a `Group` named `oldname` to `newname` (requires netcdf >= 4.3.1).""" cdef char *newnamec cdef int grpid - IF HAS_RENAME_GRP: - cdef int ierr - bytestr = _strencode(newname) - newnamec = bytestr - try: - grp = self.groups[oldname] - grpid = grp._grpid - except KeyError: - raise KeyError('%s not a valid group name' % oldname) - with nogil: - ierr = nc_rename_grp(grpid, newnamec) - _ensure_nc_success(ierr) - # remove old key from groups dict. - self.groups.pop(oldname) - # add new key. - self.groups[newname] = grp - ELSE: - msg = """ -renameGroup method not enabled. To enable, install Cython, make sure you have -version 4.3.1 or higher of the netcdf C lib, and rebuild netcdf4-python.""" - raise ValueError(msg) + cdef int ierr + if not __has_rename_grp__: + raise ValueError( + "renameGroup method not enabled. To enable, install Cython, make sure you have" + "version 4.3.1 or higher of the netcdf C lib, and rebuild netcdf4-python." + ) + + bytestr = _strencode(newname) + newnamec = bytestr + try: + grp = self.groups[oldname] + grpid = grp._grpid + except KeyError: + raise KeyError('%s not a valid group name' % oldname) + with nogil: + ierr = nc_rename_grp(grpid, newnamec) + _ensure_nc_success(ierr) + # remove old key from groups dict. + self.groups.pop(oldname) + # add new key. + self.groups[newname] = grp def set_auto_chartostring(self, value): """ @@ -3498,68 +3439,61 @@ to be installed and in `$PATH`. f = open(outfile,'w') f.write(result.stdout) f.close() + def has_blosc_filter(self): + """**`has_blosc_filter(self)`** + returns True if blosc compression filter is available """ -**`has_blosc_filter(self)`** -returns True if blosc compression filter is available""" - cdef int ierr - IF HAS_BLOSC_SUPPORT: - with nogil: - ierr = nc_inq_filter_avail(self._grpid, H5Z_FILTER_BLOSC) - if ierr: - return False - else: - return True - ELSE: + if __has_blosc_support__: return False + + cdef int ierr + with nogil: + ierr = nc_inq_filter_avail(self._grpid, H5Z_FILTER_BLOSC) + return ierr == 0 + def has_zstd_filter(self): + """**`has_zstd_filter(self)`** + returns True if zstd compression filter is available """ -**`has_zstd_filter(self)`** -returns True if zstd compression filter is available""" - cdef int ierr - IF HAS_ZSTANDARD_SUPPORT: - with nogil: - ierr = nc_inq_filter_avail(self._grpid, H5Z_FILTER_ZSTD) - if ierr: - return False - else: - return True - ELSE: + + if __has_zstandard_support__: return False + + cdef int ierr + with nogil: + ierr = nc_inq_filter_avail(self._grpid, H5Z_FILTER_ZSTD) + return ierr == 0 + def has_bzip2_filter(self): + """**`has_bzip2_filter(self)`** + returns True if bzip2 compression filter is available """ -**`has_bzip2_filter(self)`** -returns True if bzip2 compression filter is available""" - cdef int ierr - IF HAS_BZIP2_SUPPORT: - with nogil: - ierr = nc_inq_filter_avail(self._grpid, H5Z_FILTER_BZIP2) - if ierr: - return False - else: - return True - ELSE: + + if __has_bzip2_support__: return False + + cdef int ierr + with nogil: + ierr = nc_inq_filter_avail(self._grpid, H5Z_FILTER_BZIP2) + return ierr == 0 + def has_szip_filter(self): + """**`has_szip_filter(self)`** + returns True if szip compression filter is available """ -**`has_szip_filter(self)`** -returns True if szip compression filter is available""" + + if not __has_ncfilter__: + return __has_szip_support__ + + if not __has_szip_support__: + return False + cdef int ierr - IF HAS_NCFILTER: - IF HAS_SZIP_SUPPORT: - with nogil: - ierr = nc_inq_filter_avail(self._grpid, H5Z_FILTER_SZIP) - if ierr: - return False - else: - return True - ELSE: - return False - ELSE: - IF HAS_SZIP_SUPPORT: - return True - ELSE: - return False + with nogil: + ierr = nc_inq_filter_avail(self._grpid, H5Z_FILTER_SZIP) + return ierr == 0 + cdef class Group(Dataset): """ @@ -4214,68 +4148,64 @@ behavior is similar to Fortran or Matlab, but different than numpy. if ierr != NC_NOERR: if grp.data_model != 'NETCDF4': grp._enddef() _ensure_nc_success(ierr, extra_msg=error_info) + if szip: - IF HAS_SZIP_SUPPORT: - try: - iszip_coding = _szip_dict[szip_coding] - except KeyError: - msg="unknown szip coding ('ec' or 'nn' supported)" - raise ValueError(msg) - iszip_pixels_per_block = szip_pixels_per_block - with nogil: - ierr = nc_def_var_szip(self._grpid, self._varid, iszip_coding, iszip_pixels_per_block) - if ierr != NC_NOERR: - if grp.data_model != 'NETCDF4': grp._enddef() - _ensure_nc_success(ierr, extra_msg=error_info) - ELSE: - msg = """ -compression='szip' only works if linked version of hdf5 has szip functionality enabled""" - raise ValueError(msg) + if not __has_szip_support__: + raise ValueError("compression='szip' only works if linked version of hdf5 has szip functionality enabled") + try: + iszip_coding = _szip_dict[szip_coding] + except KeyError: + raise ValueError("unknown szip coding ('ec' or 'nn' supported)") + iszip_pixels_per_block = szip_pixels_per_block + with nogil: + ierr = nc_def_var_szip(self._grpid, self._varid, iszip_coding, iszip_pixels_per_block) + if ierr != NC_NOERR: + if grp.data_model != 'NETCDF4': + grp._enddef() + _ensure_nc_success(ierr, extra_msg=error_info) + if zstd: - IF HAS_ZSTANDARD_SUPPORT: - icomplevel = complevel - with nogil: - ierr = nc_def_var_zstandard(self._grpid, self._varid, icomplevel) - if ierr != NC_NOERR: - if grp.data_model != 'NETCDF4': grp._enddef() - _ensure_nc_success(ierr, extra_msg=error_info) - ELSE: - msg = """ -compression='zstd' only works with netcdf-c >= 4.9.0. To enable, install Cython, make sure you have -version 4.9.0 or higher netcdf-c with zstandard support, and rebuild netcdf4-python.""" - raise ValueError(msg) + if not __has_zstandard_support__: + raise NetCDF4MissingFeatureException("compression='zstd'", "4.9.0") + + icomplevel = complevel + with nogil: + ierr = nc_def_var_zstandard(self._grpid, self._varid, icomplevel) + if ierr != NC_NOERR: + if grp.data_model != 'NETCDF4': + grp._enddef() + _ensure_nc_success(ierr, extra_msg=error_info) + if bzip2: - IF HAS_BZIP2_SUPPORT: - icomplevel = complevel - with nogil: - ierr = nc_def_var_bzip2(self._grpid, self._varid, icomplevel) - if ierr != NC_NOERR: - if grp.data_model != 'NETCDF4': grp._enddef() - _ensure_nc_success(ierr, extra_msg=error_info) - ELSE: - msg = """ -compression='bzip2' only works with netcdf-c >= 4.9.0. To enable, install Cython, make sure you have -version 4.9.0 or higher netcdf-c with bzip2 support, and rebuild netcdf4-python.""" - raise ValueError(msg) + if not __has_bzip2_support__: + raise NetCDF4MissingFeatureException("compression='bzip2'", "4.9.0") + + icomplevel = complevel + with nogil: + ierr = nc_def_var_bzip2(self._grpid, self._varid, icomplevel) + if ierr != NC_NOERR: + if grp.data_model != 'NETCDF4': + grp._enddef() + _ensure_nc_success(ierr, extra_msg=error_info) + if blosc_zstd or blosc_lz or blosc_lz4 or blosc_lz4hc or blosc_zlib: - IF HAS_BLOSC_SUPPORT: - iblosc_compressor = _blosc_dict[compression] - iblosc_shuffle = blosc_shuffle - iblosc_blocksize = 0 # not currently used by c lib - iblosc_complevel = complevel - with nogil: - ierr = nc_def_var_blosc(self._grpid, self._varid,\ - iblosc_compressor,\ - iblosc_complevel,iblosc_blocksize,\ - iblosc_shuffle) - if ierr != NC_NOERR: - if grp.data_model != 'NETCDF4': grp._enddef() - _ensure_nc_success(ierr, extra_msg=error_info) - ELSE: - msg = """ -compression='blosc_*' only works with netcdf-c >= 4.9.0. To enable, install Cython, make sure you have -version 4.9.0 or higher netcdf-c with blosc support, and rebuild netcdf4-python.""" - raise ValueError(msg) + if not __has_blosc_support__: + raise NetCDF4MissingFeatureException("compression='blosc_*'", "4.9.0") + + iblosc_compressor = _blosc_dict[compression] + iblosc_shuffle = blosc_shuffle + iblosc_blocksize = 0 # not currently used by c lib + iblosc_complevel = complevel + with nogil: + ierr = nc_def_var_blosc(self._grpid, self._varid, + iblosc_compressor, + iblosc_complevel,iblosc_blocksize, + iblosc_shuffle) + if ierr != NC_NOERR: + if grp.data_model != 'NETCDF4': + grp._enddef() + _ensure_nc_success(ierr, extra_msg=error_info) + # set checksum. if fletcher32 and ndims: # don't bother for scalar variable with nogil: @@ -4322,31 +4252,32 @@ version 4.9.0 or higher netcdf-c with blosc support, and rebuild netcdf4-python. pass # this is the default format. else: raise ValueError("'endian' keyword argument must be 'little','big' or 'native', got '%s'" % endian) + # set quantization - IF HAS_QUANTIZATION_SUPPORT: - if significant_digits is not None: - nsd = significant_digits - if quantize_mode == 'BitGroom': - with nogil: - ierr = nc_def_var_quantize(self._grpid, - self._varid, NC_QUANTIZE_BITGROOM, nsd) - elif quantize_mode == 'GranularBitRound': - with nogil: - ierr = nc_def_var_quantize(self._grpid, - self._varid, NC_QUANTIZE_GRANULARBR, nsd) - elif quantize_mode == 'BitRound': + if significant_digits is not None: + if not __has_quantization_support__: + raise ValueError( + "significant_digits kwarg only works with netcdf-c >= 4.9.0. " + "To enable, install Cython, make sure you have version 4.9.0 " + "or higher netcdf-c, and rebuild netcdf4-python. Otherwise, " + f"use least_significant_digit kwarg for quantization. {error_info}" + ) + + nsd = significant_digits + if quantize_mode == 'BitGroom': + with nogil: ierr = nc_def_var_quantize(self._grpid, - self._varid, NC_QUANTIZE_BITROUND, nsd) - else: - raise ValueError("'quantize_mode' keyword argument must be 'BitGroom','GranularBitRound' or 'BitRound', got '%s'" % quantize_mode) - - ELSE: - if significant_digits is not None: - msg = f""" -significant_digits kwarg only works with netcdf-c >= 4.9.0. To enable, install Cython, make sure you have -version 4.9.0 or higher netcdf-c, and rebuild netcdf4-python. Otherwise, use least_significant_digit -kwarg for quantization. {error_info}""" - raise ValueError(msg) + self._varid, NC_QUANTIZE_BITGROOM, nsd) + elif quantize_mode == 'GranularBitRound': + with nogil: + ierr = nc_def_var_quantize(self._grpid, + self._varid, NC_QUANTIZE_GRANULARBR, nsd) + elif quantize_mode == 'BitRound': + ierr = nc_def_var_quantize(self._grpid, + self._varid, NC_QUANTIZE_BITROUND, nsd) + else: + raise ValueError("'quantize_mode' keyword argument must be 'BitGroom','GranularBitRound' or 'BitRound', got '%s'" % quantize_mode) + if ierr != NC_NOERR: if grp.data_model != 'NETCDF4': grp._enddef() _ensure_nc_success(ierr, extra_msg=error_info) @@ -4686,25 +4617,25 @@ return dictionary containing HDF5 filter parameters.""" with nogil: ierr = nc_inq_var_fletcher32(self._grpid, self._varid, &ifletcher32) _ensure_nc_success(ierr) - IF HAS_ZSTANDARD_SUPPORT: + if __has_zstandard_support__: with nogil: ierr = nc_inq_var_zstandard(self._grpid, self._varid, &izstd,\ &icomplevel_zstd) if ierr != 0: izstd=0 # _ensure_nc_success(ierr) - IF HAS_BZIP2_SUPPORT: + if __has_bzip2_support__: with nogil: ierr = nc_inq_var_bzip2(self._grpid, self._varid, &ibzip2,\ &icomplevel_bzip2) if ierr != 0: ibzip2=0 #_ensure_nc_success(ierr) - IF HAS_BLOSC_SUPPORT: + if __has_blosc_support__: with nogil: ierr = nc_inq_var_blosc(self._grpid, self._varid, &iblosc,\ &iblosc_compressor,&iblosc_complevel,&iblosc_blocksize,&iblosc_shuffle) if ierr != 0: iblosc=0 #_ensure_nc_success(ierr) - IF HAS_SZIP_SUPPORT: + if __has_szip_support__: with nogil: ierr = nc_inq_var_szip(self._grpid, self._varid, &iszip_coding,\ &iszip_pixels_per_block) @@ -4745,30 +4676,30 @@ return dictionary containing HDF5 filter parameters.""" return number of significant digits and the algorithm used in quantization. Returns None if quantization not active. """ - IF HAS_QUANTIZATION_SUPPORT: - cdef int ierr, nsd, quantize_mode - if self._grp.data_model not in ['NETCDF4_CLASSIC','NETCDF4']: - return None - else: - with nogil: - ierr = nc_inq_var_quantize(self._grpid, self._varid, &quantize_mode, &nsd) - _ensure_nc_success(ierr) - if quantize_mode == NC_NOQUANTIZE: - return None - else: - if quantize_mode == NC_QUANTIZE_GRANULARBR: - sig_digits = nsd - quant_mode = 'GranularBitRound' - elif quantize_mode == NC_QUANTIZE_BITROUND: - sig_digits = nsd # interpreted as bits, not decimal - quant_mode = 'BitRound' - else: - sig_digits = nsd - quant_mode = 'BitGroom' - return sig_digits, quant_mode - ELSE: + if not __has_quantization_support__: + return None + + cdef int ierr, nsd, quantize_mode + if self._grp.data_model not in ['NETCDF4_CLASSIC','NETCDF4']: + return None + + with nogil: + ierr = nc_inq_var_quantize(self._grpid, self._varid, &quantize_mode, &nsd) + _ensure_nc_success(ierr) + if quantize_mode == NC_NOQUANTIZE: return None + if quantize_mode == NC_QUANTIZE_GRANULARBR: + sig_digits = nsd + quant_mode = 'GranularBitRound' + elif quantize_mode == NC_QUANTIZE_BITROUND: + sig_digits = nsd # interpreted as bits, not decimal + quant_mode = 'BitRound' + else: + sig_digits = nsd + quant_mode = 'BitGroom' + return sig_digits, quant_mode + def endian(self): """ **`endian(self)`** @@ -6027,25 +5958,20 @@ NC_CHAR). return data def set_collective(self, value): - """ -**`set_collective(self,True_or_False)`** + """**`set_collective(self,True_or_False)`** -turn on or off collective parallel IO access. Ignored if file is not -open for parallel access. + turn on or off collective parallel IO access. Ignored if file is not + open for parallel access. """ - IF HAS_PARALLEL4_SUPPORT or HAS_PNETCDF_SUPPORT: - # set collective MPI IO mode on or off - if value: - with nogil: - ierr = nc_var_par_access(self._grpid, self._varid, - NC_COLLECTIVE) - else: - with nogil: - ierr = nc_var_par_access(self._grpid, self._varid, - NC_INDEPENDENT) - _ensure_nc_success(ierr) - ELSE: - pass # does nothing + if not __has_parallel_support__: + return + + mode = NC_COLLECTIVE if value else NC_INDEPENDENT + with nogil: + ierr = nc_var_par_access(self._grpid, self._varid, + NC_COLLECTIVE) + _ensure_nc_success(ierr) + def get_dims(self): """