From d5b2c4051996b2e8971816a0c8d41638fe0e0ffe Mon Sep 17 00:00:00 2001 From: Qianqian Fang Date: Fri, 13 Oct 2023 22:36:46 -0400 Subject: [PATCH] revert blosc2 to v2.8.0 to remove new dependency libdl --- src/Makefile | 6 +- src/blosc2/LICENSE.txt | 2 +- src/blosc2/README.rst | 113 ++-- src/blosc2/blosc/Makefile | 2 +- src/blosc2/blosc/b2nd.c | 44 +- src/blosc2/blosc/b2nd_utils.c | 8 +- src/blosc2/blosc/b2nd_utils.h | 20 +- src/blosc2/blosc/bitshuffle-altivec.c | 9 +- src/blosc2/blosc/bitshuffle-altivec.h | 19 +- src/blosc2/blosc/bitshuffle-avx2.c | 10 +- src/blosc2/blosc/bitshuffle-avx2.h | 19 +- src/blosc2/blosc/bitshuffle-generic.c | 4 +- src/blosc2/blosc/bitshuffle-generic.h | 86 ++- src/blosc2/blosc/bitshuffle-neon.c | 6 +- src/blosc2/blosc/bitshuffle-neon.h | 17 +- src/blosc2/blosc/bitshuffle-sse2.c | 6 +- src/blosc2/blosc/bitshuffle-sse2.h | 19 +- src/blosc2/blosc/blosc-private.h | 125 +--- src/blosc2/blosc/blosc2-stdio.c | 12 +- src/blosc2/blosc/blosc2.c | 654 +++++-------------- src/blosc2/blosc/blosclz.c | 15 +- src/blosc2/blosc/blosclz.h | 19 +- src/blosc2/blosc/context.h | 43 +- src/blosc2/blosc/delta.c | 6 +- src/blosc2/blosc/delta.h | 5 +- src/blosc2/blosc/directories.c | 13 +- src/blosc2/blosc/fastcopy.c | 9 +- src/blosc2/blosc/fastcopy.h | 4 +- src/blosc2/blosc/frame.c | 98 +-- src/blosc2/blosc/frame.h | 8 +- src/blosc2/blosc/schunk.c | 91 ++- src/blosc2/blosc/sframe.c | 20 +- src/blosc2/blosc/sframe.h | 9 +- src/blosc2/blosc/shuffle-altivec.c | 9 +- src/blosc2/blosc/shuffle-altivec.h | 16 +- src/blosc2/blosc/shuffle-avx2.c | 6 +- src/blosc2/blosc/shuffle-avx2.h | 10 +- src/blosc2/blosc/shuffle-generic.c | 2 +- src/blosc2/blosc/shuffle-generic.h | 23 +- src/blosc2/blosc/shuffle-neon.c | 5 +- src/blosc2/blosc/shuffle-neon.h | 16 +- src/blosc2/blosc/shuffle-sse2.c | 5 +- src/blosc2/blosc/shuffle-sse2.h | 16 +- src/blosc2/blosc/shuffle.c | 38 +- src/blosc2/blosc/shuffle.h | 19 +- src/blosc2/blosc/stune.c | 42 +- src/blosc2/blosc/stune.h | 31 +- src/blosc2/blosc/timestamp.c | 19 +- src/blosc2/blosc/transpose-altivec.h | 15 +- src/blosc2/blosc/trunc-prec.c | 14 +- src/blosc2/blosc/trunc-prec.h | 5 +- src/blosc2/include/b2nd.h | 24 +- src/blosc2/include/blosc2.h | 199 +++--- src/blosc2/include/blosc2/blosc2-common.h | 32 +- src/blosc2/include/blosc2/blosc2-export.h | 12 +- src/blosc2/include/blosc2/blosc2-stdio.h | 40 +- src/blosc2/include/blosc2/codecs-registry.h | 27 +- src/blosc2/include/blosc2/filters-registry.h | 24 +- 58 files changed, 773 insertions(+), 1397 deletions(-) diff --git a/src/Makefile b/src/Makefile index dc799e2..728bd62 100644 --- a/src/Makefile +++ b/src/Makefile @@ -95,11 +95,7 @@ else INCLUDEDIRS+=-Iblosc2/internal-complibs/zstd LIBZLIB+=-Lblosc2/internal-complibs/zstd -lzstd endif - ifeq ($(findstring _NT-,$(PLATFORM)), _NT-) - LIBZLIB+=-Lblosc2/lib -lblosc2 -pthread - else - LIBZLIB+=-Lblosc2/lib -lblosc2 -pthread -ldl - endif + LIBZLIB+=-Lblosc2/lib -lblosc2 -pthread ifeq ($(HAVE_ZLIB),yes) LIBZLIB+=-lz endif diff --git a/src/blosc2/LICENSE.txt b/src/blosc2/LICENSE.txt index 076549b..22a66d1 100644 --- a/src/blosc2/LICENSE.txt +++ b/src/blosc2/LICENSE.txt @@ -3,7 +3,7 @@ BSD License For Blosc - A blocking, shuffling and lossless compression library Copyright (C) 2009-2018 Francesc Alted -Copyright (C) 2019- The Blosc Development Team +Copyright (C) 2019-present Blosc Development team Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/src/blosc2/README.rst b/src/blosc2/README.rst index 918ab03..b6a065c 100644 --- a/src/blosc2/README.rst +++ b/src/blosc2/README.rst @@ -1,10 +1,14 @@ -====================================================================== - C-Blosc2: A fast, compressed and persistent data store library for C -====================================================================== +======== +C-Blosc2 +======== + +A fast, compressed and persistent data store library for C +========================================================== + :Author: The Blosc Development Team :Contact: blosc@blosc.org -:URL: http://www.blosc.org +:URL: https://www.blosc.org :Gitter: |gitter| :Actions: |actions| :NumFOCUS: |numfocus| @@ -17,24 +21,44 @@ .. |actions| image:: https://github.com/Blosc/c-blosc2/workflows/CI%20CMake/badge.svg :target: https://github.com/Blosc/c-blosc2/actions?query=workflow%3A%22CI+CMake%22 -.. |appveyor| image:: https://ci.appveyor.com/api/projects/status/qiaxywqrouj6nkug/branch/master?svg=true - :target: https://ci.appveyor.com/project/FrancescAlted/c-blosc2/branch/master - .. |numfocus| image:: https://img.shields.io/badge/powered%20by-NumFOCUS-orange.svg?style=flat&colorA=E1523D&colorB=007D8A :target: https://numfocus.org .. |Contributor Covenant| image:: https://img.shields.io/badge/Contributor%20Covenant-v2.0%20adopted-ff69b4.svg - :target: code_of_conduct.md + :target: https://github.com/Blosc/community/blob/master/code_of_conduct.md What is it? =========== -`Blosc `_ is a high performance compressor optimized for binary data (i.e. floating point numbers, integers and booleans). It has been designed to transmit data to the processor cache faster than the traditional, non-compressed, direct memory fetch approach via a memcpy() OS call. Blosc main goal is not just to reduce the size of large datasets on-disk or in-memory, but also to accelerate memory-bound computations. +`Blosc `_ is a high performance compressor optimized for binary data (i.e. floating point numbers, integers and booleans, although it can handle string data too). It has been designed to transmit data to the processor cache faster than the traditional, non-compressed, direct memory fetch approach via a memcpy() OS call. Blosc main goal is not just to reduce the size of large datasets on-disk or in-memory, but also to accelerate memory-bound computations. + +C-Blosc2 is the new major version of `C-Blosc `_, and is backward compatible with both the C-Blosc1 API and its in-memory format. However, the reverse thing is generally not true for the format; buffers generated with C-Blosc2 are not format-compatible with C-Blosc1 (i.e. forward compatibility is not supported). In case you want to ensure full API compatibility with C-Blosc1 API, define the `BLOSC1_COMPAT` symbol. + +See a 3 minutes `introductory video to Blosc2 `_. + +Blosc2 NDim: an N-Dimensional store +=================================== + +One of the latest and more exciting additions in C-Blosc2 is the `Blosc2 NDim layer `_ (or b2nd for short), allowing to create *and* read n-dimensional datasets in an extremely efficient way thanks to a n-dim 2-level partitioning, that allows to slice and dice arbitrary large and compressed data in a more fine-grained way: + +.. image:: https://github.com/Blosc/c-blosc2/blob/main/images/b2nd-2level-parts.png?raw=true + :width: 75% + +To wet you appetite, here it is how the `NDArray` object in the `Python wrapper`_ performs on getting slices orthogonal to the different axis of a 4-dim dataset: -C-Blosc2 is the new major version of `C-Blosc `_, and tries hard to be backward compatible with both the C-Blosc1 API and its in-memory format. However, the reverse thing is generally not true for the format; buffers generated with C-Blosc2 are not format-compatible with C-Blosc1 (i.e. forward compatibility is not supported). In case you want to ensure full API compatibility with C-Blosc1 API, define the `BLOSC1_COMPAT` symbol. +.. image:: https://github.com/Blosc/c-blosc2/blob/main/images/Read-Partial-Slices-B2ND.png?raw=true + :width: 75% -See a 3 minutes `introductory video to Blosc2 `_. +We have blogged about this: https://www.blosc.org/posts/blosc2-ndim-intro + +We also have a ~2 min explanatory video on `why slicing in a pineapple-style (aka double partition) +is useful `_: + +.. image:: https://github.com/Blosc/blogsite/blob/master/files/images/slicing-pineapple-style.png?raw=true + :width: 50% + :alt: Slicing a dataset in pineapple-style + :target: https://www.youtube.com/watch?v=LvP9zxMGBng New features in C-Blosc2 @@ -42,6 +66,8 @@ New features in C-Blosc2 * **64-bit containers:** the first-class container in C-Blosc2 is the `super-chunk` or, for brevity, `schunk`, that is made by smaller chunks which are essentially C-Blosc1 32-bit containers. The super-chunk can be backed or not by another container which is called a `frame` (see later). +* **NDim containers (b2nd):** allow to store n-dimensional data that can efficiently read datasets in slices that can be n-dimensional too. To achieve this, a n-dimensional 2-level partitioning has been implemented. This capabilities were formerly part of `Caterva `_, and now it is included in C-Blosc2 for convenience. Caterva is now deprecated. + * **More filters:** besides `shuffle` and `bitshuffle` already present in C-Blosc1, C-Blosc2 already implements: - `delta`: the stored blocks inside a chunk are diff'ed with respect to first block in the chunk. The idea is that, in some situations, the diff will have more zeros than the original data, leading to better compression. @@ -68,13 +94,13 @@ New features in C-Blosc2 * **Parallel chunk reads:** when several blocks of a chunk are to be read, this is done in parallel by the decompressing machinery. That means that every thread is responsible to read, post-filter and decompress a block by itself, leading to an efficient overlap of I/O and CPU usage that optimizes reads to a maximum. -* **Meta-layers:** optionally, the user can add meta-data for different uses and in different layers. For example, one may think on providing a meta-layer for `NumPy `_ so that most of the meta-data for it is stored in a meta-layer; then, one can place another meta-layer on top of the latter for adding more high-level info if desired (e.g. geo-spatial, meteorological...). +* **Meta-layers:** optionally, the user can add meta-data for different uses and in different layers. For example, one may think on providing a meta-layer for `NumPy `_ so that most of the meta-data for it is stored in a meta-layer; then, one can place another meta-layer on top of the latter for adding more high-level info if desired (e.g. geo-spatial, meteorological...). * **Variable length meta-layers:** the user may want to add variable-length meta information that can be potentially very large (up to 2 GB). The regular meta-layer described above is very quick to read, but meant to store fixed-length and relatively small meta information. Variable length metalayers are stored in the trailer of a frame, whereas regular meta-layers are in the header. * **Efficient support for special values:** large sequences of repeated values can be represented with an efficient, simple and fast run-length representation, without the need to use regular codecs. With that, chunks or super-chunks with values that are the same (zeros, NaNs or any value in general) can be built in constant time, regardless of the size. This can be useful in situations where a lot of zeros (or NaNs) need to be stored (e.g. sparse matrices). -* **Nice markup for documentation:** we are currently using a combination of Sphinx + Doxygen + Breathe for documenting the C-API. See https://c-blosc2.readthedocs.io. Thanks to Alberto Sabater and Aleix Alcacer for contributing the support for this. +* **Nice markup for documentation:** we are currently using a combination of Sphinx + Doxygen + Breathe for documenting the C-API. See https://www.blosc.org/c-blosc2/c-blosc2.html. Thanks to Alberto Sabater and Aleix Alcacer for contributing the support for this. * **Plugin capabilities for filters and codecs:** we have a plugin register capability inplace so that the info about the new filters and codecs can be persisted and transmitted to different machines. See https://github.com/Blosc/c-blosc2/blob/main/examples/urfilters.c for a self-contained example. Thanks to the NumFOCUS foundation for providing a grant for doing this, and Oscar Griñón and Aleix Alcacer for the implementation. @@ -88,34 +114,13 @@ New features in C-Blosc2 More info about the `improved capabilities of C-Blosc2 can be found in this talk `_. -After a long period of testing, C-Blosc2 entered production stage in 2.0.0. The API and format have been frozen, and that means that there is guarantee that your programs will continue to work with future versions of the library, and that next releases will be able to read from persistent storage generated from previous releases (as of 2.0.0). - - -Meta-compression and other advantages over existing compressors -=============================================================== - -C-Blosc2 is not like other compressors: it should rather be called a meta-compressor. This is so because it can use different codecs (libraries that can reduce the size of inputs) and filters (libraries that generally improve compression ratio). At the same time, it can also be called a compressor because it makes an actual use of the several codecs and filters, so it can actually work like so. - -Currently C-Blosc2 comes with support of BloscLZ, a compressor heavily based on `FastLZ `_, `LZ4 and LZ4HC `_, `Zstd `_, and `Zlib, via zlib-ng: `_, as well as a highly optimized (it can use SSE2, AVX2, NEON or ALTIVEC instructions, if available) shuffle and bitshuffle filters (for info on how shuffling works, see slide 17 of http://www.slideshare.net/PyData/blosc-py-data-2014). - -Blosc is in charge of coordinating the codecs and filters so that they can leverage the blocking technique (described above) as -well as multi-threaded execution (if several cores are available) automatically. That makes that every codec and filter -will work at very high speeds, even if it was not initially designed for doing blocking or multi-threading. For example, -Blosc allows you to use the ``LZ4`` codec, but in a multi-threaded way. - -Last but not least, C-Blosc2 comes with an easy-to-use plugin mechanism for codecs and filters, so anyone can inject their own code in the compression pipeline of Blosc2 and reap its benefits (like multi-threading and integration with other filters) for free (see a `self-contained example `_). In addition, we have implemented a centralized plugin system too (see the `docs in the plugins directory `_). - - -Multidimensional containers -=========================== - -As said, C-Blosc2 adds a powerful mechanism for adding different metalayers on top of its containers. `Caterva `_ is a sibling library that adds such a metalayer specifying not only the dimensionality of a dataset, but also the dimensionality of the chunks inside the dataset. In addition, Caterva adds machinery for retrieving arbitrary multi-dimensional slices (aka hyper-slices) out of the multi-dimensional containers in the most efficient way. Hence, Caterva brings the convenience of multi-dimensional containers to your application very easily. For more info, check out the `Caterva documentation `_. +C-Blosc2 API and format have been frozen, and that means that there is guarantee that your programs will continue to work with future versions of the library, and that next releases will be able to read from persistent storage generated from previous releases (as of 2.0.0). Python wrapper ============== -We are officially supporting (thanks to the Python Software Foundation) a `Python wrapper for Blosc2 `_. Although this is still in early development, it already supports all the features of the venerable `python-blosc ` package. As a bonus, the `python-blosc2` package comes with wheels and binary versions of the C-Blosc2 libraries, so anyone, even non-Python users can install C-Blosc2 binaries easily with: +We are officially supporting (thanks to the Python Software Foundation) a `Python wrapper for Blosc2 `_. It supports all the features of the predecessor `python-blosc `_ package plus most of the bells and whistles of C-Blosc2, like 64-bit and multidimensional containers. As a bonus, the `python-blosc2` package comes with wheels and binary versions of the C-Blosc2 libraries, so anyone, even non-Python users can install C-Blosc2 binaries easily with: .. code-block:: console @@ -125,7 +130,7 @@ We are officially supporting (thanks to the Python Software Foundation) a `Pytho Compiling the C-Blosc2 library with CMake ========================================= -Blosc can be built, tested and installed using `CMake `_. The following procedure describes a typical CMake build. +Blosc can be built, tested and installed using `CMake `_. The following procedure describes a typical CMake build. Create the build directory inside the sources and move into it: @@ -166,59 +171,41 @@ Once you have compiled your Blosc library, you can easily link your apps with it Handling support for codecs (LZ4, LZ4HC, Zstd, Zlib) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -C-Blosc2 comes with full sources for LZ4, LZ4HC, Zstd, and Zlib and in general, you should not worry about not having (or CMake not finding) the libraries in your system because by default the included sources will be automatically compiled and included in the C-Blosc2 library. This means that you can be confident in having a complete support for all the codecs in all the Blosc deployments (unless you are explicitly excluding support for some of them). +C-Blosc2 comes with full sources for LZ4, LZ4HC, Zstd, and Zlib and in general, you should not worry about not having (or CMake not finding) the libraries in your system because by default the included sources will be automatically compiled and included in the C-Blosc2 library. This means that you can be confident in having a complete support for all these codecs in all the official Blosc deployments. Of course, if you consider this is too bloated, you can exclude support for some of them. -If you want to force Blosc to use external libraries instead of the included compression sources: +For example, let's suppose that you want to disable support for Zstd: .. code-block:: console - cmake -DPREFER_EXTERNAL_LZ4=ON .. + cmake -DDEACTIVATE_ZSTD=ON .. -You can also disable support for some compression libraries: +Or, you may want to use a codec in an external library already in the system: .. code-block:: console - cmake -DDEACTIVATE_ZSTD=ON .. + cmake -DPREFER_EXTERNAL_LZ4=ON .. Supported platforms ~~~~~~~~~~~~~~~~~~~ -C-Blosc2 is meant to support all platforms where a C99 compliant C compiler can be found. The ones that are mostly tested are Intel (Linux, Mac OSX and Windows), ARM (Linux, Mac), and PowerPC (Linux) but exotic ones as IBM Blue Gene Q embedded "A2" processor are reported to work too. More on ARM support in `README_ARM.rst`. +C-Blosc2 is meant to support all platforms where a C99 compliant C compiler can be found. The ones that are mostly tested are Intel (Linux, Mac OSX and Windows), ARM (Linux, Mac), and PowerPC (Linux). More on ARM support in `README_ARM.rst`. For Windows, you will need at least VS2015 or higher on x86 and x64 targets (i.e. ARM is not supported on Windows). -For Mac OSX, make sure that you have installed the command line developer tools. You can always install them with: +For Mac OSX, make sure that you have the command line developer tools available. You can always install them with: .. code-block:: console xcode-select --install -For Mac OSX on arm64 architecture, you need to compile like this: +For Mac OSX on arm64 architecture, you may want to compile it like this: .. code-block:: console CC="clang -arch arm64" cmake .. -Support for the LZ4 optimized version in Intel IPP -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -C-Blosc2 comes with support for a highly optimized version of the LZ4 codec present in Intel IPP. Here it is a way to easily install Intel IPP using Conda(https://docs.conda.io): - -.. code-block:: console - - conda install -c intel ipp-static - -With that, you can enable support for LZ4/IPP (it is disabled by default) with: - -.. code-block:: console - - cmake .. -DDEACTIVATE_IPP=OFF - -In some Intel CPUs LZ4/IPP could be faster than regular LZ4, although in many cases you may experience different compression ratios depending on which version you use. See #313 for some quick and dirty benchmarks. - - Display error messages ~~~~~~~~~~~~~~~~~~~~~~ diff --git a/src/blosc2/blosc/Makefile b/src/blosc2/blosc/Makefile index 1d978ff..9ae0d7c 100644 --- a/src/blosc2/blosc/Makefile +++ b/src/blosc2/blosc/Makefile @@ -37,7 +37,7 @@ PLATFORM = $(shell uname -s) DLLFLAG=-fPIC -CPPOPT=-g -Wall -Wextra -O3 -msse2 -DNDEBUG $(DLLFLAG) -std=gnu99 #-g -Wall -std=c99 # -DUSE_OS_TIMER +CPPOPT=-g -Wall -Wextra -O3 -msse2 -DNDEBUG $(DLLFLAG) -std=gnu99 -Wno-unused-variable #-g -Wall -std=c99 # -DUSE_OS_TIMER ifeq ($(HAVE_ZLIB),yes) CPPOPT+=-DHAVE_ZLIB diff --git a/src/blosc2/blosc/b2nd.c b/src/blosc2/blosc/b2nd.c index 4c0ff70..ae85a19 100644 --- a/src/blosc2/blosc/b2nd.c +++ b/src/blosc2/blosc/b2nd.c @@ -1,23 +1,19 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ -#include "b2nd.h" -#include "b2nd_utils.h" +#include #include "context.h" -#include "blosc2/blosc2-common.h" +#include "b2nd_utils.h" #include "blosc2.h" - +#include "blosc2/blosc2-common.h" #include -#include -#include -#include int b2nd_serialize_meta(int8_t ndim, const int64_t *shape, const int32_t *chunkshape, @@ -176,8 +172,8 @@ int update_shape(b2nd_array_t *array, int8_t ndim, const int64_t *shape, b2nd_serialize_meta(array->ndim, array->shape, array->chunkshape, array->blockshape, array->dtype, array->dtype_format, &smeta); if (smeta_len < 0) { - BLOSC_TRACE_ERROR("Error during serializing dims info for Blosc2 NDim"); - BLOSC_ERROR(BLOSC2_ERROR_FAILURE); + fprintf(stderr, "error during serializing dims info for Blosc2 NDim"); + return -1; } // ... and update it in its metalayer if (blosc2_meta_exists(array->sc, "b2nd") < 0) { @@ -269,10 +265,6 @@ int array_new(b2nd_context_t *ctx, int special_value, b2nd_array_t **array) { } } - if ((*array)->extchunknitems * sc->typesize > BLOSC2_MAX_BUFFERSIZE){ - BLOSC_TRACE_ERROR("Chunksize exceeds maximum of %d", BLOSC2_MAX_BUFFERSIZE); - return BLOSC2_ERROR_MAX_BUFSIZE_EXCEEDED; - } // Fill schunk with uninit values if ((*array)->nitems != 0) { int32_t chunksize = (int32_t) (*array)->extchunknitems * sc->typesize; @@ -603,7 +595,7 @@ int get_set_slice(void *buffer, int64_t buffersize, const int64_t *start, const int64_t nchunk; blosc2_multidim_to_unidim(nchunk_ndim, ndim, chunks_in_array_strides, &nchunk); - // Check if the chunk needs to be updated + // check if the chunk needs to be updated int64_t chunk_start[B2ND_MAX_DIM] = {0}; int64_t chunk_stop[B2ND_MAX_DIM] = {0}; for (int i = 0; i < ndim; ++i) { @@ -648,7 +640,7 @@ int get_set_slice(void *buffer, int64_t buffersize, const int64_t *start, const int64_t nblock_ndim[B2ND_MAX_DIM] = {0}; blosc2_unidim_to_multidim(ndim, blocks_in_chunk, nblock, nblock_ndim); - // Check if the block needs to be updated + // check if the block needs to be updated int64_t block_start[B2ND_MAX_DIM] = {0}; int64_t block_stop[B2ND_MAX_DIM] = {0}; for (int i = 0; i < ndim; ++i) { @@ -692,7 +684,7 @@ int get_set_slice(void *buffer, int64_t buffersize, const int64_t *start, const int64_t nblock_ndim[B2ND_MAX_DIM] = {0}; blosc2_unidim_to_multidim(ndim, blocks_in_chunk, nblock, nblock_ndim); - // Check if the block needs to be updated + // check if the block needs to be updated int64_t block_start[B2ND_MAX_DIM] = {0}; int64_t block_stop[B2ND_MAX_DIM] = {0}; for (int i = 0; i < ndim; ++i) { @@ -891,7 +883,7 @@ int b2nd_get_slice(b2nd_context_t *ctx, b2nd_array_t **array, const b2nd_array_t int64_t nchunk_ndim[B2ND_MAX_DIM] = {0}; blosc2_unidim_to_multidim(ndim, chunks_in_array, nchunk, nchunk_ndim); - // Check if the chunk needs to be updated + // check if the chunk needs to be updated int64_t chunk_start[B2ND_MAX_DIM] = {0}; int64_t chunk_stop[B2ND_MAX_DIM] = {0}; int64_t chunk_shape[B2ND_MAX_DIM] = {0}; @@ -1100,7 +1092,7 @@ int b2nd_print_meta(const b2nd_array_t *array) { &dtype, &dtype_format)); free(smeta); - printf("b2nd metalayer parameters:\n Ndim: %d", ndim); + printf("b2nd metalayer parameters: \n Ndim: %d", ndim); printf("\n shape: %" PRId64 "", shape[0]); for (int i = 1; i < ndim; ++i) { printf(", %" PRId64 "", shape[i]); @@ -1824,20 +1816,6 @@ b2nd_create_ctx(const blosc2_storage *b2_storage, int8_t ndim, const int64_t *sh ctx->metalayers[i] = metalayers[i]; } -#if defined(HAVE_PLUGINS) - #include "blosc2/codecs-registry.h" - if ((ctx->b2_storage->cparams->compcode >= BLOSC_CODEC_ZFP_FIXED_ACCURACY) && - (ctx->b2_storage->cparams->compcode <= BLOSC_CODEC_ZFP_FIXED_RATE)) { - for (int i = 0; i < BLOSC2_MAX_FILTERS; ++i) { - if ((ctx->b2_storage->cparams->filters[i] == BLOSC_SHUFFLE) || - (ctx->b2_storage->cparams->filters[i] == BLOSC_BITSHUFFLE)) { - BLOSC_TRACE_ERROR("ZFP cannot be run in presence of SHUFFLE / BITSHUFFLE"); - return NULL; - } - } - } -#endif /* HAVE_PLUGINS */ - return ctx; } diff --git a/src/blosc2/blosc/b2nd_utils.c b/src/blosc2/blosc/b2nd_utils.c index bcd5e14..e6dbf82 100644 --- a/src/blosc2/blosc/b2nd_utils.c +++ b/src/blosc2/blosc/b2nd_utils.c @@ -1,18 +1,14 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ -#include "b2nd_utils.h" -#include "b2nd.h" - -#include -#include +#include // copyNdim where N = {2-8} - specializations of copy loops to be used by b2nd_copy_buffer // since we don't have c++ templates, substitute manual specializations for up to known B2ND_MAX_DIM (8) diff --git a/src/blosc2/blosc/b2nd_utils.h b/src/blosc2/blosc/b2nd_utils.h index 38fba67..f846623 100644 --- a/src/blosc2/blosc/b2nd_utils.h +++ b/src/blosc2/blosc/b2nd_utils.h @@ -1,17 +1,23 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ -#ifndef BLOSC_B2ND_UTILS_H -#define BLOSC_B2ND_UTILS_H +#ifndef B2ND_B2ND_UTILS_H_ +#define B2ND_B2ND_UTILS_H_ + +#include +#include <../plugins/plugin_utils.h> + +#ifdef __cplusplus +extern "C" { +#endif -#include int b2nd_copy_buffer(int8_t ndim, uint8_t itemsize, @@ -20,4 +26,8 @@ int b2nd_copy_buffer(int8_t ndim, void *dst, const int64_t *dst_pad_shape, int64_t *dst_start); -#endif /* BLOSC_B2ND_UTILS_H */ +#ifdef __cplusplus +} +#endif + +#endif // B2ND_B2ND_UTILS_H_ diff --git a/src/blosc2/blosc/bitshuffle-altivec.c b/src/blosc2/blosc/bitshuffle-altivec.c index 884bf90..4c7e3b2 100644 --- a/src/blosc2/blosc/bitshuffle-altivec.c +++ b/src/blosc2/blosc/bitshuffle-altivec.c @@ -1,7 +1,7 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) @@ -22,17 +22,14 @@ **********************************************************************/ -#include "bitshuffle-altivec.h" #include "bitshuffle-generic.h" +#include "bitshuffle-altivec.h" /* Make sure ALTIVEC is available for the compilation target and compiler. */ #if defined(__ALTIVEC__) -#include "transpose-altivec.h" - #include - -#include +#include "transpose-altivec.h" /* The next is useful for debugging purposes */ #if 0 diff --git a/src/blosc2/blosc/bitshuffle-altivec.h b/src/blosc2/blosc/bitshuffle-altivec.h index 5995aa7..7416c7a 100644 --- a/src/blosc2/blosc/bitshuffle-altivec.h +++ b/src/blosc2/blosc/bitshuffle-altivec.h @@ -1,7 +1,7 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) @@ -10,13 +10,15 @@ /* ALTIVEC-accelerated shuffle/unshuffle routines. */ -#ifndef BLOSC_BITSHUFFLE_ALTIVEC_H -#define BLOSC_BITSHUFFLE_ALTIVEC_H +#ifndef BITSHUFFLE_ALTIVEC_H +#define BITSHUFFLE_ALTIVEC_H #include "blosc2/blosc2-common.h" -#include -#include +#ifdef __cplusplus +extern "C" { +#endif + BLOSC_NO_EXPORT int64_t bshuf_trans_byte_elem_altivec(void* in, void* out, const size_t size, @@ -44,4 +46,9 @@ BLOSC_NO_EXPORT int64_t bshuf_untrans_bit_elem_altivec(void* in, void* out, const size_t size, const size_t elem_size, void* tmp_buf); -#endif /* BLOSC_BITSHUFFLE_ALTIVEC_H */ +#ifdef __cplusplus +} +#endif + + +#endif /* BITSHUFFLE_ALTIVEC_H */ diff --git a/src/blosc2/blosc/bitshuffle-avx2.c b/src/blosc2/blosc/bitshuffle-avx2.c index a855bf8..ee007fa 100644 --- a/src/blosc2/blosc/bitshuffle-avx2.c +++ b/src/blosc2/blosc/bitshuffle-avx2.c @@ -1,7 +1,7 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) @@ -20,17 +20,17 @@ rights to use. **********************************************************************/ -#include "bitshuffle-avx2.h" -#include "bitshuffle-sse2.h" + #include "bitshuffle-generic.h" +#include "bitshuffle-sse2.h" +#include "bitshuffle-avx2.h" + /* Make sure AVX2 is available for the compilation target and compiler. */ #if defined(__AVX2__) #include -#include - /* The next is useful for debugging purposes */ #if 0 #include diff --git a/src/blosc2/blosc/bitshuffle-avx2.h b/src/blosc2/blosc/bitshuffle-avx2.h index 2b40bd2..5d9d51b 100644 --- a/src/blosc2/blosc/bitshuffle-avx2.h +++ b/src/blosc2/blosc/bitshuffle-avx2.h @@ -1,7 +1,7 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) @@ -10,13 +10,14 @@ /* AVX2-accelerated shuffle/unshuffle routines. */ -#ifndef BLOSC_BITSHUFFLE_AVX2_H -#define BLOSC_BITSHUFFLE_AVX2_H +#ifndef BITSHUFFLE_AVX2_H +#define BITSHUFFLE_AVX2_H -#include "blosc2/blosc2-common.h" +#include -#include -#include +#ifdef __cplusplus +extern "C" { +#endif /** AVX2-accelerated bitshuffle routine. @@ -32,4 +33,8 @@ BLOSC_NO_EXPORT int64_t bshuf_untrans_bit_elem_avx2(void* in, void* out, const size_t size, const size_t elem_size, void* tmp_buf); -#endif /* BLOSC_BITSHUFFLE_AVX2_H */ +#ifdef __cplusplus +} +#endif + +#endif /* BITSHUFFLE_AVX2_H */ diff --git a/src/blosc2/blosc/bitshuffle-generic.c b/src/blosc2/blosc/bitshuffle-generic.c index 13748ef..1840045 100644 --- a/src/blosc2/blosc/bitshuffle-generic.c +++ b/src/blosc2/blosc/bitshuffle-generic.c @@ -1,7 +1,7 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) @@ -10,8 +10,6 @@ #include "bitshuffle-generic.h" -#include - #ifdef _MSC_VER #pragma warning (push) #pragma warning (disable: 4146) diff --git a/src/blosc2/blosc/bitshuffle-generic.h b/src/blosc2/blosc/bitshuffle-generic.h index 843b66a..49b1be3 100644 --- a/src/blosc2/blosc/bitshuffle-generic.h +++ b/src/blosc2/blosc/bitshuffle-generic.h @@ -1,46 +1,35 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ -/* - * Bitshuffle - Filter for improving compression of typed binary data. - * - * Author: Kiyoshi Masui - * Website: http://www.github.com/kiyo-masui/bitshuffle - * Created: 2014 - * - */ - /* Generic (non-hardware-accelerated) shuffle/unshuffle routines. These are used when hardware-accelerated functions aren't available for a particular platform; they are also used by the hardware- accelerated functions to handle any remaining elements in a block which isn't a multiple of the hardware's vector size. */ -#ifndef BLOSC_BITSHUFFLE_GENERIC_H -#define BLOSC_BITSHUFFLE_GENERIC_H +#ifndef BITSHUFFLE_GENERIC_H +#define BITSHUFFLE_GENERIC_H -#include "blosc2/blosc2-common.h" +#include +#include -#include -#include +#ifdef __cplusplus +extern "C" { +#endif -// Macros. -#define CHECK_MULT_EIGHT(n) if (n % 8) return -80; -#define MAX(X,Y) ((X) > (Y) ? (X) : (Y)) +/* Macros. */ +#define CHECK_MULT_EIGHT(n) if ((n) % 8) return -80; #define MIN(X,Y) ((X) < (Y) ? (X) : (Y)) -#define CHECK_ERR(count) \ - do { \ - if ((count) < 0) \ - return count; \ - } while (0) +#define MAX(X,Y) ((X) > (Y) ? (X) : (Y)) +#define CHECK_ERR(count) if ((count) < 0) { return count; } /* ---- Worker code not requiring special instruction sets. ---- @@ -53,41 +42,41 @@ /* Transpose 8x8 bit array packed into a single quadword *x*. * *t* is workspace. */ #define TRANS_BIT_8X8(x, t) { \ - t = (x ^ (x >> 7)) & 0x00AA00AA00AA00AALL; \ - x = x ^ t ^ (t << 7); \ - t = (x ^ (x >> 14)) & 0x0000CCCC0000CCCCLL; \ - x = x ^ t ^ (t << 14); \ - t = (x ^ (x >> 28)) & 0x00000000F0F0F0F0LL; \ - x = x ^ t ^ (t << 28); \ + (t) = ((x) ^ ((x) >> 7)) & 0x00AA00AA00AA00AALL; \ + (x) = (x) ^ (t) ^ ((t) << 7); \ + (t) = ((x) ^ ((x) >> 14)) & 0x0000CCCC0000CCCCLL; \ + (x) = (x) ^ (t) ^ ((t) << 14); \ + (t) = ((x) ^ ((x) >> 28)) & 0x00000000F0F0F0F0LL; \ + (x) = (x) ^ (t) ^ ((t) << 28); \ } /* Transpose 8x8 bit array along the diagonal from upper right to lower left */ #define TRANS_BIT_8X8_BE(x, t) { \ - t = (x ^ (x >> 9)) & 0x0055005500550055LL; \ - x = x ^ t ^ (t << 9); \ - t = (x ^ (x >> 18)) & 0x0000333300003333LL; \ - x = x ^ t ^ (t << 18); \ - t = (x ^ (x >> 36)) & 0x000000000F0F0F0FLL; \ - x = x ^ t ^ (t << 36); \ + (t) = ((x) ^ ((x) >> 9)) & 0x0055005500550055LL; \ + (x) = (x) ^ (t) ^ ((t) << 9); \ + (t) = ((x) ^ ((x) >> 18)) & 0x0000333300003333LL; \ + (x) = (x) ^ (t) ^ ((t) << 18); \ + (t) = ((x) ^ ((x) >> 36)) & 0x000000000F0F0F0FLL; \ + (x) = (x) ^ (t) ^ ((t) << 36); \ } /* Transpose of an array of arbitrarily typed elements. */ #define TRANS_ELEM_TYPE(in, out, lda, ldb, type_t) { \ + type_t* in_type = (type_t*) (in); \ + type_t* out_type = (type_t*) (out); \ size_t ii, jj, kk; \ - const type_t* in_type = (const type_t*) in; \ - type_t* out_type = (type_t*) out; \ - for(ii = 0; ii + 7 < lda; ii += 8) { \ - for(jj = 0; jj < ldb; jj++) { \ - for(kk = 0; kk < 8; kk++) { \ - out_type[jj*lda + ii + kk] = \ - in_type[ii*ldb + kk * ldb + jj]; \ + for (ii = 0; ii + 7 < (lda); ii += 8) { \ + for (jj = 0; jj < (ldb); jj++) { \ + for (kk = 0; kk < 8; kk++) { \ + out_type[jj*(lda) + ii + kk] = \ + in_type[ii*(ldb) + kk * (ldb) + jj]; \ } \ } \ } \ - for(ii = lda - lda % 8; ii < lda; ii ++) { \ - for(jj = 0; jj < ldb; jj++) { \ - out_type[jj*lda + ii] = in_type[ii*ldb + jj]; \ + for (ii = (lda) - (lda) % 8; ii < (lda); ii ++) { \ + for (jj = 0; jj < (ldb); jj++) { \ + out_type[jj*(lda) + ii] = in_type[ii*(ldb) + jj]; \ } \ } \ } @@ -169,4 +158,9 @@ BLOSC_NO_EXPORT int64_t bshuf_untrans_bit_elem_scal(const void* in, void* out, const size_t size, const size_t elem_size, void* tmp_buf); -#endif /* BLOSC_BITSHUFFLE_GENERIC_H */ + +#ifdef __cplusplus +} +#endif + +#endif /* BITSHUFFLE_GENERIC_H */ diff --git a/src/blosc2/blosc/bitshuffle-neon.c b/src/blosc2/blosc/bitshuffle-neon.c index 8b1b594..92477aa 100644 --- a/src/blosc2/blosc/bitshuffle-neon.c +++ b/src/blosc2/blosc/bitshuffle-neon.c @@ -1,23 +1,21 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ -#include "bitshuffle-neon.h" #include "bitshuffle-generic.h" +#include "bitshuffle-neon.h" /* Make sure NEON is available for the compilation target and compiler. */ #if defined(__ARM_NEON) #include -#include - /* The next is useful for debugging purposes */ #if 0 #include diff --git a/src/blosc2/blosc/bitshuffle-neon.h b/src/blosc2/blosc/bitshuffle-neon.h index 5d9f04a..4b08e99 100644 --- a/src/blosc2/blosc/bitshuffle-neon.h +++ b/src/blosc2/blosc/bitshuffle-neon.h @@ -1,7 +1,7 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) @@ -12,13 +12,14 @@ /* NEON-accelerated bitshuffle/bitunshuffle routines. */ -#ifndef BLOSC_BITSHUFFLE_NEON_H -#define BLOSC_BITSHUFFLE_NEON_H +#ifndef BITSHUFFLE_NEON_H +#define BITSHUFFLE_NEON_H #include "blosc2/blosc2-common.h" -#include -#include +#ifdef __cplusplus +extern "C" { +#endif /** NEON-accelerated bitshuffle routine. @@ -32,4 +33,8 @@ BLOSC_NO_EXPORT int64_t bitshuffle_neon(void* _src, void* _dest, const size_t bl BLOSC_NO_EXPORT int64_t bitunshuffle_neon(void* _src, void* _dest, const size_t blocksize, const size_t bytesoftype, void* tmp_buf); -#endif /* BLOSC_BITSHUFFLE_NEON_H */ +#ifdef __cplusplus +} +#endif + +#endif /* BITSHUFFLE_NEON_H */ diff --git a/src/blosc2/blosc/bitshuffle-sse2.c b/src/blosc2/blosc/bitshuffle-sse2.c index ed5769d..1da84cd 100644 --- a/src/blosc2/blosc/bitshuffle-sse2.c +++ b/src/blosc2/blosc/bitshuffle-sse2.c @@ -1,7 +1,7 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) @@ -21,16 +21,14 @@ **********************************************************************/ -#include "bitshuffle-sse2.h" #include "bitshuffle-generic.h" +#include "bitshuffle-sse2.h" /* Make sure SSE2 is available for the compilation target and compiler. */ #if defined(__SSE2__) #include -#include - /* The next is useful for debugging purposes */ #if 0 #include diff --git a/src/blosc2/blosc/bitshuffle-sse2.h b/src/blosc2/blosc/bitshuffle-sse2.h index 2d31789..7e90798 100644 --- a/src/blosc2/blosc/bitshuffle-sse2.h +++ b/src/blosc2/blosc/bitshuffle-sse2.h @@ -1,7 +1,7 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) @@ -10,13 +10,15 @@ /* SSE2-accelerated shuffle/unshuffle routines. */ -#ifndef BLOSC_BITSHUFFLE_SSE2_H -#define BLOSC_BITSHUFFLE_SSE2_H +#ifndef BITSHUFFLE_SSE2_H +#define BITSHUFFLE_SSE2_H #include "blosc2/blosc2-common.h" -#include -#include +#ifdef __cplusplus +extern "C" { +#endif + BLOSC_NO_EXPORT int64_t bshuf_trans_byte_elem_sse2(void* in, void* out, const size_t size, @@ -44,4 +46,9 @@ BLOSC_NO_EXPORT int64_t bshuf_untrans_bit_elem_sse2(void* in, void* out, const size_t size, const size_t elem_size, void* tmp_buf); -#endif /* BLOSC_BITSHUFFLE_SSE2_H */ +#ifdef __cplusplus +} +#endif + + +#endif /* BITSHUFFLE_SSE2_H */ diff --git a/src/blosc2/blosc/blosc-private.h b/src/blosc2/blosc/blosc-private.h index 306825b..cfdb17e 100644 --- a/src/blosc2/blosc/blosc-private.h +++ b/src/blosc2/blosc/blosc-private.h @@ -1,7 +1,7 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) @@ -9,15 +9,16 @@ **********************************************************************/ -#ifndef BLOSC_BLOSC_PRIVATE_H -#define BLOSC_BLOSC_PRIVATE_H +#ifndef IARRAY_BLOSC_PRIVATE_H +#define IARRAY_BLOSC_PRIVATE_H -#include "blosc2/blosc2-common.h" -#include "blosc2.h" +#ifdef __cplusplus +extern "C" { +#endif -#include -#include -#include +#include "stdbool.h" +#include "blosc2.h" +#include "blosc2/blosc2-common.h" /********************************************************************* @@ -166,112 +167,8 @@ int register_filter_private(blosc2_filter *filter); */ int register_codec_private(blosc2_codec *codec); - -/** - * @brief Register a tune in Blosc. - * - * @param tune The tune to register. - * - * @return 0 if succeeds. Else a negative code is returned. - */ -int register_tuner_private(blosc2_tuner *tuner); - -int fill_tuner(blosc2_tuner *tuner); - -extern blosc2_tuner g_tuners[256]; -extern int g_ntuners; - - -#if defined(_WIN32) -#include -#ifndef PATH_MAX -#define PATH_MAX MAX_PATH -#endif -#define RTLD_LAZY 0x000 -#define popen _popen -#define pclose _pclose - -static struct { - long lasterror; - const char *err_rutin; -} -var = { - 0, - NULL -}; - -static inline void *dlopen (const char *filename, int flags) { - BLOSC_UNUSED_PARAM(flags); - HINSTANCE hInst; - hInst = LoadLibrary(filename); - if (hInst==NULL) { - var.lasterror = GetLastError(); - var.err_rutin = "dlopen"; - } - - return hInst; -} - -static inline void *dlsym(void *handle, const char *name) { - FARPROC fp; - fp = GetProcAddress((HINSTANCE)handle, name); - if (!fp) { - var.lasterror = GetLastError (); - var.err_rutin = "dlsym"; - } - return (void *)(intptr_t)fp; +#ifdef __cplusplus } - -static inline int dlclose(void *handle) { - bool ok = FreeLibrary((HINSTANCE)handle); - if (!ok) { - var.lasterror = GetLastError(); - var.err_rutin = "dlclose"; - return BLOSC2_ERROR_FAILURE; - } - return BLOSC2_ERROR_SUCCESS; -} - -static inline const char *dlerror (void) { - static char errstr [88]; - if (var.lasterror) { - sprintf (errstr, "%s error #%ld", var.err_rutin, var.lasterror); - return errstr; - } else { - return NULL; - } -} -#else -#include #endif - -static inline void* load_lib(char *plugin_name, char *libpath) { - char python_cmd[PATH_MAX] = {0}; - sprintf(python_cmd, "python -c \"import blosc2_%s; blosc2_%s.print_libpath()\"", plugin_name, plugin_name); - FILE *fp = popen(python_cmd, "r"); - if (fp == NULL) { - BLOSC_TRACE_ERROR("Could not run python"); - return NULL; - } - if (fgets(libpath, PATH_MAX, fp) == NULL) { - BLOSC_TRACE_ERROR("Could not read python output"); - pclose(fp); - return NULL; - } - pclose(fp); - if (strlen(libpath) == 0) { - BLOSC_TRACE_ERROR("Could not find plugin libpath"); - return NULL; - } - void* loaded_lib; - BLOSC_TRACE_INFO("libpath for plugin blosc2_%s: %s\n", plugin_name, libpath); - loaded_lib = dlopen(libpath, RTLD_LAZY); - if (loaded_lib == NULL) { - BLOSC_TRACE_ERROR("Attempt to load plugin in path '%s' failed with error: %s", - libpath, dlerror()); - } - return loaded_lib; -} - -#endif /* BLOSC_BLOSC_PRIVATE_H */ +#endif //IARRAY_BLOSC_PRIVATE_H diff --git a/src/blosc2/blosc/blosc2-stdio.c b/src/blosc2/blosc/blosc2-stdio.c index 912c02d..23f34af 100644 --- a/src/blosc2/blosc/blosc2-stdio.c +++ b/src/blosc2/blosc/blosc2-stdio.c @@ -1,7 +1,7 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) @@ -12,10 +12,6 @@ #include "blosc2/blosc2-stdio.h" #include "blosc2.h" -#include -#include -#include - void *blosc2_stdio_open(const char *urlpath, const char *mode, void *params) { BLOSC_UNUSED_PARAM(params); FILE *file = fopen(urlpath, mode); @@ -36,7 +32,7 @@ int blosc2_stdio_close(void *stream) { int64_t blosc2_stdio_tell(void *stream) { blosc2_stdio_file *my_fp = (blosc2_stdio_file *) stream; int64_t pos; -#if defined(_MSC_VER) +#if defined(_MSC_VER) && (_MSC_VER >= 1400) pos = _ftelli64(my_fp->file); #else pos = (int64_t)ftell(my_fp->file); @@ -47,7 +43,7 @@ int64_t blosc2_stdio_tell(void *stream) { int blosc2_stdio_seek(void *stream, int64_t offset, int whence) { blosc2_stdio_file *my_fp = (blosc2_stdio_file *) stream; int rc; -#if defined(_MSC_VER) +#if defined(_MSC_VER) && (_MSC_VER >= 1400) rc = _fseeki64(my_fp->file, offset, whence); #else rc = fseek(my_fp->file, (long) offset, whence); @@ -71,7 +67,7 @@ int64_t blosc2_stdio_read(void *ptr, int64_t size, int64_t nitems, void *stream) int blosc2_stdio_truncate(void *stream, int64_t size) { blosc2_stdio_file *my_fp = (blosc2_stdio_file *) stream; int rc; -#if defined(_MSC_VER) +#if defined(_MSC_VER) && (_MSC_VER >= 1400) rc = _chsize_s(_fileno(my_fp->file), size); #else rc = ftruncate(fileno(my_fp->file), size); diff --git a/src/blosc2/blosc/blosc2.c b/src/blosc2/blosc/blosc2.c index 336f3ab..08751f9 100644 --- a/src/blosc2/blosc/blosc2.c +++ b/src/blosc2/blosc/blosc2.c @@ -1,7 +1,7 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) @@ -9,11 +9,20 @@ **********************************************************************/ +#include +#include +#include +#include +#include +#include +#include + #include "blosc2.h" #include "blosc-private.h" #include "../plugins/codecs/zfp/blosc2-zfp.h" #include "frame.h" + #if defined(USING_CMAKE) #include "config.h" #endif /* USING_CMAKE */ @@ -26,7 +35,6 @@ #include "stune.h" #include "blosc2/codecs-registry.h" #include "blosc2/filters-registry.h" -#include "blosc2/tuners-registry.h" #include "lz4.h" #include "lz4hc.h" @@ -50,6 +58,7 @@ #include "zdict.h" #endif /* HAVE_ZSTD */ + #if defined(_WIN32) && !defined(__MINGW32__) #include #include @@ -61,15 +70,6 @@ #include "win32/pthread.c" #endif -#include -#include -#include -#include -#include -#include -#include - - /* Synchronization variables */ /* Global context for non-contextual API */ @@ -91,13 +91,9 @@ uint8_t g_ncodecs = 0; static blosc2_filter g_filters[256] = {0}; static uint64_t g_nfilters = 0; -static blosc2_io_cb g_ios[256] = {0}; +static blosc2_io_cb g_io[256] = {0}; static uint64_t g_nio = 0; -blosc2_tuner g_tuners[256] = {0}; -int g_ntuners = 0; - -static int g_tuner = BLOSC_STUNE; // Forward declarations int init_threadpool(blosc2_context *context); @@ -107,54 +103,46 @@ int release_threadpool(blosc2_context *context); /* Wait until all threads are initialized */ #ifdef BLOSC_POSIX_BARRIERS -#define WAIT_INIT(RET_VAL, CONTEXT_PTR) \ - do { \ - rc = pthread_barrier_wait(&(CONTEXT_PTR)->barr_init); \ - if (rc != 0 && rc != PTHREAD_BARRIER_SERIAL_THREAD) { \ - BLOSC_TRACE_ERROR("Could not wait on barrier (init): %d", rc); \ - return((RET_VAL)); \ - } \ - } while (0) +#define WAIT_INIT(RET_VAL, CONTEXT_PTR) \ + rc = pthread_barrier_wait(&(CONTEXT_PTR)->barr_init); \ + if (rc != 0 && rc != PTHREAD_BARRIER_SERIAL_THREAD) { \ + BLOSC_TRACE_ERROR("Could not wait on barrier (init): %d", rc); \ + return((RET_VAL)); \ + } #else -#define WAIT_INIT(RET_VAL, CONTEXT_PTR) \ - do { \ - pthread_mutex_lock(&(CONTEXT_PTR)->count_threads_mutex); \ - if ((CONTEXT_PTR)->count_threads < (CONTEXT_PTR)->nthreads) { \ - (CONTEXT_PTR)->count_threads++; \ - pthread_cond_wait(&(CONTEXT_PTR)->count_threads_cv, \ - &(CONTEXT_PTR)->count_threads_mutex); \ - } \ - else { \ - pthread_cond_broadcast(&(CONTEXT_PTR)->count_threads_cv); \ - } \ - pthread_mutex_unlock(&(CONTEXT_PTR)->count_threads_mutex); \ - } while (0) +#define WAIT_INIT(RET_VAL, CONTEXT_PTR) \ + pthread_mutex_lock(&(CONTEXT_PTR)->count_threads_mutex); \ + if ((CONTEXT_PTR)->count_threads < (CONTEXT_PTR)->nthreads) { \ + (CONTEXT_PTR)->count_threads++; \ + pthread_cond_wait(&(CONTEXT_PTR)->count_threads_cv, \ + &(CONTEXT_PTR)->count_threads_mutex); \ + } \ + else { \ + pthread_cond_broadcast(&(CONTEXT_PTR)->count_threads_cv); \ + } \ + pthread_mutex_unlock(&(CONTEXT_PTR)->count_threads_mutex); #endif /* Wait for all threads to finish */ #ifdef BLOSC_POSIX_BARRIERS -#define WAIT_FINISH(RET_VAL, CONTEXT_PTR) \ - do { \ - rc = pthread_barrier_wait(&(CONTEXT_PTR)->barr_finish); \ - if (rc != 0 && rc != PTHREAD_BARRIER_SERIAL_THREAD) { \ - BLOSC_TRACE_ERROR("Could not wait on barrier (finish)"); \ - return((RET_VAL)); \ - } \ - } while (0) +#define WAIT_FINISH(RET_VAL, CONTEXT_PTR) \ + rc = pthread_barrier_wait(&(CONTEXT_PTR)->barr_finish); \ + if (rc != 0 && rc != PTHREAD_BARRIER_SERIAL_THREAD) { \ + BLOSC_TRACE_ERROR("Could not wait on barrier (finish)"); \ + return((RET_VAL)); \ + } #else -#define WAIT_FINISH(RET_VAL, CONTEXT_PTR) \ - do { \ - pthread_mutex_lock(&(CONTEXT_PTR)->count_threads_mutex); \ - if ((CONTEXT_PTR)->count_threads > 0) { \ - (CONTEXT_PTR)->count_threads--; \ - pthread_cond_wait(&(CONTEXT_PTR)->count_threads_cv, \ - &(CONTEXT_PTR)->count_threads_mutex); \ - } \ - else { \ - pthread_cond_broadcast(&(CONTEXT_PTR)->count_threads_cv); \ - } \ - pthread_mutex_unlock(&(CONTEXT_PTR)->count_threads_mutex); \ - } while (0) +#define WAIT_FINISH(RET_VAL, CONTEXT_PTR) \ + pthread_mutex_lock(&(CONTEXT_PTR)->count_threads_mutex); \ + if ((CONTEXT_PTR)->count_threads > 0) { \ + (CONTEXT_PTR)->count_threads--; \ + pthread_cond_wait(&(CONTEXT_PTR)->count_threads_cv, \ + &(CONTEXT_PTR)->count_threads_mutex); \ + } \ + else { \ + pthread_cond_broadcast(&(CONTEXT_PTR)->count_threads_cv); \ + } \ + pthread_mutex_unlock(&(CONTEXT_PTR)->count_threads_mutex); #endif @@ -348,7 +336,7 @@ static int compcode_to_compformat(int compcode) { default: return BLOSC_UDCODEC_FORMAT; } - BLOSC_ERROR(BLOSC2_ERROR_FAILURE); + return -1; } @@ -356,32 +344,27 @@ static int compcode_to_compformat(int compcode) { static int compcode_to_compversion(int compcode) { /* Write compressor format */ switch (compcode) { - case BLOSC_BLOSCLZ: - return BLOSC_BLOSCLZ_VERSION_FORMAT; - case BLOSC_LZ4: - return BLOSC_LZ4_VERSION_FORMAT; - case BLOSC_LZ4HC: - return BLOSC_LZ4HC_VERSION_FORMAT; + case BLOSC_BLOSCLZ: return BLOSC_BLOSCLZ_VERSION_FORMAT; + case BLOSC_LZ4: return BLOSC_LZ4_VERSION_FORMAT; + case BLOSC_LZ4HC: return BLOSC_LZ4HC_VERSION_FORMAT; #if defined(HAVE_ZLIB) - case BLOSC_ZLIB: - return BLOSC_ZLIB_VERSION_FORMAT; + case BLOSC_ZLIB: return BLOSC_ZLIB_VERSION_FORMAT; break; #endif /* HAVE_ZLIB */ #if defined(HAVE_ZSTD) - case BLOSC_ZSTD: - return BLOSC_ZSTD_VERSION_FORMAT; + case BLOSC_ZSTD: return BLOSC_ZSTD_VERSION_FORMAT; break; #endif /* HAVE_ZSTD */ default: for (int i = 0; i < g_ncodecs; ++i) { if (compcode == g_codecs[i].compcode) { - return g_codecs[i].version; + return g_codecs[i].compver; } } } - return BLOSC2_ERROR_FAILURE; + return -1; } @@ -513,7 +496,9 @@ static int zstd_wrap_compress(struct thread_context* thread_context, (void*)output, maxout, (void*)input, input_length, clevel); } if (ZSTD_isError(code) != ZSTD_error_no_error) { - // Blosc will just memcpy this buffer + // Do not print anything because blosc will just memcpy this buffer + // fprintf(stderr, "Error in ZSTD compression: '%s'. Giving up.\n", + // ZDICT_getErrorName(code)); return 0; } return (int)code; @@ -797,81 +782,6 @@ static int blosc2_initialize_context_from_header(blosc2_context* context, blosc_ } -int fill_filter(blosc2_filter *filter) { - char libpath[PATH_MAX]; - void *lib = load_lib(filter->name, libpath); - if(lib == NULL) { - BLOSC_TRACE_ERROR("Error while loading the library"); - return BLOSC2_ERROR_FAILURE; - } - - filter_info *info = dlsym(lib, "info"); - filter->forward = dlsym(lib, info->forward); - filter->backward = dlsym(lib, info->backward); - - if (filter->forward == NULL || filter->backward == NULL){ - BLOSC_TRACE_ERROR("Wrong library loaded"); - dlclose(lib); - return BLOSC2_ERROR_FAILURE; - } - - return BLOSC2_ERROR_SUCCESS; -} - - -int fill_codec(blosc2_codec *codec) { - char libpath[PATH_MAX]; - void *lib = load_lib(codec->compname, libpath); - if(lib == NULL) { - BLOSC_TRACE_ERROR("Error while loading the library for codec `%s`", codec->compname); - return BLOSC2_ERROR_FAILURE; - } - - codec_info *info = dlsym(lib, "info"); - if (info == NULL) { - BLOSC_TRACE_ERROR("`info` symbol cannot be loaded from plugin `%s`", codec->compname); - dlclose(lib); - return BLOSC2_ERROR_FAILURE; - } - - codec->encoder = dlsym(lib, info->encoder); - codec->decoder = dlsym(lib, info->decoder); - if (codec->encoder == NULL || codec->decoder == NULL) { - BLOSC_TRACE_ERROR("encoder or decoder cannot be loaded from plugin `%s`", codec->compname); - dlclose(lib); - return BLOSC2_ERROR_FAILURE; - } - - return BLOSC2_ERROR_SUCCESS; -} - - -int fill_tuner(blosc2_tuner *tuner) { - char libpath[PATH_MAX] = {0}; - void *lib = load_lib(tuner->name, libpath); - if(lib == NULL) { - BLOSC_TRACE_ERROR("Error while loading the library"); - return BLOSC2_ERROR_FAILURE; - } - - tuner_info *info = dlsym(lib, "info"); - tuner->init = dlsym(lib, info->init); - tuner->update = dlsym(lib, info->update); - tuner->next_blocksize = dlsym(lib, info->next_blocksize); - tuner->free = dlsym(lib, info->free); - tuner->next_cparams = dlsym(lib, info->next_cparams); - - if (tuner->init == NULL || tuner->update == NULL || tuner->next_blocksize == NULL || tuner->free == NULL - || tuner->next_cparams == NULL){ - BLOSC_TRACE_ERROR("Wrong library loaded"); - dlclose(lib); - return BLOSC2_ERROR_FAILURE; - } - - return BLOSC2_ERROR_SUCCESS; -} - - static int blosc2_intialize_header_from_context(blosc2_context* context, blosc_header* header, bool extended_header) { memset(header, 0, sizeof(blosc_header)); @@ -912,12 +822,6 @@ static int blosc2_intialize_header_from_context(blosc2_context* context, blosc_h return 0; } -void _cycle_buffers(uint8_t **src, uint8_t **dest, uint8_t **tmp) { - uint8_t *tmp2 = *src; - *src = *dest; - *dest = *tmp; - *tmp = tmp2; -} uint8_t* pipeline_forward(struct thread_context* thread_context, const int32_t bsize, const uint8_t* src, const int32_t offset, @@ -959,7 +863,10 @@ uint8_t* pipeline_forward(struct thread_context* thread_context, const int32_t b // No more filters are required return _dest; } - _cycle_buffers(&_src, &_dest, &_tmp); + // Cycle buffers + _src = _dest; + _dest = _tmp; + _tmp = _src; } /* Process the filter pipeline */ @@ -972,7 +879,9 @@ uint8_t* pipeline_forward(struct thread_context* thread_context, const int32_t b shuffle(typesize, bsize, _src, _dest); // Cycle filters when required if (j < filters_meta[i]) { - _cycle_buffers(&_src, &_dest, &_tmp); + _src = _dest; + _dest = _tmp; + _tmp = _src; } } break; @@ -1000,13 +909,6 @@ uint8_t* pipeline_forward(struct thread_context* thread_context, const int32_t b // Look for the filters_meta in user filters and run it for (uint64_t j = 0; j < g_nfilters; ++j) { if (g_filters[j].id == filters[i]) { - if (g_filters[j].forward == NULL) { - // Dynamically load library - if (fill_filter(&g_filters[j]) < 0) { - BLOSC_TRACE_ERROR("Could not load filter %d\n", g_filters[j].id); - return NULL; - } - } if (g_filters[j].forward != NULL) { blosc2_cparams cparams; blosc2_ctx_get_cparams(context, &cparams); @@ -1031,7 +933,9 @@ uint8_t* pipeline_forward(struct thread_context* thread_context, const int32_t b // Cycle buffers when required if (filters[i] != BLOSC_NOFILTER) { - _cycle_buffers(&_src, &_dest, &_tmp); + _src = _dest; + _dest = _tmp; + _tmp = _src; } } return _src; @@ -1191,7 +1095,7 @@ static int blosc_c(struct thread_context* thread_context, int32_t bsize, } maxout = neblock; - if (ntbytes + maxout > destsize && !instr_codec) { + if (ntbytes + maxout > destsize) { /* avoid buffer * overrun */ maxout = destsize - ntbytes; if (maxout <= 0) { @@ -1236,13 +1140,6 @@ static int blosc_c(struct thread_context* thread_context, int32_t bsize, else if (context->compcode > BLOSC2_DEFINED_CODECS_STOP) { for (int i = 0; i < g_ncodecs; ++i) { if (g_codecs[i].compcode == context->compcode) { - if (g_codecs[i].encoder == NULL) { - // Dynamically load codec plugin - if (fill_codec(&g_codecs[i]) < 0) { - BLOSC_TRACE_ERROR("Could not load codec %d.", g_codecs[i].compcode); - return BLOSC2_ERROR_CODEC_SUPPORT; - } - } blosc2_cparams cparams; blosc2_ctx_get_cparams(context, &cparams); cbytes = g_codecs[i].encoder(_src + j * neblock, @@ -1317,6 +1214,7 @@ static int blosc_c(struct thread_context* thread_context, int32_t bsize, ctbytes += cbytes; } /* Closes j < nstreams */ + //printf("c%d", ctbytes); return ctbytes; } @@ -1348,7 +1246,9 @@ int pipeline_backward(struct thread_context* thread_context, const int32_t bsize unshuffle(typesize, bsize, _src, _dest); // Cycle filters when required if (j < filters_meta[i]) { - _cycle_buffers(&_src, &_dest, &_tmp); + _src = _dest; + _dest = _tmp; + _tmp = _src; } // Check whether we have to copy the intermediate _dest buffer to final destination if (last_copy_filter && (filters_meta[i] % 2) == 1 && j == filters_meta[i]) { @@ -1397,13 +1297,6 @@ int pipeline_backward(struct thread_context* thread_context, const int32_t bsize // Look for the filters_meta in user filters and run it for (uint64_t j = 0; j < g_nfilters; ++j) { if (g_filters[j].id == filters[i]) { - if (g_filters[j].backward == NULL) { - // Dynamically load filter - if (fill_filter(&g_filters[j]) < 0) { - BLOSC_TRACE_ERROR("Could not load filter %d.", g_filters[j].id); - return BLOSC2_ERROR_FILTER_PIPELINE; - } - } if (g_filters[j].backward != NULL) { blosc2_dparams dparams; blosc2_ctx_get_dparams(context, &dparams); @@ -1426,7 +1319,9 @@ int pipeline_backward(struct thread_context* thread_context, const int32_t bsize // Cycle buffers when required if ((filters[i] != BLOSC_NOFILTER) && (filters[i] != BLOSC_TRUNC_PREC)) { - _cycle_buffers(&_src, &_dest, &_tmp); + _src = _dest; + _dest = _tmp; + _tmp = _src; } if (last_filter_index == i) { break; @@ -1461,9 +1356,9 @@ int pipeline_backward(struct thread_context* thread_context, const int32_t bsize static int32_t set_nans(int32_t typesize, uint8_t* dest, int32_t destsize) { + // destsize can only be a multiple of typesize if (destsize % typesize != 0) { - BLOSC_TRACE_ERROR("destsize can only be a multiple of typesize"); - BLOSC_ERROR(BLOSC2_ERROR_FAILURE); + return -1; } int32_t nitems = destsize / typesize; if (nitems == 0) { @@ -1504,7 +1399,7 @@ static int32_t set_values(int32_t typesize, const uint8_t* src, uint8_t* dest, i int8_t* dest1; if (destsize % typesize != 0) { - BLOSC_ERROR(BLOSC2_ERROR_FAILURE); + return -1; } int32_t nitems = destsize / typesize; if (nitems == 0) { @@ -1760,7 +1655,7 @@ static int blosc_d( neblock = bsize / nstreams; if (neblock == 0) { /* Not enough space to output bytes */ - BLOSC_ERROR(BLOSC2_ERROR_WRITE_BUFFER); + return -1; } for (int j = 0; j < nstreams; j++) { if (srcsize < (signed)sizeof(int32_t)) { @@ -1858,13 +1753,6 @@ static int blosc_d( thread_context->zfp_cell_nitems = 0; for (int i = 0; i < g_ncodecs; ++i) { if (g_codecs[i].compcode == context->compcode) { - if (g_codecs[i].decoder == NULL) { - // Dynamically load codec plugin - if (fill_codec(&g_codecs[i]) < 0) { - BLOSC_TRACE_ERROR("Could not load codec %d.", g_codecs[i].compcode); - return BLOSC2_ERROR_CODEC_SUPPORT; - } - } blosc2_dparams dparams; blosc2_ctx_get_dparams(context, &dparams); nbytes = g_codecs[i].decoder(src, @@ -1924,7 +1812,7 @@ static int serial_blosc(struct thread_context* thread_context) { blosc2_context* context = thread_context->parent_context; int32_t j, bsize, leftoverblock; int32_t cbytes; - int32_t ntbytes = context->output_bytes; + int32_t ntbytes = (int32_t)context->output_bytes; int32_t* bstarts = context->bstarts; uint8_t* tmp = thread_context->tmp; uint8_t* tmp2 = thread_context->tmp2; @@ -2146,13 +2034,13 @@ static int do_job(blosc2_context* context) { static int initialize_context_compression( - blosc2_context* context, const void* src, int32_t srcsize, void* dest, - int32_t destsize, int clevel, uint8_t const *filters, - uint8_t const *filters_meta, int32_t typesize, int compressor, - int32_t blocksize, int16_t new_nthreads, int16_t nthreads, - int32_t splitmode, - int tuner_id, void *tuner_params, - blosc2_schunk* schunk) { + blosc2_context* context, const void* src, int32_t srcsize, void* dest, + int32_t destsize, int clevel, uint8_t const *filters, + uint8_t const *filters_meta, int32_t typesize, int compressor, + int32_t blocksize, int16_t new_nthreads, int16_t nthreads, + int32_t splitmode, + blosc2_btune *udbtune, void *btune_config, + blosc2_schunk* schunk) { /* Set parameters */ context->do_compress = 1; @@ -2174,82 +2062,38 @@ static int initialize_context_compression( context->end_threads = 0; context->clevel = clevel; context->schunk = schunk; - context->tuner_params = tuner_params; - context->tuner_id = tuner_id; + context->btune = btune_config; + context->udbtune = udbtune; context->splitmode = splitmode; - /* tuner some compression parameters */ + /* Tune some compression parameters */ context->blocksize = (int32_t)blocksize; - int rc = 0; - if (context->tuner_params != NULL) { - if (context->tuner_id < BLOSC_LAST_TUNER && context->tuner_id == BLOSC_STUNE) { - if (blosc_stune_next_cparams(context) < 0) { - BLOSC_TRACE_ERROR("Error in stune next_cparams func\n"); - return BLOSC2_ERROR_TUNER; - } - } else { - for (int i = 0; i < g_ntuners; ++i) { - if (g_tuners[i].id == context->tuner_id) { - if (g_tuners[i].next_cparams == NULL) { - if (fill_tuner(&g_tuners[i]) < 0) { - BLOSC_TRACE_ERROR("Could not load tuner %d.", g_tuners[i].id); - return BLOSC2_ERROR_FAILURE; - } - } - if (g_tuners[i].next_cparams(context) < 0) { - BLOSC_TRACE_ERROR("Error in tuner %d next_cparams func\n", context->tuner_id); - return BLOSC2_ERROR_TUNER; - } - if (g_tuners[i].id == BLOSC_BTUNE && context->blocksize == 0) { - // Call stune for initializing blocksize - if (blosc_stune_next_blocksize(context) < 0) { - BLOSC_TRACE_ERROR("Error in stune next_blocksize func\n"); - return BLOSC2_ERROR_TUNER; - } - } - goto urtunersuccess; - } - } - BLOSC_TRACE_ERROR("User-defined tuner %d not found\n", context->tuner_id); - return BLOSC2_ERROR_INVALID_PARAM; - } + if (context->btune != NULL) { + context->udbtune->btune_next_cparams(context); } else { - if (context->tuner_id < BLOSC_LAST_TUNER && context->tuner_id == BLOSC_STUNE) { - rc = blosc_stune_next_blocksize(context); - } else { - for (int i = 0; i < g_ntuners; ++i) { - if (g_tuners[i].id == context->tuner_id) { - if (g_tuners[i].next_blocksize == NULL) { - if (fill_tuner(&g_tuners[i]) < 0) { - BLOSC_TRACE_ERROR("Could not load tuner %d.", g_tuners[i].id); - return BLOSC2_ERROR_FAILURE; - } - } - rc = g_tuners[i].next_blocksize(context); - goto urtunersuccess; - } - } - BLOSC_TRACE_ERROR("User-defined tuner %d not found\n", context->tuner_id); - return BLOSC2_ERROR_INVALID_PARAM; - } - } - urtunersuccess:; - if (rc < 0) { - BLOSC_TRACE_ERROR("Error in tuner next_blocksize func\n"); - return BLOSC2_ERROR_TUNER; + context->udbtune->btune_next_blocksize(context); } + char* envvar = getenv("BLOSC_WARN"); + int64_t warnlvl = 0; + if (envvar != NULL) { + warnlvl = strtol(envvar, NULL, 10); + } /* Check buffer size limits */ if (srcsize > BLOSC2_MAX_BUFFERSIZE) { - BLOSC_TRACE_ERROR("Input buffer size cannot exceed %d bytes.", - BLOSC2_MAX_BUFFERSIZE); - return BLOSC2_ERROR_MAX_BUFSIZE_EXCEEDED; + if (warnlvl > 0) { + BLOSC_TRACE_ERROR("Input buffer size cannot exceed %d bytes.", + BLOSC2_MAX_BUFFERSIZE); + } + return 0; } if (destsize < BLOSC2_MAX_OVERHEAD) { - BLOSC_TRACE_ERROR("Output buffer size should be larger than %d bytes.", - BLOSC2_MAX_OVERHEAD); - return BLOSC2_ERROR_MAX_BUFSIZE_EXCEEDED; + if (warnlvl > 0) { + BLOSC_TRACE_ERROR("Output buffer size should be larger than %d bytes.", + BLOSC2_MAX_OVERHEAD); + } + return 0; } /* Compression level */ @@ -2513,36 +2357,13 @@ static int blosc_compress_context(blosc2_context* context) { _sw32(context->dest + BLOSC2_CHUNK_BLOCKSIZE, blocksize); } - /* Set the number of bytes in dest buffer (might be useful for tuner) */ + /* Set the number of bytes in dest buffer (might be useful for btune) */ context->destsize = ntbytes; - if (context->tuner_params != NULL) { + if (context->btune != NULL) { blosc_set_timestamp(¤t); double ctime = blosc_elapsed_secs(last, current); - int rc; - if (context->tuner_id < BLOSC_LAST_TUNER && context->tuner_id == BLOSC_STUNE) { - rc = blosc_stune_update(context, ctime); - } else { - for (int i = 0; i < g_ntuners; ++i) { - if (g_tuners[i].id == context->tuner_id) { - if (g_tuners[i].update == NULL) { - if (fill_tuner(&g_tuners[i]) < 0) { - BLOSC_TRACE_ERROR("Could not load tuner %d.", g_tuners[i].id); - return BLOSC2_ERROR_FAILURE; - } - } - rc = g_tuners[i].update(context, ctime); - goto urtunersuccess; - } - } - BLOSC_TRACE_ERROR("User-defined tuner %d not found\n", context->tuner_id); - return BLOSC2_ERROR_INVALID_PARAM; - urtunersuccess:; - } - if (rc < 0) { - BLOSC_TRACE_ERROR("Error in tuner update func\n"); - return BLOSC2_ERROR_TUNER; - } + context->udbtune->btune_update(context, ctime); } return ntbytes; @@ -2560,11 +2381,11 @@ int blosc2_compress_ctx(blosc2_context* context, const void* src, int32_t srcsiz } error = initialize_context_compression( - context, src, srcsize, dest, destsize, - context->clevel, context->filters, context->filters_meta, - context->typesize, context->compcode, context->blocksize, - context->new_nthreads, context->nthreads, context->splitmode, - context->tuner_id, context->tuner_params, context->schunk); + context, src, srcsize, dest, destsize, + context->clevel, context->filters, context->filters_meta, + context->typesize, context->compcode, context->blocksize, + context->new_nthreads, context->nthreads, context->splitmode, + context->udbtune, context->btune, context->schunk); if (error <= 0) { return error; } @@ -2820,10 +2641,6 @@ int blosc2_compress(int clevel, int doshuffle, int32_t typesize, cparams.nthreads = g_nthreads; cparams.splitmode = g_splitmode; cctx = blosc2_create_cctx(cparams); - if (cctx == NULL) { - BLOSC_TRACE_ERROR("Error while creating the compression context"); - return BLOSC2_ERROR_NULL_POINTER; - } /* Do the actual compression */ result = blosc2_compress_ctx(cctx, src, srcsize, dest, destsize); /* Release context resources */ @@ -2840,9 +2657,9 @@ int blosc2_compress(int clevel, int doshuffle, int32_t typesize, BLOSC_ERROR_NULL(filters_meta, BLOSC2_ERROR_MEMORY_ALLOC); build_filters(doshuffle, g_delta, typesize, filters); error = initialize_context_compression( - g_global_context, src, srcsize, dest, destsize, clevel, filters, - filters_meta, (int32_t)typesize, g_compressor, g_force_blocksize, g_nthreads, g_nthreads, - g_splitmode, g_tuner, NULL, g_schunk); + g_global_context, src, srcsize, dest, destsize, clevel, filters, + filters_meta, (int32_t)typesize, g_compressor, g_force_blocksize, g_nthreads, g_nthreads, + g_splitmode, &BTUNE_DEFAULTS, NULL, g_schunk); free(filters); free(filters_meta); if (error <= 0) { @@ -2964,10 +2781,6 @@ int blosc2_decompress(const void* src, int32_t srcsize, void* dest, int32_t dest if (envvar != NULL) { dparams.nthreads = g_nthreads; dctx = blosc2_create_dctx(dparams); - if (dctx == NULL) { - BLOSC_TRACE_ERROR("Error while creating the decompression context"); - return BLOSC2_ERROR_NULL_POINTER; - } result = blosc2_decompress_ctx(dctx, src, srcsize, dest, destsize); blosc2_free_ctx(dctx); return result; @@ -3071,7 +2884,7 @@ int _blosc_getitem(blosc2_context* context, blosc_header* header, const void* sr break; default: BLOSC_TRACE_ERROR("Unhandled special value case"); - BLOSC_ERROR(BLOSC2_ERROR_SCHUNK_SPECIAL); + return -1; } return ntbytes; } @@ -3131,7 +2944,7 @@ int _blosc_getitem(blosc2_context* context, blosc_header* header, const void* sr // If memcpyed we don't have a bstarts section (because it is not needed) int32_t src_offset = memcpyed ? - context->header_overhead + j * header->blocksize : sw32_(context->bstarts + j); + context->header_overhead + j * bsize : sw32_(context->bstarts + j); int32_t cbytes = blosc_d(context->serial_context, bsize, leftoverblock, memcpyed, src, srcsize, src_offset, j, @@ -3542,7 +3355,7 @@ int blosc1_set_compressor(const char* compname) { int code = blosc2_compname_to_compcode(compname); if (code >= BLOSC_LAST_CODEC) { BLOSC_TRACE_ERROR("User defined codecs cannot be set here. Use Blosc2 mechanism instead."); - BLOSC_ERROR(BLOSC2_ERROR_CODEC_SUPPORT); + return -1; } g_compressor = code; @@ -3768,14 +3581,12 @@ void blosc_set_schunk(blosc2_schunk* schunk) { } blosc2_io *blosc2_io_global = NULL; -blosc2_io_cb BLOSC2_IO_CB_DEFAULTS; void blosc2_init(void) { /* Return if Blosc is already initialized */ if (g_initlib) return; BLOSC2_IO_CB_DEFAULTS.id = BLOSC2_IO_FILESYSTEM; - BLOSC2_IO_CB_DEFAULTS.name = "filesystem"; BLOSC2_IO_CB_DEFAULTS.open = (blosc2_open_cb) blosc2_stdio_open; BLOSC2_IO_CB_DEFAULTS.close = (blosc2_close_cb) blosc2_stdio_close; BLOSC2_IO_CB_DEFAULTS.tell = (blosc2_tell_cb) blosc2_stdio_tell; @@ -3786,14 +3597,12 @@ void blosc2_init(void) { g_ncodecs = 0; g_nfilters = 0; - g_ntuners = 0; #if defined(HAVE_PLUGINS) #include "blosc2/blosc2-common.h" #include "blosc2/blosc2-stdio.h" register_codecs(); register_filters(); - register_tuners(); #endif pthread_mutex_init(&global_comp_mutex, NULL); /* Create a global context */ @@ -3920,18 +3729,6 @@ blosc2_context* blosc2_create_cctx(blosc2_cparams cparams) { } } -#if defined(HAVE_PLUGINS) -#include "blosc2/codecs-registry.h" - if ((context->compcode >= BLOSC_CODEC_ZFP_FIXED_ACCURACY) && (context->compcode <= BLOSC_CODEC_ZFP_FIXED_RATE)) { - for (int i = 0; i < BLOSC2_MAX_FILTERS; ++i) { - if ((context->filters[i] == BLOSC_SHUFFLE) || (context->filters[i] == BLOSC_BITSHUFFLE)) { - BLOSC_TRACE_ERROR("ZFP cannot be run in presence of SHUFFLE / BITSHUFFLE"); - return NULL; - } - } - } -#endif /* HAVE_PLUGINS */ - /* Check for a BLOSC_SHUFFLE environment variable */ int doshuffle = -1; char* envvar = getenv("BLOSC_SHUFFLE"); @@ -4067,30 +3864,11 @@ blosc2_context* blosc2_create_cctx(blosc2_cparams cparams) { memcpy(context->preparams, cparams.preparams, sizeof(blosc2_prefilter_params)); } - if (cparams.tuner_id <= 0) { - cparams.tuner_id = g_tuner; + if (cparams.udbtune == NULL) { + context->udbtune = &BTUNE_DEFAULTS; } else { - for (int i = 0; i < g_ntuners; ++i) { - if (g_tuners[i].id == cparams.tuner_id) { - if (g_tuners[i].init == NULL) { - if (fill_tuner(&g_tuners[i]) < 0) { - BLOSC_TRACE_ERROR("Could not load tuner %d.", g_tuners[i].id); - return NULL; - } - } - if (g_tuners[i].init(cparams.tuner_params, context, NULL) < 0) { - BLOSC_TRACE_ERROR("Error in user-defined tuner %d init function\n", cparams.tuner_id); - return NULL; - } - goto urtunersuccess; - } - } - BLOSC_TRACE_ERROR("User-defined tuner %d not found\n", cparams.tuner_id); - return NULL; + context->udbtune = cparams.udbtune; } - urtunersuccess:; - - context->tuner_id = cparams.tuner_id; context->codec_params = cparams.codec_params; memcpy(context->filter_params, cparams.filter_params, BLOSC2_MAX_FILTERS * sizeof(void*)); @@ -4148,31 +3926,8 @@ void blosc2_free_ctx(blosc2_context* context) { ZSTD_freeDDict(context->dict_ddict); #endif } - if (context->tuner_params != NULL) { - int rc; - if (context->tuner_id < BLOSC_LAST_TUNER && context->tuner_id == BLOSC_STUNE) { - rc = blosc_stune_free(context); - } else { - for (int i = 0; i < g_ntuners; ++i) { - if (g_tuners[i].id == context->tuner_id) { - if (g_tuners[i].free == NULL) { - if (fill_tuner(&g_tuners[i]) < 0) { - BLOSC_TRACE_ERROR("Could not load tuner %d.", g_tuners[i].id); - return; - } - } - rc = g_tuners[i].free(context); - goto urtunersuccess; - } - } - BLOSC_TRACE_ERROR("User-defined tuner %d not found\n", context->tuner_id); - return; - urtunersuccess:; - } - if (rc < 0) { - BLOSC_TRACE_ERROR("Error in user-defined tuner free function\n"); - return; - } + if (context->btune != NULL) { + context->udbtune->btune_free(context); } if (context->prefilter != NULL) { my_free(context->preparams); @@ -4205,7 +3960,7 @@ int blosc2_ctx_get_cparams(blosc2_context *ctx, blosc2_cparams *cparams) { } cparams->prefilter = ctx->prefilter; cparams->preparams = ctx->preparams; - cparams->tuner_id = ctx->tuner_id; + cparams->udbtune = ctx->udbtune; cparams->codec_params = ctx->codec_params; return BLOSC2_ERROR_SUCCESS; @@ -4254,17 +4009,13 @@ int blosc2_chunk_zeros(blosc2_cparams cparams, const int32_t nbytes, void* dest, blosc_header header; blosc2_context* context = blosc2_create_cctx(cparams); - if (context == NULL) { - BLOSC_TRACE_ERROR("Error while creating the compression context"); - return BLOSC2_ERROR_NULL_POINTER; - } int error = initialize_context_compression( context, NULL, nbytes, dest, destsize, context->clevel, context->filters, context->filters_meta, context->typesize, context->compcode, context->blocksize, context->new_nthreads, context->nthreads, context->splitmode, - context->tuner_id, context->tuner_params, context->schunk); + context->udbtune, context->btune, context->schunk); if (error <= 0) { blosc2_free_ctx(context); return error; @@ -4301,16 +4052,12 @@ int blosc2_chunk_uninit(blosc2_cparams cparams, const int32_t nbytes, void* dest blosc_header header; blosc2_context* context = blosc2_create_cctx(cparams); - if (context == NULL) { - BLOSC_TRACE_ERROR("Error while creating the compression context"); - return BLOSC2_ERROR_NULL_POINTER; - } int error = initialize_context_compression( context, NULL, nbytes, dest, destsize, context->clevel, context->filters, context->filters_meta, context->typesize, context->compcode, context->blocksize, context->new_nthreads, context->nthreads, context->splitmode, - context->tuner_id, context->tuner_params, context->schunk); + context->udbtune, context->btune, context->schunk); if (error <= 0) { blosc2_free_ctx(context); return error; @@ -4347,17 +4094,13 @@ int blosc2_chunk_nans(blosc2_cparams cparams, const int32_t nbytes, void* dest, blosc_header header; blosc2_context* context = blosc2_create_cctx(cparams); - if (context == NULL) { - BLOSC_TRACE_ERROR("Error while creating the compression context"); - return BLOSC2_ERROR_NULL_POINTER; - } int error = initialize_context_compression( context, NULL, nbytes, dest, destsize, context->clevel, context->filters, context->filters_meta, context->typesize, context->compcode, context->blocksize, context->new_nthreads, context->nthreads, context->splitmode, - context->tuner_id, context->tuner_params, context->schunk); + context->udbtune, context->btune, context->schunk); if (error <= 0) { blosc2_free_ctx(context); return error; @@ -4396,17 +4139,13 @@ int blosc2_chunk_repeatval(blosc2_cparams cparams, const int32_t nbytes, blosc_header header; blosc2_context* context = blosc2_create_cctx(cparams); - if (context == NULL) { - BLOSC_TRACE_ERROR("Error while creating the compression context"); - return BLOSC2_ERROR_NULL_POINTER; - } int error = initialize_context_compression( context, NULL, nbytes, dest, destsize, context->clevel, context->filters, context->filters_meta, context->typesize, context->compcode, context->blocksize, context->new_nthreads, context->nthreads, context->splitmode, - context->tuner_id, context->tuner_params, context->schunk); + context->udbtune, context->btune, context->schunk); if (error <= 0) { blosc2_free_ctx(context); return error; @@ -4449,18 +4188,12 @@ int register_filter_private(blosc2_filter *filter) { } */ + // Check if the filter is already registered for (uint64_t i = 0; i < g_nfilters; ++i) { - if (g_filters[i].id == filter->id) { - if (strcmp(g_filters[i].name, filter->name) != 0) { - BLOSC_TRACE_ERROR("The filter (ID: %d) plugin is already registered with name: %s." - " Choose another one !", filter->id, g_filters[i].name); - return BLOSC2_ERROR_FAILURE; + if (g_filters[i].id == filter->id) { + BLOSC_TRACE_ERROR("The filter is already registered!"); + return BLOSC2_ERROR_FAILURE; } - else { - // Already registered, so no more actions needed - return BLOSC2_ERROR_SUCCESS; - } - } } blosc2_filter *filter_new = &g_filters[g_nfilters++]; @@ -4485,32 +4218,26 @@ int blosc2_register_filter(blosc2_filter *filter) { int register_codec_private(blosc2_codec *codec) { BLOSC_ERROR_NULL(codec, BLOSC2_ERROR_INVALID_PARAM); if (g_ncodecs == UINT8_MAX) { - BLOSC_TRACE_ERROR("Can not register more codecs"); - return BLOSC2_ERROR_CODEC_SUPPORT; + BLOSC_TRACE_ERROR("Can not register more codecs"); + return BLOSC2_ERROR_CODEC_SUPPORT; } if (codec->compcode < BLOSC2_GLOBAL_REGISTERED_CODECS_START) { - BLOSC_TRACE_ERROR("The id must be greater or equal than %d", BLOSC2_GLOBAL_REGISTERED_CODECS_START); - return BLOSC2_ERROR_FAILURE; + BLOSC_TRACE_ERROR("The id must be greater or equal than %d", BLOSC2_GLOBAL_REGISTERED_CODECS_START); + return BLOSC2_ERROR_FAILURE; } /* This condition can never be fulfilled if (codec->compcode > BLOSC2_USER_REGISTERED_CODECS_STOP) { - BLOSC_TRACE_ERROR("The id must be less or equal to %d", BLOSC2_USER_REGISTERED_CODECS_STOP); - return BLOSC2_ERROR_FAILURE; + BLOSC_TRACE_ERROR("The id must be less or equal to %d", BLOSC2_USER_REGISTERED_CODECS_STOP); + return BLOSC2_ERROR_FAILURE; } */ + // Check if the code is already registered for (int i = 0; i < g_ncodecs; ++i) { - if (g_codecs[i].compcode == codec->compcode) { - if (strcmp(g_codecs[i].compname, codec->compname) != 0) { - BLOSC_TRACE_ERROR("The codec (ID: %d) plugin is already registered with name: %s." - " Choose another one !", codec->compcode, codec->compname); - return BLOSC2_ERROR_CODEC_PARAM; - } - else { - // Already registered, so no more actions needed - return BLOSC2_ERROR_SUCCESS; + if (g_codecs[i].compcode == codec->compcode) { + BLOSC_TRACE_ERROR("The codec is already registered!"); + return BLOSC2_ERROR_CODEC_PARAM; } - } } blosc2_codec *codec_new = &g_codecs[g_ncodecs++]; @@ -4530,67 +4257,17 @@ int blosc2_register_codec(blosc2_codec *codec) { } -/* Register tuners */ - -int register_tuner_private(blosc2_tuner *tuner) { - BLOSC_ERROR_NULL(tuner, BLOSC2_ERROR_INVALID_PARAM); - if (g_ntuners == UINT8_MAX) { - BLOSC_TRACE_ERROR("Can not register more tuners"); - return BLOSC2_ERROR_CODEC_SUPPORT; - } - if (tuner->id < BLOSC2_GLOBAL_REGISTERED_TUNER_START) { - BLOSC_TRACE_ERROR("The id must be greater or equal than %d", BLOSC2_GLOBAL_REGISTERED_TUNER_START); - return BLOSC2_ERROR_FAILURE; - } - - for (int i = 0; i < g_ntuners; ++i) { - if (g_tuners[i].id == tuner->id) { - if (strcmp(g_tuners[i].name, tuner->name) != 0) { - BLOSC_TRACE_ERROR("The tuner (ID: %d) plugin is already registered with name: %s." - " Choose another one !", tuner->id, g_tuners[i].name); - return BLOSC2_ERROR_FAILURE; - } - else { - // Already registered, so no more actions needed - return BLOSC2_ERROR_SUCCESS; - } - } - } - - blosc2_tuner *tuner_new = &g_tuners[g_ntuners++]; - memcpy(tuner_new, tuner, sizeof(blosc2_tuner)); - - return BLOSC2_ERROR_SUCCESS; -} - - -int blosc2_register_tuner(blosc2_tuner *tuner) { - if (tuner->id < BLOSC2_USER_REGISTERED_TUNER_START) { - BLOSC_TRACE_ERROR("The id must be greater or equal to %d", BLOSC2_USER_REGISTERED_TUNER_START); - return BLOSC2_ERROR_FAILURE; - } - - return register_tuner_private(tuner); -} - - int _blosc2_register_io_cb(const blosc2_io_cb *io) { + // Check if the io is already registered for (uint64_t i = 0; i < g_nio; ++i) { - if (g_ios[i].id == io->id) { - if (strcmp(g_ios[i].name, io->name) != 0) { - BLOSC_TRACE_ERROR("The IO (ID: %d) plugin is already registered with name: %s." - " Choose another one !", io->id, g_ios[i].name); - return BLOSC2_ERROR_PLUGIN_IO; - } - else { - // Already registered, so no more actions needed - return BLOSC2_ERROR_SUCCESS; - } + if (g_io[i].id == io->id) { + BLOSC_TRACE_ERROR("The codec is already registered!"); + return BLOSC2_ERROR_PLUGIN_IO; } } - blosc2_io_cb *io_new = &g_ios[g_nio++]; + blosc2_io_cb *io_new = &g_io[g_nio++]; memcpy(io_new, io, sizeof(blosc2_io_cb)); return BLOSC2_ERROR_SUCCESS; @@ -4613,8 +4290,8 @@ int blosc2_register_io_cb(const blosc2_io_cb *io) { blosc2_io_cb *blosc2_get_io_cb(uint8_t id) { for (uint64_t i = 0; i < g_nio; ++i) { - if (g_ios[i].id == id) { - return &g_ios[i]; + if (g_io[i].id == id) { + return &g_io[i]; } } if (id == BLOSC2_IO_FILESYSTEM) { @@ -4628,25 +4305,24 @@ blosc2_io_cb *blosc2_get_io_cb(uint8_t id) { } void blosc2_unidim_to_multidim(uint8_t ndim, int64_t *shape, int64_t i, int64_t *index) { + int64_t strides[BLOSC2_MAX_DIM]; if (ndim == 0) { return; } - int64_t *strides = malloc(ndim * sizeof(int64_t)); strides[ndim - 1] = 1; - for (int j = ndim - 2; j >= 0; --j) { - strides[j] = shape[j + 1] * strides[j + 1]; - } + for (int j = ndim - 2; j >= 0; --j) { + strides[j] = shape[j + 1] * strides[j + 1]; + } - index[0] = i / strides[0]; - for (int j = 1; j < ndim; ++j) { - index[j] = (i % strides[j - 1]) / strides[j]; - } - free(strides); + index[0] = i / strides[0]; + for (int j = 1; j < ndim; ++j) { + index[j] = (i % strides[j - 1]) / strides[j]; + } } void blosc2_multidim_to_unidim(const int64_t *index, int8_t ndim, const int64_t *strides, int64_t *i) { - *i = 0; - for (int j = 0; j < ndim; ++j) { - *i += index[j] * strides[j]; - } + *i = 0; + for (int j = 0; j < ndim; ++j) { + *i += index[j] * strides[j]; + } } diff --git a/src/blosc2/blosc/blosclz.c b/src/blosc2/blosc/blosclz.c index 00eda33..d55a7cd 100644 --- a/src/blosc2/blosc/blosclz.c +++ b/src/blosc2/blosc/blosclz.c @@ -1,7 +1,7 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) @@ -14,14 +14,12 @@ **********************************************************************/ +#include +#include #include "blosclz.h" #include "fastcopy.h" #include "blosc2/blosc2-common.h" -#include -#include -#include -#include /* * Give hints to the compiler for branch prediction optimization. @@ -438,7 +436,7 @@ int blosclz_compress(const int clevel, const void* input, int length, // Minimum lengths for encoding (normally it is good to match the shift value) unsigned minlen = 3; - uint8_t hashlog_[10] = {0, HASH_LOG - 1, HASH_LOG - 1, HASH_LOG, HASH_LOG, + uint8_t hashlog_[10] = {0, HASH_LOG - 2, HASH_LOG - 1, HASH_LOG, HASH_LOG, HASH_LOG, HASH_LOG, HASH_LOG, HASH_LOG, HASH_LOG}; uint8_t hashlog = hashlog_[clevel]; @@ -447,10 +445,7 @@ int blosclz_compress(const int clevel, const void* input, int length, // is better (specially when combined with bitshuffle). // The loss in speed for checking the whole buffer is pretty negligible too. int maxlen = length; - if (clevel < 2) { - maxlen /= 8; - } - else if (clevel < 4) { + if (clevel < 4) { maxlen /= 4; } else if (clevel < 7) { diff --git a/src/blosc2/blosc/blosclz.h b/src/blosc2/blosc/blosclz.h index dfe49bf..c3248c9 100644 --- a/src/blosc2/blosc/blosclz.h +++ b/src/blosc2/blosc/blosclz.h @@ -1,7 +1,7 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) @@ -14,12 +14,17 @@ about copyright and rights to use. **********************************************************************/ -#ifndef BLOSC_BLOSCLZ_H -#define BLOSC_BLOSCLZ_H +#ifndef BLOSCLZ_H +#define BLOSCLZ_H #include "context.h" -#define BLOSCLZ_VERSION_STRING "2.5.3" +#if defined (__cplusplus) +extern "C" { +#endif + +#define BLOSCLZ_VERSION_STRING "2.5.2" + /** Compress a block of data in the input buffer and returns the size of @@ -58,4 +63,8 @@ int blosclz_compress(int opt_level, const void* input, int length, int blosclz_decompress(const void* input, int length, void* output, int maxout); -#endif /* BLOSC_BLOSCLZ_H */ +#if defined (__cplusplus) +} +#endif + +#endif /* BLOSCLZ_H */ diff --git a/src/blosc2/blosc/context.h b/src/blosc2/blosc/context.h index 4941df9..971e5fd 100644 --- a/src/blosc2/blosc/context.h +++ b/src/blosc2/blosc/context.h @@ -1,42 +1,39 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ -#ifndef BLOSC_CONTEXT_H -#define BLOSC_CONTEXT_H - -#include "b2nd.h" -#include "blosc2.h" - -#if defined(HAVE_ZSTD) -#include "zstd.h" -#endif - -#ifdef HAVE_IPP -#include -#endif +#ifndef CONTEXT_H +#define CONTEXT_H #if defined(_WIN32) && !defined(__GNUC__) -#include "win32/pthread.h" + #include "win32/pthread.h" #else -#include + #include #endif -#include -#include - /* Have problems using posix barriers when symbol value is 200112L */ /* Requires more investigation, but this will work for the moment */ #if defined(_POSIX_BARRIERS) && ( (_POSIX_BARRIERS - 20012L) >= 0 && _POSIX_BARRIERS != 200112L) #define BLOSC_POSIX_BARRIERS #endif +#include "blosc2.h" +#include "b2nd.h" + +#if defined(HAVE_ZSTD) + #include "zstd.h" +#endif /* HAVE_ZSTD */ + +#ifdef HAVE_IPP + #include +#endif /* HAVE_IPP */ + struct blosc2_context_s { const uint8_t* src; /* The source buffer */ uint8_t* dest; /* The destination buffer */ @@ -77,8 +74,8 @@ struct blosc2_context_s { blosc2_schunk* schunk; /* Associated super-chunk (if available) */ struct thread_context* serial_context; /* Cache for temporaries for serial operation */ int do_compress; /* 1 if we are compressing, 0 if decompressing */ - void *tuner_params; /* Entry point for tuner persistence between runs */ - int tuner_id; /* User-defined tuner id */ + void *btune; /* Entry point for BTune persistence between runs */ + blosc2_btune *udbtune; /* User-defined BTune parameters */ void *codec_params; /* User defined parameters for the codec */ void *filter_params[BLOSC2_MAX_FILTERS]; /* User defined parameters for the filters */ /* Threading */ @@ -106,7 +103,6 @@ struct blosc2_context_s { int dref_not_init; /* data ref in delta not initialized */ pthread_mutex_t delta_mutex; pthread_cond_t delta_cv; - // Add new fields here to avoid breaking the ABI. }; struct b2nd_context_s { @@ -151,4 +147,5 @@ struct thread_context { #endif }; -#endif /* BLOSC_CONTEXT_H */ + +#endif /* CONTEXT_H */ diff --git a/src/blosc2/blosc/delta.c b/src/blosc2/blosc/delta.c index adddfbf..1559075 100644 --- a/src/blosc2/blosc/delta.c +++ b/src/blosc2/blosc/delta.c @@ -1,17 +1,15 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ -#include "delta.h" - #include -#include +#include "delta.h" /* Apply the delta filters to src. This can never fail. */ diff --git a/src/blosc2/blosc/delta.h b/src/blosc2/blosc/delta.h index 2e9ce9a..8ca26de 100644 --- a/src/blosc2/blosc/delta.h +++ b/src/blosc2/blosc/delta.h @@ -1,7 +1,7 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) @@ -11,6 +11,7 @@ #ifndef BLOSC_DELTA_H #define BLOSC_DELTA_H +#include #include void delta_encoder(const uint8_t* dref, int32_t offset, int32_t nbytes, @@ -19,4 +20,4 @@ void delta_encoder(const uint8_t* dref, int32_t offset, int32_t nbytes, void delta_decoder(const uint8_t* dref, int32_t offset, int32_t nbytes, int32_t typesize, uint8_t* dest); -#endif /* BLOSC_DELTA_H */ +#endif //BLOSC_DELTA_H diff --git a/src/blosc2/blosc/directories.c b/src/blosc2/blosc/directories.c index 13a3ac0..3fb4af6 100644 --- a/src/blosc2/blosc/directories.c +++ b/src/blosc2/blosc/directories.c @@ -1,21 +1,16 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ +#include #include "blosc2.h" - #include -#include - -#include -#include -#include #if defined(_WIN32) || defined(__MINGW32__) #include @@ -131,10 +126,6 @@ int blosc2_remove_urlpath(const char* urlpath){ if (urlpath != NULL) { struct stat statbuf; if (stat(urlpath, &statbuf) != 0){ - if (errno == ENOENT) { - // Path does not exist - return BLOSC2_ERROR_SUCCESS; - } BLOSC_TRACE_ERROR("Could not access %s", urlpath); return BLOSC2_ERROR_FAILURE; } diff --git a/src/blosc2/blosc/fastcopy.c b/src/blosc2/blosc/fastcopy.c index 81e866c..43f946c 100644 --- a/src/blosc2/blosc/fastcopy.c +++ b/src/blosc2/blosc/fastcopy.c @@ -1,7 +1,7 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) @@ -18,13 +18,8 @@ * Support for SSE2/AVX2 copy instructions for these routines **********************************************************************/ -#include "blosc2/blosc2-common.h" - #include -#include -#if defined(BLOSC_STRICT_ALIGN) -#include -#endif +#include "blosc2/blosc2-common.h" /* * Use inlined functions for supported systems. diff --git a/src/blosc2/blosc/fastcopy.h b/src/blosc2/blosc/fastcopy.h index ae2c27a..0993486 100644 --- a/src/blosc2/blosc/fastcopy.h +++ b/src/blosc2/blosc/fastcopy.h @@ -1,7 +1,7 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) @@ -17,4 +17,4 @@ unsigned char *fastcopy(unsigned char *out, const unsigned char *from, unsigned /* Same as fastcopy() but without overwriting origin or destination when they overlap */ unsigned char* copy_match(unsigned char *out, const unsigned char *from, unsigned len); -#endif /* BLOSC_FASTCOPY_H */ +#endif //BLOSC_FASTCOPY_H diff --git a/src/blosc2/blosc/frame.c b/src/blosc2/blosc/frame.c index 4808a6c..c129479 100644 --- a/src/blosc2/blosc/frame.c +++ b/src/blosc2/blosc/frame.c @@ -1,32 +1,37 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ +#include +#include +#include +#include +#include +#include "blosc2.h" +#include "blosc-private.h" +#include "context.h" #include "frame.h" #include "sframe.h" -#include "context.h" -#include "blosc-private.h" -#include "blosc2.h" +#include #if defined(_WIN32) #include -#include -#endif /* _WIN32 */ + #include -#include +/* stdint.h only available in VS2010 (VC++ 16.0) and newer */ + #if defined(_MSC_VER) && _MSC_VER < 1600 + #include "win32/stdint-windows.h" + #else + #include + #endif -#include -#include -#include -#include -#include -#include +#endif /* _WIN32 */ /* If C11 is supported, use it's built-in aligned allocation. */ #if __STDC_VERSION__ >= 201112L @@ -961,10 +966,6 @@ int64_t frame_from_schunk(blosc2_schunk *schunk, blosc2_frame_s *frame) { // Compress the chunk of offsets off_chunk = malloc(off_nbytes + BLOSC2_MAX_OVERHEAD); blosc2_context *cctx = blosc2_create_cctx(BLOSC2_CPARAMS_DEFAULTS); - if (cctx == NULL) { - BLOSC_TRACE_ERROR("Error while creating the compression context"); - return BLOSC2_ERROR_NULL_POINTER; - } cctx->typesize = sizeof(int64_t); off_cbytes = blosc2_compress_ctx(cctx, data_tmp, off_nbytes, off_chunk, off_nbytes + BLOSC2_MAX_OVERHEAD); @@ -1195,10 +1196,6 @@ int64_t* blosc2_frame_get_offsets(blosc2_schunk *schunk) { // Decompress offsets blosc2_dparams off_dparams = BLOSC2_DPARAMS_DEFAULTS; blosc2_context *dctx = blosc2_create_dctx(off_dparams); - if (dctx == NULL) { - BLOSC_TRACE_ERROR("Error while creating the decompression context"); - return NULL; - } int32_t prev_nbytes = blosc2_decompress_ctx(dctx, coffsets, coffsets_cbytes, offsets, off_nbytes); blosc2_free_ctx(dctx); @@ -1354,7 +1351,7 @@ static int get_meta_from_header(blosc2_frame_s* frame, blosc2_schunk* schunk, ui if ((*idxp & 0xe0u) != 0xa0u) { // sanity check return BLOSC2_ERROR_DATA; } - blosc2_metalayer* metalayer = calloc(1, sizeof(blosc2_metalayer)); + blosc2_metalayer* metalayer = calloc(sizeof(blosc2_metalayer), 1); schunk->metalayers[nmetalayer] = metalayer; // Populate the metalayer string @@ -1534,7 +1531,7 @@ static int get_vlmeta_from_trailer(blosc2_frame_s* frame, blosc2_schunk* schunk, if ((*idxp & 0xe0u) != 0xa0u) { // sanity check return BLOSC2_ERROR_DATA; } - blosc2_metalayer* metalayer = calloc(1, sizeof(blosc2_metalayer)); + blosc2_metalayer* metalayer = calloc(sizeof(blosc2_metalayer), 1); schunk->vlmetalayers[nmetalayer] = metalayer; // Populate the metalayer string @@ -1643,12 +1640,11 @@ int frame_get_vlmetalayers(blosc2_frame_s* frame, blosc2_schunk* schunk) { char* eframe_name = malloc(strlen(frame->urlpath) + strlen("/chunks.b2frame") + 1); sprintf(eframe_name, "%s/chunks.b2frame", frame->urlpath); fp = io_cb->open(eframe_name, "rb", frame->schunk->storage->io->params); + free(eframe_name); if (fp == NULL) { BLOSC_TRACE_ERROR("Error opening file in: %s", eframe_name); - free(eframe_name); return BLOSC2_ERROR_FILE_OPEN; } - free(eframe_name); io_cb->seek(fp, trailer_offset, SEEK_SET); } else { @@ -1754,17 +1750,9 @@ blosc2_schunk* frame_to_schunk(blosc2_frame_s* frame, bool copy, const blosc2_io blosc2_cparams *cparams; blosc2_schunk_get_cparams(schunk, &cparams); schunk->cctx = blosc2_create_cctx(*cparams); - if (schunk->cctx == NULL) { - BLOSC_TRACE_ERROR("Error while creating the compression context"); - return NULL; - } blosc2_dparams *dparams; blosc2_schunk_get_dparams(schunk, &dparams); schunk->dctx = blosc2_create_dctx(*dparams); - if (schunk->dctx == NULL) { - BLOSC_TRACE_ERROR("Error while creating the decompression context"); - return NULL; - } blosc2_storage storage = {.contiguous = copy ? false : true}; schunk->storage = get_new_storage(&storage, cparams, dparams, udio); free(cparams); @@ -1793,10 +1781,6 @@ blosc2_schunk* frame_to_schunk(blosc2_frame_s* frame, bool copy, const blosc2_io // Decompress offsets blosc2_dparams off_dparams = BLOSC2_DPARAMS_DEFAULTS; blosc2_context *dctx = blosc2_create_dctx(off_dparams); - if (dctx == NULL) { - BLOSC_TRACE_ERROR("Error while creating the decompression context"); - return NULL; - } int64_t* offsets = (int64_t *) malloc((size_t)nchunks * sizeof(int64_t)); int32_t off_nbytes = blosc2_decompress_ctx(dctx, coffsets, coffsets_cbytes, offsets, (int32_t)(nchunks * sizeof(int64_t))); @@ -2644,10 +2628,6 @@ void* frame_append_chunk(blosc2_frame_s* frame, void* chunk, blosc2_schunk* schu // Decompress offsets blosc2_dparams off_dparams = BLOSC2_DPARAMS_DEFAULTS; blosc2_context *dctx = blosc2_create_dctx(off_dparams); - if (dctx == NULL) { - BLOSC_TRACE_ERROR("Error while creating the decompression context"); - return NULL; - } int32_t prev_nbytes = blosc2_decompress_ctx(dctx, coffsets, coffsets_cbytes, offsets, off_nbytes); blosc2_free_ctx(dctx); @@ -2704,10 +2684,6 @@ void* frame_append_chunk(blosc2_frame_s* frame, void* chunk, blosc2_schunk* schu cparams.nthreads = 4; // 4 threads seems a decent default for nowadays CPUs cparams.compcode = BLOSC_BLOSCLZ; blosc2_context* cctx = blosc2_create_cctx(cparams); - if (cctx == NULL) { - BLOSC_TRACE_ERROR("Error while creating the compression context"); - return NULL; - } cctx->typesize = sizeof(int64_t); // override a possible BLOSC_TYPESIZE env variable (or chaos may appear) void* off_chunk = malloc((size_t)off_nbytes + BLOSC2_MAX_OVERHEAD); int32_t new_off_cbytes = blosc2_compress_ctx(cctx, offsets, off_nbytes, @@ -2856,10 +2832,6 @@ void* frame_insert_chunk(blosc2_frame_s* frame, int64_t nchunk, void* chunk, blo // Decompress offsets blosc2_dparams off_dparams = BLOSC2_DPARAMS_DEFAULTS; blosc2_context *dctx = blosc2_create_dctx(off_dparams); - if (dctx == NULL) { - BLOSC_TRACE_ERROR("Error while creating the decompression context"); - return NULL; - } int32_t prev_nbytes = blosc2_decompress_ctx(dctx, coffsets, coffsets_cbytes, offsets, off_nbytes); blosc2_free_ctx(dctx); if (prev_nbytes < 0) { @@ -2921,10 +2893,6 @@ void* frame_insert_chunk(blosc2_frame_s* frame, int64_t nchunk, void* chunk, blo cparams.nthreads = 4; // 4 threads seems a decent default for nowadays CPUs cparams.compcode = BLOSC_BLOSCLZ; blosc2_context* cctx = blosc2_create_cctx(cparams); - if (cctx == NULL) { - BLOSC_TRACE_ERROR("Error while creating the compression context"); - return NULL; - } void* off_chunk = malloc((size_t)off_nbytes + BLOSC2_MAX_OVERHEAD); int32_t new_off_cbytes = blosc2_compress_ctx(cctx, offsets, off_nbytes, off_chunk, off_nbytes + BLOSC2_MAX_OVERHEAD); @@ -3079,10 +3047,6 @@ void* frame_update_chunk(blosc2_frame_s* frame, int64_t nchunk, void* chunk, blo // Decompress offsets blosc2_dparams off_dparams = BLOSC2_DPARAMS_DEFAULTS; blosc2_context *dctx = blosc2_create_dctx(off_dparams); - if (dctx == NULL) { - BLOSC_TRACE_ERROR("Error while creating the decompression context"); - return NULL; - } int32_t prev_nbytes = blosc2_decompress_ctx(dctx, coffsets, coffsets_cbytes, offsets, off_nbytes); blosc2_free_ctx(dctx); if (prev_nbytes < 0) { @@ -3179,10 +3143,6 @@ void* frame_update_chunk(blosc2_frame_s* frame, int64_t nchunk, void* chunk, blo cparams.nthreads = 4; // 4 threads seems a decent default for nowadays CPUs cparams.compcode = BLOSC_BLOSCLZ; blosc2_context* cctx = blosc2_create_cctx(cparams); - if (cctx == NULL) { - BLOSC_TRACE_ERROR("Error while creating the compression context"); - return NULL; - } void* off_chunk = malloc((size_t)off_nbytes + BLOSC2_MAX_OVERHEAD); int32_t new_off_cbytes = blosc2_compress_ctx(cctx, offsets, off_nbytes, off_chunk, off_nbytes + BLOSC2_MAX_OVERHEAD); @@ -3322,10 +3282,6 @@ void* frame_delete_chunk(blosc2_frame_s* frame, int64_t nchunk, blosc2_schunk* s // Decompress offsets blosc2_dparams off_dparams = BLOSC2_DPARAMS_DEFAULTS; blosc2_context *dctx = blosc2_create_dctx(off_dparams); - if (dctx == NULL) { - BLOSC_TRACE_ERROR("Error while creating the decompression context"); - return NULL; - } int32_t prev_nbytes = blosc2_decompress_ctx(dctx, coffsets, coffsets_cbytes, offsets, off_nbytes); blosc2_free_ctx(dctx); if (prev_nbytes < 0) { @@ -3349,10 +3305,6 @@ void* frame_delete_chunk(blosc2_frame_s* frame, int64_t nchunk, blosc2_schunk* s cparams.nthreads = 4; // 4 threads seems a decent default for nowadays CPUs cparams.compcode = BLOSC_BLOSCLZ; blosc2_context* cctx = blosc2_create_cctx(cparams); - if (cctx == NULL) { - BLOSC_TRACE_ERROR("Error while creating the compression context"); - return NULL; - } void* off_chunk = malloc((size_t)off_nbytes + BLOSC2_MAX_OVERHEAD); int32_t new_off_cbytes = blosc2_compress_ctx(cctx, offsets, off_nbytes - (int32_t)sizeof(int64_t), off_chunk, off_nbytes + BLOSC2_MAX_OVERHEAD); @@ -3488,10 +3440,6 @@ int frame_reorder_offsets(blosc2_frame_s* frame, const int64_t* offsets_order, b // Decompress offsets blosc2_dparams off_dparams = BLOSC2_DPARAMS_DEFAULTS; blosc2_context *dctx = blosc2_create_dctx(off_dparams); - if (dctx == NULL) { - BLOSC_TRACE_ERROR("Error while creating the decompression context"); - return BLOSC2_ERROR_NULL_POINTER; - } int32_t prev_nbytes = blosc2_decompress_ctx(dctx, coffsets, coffsets_cbytes, offsets, off_nbytes); blosc2_free_ctx(dctx); @@ -3518,10 +3466,6 @@ int frame_reorder_offsets(blosc2_frame_s* frame, const int64_t* offsets_order, b cparams.nthreads = 4; // 4 threads seems a decent default for nowadays CPUs cparams.compcode = BLOSC_BLOSCLZ; blosc2_context* cctx = blosc2_create_cctx(cparams); - if (cctx == NULL) { - BLOSC_TRACE_ERROR("Error while creating the compression context"); - return BLOSC2_ERROR_NULL_POINTER; - } void* off_chunk = malloc((size_t)off_nbytes + BLOSC2_MAX_OVERHEAD); int32_t new_off_cbytes = blosc2_compress_ctx(cctx, offsets, off_nbytes, off_chunk, off_nbytes + BLOSC2_MAX_OVERHEAD); diff --git a/src/blosc2/blosc/frame.h b/src/blosc2/blosc/frame.h index 20e7687..18eb4f3 100644 --- a/src/blosc2/blosc/frame.h +++ b/src/blosc2/blosc/frame.h @@ -1,7 +1,7 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) @@ -11,9 +11,7 @@ #ifndef BLOSC_FRAME_H #define BLOSC_FRAME_H -#include "blosc2.h" - -#include +#include #include // Different types of frames @@ -167,4 +165,4 @@ int frame_update_trailer(blosc2_frame_s* frame, blosc2_schunk* schunk); int64_t frame_fill_special(blosc2_frame_s* frame, int64_t nitems, int special_value, int32_t chunksize, blosc2_schunk* schunk); -#endif /* BLOSC_FRAME_H */ +#endif //BLOSC_FRAME_H diff --git a/src/blosc2/blosc/schunk.c b/src/blosc2/blosc/schunk.c index 20f0bf3..e4168fb 100644 --- a/src/blosc2/blosc/schunk.c +++ b/src/blosc2/blosc/schunk.c @@ -1,33 +1,40 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ + +#include +#include +#include +#include +#include "blosc2.h" #include "frame.h" #include "stune.h" +#include #include "blosc-private.h" -#include "blosc2/tuners-registry.h" -#include "blosc2.h" #if defined(_WIN32) -#include -#include -#include -#define mkdir(D, M) _mkdir(D) -#endif /* _WIN32 */ + #include + #include + #include -#include + #define mkdir(D, M) _mkdir(D) + +/* stdint.h only available in VS2010 (VC++ 16.0) and newer */ + #if defined(_MSC_VER) && _MSC_VER < 1600 + #include "win32/stdint-windows.h" + #else + #include + #endif + +#endif /* _WIN32 */ -#include -#include -#include -#include -#include /* If C11 is supported, use it's built-in aligned allocation. */ #if __STDC_VERSION__ >= 201112L @@ -37,7 +44,7 @@ /* Get the cparams associated with a super-chunk */ int blosc2_schunk_get_cparams(blosc2_schunk *schunk, blosc2_cparams **cparams) { - *cparams = calloc(1, sizeof(blosc2_cparams)); + *cparams = calloc(sizeof(blosc2_cparams), 1); (*cparams)->schunk = schunk; for (int i = 0; i < BLOSC2_MAX_FILTERS; i++) { (*cparams)->filters[i] = schunk->filters[i]; @@ -61,7 +68,7 @@ int blosc2_schunk_get_cparams(blosc2_schunk *schunk, blosc2_cparams **cparams) { /* Get the dparams associated with a super-chunk */ int blosc2_schunk_get_dparams(blosc2_schunk *schunk, blosc2_dparams **dparams) { - *dparams = calloc(1, sizeof(blosc2_dparams)); + *dparams = calloc(sizeof(blosc2_dparams), 1); (*dparams)->schunk = schunk; if (schunk->dctx == NULL) { (*dparams)->nthreads = blosc2_get_nthreads(); @@ -73,7 +80,7 @@ int blosc2_schunk_get_dparams(blosc2_schunk *schunk, blosc2_dparams **dparams) { } -int update_schunk_properties(struct blosc2_schunk* schunk) { +void update_schunk_properties(struct blosc2_schunk* schunk) { blosc2_cparams* cparams = schunk->storage->cparams; blosc2_dparams* dparams = schunk->storage->dparams; @@ -88,21 +95,13 @@ int update_schunk_properties(struct blosc2_schunk* schunk) { schunk->typesize = cparams->typesize; schunk->blocksize = cparams->blocksize; schunk->chunksize = -1; - schunk->tuner_params = cparams->tuner_params; - schunk->tuner_id = cparams->tuner_id; - if (cparams->tuner_id == BLOSC_BTUNE) { - cparams->use_dict = 0; - } + /* The compression context */ if (schunk->cctx != NULL) { blosc2_free_ctx(schunk->cctx); } cparams->schunk = schunk; schunk->cctx = blosc2_create_cctx(*cparams); - if (schunk->cctx == NULL) { - BLOSC_TRACE_ERROR("Could not create compression ctx"); - return BLOSC2_ERROR_NULL_POINTER; - } /* The decompression context */ if (schunk->dctx != NULL) { @@ -110,12 +109,6 @@ int update_schunk_properties(struct blosc2_schunk* schunk) { } dparams->schunk = schunk; schunk->dctx = blosc2_create_dctx(*dparams); - if (schunk->dctx == NULL) { - BLOSC_TRACE_ERROR("Could not create decompression ctx"); - return BLOSC2_ERROR_NULL_POINTER; - } - - return BLOSC2_ERROR_SUCCESS; } @@ -135,17 +128,18 @@ blosc2_schunk* blosc2_schunk_new(blosc2_storage *storage) { // Update the (local variable) storage storage = schunk->storage; - char* tradeoff = getenv("BTUNE_TRADEOFF"); - if (tradeoff != NULL) { - // If BTUNE_TRADEOFF passed, automatically use btune - storage->cparams->tuner_id = BLOSC_BTUNE; + schunk->udbtune = malloc(sizeof(blosc2_btune)); + if (schunk->storage->cparams->udbtune == NULL) { + memcpy(schunk->udbtune, &BTUNE_DEFAULTS, sizeof(blosc2_btune)); + } else { + memcpy(schunk->udbtune, schunk->storage->cparams->udbtune, sizeof(blosc2_btune)); } + schunk->storage->cparams->udbtune = schunk->udbtune; // ...and update internal properties - if (update_schunk_properties(schunk) < 0) { - BLOSC_TRACE_ERROR("Error when updating schunk properties"); - return NULL; - } + update_schunk_properties(schunk); + + schunk->cctx->udbtune->btune_init(schunk->udbtune->btune_config, schunk->cctx, schunk->dctx); if (!storage->contiguous && storage->urlpath != NULL){ char* urlpath; @@ -337,7 +331,7 @@ blosc2_schunk* blosc2_schunk_open(const char* urlpath) { return blosc2_schunk_open_udio(urlpath, &BLOSC2_IO_DEFAULTS); } -blosc2_schunk* blosc2_schunk_open_offset(const char* urlpath, int64_t offset) { +BLOSC_EXPORT blosc2_schunk* blosc2_schunk_open_offset(const char* urlpath, int64_t offset) { if (urlpath == NULL) { BLOSC_TRACE_ERROR("You need to supply a urlpath."); return NULL; @@ -546,6 +540,9 @@ int blosc2_schunk_free(blosc2_schunk *schunk) { } } + if (schunk->udbtune != NULL) { + free(schunk->udbtune); + } free(schunk); return 0; @@ -1593,10 +1590,6 @@ int blosc2_vlmeta_add(blosc2_schunk *schunk, const char *name, uint8_t *content, } else { cctx = blosc2_create_cctx(BLOSC2_CPARAMS_DEFAULTS); } - if (cctx == NULL) { - BLOSC_TRACE_ERROR("Error while creating the compression context"); - return BLOSC2_ERROR_NULL_POINTER; - } int csize = blosc2_compress_ctx(cctx, content, content_len, content_buf, content_len + BLOSC2_MAX_OVERHEAD); if (csize < 0) { @@ -1638,10 +1631,6 @@ int blosc2_vlmeta_get(blosc2_schunk *schunk, const char *name, uint8_t **content *content_len = nbytes; *content = malloc((size_t) nbytes); blosc2_context *dctx = blosc2_create_dctx(*schunk->storage->dparams); - if (dctx == NULL) { - BLOSC_TRACE_ERROR("Error while creating the decompression context"); - return BLOSC2_ERROR_NULL_POINTER; - } int nbytes_ = blosc2_decompress_ctx(dctx, meta->content, meta->content_len, *content, nbytes); blosc2_free_ctx(dctx); if (nbytes_ != nbytes) { @@ -1669,10 +1658,6 @@ int blosc2_vlmeta_update(blosc2_schunk *schunk, const char *name, uint8_t *conte } else { cctx = blosc2_create_cctx(BLOSC2_CPARAMS_DEFAULTS); } - if (cctx == NULL) { - BLOSC_TRACE_ERROR("Error while creating the compression context"); - return BLOSC2_ERROR_NULL_POINTER; - } int csize = blosc2_compress_ctx(cctx, content, content_len, content_buf, content_len + BLOSC2_MAX_OVERHEAD); if (csize < 0) { diff --git a/src/blosc2/blosc/sframe.c b/src/blosc2/blosc/sframe.c index efac04e..f852422 100644 --- a/src/blosc2/blosc/sframe.c +++ b/src/blosc2/blosc/sframe.c @@ -1,20 +1,18 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ -#include "frame.h" -#include "blosc2.h" - #include -#include #include #include +#include "blosc2.h" +#include "frame.h" /* If C11 is supported, use it's built-in aligned allocation. */ @@ -35,9 +33,11 @@ void* sframe_open_index(const char* urlpath, const char* mode, const blosc2_io * return NULL; } fp = io_cb->open(index_path, mode, io->params); - if (fp == NULL) - BLOSC_TRACE_ERROR("Error creating index path in: %s", index_path); free(index_path); + if (fp == NULL) { + BLOSC_TRACE_ERROR("Error creating index path in: %s", index_path); + return NULL; + } } return fp; } @@ -54,9 +54,11 @@ void* sframe_open_chunk(const char* urlpath, int64_t nchunk, const char* mode, c return NULL; } fp = io_cb->open(chunk_path, mode, io->params); - if (fp == NULL) - BLOSC_TRACE_ERROR("Error opening chunk path in: %s", chunk_path); free(chunk_path); + if (fp == NULL) { + BLOSC_TRACE_ERROR("Error opening chunk path in: %s", chunk_path); + return NULL; + } } return fp; } diff --git a/src/blosc2/blosc/sframe.h b/src/blosc2/blosc/sframe.h index b62979f..75ea239 100644 --- a/src/blosc2/blosc/sframe.h +++ b/src/blosc2/blosc/sframe.h @@ -1,7 +1,7 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) @@ -11,15 +11,10 @@ #ifndef BLOSC_SFRAME_H #define BLOSC_SFRAME_H -#include "frame.h" - -#include -#include - void* sframe_open_index(const char* urlpath, const char* mode, const blosc2_io *io); void* sframe_open_chunk(const char* urlpath, int64_t nchunk, const char* mode, const blosc2_io *io); int sframe_delete_chunk(const char* urlpath, int64_t nchunk); void* sframe_create_chunk(blosc2_frame_s* frame, uint8_t* chunk, int64_t nchunk, int64_t cbytes); int32_t sframe_get_chunk(blosc2_frame_s* frame, int64_t nchunk, uint8_t** chunk, bool* needs_free); -#endif /* BLOSC_SFRAME_H */ +#endif //BLOSC_SFRAME_H diff --git a/src/blosc2/blosc/shuffle-altivec.c b/src/blosc2/blosc/shuffle-altivec.c index d0077c5..2de003b 100644 --- a/src/blosc2/blosc/shuffle-altivec.c +++ b/src/blosc2/blosc/shuffle-altivec.c @@ -1,24 +1,21 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team and Jerome Kieffer + Copyright (C) 2021 The Blosc developers and Jerome Kieffer https://blosc.org License: BSD 3-Clause (see LICENSE.txt) See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ -#include "shuffle-altivec.h" #include "shuffle-generic.h" +#include "shuffle-altivec.h" /* Make sure ALTIVEC is available for the compilation target and compiler. */ #if defined(__ALTIVEC__) -#include "transpose-altivec.h" - #include - -#include +#include "transpose-altivec.h" /* Routine optimized for shuffling a buffer for a type size of 2 bytes. */ static void diff --git a/src/blosc2/blosc/shuffle-altivec.h b/src/blosc2/blosc/shuffle-altivec.h index 1fd98a0..3825557 100644 --- a/src/blosc2/blosc/shuffle-altivec.h +++ b/src/blosc2/blosc/shuffle-altivec.h @@ -1,7 +1,7 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) @@ -10,12 +10,14 @@ /* ALTIVEC-accelerated shuffle/unshuffle routines. */ -#ifndef BLOSC_SHUFFLE_ALTIVEC_H -#define BLOSC_SHUFFLE_ALTIVEC_H +#ifndef SHUFFLE_ALTIVEC_H +#define SHUFFLE_ALTIVEC_H #include "blosc2/blosc2-common.h" -#include +#ifdef __cplusplus +extern "C" { +#endif /** ALTIVEC-accelerated shuffle routine. @@ -29,4 +31,8 @@ BLOSC_NO_EXPORT void shuffle_altivec(const int32_t bytesoftype, const int32_t bl BLOSC_NO_EXPORT void unshuffle_altivec(const int32_t bytesoftype, const int32_t blocksize, const uint8_t *_src, uint8_t *_dest); -#endif /* BLOSC_SHUFFLE_ALTIVEC_H */ +#ifdef __cplusplus +} +#endif + +#endif /* SHUFFLE_ALTIVEC_H */ diff --git a/src/blosc2/blosc/shuffle-avx2.c b/src/blosc2/blosc/shuffle-avx2.c index efb10bf..ae7dad3 100644 --- a/src/blosc2/blosc/shuffle-avx2.c +++ b/src/blosc2/blosc/shuffle-avx2.c @@ -1,23 +1,21 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ -#include "shuffle-avx2.h" #include "shuffle-generic.h" +#include "shuffle-avx2.h" /* Make sure AVX2 is available for the compilation target and compiler. */ #if defined(__AVX2__) #include -#include -#include /* The next is useful for debugging purposes */ #if 0 diff --git a/src/blosc2/blosc/shuffle-avx2.h b/src/blosc2/blosc/shuffle-avx2.h index 1159ad5..98c053f 100644 --- a/src/blosc2/blosc/shuffle-avx2.h +++ b/src/blosc2/blosc/shuffle-avx2.h @@ -1,7 +1,7 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) @@ -15,7 +15,9 @@ #include "blosc2/blosc2-common.h" -#include +#ifdef __cplusplus +extern "C" { +#endif /** AVX2-accelerated shuffle routine. @@ -29,4 +31,8 @@ BLOSC_NO_EXPORT void shuffle_avx2(const int32_t bytesoftype, const int32_t block BLOSC_NO_EXPORT void unshuffle_avx2(const int32_t bytesoftype, const int32_t blocksize, const uint8_t *_src, uint8_t *_dest); +#ifdef __cplusplus +} +#endif + #endif /* SHUFFLE_AVX2_H */ diff --git a/src/blosc2/blosc/shuffle-generic.c b/src/blosc2/blosc/shuffle-generic.c index 1c50a6a..ebd0720 100644 --- a/src/blosc2/blosc/shuffle-generic.c +++ b/src/blosc2/blosc/shuffle-generic.c @@ -1,7 +1,7 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) diff --git a/src/blosc2/blosc/shuffle-generic.h b/src/blosc2/blosc/shuffle-generic.h index 5c2e012..6ed95fa 100644 --- a/src/blosc2/blosc/shuffle-generic.h +++ b/src/blosc2/blosc/shuffle-generic.h @@ -1,7 +1,7 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) @@ -16,13 +16,16 @@ which isn't a multiple of the hardware's vector size. **********************************************************************/ -#ifndef BLOSC_SHUFFLE_GENERIC_H -#define BLOSC_SHUFFLE_GENERIC_H + +#ifndef SHUFFLE_GENERIC_H +#define SHUFFLE_GENERIC_H #include "blosc2/blosc2-common.h" +#include -#include -#include +#ifdef __cplusplus +extern "C" { +#endif /** Generic (non-hardware-accelerated) shuffle routine. @@ -31,7 +34,7 @@ implementations to process any remaining elements in a block which is not a multiple of (type_size * vector_size). */ -static inline void shuffle_generic_inline(const int32_t type_size, +inline static void shuffle_generic_inline(const int32_t type_size, const int32_t vectorizable_blocksize, const int32_t blocksize, const uint8_t *_src, uint8_t *_dest) { int32_t i, j; @@ -59,7 +62,7 @@ static inline void shuffle_generic_inline(const int32_t type_size, implementations to process any remaining elements in a block which is not a multiple of (type_size * vector_size). */ -static inline void unshuffle_generic_inline(const int32_t type_size, +inline static void unshuffle_generic_inline(const int32_t type_size, const int32_t vectorizable_blocksize, const int32_t blocksize, const uint8_t *_src, uint8_t *_dest) { int32_t i, j; @@ -92,4 +95,8 @@ BLOSC_NO_EXPORT void shuffle_generic(const int32_t bytesoftype, const int32_t bl BLOSC_NO_EXPORT void unshuffle_generic(const int32_t bytesoftype, const int32_t blocksize, const uint8_t *_src, uint8_t *_dest); -#endif /* BLOSC_SHUFFLE_GENERIC_H */ +#ifdef __cplusplus +} +#endif + +#endif /* SHUFFLE_GENERIC_H */ diff --git a/src/blosc2/blosc/shuffle-neon.c b/src/blosc2/blosc/shuffle-neon.c index c1940c1..c7cd20e 100644 --- a/src/blosc2/blosc/shuffle-neon.c +++ b/src/blosc2/blosc/shuffle-neon.c @@ -1,7 +1,7 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 Lucian Marc + Copyright (C) 2021 Lucian Marc https://blosc.org License: BSD 3-Clause (see LICENSE.txt) @@ -9,15 +9,14 @@ **********************************************************************/ -#include "shuffle-neon.h" #include "shuffle-generic.h" +#include "shuffle-neon.h" /* Make sure NEON is available for the compilation target and compiler. */ #if defined(__ARM_NEON) #include -#include /* The next is useful for debugging purposes */ #if 0 diff --git a/src/blosc2/blosc/shuffle-neon.h b/src/blosc2/blosc/shuffle-neon.h index b7cb6a2..3b54655 100644 --- a/src/blosc2/blosc/shuffle-neon.h +++ b/src/blosc2/blosc/shuffle-neon.h @@ -1,7 +1,7 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) @@ -12,12 +12,14 @@ /* NEON-accelerated shuffle/unshuffle routines. */ -#ifndef BLOSC_SHUFFLE_NEON_H -#define BLOSC_SHUFFLE_NEON_H +#ifndef SHUFFLE_NEON_H +#define SHUFFLE_NEON_H #include "blosc2/blosc2-common.h" -#include +#ifdef __cplusplus +extern "C" { +#endif /** NEON-accelerated shuffle routine. @@ -31,4 +33,8 @@ BLOSC_NO_EXPORT void shuffle_neon(const int32_t bytesoftype, const int32_t block BLOSC_NO_EXPORT void unshuffle_neon(const int32_t bytesoftype, const int32_t blocksize, const uint8_t *_src, uint8_t *_dest); -#endif /* BLOSC_SHUFFLE_NEON_H */ +#ifdef __cplusplus +} +#endif + +#endif /* SHUFFLE_NEON_H */ diff --git a/src/blosc2/blosc/shuffle-sse2.c b/src/blosc2/blosc/shuffle-sse2.c index deee4a4..7b804ee 100644 --- a/src/blosc2/blosc/shuffle-sse2.c +++ b/src/blosc2/blosc/shuffle-sse2.c @@ -1,22 +1,21 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ -#include "shuffle-sse2.h" #include "shuffle-generic.h" +#include "shuffle-sse2.h" /* Make sure SSE2 is available for the compilation target and compiler. */ #if defined(__SSE2__) #include -#include /* The next is useful for debugging purposes */ #if 0 diff --git a/src/blosc2/blosc/shuffle-sse2.h b/src/blosc2/blosc/shuffle-sse2.h index 9962b3f..783641d 100644 --- a/src/blosc2/blosc/shuffle-sse2.h +++ b/src/blosc2/blosc/shuffle-sse2.h @@ -1,7 +1,7 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) @@ -10,12 +10,14 @@ /* SSE2-accelerated shuffle/unshuffle routines. */ -#ifndef BLOSC_SHUFFLE_SSE2_H -#define BLOSC_SHUFFLE_SSE2_H +#ifndef SHUFFLE_SSE2_H +#define SHUFFLE_SSE2_H #include "blosc2/blosc2-common.h" -#include +#ifdef __cplusplus +extern "C" { +#endif /** SSE2-accelerated shuffle routine. @@ -29,4 +31,8 @@ BLOSC_NO_EXPORT void shuffle_sse2(const int32_t bytesoftype, const int32_t block BLOSC_NO_EXPORT void unshuffle_sse2(const int32_t bytesoftype, const int32_t blocksize, const uint8_t *_src, uint8_t *_dest); -#endif /* BLOSC_SHUFFLE_SSE2_H */ +#ifdef __cplusplus +} +#endif + +#endif /* SHUFFLE_SSE2_H */ diff --git a/src/blosc2/blosc/shuffle.c b/src/blosc2/blosc/shuffle.c index 2fff396..ca91b59 100644 --- a/src/blosc2/blosc/shuffle.c +++ b/src/blosc2/blosc/shuffle.c @@ -1,14 +1,26 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ -#include "shuffle.h" /* needs to be included first to define macros */ +#include "shuffle.h" +#include "blosc2/blosc2-common.h" +#include "shuffle-generic.h" +#include "bitshuffle-generic.h" +#include +#include +#include + + +#if !defined(__clang__) && defined(__GNUC__) && defined(__GNUC_MINOR__) && \ + __GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8) +#define HAVE_CPU_FEAT_INTRIN +#endif /* Include hardware-accelerated shuffle/unshuffle routines based on the target architecture. Note that a target architecture may support @@ -39,22 +51,6 @@ #include "bitshuffle-altivec.h" #endif /* defined(SHUFFLE_USE_ALTIVEC) */ -#include "shuffle-generic.h" -#include "bitshuffle-generic.h" -#include "blosc2/blosc2-common.h" -#include "blosc2.h" - -#include -#include -#include -#include - - -#if !defined(__clang__) && defined(__GNUC__) && defined(__GNUC_MINOR__) && \ - __GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8) -#define HAVE_CPU_FEAT_INTRIN -#endif - /* Define function pointer types for shuffle/unshuffle routines. */ typedef void(* shuffle_func)(const int32_t, const int32_t, const uint8_t*, const uint8_t*); @@ -437,7 +433,7 @@ bitshuffle(const int32_t bytesoftype, const int32_t blocksize, size, bytesoftype, (void *) _tmp); if (ret < 0) { // Some error in bitshuffle (should not happen) - BLOSC_TRACE_ERROR("the impossible happened: the bitshuffle filter failed!"); + fprintf(stderr, "the impossible happened: the bitshuffle filter failed!"); return ret; } @@ -467,7 +463,7 @@ int32_t bitunshuffle(const int32_t bytesoftype, const int32_t blocksize, bytesoftype, (void *) _tmp); if (ret < 0) { // Some error in bitshuffle (should not happen) - BLOSC_TRACE_ERROR("the impossible happened: the bitunshuffle filter failed!"); + fprintf(stderr, "the impossible happened: the bitunshuffle filter failed!"); return ret; } /* Copy the leftovers (we do so starting from c-blosc 1.18 on) */ @@ -484,7 +480,7 @@ int32_t bitunshuffle(const int32_t bytesoftype, const int32_t blocksize, int ret = (int) (host_implementation.bitunshuffle)((void *) _src, (void *) _dest, size, bytesoftype, (void *) _tmp); if (ret < 0) { - BLOSC_TRACE_ERROR("the impossible happened: the bitunshuffle filter failed!"); + fprintf(stderr, "the impossible happened: the bitunshuffle filter failed!"); return ret; } diff --git a/src/blosc2/blosc/shuffle.h b/src/blosc2/blosc/shuffle.h index 24784ea..7f98d29 100644 --- a/src/blosc2/blosc/shuffle.h +++ b/src/blosc2/blosc/shuffle.h @@ -1,7 +1,7 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) @@ -16,12 +16,11 @@ these are cross-platform and future-proof. **********************************************************************/ -#ifndef BLOSC_SHUFFLE_H -#define BLOSC_SHUFFLE_H -#include "blosc2/blosc2-common.h" +#ifndef SHUFFLE_H +#define SHUFFLE_H -#include +#include "blosc2/blosc2-common.h" /* Toggle hardware-accelerated routines based on SHUFFLE_*_ENABLED macros and availability on the target architecture. @@ -42,6 +41,10 @@ #define SHUFFLE_USE_NEON #endif +#ifdef __cplusplus +extern "C" { +#endif + /** Primary shuffle and bitshuffle routines. This function dynamically dispatches to the appropriate hardware-accelerated @@ -81,4 +84,8 @@ BLOSC_NO_EXPORT int32_t const uint8_t *_src, const uint8_t *_dest, const uint8_t *_tmp, const uint8_t format_version); -#endif /* BLOSC_SHUFFLE_H */ +#ifdef __cplusplus +} +#endif + +#endif /* SHUFFLE_H */ diff --git a/src/blosc2/blosc/stune.c b/src/blosc2/blosc/stune.c index 4a8b2f3..de43f6c 100644 --- a/src/blosc2/blosc/stune.c +++ b/src/blosc2/blosc/stune.c @@ -1,17 +1,16 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ -#include "stune.h" - #include #include +#include "stune.h" /* Whether a codec is meant for High Compression Ratios @@ -34,16 +33,14 @@ static bool is_HCR(blosc2_context * context) { } } -int blosc_stune_init(void * config, blosc2_context* cctx, blosc2_context* dctx) { +void blosc_stune_init(void * config, blosc2_context* cctx, blosc2_context* dctx) { BLOSC_UNUSED_PARAM(config); BLOSC_UNUSED_PARAM(cctx); BLOSC_UNUSED_PARAM(dctx); - - return BLOSC2_ERROR_SUCCESS; } // Set the automatic blocksize 0 to its real value -int blosc_stune_next_blocksize(blosc2_context *context) { +void blosc_stune_next_blocksize(blosc2_context *context) { int32_t clevel = context->clevel; int32_t typesize = context->typesize; int32_t nbytes = context->sourcesize; @@ -53,7 +50,7 @@ int blosc_stune_next_blocksize(blosc2_context *context) { // Protection against very small buffers if (nbytes < typesize) { context->blocksize = 1; - return BLOSC2_ERROR_SUCCESS; + return; } if (user_blocksize) { @@ -108,8 +105,7 @@ int blosc_stune_next_blocksize(blosc2_context *context) { } /* Now the blocksize for splittable codecs */ - int splitmode = split_block(context, typesize, blocksize); - if (clevel > 0 && splitmode) { + if (clevel > 0 && split_block(context, typesize, blocksize)) { // For performance reasons, do not exceed 256 KB (it must fit in L2 cache) switch (clevel) { case 1: @@ -123,8 +119,6 @@ int blosc_stune_next_blocksize(blosc2_context *context) { blocksize = 64 * 1024; break; case 7: - blocksize = 128 * 1024; - break; case 8: blocksize = 256 * 1024; break; @@ -157,29 +151,19 @@ int blosc_stune_next_blocksize(blosc2_context *context) { } context->blocksize = blocksize; - BLOSC_INFO("compcode: %d, clevel: %d, blocksize: %d, splitmode: %d, typesize: %d", - context->compcode, context->clevel, blocksize, splitmode, typesize); - - return BLOSC2_ERROR_SUCCESS; } -int blosc_stune_next_cparams(blosc2_context * context) { - BLOSC_UNUSED_PARAM(context); - - return BLOSC2_ERROR_SUCCESS; +void blosc_stune_next_cparams(blosc2_context * context) { + BLOSC_UNUSED_PARAM(context); } -int blosc_stune_update(blosc2_context * context, double ctime) { - BLOSC_UNUSED_PARAM(context); - BLOSC_UNUSED_PARAM(ctime); - - return BLOSC2_ERROR_SUCCESS; +void blosc_stune_update(blosc2_context * context, double ctime) { + BLOSC_UNUSED_PARAM(context); + BLOSC_UNUSED_PARAM(ctime); } -int blosc_stune_free(blosc2_context * context) { - BLOSC_UNUSED_PARAM(context); - - return BLOSC2_ERROR_SUCCESS; +void blosc_stune_free(blosc2_context * context) { + BLOSC_UNUSED_PARAM(context); } int split_block(blosc2_context *context, int32_t typesize, int32_t blocksize) { diff --git a/src/blosc2/blosc/stune.h b/src/blosc2/blosc/stune.h index c4ac148..cf78cbf 100644 --- a/src/blosc2/blosc/stune.h +++ b/src/blosc2/blosc/stune.h @@ -1,20 +1,18 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ -#ifndef BLOSC_STUNE_H -#define BLOSC_STUNE_H +#ifndef STUNE_H +#define STUNE_H #include "context.h" -#include - /* The size of L1 cache. 32 KB is quite common nowadays. */ #define L1 (32 * 1024) /* The size of L2 cache. 256 KB is quite common nowadays. */ @@ -23,19 +21,28 @@ /* The maximum number of compressed data streams in a block for compression */ #define MAX_STREAMS 16 /* Cannot be larger than 128 */ -#define BLOSC_STUNE 0 -int blosc_stune_init(void * config, blosc2_context* cctx, blosc2_context* dctx); +void blosc_stune_init(void * config, blosc2_context* cctx, blosc2_context* dctx); + +void blosc_stune_next_blocksize(blosc2_context * context); + +void blosc_stune_next_cparams(blosc2_context * context); -int blosc_stune_next_blocksize(blosc2_context * context); +void blosc_stune_update(blosc2_context * context, double ctime); -int blosc_stune_next_cparams(blosc2_context * context); +void blosc_stune_free(blosc2_context * context); -int blosc_stune_update(blosc2_context * context, double ctime); +static blosc2_btune BTUNE_DEFAULTS = { + .btune_init = blosc_stune_init, + .btune_free = blosc_stune_free, + .btune_update = blosc_stune_update, + .btune_next_cparams = blosc_stune_next_cparams, + .btune_next_blocksize = blosc_stune_next_blocksize, + .btune_config = NULL, +}; -int blosc_stune_free(blosc2_context * context); /* Conditions for splitting a block before compressing with a codec. */ int split_block(blosc2_context *context, int32_t typesize, int32_t blocksize); -#endif /* BLOSC_STUNE_H */ +#endif /* STUNE_H */ diff --git a/src/blosc2/blosc/timestamp.c b/src/blosc2/blosc/timestamp.c index e14d07e..c143971 100644 --- a/src/blosc2/blosc/timestamp.c +++ b/src/blosc2/blosc/timestamp.c @@ -1,7 +1,7 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) @@ -13,8 +13,6 @@ /* System-specific high-precision timing functions. */ #if defined(_WIN32) -#include - /* Set a timestamp value to the current time. */ void blosc_set_timestamp(blosc_timestamp_t* timestamp) { /* Ignore the return value, assume the call always succeeds. */ @@ -33,14 +31,9 @@ double blosc_elapsed_nsecs(blosc_timestamp_t start_time, #else -#include - -#if defined(__MACH__) // OS X does not have clock_gettime, use clock_get_time - -#include - /* Set a timestamp value to the current time. */ void blosc_set_timestamp(blosc_timestamp_t* timestamp) { +#ifdef __MACH__ // OS X does not have clock_gettime, use clock_get_time clock_serv_t cclock; mach_timespec_t mts; host_get_clock_service(mach_host_self(), CALENDAR_CLOCK, &cclock); @@ -48,16 +41,10 @@ void blosc_set_timestamp(blosc_timestamp_t* timestamp) { mach_port_deallocate(mach_task_self(), cclock); timestamp->tv_sec = mts.tv_sec; timestamp->tv_nsec = mts.tv_nsec; -} - #else - -/* Set a timestamp value to the current time. */ -void blosc_set_timestamp(blosc_timestamp_t* timestamp) { clock_gettime(CLOCK_MONOTONIC, timestamp); -} - #endif +} /* Given two timestamp values, return the difference in nanoseconds. */ double blosc_elapsed_nsecs(blosc_timestamp_t start_time, diff --git a/src/blosc2/blosc/transpose-altivec.h b/src/blosc2/blosc/transpose-altivec.h index 6a8b086..fab0e5d 100644 --- a/src/blosc2/blosc/transpose-altivec.h +++ b/src/blosc2/blosc/transpose-altivec.h @@ -1,7 +1,7 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team and Jerome Kieffer + Copyright (C) 2021 The Blosc developers and Jerome Kieffer https://blosc.org License: BSD 3-Clause (see LICENSE.txt) @@ -11,9 +11,9 @@ #ifndef BLOSC_TRANSPOSE_ALTIVEC_H #define BLOSC_TRANSPOSE_ALTIVEC_H -#include - -#include +#ifdef __cplusplus +extern "C" { +#endif static const __vector uint8_t even = (const __vector uint8_t) { 0x00, 0x02, 0x04, 0x06, 0x08, 0x0a, 0x0c, 0x0e, @@ -113,4 +113,9 @@ static void transpose16x16(__vector uint8_t * xmm0){ } } -#endif /* BLOSC_TRANSPOSE_ALTIVEC_H */ + +#ifdef __cplusplus +} +#endif + +#endif //BLOSC_TRANSPOSE_ALTIVEC_H diff --git a/src/blosc2/blosc/trunc-prec.c b/src/blosc2/blosc/trunc-prec.c index 670316d..f4891fc 100644 --- a/src/blosc2/blosc/trunc-prec.c +++ b/src/blosc2/blosc/trunc-prec.c @@ -1,20 +1,18 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ -#include "trunc-prec.h" -#include "blosc2.h" - -#include #include -#include #include +#include "assert.h" +#include "trunc-prec.h" +#include "blosc2.h" #define BITS_MANTISSA_FLOAT 23 #define BITS_MANTISSA_DOUBLE 52 @@ -79,8 +77,8 @@ int truncate_precision(int8_t prec_bits, int32_t typesize, int32_t nbytes, return truncate_precision64(prec_bits, nbytes / typesize, (int64_t *)src, (int64_t *)dest); default: - BLOSC_TRACE_ERROR("Error in trunc-prec filter: Precision for typesize %d not handled", - (int)typesize); + fprintf(stderr, "Error in trunc-prec filter: Precision for typesize %d " + "not handled", (int)typesize); return -1; } } diff --git a/src/blosc2/blosc/trunc-prec.h b/src/blosc2/blosc/trunc-prec.h index 4007463..50d53de 100644 --- a/src/blosc2/blosc/trunc-prec.h +++ b/src/blosc2/blosc/trunc-prec.h @@ -1,7 +1,7 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) @@ -11,9 +11,10 @@ #ifndef BLOSC_TRUNC_PREC_H #define BLOSC_TRUNC_PREC_H +#include #include int truncate_precision(int8_t prec_bits, int32_t typesize, int32_t nbytes, const uint8_t* src, uint8_t* dest); -#endif /* BLOSC_TRUNC_PREC_H */ +#endif //BLOSC_TRUNC_PREC_H diff --git a/src/blosc2/include/b2nd.h b/src/blosc2/include/b2nd.h index 0e6a43a..b2ff31a 100644 --- a/src/blosc2/include/b2nd.h +++ b/src/blosc2/include/b2nd.h @@ -1,7 +1,7 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) @@ -12,28 +12,20 @@ * @brief Blosc2 NDim header file. * * This file contains Blosc2 NDim public API and the structures needed to use it. - * @author Blosc Development Team + * @author Blosc Development team */ -#ifndef BLOSC_B2ND_H -#define BLOSC_B2ND_H - -#ifdef __cplusplus -extern "C" { -#endif -#include "blosc2/blosc2-export.h" -#ifdef __cplusplus -} -#endif - -#include "blosc2.h" +#ifndef B2ND_B2ND_H_ +#define B2ND_B2ND_H_ +#include #include #include #ifdef __cplusplus extern "C" { #endif +#include "blosc2/blosc2-export.h" /* The version for metalayer format; starts from 0 and it must not exceed 127 */ @@ -198,8 +190,8 @@ BLOSC_EXPORT int b2nd_zeros(b2nd_context_t *ctx, b2nd_array_t **array); * uninitialized portions of the array. * * @param ctx The b2nd context for the new array. - * @param array The memory pointer where the array will be created. * @param fill_value Default value for uninitialized portions of the array. + * @param array The memory pointer where the array will be created. * * @return An error code. */ @@ -603,4 +595,4 @@ static inline int b2nd_deserialize_meta( } #endif -#endif /* BLOSC_B2ND_H */ +#endif // B2ND_B2ND_H_ diff --git a/src/blosc2/include/blosc2.h b/src/blosc2/include/blosc2.h index dd1fea4..96df9d7 100644 --- a/src/blosc2/include/blosc2.h +++ b/src/blosc2/include/blosc2.h @@ -1,7 +1,7 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) @@ -13,28 +13,36 @@ @brief Blosc2 header file. This file contains Blosc2 public API and the structures needed to use it. - @author The Blosc Development Team + @author The Blosc Developers **********************************************************************/ -#ifndef BLOSC_BLOSC2_H -#define BLOSC_BLOSC2_H +#ifndef BLOSC2_H +#define BLOSC2_H + +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif #include "blosc2/blosc2-export.h" #include "blosc2/blosc2-common.h" #include "blosc2/blosc2-stdio.h" +#ifdef __cplusplus +} +#endif #if defined(_WIN32) && !defined(__MINGW32__) #include -#include -#include -#define getpid _getpid -#endif + #include -#include -#include -#include -#include -#include + #include + #define getpid _getpid +#endif #ifdef __cplusplus extern "C" { @@ -82,11 +90,11 @@ extern "C" { /* Version numbers */ #define BLOSC2_VERSION_MAJOR 2 /* for major interface/format changes */ -#define BLOSC2_VERSION_MINOR 10 /* for minor interface/format changes */ -#define BLOSC2_VERSION_RELEASE 5 /* for tweaks, bug-fixes, or development */ +#define BLOSC2_VERSION_MINOR 8 /* for minor interface/format changes */ +#define BLOSC2_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */ -#define BLOSC2_VERSION_STRING "2.10.5" /* string version. Sync with above! */ -#define BLOSC2_VERSION_DATE "$Date:: 2023-10-05 #$" /* date version */ +#define BLOSC2_VERSION_STRING "2.8.0" /* string version. Sync with above! */ +#define BLOSC2_VERSION_DATE "$Date:: 2023-03-24 #$" /* date version */ /* The maximum number of dimensions for Blosc2 NDim arrays */ @@ -96,20 +104,19 @@ extern "C" { /* Tracing macros */ #define BLOSC_TRACE_ERROR(msg, ...) BLOSC_TRACE(error, msg, ##__VA_ARGS__) #define BLOSC_TRACE_WARNING(msg, ...) BLOSC_TRACE(warning, msg, ##__VA_ARGS__) -#define BLOSC_TRACE_INFO(msg, ...) BLOSC_TRACE(info, msg, ##__VA_ARGS__) -#define BLOSC_TRACE(cat, msg, ...) \ - do { \ - const char *__e = getenv("BLOSC_TRACE"); \ - if (!__e) { break; } \ - fprintf(stderr, "[%s] - " msg " (%s:%d)\n", #cat, ##__VA_ARGS__, __FILE__, __LINE__); \ - } while(0) - -#define BLOSC_ERROR_NULL(pointer, rc) \ - do { \ - if ((pointer) == NULL) { \ - BLOSC_TRACE_ERROR("Pointer is null"); \ - return rc; \ - } \ +#define BLOSC_TRACE(cat, msg, ...) \ + do { \ + const char *__e = getenv("BLOSC_TRACE"); \ + if (!__e) { break; } \ + fprintf(stderr, "[%s] - " msg " (%s:%d)\n", #cat, ##__VA_ARGS__, __FILE__, __LINE__); \ + } while(0) + +#define BLOSC_ERROR_NULL(pointer, rc) \ + do { \ + if ((pointer) == NULL) { \ + BLOSC_TRACE_ERROR("Pointer is null"); \ + return rc; \ + }\ } while (0) #define BLOSC_ERROR(rc) \ do { \ @@ -121,13 +128,6 @@ extern "C" { } \ } while (0) -#define BLOSC_INFO(msg, ...) \ - do { \ - const char *__e = getenv("BLOSC_INFO"); \ - if (!__e) { break; } \ - fprintf(stderr, "[INFO] - " msg "\n", ##__VA_ARGS__); \ - } while(0) - /* The VERSION_FORMAT symbols below should be just 1-byte long */ enum { @@ -194,30 +194,6 @@ enum { //!< Minimum buffer size to be compressed. }; -enum { - BLOSC2_DEFINED_TUNER_START = 0, - BLOSC2_DEFINED_TUNER_STOP = 31, - //!< Blosc-defined tuners must be between 0 - 31. - BLOSC2_GLOBAL_REGISTERED_TUNER_START = 32, - BLOSC2_GLOBAL_REGISTERED_TUNER_STOP = 159, - //!< Blosc-registered tuners must be between 31 - 159. - BLOSC2_GLOBAL_REGISTERED_TUNERS = 0, - //!< Number of Blosc-registered tuners at the moment. - BLOSC2_USER_REGISTERED_TUNER_START = 160, - BLOSC2_USER_REGISTERED_TUNER_STOP = 255, - //!< User-defined tuners must be between 160 - 255. -}; - -/** - * @brief Codes for the different tuners shipped with Blosc - */ -enum { - BLOSC_STUNE = 0, - BLOSC_LAST_TUNER = 1, - //!< Determine the last tuner defined by Blosc. - BLOSC_LAST_REGISTERED_TUNE = BLOSC2_GLOBAL_REGISTERED_TUNER_START + BLOSC2_GLOBAL_REGISTERED_TUNERS - 1, - //!< Determine the last registered tuner. It is used to check if a tuner is registered or not. -}; enum { BLOSC2_DEFINED_FILTERS_START = 0, @@ -226,7 +202,7 @@ enum { BLOSC2_GLOBAL_REGISTERED_FILTERS_START = 32, BLOSC2_GLOBAL_REGISTERED_FILTERS_STOP = 159, //!< Blosc-registered filters must be between 32 - 159. - BLOSC2_GLOBAL_REGISTERED_FILTERS = 4, + BLOSC2_GLOBAL_REGISTERED_FILTERS = 3, //!< Number of Blosc-registered filters at the moment. BLOSC2_USER_REGISTERED_FILTERS_START = 160, BLOSC2_USER_REGISTERED_FILTERS_STOP = 255, @@ -294,7 +270,7 @@ enum { BLOSC2_GLOBAL_REGISTERED_CODECS_START = 32, BLOSC2_GLOBAL_REGISTERED_CODECS_STOP = 159, //!< Blosc-registered codecs must be between 31 - 159. - BLOSC2_GLOBAL_REGISTERED_CODECS = 5, + BLOSC2_GLOBAL_REGISTERED_CODECS = 1, //!< Number of Blosc-registered codecs at the moment. BLOSC2_USER_REGISTERED_CODECS_START = 160, BLOSC2_USER_REGISTERED_CODECS_STOP = 255, @@ -429,8 +405,6 @@ enum { /** * @brief Error codes - * Each time an error code is added here, its corresponding message error should be added in - * print_error() */ enum { BLOSC2_ERROR_SUCCESS = 0, //tuner_params. - int (*next_blocksize)(blosc2_context * context); - //!< Only compute the next blocksize. Only it is executed if tuner is not initialized. - int (*next_cparams)(blosc2_context * context); - //!< Compute the next cparams. Only is executed if tuner is initialized. - int (*update)(blosc2_context * context, double ctime); - //!< Update the tuner parameters. - int (*free)(blosc2_context * context); - //!< Free the tuner. - int id; - //!< The tuner id - char *name; - //!< The tuner name -} blosc2_tuner; - - -/** - * @brief Register locally a user-defined tuner in Blosc. - * - * @param tuner The tuner to register. - * - * @return 0 if succeeds. Else a negative code is returned. - */ -BLOSC_EXPORT int register_tuner_private(blosc2_tuner *tuner); + void (*btune_init)(void * config, blosc2_context* cctx, blosc2_context* dctx); + //!< Initialize BTune. + void (*btune_next_blocksize)(blosc2_context * context); + //!< Only compute the next blocksize. Only it is executed if BTune is not initialized. + void (*btune_next_cparams)(blosc2_context * context); + //!< Compute the next cparams. Only is executed if BTune is initialized. + void (*btune_update)(blosc2_context * context, double ctime); + //!< Update the BTune parameters. + void (*btune_free)(blosc2_context * context); + //!< Free the BTune. + void *btune_config; + //!> BTune configuration. +}blosc2_btune; /** @@ -1197,10 +1153,8 @@ typedef struct { //!< The prefilter function. blosc2_prefilter_params *preparams; //!< The prefilter parameters. - void *tuner_params; - //!< Tune configuration. - int tuner_id; - //!< The tuner id. + blosc2_btune *udbtune; + //!< The user-defined BTune parameters. bool instr_codec; //!< Whether the codec is instrumented or not void *codec_params; @@ -1217,7 +1171,7 @@ static const blosc2_cparams BLOSC2_CPARAMS_DEFAULTS = { BLOSC_FORWARD_COMPAT_SPLIT, NULL, {0, 0, 0, 0, 0, BLOSC_SHUFFLE}, {0, 0, 0, 0, 0, 0}, - NULL, NULL, NULL, 0, 0, + NULL, NULL, NULL, 0, NULL, {NULL, NULL, NULL, NULL, NULL, NULL} }; @@ -1251,7 +1205,7 @@ static const blosc2_dparams BLOSC2_DPARAMS_DEFAULTS = {1, NULL, NULL, NULL}; * * @return A pointer to the new context. NULL is returned if this fails. * - * @note This supports the same environment variables than #blosc2_compress + * @note This support the same environment variables than #blosc2_compress * for overriding the programmatic compression values. * * @sa #blosc2_compress @@ -1265,7 +1219,7 @@ BLOSC_EXPORT blosc2_context* blosc2_create_cctx(blosc2_cparams cparams); * * @return A pointer to the new context. NULL is returned if this fails. * - * @note This supports the same environment variables than #blosc2_decompress + * @note This support the same environment variables than #blosc2_decompress * for overriding the programmatic decompression values. * * @sa #blosc2_decompress @@ -1356,11 +1310,9 @@ BLOSC_EXPORT int blosc2_set_maskout(blosc2_context *ctx, bool *maskout, int nblo * @return The number of bytes compressed. * If @p src buffer cannot be compressed into @p destsize, the return * value is zero and you should discard the contents of the @p dest - * buffer. A negative return value means that either a parameter is not correct - * or that an internal error happened. Set the BLOSC_TRACE environment variable - * for getting more info on what is happening. If the error is not related with - * wrong params, please report it back together with the buffer data causing this, - * as well as the compression params used. + * buffer. A negative return value means that an internal error happened. This + * should never happen. If you see this, please report it back + * together with the buffer data causing this and compression settings. */ /* * Environment variables @@ -1711,10 +1663,8 @@ typedef struct blosc2_schunk { // + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ -#ifndef BLOSC_BLOSC2_BLOSC2_COMMON_H -#define BLOSC_BLOSC2_BLOSC2_COMMON_H +#ifndef SHUFFLE_COMMON_H +#define SHUFFLE_COMMON_H #include "blosc2-export.h" - -#include #include // For shutting up stupid compiler warning about some 'unused' variables in GCC @@ -27,10 +25,24 @@ // For shutting up compiler warning about unused parameters #define BLOSC_UNUSED_PARAM(x) ((void)(x)) -/* Use inlined functions for supported systems */ -#if defined(_MSC_VER) && !defined(__cplusplus) /* Visual Studio */ - #define inline __inline /* Visual C is not C99, but supports some kind of inline */ -#endif +/* Import standard integer type definitions */ +#if defined(_WIN32) && !defined(__MINGW32__) + + /* stdint.h only available in VS2010 (VC++ 16.0) and newer */ + #if defined(_MSC_VER) && _MSC_VER < 1600 + #include "win32/stdint-windows.h" + #else + #include + #endif + + /* Use inlined functions for supported systems */ + #if defined(_MSC_VER) && !defined(__cplusplus) /* Visual Studio */ + #define inline __inline /* Visual C is not C99, but supports some kind of inline */ + #endif + +#else + #include +#endif /* _WIN32 */ /* Define the __SSE2__ symbol if compiling with Visual C++ and @@ -77,4 +89,4 @@ #include #endif -#endif /* BLOSC_BLOSC2_BLOSC2_COMMON_H */ +#endif /* SHUFFLE_COMMON_H */ diff --git a/src/blosc2/include/blosc2/blosc2-export.h b/src/blosc2/include/blosc2/blosc2-export.h index 99eccbe..3923651 100644 --- a/src/blosc2/include/blosc2/blosc2-export.h +++ b/src/blosc2/include/blosc2/blosc2-export.h @@ -1,15 +1,15 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ -#ifndef BLOSC_BLOSC2_BLOSC2_EXPORT_H -#define BLOSC_BLOSC2_BLOSC2_EXPORT_H +#ifndef BLOSC_EXPORT_H +#define BLOSC_EXPORT_H /* Macros for specifying exported symbols. BLOSC_EXPORT is used to decorate symbols that should be @@ -33,11 +33,11 @@ #define BLOSC_EXPORT #endif /* defined(BLOSC_SHARED_LIBRARY) */ -#if (defined(__GNUC__) || defined(__clang__)) && !defined(__MINGW32__) +#if defined(__GNUC__) || defined(__clang__) #define BLOSC_NO_EXPORT __attribute__((visibility("hidden"))) #else #define BLOSC_NO_EXPORT -#endif /* (defined(__GNUC__) || defined(__clang__)) && !defined(__MINGW32__) */ +#endif /* defined(__GNUC__) || defined(__clang__) */ /* When testing, export everything to make it easier to implement tests. */ #if defined(BLOSC_TESTING) @@ -45,4 +45,4 @@ #define BLOSC_NO_EXPORT BLOSC_EXPORT #endif /* defined(BLOSC_TESTING) */ -#endif /* BLOSC_BLOSC2_BLOSC2_EXPORT_H */ +#endif /* BLOSC_EXPORT_H */ diff --git a/src/blosc2/include/blosc2/blosc2-stdio.h b/src/blosc2/include/blosc2/blosc2-stdio.h index 0556f15..09368cf 100644 --- a/src/blosc2/include/blosc2/blosc2-stdio.h +++ b/src/blosc2/include/blosc2/blosc2-stdio.h @@ -1,30 +1,40 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ -#ifndef BLOSC_BLOSC2_BLOSC2_STDIO_H -#define BLOSC_BLOSC2_BLOSC2_STDIO_H -#include "blosc2-export.h" +#ifndef BLOSC_BLOSC2_STDIO_H +#define BLOSC_BLOSC2_STDIO_H -#if defined(_MSC_VER) -#include -#else -#include -#endif #include -#include #include +#include "blosc2-export.h" + + +#if defined(_WIN32) && !defined(__MINGW32__) + +/* stdint.h only available in VS2010 (VC++ 16.0) and newer */ + #if defined(_MSC_VER) && _MSC_VER < 1600 + #include "win32/stdint-windows.h" + #else + #include + #endif -#ifdef __cplusplus -extern "C" { +#else +#include +#endif + +#if defined(_MSC_VER) && (_MSC_VER >= 1400) +#include +#else +#include #endif typedef struct { @@ -39,8 +49,4 @@ BLOSC_EXPORT int64_t blosc2_stdio_write(const void *ptr, int64_t size, int64_t n BLOSC_EXPORT int64_t blosc2_stdio_read(void *ptr, int64_t size, int64_t nitems, void *stream); BLOSC_EXPORT int blosc2_stdio_truncate(void *stream, int64_t size); -#ifdef __cplusplus -} -#endif - -#endif /* BLOSC_BLOSC2_BLOSC2_STDIO_H */ +#endif //BLOSC_BLOSC2_STDIO_H diff --git a/src/blosc2/include/blosc2/codecs-registry.h b/src/blosc2/include/blosc2/codecs-registry.h index 058fe74..8b5e78d 100644 --- a/src/blosc2/include/blosc2/codecs-registry.h +++ b/src/blosc2/include/blosc2/codecs-registry.h @@ -1,43 +1,18 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ -#ifndef BLOSC_BLOSC2_CODECS_REGISTRY_H -#define BLOSC_BLOSC2_CODECS_REGISTRY_H - -#include "blosc2.h" - -#ifdef __cplusplus -extern "C" { -#endif - enum { BLOSC_CODEC_NDLZ = 32, BLOSC_CODEC_ZFP_FIXED_ACCURACY = 33, BLOSC_CODEC_ZFP_FIXED_PRECISION = 34, BLOSC_CODEC_ZFP_FIXED_RATE = 35, - BLOSC_CODEC_OPENHTJ2K = 36, }; void register_codecs(void); - -// For dynamically loaded codecs -typedef struct { - char *encoder; - char *decoder; -} codec_info; - -// Silence unused codec_info typedef warning -static codec_info codec_info_defaults BLOSC_ATTRIBUTE_UNUSED = {0}; - -#ifdef __cplusplus -} -#endif - -#endif /* BLOSC_BLOSC2_CODECS_REGISTRY_H */ diff --git a/src/blosc2/include/blosc2/filters-registry.h b/src/blosc2/include/blosc2/filters-registry.h index b39e2aa..329132b 100644 --- a/src/blosc2/include/blosc2/filters-registry.h +++ b/src/blosc2/include/blosc2/filters-registry.h @@ -1,37 +1,17 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Copyright (c) 2021 The Blosc Development Team + Copyright (C) 2021 The Blosc Developers https://blosc.org License: BSD 3-Clause (see LICENSE.txt) See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ -#ifndef BLOSC_BLOSC2_FILTERS_REGISTRY_H -#define BLOSC_BLOSC2_FILTERS_REGISTRY_H - -#ifdef __cplusplus -extern "C" { -#endif - enum { BLOSC_FILTER_NDCELL = 32, BLOSC_FILTER_NDMEAN = 33, - BLOSC_FILTER_BYTEDELTA_BUGGY = 34, // buggy version. See #524 - BLOSC_FILTER_BYTEDELTA = 35, // fixed version + BLOSC_FILTER_BYTEDELTA = 34, }; void register_filters(void); - -// For dynamically loaded filters -typedef struct { - char *forward; - char *backward; -} filter_info; - -#ifdef __cplusplus -} -#endif - -#endif /* BLOSC_BLOSC2_FILTERS_REGISTRY_H */