Merge pull request #109 from DataDog/dopuskh3/zstd-external-library

Add support for building zstd binding against an external libzstd library

.circleci/config.yml

@@ -1,21 +1,45 @@
 version: 2
 
 jobs:
-  "golang-1.14":
+  "golang-1.16":
     docker:
-      - image: circleci/golang:1.14
+      - image: circleci/golang:1.16
     steps:
       - checkout
       - run: 'wget https://github.com/DataDog/zstd/files/2246767/mr.zip'
       - run: 'unzip mr.zip'
       - run: 'go build'
       - run: 'PAYLOAD=`pwd`/mr go test -v'
       - run: 'PAYLOAD=`pwd`/mr go test -bench .'
-  "golang-1.15":
+  "golang-1.16-external-libzstd":
     docker:
-      - image: circleci/golang:1.15
+      - image: circleci/golang:1.16
     steps:
       - checkout
+      - run: 'sudo apt update'
+      - run: 'sudo apt install libzstd-dev'
+      - run: 'wget https://github.com/DataDog/zstd/files/2246767/mr.zip'
+      - run: 'unzip mr.zip'
+      - run: 'go build'
+      - run: 'PAYLOAD=`pwd`/mr go test -v'
+      - run: 'PAYLOAD=`pwd`/mr go test -bench .'
+  "golang-1.17":
+    docker:
+      - image: circleci/golang:1.17
+    steps:
+      - checkout
+      - run: 'wget https://github.com/DataDog/zstd/files/2246767/mr.zip'
+      - run: 'unzip mr.zip'
+      - run: 'go build'
+      - run: 'PAYLOAD=`pwd`/mr go test -v'
+      - run: 'PAYLOAD=`pwd`/mr go test -bench .'
+  "golang-1.17-external-libzstd":
+    docker:
+      - image: circleci/golang:1.17
+    steps:
+      - checkout
+      - run: 'sudo apt update'
+      - run: 'sudo apt install libzstd-dev'
       - run: 'wget https://github.com/DataDog/zstd/files/2246767/mr.zip'
       - run: 'unzip mr.zip'
       - run: 'go build'
@@ -31,6 +55,18 @@ jobs:
       - run: 'go build'
       - run: 'PAYLOAD=`pwd`/mr go test -v'
       - run: 'PAYLOAD=`pwd`/mr go test -bench .'
+  "golang-latest-external-libzstd":
+    docker:
+      - image: circleci/golang:latest
+    steps:
+      - checkout
+      - run: 'sudo apt update'
+      - run: 'sudo apt install libzstd-dev'
+      - run: 'wget https://github.com/DataDog/zstd/files/2246767/mr.zip'
+      - run: 'unzip mr.zip'
+      - run: 'go build -tags external_libzstd'
+      - run: 'PAYLOAD=`pwd`/mr go test -tags external_libzstd -v'
+      - run: 'PAYLOAD=`pwd`/mr go test -tags external_libzstd -bench .'
   "golang-efence":
     resource_class: xlarge
     docker:
@@ -41,6 +77,18 @@ jobs:
       - run: 'unzip mr.zip'
       - run: 'go build'
       - run: 'PAYLOAD=`pwd`/mr GODEBUG=efence=1 go test -v'
+  "golang-efence-external-libzstd":
+    resource_class: xlarge
+    docker:
+      - image: circleci/golang:latest
+    steps:
+      - checkout
+      - run: 'sudo apt update'
+      - run: 'sudo apt install libzstd-dev'
+      - run: 'wget https://github.com/DataDog/zstd/files/2246767/mr.zip'
+      - run: 'unzip mr.zip'
+      - run: 'go build -tags external_libzstd'
+      - run: 'PAYLOAD=`pwd`/mr GODEBUG=efence=1 go test -tags external_libzstd -v'
   "golang-zstd-legacy-support":
     docker:
       - image: circleci/golang:latest
@@ -61,9 +109,13 @@ workflows:
   version: 2
   build:
     jobs:
-      - "golang-1.14"
-      - "golang-1.15"
+      - "golang-1.16"
+      - "golang-1.16-external-libzstd"
+      - "golang-1.17"
+      - "golang-1.17-external-libzstd"
       - "golang-latest"
+      - "golang-latest-external-libzstd"
       - "golang-efence"
+      - "golang-efence-external-libzstd"
       - "golang-i386"
       - "golang-zstd-legacy-support"

README.md

@@ -19,6 +19,21 @@ There are two main APIs:
 The compress/decompress APIs mirror that of lz4, while the streaming API was
 designed to be a drop-in replacement for zlib.
 
+### Building against an external libzstd
+
+By default, zstd source code is vendored in this repository and the binding will be built with
+the vendored source code bundled.
+
+If you want to build this binding against an external static or shared libzstd library, you can
+use the `external_libzstd` build tag. This will look for the libzstd pkg-config file and extract
+build and linking parameters from that pkg-config file.
+
+Note that it requires at least libzstd 1.4.0.
+
+```bash
+go build -tags external_libzstd
+```
+
 ### Simple `Compress/Decompress`
 
 

bitstream.h

@@ -1,3 +1,4 @@
+#ifndef USE_EXTERNAL_ZSTD
 /* ******************************************************************
  * bitstream
  * Part of FSE library
@@ -461,3 +462,5 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
 #endif
 
 #endif /* BITSTREAM_H_MODULE */
+
+#endif /* USE_EXTERNAL_ZSTD */

compiler.h

@@ -1,3 +1,4 @@
+#ifndef USE_EXTERNAL_ZSTD
 /*
  * Copyright (c) Yann Collet, Facebook, Inc.
  * All rights reserved.
@@ -287,3 +288,5 @@ void __asan_unpoison_memory_region(void const volatile *addr, size_t size);
 #endif
 
 #endif /* ZSTD_COMPILER_H */
+
+#endif /* USE_EXTERNAL_ZSTD */

cover.c

@@ -1,3 +1,4 @@
+#ifndef USE_EXTERNAL_ZSTD
 /*
  * Copyright (c) Yann Collet, Facebook, Inc.
  * All rights reserved.
@@ -1244,3 +1245,5 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
     return dictSize;
   }
 }
+
+#endif /* USE_EXTERNAL_ZSTD */

cover.h

@@ -1,3 +1,4 @@
+#ifndef USE_EXTERNAL_ZSTD
 /*
  * Copyright (c) Facebook, Inc.
  * All rights reserved.
@@ -156,3 +157,5 @@ void COVER_dictSelectionFree(COVER_dictSelection_t selection);
  COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBufferCapacity,
                        size_t dictContentSize, const BYTE* samplesBuffer, const size_t* samplesSizes, unsigned nbFinalizeSamples,
                        size_t nbCheckSamples, size_t nbSamples, ZDICT_cover_params_t params, size_t* offsets, size_t totalCompressedSize);
+
+#endif /* USE_EXTERNAL_ZSTD */

cpu.h

@@ -1,3 +1,4 @@
+#ifndef USE_EXTERNAL_ZSTD
 /*
  * Copyright (c) Facebook, Inc.
  * All rights reserved.
@@ -211,3 +212,5 @@ MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) {
 #undef X
 
 #endif /* ZSTD_COMMON_CPU_H */
+
+#endif /* USE_EXTERNAL_ZSTD */

debug.c

@@ -1,3 +1,4 @@
+#ifndef USE_EXTERNAL_ZSTD
 /* ******************************************************************
  * debug
  * Part of FSE library
@@ -22,3 +23,5 @@
 #include "debug.h"
 
 int g_debuglevel = DEBUGLEVEL;
+
+#endif /* USE_EXTERNAL_ZSTD */

debug.h

@@ -1,3 +1,4 @@
+#ifndef USE_EXTERNAL_ZSTD
 /* ******************************************************************
  * debug
  * Part of FSE library
@@ -105,3 +106,5 @@ extern int g_debuglevel; /* the variable is only declared,
 #endif
 
 #endif /* DEBUG_H_12987983217 */
+
+#endif /* USE_EXTERNAL_ZSTD */

divsufsort.c

@@ -1,3 +1,4 @@
+#ifndef USE_EXTERNAL_ZSTD
 /*
  * divsufsort.c for libdivsufsort-lite
  * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved.
@@ -1911,3 +1912,5 @@ divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char *
 
   return pidx;
 }
+
+#endif /* USE_EXTERNAL_ZSTD */

divsufsort.h

@@ -1,3 +1,4 @@
+#ifndef USE_EXTERNAL_ZSTD
 /*
  * divsufsort.h for libdivsufsort-lite
  * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved.
@@ -65,3 +66,5 @@ divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char *
 #endif /* __cplusplus */
 
 #endif /* _DIVSUFSORT_H */
+
+#endif /* USE_EXTERNAL_ZSTD */

entropy_common.c

@@ -1,3 +1,4 @@
+#ifndef USE_EXTERNAL_ZSTD
 /* ******************************************************************
  * Common functions of New Generation Entropy library
  * Copyright (c) Yann Collet, Facebook, Inc.
@@ -360,3 +361,5 @@ size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats,
     (void)bmi2;
     return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize);
 }
+
+#endif /* USE_EXTERNAL_ZSTD */

error_private.c

@@ -1,3 +1,4 @@
+#ifndef USE_EXTERNAL_ZSTD
 /*
  * Copyright (c) Yann Collet, Facebook, Inc.
  * All rights reserved.
@@ -54,3 +55,5 @@ const char* ERR_getErrorString(ERR_enum code)
     }
 #endif
 }
+
+#endif /* USE_EXTERNAL_ZSTD */

error_private.h

@@ -1,3 +1,4 @@
+#ifndef USE_EXTERNAL_ZSTD
 /*
  * Copyright (c) Yann Collet, Facebook, Inc.
  * All rights reserved.
@@ -78,3 +79,5 @@ ERR_STATIC const char* ERR_getErrorName(size_t code)
 #endif
 
 #endif /* ERROR_H_MODULE */
+
+#endif /* USE_EXTERNAL_ZSTD */

errors.go

@@ -1,7 +1,6 @@
 package zstd
 
 /*
-#define ZSTD_STATIC_LINKING_ONLY
 #include "zstd.h"
 */
 import "C"

external_zstd.go

@@ -0,0 +1,14 @@
+//go:build external_libzstd
+// +build external_libzstd
+
+package zstd
+
+// #cgo CFLAGS: -DUSE_EXTERNAL_ZSTD
+// #cgo pkg-config: libzstd
+/*
+#include<zstd.h>
+#if ZSTD_VERSION_NUMBER < 10400
+#error "ZSTD version >= 1.4 is required"
+#endif
+*/
+import "C"

fastcover.c

@@ -1,3 +1,4 @@
+#ifndef USE_EXTERNAL_ZSTD
 /*
  * Copyright (c) Facebook, Inc.
  * All rights reserved.
@@ -757,3 +758,5 @@ ZDICT_optimizeTrainFromBuffer_fastCover(
     }
 
 }
+
+#endif /* USE_EXTERNAL_ZSTD */

fse.h

@@ -1,3 +1,4 @@
+#ifndef USE_EXTERNAL_ZSTD
 /* ******************************************************************
  * FSE : Finite State Entropy codec
  * Public Prototypes declaration
@@ -714,3 +715,5 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
 #if defined (__cplusplus)
 }
 #endif
+
+#endif /* USE_EXTERNAL_ZSTD */

fse_compress.c

@@ -1,3 +1,4 @@
+#ifndef USE_EXTERNAL_ZSTD
 /* ******************************************************************
  * FSE : Finite State Entropy encoder
  * Copyright (c) Yann Collet, Facebook, Inc.
@@ -703,3 +704,5 @@ size_t FSE_compress (void* dst, size_t dstCapacity, const void* src, size_t srcS
 #endif
 
 #endif   /* FSE_COMMONDEFS_ONLY */
+
+#endif /* USE_EXTERNAL_ZSTD */

fse_decompress.c

@@ -1,3 +1,4 @@
+#ifndef USE_EXTERNAL_ZSTD
 /* ******************************************************************
  * FSE : Finite State Entropy decoder
  * Copyright (c) Yann Collet, Facebook, Inc.
@@ -401,3 +402,5 @@ size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cS
 
 
 #endif   /* FSE_COMMONDEFS_ONLY */
+
+#endif /* USE_EXTERNAL_ZSTD */

hist.c

@@ -1,3 +1,4 @@
+#ifndef USE_EXTERNAL_ZSTD
 /* ******************************************************************
  * hist : Histogram functions
  * part of Finite State Entropy project
@@ -179,3 +180,5 @@ size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr,
     return HIST_count_wksp(count, maxSymbolValuePtr, src, srcSize, tmpCounters, sizeof(tmpCounters));
 }
 #endif
+
+#endif /* USE_EXTERNAL_ZSTD */

hist.h

@@ -1,3 +1,4 @@
+#ifndef USE_EXTERNAL_ZSTD
 /* ******************************************************************
  * hist : Histogram functions
  * part of Finite State Entropy project
@@ -73,3 +74,5 @@ size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
  */
 unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
                            const void* src, size_t srcSize);
+
+#endif /* USE_EXTERNAL_ZSTD */

huf.h

@@ -1,3 +1,4 @@
+#ifndef USE_EXTERNAL_ZSTD
 /* ******************************************************************
  * huff0 huffman codec,
  * part of Finite State Entropy library
@@ -360,3 +361,5 @@ size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t sr
 #if defined (__cplusplus)
 }
 #endif
+
+#endif /* USE_EXTERNAL_ZSTD */

huf_compress.c

@@ -1,3 +1,4 @@
+#ifndef USE_EXTERNAL_ZSTD
 /* ******************************************************************
  * Huffman encoder, part of New Generation Entropy library
  * Copyright (c) Yann Collet, Facebook, Inc.
@@ -935,3 +936,5 @@ size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSi
     return HUF_compress2(dst, maxDstSize, src, srcSize, 255, HUF_TABLELOG_DEFAULT);
 }
 #endif
+
+#endif /* USE_EXTERNAL_ZSTD */

huf_decompress.c

@@ -1,3 +1,4 @@
+#ifndef USE_EXTERNAL_ZSTD
 /* ******************************************************************
  * huff0 huffman decoder,
  * part of Finite State Entropy library
@@ -1349,3 +1350,5 @@ size_t HUF_decompress1X_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize,
                                       workSpace, sizeof(workSpace));
 }
 #endif
+
+#endif /* USE_EXTERNAL_ZSTD */

mem.h

@@ -1,3 +1,4 @@
+#ifndef USE_EXTERNAL_ZSTD
 /*
  * Copyright (c) Yann Collet, Facebook, Inc.
  * All rights reserved.
@@ -422,3 +423,5 @@ MEM_STATIC void MEM_check(void) { DEBUG_STATIC_ASSERT((sizeof(size_t)==4) || (si
 #endif
 
 #endif /* MEM_H_MODULE */
+
+#endif /* USE_EXTERNAL_ZSTD */