More reasonable and accurate benchmarks

CMakeLists.txt

@@ -31,7 +31,7 @@ endif()
 option(ROARING_DISABLE_X64 "Forcefully disable x64 optimizations even if hardware supports it (this disables AVX)" OFF)
 option(ROARING_DISABLE_AVX "Forcefully disable AVX even if hardware supports it " OFF)
 option(ROARING_DISABLE_NEON "Forcefully disable NEON even if hardware supports it" OFF)
-option(ROARING_DISABLE_NATIVE "Forcefully disable -march optimizations (obsolete)" OFF)
+option(ROARING_DISABLE_AVX512 "Forcefully disable AVX512 even if compiler supports it" OFF)
 
 option(ROARING_BUILD_STATIC "Build a static library" ON)
 if(BUILD_SHARED_LIBS)
@@ -62,7 +62,7 @@ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/roaring.pc" DESTINATION ${CMAKE_INSTA
 
 add_library(roaring-headers INTERFACE)
 target_include_directories(roaring-headers INTERFACE
-  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include/${ROARING_LIB_NAME}>
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include/roaring>
   $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCDIR}>)
 add_library(roaring-headers-cpp INTERFACE)
 target_include_directories(roaring-headers-cpp INTERFACE
@@ -73,11 +73,11 @@ target_include_directories(roaring-headers-cpp INTERFACE
 ### Some users want the C++ header files to be installed as well.
 ### C++ header files get installed to /usr/local/include/roaring typically
 SET(CPP_ROARING_HEADERS cpp/roaring64map.hh  cpp/roaring.hh) # needs to be updated if we add more files
-install(FILES ${CPP_ROARING_HEADERS} DESTINATION include/${ROARING_LIB_NAME})
-install(DIRECTORY include/${ROARING_LIB_NAME} DESTINATION include)
+install(FILES ${CPP_ROARING_HEADERS} DESTINATION include/roaring)
+install(DIRECTORY include/roaring DESTINATION include)
 
 install(TARGETS roaring-headers roaring-headers-cpp
-   EXPORT ${ROARING_LIB_NAME}-config
+   EXPORT roaring-config
    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
@@ -98,7 +98,10 @@ configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/tests/config.h.in"
 
 add_subdirectory(src)
 if(ENABLE_ROARING_TESTS)
-  add_subdirectory(benchmarks)
+  if(CMAKE_SIZEOF_VOID_P EQUAL 8) # we only include the benchmarks on 64-bit systems.
+    add_subdirectory(microbenchmarks)
+    add_subdirectory(benchmarks)
+  endif()
   add_subdirectory(tests)
 endif()
 # Being terse is good, but knowing how the build is configured is important

README.md

@@ -179,6 +179,32 @@ The C interface is found in the file ``include/roaring/roaring.h``. We have C++
 
 Some users have to deal with large volumes of data. It  may be important for these users to be aware of the `addMany` (C++) `roaring_bitmap_or_many` (C) functions as it is much faster and economical to add values in batches when possible. Furthermore, calling periodically the `runOptimize` (C++) or `roaring_bitmap_run_optimize` (C) functions may help.
 
+
+# Running microbenchmarks
+
+We have microbenchmarks constructed with the Google Benchmarks.
+Under Linux or macOS, you may run them as follows:
+
+```
+cmake --build build
+./build/microbenchmarks/bench
+```
+
+By default, the benchmark tools picks one data set (e.g., `CRoaring/benchmarks/realdata/census1881`).
+We have several data sets and you may pick others:
+
+```
+./build/microbenchmarks/bench benchmarks/realdata/wikileaks-noquotes 
+```
+
+You may disable some functionality for the purpose of benchmarking. For example, you could
+benchmark the code without AVX-512 even if both your processor and compiler supports it:
+
+```
+cmake --buildnoavx512 -D ROARING_DISABLE_AVX512=OFF
+./buildnoavx512/microbenchmarks/bench
+```
+
 # Custom memory allocators
 For general users, CRoaring would apply default allocator without extra codes. But global memory hook is also provided for those who want a custom memory allocator. Here is an example:
 ```C
@@ -575,14 +601,6 @@ ctest
 ```
 
 
-To run real-data benchmark
-
-```
-./real_bitmaps_benchmark ../benchmarks/realdata/census1881
-```
-where you must adjust the path "../benchmarks/realdata/census1881" so that it points to one of the directories in the benchmarks/realdata directory.
-
-
 To check that your code abides by the style convention (make sure that ``clang-format`` is installed):
 
 ```

amalgamation.sh

@@ -82,7 +82,6 @@ ALL_PRIVATE_C=$( ( \
         && ( type git >/dev/null 2>&1 ) \
         && ( git ls-files $SCRIPTPATH/src/*.c $SCRIPTPATH/src/**/*c ) \
     ) || ( find $SCRIPTPATH/src -name '*.c' ) )
-
 # Verify up-front that all the files exist
 #
 for i in ${ALL_PUBLIC_H} ${ALL_PUBLIC_HH} ${ALL_PRIVATE_H} ${ALL_PRIVATE_C}; do

cmake/import.cmake

@@ -0,0 +1,52 @@
+set(dep_root "${PROJEC_SOURCE_DIR}/dependencies/.cache")
+if(DEFINED ENV{roaring_DEPENDENCY_CACHE_DIR})
+  set(dep_root "$ENV{roaring_DEPENDENCY_CACHE_DIR}")
+endif()
+
+function(import_dependency NAME GITHUB_REPO COMMIT)
+  message(STATUS "Importing ${NAME} (${GITHUB_REPO}@${COMMIT})")
+  set(target "${dep_root}/${NAME}")
+
+  # If the folder exists in the cache, then we assume that everything is as
+  # should be and do nothing
+  if(EXISTS "${target}")
+    set("${NAME}_SOURCE_DIR" "${target}" PARENT_SCOPE)
+    return()
+  endif()
+
+  set(zip_url "https://github.com/${GITHUB_REPO}/archive/${COMMIT}.zip")
+  set(archive "${dep_root}/archive.zip")
+  set(dest "${dep_root}/_extract")
+
+  file(DOWNLOAD "${zip_url}" "${archive}")
+  file(MAKE_DIRECTORY "${dest}")
+  execute_process(
+          WORKING_DIRECTORY "${dest}"
+          COMMAND "${CMAKE_COMMAND}" -E tar xf "${archive}")
+  file(REMOVE "${archive}")
+
+  # GitHub archives only ever have one folder component at the root, so this
+  # will always match that single folder
+  file(GLOB dir LIST_DIRECTORIES YES "${dest}/*")
+
+  file(RENAME "${dir}" "${target}")
+
+  set("${NAME}_SOURCE_DIR" "${target}" PARENT_SCOPE)
+endfunction()
+
+# Delegates to the dependency
+macro(add_dependency NAME)
+  if(NOT DEFINED "${NAME}_SOURCE_DIR")
+    message(FATAL_ERROR "Missing ${NAME}_SOURCE_DIR variable")
+  endif()
+
+  add_subdirectory("${${NAME}_SOURCE_DIR}" "${PROJECT_BINARY_DIR}/_deps/${NAME}" EXCLUDE_FROM_ALL)
+endmacro()
+
+function(set_off NAME)
+  set("${NAME}" OFF CACHE INTERNAL "")
+endfunction()
+
+function(set_on NAME)
+  set("${NAME}" ON CACHE INTERNAL "")
+endfunction()

include/roaring/bitset_util.h

@@ -6,6 +6,12 @@
 #include <roaring/portability.h>
 #include <roaring/utilasm.h>
 
+#if CROARING_IS_X64
+#ifndef CROARING_COMPILER_SUPPORTS_AVX512
+#error "CROARING_COMPILER_SUPPORTS_AVX512 needs to be defined."
+#endif // CROARING_COMPILER_SUPPORTS_AVX512
+#endif
+
 #ifdef __cplusplus
 extern "C" { namespace roaring { namespace internal {
 #endif
@@ -631,7 +637,7 @@ CROARING_UNTARGET_AVX512
         __m512i total = _mm512_setzero_si512();                                \
         const uint64_t limit = size - size % 4;                                \
         uint64_t i = 0;                                                        \
-	    for (; i < limit; i += 4) {                                        \
+	    for (; i < limit; i += 4) {                                            \
             __m512i a1 = avx_intrinsic(_mm512_loadu_si512(data1 + i),          \
                                        _mm512_loadu_si512(data2 + i));         \
             total = _mm512_add_epi64(total, _mm512_popcnt_epi64(a1));          \