Relaxed ABI tagging

This commit incorporates some of the suggestions from pybind11 PR
pybind/pybind11#4953 (and linked discussion) to
relax ABI tagging for libstdc++ and MSVC. The goal is to ensure that
ABI-incompatible extensions are separated from each other. These checks
were previously too fine-grained and caused isolation in cases where an
actual ABI incompatibility was not present. This is a long-standing
problem in both pybind11 and nanobind causing inconvenience for users.

While looking into this topic, I realized that libc++ has an opt-in
unstable ABI feature
(https://libcxx.llvm.org/DesignDocs/ABIVersioning.html). The PR also
adds checks for this.

Merging this PR will imply an ABI version bump for nanobind due to the
differnt naming convention of the associated ABI tag.

.github/workflows/ci.yml

@@ -55,11 +55,23 @@ jobs:
 
     - name: Configure
       run: >
-        cmake -S . -B build -DNB_TEST_STABLE_ABI=ON -DNB_TEST_SHARED_BUILD="$(python3 -c 'import sys; print(int(sys.version_info.minor>=11))')"
+        cmake -S . -B build -DNB_TEST_STABLE_ABI=ON -DNB_TEST_SHARED_BUILD="$(python -c 'import sys; print(int(sys.version_info.minor>=11))')"
 
     - name: Build C++
       run: cmake --build build -j 2
 
+    - name: Check ABI tag
+      if: ${{ !startsWith(matrix.os, 'windows')  }}
+      run: >
+        cd build/tests;
+        python -c 'import test_functions_ext as t; print(f"ABI tag is \"{ t.abi_tag() }\"")'
+
+    - name: Check ABI tag
+      if: ${{ startsWith(matrix.os, 'windows')  }}
+      run: >
+        cd build/tests/Debug;
+        python -c 'import test_functions_ext as t; print(f"ABI tag is \"{ t.abi_tag() }\"")'
+
     - name: Run tests
       run: >
         cd build;
@@ -85,6 +97,11 @@ jobs:
     - name: Build C++
       run: cmake --build build -j 2
 
+    - name: Check ABI tag
+      run: >
+        cd build/tests;
+        python3 -c 'import test_functions_ext as t; print(f"ABI tag is \"{ t.abi_tag() }\"")'
+
     - name: Run tests
       run: >
         cd build;
@@ -134,6 +151,11 @@ jobs:
     - name: Build C++
       run: cmake --build build -j 2
 
+    - name: Check ABI tag
+      run: >
+        cd build/tests;
+        python -c 'import test_functions_ext as t; print(f"ABI tag is \"{ t.abi_tag() }\"")'
+
     - name: Run tests
       run: >
         cd build;
@@ -165,7 +187,13 @@ jobs:
         cmake -S . -B build -DNB_TEST_FREE_THREADED=ON
 
     - name: Build C++
-      run: cmake --build build -j 2
+      run: >
+        cmake --build build -j 2
+
+    - name: Check ABI tag
+      run: >
+        cd build/tests;
+        python -c 'import test_functions_ext as t; print(f"ABI tag is \"{ t.abi_tag() }\"")'
 
     - name: Run tests
       run: >

include/nanobind/nb_lib.h

@@ -557,6 +557,8 @@ NB_CORE void *type_get_slot(PyTypeObject *t, int slot_id);
 
 NB_CORE PyObject *dict_get_item_ref_or_fail(PyObject *d, PyObject *k);
 
+NB_CORE const char *abi_tag();
+
 NAMESPACE_END(detail)
 
 using detail::raise;

src/nb_internals.cpp

@@ -18,7 +18,7 @@
 
 /// Tracks the ABI of nanobind
 #ifndef NB_INTERNALS_VERSION
-#  define NB_INTERNALS_VERSION 15
+#  define NB_INTERNALS_VERSION 16
 #endif
 
 /// On MSVC, debug and release builds are not ABI-compatible!
@@ -49,30 +49,52 @@
 
 /// Also standard libs
 #if defined(_LIBCPP_VERSION)
-#  define NB_STDLIB "_libcpp"
-#elif defined(__GLIBCXX__) || defined(__GLIBCPP__)
-#  define NB_STDLIB "_libstdcpp"
+#  define NB_STDLIB "_libc++"
+#elif defined(__GLIBCXX__)
+#  define NB_STDLIB "_libstdc++"
 #else
 #  define NB_STDLIB ""
 #endif
 
-/// On Linux/OSX, changes in __GXX_ABI_VERSION__ indicate ABI incompatibility.
-/// Also keep potentially ABI-incompatible visual studio builds apart.
-#if defined(__GXX_ABI_VERSION)
-#  define NB_BUILD_ABI "_cxxabi" NB_TOSTRING(__GXX_ABI_VERSION)
-#elif defined(_MSC_VER)
-#  define NB_BUILD_ABI "_mscver" NB_TOSTRING(_MSC_VER)
+// Catch other conditions that imply ABI incompatibility
+// - MSVC builds with different CRT versions
+// - An anticipated MSVC ABI break ("vNext")
+// - Builds using libc++ with unstable ABIs
+// - Builds using libstdc++ with the legacy (pre-C++11) ABI
+#if defined(_MSC_VER)
+#  if defined(_MT) && defined(_DLL) // catches /MD or /MDd
+#    define NB_BUILD_LIB "_md"
+#  elif defined(_MT)
+#    define NB_BUILD_LIB "_mt"      // catches /MT or /MTd
+#  else
+#    define NB_BUILD_LIB ""
+#  endif
+#  if (_MSC_VER) / 100 == 19
+#    define NB_BUILD_ABI NB_BUILD_LIB "_19"
+#  else
+#    define NB_BUILD_ABI NB_BUILD_LIB "_unknown"
+#  endif
+#elif defined(_LIBCPP_ABI_VERSION)
+#  define NB_BUILD_ABI "_abi" NB_TOSTRING(_LIBCPP_ABI_VERSION)
+#elif defined(__GLIBCXX__)
+#  if _GLIBCXX_USE_CXX11_ABI
+#    define NB_BUILD_ABI ""
+#  else
+#    define NB_BUILD_ABI "_legacy"
+#  endif
 #else
 #  define NB_BUILD_ABI ""
 #endif
 
-// Can have limited and non-limited-API extensions in the same process, and they might be incompatible
+// Can have limited and non-limited-API extensions in the same process.
+// Nanobind data structures will differ, so these can't talk to each other
 #if defined(Py_LIMITED_API)
 #  define NB_STABLE_ABI "_stable"
 #else
 #  define NB_STABLE_ABI ""
 #endif
 
+// As above, but for free-threaded extensions
 #if defined(NB_FREE_THREADED)
 #  define NB_FREE_THREADED_ABI "_ft"
 #else
@@ -85,7 +107,7 @@
   #define NB_VERSION_DEV_STR ""
 #endif
 
-#define NB_INTERNALS_ID                                                        \
+#define NB_ABI_TAG                                                        \
     "v" NB_TOSTRING(NB_INTERNALS_VERSION)                                      \
         NB_VERSION_DEV_STR NB_COMPILER_TYPE NB_STDLIB NB_BUILD_ABI             \
             NB_BUILD_TYPE NB_STABLE_ABI NB_FREE_THREADED_ABI
@@ -241,6 +263,8 @@ static bool is_alive_value = false;
 static bool *is_alive_ptr = &is_alive_value;
 bool is_alive() noexcept { return *is_alive_ptr; }
 
+const char *abi_tag() { return NB_ABI_TAG; }
+
 static void internals_cleanup() {
     nb_internals *p = internals;
     if (!p)
@@ -382,7 +406,7 @@ NB_NOINLINE void init(const char *name) {
     check(dict, "nanobind::detail::init(): could not access internals dictionary!");
 
     PyObject *key = PyUnicode_FromFormat("__nb_internals_%s_%s__",
-                                         NB_INTERNALS_ID, name ? name : "");
+                                         abi_tag(), name ? name : "");
     check(key, "nanobind::detail::init(): could not create dictionary key!");
 
     PyObject *capsule = dict_get_item_ref_or_fail(dict, key);

tests/test_functions.cpp

@@ -377,4 +377,6 @@ NB_MODULE(test_functions_ext, m) {
     m.def("test_bytearray_c_str",   [](nb::bytearray o) -> const char * { return o.c_str(); });
     m.def("test_bytearray_size",    [](nb::bytearray o) { return o.size(); });
     m.def("test_bytearray_resize",  [](nb::bytearray c, int size) { return c.resize(size); });
+
+    m.def("abi_tag", [](){ return nb::detail::abi_tag(); });
 }

tests/test_functions_ext.pyi.ref

@@ -3,6 +3,8 @@ import types
 from typing import Annotated, Any, overload
 
 
+def abi_tag() -> str: ...
+
 def call_guard_value() -> int: ...
 
 def hash_it(arg: object, /) -> int: ...