Fix OSS build for Python extensions (#2370)

Summary: Pull Request resolved: #2370 This diff fixes the OSS build for folly's Python extensions (`folly.iobuf` and `folly.executor` libraries), which seem to have been in an unmaintained state for a while now. Reviewed By: skcoirz Differential Revision: D68169324 fbshipit-source-id: c65f5bd50ffeac32d3584b2b348ce5945863952e
facebook · Jan 17, 2025 · f1c1542 · f1c1542
1 parent 4c5ab8b
commit f1c1542
Show file tree

Hide file tree

Showing 3 changed files with 91 additions and 26 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -466,6 +466,12 @@ option(PYTHON_EXTENSIONS
   "Build Python Bindings for Folly, requires Cython and (BUILD_SHARED_LIBS=ON)"
   OFF
 )
+set(PYTHON_PACKAGE_INSTALL_DIR "" CACHE STRING
+  "Installation directory for folly Python packages. \
+  If empty, CMAKE_INSTALL_PREFIX will be used. \
+  No effect if PYTHON_EXTENSIONS=OFF."
+)
+
 
 add_library(folly
   $<TARGET_OBJECTS:folly_base>

diff --git a/folly/CMakeLists.txt b/folly/CMakeLists.txt
@@ -40,69 +40,122 @@ if (PYTHON_EXTENSIONS)
 
   add_custom_target(create_binding_symlink ALL)
   file(GLOB BindingFiles
-    "${CMAKE_CURRENT_SOURCE_DIR}/python/*.pyx"
+    "${CMAKE_CURRENT_SOURCE_DIR}/python/executor.pyx"
+    "${CMAKE_CURRENT_SOURCE_DIR}/python/iobuf.pyx"
+    "${CMAKE_CURRENT_SOURCE_DIR}/python/iobuf_ext.cpp"
+    "${CMAKE_CURRENT_SOURCE_DIR}/python/ProactorExecutor.cpp"
     "${CMAKE_CURRENT_SOURCE_DIR}/python/*.pxd"
     "${CMAKE_CURRENT_SOURCE_DIR}/python/*.h"
-    "${CMAKE_CURRENT_SOURCE_DIR}/python/*.cpp"
   )
   file(MAKE_DIRECTORY "${_cybld}/folly/")
+  file(MAKE_DIRECTORY "${_cybld}/folly/python/")
 
   foreach(_src ${BindingFiles})
     get_filename_component(_target_file "${_src}" NAME)
 
-  message(
-    STATUS
-    "Linking ${_src} "
-    "to ${_cybld}/folly/${_target_file}"
-  )
-  add_custom_command(TARGET create_binding_symlink PRE_BUILD
-    COMMAND
-      ${CMAKE_COMMAND} -E create_symlink
-      "${_src}"
-      "${_cybld}/folly/${_target_file}"
-  )
+    message(
+      STATUS
+      "Linking ${_src} "
+      "to ${_cybld}/folly/${_target_file}"
+    )
+    add_custom_command(TARGET create_binding_symlink PRE_BUILD
+      COMMAND
+        ${CMAKE_COMMAND} -E create_symlink
+        "${_src}"
+        "${_cybld}/folly/${_target_file}"
+    )
   endforeach()
 
   # Tell setup.py where to find includes and libfolly.so
   set(prop "$<TARGET_PROPERTY:folly_base,INCLUDE_DIRECTORIES>")
   set(incs "$<$<BOOL:${prop}>:-I$<JOIN:${prop},:>>")
-  set(libs "-L${CMAKE_BINARY_DIR}")
+  set(glog_lib "${GLOG_LIBRARY}")
+  cmake_path(REMOVE_FILENAME glog_lib)
+  set(libs "-L${CMAKE_BINARY_DIR}:${glog_lib}")
 
   add_custom_target(folly_python_bindings ALL
     DEPENDS folly create_binding_symlink
     WORKING_DIRECTORY ${_cybld})
 
   add_custom_command(TARGET folly_python_bindings POST_BUILD
     COMMAND
+      CFLAGS="${CMAKE_C_FLAGS}" CXXFLAGS="${CMAKE_CXX_FLAGS}"
+      CC="${CMAKE_C_COMPILER}" CXX="${CMAKE_CXX_COMPILER}"
       python3 ${CMAKE_CURRENT_SOURCE_DIR}/python/setup.py
       build_ext -f ${incs} ${libs}
-    BYPRODUCTS ${_cybld}/folly/executor_api.h
+    BYPRODUCTS
+      ${_cybld}/folly/executor_api.h
+      ${_cybld}/folly/iobuf_api.h
     WORKING_DIRECTORY ${_cybld}
   )
 
-  add_custom_command(TARGET folly_python_bindings POST_BUILD
+  add_custom_target(create_post_binding_symlink ALL)
+  add_custom_command(TARGET create_post_binding_symlink
+    COMMAND
+      ${CMAKE_COMMAND} -E create_symlink
+      "${_cybld}/folly/executor_api.h"
+      "${_cybld}/folly/python/executor_api.h"
+  )
+  add_custom_command(TARGET create_post_binding_symlink
+    COMMAND
+      ${CMAKE_COMMAND} -E create_symlink
+      "${_cybld}/folly/iobuf_api.h"
+      "${_cybld}/folly/python/iobuf_api.h"
+  )
+
+  add_library(
+    folly_python_cpp
+      python/error.cpp
+      python/executor.cpp
+      python/iobuf.cpp
+  )
+
+  add_dependencies(folly_python_cpp folly_python_bindings create_post_binding_symlink)
+  set_property(TARGET folly_python_cpp PROPERTY VERSION ${PACKAGE_VERSION})
+  target_compile_definitions(folly_python_cpp PRIVATE BOOST_NO_AUTO_PTR)
+  target_include_directories(folly_python_cpp PRIVATE "${_cybld}")
+  target_link_libraries(folly_python_cpp PUBLIC folly)
+  apply_folly_compile_options_to_target(folly_python_cpp)
+  install(
+    TARGETS folly_python_cpp
+    EXPORT folly
+    DESTINATION ${LIB_INSTALL_DIR}
+  )
+  install(
+    DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    DESTINATION ${INCLUDE_INSTALL_DIR}
+    FILES_MATCHING
+      PATTERN "*.h"
+  )
+
+  add_custom_command(TARGET folly_python_bindings
     COMMAND
       python3 ${CMAKE_CURRENT_SOURCE_DIR}/python/setup.py
       bdist_wheel
     WORKING_DIRECTORY ${_cybld}
   )
 
   install(
-    FILES ${_cybld}/folly/executor_api.h
+    FILES
+      ${_cybld}/folly/executor_api.h
+      ${_cybld}/folly/iobuf_api.h
     DESTINATION ${INCLUDE_INSTALL_DIR}/folly/python
     COMPONENT dev
   )
 
-  # Install Folly Python Bindings
-  if (UNIX)
-    set(ROOT_ARG "--root \$DESTDIR/")
-  endif ()
+  # Install Folly Python bindings. getdeps.py does not allow sufficient
+  # control of CMAKE_INSTALL_PREFIX, so PYTHON_PACKAGE_INSTALL_DIR can
+  # be used to control the installation location of Python packages.
+  set(INSTALL_DIR ${CMAKE_INSTALL_PREFIX})
+  if (NOT ${PYTHON_PACKAGE_INSTALL_DIR} STREQUAL "")
+    set(INSTALL_DIR ${PYTHON_PACKAGE_INSTALL_DIR})
+  endif()
 
   install(CODE "
     execute_process(
       COMMAND
         python3 ${CMAKE_CURRENT_SOURCE_DIR}/python/setup.py install
-        --prefix ${CMAKE_INSTALL_PREFIX} ${ROOT_ARG}
+        --prefix ${INSTALL_DIR}
       COMMAND_ECHO STDOUT
       WORKING_DIRECTORY ${_cybld}
     )"

diff --git a/folly/python/setup.py b/folly/python/setup.py
@@ -28,13 +28,19 @@
 exts = [
     Extension(
         "folly.executor",
-        sources=["folly/executor.pyx"],
-        libraries=["folly", "glog", "double-conversion", "iberty"],
+        sources=["folly/executor.pyx", "folly/ProactorExecutor.cpp"],
+        libraries=["folly", "glog"],
+        extra_compile_args=[
+            "-std=c++17",
+        ],
     ),
     Extension(
         "folly.iobuf",
-        sources=["folly/iobuf.pyx"],
-        libraries=["folly", "glog", "double-conversion", "iberty"],
+        sources=["folly/iobuf.pyx", "folly/iobuf_ext.cpp"],
+        libraries=["folly", "glog"],
+        extra_compile_args=[
+            "-std=c++17",
+        ],
     ),
 ]