Merge pull request #53 from bluescarni/function

(WIP) Serializable function

.gitignore

@@ -30,7 +30,7 @@
 .buildinfo
 
 # Directories
-/build
+/build*
 /doc/doxygen/html
 /doc/doxygen/latex
 /doc/doxygen/xml

CMakeLists.txt

@@ -208,15 +208,16 @@ if(DCGP_BUILD_DCGP)
 
     # Setup of the header-only dcgp library.
     add_library(dcgp INTERFACE)
-    target_link_libraries(dcgp INTERFACE Boost::boost Boost::serialization Eigen3::eigen3 TBB::tbb)
-    target_link_libraries(dcgp INTERFACE Audi::audi Pagmo::pagmo ${SYMENGINE_LIBRARIES})
-
 
     # This sets up the include directory to be different if we build
     target_include_directories(dcgp INTERFACE
-    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
-    $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/include>
-    $<INSTALL_INTERFACE:include>)
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/include>
+        $<INSTALL_INTERFACE:include>
+    )
+
+    target_link_libraries(dcgp INTERFACE Boost::boost Boost::serialization Eigen3::eigen3 TBB::tbb)
+    target_link_libraries(dcgp INTERFACE Audi::audi Pagmo::pagmo ${SYMENGINE_LIBRARIES})
 
     # Build main
     if(DCGP_BUILD_MAIN)

README.md

@@ -65,4 +65,4 @@ If all below statements are true:
  * You do not care about the possibility of defining your kernel functions as complex functors (e.g. CGP expressions.)
  * You do not care about thread-safety
 
-then you should consider using, instead, Andrew Turner's CGP-Library (http://www.cgplibrary.co.uk/files2/About-txt.html) which is, roughly, twice as fast to compute a CGP expression as it makes use of function pointers rather than a std::function to define the kernel functions.
+then you should consider using, instead, Andrew Turner's CGP-Library (http://www.cgplibrary.co.uk/files2/About-txt.html) which is, roughly, twice as fast to compute a CGP expression as it makes use of function pointers rather than a type-erased function wrapper to define the kernel functions.

azure-pipelines.yml

@@ -58,6 +58,8 @@ jobs:
                         tbb-devel ^
                         audi ^
                         pyaudi ^
+                        matplotlib ^
+                        sympy ^
                         python=3.7
           conda list
         displayName: "Install conda packages"
@@ -78,7 +80,7 @@ jobs:
                 -DDCGP_BUILD_DCGP=yes ^
                 -DDCGP_BUILD_TESTS=yes
           cmake --build . -- -v
-          ctest -j4 -V .
+          ctest -V .
           cmake --build . --target install
           cd ..
           mkdir build_pyaudi
@@ -96,4 +98,7 @@ jobs:
           cmake --build . -- -v
           cmake --build . --target install
           python -c "import dcgpy.test; dcgpy.test.run_test_suite()"
+          cd ..
+          cd tools
+          python tutorial1.py
         displayName: "Configure, build and test"

dcgpy/expose_kernels.cpp

@@ -5,13 +5,23 @@
 #define PY_ARRAY_UNIQUE_SYMBOL dcgpy_ARRAY_API
 #include "numpy.hpp"
 
+#include <boost/numeric/conversion/cast.hpp>
 #include <boost/python.hpp>
+
+#include <memory>
 #include <sstream>
 #include <string>
 #include <vector>
 
+#include <audi/audi.hpp>
+
+#include <pagmo/threading.hpp>
+
+#include <dcgp/function.hpp>
 #include <dcgp/kernel.hpp>
 #include <dcgp/kernel_set.hpp>
+#include <dcgp/s11n.hpp>
+#include <dcgp/wrapped_functions_s11n_implement.hpp>
 
 #include "common_utils.hpp"
 #include "docstrings.hpp"
@@ -24,24 +34,193 @@ namespace bp = boost::python;
 namespace dcgpy
 {
 
+// Wrapper around the CPython function to create a bytes object from raw data.
+inline bp::object make_bytes(const char *ptr, Py_ssize_t len)
+{
+    PyObject *retval;
+    if (len) {
+        retval = PyBytes_FromStringAndSize(ptr, len);
+    } else {
+        retval = PyBytes_FromStringAndSize(nullptr, 0);
+    }
+    if (!retval) {
+        dcgpy_throw(PyExc_RuntimeError, "unable to create a bytes object: the 'PyBytes_FromStringAndSize()' "
+                                        "function returned NULL");
+    }
+    return bp::object(bp::handle<>(retval));
+}
+
+// Perform a deep copy of input object o.
+inline bp::object deepcopy(const bp::object &o)
+{
+    return bp::import("copy").attr("deepcopy")(o);
+}
+
+inline std::vector<char> object_to_vchar(const bp::object &o)
+{
+    // This will dump to a bytes object.
+    bp::object tmp = bp::import("cloudpickle").attr("dumps")(o);
+    // This gives a null-terminated char * to the internal
+    // content of the bytes object.
+    auto ptr = PyBytes_AsString(tmp.ptr());
+    if (!ptr) {
+        dcgpy_throw(PyExc_TypeError, "the serialization backend's dumps() function did not return a bytes object");
+    }
+    // NOTE: this will be the length of the bytes object *without* the terminator.
+    const auto size = len(tmp);
+    // NOTE: we store as char here because that's what is returned by the CPython function.
+    // From Python it seems like these are unsigned chars, but this should not concern us.
+    return std::vector<char>(ptr, ptr + size);
+}
+
+inline bp::object vchar_to_object(const std::vector<char> &v)
+{
+    auto b = make_bytes(v.data(), boost::numeric_cast<Py_ssize_t>(v.size()));
+    return bp::import("cloudpickle").attr("loads")(b);
+}
+
+} // namespace dcgpy
+
+namespace dcgp::detail
+{
+
+template <typename T>
+struct function_inner<bp::object, T, const std::vector<T> &> final : function_inner_base<T, const std::vector<T> &> {
+    // We just need the def ctor, delete everything else.
+    function_inner() = default;
+    function_inner(const function_inner &) = delete;
+    function_inner(function_inner &&) = delete;
+    function_inner &operator=(const function_inner &) = delete;
+    function_inner &operator=(function_inner &&) = delete;
+
+    // Constructor from generic python object.
+    explicit function_inner(const bp::object &o)
+    {
+        m_value = dcgpy::deepcopy(o);
+    }
+
+    // Clone method.
+    virtual std::unique_ptr<function_inner_base<T, const std::vector<T> &>> clone() const override final
+    {
+        // This will make a deep copy using the ctor above.
+        return std::make_unique<function_inner>(m_value);
+    }
+
+    // Mandatory methods.
+    virtual T operator()(const std::vector<T> &v) const override final
+    {
+        return bp::extract<T>(m_value(dcgpy::v_to_l(v)));
+    }
+
+    virtual pagmo::thread_safety get_thread_safety() const override final
+    {
+        return pagmo::thread_safety::none;
+    }
+
+    template <typename Archive>
+    void save(Archive &ar, unsigned) const
+    {
+        ar << boost::serialization::base_object<function_inner_base<T, const std::vector<T> &>>(*this);
+        ar << dcgpy::object_to_vchar(m_value);
+    }
+    template <typename Archive>
+    void load(Archive &ar, unsigned)
+    {
+        ar >> boost::serialization::base_object<function_inner_base<T, const std::vector<T> &>>(*this);
+        std::vector<char> v;
+        ar >> v;
+        m_value = dcgpy::vchar_to_object(v);
+    }
+    BOOST_SERIALIZATION_SPLIT_MEMBER()
+
+    bp::object m_value;
+};
+
+} // namespace dcgp::detail
+
+namespace dcgp::s11n_names
+{
+
+using udf_bp_object_double = dcgp::detail::function_inner<bp::object, double, const std::vector<double> &>;
+using udf_bp_object_string = dcgp::detail::function_inner<bp::object, std::string, const std::vector<std::string> &>;
+using udf_bp_object_gdual_d
+    = dcgp::detail::function_inner<bp::object, audi::gdual_d, const std::vector<audi::gdual_d> &>;
+using udf_bp_object_gdual_v
+    = dcgp::detail::function_inner<bp::object, audi::gdual_v, const std::vector<audi::gdual_v> &>;
+
+} // namespace dcgp::s11n_names
+
+BOOST_CLASS_EXPORT_KEY2(dcgp::s11n_names::udf_bp_object_double, "udf bp::object double")
+BOOST_CLASS_TRACKING(dcgp::s11n_names::udf_bp_object_double, boost::serialization::track_never)
+BOOST_CLASS_EXPORT_IMPLEMENT(dcgp::s11n_names::udf_bp_object_double)
+
+BOOST_CLASS_EXPORT_KEY2(dcgp::s11n_names::udf_bp_object_gdual_d, "udf bp::object gdual_d")
+BOOST_CLASS_TRACKING(dcgp::s11n_names::udf_bp_object_gdual_d, boost::serialization::track_never)
+BOOST_CLASS_EXPORT_IMPLEMENT(dcgp::s11n_names::udf_bp_object_gdual_d)
+
+BOOST_CLASS_EXPORT_KEY2(dcgp::s11n_names::udf_bp_object_gdual_v, "udf bp::object gdual_v")
+BOOST_CLASS_TRACKING(dcgp::s11n_names::udf_bp_object_gdual_v, boost::serialization::track_never)
+BOOST_CLASS_EXPORT_IMPLEMENT(dcgp::s11n_names::udf_bp_object_gdual_v)
+
+BOOST_CLASS_EXPORT_KEY2(dcgp::s11n_names::udf_bp_object_string, "udf bp::object string")
+BOOST_CLASS_TRACKING(dcgp::s11n_names::udf_bp_object_string, boost::serialization::track_never)
+BOOST_CLASS_EXPORT_IMPLEMENT(dcgp::s11n_names::udf_bp_object_string)
+
+namespace dcgpy
+{
+
+template <typename T>
+struct kernel_pickle_suite : bp::pickle_suite {
+    static bp::tuple getstate(const T &k)
+    {
+        // The idea here is that first we extract a char array
+        // into which the kernel has been serialized, then we turn
+        // this object into a Python bytes object and return that.
+        std::ostringstream oss;
+        {
+            boost::archive::binary_oarchive oarchive(oss);
+            oarchive << k;
+        }
+        auto s = oss.str();
+        // Store the serialized kernel.
+        return bp::make_tuple(make_bytes(s.data(), boost::numeric_cast<Py_ssize_t>(s.size())));
+    }
+    static void setstate(T &k, const bp::tuple &state)
+    {
+        // Similarly, first we extract a bytes object from the Python state,
+        // and then we build a C++ string from it. The string is then used
+        // to deserialize the object.
+        if (len(state) != 1) {
+            dcgpy_throw(PyExc_ValueError, ("the state tuple passed for kernel deserialization "
+                                           "must have 1 element, but instead it has "
+                                           + std::to_string(len(state)) + " elements")
+                                              .c_str());
+        }
+
+        auto ptr = PyBytes_AsString(bp::object(state[0]).ptr());
+        if (!ptr) {
+            dcgpy_throw(PyExc_TypeError, "a bytes object is needed to deserialize a kernel");
+        }
+        const auto size = len(state[0]);
+        std::string s(ptr, ptr + size);
+        std::istringstream iss;
+        iss.str(s);
+        {
+            boost::archive::binary_iarchive iarchive(iss);
+            iarchive >> k;
+        }
+    }
+};
+
 template <typename T>
 void expose_kernel(const std::string &type)
 {
     std::string class_name = "kernel_" + type;
-    bp::class_<kernel<T>>(class_name.c_str(), "The function defining the generic CGP node", bp::no_init)
+    bp::class_<kernel<T>>(class_name.c_str(), "The function defining the generic CGP node", bp::init<>())
         .def("__init__",
              bp::make_constructor(
                  +[](const bp::object &obj1, const bp::object &obj2, const std::string &name) {
-                     std::function<T(const std::vector<T> &)> my_function = [obj1](const std::vector<T> &x) {
-                         T in = bp::extract<T>(obj1(v_to_l(x)));
-                         return in;
-                     };
-                     std::function<std::string(const std::vector<std::string> &)> my_print_function
-                         = [obj2](const std::vector<std::string> &x) {
-                               std::string in = bp::extract<std::string>(obj2(v_to_l(x)));
-                               return in;
-                           };
-                     return ::new kernel<T>(my_function, my_print_function, name);
+                     return ::new kernel<T>(obj1, obj2, name);
                  },
                  bp::default_call_policies(), (bp::arg("callable_f"), bp::arg("callable_s"), bp::arg("name"))),
              kernel_init_doc(type).c_str())
@@ -58,11 +237,13 @@ void expose_kernel(const std::string &type)
                 }
             })
         .def(
-            "__repr__", +[](const kernel<T> &instance) -> std::string {
+            "__repr__",
+            +[](const kernel<T> &instance) -> std::string {
                 std::ostringstream oss;
                 oss << instance;
                 return oss.str();
-            });
+            })
+        .def_pickle(kernel_pickle_suite<kernel<T>>());
     ;
 }
 
@@ -77,7 +258,7 @@ void expose_kernel_set(std::string type)
 {
     std::string class_name = "kernel_set_" + type;
     bp::class_<kernel_set<T>>(class_name.c_str(),
-                              "Helper to construct a set of kernel functions from their common name", bp::no_init)
+                              "Helper to construct a set of kernel functions from their common name", bp::init<>())
         .def("__init__",
              bp::make_constructor(
                  +[](const bp::object &obj1) {
@@ -99,7 +280,8 @@ void expose_kernel_set(std::string type)
              kernel_set_push_back_str_doc().c_str(), bp::arg("kernel_name"))
         .def("push_back", (void (kernel_set<T>::*)(const kernel<T> &)) & kernel_set<T>::push_back,
              kernel_set_push_back_ker_doc(type).c_str(), bp::arg("kernel"))
-        .def("__getitem__", &wrap_operator<T>);
+        .def("__getitem__", &wrap_operator<T>)
+        .def_pickle(kernel_pickle_suite<kernel_set<T>>());
 }
 
 void expose_kernels()