From 228624b1d26bd243ab340f8b4083565b424dcd6d Mon Sep 17 00:00:00 2001
From: Timothy Mastny <tim.mastny@gmail.com>
Date: Sun, 25 Feb 2018 20:09:16 -0600
Subject: [PATCH 01/26] added unit tests that cover knitr #1505

---
 .../resources/eng-reticulate-cache-test.Rmd   | 22 ++++++++++
 tests/testthat/test-python-cache-engine.R     | 44 +++++++++++++++++++
 tests/testthat/test-python-dill.R             | 37 ++++++++++++++++
 tests/testthat/test-python-globals.R          | 27 ++++++++++++
 4 files changed, 130 insertions(+)
 create mode 100644 tests/testthat/resources/eng-reticulate-cache-test.Rmd
 create mode 100644 tests/testthat/test-python-cache-engine.R
 create mode 100644 tests/testthat/test-python-dill.R
 create mode 100644 tests/testthat/test-python-globals.R

diff --git a/tests/testthat/resources/eng-reticulate-cache-test.Rmd b/tests/testthat/resources/eng-reticulate-cache-test.Rmd
new file mode 100644
index 000000000..a878f8b98
--- /dev/null
+++ b/tests/testthat/resources/eng-reticulate-cache-test.Rmd
@@ -0,0 +1,22 @@
+---
+title: "Using reticulate's Python Engine with knitr"
+---
+
+```{r setup, include = FALSE}
+library(reticulate)
+knitr::opts_chunk$set(cache=TRUE)
+knitr::knit_engines$set(python = eng_python)
+knitr::cache_engines$set(python = cache_eng_python)
+```
+
+Cache can handle changes to second chunk:
+
+```{python}
+x = 1
+```
+
+```{python}
+print(x + 1)
+```
+
+
diff --git a/tests/testthat/test-python-cache-engine.R b/tests/testthat/test-python-cache-engine.R
new file mode 100644
index 000000000..2c4caa6e0
--- /dev/null
+++ b/tests/testthat/test-python-cache-engine.R
@@ -0,0 +1,44 @@
+context("knitr-cache")
+
+test_that("An R Markdown document can be rendered with cache using reticulate", {
+  
+  skip_on_cran()
+  skip_if_not_installed("rmarkdown")
+  skip_if_not_installed("callr")
+  
+  path <- callr::r(
+    function() {
+      rmarkdown::render("resources/eng-reticulate-cache-test.Rmd", quiet = TRUE, envir = new.env())
+    })
+  expect_true(file.exists(path))
+  on.exit(unlink(path), add = TRUE)
+})
+
+test_that("An R Markdown document builds if a cache is modified", {
+  
+  skip_on_cran()
+  skip_if_not_installed("rmarkdown")
+  skip_if_not_installed("callr")
+  
+  old_var <- "1"
+  new_var <- "0"
+  mutate_chunk <- function(x) {
+    print_line <- 19
+    file_text <- readLines("resources/eng-reticulate-cache-test.Rmd")
+    file_text[print_line] <- paste("print(x + ", x, ")", sep = "")
+    writeLines(file_text, "resources/eng-reticulate-cache-test.Rmd")
+  }
+  mutate_chunk(old_var)
+  mutate_chunk(new_var)
+  path <- callr::r(
+    function() {
+      rmarkdown::render("resources/eng-reticulate-cache-test.Rmd", quiet = TRUE, envir = new.env())
+    })
+  mutate_chunk(old_var)
+  expect_true(file.exists(path))
+  on.exit(unlink(path), add = TRUE)
+  on.exit(unlink("resources/eng-reticulate-cache-test_cache/", recursive = TRUE), add = TRUE)
+})
+
+
+
diff --git a/tests/testthat/test-python-dill.R b/tests/testthat/test-python-dill.R
new file mode 100644
index 000000000..56efa9d99
--- /dev/null
+++ b/tests/testthat/test-python-dill.R
@@ -0,0 +1,37 @@
+context("dill")
+
+source("utils.R")
+
+test_that("Interpreter sessions can be saved and loaded with dill", {
+  skip_if_no_python()
+  skip_if_not_installed("callr")
+  
+  session_one_vars <- callr::r(
+    function() {
+      module_load <- tryCatch(
+        dill <- reticulate::import("dill"),
+        error = function(c) {
+          py_error <- reticulate::py_last_error()
+          if(py_error$type == "ImportError" && py_error$value == "No module named dill") {
+            "No dill"
+          }})
+      if (module_load == "No dill") return(module_load)
+      main <- reticulate::py_run_string("x = 1")
+      reticulate::py_run_string("y = x + 1")
+      dill$dump_session(filename = "x.dill", byref = TRUE)
+      c(main$x, main$y)
+    })
+  if (session_one_vars[1] == "No dill")
+    skip("The dill Python module is not installed")
+  
+  session_two_vars <- callr::r(
+    function() {
+      dill <- reticulate::import("dill")
+      dill$load_session(filename = "x.dill")
+      main <- reticulate::py_run_string("pass")
+      c(main$x, main$y)
+    })
+  on.exit(unlink("x.dill"), add = TRUE)
+  expect_equal(session_one_vars, session_two_vars)
+})
+
diff --git a/tests/testthat/test-python-globals.R b/tests/testthat/test-python-globals.R
new file mode 100644
index 000000000..d9beb36ab
--- /dev/null
+++ b/tests/testthat/test-python-globals.R
@@ -0,0 +1,27 @@
+context("globals")
+
+source("utils.R")
+
+test_that("Interpreter sessions can be saved and loaded with dill", {
+  skip_if_no_python()
+  
+  py_run_string("x = 1")
+  py_run_string("y = 1")
+  py_run_string("[globals().pop(i) for i in ['x', 'y']]")
+  
+  test_x <- tryCatch(
+    py_run_string("x = x + 1"),
+    error = function(e) {
+      py_last_error()$value
+    }
+  )
+  test_y <- tryCatch(
+    py_run_string("y = y + 1"),
+    error = function(e) {
+      py_last_error()$value
+    }
+  )
+  expect_equal(test_x, "name 'x' is not defined")
+  expect_equal(test_y, "name 'y' is not defined")
+})
+

From b52ecaa4f25c24ab9915c3d2e172daa08e3be381 Mon Sep 17 00:00:00 2001
From: Timothy Mastny <tim.mastny@gmail.com>
Date: Sun, 25 Feb 2018 20:10:17 -0600
Subject: [PATCH 02/26] added cache_eng_python to add Python session caching
 between chunks. Addresses knitr #1505

---
 NAMESPACE               |  1 +
 R/knitr-engine.R        | 67 +++++++++++++++++++++++++++++++++++++++--
 man/cache_eng_python.Rd | 24 +++++++++++++++
 3 files changed, 89 insertions(+), 3 deletions(-)
 create mode 100644 man/cache_eng_python.Rd

diff --git a/NAMESPACE b/NAMESPACE
index 512e13a52..01e133f27 100644
--- a/NAMESPACE
+++ b/NAMESPACE
@@ -66,6 +66,7 @@ S3method(summary,python.builtin.object)
 S3method(with,python.builtin.object)
 export("%as%")
 export(array_reshape)
+export(cache_eng_python)
 export(conda_binary)
 export(conda_create)
 export(conda_install)
diff --git a/R/knitr-engine.R b/R/knitr-engine.R
index a3e4b22e9..f4db73d31 100644
--- a/R/knitr-engine.R
+++ b/R/knitr-engine.R
@@ -175,7 +175,9 @@ eng_python <- function(options) {
   }
   
   eng_python_synchronize_after()
-  
+  if(options$cache > 0) {
+    save_python_session(options$hash)
+  }
   wrap <- getOption("reticulate.engine.wrap", eng_python_wrap)
   wrap(outputs, options)
   
@@ -186,8 +188,18 @@ eng_python_initialize <- function(options, context, envir) {
   if (is.character(options$engine.path))
     use_python(options$engine.path[[1]])
   
-  ensure_python_initialized()
-  
+  if (options$cache > 0) {
+    load_module <- tryCatch(
+      import("dill"),
+      error = function(c) {
+        py_error <- py_last_error()
+        if(py_error$type == "ImportError" && py_error$value == "No module named dill") {
+          warning("The Python module dill was not found. This module is needed for full cache functionality.")
+          "No dill"
+        }})
+    if (load_module != "No dill") 
+      py_run_string("import dill")
+  } 
   eng_python_initialize_matplotlib(options, context, envir)
 }
 
@@ -278,3 +290,52 @@ eng_python_wrap <- function(outputs, options) {
   wrap <- yoink("knitr", "wrap")
   wrap(outputs, options)
 }
+
+save_python_session <- function(cache_path) {
+  dill <- tryCatch(
+    import("dill"),
+    error = function(c) {
+      py_error <- py_last_error()
+      if(py_error$type == "ImportError" && py_error$value == "No module named dill") {
+        "No dill"
+      }})
+  if (dill == "No dill") return()
+
+  py_run_string("globals().pop('r')")
+  py_run_string("globals().pop('R')")
+  dill$dump_session(filename = paste0(cache_path, ".pkl"), byref = TRUE)
+}
+
+#' A reticulate cache engine for Knitr
+#' 
+#' This provides a `reticulate` cache engine for `knitr`. The cache engine
+#' allows `knitr` to save and load Python sessions between cached chunks. The
+#' cache engine depends on the `dill` Python module. Therefore, you must have
+#' `dill` installed in your Python environment.
+#' 
+#' The engine can be activated by setting (for example)
+#' 
+#' ```
+#' knitr::cache_engines$set(python = reticulate::cache_eng_python)
+#' ```
+#' 
+#' Typically, this will be set within a document's setup chunk, or by the
+#' environment requesting that Python chunks be processed by this engine.
+#' 
+#' @param cache_path
+#'   The path to save the chunk cache, as provided by `knitr` during chunk execution.
+#'   
+#' @export
+cache_eng_python <- function(cache_path) {
+  dill <- tryCatch(
+    import("dill"),
+    error = function(c) {
+      py_error <- py_last_error()
+      if(py_error$type == "ImportError" && py_error$value == "No module named dill") {
+        "No dill"
+      }})
+  if (dill == "No dill") return()
+  dill$load_session(filename = paste0(cache_path, ".pkl"))
+}
+
+
diff --git a/man/cache_eng_python.Rd b/man/cache_eng_python.Rd
new file mode 100644
index 000000000..d8fd64ef0
--- /dev/null
+++ b/man/cache_eng_python.Rd
@@ -0,0 +1,24 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/knitr-engine.R
+\name{cache_eng_python}
+\alias{cache_eng_python}
+\title{A reticulate cache engine for Knitr}
+\usage{
+cache_eng_python(cache_path)
+}
+\arguments{
+\item{cache_path}{The path to save the chunk cache, as provided by \code{knitr} during chunk execution.}
+}
+\description{
+This provides a \code{reticulate} cache engine for \code{knitr}. The cache engine
+allows \code{knitr} to save and load Python sessions between cached chunks. The
+cache engine depends on the \code{dill} Python module. Therefore, you must have
+\code{dill} installed in your Python environment.
+}
+\details{
+The engine can be activated by setting (for example)\preformatted{knitr::cache_engines$set(python = reticulate::cache_eng_python)
+}
+
+Typically, this will be set within a document's setup chunk, or by the
+environment requesting that Python chunks be processed by this engine.
+}

From c345ce2fde6555f2f71138de18d219b9e78b59c1 Mon Sep 17 00:00:00 2001
From: Timothy Mastny <tim.mastny@gmail.com>
Date: Wed, 28 Feb 2018 17:14:11 -0600
Subject: [PATCH 03/26] dill caching engine for knitr, with tests

---
 R/knitr-engine.R                          | 60 +++++++++++------------
 R/python.R                                | 60 +++++++++++++++++++++++
 tests/testthat/test-python-cache-engine.R |  2 +
 3 files changed, 91 insertions(+), 31 deletions(-)

diff --git a/R/knitr-engine.R b/R/knitr-engine.R
index f4db73d31..c73d71554 100644
--- a/R/knitr-engine.R
+++ b/R/knitr-engine.R
@@ -189,16 +189,14 @@ eng_python_initialize <- function(options, context, envir) {
     use_python(options$engine.path[[1]])
   
   if (options$cache > 0) {
-    load_module <- tryCatch(
-      import("dill"),
-      error = function(c) {
-        py_error <- py_last_error()
-        if(py_error$type == "ImportError" && py_error$value == "No module named dill") {
-          warning("The Python module dill was not found. This module is needed for full cache functionality.")
-          "No dill"
-        }})
-    if (load_module != "No dill") 
-      py_run_string("import dill")
+    module <- tryCatch(import("dill"), error = identity)
+    if (inherits(module, "error")) {
+      if (module$message == "ImportError: No module named dill") {
+        warning("The Python module dill was not found. This module is needed for full cache functionality.")
+      } else {
+        stop(module$message)
+      }
+    }
   } 
   eng_python_initialize_matplotlib(options, context, envir)
 }
@@ -292,18 +290,20 @@ eng_python_wrap <- function(outputs, options) {
 }
 
 save_python_session <- function(cache_path) {
-  dill <- tryCatch(
-    import("dill"),
-    error = function(c) {
-      py_error <- py_last_error()
-      if(py_error$type == "ImportError" && py_error$value == "No module named dill") {
-        "No dill"
-      }})
-  if (dill == "No dill") return()
-
-  py_run_string("globals().pop('r')")
-  py_run_string("globals().pop('R')")
-  dill$dump_session(filename = paste0(cache_path, ".pkl"), byref = TRUE)
+  module <- tryCatch(import("dill"), error = identity)
+  if (inherits(module, "error")) {
+    if (module$message == "ImportError: No module named dill") return()
+    signalCondition(module$message)
+  }
+  
+  r_objs_exist <- "all(r_obj in globals() for r_obj in ('r', 'R'))"
+  r_is_R <- "isinstance(r, R)"
+  if (py_eval(r_objs_exist) && py_eval(r_is_R)) {
+    py_run_string("globals().pop('r')")
+    py_run_string("globals().pop('R')")
+  }
+  
+  module$dump_session(filename = paste0(cache_path, ".pkl"), byref = TRUE)
 }
 
 #' A reticulate cache engine for Knitr
@@ -327,15 +327,13 @@ save_python_session <- function(cache_path) {
 #'   
 #' @export
 cache_eng_python <- function(cache_path) {
-  dill <- tryCatch(
-    import("dill"),
-    error = function(c) {
-      py_error <- py_last_error()
-      if(py_error$type == "ImportError" && py_error$value == "No module named dill") {
-        "No dill"
-      }})
-  if (dill == "No dill") return()
-  dill$load_session(filename = paste0(cache_path, ".pkl"))
+  module <- tryCatch(import("dill"), error = identity)
+  if (inherits(module, "error")) {
+    if (module$message == "ImportError: No module named dill") return()
+    stop(module$message)
+  }
+
+  module$load_session(filename = paste0(cache_path, ".pkl"))
 }
 
 
diff --git a/R/python.R b/R/python.R
index d6bfeacf7..2440e1a5f 100644
--- a/R/python.R
+++ b/R/python.R
@@ -198,6 +198,66 @@ summary.python.builtin.object <- function(object, ...) {
   str(object)
 }
 
+
+#' Convert between Python and R objects
+#' 
+#' @inheritParams import
+#' @param x Object to convert
+#' 
+#' @return Converted object
+#' 
+#' @name r-py-conversion
+#' @export
+py_to_r <- function(x) {
+  
+  ensure_python_initialized()
+  
+  if (!inherits(x, "python.builtin.object"))
+    stop("Object to convert is not a Python object")
+  
+  # get the default wrapper
+  x <- py_ref_to_r(x)
+  
+  # allow customization of the wrapper
+  wrapper <- py_to_r_wrapper(x)
+  attributes(wrapper) <- attributes(x)
+  
+  # return the wrapper
+  wrapper 
+}
+
+#' R wrapper for Python objects 
+#' 
+#' S3 method to create a custom R wrapper for a Python object.
+#' The default wrapper is either an R environment or an R function
+#' (for callable python objects).
+#' 
+#' @param x Python object 
+#' 
+#' @export
+py_to_r_wrapper <- function(x) {
+  UseMethod("py_to_r_wrapper")
+}
+
+#' @export
+py_to_r_wrapper.default <- function(x) {
+  x
+}
+
+
+
+
+
+#' @rdname r-py-conversion
+#' @export
+r_to_py <- function(x, convert = FALSE) {
+  
+  ensure_python_initialized()
+  
+  r_to_py_impl(x, convert = convert)
+}
+
+
 #' @export
 `$.python.builtin.module` <- function(x, name) {
  
diff --git a/tests/testthat/test-python-cache-engine.R b/tests/testthat/test-python-cache-engine.R
index 2c4caa6e0..940d7016d 100644
--- a/tests/testthat/test-python-cache-engine.R
+++ b/tests/testthat/test-python-cache-engine.R
@@ -6,6 +6,8 @@ test_that("An R Markdown document can be rendered with cache using reticulate",
   skip_if_not_installed("rmarkdown")
   skip_if_not_installed("callr")
   
+  unlink("resources/eng-reticulate-cache-test_cache/", recursive = TRUE)
+  
   path <- callr::r(
     function() {
       rmarkdown::render("resources/eng-reticulate-cache-test.Rmd", quiet = TRUE, envir = new.env())

From ff39889bb09eae41f9c07aa1a6d12c6ac8ebc14f Mon Sep 17 00:00:00 2001
From: Timothy Mastny <tim.mastny@gmail.com>
Date: Wed, 18 Apr 2018 17:19:02 -0500
Subject: [PATCH 04/26] changes from feedback on knitr #1518 with updated tests

---
 R/knitr-engine.R                                       | 9 +++++----
 man/cache_eng_python.Rd                                | 5 +++--
 tests/testthat/resources/eng-reticulate-cache-test.Rmd | 2 --
 tests/testthat/test-python-cache-engine.R              | 4 ++--
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/R/knitr-engine.R b/R/knitr-engine.R
index c73d71554..50cb5e1dc 100644
--- a/R/knitr-engine.R
+++ b/R/knitr-engine.R
@@ -322,18 +322,19 @@ save_python_session <- function(cache_path) {
 #' Typically, this will be set within a document's setup chunk, or by the
 #' environment requesting that Python chunks be processed by this engine.
 #' 
-#' @param cache_path
-#'   The path to save the chunk cache, as provided by `knitr` during chunk execution.
+#' @param options
+#'   List of chunk options provided by `knitr` during chunk execution. 
+#'   Contains the caching path.
 #'   
 #' @export
-cache_eng_python <- function(cache_path) {
+cache_eng_python <- function(options) {
   module <- tryCatch(import("dill"), error = identity)
   if (inherits(module, "error")) {
     if (module$message == "ImportError: No module named dill") return()
     stop(module$message)
   }
 
-  module$load_session(filename = paste0(cache_path, ".pkl"))
+  module$load_session(filename = paste0(options$hash, ".pkl"))
 }
 
 
diff --git a/man/cache_eng_python.Rd b/man/cache_eng_python.Rd
index d8fd64ef0..ff8067b7b 100644
--- a/man/cache_eng_python.Rd
+++ b/man/cache_eng_python.Rd
@@ -4,10 +4,11 @@
 \alias{cache_eng_python}
 \title{A reticulate cache engine for Knitr}
 \usage{
-cache_eng_python(cache_path)
+cache_eng_python(options)
 }
 \arguments{
-\item{cache_path}{The path to save the chunk cache, as provided by \code{knitr} during chunk execution.}
+\item{options}{List of chunk options provided by \code{knitr} during chunk execution.
+Contains the caching path.}
 }
 \description{
 This provides a \code{reticulate} cache engine for \code{knitr}. The cache engine
diff --git a/tests/testthat/resources/eng-reticulate-cache-test.Rmd b/tests/testthat/resources/eng-reticulate-cache-test.Rmd
index a878f8b98..2017d584a 100644
--- a/tests/testthat/resources/eng-reticulate-cache-test.Rmd
+++ b/tests/testthat/resources/eng-reticulate-cache-test.Rmd
@@ -5,8 +5,6 @@ title: "Using reticulate's Python Engine with knitr"
 ```{r setup, include = FALSE}
 library(reticulate)
 knitr::opts_chunk$set(cache=TRUE)
-knitr::knit_engines$set(python = eng_python)
-knitr::cache_engines$set(python = cache_eng_python)
 ```
 
 Cache can handle changes to second chunk:
diff --git a/tests/testthat/test-python-cache-engine.R b/tests/testthat/test-python-cache-engine.R
index 940d7016d..6d8f17422 100644
--- a/tests/testthat/test-python-cache-engine.R
+++ b/tests/testthat/test-python-cache-engine.R
@@ -25,9 +25,9 @@ test_that("An R Markdown document builds if a cache is modified", {
   old_var <- "1"
   new_var <- "0"
   mutate_chunk <- function(x) {
-    print_line <- 19
+    print_line <- 17
     file_text <- readLines("resources/eng-reticulate-cache-test.Rmd")
-    file_text[print_line] <- paste("print(x + ", x, ")", sep = "")
+    file_text[print_line] <- paste0("print(x + ", x, ")")
     writeLines(file_text, "resources/eng-reticulate-cache-test.Rmd")
   }
   mutate_chunk(old_var)

From 8e07779631d2089310e46cecd38cf5a1c66a79c4 Mon Sep 17 00:00:00 2001
From: Timothy Mastny <tim.mastny@gmail.com>
Date: Wed, 18 Apr 2018 19:18:23 -0500
Subject: [PATCH 05/26] fixed testing utils source in dill tests

---
 tests/testthat/test-python-dill.R | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/testthat/test-python-dill.R b/tests/testthat/test-python-dill.R
index 56efa9d99..e14b9c04d 100644
--- a/tests/testthat/test-python-dill.R
+++ b/tests/testthat/test-python-dill.R
@@ -1,6 +1,6 @@
 context("dill")
 
-source("utils.R")
+source("helper-utils.R")
 
 test_that("Interpreter sessions can be saved and loaded with dill", {
   skip_if_no_python()

From 975387094716b46a4fc713ee621d9e329c8a2538 Mon Sep 17 00:00:00 2001
From: Leonardo Gama <leonardo.reis.gama@gmail.com>
Date: Wed, 13 Apr 2022 21:40:07 -0300
Subject: [PATCH 06/26] cache engine: update 'r' object identification logic

---
 R/knitr-engine.R | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/R/knitr-engine.R b/R/knitr-engine.R
index 53b8004f2..fdf26ff8e 100644
--- a/R/knitr-engine.R
+++ b/R/knitr-engine.R
@@ -649,12 +649,11 @@ save_python_session <- function(cache_path) {
     if (module$message == "ImportError: No module named dill") return()
     signalCondition(module$message)
   }
-  
-  r_objs_exist <- "all(r_obj in globals() for r_obj in ('r', 'R'))"
-  r_is_R <- "isinstance(r, R)"
-  if (py_eval(r_objs_exist) && py_eval(r_is_R)) {
-    py_run_string("globals().pop('r')")
-    py_run_string("globals().pop('R')")
+
+  r_obj_exists <- "'r' in globals()"
+  r_is_R <- "type(r).__module__ == '__main__' and type(r).__name__ == 'R'"
+  if (py_eval(r_obj_exists) && py_eval(r_is_R)) {
+    py_run_string("del globals()['r']")
   }
   
   module$dump_session(filename = paste0(cache_path, ".pkl"), byref = TRUE)

From 02c177166a2b3387aa2910fd02c65727eed22a08 Mon Sep 17 00:00:00 2001
From: Leonardo Gama <leonardo.reis.gama@gmail.com>
Date: Wed, 20 Apr 2022 14:52:16 -0300
Subject: [PATCH 07/26] fix 'cache_path' when 'output.dir' is different from
 'knitr:::input_dir()'

---
 R/knitr-engine.R | 1 +
 1 file changed, 1 insertion(+)

diff --git a/R/knitr-engine.R b/R/knitr-engine.R
index fdf26ff8e..07a87eefb 100644
--- a/R/knitr-engine.R
+++ b/R/knitr-engine.R
@@ -656,6 +656,7 @@ save_python_session <- function(cache_path) {
     py_run_string("del globals()['r']")
   }
   
+  cache_path <- file.path(knitr::opts_knit$get("output.dir"), cache_path)
   module$dump_session(filename = paste0(cache_path, ".pkl"), byref = TRUE)
 }
 

From dbebab3ffce4a8e8421bc47fd23e3e7060ffb0cb Mon Sep 17 00:00:00 2001
From: Leonardo Gama <leonardo.reis.gama@gmail.com>
Date: Wed, 20 Apr 2022 17:20:57 -0300
Subject: [PATCH 08/26] cache loading should run in the input directory

This is what cache saving does, therefore it is necesseray that load_session()
runs in the same dir or it won't find local python modules.
---
 R/knitr-engine.R | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/R/knitr-engine.R b/R/knitr-engine.R
index 07a87eefb..449fe4241 100644
--- a/R/knitr-engine.R
+++ b/R/knitr-engine.R
@@ -688,7 +688,8 @@ cache_eng_python <- function(options) {
     stop(module$message)
   }
 
-  module$load_session(filename = paste0(options$hash, ".pkl"))
+  cache_path <- normalizePath(paste0(options$hash, ".pkl"), mustWork = TRUE)
+  knitr:::in_input_dir(module$load_session(filename = cache_path))
 }
 
 

From bd29f84808f9cd1b792bd4dcf3cbf9269d803ee0 Mon Sep 17 00:00:00 2001
From: Leonardo Gama <leonardo.reis.gama@gmail.com>
Date: Thu, 26 May 2022 19:50:13 -0300
Subject: [PATCH 09/26] remove duplicated conversion functions

---
 R/python.R | 60 ------------------------------------------------------
 1 file changed, 60 deletions(-)

diff --git a/R/python.R b/R/python.R
index 6ba597d29..0a362a51b 100644
--- a/R/python.R
+++ b/R/python.R
@@ -88,66 +88,6 @@ summary.python.builtin.object <- function(object, ...) {
   str(object)
 }
 
-
-#' Convert between Python and R objects
-#' 
-#' @inheritParams import
-#' @param x Object to convert
-#' 
-#' @return Converted object
-#' 
-#' @name r-py-conversion
-#' @export
-py_to_r <- function(x) {
-  
-  ensure_python_initialized()
-  
-  if (!inherits(x, "python.builtin.object"))
-    stop("Object to convert is not a Python object")
-  
-  # get the default wrapper
-  x <- py_ref_to_r(x)
-  
-  # allow customization of the wrapper
-  wrapper <- py_to_r_wrapper(x)
-  attributes(wrapper) <- attributes(x)
-  
-  # return the wrapper
-  wrapper 
-}
-
-#' R wrapper for Python objects 
-#' 
-#' S3 method to create a custom R wrapper for a Python object.
-#' The default wrapper is either an R environment or an R function
-#' (for callable python objects).
-#' 
-#' @param x Python object 
-#' 
-#' @export
-py_to_r_wrapper <- function(x) {
-  UseMethod("py_to_r_wrapper")
-}
-
-#' @export
-py_to_r_wrapper.default <- function(x) {
-  x
-}
-
-
-
-
-
-#' @rdname r-py-conversion
-#' @export
-r_to_py <- function(x, convert = FALSE) {
-  
-  ensure_python_initialized()
-  
-  r_to_py_impl(x, convert = convert)
-}
-
-
 #' @export
 `$.python.builtin.module` <- function(x, name) {
 

From fe4cd9f7ad9d57d3378ca04f4a098c7d5d098c3f Mon Sep 17 00:00:00 2001
From: Leonardo Gama <leonardo.reis.gama@gmail.com>
Date: Sat, 3 Sep 2022 14:44:33 -0300
Subject: [PATCH 10/26] remove trailing whitespaces and empty line

---
 R/knitr-engine.R | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/R/knitr-engine.R b/R/knitr-engine.R
index 7847f6270..7f5ef7e8e 100644
--- a/R/knitr-engine.R
+++ b/R/knitr-engine.R
@@ -253,7 +253,7 @@ eng_python <- function(options) {
     outputs$push(output)
   }
 
-  if(options$cache > 0) {
+  if (options$cache > 0) {
     save_python_session(options$hash)
   }
 
@@ -314,7 +314,7 @@ eng_python_initialize <- function(options, envir) {
         stop(module$message)
       }
     }
-  } 
+  }
 }
 
 eng_python_knit_figure_path <- function(options, suffix = NULL) {
@@ -674,31 +674,31 @@ save_python_session <- function(cache_path) {
   if (py_eval(r_obj_exists) && py_eval(r_is_R)) {
     py_run_string("del globals()['r']")
   }
-  
+
   cache_path <- file.path(knitr::opts_knit$get("output.dir"), cache_path)
   module$dump_session(filename = paste0(cache_path, ".pkl"), byref = TRUE)
 }
 
 #' A reticulate cache engine for Knitr
-#' 
+#'
 #' This provides a `reticulate` cache engine for `knitr`. The cache engine
 #' allows `knitr` to save and load Python sessions between cached chunks. The
 #' cache engine depends on the `dill` Python module. Therefore, you must have
 #' `dill` installed in your Python environment.
-#' 
+#'
 #' The engine can be activated by setting (for example)
-#' 
+#'
 #' ```
 #' knitr::cache_engines$set(python = reticulate::cache_eng_python)
 #' ```
-#' 
+#'
 #' Typically, this will be set within a document's setup chunk, or by the
 #' environment requesting that Python chunks be processed by this engine.
-#' 
+#'
 #' @param options
-#'   List of chunk options provided by `knitr` during chunk execution. 
+#'   List of chunk options provided by `knitr` during chunk execution.
 #'   Contains the caching path.
-#'   
+#'
 #' @export
 cache_eng_python <- function(options) {
   module <- tryCatch(import("dill"), error = identity)
@@ -710,5 +710,3 @@ cache_eng_python <- function(options) {
   cache_path <- normalizePath(paste0(options$hash, ".pkl"), mustWork = TRUE)
   knitr:::in_input_dir(module$load_session(filename = cache_path))
 }
-
-

From 5d6f7a7bd9b9c077279156836f7bf35e8a0df0b7 Mon Sep 17 00:00:00 2001
From: Leonardo Gama <leonardo.reis.gama@gmail.com>
Date: Mon, 12 Sep 2022 21:42:16 -0300
Subject: [PATCH 11/26] First version of cache implementation with new knitr
 API

---
 R/config.R           | 10 ++---
 R/knitr-cache.R      | 99 ++++++++++++++++++++++++++++++++++++++++++++
 R/knitr-engine.R     | 65 +----------------------------
 R/python.R           | 60 ++++++++++++++-------------
 R/testthat-helpers.R |  2 +-
 5 files changed, 135 insertions(+), 101 deletions(-)
 create mode 100644 R/knitr-cache.R

diff --git a/R/config.R b/R/config.R
index 93c44597e..bbee1b4a3 100644
--- a/R/config.R
+++ b/R/config.R
@@ -756,11 +756,6 @@ python_config <- function(python,
     }
   }
 
-  as_numeric_version <- function(version) {
-    version <- clean_version(version)
-    numeric_version(version)
-  }
-
   # check for numpy
   numpy <- NULL
   if (!is.null(config$NumpyPath)) {
@@ -909,8 +904,9 @@ is_rstudio_desktop <- function() {
   identical(version$mode, "desktop")
 }
 
-clean_version <- function(version) {
-  gsub("\\.$", "", gsub("[A-Za-z_+].*$", "", version))
+as_numeric_version <- function(version) {
+  version <- sub("\\.$", "", sub("[A-Za-z_+].*$", "", version))
+  numeric_version(version)
 }
 
 reticulate_python_versions <- function() {
diff --git a/R/knitr-cache.R b/R/knitr-cache.R
new file mode 100644
index 000000000..c7fc6e6dc
--- /dev/null
+++ b/R/knitr-cache.R
@@ -0,0 +1,99 @@
+#' A reticulate cache engine for Knitr
+#'
+#' This provides a `reticulate` cache engine for `knitr`. The cache engine
+#' allows `knitr` to save and load Python sessions between cached chunks. The
+#' cache engine depends on the `dill` Python module. Therefore, you must have
+#' `dill` installed in your Python environment.
+#'
+#' The engine can be activated by setting (for example)
+#'
+#' ```
+#' knitr::cache_engines$set(python = reticulate::cache_eng_python)
+#' ```
+#'
+#' Typically, this will be set within a document's setup chunk, or by the
+#' environment requesting that Python chunks be processed by this engine.
+#'
+#' @param options
+#'   List of chunk options provided by `knitr` during chunk execution.
+#'   Contains the caching path.
+#'
+#' @export
+cache_eng_python <- (function() {
+  check_cache_available <- function() {
+    # does the python version is supported by 'dill'?
+    if (py_version() < "3.7") {
+      warning("Python cache requires Python version >= 3.7")
+      return(FALSE)
+    }
+
+    # is the module 'dill' loadable?
+    dill <- tryCatch(import("dill"), error = identity)
+    if (inherits(dill, "error")) {
+      error <- reticulate::py_last_error()
+      if (!error$type %in% c("ImportError", "ModuleNotFoundError"))
+        stop(error$value, call. = FALSE)
+      warning("The Python module 'dill' was not found, it's required for Python cache")
+      return(FALSE)
+    }
+
+    # is the 'dill' version recent enough?
+    dill_version <- as_numeric_version(dill$`__version__`)
+    if (dill_version < "0.3.6") {
+      warning("Python cache requires module dill>=0.3.6")
+      return(FALSE)
+    }
+
+    # Python cache is available
+    TRUE
+  }
+
+  cache_available <- function() {
+    available <- knitr::opts_knit$get("reticulate.cache")
+    if (is.null(available)) {
+      available <- check_cache_available()
+      knitr::opts_knit$set(reticulate.cache = available)
+    }
+    available
+  }
+
+  cache_path <- function(path) {
+    paste(path, "pkl", sep=".")
+  }
+
+  cache_exists <- function(options) {
+    file.exists(cache_path(options$hash))
+  }
+
+  cache_load <- function(options) {
+    eng_python_initialize(options, envir = environment())
+    if (!cache_available()) return()
+    dill <- import("dill")
+    dill$load_module(filename = cache_path(options$hash), module = "__main__")
+  }
+
+  filter <- NULL
+  r_obj_filter <- function() {
+    if (is.null(filter)) {
+      filter <<- py_eval("lambda obj: obj.name == 'r' and type(obj.value) is __builtins__.__R__")
+    }
+    filter
+  }
+
+  cache_save <- function(options) {
+    if (!cache_available()) return()
+    dill <- import("dill")
+    tryCatch({
+      dill$dump_module(cache_path(options$hash), refimported = TRUE, exclude = r_obj_filter())
+    }, error = function(e) {
+      cache_purge(options$hash)
+      stop(e)
+    })
+  }
+
+  cache_purge <- function(glob_path) {
+    unlink(cache_path(glob_path))
+  }
+
+  list(exists = cache_exists, load = cache_load, save = cache_save, purge = cache_purge)
+})()
diff --git a/R/knitr-engine.R b/R/knitr-engine.R
index 7f5ef7e8e..0e33318de 100644
--- a/R/knitr-engine.R
+++ b/R/knitr-engine.R
@@ -253,10 +253,6 @@ eng_python <- function(options) {
     outputs$push(output)
   }
 
-  if (options$cache > 0) {
-    save_python_session(options$hash)
-  }
-
   # if we had held outputs, add those in now (merging text output as appropriate)
   text_output <- character()
 
@@ -305,16 +301,6 @@ eng_python_initialize <- function(options, envir) {
   ensure_python_initialized()
   eng_python_initialize_hooks(options, envir)
 
-  if (options$cache > 0) {
-    module <- tryCatch(import("dill"), error = identity)
-    if (inherits(module, "error")) {
-      if (module$message == "ImportError: No module named dill") {
-        warning("The Python module dill was not found. This module is needed for full cache functionality.")
-      } else {
-        stop(module$message)
-      }
-    }
-  }
 }
 
 eng_python_knit_figure_path <- function(options, suffix = NULL) {
@@ -416,7 +402,7 @@ eng_python_initialize_matplotlib <- function(options, envir) {
     if ("matplotlib.backends" %in% names(sys$modules)) {
       matplotlib$pyplot$switch_backend("agg")
     } else {
-      version <- numeric_version(matplotlib$`__version__`)
+      version <- as_numeric_version(matplotlib$`__version__`)
       if (version < "3.3.0")
         matplotlib$use("agg", warn = FALSE, force = TRUE)
       else
@@ -661,52 +647,3 @@ eng_python_autoprint <- function(captured, options, autoshow) {
   }
 
 }
-
-save_python_session <- function(cache_path) {
-  module <- tryCatch(import("dill"), error = identity)
-  if (inherits(module, "error")) {
-    if (module$message == "ImportError: No module named dill") return()
-    signalCondition(module$message)
-  }
-
-  r_obj_exists <- "'r' in globals()"
-  r_is_R <- "type(r).__module__ == '__main__' and type(r).__name__ == 'R'"
-  if (py_eval(r_obj_exists) && py_eval(r_is_R)) {
-    py_run_string("del globals()['r']")
-  }
-
-  cache_path <- file.path(knitr::opts_knit$get("output.dir"), cache_path)
-  module$dump_session(filename = paste0(cache_path, ".pkl"), byref = TRUE)
-}
-
-#' A reticulate cache engine for Knitr
-#'
-#' This provides a `reticulate` cache engine for `knitr`. The cache engine
-#' allows `knitr` to save and load Python sessions between cached chunks. The
-#' cache engine depends on the `dill` Python module. Therefore, you must have
-#' `dill` installed in your Python environment.
-#'
-#' The engine can be activated by setting (for example)
-#'
-#' ```
-#' knitr::cache_engines$set(python = reticulate::cache_eng_python)
-#' ```
-#'
-#' Typically, this will be set within a document's setup chunk, or by the
-#' environment requesting that Python chunks be processed by this engine.
-#'
-#' @param options
-#'   List of chunk options provided by `knitr` during chunk execution.
-#'   Contains the caching path.
-#'
-#' @export
-cache_eng_python <- function(options) {
-  module <- tryCatch(import("dill"), error = identity)
-  if (inherits(module, "error")) {
-    if (module$message == "ImportError: No module named dill") return()
-    stop(module$message)
-  }
-
-  cache_path <- normalizePath(paste0(options$hash, ".pkl"), mustWork = TRUE)
-  knitr:::in_input_dir(module$load_session(filename = cache_path))
-}
diff --git a/R/python.R b/R/python.R
index c549fe73f..72fa520de 100644
--- a/R/python.R
+++ b/R/python.R
@@ -274,14 +274,14 @@ as.environment.python.builtin.object <- function(x) {
   if (inherits(x, "python.builtin.dict")) {
 
     names <- py_dict_get_keys_as_str(x)
-    names <- names[substr(names, 1, 1) != '_']
+    names <- names[substr(names, 1, 1) != "_"]
     Encoding(names) <- "UTF-8"
     types <- rep_len(0L, length(names))
 
   } else {
     # get the names and filter out internal attributes (_*)
     names <- py_suppress_warnings(py_list_attributes(x))
-    names <- names[substr(names, 1, 1) != '_']
+    names <- names[substr(names, 1, 1) != "_"]
     # replace function with `function`
     names <- sub("^function$", "`function`", names)
     names <- sort(names, decreasing = FALSE)
@@ -1351,43 +1351,45 @@ py_filter_classes <- function(classes) {
 }
 
 py_inject_r <- function() {
-
   # don't inject 'r' if there's already an 'r' object defined
   main <- import_main(convert = FALSE)
   if (py_has_attr(main, "r"))
     return(FALSE)
 
-  # define our 'R' class
-  py_run_string("class R(object): pass")
-
-  # extract it from the main module
-  main <- import_main(convert = FALSE)
-  R <- main$R
+  builtins <- import_builtins(convert = FALSE)
+  if (!py_has_attr(builtins, "__R__")) {
+    # define our 'R' class
+    py_run_string("class R(object): pass")
+    R <- main$R
+
+    # copy it to 'builtins'
+    py_set_attr(builtins, "__R__", R)
+
+    # remove the 'R' class object from '__main__'
+    py_del_attr(main, "R")
+
+    # define the getters, setters we'll attach to the Python class
+    getter <- function(self, code) {
+      envir <- py_resolve_envir()
+      object <- eval(parse(text = as_r_value(code)), envir = envir)
+      r_to_py(object, convert = is.function(object))
+    }
 
-  # define the getters, setters we'll attach to the Python class
-  getter <- function(self, code) {
-    envir <- py_resolve_envir()
-    object <- eval(parse(text = as_r_value(code)), envir = envir)
-    r_to_py(object, convert = is.function(object))
-  }
+    setter <- function(self, name, value) {
+      envir <- py_resolve_envir()
+      name  <- as_r_value(name)
+      value <- as_r_value(value)
+      assign(name, value, envir = envir)
+    }
 
-  setter <- function(self, name, value) {
-    envir <- py_resolve_envir()
-    name  <- as_r_value(name)
-    value <- as_r_value(value)
-    assign(name, value, envir = envir)
+    py_set_attr(R, "__getattr__", getter)
+    py_set_attr(R, "__setattr__", setter)
+    py_set_attr(R, "__getitem__", getter)
+    py_set_attr(R, "__setitem__", setter)
   }
 
-  py_set_attr(R, "__getattr__", getter)
-  py_set_attr(R, "__setattr__", setter)
-  py_set_attr(R, "__getitem__", getter)
-  py_set_attr(R, "__setitem__", setter)
-
   # now define the R object
-  py_run_string("r = R()")
-
-  # remove the 'R' class object
-  py_del_attr(main, "R")
+  py_run_string("r = __R__()")
 
   # indicate success
   TRUE
diff --git a/R/testthat-helpers.R b/R/testthat-helpers.R
index 67b4b5f37..e7a3dc037 100644
--- a/R/testthat-helpers.R
+++ b/R/testthat-helpers.R
@@ -109,7 +109,7 @@ skip_if_no_scipy <- function() {
     skip("scipy not available for testing")
 
   scipy <- import("scipy")
-  if (clean_version(scipy$`__version__`) < "1.0")
+  if (as_numeric_version(scipy$`__version__`) < "1.0")
     skip("scipy version is less than v1.0")
 
 }

From a33ed395dccc7c9c77ed8a88a74af1e2f05c5ace Mon Sep 17 00:00:00 2001
From: Leonardo Gama <leonardo.reis.gama@gmail.com>
Date: Wed, 14 Sep 2022 09:44:47 -0300
Subject: [PATCH 12/26] Expose the cache$available() method to knitr

---
 R/knitr-cache.R  | 16 +++++++++-------
 R/knitr-engine.R |  3 ++-
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/R/knitr-cache.R b/R/knitr-cache.R
index c7fc6e6dc..ca786ca94 100644
--- a/R/knitr-cache.R
+++ b/R/knitr-cache.R
@@ -20,7 +20,9 @@
 #'
 #' @export
 cache_eng_python <- (function() {
-  check_cache_available <- function() {
+  check_cache_available <- function(options) {
+    eng_python_initialize(options)
+
     # does the python version is supported by 'dill'?
     if (py_version() < "3.7") {
       warning("Python cache requires Python version >= 3.7")
@@ -48,10 +50,10 @@ cache_eng_python <- (function() {
     TRUE
   }
 
-  cache_available <- function() {
+  cache_available <- function(options) {
     available <- knitr::opts_knit$get("reticulate.cache")
     if (is.null(available)) {
-      available <- check_cache_available()
+      available <- check_cache_available(options)
       knitr::opts_knit$set(reticulate.cache = available)
     }
     available
@@ -66,8 +68,7 @@ cache_eng_python <- (function() {
   }
 
   cache_load <- function(options) {
-    eng_python_initialize(options, envir = environment())
-    if (!cache_available()) return()
+    if (!cache_available(options)) return()
     dill <- import("dill")
     dill$load_module(filename = cache_path(options$hash), module = "__main__")
   }
@@ -81,7 +82,7 @@ cache_eng_python <- (function() {
   }
 
   cache_save <- function(options) {
-    if (!cache_available()) return()
+    if (!cache_available(options)) return()
     dill <- import("dill")
     tryCatch({
       dill$dump_module(cache_path(options$hash), refimported = TRUE, exclude = r_obj_filter())
@@ -95,5 +96,6 @@ cache_eng_python <- (function() {
     unlink(cache_path(glob_path))
   }
 
-  list(exists = cache_exists, load = cache_load, save = cache_save, purge = cache_purge)
+  list(available = cache_available, exists = cache_exists, load = cache_load, save = cache_save,
+       purge = cache_purge)
 })()
diff --git a/R/knitr-engine.R b/R/knitr-engine.R
index 0e33318de..99bef9b4a 100644
--- a/R/knitr-engine.R
+++ b/R/knitr-engine.R
@@ -70,7 +70,7 @@ eng_python <- function(options) {
   # a list of pending plots / outputs
   .engine_context$pending_plots <- stack()
 
-  eng_python_initialize(options = options, envir = environment())
+  eng_python_initialize(options)
 
   # helper function for extracting range of code, dropping blank lines
   extract <- function(code, range) {
@@ -294,6 +294,7 @@ eng_python <- function(options) {
 }
 
 eng_python_initialize <- function(options, envir) {
+  if (missing(envir)) envir <- environment()
 
   if (is.character(options$engine.path))
     use_python(options$engine.path[[1]])

From 266463c2d0ce36fba0bdc4ab3ad90affefd17351 Mon Sep 17 00:00:00 2001
From: Leonardo Gama <leonardo.reis.gama@gmail.com>
Date: Thu, 15 Sep 2022 20:00:34 -0300
Subject: [PATCH 13/26] Use the same warning for missing and old dill module
 cases

---
 R/knitr-cache.R | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/R/knitr-cache.R b/R/knitr-cache.R
index ca786ca94..eca5988ea 100644
--- a/R/knitr-cache.R
+++ b/R/knitr-cache.R
@@ -21,33 +21,33 @@
 #' @export
 cache_eng_python <- (function() {
   check_cache_available <- function(options) {
+    MINIMUM_PYTHON_VERSION <- "3.7"
+    MINIMUM_DILL_VERSION <- "0.3.6"
+
     eng_python_initialize(options)
 
     # does the python version is supported by 'dill'?
-    if (py_version() < "3.7") {
-      warning("Python cache requires Python version >= 3.7")
+    if (py_version() < MINIMUM_PYTHON_VERSION) {
+      warning("Python cache requires Python version >= ", MINIMUM_PYTHON_VERSION)
       return(FALSE)
     }
 
-    # is the module 'dill' loadable?
+    # is the module 'dill' loadable and recent enough?
     dill <- tryCatch(import("dill"), error = identity)
-    if (inherits(dill, "error")) {
+    if (!inherits(dill, "error")) {
+      dill_version <- as_numeric_version(dill$`__version__`)
+      if (dill_version >= MINIMUM_DILL_VERSION)
+        return(TRUE)
+    } else {
+      # handle non-import error
       error <- reticulate::py_last_error()
       if (!error$type %in% c("ImportError", "ModuleNotFoundError"))
         stop(error$value, call. = FALSE)
-      warning("The Python module 'dill' was not found, it's required for Python cache")
-      return(FALSE)
-    }
-
-    # is the 'dill' version recent enough?
-    dill_version <- as_numeric_version(dill$`__version__`)
-    if (dill_version < "0.3.6") {
-      warning("Python cache requires module dill>=0.3.6")
-      return(FALSE)
     }
 
-    # Python cache is available
-    TRUE
+    # 'dill' isn't available
+    warning("Python cache requires module dill>=", MINIMUM_DILL_VERSION)
+    FALSE
   }
 
   cache_available <- function(options) {

From 445a5ca2ee3954eea414f00793fb1b7648f77a0c Mon Sep 17 00:00:00 2001
From: Leonardo Gama <leonardo.reis.gama@gmail.com>
Date: Thu, 15 Sep 2022 20:06:59 -0300
Subject: [PATCH 14/26] Set environment() as default argument in
 eng_python_initialize()

---
 R/knitr-engine.R | 3 +--
 R/python.R       | 2 ++
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/R/knitr-engine.R b/R/knitr-engine.R
index 99bef9b4a..016abde12 100644
--- a/R/knitr-engine.R
+++ b/R/knitr-engine.R
@@ -293,8 +293,7 @@ eng_python <- function(options) {
 
 }
 
-eng_python_initialize <- function(options, envir) {
-  if (missing(envir)) envir <- environment()
+eng_python_initialize <- function(options, envir = environment()) {
 
   if (is.character(options$engine.path))
     use_python(options$engine.path[[1]])
diff --git a/R/python.R b/R/python.R
index 72fa520de..889cc079a 100644
--- a/R/python.R
+++ b/R/python.R
@@ -1351,6 +1351,7 @@ py_filter_classes <- function(classes) {
 }
 
 py_inject_r <- function() {
+
   # don't inject 'r' if there's already an 'r' object defined
   main <- import_main(convert = FALSE)
   if (py_has_attr(main, "r"))
@@ -1358,6 +1359,7 @@ py_inject_r <- function() {
 
   builtins <- import_builtins(convert = FALSE)
   if (!py_has_attr(builtins, "__R__")) {
+
     # define our 'R' class
     py_run_string("class R(object): pass")
     R <- main$R

From c6a88ad49d175dabe988ff7d7bfce9133a79e1d1 Mon Sep 17 00:00:00 2001
From: Leonardo Gama <leonardo.reis.gama@gmail.com>
Date: Fri, 16 Sep 2022 22:15:13 -0300
Subject: [PATCH 15/26] Basic test for knitr engine cache

---
 .../resources/eng-reticulate-cache-test.Rmd   | 20 --------
 .../resources/eng-reticulate-cache.Rmd        | 22 +++++++++
 tests/testthat/test-python-cache-engine.R     | 46 -------------------
 tests/testthat/test-python-dill.R             | 37 ---------------
 tests/testthat/test-python-globals.R          | 27 -----------
 tests/testthat/test-python-knitr-cache.R      | 30 ++++++++++++
 tests/testthat/test-python-knitr-engine.R     | 12 ++---
 7 files changed, 57 insertions(+), 137 deletions(-)
 delete mode 100644 tests/testthat/resources/eng-reticulate-cache-test.Rmd
 create mode 100644 tests/testthat/resources/eng-reticulate-cache.Rmd
 delete mode 100644 tests/testthat/test-python-cache-engine.R
 delete mode 100644 tests/testthat/test-python-dill.R
 delete mode 100644 tests/testthat/test-python-globals.R
 create mode 100644 tests/testthat/test-python-knitr-cache.R

diff --git a/tests/testthat/resources/eng-reticulate-cache-test.Rmd b/tests/testthat/resources/eng-reticulate-cache-test.Rmd
deleted file mode 100644
index 2017d584a..000000000
--- a/tests/testthat/resources/eng-reticulate-cache-test.Rmd
+++ /dev/null
@@ -1,20 +0,0 @@
----
-title: "Using reticulate's Python Engine with knitr"
----
-
-```{r setup, include = FALSE}
-library(reticulate)
-knitr::opts_chunk$set(cache=TRUE)
-```
-
-Cache can handle changes to second chunk:
-
-```{python}
-x = 1
-```
-
-```{python}
-print(x + 1)
-```
-
-
diff --git a/tests/testthat/resources/eng-reticulate-cache.Rmd b/tests/testthat/resources/eng-reticulate-cache.Rmd
new file mode 100644
index 000000000..25ea306cc
--- /dev/null
+++ b/tests/testthat/resources/eng-reticulate-cache.Rmd
@@ -0,0 +1,22 @@
+---
+title: "Using reticulate's Python Engine with cache enabled"
+---
+
+```{r setup, include = FALSE}
+knitr::opts_chunk$set(cache = TRUE)
+```
+
+Cache can handle changes to second chunk:
+
+```{python}
+x = 1
+
+# empty file that flags the chunk was executed
+file = open('py_chunk_executed', 'w')
+file.close()
+del file
+```
+
+```{python, cache = FALSE}
+print(x + 1)
+```
diff --git a/tests/testthat/test-python-cache-engine.R b/tests/testthat/test-python-cache-engine.R
deleted file mode 100644
index 6d8f17422..000000000
--- a/tests/testthat/test-python-cache-engine.R
+++ /dev/null
@@ -1,46 +0,0 @@
-context("knitr-cache")
-
-test_that("An R Markdown document can be rendered with cache using reticulate", {
-  
-  skip_on_cran()
-  skip_if_not_installed("rmarkdown")
-  skip_if_not_installed("callr")
-  
-  unlink("resources/eng-reticulate-cache-test_cache/", recursive = TRUE)
-  
-  path <- callr::r(
-    function() {
-      rmarkdown::render("resources/eng-reticulate-cache-test.Rmd", quiet = TRUE, envir = new.env())
-    })
-  expect_true(file.exists(path))
-  on.exit(unlink(path), add = TRUE)
-})
-
-test_that("An R Markdown document builds if a cache is modified", {
-  
-  skip_on_cran()
-  skip_if_not_installed("rmarkdown")
-  skip_if_not_installed("callr")
-  
-  old_var <- "1"
-  new_var <- "0"
-  mutate_chunk <- function(x) {
-    print_line <- 17
-    file_text <- readLines("resources/eng-reticulate-cache-test.Rmd")
-    file_text[print_line] <- paste0("print(x + ", x, ")")
-    writeLines(file_text, "resources/eng-reticulate-cache-test.Rmd")
-  }
-  mutate_chunk(old_var)
-  mutate_chunk(new_var)
-  path <- callr::r(
-    function() {
-      rmarkdown::render("resources/eng-reticulate-cache-test.Rmd", quiet = TRUE, envir = new.env())
-    })
-  mutate_chunk(old_var)
-  expect_true(file.exists(path))
-  on.exit(unlink(path), add = TRUE)
-  on.exit(unlink("resources/eng-reticulate-cache-test_cache/", recursive = TRUE), add = TRUE)
-})
-
-
-
diff --git a/tests/testthat/test-python-dill.R b/tests/testthat/test-python-dill.R
deleted file mode 100644
index e14b9c04d..000000000
--- a/tests/testthat/test-python-dill.R
+++ /dev/null
@@ -1,37 +0,0 @@
-context("dill")
-
-source("helper-utils.R")
-
-test_that("Interpreter sessions can be saved and loaded with dill", {
-  skip_if_no_python()
-  skip_if_not_installed("callr")
-  
-  session_one_vars <- callr::r(
-    function() {
-      module_load <- tryCatch(
-        dill <- reticulate::import("dill"),
-        error = function(c) {
-          py_error <- reticulate::py_last_error()
-          if(py_error$type == "ImportError" && py_error$value == "No module named dill") {
-            "No dill"
-          }})
-      if (module_load == "No dill") return(module_load)
-      main <- reticulate::py_run_string("x = 1")
-      reticulate::py_run_string("y = x + 1")
-      dill$dump_session(filename = "x.dill", byref = TRUE)
-      c(main$x, main$y)
-    })
-  if (session_one_vars[1] == "No dill")
-    skip("The dill Python module is not installed")
-  
-  session_two_vars <- callr::r(
-    function() {
-      dill <- reticulate::import("dill")
-      dill$load_session(filename = "x.dill")
-      main <- reticulate::py_run_string("pass")
-      c(main$x, main$y)
-    })
-  on.exit(unlink("x.dill"), add = TRUE)
-  expect_equal(session_one_vars, session_two_vars)
-})
-
diff --git a/tests/testthat/test-python-globals.R b/tests/testthat/test-python-globals.R
deleted file mode 100644
index d9beb36ab..000000000
--- a/tests/testthat/test-python-globals.R
+++ /dev/null
@@ -1,27 +0,0 @@
-context("globals")
-
-source("utils.R")
-
-test_that("Interpreter sessions can be saved and loaded with dill", {
-  skip_if_no_python()
-  
-  py_run_string("x = 1")
-  py_run_string("y = 1")
-  py_run_string("[globals().pop(i) for i in ['x', 'y']]")
-  
-  test_x <- tryCatch(
-    py_run_string("x = x + 1"),
-    error = function(e) {
-      py_last_error()$value
-    }
-  )
-  test_y <- tryCatch(
-    py_run_string("y = y + 1"),
-    error = function(e) {
-      py_last_error()$value
-    }
-  )
-  expect_equal(test_x, "name 'x' is not defined")
-  expect_equal(test_y, "name 'y' is not defined")
-})
-
diff --git a/tests/testthat/test-python-knitr-cache.R b/tests/testthat/test-python-knitr-cache.R
new file mode 100644
index 000000000..a4a563c49
--- /dev/null
+++ b/tests/testthat/test-python-knitr-cache.R
@@ -0,0 +1,30 @@
+context("knitr-cache")
+
+test_that("An R Markdown document using reticulate can be rendered with cache feature", {
+
+  skip_on_cran()
+  skip_if_not_installed("rmarkdown")
+  skip_if_not(cache_eng_python$available(knitr::opts_chunk$get()))
+
+  flag_file <- "py_chunk_executed"
+  rmd_prefix <- "eng-reticulate-cache"
+  rmd_file <- paste(rmd_prefix, "Rmd", sep=".")
+
+  withr::with_dir("resources", local({
+    withr::defer({
+      unlink(flag_file)
+      unlink(paste(rmd_prefix, "cache", sep="_"), recursive = TRUE)
+    })
+
+    # cache file is created
+    output <- rmarkdown::render(rmd_file, quiet = TRUE)
+    expect_true(file.exists(flag_file))
+    expect_true(file.exists(output))
+    unlink(c(output, flag_file))
+
+    # cached results should be used
+    output <- rmarkdown::render(rmd_file, quiet = TRUE)
+    expect_false(file.exists(flag_file))
+    expect_true(file.exists(output))
+  }))
+})
diff --git a/tests/testthat/test-python-knitr-engine.R b/tests/testthat/test-python-knitr-engine.R
index 127be47f9..8c7c2329d 100644
--- a/tests/testthat/test-python-knitr-engine.R
+++ b/tests/testthat/test-python-knitr-engine.R
@@ -1,4 +1,4 @@
-context("knitr")
+context("knitr-engine")
 
 test_that("An R Markdown document can be rendered using reticulate", {
 
@@ -17,11 +17,9 @@ test_that("An R Markdown document can be rendered using reticulate", {
     }
   }
 
-  owd <- setwd("resources")
-  status <- rmarkdown::render("eng-reticulate-example.Rmd", quiet = TRUE)
-  setwd(owd)
+  output <- withr::with_dir("resources", {
+    rmarkdown::render("eng-reticulate-example.Rmd", quiet = TRUE)
+  })
 
-  expect_true(file.exists(status), "example.Rmd rendered successfully")
+  expect_true(file.exists(output), "example.Rmd rendered successfully")
 })
-
-

From 975c1b06c6fe46d679bbeea42b111c5183665d84 Mon Sep 17 00:00:00 2001
From: Leonardo Gama <leonardo.reis.gama@gmail.com>
Date: Fri, 16 Sep 2022 22:44:40 -0300
Subject: [PATCH 16/26] minor

---
 tests/testthat/resources/eng-reticulate-cache.Rmd | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/tests/testthat/resources/eng-reticulate-cache.Rmd b/tests/testthat/resources/eng-reticulate-cache.Rmd
index 25ea306cc..c46725981 100644
--- a/tests/testthat/resources/eng-reticulate-cache.Rmd
+++ b/tests/testthat/resources/eng-reticulate-cache.Rmd
@@ -12,9 +12,7 @@ Cache can handle changes to second chunk:
 x = 1
 
 # empty file that flags the chunk was executed
-file = open('py_chunk_executed', 'w')
-file.close()
-del file
+open('py_chunk_executed', 'w').close()
 ```
 
 ```{python, cache = FALSE}

From 401b1ba095951ef5f19fe619b115f664d233da3e Mon Sep 17 00:00:00 2001
From: Leonardo Gama <leonardo.reis.gama@gmail.com>
Date: Sat, 17 Sep 2022 14:33:31 -0300
Subject: [PATCH 17/26] Workflows: install module dill in the testing
 virtualenv

---
 .github/workflows/R-CMD-check.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml
index f555269e0..8ee96ee1e 100644
--- a/.github/workflows/R-CMD-check.yaml
+++ b/.github/workflows/R-CMD-check.yaml
@@ -66,7 +66,7 @@ jobs:
           reticulate::virtualenv_create("r-reticulate", Sys.which("python"))
           reticulate::virtualenv_install("r-reticulate",
             c("docutils", "pandas", "scipy", "matplotlib", "ipython",
-              "tabulate", "plotly", "psutil", "kaleido"))
+              "tabulate", "plotly", "psutil", "kaleido", "dill"))
           python <- reticulate::virtualenv_python("r-reticulate")
           writeLines(sprintf("RETICULATE_PYTHON=%s", python),
                      Sys.getenv("GITHUB_ENV"))

From 62c77d8c927e2b6a4158c90e49f7db437f9a1e4a Mon Sep 17 00:00:00 2001
From: Leonardo Gama <leonardo.reis.gama@gmail.com>
Date: Sat, 17 Sep 2022 14:51:41 -0300
Subject: [PATCH 18/26] Docs: remove @params from cache_eng_python, add it to
 pkgdown index

---
 R/knitr-cache.R | 4 ----
 _pkgdown.yml    | 1 +
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/R/knitr-cache.R b/R/knitr-cache.R
index eca5988ea..9a0dbd54c 100644
--- a/R/knitr-cache.R
+++ b/R/knitr-cache.R
@@ -14,10 +14,6 @@
 #' Typically, this will be set within a document's setup chunk, or by the
 #' environment requesting that Python chunks be processed by this engine.
 #'
-#' @param options
-#'   List of chunk options provided by `knitr` during chunk execution.
-#'   Contains the caching path.
-#'
 #' @export
 cache_eng_python <- (function() {
   check_cache_available <- function(options) {
diff --git a/_pkgdown.yml b/_pkgdown.yml
index 1c215712b..93111cbe7 100644
--- a/_pkgdown.yml
+++ b/_pkgdown.yml
@@ -53,6 +53,7 @@ reference:
     contents:
       - py_save_object
       - py_load_object
+      - cache_eng_python
 
   - title: "Low-Level Interface"
     contents:

From f487b5270dfffee082613f5dd8a67a2fa8cf6936 Mon Sep 17 00:00:00 2001
From: Leonardo Gama <leonardo.reis.gama@gmail.com>
Date: Sun, 18 Sep 2022 22:10:52 -0300
Subject: [PATCH 19/26] Correctly initialize Python in knitr, honoring
 'engine.path'

---
 R/knitr-engine.R | 39 +++++++++++++++++++--------------------
 1 file changed, 19 insertions(+), 20 deletions(-)

diff --git a/R/knitr-engine.R b/R/knitr-engine.R
index 016abde12..7890a10f1 100644
--- a/R/knitr-engine.R
+++ b/R/knitr-engine.R
@@ -38,22 +38,12 @@ eng_python <- function(options) {
     return(wrap(outputs, options))
   }
 
-  engine.path <- if (is.list(options[["engine.path"]]))
-    options[["engine.path"]][["python"]]
-  else
-    options[["engine.path"]]
-
-  # if the user has requested a custom Python, attempt
-  # to honor that request (warn if Python already initialized
-  # to a different version)
-  if (is.character(engine.path)) {
-
-    # if Python has not yet been loaded, then try
-    # to load it with the requested version of Python
-    if (!py_available())
-      use_python(engine.path, required = TRUE)
-
-    # double-check that we've loaded the requested Python
+  # if the user has requested a custom Python, attempt to honor that request
+  eng_python_initialize(options)
+
+  # double-check that we've loaded the requested Python (warn if Python already
+  # initialized to a different version)
+  if (is.character(engine.path <- get_engine_path(options))) {
     conf <- py_config()
     requestedPython <- normalizePath(engine.path)
     actualPython <- normalizePath(conf$python)
@@ -70,8 +60,6 @@ eng_python <- function(options) {
   # a list of pending plots / outputs
   .engine_context$pending_plots <- stack()
 
-  eng_python_initialize(options)
-
   # helper function for extracting range of code, dropping blank lines
   extract <- function(code, range) {
     snippet <- code[range[1]:range[2]]
@@ -293,10 +281,21 @@ eng_python <- function(options) {
 
 }
 
+get_engine_path <- function(options) {
+  option <- options[["engine.path"]]
+  engine.path <- if (is.list(option)) option[["python"]] else option
+  if (is.character(engine.path))
+    stopifnot(length(engine.path) == 1L)
+  engine.path
+}
+
 eng_python_initialize <- function(options, envir = environment()) {
 
-  if (is.character(options$engine.path))
-    use_python(options$engine.path[[1]])
+  # if Python has not yet been loaded, then try
+  # to load it with the requested version of Python
+  engine.path <- get_engine_path(options)
+  if (is.character(engine.path) && !py_available())
+    use_python(engine_path, required = TRUE)
 
   ensure_python_initialized()
   eng_python_initialize_hooks(options, envir)

From cb9ee1f2477cd0a691fe76f0f887e135fef5b89c Mon Sep 17 00:00:00 2001
From: Leonardo Gama <leonardo.reis.gama@gmail.com>
Date: Mon, 19 Sep 2022 12:07:42 -0300
Subject: [PATCH 20/26] Implement the 'cache.vars' chunk option; some style
 changes

---
 R/knitr-cache.R                               | 47 ++++++++++++-------
 .../resources/eng-reticulate-cache.Rmd        | 18 +++++++
 tests/testthat/test-python-knitr-cache.R      | 11 ++++-
 3 files changed, 59 insertions(+), 17 deletions(-)

diff --git a/R/knitr-cache.R b/R/knitr-cache.R
index 9a0dbd54c..453741ba6 100644
--- a/R/knitr-cache.R
+++ b/R/knitr-cache.R
@@ -16,6 +16,13 @@
 #'
 #' @export
 cache_eng_python <- (function() {
+  closure <- environment()
+  dill <- NULL
+
+  cache_path <- function(path) {
+    paste(path, "pkl", sep=".")
+  }
+
   check_cache_available <- function(options) {
     MINIMUM_PYTHON_VERSION <- "3.7"
     MINIMUM_DILL_VERSION <- "0.3.6"
@@ -29,10 +36,11 @@ cache_eng_python <- (function() {
     }
 
     # is the module 'dill' loadable and recent enough?
-    dill <- tryCatch(import("dill"), error = identity)
+    closure$dill <- tryCatch(import("dill"), error = identity)
     if (!inherits(dill, "error")) {
       dill_version <- as_numeric_version(dill$`__version__`)
       if (dill_version >= MINIMUM_DILL_VERSION)
+        cache_initialize()
         return(TRUE)
     } else {
       # handle non-import error
@@ -47,16 +55,15 @@ cache_eng_python <- (function() {
   }
 
   cache_available <- function(options) {
-    available <- knitr::opts_knit$get("reticulate.cache")
-    if (is.null(available)) {
-      available <- check_cache_available(options)
-      knitr::opts_knit$set(reticulate.cache = available)
-    }
-    available
+    if (is.null(closure$.cache_available))
+      closure$.cache_available <- check_cache_available(options)
+    .cache_available
   }
 
-  cache_path <- function(path) {
-    paste(path, "pkl", sep=".")
+  cache_initialize <- function() {
+    # save imported objects by reference when possible
+    dill.session <- import("dill.session")
+    dill.session[["settings"]][["refimported"]] <- TRUE
   }
 
   cache_exists <- function(options) {
@@ -65,23 +72,31 @@ cache_eng_python <- (function() {
 
   cache_load <- function(options) {
     if (!cache_available(options)) return()
-    dill <- import("dill")
     dill$load_module(filename = cache_path(options$hash), module = "__main__")
   }
 
-  filter <- NULL
   r_obj_filter <- function() {
-    if (is.null(filter)) {
-      filter <<- py_eval("lambda obj: obj.name == 'r' and type(obj.value) is __builtins__.__R__")
+    if (is.null(closure$.r_obj_filter)) {
+      expr <- "lambda obj: obj.name == 'r' and type(obj.value) is __builtins__.__R__"
+      closure$.r_obj_filter <- py_eval(expr)
     }
-    filter
+    .r_obj_filter
   }
 
   cache_save <- function(options) {
     if (!cache_available(options)) return()
-    dill <- import("dill")
+
+    # when only inclusion filters are specified, it works as an allowlist
+    if (!is.null(options$cache.vars)) {
+      exclude <- NULL  # the R object won't be saved unless specified by cache.vars
+      include <- options$cache.vars
+    } else {
+      exclude <- r_obj_filter()
+      include <- NULL
+    }
+
     tryCatch({
-      dill$dump_module(cache_path(options$hash), refimported = TRUE, exclude = r_obj_filter())
+      dill$dump_module(cache_path(options$hash), exclude = exclude, include = include)
     }, error = function(e) {
       cache_purge(options$hash)
       stop(e)
diff --git a/tests/testthat/resources/eng-reticulate-cache.Rmd b/tests/testthat/resources/eng-reticulate-cache.Rmd
index c46725981..f83a9f8b2 100644
--- a/tests/testthat/resources/eng-reticulate-cache.Rmd
+++ b/tests/testthat/resources/eng-reticulate-cache.Rmd
@@ -18,3 +18,21 @@ open('py_chunk_executed', 'w').close()
 ```{python, cache = FALSE}
 print(x + 1)
 ```
+
+The `cache.vars` chunk option may be used to select only a subset of variables
+from the global environment to be cached.
+
+```{python cache-vars, cache.vars = 'x'}
+x = 42
+spam = "Lovely SPAM! Wonderful SPAM!"
+```
+
+In a second excution of the document, `spam` should not be present, because it wasn't saved.
+
+```{python, cache = FALSE}
+def print_globals():
+    for name, value in globals().items():
+        if not name.startswith('__'):
+            print(name, ": ", value, sep="")
+print_globals()
+```
diff --git a/tests/testthat/test-python-knitr-cache.R b/tests/testthat/test-python-knitr-cache.R
index a4a563c49..16a3da0a1 100644
--- a/tests/testthat/test-python-knitr-cache.R
+++ b/tests/testthat/test-python-knitr-cache.R
@@ -9,11 +9,12 @@ test_that("An R Markdown document using reticulate can be rendered with cache fe
   flag_file <- "py_chunk_executed"
   rmd_prefix <- "eng-reticulate-cache"
   rmd_file <- paste(rmd_prefix, "Rmd", sep=".")
+  cache_path <- paste(rmd_prefix, "cache", sep="_")
 
   withr::with_dir("resources", local({
     withr::defer({
       unlink(flag_file)
-      unlink(paste(rmd_prefix, "cache", sep="_"), recursive = TRUE)
+      unlink(cache_path, recursive = TRUE)
     })
 
     # cache file is created
@@ -26,5 +27,13 @@ test_that("An R Markdown document using reticulate can be rendered with cache fe
     output <- rmarkdown::render(rmd_file, quiet = TRUE)
     expect_false(file.exists(flag_file))
     expect_true(file.exists(output))
+
+    # the 'spam' variable should not be cached in the 'cache-vars' block
+    main <- import_main()
+    dill <- import("dill")
+    py_del_attr(main, "spam")
+    session_file <- Sys.glob(paste0(cache_path, "/*/cache-vars_*.pkl"))
+    dill$load_module(session_file)
+    expect_false("spam" %in% names(main))
   }))
 })

From 7d4eeec876edf18ea6e8840ca3078c8e4a38ca41 Mon Sep 17 00:00:00 2001
From: Leonardo Gama <leonardo.reis.gama@gmail.com>
Date: Mon, 19 Sep 2022 22:11:56 -0300
Subject: [PATCH 21/26] Remove unused 'envir' parameter from
 'eng_python_initialize*' functions

---
 R/knitr-engine.R | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/R/knitr-engine.R b/R/knitr-engine.R
index 7890a10f1..7ca8a9f08 100644
--- a/R/knitr-engine.R
+++ b/R/knitr-engine.R
@@ -289,7 +289,7 @@ get_engine_path <- function(options) {
   engine.path
 }
 
-eng_python_initialize <- function(options, envir = environment()) {
+eng_python_initialize <- function(options) {
 
   # if Python has not yet been loaded, then try
   # to load it with the requested version of Python
@@ -298,7 +298,7 @@ eng_python_initialize <- function(options, envir = environment()) {
     use_python(engine_path, required = TRUE)
 
   ensure_python_initialized()
-  eng_python_initialize_hooks(options, envir)
+  eng_python_initialize_hooks(options)
 
 }
 
@@ -348,7 +348,7 @@ eng_python_matplotlib_show <- function(plt, options) {
 
 }
 
-eng_python_initialize_hooks <- function(options, envir) {
+eng_python_initialize_hooks <- function(options) {
 
   # set up hooks for matplotlib modules
   matplotlib_modules <- c(
@@ -359,7 +359,7 @@ eng_python_initialize_hooks <- function(options, envir) {
 
   for (module in matplotlib_modules) {
     py_register_load_hook(module, function(...) {
-      eng_python_initialize_matplotlib(options, envir)
+      eng_python_initialize_matplotlib(options)
     })
   }
 
@@ -371,13 +371,13 @@ eng_python_initialize_hooks <- function(options, envir) {
 
   for (module in plotly_modules) {
     py_register_load_hook(module, function(...) {
-      eng_python_initialize_plotly(options, envir)
+      eng_python_initialize_plotly(options)
     })
   }
 
 }
 
-eng_python_initialize_matplotlib <- function(options, envir) {
+eng_python_initialize_matplotlib <- function(options) {
 
   # mark initialization done
   if (identical(.globals$matplotlib_initialized, TRUE))
@@ -440,7 +440,7 @@ eng_python_initialize_matplotlib <- function(options, envir) {
 
 }
 
-eng_python_initialize_plotly <- function(options, envir) {
+eng_python_initialize_plotly <- function(options) {
 
   # mark initialization done
   if (identical(.globals$plotly_initialized, TRUE))

From 395627e7e8c45065476e7e56273fc7a85b065368 Mon Sep 17 00:00:00 2001
From: Leonardo Gama <leonardo.reis.gama@gmail.com>
Date: Mon, 12 Dec 2022 21:47:31 -0300
Subject: [PATCH 22/26] update cache engine docs

---
 R/knitr-cache.R | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/R/knitr-cache.R b/R/knitr-cache.R
index 453741ba6..4022ef199 100644
--- a/R/knitr-cache.R
+++ b/R/knitr-cache.R
@@ -1,18 +1,23 @@
 #' A reticulate cache engine for Knitr
 #'
-#' This provides a `reticulate` cache engine for `knitr`. The cache engine
-#' allows `knitr` to save and load Python sessions between cached chunks. The
-#' cache engine depends on the `dill` Python module. Therefore, you must have
-#' `dill` installed in your Python environment.
+#' This provides caching of Python variables to the `reticulate` engine for
+#' `knitr`. The cache allows `knitr` to save and load the state of Python
+#' variables between cached chunks.  The cache engine depends on the `dill`
+#' Python module. Therefore, you must have a recent version of `dill` installed
+#' in your Python environment.
 #'
-#' The engine can be activated by setting (for example)
+#' The Python cache is activated the same way as the R cache, by setting the
+#' `cache` chunk option to `TRUE`.  To _deactivate_ the Python cache globally
+#' while keeping the R cache active, one may set the option `reticulate.cache`
+#' to `FALSE`.  For example:
 #'
 #' ```
-#' knitr::cache_engines$set(python = reticulate::cache_eng_python)
+#' knitr::opts_knit$set(reticulate.cache = FALSE)
 #' ```
 #'
-#' Typically, this will be set within a document's setup chunk, or by the
-#' environment requesting that Python chunks be processed by this engine.
+#' @note Different from `knitr`'s R cache, the Python cache is capable of saving
+#' most, but not all types of Python objects.  Some Python objects are
+#' "unpickleable" and will rise an error when attepmted to be saved.
 #'
 #' @export
 cache_eng_python <- (function() {

From 55d1e034f2c46abe3862596ac911c03179b1322a Mon Sep 17 00:00:00 2001
From: Leonardo Gama <leonardo.reis.gama@gmail.com>
Date: Tue, 13 Dec 2022 14:15:08 -0300
Subject: [PATCH 23/26] cache: adapt code and tests to dill package v0.3.6

---
 R/knitr-cache.R                               | 31 +++++--------------
 .../resources/eng-reticulate-cache.Rmd        | 14 +++++----
 tests/testthat/test-python-knitr-cache.R      | 19 ++++++------
 3 files changed, 26 insertions(+), 38 deletions(-)

diff --git a/R/knitr-cache.R b/R/knitr-cache.R
index 4022ef199..e2fffcf81 100644
--- a/R/knitr-cache.R
+++ b/R/knitr-cache.R
@@ -45,7 +45,6 @@ cache_eng_python <- (function() {
     if (!inherits(dill, "error")) {
       dill_version <- as_numeric_version(dill$`__version__`)
       if (dill_version >= MINIMUM_DILL_VERSION)
-        cache_initialize()
         return(TRUE)
     } else {
       # handle non-import error
@@ -65,12 +64,6 @@ cache_eng_python <- (function() {
     .cache_available
   }
 
-  cache_initialize <- function() {
-    # save imported objects by reference when possible
-    dill.session <- import("dill.session")
-    dill.session[["settings"]][["refimported"]] <- TRUE
-  }
-
   cache_exists <- function(options) {
     file.exists(cache_path(options$hash))
   }
@@ -80,28 +73,20 @@ cache_eng_python <- (function() {
     dill$load_module(filename = cache_path(options$hash), module = "__main__")
   }
 
-  r_obj_filter <- function() {
-    if (is.null(closure$.r_obj_filter)) {
-      expr <- "lambda obj: obj.name == 'r' and type(obj.value) is __builtins__.__R__"
-      closure$.r_obj_filter <- py_eval(expr)
-    }
-    .r_obj_filter
-  }
-
   cache_save <- function(options) {
     if (!cache_available(options)) return()
 
-    # when only inclusion filters are specified, it works as an allowlist
-    if (!is.null(options$cache.vars)) {
-      exclude <- NULL  # the R object won't be saved unless specified by cache.vars
-      include <- options$cache.vars
-    } else {
-      exclude <- r_obj_filter()
-      include <- NULL
+    # remove injected 'r' object before saving session (and after executing block)
+    main <- import_main(convert = FALSE)
+    if (py_has_attr(main, "r")) {
+      builtins <- import_builtins(convert = TRUE)
+      if (builtins$isinstance(main$r, builtins[["__R__"]]))
+        py_del_attr(main, "r")
     }
 
     tryCatch({
-      dill$dump_module(cache_path(options$hash), exclude = exclude, include = include)
+      # refimported: save imported objects by reference when possible
+      dill$dump_module(cache_path(options$hash), refimported = TRUE)
     }, error = function(e) {
       cache_purge(options$hash)
       stop(e)
diff --git a/tests/testthat/resources/eng-reticulate-cache.Rmd b/tests/testthat/resources/eng-reticulate-cache.Rmd
index f83a9f8b2..f4a75f7d9 100644
--- a/tests/testthat/resources/eng-reticulate-cache.Rmd
+++ b/tests/testthat/resources/eng-reticulate-cache.Rmd
@@ -9,6 +9,7 @@ knitr::opts_chunk$set(cache = TRUE)
 Cache can handle changes to second chunk:
 
 ```{python}
+# define a variable
 x = 1
 
 # empty file that flags the chunk was executed
@@ -19,18 +20,19 @@ open('py_chunk_executed', 'w').close()
 print(x + 1)
 ```
 
-The `cache.vars` chunk option may be used to select only a subset of variables
-from the global environment to be cached.
+The special "`r` object" is _deleted_ before saving the session, but a custom
+user-defined `r` variable should be kept.
 
-```{python cache-vars, cache.vars = 'x'}
-x = 42
-spam = "Lovely SPAM! Wonderful SPAM!"
+```{python custom-r-obj}
+# overwrites the special 'r' object
+r = 'awesome'
 ```
 
-In a second excution of the document, `spam` should not be present, because it wasn't saved.
+In a second excution of the document, the `r` variable should be loaded from cache.
 
 ```{python, cache = FALSE}
 def print_globals():
+    # avoid 'dictionary changed size during iteration' error
     for name, value in globals().items():
         if not name.startswith('__'):
             print(name, ": ", value, sep="")
diff --git a/tests/testthat/test-python-knitr-cache.R b/tests/testthat/test-python-knitr-cache.R
index 16a3da0a1..f193285e6 100644
--- a/tests/testthat/test-python-knitr-cache.R
+++ b/tests/testthat/test-python-knitr-cache.R
@@ -23,17 +23,18 @@ test_that("An R Markdown document using reticulate can be rendered with cache fe
     expect_true(file.exists(output))
     unlink(c(output, flag_file))
 
-    # cached results should be used
+    # re-run using cache
+    main <- import_main()
+    py_del_attr(main, "r")
     output <- rmarkdown::render(rmd_file, quiet = TRUE)
-    expect_false(file.exists(flag_file))
+
+    # output file is generated with cache
     expect_true(file.exists(output))
 
-    # the 'spam' variable should not be cached in the 'cache-vars' block
-    main <- import_main()
-    dill <- import("dill")
-    py_del_attr(main, "spam")
-    session_file <- Sys.glob(paste0(cache_path, "/*/cache-vars_*.pkl"))
-    dill$load_module(session_file)
-    expect_false("spam" %in% names(main))
+    # the cached results must be used, cached blocks should not run
+    expect_false(file.exists(flag_file))
+
+    # a user-defined 'r' variable should be included with the saved variables
+    expect_true(is.character(main$r))
   }))
 })

From d43b5934e9f31c4b68ff44c0e30d180d22153487 Mon Sep 17 00:00:00 2001
From: Leonardo Gama <leonardo.reis.gama@gmail.com>
Date: Tue, 13 Dec 2022 14:43:12 -0300
Subject: [PATCH 24/26] fix typo, update generated documentation

---
 man/cache_eng_python.Rd                       | 25 ++++++++++++-------
 .../resources/eng-reticulate-cache.Rmd        |  2 +-
 2 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/man/cache_eng_python.Rd b/man/cache_eng_python.Rd
index ff8067b7b..50495d78d 100644
--- a/man/cache_eng_python.Rd
+++ b/man/cache_eng_python.Rd
@@ -1,5 +1,5 @@
 % Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/knitr-engine.R
+% Please edit documentation in R/knitr-cache.R
 \name{cache_eng_python}
 \alias{cache_eng_python}
 \title{A reticulate cache engine for Knitr}
@@ -11,15 +11,22 @@ cache_eng_python(options)
 Contains the caching path.}
 }
 \description{
-This provides a \code{reticulate} cache engine for \code{knitr}. The cache engine
-allows \code{knitr} to save and load Python sessions between cached chunks. The
-cache engine depends on the \code{dill} Python module. Therefore, you must have
-\code{dill} installed in your Python environment.
+This provides caching of Python variables to the \code{reticulate} engine for
+\code{knitr}. The cache allows \code{knitr} to save and load the state of Python
+variables between cached chunks.  The cache engine depends on the \code{dill}
+Python module. Therefore, you must have a recent version of \code{dill} installed
+in your Python environment.
 }
 \details{
-The engine can be activated by setting (for example)\preformatted{knitr::cache_engines$set(python = reticulate::cache_eng_python)
-}
+The Python cache is activated the same way as the R cache, by setting the
+\code{cache} chunk option to \code{TRUE}.  To \emph{deactivate} the Python cache globally
+while keeping the R cache active, one may set the option \code{reticulate.cache}
+to \code{FALSE}.  For example:
 
-Typically, this will be set within a document's setup chunk, or by the
-environment requesting that Python chunks be processed by this engine.
+\preformatted{knitr::opts_knit$set(reticulate.cache = FALSE)}
+}
+\note{
+Different from \code{knitr}'s R cache, the Python cache is capable of saving
+most, but not all types of Python objects.  Some Python objects are
+"unpickleable" and will rise an error when attepmted to be saved.
 }
diff --git a/tests/testthat/resources/eng-reticulate-cache.Rmd b/tests/testthat/resources/eng-reticulate-cache.Rmd
index f4a75f7d9..9f6335936 100644
--- a/tests/testthat/resources/eng-reticulate-cache.Rmd
+++ b/tests/testthat/resources/eng-reticulate-cache.Rmd
@@ -28,7 +28,7 @@ user-defined `r` variable should be kept.
 r = 'awesome'
 ```
 
-In a second excution of the document, the `r` variable should be loaded from cache.
+In a second execution of the document, the `r` variable should be loaded from cache.
 
 ```{python, cache = FALSE}
 def print_globals():

From f354f60605a7ca11991ae58090e26a08caabc4b7 Mon Sep 17 00:00:00 2001
From: Leonardo Gama <leonardo.reis.gama@gmail.com>
Date: Fri, 16 Dec 2022 10:08:27 -0300
Subject: [PATCH 25/26] Workflow: use PR branch from knitr for testing

---
 .github/workflows/R-CMD-check.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml
index 8ee96ee1e..8bc6674c9 100644
--- a/.github/workflows/R-CMD-check.yaml
+++ b/.github/workflows/R-CMD-check.yaml
@@ -57,7 +57,7 @@ jobs:
 
       - uses: r-lib/actions/setup-r-dependencies@v2
         with:
-          extra-packages: rcmdcheck remotes
+          extra-packages: rcmdcheck remotes yihui/knitr#2170
 
       - name: setup r-reticulate venv
         shell: Rscript {0}

From 38ef3ceae53e0e7c22e77042c29c00c87142b2ae Mon Sep 17 00:00:00 2001
From: Leonardo Gama <leonardo.reis.gama@gmail.com>
Date: Mon, 19 Dec 2022 21:33:30 -0300
Subject: [PATCH 26/26] fix typo

---
 R/knitr-engine.R | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/R/knitr-engine.R b/R/knitr-engine.R
index 7ca8a9f08..39048797e 100644
--- a/R/knitr-engine.R
+++ b/R/knitr-engine.R
@@ -295,7 +295,7 @@ eng_python_initialize <- function(options) {
   # to load it with the requested version of Python
   engine.path <- get_engine_path(options)
   if (is.character(engine.path) && !py_available())
-    use_python(engine_path, required = TRUE)
+    use_python(engine.path, required = TRUE)
 
   ensure_python_initialized()
   eng_python_initialize_hooks(options)