diff --git a/pytest_black.py b/pytest_black.py
index 04c80cb..d8f706e 100644
--- a/pytest_black.py
+++ b/pytest_black.py
@@ -39,7 +39,9 @@ def pytest_configure(config):
 def pytest_unconfigure(config):
     # save cached mtimes at end of session
     if hasattr(config, "_blackmtimes"):
-        config.cache.set(HISTKEY, config._blackmtimes)
+        cache = config.cache.get(HISTKEY, {})
+        cache.update(config._blackmtimes)
+        config.cache.set(HISTKEY, cache)
 
 
 class BlackItem(pytest.Item, pytest.File):