re-implement (?) disk-only database

big_scape/cli/cli_common_options.py

@@ -103,6 +103,15 @@ def common_all(fn):
             "but in case of a crashed run no info will be stored and you'll have to "
             "re-start the run from scratch",
         ),
+        click.option(
+            "--disk-only",
+            type=bool,
+            is_flag=True,
+            default=False,
+            help="Do not store any results in memory, only on disk. This is almost certainly "
+            "slower than the default behaviour, but can be useful for very large runs or "
+            "runs with limited memory.",
+        ),
         click.option(
             "--no-interactive",
             type=bool,

big_scape/data/sqlite.py

@@ -96,6 +96,26 @@ def open_memory_connection() -> None:
         )
         DB.connection = DB.engine.connect()
 
+    def open_disk_connection(db_path: Path) -> None:
+        if DB.opened():
+            raise DBAlreadyOpenError()
+
+        DB.engine = create_engine(
+            "sqlite:///" + str(db_path),
+            connect_args={"check_same_thread": False},
+            poolclass=StaticPool,
+        )
+        DB.connection = DB.engine.connect()
+
+    @staticmethod
+    def create_on_disk(db_path: Path) -> None:
+        """Open a connection to a database file"""
+        DB.open_disk_connection(db_path)
+
+        DB.create_tables()
+
+        DB.reflect()
+
     @staticmethod
     def create_in_mem() -> None:
         """Create a new database in-memory"""
@@ -118,6 +138,10 @@ def save_to_disk(db_path: Path) -> None:
         if click_context and click_context.obj["no_db_dump"]:
             return
 
+        # skip this if we are using disk-only mode
+        if click_context and click_context.obj["disk_only"]:
+            return
+
         if not DB.opened():
             raise DBClosedError()
 
@@ -175,6 +199,12 @@ def load_from_disk(db_path: Path) -> None:
         if not db_path.exists():
             raise FileNotFoundError()
 
+        # disk only means we don't have to dump to memory
+        click_context = click.get_current_context(silent=True)
+        if click_context and click_context.obj["disk_only"]:
+            DB.create_on_disk(db_path)
+            return
+
         file_engine = create_engine("sqlite:///" + str(db_path))
         file_engine.connect()
 

big_scape/run_bigscape.py

@@ -111,8 +111,14 @@ def signal_handler(sig, frame):
 
     # INPUT - create an in memory bs_data.DB or load from disk
     if not run["db_path"].exists():
-        bs_data.DB.create_in_mem()
+        if run["disk_only"]:
+            logging.info("Creating on disk database")
+            bs_data.DB.create_on_disk(run["db_path"])
+        else:
+            logging.info("Creating in memory database")
+            bs_data.DB.create_in_mem()
     else:
+        logging.info("Loading database from disk")
         bs_data.DB.load_from_disk(run["db_path"])
         bs_data.DB.check_config_hash()