Closes #166 : re-execute failures (not report them from cache)

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = pytest-testmon
-version = 1.1.3
+version = 1.2.0
 license = AGPL
 author_email = [email protected]
 author = Tibor Arpas, Tomas Matlovic, Daniel Hahler, Martin Racak

test/test_core.py

@@ -36,6 +36,13 @@ def create_report(
     ):
         phases = ["setup", "call", "teardown"]
         result = {}
+        name = "".join(
+            [
+                node_module,
+                "::",
+                f"{node_class}::{node_name}" if node_class else node_name,
+            ]
+        )
         location = (
             node_module,
             1,
@@ -47,7 +54,7 @@ def create_report(
                 duration / phases_count, location
             )
 
-        self.write(node_name, {node_module: encode_lines([""])}, result)
+        self.write(name, {node_module: encode_lines([""])}, result)
 
     def write(self, node, files, result=None, failed=False):
         records = []
@@ -124,7 +131,12 @@ def test_write_read_attribute(self, testdir):
 
     def test_write_read_nodedata(self, tmdata):
         tmdata.write("test_a.py::n1", {"test_a.py": encode_lines(["1"])})
-        assert tmdata.all_nodes == {"test_a.py::n1": {}}
+        assert tmdata.all_nodes == {
+            "test_a.py::n1": {
+                "durations": {"call": 0.0, "setup": 0.0, "teardown": 0.0},
+                "failed": 0,
+            }
+        }
         assert tmdata.all_files == {"test_a.py"}
 
     def test_filenames_fingerprints(self, tmdata):
@@ -299,9 +311,10 @@ def test_nodes_classes_modules_durations(self, tmdata: CoreTestmonDataForTest):
         tmdata.create_report(1, 5, "test_b1", "tests.py", "TestB")
 
         avg_durations = tmdata.nodes_classes_modules_avg_durations
-        assert avg_durations["test_a1"] == 3
-        assert avg_durations["test_a2"] == 4
-        assert avg_durations["test_b1"] == 5
+        print(avg_durations)
+        assert avg_durations["tests.py::TestA::test_a1"] == 3
+        assert avg_durations["tests.py::TestA::test_a2"] == 4
+        assert avg_durations["tests.py::TestB::test_b1"] == 5
         assert avg_durations["TestA"] == 3.5
         assert avg_durations["TestB"] == 5
         assert avg_durations["tests.py"] == 4

test/test_testmon.py

@@ -1,11 +1,11 @@
-import sqlite3
 import os
 
 import pkg_resources
 import sys
 import textwrap
 import time
 import pytest
+import sqlite3
 
 from testmon import db
 from testmon.process_code import Module, encode_lines
@@ -24,6 +24,8 @@
 from .coveragepy import coveragetest
 from .test_core import CoreTestmonDataForTest
 
+from threading import Thread, Condition
+
 pytest_plugins = ("pytester",)
 
 datafilename = os.environ.get("TESTMON_DATAFILE", DB_FILENAME)
@@ -289,6 +291,41 @@ def test_2():
             ]
         )
 
+    def test_re_executing_failed(self, testdir):
+        testdir.makepyfile(
+            test_a="""
+            import os
+            
+            def test_file(): # test that on first run fails, but on second one passes
+                if os.path.exists('check'): # if file exists then pass the test
+                    assert True
+                else: # otherwise create the file and fail the test
+                    open('check', 'a').close()
+                    assert False
+                    """
+        )
+
+        result = testdir.runpytest_inprocess("--testmon")
+        result.stdout.fnmatch_lines(
+            [
+                "*1 failed*",
+            ]
+        )
+
+        result = testdir.runpytest_inprocess("--testmon")
+        result.stdout.fnmatch_lines(
+            [
+                "*1 passed*",
+            ]
+        )
+
+        result = testdir.runpytest_inprocess("--testmon")
+        result.stdout.fnmatch_lines(
+            [
+                "*1 deselected*",
+            ]
+        )
+
     def test_simple_change_1_of_2_with_decorator(self, testdir):
         testdir.makepyfile(
             test_a="""
@@ -458,7 +495,10 @@ def test_add():
         )
         testdir.runpytest_inprocess("--testmon", "-v")
         testmon_data = CoreTestmonData(testdir.tmpdir.strpath)
-        assert len(testmon_data.all_nodes["test_a.py::test_add"]) == 3
+        assert testmon_data.all_nodes["test_a.py::test_add"]["failed"]
+        assert testmon_data.all_nodes["test_a.py::test_add"]["durations"]["setup"]
+        assert testmon_data.all_nodes["test_a.py::test_add"]["durations"]["call"]
+        assert testmon_data.all_nodes["test_a.py::test_add"]["durations"]["teardown"]
 
         tf = testdir.makepyfile(
             test_a="""
@@ -473,7 +513,9 @@ def test_add():
 
         testmon_data.close_connection()
         testmon_data = CoreTestmonData(testdir.tmpdir.strpath)
-        assert len(testmon_data.all_nodes["test_a.py::test_add"]) == 2
+        assert testmon_data.all_nodes["test_a.py::test_add"]["durations"]["setup"]
+        assert testmon_data.all_nodes["test_a.py::test_add"]["durations"]["call"] == 0
+        assert testmon_data.all_nodes["test_a.py::test_add"]["durations"]["teardown"]
 
         tf = testdir.makepyfile(
             test_a="""
@@ -487,7 +529,9 @@ def test_add():
 
         testmon_data.close_connection()
         testmon_data = CoreTestmonData(testdir.tmpdir.strpath)
-        assert len(testmon_data.all_nodes["test_a.py::test_add"]) == 3
+        assert testmon_data.all_nodes["test_a.py::test_add"]["durations"]["setup"]
+        assert testmon_data.all_nodes["test_a.py::test_add"]["durations"]["call"]
+        assert testmon_data.all_nodes["test_a.py::test_add"]["durations"]["teardown"]
 
     def test_lf(self, testdir):
         testdir.makepyfile(
@@ -501,7 +545,7 @@ def test_add():
         result = testdir.runpytest_inprocess("--testmon", "-v")
         result.stdout.fnmatch_lines(
             [
-                "*1 failed, 1 deselected*",
+                "*1 failed*",
             ]
         )
 
@@ -1357,7 +1401,92 @@ def test_2():
         )
 
 
+while_running_condition = Condition()
+
+
 class TestmonCollect:
+    @pytest.mark.xfail
+    def test_change_while_running_no_data(self, testdir):
+        def make_second_version(condition):
+            with condition:
+                condition.wait()
+                t = testdir.makepyfile(test_a=test_template.replace("$r", "2 == 3"))
+                t.setmtime(2640053809)
+
+        test_template = """
+                from test import test_testmon
+
+                with test_testmon.while_running_condition:
+                    test_testmon.while_running_condition.notify()
+
+                def test_1():
+                    assert $r
+            """
+
+        testdir.makepyfile(test_a=test_template.replace("$r", "1 == 1"))
+        thread = Thread(target=make_second_version, args=(while_running_condition,))
+        thread.start()
+        testdir.runpytest_inprocess(
+            "--testmon",
+        )
+        thread.join()
+
+        result = testdir.runpytest_inprocess("--testmon")
+        result.stdout.fnmatch_lines(
+            [
+                "*1 failed*",
+            ]
+        )
+
+    def test_change_while_running_with_data(self, testdir):
+        def make_third_version(condition):
+            with condition:
+                condition.wait()
+                t = testdir.makepyfile(test_a=test_template.replace("$r", "2 == 3"))
+                t.setmtime(2640053809)
+
+        test_template = """
+                    from test import test_testmon
+
+                    with test_testmon.while_running_condition:
+                        test_testmon.while_running_condition.notify()
+
+                    def test_1():
+                        assert $r
+                """
+
+        testdir.makepyfile(test_a=test_template.replace("$r", "1 == 1"))
+
+        result = testdir.runpytest_inprocess("--testmon")
+        result.stdout.fnmatch_lines(
+            [
+                "*1 passed*",
+            ]
+        )
+
+        file = testdir.makepyfile(test_a=test_template.replace("$r", "2 == 2"))
+        file.setmtime(2640044809)
+
+        thread = Thread(target=make_third_version, args=(while_running_condition,))
+        thread.start()
+
+        result = testdir.runpytest_inprocess("--testmon")
+
+        result.stdout.fnmatch_lines(
+            [
+                "*1 passed*",
+            ]
+        )
+
+        thread.join()
+
+        result = testdir.runpytest_inprocess("--testmon")
+        result.stdout.fnmatch_lines(
+            [
+                "*1 failed*",
+            ]
+        )
+
     def test_failed_setup_phase(self, testdir):
         testdir.makepyfile(
             fixture="""
@@ -1766,7 +1895,6 @@ def test_b():
         )
 
     def test_interrupted2(self, testdir):
-
         testdir.makepyfile(
             test_m="""
                 import time     

testmon/db.py

@@ -1,10 +1,10 @@
 import json
 import sqlite3
 from functools import lru_cache
-from collections import namedtuple
+from collections import namedtuple, defaultdict
 import os
 
-DATA_VERSION = 7
+DATA_VERSION = 8
 
 from testmon.process_code import blob_to_checksums
 
@@ -102,14 +102,25 @@ def fetch_or_create_fingerprint(self, filename, mtime, checksum, fingerprint):
     def insert_node_fingerprints(self, nodeid: str, fingerprint_records, result={}):
         with self.con as con:
             failed = any(r.get("outcome") == "failed" for r in result.values())
+            durations = defaultdict(
+                float,
+                {key: value.get("duration", 0.0) for key, value in result.items()},
+            )
             cursor = con.cursor()
             cursor.execute(
                 """ 
                 INSERT OR REPLACE INTO node 
-                (environment, name, result, failed) 
-                VALUES (?, ?, ?, ?)
+                (environment, name, setup_duration, call_duration, teardown_duration, failed) 
+                VALUES (?, ?, ?, ?, ?, ?)
                 """,
-                (self.env, nodeid, json.dumps(result), failed),
+                (
+                    self.env,
+                    nodeid,
+                    durations["setup"],
+                    durations["call"],
+                    durations["teardown"],
+                    failed,
+                ),
             )
             node_id = cursor.lastrowid
 
@@ -156,7 +167,9 @@ def init_tables(self, DATA_VERSION):
                 id INTEGER PRIMARY KEY ASC,
                 environment TEXT,
                 name TEXT,
-                result TEXT,
+                setup_duration FLOAT,
+                call_duration FLOAT,
+                teardown_duration FLOAT,
                 failed BIT,
                 UNIQUE (environment, name)
             )
@@ -240,9 +253,12 @@ def delete_nodes(self, nodeids):
 
     def all_nodes(self):
         return {
-            row[0]: json.loads(row[1])
+            row[0]: {
+                "durations": {"setup": row[1], "call": row[2], "teardown": row[3]},
+                "failed": row[4],
+            }
             for row in self.con.execute(
-                """  SELECT name, result
+                """  SELECT name, setup_duration, call_duration, teardown_duration, failed
                                     FROM node 
                                     WHERE environment = ?
                                    """,