Merge pull request #263 from xlab-uiuc/refactor-snapshot

Refactor snapshot
xlab-uiuc · Aug 29, 2023 · 4cf52dd · 4cf52dd · github-actions · Aug 29, 2023
2 parents 8430644 + 7661d3d
commit 4cf52dd
Show file tree

Hide file tree

Showing 7 changed files with 65 additions and 21 deletions.
diff --git a/acto/checker/impl/tests/test_crash.py b/acto/checker/impl/tests/test_crash.py
@@ -3,15 +3,15 @@
 from acto.checker.impl.crash import CrashChecker
 from acto.checker.impl.tests import load_snapshot
 from acto.common import Oracle, OracleResult, PassResult
-from acto.snapshot import EmptySnapshot, Snapshot
+from acto.snapshot import Snapshot
 
 # Crash checker is stateless, so we can use the same instance for all tests
 checker = CrashChecker()
 
 
 def checker_func(s: Snapshot) -> OracleResult:
     assert s.system_state != {}
-    return checker.check(0, s, EmptySnapshot({}))
+    return checker.check(0, s, Snapshot({}))
 
 
 @pytest.mark.parametrize("test_case_id,result_dict", list(enumerate([

diff --git a/acto/checker/impl/tests/test_health.py b/acto/checker/impl/tests/test_health.py
@@ -3,15 +3,15 @@
 from acto.checker.impl.health import HealthChecker
 from acto.checker.impl.tests import load_snapshot
 from acto.common import OracleResult, PassResult, UnhealthyResult, Oracle
-from acto.snapshot import Snapshot, EmptySnapshot
+from acto.snapshot import Snapshot
 
 # Crash checker is stateless, so we can use the same instance for all tests
 checker = HealthChecker()
 
 
 def checker_func(s: Snapshot) -> OracleResult:
     assert s.system_state != {}
-    return checker.check(0, s, EmptySnapshot({}))
+    return checker.check(0, s, Snapshot({}))
 
 
 @pytest.mark.parametrize("test_case_id,result_dict", list(enumerate([

diff --git a/acto/checker/impl/tests/test_operator_log.py b/acto/checker/impl/tests/test_operator_log.py
@@ -3,7 +3,7 @@
 from acto.checker.impl.operator_log import OperatorLogChecker
 from acto.checker.impl.tests import load_snapshot
 from acto.common import OracleResult, PassResult
-from acto.snapshot import Snapshot, EmptySnapshot
+from acto.snapshot import Snapshot
 
 # Crash checker is stateless, so we can use the same instance for all tests
 checker = OperatorLogChecker()

diff --git a/acto/checker/test_checker.py b/acto/checker/test_checker.py
@@ -7,7 +7,7 @@
 from acto.common import RunResult, RecoveryResult, ErrorResult, Oracle
 from acto.input import InputModel
 from acto.serialization import ActoEncoder
-from acto.snapshot import EmptySnapshot, Snapshot
+from acto.snapshot import Snapshot
 
 if __name__ == "__main__":
     import argparse
@@ -70,7 +70,7 @@ def check_trial_worker(workqueue: multiprocessing.Queue, checker_class: type,
                                              trial_dir=trial_dir,
                                              input_model=input_model)
             snapshots = []
-            snapshots.append(EmptySnapshot(seed))
+            snapshots.append(Snapshot(seed))
 
             alarm = False
             for generation in range(0, 20):

diff --git a/acto/engine.py b/acto/engine.py
@@ -27,7 +27,7 @@
 from acto.oracle_handle import OracleHandle
 from acto.runner import Runner
 from acto.serialization import ActoEncoder, ContextEncoder
-from acto.snapshot import EmptySnapshot, Snapshot
+from acto.snapshot import Snapshot
 from acto.utils import (delete_operator_pod, process_crd,
                         update_preload_images)
 from acto.lib.operator_config import OperatorConfig
@@ -310,7 +310,7 @@ def run_trial(self,
         checker: CheckerSet = self.checker_t(self.context, trial_dir, self.input_model, oracle_handle, self.custom_oracle)
 
         curr_input = self.input_model.get_seed_input()
-        self.snapshots.append(EmptySnapshot(curr_input))
+        self.snapshots.append(Snapshot(curr_input))
 
         generation = 0
         while generation < num_mutation:  # every iteration gets a new list of next tests

diff --git a/acto/snapshot.py b/acto/snapshot.py
@@ -1,17 +1,32 @@
-from functools import lru_cache
-from typing import Tuple, Dict, List
+import json
+import os
+from dataclasses import dataclass, field
+from typing import Tuple, Dict, List, TypedDict, Optional, Literal, Any
 
+import yaml
 from deepdiff import DeepDiff
 
 from acto.common import postprocess_diff, EXCLUDE_PATH_REGEX, Diff
 
 
+class PodLog(TypedDict):
+    log: Optional[List[str]]
+    previous_log: Optional[List[str]]
+
+
+@dataclass
 class Snapshot:
-    def __init__(self, input: dict, cli_result: dict, system_state: dict, operator_log: List[str]):
-        self.input = input
-        self.cli_result = cli_result
-        self.system_state = system_state
-        self.operator_log = operator_log
+    input: dict = field(default_factory=dict)
+    cli_result: dict = field(default_factory=dict)
+    system_state: dict = field(default_factory=dict)
+    operator_log: List[str] = field(default_factory=list)
+    events: dict = field(default_factory=dict)
+    not_ready_pods_logs: dict = field(default_factory=dict)
+    coverage_files: Dict[str, bytes] = field(default_factory=dict)
+    generation: int = 0
+    trial_state: Literal['normal', 'recovering', 'terminated', 'runtime_exception'] = 'normal'
+    parent: Optional['Snapshot'] = None
+    __context: Dict[str, Any] = field(default_factory=dict)
 
     def to_dict(self):
         return {
@@ -21,7 +36,6 @@ def to_dict(self):
             'operator_log': self.operator_log
         }
 
-    @lru_cache
     def delta(self, prev: 'Snapshot') -> Tuple[Dict[str, Dict[str, Dict[str, Diff]]], Dict[str, Dict[str, Dict[str, Diff]]]]:
         curr_input, curr_system_state = self.input, self.system_state
         prev_input, prev_system_state = prev.input, prev.system_state
@@ -40,6 +54,36 @@ def delta(self, prev: 'Snapshot') -> Tuple[Dict[str, Dict[str, Dict[str, Diff]]]
 
         return input_delta, system_state_delta
 
+    def save(self, base_dir: str):
+        yaml.safe_dump(self.input, open(os.path.join(base_dir, f'mutated-{self.generation}.yaml'), 'w'))
+        json.dump(self.cli_result, open(os.path.join(base_dir, f'cli-output-{self.generation}.log'), 'w'))
+        json.dump(self.system_state, open(os.path.join(base_dir, f'system-state-{self.generation}.json'), 'w'), default=str)
+        open(os.path.join(base_dir, f'operator-{self.generation}.json'), 'w').write('\n'.join(self.operator_log))
+        json.dump(self.events, open(os.path.join(base_dir, f'events-{self.generation}.json'), 'w'))
+        json.dump(self.not_ready_pods_logs, open(os.path.join(base_dir, f'not-ready-pods-logs-{self.generation}.json'), 'w'))
+        open(os.path.join(base_dir, f'trial-state-{self.generation}.txt'), 'w').write(self.trial_state)
+        # dump coverage info
+        # converge_index is a dict to map from filename to its generation
+        coverage_index_path = os.path.join(base_dir, f'coverage-index.json')
+        if not os.path.exists(coverage_index_path):
+            open(coverage_index_path, 'w').write('{}')
+            coverage_index = {}
+        else:
+            coverage_index = json.load(open(coverage_index_path))
+
+        for filename, content in self.coverage_files.items():
+            if filename not in coverage_index:
+                with open(os.path.join(base_dir, filename), 'wb') as f:
+                    f.write(content)
+                coverage_index[filename] = self.generation
+        json.dump(coverage_index, open(coverage_index_path, 'w'))
+
+    def get_context_value(self, key: str):
+        if key in self.__context:
+            return self.__context[key]
+        if self.parent:
+            return self.parent.get_context_value(key)
+        return None
 
-def EmptySnapshot(input: dict):
-    return Snapshot(input, {}, {}, [])
+    def set_context_value(self, key: str, value: Any):
+        self.__context[key] = value
diff --git a/test/utils.py b/test/utils.py
@@ -8,7 +8,7 @@
 from acto.checker.impl.health import HealthChecker
 from acto.common import PassResult
 
-from acto.snapshot import EmptySnapshot, Snapshot
+from acto.snapshot import Snapshot
 
 
 def construct_snapshot(trial_dir: str, generation: int):
@@ -593,7 +593,7 @@ def check_postdiff_runtime_error(workdir_path: str) -> bool:
             with open(compare_result) as f:
                 result = json.load(f)[0]
                 to_state = result['to']
-                snapshot = EmptySnapshot({})
+                snapshot = Snapshot({})
                 snapshot.system_state = to_state
                 health_result = HealthChecker().check(0, snapshot, {})
                 if not isinstance(health_result, PassResult):
File	Stmts	Miss	Cover	Missing
acto
__main__.py	86	86	0%	1–174
common.py	354	101	71%	97–98, 102–116, 119–126, 129, 132, 136, 138, 140, 142, 144, 148, 153–162, 203, 236, 249, 291, 315–316, 325, 327, 330, 334–343, 363–366, 415, 418, 432, 460–461, 498–503, 505–507, 514–517, 521, 537–538, 544–555, 614–623, 627, 631, 634–639
deploy.py	147	121	18%	29–32, 35–52, 56, 59–69, 77–107, 110–117, 123–145, 148–161, 173–199, 206–217, 225–243
engine.py	530	484	9%	39–56, 63–79, 91–191, 201–226, 229–284, 300–423, 427–528, 538–563, 567–575, 578–591, 616–732, 735–818, 822–912
oracle_handle.py	24	13	46%	15–18, 26–27, 39–46, 54
reproduce.py	121	80	34%	30–40, 48–52, 58–59, 72–93, 96, 99, 102, 106–108, 111, 114, 122–123, 126, 130, 134, 138, 142–176, 179–189, 192–219
serialization.py	30	16	47%	19–27, 33–39
snapshot.py	62	26	58%	32, 48, 58–79, 82–86, 89
acto/checker
checker.py	13	2	85%	12, 19
checker_set.py	57	12	79%	41–42, 67–78
test_checker.py	162	152	6%	13–246
acto/checker/impl
crash.py	30	2	93%	14, 16
health.py	49	3	94%	36, 65, 81
kubectl_cli.py	29	4	86%	32–33, 44–46
operator_log.py	24	2	92%	24, 32
state.py	225	51	77%	55, 63–64, 69–93, 104–105, 133, 140, 147–149, 166, 265, 288, 317, 319–323, 329–336
state_compare.py	76	3	96%	62, 73, 80
state_condition.py	39	13	67%	18, 24, 29–37, 47, 53
acto/input
get_matched_schemas.py	54	22	59%	12, 47–51, 55–74
input.py	609	489	20%	30–37, 74, 92, 112, 123–126, 129, 159–171, 174–185, 189, 193, 200, 206, 210–389, 410–424, 432–437, 441–454, 463, 479, 482–489, 501, 503, 505–507, 510–523, 528, 536–547, 550–556, 564–591, 595–899, 922–935, 939–986
testcase.py	55	20	64%	40–50, 53, 56, 59, 62–66, 95–96, 100, 103, 109, 116, 119
testplan.py	183	137	25%	14–24, 27–29, 32, 35–42, 45, 48–67, 70–74, 78, 82, 85–91, 99–107, 110–121, 124, 127, 130, 133, 136–138, 141–151, 154–164, 167, 174–185, 188–194, 200, 203–219, 222, 225, 231, 239–244, 247, 250, 253, 259–260, 263–269, 272, 275, 278
value_with_schema.py	337	218	35%	16, 20, 24, 28, 32, 36, 40, 54, 58, 61–67, 70–76, 86–114, 117–124, 128–137, 141–149, 152–156, 159, 162, 166, 180, 183, 186–192, 195–201, 211–230, 233–240, 243, 247–256, 260–272, 275–279, 282, 285, 289, 301, 307, 312, 314, 316, 320, 324, 329–333, 337–340, 349–359, 362–369, 373–376, 380–385, 388–391, 405, 408–412, 416, 421–428, 431–434, 437–439, 442–445, 448–451, 462, 465, 468, 485–498
valuegenerator.py	620	389	37%	20, 24, 28, 32, 43, 47, 51, 55, 76–86, 90–104, 107, 110, 114, 117, 121–130, 133–139, 142, 145, 148, 151, 154, 157–167, 194–199, 202–213, 216, 219, 223, 226–231, 234–237, 240, 243–248, 251–254, 257, 260, 263, 266, 269, 273–282, 285, 288, 291, 294–304, 331–343, 346–347, 350, 353, 357, 360–365, 368–371, 374, 377–382, 385–388, 391, 394, 397, 400, 403, 406, 409–419, 450–482, 485–496, 499–502, 505–508, 512–520, 523, 526, 529, 532, 535, 538–548, 573–589, 592–609, 612, 615, 619–621, 624–628, 631–632, 635–644, 647–653, 656–657, 660–670, 673, 676, 679, 682, 685, 688–698, 710–711, 714–724, 727–730, 733–736, 740, 748, 751–752, 755–765, 768–771, 774–777, 781, 794–797, 800–814, 817, 820, 824, 827–832, 835, 838, 841–846, 849, 852, 855, 858, 861–871, 881, 884, 887, 890, 894, 913, 919, 931, 933, 936–940, 943–946, 951, 955, 957, 961–962
acto/input/known_schemas
base.py	53	13	75%	17–18, 28, 37, 46–47, 56–57, 66, 75, 84–85, 93
cronjob_schemas.py	76	33	57%	13, 16–19, 22, 25, 36–39, 42–47, 50, 53, 59, 62–65, 68, 71, 82, 85–90, 93, 96, 113, 117–119, 131, 137, 140
deployment_schemas.py	59	25	58%	16, 22–27, 30–32, 35, 38, 54–57, 65–67, 70, 78–81, 91, 94
known_schema.py	75	39	48%	28, 31–34, 37, 43, 46–48, 51, 54, 81–84, 102–113, 117–135
pod_disruption_budget_schemas.py	56	22	61%	14–17, 21, 25–27, 30, 41–44, 48, 54, 57, 68–71, 81, 84
pod_schemas.py	797	271	66%	16–19, 23, 28, 32, 40–43, 47, 51–53, 61–64, 68, 73, 83, 92, 151, 156, 160, 167, 171, 178, 182, 238, 242, 247, 251, 294, 298, 303, 307, 335, 338, 341, 344, 347, 350, 353, 356, 359, 362, 365, 368, 393, 397, 400–405, 408, 414, 417, 420, 428, 431, 434, 437, 445, 453, 481, 488–494, 503, 507, 539, 543, 572, 575, 578, 581, 584, 587, 590, 619, 623, 626–631, 634, 642, 645, 648, 664–667, 670–672, 675, 684, 690, 693–696, 699, 702, 713–714, 717–719, 722, 725, 736, 739, 742, 745–748, 752, 756–758, 761–765, 768, 774, 777, 780, 783, 786, 789, 792, 795, 798, 801, 804, 807, 810, 813, 816, 840, 845, 849–857, 860, 866, 869, 872, 875, 878, 881, 884, 887, 890, 893, 896, 899, 902, 905, 908, 929, 933–935, 938–943, 946, 959, 964, 974, 980, 986, 989, 992, 1032, 1036–1038, 1041, 1054, 1059, 1065, 1068–1071, 1074, 1077, 1088–1091, 1094–1096, 1102, 1108, 1111–1114, 1117, 1120, 1133–1136, 1139–1141, 1147, 1153, 1156–1159, 1162, 1165, 1176–1179, 1182–1184, 1190, 1196, 1199–1202, 1205, 1212, 1215–1217, 1223, 1238, 1243, 1247, 1274, 1278, 1291, 1296, 1302, 1305, 1308, 1314, 1317, 1320–1322, 1325, 1342, 1345, 1348, 1374, 1378–1381, 1384, 1402, 1408, 1411, 1414, 1421, 1427, 1469, 1473
resource_schemas.py	149	50	66%	15–19, 22, 25, 28–32, 35, 38, 45, 49, 54, 57–59, 62, 76, 78, 82, 96, 102–105, 108, 111, 121, 134, 147, 154, 159–162, 165, 173–176, 179, 187, 194, 199, 213, 231, 236
service_schemas.py	178	67	62%	13, 16, 19, 25–30, 33, 36, 42, 45–48, 51, 54, 64–67, 70–73, 79, 85, 88–91, 94, 101–102, 105–108, 111, 114, 127, 142, 180, 184, 208, 214, 217, 220, 228–231, 235, 240, 244–246, 249, 257–258, 261, 264, 277–280, 284, 288–290, 293
statefulset_schemas.py	186	61	67%	15–18, 21, 31–36, 42, 49–53, 56–58, 61–63, 66–70, 73–75, 78–80, 83, 90–93, 99–102, 105, 132, 149, 154, 158, 164, 167–170, 173, 176, 190–193, 196–201, 207, 225, 230, 234, 262, 266, 290
storage_schemas.py	179	79	56%	13, 16, 25–30, 33, 36, 42, 48–53, 59, 67–70, 74, 79–80, 83, 89, 92–95, 98, 104–105, 108–111, 114–116, 122, 130–131, 135, 140, 145–148, 154–155, 158, 164, 181–184, 193, 197, 203–205, 211, 214, 228–231, 244, 250–252, 255, 258, 266–269, 273, 277–279, 282
acto/kubectl_client
kubectl.py	23	18	22%	8–14, 23–29, 37–44
acto/kubernetes_engine
base.py	48	30	38%	12, 16, 20, 24, 28, 31–49, 56–70
k3d.py	85	68	20%	17, 21–39, 45, 54–78, 81–97, 100–110, 117–133, 137–139
kind.py	88	71	19%	18, 22–51, 57, 66–99, 102–114, 117–130, 137–151
minikube.py	3	3	0%	1–5
acto/monkey_patch
monkey_patch.py	79	24	70%	9, 18, 36, 39, 41, 45–56, 76, 90–95, 106
acto/parse_log
parse_log.py	78	19	76%	71–74, 84–86, 89–91, 96–98, 116–124
acto/post_process
post_diff_test.py	390	231	41%	40, 54–55, 94–95, 166, 176, 180, 185–194, 229, 231, 233–237, 243–260, 268–276, 279–301, 308–317, 320–378, 390–394, 414, 422–452, 455–472, 476–525, 537, 541, 547–548, 550–571, 581–617
post_process.py	104	22	79%	51, 55, 67, 77–88, 100, 103, 138–142, 146, 150, 158, 162
test_post_process.py	28	1	96%	53
acto/runner
runner.py	286	258	10%	24–44, 72–117, 120–129, 132–163, 170–197, 204–227, 230–236, 239–258, 261–266, 277–282, 296–301, 314–390, 395–399, 405–415, 426–449, 454–487
acto/schema
anyof.py	44	20	55%	26, 30–38, 41, 44, 47–49, 52, 55–60
array.py	70	29	59%	31, 43–52, 57, 59–62, 71–78, 81–83, 86–88, 91, 94, 97, 103
base.py	102	57	44%	13–15, 18–20, 23, 26–37, 40, 43, 46–48, 51–61, 64–74, 77, 80–86, 95, 100, 105, 110, 114, 118, 142, 145–149
boolean.py	27	10	63%	16, 20, 23, 26, 29–32, 35, 38
integer.py	26	9	65%	17, 21–23, 26, 29, 32, 35, 38
number.py	30	11	63%	30–32, 35–37, 40, 43, 46, 49, 52
object.py	117	48	59%	44, 46, 51, 66–75, 80, 83, 94–109, 112–120, 123–126, 129, 132, 148, 151–158, 168
oneof.py	44	30	32%	13–19, 22, 25–27, 30–38, 41, 44, 47–49, 52, 55–60
opaque.py	17	6	65%	13, 16, 19, 22, 25, 28
schema.py	42	8	81%	21, 25, 31–34, 49–50
string.py	27	9	67%	25, 29–31, 34, 37, 40, 43, 46
acto/utils
__init__.py	13	1	92%	11
acto_timer.py	31	22	29%	10–15, 19, 22–33, 38–40, 44–47
error_handler.py	43	33	23%	13–30, 38–52, 56–74
k8s_helper.py	64	51	20%	21–27, 39–45, 57–61, 73–79, 83–91, 95–102, 106–127
preprocess.py	69	59	14%	16–62, 74–123, 129–174
process_with_except.py	9	9	0%	1–13
thread_logger.py	15	3	80%	9, 18, 28
TOTAL	8094	4266	47%