Converts config tests to Python 3

We've already converted the application code to Python 3, but the config
tests (run in dom0) were still using Python 2. Fixed.

In the process, realized that the "integration" tests (targeting
`sd-journalist`) were broken. We don't run those regularly, but updated
them anyway to maximize testing. Some of that logic may be useful even
after the port to `sd-proxy`; at the very least, it's a useful test
baseline for how the submission workflow currently works.

Prunes all the colorized output because it doesn't work in dom0 (unless
using unsafe options).

Makefile

@@ -88,22 +88,28 @@ remove-sd-gpg: assert-dom0 ## Destroys SD GPG keystore VM
 clean: assert-dom0 destroy-all clean-salt ## Destroys all SD VMs
 
 test: assert-dom0 ## Runs all application tests (no integration tests yet)
-	python -m unittest discover tests
+	python3 -m unittest discover -v tests
 
 test-base: assert-dom0 ## Runs tests for VMs layout
-	python -m unittest -v tests.test_vms_exist.SD_VM_Tests
+	python3 -m unittest -v tests.test_vms_exist.SD_VM_Tests
 
 test-svs: assert-dom0 ## Runs tests for SD SVS VM config
-	python -m unittest -v tests.test_svs.SD_SVS_Tests
+	python3 -m unittest -v tests.test_svs.SD_SVS_Tests
+
+test-integration: assert-dom0 ## Runs integration tests inside sd-journalist VM
+	qvm-run -a sd-journalist "rm -rf ~/QubesIncoming/dom0/integration"
+	qvm-copy-to-vm sd-journalist tests/integration
+	qvm-run --no-color-output --no-color-stderr \
+		-a -p sd-journalist "cd ~/QubesIncoming/dom0/integration && ./test_integration"
 
 test-journalist: assert-dom0 ## Runs tests for SD Journalist VM
-	python -m unittest -v tests.test_journalist_vm
+	python3 -m unittest -v tests.test_journalist_vm
 
 test-whonix: assert-dom0 ## Runs tests for SD Whonix VM
-	python -m unittest -v tests.test_sd_whonix
+	python3 -m unittest -v tests.test_sd_whonix
 
 test-gpg: assert-dom0 ## Runs tests for SD GPG functionality
-	python -m unittest -v tests.test_gpg
+	python3 -m unittest -v tests.test_gpg
 
 validate: assert-dom0 ## Checks for local requirements in dev env
 	@bash -c "test -e config.json" || \

tests/base.py

@@ -58,8 +58,9 @@ def _reboot(self):
         self.vm.start()
 
     def _get_file_contents(self, path):
-        contents = subprocess.check_output(["qvm-run", "-p", self.vm_name,
-                                            "/bin/cat {}".format(path)])
+        cmd = ["qvm-run", "-p", self.vm_name,
+               "/bin/cat {}".format(path)]
+        contents = subprocess.check_output(cmd).decode("utf-8")
         return contents
 
     def _package_is_installed(self, pkg):
@@ -79,7 +80,7 @@ def assertFilesMatch(self, remote_path, local_path):
         with open(local_path) as f:
             content = f.read()
         import difflib
-        print("".join(difflib.unified_diff(remote_content, content)))
+        print("".join(difflib.unified_diff(remote_content, content)), end="")
         self.assertTrue(remote_content == content)
 
     def assertFileHasLine(self, remote_path, wanted_line):

tests/test_gpg.py

@@ -7,7 +7,7 @@
 
 def find_fp_from_gpg_output(gpg):
 
-    lines = gpg.split("\n")
+    lines = gpg.decode("utf-8").split("\n")
 
     for line in lines:
         # dom0 uses Fedora25 with gpg 1.4.22, whereas AppVMs

tests/test_vms_exist.py

@@ -28,9 +28,8 @@ def _check_kernel(self, vm):
         self.assertTrue(vm.kernel == "")
 
         # Check exact kernel version in VM
-        raw_output = vm.run("uname -r")
-        # Response is a tuple of e.g. ('4.14.74-grsec\n', '')
-        kernel_version = raw_output[0].rstrip()
+        stdout, stderr = vm.run("uname -r")
+        kernel_version = stdout.decode("utf-8").rstrip()
         assert kernel_version.endswith("-grsec")
         assert kernel_version == EXPECTED_KERNEL_VERSION
 

tests/test_vms_platform.py

@@ -27,7 +27,7 @@ def _get_platform_info(self, vm):
         # let's maintain the default config and retrieve the value elsewise.
         cmd = "perl -nE '/^PRETTY_NAME=\"(.*)\"$/ and say $1' /etc/os-release"
         stdout, stderr = vm.run(cmd)
-        platform = stdout.rstrip("\n")
+        platform = stdout.decode("utf-8").rstrip("\n")
         return platform
 
     def _validate_vm_platform(self, vm):
@@ -47,7 +47,7 @@ def _ensure_packages_up_to_date(self, vm, fedora=False):
         if not fedora:
             cmd = "apt list --upgradable"
             stdout, stderr = vm.run(cmd)
-            results = stdout.rstrip()
+            results = stdout.rstrip().decode("utf-8")
             # `apt list` will always print "Listing..." to stdout,
             # so expect only that string.
             self.assertEqual(results, "Listing...")
@@ -56,7 +56,7 @@ def _ensure_packages_up_to_date(self, vm, fedora=False):
             # Will raise CalledProcessError if updates available
             stdout, stderr = vm.run(cmd)
             # 'stdout' will contain timestamped progress info; ignore it
-            results = stderr.rstrip()
+            results = stderr.rstrip().decode("utf-8")
             self.assertEqual(results, "")
 
     def test_all_sd_vms_uptodate(self):
@@ -112,7 +112,7 @@ def test_dispvm_default_platform(self):
         test because DispVMs may not be running at present.
         """
         cmd = ["qubes-prefs", "default_dispvm"]
-        result = subprocess.check_output(cmd).rstrip("\n")
+        result = subprocess.check_output(cmd).decode("utf-8").rstrip("\n")
         self.assertEqual(result, "fedora-28-dvm")