Fix CI tests

.github/workflows/ci.yaml

@@ -1,27 +1,26 @@
 name: Python CI
-on: [pull_request]
+on: [pull_request, push]
 jobs:
   test:
     runs-on: ubuntu-latest
     timeout-minutes: 90
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.7']
+        python-version: ['3.8']
         tox-action:
           - lint
           - pytest
           - integration
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 1
-      - uses: actions/setup-python@v1
+      - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
           pip install tox coveralls
       - name: Test with tox
         run: |
@@ -31,11 +30,11 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 1
           submodules: true
-      - uses: actions/setup-python@v1
+      - uses: actions/setup-python@v4
         with:
           python-version: '3.7'
       - name: Install dependencies

dev-requirements.txt

@@ -3,7 +3,7 @@ tox
 coverage
 pytest
 pytest-cov
-docker
+galaxy-test-driver
 
 #Building Docs
 sphinx_rtd_theme

tests/conftest.py

@@ -1,65 +1,58 @@
 from collections import namedtuple
 
-import docker
 import pytest
 from bioblend.galaxy import GalaxyInstance
-
-from ephemeris.sleep import galaxy_wait
-
-# It needs to work well with dev. Alternatively we can pin this to 'master' or another stable branch.
-# Preferably a branch that updates with each stable release
-GALAXY_IMAGE = "bgruening/galaxy-stable:20.05"
-GALAXY_ADMIN_KEY = "fakekey"
-GALAXY_ADMIN_PASSWORD = "password"
-GALAXY_ADMIN_USER = "[email protected]"
-
-client = docker.from_env()
+from galaxy.tool_util.verify.interactor import GalaxyInteractorApi
+from galaxy_test.driver.driver_util import GalaxyTestDriver
 
 GalaxyContainer = namedtuple(
-    "GalaxyContainer", ["url", "container", "attributes", "gi"]
+    "GalaxyContainer", ["url", "gi", "password", "username", "api_key"]
 )
 
 
-# Class scope is chosen here so we can group tests on the same galaxy in a class.
 @pytest.fixture(scope="class")
-def start_container(**kwargs):
-    """Starts a docker container with the galaxy image. Returns a named tuple with the url, a GalaxyInstance object,
-    the container attributes, and the container itself."""
-    # We start a container from the galaxy image. We detach it. Port 80 is exposed to the host at a random port.
-    # The random port is because we need mac compatibility. On GNU/linux a better option would be not to expose it
-    # and use the internal ip address instead.
-    # But alas, the trappings of a proprietary BSD kernel compel us to do ugly workarounds.
-    key = kwargs.get("api_key", GALAXY_ADMIN_KEY)
-    ensure_admin = kwargs.get("ensure_admin", True)
-
-    container = client.containers.run(
-        GALAXY_IMAGE, detach=True, ports={"80/tcp": None}, **kwargs
-    )
-    container_id = container.attrs.get("Id")
-    print(container_id)
-
-    # This seems weird as we also can just get container.attrs but for some reason
-    # the network settings are not loaded in container.attrs. With the get request
-    # these attributes are loaded
-    container_attributes = client.containers.get(container_id).attrs
-
-    # Venturing into deep nested dictionaries.
-    exposed_port = (
-        container_attributes.get("NetworkSettings")
-        .get("Ports")
-        .get("80/tcp")[0]
-        .get("HostPort")
-    )
-
-    container_url = "http://localhost:{0}".format(exposed_port)
-    assert key
-    ready = galaxy_wait(
-        container_url, timeout=180, api_key=key, ensure_admin=ensure_admin
-    )
-    if not ready:
-        raise Exception("Failed to wait on Galaxy to start.")
-    gi = GalaxyInstance(container_url, key=key)
-    yield GalaxyContainer(
-        url=container_url, container=container, attributes=container_attributes, gi=gi
+def start_container(tmpdir_factory):
+    config_dir = tmpdir_factory.mktemp("config")
+    database = config_dir.join("universe.sqlite")
+    test_driver = GalaxyTestDriver()
+    test_driver.galaxy_config = {
+        "job_config": {
+            "runners": {
+                "local": {
+                    "load": "galaxy.jobs.runners.local:LocalJobRunner",
+                    "workers": 1,
+                }
+            },
+            "execution": {
+                "default": "local_docker",
+                "environments": {
+                    "local_docker": {"runner": "local", "docker_enabled": True},
+                },
+            },
+        },
+        "admin_users": "[email protected]",
+        "bootstrap_admin_api_key": "123456789",
+        "conda_auto_init": False,
+        "config_dir": str(config_dir),
+        "database_connection": f"sqlite:///{database}?isolation_level=IMMEDIATE",
+    }
+    test_driver.setup()
+    server_wrapper = test_driver.server_wrappers[0]
+    host = server_wrapper.host
+    port = server_wrapper.port
+    prefix = server_wrapper.prefix or ""
+    url = f"http://{host}:{port}{prefix.rstrip('/')}/"
+    interactor = GalaxyInteractorApi(
+        galaxy_url=url, master_api_key="123456789", test_user="[email protected]"
     )
-    container.remove(force=True)
+    gi = GalaxyInstance(url, key=interactor.api_key, password="testpass")
+    try:
+        yield GalaxyContainer(
+            url=url,
+            gi=gi,
+            password="testpass",
+            username="[email protected]",
+            api_key=interactor.api_key,
+        )
+    finally:
+        test_driver.stop_servers()

tests/test_run_data_managers.py

@@ -3,20 +3,13 @@
 # for pytest to work.
 # pylint: disable=no-self-use,unused-import
 import sys
-import time
 
 import pytest
 import yaml
-from conftest import (
-    GALAXY_ADMIN_KEY,
-    GALAXY_ADMIN_PASSWORD,
-    GALAXY_ADMIN_USER,
-)
 
 from ephemeris import run_data_managers
 from ephemeris.run_data_managers import DataManagers
 from ephemeris.shed_tools import InstallRepositoryManager
-from ephemeris.sleep import galaxy_wait
 
 AUTH_BY = "key"
 
@@ -36,10 +29,6 @@ def test_install_data_managers(
         ]
         irm = InstallRepositoryManager(container.gi)
         irm.install_repositories(data_managers)
-        # Galaxy is restarted because otherwise data tables are not watched.
-        container.container.exec_run("supervisorctl restart galaxy:")
-        time.sleep(10)  # give time for the services to go down
-        galaxy_wait(container.url)
 
     def test_run_data_managers(
         self, start_container
@@ -51,13 +40,13 @@ def test_run_data_managers(
             argv.extend(
                 [
                     "--user",
-                    GALAXY_ADMIN_USER,
+                    container.username,
                     "-p",
-                    GALAXY_ADMIN_PASSWORD,
+                    container.password,
                 ]
             )
         else:
-            argv.extend(["-a", GALAXY_ADMIN_KEY])
+            argv.extend(["-a", container.api_key])
         argv.extend(
             ["-g", container.url, "--config", "tests/run_data_managers.yaml.test"]
         )

tox.ini

@@ -1,28 +1,23 @@
 # TODO: implement doc linting
 [tox]
-envlist = py{37}-lint, py{37}-pytest, py{37}, py{37}-integration
+envlist = py{38}-lint, py{38}-pytest, py{38}, py{38}-integration
 source_dir = src/ephemeris
 test_dir = tests
 
 [testenv]
-commands = {envpython} setup.py nosetests []
 allowlist_externals = bash
 
-[testenv:py37-lint]
+[testenv:py38-lint]
 commands = flake8 {[tox]source_dir} {[tox]test_dir}
 skip_install = True
 deps =
     flake8
     flake8-import-order
 
-[testenv:py37-pytest]
+[testenv:py38-pytest]
 deps =
     -r requirements.txt
-    pytest
-    pytest-cov
-    coverage
-    codacy-coverage
-    docker
+    -r dev-requirements.txt
 allowlist_externals =
     sed
     bash
@@ -33,12 +28,12 @@ commands =
     # Unfortunately this has to run in the tox env to have access to envsitepackagesdir
     sed -i 's|{envsitepackagesdir}|src|' coverage.xml
 
-[testenv:py37]
+[testenv:py38]
 deps =
     -r requirements.txt
 commands = bash {[tox]test_dir}/test.sh
 
-[testenv:py37-integration]
+[testenv:py38-integration]
 deps =
     -r requirements.txt
 commands = bash {[tox]test_dir}/test.sh