Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

use galaxy-tool-test functions for shed-tools test #214

Draft
wants to merge 2 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
176 changes: 35 additions & 141 deletions src/ephemeris/shed_tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,10 +41,6 @@
import re
import time
from collections import namedtuple
from concurrent.futures import (
thread,
ThreadPoolExecutor,
)
from typing import (
Iterable,
List,
Expand All @@ -58,8 +54,8 @@
from galaxy.tool_util.verify.interactor import (
DictClientTestConfig,
GalaxyInteractorApi,
verify_tool,
)
from galaxy.tool_util.verify.script import build_case_references, test_tools, Results, setup_global_logger
from galaxy.util import unicodify
from typing_extensions import (
NamedTuple,
Expand Down Expand Up @@ -294,12 +290,10 @@ def test_tools(
test_history_name=None,
parallel_tests=1,
test_all_versions=False,
skip_with_reference_data=False,
client_test_config_path=None,
):
"""Run tool tests for all tools in each repository in supplied tool list or ``self.installed_repositories()``."""
tool_test_start = dt.datetime.now()
tests_passed = []
test_exceptions = []

if not repositories: # If repositories is None or empty list
# Consider a variant of this that doesn't even consume a tool list YAML? target
Expand All @@ -313,7 +307,6 @@ def test_tools(
repo_tools = tools_for_repository(self.gi, target_repository, all_tools=test_all_versions)
installed_tools.extend(repo_tools)

all_test_results = []
galaxy_interactor = self._get_interactor(test_user, test_user_api_key)
if client_test_config_path is not None:
with open(client_test_config_path) as f:
Expand All @@ -322,61 +315,39 @@ def test_tools(
else:
client_test_config = None

if test_history_name:
for history in self.gi.histories.get_histories(name=test_history_name, deleted=False):
test_history = history["id"]
log.debug(
"Using existing history with id '%s', last updated: %s",
test_history,
history["update_time"],
)
break
else:
test_history = galaxy_interactor.new_history(history_name=test_history_name)
else:
test_history = galaxy_interactor.new_history()
test_references = []
for installed_tool in installed_tools:
test_references.extend(
build_case_references(
galaxy_interactor=galaxy_interactor,
tool_id=re.sub(f"/{installed_tool['version']}$", "", installed_tool["id"]),
tool_version=installed_tool["version"],
),
)

with ThreadPoolExecutor(max_workers=parallel_tests) as executor:
try:
for tool in installed_tools:
self._test_tool(
executor=executor,
tool=tool,
galaxy_interactor=galaxy_interactor,
test_history=test_history,
log=log,
tool_test_results=all_test_results,
tests_passed=tests_passed,
test_exceptions=test_exceptions,
client_test_config=client_test_config,
)
finally:
# Always write report, even if test was cancelled.
try:
executor.shutdown(wait=True)
except KeyboardInterrupt:
executor._threads.clear()
thread._threads_queues.clear()
n_passed = len(tests_passed)
n_failed = len(test_exceptions)
report_obj = {
"version": "0.1",
"suitename": "Ephemeris tool tests targeting %s" % self.gi.base_url,
"results": {
"total": n_passed + n_failed,
"errors": n_failed,
"failures": 0,
"skips": 0,
},
"tests": sorted(all_test_results, key=lambda el: el["id"]),
}
with open(test_json, "w") as f:
json.dump(report_obj, f)
if log:
log.info("Report written to '%s'", os.path.abspath(test_json))
log.info(f"Passed tool tests ({n_passed}): {[t for t in tests_passed]}")
log.info(f"Failed tool tests ({n_failed}): {[t[0] for t in test_exceptions]}")
log.info(f"Total tool test time: {dt.datetime.now() - tool_test_start}")
results = Results(
f"Ephemeris tool tests targeting {self.gi.base_url}",
test_json,
append=False, # ?
galaxy_url=self.gi.base_url,
)

verify_kwds = {
"skip_with_reference_data": skip_with_reference_data,
"client_test_config": client_test_config,
}

test_tools(
galaxy_interactor=galaxy_interactor,
test_references=test_references,
results=results,
log=log,
parallel_tests=parallel_tests,
history_name=test_history_name or "test_history",
no_history_cleanup=True, # TODO: Add command line option to make this False
retries=0, # TODO: Add command line option for this, would benefit Galaxy Australia
verify_kwds=verify_kwds,
)

def _get_interactor(self, test_user, test_user_api_key):
if test_user_api_key is None:
Expand All @@ -394,84 +365,6 @@ def _get_interactor(self, test_user, test_user_api_key):
galaxy_interactor = GalaxyInteractorApi(**galaxy_interactor_kwds)
return galaxy_interactor

@staticmethod
def _test_tool(
executor,
tool,
galaxy_interactor,
tool_test_results,
tests_passed,
test_exceptions,
log,
test_history=None,
client_test_config=None,
):
if test_history is None:
test_history = galaxy_interactor.new_history()
tool_id = tool["id"]
tool_version = tool["version"]
# If given a tool_id with a version suffix, strip it off so we can treat tool_version
# correctly at least in client_test_config.
if tool_version and tool_id.endswith("/" + tool_version):
tool_id = tool_id[: -len("/" + tool_version)]

label_base = tool_id
if tool_version:
label_base += "/" + str(tool_version)
try:
tool_test_dicts = galaxy_interactor.get_tool_tests(tool_id, tool_version=tool_version)
except Exception as e:
if log:
log.warning(
"Fetching test definition for tool '%s' failed",
label_base,
exc_info=True,
)
test_exceptions.append((label_base, e))
Results = namedtuple("Results", ["tool_test_results", "tests_passed", "test_exceptions"])
return Results(
tool_test_results=tool_test_results,
tests_passed=tests_passed,
test_exceptions=test_exceptions,
)
test_indices = list(range(len(tool_test_dicts)))

for test_index in test_indices:
test_id = label_base + "-" + str(test_index)

def run_test(index, test_id):
def register(job_data):
tool_test_results.append(
{
"id": test_id,
"has_data": True,
"data": job_data,
}
)

try:
if log:
log.info("Executing test '%s'", test_id)
verify_tool(
tool_id,
galaxy_interactor,
test_index=index,
tool_version=tool_version,
register_job_data=register,
quiet=True,
test_history=test_history,
client_test_config=client_test_config,
)
tests_passed.append(test_id)
if log:
log.info("Test '%s' passed", test_id)
except Exception as e:
if log:
log.warning("Test '%s' failed", test_id, exc_info=True)
test_exceptions.append((test_id, e))

executor.submit(run_test, test_index, test_id)

def install_repository_revision(self, repository: InstallRepoDict, log):
default_err_msg = "All repositories that you are attempting to install " "have been previously installed."
start = dt.datetime.now()
Expand Down Expand Up @@ -718,6 +611,7 @@ def main(argv=None):
test_history_name=args.test_history_name,
parallel_tests=args.parallel_tests,
test_all_versions=args.test_all_versions,
skip_with_reference_data=args.skip_with_reference_data,
client_test_config_path=args.client_test_config,
)
else:
Expand Down
8 changes: 8 additions & 0 deletions src/ephemeris/shed_tools_args.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ def parser():
test_existing=False,
parallel_tests=1,
client_test_config=None,
skip_with_reference_data=False,
)

# SUBPARSERS
Expand Down Expand Up @@ -287,6 +288,13 @@ def parser():
"apply for tools where revisions have not been provided through "
"the --revisions arg, --tool_file or --tool_yaml.",
)
test_command_parser.add_argument(
"--skip-with-reference-data",
"--skip_with_reference_data",
action="store_true",
dest="skip_with_reference_data",
help="Skip tests the Galaxy server believes use data tables or loc files.",
)
test_command_parser.add_argument(
"--client-test-config",
"--client_test_config",
Expand Down
Loading