From e659ac5135e2321d2c5e59419e3d9c0df268553a Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Mon, 8 Jun 2020 21:57:51 -0700 Subject: [PATCH] Allocate more workers to parse module docs. There are vastly more modules than other types of plugins so we want to allocate the bulk of our threads to processing documentation from them. This is an intermediate fix for #89 We may want to fix this in a better way later on. --- antsibull/docs_parsing/ansible_doc.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/antsibull/docs_parsing/ansible_doc.py b/antsibull/docs_parsing/ansible_doc.py index 394780ba..b22beede 100644 --- a/antsibull/docs_parsing/ansible_doc.py +++ b/antsibull/docs_parsing/ansible_doc.py @@ -162,12 +162,19 @@ async def get_ansible_plugin_info(venv: Union['VenvRunner', 'FakeVenvRunner'], # Why use THREAD_MAX instead of process max? Even though this ultimately invokes separate # ansible-doc processes, the limiting factor is IO as ansible-doc reads from disk. So it makes # sense to scale up to THREAD_MAX instead of PROCESS_MAX. - max_workers = int(THREAD_MAX / len(DOCUMENTABLE_PLUGINS)) - if max_workers < 1: - max_workers = 1 + + # Allocate more for modules because the vast majority of plugins are modules + module_workers = max(int(.7 * THREAD_MAX), 1) + other_workers = int((THREAD_MAX - module_workers) / (len(DOCUMENTABLE_PLUGINS) - 1)) + if other_workers < 1: + other_workers = 1 extractors = {} for plugin_type in DOCUMENTABLE_PLUGINS: + if plugin_type == 'module': + max_workers = module_workers + else: + max_workers = other_workers extractors[plugin_type] = asyncio.create_task( _get_plugin_info(plugin_type, venv_ansible_doc, max_workers))