Split existing code into modules

Add modules: configs, locking, outputs, runners, validators, misc and globalvars
Remove ansible_deployer/command_line.py
Adjust setup and increase version

setup.py

@@ -19,7 +19,7 @@ def read(fname):
 # This call to setup() does all the work
 setup(
     name="ansible-deployer",
-    version="0.0.23",
+    version="0.0.24",
     description="Wrapper around ansible-playbook allowing configurable tasks and permissions",
     long_description=README,
     long_description_content_type="text/markdown",
@@ -28,12 +28,21 @@ def read(fname):
     classifiers=[
         "Programming Language :: Python :: 3.9"
     ],
-    packages=["ansible_deployer"],
+    package_dir={"ansible_deployer": "source"},
+    packages=[
+    "ansible_deployer",
+    "ansible_deployer.modules",
+    "ansible_deployer.modules.configs",
+    "ansible_deployer.modules.outputs",
+    "ansible_deployer.modules.locking",
+    "ansible_deployer.modules.runners",
+    "ansible_deployer.modules.validators"
+    ],
     include_package_data=True,
     install_requires=["pyyaml>=5.3.1", "cerberus>=1.3.1", "pytest-testinfra>=6.6.0"],
     entry_points={
         "console_scripts": [
-            "ansible-deployer = ansible_deployer.command_line:main",
+            "ansible-deployer = ansible_deployer.main:main",
         ]
     },
 )

source/main.py

@@ -0,0 +1,130 @@
+"""Main module for ansible-deploy"""
+
+import os
+import sys
+import argparse
+import datetime
+import errno
+from ansible_deployer.modules.globalvars import SUBCOMMANDS
+from ansible_deployer.modules.configs.config import Config
+from ansible_deployer.modules.locking.locking import Locking
+from ansible_deployer.modules.outputs.logging import Loggers
+from ansible_deployer.modules.validators.validate import Validators
+from ansible_deployer.modules.runners.run import Runners
+from ansible_deployer.modules import misc
+
+
+def parse_options(argv):
+    """Generic function to parse options for all commands, we validate if the option was allowed for
+    specific subcommand outside"""
+    parser = argparse.ArgumentParser(add_help=True)
+
+    parser.add_argument("subcommand", nargs='*', default=None, metavar="SUBCOMMAND",
+                        help='Specify subcommand to execute. Available commands: '+str(SUBCOMMANDS))
+    parser.add_argument("--infrastructure", "-i", nargs=1, default=[None], metavar="INFRASTRUCTURE",
+                        help='Specify infrastructure for deploy.')
+    parser.add_argument("--stage", "-s", nargs=1, default=[None], metavar="STAGE",
+                        help='Specify stage type. Available types are: "testing" and "prod".')
+    parser.add_argument("--commit", "-c", nargs=1, default=[None], metavar="COMMIT",
+                        help='Provide commit ID.')
+    parser.add_argument("--task", "-t", nargs=1, default=[None], metavar='TASK_NAME',
+                        help='Provide task_name.')
+    parser.add_argument("--dry", "-C", default=False, action='store_true', help='Perform dry run.')
+    parser.add_argument("--keep-locked", "-k", default=False, action='store_true', help='Keep'
+                        ' infrastructure locked after task execution.')
+    parser.add_argument("--debug", "-d", default=False, action="store_true",
+                        help='Print debug output.')
+    parser.add_argument("--syslog", "-v", default=False, action="store_true", help='Log warnings '
+                        'and errors to syslog. --debug doesn\'t affect this option!')
+    parser.add_argument("--limit", "-l", nargs=1, default=[None], metavar="[LIMIT]",
+                        help='Limit task execution to specified host.')
+    parser.add_argument("--conf-dir", nargs=1, default=[None], metavar="conf_dir",
+                        help='Use non-default configuration directory, only allowed for \
+                              non-binarized exec')
+
+    arguments = parser.parse_args(argv)
+
+    if not arguments.subcommand:
+        sub_string = ", ".join(SUBCOMMANDS).strip(", ")
+        print(f"[CRITICAL]: First positional argument (subcommand) is required! Available commands"
+              f" are: {sub_string}.")
+        sys.exit(57)
+
+    options = {}
+    options["subcommand"] = arguments.subcommand[0].lower()
+    Validators.verify_subcommand(options["subcommand"])
+    Validators.verify_switches(arguments.subcommand)
+
+    options["switches"] = arguments.subcommand[1:]
+    options["infra"] = arguments.infrastructure[0]
+    options["stage"] = arguments.stage[0]
+    options["commit"] = arguments.commit[0]
+    options["task"] = arguments.task[0]
+    options["dry"] = arguments.dry
+    options["keep_locked"] = arguments.keep_locked
+    options["debug"] = arguments.debug
+    options["syslog"] = arguments.syslog
+    options["limit"] = arguments.limit[0]
+    if arguments.conf_dir[0]:
+        options["conf_dir"] = os.path.abspath(arguments.conf_dir[0])
+    else:
+        options["conf_dir"] = None
+
+    return options
+
+def main():
+    """ansible-deploy endpoint function"""
+    start_ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+
+    if len(sys.argv) < 2:
+        print("[CRITICAL]: Too few arguments", file=sys.stderr)
+        sys.exit(2)
+    options = parse_options(sys.argv[1:])
+
+    logger = Loggers(options)
+
+    configuration = Config(logger.logger, options["conf_dir"])
+    conf = configuration.conf
+    config = configuration.load_configuration()
+
+    if options["subcommand"] in ("run", "verify"):
+        workdir = misc.create_workdir(start_ts, conf, logger.logger)
+        Loggers.set_logging_to_file(logger, workdir, start_ts, conf)
+
+    validators = Validators(logger.logger)
+    validators.validate_options(options)
+    selected_items = validators.validate_option_values_against_config(config, options)
+
+    user_groups = misc.get_all_user_groups(logger.logger)
+
+    if options["dry"]:
+        logger.logger.info("Skipping execution because of --dry-run option")
+        sys.exit(0)
+
+    if options["subcommand"] == "list":
+        misc.list_tasks(config, options)
+    elif options["subcommand"] == "show":
+        misc.show_deployer(config, options)
+    else:
+        lockdir = os.path.join(conf["global_paths"]["work_dir"], "locks")
+        inv_file = misc.get_inventory_file(config, options)
+        lockpath = os.path.join(lockdir, inv_file.lstrip(f".{os.sep}").replace(os.sep, "_"))
+        lock = Locking(logger.logger, options["keep_locked"], (options["infra"], options["stage"]))
+        if options["subcommand"] in ("run", "verify"):
+            if not validators.verify_task_permissions(selected_items, user_groups, config):
+                logger.logger.critical("Task forbidden")
+                sys.exit(errno.EPERM)
+            runner = Runners(logger.logger, lock)
+            runner.setup_ansible(config["tasks"]["setup_hooks"], options["commit"], workdir)
+            lock.lock_inventory(lockdir, lockpath)
+            runner.run_playitem(config, options, inv_file, lockpath)
+            lock.unlock_inventory(lockpath)
+        elif options["subcommand"] == "lock":
+            lock.lock_inventory(lockdir, lockpath)
+        elif options["subcommand"] == "unlock":
+            lock.unlock_inventory(lockpath)
+
+    sys.exit(0)
+
+if __name__ == "__main__":
+    main()

source/modules/configs/config.py

@@ -0,0 +1,150 @@
+"""Module for configuration files handling"""
+
+import os
+import sys
+import stat
+import yaml
+from cerberus import Validator
+from ansible_deployer.modules.globalvars import APP_CONF, CFG_PERMISSIONS
+
+
+class Config:
+    """Class handling global configuration, and other configs: tasks, infrastructures,
+    permissions"""
+
+    def __init__(self, logger, conf_dir):
+        self.logger = logger
+        self.conf_dir = conf_dir if conf_dir else APP_CONF
+        self.conf = self.load_global_configuration()
+
+    def load_configuration_file(self, config_path: str):
+        """Function responsible for single file loading and validation"""
+        #TODO: Add verification of owner/group/persmissions
+        if config_path == APP_CONF:
+            self.check_cfg_permissions_and_owner(config_path)
+        config_file = os.path.basename(config_path)
+        self.logger.debug("Loading :%s", config_file)
+
+        with open(config_path, "r", encoding="utf8") as config_stream:
+            try:
+                config = yaml.safe_load(config_stream)
+            except yaml.YAMLError as e:
+                self.logger.critical("Yaml loading failed for %s due to %s.", config_path, e)
+                sys.exit(51)
+
+        schema_path = os.path.join(self.conf_dir, "schema", config_file)
+        with open(schema_path, "r", encoding="utf8") as schema_stream:
+            try:
+                schema = yaml.safe_load(schema_stream)
+            except yaml.YAMLError as e:
+                self.logger.critical("Yaml loading failed for %s due to %s.", config_path, e)
+                sys.exit(52)
+
+        validator = Validator(schema)
+        if not validator.validate(config, schema):
+            self.logger.critical("Yaml validation failed for %s due to %s.", config_path,
+                                 validator.errors)
+            sys.exit(53)
+
+        self.logger.debug("Loaded:\n%s", str(config))
+        return config
+
+    def check_cfg_permissions_and_owner(self, cfg_path: str):
+        """Function to verify permissions and owner for config files"""
+        stat_info = os.stat(cfg_path)
+
+        if stat_info.st_uid == 0:
+            if oct(stat.S_IMODE(stat_info.st_mode)) == CFG_PERMISSIONS:
+                self.logger.debug("Correct permissions and owner for config file %s.", cfg_path)
+            else:
+                self.logger.error("File %s permissions are incorrect! Contact your sys admin.",
+                                  cfg_path)
+                self.logger.error("Program will exit now.")
+                sys.exit(40)
+        else:
+            self.logger.error("File %s owner is not root! Contact your sys admin.", cfg_path)
+            self.logger.error("Program will exit now.")
+            sys.exit(41)
+
+    def get_config_paths(self):
+        """Function to create absolute config paths and check their extension compatibility"""
+        ymls = []
+        yamls = []
+        infra_cfg = None
+        tasks_cfg = None
+        acl_cfg = None
+
+        for config in os.listdir(self.conf_dir):
+            if config != "ansible-deploy.yaml":
+                if config.endswith(".yml"):
+                    ymls.append(config)
+                elif config.endswith(".yaml"):
+                    yamls.append(config)
+
+                if config.startswith("infra"):
+                    infra_cfg = os.path.join(self.conf_dir, config)
+                elif config.startswith("tasks"):
+                    tasks_cfg = os.path.join(self.conf_dir, config)
+                elif config.startswith("acl"):
+                    acl_cfg = os.path.join(self.conf_dir, config)
+
+        if len(ymls) > 0 and len(yamls) > 0:
+            self.logger.debug("Config files with yml extensions: %s", " ".join(ymls))
+            self.logger.debug("Config files with yaml extensions: %s", " ".join(yamls))
+            self.logger.critical("Config files with different extensions (.yml and .yaml) are not"
+                                 " allowed in conf dir %s !", self.conf_dir)
+            sys.exit(42)
+
+        if not infra_cfg:
+            self.logger.critical("Infrastructure configuration file does not exist in %s!",
+                            self.conf_dir)
+            sys.exit(43)
+
+        if not tasks_cfg:
+            self.logger.critical("Tasks configuration file does not exist in %s!",
+                            self.conf_dir)
+            sys.exit(44)
+
+        if not acl_cfg:
+            self.logger.critical("Permission configuration file does not exist in %s!",
+                            self.conf_dir)
+            sys.exit(45)
+
+        return infra_cfg, tasks_cfg, acl_cfg
+
+    def load_configuration(self):
+        """Function responsible for reading configuration files and running a schema validator
+        against it"""
+        self.logger.debug("load_configuration called")
+        #TODO: validate files/directories permissions - should be own end editable only by
+        #special user
+        infra_cfg, tasks_cfg, acl_cfg = self.get_config_paths()
+
+        infra = self.load_configuration_file(infra_cfg)
+        tasks = self.load_configuration_file(tasks_cfg)
+        acl = self.load_configuration_file(acl_cfg)
+
+        config = {}
+        config["infra"] = infra["infrastructures"]
+        config["tasks"] = tasks
+
+        config["acl"] = {}
+        for group in acl["acl_lists"]:
+            key = group["name"]
+            group.pop("name")
+            config["acl"][key] = group
+
+        return config
+
+    def load_global_configuration(self):
+        """Function responsible for single file loading and validation"""
+        main_config_file = os.path.join(self.conf_dir, "ansible-deploy.yaml")
+        if self.conf_dir == APP_CONF:
+            self.check_cfg_permissions_and_owner(main_config_file)
+        with open(main_config_file, "r", encoding="utf8") as config_stream:
+            try:
+                config = yaml.safe_load(config_stream)
+                return config
+            except yaml.YAMLError as e:
+                self.logger.critical(e, file=sys.stderr)
+                sys.exit(51)

source/modules/globalvars.py

@@ -0,0 +1,5 @@
+"""Global variables"""
+
+APP_CONF = "/etc/ansible-deployer"
+CFG_PERMISSIONS = "0o644"
+SUBCOMMANDS = ("run", "list", "lock", "unlock", "verify", "show")

source/modules/locking/locking.py

@@ -0,0 +1,66 @@
+"""Module for handling infrastructure locks"""
+
+import os
+import sys
+import pwd
+
+
+class Locking:
+    """Class handling infrastructure locks"""
+
+    def __init__(self, logger, keep_lock, infra):
+        self.logger = logger
+        self.keep_lock = keep_lock
+        self.infra = infra
+
+    def lock_inventory(self, lockdir: str, lockpath: str):
+        """
+        Function responsible for locking inventory file.
+        The goal is to prevent two parallel ansible-deploy's running on the same inventory
+        This needs to be done by the use of additional directory under PARNT_WORKDIR,, for instance:
+        PARENT_WORKDIR/locks.
+        We shouldn't check if file exists, but rather attempt to open it for writing, until we're
+        done every other process should be rejected this access.
+        The file should match inventory file name.
+        """
+
+        self.logger.debug("Started lock_inventory for lockdir: %s and lockpath %s.", lockdir,
+                          lockpath)
+        os.makedirs(lockdir, exist_ok=True)
+
+        try:
+            with open(lockpath, "x", encoding="utf8") as fh:
+                fh.write(str(os.getpid()))
+                fh.write(str("\n"))
+                fh.write(str(pwd.getpwuid(os.getuid()).pw_name))
+            self.logger.info("Infra locked.")
+        except FileExistsError:
+            with open(lockpath, "r", encoding="utf8") as fh:
+                proc_pid, proc_user = fh.readlines()
+            self.logger.critical("Another process (PID: %s) started by user %s is using this"
+                                " infrastructure, please try again later.", proc_pid.strip(),
+                                proc_user.strip())
+            sys.exit(61)
+        except Exception as exc:
+            self.logger.critical(exc)
+            sys.exit(62)
+
+    def unlock_inventory(self, lockpath: str):
+        """
+        Function responsible for unlocking inventory file, See also lock_inventory
+        """
+
+        self.logger.debug("Started unlock_inventory for lockpath %s.", lockpath)
+
+        if not self.keep_lock:
+            try:
+                os.remove(lockpath)
+                self.logger.info("Lock %s has been removed.", lockpath)
+            except FileNotFoundError:
+                self.logger.critical("Requested lock %s was not found. Nothing to do.", lockpath)
+                sys.exit(63)
+            except Exception as exc:
+                self.logger.critical(exc)
+                sys.exit(64)
+        else:
+            self.logger.debug("Keep locked infra %s:%s .", self.infra[0], self.infra[1])