updates to podman containers to pull digest and then tag and modulefiles

Signed-off-by: vsoch <[email protected]>

docs/getting_started/user-guide.rst

@@ -13,16 +13,17 @@ you should do that first.
 Why shpc?
 =========
 
-While the library is currently focused on Singularity containers (hence
-the name) it's created to be modular, meaning that if another container technology
-is wanted, it can be added. The module name would still be appropriate, as
-singularity does imply a single entity that is "one library to rule them all!"
+Singularity Registry HPC is created to be modular, meaning that we support a distinct
+set of container technologies and module systems. The name of the library "Singularity
+Registry HPC" does not refer specifically to the container technology "Singularity,"
+but more generally implies the same spirit -- a single entity that is "one library to rule them all!"
+
 
 What is a registry?
 ===================
 
 A registry consists of a database of local containers configuration files, ``container.yaml``
-files organized in the root of the shpc install in the ``registry`` folder. The namespace
+files organized in the root of the shpc install in one of the ``registry`` folders. The namespace
 is organized by Docker unique resources identifiers. When you install an identifier
 as we saw above, the container binaries and customized module files are added to 
 the ``module_dir`` defined in your settings, which defaults to ``modules`` in the

shpc/client/__init__.py

@@ -142,6 +142,16 @@ def get_parser():
         help="update configuration settings. Use set or get to see or set information.",
         formatter_class=argparse.RawTextHelpFormatter,
     )
+
+    config.add_argument(
+        "--central",
+        "-c",
+        dest="central",
+        help="make edits to the central config file.",
+        default=False,
+        action="store_true",
+    )
+
     config.add_argument(
         "params",
         nargs="*",

shpc/client/config.py

@@ -2,6 +2,7 @@
 __copyright__ = "Copyright 2021, Vanessa Sochat"
 __license__ = "MPL 2.0"
 
+import shpc.defaults as defaults
 from shpc.logger import logger
 import sys
 
@@ -18,6 +19,10 @@ def main(args, parser, extra, subparser):
     # The first "param" is either set of get
     command = args.params.pop(0)
 
+    # If the user wants the central config file
+    if args.central:
+        args.settings_file = defaults.default_settings_file
+
     validate = True if not command == "edit" else False
     cli = get_client(
         quiet=args.quiet, settings_file=args.settings_file, validate=validate

shpc/main/container/docker.py

@@ -60,9 +60,11 @@ def registry_pull(self, module_dir, container_dir, config, tag):
         if pull_type != "docker":
             logger.exit("%s only supports Docker (oci registry) pulls." % self.command)
 
-        # Podman doesn't keep a record of digest->tag, so we use tag
-        uri = "%s:%s" % (self.add_registry(config.docker), tag.name)
-        return self.pull(uri)
+        tag_uri = "%s:%s" % (self.add_registry(config.docker), tag.name)
+        tag_digest = "%s@%s" % (self.add_registry(config.docker), tag.digest)
+        self.pull(tag_digest)
+        # Podman doesn't keep a record of digest->tag, so we tag after
+        return self.tag(tag_digest, tag_uri)
 
     def pull(self, uri):
         """
@@ -73,6 +75,15 @@ def pull(self, uri):
             logger.exit("There was an issue pulling %s" % uri)
         return uri
 
+    def tag(self, image, tag_as):
+        """
+        Given a container URI, tag as something else.
+        """
+        res = shpc.utils.run_command([self.command, "tag", image, tag_as])
+        if res["return_code"] != 0:
+            logger.exit("There was an issue tagging %s as %s" % (image, tag_as))
+        return tag_as
+
     def inspect(self, image):
         """
         Inspect an image

shpc/main/modules/templates/docker.lua

@@ -16,11 +16,11 @@ Container:
 Commands include:
 
  - {|module_name|}-run:
-       {{ command }} run -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if envfile %}--env-file {{ module_dir }}/{{ envfile }}{% endif %} {% if bindpaths %}-v {{ bindpaths }} -v ${PWD} -w ${PWD} {% endif %}<container> "$@"
+       {{ command }} run -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if envfile %}--env-file {{ module_dir }}/{{ envfile }} {% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %}-v ${PWD} -w ${PWD} <container> "$@"
  - {|module_name|}-shell:
-       {{ command }} run -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if envfile %}--env-file {{ module_dir }}/{{ envfile }}{% endif %} {% if bindpaths %}-v {{ bindpaths }} --entrypoint {{ shell }} -v ${PWD} -w ${PWD} {% endif %}<container>
+       {{ command }} run -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if envfile %}--env-file {{ module_dir }}/{{ envfile }} {% endif %} {% if bindpaths %}-v {{ bindpaths }} --entrypoint {{ shell }} {% endif %}-v ${PWD} -w ${PWD}<container>
  - {|module_name|}-exec:
-       {{ command }} run -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm --entrypoint "" {% if envfile %}--env-file {{ module_dir }}/{{ envfile }}{% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %} -v ${PWD} -w ${PWD} <container> "$@"
+       {{ command }} run -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm --entrypoint "" {% if envfile %}--env-file {{ module_dir }}/{{ envfile }} {% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %} -v ${PWD} -w ${PWD} <container> "$@"
  - {|module_name|}-inspect:
        {{ command }} inspect <container>
 

shpc/main/modules/templates/docker.tcl

@@ -15,15 +15,15 @@ proc ModulesHelp { } {
     puts stderr " - {{ image }}"
     puts stderr "Commands include:"
     puts stderr " - {|module_name|}-run:"
-    puts stderr "       {{ command }} run -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if envfile %}--env-file  {{ module_dir }}/{{ envfile }}{% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %} -v . -w . <container>"
+    puts stderr "       {{ command }} run -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if envfile %}--env-file  {{ module_dir }}/{{ envfile }} {% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %} -v . -w . <container>"
     puts stderr " - {|module_name|}-shell:"
-    puts stderr "       {{ command }} run -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm --entrypoint {{ shell }}{% if envfile %} --env-file {{ module_dir }}/{{ envfile }}{% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %} -v . -w . <container>"
+    puts stderr "       {{ command }} run -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm --entrypoint \"\"{% if envfile %} --env-file {{ module_dir }}/{{ envfile }} {% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %} -v . -w . <container>"
     puts stderr " - {|module_name|}-exec:"
-    puts stderr "       {{ command }} run -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm --entrypoint \"\" {% if envfile %} --env-file  {{ module_dir }}/{{ envfile }}{% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %} -v . -w . <container> $*"
+    puts stderr "       {{ command }} run -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm --entrypoint \"\" {% if envfile %} --env-file  {{ module_dir }}/{{ envfile }} {% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %} -v . -w . <container> $*"
     puts stderr " - {|module_name|}-inspect:"
     puts stderr "       {{ command }} inspect <container>"
 {% if aliases %}{% for alias in aliases %}    puts stderr " - {{ alias.name }}:"
-    puts stderr "       {{ command }} run -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --entrypoint {{ alias.entrypoint }} {% if envfile %}--envfile  {{ module_dir }}/{{ envfile }} {% endif %}{% if bindpaths %}-v {{ bindpaths }} {% endif %}{% if alias.options %}{{ alias.options }} {% endif %} -v . -w . <container> {{ alias.args }}"
+    puts stderr "       {{ command }} run -i{% if tty %}t{% endif %} --rm -u `id -u`:`id -g` --entrypoint {{ alias.entrypoint }} {% if envfile %}--envfile  {{ module_dir }}/{{ envfile }} {% endif %}{% if bindpaths %}-v {{ bindpaths }} {% endif %}{% if alias.options %}{{ alias.options }} {% endif %} -v . -w . <container> {{ alias.args }}"
 {% endfor %}{% endif %}
 
     puts stderr "For each of the above, you can export:"
@@ -59,7 +59,7 @@ conflict {{ name }}
 set shellCmd "{{ command }} \${PODMAN_OPTS} run \${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm -i{% if tty %}t{% endif %} --entrypoint {{ shell }} {% if envfile %}--env-file {{ module_dir }}/{{ envfile }}{% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %} -v $workdir -w $workdir ${containerPath}" 
 
 # execCmd needs entrypoint to be the executor
-set execCmd "{{ command }} \${PODMAN_OPTS} run -i{% if tty %}t{% endif %} \${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm {% if envfile %} --env-file {{ module_dir }}/{{ envfile }}{% endif %} {% if bindpaths %}-v {{ bindpaths }} -v $workdir -w $workdir {% endif %} "
+set execCmd "{{ command }} \${PODMAN_OPTS} run -i{% if tty %}t{% endif %} \${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm {% if envfile %} --env-file {{ module_dir }}/{{ envfile }}{% endif %} {% if bindpaths %}-v {{ bindpaths }}{% endif %} -v $workdir -w $workdir"
 set runCmd "{{ command }} \${PODMAN_OPTS} run -i{% if tty %}t{% endif %} \${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm {% if envfile %}--env-file  {{ module_dir }}/{{ envfile }}{% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %} -v $workdir -w $workdir ${containerPath}"
 set inspectCmd "{{ command }} \${PODMAN_OPTS} inspect ${containerPath}" 
 
@@ -68,7 +68,7 @@ set-alias {|module_name|}-shell "${shellCmd}"
 
 # exec functions to provide "alias" to module commands
 {% if aliases %}{% for alias in aliases %}
-set-alias {{ alias.name }} "${execCmd} {% if alias.options %} {{ alias.options }} {% endif %} --entrypoint {{ alias.entrypoint }} ${containerPath} {{ alias.command }}"
+set-alias {{ alias.name }} "${execCmd} {% if alias.options %} {{ alias.options }} {% endif %} --entrypoint {{ alias.entrypoint }} ${containerPath} {{ alias.args }}"
 {% endfor %}{% endif %}
 
 # A customizable exec function

shpc/main/settings.py

@@ -121,7 +121,8 @@ def add(self, key, value):
         current = self._settings.get(key)
         if current and not isinstance(current, list):
             logger.exit("You cannot only add to a list variable.")
-        current.append(value)
+        # Add to the beginning of the list
+        current = current + [value]
         value = list(set(current))
         self._settings[key] = value
         self.change_validate(key, value)
@@ -197,12 +198,21 @@ def delete(self, key):
             del self._settings[key]
 
     def save(self, filename=None):
+        """
+        Save settings, but do not change order of anything.
+        """
         filename = filename or self.settings_file
         if not filename:
             logger.exit("A filename is required to save to.")
         yaml = YAML()
-        with open(filename, "w") as fd:
-            yaml.dump(self._settings, fd)
+
+        # This requires Python 3.7 support
+        try:
+            with open(filename, "w") as fd:
+                yaml.dump(self._settings, fd, sort_keys=False)
+        except:
+            with open(filename, "w") as fd:
+                yaml.dump(self._settings, fd)
 
     def __iter__(self):
         for key, value in self.__dict__.items():