Merge pull request #109 from kbase/dev-service

Save image details to mongo

cdmtaskservice/error_mapping.py

@@ -11,6 +11,7 @@
 from cdmtaskservice.image_remote_lookup import ImageNameParseError, ImageInfoFetchError
 from cdmtaskservice.job_state import ETagMismatchError
 from cdmtaskservice.kb_auth import InvalidTokenError, MissingRoleError
+from cdmtaskservice.mongo import ImageTagExistsError, ImageDigestExistsError
 from cdmtaskservice.routes import UnauthorizedError, ClientLifeTimeError
 from cdmtaskservice.s3.client import (
     S3BucketInaccessibleError,
@@ -50,8 +51,11 @@ class ErrorMapping(NamedTuple):
     NoEntrypointError: ErrorMapping(ErrorType.MISSING_ENTRYPOINT, _H400),
     ImageInfoFetchError: ErrorMapping(ErrorType.IMAGE_FETCH, _H400),
     ImageNameParseError: ErrorMapping(ErrorType.IMAGE_NAME_PARSE, _H400),
+    ImageTagExistsError: ErrorMapping(ErrorType.IMAGE_TAG_EXISTS, _H400),
+    ImageDigestExistsError: ErrorMapping(ErrorType.IMAGE_DIGEST_EXISTS, _H400),
 }
 
+
 def map_error(err: Exception) -> tuple[ErrorType, int]:
     """
     Map an error to an optional error type and a HTTP code.

cdmtaskservice/errors.py

@@ -69,6 +69,12 @@ class ErrorType(Enum):
     S3_BUCKET_NOT_FOUND =        (40020, "S3 bucket not Found")  # noqa: E222 @IgnorePep8
     """ The S3 bucket was not found. """
 
+    IMAGE_TAG_EXISTS =           (50000, "Image tag exists")  # noqa: E222 @IgnorePep
+    """ The tag for the image already exists in the system. """
+
+    IMAGE_DIGEST_EXISTS =        (50010, "Image digest exists")  # noqa: E222 @IgnorePep
+    """ The digest for the image already exists in the system. """
+
     UNSUPPORTED_OP =             (100000, "Unsupported operation")  # noqa: E222 @IgnorePep8
     """ The requested operation is not supported. """
 

cdmtaskservice/images.py

@@ -29,17 +29,16 @@ async def register(self, imagename: str):
         entrypoint = await self._iminfo.get_entrypoint_from_name(normedname.name_with_digest)
         if not entrypoint:
             raise NoEntrypointError(f"Image {imagename} does not have an entrypoint")
-        # TODO IMAGEREG save image to mongo
-        #       need unique index on image name + tag for lookups and uniqueness
-        #       need unique index on image name + sha for the same
         # TODO DATAINTEG add username and date created
         # TODO REFDATA allow specifying refdata for image
-        return models.Image(
+        img = models.Image(
             name=normedname.name,
             digest = normedname.digest,
             tag=normedname.tag,
             entrypoint=entrypoint
         )
+        await self._mongo.save_image(img)
+        return img
 
 
 class NoEntrypointError(Exception):

cdmtaskservice/models.py

@@ -26,6 +26,17 @@
 #               still doesn't seem to work as of 24/11/11
 #               https://github.com/fastapi/fastapi/discussions/11137
 
+# WARNNING: Model field names also define field names in the MOngo database.
+# As such, field names cannot change without creating a mapping for field names in the mongo
+# layer, or data will not be returned correclty and corruption of old datq may occur.
+# The strings below are used in the mongo interface to define fields and indexes.
+# They must match with the field names in the models.
+
+FLD_IMAGE_NAME = "name"
+FLD_IMAGE_DIGEST = "digest"
+FLD_IMAGE_TAG = "tag"
+
+
 # https://en.wikipedia.org/wiki/Filename#Comparison_of_filename_limitations
 # POSiX fully portable filenames and /
 _PATH_REGEX=r"^[\w.-/]+$"

cdmtaskservice/mongo.py

@@ -6,11 +6,14 @@
 
 from motor.motor_asyncio import AsyncIOMotorDatabase
 from pymongo import IndexModel, ASCENDING, DESCENDING
+from pymongo.errors import DuplicateKeyError
 
-_FLD_IMAGE_NAME = "name"
-_FLD_IMAGE_HASH = "hash"
-_FLD_IMAGE_TAG = "tag"
-_FLD_IMAGE_ENTRYPOINT = "entrypoint"
+from cdmtaskservice.arg_checkers import not_falsy as _not_falsy
+from cdmtaskservice import models
+
+
+_INDEX_TAG = "UNIQUE_IMAGE_TAG_INDEX"
+_INDEX_DIGEST = "UNIQUE_IMAGE_DIGEST_INDEX"
 
 
 class MongoDAO:
@@ -42,12 +45,47 @@ async def _create_indexes(self):
         #                    https://www.mongodb.com/docs/manual/core/index-creation/#constraint-violations-during-index-build
         #                    Otherwise, create a polling loop to wait until indexes exist
 
+        # NOTE that since there's two unique indexes means this collection can't be sharded
+        # but it's likely to be very small so shouldn't be an issue
         await self._col_images.create_indexes([
-            IndexModel([(_FLD_IMAGE_NAME, ASCENDING), (_FLD_IMAGE_HASH, ASCENDING)], unique=True),
+            IndexModel(
+                [(models.FLD_IMAGE_NAME, ASCENDING), (models.FLD_IMAGE_DIGEST, ASCENDING)],
+                 unique=True,
+                 name=_INDEX_DIGEST
+            ),
             # Only allow one instance of a tag per image name to avoid confusion
             IndexModel(
-                [(_FLD_IMAGE_NAME, ASCENDING), (_FLD_IMAGE_TAG, ASCENDING)],
+                [(models.FLD_IMAGE_NAME, ASCENDING), (models.FLD_IMAGE_TAG, ASCENDING)],
                 unique=True,
-                sparse=True  # tags are optional
+                sparse=True,  # tags are optional
+                name=_INDEX_TAG
             ),
         ])
+
+    async def save_image(self, image: models.Image):
+        """
+        Save details about a Docker image.
+        
+        Only one record per digest or tag is allowed per image
+        """
+        _not_falsy(image, "image")
+        try:
+            await self._col_images.insert_one(image.model_dump())
+        except DuplicateKeyError as e:
+            if _INDEX_TAG in e.args[0]:
+                raise ImageTagExistsError(f"The tag {image.tag} for image {image.name} "
+                                          + "already exists in the system")
+            elif _INDEX_DIGEST in e.args[0]:
+                raise ImageDigestExistsError(f"The digest {image.digest} for image {image.name} "
+                                             + "already exists in the system")
+            else:
+                # no way to test this, but just in case
+                raise ValueError(f"Unexpected duplicate key collision for image {image}") from e
+
+
+class ImageTagExistsError(Exception):
+    """ The tag for the image already exists in the system. """
+
+
+class ImageDigestExistsError(Exception):
+    """ The digest for the image already exists in the system. """