Add checkpoint support to Pulp core and file

Introduce a checkpoint field for Publication and Distribution models.
Handle serving checkpoint Publications via checkpoint Distributions.
Protect checkpoint Publications' RepositoryVersions from cleanup.
Enable checkpoint support in pulp_file.

closes pulp#6244

CHANGES/6244.feature

@@ -0,0 +1 @@
+Added support to create and distribute checkpoint publications in Pulp.

CHANGES/plugin_api/6244.feature

@@ -0,0 +1,3 @@
+Added support to create and distribute checkpoint publications in Pulp.
+Plugins can choose to enable this feature by exposing the checkpoint field in their inherited PublicationSerializer and DistributionSerializer.
+Checkpoint publications and distributions can be created by passing checkpoint=True when creating them. 

CHANGES/pulp_file/6244.feature

@@ -0,0 +1 @@
+Added support to create checkpoint file publications and distribute them through checkpoint file distributions.

pulp_file/app/serializers.py

@@ -115,10 +115,11 @@ class FilePublicationSerializer(PublicationSerializer):
         required=False,
         allow_null=True,
     )
+    checkpoint = serializers.BooleanField(default=False)
 
     class Meta:
         model = FilePublication
-        fields = PublicationSerializer.Meta.fields + ("distributions", "manifest")
+        fields = PublicationSerializer.Meta.fields + ("distributions", "manifest", "checkpoint")
 
 
 class FileDistributionSerializer(DistributionSerializer):
@@ -133,9 +134,10 @@ class FileDistributionSerializer(DistributionSerializer):
         queryset=models.Publication.objects.exclude(complete=False),
         allow_null=True,
     )
+    checkpoint = serializers.BooleanField(default=False)
 
     class Meta:
-        fields = DistributionSerializer.Meta.fields + ("publication",)
+        fields = DistributionSerializer.Meta.fields + ("publication", "checkpoint")
         model = FileDistribution
 
 

pulp_file/app/tasks/publishing.py

@@ -19,7 +19,7 @@
 log = logging.getLogger(__name__)
 
 
-def publish(manifest, repository_version_pk):
+def publish(manifest, repository_version_pk, checkpoint=False):
     """
     Create a Publication based on a RepositoryVersion.
 
@@ -37,7 +37,9 @@ def publish(manifest, repository_version_pk):
     )
 
     with tempfile.TemporaryDirectory(dir="."):
-        with FilePublication.create(repo_version, pass_through=True) as publication:
+        with FilePublication.create(
+            repo_version, pass_through=True, checkpoint=checkpoint
+        ) as publication:
             publication.manifest = manifest
             if manifest:
                 manifest = Manifest(manifest)

pulp_file/app/viewsets.py

@@ -433,11 +433,16 @@ def create(self, request):
         serializer.is_valid(raise_exception=True)
         repository_version = serializer.validated_data.get("repository_version")
         manifest = serializer.validated_data.get("manifest")
+        checkpoint = serializer.validated_data.get("checkpoint")
 
         result = dispatch(
             tasks.publish,
             shared_resources=[repository_version.repository],
-            kwargs={"repository_version_pk": str(repository_version.pk), "manifest": manifest},
+            kwargs={
+                "repository_version_pk": str(repository_version.pk),
+                "manifest": manifest,
+                "checkpoint": checkpoint,
+            },
         )
         return OperationPostponedResponse(result, request)
 

pulpcore/app/migrations/0128_distribution_checkpoint_publication_checkpoint.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.2.18 on 2025-01-30 19:14
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("core", "0127_remove_upstreampulp_pulp_label_select"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="distribution",
+            name="checkpoint",
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name="publication",
+            name="checkpoint",
+            field=models.BooleanField(default=False, editable=False),
+        ),
+    ]

pulpcore/app/models/publication.py

@@ -73,6 +73,7 @@ class Publication(MasterModel):
         pass_through (models.BooleanField): Indicates that the publication is a pass-through
             to the repository version. Enabling pass-through has the same effect as creating
             a PublishedArtifact for all of the content (artifacts) in the repository.
+        checkpoint (models.BooleanField): Indicates a checkpoint publication.
 
     Relations:
         repository_version (models.ForeignKey): The RepositoryVersion used to
@@ -98,12 +99,13 @@ class Publication(MasterModel):
 
     complete = models.BooleanField(db_index=True, default=False)
     pass_through = models.BooleanField(default=False)
+    checkpoint = models.BooleanField(default=False, editable=False)
 
     repository_version = models.ForeignKey("RepositoryVersion", on_delete=models.CASCADE)
     pulp_domain = models.ForeignKey("Domain", default=get_domain_pk, on_delete=models.PROTECT)
 
     @classmethod
-    def create(cls, repository_version, pass_through=False):
+    def create(cls, repository_version, pass_through=False, checkpoint=False):
         """
         Create a publication.
 
@@ -125,7 +127,11 @@ def create(cls, repository_version, pass_through=False):
             Adds a Task.created_resource for the publication.
         """
         with transaction.atomic():
-            publication = cls(pass_through=pass_through, repository_version=repository_version)
+            publication = cls(
+                pass_through=pass_through,
+                repository_version=repository_version,
+                checkpoint=checkpoint,
+            )
             publication.save()
             resource = CreatedResource(content_object=publication)
             resource.save()
@@ -159,6 +165,10 @@ def delete(self, **kwargs):
             # It's possible for errors to occur before any publication has been completed,
             # so we need to handle the case when no Publication exists.
             try:
+                if self.checkpoint:
+                    base_paths |= Distribution.objects.filter(
+                        checkpoint=self.checkpoint, repository=self.repository_version.repository
+                    ).values_list("base_path", flat=True)
                 versions = self.repository.versions.all()
                 pubs = Publication.objects.filter(repository_version__in=versions, complete=True)
                 publication = pubs.latest("repository_version", "pulp_created")
@@ -629,6 +639,7 @@ class Distribution(MasterModel):
         pulp_labels (HStoreField): Dictionary of string values.
         base_path (models.TextField): The base (relative) path component of the published url.
         hidden (models.BooleanField): Whether this distribution should be hidden in the content app.
+        checkpoint (models.BooleanField): Whether this distribution serves checkpoint publications.
 
     Relations:
         content_guard (models.ForeignKey): An optional content-guard.
@@ -649,6 +660,7 @@ class Distribution(MasterModel):
     base_path = models.TextField()
     pulp_domain = models.ForeignKey("Domain", default=get_domain_pk, on_delete=models.PROTECT)
     hidden = models.BooleanField(default=False, null=True)
+    checkpoint = models.BooleanField(default=False)
 
     content_guard = models.ForeignKey(ContentGuard, null=True, on_delete=models.SET_NULL)
     publication = models.ForeignKey(Publication, null=True, on_delete=models.SET_NULL)
@@ -706,6 +718,7 @@ def content_headers_for(self, path):
             "remote",
             "repository",
             "repository_version",
+            "checkpoint",
         ],
         has_changed=True,
     )

pulpcore/app/models/repository.py

@@ -328,7 +328,15 @@ def protected_versions(self):
             publication__pk__in=Distribution.objects.values_list("publication_id")
         )
 
-        if distro := Distribution.objects.filter(repository=self.pk).first():
+        # Protect repo versions of distributed checkpoint publications.
+        if Distribution.objects.filter(repository=self.pk, checkpoint=True).exists():
+            qs |= self.versions.filter(
+                publication__pk__in=Publication.objects.filter(checkpoint=True).values_list(
+                    "pulp_id"
+                )
+            )
+
+        if distro := Distribution.objects.filter(repository=self.pk, checkpoint=False).first():
             if distro.detail_model().SERVE_FROM_PUBLICATION:
                 # if the distro serves publications, protect the latest published repo version
                 version = self.versions.filter(

pulpcore/app/serializers/publication.py

@@ -294,6 +294,8 @@ def validate(self, data):
             "publication", (self.partial and self.instance.publication) or None
         )
 
+        checkpoint = data.get("checkpoint", (self.partial and self.instance.checkpoint) or None)
+
         if publication_provided and repository_version_provided:
             raise serializers.ValidationError(
                 _(
@@ -316,6 +318,12 @@ def validate(self, data):
                     "may be used simultaneously."
                 )
             )
+        elif checkpoint and (
+            not repository_provided or publication_provided or repository_version_provided
+        ):
+            raise serializers.ValidationError(
+                _("The 'checkpoint' attribute may only be used with the 'repository' attribute.")
+            )
 
         return data
 

pulpcore/app/viewsets/publication.py

@@ -70,6 +70,7 @@ class Meta:
         model = Publication
         fields = {
             "pulp_created": DATETIME_FILTER_OPTIONS,
+            "checkpoint": ["exact"],
         }
 
 
@@ -497,6 +498,7 @@ class Meta:
             "name": NAME_FILTER_OPTIONS,
             "base_path": ["exact", "contains", "icontains", "in"],
             "repository": ["exact", "in"],
+            "checkpoint": ["exact"],
         }