Merge pull request #448 from i-dot-ai/feature/REDBOX-234-keep-deleted…

…-file-django-reference

Feature/redbox 234 keep deleted file django reference

django_app/redbox_app/redbox_core/admin.py

@@ -9,7 +9,7 @@ class UserResource(admin.ModelAdmin):
 
 
 class FileResource(admin.ModelAdmin):
-    list_display = ["original_file_name", "user", "processing_status"]
+    list_display = ["original_file_name", "user", "status"]
 
 
 class ChatMessageInline(admin.StackedInline):

django_app/redbox_app/redbox_core/migrations/0011_alter_file_processing_status.py

@@ -0,0 +1,33 @@
+# Generated by Django 5.0.6 on 2024-05-23 14:09
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("redbox_core", "0010_rename_expiry_date_file_last_referenced"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="file",
+            name="processing_status",
+            field=models.CharField(
+                choices=[
+                    ("uploaded", "Uploaded"),
+                    ("parsing", "Parsing"),
+                    ("chunking", "Chunking"),
+                    ("embedding", "Embedding"),
+                    ("indexing", "Indexing"),
+                    ("complete", "Complete"),
+                    ("unknown", "Unknown"),
+                    ("deleted", "Deleted"),
+                ]
+            ),
+        ),
+        migrations.RenameField(
+            model_name="file",
+            old_name="processing_status",
+            new_name="status",
+        ),
+    ]

django_app/redbox_app/redbox_core/models.py

@@ -50,18 +50,19 @@ def get_bearer_token(self) -> str:
         return f"Bearer {bearer_token}"
 
 
-class ProcessingStatusEnum(models.TextChoices):
+class StatusEnum(models.TextChoices):
     uploaded = "uploaded"
     parsing = "parsing"
     chunking = "chunking"
     embedding = "embedding"
     indexing = "indexing"
     complete = "complete"
     unknown = "unknown"
+    deleted = "deleted"
 
 
 class File(UUIDPrimaryKeyBase, TimeStampedModel):
-    processing_status = models.CharField(choices=ProcessingStatusEnum.choices, null=False, blank=False)
+    status = models.CharField(choices=StatusEnum.choices, null=False, blank=False)
     original_file = models.FileField(storage=settings.STORAGES["default"]["BACKEND"])
     user = models.ForeignKey(User, on_delete=models.CASCADE)
     original_file_name = models.TextField(max_length=2048, blank=True, null=True)
@@ -85,6 +86,10 @@ def delete(self, using=None, keep_parents=False):  # noqa: ARG002  # remove at P
         self.original_file.storage.delete(self.original_file.name)
         super().delete()
 
+    def delete_from_s3(self):
+        """Manually deletes the file from S3 storage."""
+        self.original_file.delete(save=False)
+
     @property
     def file_type(self) -> str:
         name = self.original_file.name
@@ -124,9 +129,9 @@ def unique_name(self) -> str:
         # Name used by core-api
         return self.original_file.file.name
 
-    def get_processing_status_text(self) -> str:
+    def get_status_text(self) -> str:
         return next(
-            (status[1] for status in ProcessingStatusEnum.choices if self.processing_status == status[0]),
+            (status[1] for status in StatusEnum.choices if self.status == status[0]),
             "Unknown",
         )
 

django_app/redbox_app/redbox_core/views.py

@@ -18,7 +18,7 @@
     ChatMessage,
     ChatRoleEnum,
     File,
-    ProcessingStatusEnum,
+    StatusEnum,
     User,
 )
 from requests.exceptions import HTTPError
@@ -118,7 +118,7 @@ def ingest_file(uploaded_file: UploadedFile, user: User) -> list[str]:
     errors: list[str] = []
     try:
         file = File.objects.create(
-            processing_status=ProcessingStatusEnum.uploaded.value,
+            status=StatusEnum.uploaded.value,
             user=user,
             original_file=uploaded_file,
             original_file_name=uploaded_file.name,
@@ -154,7 +154,9 @@ def remove_doc_view(request, doc_id: uuid):
 
         else:
             logger.info("Removing document: %s", request.POST["doc_id"])
-            file.delete()
+            file.delete_from_s3()
+            file.status = StatusEnum.deleted
+            file.save()
             return redirect("documents")
 
     return render(
@@ -229,23 +231,23 @@ def file_status_api_view(request: HttpRequest) -> JsonResponse:
     file_id = request.GET.get("id", None)
     if not file_id:
         logger.error("Error getting file object information - no file ID provided %s.")
-        return JsonResponse({"status": ProcessingStatusEnum.unknown.label})
+        return JsonResponse({"status": StatusEnum.unknown.label})
     try:
         file = File.objects.get(pk=file_id)
     except File.DoesNotExist as ex:
         logger.exception("File object information not found in django - file does not exist %s.", file_id, exc_info=ex)
-        return JsonResponse({"status": ProcessingStatusEnum.unknown.label})
+        return JsonResponse({"status": StatusEnum.unknown.label})
     try:
         core_file_status_response = core_api.get_file_status(file_id=file.core_file_uuid, user=request.user)
     except HTTPError as ex:
         logger.exception("File object information from core not found - file does not exist %s.", file_id, exc_info=ex)
-        if not file.processing_status:
-            file.processing_status = ProcessingStatusEnum.unknown.label
+        if not file.status:
+            file.status = StatusEnum.unknown.label
             file.save()
-        return JsonResponse({"status": file.processing_status})
-    file.processing_status = core_file_status_response.processing_status
+        return JsonResponse({"status": file.status})
+    file.status = core_file_status_response.processing_status
     file.save()
-    return JsonResponse({"status": file.get_processing_status_text()})
+    return JsonResponse({"status": file.get_status_text()})
 
 
 @require_http_methods(["GET"])

django_app/redbox_app/templates/documents.html

@@ -44,19 +44,23 @@ <h1 class="govuk-heading-l govuk-!-margin-bottom-0">Documents</h1>
                 <div class="govuk-summary-list__row">
                   <dt class="govuk-summary-list__key">Status</dt>
                   <dd class="govuk-summary-list__value">
-                    {% if file.get_processing_status_text() | lower == 'complete' %}
-                      <strong class="govuk-tag govuk-tag--green iai-docs__status">{{ file.get_processing_status_text() }}</strong>
+                    {% if file.get_status_text() | lower == 'complete' %}
+                      <strong class="govuk-tag govuk-tag--green iai-docs__status">{{ file.get_status_text() }}</strong>
+                    {% elif file.get_status_text() | lower == 'deleted' %}
+                      <strong class="govuk-tag govuk-tag--red iai-docs__status">{{ file.get_status_text() }}</strong>
                     {% else %}
-                      <strong class="govuk-tag govuk-tag--yellow iai-docs__status" x-data="file-status" data-id="{{file.id}}" x-text="status">{{ file.get_processing_status_text() }}</strong>
+                      <strong class="govuk-tag govuk-tag--yellow iai-docs__status" x-data="file-status" data-id="{{file.id}}" x-text="status">{{ file.get_status_text() }}</strong>
                     {% endif %}
                   </dd>
                 </div>
               </dl>
-              {{ govukButton(
-                text="Remove" + "<span class=\"govuk-visually-hidden\">" + file.name + "</span>",
-                href=url('remove-doc', file.id),
-                classes="govuk-button--warning govuk-!-margin-bottom-0"
-              ) }}
+              {% if file.get_status_text() | lower != 'deleted' %}
+                {{ govukButton(
+                  text="Remove" + "<span class=\"govuk-visually-hidden\">" + file.name + "</span>",
+                  href=url('remove-doc', file.id),
+                  classes="govuk-button--warning govuk-!-margin-bottom-0"
+                ) }}
+              {% endif %}
             </li>
           {% endfor %}
         </ul>
@@ -81,26 +85,30 @@ <h1 class="govuk-heading-l govuk-!-margin-bottom-0">Documents</h1>
               <td class="govuk-table__cell govuk-!-padding-top-3">{{ file.name }}</td>
               <td class="govuk-table__cell govuk-!-padding-top-3">{{ file.created_at | to_user_timezone }}</td>
               <td class="govuk-table__cell govuk-!-padding-top-3">
-                {% if file.get_processing_status_text() | lower == 'complete' %}
-                  <strong class="govuk-tag govuk-tag--green iai-docs__status govuk-!-margin-top-0">{{ file.get_processing_status_text() }}</strong>
+                {% if file.get_status_text() | lower == 'complete' %}
+                  <strong class="govuk-tag govuk-tag--green iai-docs__status govuk-!-margin-top-0">{{ file.get_status_text() }}</strong>
+                {% elif file.get_status_text() | lower == 'deleted' %}
+                  <strong class="govuk-tag govuk-tag--red iai-docs__status govuk-!-margin-top-0">{{file.get_status_text()}}</strong>
                 {% else %}
-                  <strong class="govuk-tag govuk-tag--yellow iai-docs__status govuk-!-margin-top-0" x-data="file-status" data-id="{{file.id}}" x-text="status">{{ file.get_processing_status_text() }}</strong>
+                  <strong class="govuk-tag govuk-tag--yellow iai-docs__status govuk-!-margin-top-0" x-data="file-status" data-id="{{file.id}}" x-text="status">{{ file.get_status_text() }}</strong>
                 {% endif %}
               </td>
               <td class="govuk-table__cell iai-docs__actions">
-                {# Removed for now until we've validated a user need for this
-                {{ govukButton(
-                  text="Download" + "<span class=\"govuk-visually-hidden\">" + file.name + "</span>",
-                  href=file.url,
-                  classes="govuk-button--secondary govuk-!-margin-bottom-0",
-                  download=True
-                ) }}
-                #}
-                {{ govukButton(
-                  text="Remove" + "<span class=\"govuk-visually-hidden\">" + file.name + "</span>",
-                  href=url('remove-doc', file.id),
-                  classes="govuk-button--warning govuk-!-margin-bottom-0"
-                ) }}
+                {% if file.get_status_text() | lower != 'deleted' %}
+                  {# Removed for now until we've validated a user need for this
+                  {{ govukButton(
+                    text="Download" + "<span class=\"govuk-visually-hidden\">" + file.name + "</span>",
+                    href=file.url,
+                    classes="govuk-button--secondary govuk-!-margin-bottom-0",
+                    download=True
+                  ) }}
+                  #}
+                  {{ govukButton(
+                    text="Remove" + "<span class=\"govuk-visually-hidden\">" + file.name + "</span>",
+                    href=url('remove-doc', file.id),
+                    classes="govuk-button--warning govuk-!-margin-bottom-0"
+                  ) }}
+                {% endif %}
               </td>
             </tr>
           {% endfor %}

django_app/tests/test_models.py

@@ -2,15 +2,15 @@
 
 import pytest
 from django.core.files.uploadedfile import SimpleUploadedFile
-from redbox_app.redbox_core.models import File, ProcessingStatusEnum
+from redbox_app.redbox_core.models import File, StatusEnum
 
 
 @pytest.mark.django_db()
 def test_file_model_last_referenced(peter_rabbit, s3_client):  # noqa: ARG001
     mock_file = SimpleUploadedFile("test.txt", b"these are the file contents")
 
     new_file = File.objects.create(
-        processing_status=ProcessingStatusEnum.uploaded,
+        status=StatusEnum.uploaded,
         original_file=mock_file,
         user=peter_rabbit,
         original_file_name="test.txt",

django_app/tests/test_views.py

@@ -12,7 +12,7 @@
     ChatMessage,
     ChatRoleEnum,
     File,
-    ProcessingStatusEnum,
+    StatusEnum,
     User,
 )
 from requests_mock import Mocker
@@ -112,7 +112,7 @@ def test_document_upload_status(client, alice, file_pdf_path: Path, s3_client, r
         assert response.url == "/documents/"
         assert count_s3_objects(s3_client) == previous_count + 1
         uploaded_file = File.objects.filter(user=alice).order_by("-created_at")[0]
-        assert uploaded_file.processing_status == ProcessingStatusEnum.uploaded
+        assert uploaded_file.status == StatusEnum.uploaded
 
 
 @pytest.mark.django_db()
@@ -215,9 +215,7 @@ def test_remove_doc_view(client: Client, alice: User, file_pdf_path: Path, s3_cl
         assert not file_exists(s3_client, file_name)
         assert count_s3_objects(s3_client) == previous_count
         assert requests_mock.request_history[-1].method == "DELETE"
-
-        with pytest.raises(File.DoesNotExist):
-            File.objects.get(id=new_file.id)
+        assert File.objects.get(id=new_file.id).status == StatusEnum.deleted
 
 
 @pytest.mark.django_db()