Feature/document status check

django_app/frontend/js/documents.js

@@ -15,15 +15,15 @@ Alpine.data('file-status', () => ({
         const checkStatus = async () => {
 
             // UPDATE THESE AS REQUIRED
-            const FILE_STATUS_ENDPOINT = '/api/get-status'; 
+            const FILE_STATUS_ENDPOINT = '/file-status';
             const CHECK_INTERVAL_MS = 5000;
 
             const response = await fetch(`${FILE_STATUS_ENDPOINT}?id=${this.$el.dataset.id}`);
-            const responseText = await response.text();
+            const responseText = await response.json();
             if (response.ok) {
-                this.status = responseText;
+                this.status = responseText["status"];
             }
-            if (responseText !== 'complete') {
+            if (this.status !== 'complete') {
                 window.setTimeout(checkStatus, CHECK_INTERVAL_MS);
             }
 

django_app/redbox_app/redbox_core/client.py

@@ -67,3 +67,10 @@ def rag_chat(self, message_history: list[dict[str, str]], token: str) -> SimpleN
         logger.debug("response_data: %s", response_data)
 
         return response_data
+
+    def get_file_status(self, file_id: uuid, token: str) -> SimpleNamespace:
+        url = f"{self.url}/file/{file_id}/status"
+        response = requests.get(url, headers={"Authorization": token}, timeout=60)
+        response.raise_for_status()
+        response_data = response.json(object_hook=lambda d: SimpleNamespace(**d))
+        return response_data

django_app/redbox_app/redbox_core/migrations/0007_alter_file_processing_status.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0.6 on 2024-05-15 15:20
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('redbox_core', '0006_file_core_file_uuid'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='file',
+            name='processing_status',
+            field=models.CharField(choices=[('uploaded', 'Uploaded'), ('parsing', 'Parsing'), ('chunking', 'Chunking'), ('embedding', 'Embedding'), ('indexing', 'Indexing'), ('complete', 'Complete'), ('unknown', 'Unknown')]),
+        ),
+    ]

django_app/redbox_app/redbox_core/models.py

@@ -52,6 +52,7 @@ class ProcessingStatusEnum(models.TextChoices):
     embedding = "embedding"
     indexing = "indexing"
     complete = "complete"
+    unknown = "unknown"
 
 
 class File(UUIDPrimaryKeyBase, TimeStampedModel):

django_app/redbox_app/redbox_core/views.py

@@ -1,20 +1,28 @@
 import logging
 import os
 import uuid
-from urllib.error import HTTPError
 
 from django.conf import settings
 from django.contrib.auth.decorators import login_required
-from django.core.exceptions import FieldError, ValidationError
+from django.core.exceptions import FieldError, ObjectDoesNotExist, ValidationError
 from django.core.files.uploadedfile import UploadedFile
-from django.http import HttpRequest, HttpResponse
+from django.http import HttpRequest, HttpResponse, JsonResponse
 from django.shortcuts import redirect, render
 from django.urls import reverse
 from django.views.decorators.http import require_http_methods
-from redbox_app.redbox_core.client import CoreApiClient
-from redbox_app.redbox_core.models import ChatHistory, ChatMessage, ChatRoleEnum, File, ProcessingStatusEnum, User
+from requests.exceptions import HTTPError
 from yarl import URL
 
+from redbox_app.redbox_core.client import CoreApiClient
+from redbox_app.redbox_core.models import (
+    ChatHistory,
+    ChatMessage,
+    ChatRoleEnum,
+    File,
+    ProcessingStatusEnum,
+    User,
+)
+
 logger = logging.getLogger(__name__)
 
 CHUNK_SIZE = 1024
@@ -92,7 +100,7 @@ def upload_view(request):
             errors.append(f"File type {file_extension} not supported")
 
         if not errors:
-            errors += injest_file(uploaded_file, request.user)
+            errors += ingest_file(uploaded_file, request.user)
 
         if not errors:
             return redirect(documents_view)
@@ -104,7 +112,7 @@ def upload_view(request):
     )
 
 
-def injest_file(uploaded_file: UploadedFile, user: User) -> list[str]:
+def ingest_file(uploaded_file: UploadedFile, user: User) -> list[str]:
     errors: list[str] = []
     api = CoreApiClient(host=settings.CORE_API_HOST, port=settings.CORE_API_PORT)
     try:
@@ -130,6 +138,7 @@ def injest_file(uploaded_file: UploadedFile, user: User) -> list[str]:
             file.save()
     return errors
 
+
 @login_required
 def remove_doc_view(request, doc_id: uuid):
     file = File.objects.get(pk=doc_id)
@@ -200,6 +209,33 @@ def post_message(request: HttpRequest) -> HttpResponse:
     return redirect(reverse(sessions_view, args=(session.id,)))
 
 
+@require_http_methods(["GET"])
+@login_required
+def file_status_api_view(request: HttpRequest) -> JsonResponse:
+    file_id = request.GET.get("id", None)
+    if not file_id:
+        logger.error("Error getting file object information - no file ID provided %s.")
+        return JsonResponse({"status": ProcessingStatusEnum.unknown.label})
+    try:
+        file = File.objects.get(pk=file_id)
+    except ObjectDoesNotExist as ex:
+        logger.error("File object information not found in django - file does not exist %s.", file_id, exc_info=ex)
+        return JsonResponse({"status": ProcessingStatusEnum.unknown.label})
+    core_api = CoreApiClient(host=settings.CORE_API_HOST, port=settings.CORE_API_PORT)
+    token = request.user.get_bearer_token()
+    try:
+        core_file_status_response = core_api.get_file_status(file_id=file.core_file_uuid, token=token)
+    except HTTPError as ex:
+        logger.error("File object information from core not found - file does not exist %s.", file_id, exc_info=ex)
+        if not file.processing_status:
+            file.processing_status = ProcessingStatusEnum.unknown.label
+            file.save()
+        return JsonResponse({"status": file.processing_status})
+    file.processing_status = core_file_status_response.processing_status
+    file.save()
+    return JsonResponse({"status": file.get_processing_status_text()})
+
+
 @require_http_methods(["GET"])
 def health(_request: HttpRequest) -> HttpResponse:
     """this required by ECS Fargate"""

django_app/redbox_app/urls.py

@@ -47,6 +47,7 @@
 other_urlpatterns = [
     path("", views.homepage_view, name="homepage"),
     path("health/", views.health, name="health"),
+    path("file-status/", views.file_status_api_view, name="file-status"),
 ]
 
 urlpatterns = (