Merge pull request #1310 from pierotofy/dropimageupload

Drop ImageUpload model

app/admin.py

@@ -16,7 +16,7 @@
 from app.models import Plugin
 from app.plugins import get_plugin_by_name, enable_plugin, disable_plugin, delete_plugin, valid_plugin, \
     get_plugins_persistent_path, clear_plugins_cache, init_plugins
-from .models import Project, Task, ImageUpload, Setting, Theme
+from .models import Project, Task, Setting, Theme
 from django import forms
 from codemirror2.widgets import CodeMirrorEditor
 from webodm import settings
@@ -37,12 +37,6 @@ def has_add_permission(self, request):
 
 admin.site.register(Task, TaskAdmin)
 
-
-class ImageUploadAdmin(admin.ModelAdmin):
-    readonly_fields = ('image',)
-
-admin.site.register(ImageUpload, ImageUploadAdmin)
-
 admin.site.register(Preset, admin.ModelAdmin)
 
 

app/api/imageuploads.py

@@ -4,7 +4,6 @@
 
 from .tasks import TaskNestedView
 from rest_framework import exceptions
-from app.models import ImageUpload
 from app.models.task import assets_directory_path
 from PIL import Image, ImageDraw, ImageOps
 from django.http import HttpResponse
@@ -33,12 +32,7 @@ def get(self, request, pk=None, project_pk=None, image_filename=""):
         Generate a thumbnail on the fly for a particular task's image
         """
         task = self.get_and_check_task(request, pk)
-        image = ImageUpload.objects.filter(task=task, image=assets_directory_path(task.id, task.project.id, image_filename)).first()
-
-        if image is None:
-            raise exceptions.NotFound()
-
-        image_path = image.path()
+        image_path = task.get_image_path(image_filename)
         if not os.path.isfile(image_path):
             raise exceptions.NotFound()
 
@@ -146,12 +140,7 @@ def get(self, request, pk=None, project_pk=None, image_filename=""):
         Download a task's image
         """
         task = self.get_and_check_task(request, pk)
-        image = ImageUpload.objects.filter(task=task, image=assets_directory_path(task.id, task.project.id, image_filename)).first()
-
-        if image is None:
-            raise exceptions.NotFound()
-
-        image_path = image.path()
+        image_path = task.get_image_path(image_filename)
         if not os.path.isfile(image_path):
             raise exceptions.NotFound()
 

app/api/tasks.py

@@ -179,7 +179,7 @@ def commit(self, request, pk=None, project_pk=None):
             raise exceptions.NotFound()
 
         task.partial = False
-        task.images_count = models.ImageUpload.objects.filter(task=task).count()
+        task.images_count = len(task.scan_images())
 
         if task.images_count < 2:
             raise exceptions.ValidationError(detail=_("You need to upload at least 2 images before commit"))
@@ -206,11 +206,8 @@ def upload(self, request, pk=None, project_pk=None):
         if len(files) == 0:
             raise exceptions.ValidationError(detail=_("No files uploaded"))
 
-        with transaction.atomic():
-            for image in files:
-                models.ImageUpload.objects.create(task=task, image=image)
-
-        task.images_count = models.ImageUpload.objects.filter(task=task).count()
+        task.handle_images_upload(files)
+        task.images_count = len(task.scan_images())
         # Update other parameters such as processing node, task name, etc.
         serializer = TaskSerializer(task, data=request.data, partial=True)
         serializer.is_valid(raise_exception=True)
@@ -256,9 +253,8 @@ def create(self, request, project_pk=None):
                 task = models.Task.objects.create(project=project,
                                                   pending_action=pending_actions.RESIZE if 'resize_to' in request.data else None)
 
-                for image in files:
-                    models.ImageUpload.objects.create(task=task, image=image)
-                task.images_count = len(files)
+                task.handle_images_upload(files)
+                task.images_count = len(task.scan_images())
 
                 # Update other parameters such as processing node, task name, etc.
                 serializer = TaskSerializer(task, data=request.data, partial=True)

app/migrations/0012_public_task_uuids.py

@@ -8,17 +8,14 @@
 from webodm import settings
 
 tasks = []
-imageuploads = []
 task_ids = {} # map old task IDs --> new task IDs
 
 def dump(apps, schema_editor):
-    global tasks, imageuploads, task_ids
+    global tasks, task_ids
 
     Task = apps.get_model('app', 'Task')
-    ImageUpload = apps.get_model('app', 'ImageUpload')
 
     tasks = list(Task.objects.all().values('id', 'project'))
-    imageuploads = list(ImageUpload.objects.all().values('id', 'task'))
 
     # Generate UUIDs
     for task in tasks:
@@ -31,9 +28,9 @@ def dump(apps, schema_editor):
         task_ids[task['id']] = new_id
 
     tmp_path = os.path.join(tempfile.gettempdir(), "public_task_uuids_migration.pickle")
-    pickle.dump((tasks, imageuploads, task_ids), open(tmp_path, 'wb'))
+    pickle.dump((tasks, task_ids), open(tmp_path, 'wb'))
 
-    if len(tasks) > 0: print("Dumped tasks and imageuploads")
+    if len(tasks) > 0: print("Dumped tasks")
 
 
 class Migration(migrations.Migration):

app/migrations/0013_public_task_uuids.py

@@ -8,7 +8,6 @@
 from webodm import settings
 
 tasks = []
-imageuploads = []
 task_ids = {} # map old task IDs --> new task IDs
 
 def task_path(project_id, task_id):
@@ -44,10 +43,10 @@ def create_uuids(apps, schema_editor):
 
 
 def restore(apps, schema_editor):
-    global tasks, imageuploads, task_ids
+    global tasks, task_ids
 
     tmp_path = os.path.join(tempfile.gettempdir(), "public_task_uuids_migration.pickle")
-    tasks, imageuploads, task_ids = pickle.load(open(tmp_path, 'rb'))
+    tasks, task_ids = pickle.load(open(tmp_path, 'rb'))
 
 
 class Migration(migrations.Migration):

app/migrations/0016_public_task_uuids.py

@@ -9,7 +9,7 @@
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('app', '0015_public_task_uuids'),
+        ('app', '0014_public_task_uuids'),
     ]
 
     operations = [

app/migrations/0026_update_images_count.py

@@ -10,7 +10,7 @@ def update_images_count(apps, schema_editor):
 
     for t in Task.objects.all():
         print("Updating {}".format(t))
-        t.images_count = t.imageupload_set.count()
+        t.images_count = len(t.scan_images())
         t.save()
 
 

app/migrations/0029_auto_20190907_1348.py

@@ -1,6 +1,6 @@
 # Generated by Django 2.1.11 on 2019-09-07 13:48
 
-import app.models.image_upload
+import app.models
 from django.db import migrations, models
 
 
@@ -14,6 +14,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='imageupload',
             name='image',
-            field=models.ImageField(help_text='File uploaded by a user', max_length=512, upload_to=app.models.image_upload.image_directory_path),
+            field=models.ImageField(help_text='File uploaded by a user', max_length=512, upload_to=app.models.image_directory_path),
         ),
     ]

app/migrations/0031_auto_20210610_1850.py

@@ -1,7 +1,7 @@
 # Generated by Django 2.1.15 on 2021-06-10 18:50
 
-import app.models.image_upload
 import app.models.task
+from app.models import image_directory_path
 import colorfield.fields
 from django.conf import settings
 import django.contrib.gis.db.models.fields
@@ -60,7 +60,7 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='imageupload',
             name='image',
-            field=models.ImageField(help_text='File uploaded by a user', max_length=512, upload_to=app.models.image_upload.image_directory_path, verbose_name='Image'),
+            field=models.ImageField(help_text='File uploaded by a user', max_length=512, upload_to=image_directory_path, verbose_name='Image'),
         ),
         migrations.AlterField(
             model_name='imageupload',

app/migrations/0034_delete_imageupload.py

@@ -0,0 +1,16 @@
+# Generated by Django 2.2.27 on 2023-03-23 17:10
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('app', '0033_auto_20230307_1532'),
+    ]
+
+    operations = [
+        migrations.DeleteModel(
+            name='ImageUpload',
+        ),
+    ]

app/models/__init__.py

@@ -1,4 +1,3 @@
-from .image_upload import ImageUpload, image_directory_path
 from .project import Project
 from .task import Task, validate_task_options, gcp_directory_path
 from .preset import Preset
@@ -7,3 +6,6 @@
 from .plugin_datum import PluginDatum
 from .plugin import Plugin
 
+# deprecated
+def image_directory_path(image_upload, filename):
+    raise Exception("Deprecated")

app/models/task.py

@@ -21,6 +21,7 @@
 from django.contrib.gis.gdal import OGRGeometry
 from django.contrib.gis.geos import GEOSGeometry
 from django.contrib.postgres import fields
+from django.core.files.uploadedfile import InMemoryUploadedFile
 from django.core.exceptions import ValidationError, SuspiciousFileOperation
 from django.db import models
 from django.db import transaction
@@ -310,15 +311,6 @@ def move_assets(self, old_project_id, new_project_id):
                 shutil.move(old_task_folder, new_task_folder_parent)
 
                 logger.info("Moved task folder from {} to {}".format(old_task_folder, new_task_folder))
-
-                with transaction.atomic():
-                    for img in self.imageupload_set.all():
-                        prev_name = img.image.name
-                        img.image.name = assets_directory_path(self.id, new_project_id,
-                                                               os.path.basename(img.image.name))
-                        logger.info("Changing {} to {}".format(prev_name, img))
-                        img.save()
-
             else:
                 logger.warning("Project changed for task {}, but either {} doesn't exist, or {} already exists. This doesn't look right, so we will not move any files.".format(self,
                                                                                                              old_task_folder,
@@ -430,16 +422,6 @@ def duplicate(self, set_new_name=True):
 
                 logger.info("Duplicating {} to {}".format(self, task))
 
-                for img in self.imageupload_set.all():
-                    img.pk = None
-                    img.task = task
-
-                    prev_name = img.image.name
-                    img.image.name = assets_directory_path(task.id, task.project.id,
-                                                            os.path.basename(img.image.name))
-
-                    img.save()
-
                 if os.path.isdir(self.task_path()):
                     try:
                         # Try to use hard links first
@@ -629,7 +611,8 @@ def process(self):
                 if not self.uuid and self.pending_action is None and self.status is None:
                     logger.info("Processing... {}".format(self))
 
-                    images = [image.path() for image in self.imageupload_set.all()]
+                    images_path = self.task_path()
+                    images = [os.path.join(images_path, i) for i in self.scan_images()]
 
                     # Track upload progress, but limit the number of DB updates
                     # to every 2 seconds (and always record the 100% progress)
@@ -1122,3 +1105,34 @@ def create_task_directories(self):
                 pass
             else:
                 raise
+
+    def scan_images(self):
+        tp = self.task_path()
+        try:
+            return [e.name for e in os.scandir(tp) if e.is_file()]
+        except:
+            return []
+
+    def get_image_path(self, filename):
+        p = self.task_path(filename)
+        return path_traversal_check(p, self.task_path())
+
+    def handle_images_upload(self, files):
+        for file in files:
+            name = file.name
+            if name is None:
+                continue
+
+            tp = self.task_path()
+            if not os.path.exists(tp):
+                os.makedirs(tp, exist_ok=True)
+
+            dst_path = self.get_image_path(name)
+
+            with open(dst_path, 'wb+') as fd:
+                if isinstance(file, InMemoryUploadedFile):
+                    for chunk in file.chunks():
+                        fd.write(chunk)
+                else:
+                    with open(file.temporary_file_path(), 'rb') as f:
+                        copyfileobj(f, fd)