Merge pull request #290 from gnes-ai/frame_selector_

feat(preprocessor): add frame selector

gnes/preprocessor/__init__.py

@@ -38,7 +38,8 @@
     'RawChunkPreprocessor': 'base',
     'GifChunkPreprocessor': 'video.ffmpeg',
     'VggishPreprocessor': 'audio.vggish_example',
-    'VideoDecodePreprocessor': 'video.video_decode'
+    'VideoDecodePreprocessor': 'video.video_decode',
+    'FrameSelectPreprocessor': 'video.frame_select'
 }
 
 register_all_class(_cls2file_map, 'preprocessor')

gnes/preprocessor/video/frame_select.py

@@ -0,0 +1,51 @@
+#  Tencent is pleased to support the open source community by making GNES available.
+#
+#  Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import numpy as np
+import math
+
+from gnes.preprocessor.base import BaseVideoPreprocessor
+from gnes.proto import gnes_pb2, array2blob, blob2array
+
+
+class FrameSelectPreprocessor(BaseVideoPreprocessor):
+
+    def __init__(self,
+                 sframes: int = 1,
+                 *args,
+                 **kwargs):
+        super().__init__(*args, **kwargs)
+        self.sframes = sframes
+
+    def apply(self, doc: 'gnes_pb2.Document') -> None:
+        super().apply(doc)
+        if len(doc.chunks) > 0:
+            for chunk in doc.chunks:
+                images = blob2array(chunk.blob)
+                if len(images) == 0:
+                    self.logger.warning("this chunk has no frame!")
+                elif self.sframes == 1:
+                    idx = [int(len(images) / 2)]
+                    chunk.blob.CopyFrom(array2blob(images[idx]))
+                elif self.sframes > 0 and len(images) > self.sframes:
+                    if len(images) >= 2 * self.sframes:
+                        step = math.ceil(len(images) / self.sframes)
+                        chunk.blob.CopyFrom(array2blob(images[::step]))
+                    else:
+                        idx = np.sort(np.random.choice(len(images), self.sframes, replace=False))
+                        chunk.blob.CopyFrom(array2blob(images[idx]))
+        else:
+            self.logger.error(
+                'bad document: "doc.chunks" is empty!')

gnes/preprocessor/video/shotdetect.py

@@ -14,6 +14,7 @@
 #  limitations under the License.
 
 import numpy as np
+import math
 from typing import List
 
 from gnes.preprocessor.base import BaseVideoPreprocessor
@@ -99,11 +100,12 @@ def apply(self, doc: 'gnes_pb2.Document') -> None:
                     shot_len = len(frames)
                     c.weight = shot_len / num_frames
                     if self.sframes > 0 and shot_len > self.sframes:
-                        begin = 0
-                        if self.sframes < 3:
-                            begin = (shot_len - self.sframes) // 2
-                        step = (shot_len) // self.sframes
-                        frames = [frames[_] for _ in range(begin, shot_len, step)]
+                        if shot_len >= 2 * self.sframes:
+                            step = math.ceil(shot_len / self.sframes)
+                            frames = frames[::step]
+                        else:
+                            idx = np.sort(np.random.choice(shot_len, self.sframes, replace=False))
+                            frames = [frames[idx_] for idx_ in idx]
 
                     chunk_data = np.array(frames)
                     c.blob.CopyFrom(array2blob(chunk_data))

tests/test_frame_selector.py

@@ -0,0 +1,49 @@
+import unittest
+from gnes.proto import gnes_pb2, array2blob, blob2array
+from gnes.preprocessor.video.frame_select import FrameSelectPreprocessor
+import numpy as np
+import copy
+
+
+class TestFrameSelector(unittest.TestCase):
+    def setUp(self) -> None:
+        self.doc = gnes_pb2.Document()
+
+        c1 = self.doc.chunks.add()
+        c1.blob.CopyFrom(array2blob(np.array([[1,2,3], [2,3,4]])))
+
+        c2 = self.doc.chunks.add()
+        c2.blob.CopyFrom(array2blob(np.array([[1,2,3], [2,3,4], [1,2,3]])))
+
+        c3 = self.doc.chunks.add()
+        c3.blob.CopyFrom(array2blob(np.array([[1,2,3], [2,3,4], [1,2,3], [2,3,4]])))
+
+    def test_emtpy_document(self):
+        frame_selector = FrameSelectPreprocessor(sframes=-1)
+        frame_selector.apply(gnes_pb2.Document())
+
+    def test_big_sframe(self):
+        doc = copy.deepcopy(self.doc)
+        frame_selector = FrameSelectPreprocessor(sframes=100)
+        frame_selector.apply(doc)
+
+    def test_get_frames(self):
+        doc = copy.deepcopy(self.doc)
+        frame_selector = FrameSelectPreprocessor(sframes=3)
+        frame_selector.apply(doc)
+        for idx, chunk in enumerate(doc.chunks):
+            if idx == 0:
+                self.assertEqual(blob2array(chunk.blob).shape[0], 2)
+            else:
+                self.assertEqual(blob2array(chunk.blob).shape[0], 3)
+
+    def test_get_one_frame(self):
+        doc = copy.deepcopy(self.doc)
+        frame_selector = FrameSelectPreprocessor(sframes=1)
+        frame_selector.apply(doc)
+        for chunk in doc.chunks:
+            self.assertEqual(blob2array(chunk.blob).shape[0], 1)
+
+
+
+