Support for extracting a series of video thumbnails (#146)

* [WIP] BROKEN!!! (VideoFiltersFragment test bed) Initial WIP via mediacodec

* WIP thumbnail extraction

* [WIP] Broken - use custom MediaCodec extraction

* Use MediaMetadataRetriever for frame extraction

* Cleanup

* Extract interface from renderer

* Add comments

* Comments, refactor

* Add notices

* Do ByteBuffer initialization once

* Remove two-pass extraction method

* Extract 1 frame per request
Add support for priority
Simplify API
Update demo to use a RecyclerView, and in-memory cache
Behavior now only responsible for retrieving the frame at original resolution

* Add experimental annotations for thumbnails APIs

* Add comments

* Add view comments

* Add compression for the in-memory cache in demo

* Refactor: Rename file to PriorityExecutorUtil.kt

* [PR] Use const val for tag
[PR] Fix newlines for arguments

* FIXUP: Add missing import

* [PR] Refactor: "thumbnails" becomes "frames" in the lib to be less prescriptive about how this should be used
Remove unused imports
Add OptIn annotations to demo classes using the experimental frame extract feature

* [PR] Make several classes internal
[PR] Add experimental annotation for SingleFrameRenderer

* [PR] Add copyright notice to annotations

* Revert kotlin version bump

litr-demo/build.gradle

@@ -31,6 +31,10 @@ android {
     buildFeatures {
         dataBinding true
     }
+
+    kotlinOptions {
+        freeCompilerArgs += "-Xopt-in=kotlin.RequiresOptIn"
+    }
 }
 
 dependencies {

litr-demo/src/main/java/com/linkedin/android/litr/demo/DemoCase.java

@@ -21,7 +21,8 @@ public enum DemoCase {
     VIDEO_FILTERS(R.string.demo_case_video_filters, "VideoFilters", new VideoFiltersFragment()),
     VIDEO_FILTERS_PREVIEW(R.string.demo_case_video_filters_preview, "VideoFiltersPreview", new VideoFilterPreviewFragment()),
     TRANSCODE_VIDEO_MOCK(R.string.demo_case_mock_transcode_video, "TranscodeVideoMock", new MockTranscodeFragment()),
-    TRANSCODE_AUDIO(R.string.demo_case_transcode_audio, "TranscodeAudio", new TranscodeAudioFragment());
+    TRANSCODE_AUDIO(R.string.demo_case_transcode_audio, "TranscodeAudio", new TranscodeAudioFragment()),
+    EXTRACT_FRAMES(R.string.demo_case_extract_frames, "ExtractFramesFragment", new ExtractFramesFragment());
 
     @StringRes int displayName;
     String fragmentTag;

litr-demo/src/main/java/com/linkedin/android/litr/demo/ExtractFramesFragment.kt

@@ -0,0 +1,157 @@
+/*
+ * Copyright 2021 LinkedIn Corporation
+ * All Rights Reserved.
+ *
+ * Licensed under the BSD 2-Clause License (the "License").  See License in the project root for
+ * license information.
+ */
+package com.linkedin.android.litr.demo
+
+import android.R
+import android.graphics.Bitmap
+import android.graphics.Point
+import android.net.Uri
+import android.os.Bundle
+import android.util.LruCache
+import android.view.LayoutInflater
+import android.view.View
+import android.view.ViewGroup
+import android.widget.AdapterView
+import android.widget.AdapterView.OnItemSelectedListener
+import android.widget.ArrayAdapter
+import androidx.recyclerview.widget.LinearLayoutManager
+import androidx.recyclerview.widget.RecyclerView
+import com.linkedin.android.litr.ExperimentalFrameExtractorApi
+import com.linkedin.android.litr.demo.data.SourceMedia
+import com.linkedin.android.litr.demo.databinding.FragmentExtractFramesBinding
+import com.linkedin.android.litr.filter.GlFilter
+import com.linkedin.android.litr.render.GlSingleFrameRenderer
+import com.linkedin.android.litr.frameextract.FrameExtractMode
+import com.linkedin.android.litr.frameextract.FrameExtractListener
+import com.linkedin.android.litr.frameextract.FrameExtractParameters
+import com.linkedin.android.litr.frameextract.VideoFrameExtractor
+import java.io.ByteArrayOutputStream
+import java.util.*
+
+@OptIn(ExperimentalFrameExtractorApi::class)
+class ExtractFramesFragment : BaseTransformationFragment(), MediaPickerListener {
+    private lateinit var binding: FragmentExtractFramesBinding
+    private lateinit var filtersAdapter: ArrayAdapter<DemoFilter>
+    private lateinit var frameExtractor: VideoFrameExtractor
+    private lateinit var framesAdapter: ExtractedFramesAdapter
+    private lateinit var bitmapInMemoryCache: LruCache<Long, ByteArray>
+
+    override fun onCreate(savedInstanceState: Bundle?) {
+        super.onCreate(savedInstanceState)
+        val cacheSize = (Runtime.getRuntime().maxMemory() / 1024L) / 8L
+        bitmapInMemoryCache = object : LruCache<Long, ByteArray>(cacheSize.toInt()) {
+            override fun sizeOf(key: Long, byteArray: ByteArray): Int {
+                return byteArray.size / 1024
+            }
+        }
+
+        filtersAdapter = ArrayAdapter(requireContext(), R.layout.simple_spinner_item, DemoFilter.values()).apply {
+            setDropDownViewResource(R.layout.simple_spinner_dropdown_item)
+        }
+
+        frameExtractor = VideoFrameExtractor(requireContext())
+        framesAdapter = ExtractedFramesAdapter(frameExtractor, bitmapInMemoryCache)
+    }
+
+    override fun onDestroy() {
+        super.onDestroy()
+        frameExtractor.release()
+    }
+
+    override fun onCreateView(
+        inflater: LayoutInflater,
+        container: ViewGroup?, savedInstanceState: Bundle?
+    ): View {
+        binding = FragmentExtractFramesBinding.inflate(inflater, container, false).also { binding ->
+            binding.fragment = this@ExtractFramesFragment
+            binding.sourceMedia = SourceMedia()
+            binding.sectionPickVideo.buttonPickVideo.setOnClickListener { pickVideo(this@ExtractFramesFragment) }
+            binding.spinnerFilters.adapter = filtersAdapter
+            binding.spinnerFilters.onItemSelectedListener = object : OnItemSelectedListener {
+                override fun onItemSelected(parent: AdapterView<*>?, view: View?, position: Int, id: Long) {
+                    filtersAdapter.getItem(position)?.filter?.let {
+                        binding.filter = it
+                    }
+                }
+
+                override fun onNothingSelected(parent: AdapterView<*>?) {}
+            }
+        }
+        binding.framesRecycler.layoutManager = LinearLayoutManager(requireContext(), RecyclerView.HORIZONTAL, false)
+        binding.framesRecycler.adapter = framesAdapter
+        return binding.root
+    }
+
+    override fun onDestroyView() {
+        binding.framesRecycler.adapter = null
+        bitmapInMemoryCache.evictAll()
+        super.onDestroyView()
+    }
+
+    fun extractThumbnails(sourceMedia: SourceMedia, filter: GlFilter?) {
+        frameExtractor.stopAll()
+        bitmapInMemoryCache.evictAll()
+
+        val thumbCount = 50
+
+        val videoDurationSec = sourceMedia.duration.toDouble()
+        val thumbHeight = binding.framesRecycler.height
+
+        if (videoDurationSec <= 0 || thumbHeight <= 0) {
+            return
+        }
+
+        // Based on the video duration and number of thumbnails, obtain the list of timestamps (in microseconds),
+        // of each frame we should extract.
+        val secPerThumbnail = videoDurationSec / thumbCount
+        val timestamps = (0 until thumbCount).map {
+            (secPerThumbnail * 1000000.0).toLong() * it
+        }
+
+        // Because we want to apply the same filter to each frame, the renderer may be shared between all thumbnail requests.
+        val renderer = GlSingleFrameRenderer(filter?.let { listOf(it) })
+
+        // From each timestamp, construct the parameters to send to the thumbnail extractor.
+        val params = timestamps.map {
+            FrameExtractParameters(
+                sourceMedia.uri,
+                it,
+                renderer,
+                FrameExtractMode.Fast,
+                Point(thumbHeight, thumbHeight),
+                0L
+            )
+        }
+
+        // Set the list of extraction params to the adapter. Note that frame extraction will only start when the adapter binds an item to a view.
+        framesAdapter.loadData(params)
+
+        // What follows is an optimization.
+        // We request all the thumbnails with a low priority, and cache the resulting bitmaps in an in-memory cache.
+        // Because of the lower priority, these frames will only load when the RecyclerView adapter is not requesting frames at a higher priority.
+        // In a production application, an on-disk bitmap cache may be preferred.
+        params.forEach {
+            frameExtractor.extract(UUID.randomUUID().toString(), it.copy(priority = 100L), object: FrameExtractListener {
+                override fun onExtracted(id: String, timestampUs: Long, bitmap: Bitmap) {
+                    // Compress bitmap
+                    val baos = ByteArrayOutputStream()
+                    bitmap.compress(Bitmap.CompressFormat.JPEG, 90, baos)
+                    val byteArray = baos.toByteArray()
+                    // Add compressed bytes to in-memory cache
+                    bitmapInMemoryCache.put(timestampUs, byteArray)
+                }
+            })
+        }
+    }
+
+    override fun onMediaPicked(uri: Uri) {
+        binding.sourceMedia?.let {
+            updateSourceMedia(it, uri)
+        }
+    }
+}

litr-demo/src/main/java/com/linkedin/android/litr/demo/ExtractedFramesAdapter.kt

@@ -0,0 +1,105 @@
+/*
+ * Copyright 2021 LinkedIn Corporation
+ * All Rights Reserved.
+ *
+ * Licensed under the BSD 2-Clause License (the "License").  See License in the project root for
+ * license information.
+ */
+package com.linkedin.android.litr.demo
+
+import android.graphics.Bitmap
+import android.graphics.BitmapFactory
+import android.graphics.Color
+import android.util.LruCache
+import android.view.LayoutInflater
+import android.view.View
+import android.view.ViewGroup
+import android.widget.ImageView
+import androidx.recyclerview.widget.RecyclerView
+import com.linkedin.android.litr.ExperimentalFrameExtractorApi
+import com.linkedin.android.litr.frameextract.FrameExtractListener
+import com.linkedin.android.litr.frameextract.FrameExtractParameters
+import com.linkedin.android.litr.frameextract.VideoFrameExtractor
+import java.util.*
+
+@OptIn(ExperimentalFrameExtractorApi::class)
+class ExtractedFramesAdapter(private val extractor: VideoFrameExtractor, private val cache: LruCache<Long, ByteArray>) :
+        RecyclerView.Adapter<ExtractedFramesAdapter.FrameViewHolder>() {
+
+    private val frames = mutableListOf<FrameExtractParameters>()
+
+    fun loadData(frames: List<FrameExtractParameters>) {
+        this.frames.clear()
+        this.frames.addAll(frames)
+        notifyDataSetChanged()
+    }
+
+    override fun onCreateViewHolder(parent: ViewGroup, viewType: Int): FrameViewHolder {
+        val view = LayoutInflater.from(parent.context)
+                .inflate(R.layout.item_frame, parent, false)
+
+        return FrameViewHolder(view, null)
+    }
+
+    override fun onBindViewHolder(holder: FrameViewHolder, position: Int) {
+
+        val frameParams = frames[position]
+
+        val requestId = UUID.randomUUID().toString()
+
+        holder.requestId = requestId
+        holder.imageView.visibility = View.GONE
+        holder.indicator.setBackgroundColor(Color.GRAY)
+
+        val cachedByteArray = cache.get(frameParams.timestampUs)
+        if (cachedByteArray != null) {
+            // Cache hit: just use the cached bitmap.
+            val cachedBitmap = BitmapFactory.decodeByteArray(cachedByteArray, 0, cachedByteArray.size)
+            holder.imageView.setImageBitmap(cachedBitmap)
+            holder.imageView.visibility = View.VISIBLE
+            holder.indicator.setBackgroundColor(Color.BLACK)
+        } else {
+            // Cache miss: start the frame extraction job.
+            extractor.extract(
+                    requestId,
+                    frameParams,
+                    object : FrameExtractListener {
+                        override fun onStarted(id: String, timestampUs: Long) {
+                            holder.indicator.setBackgroundColor(Color.BLUE)
+                        }
+
+                        override fun onExtracted(id: String, timestampUs: Long, bitmap: Bitmap) {
+                            holder.imageView.setImageBitmap(bitmap)
+                            holder.imageView.visibility = View.VISIBLE
+                            holder.indicator.setBackgroundColor(Color.GREEN)
+                        }
+
+                        override fun onCancelled(id: String, timestampUs: Long) {
+                            holder.indicator.setBackgroundColor(Color.YELLOW)
+                        }
+
+                        override fun onError(id: String, timestampUs: Long, cause: Throwable?) {
+                            holder.indicator.setBackgroundColor(Color.RED)
+
+                        }
+                    })
+        }
+    }
+
+    override fun onViewRecycled(holder: FrameViewHolder) {
+        holder.requestId?.let {
+            extractor.stop(it)
+        }
+        holder.indicator.setBackgroundColor(Color.MAGENTA)
+        super.onViewRecycled(holder)
+    }
+
+    override fun getItemCount(): Int {
+        return frames.size
+    }
+
+    class FrameViewHolder(itemView: View, var requestId: String?) : RecyclerView.ViewHolder(itemView) {
+        val imageView: ImageView = this.itemView.findViewById(R.id.frameImageView)
+        val indicator: View = this.itemView.findViewById(R.id.debugIndicator)
+    }
+}

litr-demo/src/main/java/com/linkedin/android/litr/demo/VideoFilmStripView.kt

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2021 LinkedIn Corporation
+ * All Rights Reserved.
+ *
+ * Licensed under the BSD 2-Clause License (the "License").  See License in the project root for
+ * license information.
+ */
+package com.linkedin.android.litr.demo
+
+import android.content.Context
+import android.graphics.Bitmap
+import android.graphics.Canvas
+import android.util.AttributeSet
+import android.view.View
+import androidx.annotation.NonNull
+
+
+class VideoFilmStripView @JvmOverloads constructor(context: Context, attrs: AttributeSet? = null, defStyleAttr: Int = 0) : View(context, attrs, defStyleAttr) {
+
+    private var bitmapList: MutableList<Bitmap?>? = null
+
+    fun setFrameList(list: List<Bitmap?>) {
+        bitmapList = list.toMutableList()
+        invalidate()
+    }
+
+    override fun onDraw(@NonNull canvas: Canvas) {
+        super.onDraw(canvas)
+        var x = 0f
+        bitmapList?.filterNotNull()?.forEach { bitmap ->
+            canvas.drawBitmap(bitmap, x, 0f, null)
+            x += bitmap.width
+        }
+    }
+
+    fun setFrameAt(index: Int, bitmap: Bitmap?) {
+        bitmapList?.set(index, bitmap)
+        invalidate()
+    }
+}

litr-demo/src/main/res/layout/fragment_extract_frames.xml

@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright 2021 LinkedIn Corporation -->
+<!-- All Rights Reserved. -->
+<!-- -->
+<!-- Licensed under the BSD 2-Clause License (the "License").  See License in the project root -->
+<!-- for license information. -->
+<layout xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:app="http://schemas.android.com/apk/res-auto">
+
+    <data>
+
+        <import type="android.view.View" />
+
+        <variable
+            name="sourceMedia"
+            type="com.linkedin.android.litr.demo.data.SourceMedia" />
+
+        <variable
+            name="filter"
+            type="com.linkedin.android.litr.filter.GlFilter" />
+
+        <variable
+            name="fragment"
+            type="com.linkedin.android.litr.demo.ExtractFramesFragment" />
+
+    </data>
+
+    <androidx.core.widget.NestedScrollView
+        android:layout_width="match_parent"
+        android:layout_height="match_parent">
+
+        <LinearLayout
+            android:layout_width="match_parent"
+            android:layout_height="wrap_content"
+            android:orientation="vertical">
+
+            <include
+                android:id="@+id/section_pick_video"
+                layout="@layout/section_pick_video"
+                android:layout_width="match_parent"
+                android:layout_height="wrap_content"
+                app:sourceMedia="@{sourceMedia}" />
+
+            <Spinner
+                android:id="@+id/spinner_filters"
+                android:layout_width="match_parent"
+                android:layout_height="wrap_content"
+                android:enabled="@{sourceMedia != null}"
+                android:padding="@dimen/cell_padding" />
+
+            <Button
+                android:id="@+id/button_extract"
+                android:layout_width="match_parent"
+                android:layout_height="wrap_content"
+                android:enabled="@{sourceMedia != null}"
+                android:onClick="@{() -> fragment.extractThumbnails(sourceMedia, filter)}"
+                android:padding="@dimen/cell_padding"
+                android:text="@string/extract" />
+
+
+            <androidx.recyclerview.widget.RecyclerView
+                android:id="@+id/frames_recycler"
+                android:layout_width="match_parent"
+                android:layout_height="40dp"
+                />
+
+        </LinearLayout>
+
+    </androidx.core.widget.NestedScrollView>
+
+</layout>