From a39903087a0ea996342c53133c4a954510afbde1 Mon Sep 17 00:00:00 2001 From: "Hecker, Ronald" Date: Tue, 19 Nov 2024 16:23:50 +0100 Subject: [PATCH 01/33] Reimplement part of speech to text Enabled the c++ part and added the base UI. Actual video and subtitles showing etc needs to be implemented --- lib/interop/generated_bindings.dart | 71 +++++++++++ lib/interop/speech_to_text.dart | 109 ++++++++-------- .../computer_vision/batch_inference.dart | 2 +- lib/pages/computer_vision/live_inference.dart | 2 +- .../widgets/model_properties.dart | 107 ++++++++-------- lib/pages/models/inference.dart | 3 +- lib/pages/transcription/playground.dart | 101 +++++++++++++++ .../providers/speech_inference_provider.dart | 88 +++++++++++++ lib/pages/transcription/transcription.dart | 120 ++++++++++++++++++ lib/pages/transcription/utils/section.dart | 98 ++++++++++++++ lib/utils/drop_area.dart | 84 ++++++------ macos/Runner.xcodeproj/project.pbxproj | 30 +++++ openvino_bindings/src/BUILD | 1 + openvino_bindings/src/bindings.cc | 86 ++++++------- openvino_bindings/src/bindings.h | 8 +- 15 files changed, 706 insertions(+), 204 deletions(-) create mode 100644 lib/pages/transcription/playground.dart create mode 100644 lib/pages/transcription/providers/speech_inference_provider.dart create mode 100644 lib/pages/transcription/transcription.dart create mode 100644 lib/pages/transcription/utils/section.dart diff --git a/lib/interop/generated_bindings.dart b/lib/interop/generated_bindings.dart index f37fd89b..ea541a85 100644 --- a/lib/interop/generated_bindings.dart +++ b/lib/interop/generated_bindings.dart @@ -569,6 +569,77 @@ class OpenVINO { late final _graphRunnerStop = _graphRunnerStopPtr .asFunction Function(CGraphRunner)>(); + ffi.Pointer speechToTextOpen( + ffi.Pointer model_path, + ffi.Pointer device, + ) { + return _speechToTextOpen( + model_path, + device, + ); + } + + late final _speechToTextOpenPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('speechToTextOpen'); + late final _speechToTextOpen = _speechToTextOpenPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); + + ffi.Pointer speechToTextLoadVideo( + CSpeechToText instance, + ffi.Pointer video_path, + ) { + return _speechToTextLoadVideo( + instance, + video_path, + ); + } + + late final _speechToTextLoadVideoPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(CSpeechToText, + ffi.Pointer)>>('speechToTextLoadVideo'); + late final _speechToTextLoadVideo = _speechToTextLoadVideoPtr.asFunction< + ffi.Pointer Function(CSpeechToText, ffi.Pointer)>(); + + ffi.Pointer speechToTextVideoDuration( + CSpeechToText instance, + ) { + return _speechToTextVideoDuration( + instance, + ); + } + + late final _speechToTextVideoDurationPtr = _lookup< + ffi.NativeFunction Function(CSpeechToText)>>( + 'speechToTextVideoDuration'); + late final _speechToTextVideoDuration = _speechToTextVideoDurationPtr + .asFunction Function(CSpeechToText)>(); + + ffi.Pointer speechToTextTranscribe( + CSpeechToText instance, + int start, + int duration, + ffi.Pointer language, + ) { + return _speechToTextTranscribe( + instance, + start, + duration, + language, + ); + } + + late final _speechToTextTranscribePtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(CSpeechToText, ffi.Int, + ffi.Int, ffi.Pointer)>>('speechToTextTranscribe'); + late final _speechToTextTranscribe = _speechToTextTranscribePtr.asFunction< + ffi.Pointer Function( + CSpeechToText, int, int, ffi.Pointer)>(); + ffi.Pointer getAvailableDevices() { return _getAvailableDevices(); } diff --git a/lib/interop/speech_to_text.dart b/lib/interop/speech_to_text.dart index c8635ae1..b81ed02f 100644 --- a/lib/interop/speech_to_text.dart +++ b/lib/interop/speech_to_text.dart @@ -14,67 +14,64 @@ class SpeechToText { SpeechToText(this.instance); static Future init(String modelPath, String device) async { - throw UnimplementedError(); - //final result = await Isolate.run(() { - // final modelPathPtr = modelPath.toNativeUtf8(); - // final devicePtr = device.toNativeUtf8(); - // final status = ov.speechToTextOpen(modelPathPtr, devicePtr); - // calloc.free(modelPathPtr); - // calloc.free(devicePtr); - - // return status; - //}); - - //print("${result.ref.status}, ${result.ref.message}"); - //if (StatusEnum.fromValue(result.ref.status) != StatusEnum.OkStatus) { - // throw "SpeechToText open error: ${result.ref.status} ${result.ref.message.toDartString()}"; - //} - - //return SpeechToText(result); + final result = await Isolate.run(() { + final modelPathPtr = modelPath.toNativeUtf8(); + final devicePtr = device.toNativeUtf8(); + final status = ov.speechToTextOpen(modelPathPtr, devicePtr); + calloc.free(modelPathPtr); + calloc.free(devicePtr); + + return status; + }); + + print("${result.ref.status}, ${result.ref.message}"); + if (StatusEnum.fromValue(result.ref.status) != StatusEnum.OkStatus) { + throw "SpeechToText open error: ${result.ref.status} ${result.ref.message.toDartString()}"; + } + + return SpeechToText(result); } Future loadVideo(String videoPath) async{ - throw UnimplementedError(); - //int instanceAddress = instance.ref.value.address; - //{ - // final result = await Isolate.run(() { - // final videoPathPtr = videoPath.toNativeUtf8(); - // final status = ov.speechToTextLoadVideo(Pointer.fromAddress(instanceAddress), videoPathPtr); - // calloc.free(videoPathPtr); - // return status; - // }); - - // if (StatusEnum.fromValue(result.ref.status) != StatusEnum.OkStatus) { - // throw "SpeechToText LoadVideo error: ${result.ref.status} ${result.ref.message.toDartString()}"; - // } - //} - - //{ - // final result = await Isolate.run(() { - // final status = ov.speechToTextVideoDuration(Pointer.fromAddress(instanceAddress)); - // return status; - // }); - // if (StatusEnum.fromValue(result.ref.status) != StatusEnum.OkStatus) { - // throw "SpeechToText VideoDuration error: ${result.ref.status} ${result.ref.message.toDartString()}"; - // } - // return result.ref.value; - //} + int instanceAddress = instance.ref.value.address; + { + final result = await Isolate.run(() { + final videoPathPtr = videoPath.toNativeUtf8(); + final status = ov.speechToTextLoadVideo(Pointer.fromAddress(instanceAddress), videoPathPtr); + calloc.free(videoPathPtr); + return status; + }); + + if (StatusEnum.fromValue(result.ref.status) != StatusEnum.OkStatus) { + throw "SpeechToText LoadVideo error: ${result.ref.status} ${result.ref.message.toDartString()}"; + } + } + + { + final result = await Isolate.run(() { + final status = ov.speechToTextVideoDuration(Pointer.fromAddress(instanceAddress)); + return status; + }); + if (StatusEnum.fromValue(result.ref.status) != StatusEnum.OkStatus) { + throw "SpeechToText VideoDuration error: ${result.ref.status} ${result.ref.message.toDartString()}"; + } + return result.ref.value; + } } Future transcribe(int start, int duration, String language) async{ - throw UnimplementedError(); - //int instanceAddress = instance.ref.value.address; - //final result = await Isolate.run(() { - // final languagePtr = language.toNativeUtf8(); - // final status = ov.speechToTextTranscribe(Pointer.fromAddress(instanceAddress), start, duration, languagePtr); - // calloc.free(languagePtr); - // return status; - //}); - - //if (StatusEnum.fromValue(result.ref.status) != StatusEnum.OkStatus) { - // throw "SpeechToText LoadVideo error: ${result.ref.status} ${result.ref.message.toDartString()}"; - //} - - //return result.ref.value.toDartString(); + int instanceAddress = instance.ref.value.address; + final result = await Isolate.run(() { + final languagePtr = language.toNativeUtf8(); + final status = ov.speechToTextTranscribe(Pointer.fromAddress(instanceAddress), start, duration, languagePtr); + calloc.free(languagePtr); + return status; + }); + + if (StatusEnum.fromValue(result.ref.status) != StatusEnum.OkStatus) { + throw "SpeechToText LoadVideo error: ${result.ref.status} ${result.ref.message.toDartString()}"; + } + + return result.ref.value.toDartString(); } } diff --git a/lib/pages/computer_vision/batch_inference.dart b/lib/pages/computer_vision/batch_inference.dart index d2f34cfa..53c74ff5 100644 --- a/lib/pages/computer_vision/batch_inference.dart +++ b/lib/pages/computer_vision/batch_inference.dart @@ -99,7 +99,7 @@ class BatchInference extends StatelessWidget { ), ), ), - const ModelProperties(), + ModelProperties(project: batchInference.imageInference.project), ], ); } diff --git a/lib/pages/computer_vision/live_inference.dart b/lib/pages/computer_vision/live_inference.dart index 0b089bfd..9c78f25c 100644 --- a/lib/pages/computer_vision/live_inference.dart +++ b/lib/pages/computer_vision/live_inference.dart @@ -135,7 +135,7 @@ class _LiveInferenceState extends State { ], ), ), - const ModelProperties(), + ModelProperties(project: widget.project), ], ); } diff --git a/lib/pages/computer_vision/widgets/model_properties.dart b/lib/pages/computer_vision/widgets/model_properties.dart index d333243f..5d7e932d 100644 --- a/lib/pages/computer_vision/widgets/model_properties.dart +++ b/lib/pages/computer_vision/widgets/model_properties.dart @@ -1,70 +1,67 @@ import 'package:fluent_ui/fluent_ui.dart'; +import 'package:inference/project.dart'; import 'package:inference/theme_fluent.dart'; -import 'package:inference/utils.dart'; import 'package:inference/pages/models/widgets/grid_container.dart'; -import 'package:inference/providers/image_inference_provider.dart'; import 'package:intl/intl.dart'; -import 'package:provider/provider.dart'; +import 'package:inference/utils.dart'; class ModelProperties extends StatelessWidget { - const ModelProperties({super.key}); + final Project project; + const ModelProperties({super.key, required this.project}); @override Widget build(BuildContext context) { - return Consumer(builder: (context, inference, child) { - Locale locale = Localizations.localeOf(context); - final formatter = NumberFormat.percentPattern(locale.languageCode); + Locale locale = Localizations.localeOf(context); + final formatter = NumberFormat.percentPattern(locale.languageCode); - return SizedBox( - width: 280, - child: GridContainer( - padding: const EdgeInsets.symmetric(vertical: 18, horizontal: 24), - child: Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - const Text("Model parameters", style: TextStyle( - fontSize: 20, - )), - Container( - padding: const EdgeInsets.only(top: 16), - child: Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - ModelProperty( - title: "Model name", - value: inference.project.name, - ), - ModelProperty( - title: "Task", - value: inference.project.taskName(), - ), - ModelProperty( - title: "Architecture", - value: inference.project.architecture, - ), - ModelProperty( - title: "Size", - value: inference.project.size?.readableFileSize() ?? "", - ), - Builder( - builder: (context) { - if (inference.project.tasks.first.performance == null) { - return Container(); - } - return ModelProperty( - title: "Accuracy", - value: formatter.format(inference.project.tasks.first.performance!.score) - ); - } - ), - ], + return SizedBox( + width: 280, + child: GridContainer( + padding: const EdgeInsets.symmetric(vertical: 18, horizontal: 24), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + const Text("Model parameters", style: TextStyle( + fontSize: 20, + )), + Container( + padding: const EdgeInsets.only(top: 16), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + ModelProperty( + title: "Model name", + value: project.name, + ), + ModelProperty( + title: "Task", + value: project.taskName(), + ), + ModelProperty( + title: "Architecture", + value: project.architecture, + ), + ModelProperty( + title: "Size", + value: project.size?.readableFileSize() ?? "", + ), + Builder( + builder: (context) { + if (project.tasks.first.performance == null) { + return Container(); + } + return ModelProperty( + title: "Accuracy", + value: formatter.format(project.tasks.first.performance!.score) + ); + } ), - ) - ], + ], + ), ) - ), - ); - } + ], + ) + ), ); } } diff --git a/lib/pages/models/inference.dart b/lib/pages/models/inference.dart index 1445420b..18b022a2 100644 --- a/lib/pages/models/inference.dart +++ b/lib/pages/models/inference.dart @@ -1,5 +1,6 @@ import 'package:fluent_ui/fluent_ui.dart'; import 'package:inference/pages/computer_vision/computer_vision.dart'; +import 'package:inference/pages/transcription/transcription.dart'; import 'package:inference/project.dart'; class InferencePage extends StatelessWidget { @@ -14,7 +15,7 @@ class InferencePage extends StatelessWidget { case ProjectType.text: return Container(); case ProjectType.speech: - return Container(); + return TranscriptionPage(project); } } diff --git a/lib/pages/transcription/playground.dart b/lib/pages/transcription/playground.dart new file mode 100644 index 00000000..28417b8f --- /dev/null +++ b/lib/pages/transcription/playground.dart @@ -0,0 +1,101 @@ +import 'package:file_picker/file_picker.dart'; +import 'package:fluent_ui/fluent_ui.dart'; +import 'package:inference/pages/computer_vision/widgets/model_properties.dart'; +import 'package:inference/pages/models/widgets/grid_container.dart'; +import 'package:inference/project.dart'; +import 'package:inference/pages/transcription/providers/speech_inference_provider.dart'; +import 'package:inference/theme_fluent.dart'; +import 'package:inference/utils/drop_area.dart'; +import 'package:inference/widgets/controls/no_outline_button.dart'; +import 'package:inference/widgets/device_selector.dart'; +//import 'package:media_kit/media_kit.dart'; +//import 'package:media_kit_video/media_kit_video.dart'; +import 'package:provider/provider.dart'; + +class Playground extends StatefulWidget { + final Project project; + const Playground({super.key, required this.project}); + + @override + State createState() => _PlaygroundState(); +} + +class _PlaygroundState extends State { + //late final player = Player(); + //late final controller = VideoController(player); + + void showUploadMenu() async { + FilePickerResult? result = await FilePicker.platform.pickFiles(type: FileType.video); + + if (result != null) { + uploadFile(result.files.single.path!); + } + } + + void uploadFile(String file) async { + final inference = Provider.of(context, listen: false); + await inference.loadVideo(file); + inference.startTranscribing(); + } + + @override + Widget build(BuildContext context) { + final theme = FluentTheme.of(context); + return Row( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Expanded( + child: Column( + children: [ + SizedBox( + height: 64, + child: GridContainer( + child: Padding( + padding: const EdgeInsets.symmetric(horizontal: 16), + child: Row( + children: [ + NoOutlineButton( + onPressed: showUploadMenu, + child: Row( + children: [ + const Text("Choose video"), + const Padding( + padding: EdgeInsets.only(left: 8), + child: Icon(FluentIcons.chevron_down, size: 12), + ), + ], + ), + ), + const DeviceSelector(), + ], + ), + ), + ), + ), + Expanded( + child: GridContainer( + color: backgroundColor.of(theme), + child: Builder( + builder: (context) { + return DropArea( + type: "video", + showChild: false, + onUpload: (String file) { uploadFile(file); }, + extensions: const [], + child: Padding( + padding: const EdgeInsets.all(8.0), + child: Container(), + ), + ); + } + ), + ), + ) + ], + ), + ), + ModelProperties(project: widget.project), + ] + ); + } +} diff --git a/lib/pages/transcription/providers/speech_inference_provider.dart b/lib/pages/transcription/providers/speech_inference_provider.dart new file mode 100644 index 00000000..9f658fe6 --- /dev/null +++ b/lib/pages/transcription/providers/speech_inference_provider.dart @@ -0,0 +1,88 @@ +import 'dart:async'; + +import 'package:flutter/material.dart'; +import 'package:inference/interop/speech_to_text.dart'; +import 'package:inference/pages/transcription/utils/section.dart'; +import 'package:inference/project.dart'; + +const transcriptionPeriod = 10; + +class SpeechInferenceProvider extends ChangeNotifier { + Completer loaded = Completer(); + + + Project? _project; + String? _device; + + String? _videoPath; + String? get videoPath => _videoPath; + + bool get videoLoaded => _videoPath != null; + + DynamicRangeLoading>? _transcription; + Map>? get transcription => _transcription?.data; + + String _language = ""; + + String get language => _language; + set language(String val) { + _language = val; + notifyListeners(); + } + + SpeechToText? _inference; + + SpeechInferenceProvider(Project? project, String? device) { + _project = project; + _device = device; + + if (project != null && device != null) { + SpeechToText.init(project.storagePath, device).then((instance) { + _inference = instance; + loaded.complete(); + notifyListeners(); + }); + } + } + + void skipTo(int index) { + _transcription!.skipTo(index); + } + + Future loadVideo(String path) async { + await loaded.future; + _videoPath = path; + final duration = await _inference!.loadVideo(path); + final sections = (duration / transcriptionPeriod).ceil(); + _transcription = DynamicRangeLoading>(Section(0, sections)); + notifyListeners(); + } + + Future startTranscribing() async { + if (_transcription == null) { + throw Exception("Can't transcribe before loading video"); + } + + while (!_transcription!.complete) { + if (_transcription == null) { + return; + } + await _transcription!.process((int i) { + return transcribe(i * transcriptionPeriod, transcriptionPeriod); + }); + if (hasListeners) { + notifyListeners(); + } + } + } + + Future transcribe(int start, int duration) async { + await loaded.future; + return await _inference!.transcribe(start, duration, _language); + } + + bool sameProps(Project? project, String? device) { + return _project == project && _device == device; + } + +} diff --git a/lib/pages/transcription/transcription.dart b/lib/pages/transcription/transcription.dart new file mode 100644 index 00000000..14353d9d --- /dev/null +++ b/lib/pages/transcription/transcription.dart @@ -0,0 +1,120 @@ +import 'package:fluent_ui/fluent_ui.dart'; +import 'package:go_router/go_router.dart'; +import 'package:inference/project.dart'; +import 'package:inference/providers/preference_provider.dart'; +import 'package:inference/pages/transcription/providers/speech_inference_provider.dart'; +import 'package:inference/pages/transcription/playground.dart'; +import 'package:provider/provider.dart'; + +class TranscriptionPage extends StatefulWidget { + final Project project; + const TranscriptionPage(this.project, {super.key}); + + @override + State createState() => _TranscriptionPageState(); +} + +class _TranscriptionPageState extends State { + + + int selected = 0; + @override + Widget build(BuildContext context) { + final theme = FluentTheme.of(context); + final updatedTheme = theme.copyWith( + navigationPaneTheme: theme.navigationPaneTheme.merge(NavigationPaneThemeData( + backgroundColor: theme.scaffoldBackgroundColor, + )) + ); + return ChangeNotifierProxyProvider( + lazy: false, + create: (_) { + final device = Provider.of(context, listen: false).device; + return SpeechInferenceProvider(widget.project, device); + }, + update: (_, preferences, imageInferenceProvider) { + if (imageInferenceProvider != null && imageInferenceProvider.sameProps(widget.project, preferences.device)) { + return imageInferenceProvider; + } + return SpeechInferenceProvider(widget.project, preferences.device); + }, + child: Stack( + children: [ + FluentTheme( + data: updatedTheme, + child: NavigationView( + pane: NavigationPane( + size: const NavigationPaneSize(topHeight: 64), + header: Row( + children: [ + Padding( + padding: const EdgeInsets.only(left: 12.0), + child: ClipRRect( + borderRadius: BorderRadius.circular(4.0), + child: Container( + width: 40, + height: 40, + decoration: BoxDecoration( + image: DecorationImage( + image: widget.project.thumbnailImage(), + fit: BoxFit.cover), + ), + ), + ), + ), + Padding( + padding: const EdgeInsets.symmetric(horizontal: 16), + child: Text(widget.project.name, + style: const TextStyle(fontSize: 20, fontWeight: FontWeight.bold), + ), + ), + ], + ), + //customPane: CustomNavigationPane(), + selected: selected, + onChanged: (i) => setState(() {selected = i;}), + displayMode: PaneDisplayMode.top, + items: [ + PaneItem( + icon: const Icon(FluentIcons.processing), + title: const Text("Playground"), + body: Playground(project: widget.project), + ), + PaneItem( + icon: const Icon(FluentIcons.project_collection), + title: const Text("Performance metrics"), + body: Container(), + ), + ], + ) + ), + ), + SizedBox( + height: 64, + child: Padding( + padding: const EdgeInsets.symmetric(horizontal: 25), + child: Row( + mainAxisAlignment: MainAxisAlignment.end, + children: [ + Padding( + padding: const EdgeInsets.all(4), + child: OutlinedButton( + style: ButtonStyle( + shape:WidgetStatePropertyAll(RoundedRectangleBorder( + borderRadius: BorderRadius.circular(4.0), + side: const BorderSide(color: Color(0XFF545454)), + )), + ), + child: const Text("Close"), + onPressed: () => GoRouter.of(context).go("/models"), + ), + ), + ] + ), + ), + ) + ], + ) + ); + } +} diff --git a/lib/pages/transcription/utils/section.dart b/lib/pages/transcription/utils/section.dart new file mode 100644 index 00000000..27ede1d7 --- /dev/null +++ b/lib/pages/transcription/utils/section.dart @@ -0,0 +1,98 @@ +void moveToFront(List list, I item) { + list.remove(item); + list.insert(0, item); +} + +void moveToEnd(List list, I item) { + list.remove(item); + list.add(item); +} + +class DynamicRangeLoading { + List
sections = []; + Map data = {}; + + DynamicRangeLoading(Section section): sections = [section]; + + Section get activeSection => sections.first; + + // The incomplete sections will always be in front + bool get complete => activeSection.complete; + + void skipTo(int i) { + for (var section in sections) { + if (section.contains(i)) { + if (i > section.index) { + // Section has not progressed until the requested index + // Split the section and move the new section to the front + final newSection = section.split(i); + sections.insert(0, newSection); + } else { + // Section is further ahead than requested skipTo + // move section to front since that work has higher prio + if (!section.complete && section != activeSection) { + moveToFront(sections, section); + } + } + return; + } + } + + throw Exception("Out of range"); + } + + int getNextIndex() { + if (complete) { + throw Exception("Cannot get next index. All work is done"); + } + return activeSection.index; + } + + void pumpIndex() { + if (activeSection.pump()) { + //activeSection has ended + if (sections.length > 1) { + moveToEnd(sections,activeSection); + } + } + } + + Future process(Future Function(int) func) async{ + final index = getNextIndex(); + final val = await func(index); + data[index] = val; + pumpIndex(); + return val; + } + + void setData(I value) { + data[activeSection.index] = value; + activeSection.index += 1; + } +} + +class Section { + int begin; + int? end; + int index; + + Section(this.begin, this.end): index = begin; + + bool contains(int i) => begin <= i && (end == null ? true : i < end!); + + Section split(int i) { + final newSection = Section(i, end); + end = i; + return newSection; + } + + bool get complete => index == end; + + //returns false if there is still work to do in the section + bool pump() { + if (end == null || index < end!) { + index += 1; + } + return complete; + } +} diff --git a/lib/utils/drop_area.dart b/lib/utils/drop_area.dart index 61bb2f8d..dcdf7619 100644 --- a/lib/utils/drop_area.dart +++ b/lib/utils/drop_area.dart @@ -50,51 +50,49 @@ class _DropAreaState extends State { @override Widget build(BuildContext context) { - return Expanded( - child: DropTarget( - onDragDone: (details) => handleDrop(details), - onDragExited: (val) => hideReleaseMessage(), - onDragEntered: (val) => showReleaseMessage(), - child: Container( - decoration: BoxDecoration( - borderRadius: BorderRadius.circular(4.0), - color: intelGray, - ), - child: Builder( - builder: (context) { - if (!_showReleaseMessage && widget.showChild) { - return widget.child!; - } - return Center( - child: SizedBox( - height: 310, - child: Column( - crossAxisAlignment: CrossAxisAlignment.center, - mainAxisAlignment: MainAxisAlignment.spaceBetween, - children: [ - SvgPicture.asset('images/drop.svg'), - ( _showReleaseMessage - ? const Text("Release to drop media") - : Text("Drop ${widget.type} here") - ), - ElevatedButton( - onPressed: () => showUploadMenu(), - child: const Text("Upload") - ), - Builder( - builder: (context) { - if (widget.extensions == null) { - return Container(); - } - return Text(widget.extensions!.join(", ")); + return DropTarget( + onDragDone: (details) => handleDrop(details), + onDragExited: (val) => hideReleaseMessage(), + onDragEntered: (val) => showReleaseMessage(), + child: Container( + decoration: BoxDecoration( + borderRadius: BorderRadius.circular(4.0), + color: intelGray, + ), + child: Builder( + builder: (context) { + if (!_showReleaseMessage && widget.showChild) { + return widget.child!; + } + return Center( + child: SizedBox( + height: 310, + child: Column( + crossAxisAlignment: CrossAxisAlignment.center, + mainAxisAlignment: MainAxisAlignment.spaceBetween, + children: [ + SvgPicture.asset('images/drop.svg'), + ( _showReleaseMessage + ? const Text("Release to drop media") + : Text("Drop ${widget.type} here") + ), + ElevatedButton( + onPressed: () => showUploadMenu(), + child: const Text("Upload") + ), + Builder( + builder: (context) { + if (widget.extensions == null) { + return Container(); } - ) - ], - ), + return Text(widget.extensions!.join(", ")); + } + ) + ], ), - ); - } - ), + ), + ); + } ), ), ); diff --git a/macos/Runner.xcodeproj/project.pbxproj b/macos/Runner.xcodeproj/project.pbxproj index 8e01daa8..9d53cd0c 100644 --- a/macos/Runner.xcodeproj/project.pbxproj +++ b/macos/Runner.xcodeproj/project.pbxproj @@ -39,6 +39,16 @@ 0C42C76A2CE386680079F72B /* libopenvino_tensorflow_lite_frontend.2450.dylib in Bundle Framework */ = {isa = PBXBuildFile; fileRef = 0C42C7592CE386520079F72B /* libopenvino_tensorflow_lite_frontend.2450.dylib */; }; 0C42C76B2CE388D90079F72B /* libopenvino_c.2450.dylib in Bundle Framework */ = {isa = PBXBuildFile; fileRef = 0C42C7522CE386520079F72B /* libopenvino_c.2450.dylib */; }; 0C42C76C2CE388DC0079F72B /* libopenvino.2450.dylib in Bundle Framework */ = {isa = PBXBuildFile; fileRef = 0C42C75A2CE386520079F72B /* libopenvino.2450.dylib */; }; + 0C4E1F6C2CECC22800124339 /* libavformat.60.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = 0C4E1F692CECC22800124339 /* libavformat.60.dylib */; }; + 0C4E1F6D2CECC22800124339 /* libavutil.58.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = 0C4E1F6A2CECC22800124339 /* libavutil.58.dylib */; }; + 0C4E1F6E2CECC22800124339 /* libswresample.4.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = 0C4E1F6B2CECC22800124339 /* libswresample.4.dylib */; }; + 0C4E1F6F2CECC22800124339 /* libavcodec.60.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = 0C4E1F672CECC22800124339 /* libavcodec.60.dylib */; }; + 0C4E1F702CECC22800124339 /* libavdevice.60.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = 0C4E1F682CECC22800124339 /* libavdevice.60.dylib */; }; + 0C4E1F712CECC24900124339 /* libswresample.4.dylib in Bundle Framework */ = {isa = PBXBuildFile; fileRef = 0C4E1F6B2CECC22800124339 /* libswresample.4.dylib */; }; + 0C4E1F722CECC25400124339 /* libavcodec.60.dylib in Bundle Framework */ = {isa = PBXBuildFile; fileRef = 0C4E1F672CECC22800124339 /* libavcodec.60.dylib */; }; + 0C4E1F732CECC25400124339 /* libavdevice.60.dylib in Bundle Framework */ = {isa = PBXBuildFile; fileRef = 0C4E1F682CECC22800124339 /* libavdevice.60.dylib */; }; + 0C4E1F742CECC25400124339 /* libavformat.60.dylib in Bundle Framework */ = {isa = PBXBuildFile; fileRef = 0C4E1F692CECC22800124339 /* libavformat.60.dylib */; }; + 0C4E1F752CECC25400124339 /* libavutil.58.dylib in Bundle Framework */ = {isa = PBXBuildFile; fileRef = 0C4E1F6A2CECC22800124339 /* libavutil.58.dylib */; }; 0C5D47382C6F2F9500307B37 /* libmacos_bindings.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = 0C5D47372C6F2F9500307B37 /* libmacos_bindings.dylib */; settings = {ATTRIBUTES = (Weak, ); }; }; 0C5D47392C6F2FB200307B37 /* libmacos_bindings.dylib in Resources */ = {isa = PBXBuildFile; fileRef = 0C5D47372C6F2F9500307B37 /* libmacos_bindings.dylib */; }; 0C5D473A2C6F308000307B37 /* libmacos_bindings.dylib in Bundle Framework */ = {isa = PBXBuildFile; fileRef = 0C5D47372C6F2F9500307B37 /* libmacos_bindings.dylib */; settings = {ATTRIBUTES = (CodeSignOnCopy, ); }; }; @@ -117,15 +127,19 @@ dstSubfolderSpec = 10; files = ( 0C42C7672CE386680079F72B /* libopenvino_paddle_frontend.2450.dylib in Bundle Framework */, + 0C4E1F712CECC24900124339 /* libswresample.4.dylib in Bundle Framework */, 0C5D47B32C6F5C1300307B37 /* libopenvino_hetero_plugin.so in Bundle Framework */, + 0C4E1F752CECC25400124339 /* libavutil.58.dylib in Bundle Framework */, 0C42C76C2CE388DC0079F72B /* libopenvino.2450.dylib in Bundle Framework */, 0C42C7662CE386680079F72B /* libopenvino_onnx_frontend.2450.dylib in Bundle Framework */, 0C5D47B12C6F5C0A00307B37 /* libopenvino_auto_batch_plugin.so in Bundle Framework */, 0C5D47B22C6F5C0E00307B37 /* libopenvino_auto_plugin.so in Bundle Framework */, 0C5D473E2C6F35E500307B37 /* libblend2d.dylib in Bundle Framework */, + 0C4E1F732CECC25400124339 /* libavdevice.60.dylib in Bundle Framework */, 0C5D47782C6F398400307B37 /* libopencv_core.407.dylib in Bundle Framework */, 0C42C7642CE386680079F72B /* libopenvino_genai.2450.dylib in Bundle Framework */, 0C5D47B02C6F5C0200307B37 /* libopenvino_arm_cpu_plugin.so in Bundle Framework */, + 0C4E1F742CECC25400124339 /* libavformat.60.dylib in Bundle Framework */, 0C5D47802C6F398400307B37 /* libopencv_videoio.407.dylib in Bundle Framework */, 0C5D47792C6F398400307B37 /* libopencv_features2d.407.dylib in Bundle Framework */, 0C42C7682CE386680079F72B /* libopenvino_pytorch_frontend.2450.dylib in Bundle Framework */, @@ -139,6 +153,7 @@ 0C5D477F2C6F398400307B37 /* libopencv_video.407.dylib in Bundle Framework */, 0C5D47812C6F398400307B37 /* libopencv_ximgproc.407.dylib in Bundle Framework */, 0C5D473A2C6F308000307B37 /* libmacos_bindings.dylib in Bundle Framework */, + 0C4E1F722CECC25400124339 /* libavcodec.60.dylib in Bundle Framework */, 0C42C7692CE386680079F72B /* libopenvino_tensorflow_frontend.2450.dylib in Bundle Framework */, 0C5D47A52C6F3B7700307B37 /* libtbb.12.dylib in Bundle Framework */, 0C5D477C2C6F398400307B37 /* libopencv_imgcodecs.407.dylib in Bundle Framework */, @@ -161,6 +176,11 @@ 0C42C7582CE386520079F72B /* libopenvino_tensorflow_frontend.2450.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libopenvino_tensorflow_frontend.2450.dylib; path = ../bindings/libopenvino_tensorflow_frontend.2450.dylib; sourceTree = SOURCE_ROOT; }; 0C42C7592CE386520079F72B /* libopenvino_tensorflow_lite_frontend.2450.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libopenvino_tensorflow_lite_frontend.2450.dylib; path = ../bindings/libopenvino_tensorflow_lite_frontend.2450.dylib; sourceTree = SOURCE_ROOT; }; 0C42C75A2CE386520079F72B /* libopenvino.2450.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libopenvino.2450.dylib; path = ../bindings/libopenvino.2450.dylib; sourceTree = SOURCE_ROOT; }; + 0C4E1F672CECC22800124339 /* libavcodec.60.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libavcodec.60.dylib; path = ../bindings/libavcodec.60.dylib; sourceTree = SOURCE_ROOT; }; + 0C4E1F682CECC22800124339 /* libavdevice.60.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libavdevice.60.dylib; path = ../bindings/libavdevice.60.dylib; sourceTree = SOURCE_ROOT; }; + 0C4E1F692CECC22800124339 /* libavformat.60.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libavformat.60.dylib; path = ../bindings/libavformat.60.dylib; sourceTree = SOURCE_ROOT; }; + 0C4E1F6A2CECC22800124339 /* libavutil.58.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libavutil.58.dylib; path = ../bindings/libavutil.58.dylib; sourceTree = SOURCE_ROOT; }; + 0C4E1F6B2CECC22800124339 /* libswresample.4.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libswresample.4.dylib; path = ../bindings/libswresample.4.dylib; sourceTree = SOURCE_ROOT; }; 0C5D47372C6F2F9500307B37 /* libmacos_bindings.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libmacos_bindings.dylib; path = ../bindings/libmacos_bindings.dylib; sourceTree = ""; }; 0C5D473B2C6F357C00307B37 /* libblend2d.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libblend2d.dylib; path = ../bindings/libblend2d.dylib; sourceTree = ""; }; 0C5D47602C6F382800307B37 /* libopencv_calib3d.407.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libopencv_calib3d.407.dylib; path = ../bindings/libopencv_calib3d.407.dylib; sourceTree = ""; }; @@ -241,6 +261,11 @@ 0C42C75D2CE386520079F72B /* libopenvino_paddle_frontend.2450.dylib in Frameworks */, 0C42C75E2CE386520079F72B /* libopenvino_onnx_frontend.2450.dylib in Frameworks */, 0C42C75F2CE386520079F72B /* libopenvino_c.2450.dylib in Frameworks */, + 0C4E1F6C2CECC22800124339 /* libavformat.60.dylib in Frameworks */, + 0C4E1F6D2CECC22800124339 /* libavutil.58.dylib in Frameworks */, + 0C4E1F6E2CECC22800124339 /* libswresample.4.dylib in Frameworks */, + 0C4E1F6F2CECC22800124339 /* libavcodec.60.dylib in Frameworks */, + 0C4E1F702CECC22800124339 /* libavdevice.60.dylib in Frameworks */, 0C42C7602CE386520079F72B /* libopenvino_genai.2450.dylib in Frameworks */, 0C42C7612CE386520079F72B /* libopenvino.2450.dylib in Frameworks */, 0C42C7622CE386520079F72B /* libopenvino_ir_frontend.2450.dylib in Frameworks */, @@ -361,6 +386,11 @@ 0C5D47642C6F397900307B37 /* libopencv_ximgproc.407.dylib */, 0C5D473B2C6F357C00307B37 /* libblend2d.dylib */, 0C5D47372C6F2F9500307B37 /* libmacos_bindings.dylib */, + 0C4E1F672CECC22800124339 /* libavcodec.60.dylib */, + 0C4E1F682CECC22800124339 /* libavdevice.60.dylib */, + 0C4E1F692CECC22800124339 /* libavformat.60.dylib */, + 0C4E1F6A2CECC22800124339 /* libavutil.58.dylib */, + 0C4E1F6B2CECC22800124339 /* libswresample.4.dylib */, 11E6C6B7198D7B3B20F4A75C /* Pods_Runner.framework */, CB5E7865DB70376BADAAEAE6 /* Pods_RunnerTests.framework */, ); diff --git a/openvino_bindings/src/BUILD b/openvino_bindings/src/BUILD index 13b1000b..3bb179c1 100644 --- a/openvino_bindings/src/BUILD +++ b/openvino_bindings/src/BUILD @@ -9,6 +9,7 @@ cc_library( "//src/utils:utils", "//src/image:image_inference", "//src/llm:llm_inference", + "//src/audio:speech_to_text", "//src/mediapipe:graph_runner", ], ) diff --git a/openvino_bindings/src/bindings.cc b/openvino_bindings/src/bindings.cc index fcefbec0..04b0efa4 100644 --- a/openvino_bindings/src/bindings.cc +++ b/openvino_bindings/src/bindings.cc @@ -4,7 +4,7 @@ #include #include -//#include "src/audio/speech_to_text.h" +#include "src/audio/speech_to_text.h" #include "src/image/image_inference.h" #include "src/mediapipe/graph_runner.h" #include "src/mediapipe/serialization/serialization_calculators.h" @@ -290,48 +290,48 @@ Status* graphRunnerStop(CGraphRunner instance) { } } -//StatusOrSpeechToText* speechToTextOpen(const char* model_path, const char* device) { -// try { -// auto instance = new SpeechToText(model_path, device); -// return new StatusOrSpeechToText{OkStatus, "", instance}; -// } catch (...) { -// auto except = handle_exceptions(); -// return new StatusOrSpeechToText{except->status, except->message}; -// } -//} -// -//Status* speechToTextLoadVideo(CSpeechToText instance, const char* video_path) { -// try { -// auto object = reinterpret_cast(instance); -// object->load_video(video_path); -// return new Status{OkStatus, ""}; -// } catch (...) { -// return handle_exceptions(); -// } -//} -// -//StatusOrInt* speechToTextVideoDuration(CSpeechToText instance) { -// try { -// auto object = reinterpret_cast(instance); -// object->video_duration(); -// // Deal with long in the future -// return new StatusOrInt{OkStatus, "", (int)object->video_duration()}; -// } catch (...) { -// return new StatusOrInt{OkStatus, ""}; -// } -//} -// -//StatusOrModelResponse* speechToTextTranscribe(CSpeechToText instance, int start, int duration, const char* language) { -// try { -// auto object = reinterpret_cast(instance); -// auto result = object->transcribe(start, duration, language); -// std::string text = result; -// return new StatusOrModelResponse{OkStatus, "", convertToMetricsStruct(result.perf_metrics), strdup(text.c_str())}; -// } catch (...) { -// auto except = handle_exceptions(); -// return new StatusOrModelResponse{except->status, except->message}; -// } -//} +StatusOrSpeechToText* speechToTextOpen(const char* model_path, const char* device) { + try { + auto instance = new SpeechToText(model_path, device); + return new StatusOrSpeechToText{OkStatus, "", instance}; + } catch (...) { + auto except = handle_exceptions(); + return new StatusOrSpeechToText{except->status, except->message}; + } +} + +Status* speechToTextLoadVideo(CSpeechToText instance, const char* video_path) { + try { + auto object = reinterpret_cast(instance); + object->load_video(video_path); + return new Status{OkStatus, ""}; + } catch (...) { + return handle_exceptions(); + } +} + +StatusOrInt* speechToTextVideoDuration(CSpeechToText instance) { + try { + auto object = reinterpret_cast(instance); + object->video_duration(); + // Deal with long in the future + return new StatusOrInt{OkStatus, "", (int)object->video_duration()}; + } catch (...) { + return new StatusOrInt{OkStatus, ""}; + } +} + +StatusOrModelResponse* speechToTextTranscribe(CSpeechToText instance, int start, int duration, const char* language) { + try { + auto object = reinterpret_cast(instance); + auto result = object->transcribe(start, duration, language); + std::string text = result; + return new StatusOrModelResponse{OkStatus, "", convertToMetricsStruct(result.perf_metrics), strdup(text.c_str())}; + } catch (...) { + auto except = handle_exceptions(); + return new StatusOrModelResponse{except->status, except->message}; + } +} //void report_rss() { // struct rusage r_usage; diff --git a/openvino_bindings/src/bindings.h b/openvino_bindings/src/bindings.h index e496c5aa..45289160 100644 --- a/openvino_bindings/src/bindings.h +++ b/openvino_bindings/src/bindings.h @@ -123,10 +123,10 @@ EXPORT Status* graphRunnerQueueSerializationOutput(CGraphRunner instance, const EXPORT StatusOrString* graphRunnerGet(CGraphRunner instance); EXPORT Status* graphRunnerStop(CGraphRunner instance); -//EXPORT StatusOrSpeechToText* speechToTextOpen(const char* model_path, const char* device); -//EXPORT Status* speechToTextLoadVideo(CSpeechToText instance, const char* video_path); -//EXPORT StatusOrInt* speechToTextVideoDuration(CSpeechToText instance); -//EXPORT StatusOrModelResponse* speechToTextTranscribe(CSpeechToText instance, int start, int duration, const char* language); +EXPORT StatusOrSpeechToText* speechToTextOpen(const char* model_path, const char* device); +EXPORT Status* speechToTextLoadVideo(CSpeechToText instance, const char* video_path); +EXPORT StatusOrInt* speechToTextVideoDuration(CSpeechToText instance); +EXPORT StatusOrModelResponse* speechToTextTranscribe(CSpeechToText instance, int start, int duration, const char* language); EXPORT StatusOrDevices* getAvailableDevices(); Status* handle_exceptions(); From e32515493b7e55a64d7034b6a384f2c7e7e05a19 Mon Sep 17 00:00:00 2001 From: "Hecker, Ronald" Date: Tue, 19 Nov 2024 16:40:29 +0100 Subject: [PATCH 02/33] Working video and subtitles --- lib/main.dart | 2 + lib/pages/transcription/playground.dart | 79 +++++-- .../transcription/widgets/subtitles.dart | 59 +++++ linux/flutter/generated_plugin_registrant.cc | 8 + linux/flutter/generated_plugins.cmake | 3 + macos/Flutter/GeneratedPluginRegistrant.swift | 10 + macos/Podfile.lock | 36 ++++ pubspec.lock | 204 +++++++++++++++++- pubspec.yaml | 3 + .../flutter/generated_plugin_registrant.cc | 9 + windows/flutter/generated_plugins.cmake | 4 + 11 files changed, 392 insertions(+), 25 deletions(-) create mode 100644 lib/pages/transcription/widgets/subtitles.dart diff --git a/lib/main.dart b/lib/main.dart index 9f019f04..fe0ba36b 100644 --- a/lib/main.dart +++ b/lib/main.dart @@ -6,6 +6,7 @@ import 'package:inference/theme_fluent.dart'; import 'package:inference/providers/preference_provider.dart'; import 'package:inference/providers/project_provider.dart'; import 'package:inference/public_models.dart'; +import 'package:media_kit/media_kit.dart'; import 'package:provider/provider.dart'; @@ -25,6 +26,7 @@ void testConnection() async { } void main() { + MediaKit.ensureInitialized(); testConnection(); runApp(const App()); } diff --git a/lib/pages/transcription/playground.dart b/lib/pages/transcription/playground.dart index 28417b8f..0b2d74c4 100644 --- a/lib/pages/transcription/playground.dart +++ b/lib/pages/transcription/playground.dart @@ -1,15 +1,19 @@ +import 'dart:async'; + import 'package:file_picker/file_picker.dart'; import 'package:fluent_ui/fluent_ui.dart'; import 'package:inference/pages/computer_vision/widgets/model_properties.dart'; import 'package:inference/pages/models/widgets/grid_container.dart'; +import 'package:inference/pages/transcription/widgets/subtitles.dart'; import 'package:inference/project.dart'; import 'package:inference/pages/transcription/providers/speech_inference_provider.dart'; import 'package:inference/theme_fluent.dart'; import 'package:inference/utils/drop_area.dart'; import 'package:inference/widgets/controls/no_outline_button.dart'; import 'package:inference/widgets/device_selector.dart'; -//import 'package:media_kit/media_kit.dart'; -//import 'package:media_kit_video/media_kit_video.dart'; +import 'package:intl/date_symbol_data_local.dart'; +import 'package:media_kit/media_kit.dart'; +import 'package:media_kit_video/media_kit_video.dart'; import 'package:provider/provider.dart'; class Playground extends StatefulWidget { @@ -20,9 +24,12 @@ class Playground extends StatefulWidget { State createState() => _PlaygroundState(); } -class _PlaygroundState extends State { - //late final player = Player(); - //late final controller = VideoController(player); +class _PlaygroundState extends State with TickerProviderStateMixin{ + final player = Player(); + late final controller = VideoController(player); + int subtitleIndex = 0; + StreamSubscription? listener; + void showUploadMenu() async { FilePickerResult? result = await FilePicker.platform.pickFiles(type: FileType.video); @@ -32,10 +39,29 @@ class _PlaygroundState extends State { } } + void positionListener(Duration position) { + int index = (position.inSeconds / transcriptionPeriod).floor(); + if (index != subtitleIndex) { + final inference = Provider.of(context, listen: false); + inference.skipTo(index); + setState(() { + subtitleIndex = index; + }); + } + } + + void initializeVideoAndListeners(String source) async { + await listener?.cancel(); + player.open(Media(source)); + player.setVolume(0); // TODO: Disable this for release. This is for our sanity + listener = player.stream.position.listen(positionListener); + } + void uploadFile(String file) async { final inference = Provider.of(context, listen: false); await inference.loadVideo(file); inference.startTranscribing(); + initializeVideoAndListeners(file); } @override @@ -72,24 +98,31 @@ class _PlaygroundState extends State { ), ), ), - Expanded( - child: GridContainer( - color: backgroundColor.of(theme), - child: Builder( - builder: (context) { - return DropArea( - type: "video", - showChild: false, - onUpload: (String file) { uploadFile(file); }, - extensions: const [], - child: Padding( - padding: const EdgeInsets.all(8.0), - child: Container(), - ), - ); - } - ), - ), + Consumer( + builder: (context, inference, child) { + return Expanded( + child: GridContainer( + color: backgroundColor.of(theme), + child: Builder( + builder: (context) { + return DropArea( + type: "video", + showChild: inference.videoLoaded, + onUpload: (String file) { uploadFile(file); }, + extensions: const [], + child: Stack( + alignment: Alignment.bottomCenter, + children: [ + Video(controller: controller), + Subtitles(transcription: inference.transcription, subtitleIndex: subtitleIndex), + ] + ), + ); + } + ), + ), + ); + } ) ], ), diff --git a/lib/pages/transcription/widgets/subtitles.dart b/lib/pages/transcription/widgets/subtitles.dart new file mode 100644 index 00000000..21c609b8 --- /dev/null +++ b/lib/pages/transcription/widgets/subtitles.dart @@ -0,0 +1,59 @@ +import 'dart:async'; + +import 'package:fluent_ui/fluent_ui.dart'; + +class Subtitles extends StatelessWidget { + const Subtitles({ + super.key, + required this.transcription, + required this.subtitleIndex, + }); + + final Map>? transcription; + final int subtitleIndex; + + static const double fontSize = 18; + + @override + Widget build(BuildContext context) { + return Padding( + padding: const EdgeInsets.only(left: 8, right: 8, bottom: 60), + child: SizedBox( + height: 100, + child: Builder( + builder: (context) { + if (transcription == null ) { + return Container(); + } + if (transcription![subtitleIndex] is String) { + return Stack( + alignment: Alignment.bottomCenter, + children: [ + Text( + transcription![subtitleIndex] as String, + textAlign: TextAlign.center, + style: TextStyle( + fontSize: fontSize, + foreground: Paint() + ..style = PaintingStyle.stroke + ..strokeWidth = 2 + ..color = Colors.black, + ) + ), + Text( + transcription![subtitleIndex] as String, + textAlign: TextAlign.center, + style: const TextStyle( + fontSize: fontSize + ) + ) + ], + ); + } + return Container(); + } + ), + ), + ); + } +} diff --git a/linux/flutter/generated_plugin_registrant.cc b/linux/flutter/generated_plugin_registrant.cc index 8e89f019..17066f6c 100644 --- a/linux/flutter/generated_plugin_registrant.cc +++ b/linux/flutter/generated_plugin_registrant.cc @@ -8,6 +8,8 @@ #include #include +#include +#include #include void fl_register_plugins(FlPluginRegistry* registry) { @@ -17,6 +19,12 @@ void fl_register_plugins(FlPluginRegistry* registry) { g_autoptr(FlPluginRegistrar) flutter_acrylic_registrar = fl_plugin_registry_get_registrar_for_plugin(registry, "FlutterAcrylicPlugin"); flutter_acrylic_plugin_register_with_registrar(flutter_acrylic_registrar); + g_autoptr(FlPluginRegistrar) media_kit_libs_linux_registrar = + fl_plugin_registry_get_registrar_for_plugin(registry, "MediaKitLibsLinuxPlugin"); + media_kit_libs_linux_plugin_register_with_registrar(media_kit_libs_linux_registrar); + g_autoptr(FlPluginRegistrar) media_kit_video_registrar = + fl_plugin_registry_get_registrar_for_plugin(registry, "MediaKitVideoPlugin"); + media_kit_video_plugin_register_with_registrar(media_kit_video_registrar); g_autoptr(FlPluginRegistrar) system_theme_registrar = fl_plugin_registry_get_registrar_for_plugin(registry, "SystemThemePlugin"); system_theme_plugin_register_with_registrar(system_theme_registrar); diff --git a/linux/flutter/generated_plugins.cmake b/linux/flutter/generated_plugins.cmake index cc87f3ae..386a1ebb 100644 --- a/linux/flutter/generated_plugins.cmake +++ b/linux/flutter/generated_plugins.cmake @@ -5,10 +5,13 @@ list(APPEND FLUTTER_PLUGIN_LIST desktop_drop flutter_acrylic + media_kit_libs_linux + media_kit_video system_theme ) list(APPEND FLUTTER_FFI_PLUGIN_LIST + media_kit_native_event_loop ) set(PLUGIN_BUNDLED_LIBRARIES) diff --git a/macos/Flutter/GeneratedPluginRegistrant.swift b/macos/Flutter/GeneratedPluginRegistrant.swift index dc088718..c7199431 100644 --- a/macos/Flutter/GeneratedPluginRegistrant.swift +++ b/macos/Flutter/GeneratedPluginRegistrant.swift @@ -7,12 +7,22 @@ import Foundation import desktop_drop import macos_window_utils +import media_kit_libs_macos_video +import media_kit_video +import package_info_plus import path_provider_foundation +import screen_brightness_macos import system_theme +import wakelock_plus func RegisterGeneratedPlugins(registry: FlutterPluginRegistry) { DesktopDropPlugin.register(with: registry.registrar(forPlugin: "DesktopDropPlugin")) MacOSWindowUtilsPlugin.register(with: registry.registrar(forPlugin: "MacOSWindowUtilsPlugin")) + MediaKitLibsMacosVideoPlugin.register(with: registry.registrar(forPlugin: "MediaKitLibsMacosVideoPlugin")) + MediaKitVideoPlugin.register(with: registry.registrar(forPlugin: "MediaKitVideoPlugin")) + FPPPackageInfoPlusPlugin.register(with: registry.registrar(forPlugin: "FPPPackageInfoPlusPlugin")) PathProviderPlugin.register(with: registry.registrar(forPlugin: "PathProviderPlugin")) + ScreenBrightnessMacosPlugin.register(with: registry.registrar(forPlugin: "ScreenBrightnessMacosPlugin")) SystemThemePlugin.register(with: registry.registrar(forPlugin: "SystemThemePlugin")) + WakelockPlusMacosPlugin.register(with: registry.registrar(forPlugin: "WakelockPlusMacosPlugin")) } diff --git a/macos/Podfile.lock b/macos/Podfile.lock index 5d8310ba..c8104321 100644 --- a/macos/Podfile.lock +++ b/macos/Podfile.lock @@ -4,18 +4,36 @@ PODS: - FlutterMacOS (1.0.0) - macos_window_utils (1.0.0): - FlutterMacOS + - media_kit_libs_macos_video (1.0.4): + - FlutterMacOS + - media_kit_native_event_loop (1.0.0): + - FlutterMacOS + - media_kit_video (0.0.1): + - FlutterMacOS + - package_info_plus (0.0.1): + - FlutterMacOS - path_provider_foundation (0.0.1): - Flutter - FlutterMacOS + - screen_brightness_macos (0.1.0): + - FlutterMacOS - system_theme (0.0.1): - FlutterMacOS + - wakelock_plus (0.0.1): + - FlutterMacOS DEPENDENCIES: - desktop_drop (from `Flutter/ephemeral/.symlinks/plugins/desktop_drop/macos`) - FlutterMacOS (from `Flutter/ephemeral`) - macos_window_utils (from `Flutter/ephemeral/.symlinks/plugins/macos_window_utils/macos`) + - media_kit_libs_macos_video (from `Flutter/ephemeral/.symlinks/plugins/media_kit_libs_macos_video/macos`) + - media_kit_native_event_loop (from `Flutter/ephemeral/.symlinks/plugins/media_kit_native_event_loop/macos`) + - media_kit_video (from `Flutter/ephemeral/.symlinks/plugins/media_kit_video/macos`) + - package_info_plus (from `Flutter/ephemeral/.symlinks/plugins/package_info_plus/macos`) - path_provider_foundation (from `Flutter/ephemeral/.symlinks/plugins/path_provider_foundation/darwin`) + - screen_brightness_macos (from `Flutter/ephemeral/.symlinks/plugins/screen_brightness_macos/macos`) - system_theme (from `Flutter/ephemeral/.symlinks/plugins/system_theme/macos`) + - wakelock_plus (from `Flutter/ephemeral/.symlinks/plugins/wakelock_plus/macos`) EXTERNAL SOURCES: desktop_drop: @@ -24,17 +42,35 @@ EXTERNAL SOURCES: :path: Flutter/ephemeral macos_window_utils: :path: Flutter/ephemeral/.symlinks/plugins/macos_window_utils/macos + media_kit_libs_macos_video: + :path: Flutter/ephemeral/.symlinks/plugins/media_kit_libs_macos_video/macos + media_kit_native_event_loop: + :path: Flutter/ephemeral/.symlinks/plugins/media_kit_native_event_loop/macos + media_kit_video: + :path: Flutter/ephemeral/.symlinks/plugins/media_kit_video/macos + package_info_plus: + :path: Flutter/ephemeral/.symlinks/plugins/package_info_plus/macos path_provider_foundation: :path: Flutter/ephemeral/.symlinks/plugins/path_provider_foundation/darwin + screen_brightness_macos: + :path: Flutter/ephemeral/.symlinks/plugins/screen_brightness_macos/macos system_theme: :path: Flutter/ephemeral/.symlinks/plugins/system_theme/macos + wakelock_plus: + :path: Flutter/ephemeral/.symlinks/plugins/wakelock_plus/macos SPEC CHECKSUMS: desktop_drop: 69eeff437544aa619c8db7f4481b3a65f7696898 FlutterMacOS: 8f6f14fa908a6fb3fba0cd85dbd81ec4b251fb24 macos_window_utils: 933f91f64805e2eb91a5bd057cf97cd097276663 + media_kit_libs_macos_video: b3e2bbec2eef97c285f2b1baa7963c67c753fb82 + media_kit_native_event_loop: 81fd5b45192b72f8b5b69eaf5b540f45777eb8d5 + media_kit_video: c75b07f14d59706c775778e4dd47dd027de8d1e5 + package_info_plus: 12f1c5c2cfe8727ca46cbd0b26677728972d9a5b path_provider_foundation: 2b6b4c569c0fb62ec74538f866245ac84301af46 + screen_brightness_macos: 2d6d3af2165592d9a55ffcd95b7550970e41ebda system_theme: c7b9f6659a5caa26c9bc2284da096781e9a6fcbc + wakelock_plus: 4783562c9a43d209c458cb9b30692134af456269 PODFILE CHECKSUM: 16208599a12443d53889ba2270a4985981cfb204 diff --git a/pubspec.lock b/pubspec.lock index 9e93ab40..d1a1ee89 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -230,6 +230,14 @@ packages: url: "https://pub.dev" source: hosted version: "2.3.7" + dbus: + dependency: transitive + description: + name: dbus + sha256: "365c771ac3b0e58845f39ec6deebc76e3276aa9922b0cc60840712094d9047ac" + url: "https://pub.dev" + source: hosted + version: "0.7.10" desktop_drop: dependency: "direct main" description: @@ -477,10 +485,10 @@ packages: dependency: transitive description: name: js - sha256: c1b2e9b5ea78c45e1a0788d29606ba27dc5f71f019f32ca5140f61ef071838cf + sha256: f2c445dce49627136094980615a031419f7f3eb393237e4ecd97ac15dea343f3 url: "https://pub.dev" source: hosted - version: "0.7.1" + version: "0.6.7" json_annotation: dependency: transitive description: @@ -569,6 +577,78 @@ packages: url: "https://pub.dev" source: hosted version: "2.6.0" + media_kit: + dependency: "direct main" + description: + name: media_kit + sha256: "1f1deee148533d75129a6f38251ff8388e33ee05fc2d20a6a80e57d6051b7b62" + url: "https://pub.dev" + source: hosted + version: "1.1.11" + media_kit_libs_android_video: + dependency: transitive + description: + name: media_kit_libs_android_video + sha256: "9dd8012572e4aff47516e55f2597998f0a378e3d588d0fad0ca1f11a53ae090c" + url: "https://pub.dev" + source: hosted + version: "1.3.6" + media_kit_libs_ios_video: + dependency: transitive + description: + name: media_kit_libs_ios_video + sha256: b5382994eb37a4564c368386c154ad70ba0cc78dacdd3fb0cd9f30db6d837991 + url: "https://pub.dev" + source: hosted + version: "1.1.4" + media_kit_libs_linux: + dependency: transitive + description: + name: media_kit_libs_linux + sha256: e186891c31daa6bedab4d74dcdb4e8adfccc7d786bfed6ad81fe24a3b3010310 + url: "https://pub.dev" + source: hosted + version: "1.1.3" + media_kit_libs_macos_video: + dependency: transitive + description: + name: media_kit_libs_macos_video + sha256: f26aa1452b665df288e360393758f84b911f70ffb3878032e1aabba23aa1032d + url: "https://pub.dev" + source: hosted + version: "1.1.4" + media_kit_libs_video: + dependency: "direct main" + description: + name: media_kit_libs_video + sha256: "20bb4aefa8fece282b59580e1cd8528117297083a6640c98c2e98cfc96b93288" + url: "https://pub.dev" + source: hosted + version: "1.0.5" + media_kit_libs_windows_video: + dependency: transitive + description: + name: media_kit_libs_windows_video + sha256: "32654572167825c42c55466f5d08eee23ea11061c84aa91b09d0e0f69bdd0887" + url: "https://pub.dev" + source: hosted + version: "1.0.10" + media_kit_native_event_loop: + dependency: transitive + description: + name: media_kit_native_event_loop + sha256: "7d82e3b3e9ded5c35c3146c5ba1da3118d1dd8ac3435bac7f29f458181471b40" + url: "https://pub.dev" + source: hosted + version: "1.0.9" + media_kit_video: + dependency: "direct main" + description: + name: media_kit_video + sha256: "2cc3b966679963ba25a4ce5b771e532a521ebde7c6aa20e9802bec95d9916c8f" + url: "https://pub.dev" + source: hosted + version: "1.2.5" meta: dependency: transitive description: @@ -609,6 +689,22 @@ packages: url: "https://pub.dev" source: hosted version: "2.1.0" + package_info_plus: + dependency: transitive + description: + name: package_info_plus + sha256: da8d9ac8c4b1df253d1a328b7bf01ae77ef132833479ab40763334db13b91cce + url: "https://pub.dev" + source: hosted + version: "8.1.1" + package_info_plus_platform_interface: + dependency: transitive + description: + name: package_info_plus_platform_interface + sha256: ac1f4a4847f1ade8e6a87d1f39f5d7c67490738642e2542f559ec38c37489a66 + url: "https://pub.dev" + source: hosted + version: "3.0.1" path: dependency: "direct main" description: @@ -753,6 +849,62 @@ packages: url: "https://pub.dev" source: hosted version: "4.1.0" + safe_local_storage: + dependency: transitive + description: + name: safe_local_storage + sha256: ede4eb6cb7d88a116b3d3bf1df70790b9e2038bc37cb19112e381217c74d9440 + url: "https://pub.dev" + source: hosted + version: "1.0.2" + screen_brightness: + dependency: transitive + description: + name: screen_brightness + sha256: ed8da4a4511e79422fc1aa88138e920e4008cd312b72cdaa15ccb426c0faaedd + url: "https://pub.dev" + source: hosted + version: "0.2.2+1" + screen_brightness_android: + dependency: transitive + description: + name: screen_brightness_android + sha256: "3df10961e3a9e968a5e076fe27e7f4741fa8a1d3950bdeb48cf121ed529d0caf" + url: "https://pub.dev" + source: hosted + version: "0.1.0+2" + screen_brightness_ios: + dependency: transitive + description: + name: screen_brightness_ios + sha256: "99adc3ca5490b8294284aad5fcc87f061ad685050e03cf45d3d018fe398fd9a2" + url: "https://pub.dev" + source: hosted + version: "0.1.0" + screen_brightness_macos: + dependency: transitive + description: + name: screen_brightness_macos + sha256: "64b34e7e3f4900d7687c8e8fb514246845a73ecec05ab53483ed025bd4a899fd" + url: "https://pub.dev" + source: hosted + version: "0.1.0+1" + screen_brightness_platform_interface: + dependency: transitive + description: + name: screen_brightness_platform_interface + sha256: b211d07f0c96637a15fb06f6168617e18030d5d74ad03795dd8547a52717c171 + url: "https://pub.dev" + source: hosted + version: "0.1.0" + screen_brightness_windows: + dependency: transitive + description: + name: screen_brightness_windows + sha256: "9261bf33d0fc2707d8cf16339ce25768100a65e70af0fcabaf032fc12408ba86" + url: "https://pub.dev" + source: hosted + version: "0.1.3" scroll_pos: dependency: transitive description: @@ -846,6 +998,14 @@ packages: url: "https://pub.dev" source: hosted version: "0.3.1" + synchronized: + dependency: transitive + description: + name: synchronized + sha256: "69fe30f3a8b04a0be0c15ae6490fc859a78ef4c43ae2dd5e8a623d45bfcf9225" + url: "https://pub.dev" + source: hosted + version: "3.3.0+3" system_theme: dependency: "direct main" description: @@ -894,6 +1054,22 @@ packages: url: "https://pub.dev" source: hosted version: "1.4.0" + universal_platform: + dependency: transitive + description: + name: universal_platform + sha256: "64e16458a0ea9b99260ceb5467a214c1f298d647c659af1bff6d3bf82536b1ec" + url: "https://pub.dev" + source: hosted + version: "1.1.0" + uri_parser: + dependency: transitive + description: + name: uri_parser + sha256: "6543c9fd86d2862fac55d800a43e67c0dcd1a41677cb69c2f8edfe73bbcf1835" + url: "https://pub.dev" + source: hosted + version: "2.0.2" uuid: dependency: "direct main" description: @@ -942,6 +1118,30 @@ packages: url: "https://pub.dev" source: hosted version: "14.2.5" + volume_controller: + dependency: transitive + description: + name: volume_controller + sha256: c71d4c62631305df63b72da79089e078af2659649301807fa746088f365cb48e + url: "https://pub.dev" + source: hosted + version: "2.0.8" + wakelock_plus: + dependency: transitive + description: + name: wakelock_plus + sha256: bf4ee6f17a2fa373ed3753ad0e602b7603f8c75af006d5b9bdade263928c0484 + url: "https://pub.dev" + source: hosted + version: "1.2.8" + wakelock_plus_platform_interface: + dependency: transitive + description: + name: wakelock_plus_platform_interface + sha256: "422d1cdbb448079a8a62a5a770b69baa489f8f7ca21aef47800c726d404f9d16" + url: "https://pub.dev" + source: hosted + version: "1.2.1" watcher: dependency: transitive description: diff --git a/pubspec.yaml b/pubspec.yaml index 5e35670e..c3233b89 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -56,6 +56,9 @@ dependencies: fluent_ui: ^4.9.2 system_theme: ^3.1.2 flutter_acrylic: ^1.1.4 + media_kit: ^1.1.11 # Primary package. + media_kit_video: ^1.2.5 # For video rendering. + media_kit_libs_video: ^1.0.5 # Native video dependencies. dev_dependencies: flutter_test: diff --git a/windows/flutter/generated_plugin_registrant.cc b/windows/flutter/generated_plugin_registrant.cc index 909a92e0..054d5c64 100644 --- a/windows/flutter/generated_plugin_registrant.cc +++ b/windows/flutter/generated_plugin_registrant.cc @@ -8,6 +8,9 @@ #include #include +#include +#include +#include #include void RegisterPlugins(flutter::PluginRegistry* registry) { @@ -15,6 +18,12 @@ void RegisterPlugins(flutter::PluginRegistry* registry) { registry->GetRegistrarForPlugin("DesktopDropPlugin")); FlutterAcrylicPluginRegisterWithRegistrar( registry->GetRegistrarForPlugin("FlutterAcrylicPlugin")); + MediaKitLibsWindowsVideoPluginCApiRegisterWithRegistrar( + registry->GetRegistrarForPlugin("MediaKitLibsWindowsVideoPluginCApi")); + MediaKitVideoPluginCApiRegisterWithRegistrar( + registry->GetRegistrarForPlugin("MediaKitVideoPluginCApi")); + ScreenBrightnessWindowsPluginRegisterWithRegistrar( + registry->GetRegistrarForPlugin("ScreenBrightnessWindowsPlugin")); SystemThemePluginRegisterWithRegistrar( registry->GetRegistrarForPlugin("SystemThemePlugin")); } diff --git a/windows/flutter/generated_plugins.cmake b/windows/flutter/generated_plugins.cmake index 1f4b61fd..3c6f76d0 100644 --- a/windows/flutter/generated_plugins.cmake +++ b/windows/flutter/generated_plugins.cmake @@ -5,10 +5,14 @@ list(APPEND FLUTTER_PLUGIN_LIST desktop_drop flutter_acrylic + media_kit_libs_windows_video + media_kit_video + screen_brightness_windows system_theme ) list(APPEND FLUTTER_FFI_PLUGIN_LIST + media_kit_native_event_loop ) set(PLUGIN_BUNDLED_LIBRARIES) From df3b0903b1e192d76489d762adb5375305d24f0b Mon Sep 17 00:00:00 2001 From: "Hecker, Ronald" Date: Wed, 20 Nov 2024 13:29:07 +0100 Subject: [PATCH 03/33] Implement speech to text using chunks Gen AI allows you to output chunks when return_timestamps is true. The chunks are closer to actual sentences and the timestamps are better. I parse these chunks to get to combine them into better sentences. --- lib/interop/generated_bindings.dart | 68 +++++++++++- lib/interop/openvino_bindings.dart | 14 +++ lib/interop/speech_to_text.dart | 16 ++- lib/pages/transcription/playground.dart | 58 ++++++---- .../providers/speech_inference_provider.dart | 9 +- lib/pages/transcription/utils/message.dart | 38 +++++++ .../transcription/widgets/subtitles.dart | 12 +- .../transcription/widgets/transcription.dart | 104 ++++++++++++++++++ openvino_bindings/README.md | 2 +- openvino_bindings/WORKSPACE | 2 +- openvino_bindings/src/audio/speech_to_text.cc | 3 +- openvino_bindings/src/audio/speech_to_text.h | 3 +- openvino_bindings/src/bindings.cc | 29 ++++- openvino_bindings/src/bindings.h | 18 ++- 14 files changed, 330 insertions(+), 46 deletions(-) create mode 100644 lib/pages/transcription/utils/message.dart create mode 100644 lib/pages/transcription/widgets/transcription.dart diff --git a/lib/interop/generated_bindings.dart b/lib/interop/generated_bindings.dart index ea541a85..5699581b 100644 --- a/lib/interop/generated_bindings.dart +++ b/lib/interop/generated_bindings.dart @@ -92,6 +92,37 @@ class OpenVINO { late final _freeStatusOrSpeechToText = _freeStatusOrSpeechToTextPtr .asFunction)>(); + void freeStatusOrModelResponse( + ffi.Pointer status, + ) { + return _freeStatusOrModelResponse( + status, + ); + } + + late final _freeStatusOrModelResponsePtr = _lookup< + ffi.NativeFunction< + ffi.Void Function(ffi.Pointer)>>( + 'freeStatusOrModelResponse'); + late final _freeStatusOrModelResponse = _freeStatusOrModelResponsePtr + .asFunction)>(); + + void freeStatusOrWhisperModelResponse( + ffi.Pointer status, + ) { + return _freeStatusOrWhisperModelResponse( + status, + ); + } + + late final _freeStatusOrWhisperModelResponsePtr = _lookup< + ffi.NativeFunction< + ffi.Void Function(ffi.Pointer)>>( + 'freeStatusOrWhisperModelResponse'); + late final _freeStatusOrWhisperModelResponse = + _freeStatusOrWhisperModelResponsePtr.asFunction< + void Function(ffi.Pointer)>(); + void freeStatusOrDevices( ffi.Pointer status, ) { @@ -618,7 +649,7 @@ class OpenVINO { late final _speechToTextVideoDuration = _speechToTextVideoDurationPtr .asFunction Function(CSpeechToText)>(); - ffi.Pointer speechToTextTranscribe( + ffi.Pointer speechToTextTranscribe( CSpeechToText instance, int start, int duration, @@ -634,10 +665,13 @@ class OpenVINO { late final _speechToTextTranscribePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(CSpeechToText, ffi.Int, - ffi.Int, ffi.Pointer)>>('speechToTextTranscribe'); + ffi.Pointer Function( + CSpeechToText, + ffi.Int, + ffi.Int, + ffi.Pointer)>>('speechToTextTranscribe'); late final _speechToTextTranscribe = _speechToTextTranscribePtr.asFunction< - ffi.Pointer Function( + ffi.Pointer Function( CSpeechToText, int, int, ffi.Pointer)>(); ffi.Pointer getAvailableDevices() { @@ -744,6 +778,16 @@ final class Device extends ffi.Struct { external ffi.Pointer name; } +final class TranscriptionChunk extends ffi.Struct { + @ffi.Float() + external double start_ts; + + @ffi.Float() + external double end_ts; + + external ffi.Pointer text; +} + final class Status extends ffi.Struct { @ffi.Int() external int status; @@ -835,6 +879,22 @@ final class StatusOrModelResponse extends ffi.Struct { external ffi.Pointer value; } +final class StatusOrWhisperModelResponse extends ffi.Struct { + @ffi.Int() + external int status; + + external ffi.Pointer message; + + external Metrics metrics; + + external ffi.Pointer value; + + @ffi.Int() + external int size; + + external ffi.Pointer text; +} + final class StatusOrDevices extends ffi.Struct { @ffi.Int() external int status; diff --git a/lib/interop/openvino_bindings.dart b/lib/interop/openvino_bindings.dart index e11cc935..defe9771 100644 --- a/lib/interop/openvino_bindings.dart +++ b/lib/interop/openvino_bindings.dart @@ -18,6 +18,20 @@ class SerializationOutput { } +class Chunk { + final double start; + final double end; + final String text; + const Chunk(this.start, this.end, this.text); +} + +class TranscriptionModelResponse { + final List chunks; + final Metrics metrics; + final String text; + const TranscriptionModelResponse(this.chunks, this.metrics, this.text); +} + class ModelResponse { final String content; final Metrics metrics; diff --git a/lib/interop/speech_to_text.dart b/lib/interop/speech_to_text.dart index b81ed02f..1e07f178 100644 --- a/lib/interop/speech_to_text.dart +++ b/lib/interop/speech_to_text.dart @@ -59,7 +59,7 @@ class SpeechToText { } } - Future transcribe(int start, int duration, String language) async{ + Future transcribe(int start, int duration, String language) async{ int instanceAddress = instance.ref.value.address; final result = await Isolate.run(() { final languagePtr = language.toNativeUtf8(); @@ -72,6 +72,18 @@ class SpeechToText { throw "SpeechToText LoadVideo error: ${result.ref.status} ${result.ref.message.toDartString()}"; } - return result.ref.value.toDartString(); + List chunks = []; + for (int i = 0; i < result.ref.size; i++) { + chunks.add(Chunk( + result.ref.value[i].start_ts, + result.ref.value[i].end_ts, + result.ref.value[i].text.toDartString() + )); + } + final metrics = result.ref.metrics; + final text = result.ref.text.toDartString(); + ov.freeStatusOrWhisperModelResponse(result); + + return TranscriptionModelResponse(chunks, metrics, text); } } diff --git a/lib/pages/transcription/playground.dart b/lib/pages/transcription/playground.dart index 0b2d74c4..4af04e3d 100644 --- a/lib/pages/transcription/playground.dart +++ b/lib/pages/transcription/playground.dart @@ -5,13 +5,13 @@ import 'package:fluent_ui/fluent_ui.dart'; import 'package:inference/pages/computer_vision/widgets/model_properties.dart'; import 'package:inference/pages/models/widgets/grid_container.dart'; import 'package:inference/pages/transcription/widgets/subtitles.dart'; +import 'package:inference/pages/transcription/widgets/transcription.dart'; import 'package:inference/project.dart'; import 'package:inference/pages/transcription/providers/speech_inference_provider.dart'; import 'package:inference/theme_fluent.dart'; import 'package:inference/utils/drop_area.dart'; import 'package:inference/widgets/controls/no_outline_button.dart'; import 'package:inference/widgets/device_selector.dart'; -import 'package:intl/date_symbol_data_local.dart'; import 'package:media_kit/media_kit.dart'; import 'package:media_kit_video/media_kit_video.dart'; import 'package:provider/provider.dart'; @@ -101,25 +101,42 @@ class _PlaygroundState extends State with TickerProviderStateMixin{ Consumer( builder: (context, inference, child) { return Expanded( - child: GridContainer( - color: backgroundColor.of(theme), - child: Builder( - builder: (context) { - return DropArea( - type: "video", - showChild: inference.videoLoaded, - onUpload: (String file) { uploadFile(file); }, - extensions: const [], - child: Stack( - alignment: Alignment.bottomCenter, - children: [ - Video(controller: controller), - Subtitles(transcription: inference.transcription, subtitleIndex: subtitleIndex), - ] - ), - ); - } - ), + child: Builder( + builder: (context) { + return DropArea( + type: "video", + showChild: inference.videoLoaded, + onUpload: (String file) { uploadFile(file); }, + extensions: const [], + child: Row( + crossAxisAlignment: CrossAxisAlignment.stretch, + children: [ + Expanded( + child: GridContainer( + color: backgroundColor.of(theme), + child: Stack( + alignment: Alignment.bottomCenter, + children: [ + Video(controller: controller), + Subtitles(transcription: inference.transcription, subtitleIndex: subtitleIndex), + ] + ), + ), + ), + SizedBox( + width: 312, + child: GridContainer( + color: backgroundColor.of(theme), + child: Transcription( + onSeek: player.seek, + transcription: inference.transcription + ), + ), + ) + ], + ), + ); + } ), ); } @@ -132,3 +149,4 @@ class _PlaygroundState extends State with TickerProviderStateMixin{ ); } } + diff --git a/lib/pages/transcription/providers/speech_inference_provider.dart b/lib/pages/transcription/providers/speech_inference_provider.dart index 9f658fe6..2302b341 100644 --- a/lib/pages/transcription/providers/speech_inference_provider.dart +++ b/lib/pages/transcription/providers/speech_inference_provider.dart @@ -1,6 +1,7 @@ import 'dart:async'; import 'package:flutter/material.dart'; +import 'package:inference/interop/openvino_bindings.dart'; import 'package:inference/interop/speech_to_text.dart'; import 'package:inference/pages/transcription/utils/section.dart'; import 'package:inference/project.dart'; @@ -19,8 +20,8 @@ class SpeechInferenceProvider extends ChangeNotifier { bool get videoLoaded => _videoPath != null; - DynamicRangeLoading>? _transcription; - Map>? get transcription => _transcription?.data; + DynamicRangeLoading>? _transcription; + Map>? get transcription => _transcription?.data; String _language = ""; @@ -54,7 +55,7 @@ class SpeechInferenceProvider extends ChangeNotifier { _videoPath = path; final duration = await _inference!.loadVideo(path); final sections = (duration / transcriptionPeriod).ceil(); - _transcription = DynamicRangeLoading>(Section(0, sections)); + _transcription = DynamicRangeLoading>(Section(0, sections)); notifyListeners(); } @@ -76,7 +77,7 @@ class SpeechInferenceProvider extends ChangeNotifier { } } - Future transcribe(int start, int duration) async { + Future transcribe(int start, int duration) async { await loaded.future; return await _inference!.transcribe(start, duration, _language); } diff --git a/lib/pages/transcription/utils/message.dart b/lib/pages/transcription/utils/message.dart new file mode 100644 index 00000000..95687320 --- /dev/null +++ b/lib/pages/transcription/utils/message.dart @@ -0,0 +1,38 @@ +import 'dart:async'; + +import 'package:inference/interop/openvino_bindings.dart'; + +class Message { + String message; + final Duration position; + + Message(this.message, this.position); + + static List parse(Map> transcriptions, int indexDuration) { + final indices = transcriptions.keys.toList()..sort(); + if (indices.isEmpty) { + return []; + } + + List output = []; + + bool lastChunkIsOpenEnded = false; + + for (int i in indices) { + if (transcriptions[i] is Future) { + continue; + } + final part = transcriptions[i] as TranscriptionModelResponse; + for (final chunk in part.chunks) { + String text = chunk.text; + if (lastChunkIsOpenEnded) { + output.last.message += text; + } else { + output.add(Message(text.substring(1), Duration(seconds: chunk.start.toInt()))); + } + lastChunkIsOpenEnded = text[text.length - 1] != "."; + } + } + return output; + } +} diff --git a/lib/pages/transcription/widgets/subtitles.dart b/lib/pages/transcription/widgets/subtitles.dart index 21c609b8..9971c9a2 100644 --- a/lib/pages/transcription/widgets/subtitles.dart +++ b/lib/pages/transcription/widgets/subtitles.dart @@ -1,6 +1,7 @@ import 'dart:async'; import 'package:fluent_ui/fluent_ui.dart'; +import 'package:inference/interop/openvino_bindings.dart'; class Subtitles extends StatelessWidget { const Subtitles({ @@ -9,7 +10,7 @@ class Subtitles extends StatelessWidget { required this.subtitleIndex, }); - final Map>? transcription; + final Map>? transcription; final int subtitleIndex; static const double fontSize = 18; @@ -25,12 +26,12 @@ class Subtitles extends StatelessWidget { if (transcription == null ) { return Container(); } - if (transcription![subtitleIndex] is String) { + if (transcription![subtitleIndex] is TranscriptionModelResponse) { + final text = (transcription![subtitleIndex] as TranscriptionModelResponse).text; return Stack( alignment: Alignment.bottomCenter, children: [ - Text( - transcription![subtitleIndex] as String, + Text(text, textAlign: TextAlign.center, style: TextStyle( fontSize: fontSize, @@ -40,8 +41,7 @@ class Subtitles extends StatelessWidget { ..color = Colors.black, ) ), - Text( - transcription![subtitleIndex] as String, + Text(text, textAlign: TextAlign.center, style: const TextStyle( fontSize: fontSize diff --git a/lib/pages/transcription/widgets/transcription.dart b/lib/pages/transcription/widgets/transcription.dart new file mode 100644 index 00000000..f521fb8b --- /dev/null +++ b/lib/pages/transcription/widgets/transcription.dart @@ -0,0 +1,104 @@ +import 'dart:async'; + +import 'package:fluent_ui/fluent_ui.dart'; +import 'package:inference/interop/openvino_bindings.dart'; +import 'package:inference/pages/transcription/utils/message.dart'; +import 'package:inference/pages/transcription/providers/speech_inference_provider.dart'; +import 'package:inference/theme_fluent.dart'; + +String formatDuration(int totalSeconds) { + final duration = Duration(seconds: totalSeconds); + final minutes = duration.inMinutes; + final seconds = totalSeconds % 60; + + final minutesString = '$minutes'.padLeft(2, '0'); + final secondsString = '$seconds'.padLeft(2, '0'); + return '$minutesString:$secondsString'; +} + + + +class Transcription extends StatelessWidget { + final Map>? transcription; + final Function(Duration)? onSeek; + const Transcription({super.key, this.transcription, this.onSeek}); + + @override + Widget build(BuildContext context) { + if (transcription == null) { + return Container(); + } + + final messages = Message.parse(transcription!, transcriptionPeriod); + + return SingleChildScrollView( + child: Padding( + padding: const EdgeInsets.symmetric(horizontal: 8), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + for (final message in messages) + TranscriptionMessage(message: message, onSeek: onSeek) + ], + ), + ), + ); + } +} + +class TranscriptionMessage extends StatefulWidget { + final Function(Duration)? onSeek; + final Message message; + + const TranscriptionMessage({super.key, required this.message, this.onSeek}); + + @override + State createState() => _TranscriptionMessageState(); +} + +class _TranscriptionMessageState extends State { + bool hover = false; + + @override + Widget build(BuildContext context) { + final theme = FluentTheme.of(context); + return MouseRegion( + onEnter: (_) { + setState(() => hover = true); + }, + onExit: (_) { + setState(() => hover = false); + }, + child: GestureDetector( + onTap: () { + widget.onSeek?.call(widget.message.position); + }, + child: Padding( + padding: const EdgeInsets.symmetric(vertical: 20), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Align( + alignment: Alignment.bottomRight, + child: Text(formatDuration(widget.message.position.inSeconds), + style: TextStyle( + fontSize: 9, + color: subtleTextColor.of(theme), + ) + ) + ), + Container( + decoration: BoxDecoration( + color: hover ? subtleTextColor.of(theme).withOpacity(0.3) : null, + borderRadius: const BorderRadius.all(Radius.circular(4)), + ), + padding: const EdgeInsets.symmetric(horizontal: 4, vertical: 2), + child: Text(widget.message.message) + ), + ], + ), + ), + ), + ); + } +} diff --git a/openvino_bindings/README.md b/openvino_bindings/README.md index 952b68b7..d6f8c219 100644 --- a/openvino_bindings/README.md +++ b/openvino_bindings/README.md @@ -108,7 +108,7 @@ The DLLs (with dependencies) will be in `bazel-bin/windows_bindings.tar` [Install OpenVINO Runtime 24.5.0](https://docs.openvino.ai/2024/get-started/install-openvino.html?PACKAGE=OPENVINO_GENAI&VERSION=v_2024_4_0&OP_SYSTEM=MACOS&DISTRIBUTION=ARCHIVE) with GenAI flavor in `/opt/intel/openvino_24.5.0` and symlink to `/opt/intel/openvino`. Install OpenCV: `brew install opencv` -Install ffmpeg: `brew install ffmpeg@6` +Install ffmpeg: `brew install ffmpeg@6 && brew link ffmpeg@6` Run: `bazel build :macos_bindings` diff --git a/openvino_bindings/WORKSPACE b/openvino_bindings/WORKSPACE index 17409650..6a1707a4 100644 --- a/openvino_bindings/WORKSPACE +++ b/openvino_bindings/WORKSPACE @@ -115,7 +115,7 @@ git_repository( new_local_repository( name = "mac_ffmpeg", build_file = "//third_party/ffmpeg:mac.BUILD", - path = "/opt/homebrew/Cellar/ffmpeg@6/6.1.2_3", + path = "/opt/homebrew/opt/ffmpeg@6", ) # #new_local_repository( diff --git a/openvino_bindings/src/audio/speech_to_text.cc b/openvino_bindings/src/audio/speech_to_text.cc index e39cb4d4..4a2e1012 100644 --- a/openvino_bindings/src/audio/speech_to_text.cc +++ b/openvino_bindings/src/audio/speech_to_text.cc @@ -8,7 +8,7 @@ void SpeechToText::load_video(std::string video_path) { audio_grabber = std::make_unique(video_path); } -ov::genai::DecodedResults SpeechToText::transcribe(int start, int duration, std::string language) { +ov::genai::WhisperDecodedResults SpeechToText::transcribe(int start, int duration, std::string language) { auto video_duration = audio_grabber->get_duration(); if (start > video_duration) { throw api_error(SpeechToTextChunkOutOfBounds); @@ -23,6 +23,7 @@ ov::genai::DecodedResults SpeechToText::transcribe(int start, int duration, std: if (data.empty()) { throw api_error(SpeechToTextChunkHasNoData); } + config.return_timestamps = true; config.max_new_tokens = 100; if (!language.empty()){ config.language = language; diff --git a/openvino_bindings/src/audio/speech_to_text.h b/openvino_bindings/src/audio/speech_to_text.h index c0c7c1ed..f119ca72 100644 --- a/openvino_bindings/src/audio/speech_to_text.h +++ b/openvino_bindings/src/audio/speech_to_text.h @@ -1,6 +1,7 @@ #ifndef SPEECH_TO_TEXT_H_ #define SPEECH_TO_TEXT_H_ + #include #include "openvino/genai/whisper_pipeline.hpp" #include "audio_grabber.h" @@ -14,7 +15,7 @@ class SpeechToText { SpeechToText(std::string model_path, std::string device): pipe(model_path, device), config(model_path + "/generation_config.json") {} void load_video(std::string video_path); int64_t video_duration(); - ov::genai::DecodedResults transcribe(int start, int duration, std::string language); + ov::genai::WhisperDecodedResults transcribe(int start, int duration, std::string language); }; diff --git a/openvino_bindings/src/bindings.cc b/openvino_bindings/src/bindings.cc index 04b0efa4..89ee853c 100644 --- a/openvino_bindings/src/bindings.cc +++ b/openvino_bindings/src/bindings.cc @@ -39,6 +39,19 @@ void freeStatusOrImageInference(StatusOrString *status) { delete status; } +void freeStatusOrModelResponse(StatusOrModelResponse *status) { + //std::cout << "Freeing StatusOrImageInference" << std::endl; + delete status; +} + +void freeStatusOrWhisperModelResponse(StatusOrWhisperModelResponse *status) { + if (status->status == StatusEnum::OkStatus) { + delete [] status->value; + status->value = NULL; // Prevent dangling pointers + } + delete status; +} + void freeStatusOrDevices(StatusOrDevices *status) { if (status->status == StatusEnum::OkStatus) { delete [] status->value; @@ -321,15 +334,21 @@ StatusOrInt* speechToTextVideoDuration(CSpeechToText instance) { } } -StatusOrModelResponse* speechToTextTranscribe(CSpeechToText instance, int start, int duration, const char* language) { +StatusOrWhisperModelResponse* speechToTextTranscribe(CSpeechToText instance, int start, int duration, const char* language) { try { auto object = reinterpret_cast(instance); - auto result = object->transcribe(start, duration, language); - std::string text = result; - return new StatusOrModelResponse{OkStatus, "", convertToMetricsStruct(result.perf_metrics), strdup(text.c_str())}; + auto transcription_result = object->transcribe(start, duration, language); + auto chunks = transcription_result.chunks.value(); + std::string text = transcription_result; + TranscriptionChunk* result = new TranscriptionChunk[chunks.size()]; + for (int i = 0; i < chunks.size(); i++) { + auto r = chunks[i]; + result[i] = TranscriptionChunk{r.start_ts + start, r.end_ts + start, strdup(r.text.c_str())}; + } + return new StatusOrWhisperModelResponse{OkStatus, "", convertToMetricsStruct(transcription_result.perf_metrics), result, (int)chunks.size(), strdup(text.c_str())}; } catch (...) { auto except = handle_exceptions(); - return new StatusOrModelResponse{except->status, except->message}; + return new StatusOrWhisperModelResponse{except->status, except->message}; } } diff --git a/openvino_bindings/src/bindings.h b/openvino_bindings/src/bindings.h index 45289160..1f27f624 100644 --- a/openvino_bindings/src/bindings.h +++ b/openvino_bindings/src/bindings.h @@ -26,6 +26,11 @@ typedef struct { const char* name; } Device; +typedef struct { + float start_ts; + float end_ts; + const char* text; +} TranscriptionChunk; typedef struct { enum StatusEnum status; @@ -81,6 +86,15 @@ typedef struct { const char* value; } StatusOrModelResponse; +typedef struct { + enum StatusEnum status; + const char* message; + Metrics metrics; + TranscriptionChunk* value; + int size; + const char* text; +} StatusOrWhisperModelResponse; + typedef struct { enum StatusEnum status; const char* message; @@ -96,6 +110,8 @@ EXPORT void freeStatusOrString(StatusOrString *status); EXPORT void freeStatusOrImageInference(StatusOrImageInference *status); EXPORT void freeStatusOrLLMInference(StatusOrLLMInference *status); EXPORT void freeStatusOrSpeechToText(StatusOrSpeechToText *status); +EXPORT void freeStatusOrModelResponse(StatusOrModelResponse *status); +EXPORT void freeStatusOrWhisperModelResponse(StatusOrWhisperModelResponse *status); EXPORT void freeStatusOrDevices(StatusOrDevices *status); EXPORT StatusOrImageInference* imageInferenceOpen(const char* model_path, const char* task, const char* device, const char* label_definitions_json); @@ -126,7 +142,7 @@ EXPORT Status* graphRunnerStop(CGraphRunner instance); EXPORT StatusOrSpeechToText* speechToTextOpen(const char* model_path, const char* device); EXPORT Status* speechToTextLoadVideo(CSpeechToText instance, const char* video_path); EXPORT StatusOrInt* speechToTextVideoDuration(CSpeechToText instance); -EXPORT StatusOrModelResponse* speechToTextTranscribe(CSpeechToText instance, int start, int duration, const char* language); +EXPORT StatusOrWhisperModelResponse* speechToTextTranscribe(CSpeechToText instance, int start, int duration, const char* language); EXPORT StatusOrDevices* getAvailableDevices(); Status* handle_exceptions(); From c77fe809254be7db445a1f421428c494ea9a3d63 Mon Sep 17 00:00:00 2001 From: "Hecker, Ronald" Date: Wed, 20 Nov 2024 14:16:42 +0100 Subject: [PATCH 04/33] Add download button and stub search bar to transcription --- lib/interop/speech_to_text.dart | 2 - lib/pages/transcription/playground.dart | 9 +- .../providers/speech_inference_provider.dart | 19 ++-- .../transcription/widgets/transcription.dart | 86 +++++++++++++++---- lib/widgets/controls/search_bar.dart | 2 +- 5 files changed, 89 insertions(+), 29 deletions(-) diff --git a/lib/interop/speech_to_text.dart b/lib/interop/speech_to_text.dart index 1e07f178..4f57cb2d 100644 --- a/lib/interop/speech_to_text.dart +++ b/lib/interop/speech_to_text.dart @@ -9,8 +9,6 @@ final ov = getBindings(); class SpeechToText { final Pointer instance; - - SpeechToText(this.instance); static Future init(String modelPath, String device) async { diff --git a/lib/pages/transcription/playground.dart b/lib/pages/transcription/playground.dart index 4af04e3d..b1f8b549 100644 --- a/lib/pages/transcription/playground.dart +++ b/lib/pages/transcription/playground.dart @@ -118,18 +118,21 @@ class _PlaygroundState extends State with TickerProviderStateMixin{ alignment: Alignment.bottomCenter, children: [ Video(controller: controller), - Subtitles(transcription: inference.transcription, subtitleIndex: subtitleIndex), + Subtitles( + transcription: inference.transcription?.data, + subtitleIndex: subtitleIndex, + ), ] ), ), ), SizedBox( - width: 312, + width: 360, child: GridContainer( color: backgroundColor.of(theme), child: Transcription( onSeek: player.seek, - transcription: inference.transcription + transcription: inference.transcription, ), ), ) diff --git a/lib/pages/transcription/providers/speech_inference_provider.dart b/lib/pages/transcription/providers/speech_inference_provider.dart index 2302b341..8e574f2b 100644 --- a/lib/pages/transcription/providers/speech_inference_provider.dart +++ b/lib/pages/transcription/providers/speech_inference_provider.dart @@ -20,8 +20,11 @@ class SpeechInferenceProvider extends ChangeNotifier { bool get videoLoaded => _videoPath != null; - DynamicRangeLoading>? _transcription; - Map>? get transcription => _transcription?.data; + DynamicRangeLoading>? transcription; + + bool get transcriptionComplete { + return transcription?.complete ?? false; + } String _language = ""; @@ -47,7 +50,7 @@ class SpeechInferenceProvider extends ChangeNotifier { } void skipTo(int index) { - _transcription!.skipTo(index); + transcription!.skipTo(index); } Future loadVideo(String path) async { @@ -55,20 +58,20 @@ class SpeechInferenceProvider extends ChangeNotifier { _videoPath = path; final duration = await _inference!.loadVideo(path); final sections = (duration / transcriptionPeriod).ceil(); - _transcription = DynamicRangeLoading>(Section(0, sections)); + transcription = DynamicRangeLoading>(Section(0, sections)); notifyListeners(); } Future startTranscribing() async { - if (_transcription == null) { + if (transcription == null) { throw Exception("Can't transcribe before loading video"); } - while (!_transcription!.complete) { - if (_transcription == null) { + while (!transcription!.complete) { + if (transcription == null) { return; } - await _transcription!.process((int i) { + await transcription!.process((int i) { return transcribe(i * transcriptionPeriod, transcriptionPeriod); }); if (hasListeners) { diff --git a/lib/pages/transcription/widgets/transcription.dart b/lib/pages/transcription/widgets/transcription.dart index f521fb8b..9301aa11 100644 --- a/lib/pages/transcription/widgets/transcription.dart +++ b/lib/pages/transcription/widgets/transcription.dart @@ -1,10 +1,14 @@ import 'dart:async'; +import 'dart:io'; +import 'package:file_picker/file_picker.dart'; import 'package:fluent_ui/fluent_ui.dart'; import 'package:inference/interop/openvino_bindings.dart'; import 'package:inference/pages/transcription/utils/message.dart'; import 'package:inference/pages/transcription/providers/speech_inference_provider.dart'; +import 'package:inference/pages/transcription/utils/section.dart'; import 'package:inference/theme_fluent.dart'; +import 'package:inference/widgets/controls/search_bar.dart'; String formatDuration(int totalSeconds) { final duration = Duration(seconds: totalSeconds); @@ -19,9 +23,30 @@ String formatDuration(int totalSeconds) { class Transcription extends StatelessWidget { - final Map>? transcription; + final DynamicRangeLoading>? transcription; final Function(Duration)? onSeek; - const Transcription({super.key, this.transcription, this.onSeek}); + const Transcription({super.key, this.onSeek, this.transcription}); + + void saveTranscript() async { + final file = await FilePicker.platform.saveFile( + dialogTitle: "Please select an output file:", + fileName: "transcription.txt", + ); + if (file == null){ + return; + } + + String contents = ""; + final indices = transcription!.data.keys.toList()..sort(); + for (int i in indices) { + final part = transcription!.data[i] as TranscriptionModelResponse; + for (final chunk in part.chunks) { + contents += chunk.text; + } + } + + await File(file).writeAsString(contents); + } @override Widget build(BuildContext context) { @@ -29,19 +54,50 @@ class Transcription extends StatelessWidget { return Container(); } - final messages = Message.parse(transcription!, transcriptionPeriod); - - return SingleChildScrollView( - child: Padding( - padding: const EdgeInsets.symmetric(horizontal: 8), - child: Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - for (final message in messages) - TranscriptionMessage(message: message, onSeek: onSeek) - ], + final messages = Message.parse(transcription!.data, transcriptionPeriod); + + return Column( + children: [ + Padding( + padding: const EdgeInsets.symmetric(vertical: 25, horizontal: 14), + child: Row( + children: [ + SearchBar(onChange: (p) {}, placeholder: "Search in transcript",), + Padding( + padding: const EdgeInsets.only(left: 8.0), + child: Tooltip( + message: transcription!.complete + ? "Download transcript" + : "Transcribing...", + child: Button( + onPressed: transcription!.complete + ? () => saveTranscript() + : null, + child: const Padding( + padding: EdgeInsets.symmetric(vertical: 2), + child: Icon(FluentIcons.download), + ), + ), + ), + ) + ], + ), ), - ), + Expanded( + child: SingleChildScrollView( + child: Padding( + padding: const EdgeInsets.only(left: 10, right: 18), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + for (final message in messages) + TranscriptionMessage(message: message, onSeek: onSeek) + ], + ), + ), + ), + ), + ], ); } } @@ -74,7 +130,7 @@ class _TranscriptionMessageState extends State { widget.onSeek?.call(widget.message.position); }, child: Padding( - padding: const EdgeInsets.symmetric(vertical: 20), + padding: const EdgeInsets.symmetric(vertical: 20, horizontal: 4), child: Column( crossAxisAlignment: CrossAxisAlignment.start, children: [ diff --git a/lib/widgets/controls/search_bar.dart b/lib/widgets/controls/search_bar.dart index 71424061..7b3e6a7a 100644 --- a/lib/widgets/controls/search_bar.dart +++ b/lib/widgets/controls/search_bar.dart @@ -50,4 +50,4 @@ class _SearchBarState extends State { ), ); } -} \ No newline at end of file +} From 960de3d378488fb6fda660692dce55b35e85850c Mon Sep 17 00:00:00 2001 From: "Hecker, Ronald" Date: Wed, 20 Nov 2024 16:16:15 +0100 Subject: [PATCH 05/33] Implement search No tests and implementation is a bit ugly. But it works. --- lib/pages/transcription/playground.dart | 15 +- lib/pages/transcription/utils/section.dart | 3 +- .../transcription/widgets/paragraph.dart | 95 +++++++++++ .../transcription/widgets/transcription.dart | 155 ++++++++---------- 4 files changed, 177 insertions(+), 91 deletions(-) create mode 100644 lib/pages/transcription/widgets/paragraph.dart diff --git a/lib/pages/transcription/playground.dart b/lib/pages/transcription/playground.dart index b1f8b549..e9b76721 100644 --- a/lib/pages/transcription/playground.dart +++ b/lib/pages/transcription/playground.dart @@ -6,6 +6,7 @@ import 'package:inference/pages/computer_vision/widgets/model_properties.dart'; import 'package:inference/pages/models/widgets/grid_container.dart'; import 'package:inference/pages/transcription/widgets/subtitles.dart'; import 'package:inference/pages/transcription/widgets/transcription.dart'; +import 'package:inference/pages/transcription/utils/message.dart'; import 'package:inference/project.dart'; import 'package:inference/pages/transcription/providers/speech_inference_provider.dart'; import 'package:inference/theme_fluent.dart'; @@ -130,9 +131,17 @@ class _PlaygroundState extends State with TickerProviderStateMixin{ width: 360, child: GridContainer( color: backgroundColor.of(theme), - child: Transcription( - onSeek: player.seek, - transcription: inference.transcription, + child: Builder( + builder: (context) { + if (inference.transcription == null) { + return Container(); + } + return Transcription( + onSeek: player.seek, + transcription: inference.transcription!, + messages: Message.parse(inference.transcription!.data, transcriptionPeriod), + ); + } ), ), ) diff --git a/lib/pages/transcription/utils/section.dart b/lib/pages/transcription/utils/section.dart index 27ede1d7..5c731b13 100644 --- a/lib/pages/transcription/utils/section.dart +++ b/lib/pages/transcription/utils/section.dart @@ -10,9 +10,10 @@ void moveToEnd(List list, I item) { class DynamicRangeLoading { List
sections = []; + int? size; Map data = {}; - DynamicRangeLoading(Section section): sections = [section]; + DynamicRangeLoading(Section section): sections = [section], size = section.end; Section get activeSection => sections.first; diff --git a/lib/pages/transcription/widgets/paragraph.dart b/lib/pages/transcription/widgets/paragraph.dart new file mode 100644 index 00000000..1189580f --- /dev/null +++ b/lib/pages/transcription/widgets/paragraph.dart @@ -0,0 +1,95 @@ + +import 'package:fluent_ui/fluent_ui.dart'; +import 'package:inference/theme_fluent.dart'; +import '../utils/message.dart'; + +String formatDuration(int totalSeconds) { + final duration = Duration(seconds: totalSeconds); + final minutes = duration.inMinutes; + final seconds = totalSeconds % 60; + + final minutesString = '$minutes'.padLeft(2, '0'); + final secondsString = '$seconds'.padLeft(2, '0'); + return '$minutesString:$secondsString'; +} + +class Paragraph extends StatefulWidget { + final Function(Duration)? onSeek; + final Message message; + final String? highlightedText; + + const Paragraph({super.key, required this.message, this.onSeek, this.highlightedText}); + + @override + State createState() => _ParagraphState(); +} + +class _ParagraphState extends State { + bool hover = false; + + @override + Widget build(BuildContext context) { + final theme = FluentTheme.of(context); + List pieces = []; + if (widget.highlightedText != null) { + final pattern = RegExp(widget.highlightedText!, caseSensitive: false); + final sections = widget.message.message.split(pattern); + if (sections.isNotEmpty) { + pieces.add(TextSpan(text: sections.first)); + for (int i = 1; i < sections.length; i++) { + pieces.add( + TextSpan( + text: widget.highlightedText!, + style: TextStyle(backgroundColor: theme.accentColor), + ) + ); + pieces.add(TextSpan(text: sections[i])); + } + } + } else { + pieces.add(TextSpan(text: widget.message.message)); + } + return MouseRegion( + onEnter: (_) { + setState(() => hover = true); + }, + onExit: (_) { + setState(() => hover = false); + }, + child: GestureDetector( + onTap: () { + widget.onSeek?.call(widget.message.position); + }, + child: Padding( + padding: const EdgeInsets.symmetric(vertical: 20, horizontal: 4), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Align( + alignment: Alignment.bottomRight, + child: Text(formatDuration(widget.message.position.inSeconds), + style: TextStyle( + fontSize: 9, + color: subtleTextColor.of(theme), + ) + ) + ), + Container( + decoration: BoxDecoration( + color: hover ? subtleTextColor.of(theme).withOpacity(0.3) : null, + borderRadius: const BorderRadius.all(Radius.circular(4)), + ), + padding: const EdgeInsets.symmetric(horizontal: 4, vertical: 2), + child: RichText( + text: TextSpan( + children: pieces + ) + ) + ), + ], + ), + ), + ), + ); + } +} diff --git a/lib/pages/transcription/widgets/transcription.dart b/lib/pages/transcription/widgets/transcription.dart index 9301aa11..8c040da0 100644 --- a/lib/pages/transcription/widgets/transcription.dart +++ b/lib/pages/transcription/widgets/transcription.dart @@ -5,27 +5,26 @@ import 'package:file_picker/file_picker.dart'; import 'package:fluent_ui/fluent_ui.dart'; import 'package:inference/interop/openvino_bindings.dart'; import 'package:inference/pages/transcription/utils/message.dart'; -import 'package:inference/pages/transcription/providers/speech_inference_provider.dart'; import 'package:inference/pages/transcription/utils/section.dart'; -import 'package:inference/theme_fluent.dart'; +import 'package:inference/pages/transcription/widgets/paragraph.dart'; import 'package:inference/widgets/controls/search_bar.dart'; -String formatDuration(int totalSeconds) { - final duration = Duration(seconds: totalSeconds); - final minutes = duration.inMinutes; - final seconds = totalSeconds % 60; - final minutesString = '$minutes'.padLeft(2, '0'); - final secondsString = '$seconds'.padLeft(2, '0'); - return '$minutesString:$secondsString'; -} - - - -class Transcription extends StatelessWidget { +class Transcription extends StatefulWidget { final DynamicRangeLoading>? transcription; final Function(Duration)? onSeek; - const Transcription({super.key, this.onSeek, this.transcription}); + final List messages; + const Transcription({super.key, this.onSeek, this.transcription, required this.messages}); + + @override + State createState() => _TranscriptionState(); +} + +class _TranscriptionState extends State { + final List _paragraphKeys = []; + final ScrollController _scrollController = ScrollController(); + final GlobalKey scrollKey = GlobalKey(); + String? searchText; void saveTranscript() async { final file = await FilePicker.platform.saveFile( @@ -37,9 +36,9 @@ class Transcription extends StatelessWidget { } String contents = ""; - final indices = transcription!.data.keys.toList()..sort(); + final indices = widget.transcription!.data.keys.toList()..sort(); for (int i in indices) { - final part = transcription!.data[i] as TranscriptionModelResponse; + final part = widget.transcription!.data[i] as TranscriptionModelResponse; for (final chunk in part.chunks) { contents += chunk.text; } @@ -48,29 +47,55 @@ class Transcription extends StatelessWidget { await File(file).writeAsString(contents); } - @override - Widget build(BuildContext context) { - if (transcription == null) { - return Container(); - } + void search(String text) { + setState(() { + searchText = text; + }); + + final pattern = RegExp(text, caseSensitive: false); + int? index; + for (int i = 0; i < widget.messages.length; i++) { + if (widget.messages[i].message.contains(pattern)) { + index = i; + break; + } - final messages = Message.parse(transcription!.data, transcriptionPeriod); + } + if (index != null){ + final context = _paragraphKeys[index].currentContext; + + if (context != null) { + final renderBox = context.findRenderObject() as RenderBox?; + if (renderBox != null) { + final position = renderBox.localToGlobal(Offset.zero, ancestor: scrollKey.currentContext?.findRenderObject()); + final offset = _scrollController.offset + position.dy; + _scrollController.animateTo( + offset, + duration: const Duration(milliseconds: 500), + curve: Curves.easeInOut, + ); + } + } + } + } + @override + Widget build(BuildContext context) { return Column( children: [ Padding( padding: const EdgeInsets.symmetric(vertical: 25, horizontal: 14), child: Row( children: [ - SearchBar(onChange: (p) {}, placeholder: "Search in transcript",), + SearchBar(onChange: search, placeholder: "Search in transcript",), Padding( padding: const EdgeInsets.only(left: 8.0), child: Tooltip( - message: transcription!.complete + message: widget.transcription!.complete ? "Download transcript" : "Transcribing...", child: Button( - onPressed: transcription!.complete + onPressed: widget.transcription?.complete ?? false ? () => saveTranscript() : null, child: const Padding( @@ -85,14 +110,27 @@ class Transcription extends StatelessWidget { ), Expanded( child: SingleChildScrollView( + key: scrollKey, + controller: _scrollController, child: Padding( padding: const EdgeInsets.only(left: 10, right: 18), child: Column( crossAxisAlignment: CrossAxisAlignment.start, - children: [ - for (final message in messages) - TranscriptionMessage(message: message, onSeek: onSeek) - ], + children: List.generate(widget.messages.length, (index) { + //Adjusting state in render is ugly. But might just work + if (_paragraphKeys.length <= index) { + print("length: ${_paragraphKeys.length}, index: $index"); + _paragraphKeys.add(GlobalKey()); + } + + return Paragraph( + key: _paragraphKeys[index], + message: widget.messages[index], + highlightedText: searchText, + onSeek: widget.onSeek, + ); + + }), ), ), ), @@ -101,60 +139,3 @@ class Transcription extends StatelessWidget { ); } } - -class TranscriptionMessage extends StatefulWidget { - final Function(Duration)? onSeek; - final Message message; - - const TranscriptionMessage({super.key, required this.message, this.onSeek}); - - @override - State createState() => _TranscriptionMessageState(); -} - -class _TranscriptionMessageState extends State { - bool hover = false; - - @override - Widget build(BuildContext context) { - final theme = FluentTheme.of(context); - return MouseRegion( - onEnter: (_) { - setState(() => hover = true); - }, - onExit: (_) { - setState(() => hover = false); - }, - child: GestureDetector( - onTap: () { - widget.onSeek?.call(widget.message.position); - }, - child: Padding( - padding: const EdgeInsets.symmetric(vertical: 20, horizontal: 4), - child: Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - Align( - alignment: Alignment.bottomRight, - child: Text(formatDuration(widget.message.position.inSeconds), - style: TextStyle( - fontSize: 9, - color: subtleTextColor.of(theme), - ) - ) - ), - Container( - decoration: BoxDecoration( - color: hover ? subtleTextColor.of(theme).withOpacity(0.3) : null, - borderRadius: const BorderRadius.all(Radius.circular(4)), - ), - padding: const EdgeInsets.symmetric(horizontal: 4, vertical: 2), - child: Text(widget.message.message) - ), - ], - ), - ), - ), - ); - } -} From 56ec80d5509ebf65678e75baa71a3829058025e9 Mon Sep 17 00:00:00 2001 From: "Hecker, Ronald" Date: Wed, 20 Nov 2024 16:28:02 +0100 Subject: [PATCH 06/33] Explain ugly state change on build --- lib/pages/transcription/widgets/transcription.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pages/transcription/widgets/transcription.dart b/lib/pages/transcription/widgets/transcription.dart index 8c040da0..b02fcdc2 100644 --- a/lib/pages/transcription/widgets/transcription.dart +++ b/lib/pages/transcription/widgets/transcription.dart @@ -117,9 +117,9 @@ class _TranscriptionState extends State { child: Column( crossAxisAlignment: CrossAxisAlignment.start, children: List.generate(widget.messages.length, (index) { - //Adjusting state in render is ugly. But might just work + // Adjusting state in render is ugly. But works. + // This is done because we need a global key but the paragraphs are added as you go. if (_paragraphKeys.length <= index) { - print("length: ${_paragraphKeys.length}, index: $index"); _paragraphKeys.add(GlobalKey()); } From 2c5db4afbbc55cbe286ae841ac3facddaf49f643 Mon Sep 17 00:00:00 2001 From: "Hecker, Ronald" Date: Wed, 20 Nov 2024 16:34:11 +0100 Subject: [PATCH 07/33] Hide performance metrics page for now --- lib/pages/transcription/transcription.dart | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/pages/transcription/transcription.dart b/lib/pages/transcription/transcription.dart index 14353d9d..0ea6f149 100644 --- a/lib/pages/transcription/transcription.dart +++ b/lib/pages/transcription/transcription.dart @@ -80,11 +80,11 @@ class _TranscriptionPageState extends State { title: const Text("Playground"), body: Playground(project: widget.project), ), - PaneItem( - icon: const Icon(FluentIcons.project_collection), - title: const Text("Performance metrics"), - body: Container(), - ), + //PaneItem( + // icon: const Icon(FluentIcons.project_collection), + // title: const Text("Performance metrics"), + // body: Container(), + //), ], ) ), From d4195a3a966511c54e2b5d1909463ec1c3107388 Mon Sep 17 00:00:00 2001 From: "Hecker, Ronald" Date: Wed, 20 Nov 2024 16:43:07 +0100 Subject: [PATCH 08/33] Fix old drop area from being used --- lib/pages/transcription/playground.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pages/transcription/playground.dart b/lib/pages/transcription/playground.dart index e9b76721..55771704 100644 --- a/lib/pages/transcription/playground.dart +++ b/lib/pages/transcription/playground.dart @@ -10,7 +10,7 @@ import 'package:inference/pages/transcription/utils/message.dart'; import 'package:inference/project.dart'; import 'package:inference/pages/transcription/providers/speech_inference_provider.dart'; import 'package:inference/theme_fluent.dart'; -import 'package:inference/utils/drop_area.dart'; +import 'package:inference/widgets/controls/drop_area.dart'; import 'package:inference/widgets/controls/no_outline_button.dart'; import 'package:inference/widgets/device_selector.dart'; import 'package:media_kit/media_kit.dart'; From f1084d6123cccf1605a0e234587f38f1119711af Mon Sep 17 00:00:00 2001 From: "Hecker, Ronald" Date: Wed, 20 Nov 2024 17:03:39 +0100 Subject: [PATCH 09/33] Fix color for paragraph in transcription --- lib/pages/transcription/widgets/paragraph.dart | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/pages/transcription/widgets/paragraph.dart b/lib/pages/transcription/widgets/paragraph.dart index 1189580f..c6ca4f16 100644 --- a/lib/pages/transcription/widgets/paragraph.dart +++ b/lib/pages/transcription/widgets/paragraph.dart @@ -82,6 +82,9 @@ class _ParagraphState extends State { padding: const EdgeInsets.symmetric(horizontal: 4, vertical: 2), child: RichText( text: TextSpan( + style: TextStyle( + color: theme.inactiveColor + ), children: pieces ) ) From 60e398665d6d39a08d889895401fd0a41074c82c Mon Sep 17 00:00:00 2001 From: "Hecker, Ronald" Date: Wed, 20 Nov 2024 17:05:17 +0100 Subject: [PATCH 10/33] Add dispose for video player --- lib/pages/transcription/playground.dart | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/pages/transcription/playground.dart b/lib/pages/transcription/playground.dart index 55771704..344d00fa 100644 --- a/lib/pages/transcription/playground.dart +++ b/lib/pages/transcription/playground.dart @@ -65,6 +65,12 @@ class _PlaygroundState extends State with TickerProviderStateMixin{ initializeVideoAndListeners(file); } + @override + void dispose() { + player.dispose(); + super.dispose(); + } + @override Widget build(BuildContext context) { final theme = FluentTheme.of(context); From a8ae6abbfa2873eb10777f80904d3e1037bdc05b Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Wed, 20 Nov 2024 17:08:55 +0100 Subject: [PATCH 11/33] Added model download screen and related tests --- integration_test/app_test.dart | 22 +- lib/importers/manifest_importer.dart | 47 ++++ lib/main.dart | 2 +- lib/openvino_console_app.dart | 2 +- lib/pages/download_model/download_model.dart | 218 +++++++++++++++++++ lib/pages/home/widgets/project_card.dart | 6 +- lib/pages/import/import.dart | 17 +- lib/pages/models/widgets/model_card.dart | 8 +- lib/project.dart | 2 +- lib/projects/projects_page.dart | 8 +- lib/providers/download_provider.dart | 11 +- lib/public_models.dart | 6 +- lib/router.dart | 9 +- lib/theme_fluent.dart | 2 + 14 files changed, 324 insertions(+), 36 deletions(-) create mode 100644 lib/pages/download_model/download_model.dart diff --git a/integration_test/app_test.dart b/integration_test/app_test.dart index 2c6bd98f..01eb345b 100644 --- a/integration_test/app_test.dart +++ b/integration_test/app_test.dart @@ -1,4 +1,3 @@ -import 'package:flutter/foundation.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:inference/main.dart'; import 'package:integration_test/integration_test.dart'; @@ -7,28 +6,27 @@ void main() { IntegrationTestWidgetsFlutterBinding.ensureInitialized(); testWidgets('Download model from HF', (tester) async { - final originalOnError = FlutterError.onError!; - const app = App(); await tester.pumpWidget(app); - FlutterError.onError = originalOnError; - - await tester.tap(find.text('Import Model')); + await tester.tap(find.text('Import model')); await tester.pumpAndSettle(); - await tester.tap(find.text('Huggingface')); + await tester.tap(find.text('Hugging Face')); await tester.pumpAndSettle(); - await tester.tap(find.bySemanticsLabel('Search by name')); + final searchBarFinder = find.bySemanticsLabel('Find a model').first; + await tester.tap(searchBarFinder, warnIfMissed: false); await tester.pumpAndSettle(); - await tester.enterText(find.bySemanticsLabel('Search by name'), 'tiny'); + await tester.enterText(searchBarFinder, 'tiny'); await tester.pumpAndSettle(); - await tester.tap(find.text('TinyLlama-1.1B-Chat-v1.0-int4-ov')); + await tester.tap(find.text('TinyLlama 1.1B Chat V1.0').first); await tester.pumpAndSettle(); - await tester.tap(find.text('Add model')); + await tester.tap(find.text('Import selected model')); await tester.pumpFrames(app, const Duration(seconds: 1)); - expect(find.textContaining(RegExp(r'^[1-9]\d* MB$')), findsNWidgets(2)); + expect(find.textContaining(RegExp(r'^[1-9][\d,]* MB$')), findsNWidgets(2)); + + await tester.pumpAndSettle(); }); } \ No newline at end of file diff --git a/lib/importers/manifest_importer.dart b/lib/importers/manifest_importer.dart index 116f272c..7391c7d5 100644 --- a/lib/importers/manifest_importer.dart +++ b/lib/importers/manifest_importer.dart @@ -1,9 +1,14 @@ import 'dart:convert'; +import 'dart:io'; import 'package:flutter/services.dart'; import 'package:flutter/widgets.dart'; +import 'package:inference/project.dart'; +import 'package:inference/public_model_info.dart'; import 'package:inference/utils/get_public_thumbnail.dart'; import 'package:inference/utils.dart'; +import 'package:path_provider/path_provider.dart'; +import 'package:uuid/uuid.dart'; class Model { final String name; @@ -45,6 +50,48 @@ class Model { task: json['task'], ); } + + Future convertToProject() async { + final directory = await getApplicationSupportDirectory(); + final projectId = const Uuid().v4(); + final storagePath = platformContext.join(directory.path, projectId.toString()); + await Directory(storagePath).create(recursive: true); + final projectType = parseProjectType(task); + + final project = PublicProject( + projectId, + "OpenVINO/$id", + "1.0.0", + name, + DateTime.now().toIso8601String(), + projectType, + storagePath, + thumbnail, + PublicModelInfo( + id, + DateTime.now().toIso8601String(), + 0, + 0, + task, + const Collection("https://huggingface.co/api/collections/OpenVINO/llm-6687aaa2abca3bbcec71a9bd", "", "text"), + ), + ); + + project.tasks.add( + Task( + genUUID(), + task, + task, + [], + null, + [], + "", + "", + ), + ); + + return project; + } } class ManifestImporter { diff --git a/lib/main.dart b/lib/main.dart index 37a169a3..d412bbe0 100644 --- a/lib/main.dart +++ b/lib/main.dart @@ -13,7 +13,7 @@ const String title = 'OpenVINO TestDrive'; void testConnection() async { final dio = Dio(BaseOptions(connectTimeout: Duration(seconds: 10))); - + try { await dio.get(collections[0].path); } on DioException catch(ex) { diff --git a/lib/openvino_console_app.dart b/lib/openvino_console_app.dart index 7fecf11f..f3c93cd4 100644 --- a/lib/openvino_console_app.dart +++ b/lib/openvino_console_app.dart @@ -60,7 +60,7 @@ class _OpenVINOTestDriveAppState extends State { fontPath().then((font) => ImageInference.setupFont(font)); }); - setupErrors(); + // setupErrors(); Device.getDevices().then((devices) { devices.forEach((p) => print("${p.id}, ${p.name}")); diff --git a/lib/pages/download_model/download_model.dart b/lib/pages/download_model/download_model.dart new file mode 100644 index 00000000..cd9ac830 --- /dev/null +++ b/lib/pages/download_model/download_model.dart @@ -0,0 +1,218 @@ +import 'dart:math'; + +import 'package:fluent_ui/fluent_ui.dart'; +import 'package:go_router/go_router.dart'; +import 'package:inference/project.dart'; +import 'package:inference/providers/download_provider.dart'; +import 'package:inference/providers/project_provider.dart'; +import 'package:inference/public_models.dart'; +import 'package:inference/theme_fluent.dart'; +import 'package:intl/intl.dart'; +import 'package:provider/provider.dart'; + +String formatBytes(int bytes) { + return "${NumberFormat("#,##0").format(bytes / pow(1024, 2))} MB"; +} + +class DownloadModelPage extends StatefulWidget { + final PublicProject project; + const DownloadModelPage({super.key, required this.project}); + @override + _DownloadModelPageState createState() => _DownloadModelPageState(); +} + +class _DownloadModelPageState extends State { + bool _isInitialized = false; + + @override + void didChangeDependencies() { + super.didChangeDependencies(); + if (!_isInitialized) { + _isInitialized = true; + Future.microtask(() { startDownload(); }); + } + } + + void startDownload() async { + final downloadProvider = Provider.of(context, listen: false); + final projectProvider = Provider.of(context, listen: false); + final router = GoRouter.of(context); + late Map files; + + projectProvider.addProject(widget.project); + + try { + files = await downloadFiles(widget.project); + } catch (e) { + await showDialog(context: context, builder: (BuildContext context) => ContentDialog( + title: const Text('Model was not found'), + actions: [ + Button( + onPressed: () { + router.canPop() ? router.pop() : router.go('/home'); + }, + child: const Text('Close'), + ), + ], + )); + return; + } + + try { + await downloadProvider.queue(files, widget.project.modelInfo?.collection.token); + await getAdditionalModelInfo(widget.project); + projectProvider.completeLoading(widget.project); + router.go("/models/inference", extra: widget.project); + } catch(e) { + if (mounted) { + await showDialog(context: context, builder: (BuildContext context) => ContentDialog( + title: Text('An error occurred trying to download ${widget.project.name}'), + content: Text(e.toString()), + actions: [ + Button( + onPressed: () { + router.canPop() ? router.pop() : router.go('/home'); + }, + child: const Text('Close'), + ), + ], + )); + } + } + } + + @override + Widget build(BuildContext context) { + final theme = FluentTheme.of(context); + return ScaffoldPage( + padding: const EdgeInsets.symmetric(horizontal: 24, vertical: 8.0), + header: Container( + decoration: BoxDecoration( + border: Border( + bottom: BorderSide( + color: theme.resources.controlStrokeColorDefault, + width: 1.0 + ) + ) + ), + height: 56, + padding: const EdgeInsets.symmetric(horizontal: 12.0), + child: Row( + mainAxisAlignment: MainAxisAlignment.spaceBetween, + crossAxisAlignment: CrossAxisAlignment.center, + children: [ + Row( + children: [ + Padding( + padding: const EdgeInsets.only(left: 12.0, bottom: 8), + child: ClipRRect( + borderRadius: BorderRadius.circular(4.0), + child: Container( + width: 40, + height: 40, + decoration: BoxDecoration( + image: DecorationImage( + image: widget.project.thumbnailImage(), + fit: BoxFit.fitWidth), + ), + ), + ), + ), + Padding( + padding: const EdgeInsets.symmetric(horizontal: 16), + child: Text(widget.project.name, + style: const TextStyle(fontSize: 20, fontWeight: FontWeight.bold), + ), + ), + ], + ), + Button(child: const Text("Close"), onPressed: () { + GoRouter.of(context).canPop() ? GoRouter.of(context).pop() : GoRouter.of(context).go('/home'); + }), + ], + ), + ), + content: Row( + children: [ + Expanded( + child: Column( + mainAxisAlignment: MainAxisAlignment.center, + crossAxisAlignment: CrossAxisAlignment.center, + children: [ + Padding( + padding: const EdgeInsets.all(16.0), + child: Consumer(builder: (context, downloadProvider, child) { + final stats = downloadProvider.stats; + return Column( + children: [ + ProgressRing( + value: stats.percentage * 100, + strokeWidth: 8, + ), + SizedBox( + width: 140, + child: Padding( + padding: const EdgeInsets.only(top: 8.0), + child: Row( + mainAxisAlignment: MainAxisAlignment.spaceBetween, + children: [ + Text(formatBytes(stats.received), textAlign: TextAlign.end,), + const Text("/"), + Text(formatBytes(stats.total)) + ], + ), + ), + ), + const Padding( + padding: EdgeInsets.all(8.0), + child: Text("Downloading model weights"), + ) + ] + ); + } + ), + ) + ], + ), + ), + Container( + width: 280, + decoration: BoxDecoration( + border: Border( + left: BorderSide( + color: theme.resources.controlStrokeColorDefault, + width: 1.0, + ), + ), + ), + child: Padding( + padding: const EdgeInsets.symmetric(horizontal: 24), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + const Padding( + padding: EdgeInsets.symmetric(vertical: 18.0), + child: Text("Model parameters", style: TextStyle(fontSize: 20, fontWeight: FontWeight.w600)), + ), + const Padding( + padding: EdgeInsets.only(top: 16), + child: Text("Model name", style: TextStyle(fontSize: 14),), + ), + Padding(padding: const EdgeInsets.only(top: 4), + child: Text(widget.project.modelId, style: const TextStyle(fontSize: 14, color: foreground3Color),), + ), + const Padding(padding: EdgeInsets.only(top: 16), + child: Text("Task", style: TextStyle(fontSize: 14),), + ), + Padding(padding: const EdgeInsets.only(top: 4), + child: Text(widget.project.taskName(), style: const TextStyle(fontSize: 14, color: foreground3Color),), + ), + ], + ), + ), + ), + ], + ) + ); + } +} diff --git a/lib/pages/home/widgets/project_card.dart b/lib/pages/home/widgets/project_card.dart index 5177d32d..7654b5e3 100644 --- a/lib/pages/home/widgets/project_card.dart +++ b/lib/pages/home/widgets/project_card.dart @@ -1,4 +1,5 @@ import 'package:fluent_ui/fluent_ui.dart'; +import 'package:go_router/go_router.dart'; import 'package:inference/project.dart'; import 'package:inference/providers/project_provider.dart'; import 'package:inference/widgets/elevation.dart'; @@ -11,6 +12,7 @@ class ProjectCard extends StatelessWidget { @override Widget build(BuildContext context) { final theme = FluentTheme.of(context); + final router = GoRouter.of(context); return Elevation( backgroundColor: theme.cardColor, shape: RoundedRectangleBorder(borderRadius: BorderRadius.circular(4)), @@ -87,7 +89,9 @@ class ProjectCard extends StatelessWidget { child: Row( mainAxisAlignment: MainAxisAlignment.spaceBetween, children: [ - FilledButton(onPressed: () {}, child: const Row(children: [ + FilledButton(onPressed: () { + project.isDownloaded ? router.push('/models/inference', extra: project) : router.push('/models/download', extra: project); + }, child: const Row(children: [ Icon(FluentIcons.pop_expand,), Padding( padding: EdgeInsets.symmetric(horizontal: 8.0), diff --git a/lib/pages/import/import.dart b/lib/pages/import/import.dart index bf3f540f..97ffd82a 100644 --- a/lib/pages/import/import.dart +++ b/lib/pages/import/import.dart @@ -42,6 +42,7 @@ class _ImportPageState extends State { @override Widget build(BuildContext context) { final theme = FluentTheme.of(context); + final router = GoRouter.of(context); return ScaffoldPage.scrollable( padding: const EdgeInsets.symmetric(horizontal: 24, vertical: 8.0), header: Container( @@ -68,11 +69,14 @@ class _ImportPageState extends State { Row( children: [ FilledButton(onPressed: selectedModel == null ? (null) : () { - GoRouter.of(context).go('/models/download', extra: selectedModel); + selectedModel?.convertToProject().then((project) { + router.push('/models/download', extra: project); + }); + }, child: const Text('Import selected model'),), Padding( padding: const EdgeInsets.only(left: 8.0), - child: Button(child: const Text('Close'), onPressed: () { GoRouter.of(context).pop(); }), + child: Button(child: const Text('Close'), onPressed: () { router.pop(); }), ) ], ) @@ -89,9 +93,12 @@ class _ImportPageState extends State { children: [ ConstrainedBox( constraints: const BoxConstraints(maxWidth: 280), - child: SearchBar(onChange: (value) { setState(() { - searchValue = value; - }); }, placeholder: 'Find a model',), + child: Semantics( + label: 'Find a model', + child: SearchBar(onChange: (value) { setState(() { + searchValue = value; + }); }, placeholder: 'Find a model',), + ), ), Padding( padding: const EdgeInsets.only(left: 8), diff --git a/lib/pages/models/widgets/model_card.dart b/lib/pages/models/widgets/model_card.dart index e3f0a4c5..d6817f97 100644 --- a/lib/pages/models/widgets/model_card.dart +++ b/lib/pages/models/widgets/model_card.dart @@ -22,7 +22,13 @@ class _ModelCardState extends State{ final theme = FluentTheme.of(context); return GestureDetector( - onTap: () => GoRouter.of(context).go("/models/inference", extra: widget.project), + onTap: () { + if (widget.project.isDownloaded) { + GoRouter.of(context).go("/models/inference", extra: widget.project); + } else { + GoRouter.of(context).go("/models/download", extra: widget.project); + } + }, child: MouseRegion( cursor: SystemMouseCursors.click, child: Elevation( diff --git a/lib/project.dart b/lib/project.dart index cfe9c0da..09ae9411 100644 --- a/lib/project.dart +++ b/lib/project.dart @@ -117,7 +117,7 @@ ProjectType parseProjectType(String name) { if (name == "image") { return ProjectType.image; } - if (name == "text"){ + if (name == "text" || name == "text-generation"){ return ProjectType.text; } if (name == "speech") { diff --git a/lib/projects/projects_page.dart b/lib/projects/projects_page.dart index 7c80609b..e5ff9738 100644 --- a/lib/projects/projects_page.dart +++ b/lib/projects/projects_page.dart @@ -44,10 +44,10 @@ class _ProjectPageState extends State with TickerProviderStateMixi void initState() { super.initState(); - FlutterError.onError = (details) { - FlutterError.presentError(details); - //showErrorDialog(details.exception.toString(), details.stack.toString()); - }; + // FlutterError.onError = (details) { + // FlutterError.presentError(details); + // //showErrorDialog(details.exception.toString(), details.stack.toString()); + // }; //PlatformDispatcher.instance.onError = (error, stack) { // showErrorDialog(error.toString(), stack.toString()); // return true; diff --git a/lib/providers/download_provider.dart b/lib/providers/download_provider.dart index 1a109c41..5a36836f 100644 --- a/lib/providers/download_provider.dart +++ b/lib/providers/download_provider.dart @@ -39,7 +39,7 @@ class DownloadProvider extends ChangeNotifier { _downloads[url] = state; final destination = downloads[url]; Map headers = {}; - if (token != null) { + if (token != null && token.isNotEmpty) { headers["Authorization"] = "Bearer $token"; } final promise = dio.download(url, destination, @@ -51,15 +51,16 @@ class DownloadProvider extends ChangeNotifier { state.total = total; notifyListeners(); } - }); - promise.catchError((e) { + }, + ).catchError((e) { if (e is DioException && e.type == DioExceptionType.cancel) { print("Download cancelled: $url"); + return Response(requestOptions: RequestOptions(path: url)); } else { _cancelToken?.cancel(); + throw e; } - }); - promise.then((_) => state.done); + }).then((_) => state.done); promises.add(promise); } diff --git a/lib/public_models.dart b/lib/public_models.dart index 0305bb62..25eadf5f 100644 --- a/lib/public_models.dart +++ b/lib/public_models.dart @@ -30,7 +30,7 @@ void writeProjectJson(PublicProject project) { } Future getAdditionalModelInfo(PublicProject project) async { - final configJsonURL = huggingFaceModelFileUrl(project.id, "config.json"); + final configJsonURL = huggingFaceModelFileUrl(project.modelId, "config.json"); final config = jsonDecode((await http.get( Uri.parse(configJsonURL), headers: { @@ -48,8 +48,8 @@ Future> getFilesForModel(String modelId) async { } Future> downloadFiles(PublicProject project) async { - final files = await getFilesForModel(project.id); - return { for (var v in files) huggingFaceModelFileUrl(project.id, v) : platformContext.join(project.storagePath, v) }; + final files = await getFilesForModel(project.modelId); + return { for (var v in files) huggingFaceModelFileUrl(project.modelId, v) : platformContext.join(project.storagePath, v) }; } String huggingFaceModelFileUrl(String modelId, String name) { diff --git a/lib/router.dart b/lib/router.dart index b3498239..8a9bd750 100644 --- a/lib/router.dart +++ b/lib/router.dart @@ -1,11 +1,14 @@ import 'package:fluent_ui/fluent_ui.dart'; import 'package:go_router/go_router.dart'; -import 'package:inference/importers/manifest_importer.dart'; import 'package:inference/openvino_console_app.dart'; +import 'package:inference/pages/download_model/download_model.dart'; import 'package:inference/pages/home/home.dart'; import 'package:inference/pages/import/import.dart'; import 'package:inference/pages/models/models.dart'; +import 'package:inference/project.dart'; +import 'package:inference/providers/download_provider.dart'; +import 'package:provider/provider.dart'; final rootNavigatorKey = GlobalKey(); final _shellNavigatorKey = GlobalKey(); @@ -22,7 +25,9 @@ final router = GoRouter(navigatorKey: rootNavigatorKey, GoRoute(path: '/home', builder: (context, state) => const HomePage()), GoRoute(path: '/models', builder: (context, state) => const ModelsPage()), GoRoute(path: '/models/import', builder: (context, state) => const ImportPage()), - GoRoute(path: '/models/download', builder: (context, state) => Container(color: Colors.blue, child: Text('Downloading model: ${(state.extra as Model).id}'))), + GoRoute(path: '/models/download', builder: (context, state) => ChangeNotifierProvider(create: (_) => + DownloadProvider(state.extra as PublicProject), child: DownloadModelPage(project: state.extra as PublicProject)), + ), ], ) ] diff --git a/lib/theme_fluent.dart b/lib/theme_fluent.dart index d3ea728c..c1e26b99 100644 --- a/lib/theme_fluent.dart +++ b/lib/theme_fluent.dart @@ -85,6 +85,8 @@ class AppTheme extends ChangeNotifier { } +const foreground3Color = Color(0xFF616161); + final AccentColor electricCosmos = AccentColor.swatch(const { 'normal': Color(0xFF7000FF), }); From 72de8efb51dba110406af094612147c97fdf977f Mon Sep 17 00:00:00 2001 From: "Hecker, Ronald" Date: Wed, 20 Nov 2024 17:10:27 +0100 Subject: [PATCH 12/33] Reimplement section tests and fix test for computer vision --- .../computer_vision/computer_vision.dart | 1 - .../model_properties_test.dart | 4 +- .../transcriptions/utils/section_test.dart | 99 +++++++++++++++++ test/section_test.dart | 100 ------------------ 4 files changed, 101 insertions(+), 103 deletions(-) create mode 100644 test/pages/transcriptions/utils/section_test.dart delete mode 100644 test/section_test.dart diff --git a/lib/pages/computer_vision/computer_vision.dart b/lib/pages/computer_vision/computer_vision.dart index 8c7522c3..715a270e 100644 --- a/lib/pages/computer_vision/computer_vision.dart +++ b/lib/pages/computer_vision/computer_vision.dart @@ -2,7 +2,6 @@ import 'package:fluent_ui/fluent_ui.dart'; import 'package:go_router/go_router.dart'; import 'package:inference/pages/computer_vision/batch_inference.dart'; import 'package:inference/pages/computer_vision/live_inference.dart'; -import 'package:inference/pages/models/widgets/grid_container.dart'; import 'package:inference/project.dart'; import 'package:inference/providers/image_inference_provider.dart'; import 'package:inference/providers/preference_provider.dart'; diff --git a/test/pages/computer_vision/model_properties_test.dart b/test/pages/computer_vision/model_properties_test.dart index 70d1e6ef..d034774f 100644 --- a/test/pages/computer_vision/model_properties_test.dart +++ b/test/pages/computer_vision/model_properties_test.dart @@ -22,8 +22,8 @@ Widget testWidget(ImageInferenceProvider provider) { ), ], child: FluentApp( - home: const Center( - child: ModelProperties() + home: Center( + child: ModelProperties(project: provider.project) ), ), ); diff --git a/test/pages/transcriptions/utils/section_test.dart b/test/pages/transcriptions/utils/section_test.dart new file mode 100644 index 00000000..3452a625 --- /dev/null +++ b/test/pages/transcriptions/utils/section_test.dart @@ -0,0 +1,99 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:inference/pages/transcription/utils/section.dart'; + +void main() { + group("Section", () { + group("process", () { + test("process sets values in data", () async { + final state = DynamicRangeLoading(Section(0, 10)); + for (int j = 0; j < 10; j++) { + await state.process((i) async { + return j; + }); + + expect(state.data[j], j); + } + }); + + test("process out of bounds throws error", () async { + final state = DynamicRangeLoading(Section(0, 10)); + for (int j = 0; j < 10; j++) { + await state.process((i) async { + return j; + }); + } + + expect(() async { + await state.process((i) async { + return 1; + }); + }, throwsException); + }); + + test("process continues after skip is done", () async { + final state = DynamicRangeLoading(Section(0, 10)); + state.skipTo(8); + for (int j = 0; j < 2; j++) { + await state.process((i) async { + return j; + }); + } + expect(state.getNextIndex(), 0); + }); + + }); + + test('getNextIndex throws error when state is complete', () { + final state = DynamicRangeLoading(Section(0, 0)); + expect(() { + state.getNextIndex(); + },throwsException); + }); + + test('complete', () async { + final state = DynamicRangeLoading(Section(0, 10)); + for (int j = 0; j < 10; j++) { + expect(state.complete, false); + await state.process((i) async { + return j; + }); + } + expect(state.complete, true); + }); + + group("skip", () { + test("skips to specific index", () async { + final state = DynamicRangeLoading(Section(0, 10)); + state.skipTo(5); + expect(state.getNextIndex(), 5); + expect(state.activeSection.begin, 5); + expect(state.activeSection.end, 10); + }); + + test("skips to partially complete section will go to end of that section ", () async { + final state = DynamicRangeLoading(Section(0, 10)); + + for (int j = 0; j < 8; j++) { + await state.process((i) async { + return j; + }); + } + state.skipTo(5); + expect(state.getNextIndex(), 8); + }); + + test("skips to fully complete section will not shift next index", () async { + final state = DynamicRangeLoading(Section(0, 10)); + state.skipTo(5); + + for (int j = 0; j < 5; j++) { + await state.process((i) async { + return j; + }); + } + state.skipTo(5); + expect(state.getNextIndex(), 0); + }); + }); + }); +} diff --git a/test/section_test.dart b/test/section_test.dart deleted file mode 100644 index 7aebd689..00000000 --- a/test/section_test.dart +++ /dev/null @@ -1,100 +0,0 @@ -import 'package:flutter_test/flutter_test.dart'; - -void main() { - /* - group("Section", () { - group("process", () { - // test("process sets values in data", () async { - // final state = DynamicRangeLoading(Section(0, 10)); - // for (int j = 0; j < 10; j++) { - // await state.process((i) async { - // return j; - // }); - - // expect(state.data[j], j); - // } - // }); - - // test("process out of bounds throws error", () async { - // final state = DynamicRangeLoading(Section(0, 10)); - // for (int j = 0; j < 10; j++) { - // await state.process((i) async { - // return j; - // }); - // } - - // expect(() async { - // await state.process((i) async { - // return 1; - // }); - // }, throwsException); - // }); - - // test("process continues after skip is done", () async { - // final state = DynamicRangeLoading(Section(0, 10)); - // state.skipTo(8); - // for (int j = 0; j < 2; j++) { - // await state.process((i) async { - // return j; - // }); - // } - // expect(state.getNextIndex(), 0); - // }); - - // }); - - // test('getNextIndex throws error when state is complete', () { - // final state = DynamicRangeLoading(Section(0, 0)); - // expect(() { - // state.getNextIndex(); - // },throwsException); - // }); - - // test('complete', () async { - // final state = DynamicRangeLoading(Section(0, 10)); - // for (int j = 0; j < 10; j++) { - // expect(state.complete, false); - // await state.process((i) async { - // return j; - // }); - // } - // expect(state.complete, true); - // }); - - // group("skip", () { - // test("skips to specific index", () async { - // final state = DynamicRangeLoading(Section(0, 10)); - // state.skipTo(5); - // expect(state.getNextIndex(), 5); - // expect(state.activeSection.begin, 5); - // expect(state.activeSection.end, 10); - // }); - - // test("skips to partially complete section will go to end of that section ", () async { - // final state = DynamicRangeLoading(Section(0, 10)); - - // for (int j = 0; j < 8; j++) { - // await state.process((i) async { - // return j; - // }); - // } - // state.skipTo(5); - // expect(state.getNextIndex(), 8); - // }); - - // test("skips to fully complete section will not shift next index", () async { - // final state = DynamicRangeLoading(Section(0, 10)); - // state.skipTo(5); - - // for (int j = 0; j < 5; j++) { - // await state.process((i) async { - // return j; - // }); - // } - // state.skipTo(5); - // expect(state.getNextIndex(), 0); - // }); - }); - }); - */ -} From fb63326cf0ca20ab13a6c510371c08e56acdb068 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Wed, 20 Nov 2024 17:15:06 +0100 Subject: [PATCH 13/33] Reverted errors debug --- lib/openvino_console_app.dart | 2 +- lib/projects/projects_page.dart | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/openvino_console_app.dart b/lib/openvino_console_app.dart index b31778fb..4cc19a85 100644 --- a/lib/openvino_console_app.dart +++ b/lib/openvino_console_app.dart @@ -60,7 +60,7 @@ class _OpenVINOTestDriveAppState extends State { fontPath().then((font) => ImageInference.setupFont(font)); }); - // setupErrors(); + setupErrors(); Device.getDevices().then((devices) { devices.forEach((p) => print("${p.id}, ${p.name}")); diff --git a/lib/projects/projects_page.dart b/lib/projects/projects_page.dart index e5ff9738..ac6daf5d 100644 --- a/lib/projects/projects_page.dart +++ b/lib/projects/projects_page.dart @@ -9,7 +9,6 @@ import 'package:inference/config.dart'; import 'package:inference/header.dart'; import 'package:inference/importers/importer.dart'; import 'package:inference/project.dart'; -import 'package:inference/projects/task_type_filter.dart'; import 'package:inference/providers/project_filter_provider.dart'; import 'package:inference/providers/project_provider.dart'; import 'package:inference/searchbar.dart'; @@ -44,10 +43,10 @@ class _ProjectPageState extends State with TickerProviderStateMixi void initState() { super.initState(); - // FlutterError.onError = (details) { - // FlutterError.presentError(details); - // //showErrorDialog(details.exception.toString(), details.stack.toString()); - // }; + FlutterError.onError = (details) { + FlutterError.presentError(details); + //showErrorDialog(details.exception.toString(), details.stack.toString()); + }; //PlatformDispatcher.instance.onError = (error, stack) { // showErrorDialog(error.toString(), stack.toString()); // return true; From ca749ac872b6d8e714ddc35572c00e812f3fedb0 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Wed, 20 Nov 2024 17:33:44 +0100 Subject: [PATCH 14/33] Moved project creation after files download --- lib/pages/download_model/download_model.dart | 3 +-- lib/providers/download_provider.dart | 3 --- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/pages/download_model/download_model.dart b/lib/pages/download_model/download_model.dart index cd9ac830..9a4f60b9 100644 --- a/lib/pages/download_model/download_model.dart +++ b/lib/pages/download_model/download_model.dart @@ -39,8 +39,6 @@ class _DownloadModelPageState extends State { final router = GoRouter.of(context); late Map files; - projectProvider.addProject(widget.project); - try { files = await downloadFiles(widget.project); } catch (e) { @@ -60,6 +58,7 @@ class _DownloadModelPageState extends State { try { await downloadProvider.queue(files, widget.project.modelInfo?.collection.token); + projectProvider.addProject(widget.project); await getAdditionalModelInfo(widget.project); projectProvider.completeLoading(widget.project); router.go("/models/inference", extra: widget.project); diff --git a/lib/providers/download_provider.dart b/lib/providers/download_provider.dart index 5a36836f..0c61d2db 100644 --- a/lib/providers/download_provider.dart +++ b/lib/providers/download_provider.dart @@ -1,7 +1,4 @@ -import 'dart:io'; - import 'package:dio/dio.dart'; -import 'package:dio/io.dart'; import 'package:flutter/foundation.dart'; import 'package:inference/deployment_processor.dart'; import 'package:inference/project.dart'; From bb635ac6498cd707cdd80b3a10b8284595fba07c Mon Sep 17 00:00:00 2001 From: Ronald Hecker Date: Thu, 21 Nov 2024 14:51:39 +0100 Subject: [PATCH 15/33] Lock ffmpeg in windows to 6.1.1 --- .github/workflows/windows-build.yaml | 3 ++- openvino_bindings/README.md | 6 +++++- openvino_bindings/WORKSPACE | 24 ++++++++++++------------ openvino_bindings/third_party/.gitignore | 1 + openvino_bindings/third_party/vcpkg.json | 10 ++++++++++ 5 files changed, 30 insertions(+), 14 deletions(-) create mode 100644 openvino_bindings/third_party/.gitignore create mode 100644 openvino_bindings/third_party/vcpkg.json diff --git a/.github/workflows/windows-build.yaml b/.github/workflows/windows-build.yaml index e0beccd2..8b47c6d4 100644 --- a/.github/workflows/windows-build.yaml +++ b/.github/workflows/windows-build.yaml @@ -76,7 +76,8 @@ jobs: run: | git clone https://github.com/microsoft/vcpkg.git C:\vcpkg C:\vcpkg\bootstrap-vcpkg.bat - C:\vcpkg\vcpkg install ffmpeg + cd openvino_bindings/third_party + C:\vcpkg\vcpkg install shell: cmd # Step 10: Download and Install OpenVINO Runtime diff --git a/openvino_bindings/README.md b/openvino_bindings/README.md index d6f8c219..4715bef7 100644 --- a/openvino_bindings/README.md +++ b/openvino_bindings/README.md @@ -95,7 +95,11 @@ A step by step guide can be found [here]('./docs/WINDOWS.md'). [Install OpenVINO Runtime 24.5.0]( https://docs.openvino.ai/2024/get-started/install-openvino.html?PACKAGE=OPENVINO_GENAI&VERSION=v_2024_4_0&OP_SYSTEM=WINDOWS&DISTRIBUTION=ARCHIVE) with GenAI flavor in `C:/Intel/openvino_24.5.0`. Build OpenCV in `C:/opencv/build`. -Install ffmpeg: `vcpkg install ffmpeg`. +Install ffmpeg: +```sh +cd openvino_bindings/third_party +vcpkg install +``` Install [mediapipe requirements](https://ai.google.dev/edge/mediapipe/framework/getting_started/install#installing_on_windows) and setup the environment variables. diff --git a/openvino_bindings/WORKSPACE b/openvino_bindings/WORKSPACE index 6a1707a4..1238a9d8 100644 --- a/openvino_bindings/WORKSPACE +++ b/openvino_bindings/WORKSPACE @@ -106,23 +106,23 @@ git_repository( tag = "v3.11.3", ) -#new_local_repository( -# name = "linux_ffmpeg", -# build_file = "//third_party/ffmpeg:linux.BUILD", -# path = "/usr" -#) -# +new_local_repository( + name = "linux_ffmpeg", + build_file = "//third_party/ffmpeg:linux.BUILD", + path = "/usr" +) + new_local_repository( name = "mac_ffmpeg", build_file = "//third_party/ffmpeg:mac.BUILD", path = "/opt/homebrew/opt/ffmpeg@6", ) -# -#new_local_repository( -# name = "windows_ffmpeg", -# build_file = "//third_party/ffmpeg:windows.BUILD", -# path = "C:/vcpkg/packages/ffmpeg_x64-windows", -#) + +new_local_repository( + name = "windows_ffmpeg", + build_file = "//third_party/ffmpeg:windows.BUILD", + path = "./third_party/vcpkg_installed/x64-windows", +) http_archive( name = "rules_pkg", diff --git a/openvino_bindings/third_party/.gitignore b/openvino_bindings/third_party/.gitignore new file mode 100644 index 00000000..8a1403ea --- /dev/null +++ b/openvino_bindings/third_party/.gitignore @@ -0,0 +1 @@ +vcpkg_installed diff --git a/openvino_bindings/third_party/vcpkg.json b/openvino_bindings/third_party/vcpkg.json new file mode 100644 index 00000000..2497f47c --- /dev/null +++ b/openvino_bindings/third_party/vcpkg.json @@ -0,0 +1,10 @@ +{ + "name": "openvinotestdrivebindings", + "builtin-baseline": "c8582b4d83dbd36e1bebc08bf166b5eb807996b0", + "dependencies": [ + "ffmpeg" + ], + "overrides": [ + { "name": "ffmpeg", "version": "6.1.1" } + ] +} From bc223d4afb59a6e6b5fbd23241b2a2cfc997eb3c Mon Sep 17 00:00:00 2001 From: Ronald Hecker Date: Thu, 21 Nov 2024 15:00:27 +0100 Subject: [PATCH 16/33] fix windows build flutter error --- .github/workflows/windows-build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows-build.yaml b/.github/workflows/windows-build.yaml index 8b47c6d4..70d7c3ef 100644 --- a/.github/workflows/windows-build.yaml +++ b/.github/workflows/windows-build.yaml @@ -127,7 +127,7 @@ jobs: - uses: subosito/flutter-action@v2 with: channel: 'stable' - flutter-version: '3.24.0' + flutter-version: '3.24.5' - name: Install project dependencies run: flutter pub get - name: Generate intermediates From 3491c2a56dad2dba83ddd1985386dc786a2880b3 Mon Sep 17 00:00:00 2001 From: Ronald Hecker Date: Thu, 21 Nov 2024 15:17:05 +0100 Subject: [PATCH 17/33] fix ffmpeg vcpkg hopefully --- .github/workflows/windows-build.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/windows-build.yaml b/.github/workflows/windows-build.yaml index 70d7c3ef..440b880c 100644 --- a/.github/workflows/windows-build.yaml +++ b/.github/workflows/windows-build.yaml @@ -73,12 +73,12 @@ jobs: # Step 9: Install vcpkg and ffmpeg - name: Install vcpkg and ffmpeg + shell: powershell run: | - git clone https://github.com/microsoft/vcpkg.git C:\vcpkg + if (!(Test-Path "C:\vcpkg")) { git clone https://github.com/microsoft/vcpkg.git C:\vcpkg } C:\vcpkg\bootstrap-vcpkg.bat cd openvino_bindings/third_party C:\vcpkg\vcpkg install - shell: cmd # Step 10: Download and Install OpenVINO Runtime - name: Download and Install OpenVINO Runtime 24.5.0 From f58ea0cd71e66a93464493b7aff86fe9d0fa43ac Mon Sep 17 00:00:00 2001 From: Ronald Hecker Date: Fri, 22 Nov 2024 13:52:22 +0100 Subject: [PATCH 18/33] Implement performance metrics page Added metrics for transcription page Fixes subtitles for light colorscheme --- .../transcription/performance_metrics.dart | 87 +++++++++++++++++++ lib/pages/transcription/playground.dart | 9 ++ .../providers/speech_inference_provider.dart | 16 +++- lib/pages/transcription/transcription.dart | 11 +-- lib/pages/transcription/utils/metrics.dart | 54 ++++++++++++ .../transcription/widgets/subtitles.dart | 3 +- lib/widgets/performance_tile.dart | 65 ++++++++++++++ 7 files changed, 238 insertions(+), 7 deletions(-) create mode 100644 lib/pages/transcription/performance_metrics.dart create mode 100644 lib/pages/transcription/utils/metrics.dart create mode 100644 lib/widgets/performance_tile.dart diff --git a/lib/pages/transcription/performance_metrics.dart b/lib/pages/transcription/performance_metrics.dart new file mode 100644 index 00000000..a12b5548 --- /dev/null +++ b/lib/pages/transcription/performance_metrics.dart @@ -0,0 +1,87 @@ +import 'package:fluent_ui/fluent_ui.dart'; +import 'package:inference/pages/computer_vision/widgets/horizontal_rule.dart'; +import 'package:inference/pages/transcription/providers/speech_inference_provider.dart'; +import 'package:inference/widgets/performance_tile.dart'; +import 'package:intl/intl.dart'; +import 'package:provider/provider.dart'; + +class PerformanceMetrics extends StatelessWidget { + const PerformanceMetrics({super.key}); + + @override + Widget build(BuildContext context) { + return Consumer( + builder: (context, inference, child) { + final metrics = inference.metrics; + if (metrics == null) { + return Container(); + } + + Locale locale = Localizations.localeOf(context); + final nf = NumberFormat.decimalPatternDigits( + locale: locale.languageCode, decimalDigits: 0); + + return Padding( + padding: const EdgeInsets.symmetric(vertical: 80), + child: Center( + child: SizedBox( + width: 887, + child: Column( + children: [ + Row( + mainAxisAlignment: MainAxisAlignment.spaceEvenly, + children: [ + PerformanceTile( + title: "Time to first token (TTFT)", + value: nf.format(metrics.ttft), + unit: "ms", + tall: true, + ), + PerformanceTile( + title: "Time per output token (TPOT)", + value: nf.format(metrics.tpot), + unit: "ms", + tall: true, + ), + PerformanceTile( + title: "Generate total duration", + value: nf.format(metrics.generateTime), + unit: "ms", + tall: true, + ), + ], + ), + const Padding( + padding: EdgeInsets.symmetric(horizontal: 16.0, vertical: 16), + child: HorizontalRule(), + ), + Row( + mainAxisAlignment: MainAxisAlignment.spaceEvenly, + children: [ + PerformanceTile( + title: "Load time", + value: nf.format(metrics.loadTime), + unit: "ms", + ), + PerformanceTile( + title: "Detokenization duration", + value: nf.format(metrics.detokenizationTime), + unit: "ms", + ), + PerformanceTile( + title: "Throughput", + value: nf.format(metrics.throughput), + unit: "tokens/sec", + ), + ], + ), + ], + ), + ), + ), + ); + } + ); + } +} + diff --git a/lib/pages/transcription/playground.dart b/lib/pages/transcription/playground.dart index 344d00fa..5f3e03b1 100644 --- a/lib/pages/transcription/playground.dart +++ b/lib/pages/transcription/playground.dart @@ -65,6 +65,15 @@ class _PlaygroundState extends State with TickerProviderStateMixin{ initializeVideoAndListeners(file); } + @override + void initState() { + super.initState(); + final inference = Provider.of(context, listen: false); + if (inference.videoPath != null) { + initializeVideoAndListeners(inference.videoPath!); + } + } + @override void dispose() { player.dispose(); diff --git a/lib/pages/transcription/providers/speech_inference_provider.dart b/lib/pages/transcription/providers/speech_inference_provider.dart index 8e574f2b..fedbf723 100644 --- a/lib/pages/transcription/providers/speech_inference_provider.dart +++ b/lib/pages/transcription/providers/speech_inference_provider.dart @@ -3,9 +3,11 @@ import 'dart:async'; import 'package:flutter/material.dart'; import 'package:inference/interop/openvino_bindings.dart'; import 'package:inference/interop/speech_to_text.dart'; +import 'package:inference/pages/transcription/utils/metrics.dart'; import 'package:inference/pages/transcription/utils/section.dart'; import 'package:inference/project.dart'; + const transcriptionPeriod = 10; class SpeechInferenceProvider extends ChangeNotifier { @@ -21,6 +23,7 @@ class SpeechInferenceProvider extends ChangeNotifier { bool get videoLoaded => _videoPath != null; DynamicRangeLoading>? transcription; + DMetrics? metrics; bool get transcriptionComplete { return transcription?.complete ?? false; @@ -62,6 +65,15 @@ class SpeechInferenceProvider extends ChangeNotifier { notifyListeners(); } + void addMetrics(TranscriptionModelResponse response) { + if (metrics == null) { + metrics = DMetrics.fromCMetrics(response.metrics); + } else { + metrics!.addCMetrics(response.metrics); + } + notifyListeners(); + } + Future startTranscribing() async { if (transcription == null) { throw Exception("Can't transcribe before loading video"); @@ -72,7 +84,9 @@ class SpeechInferenceProvider extends ChangeNotifier { return; } await transcription!.process((int i) { - return transcribe(i * transcriptionPeriod, transcriptionPeriod); + final request = transcribe(i * transcriptionPeriod, transcriptionPeriod); + request.then(addMetrics); + return request; }); if (hasListeners) { notifyListeners(); diff --git a/lib/pages/transcription/transcription.dart b/lib/pages/transcription/transcription.dart index 0ea6f149..1d9fd007 100644 --- a/lib/pages/transcription/transcription.dart +++ b/lib/pages/transcription/transcription.dart @@ -3,6 +3,7 @@ import 'package:go_router/go_router.dart'; import 'package:inference/project.dart'; import 'package:inference/providers/preference_provider.dart'; import 'package:inference/pages/transcription/providers/speech_inference_provider.dart'; +import 'package:inference/pages/transcription/performance_metrics.dart'; import 'package:inference/pages/transcription/playground.dart'; import 'package:provider/provider.dart'; @@ -80,11 +81,11 @@ class _TranscriptionPageState extends State { title: const Text("Playground"), body: Playground(project: widget.project), ), - //PaneItem( - // icon: const Icon(FluentIcons.project_collection), - // title: const Text("Performance metrics"), - // body: Container(), - //), + PaneItem( + icon: const Icon(FluentIcons.line_chart), + title: const Text("Performance metrics"), + body: const PerformanceMetrics(), + ), ], ) ), diff --git a/lib/pages/transcription/utils/metrics.dart b/lib/pages/transcription/utils/metrics.dart new file mode 100644 index 00000000..481c9f30 --- /dev/null +++ b/lib/pages/transcription/utils/metrics.dart @@ -0,0 +1,54 @@ +import 'package:inference/interop/generated_bindings.dart'; + +class DMetrics { + double loadTime; + double generateTime; + double tokenizationTime; + double detokenizationTime; + double ttft; + double tpot; + double throughput; + int numberOfGeneratedTokens; + int numberOfInputTokens; + + int n = 1; // number of added metrics + + DMetrics({ + required this.loadTime, + required this.generateTime, + required this.tokenizationTime, + required this.detokenizationTime, + required this.ttft, + required this.tpot, + required this.throughput, + required this.numberOfGeneratedTokens, + required this.numberOfInputTokens, + }); + + void addCMetrics(Metrics metrics) { + //loadTime = metrics.load_time; + generateTime += metrics.generate_time; + tokenizationTime += metrics.tokenization_time; + detokenizationTime += metrics.detokenization_time; + ttft = (ttft * (n / (n + 1))) + metrics.ttft / n; + tpot = (tpot * (n / (n + 1))) + metrics.tpot / n; + throughput = (throughput * (n / (n + 1))) + metrics.throughput / n; + numberOfGeneratedTokens += metrics.number_of_generated_tokens; + numberOfInputTokens += metrics.number_of_input_tokens; + n += 1; + } + + factory DMetrics.fromCMetrics(Metrics metrics) { + return DMetrics( + loadTime: metrics.load_time, + generateTime: metrics.generate_time, + tokenizationTime: metrics.tokenization_time, + detokenizationTime: metrics.detokenization_time, + ttft: metrics.ttft, + tpot: metrics.tpot, + throughput: metrics.throughput, + numberOfGeneratedTokens: metrics.number_of_generated_tokens, + numberOfInputTokens: metrics.number_of_input_tokens, + ); + } +} diff --git a/lib/pages/transcription/widgets/subtitles.dart b/lib/pages/transcription/widgets/subtitles.dart index 9971c9a2..da17b0c9 100644 --- a/lib/pages/transcription/widgets/subtitles.dart +++ b/lib/pages/transcription/widgets/subtitles.dart @@ -44,7 +44,8 @@ class Subtitles extends StatelessWidget { Text(text, textAlign: TextAlign.center, style: const TextStyle( - fontSize: fontSize + fontSize: fontSize, + color: Colors.white, ) ) ], diff --git a/lib/widgets/performance_tile.dart b/lib/widgets/performance_tile.dart new file mode 100644 index 00000000..6ab5dd06 --- /dev/null +++ b/lib/widgets/performance_tile.dart @@ -0,0 +1,65 @@ +import 'package:fluent_ui/fluent_ui.dart'; + +class PerformanceTile extends StatelessWidget { + final String title; + final String value; + final String unit; + final bool tall; + + const PerformanceTile({ + super.key, + required this.title, + required this.value, + required this.unit, + this.tall = false, + }); + + @override + Widget build(BuildContext context) { + final theme = FluentTheme.of(context); + return Padding( + padding: const EdgeInsets.all(8.0), + child: Acrylic( + elevation: 5, + shadowColor: Colors.black, + shape: RoundedRectangleBorder ( + borderRadius: BorderRadius.circular(4), + ), + child: SizedBox( + width: 268, + height: tall ? 200 : 124, + child: Center( + child: Column( + mainAxisAlignment: MainAxisAlignment.center, + crossAxisAlignment: CrossAxisAlignment.center, + children: [ + Text( + title, + style: const TextStyle( + fontSize: 14, + ), + ), + RichText( + text: TextSpan( + style: TextStyle( + color: theme.inactiveColor, + ), + children: [ + TextSpan(text: value, + style: const TextStyle( + fontSize: 30, + ) + ), + TextSpan(text: " $unit"), + ] + ) + ), + ], + ) + ) + ), + ), + ); + } + +} From 6bddb727891a8b9fa28f217bbefc2d2564020b6c Mon Sep 17 00:00:00 2001 From: Ronald Hecker Date: Fri, 22 Nov 2024 13:58:09 +0100 Subject: [PATCH 19/33] Add model properties to performance metrics --- .../transcription/performance_metrics.dart | 147 ++++++++++-------- lib/pages/transcription/transcription.dart | 2 +- 2 files changed, 79 insertions(+), 70 deletions(-) diff --git a/lib/pages/transcription/performance_metrics.dart b/lib/pages/transcription/performance_metrics.dart index a12b5548..e1795a13 100644 --- a/lib/pages/transcription/performance_metrics.dart +++ b/lib/pages/transcription/performance_metrics.dart @@ -1,87 +1,96 @@ import 'package:fluent_ui/fluent_ui.dart'; import 'package:inference/pages/computer_vision/widgets/horizontal_rule.dart'; +import 'package:inference/pages/computer_vision/widgets/model_properties.dart'; import 'package:inference/pages/transcription/providers/speech_inference_provider.dart'; +import 'package:inference/project.dart'; import 'package:inference/widgets/performance_tile.dart'; import 'package:intl/intl.dart'; import 'package:provider/provider.dart'; class PerformanceMetrics extends StatelessWidget { - const PerformanceMetrics({super.key}); + final Project project; + const PerformanceMetrics({super.key, required this.project}); @override Widget build(BuildContext context) { - return Consumer( - builder: (context, inference, child) { - final metrics = inference.metrics; - if (metrics == null) { - return Container(); - } + return Row( + children: [ + Expanded( + child: Consumer( + builder: (context, inference, child) { + final metrics = inference.metrics; + if (metrics == null) { + return Container(); + } - Locale locale = Localizations.localeOf(context); - final nf = NumberFormat.decimalPatternDigits( - locale: locale.languageCode, decimalDigits: 0); + Locale locale = Localizations.localeOf(context); + final nf = NumberFormat.decimalPatternDigits( + locale: locale.languageCode, decimalDigits: 0); - return Padding( - padding: const EdgeInsets.symmetric(vertical: 80), - child: Center( - child: SizedBox( - width: 887, - child: Column( - children: [ - Row( - mainAxisAlignment: MainAxisAlignment.spaceEvenly, - children: [ - PerformanceTile( - title: "Time to first token (TTFT)", - value: nf.format(metrics.ttft), - unit: "ms", - tall: true, - ), - PerformanceTile( - title: "Time per output token (TPOT)", - value: nf.format(metrics.tpot), - unit: "ms", - tall: true, - ), - PerformanceTile( - title: "Generate total duration", - value: nf.format(metrics.generateTime), - unit: "ms", - tall: true, - ), - ], + return Padding( + padding: const EdgeInsets.symmetric(vertical: 80), + child: Center( + child: SizedBox( + width: 887, + child: Column( + children: [ + Row( + mainAxisAlignment: MainAxisAlignment.spaceEvenly, + children: [ + PerformanceTile( + title: "Time to first token (TTFT)", + value: nf.format(metrics.ttft), + unit: "ms", + tall: true, + ), + PerformanceTile( + title: "Time per output token (TPOT)", + value: nf.format(metrics.tpot), + unit: "ms", + tall: true, + ), + PerformanceTile( + title: "Generate total duration", + value: nf.format(metrics.generateTime), + unit: "ms", + tall: true, + ), + ], + ), + const Padding( + padding: EdgeInsets.symmetric(horizontal: 16.0, vertical: 16), + child: HorizontalRule(), + ), + Row( + mainAxisAlignment: MainAxisAlignment.spaceEvenly, + children: [ + PerformanceTile( + title: "Load time", + value: nf.format(metrics.loadTime), + unit: "ms", + ), + PerformanceTile( + title: "Detokenization duration", + value: nf.format(metrics.detokenizationTime), + unit: "ms", + ), + PerformanceTile( + title: "Throughput", + value: nf.format(metrics.throughput), + unit: "tokens/sec", + ), + ], + ), + ], + ), ), - const Padding( - padding: EdgeInsets.symmetric(horizontal: 16.0, vertical: 16), - child: HorizontalRule(), - ), - Row( - mainAxisAlignment: MainAxisAlignment.spaceEvenly, - children: [ - PerformanceTile( - title: "Load time", - value: nf.format(metrics.loadTime), - unit: "ms", - ), - PerformanceTile( - title: "Detokenization duration", - value: nf.format(metrics.detokenizationTime), - unit: "ms", - ), - PerformanceTile( - title: "Throughput", - value: nf.format(metrics.throughput), - unit: "tokens/sec", - ), - ], - ), - ], - ), - ), + ), + ); + } ), - ); - } + ), + ModelProperties(project: project), + ], ); } } - diff --git a/lib/pages/transcription/transcription.dart b/lib/pages/transcription/transcription.dart index 1d9fd007..46f54af6 100644 --- a/lib/pages/transcription/transcription.dart +++ b/lib/pages/transcription/transcription.dart @@ -84,7 +84,7 @@ class _TranscriptionPageState extends State { PaneItem( icon: const Icon(FluentIcons.line_chart), title: const Text("Performance metrics"), - body: const PerformanceMetrics(), + body: PerformanceMetrics(project: widget.project), ), ], ) From c9788eaceac6ba4dcf1ba01649e1b5f4c3f894ca Mon Sep 17 00:00:00 2001 From: Ronald Hecker Date: Fri, 22 Nov 2024 17:09:54 +0100 Subject: [PATCH 20/33] Fix transcription from continuening after dispose --- .../transcription/performance_metrics.dart | 137 +++++++++--------- .../providers/speech_inference_provider.dart | 16 +- 2 files changed, 84 insertions(+), 69 deletions(-) diff --git a/lib/pages/transcription/performance_metrics.dart b/lib/pages/transcription/performance_metrics.dart index e1795a13..0abf3e19 100644 --- a/lib/pages/transcription/performance_metrics.dart +++ b/lib/pages/transcription/performance_metrics.dart @@ -1,6 +1,7 @@ import 'package:fluent_ui/fluent_ui.dart'; import 'package:inference/pages/computer_vision/widgets/horizontal_rule.dart'; import 'package:inference/pages/computer_vision/widgets/model_properties.dart'; +import 'package:inference/pages/models/widgets/grid_container.dart'; import 'package:inference/pages/transcription/providers/speech_inference_provider.dart'; import 'package:inference/project.dart'; import 'package:inference/widgets/performance_tile.dart'; @@ -16,77 +17,79 @@ class PerformanceMetrics extends StatelessWidget { return Row( children: [ Expanded( - child: Consumer( - builder: (context, inference, child) { - final metrics = inference.metrics; - if (metrics == null) { - return Container(); - } + child: GridContainer( + child: Consumer( + builder: (context, inference, child) { + final metrics = inference.metrics; + if (metrics == null) { + return Container(); + } - Locale locale = Localizations.localeOf(context); - final nf = NumberFormat.decimalPatternDigits( - locale: locale.languageCode, decimalDigits: 0); + Locale locale = Localizations.localeOf(context); + final nf = NumberFormat.decimalPatternDigits( + locale: locale.languageCode, decimalDigits: 0); - return Padding( - padding: const EdgeInsets.symmetric(vertical: 80), - child: Center( - child: SizedBox( - width: 887, - child: Column( - children: [ - Row( - mainAxisAlignment: MainAxisAlignment.spaceEvenly, - children: [ - PerformanceTile( - title: "Time to first token (TTFT)", - value: nf.format(metrics.ttft), - unit: "ms", - tall: true, - ), - PerformanceTile( - title: "Time per output token (TPOT)", - value: nf.format(metrics.tpot), - unit: "ms", - tall: true, - ), - PerformanceTile( - title: "Generate total duration", - value: nf.format(metrics.generateTime), - unit: "ms", - tall: true, - ), - ], - ), - const Padding( - padding: EdgeInsets.symmetric(horizontal: 16.0, vertical: 16), - child: HorizontalRule(), - ), - Row( - mainAxisAlignment: MainAxisAlignment.spaceEvenly, - children: [ - PerformanceTile( - title: "Load time", - value: nf.format(metrics.loadTime), - unit: "ms", - ), - PerformanceTile( - title: "Detokenization duration", - value: nf.format(metrics.detokenizationTime), - unit: "ms", - ), - PerformanceTile( - title: "Throughput", - value: nf.format(metrics.throughput), - unit: "tokens/sec", - ), - ], - ), - ], + return Padding( + padding: const EdgeInsets.symmetric(vertical: 80), + child: Center( + child: SizedBox( + width: 887, + child: Column( + children: [ + Row( + mainAxisAlignment: MainAxisAlignment.spaceEvenly, + children: [ + PerformanceTile( + title: "Time to first token (TTFT)", + value: nf.format(metrics.ttft), + unit: "ms", + tall: true, + ), + PerformanceTile( + title: "Time per output token (TPOT)", + value: nf.format(metrics.tpot), + unit: "ms", + tall: true, + ), + PerformanceTile( + title: "Generate total duration", + value: nf.format(metrics.generateTime), + unit: "ms", + tall: true, + ), + ], + ), + const Padding( + padding: EdgeInsets.symmetric(horizontal: 16.0, vertical: 16), + child: HorizontalRule(), + ), + Row( + mainAxisAlignment: MainAxisAlignment.spaceEvenly, + children: [ + PerformanceTile( + title: "Load time", + value: nf.format(metrics.loadTime), + unit: "ms", + ), + PerformanceTile( + title: "Detokenization duration", + value: nf.format(metrics.detokenizationTime), + unit: "ms", + ), + PerformanceTile( + title: "Throughput", + value: nf.format(metrics.throughput), + unit: "tokens/sec", + ), + ], + ), + ], + ), ), ), - ), - ); - } + ); + } + ), ), ), ModelProperties(project: project), diff --git a/lib/pages/transcription/providers/speech_inference_provider.dart b/lib/pages/transcription/providers/speech_inference_provider.dart index fedbf723..606b9e76 100644 --- a/lib/pages/transcription/providers/speech_inference_provider.dart +++ b/lib/pages/transcription/providers/speech_inference_provider.dart @@ -20,6 +20,8 @@ class SpeechInferenceProvider extends ChangeNotifier { String? _videoPath; String? get videoPath => _videoPath; + bool forceStop = false; + bool get videoLoaded => _videoPath != null; DynamicRangeLoading>? transcription; @@ -79,13 +81,17 @@ class SpeechInferenceProvider extends ChangeNotifier { throw Exception("Can't transcribe before loading video"); } - while (!transcription!.complete) { + forceStop = false; + + while ((!transcription!.complete) || !forceStop) { if (transcription == null) { return; } await transcription!.process((int i) { final request = transcribe(i * transcriptionPeriod, transcriptionPeriod); - request.then(addMetrics); + if (!forceStop) { + request.then(addMetrics); + } return request; }); if (hasListeners) { @@ -103,4 +109,10 @@ class SpeechInferenceProvider extends ChangeNotifier { return _project == project && _device == device; } + @override + void dispose() { + forceStop = true; + super.dispose(); + } + } From 5a876154251f589fc7168cd897e7dc7d2162b6e5 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Mon, 25 Nov 2024 09:18:47 +0100 Subject: [PATCH 21/33] Fixed comment --- lib/inference/download_page.dart | 2 +- lib/pages/download_model/download_model.dart | 2 +- lib/public_models.dart | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/inference/download_page.dart b/lib/inference/download_page.dart index fac683be..c228c99e 100644 --- a/lib/inference/download_page.dart +++ b/lib/inference/download_page.dart @@ -38,7 +38,7 @@ class _DownloadPageState extends State { final downloadProvider = Provider.of(context, listen: false); final projectProvider = Provider.of(context, listen: false); - final files = await downloadFiles(widget.project); + final files = await listDownloadFiles(widget.project); try { await downloadProvider.queue(files, widget.project.modelInfo?.collection.token); diff --git a/lib/pages/download_model/download_model.dart b/lib/pages/download_model/download_model.dart index 9a4f60b9..7a2e9e77 100644 --- a/lib/pages/download_model/download_model.dart +++ b/lib/pages/download_model/download_model.dart @@ -40,7 +40,7 @@ class _DownloadModelPageState extends State { late Map files; try { - files = await downloadFiles(widget.project); + files = await listDownloadFiles(widget.project); } catch (e) { await showDialog(context: context, builder: (BuildContext context) => ContentDialog( title: const Text('Model was not found'), diff --git a/lib/public_models.dart b/lib/public_models.dart index 25eadf5f..0efd70fe 100644 --- a/lib/public_models.dart +++ b/lib/public_models.dart @@ -47,7 +47,7 @@ Future> getFilesForModel(String modelId) async { return List.from(result.data[0]["siblings"].map((m) => m.values.first)); } -Future> downloadFiles(PublicProject project) async { +Future> listDownloadFiles(PublicProject project) async { final files = await getFilesForModel(project.modelId); return { for (var v in files) huggingFaceModelFileUrl(project.modelId, v) : platformContext.join(project.storagePath, v) }; } From 316fc1c88d32460b33d6b74d10581ddf00cec834 Mon Sep 17 00:00:00 2001 From: Ronald Hecker Date: Tue, 26 Nov 2024 10:06:19 +0100 Subject: [PATCH 22/33] Add loading to transcription and fix dispose Wait for the transcription process to be done before disposing. Same goes for starting a new transcriptionProcess. Wait for old one to be done after force stopped and then start. --- lib/pages/transcription/playground.dart | 80 ++++++++++--------- .../providers/speech_inference_provider.dart | 17 ++-- 2 files changed, 55 insertions(+), 42 deletions(-) diff --git a/lib/pages/transcription/playground.dart b/lib/pages/transcription/playground.dart index 5f3e03b1..c2224817 100644 --- a/lib/pages/transcription/playground.dart +++ b/lib/pages/transcription/playground.dart @@ -61,7 +61,6 @@ class _PlaygroundState extends State with TickerProviderStateMixin{ void uploadFile(String file) async { final inference = Provider.of(context, listen: false); await inference.loadVideo(file); - inference.startTranscribing(); initializeVideoAndListeners(file); } @@ -121,46 +120,53 @@ class _PlaygroundState extends State with TickerProviderStateMixin{ builder: (context) { return DropArea( type: "video", - showChild: inference.videoLoaded, + showChild: inference.videoPath != null, onUpload: (String file) { uploadFile(file); }, extensions: const [], - child: Row( - crossAxisAlignment: CrossAxisAlignment.stretch, - children: [ - Expanded( - child: GridContainer( - color: backgroundColor.of(theme), - child: Stack( - alignment: Alignment.bottomCenter, - children: [ - Video(controller: controller), - Subtitles( - transcription: inference.transcription?.data, - subtitleIndex: subtitleIndex, + child: Builder( + builder: (context) { + if (!inference.loaded.isCompleted) { + return Center(child: Image.asset('images/intel-loading.gif', width: 100)); + } + return Row( + crossAxisAlignment: CrossAxisAlignment.stretch, + children: [ + Expanded( + child: GridContainer( + color: backgroundColor.of(theme), + child: Stack( + alignment: Alignment.bottomCenter, + children: [ + Video(controller: controller), + Subtitles( + transcription: inference.transcription?.data, + subtitleIndex: subtitleIndex, + ), + ] ), - ] - ), - ), - ), - SizedBox( - width: 360, - child: GridContainer( - color: backgroundColor.of(theme), - child: Builder( - builder: (context) { - if (inference.transcription == null) { - return Container(); - } - return Transcription( - onSeek: player.seek, - transcription: inference.transcription!, - messages: Message.parse(inference.transcription!.data, transcriptionPeriod), - ); - } + ), ), - ), - ) - ], + SizedBox( + width: 360, + child: GridContainer( + color: backgroundColor.of(theme), + child: Builder( + builder: (context) { + if (inference.transcription == null) { + return Container(); + } + return Transcription( + onSeek: player.seek, + transcription: inference.transcription!, + messages: Message.parse(inference.transcription!.data, transcriptionPeriod), + ); + } + ), + ), + ) + ], + ); + } ), ); } diff --git a/lib/pages/transcription/providers/speech_inference_provider.dart b/lib/pages/transcription/providers/speech_inference_provider.dart index 606b9e76..84927b1b 100644 --- a/lib/pages/transcription/providers/speech_inference_provider.dart +++ b/lib/pages/transcription/providers/speech_inference_provider.dart @@ -25,6 +25,7 @@ class SpeechInferenceProvider extends ChangeNotifier { bool get videoLoaded => _videoPath != null; DynamicRangeLoading>? transcription; + Future? activeTranscriptionProcess; DMetrics? metrics; bool get transcriptionComplete { @@ -60,10 +61,15 @@ class SpeechInferenceProvider extends ChangeNotifier { Future loadVideo(String path) async { await loaded.future; + forceStop = true; + if (activeTranscriptionProcess != null) { + await activeTranscriptionProcess!; + } _videoPath = path; final duration = await _inference!.loadVideo(path); final sections = (duration / transcriptionPeriod).ceil(); transcription = DynamicRangeLoading>(Section(0, sections)); + activeTranscriptionProcess = startTranscribing(); notifyListeners(); } @@ -83,15 +89,13 @@ class SpeechInferenceProvider extends ChangeNotifier { forceStop = false; - while ((!transcription!.complete) || !forceStop) { + while (!forceStop && (!transcription!.complete)) { if (transcription == null) { return; } await transcription!.process((int i) { final request = transcribe(i * transcriptionPeriod, transcriptionPeriod); - if (!forceStop) { - request.then(addMetrics); - } + request.then(addMetrics); return request; }); if (hasListeners) { @@ -110,8 +114,11 @@ class SpeechInferenceProvider extends ChangeNotifier { } @override - void dispose() { + void dispose() async { forceStop = true; + if (activeTranscriptionProcess != null) { + await activeTranscriptionProcess!; + } super.dispose(); } From f03c26ee4d2a0ed438122820e222e71868203598 Mon Sep 17 00:00:00 2001 From: Ronald Hecker Date: Tue, 26 Nov 2024 10:29:09 +0100 Subject: [PATCH 23/33] No need to check for null for await Future? --- .../providers/speech_inference_provider.dart | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/pages/transcription/providers/speech_inference_provider.dart b/lib/pages/transcription/providers/speech_inference_provider.dart index 84927b1b..2f9a7895 100644 --- a/lib/pages/transcription/providers/speech_inference_provider.dart +++ b/lib/pages/transcription/providers/speech_inference_provider.dart @@ -62,9 +62,7 @@ class SpeechInferenceProvider extends ChangeNotifier { Future loadVideo(String path) async { await loaded.future; forceStop = true; - if (activeTranscriptionProcess != null) { - await activeTranscriptionProcess!; - } + await activeTranscriptionProcess; _videoPath = path; final duration = await _inference!.loadVideo(path); final sections = (duration / transcriptionPeriod).ceil(); @@ -116,9 +114,7 @@ class SpeechInferenceProvider extends ChangeNotifier { @override void dispose() async { forceStop = true; - if (activeTranscriptionProcess != null) { - await activeTranscriptionProcess!; - } + await activeTranscriptionProcess; super.dispose(); } From 105b9155797ae2431704eb651da097c3e587f9c7 Mon Sep 17 00:00:00 2001 From: Ronald Hecker Date: Tue, 26 Nov 2024 14:34:40 +0100 Subject: [PATCH 24/33] Move provider logic from router Created a small wrapper for the provider to get the logic out of router Added link in home page to download model directly. Should probably check if model is already installed in future. --- lib/inference/inference_page.dart | 2 +- lib/pages/download_model/download_model.dart | 30 ++++++++++++++------ lib/pages/home/widgets/featured_card.dart | 11 ++++++- lib/pages/import/import.dart | 4 +-- lib/providers/download_provider.dart | 6 ++-- lib/router.dart | 4 +-- 6 files changed, 38 insertions(+), 19 deletions(-) diff --git a/lib/inference/inference_page.dart b/lib/inference/inference_page.dart index 9bd08cbb..e8e58f61 100644 --- a/lib/inference/inference_page.dart +++ b/lib/inference/inference_page.dart @@ -28,7 +28,7 @@ class _InferencePageState extends State { } } else { return ChangeNotifierProvider( - create: (_) => DownloadProvider(widget.project), + create: (_) => DownloadProvider(), child: DownloadPage(widget.project as PublicProject, onDone: () => setState(() {}), //trigger rerender. ) diff --git a/lib/pages/download_model/download_model.dart b/lib/pages/download_model/download_model.dart index 7a2e9e77..a6fb94de 100644 --- a/lib/pages/download_model/download_model.dart +++ b/lib/pages/download_model/download_model.dart @@ -2,6 +2,7 @@ import 'dart:math'; import 'package:fluent_ui/fluent_ui.dart'; import 'package:go_router/go_router.dart'; +import 'package:inference/deployment_processor.dart'; import 'package:inference/project.dart'; import 'package:inference/providers/download_provider.dart'; import 'package:inference/providers/project_provider.dart'; @@ -14,23 +15,33 @@ String formatBytes(int bytes) { return "${NumberFormat("#,##0").format(bytes / pow(1024, 2))} MB"; } + +class DownloadPage extends StatelessWidget { + final PublicProject project; + const DownloadPage({super.key, required this.project}); + + @override + Widget build(BuildContext context) { + return ChangeNotifierProvider( + create: (_) => DownloadProvider(), + child: DownloadModelPage(project: project), + ); + } +} + class DownloadModelPage extends StatefulWidget { final PublicProject project; const DownloadModelPage({super.key, required this.project}); + @override - _DownloadModelPageState createState() => _DownloadModelPageState(); + State createState() => _DownloadModelPageState(); } class _DownloadModelPageState extends State { - bool _isInitialized = false; - @override - void didChangeDependencies() { - super.didChangeDependencies(); - if (!_isInitialized) { - _isInitialized = true; - Future.microtask(() { startDownload(); }); - } + void initState() { + super.initState(); + startDownload(); } void startDownload() async { @@ -57,6 +68,7 @@ class _DownloadModelPageState extends State { } try { + downloadProvider.onCancel = () => deleteProjectData(widget.project); await downloadProvider.queue(files, widget.project.modelInfo?.collection.token); projectProvider.addProject(widget.project); await getAdditionalModelInfo(widget.project); diff --git a/lib/pages/home/widgets/featured_card.dart b/lib/pages/home/widgets/featured_card.dart index 7e24a0c6..c756ac18 100644 --- a/lib/pages/home/widgets/featured_card.dart +++ b/lib/pages/home/widgets/featured_card.dart @@ -1,11 +1,20 @@ import 'package:fluent_ui/fluent_ui.dart'; import 'package:inference/importers/manifest_importer.dart'; import 'package:inference/widgets/elevation.dart'; +import 'package:go_router/go_router.dart'; class FeaturedCard extends StatelessWidget { final Model model; const FeaturedCard({required this.model, super.key}); + void downloadModel(BuildContext context) { + model.convertToProject().then((project) { + if (context.mounted) { + GoRouter.of(context).go('/models/download', extra: project); + } + }); + } + @override Widget build(BuildContext context) { final theme = FluentTheme.of(context); @@ -72,7 +81,7 @@ class FeaturedCard extends StatelessWidget { alignment: Alignment.centerRight, child: Padding( padding: const EdgeInsets.only(top: 2), - child: IconButton(icon: const Icon(FluentIcons.pop_expand, size: 14), onPressed: () {}), + child: IconButton(icon: const Icon(FluentIcons.pop_expand, size: 14), onPressed: () => downloadModel(context)), ), ), ], diff --git a/lib/pages/import/import.dart b/lib/pages/import/import.dart index 97ffd82a..82af278c 100644 --- a/lib/pages/import/import.dart +++ b/lib/pages/import/import.dart @@ -70,7 +70,7 @@ class _ImportPageState extends State { children: [ FilledButton(onPressed: selectedModel == null ? (null) : () { selectedModel?.convertToProject().then((project) { - router.push('/models/download', extra: project); + router.go('/models/download', extra: project); }); }, child: const Text('Import selected model'),), @@ -180,4 +180,4 @@ class _ImportPageState extends State { ] ); } -} \ No newline at end of file +} diff --git a/lib/providers/download_provider.dart b/lib/providers/download_provider.dart index 0c61d2db..3da5b828 100644 --- a/lib/providers/download_provider.dart +++ b/lib/providers/download_provider.dart @@ -19,11 +19,11 @@ class DownloadStats { } class DownloadProvider extends ChangeNotifier { - final Project? project; final Map _downloads = {}; CancelToken? _cancelToken; - DownloadProvider(this.project); + Function? onCancel; + DownloadProvider(); Future queue(Map downloads, String? token) async{ List promises = []; @@ -90,7 +90,7 @@ class DownloadProvider extends ChangeNotifier { void cancel() { _cancelToken?.cancel(); - deleteProjectData(project!); + onCancel?.call(); } @override diff --git a/lib/router.dart b/lib/router.dart index a8d371c9..1c32c68f 100644 --- a/lib/router.dart +++ b/lib/router.dart @@ -26,9 +26,7 @@ final router = GoRouter(navigatorKey: rootNavigatorKey, GoRoute(path: '/home', builder: (context, state) => const HomePage()), GoRoute(path: '/models', builder: (context, state) => const ModelsPage()), GoRoute(path: '/models/import', builder: (context, state) => const ImportPage()), - GoRoute(path: '/models/download', builder: (context, state) => ChangeNotifierProvider(create: (_) => - DownloadProvider(state.extra as PublicProject), child: DownloadModelPage(project: state.extra as PublicProject)), - ), + GoRoute(path: '/models/download', builder: (context, state) => DownloadPage(project: state.extra as PublicProject)), GoRoute(path: '/models/inference', builder: (context, state) => InferencePage(state.extra as Project)), ], ) From 06cee736950384c0e042696e61ec78ccddbbb4cd Mon Sep 17 00:00:00 2001 From: "Hecker, Ronald" Date: Tue, 26 Nov 2024 15:19:47 +0100 Subject: [PATCH 25/33] Add whisper models to manifest Sadly many of the models are still broken, but the featured one is working --- assets/manifest.json | 1521 ++++++++++++++++++++++++++-------- scripts/create_manifest.dart | 2 +- 2 files changed, 1163 insertions(+), 360 deletions(-) diff --git a/assets/manifest.json b/assets/manifest.json index 10592396..50377b65 100644 --- a/assets/manifest.json +++ b/assets/manifest.json @@ -1,996 +1,1799 @@ { "popular_models": [ { - "name": "Mistral 7b Instruct", + "name": "Mistral 7b Instruct V0.1", "id": "mistral-7b-instruct-v0.1-int8-ov", - "fileSize": 7824223166, + "fileSize": 7824223238, "optimizationPrecision": "int8", "contextWindow": 32768, - "description": "Chat with Mistral 7b Instruct V0.1 model", - "task": "text-generation" + "description": "Chat with Mistral 7b Instruct V0.1", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { - "name": "Phi 3 Mini 4k", + "name": "Phi 3 Mini 4k Instruct", "id": "Phi-3-mini-4k-instruct-int4-ov", - "fileSize": 2637358233, + "fileSize": 2317366276, "optimizationPrecision": "int4", "contextWindow": 4096, - "description": "Chat with Phi 3 Mini 4k Instruct model", - "task": "text-generation" + "description": "Chat with Phi 3 Mini 4k Instruct", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" + }, + { + "name": "Whisper Base", + "id": "whisper-base-fp16-ov", + "fileSize": 263786768, + "optimizationPrecision": "fp16", + "contextWindow": 0, + "description": "Transcribe video with Whisper Base", + "task": "speech", + "author": "OpenVINO", + "collection": "speech-to-text-672321d5c070537a178a8aeb" }, { "name": "Open_llama_3b_v2", "id": "open_llama_3b_v2-int8-ov", - "fileSize": 3689232132, + "fileSize": 3689232204, "optimizationPrecision": "int8", "contextWindow": 2048, - "description": "Chat with Open_llama_3b_v2 model", - "task": "text-generation" + "description": "Chat with Open_llama_3b_v2", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" + } + ], + "all_models": [ + { + "name": "Distil Large V2", + "id": "distil-large-v2-fp16-ov", + "fileSize": 1623693910, + "optimizationPrecision": "fp16", + "contextWindow": 0, + "description": "Transcribe video with Distil Large V2", + "task": "speech", + "author": "OpenVINO", + "collection": "speech-to-text-672321d5c070537a178a8aeb" }, { - "name": "Open_llama_3b_v2", - "id": "open_llama_3b_v2-int8-ov", - "fileSize": 3689232132, + "name": "Distil Large V2", + "id": "distil-large-v2-int8-ov", + "fileSize": 811967934, "optimizationPrecision": "int8", - "contextWindow": 2048, - "description": "Chat with Open_llama_3b_v2 model", - "task": "text-generation" + "contextWindow": 0, + "description": "Transcribe video with Distil Large V2", + "task": "speech", + "author": "OpenVINO", + "collection": "speech-to-text-672321d5c070537a178a8aeb" }, { - "name": "Open_llama_3b_v2", - "id": "open_llama_3b_v2-int8-ov", - "fileSize": 3689232132, + "name": "Distil Large V3", + "id": "distil-large-v3-int4-ov", + "fileSize": 470886140, + "optimizationPrecision": "int4", + "contextWindow": 0, + "description": "Transcribe video with Distil Large V3", + "task": "speech", + "author": "OpenVINO", + "collection": "speech-to-text-672321d5c070537a178a8aeb" + }, + { + "name": "Distil Large V3", + "id": "distil-large-v3-fp16-ov", + "fileSize": 1623692375, + "optimizationPrecision": "fp16", + "contextWindow": 0, + "description": "Transcribe video with Distil Large V3", + "task": "speech", + "author": "OpenVINO", + "collection": "speech-to-text-672321d5c070537a178a8aeb" + }, + { + "name": "Distil Large V3", + "id": "distil-large-v3-int8-ov", + "fileSize": 811966358, "optimizationPrecision": "int8", - "contextWindow": 2048, - "description": "Chat with Open_llama_3b_v2 model", - "task": "text-generation" - } - ], - "all_models": [ + "contextWindow": 0, + "description": "Transcribe video with Distil Large V3", + "task": "speech", + "author": "OpenVINO", + "collection": "speech-to-text-672321d5c070537a178a8aeb" + }, + { + "name": "Distil Large V2", + "id": "distil-large-v2-int4-ov", + "fileSize": 470887685, + "optimizationPrecision": "int4", + "contextWindow": 0, + "description": "Transcribe video with Distil Large V2", + "task": "speech", + "author": "OpenVINO", + "collection": "speech-to-text-672321d5c070537a178a8aeb" + }, + { + "name": "Distil Whisper Base", + "id": "distil-whisper-base-fp16-ov", + "fileSize": 160942998, + "optimizationPrecision": "fp16", + "contextWindow": 0, + "description": "Transcribe video with Distil Whisper Base", + "task": "speech", + "author": "OpenVINO", + "collection": "speech-to-text-672321d5c070537a178a8aeb" + }, + { + "name": "Distil Whisper Base", + "id": "distil-whisper-base-int4-ov", + "fileSize": 66325080, + "optimizationPrecision": "int4", + "contextWindow": 0, + "description": "Transcribe video with Distil Whisper Base", + "task": "speech", + "author": "OpenVINO", + "collection": "speech-to-text-672321d5c070537a178a8aeb" + }, + { + "name": "Distil Whisper Base", + "id": "distil-whisper-base-int8-ov", + "fileSize": 88033050, + "optimizationPrecision": "int8", + "contextWindow": 0, + "description": "Transcribe video with Distil Whisper Base", + "task": "speech", + "author": "OpenVINO", + "collection": "speech-to-text-672321d5c070537a178a8aeb" + }, + { + "name": "Distil Whisper Large V3", + "id": "distil-whisper-large-v3-int4-ov", + "fileSize": 905249204, + "optimizationPrecision": "int4", + "contextWindow": 0, + "description": "Transcribe video with Distil Whisper Large V3", + "task": "speech", + "author": "OpenVINO", + "collection": "speech-to-text-672321d5c070537a178a8aeb" + }, + { + "name": "Distil Whisper Medium", + "id": "distil-whisper-medium-fp16-ov", + "fileSize": 1611734288, + "optimizationPrecision": "fp16", + "contextWindow": 0, + "description": "Transcribe video with Distil Whisper Medium", + "task": "speech", + "author": "OpenVINO", + "collection": "speech-to-text-672321d5c070537a178a8aeb" + }, + { + "name": "Distil Whisper Large V3", + "id": "distil-whisper-large-v3-int8-ov", + "fileSize": 1642880457, + "optimizationPrecision": "int8", + "contextWindow": 0, + "description": "Transcribe video with Distil Whisper Large V3", + "task": "speech", + "author": "OpenVINO", + "collection": "speech-to-text-672321d5c070537a178a8aeb" + }, + { + "name": "Distil Whisper Large V3", + "id": "distil-whisper-large-v3-fp16-ov", + "fileSize": 3318535654, + "optimizationPrecision": "fp16", + "contextWindow": 0, + "description": "Transcribe video with Distil Whisper Large V3", + "task": "speech", + "author": "OpenVINO", + "collection": "speech-to-text-672321d5c070537a178a8aeb" + }, + { + "name": "Distil Whisper Medium", + "id": "distil-whisper-medium-int4-ov", + "fileSize": 467835674, + "optimizationPrecision": "int4", + "contextWindow": 0, + "description": "Transcribe video with Distil Whisper Medium", + "task": "speech", + "author": "OpenVINO", + "collection": "speech-to-text-672321d5c070537a178a8aeb" + }, + { + "name": "Distil Whisper Medium", + "id": "distil-whisper-medium-int8-ov", + "fileSize": 820786325, + "optimizationPrecision": "int8", + "contextWindow": 0, + "description": "Transcribe video with Distil Whisper Medium", + "task": "speech", + "author": "OpenVINO", + "collection": "speech-to-text-672321d5c070537a178a8aeb" + }, + { + "name": "Distil Whisper Tiny", + "id": "distil-whisper-tiny-int4-ov", + "fileSize": 42419245, + "optimizationPrecision": "int4", + "contextWindow": 0, + "description": "Transcribe video with Distil Whisper Tiny", + "task": "speech", + "author": "OpenVINO", + "collection": "speech-to-text-672321d5c070537a178a8aeb" + }, + { + "name": "Distil Whisper Tiny", + "id": "distil-whisper-tiny-int8-ov", + "fileSize": 50341791, + "optimizationPrecision": "int8", + "contextWindow": 0, + "description": "Transcribe video with Distil Whisper Tiny", + "task": "speech", + "author": "OpenVINO", + "collection": "speech-to-text-672321d5c070537a178a8aeb" + }, + { + "name": "Distil Whisper Tiny", + "id": "distil-whisper-tiny-fp16-ov", + "fileSize": 87657508, + "optimizationPrecision": "fp16", + "contextWindow": 0, + "description": "Transcribe video with Distil Whisper Tiny", + "task": "speech", + "author": "OpenVINO", + "collection": "speech-to-text-672321d5c070537a178a8aeb" + }, + { + "name": "Whisper Tiny", + "id": "whisper-tiny-int4-ov", + "fileSize": 68582454, + "optimizationPrecision": "int4", + "contextWindow": 0, + "description": "Transcribe video with Whisper Tiny", + "task": "speech", + "author": "OpenVINO", + "collection": "speech-to-text-672321d5c070537a178a8aeb" + }, + { + "name": "Whisper Base", + "id": "whisper-base-int8-ov", + "fileSize": 140207692, + "optimizationPrecision": "int8", + "contextWindow": 0, + "description": "Transcribe video with Whisper Base", + "task": "speech", + "author": "OpenVINO", + "collection": "speech-to-text-672321d5c070537a178a8aeb" + }, + { + "name": "Whisper Medium", + "id": "whisper-medium-int8-ov", + "fileSize": 1250473365, + "optimizationPrecision": "int8", + "contextWindow": 0, + "description": "Transcribe video with Whisper Medium", + "task": "speech", + "author": "OpenVINO", + "collection": "speech-to-text-672321d5c070537a178a8aeb" + }, + { + "name": "Whisper Medium", + "id": "whisper-medium-int4-ov", + "fileSize": 719474580, + "optimizationPrecision": "int4", + "contextWindow": 0, + "description": "Transcribe video with Whisper Medium", + "task": "speech", + "author": "OpenVINO", + "collection": "speech-to-text-672321d5c070537a178a8aeb" + }, + { + "name": "Whisper Tiny", + "id": "whisper-tiny-int8-ov", + "fileSize": 80769346, + "optimizationPrecision": "int8", + "contextWindow": 0, + "description": "Transcribe video with Whisper Tiny", + "task": "speech", + "author": "OpenVINO", + "collection": "speech-to-text-672321d5c070537a178a8aeb" + }, + { + "name": "Whisper Base", + "id": "whisper-base-int4-ov", + "fileSize": 107331169, + "optimizationPrecision": "int4", + "contextWindow": 0, + "description": "Transcribe video with Whisper Base", + "task": "speech", + "author": "OpenVINO", + "collection": "speech-to-text-672321d5c070537a178a8aeb" + }, + { + "name": "Whisper Large V3", + "id": "whisper-large-v3-fp16-ov", + "fileSize": 5038558437, + "optimizationPrecision": "fp16", + "contextWindow": 0, + "description": "Transcribe video with Whisper Large V3", + "task": "speech", + "author": "OpenVINO", + "collection": "speech-to-text-672321d5c070537a178a8aeb" + }, + { + "name": "Whisper Medium", + "id": "whisper-medium-fp16-ov", + "fileSize": 2465759114, + "optimizationPrecision": "fp16", + "contextWindow": 0, + "description": "Transcribe video with Whisper Medium", + "task": "speech", + "author": "OpenVINO", + "collection": "speech-to-text-672321d5c070537a178a8aeb" + }, + { + "name": "Whisper Tiny", + "id": "whisper-tiny-fp16-ov", + "fileSize": 147388830, + "optimizationPrecision": "fp16", + "contextWindow": 0, + "description": "Transcribe video with Whisper Tiny", + "task": "speech", + "author": "OpenVINO", + "collection": "speech-to-text-672321d5c070537a178a8aeb" + }, + { + "name": "Whisper Base", + "id": "whisper-base-fp16-ov", + "fileSize": 263786768, + "optimizationPrecision": "fp16", + "contextWindow": 0, + "description": "Transcribe video with Whisper Base", + "task": "speech", + "author": "OpenVINO", + "collection": "speech-to-text-672321d5c070537a178a8aeb" + }, { "name": "Phi 2", "id": "phi-2-fp16-ov", - "fileSize": 5978786593, + "fileSize": 5978786613, "optimizationPrecision": "fp16", "contextWindow": 2048, - "description": "Chat with Phi 2 model", - "task": "text-generation" + "description": "Chat with Phi 2", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Phi 2", "id": "phi-2-int8-ov", - "fileSize": 3004595529, + "fileSize": 3004595590, "optimizationPrecision": "int8", "contextWindow": 2048, - "description": "Chat with Phi 2 model", - "task": "text-generation" + "description": "Chat with Phi 2", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Mistral 7b Instruct V0.1", "id": "mistral-7b-instruct-v0.1-int8-ov", - "fileSize": 7824223166, + "fileSize": 7824223238, "optimizationPrecision": "int8", "contextWindow": 32768, - "description": "Chat with Mistral 7b Instruct V0.1 model", - "task": "text-generation" + "description": "Chat with Mistral 7b Instruct V0.1", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Mistral 7b Instruct V0.1", "id": "mistral-7b-instruct-v0.1-fp16-ov", - "fileSize": 15576387089, + "fileSize": 15576387130, "optimizationPrecision": "fp16", "contextWindow": 32768, - "description": "Chat with Mistral 7b Instruct V0.1 model", - "task": "text-generation" + "description": "Chat with Mistral 7b Instruct V0.1", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Mistral 7b Instruct V0.1", "id": "mistral-7b-instruct-v0.1-int4-ov", - "fileSize": 4967917794, + "fileSize": 4967917876, "optimizationPrecision": "int4", "contextWindow": 32768, - "description": "Chat with Mistral 7b Instruct V0.1 model", - "task": "text-generation" + "description": "Chat with Mistral 7b Instruct V0.1", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Codegen25 7b Multi", "id": "codegen25-7b-multi-fp16-ov", - "fileSize": 14822539137, + "fileSize": 14822064608, "optimizationPrecision": "fp16", "contextWindow": 2048, - "description": "Chat with Codegen25 7b Multi model", - "task": "text-generation" + "description": "Chat with Codegen25 7b Multi", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Codegen25 7b Multi", "id": "codegen25-7b-multi-int8-ov", - "fileSize": 7414035410, + "fileSize": 7413624227, "optimizationPrecision": "int8", "contextWindow": 2048, - "description": "Chat with Codegen25 7b Multi model", - "task": "text-generation" + "description": "Chat with Codegen25 7b Multi", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Mixtral 8x7b Instruct V0.1", "id": "mixtral-8x7b-instruct-v0.1-int4-ov", - "fileSize": 30833964831, + "fileSize": 30833964913, "optimizationPrecision": "int4", "contextWindow": 32768, - "description": "Chat with Mixtral 8x7b Instruct V0.1 model", - "task": "text-generation" + "description": "Chat with Mixtral 8x7b Instruct V0.1", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Mixtral 8x7B Instruct V0.1", "id": "Mixtral-8x7B-Instruct-v0.1-int8-ov", - "fileSize": 50160688476, + "fileSize": 50160688558, "optimizationPrecision": "int8", "contextWindow": 32768, - "description": "Chat with Mixtral 8x7B Instruct V0.1 model", - "task": "text-generation" + "description": "Chat with Mixtral 8x7B Instruct V0.1", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Notus 7b V1", "id": "notus-7b-v1-fp16-ov", - "fileSize": 15576386988, + "fileSize": 15576387018, "optimizationPrecision": "fp16", "contextWindow": 32768, - "description": "Chat with Notus 7b V1 model", - "task": "text-generation" + "description": "Chat with Notus 7b V1", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Notus 7b V1", "id": "notus-7b-v1-int8-ov", - "fileSize": 7803125798, + "fileSize": 7803125869, "optimizationPrecision": "int8", "contextWindow": 32768, - "description": "Chat with Notus 7b V1 model", - "task": "text-generation" + "description": "Chat with Notus 7b V1", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Neural Chat 7b V3 3", "id": "neural-chat-7b-v3-3-fp16-ov", - "fileSize": 15576386599, + "fileSize": 15576386640, "optimizationPrecision": "fp16", "contextWindow": 32768, - "description": "Chat with Neural Chat 7b V3 3 model", - "task": "text-generation" + "description": "Chat with Neural Chat 7b V3 3", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Neural Chat 7b V3 3", "id": "neural-chat-7b-v3-3-int8-ov", - "fileSize": 7803125410, + "fileSize": 7803125481, "optimizationPrecision": "int8", "contextWindow": 32768, - "description": "Chat with Neural Chat 7b V3 3 model", - "task": "text-generation" + "description": "Chat with Neural Chat 7b V3 3", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Zephyr 7b Beta", "id": "zephyr-7b-beta-int8-ov", - "fileSize": 7803126061, + "fileSize": 7803126133, "optimizationPrecision": "int8", "contextWindow": 32768, - "description": "Chat with Zephyr 7b Beta model", - "task": "text-generation" + "description": "Chat with Zephyr 7b Beta", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Zephyr 7b Beta", "id": "zephyr-7b-beta-int4-ov", - "fileSize": 4904138531, + "fileSize": 4904138613, "optimizationPrecision": "int4", "contextWindow": 32768, - "description": "Chat with Zephyr 7b Beta model", - "task": "text-generation" + "description": "Chat with Zephyr 7b Beta", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Dolly V2 3b", "id": "dolly-v2-3b-int4-ov", - "fileSize": 2434908470, + "fileSize": 1694037426, "optimizationPrecision": "int4", "contextWindow": 2048, - "description": "Chat with Dolly V2 3b model", - "task": "text-generation" + "description": "Chat with Dolly V2 3b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Dolly V2 3b", "id": "dolly-v2-3b-int8-ov", - "fileSize": 2993180745, + "fileSize": 2993264386, "optimizationPrecision": "int8", "contextWindow": 2048, - "description": "Chat with Dolly V2 3b model", - "task": "text-generation" + "description": "Chat with Dolly V2 3b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Dolly V2 3b", "id": "dolly-v2-3b-fp16-ov", - "fileSize": 5967078157, + "fileSize": 5967350336, "optimizationPrecision": "fp16", "contextWindow": 2048, - "description": "Chat with Dolly V2 3b model", - "task": "text-generation" + "description": "Chat with Dolly V2 3b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Codegen2 3_7B_P", "id": "codegen2-3_7B_P-int4-ov", - "fileSize": 2252764320, + "fileSize": 2252764402, "optimizationPrecision": "int4", "contextWindow": 2048, - "description": "Chat with Codegen2 3_7B_P model", - "task": "text-generation" + "description": "Chat with Codegen2 3_7B_P", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Codegen2 3_7B_P", "id": "codegen2-3_7B_P-fp16-ov", - "fileSize": 7835969716, + "fileSize": 7835969757, "optimizationPrecision": "fp16", "contextWindow": 2048, - "description": "Chat with Codegen2 3_7B_P model", - "task": "text-generation" + "description": "Chat with Codegen2 3_7B_P", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Zephyr 7b Beta", "id": "zephyr-7b-beta-fp16-ov", - "fileSize": 15576387241, + "fileSize": 15576387282, "optimizationPrecision": "fp16", "contextWindow": 32768, - "description": "Chat with Zephyr 7b Beta model", - "task": "text-generation" + "description": "Chat with Zephyr 7b Beta", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Codegen2 3_7B_P", "id": "codegen2-3_7B_P-int8-ov", - "fileSize": 3927738589, + "fileSize": 3927738671, "optimizationPrecision": "int8", "contextWindow": 2048, - "description": "Chat with Codegen2 3_7B_P model", - "task": "text-generation" + "description": "Chat with Codegen2 3_7B_P", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "TinyLlama 1.1B Chat V1.0", "id": "TinyLlama-1.1B-Chat-v1.0-fp16-ov", - "fileSize": 2368272475, + "fileSize": 2368272527, "optimizationPrecision": "fp16", "contextWindow": 2048, - "description": "Chat with TinyLlama 1.1B Chat V1.0 model", - "task": "text-generation" + "description": "Chat with TinyLlama 1.1B Chat V1.0", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "TinyLlama 1.1B Chat V1.0", "id": "TinyLlama-1.1B-Chat-v1.0-int4-ov", - "fileSize": 668269097, + "fileSize": 668269179, "optimizationPrecision": "int4", "contextWindow": 2048, - "description": "Chat with TinyLlama 1.1B Chat V1.0 model", - "task": "text-generation" + "description": "Chat with TinyLlama 1.1B Chat V1.0", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "TinyLlama 1.1B Chat V1.0", "id": "TinyLlama-1.1B-Chat-v1.0-int8-ov", - "fileSize": 1187586826, + "fileSize": 1187586908, "optimizationPrecision": "int8", "contextWindow": 2048, - "description": "Chat with TinyLlama 1.1B Chat V1.0 model", - "task": "text-generation" + "description": "Chat with TinyLlama 1.1B Chat V1.0", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Gpt Neox 20b", "id": "gpt-neox-20b-int8-ov", - "fileSize": 22128276574, + "fileSize": 22128276646, "optimizationPrecision": "int8", "contextWindow": 2048, - "description": "Chat with Gpt Neox 20b model", - "task": "text-generation" + "description": "Chat with Gpt Neox 20b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Gpt Neox 20b", "id": "gpt-neox-20b-fp16-ov", - "fileSize": 44140098869, + "fileSize": 44140098910, "optimizationPrecision": "fp16", "contextWindow": 2048, - "description": "Chat with Gpt Neox 20b model", - "task": "text-generation" - }, - { - "name": "Gpt Neox 20b", - "id": "gpt-neox-20b-int4-ov", - "fileSize": 13968006447, - "optimizationPrecision": "int4", - "contextWindow": 2048, - "description": "Chat with Gpt Neox 20b model", - "task": "text-generation" + "description": "Chat with Gpt Neox 20b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Gpt J 6b", "id": "gpt-j-6b-int4-ov", - "fileSize": 4196810211, + "fileSize": 4196810272, "optimizationPrecision": "int4", "contextWindow": 2048, - "description": "Chat with Gpt J 6b model", - "task": "text-generation" + "description": "Chat with Gpt J 6b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Gpt J 6b", "id": "gpt-j-6b-int8-ov", - "fileSize": 6515945720, + "fileSize": 6515945792, "optimizationPrecision": "int8", "contextWindow": 2048, - "description": "Chat with Gpt J 6b model", - "task": "text-generation" + "description": "Chat with Gpt J 6b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Gpt J 6b", "id": "gpt-j-6b-fp16-ov", - "fileSize": 13001251300, + "fileSize": 13001251330, "optimizationPrecision": "fp16", "contextWindow": 2048, - "description": "Chat with Gpt J 6b model", - "task": "text-generation" + "description": "Chat with Gpt J 6b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Falcon 7b Instruct", "id": "falcon-7b-instruct-int4-ov", - "fileSize": 3959308647, + "fileSize": 3959308719, "optimizationPrecision": "int4", "contextWindow": 2048, - "description": "Chat with Falcon 7b Instruct model", - "task": "text-generation" + "description": "Chat with Falcon 7b Instruct", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Falcon 7b Instruct", "id": "falcon-7b-instruct-fp16-ov", - "fileSize": 14825512501, + "fileSize": 14825512542, "optimizationPrecision": "fp16", "contextWindow": 2048, - "description": "Chat with Falcon 7b Instruct model", - "task": "text-generation" + "description": "Chat with Falcon 7b Instruct", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Falcon 7b Instruct", "id": "falcon-7b-instruct-int8-ov", - "fileSize": 7449021953, + "fileSize": 7449022025, "optimizationPrecision": "int8", "contextWindow": 2048, - "description": "Chat with Falcon 7b Instruct model", - "task": "text-generation" + "description": "Chat with Falcon 7b Instruct", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Open_llama_7b_v2", "id": "open_llama_7b_v2-int8-ov", - "fileSize": 7243946706, + "fileSize": 7243946788, "optimizationPrecision": "int8", "contextWindow": 2048, - "description": "Chat with Open_llama_7b_v2 model", - "task": "text-generation" + "description": "Chat with Open_llama_7b_v2", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Open_llama_7b_v2", "id": "open_llama_7b_v2-int4-ov", - "fileSize": 4581255942, + "fileSize": 4581256014, "optimizationPrecision": "int4", "contextWindow": 2048, - "description": "Chat with Open_llama_7b_v2 model", - "task": "text-generation" + "description": "Chat with Open_llama_7b_v2", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Open_llama_7b_v2", "id": "open_llama_7b_v2-fp16-ov", - "fileSize": 14502136910, + "fileSize": 14502136961, "optimizationPrecision": "fp16", "contextWindow": 2048, - "description": "Chat with Open_llama_7b_v2 model", - "task": "text-generation" + "description": "Chat with Open_llama_7b_v2", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Open_llama_3b_v2", "id": "open_llama_3b_v2-int8-ov", - "fileSize": 3689232132, + "fileSize": 3689232204, "optimizationPrecision": "int8", "contextWindow": 2048, - "description": "Chat with Open_llama_3b_v2 model", - "task": "text-generation" + "description": "Chat with Open_llama_3b_v2", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Open_llama_3b_v2", "id": "open_llama_3b_v2-fp16-ov", - "fileSize": 7361187312, + "fileSize": 7361187363, "optimizationPrecision": "fp16", "contextWindow": 2048, - "description": "Chat with Open_llama_3b_v2 model", - "task": "text-generation" + "description": "Chat with Open_llama_3b_v2", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Phi 2", "id": "phi-2-int4-ov", - "fileSize": 1963097577, + "fileSize": 1963097638, "optimizationPrecision": "int4", "contextWindow": 2048, - "description": "Chat with Phi 2 model", - "task": "text-generation" + "description": "Chat with Phi 2", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Neural Chat 7b V3 3", "id": "neural-chat-7b-v3-3-int4-ov", - "fileSize": 4957174834, + "fileSize": 4957174906, "optimizationPrecision": "int4", "contextWindow": 32768, - "description": "Chat with Neural Chat 7b V3 3 model", - "task": "text-generation" + "description": "Chat with Neural Chat 7b V3 3", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Notus 7b V1", "id": "notus-7b-v1-int4-ov", - "fileSize": 4957175373, + "fileSize": 4957175444, "optimizationPrecision": "int4", "contextWindow": 32768, - "description": "Chat with Notus 7b V1 model", - "task": "text-generation" + "description": "Chat with Notus 7b V1", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "RedPajama INCITE Chat 3B V1", "id": "RedPajama-INCITE-Chat-3B-v1-int8-ov", - "fileSize": 3003403190, + "fileSize": 2993181070, "optimizationPrecision": "int8", "contextWindow": 2048, - "description": "Chat with RedPajama INCITE Chat 3B V1 model", - "task": "text-generation" + "description": "Chat with RedPajama INCITE Chat 3B V1", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "RedPajama INCITE 7B Instruct", "id": "RedPajama-INCITE-7B-Instruct-fp16-ov", - "fileSize": 14717999973, + "fileSize": 14718000035, "optimizationPrecision": "fp16", "contextWindow": 2048, - "description": "Chat with RedPajama INCITE 7B Instruct model", - "task": "text-generation" + "description": "Chat with RedPajama INCITE 7B Instruct", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "RedPajama INCITE 7B Instruct", "id": "RedPajama-INCITE-7B-Instruct-int4-ov", - "fileSize": 7384270376, + "fileSize": 7384270468, "optimizationPrecision": "int4", "contextWindow": 2048, - "description": "Chat with RedPajama INCITE 7B Instruct model", - "task": "text-generation" + "description": "Chat with RedPajama INCITE 7B Instruct", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "RedPajama INCITE 7B Chat", "id": "RedPajama-INCITE-7B-Chat-int4-ov", - "fileSize": 4753728620, + "fileSize": 4753728702, "optimizationPrecision": "int4", "contextWindow": 2048, - "description": "Chat with RedPajama INCITE 7B Chat model", - "task": "text-generation" + "description": "Chat with RedPajama INCITE 7B Chat", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "RedPajama INCITE 7B Instruct", "id": "RedPajama-INCITE-7B-Instruct-int8-ov", - "fileSize": 7384270355, + "fileSize": 7384270448, "optimizationPrecision": "int8", "contextWindow": 2048, - "description": "Chat with RedPajama INCITE 7B Instruct model", - "task": "text-generation" + "description": "Chat with RedPajama INCITE 7B Instruct", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "RedPajama INCITE 7B Chat", "id": "RedPajama-INCITE-7B-Chat-fp16-ov", - "fileSize": 14717999600, + "fileSize": 14717999652, "optimizationPrecision": "fp16", "contextWindow": 2048, - "description": "Chat with RedPajama INCITE 7B Chat model", - "task": "text-generation" + "description": "Chat with RedPajama INCITE 7B Chat", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "RedPajama INCITE Chat 3B V1", "id": "RedPajama-INCITE-Chat-3B-v1-int4-ov", - "fileSize": 1972726843, + "fileSize": 1693954060, "optimizationPrecision": "int4", "contextWindow": 2048, - "description": "Chat with RedPajama INCITE Chat 3B V1 model", - "task": "text-generation" + "description": "Chat with RedPajama INCITE Chat 3B V1", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "RedPajama INCITE 7B Chat", "id": "RedPajama-INCITE-7B-Chat-int8-ov", - "fileSize": 7384270239, + "fileSize": 7384270331, "optimizationPrecision": "int8", "contextWindow": 2048, - "description": "Chat with RedPajama INCITE 7B Chat model", - "task": "text-generation" + "description": "Chat with RedPajama INCITE 7B Chat", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "RedPajama INCITE Chat 3B V1", "id": "RedPajama-INCITE-Chat-3B-v1-fp16-ov", - "fileSize": 5977741177, + "fileSize": 5967266928, "optimizationPrecision": "fp16", "contextWindow": 2048, - "description": "Chat with RedPajama INCITE Chat 3B V1 model", - "task": "text-generation" - }, - { - "name": "Dolly V2 12b", - "id": "dolly-v2-12b-int4-ov", - "fileSize": 8093674841, - "optimizationPrecision": "int4", - "contextWindow": 2048, - "description": "Chat with Dolly V2 12b model", - "task": "text-generation" + "description": "Chat with RedPajama INCITE Chat 3B V1", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Dolly V2 7b", "id": "dolly-v2-7b-int4-ov", - "fileSize": 4753855475, + "fileSize": 4753855546, "optimizationPrecision": "int4", "contextWindow": 2048, - "description": "Chat with Dolly V2 7b model", - "task": "text-generation" + "description": "Chat with Dolly V2 7b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Dolly V2 7b", "id": "dolly-v2-7b-int8-ov", - "fileSize": 7384407579, + "fileSize": 7384407651, "optimizationPrecision": "int8", "contextWindow": 2048, - "description": "Chat with Dolly V2 7b model", - "task": "text-generation" + "description": "Chat with Dolly V2 7b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Dolly V2 7b", "id": "dolly-v2-7b-fp16-ov", - "fileSize": 14718147683, + "fileSize": 14718147724, "optimizationPrecision": "fp16", "contextWindow": 2048, - "description": "Chat with Dolly V2 7b model", - "task": "text-generation" + "description": "Chat with Dolly V2 7b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Mistral 7B Instruct V0.2", "id": "Mistral-7B-Instruct-v0.2-int8-ov", - "fileSize": 7823897819, + "fileSize": 7823897901, "optimizationPrecision": "int8", "contextWindow": 32768, - "description": "Chat with Mistral 7B Instruct V0.2 model", - "task": "text-generation" + "description": "Chat with Mistral 7B Instruct V0.2", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Dolly V2 12b", "id": "dolly-v2-12b-int8-ov", - "fileSize": 12785790267, + "fileSize": 12785790338, "optimizationPrecision": "int8", "contextWindow": 2048, - "description": "Chat with Dolly V2 12b model", - "task": "text-generation" + "description": "Chat with Dolly V2 12b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Mistral 7B Instruct V0.2", "id": "Mistral-7B-Instruct-v0.2-int4-ov", - "fileSize": 4957164416, + "fileSize": 4957164498, "optimizationPrecision": "int4", "contextWindow": 32768, - "description": "Chat with Mistral 7B Instruct V0.2 model", - "task": "text-generation" + "description": "Chat with Mistral 7B Instruct V0.2", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Mistral 7B Instruct V0.2", "id": "Mistral-7B-Instruct-v0.2-fp16-ov", - "fileSize": 15576062131, + "fileSize": 15576062183, "optimizationPrecision": "fp16", "contextWindow": 32768, - "description": "Chat with Mistral 7B Instruct V0.2 model", - "task": "text-generation" + "description": "Chat with Mistral 7B Instruct V0.2", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Codegen25 7b Multi", "id": "codegen25-7b-multi-int4-ov", - "fileSize": 4760257312, + "fileSize": 4074538822, "optimizationPrecision": "int4", "contextWindow": 2048, - "description": "Chat with Codegen25 7b Multi model", - "task": "text-generation" + "description": "Chat with Codegen25 7b Multi", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Persimmon 8b Chat", "id": "persimmon-8b-chat-int4-ov", - "fileSize": 6896839595, + "fileSize": 6896839666, "optimizationPrecision": "int4", "contextWindow": 2048, - "description": "Chat with Persimmon 8b Chat model", - "task": "text-generation" + "description": "Chat with Persimmon 8b Chat", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Persimmon 8b Chat", "id": "persimmon-8b-chat-int8-ov", - "fileSize": 12791514405, + "fileSize": 12791514477, "optimizationPrecision": "int8", "contextWindow": 2048, - "description": "Chat with Persimmon 8b Chat model", - "task": "text-generation" + "description": "Chat with Persimmon 8b Chat", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Pythia 1.4b", "id": "pythia-1.4b-int4-ov", - "fileSize": 6890411793, + "fileSize": 6890411865, "optimizationPrecision": "int4", "contextWindow": 2048, - "description": "Chat with Pythia 1.4b model", - "task": "text-generation" + "description": "Chat with Pythia 1.4b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Pythia 1.4b", "id": "pythia-1.4b-int8-ov", - "fileSize": 12785086603, + "fileSize": 12785086675, "optimizationPrecision": "int8", "contextWindow": 2048, - "description": "Chat with Pythia 1.4b model", - "task": "text-generation" + "description": "Chat with Pythia 1.4b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Persimmon 8b Chat", "id": "persimmon-8b-chat-fp16-ov", - "fileSize": 25461688234, + "fileSize": 25461688275, "optimizationPrecision": "fp16", "contextWindow": 2048, - "description": "Chat with Persimmon 8b Chat model", - "task": "text-generation" + "description": "Chat with Persimmon 8b Chat", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Pythia 1.4b", "id": "pythia-1.4b-fp16-ov", - "fileSize": 25455260443, + "fileSize": 25455260474, "optimizationPrecision": "fp16", "contextWindow": 2048, - "description": "Chat with Pythia 1.4b model", - "task": "text-generation" - }, - { - "name": "Pythia 12b", - "id": "pythia-12b-int4-ov", - "fileSize": 3824586206, - "optimizationPrecision": "int4", - "contextWindow": 2048, - "description": "Chat with Pythia 12b model", - "task": "text-generation" + "description": "Chat with Pythia 1.4b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Pythia 12b", "id": "pythia-12b-int8-ov", - "fileSize": 7153039029, + "fileSize": 7153039101, "optimizationPrecision": "int8", "contextWindow": 2048, - "description": "Chat with Pythia 12b model", - "task": "text-generation" + "description": "Chat with Pythia 12b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Pythia 2.8b", "id": "pythia-2.8b-int8-ov", - "fileSize": 7153039039, + "fileSize": 7153039111, "optimizationPrecision": "int8", "contextWindow": 2048, - "description": "Chat with Pythia 2.8b model", - "task": "text-generation" + "description": "Chat with Pythia 2.8b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Pythia 2.8b", "id": "pythia-2.8b-int4-ov", - "fileSize": 3824586216, + "fileSize": 3824586288, "optimizationPrecision": "int4", "contextWindow": 2048, - "description": "Chat with Pythia 2.8b model", - "task": "text-generation" + "description": "Chat with Pythia 2.8b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Pythia 12b", "id": "pythia-12b-fp16-ov", - "fileSize": 14293243461, + "fileSize": 14293243502, "optimizationPrecision": "fp16", "contextWindow": 2048, - "description": "Chat with Pythia 12b model", - "task": "text-generation" + "description": "Chat with Pythia 12b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Pythia 6.9b", "id": "pythia-6.9b-int4-ov", - "fileSize": 3824586216, + "fileSize": 3824586288, "optimizationPrecision": "int4", "contextWindow": 2048, - "description": "Chat with Pythia 6.9b model", - "task": "text-generation" + "description": "Chat with Pythia 6.9b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Pythia 6.9b", "id": "pythia-6.9b-int8-ov", - "fileSize": 7153039039, + "fileSize": 7153039111, "optimizationPrecision": "int8", "contextWindow": 2048, - "description": "Chat with Pythia 6.9b model", - "task": "text-generation" + "description": "Chat with Pythia 6.9b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Pythia 2.8b", "id": "pythia-2.8b-fp16-ov", - "fileSize": 14293243256, + "fileSize": 14293243287, "optimizationPrecision": "fp16", "contextWindow": 2048, - "description": "Chat with Pythia 2.8b model", - "task": "text-generation" + "description": "Chat with Pythia 2.8b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Pythia 6.9b", "id": "pythia-6.9b-fp16-ov", - "fileSize": 14293243256, + "fileSize": 14293243287, "optimizationPrecision": "fp16", "contextWindow": 2048, - "description": "Chat with Pythia 6.9b model", - "task": "text-generation" + "description": "Chat with Pythia 6.9b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Pythia 1b", "id": "pythia-1b-int4-ov", - "fileSize": 669587847, + "fileSize": 669587919, "optimizationPrecision": "int4", "contextWindow": 2048, - "description": "Chat with Pythia 1b model", - "task": "text-generation" + "description": "Chat with Pythia 1b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Pythia 1b", "id": "pythia-1b-int8-ov", - "fileSize": 1107284420, + "fileSize": 1107284481, "optimizationPrecision": "int8", "contextWindow": 2048, - "description": "Chat with Pythia 1b model", - "task": "text-generation" + "description": "Chat with Pythia 1b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Pythia 1b", "id": "pythia-1b-fp16-ov", - "fileSize": 2181025578, + "fileSize": 2181025619, "optimizationPrecision": "fp16", "contextWindow": 2048, - "description": "Chat with Pythia 1b model", - "task": "text-generation" + "description": "Chat with Pythia 1b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Neural Chat 7b V1 1", "id": "neural-chat-7b-v1-1-int4-ov", - "fileSize": 3824586268, + "fileSize": 3824586350, "optimizationPrecision": "int4", "contextWindow": 2048, - "description": "Chat with Neural Chat 7b V1 1 model", - "task": "text-generation" + "description": "Chat with Neural Chat 7b V1 1", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Neural Chat 7b V1 1", "id": "neural-chat-7b-v1-1-int8-ov", - "fileSize": 7153039101, + "fileSize": 7153039173, "optimizationPrecision": "int8", "contextWindow": 2048, - "description": "Chat with Neural Chat 7b V1 1 model", - "task": "text-generation" + "description": "Chat with Neural Chat 7b V1 1", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Neural Chat 7b V1 1", "id": "neural-chat-7b-v1-1-fp16-ov", - "fileSize": 14293243287, + "fileSize": 14293243328, "optimizationPrecision": "fp16", "contextWindow": 2048, - "description": "Chat with Neural Chat 7b V1 1 model", - "task": "text-generation" + "description": "Chat with Neural Chat 7b V1 1", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Phi 3 Medium 4k Instruct", "id": "Phi-3-medium-4k-instruct-fp16-ov", - "fileSize": 29965606794, + "fileSize": 29965606845, "optimizationPrecision": "fp16", "contextWindow": 4096, - "description": "Chat with Phi 3 Medium 4k Instruct model", - "task": "text-generation" + "description": "Chat with Phi 3 Medium 4k Instruct", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Phi 3 Medium 4k Instruct", "id": "Phi-3-medium-4k-instruct-int4-ov", - "fileSize": 7964805299, + "fileSize": 7964805381, "optimizationPrecision": "int4", "contextWindow": 4096, - "description": "Chat with Phi 3 Medium 4k Instruct model", - "task": "text-generation" + "description": "Chat with Phi 3 Medium 4k Instruct", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Phi 3 Medium 4k Instruct", "id": "Phi-3-medium-4k-instruct-int8-ov", - "fileSize": 15040585641, + "fileSize": 15040585723, "optimizationPrecision": "int8", "contextWindow": 4096, - "description": "Chat with Phi 3 Medium 4k Instruct model", - "task": "text-generation" + "description": "Chat with Phi 3 Medium 4k Instruct", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Mpt 7b", "id": "mpt-7b-int8-ov", - "fileSize": 7146788625, + "fileSize": 7146788697, "optimizationPrecision": "int8", "contextWindow": 2048, - "description": "Chat with Mpt 7b model", - "task": "text-generation" + "description": "Chat with Mpt 7b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Mpt 7b", "id": "mpt-7b-fp16-ov", - "fileSize": 14286814799, + "fileSize": 14286814840, "optimizationPrecision": "fp16", "contextWindow": 2048, - "description": "Chat with Mpt 7b model", - "task": "text-generation" + "description": "Chat with Mpt 7b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Starcoder2 15b", "id": "starcoder2-15b-int8-ov", - "fileSize": 17190289720, + "fileSize": 17190289792, "optimizationPrecision": "int8", "contextWindow": 16384, - "description": "Chat with Starcoder2 15b model", - "task": "text-generation" + "description": "Chat with Starcoder2 15b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Starcoder2 15b", "id": "starcoder2-15b-int4-ov", - "fileSize": 9169658515, + "fileSize": 9169658587, "optimizationPrecision": "int4", "contextWindow": 16384, - "description": "Chat with Starcoder2 15b model", - "task": "text-generation" + "description": "Chat with Starcoder2 15b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Starcoder2 15b", "id": "starcoder2-15b-fp16-ov", - "fileSize": 34262102706, + "fileSize": 34262102747, "optimizationPrecision": "fp16", "contextWindow": 16384, - "description": "Chat with Starcoder2 15b model", - "task": "text-generation" + "description": "Chat with Starcoder2 15b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Starcoder2 7b", "id": "starcoder2-7b-fp16-ov", - "fileSize": 15470719144, + "fileSize": 15470719185, "optimizationPrecision": "fp16", "contextWindow": 16384, - "description": "Chat with Starcoder2 7b model", - "task": "text-generation" + "description": "Chat with Starcoder2 7b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Starcoder2 7b", "id": "starcoder2-7b-int4-ov", - "fileSize": 4111254624, + "fileSize": 4111254696, "optimizationPrecision": "int4", "contextWindow": 16384, - "description": "Chat with Starcoder2 7b model", - "task": "text-generation" + "description": "Chat with Starcoder2 7b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Starcoder2 7b", "id": "starcoder2-7b-int8-ov", - "fileSize": 7729586294, + "fileSize": 7729586365, "optimizationPrecision": "int8", "contextWindow": 16384, - "description": "Chat with Starcoder2 7b model", - "task": "text-generation" + "description": "Chat with Starcoder2 7b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Starcoder2 3b", "id": "starcoder2-3b-int4-ov", - "fileSize": 1780962229, + "fileSize": 1780962300, "optimizationPrecision": "int4", "contextWindow": 16384, - "description": "Chat with Starcoder2 3b model", - "task": "text-generation" + "description": "Chat with Starcoder2 3b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Starcoder2 3b", "id": "starcoder2-3b-fp16-ov", - "fileSize": 6526229346, + "fileSize": 6526229387, "optimizationPrecision": "fp16", "contextWindow": 16384, - "description": "Chat with Starcoder2 3b model", - "task": "text-generation" + "description": "Chat with Starcoder2 3b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Starcoder2 3b", "id": "starcoder2-3b-int8-ov", - "fileSize": 3273295377, + "fileSize": 3273295448, "optimizationPrecision": "int8", "contextWindow": 16384, - "description": "Chat with Starcoder2 3b model", - "task": "text-generation" + "description": "Chat with Starcoder2 3b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Phi 3 Mini 4k Instruct", "id": "Phi-3-mini-4k-instruct-fp16-ov", - "fileSize": 8209920736, + "fileSize": 8212081759, "optimizationPrecision": "fp16", "contextWindow": 4096, - "description": "Chat with Phi 3 Mini 4k Instruct model", - "task": "text-generation" + "description": "Chat with Phi 3 Mini 4k Instruct", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Phi 3 Mini 128k Instruct", "id": "Phi-3-mini-128k-instruct-fp16-ov", - "fileSize": 8210027955, + "fileSize": 8210027996, "optimizationPrecision": "fp16", "contextWindow": 131072, - "description": "Chat with Phi 3 Mini 128k Instruct model", - "task": "text-generation" + "description": "Chat with Phi 3 Mini 128k Instruct", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Phi 3 Mini 128k Instruct", "id": "Phi-3-mini-128k-instruct-int4-ov", - "fileSize": 2637434178, + "fileSize": 2637434260, "optimizationPrecision": "int4", "contextWindow": 131072, - "description": "Chat with Phi 3 Mini 128k Instruct model", - "task": "text-generation" + "description": "Chat with Phi 3 Mini 128k Instruct", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Phi 3 Mini 4k Instruct", "id": "Phi-3-mini-4k-instruct-int4-ov", - "fileSize": 2637358233, + "fileSize": 2317366276, "optimizationPrecision": "int4", "contextWindow": 4096, - "description": "Chat with Phi 3 Mini 4k Instruct model", - "task": "text-generation" + "description": "Chat with Phi 3 Mini 4k Instruct", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Phi 3 Mini 4k Instruct", "id": "Phi-3-mini-4k-instruct-int8-ov", - "fileSize": 4108269167, + "fileSize": 4110513544, "optimizationPrecision": "int8", "contextWindow": 4096, - "description": "Chat with Phi 3 Mini 4k Instruct model", - "task": "text-generation" + "description": "Chat with Phi 3 Mini 4k Instruct", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Phi 3 Mini 128k Instruct", "id": "Phi-3-mini-128k-instruct-int8-ov", - "fileSize": 4108345114, + "fileSize": 4108345195, "optimizationPrecision": "int8", "contextWindow": 131072, - "description": "Chat with Phi 3 Mini 128k Instruct model", - "task": "text-generation" + "description": "Chat with Phi 3 Mini 128k Instruct", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "Open_llama_3b_v2", "id": "open_llama_3b_v2-int4-ov", - "fileSize": 1960434251, + "fileSize": 1960434322, "optimizationPrecision": "int4", "contextWindow": 2048, - "description": "Chat with Open_llama_3b_v2 model", - "task": "text-generation" + "description": "Chat with Open_llama_3b_v2", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "RedPajama INCITE Instruct 3B V1", "id": "RedPajama-INCITE-Instruct-3B-v1-fp16-ov", - "fileSize": 5975908581, + "fileSize": 5975908643, "optimizationPrecision": "fp16", "contextWindow": 2048, - "description": "Chat with RedPajama INCITE Instruct 3B V1 model", - "task": "text-generation" + "description": "Chat with RedPajama INCITE Instruct 3B V1", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "RedPajama INCITE Instruct 3B V1", "id": "RedPajama-INCITE-Instruct-3B-v1-int4-ov", - "fileSize": 1970894247, + "fileSize": 1970894350, "optimizationPrecision": "int4", "contextWindow": 2048, - "description": "Chat with RedPajama INCITE Instruct 3B V1 model", - "task": "text-generation" + "description": "Chat with RedPajama INCITE Instruct 3B V1", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" }, { "name": "RedPajama INCITE Instruct 3B V1", "id": "RedPajama-INCITE-Instruct-3B-v1-int8-ov", - "fileSize": 3001571035, + "fileSize": 3001571127, "optimizationPrecision": "int8", "contextWindow": 2048, - "description": "Chat with RedPajama INCITE Instruct 3B V1 model", - "task": "text-generation" + "description": "Chat with RedPajama INCITE Instruct 3B V1", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" + }, + { + "name": "Gemma 2b It", + "id": "gemma-2b-it-fp16-ov", + "fileSize": 5437344390, + "optimizationPrecision": "fp16", + "contextWindow": 8192, + "description": "Chat with Gemma 2b It", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" + }, + { + "name": "Gemma 2b It", + "id": "gemma-2b-it-int8-ov", + "fileSize": 2753201206, + "optimizationPrecision": "int8", + "contextWindow": 8192, + "description": "Chat with Gemma 2b It", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" + }, + { + "name": "Gemma 2b", + "id": "gemma-2b-fp16-ov", + "fileSize": 5437342382, + "optimizationPrecision": "fp16", + "contextWindow": 8192, + "description": "Chat with Gemma 2b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" + }, + { + "name": "Gemma 2b", + "id": "gemma-2b-int4-ov", + "fileSize": 1722552895, + "optimizationPrecision": "int4", + "contextWindow": 8192, + "description": "Chat with Gemma 2b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" + }, + { + "name": "Gemma 2b It", + "id": "gemma-2b-it-int4-ov", + "fileSize": 1722555876, + "optimizationPrecision": "int4", + "contextWindow": 8192, + "description": "Chat with Gemma 2b It", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" + }, + { + "name": "Gemma 7b", + "id": "gemma-7b-fp16-ov", + "fileSize": 18419687858, + "optimizationPrecision": "fp16", + "contextWindow": 8192, + "description": "Chat with Gemma 7b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" + }, + { + "name": "Gemma 7b", + "id": "gemma-7b-int4-ov", + "fileSize": 5223957293, + "optimizationPrecision": "int4", + "contextWindow": 8192, + "description": "Chat with Gemma 7b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" + }, + { + "name": "Gemma 7b", + "id": "gemma-7b-int8-ov", + "fileSize": 9228794075, + "optimizationPrecision": "int8", + "contextWindow": 8192, + "description": "Chat with Gemma 7b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" + }, + { + "name": "Gemma 7b It", + "id": "gemma-7b-it-int8-ov", + "fileSize": 9228795991, + "optimizationPrecision": "int8", + "contextWindow": 8192, + "description": "Chat with Gemma 7b It", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" + }, + { + "name": "Gemma 7b It", + "id": "gemma-7b-it-fp16-ov", + "fileSize": 18419689774, + "optimizationPrecision": "fp16", + "contextWindow": 8192, + "description": "Chat with Gemma 7b It", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" + }, + { + "name": "Gemma 7b It", + "id": "gemma-7b-it-int4-ov", + "fileSize": 5223959210, + "optimizationPrecision": "int4", + "contextWindow": 8192, + "description": "Chat with Gemma 7b It", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" + }, + { + "name": "Bloomz 1b1", + "id": "bloomz-1b1-fp16-ov", + "fileSize": 2322177211, + "optimizationPrecision": "fp16", + "contextWindow": 0, + "description": "Chat with Bloomz 1b1", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" + }, + { + "name": "Bloomz 1b1", + "id": "bloomz-1b1-int4-ov", + "fileSize": 812344015, + "optimizationPrecision": "int4", + "contextWindow": 0, + "description": "Chat with Bloomz 1b1", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" + }, + { + "name": "Bloomz 3b", + "id": "bloomz-3b-int8-ov", + "fileSize": 3267500581, + "optimizationPrecision": "int8", + "contextWindow": 0, + "description": "Chat with Bloomz 3b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" + }, + { + "name": "Bloomz 1b1", + "id": "bloomz-1b1-int8-ov", + "fileSize": 1184136430, + "optimizationPrecision": "int8", + "contextWindow": 0, + "description": "Chat with Bloomz 1b1", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" + }, + { + "name": "Bloomz 3b", + "id": "bloomz-3b-int4-ov", + "fileSize": 2043570274, + "optimizationPrecision": "int4", + "contextWindow": 0, + "description": "Chat with Bloomz 3b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" + }, + { + "name": "Bloomz 3b", + "id": "bloomz-3b-fp16-ov", + "fileSize": 6488568080, + "optimizationPrecision": "fp16", + "contextWindow": 0, + "description": "Chat with Bloomz 3b", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" + }, + { + "name": "Codegen 2B Multi", + "id": "codegen-2B-multi-int8-ov", + "fileSize": 2995146381, + "optimizationPrecision": "int8", + "contextWindow": 2048, + "description": "Chat with Codegen 2B Multi", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" + }, + { + "name": "Codegen 2B Multi", + "id": "codegen-2B-multi-fp16-ov", + "fileSize": 5980001146, + "optimizationPrecision": "fp16", + "contextWindow": 2048, + "description": "Chat with Codegen 2B Multi", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" + }, + { + "name": "Codegen 2B Multi", + "id": "codegen-2B-multi-int4-ov", + "fileSize": 1696055110, + "optimizationPrecision": "int4", + "contextWindow": 2048, + "description": "Chat with Codegen 2B Multi", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" + }, + { + "name": "Codegen 6B Multi", + "id": "codegen-6B-multi-int4-ov", + "fileSize": 4165776649, + "optimizationPrecision": "int4", + "contextWindow": 2048, + "description": "Chat with Codegen 6B Multi", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" + }, + { + "name": "Codegen 6B Multi", + "id": "codegen-6B-multi-fp16-ov", + "fileSize": 15149850695, + "optimizationPrecision": "fp16", + "contextWindow": 2048, + "description": "Chat with Codegen 6B Multi", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" + }, + { + "name": "Codegen 6B Multi", + "id": "codegen-6B-multi-int8-ov", + "fileSize": 7601603664, + "optimizationPrecision": "int8", + "contextWindow": 2048, + "description": "Chat with Codegen 6B Multi", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" + }, + { + "name": "Phi 3.5 Mini Instruct", + "id": "Phi-3.5-mini-instruct-fp16-ov", + "fileSize": 8211750487, + "optimizationPrecision": "fp16", + "contextWindow": 131072, + "description": "Chat with Phi 3.5 Mini Instruct", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" + }, + { + "name": "Phi 3.5 Mini Instruct", + "id": "Phi-3.5-mini-instruct-int8-ov", + "fileSize": 4110193740, + "optimizationPrecision": "int8", + "contextWindow": 131072, + "description": "Chat with Phi 3.5 Mini Instruct", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" + }, + { + "name": "Phi 3.5 Mini Instruct", + "id": "Phi-3.5-mini-instruct-int4-ov", + "fileSize": 2242029798, + "optimizationPrecision": "int4", + "contextWindow": 131072, + "description": "Chat with Phi 3.5 Mini Instruct", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" + }, + { + "name": "Gemma 2 9b It", + "id": "gemma-2-9b-it-int4-ov", + "fileSize": 5698219462, + "optimizationPrecision": "int4", + "contextWindow": 8192, + "description": "Chat with Gemma 2 9b It", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" + }, + { + "name": "Gemma 2 9b It", + "id": "gemma-2-9b-it-int8-ov", + "fileSize": 9992851184, + "optimizationPrecision": "int8", + "contextWindow": 8192, + "description": "Chat with Gemma 2 9b It", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" + }, + { + "name": "Gemma 2 9b It", + "id": "gemma-2-9b-it-fp16-ov", + "fileSize": 19924637310, + "optimizationPrecision": "fp16", + "contextWindow": 8192, + "description": "Chat with Gemma 2 9b It", + "task": "text-generation", + "author": "OpenVINO", + "collection": "llm-6687aaa2abca3bbcec71a9bd" } ] } diff --git a/scripts/create_manifest.dart b/scripts/create_manifest.dart index 184600cc..5013639b 100644 --- a/scripts/create_manifest.dart +++ b/scripts/create_manifest.dart @@ -169,7 +169,7 @@ void generate() async { final popular = [ "mistral-7b-instruct-v0.1-int8-ov", "Phi-3-mini-4k-instruct-int4-ov", - "distil-whisper-base-fp16-ov", + "whisper-base-fp16-ov", "open_llama_3b_v2-int8-ov", ]; final List collections = [ From 806177c1ac8a1daa791cae6c789d069a89a56142 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Thu, 28 Nov 2024 14:05:23 +0100 Subject: [PATCH 26/33] Added text generation screen --- lib/pages/models/inference.dart | 3 +- lib/pages/text_generation/playground.dart | 245 ++++++++++++++++++ .../text_generation/text_generation.dart | 125 +++++++++ .../widgets/assistant_message.dart | 160 ++++++++++++ .../widgets/model_properties.dart | 58 +++++ .../text_generation/widgets/user_message.dart | 32 +++ lib/providers/project_provider.dart | 2 - lib/theme_fluent.dart | 3 + pubspec.lock | 16 ++ pubspec.yaml | 1 + 10 files changed, 642 insertions(+), 3 deletions(-) create mode 100644 lib/pages/text_generation/playground.dart create mode 100644 lib/pages/text_generation/text_generation.dart create mode 100644 lib/pages/text_generation/widgets/assistant_message.dart create mode 100644 lib/pages/text_generation/widgets/model_properties.dart create mode 100644 lib/pages/text_generation/widgets/user_message.dart diff --git a/lib/pages/models/inference.dart b/lib/pages/models/inference.dart index 1445420b..5e7231e9 100644 --- a/lib/pages/models/inference.dart +++ b/lib/pages/models/inference.dart @@ -1,5 +1,6 @@ import 'package:fluent_ui/fluent_ui.dart'; import 'package:inference/pages/computer_vision/computer_vision.dart'; +import 'package:inference/pages/text_generation/text_generation.dart'; import 'package:inference/project.dart'; class InferencePage extends StatelessWidget { @@ -12,7 +13,7 @@ class InferencePage extends StatelessWidget { case ProjectType.image: return ComputerVisionPage(project); case ProjectType.text: - return Container(); + return TextGenerationPage(project); case ProjectType.speech: return Container(); } diff --git a/lib/pages/text_generation/playground.dart b/lib/pages/text_generation/playground.dart new file mode 100644 index 00000000..8100c290 --- /dev/null +++ b/lib/pages/text_generation/playground.dart @@ -0,0 +1,245 @@ + +import 'package:fluent_ui/fluent_ui.dart'; +import 'package:flutter/services.dart'; +import 'package:inference/pages/models/widgets/grid_container.dart'; +import 'package:inference/pages/text_generation/widgets/assistant_message.dart'; +import 'package:inference/pages/text_generation/widgets/model_properties.dart'; +import 'package:inference/pages/text_generation/widgets/user_message.dart'; +import 'package:inference/project.dart'; +import 'package:inference/providers/text_inference_provider.dart'; +import 'package:inference/theme_fluent.dart'; +import 'package:inference/widgets/device_selector.dart'; +import 'package:intl/intl.dart'; +import 'package:provider/provider.dart'; + +class Playground extends StatefulWidget { + final Project project; + + const Playground({required this.project, super.key}); + + + @override + _PlaygroundState createState() => _PlaygroundState(); +} + +class SubmitMessageIntent extends Intent {} + +class _PlaygroundState extends State { + final textController = TextEditingController(); + final scrollController = ScrollController(); + bool attachedToBottom = true; + + void jumpToBottom({ offset = 0 }) { + if (scrollController.hasClients) { + scrollController.jumpTo(scrollController.position.maxScrollExtent + offset); + } + } + + void message(String message) { + if (message.isEmpty) return; + final provider = Provider.of(context, listen: false); + if (!provider.initialized || provider.response != null) return; + textController.text = ''; + jumpToBottom(offset: 110); //move to bottom including both + // TODO: add error handling + provider.message(message).catchError((e) { print(e); }); + } + + @override + void initState() { + super.initState(); + scrollController.addListener(() { + setState(() { + attachedToBottom = scrollController.position.pixels + 0.001 >= scrollController.position.maxScrollExtent; + }); + }); + } + + @override + void dispose() { + textController.dispose(); + scrollController.dispose(); + super.dispose(); + } + + @override + void didChangeDependencies() { + super.didChangeDependencies(); + if (attachedToBottom) { + jumpToBottom(); + } + } + + @override + Widget build(BuildContext context) { + Locale locale = Localizations.localeOf(context); + final nf = NumberFormat.decimalPatternDigits( + locale: locale.languageCode, decimalDigits: 2); + final theme = FluentTheme.of(context); + + return Row( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Consumer(builder: (context, provider, child) => + Expanded(child: Column( + children: [ + SizedBox( + height: 64, + child: GridContainer( + child: Padding( + padding: const EdgeInsets.symmetric(horizontal: 16), + child: Row( + children: [ + const DeviceSelector(), + const Divider(size: 24,direction: Axis.vertical,), + const SizedBox(width: 24,), + const Text('Temperature: '), + Slider( + value: provider.temperature, + onChanged: (value) { provider.temperature = value; }, + label: nf.format(provider.temperature), + min: 0.1, + max: 2.0, + ), + const SizedBox(width: 24,), + const Text('Top P: '), + Slider( + value: provider.topP, + onChanged: (value) { provider.topP = value; }, + label: nf.format(provider.topP), + max: 1.0, + min: 0.1, + ), + ], + ) + ), + ), + ), + Expanded(child: DecoratedBox( + decoration: BoxDecoration( + color: theme.brightness.isDark ? backgroundColor.dark : theme.scaffoldBackgroundColor + ), + child: GridContainer(child: SizedBox( + width: double.infinity, + child: Builder(builder: (context) { + if (!provider.initialized) { + return const Column( + mainAxisAlignment: MainAxisAlignment.center, + children: [ + SizedBox(width: 64,height: 64, child: ProgressRing()), + Padding( + padding: EdgeInsets.only(top: 18), + child: Text("Loading model..."), + ) + ], + ); + } + return Column( + children: [ + Expanded( + child: Builder(builder: (context) { + if (provider.messages.isEmpty) { + return Center( + child: Text("Start chatting with ${provider.project?.name ?? "the model"}!"), + ); + } + return Stack( + alignment: Alignment.bottomCenter, + children: [ + SingleChildScrollView( + controller: scrollController, + child: Padding(padding: const EdgeInsets.symmetric(horizontal: 64, vertical: 20), child: Column( + children: provider.messages.map((message) { switch (message.speaker) { + case Speaker.user: return UserMessage(message); + case Speaker.system: return Text('System: ${message.message}'); + case Speaker.assistant: return AssistantMessage(message); + }}).toList(), + ),), + ), + Positioned( + bottom: 10, + child: Builder(builder: (context) => attachedToBottom + ? const SizedBox() + : Padding( + padding: const EdgeInsets.only(top:2), + child: FilledButton(child: const Row( + children: [ + Icon(FluentIcons.chevron_down, size: 12), + SizedBox(width: 4), + Text('Scroll to bottom'), + ], + ), onPressed: () { + jumpToBottom(); + setState(() { + attachedToBottom = true; + }); + }), + ) + ), + ) + ], + ); + }), + ), + Padding( + padding: const EdgeInsets.symmetric(horizontal: 64, vertical: 24), + child: Row( + crossAxisAlignment: CrossAxisAlignment.end, + mainAxisAlignment: MainAxisAlignment.center, + children: [ + Tooltip( + message: "Create new thread", + child: Button(child: const Icon(FluentIcons.rocket, size: 18,), onPressed: () {}), + ), + Expanded( + child: Padding( + padding: const EdgeInsets.symmetric(horizontal: 8), + child: Shortcuts( + shortcuts: { + LogicalKeySet(LogicalKeyboardKey.meta, LogicalKeyboardKey.enter): SubmitMessageIntent(), + LogicalKeySet(LogicalKeyboardKey.control, LogicalKeyboardKey.enter): SubmitMessageIntent(), + }, + child: Actions( + actions: >{ + SubmitMessageIntent: CallbackAction( + onInvoke: (SubmitMessageIntent intent) => message(textController.text), + ), + }, + child: TextBox( + placeholder: "Type a message...", + keyboardType: TextInputType.text, + controller: textController, + maxLines: null, + expands: true, + onSubmitted: message, + autofocus: true, + ), + ), + ), + ), + ), + Builder(builder: (context) => provider.interimResponse != null + ? Tooltip( + message: "Stop", + child: Button(child: const Icon(FluentIcons.stop, size: 18,), onPressed: () { provider.forceStop(); }), + ) + : Tooltip( + message: "Send message", + child: Button(child: const Icon(FluentIcons.send, size: 18,), onPressed: () { message(textController.text); }), + ) + ) + ] + ), + ) + ], + ); + }), + )), + )), + ], + ))), + const ModelProperties(), + ], + ); + } +} \ No newline at end of file diff --git a/lib/pages/text_generation/text_generation.dart b/lib/pages/text_generation/text_generation.dart new file mode 100644 index 00000000..fa9ba926 --- /dev/null +++ b/lib/pages/text_generation/text_generation.dart @@ -0,0 +1,125 @@ + +import 'package:fluent_ui/fluent_ui.dart'; +import 'package:go_router/go_router.dart'; +import 'package:inference/pages/text_generation/playground.dart'; +import 'package:inference/project.dart'; +import 'package:inference/providers/preference_provider.dart'; +import 'package:inference/providers/text_inference_provider.dart'; +import 'package:provider/provider.dart'; + +class TextGenerationPage extends StatefulWidget { + final Project project; + const TextGenerationPage(this.project, {super.key}); + + @override + State createState() => _TextGenerationPageState(); +} + +class _TextGenerationPageState extends State { + int selected = 0; + + @override + Widget build(BuildContext context) { + final theme = FluentTheme.of(context); + final updatedTheme = theme.copyWith( + navigationPaneTheme: theme.navigationPaneTheme.merge(NavigationPaneThemeData( + backgroundColor: theme.scaffoldBackgroundColor, + )) + ); + + return ChangeNotifierProxyProvider( + create: (_) { + return TextInferenceProvider(widget.project, null); + }, + update: (_, preferences, textInferenceProvider) { + final init = textInferenceProvider == null || + !textInferenceProvider.sameProps(widget.project, preferences.device); + if (init) { + final textInferenceProvider = TextInferenceProvider(widget.project, preferences.device); + textInferenceProvider.loadModel().catchError((e) { + // TODO: Error handling + print(e); + }); + return textInferenceProvider; + } + if (!textInferenceProvider.sameProps(widget.project, preferences.device)) { + return TextInferenceProvider(widget.project, preferences.device); + } + return textInferenceProvider; + }, + child: Stack( + children: [ + FluentTheme( + data: updatedTheme, + child: NavigationView( + pane: NavigationPane( + size: const NavigationPaneSize(topHeight: 64), + header: Row( + children: [ + Padding( + padding: const EdgeInsets.only(left: 12.0), + child: ClipRRect( + borderRadius: BorderRadius.circular(4.0), + child: Container( + width: 40, + height: 40, + decoration: BoxDecoration( + image: DecorationImage( + image: widget.project.thumbnailImage(), + fit: BoxFit.cover), + ), + ), + ), + ), + Padding( + padding: const EdgeInsets.symmetric(horizontal: 16), + child: Text(widget.project.name, + style: const TextStyle(fontSize: 20, fontWeight: FontWeight.bold), + ), + ), + ], + ), + selected: selected, + onChanged: (i) => setState(() {selected = i;}), + displayMode: PaneDisplayMode.top, + items: [ + PaneItem( + icon: const Icon(FluentIcons.game), + title: const Text("Playground"), + body: Playground(project: widget.project), + ), + ], + ) + ), + ), + SizedBox( + height: 64, + child: Padding( + padding: const EdgeInsets.symmetric(horizontal: 25), + child: Row( + mainAxisAlignment: MainAxisAlignment.end, + children: [ + Padding( + padding: const EdgeInsets.all(4), + child: OutlinedButton( + style: ButtonStyle( + shape:WidgetStatePropertyAll(RoundedRectangleBorder( + borderRadius: BorderRadius.circular(4.0), + side: const BorderSide(color: Color(0XFF545454)), + )), + ), + child: const Text("Close"), + onPressed: () => GoRouter.of(context).canPop() + ? GoRouter.of(context).pop() + : GoRouter.of(context).push('/models'), + ), + ), + ] + ), + ), + ) + ], + ), + ); + } +} \ No newline at end of file diff --git a/lib/pages/text_generation/widgets/assistant_message.dart b/lib/pages/text_generation/widgets/assistant_message.dart new file mode 100644 index 00000000..dd13cf84 --- /dev/null +++ b/lib/pages/text_generation/widgets/assistant_message.dart @@ -0,0 +1,160 @@ +import 'package:fluent_ui/fluent_ui.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter_markdown/flutter_markdown.dart'; +import 'package:inference/providers/text_inference_provider.dart'; +import 'package:inference/theme_fluent.dart'; +import 'package:intl/intl.dart'; +import 'package:provider/provider.dart'; + +class AssistantMessage extends StatefulWidget { + final Message message; + const AssistantMessage(this.message, {super.key}); + + @override + _AssistantMessageState createState() => _AssistantMessageState(); +} + +class _AssistantMessageState extends State { + bool _hovering = false; + + @override + Widget build(BuildContext context) { + Locale locale = Localizations.localeOf(context); + final nf = NumberFormat.decimalPatternDigits( + locale: locale.languageCode, decimalDigits: 0); + final theme = FluentTheme.of(context); + final backgroundColor = theme.brightness.isDark + ? theme.scaffoldBackgroundColor + : const Color(0xFFF5F5F5); + + return Consumer(builder: (context, inferenceProvider, child) => + Align( + alignment: Alignment.centerLeft, + child: Padding( + padding: const EdgeInsets.only(bottom: 20), + child: Row( + crossAxisAlignment: CrossAxisAlignment.end, + children: [ + Padding( + padding: const EdgeInsets.only(right: 10, bottom: 36), + child: ClipRRect( + borderRadius: BorderRadius.circular(16.0), + child: Container( + width: 32, + height: 32, + decoration: BoxDecoration( + image: DecorationImage( + image: inferenceProvider.project!.thumbnailImage(), + fit: BoxFit.cover, + ), + ), + ), + ), + ), + Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Padding( + padding: const EdgeInsets.only(bottom: 4), + child: Text( + inferenceProvider.project!.name, + style: TextStyle( + color: subtleTextColor.of(theme), + ), + ), + ), + MouseRegion( + onEnter: (_) => setState(() { _hovering = true; }), + onExit: (_) => setState(() { _hovering = false; }), + child: Column( + crossAxisAlignment: CrossAxisAlignment.end, + children: [ + Container( + constraints: BoxConstraints(maxWidth: MediaQuery.of(context).size.width - 502), + decoration: BoxDecoration( + color: backgroundColor, + borderRadius: BorderRadius.circular(4), + ), + child: Markdown( + data: widget.message.message, + selectable: true, + shrinkWrap: true, + padding: const EdgeInsets.all(8), + physics: const NeverScrollableScrollPhysics(), + ), + ), + if (_hovering) + Padding( + padding: const EdgeInsets.only(top: 4), + child: Row( + children: [ + if (widget.message.metrics != null) Padding( + padding: const EdgeInsets.only(right: 8), + child: Tooltip( + message: 'Time to first token', + child: Text( + 'TTF: ${nf.format(widget.message.metrics!.ttft)}ms', + style: TextStyle( + fontSize: 12, + color: subtleTextColor.of(theme), + ), + ), + ), + ), + if (widget.message.metrics != null) Padding( + padding: const EdgeInsets.only(right: 8), + child: Tooltip( + message: 'Time per output token', + child: Text( + 'TPOT: ${nf.format(widget.message.metrics!.tpot)}ms', + style: TextStyle( + fontSize: 12, + color: subtleTextColor.of(theme), + ), + ), + ), + ), + if (widget.message.metrics != null) Padding( + padding: const EdgeInsets.only(right: 8), + child: Tooltip( + message: 'Generate total duration', + child: Text( + 'Generate: ${nf.format(widget.message.metrics!.generate_time/1000)}s', + style: TextStyle( + fontSize: 12, + color: subtleTextColor.of(theme), + ), + ), + ), + ), + IconButton( + icon: const Icon(FluentIcons.copy), + onPressed: () async{ + await displayInfoBar(context, builder: (context, close) => + InfoBar( + title: const Text('Copied to clipboard'), + severity: InfoBarSeverity.info, + action: IconButton( + icon: const Icon(FluentIcons.clear), + onPressed: close, + ), + ), + ); + Clipboard.setData(ClipboardData(text: widget.message.message)); + }, + ), + ], + ), + ) else const SizedBox(height: 34) + ], + ), + ), + ], + ), + ], + ), + ) + ), + ); + } +} \ No newline at end of file diff --git a/lib/pages/text_generation/widgets/model_properties.dart b/lib/pages/text_generation/widgets/model_properties.dart new file mode 100644 index 00000000..cd6bc53b --- /dev/null +++ b/lib/pages/text_generation/widgets/model_properties.dart @@ -0,0 +1,58 @@ +import 'package:flutter/widgets.dart'; +import 'package:inference/pages/computer_vision/widgets/model_properties.dart'; +import 'package:inference/pages/models/widgets/grid_container.dart'; +import 'package:inference/providers/text_inference_provider.dart'; +import 'package:intl/intl.dart'; +import 'package:provider/provider.dart'; + +class ModelProperties extends StatelessWidget { + const ModelProperties({super.key}); + + @override + Widget build(BuildContext context) { + Locale locale = Localizations.localeOf(context); + final nf = NumberFormat.decimalPatternDigits( + locale: locale.languageCode, decimalDigits: 2); + + return Consumer(builder: (context, inference, child) { + return SizedBox( + width: 280, + child: GridContainer( + padding: const EdgeInsets.symmetric(vertical: 18, horizontal: 24), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + const Text("Model parameters", style: TextStyle( + fontSize: 20, + )), + Container( + padding: const EdgeInsets.only(top: 16), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + ModelProperty( + title: "Model name", + value: inference.project!.name, + ), + ModelProperty( + title: "Architecture", + value: inference.project!.architecture, + ), + ModelProperty( + title: "Temperature", + value: nf.format(inference.temperature), + ), + ModelProperty( + title: "Top P", + value: nf.format(inference.topP), + ), + ] + ) + ), + ] + ) + ) + ); + }); + } +} \ No newline at end of file diff --git a/lib/pages/text_generation/widgets/user_message.dart b/lib/pages/text_generation/widgets/user_message.dart new file mode 100644 index 00000000..e440d0fc --- /dev/null +++ b/lib/pages/text_generation/widgets/user_message.dart @@ -0,0 +1,32 @@ +import 'package:fluent_ui/fluent_ui.dart'; +import 'package:flutter/material.dart'; +import 'package:inference/providers/text_inference_provider.dart'; +import 'package:inference/theme_fluent.dart'; + +class UserMessage extends StatelessWidget { + final Message message; + const UserMessage(this.message, {super.key}); + + @override + Widget build(BuildContext context) { + final theme = FluentTheme.of(context); + return Align( + alignment: Alignment.centerRight, + child: Padding(padding: const EdgeInsets.only(bottom: 20), child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Container( + decoration: BoxDecoration( + color: cosmosBackground.of(theme), + borderRadius: BorderRadius.circular(4), + ), + child: Padding( + padding: const EdgeInsets.all(8.0), + child: SelectableText(message.message,), + ), + ) + ], + ),), + ); + } +} \ No newline at end of file diff --git a/lib/providers/project_provider.dart b/lib/providers/project_provider.dart index e9d94542..cc4670c7 100644 --- a/lib/providers/project_provider.dart +++ b/lib/providers/project_provider.dart @@ -1,5 +1,3 @@ -import 'dart:async'; - import 'package:collection/collection.dart'; import 'package:flutter/material.dart'; import 'package:inference/deployment_processor.dart'; diff --git a/lib/theme_fluent.dart b/lib/theme_fluent.dart index 25fff0a4..1c17a629 100644 --- a/lib/theme_fluent.dart +++ b/lib/theme_fluent.dart @@ -103,12 +103,15 @@ class DarkLightColor { const borderColor = DarkLightColor(Color(0xFFF0F0F0), Color(0xFF3B3B3B)); const backgroundColor = DarkLightColor(Color(0xFFF9F9F9), Color(0xFF282828)); const subtleTextColor = DarkLightColor(Color(0xFF616161), Color(0xFF9F9F9F)); +const neutralBackground = DarkLightColor(Color(0xFFF5F5F5), Color(0xFF343434)); +const cosmosBackground = DarkLightColor(Color(0xFFEFEAFF), Color(0xFF463d66)); final AccentColor electricCosmos = AccentColor.swatch(const { 'normal': Color(0xFF7000FF), }); final AccentColor cosmos = AccentColor.swatch(const { + 'darkest': Color(0xFF463d66), 'normal': Color(0xFFAF98FF), 'lightest': Color(0xFFEFEAFF), }); diff --git a/pubspec.lock b/pubspec.lock index 9e93ab40..498a26de 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -341,6 +341,14 @@ packages: description: flutter source: sdk version: "0.0.0" + flutter_markdown: + dependency: "direct main" + description: + name: flutter_markdown + sha256: "255b00afa1a7bad19727da6a7780cf3db6c3c12e68d302d85e0ff1fdf173db9e" + url: "https://pub.dev" + source: hosted + version: "0.7.4+3" flutter_plugin_android_lifecycle: dependency: transitive description: @@ -545,6 +553,14 @@ packages: url: "https://pub.dev" source: hosted version: "0.1.2-main.4" + markdown: + dependency: transitive + description: + name: markdown + sha256: ef2a1298144e3f985cc736b22e0ccdaf188b5b3970648f2d9dc13efd1d9df051 + url: "https://pub.dev" + source: hosted + version: "7.2.2" matcher: dependency: transitive description: diff --git a/pubspec.yaml b/pubspec.yaml index 5e35670e..6c06fb78 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -56,6 +56,7 @@ dependencies: fluent_ui: ^4.9.2 system_theme: ^3.1.2 flutter_acrylic: ^1.1.4 + flutter_markdown: ^0.7.4+3 dev_dependencies: flutter_test: From 3378fc45008231881a6f8d6aa1eec3583e6b14c2 Mon Sep 17 00:00:00 2001 From: Arend Jan Kramer Date: Thu, 28 Nov 2024 16:47:06 +0100 Subject: [PATCH 27/33] Migrate TTI to fluent --- images/clear.svg | 6 +- images/copy.svg | 33 +- images/network.svg | 7 - images/playground.svg | 4 + images/stats.svg | 13 +- images/user.svg | 7 - lib/inference/inference_page.dart | 3 +- lib/inference/textToImage/tti_playground.dart | 463 --------------- .../text_to_image_inference_page.dart | 187 ------ lib/pages/models/inference.dart | 3 +- .../text_to_image/live_inference_pane.dart | 92 +++ .../performance_metrics_pane.dart} | 25 +- .../text_to_image_inference_provider.dart | 29 +- .../text_to_image/text_to_image_page.dart | 147 +++++ .../widgets/device_selector.dart | 66 +++ .../widgets/horizontal_rule.dart | 25 + .../text_to_image/widgets/metrics_card.dart | 107 ++++ .../widgets/model_properties.dart | 105 ++++ .../widgets/toolbar_text_input.dart | 115 ++++ .../text_to_image/widgets/tti_chat_area.dart | 539 ++++++++++++++++++ .../widgets/tti_metrics_grid.dart} | 14 +- .../text_to_image/widgets/vertical_rule.dart | 25 + lib/theme_fluent.dart | 2 + lib/widgets/import_model_button.dart | 44 +- 24 files changed, 1319 insertions(+), 742 deletions(-) create mode 100644 images/playground.svg delete mode 100644 lib/inference/textToImage/tti_playground.dart delete mode 100644 lib/inference/text_to_image_inference_page.dart create mode 100644 lib/pages/text_to_image/live_inference_pane.dart rename lib/{inference/textToImage/tti_performance_metrics.dart => pages/text_to_image/performance_metrics_pane.dart} (63%) rename lib/{ => pages/text_to_image}/providers/text_to_image_inference_provider.dart (92%) create mode 100644 lib/pages/text_to_image/text_to_image_page.dart create mode 100644 lib/pages/text_to_image/widgets/device_selector.dart create mode 100644 lib/pages/text_to_image/widgets/horizontal_rule.dart create mode 100644 lib/pages/text_to_image/widgets/metrics_card.dart create mode 100644 lib/pages/text_to_image/widgets/model_properties.dart create mode 100644 lib/pages/text_to_image/widgets/toolbar_text_input.dart create mode 100644 lib/pages/text_to_image/widgets/tti_chat_area.dart rename lib/{inference/textToImage/tti_metric_widgets.dart => pages/text_to_image/widgets/tti_metrics_grid.dart} (68%) create mode 100644 lib/pages/text_to_image/widgets/vertical_rule.dart diff --git a/images/clear.svg b/images/clear.svg index a9f643c0..a7fc7e39 100644 --- a/images/clear.svg +++ b/images/clear.svg @@ -1,2 +1,4 @@ - - \ No newline at end of file + + + \ No newline at end of file diff --git a/images/copy.svg b/images/copy.svg index 7a23f242..d2b20a7e 100644 --- a/images/copy.svg +++ b/images/copy.svg @@ -1,31 +1,4 @@ - - - - - S Copy 18 N - - - - - - - - - - - - - - - - - - - - - + + + diff --git a/images/network.svg b/images/network.svg index d9d0aa05..f9764d43 100644 --- a/images/network.svg +++ b/images/network.svg @@ -1,11 +1,4 @@ - - - S SocialNetwork 18 N diff --git a/images/playground.svg b/images/playground.svg new file mode 100644 index 00000000..ef094789 --- /dev/null +++ b/images/playground.svg @@ -0,0 +1,4 @@ + + + + diff --git a/images/stats.svg b/images/stats.svg index cd039b5b..7fef770d 100644 --- a/images/stats.svg +++ b/images/stats.svg @@ -1,9 +1,4 @@ - - - - - - - - - \ No newline at end of file + + + + diff --git a/images/user.svg b/images/user.svg index 01dc6b59..f5bb8944 100644 --- a/images/user.svg +++ b/images/user.svg @@ -1,11 +1,4 @@ - - - S User 18 N diff --git a/lib/inference/inference_page.dart b/lib/inference/inference_page.dart index 6e13ac09..a9ef3970 100644 --- a/lib/inference/inference_page.dart +++ b/lib/inference/inference_page.dart @@ -2,7 +2,6 @@ import 'package:flutter/material.dart'; import 'package:inference/inference/download_page.dart'; import 'package:inference/inference/image_inference_page.dart'; import 'package:inference/inference/text_inference_page.dart'; -import 'package:inference/inference/text_to_image_inference_page.dart'; import 'package:inference/project.dart'; import 'package:inference/providers/download_provider.dart'; import 'package:provider/provider.dart'; @@ -25,7 +24,7 @@ class _InferencePageState extends State { case ProjectType.text: return TextInferencePage(widget.project); case ProjectType.textToImage: - return TextToImageInferencePage(widget.project); + return Container(); case ProjectType.speech: return Container(); } diff --git a/lib/inference/textToImage/tti_playground.dart b/lib/inference/textToImage/tti_playground.dart deleted file mode 100644 index 3a468f8c..00000000 --- a/lib/inference/textToImage/tti_playground.dart +++ /dev/null @@ -1,463 +0,0 @@ - -import 'package:flutter/material.dart'; -import 'package:flutter_svg/svg.dart'; -import 'package:inference/config.dart'; -import 'package:inference/hint.dart'; -import 'package:inference/inference/device_selector.dart'; -import 'package:inference/inference/textToImage/tti_metric_widgets.dart'; -import 'package:inference/interop/openvino_bindings.dart'; -import 'package:inference/providers/text_to_image_inference_provider.dart'; -import 'package:inference/theme.dart'; -import 'package:provider/provider.dart'; -import 'package:super_clipboard/super_clipboard.dart'; - -class TTIPlayground extends StatefulWidget { - const TTIPlayground({super.key}); - - @override - State createState() => _PlaygroundState(); -} - -class _PlaygroundState extends State { - final _controller = TextEditingController(); - final _scrollController = ScrollController(); - bool attachedToBottom = true; - - void jumpToBottom({ offset = 0 }) { - if (_scrollController.hasClients) { - _scrollController.jumpTo(_scrollController.position.maxScrollExtent + offset); - } - } - - void message(String message) async { - if (message.isEmpty) { - return; - } - final tti = provider(); - if (!tti.initialized) { - return; - } - - if (tti.response != null) { - return; - } - _controller.text = ""; - jumpToBottom(offset: 110); //move to bottom including both - tti.message(message); - } - - TextToImageInferenceProvider provider() => Provider.of(context, listen: false); - - @override - void initState() { - super.initState(); - _scrollController.addListener(() { - setState(() { - attachedToBottom = _scrollController.position.pixels + 0.001 >= _scrollController.position.maxScrollExtent; - }); - - }); - } - - @override - void dispose() { - super.dispose(); - _controller.dispose(); - _scrollController.dispose(); - } - - - @override - Widget build(BuildContext context) { - return Consumer(builder: (context, inference, child) { - WidgetsBinding.instance.addPostFrameCallback((_) { - if (attachedToBottom) { - jumpToBottom(); - } - }); - - return Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - const Padding( - padding: EdgeInsets.only(left: 8), - child: Row( - mainAxisAlignment: MainAxisAlignment.spaceBetween, - children: [ - DeviceSelector(), - Hint(hint: HintsEnum.intelCoreLLMPerformanceSuggestion), - ] - ), - ), - Builder( - builder: (context) { - if (!inference.initialized){ - return Expanded( - child: Center( - child: Column( - mainAxisAlignment: MainAxisAlignment.center, - children: [ - Image.asset('images/intel-loading.gif', width: 100), - const Text("Loading model...") - ], - ) - ), - ); - } - return Expanded( - child: Container( - decoration: const BoxDecoration( - shape: BoxShape.rectangle, - borderRadius: BorderRadius.all(Radius.circular(8)), - color: intelGray, - ), - child: Column( - children: [ - Expanded( - child: Builder(builder: (context) { - if (inference.messages.isEmpty) { - return Center( - child: Text("Type a message to ${inference.project?.name ?? "assistant"}")); - } - return Stack( - alignment: Alignment.bottomCenter, - children: [ - SingleChildScrollView( - controller: _scrollController, - child: Padding( - padding: const EdgeInsets.all(20), - child: Column( - //mainAxisAlignment: MainAxisAlignment.start, - crossAxisAlignment: CrossAxisAlignment.stretch, - children: inference.messages.map((message) { - switch (message.speaker) { - case Speaker.user: - return UserInputMessage(message); - case Speaker.assistant: - return GeneratedImageMessage(message, inference.project!.name); - } - }).toList()), - ), - ), - Positioned( - bottom: 10, - child: Builder( - builder: (context) { - if (attachedToBottom) { - return Container(); - } - return Center( - child: Padding( - padding: const EdgeInsets.only(top: 2.0), - child: SizedBox( - width: 200, - height: 20, - child: FloatingActionButton( - backgroundColor: intelGray, - child: const Text("Jump to bottom"), - onPressed: () { - jumpToBottom(); - setState(() { - attachedToBottom = true; - }); - } - ), - ), - ), - ); - } - ), - ), - - ], - ); - }), - ), - - // SizedBox( - // height: 30, - // child: Builder( - // builder: (context) { - // if (inference.interimResponse == null){ - // return Container(); - // } - // return Center( - // child: OutlinedButton.icon( - // onPressed: () => inference.forceStop(), - // icon: const Icon(Icons.stop), - // label: const Text("Stop responding") - // ), - // ); - // } - // ), - // ), - Padding( - padding: const EdgeInsets.only(left: 45, right: 45, top: 10, bottom: 25), - child: SizedBox( - height: 40, - child: Row( - crossAxisAlignment: CrossAxisAlignment.end, - children: [ - Padding( - padding: const EdgeInsets.only(right: 8), - child: IconButton( - icon: SvgPicture.asset("images/clear.svg", - colorFilter: const ColorFilter.mode(textColor, BlendMode.srcIn), - width: 20, - ), - tooltip: "Clear chat", - onPressed: () => inference.reset(), - style: IconButton.styleFrom( - backgroundColor: intelGrayReallyDark, - shape: const RoundedRectangleBorder( - borderRadius: BorderRadius.all(Radius.circular(4)), - side: BorderSide( - color: intelGrayLight, - width: 2, - ) - ) - ) - ), - ), - Expanded( - child: TextField( - maxLines: null, - keyboardType: TextInputType.text, - decoration: InputDecoration( - hintText: "Ask me anything...", - suffixIcon: IconButton( - icon: Icon(Icons.send, color: (inference.interimResponse == null ? Colors.white : intelGray)), - onPressed: () => message(_controller.text), - ), - enabledBorder: const OutlineInputBorder( - borderRadius: BorderRadius.all(Radius.circular(4)), - borderSide: BorderSide( - color: intelGrayLight, - width: 2, - ) - ), - ), - style: const TextStyle( - fontSize: 14, - ), - controller: _controller, - onSubmitted: message, - ), - ), - ], - ), - ), - ), - ], - ), - ), - ); - } - ), - ], - ); - }); - } -} - -class UserInputMessage extends StatelessWidget { - final Message message; - const UserInputMessage(this.message, {super.key}); - - @override - Widget build(BuildContext context) { - return Padding( - padding: const EdgeInsets.only(bottom: 20), - child: Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - NameRowWidget(name: "You", icon: SvgPicture.asset("images/user.svg", - colorFilter: const ColorFilter.mode(textColor, BlendMode.srcIn), - width: 20, - ), - ), - MessageWidget(message: message.message), - ], - ), - ); - } -} - -class GeneratedImageMessage extends StatelessWidget { - final Message message; - final String name; - const GeneratedImageMessage(this.message, this.name, {super.key}); - - @override - Widget build(BuildContext context) { - return Padding( - padding: const EdgeInsets.only(bottom: 20), - child: Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - NameRowWidget( - name: name, - icon: SvgPicture.asset("images/network.svg", - colorFilter: const ColorFilter.mode(textColor, BlendMode.srcIn), - width: 20, - ), - ), - ImageWidget(message: message.message, image: Image.memory(message.imageContent!.imageData, width: message.imageContent!.width.toDouble(), height: message.imageContent!.height.toDouble(), fit: message.imageContent!.boxFit)), - Padding( - padding: const EdgeInsets.only(left: 28, top: 5), - child: Builder( - builder: (context) { - if (message.speaker == Speaker.user) { - return Container(); - } - return Row( - children: [ - Opacity( - opacity: message.allowedCopy ? 1.0 : 0.25, - child: - IconButton.filled( - icon: SvgPicture.asset("images/copy.svg", - colorFilter: const ColorFilter.mode(textColor, BlendMode.srcIn), - width: 20, - ), - style: IconButton.styleFrom( - backgroundColor: intelGrayLight, - shape: const RoundedRectangleBorder( - borderRadius: BorderRadius.all(Radius.circular(4)), - ), - ), - padding: const EdgeInsets.all(4), - constraints: const BoxConstraints(), - tooltip: message.allowedCopy ? "Copy to clipboard" : null, - onPressed: message.imageContent?.imageData == null || message.allowedCopy == false ? null : () { - - final clipboard = SystemClipboard.instance; - if (clipboard == null) { - return; // Clipboard API is not supported on this platform. - } - final item = DataWriterItem(); - item.add(Formats.jpeg(message.imageContent!.imageData)); - clipboard.write([item]); - - }, - ) - ), - Padding( - padding: const EdgeInsets.only(left: 8), - child: IconButton( - style: IconButton.styleFrom( - backgroundColor: intelGrayLight, - shape: const RoundedRectangleBorder( - borderRadius: BorderRadius.all(Radius.circular(4)), - ), - ), - padding: const EdgeInsets.all(4), - constraints: const BoxConstraints(), - icon: SvgPicture.asset("images/stats.svg", - colorFilter: const ColorFilter.mode(textColor, BlendMode.srcIn), - width: 20, - ), - tooltip: "Show stats", - onPressed: () { - showMetricsDialog(context, message.metrics!); - }, - ), - ), - ], - ); - } - ), - ), - ], - ), - ); - } -} - -void showMetricsDialog(BuildContext context, TTIMetrics metrics) { - showDialog( - context: context, - builder: (BuildContext context) { - return AlertDialog( - content: TTICirclePropRow( - metrics: metrics - ) - ); - } - ); -} - -class NameRowWidget extends StatelessWidget { - final String name; - final Widget icon; - const NameRowWidget({super.key, required this.name, required this.icon}); - - @override - Widget build(BuildContext context) { - return Row( - children: [ - Container( - padding: const EdgeInsets.all(2), - decoration: BoxDecoration( - borderRadius: BorderRadius.circular(4.0), - color: intelBlueVibrant, - //color: intelGrayLight, - ), - child: icon - ), - Padding( - padding: const EdgeInsets.only(left: 10.0), - child: Text(name), - ) - ] - ); - } -} - -class MessageWidget extends StatelessWidget { - final String message; - const MessageWidget({super.key, required this.message}); - - @override - Widget build(BuildContext context) { - return Padding( - padding: const EdgeInsets.only(left: 34.0, top: 10, right: 26), - child: SelectableText( - message, - style: const TextStyle( - color: textColor, - fontSize: 12, - ), - ), - ); - } - -} - -class ImageWidget extends StatelessWidget { - final String message; - final Image? image; - const ImageWidget({super.key, required this.message, required this.image}); - - @override - Widget build(BuildContext context) { - return Padding( - padding: const EdgeInsets.only(left: 34.0, top: 10, right: 26), - child: Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - // Image widget goes here - image ?? Container(), - const SizedBox(height: 8), // Add some spacing between image and text - SelectableText( - message, - style: const TextStyle( - color: textColor, - fontSize: 12, - ), - ), - ], - ), - ); - } -} diff --git a/lib/inference/text_to_image_inference_page.dart b/lib/inference/text_to_image_inference_page.dart deleted file mode 100644 index 509725db..00000000 --- a/lib/inference/text_to_image_inference_page.dart +++ /dev/null @@ -1,187 +0,0 @@ -import 'package:flutter/material.dart'; -import 'package:inference/header.dart'; -import 'package:inference/inference/model_info.dart'; -import 'package:inference/inference/textToImage/tti_performance_metrics.dart'; -import 'package:inference/inference/textToImage/tti_playground.dart'; -import 'package:inference/project.dart'; -import 'package:inference/providers/preference_provider.dart'; -import 'package:inference/providers/text_to_image_inference_provider.dart'; -import 'package:intl/intl.dart'; -import 'package:provider/provider.dart'; - -class TextToImageInferencePage extends StatefulWidget { - final Project project; - const TextToImageInferencePage(this.project, {super.key}); - - @override - State createState() => _TextToImageInferencePageState(); -} - -class _TextToImageInferencePageState extends State with TickerProviderStateMixin { - - late TabController _tabController; - - @override - void initState() { - super.initState(); - _tabController = TabController(length: 2, animationDuration: Duration.zero, vsync: this); - } - - @override - void dispose() { - _tabController.dispose(); - super.dispose(); - } - - @override - Widget build(BuildContext context) { - Locale locale = Localizations.localeOf(context); - - return ChangeNotifierProxyProvider( - create: (_) { - return TextToImageInferenceProvider(widget.project, null); - }, - update: (_, preferences, textToImageInferenceProvider) { - if (textToImageInferenceProvider == null) { - return TextToImageInferenceProvider(widget.project, preferences.device); - } - if (!textToImageInferenceProvider.sameProps(widget.project, preferences.device)) { - return TextToImageInferenceProvider(widget.project, preferences.device); - } - return textToImageInferenceProvider; - }, - child: Scaffold( - appBar: const Header(true), - body: Padding( - padding: const EdgeInsets.only(left: 58, right: 58, bottom: 30), - child: Row( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - Consumer( - builder: (context, inference, child) { - final nf = NumberFormat.decimalPatternDigits( - locale: locale.languageCode, decimalDigits: 2); - - return SizedBox( - width: 250, - child: ModelInfo( - widget.project, - children: [ - PropertyItem( - name: "Task", - child: PropertyValue(inference.task), - ), - Padding( - padding: const EdgeInsets.only(left: 12, top: 12, right: 20.0), - child: Column( - children: [ - Row( - mainAxisAlignment: MainAxisAlignment.spaceBetween, - children: [ - const Text("Width"), - Text(nf.format(inference.width)) - ] - ), - Slider( - value: inference.width.toDouble(), - max: 1024.0, - min: 64, - divisions: (1024-64)~/64, - onChanged: (double value) { - inference.width = value.toInt(); - }, - - ), - ], - ), - ), - Padding( - padding: const EdgeInsets.only(left: 12, top: 12, right: 20.0), - child: Column( - children: [ - Row( - mainAxisAlignment: MainAxisAlignment.spaceBetween, - children: [ - const Text("Height"), - Text(nf.format(inference.height)) - ] - ), - Slider( - value: inference.height.toDouble(), - max: 1024.0, - min: 64, - divisions: (1024-64)~/64, - onChanged: (double value) { - inference.height = value.toInt(); - }, - - ), - ], - ), - ), - Padding( - padding: const EdgeInsets.only(left: 12, top: 12, right: 20.0), - child: Column( - children: [ - Row( - mainAxisAlignment: MainAxisAlignment.spaceBetween, - children: [ - const Text("Rounds"), - Text(nf.format(inference.rounds)) - ] - ), - Slider( - value: inference.rounds.toDouble(), - max: 80, - min: 1, - divisions: (80-1)~/1, - onChanged: (double value) { - inference.rounds = value.toInt(); - }, - - ), - ], - ), - ), - ] - ), - ); - }), - Expanded( - child: Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - TabBar( - isScrollable: true, - tabAlignment: TabAlignment.start, - controller: _tabController, - tabs: const [ - Tab(text: "Playground"), - Tab(text: "Performance metrics"), - //Tab(text: "Deploy"), - ] - ), - Expanded( - child: Padding( - padding: const EdgeInsets.only(top: 15.0), - child: TabBarView( - controller: _tabController, - children: const [ - TTIPlayground(), - TTIPerformanceMetricsPage(), - //Container(), - ] - ), - ) - ), - ], - ), - ) - ], - ), - ), - ), - ); - } -} - diff --git a/lib/pages/models/inference.dart b/lib/pages/models/inference.dart index b98ca725..24423861 100644 --- a/lib/pages/models/inference.dart +++ b/lib/pages/models/inference.dart @@ -1,5 +1,6 @@ import 'package:fluent_ui/fluent_ui.dart'; import 'package:inference/pages/computer_vision/computer_vision.dart'; +import 'package:inference/pages/text_to_image/text_to_image_page.dart'; import 'package:inference/pages/transcription/transcription.dart'; import 'package:inference/project.dart'; @@ -17,7 +18,7 @@ class InferencePage extends StatelessWidget { case ProjectType.speech: return TranscriptionPage(project); case ProjectType.textToImage: - return Container(); + return TextToImagePage(project); } } diff --git a/lib/pages/text_to_image/live_inference_pane.dart b/lib/pages/text_to_image/live_inference_pane.dart new file mode 100644 index 00000000..7dbad38a --- /dev/null +++ b/lib/pages/text_to_image/live_inference_pane.dart @@ -0,0 +1,92 @@ +import 'package:fluent_ui/fluent_ui.dart'; +import 'package:inference/pages/models/widgets/grid_container.dart'; +import 'package:inference/pages/text_to_image/providers/text_to_image_inference_provider.dart'; +import 'package:inference/pages/text_to_image/widgets/model_properties.dart'; +import 'package:inference/pages/text_to_image/widgets/toolbar_text_input.dart'; +import 'package:inference/pages/text_to_image/widgets/tti_chat_area.dart'; +import 'package:inference/pages/text_to_image/widgets/vertical_rule.dart'; +import 'package:inference/theme_fluent.dart'; +import 'package:provider/provider.dart'; +import 'package:inference/widgets/device_selector.dart'; + +class TTILiveInferencePane extends StatefulWidget { + const TTILiveInferencePane({super.key}); + + @override + State createState() => _PlaygroundState(); +} + +class _PlaygroundState extends State { + TextToImageInferenceProvider provider() => + Provider.of(context, listen: false); + + @override + Widget build(BuildContext context) { + final theme = FluentTheme.of(context); + + return Consumer( + builder: (context, inference, child) { + + return Row( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Expanded( + child: Column( + children: [ + SizedBox( + height: 64, + child: GridContainer( + child: Padding( + padding: const EdgeInsets.symmetric(horizontal: 16), + child: Row( + children: [ + const DeviceSelector(), + const Padding( + padding: EdgeInsets.symmetric(vertical: 16), + child: VerticalRule()), + ToolbarTextInput( + marginLeft: 0, + labelText: "Width", + suffix: "px", + initialValue: provider().width, + roundPowerOfTwo: true, + onChanged: (value) { provider().width = value; }), + ToolbarTextInput( + marginLeft: 20, + labelText: "Height", + suffix: "px", + initialValue: provider().height, + roundPowerOfTwo: true, + onChanged: (value) { provider().height = value; }), + ToolbarTextInput( + marginLeft: 20, + labelText: "Rounds", + suffix: "", + initialValue: provider().rounds, + onChanged: (value) { provider().rounds = value; }), + ], + ), + ), + ), + ), + Expanded( + child: GridContainer( + color: backgroundColor.of(theme), + child: Builder(builder: (context) { + return const TTIChatArea(); + }), + ), + ) + ], + ), + ), + const ModelProperties(), + ], + ); + }); + } +} + + + + diff --git a/lib/inference/textToImage/tti_performance_metrics.dart b/lib/pages/text_to_image/performance_metrics_pane.dart similarity index 63% rename from lib/inference/textToImage/tti_performance_metrics.dart rename to lib/pages/text_to_image/performance_metrics_pane.dart index 7279701c..8b3648e7 100644 --- a/lib/inference/textToImage/tti_performance_metrics.dart +++ b/lib/pages/text_to_image/performance_metrics_pane.dart @@ -1,19 +1,17 @@ -import 'package:flutter/material.dart'; -import 'package:inference/inference/text/metric_widgets.dart'; -import 'package:inference/inference/textToImage/tti_metric_widgets.dart'; -import 'package:inference/providers/text_to_image_inference_provider.dart'; -import 'package:inference/theme.dart'; +import 'package:fluent_ui/fluent_ui.dart'; +import 'package:inference/pages/text_to_image/providers/text_to_image_inference_provider.dart'; +import 'package:inference/pages/text_to_image/widgets/tti_metrics_grid.dart'; import 'package:intl/intl.dart'; import 'package:provider/provider.dart'; -class TTIPerformanceMetricsPage extends StatefulWidget { - const TTIPerformanceMetricsPage({super.key}); +class TTIPerformanceMetricsPane extends StatefulWidget { + const TTIPerformanceMetricsPane({super.key}); @override - State createState() => _TTIPerformanceMetricsPageState(); + State createState() => _TTIPerformanceMetricsPaneState(); } -class _TTIPerformanceMetricsPageState extends State { +class _TTIPerformanceMetricsPaneState extends State { @override void initState() { @@ -48,18 +46,17 @@ class _TTIPerformanceMetricsPageState extends State { final metrics = inference.metrics!; return Container( - decoration: BoxDecoration( + decoration: const BoxDecoration( shape: BoxShape.rectangle, - borderRadius: const BorderRadius.all(Radius.circular(8)), - color: intelGray, + borderRadius: BorderRadius.all(Radius.circular(8)), ), child: Padding( padding: const EdgeInsets.all(30.0), child: Column( mainAxisAlignment: MainAxisAlignment.start, - crossAxisAlignment: CrossAxisAlignment.stretch, + crossAxisAlignment: CrossAxisAlignment.center, children: [ - TTICirclePropRow(metrics: metrics), + TTIMetricsGrid(metrics: metrics), ], ), ), diff --git a/lib/providers/text_to_image_inference_provider.dart b/lib/pages/text_to_image/providers/text_to_image_inference_provider.dart similarity index 92% rename from lib/providers/text_to_image_inference_provider.dart rename to lib/pages/text_to_image/providers/text_to_image_inference_provider.dart index adafab4c..2ba4bf15 100644 --- a/lib/providers/text_to_image_inference_provider.dart +++ b/lib/pages/text_to_image/providers/text_to_image_inference_provider.dart @@ -2,8 +2,7 @@ import 'dart:async'; import 'dart:convert'; import 'dart:typed_data'; import 'dart:ui' as ui; - -import 'package:flutter/material.dart'; +import 'package:fluent_ui/fluent_ui.dart'; import 'package:flutter/services.dart' show rootBundle; import 'package:inference/interop/generated_bindings.dart'; import 'package:inference/interop/tti_inference.dart'; @@ -44,10 +43,10 @@ class TextToImageInferenceProvider extends ChangeNotifier { Uint8List? _imageBytes; - int _loadWidth = 512; - int _loadHeight = 512; + int _loadWidth = 256; + int _loadHeight = 256; - int _width = 512; + int _width = 256; int get width => _width; @@ -56,7 +55,7 @@ class TextToImageInferenceProvider extends ChangeNotifier { notifyListeners(); } - int _height = 512; + int _height = 256; int get height => _height; @@ -65,7 +64,7 @@ class TextToImageInferenceProvider extends ChangeNotifier { notifyListeners(); } - int _rounds = 20; + int _rounds = 12; int get rounds => _rounds; @@ -87,15 +86,19 @@ class TextToImageInferenceProvider extends ChangeNotifier { print("instantiating project: ${project.name}"); print(project.storagePath); print(device); - TTIInference.init(project.storagePath, device).then((instance) { - print("done loading"); - _inference = instance; - loaded.complete(); - notifyListeners(); - }); } } + Future init() async { + await TTIInference.init(project!.storagePath, device!).then((instance) { + print("done loading"); + _inference = instance; + }); + loaded.complete(); + notifyListeners(); + } + + void preloadImageBytes() { rootBundle.load('images/intel-loading.gif').then((data) { _imageBytes = data.buffer.asUint8List(); diff --git a/lib/pages/text_to_image/text_to_image_page.dart b/lib/pages/text_to_image/text_to_image_page.dart new file mode 100644 index 00000000..ac2848fe --- /dev/null +++ b/lib/pages/text_to_image/text_to_image_page.dart @@ -0,0 +1,147 @@ +import 'package:fluent_ui/fluent_ui.dart'; +import 'package:flutter_svg/svg.dart'; +import 'package:go_router/go_router.dart'; +import 'package:inference/pages/text_to_image/live_inference_pane.dart'; +import 'package:inference/pages/text_to_image/providers/text_to_image_inference_provider.dart'; +import 'package:inference/pages/text_to_image/performance_metrics_pane.dart'; +import 'package:inference/project.dart'; +import 'package:inference/providers/preference_provider.dart'; +import 'package:provider/provider.dart'; + +class TextToImagePage extends StatefulWidget { + final Project project; + const TextToImagePage(this.project, {super.key}); + + @override + State createState() => _TextToImagePageState(); +} + +class _TextToImagePageState extends State { + + + int selected = 0; + @override + Widget build(BuildContext context) { + final theme = FluentTheme.of(context); + final updatedTheme = theme.copyWith( + navigationPaneTheme: theme.navigationPaneTheme.merge(NavigationPaneThemeData( + backgroundColor: theme.scaffoldBackgroundColor, + )) + ); + final textColor = theme.typography.body?.color ?? Colors.black; + + const inferencePane = TTILiveInferencePane(); + const metricsPane = TTIPerformanceMetricsPane(); + return ChangeNotifierProxyProvider( + lazy: false, + create: (_) { + final device = Provider.of(context, listen: false).device; + return TextToImageInferenceProvider(widget.project, device)..init(); + }, + update: (_, preferences, imageInferenceProvider) { + if (imageInferenceProvider != null && imageInferenceProvider.sameProps(widget.project, preferences.device)) { + return imageInferenceProvider; + } + return TextToImageInferenceProvider(widget.project, preferences.device)..init(); + }, + + child: Stack( + children: [ + FluentTheme( + data: updatedTheme, + child: NavigationView( + pane: NavigationPane( + size: const NavigationPaneSize(topHeight: 64), + header: Row( + children: [ + Padding( + padding: const EdgeInsets.only(left: 12.0), + child: ClipRRect( + borderRadius: BorderRadius.circular(4.0), + child: Container( + width: 40, + height: 40, + decoration: BoxDecoration( + image: DecorationImage( + image: widget.project.thumbnailImage(), + fit: BoxFit.cover), + ), + ), + ), + ), + Padding( + padding: const EdgeInsets.symmetric(horizontal: 16), + child: Text(widget.project.name, + style: const TextStyle(fontSize: 20, fontWeight: FontWeight.bold), + ), + ), + ], + ), + //customPane: CustomNavigationPane(), + selected: selected, + onChanged: (i) => setState(() {selected = i;}), + displayMode: PaneDisplayMode.top, + items: [ + PaneItem( + icon: SvgPicture.asset("images/playground.svg", + colorFilter: ColorFilter.mode(textColor, BlendMode.srcIn), + width: 15, + ), + title: const Text("Live Inference"), + body: inferencePane, + ), + PaneItem( + icon: SvgPicture.asset("images/stats.svg", + colorFilter: ColorFilter.mode(textColor, BlendMode.srcIn), + width: 15, + ), + title: const Text("Performance metrics"), + body: metricsPane, + ), + ], + ) + ), + ), + SizedBox( + height: 64, + child: Padding( + padding: const EdgeInsets.symmetric(horizontal: 25), + child: Row( + mainAxisAlignment: MainAxisAlignment.end, + children: [ + //Padding( + // padding: const EdgeInsets.all(4), + // child: FilledButton( + // child: const Text("Download"), + // onPressed: () => print("close") + // ), + //), + //Padding( + // padding: const EdgeInsets.all(4), + // child: OutlinedButton( + // child: const Text("Fine-tune"), + // onPressed: () => print("close") + // ), + //), + Padding( + padding: const EdgeInsets.all(4), + child: OutlinedButton( + style: ButtonStyle( + shape:WidgetStatePropertyAll(RoundedRectangleBorder( + borderRadius: BorderRadius.circular(4.0), + side: const BorderSide(color: Color(0XFF545454)), + )), + ), + child: const Text("Close"), + onPressed: () => GoRouter.of(context).go("/models"), + ), + ), + ] + ), + ), + ) + ], + ), + ); + } +} diff --git a/lib/pages/text_to_image/widgets/device_selector.dart b/lib/pages/text_to_image/widgets/device_selector.dart new file mode 100644 index 00000000..8ca30d89 --- /dev/null +++ b/lib/pages/text_to_image/widgets/device_selector.dart @@ -0,0 +1,66 @@ +// ignore_for_file: unused_import + +import 'package:fluent_ui/fluent_ui.dart'; +import 'package:inference/providers/preference_provider.dart'; +import 'package:provider/provider.dart'; +import 'package:collection/collection.dart'; + +class DeviceSelector extends StatefulWidget { + const DeviceSelector({super.key}); + + @override + State createState() => _DeviceSelectorState(); +} + +class _DeviceSelectorState extends State { + String? selectedDevice; + + @override + void initState() { + super.initState(); + selectedDevice = Provider.of(context, listen: false).device; + } + + @override + Widget build(BuildContext context) { + return Consumer(builder: (context, preferences, child) { + return Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + const Padding( + padding: EdgeInsets.only(bottom: 16), + child: Text("Device", + style: TextStyle( + fontSize: 16, + fontWeight: FontWeight.bold, + ), + ), + ), + Column( + crossAxisAlignment: CrossAxisAlignment.stretch, + children: [ + ComboBox( + value: selectedDevice, + items: PreferenceProvider.availableDevices.map>((e) { + return ComboBoxItem( + value: e.id, + child: Text(e.name), + ); + }).toList(), + onChanged: (v) { + setState(() { + selectedDevice = v; + if (v != null) { + preferences.device = v; + } + }); + }, + ), + ], + ), + ], + ); + } + ); + } +} diff --git a/lib/pages/text_to_image/widgets/horizontal_rule.dart b/lib/pages/text_to_image/widgets/horizontal_rule.dart new file mode 100644 index 00000000..3eb59136 --- /dev/null +++ b/lib/pages/text_to_image/widgets/horizontal_rule.dart @@ -0,0 +1,25 @@ +import 'package:fluent_ui/fluent_ui.dart'; +import 'package:inference/theme_fluent.dart'; + +class HorizontalRule extends StatelessWidget { + const HorizontalRule({super.key}); + + @override + Widget build(BuildContext context) { + final theme = FluentTheme.of(context); + + return Padding( + padding: const EdgeInsets.symmetric(vertical: 20), + child: Container( + decoration: BoxDecoration( + border: Border( + bottom: BorderSide( + color: borderColor.of(theme), + width: 1, + ) + ) + ), + ), + ); + } +} diff --git a/lib/pages/text_to_image/widgets/metrics_card.dart b/lib/pages/text_to_image/widgets/metrics_card.dart new file mode 100644 index 00000000..5b46d6be --- /dev/null +++ b/lib/pages/text_to_image/widgets/metrics_card.dart @@ -0,0 +1,107 @@ +import 'package:fluent_ui/fluent_ui.dart'; + +class MetricsCard extends StatelessWidget { + final String header; + final String value; + final String unit; + + const MetricsCard( + {super.key, + required this.header, + required this.value, + required this.unit}); + + @override + Widget build(BuildContext context) { + final theme = FluentTheme.of(context); + + const lightGradient = LinearGradient( + begin: Alignment(-0.2, -1.0), + end: Alignment(1.0, 1.0), + colors: [ + Color(0xFFF6F2FC), + Color(0xFFF3F8FF), + Color(0xFFF3FDFB), + ], + stops: [0.0, 0.5, 1.0], // Matching the 0%, 50%, 100% stops + ); + + const darkGradient = LinearGradient( + begin: Alignment(-0.36, -1.0), + end: Alignment(1.0, 1.0), + colors: [ + Color(0xFF373639), + Color(0xFF2D2C2D), + Color(0xFF323033), + Color(0xFF363538), + ], + stops: [ + 0.0, + 0.2941, + 0.5881, + 0.8911, + ], + ); + + final gradient = theme.brightness.isDark ? darkGradient : lightGradient; + + return Padding( + padding: const EdgeInsets.all(20.0), + child: Container( + width: 268.0, + height: 200.0, + decoration: BoxDecoration( + shape: BoxShape.rectangle, + borderRadius: const BorderRadius.all(Radius.circular(8)), + gradient: gradient, + boxShadow: const [ + BoxShadow( + color: Color(0x1F000000), + blurRadius: 2, + offset: Offset(0, 0), // CSS's offset-x and offset-y + ), + BoxShadow( + color: Color(0x24000000), + blurRadius: 4, + offset: Offset(0, 2), // CSS's offset-x and offset-y + ), + ], + ), + child: Column( + mainAxisAlignment: MainAxisAlignment.center, + children: [ + Text( + header, + style: const TextStyle( + fontSize: 14, + ), + textAlign: TextAlign.center, + ), + Container( + height: 10, + ), + Row( + mainAxisAlignment: MainAxisAlignment.center, + crossAxisAlignment: CrossAxisAlignment.end, + children: [ + Text( + value, + style: const TextStyle( + fontSize: 30, + ), + ), + Padding( + padding: const EdgeInsets.only(left: 8.0), + child: Text( + unit, + style: const TextStyle( + fontSize: 12, + ), + ), + ), + ]), + ], + )), + ); + } +} diff --git a/lib/pages/text_to_image/widgets/model_properties.dart b/lib/pages/text_to_image/widgets/model_properties.dart new file mode 100644 index 00000000..77b1dfde --- /dev/null +++ b/lib/pages/text_to_image/widgets/model_properties.dart @@ -0,0 +1,105 @@ +import 'package:fluent_ui/fluent_ui.dart'; +import 'package:inference/pages/text_to_image/providers/text_to_image_inference_provider.dart'; +import 'package:inference/theme_fluent.dart'; +import 'package:inference/utils.dart'; +import 'package:inference/pages/models/widgets/grid_container.dart'; +import 'package:intl/intl.dart'; +import 'package:provider/provider.dart'; + +class ModelProperties extends StatelessWidget { + const ModelProperties({super.key}); + + @override + Widget build(BuildContext context) { + return Consumer(builder: (context, inference, child) { + Locale locale = Localizations.localeOf(context); + final formatter = NumberFormat.percentPattern(locale.languageCode); + + return SizedBox( + width: 280, + child: GridContainer( + padding: const EdgeInsets.symmetric(vertical: 18, horizontal: 24), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + const Text("Model parameters", style: TextStyle( + fontSize: 20, + )), + Container( + padding: const EdgeInsets.only(top: 16), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + ModelProperty( + title: "Model name", + value: inference.project!.name, + ), + ModelProperty( + title: "Task", + value: inference.project!.taskName(), + ), + ModelProperty( + title: "Architecture", + value: inference.project!.architecture, + ), + ModelProperty( + title: "Size", + value: inference.project!.size?.readableFileSize() ?? "", + ), + Builder( + builder: (context) { + if (inference.project!.tasks.first.performance == null) { + return Container(); + } + return ModelProperty( + title: "Accuracy", + value: formatter.format(inference.project!.tasks.first.performance!.score) + ); + } + ), + ], + ), + ) + ], + ) + ), + ); + } + ); + } +} + +class ModelProperty extends StatelessWidget { + final String title; + final String value; + + const ModelProperty({ + required this.title, + required this.value, + super.key}); + + @override + Widget build(BuildContext context) { + final theme = FluentTheme.of(context); + + return Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Padding( + padding: const EdgeInsets.only(bottom: 4), + child: Text(title, style: const TextStyle( + fontWeight: FontWeight.w600, + fontSize: 14, + )), + ), + Padding( + padding: const EdgeInsets.only(bottom: 16), + child: Text(value, style: TextStyle( + fontSize: 16, + color: subtleTextColor.of(theme), + )), + ), + ] + ); + } +} diff --git a/lib/pages/text_to_image/widgets/toolbar_text_input.dart b/lib/pages/text_to_image/widgets/toolbar_text_input.dart new file mode 100644 index 00000000..39866b86 --- /dev/null +++ b/lib/pages/text_to_image/widgets/toolbar_text_input.dart @@ -0,0 +1,115 @@ +import 'dart:math'; + +import 'package:fluent_ui/fluent_ui.dart'; + +class ToolbarTextInput extends StatefulWidget { + final String labelText; + final String suffix; + final int marginLeft; + final int initialValue; + final bool? roundPowerOfTwo; + final void Function(int)? onChanged; + + const ToolbarTextInput({ + super.key, + required this.labelText, + required this.suffix, + required this.marginLeft, + required this.initialValue, + this.roundPowerOfTwo, + this.onChanged, + }); + + @override + State createState() => _ToolbarTextInputState(); +} + +class _ToolbarTextInputState extends State { + final TextEditingController _controller = TextEditingController(); + final FocusNode _focusNode = FocusNode(); + + @override + void initState() { + super.initState(); + _controller.text = widget.initialValue.toString(); // Set the initial text + if (widget.roundPowerOfTwo ?? false) { + _focusNode.addListener(_onFocusChange); // Listen for focus changes + } + } + + void _onFocusChange() { + if (!_focusNode.hasFocus) { + // When the TextBox loses focus, round and update + final inputValue = int.tryParse(_controller.text.replaceAll(RegExp(r'[^0-9]'), '')) ?? 0; + final rounded = _nearestPowerOfTwo(inputValue); + + _controller.text = rounded.toString(); + widget.onChanged!(rounded); + + } + } + + /// Calculate the nearest power of 2 for a given number + int _nearestPowerOfTwo(int value) { + if (value <= 0) return 1; // Smallest power of 2 is 1 + int lowerPower = pow(2, (log(value) / log(2)).floor()).toInt(); + int higherPower = pow(2, (log(value) / log(2)).ceil()).toInt(); + return (value - lowerPower < higherPower - value) ? lowerPower : higherPower; + } + + + void _onTextChanged(String value) { + // Keep only digits in the input + final newValue = value.replaceAll(RegExp(r'[^0-9]'), ''); + if (value != newValue) { + // Update the controller text and cursor position + _controller.text = newValue; + _controller.selection = TextSelection.collapsed(offset: newValue.length); + } + + if (widget.onChanged != null) { + if (newValue.isNotEmpty) { + // Parse the integer and call the callback + widget.onChanged!(int.parse(newValue)); + } else { + // Optionally handle empty input + widget.onChanged!(0); // You can choose to pass null or handle differently + } + } + } + + + @override + Widget build(BuildContext context) { + return Row( + crossAxisAlignment: CrossAxisAlignment.center, + children: [ + Padding( + padding: EdgeInsets.only(left: 10 + widget.marginLeft.toDouble(), right: 10), + child: Text( + widget.labelText, + style: const TextStyle( + fontSize: 14, + fontWeight: FontWeight.normal, + ), + ), + ), + SizedBox( + width: 85, + height: 30, + child: TextBox( + controller: _controller, + focusNode: _focusNode, + maxLines: 1, + keyboardType: TextInputType.number, // Ensure numeric keyboard + suffix: Padding( + padding: const EdgeInsets.only(right: 8.0), + child: Text(widget.suffix), + ), + onChanged: _onTextChanged, // Custom handler for integer validation + ), + ), + ], + ); + } +} diff --git a/lib/pages/text_to_image/widgets/tti_chat_area.dart b/lib/pages/text_to_image/widgets/tti_chat_area.dart new file mode 100644 index 00000000..c65b50e0 --- /dev/null +++ b/lib/pages/text_to_image/widgets/tti_chat_area.dart @@ -0,0 +1,539 @@ +import 'package:fluent_ui/fluent_ui.dart'; +import 'package:flutter_svg/svg.dart'; +import 'package:inference/interop/openvino_bindings.dart'; +import 'package:inference/pages/text_to_image/providers/text_to_image_inference_provider.dart'; +import 'package:inference/pages/text_to_image/widgets/tti_metrics_grid.dart'; +import 'package:inference/theme_fluent.dart'; +import 'package:provider/provider.dart'; +import 'package:super_clipboard/super_clipboard.dart'; + +class TTIChatArea extends StatefulWidget { + const TTIChatArea({super.key}); + + @override + State createState() => _PlaygroundState(); +} + +class _PlaygroundState extends State { + final _controller = TextEditingController(); + final _scrollController = ScrollController(); + bool attachedToBottom = true; + + void jumpToBottom({offset = 0}) { + if (_scrollController.hasClients) { + _scrollController + .jumpTo(_scrollController.position.maxScrollExtent + offset); + } + } + + void message(String message) async { + if (message.isEmpty) { + return; + } + final tti = provider(); + if (!tti.initialized) { + return; + } + + if (tti.response != null) { + return; + } + _controller.text = ""; + jumpToBottom(offset: 110); //move to bottom including both + tti.message(message); + } + + TextToImageInferenceProvider provider() => + Provider.of(context, listen: false); + + @override + void initState() { + super.initState(); + _scrollController.addListener(() { + setState(() { + attachedToBottom = _scrollController.position.pixels + 0.001 >= + _scrollController.position.maxScrollExtent; + }); + }); + } + + @override + void dispose() { + super.dispose(); + _controller.dispose(); + _scrollController.dispose(); + } + + @override + Widget build(BuildContext context) { + return Consumer( + builder: (context, inference, child) { + WidgetsBinding.instance.addPostFrameCallback((_) { + if (attachedToBottom) { + jumpToBottom(); + } + }); + + final theme = FluentTheme.of(context); + final textColor = theme.typography.body?.color ?? Colors.black; + + return Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Builder(builder: (context) { + if (!inference.initialized) { + return Expanded( + child: Center( + child: Column( + mainAxisAlignment: MainAxisAlignment.center, + children: [ + Image.asset('images/intel-loading.gif', width: 100), + const Text("Loading model...") + ], + )), + ); + } + return Expanded( + child: Container( + decoration: const BoxDecoration( + shape: BoxShape.rectangle, + borderRadius: BorderRadius.all(Radius.circular(8)), + ), + child: Column( + children: [ + Expanded( + child: Builder(builder: (context) { + if (inference.messages.isEmpty) { + return Center( + child: Text( + "Type a message to ${inference.project?.name ?? "assistant"}")); + } + return Stack( + alignment: Alignment.topCenter, + children: [ + SingleChildScrollView( + controller: _scrollController, + child: Padding( + padding: const EdgeInsets.all(20), + child: Column( + + // mainAxisAlignment: MainAxisAlignment.start, + crossAxisAlignment: + CrossAxisAlignment.stretch, + children: inference.messages.map((message) { + switch (message.speaker) { + case Speaker.user: + return UserInputMessage(message); + case Speaker.assistant: + return GeneratedImageMessage( + message, + inference.project! + .thumbnailImage(), + inference.project!.name); + } + }).toList()), + ), + ), + Positioned( + bottom: 10, + child: Builder(builder: (context) { + if (attachedToBottom) { + return Container(); + } + return Center( + child: Padding( + padding: const EdgeInsets.only(top: 2.0), + child: SizedBox( + width: 200, + height: 40, + // Adjusted height to match Fluent UI's button dimensions + child: FilledButton( + child: const Text("Jump to bottom"), + onPressed: () { + jumpToBottom(); + setState(() { + attachedToBottom = true; + }); + }, + ), + ), + ), + ); + }), + ), + ], + ); + }), + ), + + // SizedBox( + // height: 30, + // child: Builder( + // builder: (context) { + // if (inference.interimResponse == null){ + // return Container(); + // } + // return Center( + // child: OutlinedButton.icon( + // onPressed: () => inference.forceStop(), + // icon: const Icon(Icons.stop), + // label: const Text("Stop responding") + // ), + // ); + // } + // ), + // ), + Padding( + padding: const EdgeInsets.only( + left: 45, right: 45, top: 10, bottom: 25), + child: SizedBox( + height: 40, + child: Row( + crossAxisAlignment: CrossAxisAlignment.center, + children: [ + Padding( + padding: const EdgeInsets.only(right: 8), + child: IconButton( + icon: SvgPicture.asset( + "images/clear.svg", + width: 20, + colorFilter: ColorFilter.mode(textColor, BlendMode.srcIn), + ), + onPressed: () => inference.reset(), + ), + ), + Expanded( + child: TextBox( + maxLines: null, + keyboardType: TextInputType.text, + placeholder: "Ask me anything...", + controller: _controller, + onSubmitted: message, + style: const TextStyle( + fontSize: 14, + ), + suffix: IconButton( + icon: Icon( + FluentIcons.send, + color: + (inference.interimResponse == null + ? textColor + : textColor.withOpacity(0.2)), + ), + onPressed: () => + inference.interimResponse != null ? null : + message(_controller.text), + ), + ), + ) + ], + ), + ), + ), + ], + ), + ), + ); + }), + ], + ); + }); + } +} + +class UserInputMessage extends StatelessWidget { + final Message message; + + const UserInputMessage(this.message, {super.key}); + + @override + Widget build(BuildContext context) { + return Padding( + padding: const EdgeInsets.only(bottom: 20), + child: Column( + crossAxisAlignment: CrossAxisAlignment.end, + children: [ + Container( + constraints: const BoxConstraints(maxWidth: 500), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Padding( + padding: const EdgeInsets.only(right: 30.0), + child: MessageWidget( + message: message.message, + innerPadding: 8, + isSender: true), + ), + ])) + ], + ), + ); + } +} + +class GeneratedImageMessage extends StatelessWidget { + final Message message; + final ImageProvider icon; + final String name; + + const GeneratedImageMessage(this.message, this.icon, this.name, {super.key}); + + @override + Widget build(BuildContext context) { + final theme = FluentTheme.of(context); + + final smartImage = SmartImage(message: message); + + return Padding( + padding: const EdgeInsets.only(bottom: 20), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + mainAxisSize: MainAxisSize.min, // Wrap content + children: [ + Table( + columnWidths: const { + 0: FixedColumnWidth(40), // Fixed width for the first column + 1: IntrinsicColumnWidth(), // Flexible width for the second column + 2: FixedColumnWidth(40), // Fixed width for the first column + }, + children: [ + TableRow(children: [ + Container(), + Padding( + padding: + const EdgeInsets.only(left: 10, top: 8.0, bottom: 8.0), + child: Text( + name, + style: TextStyle( + color: subtleTextColor.of(theme), + fontSize: 12, + ), + textAlign: TextAlign.start, + ), + ), + const SizedBox.shrink(), + ]), + TableRow(children: [ + RoundedPicture( + name: name, + icon: icon, + ), + if (message.imageContent?.imageData != null) + Padding( + padding: const EdgeInsets.only(left: 10, bottom: 20), + child: Align( + alignment: Alignment.centerLeft, // Align left + child: MessageWidget( + image: smartImage, + metrics: message.metrics, + innerPadding: 24.0, + isSender: false, + ))), + ImageOptionsWidget( + image: smartImage, + metrics: message.metrics, + ), + ]) + ], + ), + ], + ), + ); + } +} + +void showMetricsDialog(BuildContext context, TTIMetrics metrics) async { + await showDialog( + context: context, + barrierDismissible: true, + builder: (context) => ContentDialog( + constraints: const BoxConstraints(maxWidth: double.infinity), + content: TTIMetricsGrid(metrics: metrics), + ), + ); +} + +class RoundedPicture extends StatelessWidget { + final String name; + final ImageProvider icon; // Icon widget provided + + const RoundedPicture({super.key, required this.name, required this.icon}); + + @override + Widget build(BuildContext context) { + return ClipOval( + child: Container( + width: 40, + height: 40, + decoration: BoxDecoration( + image: DecorationImage( + image: icon, // Adjust this to fit your `name` field + fit: BoxFit.cover, + ), + ), + ), + ); + } +} + +class MessageWidget extends StatelessWidget { + final String? message; + final SmartImage? image; + final TTIMetrics? metrics; // If not set, no copy-paste options. + final double innerPadding; + final bool isSender; + + const MessageWidget( + {super.key, + this.message, + this.image, + this.metrics, + required this.innerPadding, + required this.isSender}); + + @override + Widget build(BuildContext context) { + final theme = FluentTheme.of(context); + final textColor = theme.typography.body?.color ?? Colors.black; + + void showContentDialog(BuildContext context) async { + if (image?.hasClipboard() ?? false) { + showDialog( + context: context, + barrierDismissible: true, + builder: (context) => ContentDialog( + constraints: const BoxConstraints(maxWidth: double.infinity), + content: image, + ), + ); + } + } + + return Container( + decoration: BoxDecoration( + color: + isSender ? userMessageColor.of(theme) : modelMessageColor.of(theme), + borderRadius: const BorderRadius.only( + topLeft: Radius.circular(4.0), + topRight: Radius.circular(4.0), + bottomLeft: Radius.circular(4.0), + bottomRight: Radius.circular(4.0), + ), + ), + padding: EdgeInsets.all(innerPadding), + child: Column(children: [ + image != null + ? Column(children: [ + // Changes cursor to hand on hover when there's a dialog that can be opened + MouseRegion( + cursor: image?.hasClipboard() ?? false + ? SystemMouseCursors.click + : SystemMouseCursors.basic, + child: GestureDetector( + onTap: () => showContentDialog(context), + // Opens dialog on click + child: ConstrainedBox( + constraints: const BoxConstraints( + maxHeight: 256.0, + maxWidth: 256.0, + ), + child: image!))), + ]) + : const SizedBox.shrink(), + message != null + ? SelectableText( + message!, + style: TextStyle( + color: textColor, + fontSize: 14, + fontWeight: FontWeight.w400, + ), + ) + : const SizedBox.shrink(), + ]), + ); + } +} + +class ImageOptionsWidget extends StatelessWidget { + final SmartImage? image; + final TTIMetrics? metrics; + + const ImageOptionsWidget({super.key, this.image, this.metrics}); + + @override + Widget build(BuildContext context) { + bool hasClipboard = image?.hasClipboard() ?? false; + bool hasMetrics = metrics != null; + final textColor = + FluentTheme.of(context).typography.body?.color ?? Colors.black; + + return Column( + mainAxisSize: MainAxisSize.min, // Wrap content + children: [ + Opacity( + opacity: hasClipboard ? 1.0 : 0.25, + child: IconButton( + icon: SvgPicture.asset( + "images/copy.svg", + colorFilter: ColorFilter.mode(textColor, BlendMode.srcIn), + width: 14, + height: 14, + ), + onPressed: hasClipboard + ? () { + image?.copyToClipboard(); + } + : null, + )), + Opacity( + opacity: hasMetrics ? 1.0 : 0.25, + child: IconButton( + icon: SvgPicture.asset( + "images/stats.svg", + colorFilter: ColorFilter.mode(textColor, BlendMode.srcIn), + width: 14, + height: 14, + ), + // tooltip: "Show stats", + onPressed: () { + metrics != null ? showMetricsDialog(context, metrics!) : null; + }, + ), + ), + ], + ); + } +} + +class SmartImage extends StatelessWidget { + final Message message; + + const SmartImage({super.key, required this.message}); + + void copyToClipboard() { + final clipboard = SystemClipboard.instance; + if (clipboard == null || message.imageContent == null) { + return; // Clipboard API is not supported on this platform. + } + final item = DataWriterItem(); + + item.add(Formats.jpeg(message.imageContent!.imageData)); + clipboard.write([item]); + } + + bool hasClipboard() { + return message.allowedCopy; + } + + @override + Widget build(BuildContext context) { + return Image.memory( + message.imageContent!.imageData, + width: message.imageContent!.width.toDouble(), + height: message.imageContent!.height.toDouble(), + fit: message.imageContent!.boxFit, + ); + } +} diff --git a/lib/inference/textToImage/tti_metric_widgets.dart b/lib/pages/text_to_image/widgets/tti_metrics_grid.dart similarity index 68% rename from lib/inference/textToImage/tti_metric_widgets.dart rename to lib/pages/text_to_image/widgets/tti_metrics_grid.dart index c7cc4e6f..46dbdf58 100644 --- a/lib/inference/textToImage/tti_metric_widgets.dart +++ b/lib/pages/text_to_image/widgets/tti_metrics_grid.dart @@ -1,12 +1,13 @@ -import 'package:flutter/material.dart'; -import 'package:inference/inference/text/metric_widgets.dart'; +import 'package:fluent_ui/fluent_ui.dart'; import 'package:inference/interop/openvino_bindings.dart'; +import 'package:inference/pages/text_to_image/widgets/metrics_card.dart'; import 'package:intl/intl.dart'; -class TTICirclePropRow extends StatelessWidget { + +class TTIMetricsGrid extends StatelessWidget { final TTIMetrics metrics; - const TTICirclePropRow({super.key, required this.metrics}); + const TTIMetricsGrid({super.key, required this.metrics}); @override Widget build(BuildContext context) { @@ -16,13 +17,14 @@ class TTICirclePropRow extends StatelessWidget { return Row( mainAxisAlignment: MainAxisAlignment.spaceBetween, + mainAxisSize: MainAxisSize.min, // Wrap content children: [ - CircleProp( + MetricsCard( header: "Time to load model", value: nf.format(metrics.load_time), unit: "ms", ), - CircleProp( + MetricsCard( header: "Time to generate image", value: nf.format(metrics.generate_time), unit: "ms", diff --git a/lib/pages/text_to_image/widgets/vertical_rule.dart b/lib/pages/text_to_image/widgets/vertical_rule.dart new file mode 100644 index 00000000..e764da17 --- /dev/null +++ b/lib/pages/text_to_image/widgets/vertical_rule.dart @@ -0,0 +1,25 @@ +import 'package:fluent_ui/fluent_ui.dart'; +import 'package:inference/theme_fluent.dart'; + +class VerticalRule extends StatelessWidget { + const VerticalRule({super.key}); + + @override + Widget build(BuildContext context) { + final theme = FluentTheme.of(context); + + return Padding( + padding: const EdgeInsets.symmetric(horizontal: 20), + child: Container( + decoration: BoxDecoration( + border: Border( + left: BorderSide( + color: borderColor.of(theme), + width: 1, + ) + ) + ), + ), + ); + } +} diff --git a/lib/theme_fluent.dart b/lib/theme_fluent.dart index edf90cec..43c7033a 100644 --- a/lib/theme_fluent.dart +++ b/lib/theme_fluent.dart @@ -105,6 +105,8 @@ class DarkLightColor { const borderColor = DarkLightColor(Color(0xFFF0F0F0), Color(0xFF3B3B3B)); const backgroundColor = DarkLightColor(Color(0xFFF9F9F9), Color(0xFF282828)); const subtleTextColor = DarkLightColor(Color(0xFF616161), Color(0xFF9F9F9F)); +const userMessageColor = DarkLightColor(Color(0xFFe0d6fd), Color(0xFF463D66)); +const modelMessageColor = DarkLightColor(Color(0xFFF5F5F5), Color(0xFF343434)); final AccentColor electricCosmos = AccentColor.swatch(const { 'normal': Color(0xFF7000FF), diff --git a/lib/widgets/import_model_button.dart b/lib/widgets/import_model_button.dart index ef6fa0f6..68ecc759 100644 --- a/lib/widgets/import_model_button.dart +++ b/lib/widgets/import_model_button.dart @@ -1,17 +1,59 @@ +import 'package:file_picker/file_picker.dart'; import 'package:fluent_ui/fluent_ui.dart'; import 'package:go_router/go_router.dart'; import 'package:inference/widgets/controls/filled_dropdown_button.dart'; +import 'package:provider/provider.dart'; + +import '../importers/importer.dart'; +import '../providers/project_provider.dart'; class ImportModelButton extends StatelessWidget { const ImportModelButton({super.key}); + + void addProject(BuildContext context) async { + + final result = await FilePicker.platform.pickFiles(allowMultiple: true); + + if (result != null) { + for (final file in result.files) { + final importer = selectMatchingImporter(file.path!); + if (importer == null) { + print("unable to process file"); + return; + } + if (context.mounted) { + if (!await importer.askUser(context)) { + print("cancelling due to user input"); + return; + } + } + importer.generateProject().then((project) async { + await importer.setupFiles(); + project.loaded.future.then((_) { + if (context.mounted) { + Provider.of(context, listen: false) + .completeLoading(project); + } + }); + if (context.mounted) { + final projectsProvider = Provider.of(context, listen: false); + projectsProvider.addProject(project); + } + }); + } + } else { + // User canceled the picker + } + } + @override Widget build(BuildContext context) { return FilledDropDownButton( title: const Text('Import model'), items: [ MenuFlyoutItem(text: const Text('Hugging Face'), onPressed: () { GoRouter.of(context).push('/models/import'); }), - MenuFlyoutItem(text: const Text('Local disk'), onPressed: () {}), + MenuFlyoutItem(text: const Text('Local disk'), onPressed: () { addProject(context); }), ] ); } From d4f149f298ceb61f88a897a409636e4e8e182f50 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Fri, 29 Nov 2024 12:09:50 +0100 Subject: [PATCH 28/33] Fixed minor issues --- lib/pages/text_generation/playground.dart | 124 +++++++++++------- .../widgets/assistant_message.dart | 40 ++++-- .../text_generation/widgets/user_message.dart | 42 +++--- lib/providers/text_inference_provider.dart | 18 ++- pubspec.lock | 2 +- pubspec.yaml | 1 + 6 files changed, 148 insertions(+), 79 deletions(-) diff --git a/lib/pages/text_generation/playground.dart b/lib/pages/text_generation/playground.dart index 8100c290..be882913 100644 --- a/lib/pages/text_generation/playground.dart +++ b/lib/pages/text_generation/playground.dart @@ -1,3 +1,4 @@ +import 'dart:io'; import 'package:fluent_ui/fluent_ui.dart'; import 'package:flutter/services.dart'; @@ -41,8 +42,18 @@ class _PlaygroundState extends State { if (!provider.initialized || provider.response != null) return; textController.text = ''; jumpToBottom(offset: 110); //move to bottom including both - // TODO: add error handling - provider.message(message).catchError((e) { print(e); }); + provider.message(message).catchError((e) async { + // ignore: use_build_context_synchronously + await displayInfoBar(context, builder: (context, close) => InfoBar( + title: const Text("An error occurred processing the message"), + content: Text(e.toString()), + severity: InfoBarSeverity.error, + action: IconButton( + icon: const Icon(FluentIcons.clear), + onPressed: close, + ), + )); + }); } @override @@ -150,7 +161,10 @@ class _PlaygroundState extends State { controller: scrollController, child: Padding(padding: const EdgeInsets.symmetric(horizontal: 64, vertical: 20), child: Column( children: provider.messages.map((message) { switch (message.speaker) { - case Speaker.user: return UserMessage(message); + case Speaker.user: return Padding( + padding: const EdgeInsets.only(left: 42), + child: UserMessage(message), + ); case Speaker.system: return Text('System: ${message.message}'); case Speaker.assistant: return AssistantMessage(message); }}).toList(), @@ -183,52 +197,74 @@ class _PlaygroundState extends State { ), Padding( padding: const EdgeInsets.symmetric(horizontal: 64, vertical: 24), - child: Row( - crossAxisAlignment: CrossAxisAlignment.end, - mainAxisAlignment: MainAxisAlignment.center, + child: Column( children: [ - Tooltip( - message: "Create new thread", - child: Button(child: const Icon(FluentIcons.rocket, size: 18,), onPressed: () {}), - ), - Expanded( - child: Padding( - padding: const EdgeInsets.symmetric(horizontal: 8), - child: Shortcuts( - shortcuts: { - LogicalKeySet(LogicalKeyboardKey.meta, LogicalKeyboardKey.enter): SubmitMessageIntent(), - LogicalKeySet(LogicalKeyboardKey.control, LogicalKeyboardKey.enter): SubmitMessageIntent(), - }, - child: Actions( - actions: >{ - SubmitMessageIntent: CallbackAction( - onInvoke: (SubmitMessageIntent intent) => message(textController.text), + Row( + crossAxisAlignment: CrossAxisAlignment.end, + mainAxisAlignment: MainAxisAlignment.center, + children: [ + Padding( + padding: const EdgeInsets.only(bottom: 20), + child: Tooltip( + message: "Create new thread", + child: Button(child: const Icon(FluentIcons.rocket, size: 18,), onPressed: () { provider.reset(); }), + ), + ), + Expanded( + child: Padding( + padding: const EdgeInsets.symmetric(horizontal: 8), + child: Shortcuts( + shortcuts: { + LogicalKeySet(LogicalKeyboardKey.meta, LogicalKeyboardKey.enter): SubmitMessageIntent(), + LogicalKeySet(LogicalKeyboardKey.control, LogicalKeyboardKey.enter): SubmitMessageIntent(), + }, + child: Actions( + actions: >{ + SubmitMessageIntent: CallbackAction( + onInvoke: (SubmitMessageIntent intent) => message(textController.text), + ), + }, + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + TextBox( + placeholder: "Type a message...", + keyboardType: TextInputType.multiline, + controller: textController, + maxLines: null, + expands: true, + onSubmitted: message, + autofocus: true, + ), + Padding( + padding: const EdgeInsets.only(top: 6, left: 10), + child: Text( + 'Press ${Platform.isMacOS ? '⌘' : 'Ctrl'} + Enter to submit, Enter for newline', + style: TextStyle(fontSize: 11, color: subtleTextColor.of(theme)), + ), + ), + ], + ), ), - }, - child: TextBox( - placeholder: "Type a message...", - keyboardType: TextInputType.text, - controller: textController, - maxLines: null, - expands: true, - onSubmitted: message, - autofocus: true, ), - ), ), - ), + ), + Padding( + padding: const EdgeInsets.only(bottom: 20), + child: Builder(builder: (context) => provider.interimResponse != null + ? Tooltip( + message: "Stop", + child: Button(child: const Icon(FluentIcons.stop, size: 18,), onPressed: () { provider.forceStop(); }), + ) + : Tooltip( + message: "Send message", + child: Button(child: const Icon(FluentIcons.send, size: 18,), onPressed: () { message(textController.text); }), + ) + ), + ) + ] ), - Builder(builder: (context) => provider.interimResponse != null - ? Tooltip( - message: "Stop", - child: Button(child: const Icon(FluentIcons.stop, size: 18,), onPressed: () { provider.forceStop(); }), - ) - : Tooltip( - message: "Send message", - child: Button(child: const Icon(FluentIcons.send, size: 18,), onPressed: () { message(textController.text); }), - ) - ) - ] + ], ), ) ], diff --git a/lib/pages/text_generation/widgets/assistant_message.dart b/lib/pages/text_generation/widgets/assistant_message.dart index dd13cf84..1ff268f1 100644 --- a/lib/pages/text_generation/widgets/assistant_message.dart +++ b/lib/pages/text_generation/widgets/assistant_message.dart @@ -1,6 +1,7 @@ import 'package:fluent_ui/fluent_ui.dart'; import 'package:flutter/services.dart'; import 'package:flutter_markdown/flutter_markdown.dart'; +import 'package:markdown/markdown.dart' as md; import 'package:inference/providers/text_inference_provider.dart'; import 'package:inference/theme_fluent.dart'; import 'package:intl/intl.dart'; @@ -31,7 +32,7 @@ class _AssistantMessageState extends State { Align( alignment: Alignment.centerLeft, child: Padding( - padding: const EdgeInsets.only(bottom: 20), + padding: const EdgeInsets.only(bottom: 8), child: Row( crossAxisAlignment: CrossAxisAlignment.end, children: [ @@ -56,11 +57,21 @@ class _AssistantMessageState extends State { children: [ Padding( padding: const EdgeInsets.only(bottom: 4), - child: Text( - inferenceProvider.project!.name, - style: TextStyle( - color: subtleTextColor.of(theme), - ), + child: Row( + children: [ + Text( + inferenceProvider.project!.name, + style: TextStyle( + color: subtleTextColor.of(theme), + ), + ), + if (widget.message.time != null) Text( + DateFormat(' | yyyy-MM-dd HH:mm:ss').format(widget.message.time!), + style: TextStyle( + color: subtleTextColor.of(theme), + ), + ), + ], ), ), MouseRegion( @@ -79,8 +90,11 @@ class _AssistantMessageState extends State { data: widget.message.message, selectable: true, shrinkWrap: true, - padding: const EdgeInsets.all(8), - physics: const NeverScrollableScrollPhysics(), + padding: const EdgeInsets.all(12), + extensionSet: md.ExtensionSet( + md.ExtensionSet.gitHubFlavored.blockSyntaxes, + [md.EmojiSyntax(), ...md.ExtensionSet.gitHubFlavored.inlineSyntaxes], + ), ), ), if (_hovering) @@ -134,10 +148,10 @@ class _AssistantMessageState extends State { InfoBar( title: const Text('Copied to clipboard'), severity: InfoBarSeverity.info, - action: IconButton( - icon: const Icon(FluentIcons.clear), - onPressed: close, - ), + action: IconButton( + icon: const Icon(FluentIcons.clear), + onPressed: close, + ), ), ); Clipboard.setData(ClipboardData(text: widget.message.message)); @@ -153,7 +167,7 @@ class _AssistantMessageState extends State { ), ], ), - ) + ), ), ); } diff --git a/lib/pages/text_generation/widgets/user_message.dart b/lib/pages/text_generation/widgets/user_message.dart index e440d0fc..5e73496b 100644 --- a/lib/pages/text_generation/widgets/user_message.dart +++ b/lib/pages/text_generation/widgets/user_message.dart @@ -1,5 +1,6 @@ import 'package:fluent_ui/fluent_ui.dart'; -import 'package:flutter/material.dart'; +import 'package:flutter_markdown/flutter_markdown.dart'; +import 'package:markdown/markdown.dart' as md; import 'package:inference/providers/text_inference_provider.dart'; import 'package:inference/theme_fluent.dart'; @@ -12,21 +13,30 @@ class UserMessage extends StatelessWidget { final theme = FluentTheme.of(context); return Align( alignment: Alignment.centerRight, - child: Padding(padding: const EdgeInsets.only(bottom: 20), child: Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - Container( - decoration: BoxDecoration( - color: cosmosBackground.of(theme), - borderRadius: BorderRadius.circular(4), - ), - child: Padding( - padding: const EdgeInsets.all(8.0), - child: SelectableText(message.message,), - ), - ) - ], - ),), + child: ConstrainedBox( + constraints: const BoxConstraints(maxWidth: 1000), + child: Padding(padding: const EdgeInsets.only(bottom: 20), child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Container( + decoration: BoxDecoration( + color: cosmosBackground.of(theme), + borderRadius: BorderRadius.circular(4), + ), + child: Markdown( + data: message.message, + selectable: true, + shrinkWrap: true, + padding: const EdgeInsets.all(12), + extensionSet: md.ExtensionSet( + md.ExtensionSet.gitHubFlavored.blockSyntaxes, + [md.EmojiSyntax(), ...md.ExtensionSet.gitHubFlavored.inlineSyntaxes], + ), + ), + ) + ], + ),), + ), ); } } \ No newline at end of file diff --git a/lib/providers/text_inference_provider.dart b/lib/providers/text_inference_provider.dart index bfa278d7..443464f3 100644 --- a/lib/providers/text_inference_provider.dart +++ b/lib/providers/text_inference_provider.dart @@ -33,7 +33,8 @@ class Message { final Speaker speaker; final String message; final Metrics? metrics; - const Message(this.speaker, this.message, this.metrics); + final DateTime? time; + const Message(this.speaker, this.message, this.metrics, this.time); } class TextInferenceProvider extends ChangeNotifier { @@ -137,7 +138,7 @@ class TextInferenceProvider extends ChangeNotifier { if (_response == null) { return null; } - return Message(Speaker.assistant, response!, null); + return Message(Speaker.assistant, response!, null, null); } List get messages { @@ -149,12 +150,12 @@ class TextInferenceProvider extends ChangeNotifier { Future message(String message) async { _response = "..."; - _messages.add(Message(Speaker.user, message, null)); + _messages.add(Message(Speaker.user, message, null, DateTime.now())); notifyListeners(); final response = await _inference!.prompt(message, temperature, topP); if (_messages.isNotEmpty) { - _messages.add(Message(Speaker.assistant, response.content, response.metrics)); + _messages.add(Message(Speaker.assistant, response.content, response.metrics, DateTime.now())); } _response = null; n = 0; @@ -173,7 +174,14 @@ class TextInferenceProvider extends ChangeNotifier { } void forceStop() { - _inference?.forceStop(); //TODO + _inference?.forceStop(); + if (_response != '...') { + _messages.add(Message(Speaker.assistant, _response!, null, DateTime.now())); + } + _response = null; + if (hasListeners) { + notifyListeners(); + } } void reset() { diff --git a/pubspec.lock b/pubspec.lock index 498a26de..0bb8d121 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -554,7 +554,7 @@ packages: source: hosted version: "0.1.2-main.4" markdown: - dependency: transitive + dependency: "direct main" description: name: markdown sha256: ef2a1298144e3f985cc736b22e0ccdaf188b5b3970648f2d9dc13efd1d9df051 diff --git a/pubspec.yaml b/pubspec.yaml index 6c06fb78..6ecc19bd 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -57,6 +57,7 @@ dependencies: system_theme: ^3.1.2 flutter_acrylic: ^1.1.4 flutter_markdown: ^0.7.4+3 + markdown: ^7.2.2 dev_dependencies: flutter_test: From 93a8a15bdcf047171e0f266a7d75c49bef685173 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Fri, 29 Nov 2024 13:09:47 +0100 Subject: [PATCH 29/33] Added proper selection area --- lib/pages/text_generation/playground.dart | 49 +++--- .../widgets/assistant_message.dart | 142 +++++++++--------- .../text_generation/widgets/user_message.dart | 33 ++-- 3 files changed, 115 insertions(+), 109 deletions(-) diff --git a/lib/pages/text_generation/playground.dart b/lib/pages/text_generation/playground.dart index be882913..84b6d44d 100644 --- a/lib/pages/text_generation/playground.dart +++ b/lib/pages/text_generation/playground.dart @@ -26,13 +26,14 @@ class Playground extends StatefulWidget { class SubmitMessageIntent extends Intent {} class _PlaygroundState extends State { - final textController = TextEditingController(); - final scrollController = ScrollController(); + final _textController = TextEditingController(); + final _scrollController = ScrollController(); + final _focusNode = FocusNode(); bool attachedToBottom = true; void jumpToBottom({ offset = 0 }) { - if (scrollController.hasClients) { - scrollController.jumpTo(scrollController.position.maxScrollExtent + offset); + if (_scrollController.hasClients) { + _scrollController.jumpTo(_scrollController.position.maxScrollExtent + offset); } } @@ -40,7 +41,7 @@ class _PlaygroundState extends State { if (message.isEmpty) return; final provider = Provider.of(context, listen: false); if (!provider.initialized || provider.response != null) return; - textController.text = ''; + _textController.text = ''; jumpToBottom(offset: 110); //move to bottom including both provider.message(message).catchError((e) async { // ignore: use_build_context_synchronously @@ -59,17 +60,17 @@ class _PlaygroundState extends State { @override void initState() { super.initState(); - scrollController.addListener(() { + _scrollController.addListener(() { setState(() { - attachedToBottom = scrollController.position.pixels + 0.001 >= scrollController.position.maxScrollExtent; + attachedToBottom = _scrollController.position.pixels + 0.001 >= _scrollController.position.maxScrollExtent; }); }); } @override void dispose() { - textController.dispose(); - scrollController.dispose(); + _textController.dispose(); + _scrollController.dispose(); super.dispose(); } @@ -158,16 +159,20 @@ class _PlaygroundState extends State { alignment: Alignment.bottomCenter, children: [ SingleChildScrollView( - controller: scrollController, - child: Padding(padding: const EdgeInsets.symmetric(horizontal: 64, vertical: 20), child: Column( - children: provider.messages.map((message) { switch (message.speaker) { - case Speaker.user: return Padding( - padding: const EdgeInsets.only(left: 42), - child: UserMessage(message), - ); - case Speaker.system: return Text('System: ${message.message}'); - case Speaker.assistant: return AssistantMessage(message); - }}).toList(), + controller: _scrollController, + child: Padding(padding: const EdgeInsets.symmetric(horizontal: 64, vertical: 20), child: SelectionArea( + child: SelectionArea( + child: Column( + children: provider.messages.map((message) { switch (message.speaker) { + case Speaker.user: return Padding( + padding: const EdgeInsets.only(left: 42), + child: UserMessage(message), + ); + case Speaker.system: return Text('System: ${message.message}'); + case Speaker.assistant: return AssistantMessage(message); + }}).toList(), + ), + ), ),), ), Positioned( @@ -221,7 +226,7 @@ class _PlaygroundState extends State { child: Actions( actions: >{ SubmitMessageIntent: CallbackAction( - onInvoke: (SubmitMessageIntent intent) => message(textController.text), + onInvoke: (SubmitMessageIntent intent) => message(_textController.text), ), }, child: Column( @@ -230,7 +235,7 @@ class _PlaygroundState extends State { TextBox( placeholder: "Type a message...", keyboardType: TextInputType.multiline, - controller: textController, + controller: _textController, maxLines: null, expands: true, onSubmitted: message, @@ -258,7 +263,7 @@ class _PlaygroundState extends State { ) : Tooltip( message: "Send message", - child: Button(child: const Icon(FluentIcons.send, size: 18,), onPressed: () { message(textController.text); }), + child: Button(child: const Icon(FluentIcons.send, size: 18,), onPressed: () { message(_textController.text); }), ) ), ) diff --git a/lib/pages/text_generation/widgets/assistant_message.dart b/lib/pages/text_generation/widgets/assistant_message.dart index 1ff268f1..6ff98113 100644 --- a/lib/pages/text_generation/widgets/assistant_message.dart +++ b/lib/pages/text_generation/widgets/assistant_message.dart @@ -57,21 +57,23 @@ class _AssistantMessageState extends State { children: [ Padding( padding: const EdgeInsets.only(bottom: 4), - child: Row( - children: [ - Text( - inferenceProvider.project!.name, - style: TextStyle( - color: subtleTextColor.of(theme), + child: SelectionContainer.disabled( + child: Row( + children: [ + Text( + inferenceProvider.project!.name, + style: TextStyle( + color: subtleTextColor.of(theme), + ), ), - ), - if (widget.message.time != null) Text( - DateFormat(' | yyyy-MM-dd HH:mm:ss').format(widget.message.time!), - style: TextStyle( - color: subtleTextColor.of(theme), + if (widget.message.time != null) Text( + DateFormat(' | yyyy-MM-dd HH:mm:ss').format(widget.message.time!), + style: TextStyle( + color: subtleTextColor.of(theme), + ), ), - ), - ], + ], + ), ), ), MouseRegion( @@ -86,78 +88,80 @@ class _AssistantMessageState extends State { color: backgroundColor, borderRadius: BorderRadius.circular(4), ), - child: Markdown( - data: widget.message.message, - selectable: true, - shrinkWrap: true, - padding: const EdgeInsets.all(12), - extensionSet: md.ExtensionSet( - md.ExtensionSet.gitHubFlavored.blockSyntaxes, - [md.EmojiSyntax(), ...md.ExtensionSet.gitHubFlavored.inlineSyntaxes], + child: Padding( + padding: const EdgeInsets.all(12.0), + child: MarkdownBody( + data: widget.message.message, + extensionSet: md.ExtensionSet( + md.ExtensionSet.gitHubFlavored.blockSyntaxes, + [md.EmojiSyntax(), ...md.ExtensionSet.gitHubFlavored.inlineSyntaxes], + ), ), ), ), if (_hovering) Padding( padding: const EdgeInsets.only(top: 4), - child: Row( - children: [ - if (widget.message.metrics != null) Padding( - padding: const EdgeInsets.only(right: 8), - child: Tooltip( - message: 'Time to first token', - child: Text( - 'TTF: ${nf.format(widget.message.metrics!.ttft)}ms', - style: TextStyle( - fontSize: 12, - color: subtleTextColor.of(theme), + child: SelectionContainer.disabled( + child: Row( + children: [ + if (widget.message.metrics != null) Padding( + padding: const EdgeInsets.only(right: 8), + child: Tooltip( + message: 'Time to first token', + child: Text( + 'TTF: ${nf.format(widget.message.metrics!.ttft)}ms', + style: TextStyle( + fontSize: 12, + color: subtleTextColor.of(theme), + ), ), ), ), - ), - if (widget.message.metrics != null) Padding( - padding: const EdgeInsets.only(right: 8), - child: Tooltip( - message: 'Time per output token', - child: Text( - 'TPOT: ${nf.format(widget.message.metrics!.tpot)}ms', - style: TextStyle( - fontSize: 12, - color: subtleTextColor.of(theme), + if (widget.message.metrics != null) Padding( + padding: const EdgeInsets.only(right: 8), + child: Tooltip( + message: 'Time per output token', + child: Text( + 'TPOT: ${nf.format(widget.message.metrics!.tpot)}ms', + style: TextStyle( + fontSize: 12, + color: subtleTextColor.of(theme), + ), ), ), ), - ), - if (widget.message.metrics != null) Padding( - padding: const EdgeInsets.only(right: 8), - child: Tooltip( - message: 'Generate total duration', - child: Text( - 'Generate: ${nf.format(widget.message.metrics!.generate_time/1000)}s', - style: TextStyle( - fontSize: 12, - color: subtleTextColor.of(theme), + if (widget.message.metrics != null) Padding( + padding: const EdgeInsets.only(right: 8), + child: Tooltip( + message: 'Generate total duration', + child: Text( + 'Generate: ${nf.format(widget.message.metrics!.generate_time/1000)}s', + style: TextStyle( + fontSize: 12, + color: subtleTextColor.of(theme), + ), ), ), ), - ), - IconButton( - icon: const Icon(FluentIcons.copy), - onPressed: () async{ - await displayInfoBar(context, builder: (context, close) => - InfoBar( - title: const Text('Copied to clipboard'), - severity: InfoBarSeverity.info, - action: IconButton( - icon: const Icon(FluentIcons.clear), - onPressed: close, + IconButton( + icon: const Icon(FluentIcons.copy), + onPressed: () async{ + await displayInfoBar(context, builder: (context, close) => + InfoBar( + title: const Text('Copied to clipboard'), + severity: InfoBarSeverity.info, + action: IconButton( + icon: const Icon(FluentIcons.clear), + onPressed: close, + ), ), - ), - ); - Clipboard.setData(ClipboardData(text: widget.message.message)); - }, - ), - ], + ); + Clipboard.setData(ClipboardData(text: widget.message.message)); + }, + ), + ], + ), ), ) else const SizedBox(height: 34) ], diff --git a/lib/pages/text_generation/widgets/user_message.dart b/lib/pages/text_generation/widgets/user_message.dart index 5e73496b..dd09a36e 100644 --- a/lib/pages/text_generation/widgets/user_message.dart +++ b/lib/pages/text_generation/widgets/user_message.dart @@ -13,30 +13,27 @@ class UserMessage extends StatelessWidget { final theme = FluentTheme.of(context); return Align( alignment: Alignment.centerRight, - child: ConstrainedBox( - constraints: const BoxConstraints(maxWidth: 1000), - child: Padding(padding: const EdgeInsets.only(bottom: 20), child: Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - Container( - decoration: BoxDecoration( - color: cosmosBackground.of(theme), - borderRadius: BorderRadius.circular(4), - ), - child: Markdown( + child: Padding(padding: const EdgeInsets.only(bottom: 20), child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Container( + decoration: BoxDecoration( + color: cosmosBackground.of(theme), + borderRadius: BorderRadius.circular(4), + ), + child: Padding( + padding: const EdgeInsets.all(12.0), + child: MarkdownBody( data: message.message, - selectable: true, - shrinkWrap: true, - padding: const EdgeInsets.all(12), extensionSet: md.ExtensionSet( md.ExtensionSet.gitHubFlavored.blockSyntaxes, [md.EmojiSyntax(), ...md.ExtensionSet.gitHubFlavored.inlineSyntaxes], ), ), - ) - ], - ),), - ), + ), + ) + ], + ),), ); } } \ No newline at end of file From e7501d7eb8df5b09db90f86d4aed59aae7e64482 Mon Sep 17 00:00:00 2001 From: "Hecker, Ronald" Date: Sun, 1 Dec 2024 16:10:52 +0100 Subject: [PATCH 30/33] Remove old code that is no longer used. --- lib/canvas/canvas.dart | 136 ------ lib/canvas/canvas_painter.dart | 154 ------ lib/example_graph.dart | 46 -- lib/header.dart | 61 --- lib/import/import_page.dart | 37 -- lib/import/optimization_filter_button.dart | 30 -- lib/import/public_model.dart | 323 ------------- lib/inference/batch_inference.dart | 507 -------------------- lib/inference/camera_page.dart | 109 ----- lib/inference/device_selector.dart | 40 -- lib/inference/download_page.dart | 124 ----- lib/inference/image_inference_page.dart | 126 ----- lib/inference/inference_page.dart | 40 -- lib/inference/live_inference.dart | 232 --------- lib/inference/model_info.dart | 114 ----- lib/inference/text/metric_widgets.dart | 147 ------ lib/inference/text/performance_metrics.dart | 97 ---- lib/inference/text/playground.dart | 426 ---------------- lib/inference/text_inference_page.dart | 165 ------- lib/projects/project_item.dart | 244 ---------- lib/projects/projects_page.dart | 299 ------------ lib/projects/task_type_filter.dart | 133 ----- lib/providers/image_inference_provider.dart | 2 +- lib/utils/drop_area.dart | 100 ---- lib/{ => utils}/image_graph_builder.dart | 0 lib/zoomable_image.dart | 86 ---- 26 files changed, 1 insertion(+), 3777 deletions(-) delete mode 100644 lib/canvas/canvas.dart delete mode 100644 lib/canvas/canvas_painter.dart delete mode 100644 lib/example_graph.dart delete mode 100644 lib/header.dart delete mode 100644 lib/import/import_page.dart delete mode 100644 lib/import/optimization_filter_button.dart delete mode 100644 lib/import/public_model.dart delete mode 100644 lib/inference/batch_inference.dart delete mode 100644 lib/inference/camera_page.dart delete mode 100644 lib/inference/device_selector.dart delete mode 100644 lib/inference/download_page.dart delete mode 100644 lib/inference/image_inference_page.dart delete mode 100644 lib/inference/inference_page.dart delete mode 100644 lib/inference/live_inference.dart delete mode 100644 lib/inference/model_info.dart delete mode 100644 lib/inference/text/metric_widgets.dart delete mode 100644 lib/inference/text/performance_metrics.dart delete mode 100644 lib/inference/text/playground.dart delete mode 100644 lib/inference/text_inference_page.dart delete mode 100644 lib/projects/project_item.dart delete mode 100644 lib/projects/projects_page.dart delete mode 100644 lib/projects/task_type_filter.dart delete mode 100644 lib/utils/drop_area.dart rename lib/{ => utils}/image_graph_builder.dart (100%) delete mode 100644 lib/zoomable_image.dart diff --git a/lib/canvas/canvas.dart b/lib/canvas/canvas.dart deleted file mode 100644 index 496ce2ff..00000000 --- a/lib/canvas/canvas.dart +++ /dev/null @@ -1,136 +0,0 @@ - -import 'dart:async'; -import 'dart:math'; -import 'dart:ui' as ui; - -import 'package:flutter/gestures.dart'; -import 'package:flutter/material.dart'; -import 'package:inference/annotation.dart'; -import 'package:inference/canvas/canvas_painter.dart'; -import 'package:vector_math/vector_math_64.dart' show Vector3; -import 'package:inference/project.dart' as project; - -class Canvas extends StatefulWidget { - - final ui.Image image; - final List? annotations; - final List labelDefinitions; - - const Canvas({required this.image, this.annotations, required this.labelDefinitions, super.key}); - - @override - State createState() => _CanvasState(); -} - -class _CanvasState extends State { - - double prevScale = 1; - Matrix4 matrix = Matrix4.identity() - ..scale(0.9); - Matrix4 inverse = Matrix4.identity(); - bool done = false; - - @override - void initState() { - super.initState(); - - Future.delayed(Duration.zero).then((_) { - setState(() { - matrix = setTransformToFit(widget.image); - }); - }); - } - - Matrix4 setTransformToFit(ui.Image image) { - if (context.size == null) { - return Matrix4.identity(); - } - final imageSize = Size(image.width.toDouble(), image.height.toDouble()); - final canvasSize = context.size!; - - final ratio = Size(imageSize.width / canvasSize.width, imageSize.height / canvasSize.height); - - final scale = 1 / max(ratio.width, ratio.height) * 0.9; - final offset = (canvasSize - imageSize * scale as Offset) / 2; - - return matrix = Matrix4.identity() - ..translate(offset.dx, offset.dy, 0.0) - ..scale(scale); - } - - void scaleCanvas(Vector3 localPosition, double scale) { - inverse.copyInverse(matrix); - final position = inverse * localPosition; - final mScale = 1 - scale; - setState(() { - matrix *= Matrix4( // row major or column major - scale, 0, 0, 0, - 0, scale, 0, 0, - 0, 0, scale, 0, - mScale * position.x, mScale * position.y, 0, 1); - }); - } - - @override - Widget build(BuildContext context) { - return NotificationListener( - onNotification: (f) { - WidgetsBinding.instance.addPostFrameCallback((_) { - setState(() { - matrix = setTransformToFit(widget.image); - }); - }); - return false; - }, - child: SizeChangedLayoutNotifier( - child: SizedBox.expand( - child: Container( - clipBehavior: Clip.hardEdge, - decoration: const BoxDecoration(shape: BoxShape.rectangle), - child: GestureDetector( - behavior: HitTestBehavior.translucent, - onScaleStart: (_) { - prevScale = 1; - }, - onDoubleTap: () { - setState(() { - matrix = setTransformToFit(widget.image); - }); - }, - onScaleUpdate: (ScaleUpdateDetails d) { - final scale = 1 - (prevScale - d.scale); - prevScale = d.scale; - final zoom = matrix.getMaxScaleOnAxis(); - scaleCanvas(Vector3(d.localFocalPoint.dx, d.localFocalPoint.dy, 0), scale); - setState(() { - matrix.translate(d.focalPointDelta.dx / zoom, d.focalPointDelta.dy / zoom, 0.0); - }); - }, - child: Listener( - behavior: HitTestBehavior.translucent, - onPointerSignal: (p) { - if (p is PointerScrollEvent) { - final scale = p.scrollDelta.dy > 0 ? 0.95 : 1.05; // lazy solution, perhaps an animation depending on the scrollDelta? - scaleCanvas(Vector3(p.localPosition.dx, p.localPosition.dy, 0.0), scale); - } - }, - child: Transform( - transform: matrix, - alignment: FractionalOffset.topLeft, - child: Builder( - builder: (context) { - return CustomPaint( - painter: CanvasPainter(widget.image, widget.annotations, widget.labelDefinitions, matrix.getMaxScaleOnAxis()), - child: Container(), - ); - } - ) - ), - ), - ), - ), - ), - ), - ); - } -} diff --git a/lib/canvas/canvas_painter.dart b/lib/canvas/canvas_painter.dart deleted file mode 100644 index 87143d30..00000000 --- a/lib/canvas/canvas_painter.dart +++ /dev/null @@ -1,154 +0,0 @@ -import 'dart:math'; -import 'dart:ui' as ui; - -import 'package:collection/collection.dart'; -import 'package:flutter/material.dart'; -import 'package:inference/annotation.dart'; -import 'package:inference/color.dart'; -import 'package:inference/project.dart' as project; -import 'package:vector_math/vector_math_64.dart' show Vector3; - -Color getColorByLabelID(String labelId, List labelDefinitions) { - final label = labelDefinitions.firstWhereOrNull((project.Label b) => b.id == labelId); - if (label == null) { - throw "Label not found"; - } - return HexColor.fromHex(label.color.substring(0, 7)); -} - -class CanvasPainter extends CustomPainter { - final ui.Image image; - final List? annotations; - final List labelDefinitions; - final double scale; - - CanvasPainter(this.image, this.annotations, this.labelDefinitions, this.scale); - - @override - void paint(Canvas canvas, Size size) { - paintImage( - alignment: Alignment.topLeft, - canvas: canvas, - rect: Rect.fromLTWH(0, 0, image.width.toDouble(), image.height.toDouble()), - fit: BoxFit.scaleDown, - image: image, - ); - for (final annotation in annotations ?? []) { - final firstLabelColor = getColorByLabelID(annotation.labels[0].id, labelDefinitions); - Paint paint = Paint() - ..color = firstLabelColor - ..strokeWidth = 2 - ..style = PaintingStyle.stroke; - - Paint transparent = Paint() - ..color = Color.fromARGB(102, firstLabelColor.red, firstLabelColor.green, firstLabelColor.blue); - - if (annotation.shape is Rectangle) { - drawRectangle(canvas, size, paint, transparent, annotation); - } - if (annotation.shape is Polygon) { - drawPolygon(canvas, size, paint, transparent, annotation); - } - if (annotation.shape is RotatedRectangle) { - drawRotatedRectangle(canvas, size, paint, transparent, annotation); - } - } - } - - void drawRectangle(Canvas canvas, Size size, Paint paint, Paint transparent, Annotation annotation){ - final imageSize = Size(image.width.toDouble(), image.height.toDouble()); - final rect = (annotation.shape as Rectangle).toRect(); - canvas.drawRect(rect, paint); - if (rect.size != imageSize) { - canvas.drawRect(rect, transparent); - } - var position = rect.topLeft; - for (final label in annotation.labels) { - final labelSize = drawLabel(canvas, size, label, position); - position += Offset(labelSize.width, 0); - } - } - - - void drawPolygon(Canvas canvas, Size size, Paint paint, Paint transparent, Annotation annotation) { - final path = ui.Path(); - final shape = (annotation.shape as Polygon); - path.addPolygon(shape.points, true); - - canvas.drawPath(path, paint); - canvas.drawPath(path, transparent); - final rect = shape.rectangle.toRect(); - final topCenter = rect.topCenter - const Offset(0, 30.0); - canvas.drawLine(rect.center, topCenter, paint); - - var position = topCenter; - for (final label in annotation.labels) { - final labelSize = drawLabel(canvas, size, label, position); - position += Offset(labelSize.width, 0); - } - } - - void drawRotatedRectangle(Canvas canvas, Size size, Paint paint, Paint transparent, Annotation annotation) { - final shape = (annotation.shape as RotatedRectangle); - final path = ui.Path(); - final rect = ui.Rect.fromCenter(center: ui.Offset.zero, width: shape.width, height: shape.height); - path.addRect(rect); - final matrix = Matrix4.identity() - ..rotateZ(shape.angleInRadians) - ..setTranslationRaw(shape.centerX, shape.centerY, 0.0); - - final corners = [rect.topLeft, rect.topRight, rect.bottomRight, rect. bottomLeft]; - - - - final rotatedPath = path.transform(matrix.storage); - canvas.drawPath(rotatedPath, paint); - canvas.drawPath(rotatedPath, transparent); - - double labelPosition = double.infinity; - for (final corner in corners) { - final transformedCorner = (matrix * Vector3(corner.dx, corner.dy, 0)) as Vector3; - labelPosition = min(transformedCorner.y, labelPosition); - } - - var position = Offset(shape.centerX, labelPosition - 30); - canvas.drawLine(Offset(shape.centerX, shape.centerY), position, paint); - for (final label in annotation.labels) { - final labelSize = drawLabel(canvas, size, label, position); - position += Offset(labelSize.width, 0); - } - } - - Size drawLabel(Canvas canvas, Size size, Label label, Offset position) { - final color = getColorByLabelID(label.id, labelDefinitions); - Paint paint = Paint() - ..color = color; - final textStyle = TextStyle( - color: foregroundColorByLuminance(color), - fontFamily: 'IntelOne', - fontSize: 14 / scale, - ); - final textSpan = TextSpan( - text: "${label.name} ${(label.probability * 100).toStringAsFixed(1)}%", - style: textStyle, - ); - final textPainter = TextPainter( - text: textSpan, - textDirection: TextDirection.ltr, - ); - textPainter.layout( - minWidth: 0, - maxWidth: size.width, - ); - canvas.drawRect(ui.Rect.fromLTWH(position.dx - 1, position.dy - textPainter.height - 1, textPainter.width + 2, textPainter.height), paint); - textPainter.paint(canvas, position - Offset(0, textPainter.height)); - return textPainter.size + const Offset(3, 0); - } - - @override - bool shouldRepaint(CanvasPainter oldDelegate) { - return false; - } - @override - bool shouldRebuildSemantics(CanvasPainter oldDelegate) => false; -} diff --git a/lib/example_graph.dart b/lib/example_graph.dart deleted file mode 100644 index 6b2b588e..00000000 --- a/lib/example_graph.dart +++ /dev/null @@ -1,46 +0,0 @@ -const String faceDetectionExampleGraph = """ -output_stream : "output" - -node { - calculator : "OpenVINOInferenceAdapterCalculator" - output_side_packet : "INFERENCE_ADAPTER:adapter" - node_options: { - [type.googleapis.com/mediapipe.OpenVINOInferenceAdapterCalculatorOptions] { - model_path: "/data/omz_models/intel/face-detection-retail-0004/face-detection-retail-0004.xml" - } - } -} - -node { - calculator : "VideoInputCalculator" - output_stream: "IMAGE:input_image" -} - -node { - calculator : "DetectionCalculator" - input_side_packet : "INFERENCE_ADAPTER:adapter" - input_stream : "IMAGE:input_image" - output_stream: "INFERENCE_RESULT:inference_detections" -} - -node { - calculator : "OverlayCalculator" - input_stream : "IMAGE:input_image" - input_stream : "INFERENCE_RESULT:inference_detections" - output_stream: "IMAGE:output" - node_options: { - [type.googleapis.com/mediapipe.OverlayCalculatorOptions] { - labels: { - id: "face" - name: "face" - color: "#f7dab3ff" - is_empty: false - } - - stroke_width: 2 - opacity: 0.4 - font_size: 1.0 - } - } -} -"""; diff --git a/lib/header.dart b/lib/header.dart deleted file mode 100644 index bede01c4..00000000 --- a/lib/header.dart +++ /dev/null @@ -1,61 +0,0 @@ -import 'package:flutter/material.dart'; -import 'package:go_router/go_router.dart'; -import 'package:flutter_svg/svg.dart'; - -class Header extends StatelessWidget implements PreferredSizeWidget { - final bool showBack; - final void Function(BuildContext context)? onBack; - const Header(this.showBack, {this.onBack, super.key}); - - - @override - Widget build(BuildContext context) { - return Column( - children: [ - SizedBox( - height: 88, - child: Stack( - children: [ - Container( - padding: const EdgeInsets.only(left: 30), - height: 64, - color: Theme.of(context).colorScheme.secondary, - child: Row( - crossAxisAlignment: CrossAxisAlignment.center, - children: [ - if(showBack) Padding( - padding: const EdgeInsets.only(right: 5), - child: IconButton( - color: Colors.white, - icon: const Icon(Icons.arrow_back), - onPressed: () { - if(onBack != null) { - onBack!(context); - } else { - context.go('/'); - } - } - ), - ), - Padding( - padding: const EdgeInsets.only(right: 10.0, top: 2), - child: SvgPicture.asset("images/openvino_logo.svg"), - ), - const Text("Test Drive", style: TextStyle( - color: Colors.white, - fontSize: 20, - )), - ], - ), - ), - SvgPicture.asset("images/bit.svg"), - ] - ), - ), - ], - ); - } - - @override - Size get preferredSize => const Size.fromHeight(88); -} diff --git a/lib/import/import_page.dart b/lib/import/import_page.dart deleted file mode 100644 index 5eaab894..00000000 --- a/lib/import/import_page.dart +++ /dev/null @@ -1,37 +0,0 @@ -import 'package:flutter/material.dart'; -import 'package:inference/header.dart'; -import 'package:inference/import/public_model.dart'; - -class ImportPage extends StatefulWidget { - const ImportPage({super.key}); - - @override - State createState() => _ImportPageState(); -} - -class _ImportPageState extends State with TickerProviderStateMixin { - late TabController _tabController; - - @override - void initState() { - super.initState(); - _tabController = TabController(length: 3, animationDuration: Duration.zero, vsync: this); - } - - @override - void dispose() { - _tabController.dispose(); - super.dispose(); - } - - @override - Widget build(BuildContext context) { - return const Scaffold( - appBar: Header(false), - body: Padding( - padding: EdgeInsets.symmetric(horizontal: 30.0), - child: PublicModelPage() - ), - ); - } -} diff --git a/lib/import/optimization_filter_button.dart b/lib/import/optimization_filter_button.dart deleted file mode 100644 index 98efce6f..00000000 --- a/lib/import/optimization_filter_button.dart +++ /dev/null @@ -1,30 +0,0 @@ -import 'package:flutter/material.dart'; -import 'package:inference/providers/project_filter_provider.dart'; - -class OptimizationFilterButton extends StatelessWidget { - - final String name; - final ProjectFilterProvider filter; - const OptimizationFilterButton(this.name, this.filter, {super.key}); - - @override - Widget build(BuildContext context) { - return Card( - child: Padding( - padding: const EdgeInsets.only(right: 12.0), - child: Row( - children: [ - Checkbox(value: filter.optimizations.contains(name), onChanged: (val) { - if (val ?? false) { - filter.addOptimization(name); - } else { - filter.removeOptimization(name); - } - }), - Text(name), - ], - ), - ), - ); - } -} diff --git a/lib/import/public_model.dart b/lib/import/public_model.dart deleted file mode 100644 index ce2bd065..00000000 --- a/lib/import/public_model.dart +++ /dev/null @@ -1,323 +0,0 @@ -import 'package:dio/dio.dart'; -import 'package:flutter/material.dart'; -import 'package:inference/import/optimization_filter_button.dart'; -import 'package:inference/projects/task_type_filter.dart'; -import 'package:inference/providers/project_filter_provider.dart'; -import 'package:inference/providers/project_provider.dart'; -import 'package:inference/public_model_info.dart'; -import 'package:inference/public_models.dart'; -import 'package:inference/searchbar.dart'; -import 'package:inference/theme.dart'; -import 'package:inference/utils/dialogs.dart'; -import 'package:provider/provider.dart'; -import 'package:shimmer/shimmer.dart'; -import 'package:go_router/go_router.dart'; - -class PublicModelPage extends StatefulWidget { - const PublicModelPage({super.key}); - - @override - State createState() => _PublicModelPageState(); -} - -class _PublicModelPageState extends State { - List? models; - - PublicModelInfo? selectedModelForImport; - - void loadModels() async { - try { - final publicModels = await getPublicModels(); - setState(() { - models = publicModels; - }); - } on DioException catch(ex) { - if (ex.type == DioExceptionType.connectionError) { - if (context.mounted) { - errorDialog(context, "Connection error","Unable to connect to HuggingFace to load the OpenVINO model collections.\nPlease disable your proxy or VPN and try again."); - } - } - } - - } - - @override - void initState() { - super.initState(); - loadModels(); - } - - List buildList(ProjectFilterProvider filter) { - if (models == null) { - return const [ - GhostProjectItem(), - GhostProjectItem(), - GhostProjectItem(), - GhostProjectItem(), - GhostProjectItem(), - GhostProjectItem(), - GhostProjectItem(), - GhostProjectItem(), - ]; - } - - final filteredModels = filter.applyFilterOnPublicModelInfo(models!); - - return filteredModels.map((model) { - return PublicModelItem(model, - key: ValueKey(model), - isSelected: selectedModelForImport == model, - onToggle: (selected) { - setState(() { - if (selected) { - selectedModelForImport = model; - } else { - selectedModelForImport = null; - } - }); - }, - ); - }).toList(); - } - - @override - Widget build(BuildContext context) { - return ChangeNotifierProvider( - create: (_) => ProjectFilterProvider(), - child: Consumer(builder: (context, filter, child) { - return Padding( - padding: const EdgeInsets.only(top: 8.0), - child: Row( - children: [ - Expanded( - child: Column( - mainAxisAlignment: MainAxisAlignment.start, - crossAxisAlignment: CrossAxisAlignment.stretch, - children: [ - Padding( - padding: const EdgeInsets.only(left: 3, bottom: 8.0, top: 8.0, right: 8.0), - child: Row( - mainAxisAlignment: MainAxisAlignment.start, - children: [ - GetiSearchBar( - onChange: (value) => filter.name = value, - ), - const Padding( - padding: EdgeInsets.only(left: 32.0), - child: Text("Optimization: "), - ), - OptimizationFilterButton("int4", filter), - OptimizationFilterButton("int8", filter), - OptimizationFilterButton("fp16", filter), - ], - ), - ), - Expanded( - child: CustomScrollView( - primary: false, - slivers: [ - SliverPadding( - padding: const EdgeInsets.all(3), - sliver: SliverGrid.extent( - crossAxisSpacing: 10, - mainAxisSpacing: 10, - maxCrossAxisExtent: 310, - childAspectRatio: 1.5, - children: buildList(filter) - ) - ) - ] - ), - ), - Padding( - padding: const EdgeInsets.symmetric(vertical: 30.0, horizontal: 8), - child: Row( - mainAxisAlignment: MainAxisAlignment.end, - children: [ - Padding( - padding: const EdgeInsets.only(right: 8.0), - child: ElevatedButton( - style: const ButtonStyle( - elevation: WidgetStatePropertyAll(0), - backgroundColor: WidgetStatePropertyAll(intelGray) - ), - onPressed: () { - setState(() { - context.pop(); - }); - }, - child: const Text("Cancel") - ), - ), - - ElevatedButton( - style: ButtonStyle( - elevation: const WidgetStatePropertyAll(0), - backgroundColor: (selectedModelForImport == null - ? const WidgetStatePropertyAll(intelGray) - : null - ) - ), - onPressed: () { - if (selectedModelForImport != null) { - final projectsProvider = Provider.of(context, listen: false); - PublicModelInfo.convertToProject(selectedModelForImport!).then((project) { - projectsProvider.addProject(project); - context.go('/inference', extra: project); - }); - } - }, - child: const Text("Add model") - ), - ], - ), - ), - ], - ), - ), - ], - ), - ); - } - ), - ); - } -} - - -class GhostProjectItem extends StatelessWidget { - const GhostProjectItem({super.key}); - - @override - Widget build(BuildContext context) { - return ClipRRect( - borderRadius: BorderRadius.circular(8.0), - child: Opacity( - opacity: 1.0, - child: ConstrainedBox( - constraints: const BoxConstraints.expand(height: 136), - child: Shimmer.fromColors( - baseColor: Theme.of(context).colorScheme.surface, - highlightColor: Theme.of(context).colorScheme.onSurfaceVariant, - child: Container( - color: Theme.of(context).colorScheme.surface, - ) - ), - ), - ) - ); - } -} - -class PublicModelItem extends StatefulWidget { - final PublicModelInfo model; - final Function onToggle; - final bool isSelected; - const PublicModelItem(this.model, {required this.onToggle, this.isSelected = false, super.key}); - - @override - State createState() => _PublicModelItemState(); -} - -class _PublicModelItemState extends State { - bool hovered = false; - - void onHover(bool? val) { - setState(() { - hovered = val ?? false; - }); - } - - void onTap() { - widget.onToggle(!widget.isSelected); - } - - Color backgroundColor(BuildContext context) { - final scheme = Theme.of(context).colorScheme; - if (hovered) { - return scheme.onSurfaceVariant; - } else { - return scheme.surface; - } - } - - Color borderColor(BuildContext context) { - if (widget.isSelected) { - return intelBlue; - } else { - return backgroundColor(context); - } - } - - double borderWidth(BuildContext context) { - if (widget.isSelected) { - return 2; - } else { - return 0; - } - } - - @override - Widget build(BuildContext context) { - return Padding( - padding: const EdgeInsets.only(top: 8, bottom: 8), - child: Card( - shape: RoundedRectangleBorder( - borderRadius: BorderRadius.circular(8.0), - side: BorderSide(width: borderWidth(context), color: borderColor(context)) - ), - child: ClipRRect( - borderRadius: BorderRadius.circular(8.0), - child: InkWell( - splashFactory: NoSplash.splashFactory, - borderRadius: BorderRadius.circular(8.0), - onHover: (val) => onHover(val), - onTap: () => onTap(), - child: Column( - mainAxisAlignment: MainAxisAlignment.start, - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - Container( - height: 110, - decoration: BoxDecoration( - image: DecorationImage( - image: widget.model.thumbnail.image, - fit: BoxFit.cover), - ), - ), - Expanded( - child: Padding( - padding: const EdgeInsets.all(8.0), - child: Column( - crossAxisAlignment: CrossAxisAlignment.start, - mainAxisAlignment: MainAxisAlignment.spaceBetween, - children: [ - Text( - widget.model.name, - overflow: TextOverflow.ellipsis, - style: const TextStyle( - color: Colors.white, - fontWeight: FontWeight.bold, - fontSize: 12, - ) - ), - //Text(formatter.format(DateTime.parse(widget.model.lastModified))), - //Column( - // crossAxisAlignment: CrossAxisAlignment.start, - // children: [ - // Text("Likes: ${widget.model.likes}"), - // Text("Downloads: ${widget.model.downloads}"), - // ], - //), - ] - ), - ), - ) - ], - ), - ), - ) - ) - ); - } -} diff --git a/lib/inference/batch_inference.dart b/lib/inference/batch_inference.dart deleted file mode 100644 index e0a59d60..00000000 --- a/lib/inference/batch_inference.dart +++ /dev/null @@ -1,507 +0,0 @@ -import 'dart:convert'; -import 'dart:io'; -import 'dart:typed_data'; - -import 'package:csv/csv.dart'; -import 'package:desktop_drop/desktop_drop.dart'; -import 'package:file_picker/file_picker.dart'; -import 'package:flutter/material.dart'; -import 'package:inference/inference/device_selector.dart'; -import 'package:inference/interop/openvino_bindings.dart'; -import 'package:inference/providers/image_inference_provider.dart'; -import 'package:inference/theme.dart'; -import 'package:mime/mime.dart'; -import 'package:path/path.dart'; -import 'package:provider/provider.dart'; - -bool isImage(String path) { - final mimeType = lookupMimeType(path); - if (mimeType == null) { - return false; - } - return mimeType.startsWith('image/'); -} - - -bool pathIsValid(String path) { - // Directory.exists will return false if the file is not a folder. - return Directory(path).existsSync(); -} - -class Progress { - int current = 0; - final int total; - - Progress(this.total); - - double percentage() { - return current.toDouble() / total; - } -} - - -class BatchInference extends StatefulWidget { - const BatchInference({super.key}); - - @override - State createState() => _BatchInferenceState(); -} - -class _BatchInferenceState extends State { - - String sourceFolder = ""; - String destinationFolder = ""; - bool forceStop = false; - Progress? progress; - - bool get isRunning { - if (progress == null) { - return false; - } - - if (forceStop) { - return false; - } - - return progress!.current < progress!.total; - } - - bool get canProcess => - pathIsValid(sourceFolder) && pathIsValid(destinationFolder) && - serializationOutput.any(); - - SerializationOutput serializationOutput = SerializationOutput(overlay: true); - - void processFolder(BuildContext context, ImageInferenceProvider inference) async { - final platformContext = Context(style: Style.platform); - final dir = Directory(sourceFolder); - final listener = dir.list(recursive: true); - - final list = await listener.toList(); - setState(() { - progress = Progress(list.where((item) => isImage(item.path)).length); - forceStop = false; - }); - - List> rows = []; - const encoder = JsonEncoder.withIndent(" "); - const converter = CsvToListConverter(); - inference.lock(); - - for (final file in list) { - if (forceStop) { - break; - } - if (isImage(file.path)) { - final outputFilename = platformContext.basename(file.path); - Uint8List imageData = File(file.path).readAsBytesSync(); - final inferenceResult = await inference.infer(imageData, serializationOutput); - await Future.delayed(Duration.zero); // For some reason ui does not update even though it's running in Isolate. This gives the UI time to run that code. - final outputPath = platformContext.join(destinationFolder, outputFilename); - if (serializationOutput.overlay) { - final outputFile = File(outputPath); - final decodedImage = base64Decode(inferenceResult.overlay!); - outputFile.writeAsBytes(decodedImage); - } - if (serializationOutput.csv) { - var csvOutput = converter.convert(inferenceResult.csv); - rows.addAll(csvOutput.map((row) { - row.insert(0, outputFilename); - return row; - })); - } - if (serializationOutput.json) { - final outputFile = File(setExtension(outputPath, ".json")); - outputFile.writeAsString(encoder.convert(inferenceResult.json)); - } - setState(() { - progress!.current += 1; - }); - } - } - inference.unlock(); - - if (serializationOutput.csv) { - List columns = ["filename", "label_name", "label_id", "probability", "shape_type", "x", "y", "width", "height", "area", "angle"]; - rows.insert(0, columns); - const converter = ListToCsvConverter(); - final outputPath = platformContext.join(destinationFolder, "predictions.csv"); - File(outputPath).writeAsStringSync(converter.convert(rows)); - } - } - - void stop() { - setState((){ - forceStop = true; - }); - } - - @override - Widget build(BuildContext context) { - return Consumer( - builder: (context, inference, child) { - return Padding( - padding: const EdgeInsets.only(top: 48.0), - child: Container( - color: Theme.of(context).colorScheme.surfaceContainer, - padding: const EdgeInsets.all(30), - child: Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - FolderRow("Source folder", - initialValue: sourceFolder, - enabled: !isRunning, - onSubmit: (path) { - setState(() { - sourceFolder = path; - }); - }), - FolderRow("Destination folder", - initialValue: destinationFolder, - enabled: !isRunning, - onSubmit: (path) { - setState(() { - destinationFolder = path; - }); - }), - Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - const Padding( - padding: EdgeInsets.only(bottom: 3.0), - child: Text("Outputs"), - ), - SwitchRow("Overlay image", - onChange: (value) { - setState(() { - serializationOutput.overlay = value; - }); - }, - initialValue: serializationOutput.overlay, - ), - SwitchRow("CSV", - onChange: (value) { - setState(() { - serializationOutput.csv = value; - }); - }, - initialValue: serializationOutput.csv, - ), - SwitchRow("JSON", - onChange: (value) { - setState(() { - serializationOutput.json = value; - }); - }, - initialValue: serializationOutput.json, - ), - ], - ), - const DeviceSelector(), - Padding( - padding: const EdgeInsets.symmetric(horizontal: 0, vertical: 20), - child: BatchProgressButton( - isRunning: isRunning, - enabled: canProcess && inference.isReady, - onStart: () { - processFolder(context, inference); - }, - onStop: () => stop(), - ), - ), - - BatchProgress(progress), - ], - ), - ), - ); - } - ); - } -} - -class BatchProgressButton extends StatelessWidget { - final bool isRunning; - final bool enabled; - final void Function() onStart; - final void Function() onStop; - - const BatchProgressButton({required this.onStart, required this.onStop, this.isRunning = false, this.enabled = true, super.key}); - - @override - Widget build(BuildContext context) { - if (isRunning) { - return ElevatedButton( - onPressed: onStop, - child: const Text("Stop")); - } - - if (!enabled) { - return ElevatedButton( - style: const ButtonStyle( - backgroundColor: MaterialStatePropertyAll(intelGrayLight), - elevation: MaterialStatePropertyAll(0), - ), - onPressed: () {}, - child: const Text("Start")); - } - - return ElevatedButton( - onPressed: onStart, - child: const Text("Start")); - } - -} - -class BatchProgress extends StatelessWidget { - final Progress? progress; - const BatchProgress(this.progress, {super.key}); - - @override - Widget build(BuildContext context) { - return Builder( - builder: (context) { - if (progress == null) { - return const SizedBox.shrink(); - } - - return SizedBox( - width: 300, - child: Column( - children: [ - Padding( - padding: const EdgeInsets.only(bottom: 8.0), - child: Row( - mainAxisAlignment: MainAxisAlignment.spaceBetween, - children: [ - const Text("Inference progress"), - Text("${progress!.current}/${progress!.total}"), - ]), - ), - LinearProgressIndicator( - value: progress!.percentage(), - color: intelBlueLight, - backgroundColor: intelGrayLight, - minHeight: 8, - borderRadius: BorderRadius.circular(4), - ) - ] - ) - ); - } - ); - } -} - -class SwitchRow extends StatefulWidget { - final String label; - final void Function(bool) onChange; - final bool initialValue; - const SwitchRow(this.label, {required this.onChange, this.initialValue = false, super.key}); - - @override - State createState() => _SwitchRowState(); -} - -class _SwitchRowState extends State { - bool switchState = false; - - @override - void initState() { - // TODO: implement initState - super.initState(); - switchState = widget.initialValue; - } - - @override - Widget build(BuildContext context) { - return Padding( - padding: const EdgeInsets.only(bottom: 8.0), - child: Row( - children: [ - SizedBox( - width: 30, - height: 20, - child: FittedBox( - fit: BoxFit.fill, - child: Switch.adaptive( - value: switchState, - activeColor: intelGrayReallyDark, - activeTrackColor: lightGray, - onChanged: (value) { - setState(() { switchState = value; }); - widget.onChange(value); - }, - ), - ), - ), - Padding( - padding: const EdgeInsets.only(left: 8.0), - child: Text(widget.label), - ), - ], - ), - ); - } -} - -class FolderRow extends StatefulWidget { - final String label; - final bool enabled; - final String initialValue; - final void Function(String) onSubmit; - const FolderRow(this.label, - {required this.onSubmit, this.initialValue = "", this.enabled = true, super.key}); - - @override - State createState() => _FolderRowState(); -} - -class _FolderRowState extends State { - bool _showReleaseMessage = false; - bool pathSet = false; - bool isValid = false; - - final controller = TextEditingController(); - final platformContext = Context(style: Style.platform); - - @override - void initState() { - super.initState(); - controller.text = widget.initialValue; - isValid = pathIsValid(controller.text); - } - - void showReleaseMessage() { - setState(() => _showReleaseMessage = true); - } - - void hideReleaseMessage() { - setState(() => _showReleaseMessage = false); - } - - void handleDrop(DropDoneDetails details) { - if (details.files.isNotEmpty) { - setPath(details.files[0].path); - } - } - - void setPath(String path) { - if (pathIsValid(path)) { - controller.text = path; - widget.onSubmit(path); - setState(() { - pathSet = true; - isValid = true; - }); - } else { - //TODO show user error that a folder must be selected - } - } - - @override - void dispose() { - // TODO: implement dispose - super.dispose(); - controller.dispose(); - } - - @override - Widget build(BuildContext context) { - return Padding( - padding: const EdgeInsets.symmetric(vertical: 8.0), - child: DropTarget( - onDragDone: (details) => handleDrop(details), - onDragExited: (_) => hideReleaseMessage(), - onDragEntered: (_) => showReleaseMessage(), - child: Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - Padding( - padding: const EdgeInsets.only(bottom: 3.0), - child: Text(widget.label), - ), - Row( - children: [ - SizedBox( - width: 300, - height: 30, - child: TextField( - onChanged: (path) { - setState(() { - isValid = pathIsValid(path); - //Submit regardless. Path can be invalid, but that's less confusing to the user. - widget.onSubmit(path); - pathSet = isValid; - }); - }, - enabled: widget.enabled, - controller: controller, - textAlign: TextAlign.start, - style: const TextStyle ( - color: textColor, - fontSize: 10, - ), - decoration: InputDecoration( - contentPadding: const EdgeInsets.only(left: 10, bottom: 0), - hintText: (_showReleaseMessage - ? "Release..." - : "Drop ${widget.label.toLowerCase()} in"), - focusedBorder: OutlineInputBorder( - borderSide: BorderSide( - color: (isValid ? Colors.green : Colors.red), - )), - ), - ), - ), - Padding( - padding: const EdgeInsets.symmetric(horizontal: 20.0), - child: FolderButton( - enabled: widget.enabled, - text: (pathSet ? "Change" : "Select"), - onSelect: setPath, - ), - ) - ], - ), - ], - ), - ), - ); - } -} - -class FolderButton extends StatelessWidget { - final String text; - final void Function(String) onSelect; - final bool enabled; - const FolderButton({required this.onSelect, required this.text, this.enabled = true, super.key}); - - void showUploadMenu() async { - final result = await FilePicker.platform.getDirectoryPath(); - if (result != null) { - onSelect(result.toString()); - } - } - - @override - Widget build(BuildContext context) { - if (!enabled) { - return ElevatedButton.icon( - icon: const Icon(Icons.folder), - onPressed: () {}, - style: const ButtonStyle( - elevation: MaterialStatePropertyAll(0), - backgroundColor: MaterialStatePropertyAll(intelGrayLight), - ), - label: Text(text), - ); - } - - return OutlinedButton.icon( - icon: const Icon(Icons.folder), - onPressed: () => showUploadMenu(), - label: Text(text), - ); - } -} diff --git a/lib/inference/camera_page.dart b/lib/inference/camera_page.dart deleted file mode 100644 index 0bd26cdc..00000000 --- a/lib/inference/camera_page.dart +++ /dev/null @@ -1,109 +0,0 @@ -import 'dart:async'; -import 'dart:typed_data'; -import 'dart:ui' as ui; -import 'dart:convert'; - -import 'package:flutter/material.dart'; -import 'package:inference/providers/image_inference_provider.dart'; -import 'package:provider/provider.dart'; - -Future createImage(Uint8List bytes) async { - final Completer completer = Completer(); - ui.decodeImageFromList(bytes, - (ui.Image img) { - completer.complete(img); - }, - ); - return completer.future; -} -class CameraPage extends StatefulWidget { - final ImageInferenceProvider inferenceProvider; - - const CameraPage(this.inferenceProvider, {super.key}); - @override - State createState() => _CameraPageState(); -} - - -class _CameraPageState extends State { - ui.Image? image; - - ImageInferenceProvider get inferenceProvider => widget.inferenceProvider; - - void openCamera() { - inferenceProvider.openCamera(0); - inferenceProvider.setListener((output) { - print(output); - createImage(base64Decode(output.overlay!)).then((frame) { - if (mounted) { - setState(() { - image = frame; - }); - } - }); - }); - } - - - @override - void initState() { - super.initState(); - openCamera(); - } - - @override - void dispose() { - inferenceProvider.closeCamera(); - super.dispose(); - } - - @override - Widget build(BuildContext context) { - return Expanded( - child: Padding( - padding: const EdgeInsets.all(8.0), - child: LayoutBuilder( - builder: (context, constraints) { - if (image == null) { - return Container(); - } - - - return SizedBox( - width: 200, - height: 200, - child: CustomPaint( - painter: ImagePainter(image!), - size: Size(100, 200), - child: Container(), - ) -, - ); - } - ), - ), - ); - } - -} -class ImagePainter extends CustomPainter { - final ui.Image image; - ImagePainter(this.image); - - @override - void paint(Canvas canvas, Size size) { - paintImage( - canvas: canvas, - rect: Rect.fromLTWH(0, 0, size.width, size.height), - image: image, - ); - //canvas.drawImage(image, Offset.zero, Paint()); - } - - @override - bool shouldRepaint(ImagePainter oldDelegate) { - return true; - } - @override - bool shouldRebuildSemantics(ImagePainter oldDelegate) => false; -} diff --git a/lib/inference/device_selector.dart b/lib/inference/device_selector.dart deleted file mode 100644 index 8ca8e53f..00000000 --- a/lib/inference/device_selector.dart +++ /dev/null @@ -1,40 +0,0 @@ -import 'package:flutter/material.dart'; -import 'package:inference/providers/preference_provider.dart'; -import 'package:inference/theme.dart'; -import 'package:provider/provider.dart'; - -class DeviceSelector extends StatelessWidget { - const DeviceSelector({super.key}); - - @override - Widget build(BuildContext context) { - return Row( - children: [ - const Text("Device: "), - Consumer(builder: (context, preferences, child) { - return DropdownButton( - onChanged: (value) { - preferences.device = value!; - }, - underline: Container( - height: 0, - ), - style: const TextStyle( - fontSize: 12.0, - ), - focusColor: intelGrayDark, - padding: const EdgeInsets.symmetric(horizontal: 10), - value: preferences.device, - items: PreferenceProvider.availableDevices.map>((value) { - return DropdownMenuItem( - value: value.id, - child: Text(value.name), - ); - }).toList() - ); - } - ), - ], - ); - } -} diff --git a/lib/inference/download_page.dart b/lib/inference/download_page.dart deleted file mode 100644 index c228c99e..00000000 --- a/lib/inference/download_page.dart +++ /dev/null @@ -1,124 +0,0 @@ -import 'dart:math'; - -import 'package:go_router/go_router.dart'; -import 'package:flutter/material.dart'; -import 'package:inference/header.dart'; -import 'package:inference/inference/device_selector.dart'; -import 'package:inference/inference/model_info.dart'; -import 'package:inference/project.dart'; -import 'package:inference/providers/download_provider.dart'; -import 'package:inference/providers/project_provider.dart'; -import 'package:inference/public_models.dart'; -import 'package:inference/theme.dart'; -import 'package:intl/intl.dart'; -import 'package:provider/provider.dart'; - -String formatBytes(int bytes) { - return "${NumberFormat("#,##0").format(bytes / pow(1024, 2))} MB"; -} - -class DownloadPage extends StatefulWidget { - final PublicProject project; - final Function() onDone; - const DownloadPage(this.project, {required this.onDone, super.key}); - - @override - State createState() => _DownloadPageState(); -} - -class _DownloadPageState extends State { - @override - void initState() { - super.initState(); - - startDownload(); - } - - void startDownload() async { - final downloadProvider = Provider.of(context, listen: false); - final projectProvider = Provider.of(context, listen: false); - - final files = await listDownloadFiles(widget.project); - - try { - await downloadProvider.queue(files, widget.project.modelInfo?.collection.token); - await getAdditionalModelInfo(widget.project); - projectProvider.completeLoading(widget.project); - widget.onDone(); - } catch(e) { - if (mounted) { - showDialog(context: context, builder: (BuildContext context) => AlertDialog( - title: Text('An error occured trying to download ${widget.project.name}'), - content: Text(e.toString()), - actions: [ - TextButton( - onPressed: () => context.go('/'), - child: const Text('Close'), - ), - ] - ), - - ); - } - } - } - - @override - Widget build(BuildContext context) { - return Scaffold( - appBar: const Header(true), - body: Padding( - padding: const EdgeInsets.only(left: 58, right: 58, bottom: 30), - child: Row( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - SizedBox( - width: 250, - child: ModelInfo(widget.project), - ), - Consumer(builder: (context, downloads, child) { - final stats = downloads.stats; - return Expanded( - child: Column( - children: [ - const DeviceSelector(), - Expanded( - child: Column( - mainAxisAlignment: MainAxisAlignment.center, - crossAxisAlignment: CrossAxisAlignment.center, - children: [ - CircularProgressIndicator( - backgroundColor: textColor, - value: stats.percentage - ), - SizedBox( - width: 140, - child: Padding( - padding: const EdgeInsets.only(top: 8.0), - child: Row( - mainAxisAlignment: MainAxisAlignment.spaceBetween, - children: [ - Text(formatBytes(stats.received), textAlign: TextAlign.end,), - const Text("/"), - Text(formatBytes(stats.total)) - ], - ), - ), - ), - const Padding( - padding: EdgeInsets.all(8.0), - child: Text("Downloading model weights"), - ) - ], - ), - ), - ], - ), - ); - }) - ], - ), - ), - ); - } -} diff --git a/lib/inference/image_inference_page.dart b/lib/inference/image_inference_page.dart deleted file mode 100644 index e9b035c5..00000000 --- a/lib/inference/image_inference_page.dart +++ /dev/null @@ -1,126 +0,0 @@ -import 'package:flutter/material.dart'; -import 'package:go_router/go_router.dart'; -import 'package:inference/header.dart'; -import 'package:inference/inference/batch_inference.dart'; -import 'package:inference/inference/live_inference.dart'; -import 'package:inference/inference/model_info.dart'; -import 'package:inference/project.dart'; -import 'package:inference/providers/image_inference_provider.dart'; -import 'package:inference/providers/project_provider.dart'; -import 'package:inference/providers/preference_provider.dart'; -import 'package:inference/utils/dialogs.dart'; -import 'package:provider/provider.dart'; - -class ImageInferencePage extends StatefulWidget { - final Project project; - const ImageInferencePage(this.project, {super.key}); - - @override - State createState() => _ImageInferencePageState(); -} - -class _ImageInferencePageState extends State - with TickerProviderStateMixin { - late TabController _tabController; - - - void onBack(BuildContext context) { - final inference = Provider.of(context, listen: false); - if (inference.isLocked) { - showDialog( - context: context, - builder: (BuildContext context) => AlertDialog( - title: const Text('Batch inference'), - content: const Text('Batch inference is running currently.'), - actions: [ - TextButton( - onPressed: () => Navigator.pop(context, 'Cancel'), - child: const Text('Cancel'), - ), - ], - ) - ); - } else { - context.go('/'); - } - } - - @override - void initState() { - super.initState(); - _tabController = TabController(length: 2, animationDuration: Duration.zero, vsync: this); - } - - @override - void dispose() { - _tabController.dispose(); - super.dispose(); - } - - @override - Widget build(BuildContext context) { - return ChangeNotifierProxyProvider( - lazy: false, - create: (_) { - final device = Provider.of(context, listen: false).device; - return ImageInferenceProvider(widget.project, device)..init().catchError(onExceptionDialog(context)); - }, - update: (_, preferences, imageInferenceProvider) { - if (imageInferenceProvider != null && imageInferenceProvider.device == preferences.device) { - return imageInferenceProvider; - } - return ImageInferenceProvider(widget.project, preferences.device)..init().catchError(onExceptionDialog(context)); - }, - child: Scaffold( - appBar: Header(true, onBack: (context) => onBack(context)), - body: Padding( - padding: const EdgeInsets.only(left: 58, right: 58, bottom: 30), - child: Consumer(builder: (context, projects, child) { - return Row( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - ModelInfo(widget.project, children: [ - PropertyItem( - name: "Task", - child: Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: widget.project.tasks.map((task) => PropertyValue(task.name)).toList() - ) - ), - ]), - ( - Expanded( - child: Padding( - padding: const EdgeInsets.only(right: 35, left: 35), - child: Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - TabBar( - isScrollable: true, - tabAlignment: TabAlignment.start, - controller: _tabController, - tabs: const [ - Tab(text: "Live inference"), - Tab(text: "Batch inference"), - ], - ), - Expanded( - child: TabBarView( - controller: _tabController, - children: [ - LiveInference(widget.project), - const BatchInference(), - ]), - ) - ], - ), - ), - ) - ) - ], - ); - }), - ), - )); - } -} diff --git a/lib/inference/inference_page.dart b/lib/inference/inference_page.dart deleted file mode 100644 index a9ef3970..00000000 --- a/lib/inference/inference_page.dart +++ /dev/null @@ -1,40 +0,0 @@ -import 'package:flutter/material.dart'; -import 'package:inference/inference/download_page.dart'; -import 'package:inference/inference/image_inference_page.dart'; -import 'package:inference/inference/text_inference_page.dart'; -import 'package:inference/project.dart'; -import 'package:inference/providers/download_provider.dart'; -import 'package:provider/provider.dart'; - -class InferencePage extends StatefulWidget { - final Project project; - const InferencePage(this.project, {super.key}); - - @override - State createState() => _InferencePageState(); -} - -class _InferencePageState extends State { - @override - Widget build(BuildContext context) { - if (widget.project.isDownloaded) { - switch(widget.project.type){ - case ProjectType.image: - return ImageInferencePage(widget.project); - case ProjectType.text: - return TextInferencePage(widget.project); - case ProjectType.textToImage: - return Container(); - case ProjectType.speech: - return Container(); - } - } else { - return ChangeNotifierProvider( - create: (_) => DownloadProvider(), - child: DownloadPage(widget.project as PublicProject, - onDone: () => setState(() {}), //trigger rerender. - ) - ); - } - } -} diff --git a/lib/inference/live_inference.dart b/lib/inference/live_inference.dart deleted file mode 100644 index 0b464165..00000000 --- a/lib/inference/live_inference.dart +++ /dev/null @@ -1,232 +0,0 @@ -import 'dart:async'; -import 'dart:convert'; -import 'dart:io'; -import 'dart:typed_data'; -import 'dart:ui' as ui; - -import 'package:file_picker/file_picker.dart'; -import 'package:flutter/material.dart'; -import 'package:inference/inference/camera_page.dart'; -import 'package:inference/canvas/canvas.dart'; -import 'package:inference/inference/device_selector.dart'; -import 'package:inference/interop/image_inference.dart'; -import 'package:inference/interop/openvino_bindings.dart'; -import 'package:inference/project.dart'; -import 'package:inference/providers/image_inference_provider.dart'; -import 'package:inference/theme.dart'; -import 'package:inference/utils/drop_area.dart'; -import 'package:provider/provider.dart'; - - -Future createImage(Uint8List bytes) async { - return await decodeImageFromList(bytes); -} - -class LiveInference extends StatefulWidget { - final Project project; - const LiveInference(this.project, {super.key}); - - @override - State createState() => _LiveInferenceState(); -} - -enum MenuButtons { camera, sample, upload } - -class _LiveInferenceState extends State { - - bool loading = false; - bool cameraMode = false; - ImageInferenceResult? inferenceResult; - ui.Image? image; - - void showUploadMenu() async { - FilePickerResult? result = await FilePicker.platform.pickFiles(type: FileType.image); - - if (result != null) { - uploadFile(result.files.single.path!); - } - } - - void uploadFile(String path) async { - Uint8List imageData = File(path).readAsBytesSync(); - setState(() { - inferenceResult = null; - loading = true; - }); - - final inferenceProvider = Provider.of(context, listen: false); - - inferenceProvider.loaded.future.then((_) async{ - final output = await inferenceProvider.infer(imageData, SerializationOutput(json: true)); - final uiImage = await decodeImageFromList(imageData); - setState(() { - loading = false; - inferenceResult = output; - image = uiImage; - }); - }); - - } - - void handleMenu(MenuButtons option) { - switch(option) { - case MenuButtons.camera: - setState(() { - cameraMode = true; - }); - break; - case MenuButtons.sample: - uploadFile(widget.project.samplePath()); - break; - case MenuButtons.upload: - showUploadMenu(); - break; - } - } - - @override - Widget build(BuildContext context) { - return Consumer( - builder: (context, inference, child) { - final isLoading = loading || inference.inference == null; - return Column( - mainAxisAlignment: MainAxisAlignment.center, - crossAxisAlignment: CrossAxisAlignment.stretch, - children: [ - Padding( - padding: const EdgeInsets.symmetric(vertical: 8.0), - child: Row( - mainAxisAlignment: MainAxisAlignment.spaceBetween, - children: [ - const DeviceSelector(), - PopupMenuButton( - onSelected: (val) => handleMenu(val), - child: ClipRRect( - borderRadius: BorderRadius.circular(4.0), - child: Container( - decoration: BoxDecoration( - border: Border.all(color: textColor), - borderRadius: BorderRadius.circular(4.0), - color: intelGrayReallyDark, - //color: intelGrayLight, - ), - width: 168, - child: Padding( - padding: const EdgeInsets.only(top: 3, bottom: 3, left: 10, right: 5), - child: Row( - mainAxisAlignment: MainAxisAlignment.spaceBetween, - children: [ - const Text("Upload image"), - const Icon( - Icons.expand_more, - color: Colors.white, - ), - ], - ), - ) - ), - ), - elevation: 0, - offset: const Offset(0, 35), - shape: RoundedRectangleBorder ( - borderRadius: BorderRadius.circular(4), - side: const BorderSide( - color: textColor, - width: 1, - ) - ), - color: intelGrayReallyDark, - itemBuilder: (BuildContext context) { - if (widget.project.hasSample) { - return >[ - const PopupMenuItem( - height: 20, - value: MenuButtons.sample, - child: MenuButton('Sample image'), - ), - const PopupMenuItem( - height: 20, - value: MenuButtons.upload, - child: MenuButton('Choose an image file'), - ), - //const PopupMenuItem( - // height: 20, - // value: MenuButtons.camera, - // child: MenuButton('Camera'), - //), - ]; - } - return >[ - const PopupMenuItem( - height: 20, - value: MenuButtons.upload, - child: MenuButton('Choose an image file'), - ), - ]; - }, - ), - ], - ), - ), - (isLoading - ? Expanded( - child: Center( - child: Image.asset('images/intel-loading.gif', width: 100) - ) - ) - : Builder( - builder: (context) { - if (cameraMode) { - return CameraPage(inference); - } - return DropArea( - type: "image", - extensions: const ["jpg, jpeg, bmp, png, tif, tiff"], - onUpload: (String path) => uploadFile(path), - showChild: inferenceResult != null, - child: Padding( - padding: const EdgeInsets.all(8.0), - child: Builder( - builder: (context) { - if (inferenceResult == null) { - return Container(); - } - return Canvas( - image: image!, - annotations: inferenceResult!.parseAnnotations(), - labelDefinitions: widget.project.labelDefinitions, - ); - } - ) - ) - ); - } - ) - ) - ], - ); - } - ); - } -} - -class MenuButton extends StatelessWidget { - final String name; - const MenuButton(this.name, { - super.key, - }); - - @override - Widget build(BuildContext context) { - return Padding( - padding: const EdgeInsets.symmetric(vertical: 8.0), - child: Text(name, - style: const TextStyle( - fontSize: 10, - ), - ), - ); - - } -} - diff --git a/lib/inference/model_info.dart b/lib/inference/model_info.dart deleted file mode 100644 index 3838834f..00000000 --- a/lib/inference/model_info.dart +++ /dev/null @@ -1,114 +0,0 @@ -import 'package:collection/collection.dart'; -import 'package:flutter/material.dart'; -import 'package:inference/project.dart'; -import 'package:inference/providers/project_provider.dart'; -import 'package:inference/theme.dart'; -import 'package:provider/provider.dart'; - -class ModelInfo extends StatelessWidget { - final List children; - final Project project; - const ModelInfo(this.project, {this.children = const [], super.key}); - - // This widget is the root of your application. - @override - Widget build(BuildContext context) { - return Consumer(builder: (context, projects, child) { - return Padding( - padding: const EdgeInsets.all(8.0), - child: Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - PropertyItem( - name: "Model name", - child: PropertyValue(project.name) - ), - PropertyItem( - name: "Architecture", - enabled: project.tasks.firstWhereOrNull((t) => t.architecture.isNotEmpty) != null, - child: Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: project.tasks.map((task) => PropertyValue(task.architecture)).toList() - ) - ), - PropertyItem( - name: "Optimization", - enabled: project.tasks.firstWhereOrNull((t) => t.optimization.isNotEmpty) != null, - child: Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: project.tasks.map((task) => PropertyValue(task.optimization)).toList() - ) - ), - ...children - ], - ), - ); - }); - } -} - -class PropertyItem extends StatelessWidget { - final String name; - final Widget child; - final bool enabled; - const PropertyItem({required this.name, required this.child, this.enabled = true, super.key}); - - @override - Widget build(BuildContext context) { - if (!enabled) { - return Container(); - } - return Padding( - padding: const EdgeInsets.symmetric(vertical: 12, horizontal: 8.0), - child: Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - Padding( - padding: const EdgeInsets.only(bottom: 8.0), - child: PropertyHeader(name), - ), - child - ] - ), - ); - } -} - -class PropertyHeader extends StatelessWidget { - final String text; - const PropertyHeader(this.text, {super.key}); - - @override - Widget build(BuildContext context) { - return Text(text, style: const TextStyle( - color: Colors.white, - fontSize: 19, - )); - } - -} - -class PropertyValue extends StatelessWidget { - final String text; - const PropertyValue(this.text, {super.key}); - - @override - Widget build(BuildContext context) { - return Padding( - padding: const EdgeInsets.only(bottom: 8.0, right: 8.0), - child: Container( - decoration: BoxDecoration( - border: Border.all(color: textColor), - borderRadius: BorderRadius.circular(4.0), - color: intelGrayReallyDark, - ), - child: Padding( - padding: const EdgeInsets.symmetric(vertical: 3, horizontal: 6), - child: Text(text, style: const TextStyle( - fontSize: 12, - )), - ) - ), - ); - } -} diff --git a/lib/inference/text/metric_widgets.dart b/lib/inference/text/metric_widgets.dart deleted file mode 100644 index 830be84f..00000000 --- a/lib/inference/text/metric_widgets.dart +++ /dev/null @@ -1,147 +0,0 @@ -import 'package:flutter/material.dart'; -import 'package:inference/interop/openvino_bindings.dart'; -import 'package:inference/theme.dart'; -import 'package:intl/intl.dart'; - -class CirclePropRow extends StatelessWidget { - final Metrics metrics; - const CirclePropRow({super.key, required this.metrics}); - - @override - Widget build(BuildContext context) { - Locale locale = Localizations.localeOf(context); - final nf = NumberFormat.decimalPatternDigits( - locale: locale.languageCode, decimalDigits: 0); - - return Row( - mainAxisAlignment: MainAxisAlignment.spaceBetween, - children: [ - CircleProp( - header: "Time to first token (ttft)", - value: nf.format(metrics.ttft), - unit: "ms", - ), - CircleProp( - header: "Time per output token (tpot)", - value: nf.format(metrics.tpot), - unit: "ms", - ), - CircleProp( - header: "Generate total duration", - value: nf.format(metrics.generate_time), - unit: "ms", - ) - ], - ); - } -} - -class CircleProp extends StatelessWidget { - final String header; - final String value; - final String unit; - - const CircleProp({super.key, required this.header, required this.value, required this.unit}); - - @override - Widget build(BuildContext context) { - return Padding( - padding: const EdgeInsets.all(16.0), - child: Container( - width: 250.0, - height: 250.0, - decoration: BoxDecoration( - shape: BoxShape.circle, - color: intelGrayDark, - border: Border.all( - color: intelBlueDark, - width: 10, - ), - ), - child: Column( - mainAxisAlignment: MainAxisAlignment.spaceBetween, - children: [ - Padding( - padding: const EdgeInsets.only(top: 36.0), - child: SizedBox( - width: 170, - child: Text(header, - style: const TextStyle( - fontSize: 20, - ), - textAlign: TextAlign.center, - ), - ), - ), - Padding( - padding: const EdgeInsets.only(bottom: 46.0), - child: Row( - textBaseline: TextBaseline.alphabetic, - crossAxisAlignment: CrossAxisAlignment.baseline, - mainAxisAlignment: MainAxisAlignment.center, - children: [ - Text(value, - style: const TextStyle( - fontSize: 30, - ), - ), - Padding( - padding: const EdgeInsets.only(left: 8.0), - child: Text(unit, - style: const TextStyle( - fontSize: 12, - ), - ), - ), - ], - ), - ), - ], - ) - - ), - ); - } -} - -class Statistic extends StatelessWidget { - const Statistic({ - super.key, - required this.header, - required this.value, - required this.unit, - }); - - final String header; - final String value; - final String unit; - - @override - Widget build(BuildContext context) { - return Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - Text(header), - Row( - textBaseline: TextBaseline.alphabetic, - crossAxisAlignment: CrossAxisAlignment.baseline, - children: [ - Text(value, - style: const TextStyle( - fontSize: 30, - ) - ), - Padding( - padding: const EdgeInsets.only(left: 8.0), - child: Text(unit, - style: const TextStyle( - fontSize: 12, - ) - ), - ), - ], - ), - ] - ); - } -} diff --git a/lib/inference/text/performance_metrics.dart b/lib/inference/text/performance_metrics.dart deleted file mode 100644 index 6fde59e2..00000000 --- a/lib/inference/text/performance_metrics.dart +++ /dev/null @@ -1,97 +0,0 @@ -import 'package:flutter/material.dart'; -import 'package:inference/inference/text/metric_widgets.dart'; -import 'package:inference/providers/text_inference_provider.dart'; -import 'package:inference/theme.dart'; -import 'package:inference/utils/dialogs.dart'; -import 'package:intl/intl.dart'; -import 'package:provider/provider.dart'; - -class PerformanceMetricsPage extends StatefulWidget { - const PerformanceMetricsPage({super.key}); - - @override - State createState() => _PerformanceMetricsPageState(); -} - -class _PerformanceMetricsPageState extends State { - - @override - void initState() { - super.initState(); - final provider = Provider.of(context, listen: false); - if (provider.metrics == null) { - provider.loaded.future.then((_) { - provider.message("What is the purpose of OpenVINO?").catchError(onExceptionDialog(context)); - }); - } - } - - @override - Widget build(BuildContext context) { - return Consumer(builder: (context, inference, child) { - if (inference.metrics == null) { - return Center( - child: Column( - mainAxisAlignment: MainAxisAlignment.center, - children: [ - Image.asset('images/intel-loading.gif', width: 100), - const Text("Running benchmark prompt...") - ], - ) - ); - } - - Locale locale = Localizations.localeOf(context); - final nf = NumberFormat.decimalPatternDigits( - locale: locale.languageCode, decimalDigits: 0); - - final metrics = inference.metrics!; - - return Container( - decoration: BoxDecoration( - shape: BoxShape.rectangle, - borderRadius: const BorderRadius.all(Radius.circular(8)), - color: intelGray, - ), - child: Align( - alignment: Alignment.topLeft, - child: ConstrainedBox( - constraints: const BoxConstraints(maxWidth: 1000), - child: Padding( - padding: const EdgeInsets.all(30.0), - child: Column( - mainAxisAlignment: MainAxisAlignment.start, - crossAxisAlignment: CrossAxisAlignment.stretch, - children: [ - CirclePropRow(metrics: metrics), - Expanded( - child: Padding( - padding: const EdgeInsets.only(top: 20.0), - child: GridView.count( - physics: const NeverScrollableScrollPhysics(), - childAspectRatio: 2.0, - padding: const EdgeInsets.only(right: 20.0), - crossAxisSpacing: 4.0, - crossAxisCount: 3, - children: [ - Statistic(header: "Tokenization duration", value: nf.format(metrics.tokenization_time), unit: "ms"), - Statistic(header: "Detokenization duration", value: nf.format(metrics.detokenization_time), unit: "ms"), - Statistic(header: "Generated tokens", value: nf.format(metrics.number_of_generated_tokens), unit: ""), - Statistic(header: "Load time", value: nf.format(metrics.load_time), unit: "ms"), - Statistic(header: "Tokens in the input prompt", value: nf.format(metrics.number_of_input_tokens), unit: ""), - Statistic(header: "Throughput", value: nf.format(metrics.throughput), unit: "tokens/sec"), - ] - ), - ), - ) - ], - ), - ), - ), - ), - ); - }); - } -} - - diff --git a/lib/inference/text/playground.dart b/lib/inference/text/playground.dart deleted file mode 100644 index 6c64dec8..00000000 --- a/lib/inference/text/playground.dart +++ /dev/null @@ -1,426 +0,0 @@ - -import 'package:flutter/material.dart'; -import 'package:flutter/services.dart'; -import 'package:flutter_svg/svg.dart'; -import 'package:inference/config.dart'; -import 'package:inference/hint.dart'; -import 'package:inference/inference/device_selector.dart'; -import 'package:inference/inference/text/metric_widgets.dart'; -import 'package:inference/interop/openvino_bindings.dart'; -import 'package:inference/providers/text_inference_provider.dart'; -import 'package:inference/theme.dart'; -import 'package:inference/utils/dialogs.dart'; -import 'package:provider/provider.dart'; - -class Playground extends StatefulWidget { - const Playground({super.key}); - - @override - State createState() => _PlaygroundState(); -} - -class _PlaygroundState extends State { - final _controller = TextEditingController(); - final _scrollController = ScrollController(); - bool attachedToBottom = true; - - void jumpToBottom({ offset = 0 }) { - if (_scrollController.hasClients) { - _scrollController.jumpTo(_scrollController.position.maxScrollExtent + offset); - } - } - - void message(String message) async { - if (message.isEmpty) { - return; - } - final llm = provider(); - if (!llm.initialized) { - return; - } - - if (llm.response != null) { - return; - } - _controller.text = ""; - jumpToBottom(offset: 110); //move to bottom including both - llm.message(message).catchError(onExceptionDialog(context)); - } - - TextInferenceProvider provider() => Provider.of(context, listen: false); - - @override - void initState() { - super.initState(); - _scrollController.addListener(() { - setState(() { - attachedToBottom = _scrollController.position.pixels + 0.001 >= _scrollController.position.maxScrollExtent; - }); - - }); - } - - @override - void dispose() { - super.dispose(); - _controller.dispose(); - _scrollController.dispose(); - } - - - @override - Widget build(BuildContext context) { - return Consumer(builder: (context, inference, child) { - WidgetsBinding.instance.addPostFrameCallback((_) { - if (attachedToBottom) { - jumpToBottom(); - } - }); - - return Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - Padding( - padding: const EdgeInsets.only(left: 8), - child: Row( - mainAxisAlignment: MainAxisAlignment.spaceBetween, - children: const [ - DeviceSelector(), - Hint(hint: HintsEnum.intelCoreLLMPerformanceSuggestion), - ] - ), - ), - Builder( - builder: (context) { - if (!inference.initialized){ - return Expanded( - child: Center( - child: Column( - mainAxisAlignment: MainAxisAlignment.center, - children: [ - Image.asset('images/intel-loading.gif', width: 100), - const Text("Loading model...") - ], - ) - ), - ); - } - return Expanded( - child: Container( - decoration: BoxDecoration( - shape: BoxShape.rectangle, - borderRadius: const BorderRadius.all(Radius.circular(8)), - color: intelGray, - ), - child: Column( - children: [ - Expanded( - child: Builder(builder: (context) { - if (inference.messages.isEmpty) { - return Center( - child: Text("Type a message to ${inference.project?.name ?? "assistant"}")); - } - return Stack( - alignment: Alignment.bottomCenter, - children: [ - SingleChildScrollView( - controller: _scrollController, - child: Padding( - padding: const EdgeInsets.all(20), - child: Column( - //mainAxisAlignment: MainAxisAlignment.start, - crossAxisAlignment: CrossAxisAlignment.stretch, - children: inference.messages.map((message) { - switch (message.speaker) { - case Speaker.system: - throw UnimplementedError(); - case Speaker.user: - return UserMessage(message); - case Speaker.assistant: - return AssistantMessage(message, inference.project!.name); - } - }).toList()), - ), - ), - Positioned( - bottom: 10, - child: Builder( - builder: (context) { - if (attachedToBottom) { - return Container(); - } - return Center( - child: Padding( - padding: const EdgeInsets.only(top: 2.0), - child: SizedBox( - width: 200, - height: 20, - child: FloatingActionButton( - backgroundColor: intelGray, - child: const Text("Jump to bottom"), - onPressed: () { - jumpToBottom(); - setState(() { - attachedToBottom = true; - }); - } - ), - ), - ), - ); - } - ), - ), - - ], - ); - }), - ), - - SizedBox( - height: 30, - child: Builder( - builder: (context) { - if (inference.interimResponse == null){ - return Container(); - } - return Center( - child: OutlinedButton.icon( - onPressed: () => inference.forceStop(), - icon: const Icon(Icons.stop), - label: const Text("Stop responding") - ), - ); - } - ), - ), - Padding( - padding: const EdgeInsets.only(left: 45, right: 45, top: 10, bottom: 25), - child: SizedBox( - height: 40, - child: Row( - crossAxisAlignment: CrossAxisAlignment.end, - children: [ - Padding( - padding: const EdgeInsets.only(right: 8), - child: IconButton( - icon: SvgPicture.asset("images/clear.svg", - colorFilter: const ColorFilter.mode(textColor, BlendMode.srcIn), - width: 20, - ), - tooltip: "Clear chat", - onPressed: () => inference.reset(), - style: IconButton.styleFrom( - backgroundColor: intelGrayReallyDark, - shape: const RoundedRectangleBorder( - borderRadius: BorderRadius.all(Radius.circular(4)), - side: BorderSide( - color: intelGrayLight, - width: 2, - ) - ) - ) - ), - ), - Expanded( - child: TextField( - maxLines: null, - keyboardType: TextInputType.text, - decoration: InputDecoration( - hintText: "Ask me anything...", - suffixIcon: IconButton( - icon: Icon(Icons.send, color: (inference.interimResponse == null ? Colors.white : intelGray)), - onPressed: () => message(_controller.text), - ), - enabledBorder: const OutlineInputBorder( - borderRadius: BorderRadius.all(Radius.circular(4)), - borderSide: BorderSide( - color: intelGrayLight, - width: 2, - ) - ), - ), - style: const TextStyle( - fontSize: 14, - ), - controller: _controller, - onSubmitted: message, - ), - ), - ], - ), - ), - ), - ], - ), - ), - ); - } - ), - ], - ); - }); - } -} - -class UserMessage extends StatelessWidget { - final Message message; - const UserMessage(this.message, {super.key}); - - @override - Widget build(BuildContext context) { - return Padding( - padding: const EdgeInsets.only(bottom: 20), - child: Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - NameRowWidget(name: "You", icon: SvgPicture.asset("images/user.svg", - colorFilter: const ColorFilter.mode(textColor, BlendMode.srcIn), - width: 20, - ), - ), - MessageWidget(message: message.message), - ], - ), - ); - } -} - -class AssistantMessage extends StatelessWidget { - final Message message; - final String name; - const AssistantMessage(this.message, this.name, {super.key}); - - @override - Widget build(BuildContext context) { - return Padding( - padding: const EdgeInsets.only(bottom: 20), - child: Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - NameRowWidget( - name: name, - icon: SvgPicture.asset("images/network.svg", - colorFilter: const ColorFilter.mode(textColor, BlendMode.srcIn), - width: 20, - ), - ), - MessageWidget(message: message.message), - Padding( - padding: const EdgeInsets.only(left: 28, top: 5), - child: Builder( - builder: (context) { - if (message.metrics == null) { - return Container(); - } - return Row( - children: [ - IconButton.filled( - icon: SvgPicture.asset("images/copy.svg", - colorFilter: const ColorFilter.mode(textColor, BlendMode.srcIn), - width: 20, - ), - style: IconButton.styleFrom( - backgroundColor: intelGrayLight, - shape: const RoundedRectangleBorder( - borderRadius: BorderRadius.all(Radius.circular(4)), - ), - ), - padding: const EdgeInsets.all(4), - constraints: const BoxConstraints(), - tooltip: "Copy to clipboard", - onPressed: () { - Clipboard.setData(ClipboardData(text: message.message)); - }, - ), - Padding( - padding: const EdgeInsets.only(left: 8), - child: IconButton( - style: IconButton.styleFrom( - backgroundColor: intelGrayLight, - shape: const RoundedRectangleBorder( - borderRadius: BorderRadius.all(Radius.circular(4)), - ), - ), - padding: const EdgeInsets.all(4), - constraints: const BoxConstraints(), - icon: SvgPicture.asset("images/stats.svg", - colorFilter: const ColorFilter.mode(textColor, BlendMode.srcIn), - width: 20, - ), - tooltip: "Show stats", - onPressed: () { - showMetricsDialog(context, message.metrics!); - }, - ), - ), - ], - ); - } - ), - ), - ], - ), - ); - } -} - -void showMetricsDialog(BuildContext context, Metrics metrics) { - showDialog( - context: context, - builder: (BuildContext context) { - return AlertDialog( - content: CirclePropRow( - metrics: metrics - ) - ); - } - ); -} - -class NameRowWidget extends StatelessWidget { - final String name; - final Widget icon; - const NameRowWidget({super.key, required this.name, required this.icon}); - - @override - Widget build(BuildContext context) { - return Row( - children: [ - Container( - padding: const EdgeInsets.all(2), - decoration: BoxDecoration( - borderRadius: BorderRadius.circular(4.0), - color: intelBlueVibrant, - //color: intelGrayLight, - ), - child: icon - ), - Padding( - padding: const EdgeInsets.only(left: 10.0), - child: Text(name), - ) - ] - ); - } -} - -class MessageWidget extends StatelessWidget { - final String message; - const MessageWidget({super.key, required this.message}); - - @override - Widget build(BuildContext context) { - return Padding( - padding: const EdgeInsets.only(left: 34.0, top: 10, right: 26), - child: SelectableText( - message, - style: const TextStyle( - color: textColor, - fontSize: 12, - ), - ), - ); - } - -} diff --git a/lib/inference/text_inference_page.dart b/lib/inference/text_inference_page.dart deleted file mode 100644 index afe0e5cd..00000000 --- a/lib/inference/text_inference_page.dart +++ /dev/null @@ -1,165 +0,0 @@ -import 'package:flutter/material.dart'; -import 'package:inference/header.dart'; -import 'package:inference/inference/model_info.dart'; -import 'package:inference/inference/text/performance_metrics.dart'; -import 'package:inference/inference/text/playground.dart'; -import 'package:inference/project.dart'; -import 'package:inference/providers/preference_provider.dart'; -import 'package:inference/providers/text_inference_provider.dart'; -import 'package:inference/theme.dart'; -import 'package:inference/utils/dialogs.dart'; -import 'package:intl/intl.dart'; -import 'package:provider/provider.dart'; - -class TextInferencePage extends StatefulWidget { - final Project project; - const TextInferencePage(this.project, {super.key}); - - @override - State createState() => _TextInferencePageState(); -} - -class _TextInferencePageState extends State with TickerProviderStateMixin { - - late TabController _tabController; - - @override - void initState() { - super.initState(); - _tabController = TabController(length: 2, animationDuration: Duration.zero, vsync: this); - } - - @override - void dispose() { - _tabController.dispose(); - super.dispose(); - } - - @override - Widget build(BuildContext context) { - Locale locale = Localizations.localeOf(context); - - return ChangeNotifierProxyProvider( - create: (_) { - return TextInferenceProvider(widget.project, null); - }, - update: (_, preferences, textInferenceProvider) { - final init = textInferenceProvider == null || - !textInferenceProvider.sameProps(widget.project, preferences.device); - if (init) { - final textInferenceProvider = TextInferenceProvider(widget.project, preferences.device); - textInferenceProvider.loadModel().catchError(onExceptionDialog(context)); - return textInferenceProvider; - } - if (!textInferenceProvider.sameProps(widget.project, preferences.device)) { - return TextInferenceProvider(widget.project, preferences.device); - } - return textInferenceProvider; - }, - child: Scaffold( - appBar: const Header(true), - body: Padding( - padding: const EdgeInsets.only(left: 58, right: 58, bottom: 30), - child: Row( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - Consumer( - builder: (context, inference, child) { - final nf = NumberFormat.decimalPatternDigits( - locale: locale.languageCode, decimalDigits: 2); - - return SizedBox( - width: 250, - child: ModelInfo( - widget.project, - children: [ - PropertyItem( - name: "Task", - child: PropertyValue(inference.task), - ), - Padding( - padding: const EdgeInsets.only(left: 12, top: 12, right: 20.0), - child: Column( - children: [ - Row( - mainAxisAlignment: MainAxisAlignment.spaceBetween, - children: [ - const Text("Temperature"), - Text(nf.format(inference.temperature)) - ] - ), - Slider( - value: inference.temperature, - max: 2.0, - onChanged: (double value) { - inference.temperature = value; - }, - - ), - ], - ), - ), - Padding( - padding: const EdgeInsets.only(left: 12, top: 12, right: 20.0), - child: Column( - children: [ - Row( - mainAxisAlignment: MainAxisAlignment.spaceBetween, - children: [ - const Text("Top P"), - Text(nf.format(inference.topP)) - ] - ), - Slider( - value: inference.topP, - max: 1.0, - onChanged: (double value) { - inference.topP = value; - }, - - ), - ], - ), - ), - ] - ), - ); - }), - Expanded( - child: Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - TabBar( - isScrollable: true, - tabAlignment: TabAlignment.start, - controller: _tabController, - tabs: const [ - Tab(text: "Playground"), - Tab(text: "Performance metrics"), - //Tab(text: "Deploy"), - ] - ), - Expanded( - child: Padding( - padding: const EdgeInsets.only(top: 15.0), - child: TabBarView( - controller: _tabController, - children: [ - const Playground(), - const PerformanceMetricsPage(), - //Container(), - ] - ), - ) - ), - ], - ), - ) - ], - ), - ), - ), - ); - } -} - diff --git a/lib/projects/project_item.dart b/lib/projects/project_item.dart deleted file mode 100644 index f735fa6b..00000000 --- a/lib/projects/project_item.dart +++ /dev/null @@ -1,244 +0,0 @@ -import 'package:flutter/material.dart'; -import 'package:go_router/go_router.dart'; -import 'package:inference/project.dart'; -import 'package:inference/providers/download_provider.dart'; -import 'package:inference/providers/project_provider.dart'; -import 'package:inference/public_models.dart'; -import 'package:inference/theme.dart'; -import 'package:inference/color.dart'; -import 'package:intl/intl.dart'; -import 'package:provider/provider.dart'; - -enum MenuButtons { delete } - -class ProjectItem extends StatefulWidget { - final Project project; - const ProjectItem(this.project, {super.key}); - - @override - State createState() => _ProjectItemState(); -} - -class _ProjectItemState extends State { - final DateFormat formatter = DateFormat('dd MMMM y | h:mm a'); - bool hovered = false; - - void onHover(bool? val) { - setState(() { - hovered = val ?? false; - }); - } - - Color backgroundColor(BuildContext context) { - final scheme = Theme.of(context).colorScheme; - if (hovered) { - return scheme.onSurfaceVariant; - } else { - return scheme.surfaceContainer; - } - } - - void onTap(Project project) async { - if (project.isPublic || project.loaded.isCompleted) { - goToProject(project); - } - } - - void goToProject(Project project) { - // uncomment this to check the other downloading style... - //if (project.isDownloaded) { // or something. - context.go('/inference', extra: project); - //} else { - // createDirectory(project as PublicProject); - // writeProjectJson(project); - - // Provider.of(context, listen: false).queue(llmDownloadFiles(project)).then((_) { - // getAdditionalModelInfo(project).then((_) { - // Provider.of(context, listen: false).completeLoading(project); - // }); - // }); - //} - } - - @override - Widget build(BuildContext context) { - return Padding( - padding: const EdgeInsets.only(top: 8, bottom: 8), - child: ClipRRect( - borderRadius: BorderRadius.circular(8.0), - child: Consumer(builder: (context, projects, child) { - return InkWell( - borderRadius: BorderRadius.circular(8.0), - onHover: (val) => onHover(val), - onTap: () => onTap(widget.project), - child: Opacity( - opacity: (widget.project is PublicProject || - widget.project.loaded.isCompleted - ? 1.0 - : 0.5), - child: Stack( - alignment: Alignment.bottomLeft, - children: [ - ConstrainedBox( - constraints: const BoxConstraints.expand(height: 136), - child: Row( - mainAxisAlignment: MainAxisAlignment.start, - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - Container( - width: 288, - decoration: BoxDecoration( - image: DecorationImage( - image: widget.project.thumbnailImage(), - fit: BoxFit.cover), - ), - ), - Expanded( - child: Container( - color: backgroundColor(context), - child: Padding( - padding: const EdgeInsets.all(16.0), - child: Column( - mainAxisAlignment: MainAxisAlignment.start, - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - Row( - mainAxisAlignment: MainAxisAlignment.spaceBetween, - children: [ - Row( - children: [ - Text(widget.project.name, - style: const TextStyle( - fontWeight: FontWeight.bold, - fontSize: 16, - )), - Text(" @ ${widget.project.taskName()}") - ], - ), - (widget.project.isDownloaded - ? Row( - children: [ - (widget.project is GetiProject && widget.project.type != ProjectType.text - ? Row(children: (widget.project as GetiProject) - .scores() - .map((p) { - if (p == null) { - return Container(); - } - return ScoreItem(p); - }).toList()) - : Container()), - PopupMenuButton( - iconColor: textColor, - iconSize: 16, - onSelected: (MenuButtons _) => projects.removeProject(widget .project), - color: intelGrayDark, - elevation: 1, - itemBuilder: (BuildContext context) { - return >[ - const PopupMenuItem( - value: MenuButtons .delete, - child: Text('Delete'), - ), - ]; - }, - ) - ], - ) - : Container()) - ], - ), - Text( - "Created: ${formatter.format(DateTime.parse(widget.project.creationTime))}"), - const Divider(color: intelGrayLight), - Row( - children: (widget.project.labels().length > 4 - ? [ - ...(widget.project - .labels() - .sublist(0, 4)) - .map((label) => - LabelItem(label)), - const Text(" ...and more") - ] - : widget.project - .labels() - .map((label) => - LabelItem(label)) - .toList())) - ], - ), - ), - ), - ), - ], - ), - ), - ], - ), - ), - ); - })), - ); - } -} - -class ScoreItem extends StatelessWidget { - final Score score; - const ScoreItem(this.score, {super.key}); - - @override - Widget build(BuildContext context) { - Locale locale = Localizations.localeOf(context); - return SizedBox( - width: 40, - child: Column(children: [ - Text(NumberFormat.percentPattern(locale.languageCode) - .format(score.score)), - Padding( - padding: const EdgeInsets.only(left: 7.0, right: 7.0), - child: Stack( - children: [ - const Divider( - color: Colors.white, - height: 2, - thickness: 2, - ), - SizedBox( - width: (score.score * (26)), - child: Divider( - color: getScoreColor(score.score), - height: 2, - thickness: 2)), - ], - ), - ) - ]), - ); - } -} - -class LabelItem extends StatelessWidget { - final Label label; - const LabelItem(this.label, {super.key}); - - @override - Widget build(BuildContext context) { - return Row( - mainAxisSize: MainAxisSize.min, - children: [ - Padding( - padding: const EdgeInsets.only(left: 8.0, right: 8.0), - child: ClipRRect( - borderRadius: BorderRadius.circular(2), - child: Container( - color: HexColor.fromHex(label.color.substring(0, 7)), - width: 10, - height: 10, - )), - ), - Text(label.name), - ], - ); - } -} diff --git a/lib/projects/projects_page.dart b/lib/projects/projects_page.dart deleted file mode 100644 index ac6daf5d..00000000 --- a/lib/projects/projects_page.dart +++ /dev/null @@ -1,299 +0,0 @@ -import 'dart:io'; - -import 'package:file_picker/file_picker.dart'; -import 'package:go_router/go_router.dart'; -import 'package:flutter/foundation.dart'; -import 'package:flutter/material.dart'; -import 'package:flutter_svg/svg.dart'; -import 'package:inference/config.dart'; -import 'package:inference/header.dart'; -import 'package:inference/importers/importer.dart'; -import 'package:inference/project.dart'; -import 'package:inference/providers/project_filter_provider.dart'; -import 'package:inference/providers/project_provider.dart'; -import 'package:inference/searchbar.dart'; -import 'package:inference/projects/project_item.dart'; -import 'package:inference/theme.dart'; -import 'package:provider/provider.dart'; -import 'package:shimmer/shimmer.dart'; - -class ProjectsPage extends StatefulWidget { - const ProjectsPage({super.key}); - - @override - State createState() => _ProjectPageState(); -} - -class _ProjectPageState extends State with TickerProviderStateMixin { - void showErrorDialog(String error, String stack) { - showDialog(context: context, builder: (BuildContext context) => AlertDialog( - title: Text('A critical error occured: $error'), - content: Text(stack), - actions: [ - TextButton( - onPressed: () => Navigator.of(context).pop(), - child: const Text('Close'), - ), - ] - ) - ); - } - - @override - void initState() { - super.initState(); - - FlutterError.onError = (details) { - FlutterError.presentError(details); - //showErrorDialog(details.exception.toString(), details.stack.toString()); - }; - //PlatformDispatcher.instance.onError = (error, stack) { - // showErrorDialog(error.toString(), stack.toString()); - // return true; - //}; - } - - void addProject(BuildContext context) async { - - final result = await FilePicker.platform.pickFiles(allowMultiple: true); - - if (result != null) { - for (final file in result.files) { - final importer = selectMatchingImporter(file.path!); - if (importer == null) { - print("unable to process file"); - return; - } - if (context.mounted) { - if (!await importer.askUser(context)) { - print("cancelling due to user input"); - return; - } - } - importer.generateProject().then((project) async { - await importer.setupFiles(); - project.loaded.future.then((_) { - if (context.mounted) { - Provider.of(context, listen: false) - .completeLoading(project); - } - }); - if (context.mounted) { - final projectsProvider = Provider.of(context, listen: false); - projectsProvider.addProject(project); - } - }); - } - } else { - // User canceled the picker - } - } - - @override - Widget build(BuildContext context) { - return ChangeNotifierProvider( - create: (context) => ProjectFilterProvider(), - child: Scaffold( - appBar: const Header(false), - body: Consumer(builder: (context, projects, child) { - return Padding( - padding: const EdgeInsets.only(bottom: 10), - child: Consumer(builder: (context, filter, child) { - final selectedProjects = filter.applyFilter(projects.projects); - return Column( - mainAxisAlignment: MainAxisAlignment.start, - crossAxisAlignment: CrossAxisAlignment.stretch, - children: [ - Padding( - padding: const EdgeInsets.symmetric(horizontal: 60, vertical: 8), - child: Row( - mainAxisAlignment: MainAxisAlignment.spaceBetween, - children: [ - GetiSearchBar( - onChange: (value) => filter.name = value, - ), - (Config.geti - ? ElevatedButton( - onPressed: () => addProject(context), - child: const Text("Import Model") - ) - : ImportModelButton( - onHuggingFace: () => context.go("/import"), - onLocal: () => addProject(context), - ) - ) - ]), - ), - Expanded( - child: GetiProjectsList(selectedProjects.toList()) - ) - ] - ); - }), - ); - })), - ); - } -} - -enum MenuButtons { huggingface, local } - -class ImportModelButton extends StatelessWidget { - final Function() onHuggingFace; - final Function() onLocal; - const ImportModelButton({required this.onHuggingFace, required this.onLocal, super.key}); - - @override - Widget build(BuildContext context) { - return PopupMenuButton( - onSelected: (MenuButtons btn) { - switch(btn.name) { - case "huggingface": onHuggingFace(); - case "local": onLocal(); - default: throw UnimplementedError(); - } - }, - offset: const Offset(0, 40), - color: intelGrayVariant, - elevation: 20, - itemBuilder: (BuildContext context) { - return >[ - PopupMenuItem( - value: MenuButtons.huggingface, - child: Row( - children: [ - Padding( - padding: const EdgeInsets.only(right: 8.0), - child: SvgPicture.asset('images/hf-logo.svg', width: 18), - ), - const Text('Huggingface'), - ], - ), - ), - const PopupMenuItem( - value: MenuButtons.local, - child: Row( - children: [ - Padding( - padding: EdgeInsets.only(right: 8.0), - child: Icon(Icons.folder_open, color: Color.fromRGBO(248, 212, 78, 1.0), size: 18), - ), - Text('Local disk'), - ], - ), - ), - ]; - }, - child: Card( - shape: RoundedRectangleBorder( - borderRadius: BorderRadius.circular(16.0), - ), - color: intelBlueVibrant, - child: Padding( - padding: const EdgeInsets.symmetric(horizontal: 24, vertical: 8.0), - child: const Text("Import Model", - style: TextStyle( - color: Colors.white - ), - ), - ) - ), - ); - } - -} - -class GetiProjectsList extends StatelessWidget { - final List projects; - const GetiProjectsList(this.projects, {super.key}); - - @override - Widget build(BuildContext context) { - if (projects.isEmpty) { - return Center( - child: SizedBox( - height: 450, - child: Column( - crossAxisAlignment: CrossAxisAlignment.center, - mainAxisAlignment: MainAxisAlignment.start, - children: [ - SvgPicture.asset('images/upload_art.svg'), - const Text("No imported models", style: TextStyle( - fontSize: 20, - )), - const Text("Import a Geti deployment using the import model button") - ], - ), - ) - ); - } - - return SingleChildScrollView( - child: Column( - children: projects.map((project) { - return Padding( - padding: const EdgeInsets.symmetric(horizontal: 60), - child: ProjectItem(project), - ); - }).toList() - ) - ); - } -} - -class PublicProjectsList extends StatelessWidget { - final List projects; - final bool publicLoaded; - const PublicProjectsList(this.projects, this.publicLoaded, {super.key}); - - @override - Widget build(BuildContext context) { - if (projects.isEmpty) { - return Container(); - } - - return SingleChildScrollView( - child: Column( - crossAxisAlignment: CrossAxisAlignment.stretch, - children: [ - ...projects.map((project) => ProjectItem(project)), - ...(publicLoaded - ? [Container()] - : [ - const GhostProjectItem(), - const GhostProjectItem(), - const GhostProjectItem() - ] - ) - ] - ) - ); - } -} - -class GhostProjectItem extends StatelessWidget { - const GhostProjectItem({super.key}); - - @override - Widget build(BuildContext context) { - return Padding( - padding: const EdgeInsets.only(top: 8), - child: ClipRRect( - borderRadius: BorderRadius.circular(8.0), - child: Opacity( - opacity: 1.0, - child: ConstrainedBox( - constraints: const BoxConstraints.expand(height: 136), - child: Shimmer.fromColors( - baseColor: Theme.of(context).colorScheme.surface, - highlightColor: Theme.of(context).colorScheme.onSurfaceVariant, - child: Container( - color: Theme.of(context).colorScheme.surface, - ) - ), - ), - ) - ) - ); - } -} diff --git a/lib/projects/task_type_filter.dart b/lib/projects/task_type_filter.dart deleted file mode 100644 index 46ab909f..00000000 --- a/lib/projects/task_type_filter.dart +++ /dev/null @@ -1,133 +0,0 @@ -import 'package:flutter/material.dart'; -import 'package:inference/providers/project_filter_provider.dart'; -import 'package:inference/theme.dart'; -import 'package:provider/provider.dart'; - -class TaskTypeFilter extends StatelessWidget { - const TaskTypeFilter({super.key}); - - @override - Widget build(BuildContext context) { - return Padding( - padding: const EdgeInsets.only(right: 50.0), - child: SizedBox( - width: 200, - child: Column( - mainAxisAlignment: MainAxisAlignment.start, - crossAxisAlignment: CrossAxisAlignment.stretch, - children: [ - ...Option.filterOptions.keys.map((key) { - return Group(key, Option.filterOptions[key]!); - - }), //options.map((e) { - ] - ), - ), - ); - } -} - -class Group extends StatefulWidget { - final String name; - final List