Add working document import dialog

lib/langchain/loaders/html_loader.dart

@@ -0,0 +1,27 @@
+import 'dart:io';
+import 'dart:math';
+import 'package:html/parser.dart';
+
+import 'package:langchain/langchain.dart';
+
+class HTMLLoader extends BaseDocumentLoader {
+
+  final String path;
+  final int windowSize;
+  const HTMLLoader(this.path, this.windowSize);
+
+  @override
+  Stream<Document> lazyLoad() async* {
+    final data = await File(path).readAsString();
+    final text = parse(data).body?.text;
+    if (text != null) {
+      for (int i = 0; i < text.length; i += windowSize) {
+        final content = text.substring(i, min(i + windowSize, text.length));
+        yield Document(
+          pageContent: content,
+          metadata: {"source": path},
+        );
+      }
+    }
+  }
+}

lib/langchain/loaders/text_loader.dart

@@ -0,0 +1,23 @@
+import 'dart:io';
+import 'dart:math';
+
+import 'package:langchain/langchain.dart';
+
+class TextLoader extends BaseDocumentLoader {
+
+  final String path;
+  final int windowSize;
+  const TextLoader(this.path, this.windowSize);
+
+  @override
+  Stream<Document> lazyLoad() async* {
+    final text = await File(path).readAsString();
+    for (int i = 0; i < text.length; i += windowSize) {
+      final content = text.substring(i, min(i + windowSize, text.length));
+      yield Document(
+        pageContent: content,
+        metadata: {"source": path},
+      );
+    }
+  }
+}

lib/pages/knowledge_base/knowledge_base.dart

@@ -68,10 +68,13 @@ class KnowledgeBase extends StatelessWidget {
           Expanded(
             child: Consumer<KnowledgeBaseProvider>(
               builder: (context, data, child) {
-                if (data.activeGroup == null) {
-                  return const Center(child: Text("Select a group from the list to the left"));
+                if (data.activeGroup != null) {
+                  final group = data.groupBox.get(data.activeGroup!);
+                  if (group != null) {
+                    return DocumentsList(group: group, key: Key(group.internalId.toString()));
+                  }
                 }
-                return DocumentsList(group: data.activeGroup!);
+                return const Center(child: Text("Select a group from the list to the left"));
               }
             ),
           ),

lib/pages/knowledge_base/providers/knowledge_base_provider.dart

@@ -8,14 +8,15 @@ class KnowledgeBaseProvider extends ChangeNotifier {
 
   List<KnowledgeGroup> _groups = [];
   List<KnowledgeGroup> get groups => _groups;
+
   set groups(List<KnowledgeGroup> value) {
     _groups = value;
     notifyListeners();
   }
 
-  KnowledgeGroup? _activeGroup;
-  KnowledgeGroup? get activeGroup => _activeGroup;
-  set activeGroup(KnowledgeGroup? value) {
+  int? _activeGroup;
+  int? get activeGroup => _activeGroup;
+  set activeGroup(int? value) {
     _activeGroup = value;
     notifyListeners();
   }
@@ -47,16 +48,19 @@ class KnowledgeBaseProvider extends ChangeNotifier {
   void addGroup() {
     isEditingId = groupBox.put(KnowledgeGroup("New group"));
     groups = groupBox.getAll();
+    if (groups.length == 1) {
+      activeGroup = isEditingId;
+    }
   }
 
   void setActiveGroup(KnowledgeGroup group) {
-    activeGroup = group;
+    activeGroup = group.internalId;
   }
 
   KnowledgeBaseProvider({required this.groupBox}) {
     groupBox.getAllAsync().then((value) {
         groups = value;
-        activeGroup = groups.firstOrNull;
+        activeGroup = groups.firstOrNull?.internalId;
     });
   }
 }

lib/pages/knowledge_base/utils/loader_selector.dart

@@ -0,0 +1,38 @@
+import 'package:inference/langchain/loaders/html_loader.dart';
+import 'package:inference/langchain/loaders/pdf_loader.dart';
+import 'package:inference/langchain/loaders/text_loader.dart';
+import 'package:langchain/langchain.dart';
+import 'package:path/path.dart';
+
+const windowSize = 400;
+
+String defaultLoaderSelector(String path) {
+  final ext = extension(path);
+  if (ext == ".pdf") {
+    return "PdfLoader";
+  }
+
+  if (ext == ".html") {
+    return "HTMLLoader";
+  }
+
+  //if (ext == ".json") {
+  //  return JsonLoader(path,)
+  //}
+
+  return "TextLoader";
+}
+
+BaseDocumentLoader loaderFromName(String name, String path) {
+  switch (name) {
+    case "PdfLoader":
+      return PdfLoader(path, windowSize);
+    case "HTMLLoader":
+      return HTMLLoader(path, windowSize);
+    case "TextLoader":
+      return TextLoader(path, windowSize);
+    default:
+      throw Exception("Unknown loader name: $name");
+  }
+
+}