From a50c2fed8ffa5b926238089891c06a1e2649a8be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=E2=9D=A4=EF=B8=8F=20=E2=98=AE=EF=B8=8F=20=E2=9C=8B?= <6723574+louisgv@users.noreply.github.com> Date: Mon, 3 Jul 2023 03:03:30 -0700 Subject: [PATCH 1/3] fix: disable GPU for Windows and Linux for now until runtime-check is implemented --- apps/desktop/package.json | 2 +- apps/desktop/src-tauri/Cargo.lock | 12 +++- apps/desktop/src-tauri/Cargo.toml | 3 +- apps/desktop/src-tauri/main.ts | 36 ++++++++++ apps/desktop/src-tauri/src/inference/gpu.rs | 9 +++ apps/desktop/src-tauri/src/inference/mod.rs | 1 + apps/desktop/src-tauri/src/main.rs | 68 +++++++++++++++++-- .../inference-server/server-config.tsx | 16 ++++- apps/desktop/src/features/invoke/_shared.ts | 11 +-- apps/desktop/src/features/invoke/server.ts | 1 + 10 files changed, 147 insertions(+), 12 deletions(-) create mode 100644 apps/desktop/src-tauri/main.ts create mode 100644 apps/desktop/src-tauri/src/inference/gpu.rs diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 5eab9d7..0eabdd3 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -1,7 +1,7 @@ { "name": "@localai/desktop", "private": true, - "version": "0.5.1", + "version": "0.5.2", "scripts": { "dev:next": "next dev -p 1470", "build:next": "next build", diff --git a/apps/desktop/src-tauri/Cargo.lock b/apps/desktop/src-tauri/Cargo.lock index de69237..e896be0 100644 --- a/apps/desktop/src-tauri/Cargo.lock +++ b/apps/desktop/src-tauri/Cargo.lock @@ -840,6 +840,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" +[[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "cookie" version = "0.16.2" @@ -1112,7 +1121,7 @@ version = "0.99.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ - "convert_case", + "convert_case 0.4.0", "proc-macro2", "quote", "rustc_version 0.4.0", @@ -2483,6 +2492,7 @@ dependencies = [ "blake3", "chrono", "cocoa", + "convert_case 0.6.0", "diesel", "digest", "flume", diff --git a/apps/desktop/src-tauri/Cargo.toml b/apps/desktop/src-tauri/Cargo.toml index 6278227..efd687b 100644 --- a/apps/desktop/src-tauri/Cargo.toml +++ b/apps/desktop/src-tauri/Cargo.toml @@ -15,7 +15,7 @@ tauri-build = { version = "1.4.0", features = [] } [dependencies] llm = { git = "https://github.com/rustformers/llm", branch = "main", package = "llm", features = [ "default", - "cublas", + # "cublas", ] } # llm = { git = "https://github.com/RedBoxing/llm.git", branch = "hf-tokenizer", package = "llm" } @@ -34,6 +34,7 @@ tauri = { version = "1.4.0", features = [ "window-show", ] } +convert_case = "0.6.0" paste = "1.0" sys-info = "0.9.1" num_cpus = "1.15.0" diff --git a/apps/desktop/src-tauri/main.ts b/apps/desktop/src-tauri/main.ts new file mode 100644 index 0000000..71eff9e --- /dev/null +++ b/apps/desktop/src-tauri/main.ts @@ -0,0 +1,36 @@ + // region:sourcegen + 0 => Error::get_config, + 1 => Error::read_directory, + 2 => Error::write_file, + 3 => Error::read_file, + 4 => Error::get_thread_config, + 5 => Error::set_thread_config, + 6 => Error::append_thread_content, + 7 => Error::read_thread_file, + 8 => Error::initialize_threads_dir, + 9 => Error::update_threads_dir, + 10 => Error::delete_thread_file, + 11 => Error::create_thread_file, + 12 => Error::rename_thread_file, + 13 => Error::update_models_dir, + 14 => Error::initialize_models_dir, + 15 => Error::delete_model_file, + 16 => Error::get_download_progress, + 17 => Error::start_download, + 18 => Error::pause_download, + 19 => Error::resume_download, + 20 => Error::get_cached_integrity, + 21 => Error::compute_model_integrity, + 22 => Error::get_model_stats, + 23 => Error::get_model_config, + 24 => Error::set_model_config, + 25 => Error::load_model, + 26 => Error::start_server, + 27 => Error::stop_server, + 28 => Error::get_server_config, + 29 => Error::set_server_config, + 30 => Error::check_gpu, + 31 => Error::open_directory, + 32 => Error::test_model, + 33 => Error::, + // endregion:sourcegen diff --git a/apps/desktop/src-tauri/src/inference/gpu.rs b/apps/desktop/src-tauri/src/inference/gpu.rs new file mode 100644 index 0000000..0e3d003 --- /dev/null +++ b/apps/desktop/src-tauri/src/inference/gpu.rs @@ -0,0 +1,9 @@ +#[tauri::command] +pub async fn check_gpu<'a>() -> Result { + #[cfg(target = "aarch64-apple-darwin")] + { + return Ok(true); + } + + Ok(false) +} diff --git a/apps/desktop/src-tauri/src/inference/mod.rs b/apps/desktop/src-tauri/src/inference/mod.rs index f6dc283..dd7c99f 100644 --- a/apps/desktop/src-tauri/src/inference/mod.rs +++ b/apps/desktop/src-tauri/src/inference/mod.rs @@ -1,4 +1,5 @@ pub mod completion; +pub mod gpu; pub mod process; pub mod server; pub mod server_config; diff --git a/apps/desktop/src-tauri/src/main.rs b/apps/desktop/src-tauri/src/main.rs index 3cbda20..6c504cf 100644 --- a/apps/desktop/src-tauri/src/main.rs +++ b/apps/desktop/src-tauri/src/main.rs @@ -38,8 +38,7 @@ fn main() { Ok(()) }) - // NOTE: New cmd should be added to src/invoke/_shared.ts - // TODO: a middleware to convert this into the ts enum would be nice + // NOTE: When adding new commands, make sure to run the generate_ts_enums test to update the TS enum .invoke_handler(tauri::generate_handler![ config::get_config, path::read_directory, @@ -66,14 +65,75 @@ fn main() { model::stats::get_model_stats, model::config::get_model_config, model::config::set_model_config, + model::pool::load_model, inference::server::start_server, inference::server::stop_server, inference::server_config::get_server_config, inference::server_config::set_server_config, - model::pool::load_model, - test::test_model, + inference::gpu::check_gpu, utils::fs::open_directory, + test::test_model, ]) .run(tauri::generate_context!()) .expect("error while running tauri application"); } + +#[test] +/** + * Generate the enum for the client side invocation + * Based on https://matklad.github.io/2022/03/26/self-modifying-code.html#Minimalist-Solution + */ +fn generate_ts_cmd_enums() { + use convert_case::{Case, Casing}; + + fn split_twice<'a>( + text: &'a str, + start_marker: &str, + end_marker: &str, + ) -> Option<(&'a str, &'a str, &'a str)> { + let (prefix, rest) = text.split_once(start_marker)?; + let (mid, suffix) = rest.split_once(end_marker)?; + Some((prefix, mid, suffix)) + } + + let main_rs_text = std::fs::read_to_string(file!()).unwrap(); + + let (_, tauri_cmds, _) = split_twice( + &main_rs_text, + ".invoke_handler(tauri::generate_handler![\n", + "])", + ) + .unwrap(); + + let arms = tauri_cmds + .lines() + .map(|line| { + line + .trim() + .trim_end_matches(',') + .split("::") + .last() + .unwrap() + }) + .enumerate() + // filter only non-empty string + .filter(|(_, cmd)| !cmd.is_empty()) + .map(|(_, cmd)| format!(" {} = \"{cmd}\"", cmd.to_case(Case::Pascal))) + .collect::>() + .join(",\n"); + + let ts_enum_path = std::path::Path::new("../src/features/invoke/_shared.ts"); + + let ts_original_text = std::fs::read_to_string(ts_enum_path).unwrap(); + + let new_text = { + let start_marker = " //#region GENERATED\n"; + let end_marker = "\n //#endregion\n"; + + let (prefix, _, suffix) = + split_twice(&ts_original_text, start_marker, end_marker).unwrap(); + format!("{prefix}{start_marker}{arms}{end_marker}{suffix}") + }; + + std::fs::write(ts_enum_path, new_text).unwrap(); +} diff --git a/apps/desktop/src/features/inference-server/server-config.tsx b/apps/desktop/src/features/inference-server/server-config.tsx index 4382628..c8d1fd8 100644 --- a/apps/desktop/src/features/inference-server/server-config.tsx +++ b/apps/desktop/src/features/inference-server/server-config.tsx @@ -4,6 +4,8 @@ import { IntInput } from "@lab/ui/int-input" import { Switch } from "@lab/ui/switch" import { useState } from "react" +import { InitState, useInit } from "~features/inference-server/use-init" +import { InvokeCommand, invoke } from "~features/invoke" import { useGlobal } from "~providers/global" export const ServerConfig = () => { @@ -15,6 +17,16 @@ export const ServerConfig = () => { } = useGlobal() const [isLoading, setIsLoading] = useState(false) + + const [hasGpu, setHasGpu] = useState(false) + const gpuCheck = useInit(async () => { + const _hasGpu = await invoke(InvokeCommand.CheckGpu) + if (!_hasGpu) { + serverConfig.update({ useGpu: false }) + } + setHasGpu(_hasGpu) + }) + return (
{/*