cool

adnanwahab · Oct 20, 2024 · f915bd1 · f915bd1
1 parent 01feca5
commit f915bd1
Show file tree

Hide file tree

Showing 7 changed files with 229 additions and 57 deletions.
diff --git a/web-ui/js/LLAMA_Tools.jsx b/web-ui/js/LLAMA_Tools.jsx
@@ -1,20 +1,25 @@
+import Grid from 'views/llama-grid.tsx'
+
+
 function LLAMA_Tools () {
   const panels = [
     //{ id: "livekit_audio", title: "LiveKit Audio" },
     //{ id: "cognition_engine", title: "Cognition Engine" },
 //    { id: "logs_viewer", title: "Logs Viewer" },
   //  { id: "import_docs", title: "Import Docs" },
     // { id: "Particle_morph_target_from_video", title: "Particle morph target from video" },
-    { id: "livekit_agent", title: "livekit" , component: livekit_agent},
-    { id: "voice_reactive_particles", title: "voice reactive particles" },
-    { id: "Git Visualier", title: "Git Visualier-screenshot->iframe" },
-    { id: "Replay analyzer", title: "Replay analyzer" , component: ReplayAnalyzer},
+    // { id: "livekit_agent", title: "livekit" , component: livekit_agent},
+    // { id: "voice_reactive_particles", title: "voice reactive particles" },
+    // { id: "Git Visualier", title: "Git Visualier-screenshot->iframe" },
+    // { id: "Replay analyzer", title: "Replay analyzer" , component: ReplayAnalyzer},
     // { id: "blag", title: "blag" , component: Blog},
   ]; 
   return (
     <>
-          <div>LLAMA Tools for Eric Levin</div>
-          <div><Dashboard panels={panels} /></div>
+          <div>LLAMA Tools for JP, Abid, Raffi, Eric, and Mark</div>
+          {/* <div><Dashboard panels={panels} /></div> */}
+
+          <Grid />
           </>
 
   );

diff --git a/web-ui/js/bun_handlers/llama-backend.tsx b/web-ui/js/bun_handlers/llama-backend.tsx
@@ -9,7 +9,7 @@ import serveMakeDenoCell from './serveMakeDenoCell.ts'
 import serveMakePythonCell from './serveMakePythonCell.ts'
 import serveMakeBunCell from './bun_helper.ts'
 import docker_run from './docker_helper.ts'
-async function livekit_connect(req: Request) { 
+
   //const identity = url.searchParams.get("identity");
   //console.log('identity', identity)
   // console.log("req url", url.pathname);
@@ -38,18 +38,7 @@ async function livekit_connect(req: Request) {
 // try gpt then just redirect to 3 bun rpoceses in one bun file - started by systemd - a script... like levles 
 // delete new relic 
 //return response_404(routes)
-const jsonData = await req.json();
-console.log('Received JSON data:', jsonData);
-
-  const identity = jsonData.identity;
-  if (!identity) {
-    return new Response("Identity parameter is missing", { status: 400 });
-  }
 
-  const json = await connect_to_livekit(jsonData);
-  console.log(json, json);
-    return new Response(JSON.stringify(json));
-}
 async function history_search(req: Request) { 
   // const jsonData = await req.json();
   // console.log('Received JSON data:', jsonData);
@@ -81,7 +70,7 @@ const routes = {
 
   "/history_search": (req: Request) => history_search(req),
 
-  "/livekit_connect": (req: Request) => livekit_connect(req),
+  //"/livekit_connect": (req: Request) => livekit_connect(req),
 
   "/livekit_speech_to_fn_call": livekit_speech_to_fn_call,
 

diff --git a/web-ui/js/course_handler.tsx b/web-ui/js/course_handler.tsx
@@ -157,7 +157,22 @@ function serve404(req: Request) {
   return new Response(`404 - not found`, { status: 404 });
 }
 
+async function livekit_connect(req: Request) { 
+  const jsonData = await req.json();
+console.log('Received JSON data:', jsonData);
+
+  const identity = jsonData.identity;
+  if (!identity) {
+    return new Response("Identity parameter is missing", { status: 400 });
+  }
+
+  const json = await connect_to_livekit(jsonData);
+  console.log(json, json);
+    return new Response(JSON.stringify(json));
+}
+
 const routes = {
+  "/livekit_connect": (req: Request) => livekit_connect(req),
   //vr if done 
   ///"/os/*": (req: Request) => os_automation(req),
 

diff --git a/web-ui/js/main.jsx b/web-ui/js/main.jsx
@@ -76,7 +76,7 @@ root.render(
   <StrictMode>
     <BrowserRouter>
       <Routes>
-        <Route path="/" element={<RoboticsOdyssey />} />
+        {/* <Route path="/" element={<RoboticsOdyssey />} /> */}
         <Route path="/docs" element={<Documentation />} />
         <Route path="/cgi" element={<CGI_Tools />} />
         <Route path="/llama" element={<LLAMA_Tools />} />

diff --git a/web-ui/package.json b/web-ui/package.json
@@ -58,7 +58,7 @@
     "fluent-ffmpeg": "^2.1.3",
     "gmath": "^0.0.6",
     "http-proxy-middleware": "^3.0.3",
-    "livekit-client": "^2.5.7",
+    "livekit-client": "^2.5.9",
     "livekit-server-sdk": "^2.7.0",
     "marked": "^14.1.3",
     "octokit": "^4.0.2",

diff --git a/web-ui/views/llama-grid.tsx b/web-ui/views/llama-grid.tsx
@@ -1,5 +1,19 @@
 import React from 'react';
+import {
+  Room,
+  RoomEvent,
+  Track,
+} from "livekit-client";
 
+let Livekit = {
+  Room,
+  RoomEvent,
+  Track,
+};
+
+// const token =
+//   "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE3MjkxOTU3MjEsImlzcyI6IkFQSXRTYndYdlNqaDRjZiIsIm5hbWUiOiJzY3JlZW5fc2hhcmUiLCJuYmYiOjE3MjkxMDkzMjEsInN1YiI6InNjcmVlbl9zaGFyZSIsInZpZGVvIjp7ImNhblVwZGF0ZU93bk1ldGFkYXRhIjp0cnVlLCJyb29tIjoicm9vbSIsInJvb21BZG1pbiI6dHJ1ZSwicm9vbUNyZWF0ZSI6dHJ1ZSwicm9vbUpvaW4iOnRydWUsInJvb21MaXN0Ijp0cnVlLCJyb29tUmVjb3JkIjp0cnVlfX0.Ub3VigeCkaL4sG4cdw7VaPfaHECuMg8buy6u38xqZPQ";
+// ramble to rewind database - lots of  bear notes -> helper can reogranize into a gant chart.
 const proxy_docs = [
   // "https://bun.sh/docs/runtime/bunfig#run-bun-auto-alias-node-to-bun", 
   // "https://google.com", 
@@ -40,55 +54,204 @@ const proxy_docs = [
   // "https://www.youtube.com/watch?v=CZim0p_etvM",
   // "https://scholar.google.com/",
 ];
+
+/// spoken word = unifies groups from 5 to 100 - written word - 100,000 - pictures = 1 billion - Proof:youtube
+
+async function getLivekitData() {
+  const livekit_connect = 'livekit_connect'
+    const response = await fetch('/api/livekit_connect', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({ identity: 'voice to prompt' }),
+    });
+
+    if (!response.ok) {
+      console.error('Failed to connect to Livekit:', response.statusText);
+      return;
+    }
+
+    const data = await response.json();
+    //console.log('Connected to Livekit:', data);
+    return data
+}
+//const liveKit_data = await postLivekitConnect();
+async function joinRoom(screenShareVideo) {
+  let room = new Room();
+  //console.log('room', room.name)
+  const liveKit_data = await getLivekitData();
+  const url = "wss://omnissiah-university-kmuz0plz.livekit.cloud";
+  await room.connect(url, liveKit_data.token);
+
+  room.on(
+    RoomEvent.TrackSubscribed,
+    (track, publication, participant) => {
+      if (
+        track.kind === Track.Kind.Video &&
+        track.source === Track.Source.ScreenShare
+      ) {
+        track.attach(screenShareVideo);
+      }
+    },
+  );
+
+  room.on(
+    RoomEvent.TrackUnsubscribed,
+    (track, publication, participant) => {
+      if (
+        track.kind === Track.Kind.Video &&
+        track.source === Track.Source.ScreenShare
+      ) {
+        track.detach(screenShareVideo);
+      }
+    },
+  );
+
+  room.on(RoomEvent.LocalTrackPublished, (publication, participant) => {
+    if (
+      publication.kind === Track.Kind.Video &&
+      publication.source === Track.Source.ScreenShare
+    ) {
+      publication.track.attach(screenShareVideo);
+    }
+  });
+
+  room.on(RoomEvent.LocalTrackUnpublished, (publication, participant) => {
+    if (
+      publication.kind === Track.Kind.Video &&
+      publication.source === Track.Source.ScreenShare
+    ) {
+      publication.track.detach(screenShareVideo);
+    }
+  });
+  toggleScreenShare(room);
+}
+
+async function toggleScreenShare(room) {
+  const enabled = room.localParticipant.isScreenShareEnabled;
+  console.log(`${enabled ? "stopping" : "starting"} screen share`);
+  try {
+    await room.localParticipant.setScreenShareEnabled(!enabled, {
+      audio: true,
+    });
+  } catch (e) {
+    console.error("error sharing screen", e);
+  }
+}
+function LivekitAudio() {
+  const ref = React.useRef<HTMLDivElement>(null);
+  const screenShareVideo = React.useRef<HTMLVideoElement>(null);
+
+  function handleButtonPress() {
+    console.log('Button pressed!');
+    console.log('screenShareVideo', screenShareVideo);
+    joinRoom(screenShareVideo);
+  }
+
+  //console.log('screenShareVideo', screenShareVideo)
+  //alert(123123)
+
+  // React.useEffect(() => {
+  //   console.log('LivekitAudio component mounted');
+  // }, []);
+
+  return (
+    <div>
+      <div ref={ref}></div>
+      <div>text to speech goes here !!</div>
+      <video ref={screenShareVideo} autoPlay muted playsInline />
+      <button onClick={handleButtonPress}>connect to livekit!</button>
+    </div>
+  );
+}
 // proxy  use(figma, 2) fun - (gmail, chatGPT)
-const actualComponents = [
+const actualComponents = Object.entries({
 
-  "livekit_audio",
+  "livekit_audio": LivekitAudio,
   // "cognition_engine", - research paper -> diagram -> robot - stixels, waymo, 
-  "logs_viewer",
-  // "import_docs",
+    // "import_docs",
 
-];
+  });
 // const llamaComponents = [
 //   "youtube"
 // ]
 
-const llamaComponents = [
-  ...actualComponents,
-  //...proxy_docs
-]
-console.log(llamaComponents)
 
 function LlamaGrid() {
-  return (
-    <>
-      <div className="bg-white">
-        <h1>anthropic artifact - chatbot blahlalblh - makes observable</h1>
-        <div className="mx-auto max-w-2xl px-6 lg:max-w-7xl lg:px-8">
-          <div className="mt-10 grid grid-cols-1 gap-4 sm:mt-16 lg:grid-cols-6 lg:grid-rows-2">
-            {llamaComponents.map((component) => (
-              <div key={component} className="relative lg:col-span-4">
-                <div className="absolute inset-px rounded-lg bg-white"></div>
-                <div className="relative flex h-full flex-col overflow-hidden">
-                  <div className={`container-${component}`}></div>
-                  <iframe width="500"  
+  return           <LivekitAudio />
+  // return (
+  //   <>
+  //     <div className="bg-white">
+  //       <h1>anthropic artifact - chatbot blahlalblh - makes observable</h1>
+  //       <div className="mx-auto max-w-2xl px-6 lg:max-w-7xl lg:px-8">
+  //         <div className="mt-10 grid grid-cols-1 gap-4 sm:mt-16 lg:grid-cols-6 lg:grid-rows-2">
+  //         <LivekitAudio />
+  //           {/* {actualComponents.map((component) => (
+  //             <div key={component} className="relative lg:col-span-4">
+  //               <div className="absolute inset-px rounded-lg bg-white"></div>
+  //               <div className="relative flex h-full flex-col overflow-hidden">
+  //                 <div className={`container-${component}`}></div>
+  //                 <LivekitAudio>
+  //                 {/* <iframe width="500"  
 
 
 
-                  height="500" src={`/deno/${component}`}></iframe>
-                  <div className="p-10 pt-4"></div>
-                </div>
-                <div className="pointer-events-none absolute inset-px rounded-lg shadow ring-1 ring-black/5"></div>
-              </div>
-            ))}
-          </div>
-        </div>
-      </div>
-      <h1>goal by nov 1 - 1001 useful tools that shorten distance dynamicland</h1>
-    </>
-  );
+  //                 height="500" src={`/deno/${component}`}></iframe> */}
+  //                 <div className="p-10 pt-4"></div>
+  //               </div>
+  //               <div className="pointer-events-none absolute inset-px rounded-lg shadow ring-1 ring-black/5"></div>
+  //             </div>
+  //           {/* ))}} */}
+  //         </div>
+  //       </div>
+  //     </div>
+  //   </>
+  // );
 }
 
 // ... (rest of the code remains unchanged)
+      // <h1>goal by nov 1 - 1001 useful tools that shorten distance dynamicland</h1>
 
 export default LlamaGrid;
+
+// AI seinfeld but with all cartoons ever + robots - questionablecontent.net
+
+
+// async function requestMicrophoneAndSpeechToText() {
+//   try {
+//     // Request microphone access
+//     const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+//     console.log('Microphone access granted');
+
+//     // Initialize SpeechRecognition
+//     const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition;
+//     const recognition = new SpeechRecognition();
+
+//     recognition.onstart = () => {
+//       console.log('Speech recognition started');
+//     };
+
+//     recognition.onspeechend = () => {
+//       console.log('Speech recognition ended');
+//       recognition.stop();
+//     };
+
+//     recognition.onresult = (event) => {
+//       const transcript = event.results[0][0].transcript;
+//       console.log('Speech to text result:', transcript);
+//     };
+
+//     recognition.onerror = (event) => {
+//       console.error('Speech recognition error:', event.error);
+//     };
+
+//     // Start speech recognition
+//     recognition.start();
+//   } catch (error) {
+//     console.error('Error accessing microphone:', error);
+//   }
+// }
+
+
+//dating = a game like mounment valley or  the game amro playerd - farm ville 
diff --git a/web-ui/vite.config.js b/web-ui/vite.config.js
@@ -9,7 +9,7 @@ export default defineConfig({
     port: 8000, // You can specify the port here
     proxy: {
       '/api': {
-        target: 'http://localhost:8003',
+        target: 'http://localhost:8080',
         changeOrigin: true,
         rewrite: (path) => path.replace(/^\/api/, '')
       }
@@ -24,7 +24,7 @@ export default defineConfig({
         }
       },
       input: {
-        voiceReactiveParticles: 'views/cgi-tools/voice_reactive_particles.html',
+        // voiceReactiveParticles: 'views/cgi-tools/voice_reactive_particles.html',
       },
     },
     target: 'esnext', // Add this line to target the latest ECMAScript version