From 6a64f56ab3880cf5589717de0f3ae638efe80d87 Mon Sep 17 00:00:00 2001 From: Yixin Date: Tue, 26 Sep 2023 18:58:59 +0200 Subject: [PATCH 1/6] fix: support utf-8 chars for chat recording --- app.py | 2 +- partygpt/partygpt.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index dd2d4e7..671dfe3 100644 --- a/app.py +++ b/app.py @@ -27,7 +27,7 @@ log_console_handler = logging.StreamHandler() log_console_handler.setLevel(logging.INFO) log_console_handler.setFormatter(log_formatter) -log_file_handler = logging.FileHandler('debug.log') +log_file_handler = logging.FileHandler('debug.log', encoding='utf-8') log_file_handler.setLevel(logging.DEBUG) log_file_handler.setFormatter(log_formatter) root_logger = logging.getLogger() diff --git a/partygpt/partygpt.py b/partygpt/partygpt.py index 5a5bf1a..8f13034 100644 --- a/partygpt/partygpt.py +++ b/partygpt/partygpt.py @@ -46,7 +46,7 @@ def _clear_messages_history(self) -> None: os.makedirs(self._conversation_record_folder_path, exist_ok=True) file_name = os.path.join(self._conversation_record_folder_path, str(self._current_conversation_start) + '.convrec') - with open(file_name, 'x') as f: + with open(file_name, 'x', encoding='utf-8') as f: for el in self._messages_history: role_ = el['role'] content_ = el['content'] From d81df9e2ac4a4718d830a1669a68f23e36f129c9 Mon Sep 17 00:00:00 2001 From: Yixin Date: Tue, 26 Sep 2023 22:37:49 +0200 Subject: [PATCH 2/6] fix: save chat after different ways of ending --- app.py | 5 +++++ partygpt/partygpt.py | 7 +++++-- static/js/script.js | 12 +++++++++++- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/app.py b/app.py index 671dfe3..25786d0 100644 --- a/app.py +++ b/app.py @@ -74,6 +74,11 @@ def refresh_session(): ai_guest.reset(persona=None) return '' # may not return None or omit return statement +@app.route('/save-records', methods=['GET']) +def save_records(): + ai_guest.save_messages_history() + return '', 200 + def trigger_session_refresh(goodbye_msg): logger.info('Session refreshed (per backend).') sockets.emit('instruction', { diff --git a/partygpt/partygpt.py b/partygpt/partygpt.py index 8f13034..c496c90 100644 --- a/partygpt/partygpt.py +++ b/partygpt/partygpt.py @@ -41,7 +41,11 @@ def get_accumulated_costs(self, verbose=True): return total_t['sum'] def _clear_messages_history(self) -> None: + self._messages_history = [] + + def save_messages_history(self) -> None: if self._conversation_record_folder_path and self._messages_history: # only record if path defined + logger.info('Chat records is now saving...') try: os.makedirs(self._conversation_record_folder_path, exist_ok=True) file_name = os.path.join(self._conversation_record_folder_path, @@ -63,8 +67,7 @@ def _clear_messages_history(self) -> None: logger.info(f'Wrote conversation to {file_name}.') except Exception as e: logger.error(f'Trouble writing conversation history to file: {type(e).__name__}: {e}') - - self._messages_history = [] + def set_functions( self, diff --git a/static/js/script.js b/static/js/script.js index 54bfefd..9ef2437 100644 --- a/static/js/script.js +++ b/static/js/script.js @@ -96,6 +96,11 @@ document.addEventListener("DOMContentLoaded", function() { userInput.disabled = false; } + function saveRecords() { + fetch('/save-records', { + method: 'GET' + }); + } function close_session(){ conversation.innerHTML = ""; @@ -109,6 +114,7 @@ document.addEventListener("DOMContentLoaded", function() { closeButton.addEventListener("click", function() { + saveRecords(); close_session(); }); @@ -120,7 +126,10 @@ document.addEventListener("DOMContentLoaded", function() { function startIdleTimer() { clearTimeout(idleTimer); - idleTimer = setTimeout(close_session, idleTimeoutDuration); + idleTimer = setTimeout (function () { + saveRecords(); + close_session(); + }, idleTimeoutDuration); } const inputElements = document.querySelectorAll('input, textarea'); @@ -143,6 +152,7 @@ document.addEventListener("DOMContentLoaded", function() { type = instruction.type if (type == 'refresh_session_timer') { addMessage(instruction.goodbye_msg, 'assistant'); + saveRecords(); lockInputField(); setTimeout(close_session, instruction.timer * 1000); // TODO: clear timer if second call here } From edc572bac8665873d121c9161ec37a47838fa0f9 Mon Sep 17 00:00:00 2001 From: Yixin Date: Tue, 26 Sep 2023 23:43:30 +0200 Subject: [PATCH 3/6] enhancement: UI change after session ended by backend --- static/js/script.js | 41 +++++++++++++++++++++++++++++++++++++---- templates/index.html | 5 ++++- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/static/js/script.js b/static/js/script.js index 9ef2437..6c210b4 100644 --- a/static/js/script.js +++ b/static/js/script.js @@ -3,8 +3,10 @@ document.addEventListener("DOMContentLoaded", function() { // ---------------- USER INTERACTION ---------------- var conversation = document.getElementById("conversation"); var userInput = document.getElementById("user-input"); + var buttonContainer = document.getElementById("button-container"); var sendButton = document.getElementById("send-button"); var closeButton = document.getElementById("close-button"); + var newSessionHint = document.getElementById("new-session-hint"); function setFocusOnInput() { @@ -87,15 +89,42 @@ document.addEventListener("DOMContentLoaded", function() { }); + function chatEndedUIChange() { + lockInputField(); + HideButtonContainer(); + } + + function chatStartUIChange() { + hideNewSessionHint(); + unlockInputField(); + setFocusOnInput(); + showButtonContainer(); + } + function lockInputField() { userInput.disabled = true; } - function unlockInputField() { userInput.disabled = false; } + function showButtonContainer() { + buttonContainer.style.display = 'block'; + } + + function HideButtonContainer() { + buttonContainer.style.display = 'none'; + } + + function showNewSessionHint() { + newSessionHint.style.display = 'block'; + } + + function hideNewSessionHint() { + newSessionHint.style.display = 'none'; + } + function saveRecords() { fetch('/save-records', { method: 'GET' @@ -108,8 +137,11 @@ document.addEventListener("DOMContentLoaded", function() { method: 'GET' }); console.log('Session refreshed'); - unlockInputField(); - setFocusOnInput(); + refreshNewSession(); + } + + function refreshNewSession() { + chatStartUIChange(); } @@ -152,9 +184,10 @@ document.addEventListener("DOMContentLoaded", function() { type = instruction.type if (type == 'refresh_session_timer') { addMessage(instruction.goodbye_msg, 'assistant'); + chatEndedUIChange(); saveRecords(); - lockInputField(); setTimeout(close_session, instruction.timer * 1000); // TODO: clear timer if second call here + showNewSessionHint(); } else { console.error('Unknown instruction:', type); diff --git a/templates/index.html b/templates/index.html index 4602792..f726b7a 100644 --- a/templates/index.html +++ b/templates/index.html @@ -14,10 +14,13 @@

PartyGPT

-
+
+
+ +
From f64e666c407f0b1b879d3887a256c9b39bd59b0c Mon Sep 17 00:00:00 2001 From: Yixin Date: Wed, 27 Sep 2023 22:51:51 +0200 Subject: [PATCH 4/6] feature: user can decide if chat will be recorded --- app.py | 19 ++++++- partygpt/partygpt.py | 7 +++ settings.yml | 7 ++- static/js/script.js | 118 ++++++++++++++++++++++++++++++------------- templates/index.html | 4 ++ 5 files changed, 116 insertions(+), 39 deletions(-) diff --git a/app.py b/app.py index 25786d0..9c9808d 100644 --- a/app.py +++ b/app.py @@ -3,7 +3,7 @@ import threading # TEST -from flask import Flask, render_template, request +from flask import Flask, render_template, request, jsonify from flask_socketio import SocketIO, emit from partygpt import AiGuest @@ -14,6 +14,7 @@ PATH_SETTINGS = 'settings.yml' APPLICATION_SETTINGS = read_yaml(path=PATH_SETTINGS)['application'] PATH_FOLDER_CONVERSATION_RECORDS = APPLICATION_SETTINGS['conversation_records_folder'] +PATH_FOLDER_CONVERSATION_RECORDS_NAME = APPLICATION_SETTINGS['records_folder_name'] REFRESH_TIMER_AI_GOODBYE = APPLICATION_SETTINGS['refresh_conversation_after_ai_says_goodbye'] app = Flask(__name__) @@ -46,6 +47,7 @@ def log_connect(data): @app.route('/') def index(): + ai_guest.set_conversation_record_folder_path(PATH_FOLDER_CONVERSATION_RECORDS_NAME) return render_template('index.html') @app.route('/process-input', methods=['POST']) @@ -79,6 +81,21 @@ def save_records(): ai_guest.save_messages_history() return '', 200 +@app.route('/set-records', methods=['GET']) +def set_records(): + flag = request.args.get('flag') + logger.info(f'Chat recording is set to: {flag}') + response_data = {} + if flag == 'true': # (?)flag param can only be interpreted as string + ai_guest.set_conversation_record_folder_path(PATH_FOLDER_CONVERSATION_RECORDS_NAME) + response_data['message'] = 'Chat will be recorded' + else: + ai_guest.set_conversation_record_folder_path('') + response_data['message'] = 'Chat will not be recorded' + + response = jsonify(response_data) + return response, 200, {'Content-Type': 'application/json'} + def trigger_session_refresh(goodbye_msg): logger.info('Session refreshed (per backend).') sockets.emit('instruction', { diff --git a/partygpt/partygpt.py b/partygpt/partygpt.py index c496c90..f44f4cb 100644 --- a/partygpt/partygpt.py +++ b/partygpt/partygpt.py @@ -90,6 +90,13 @@ def reset(self): self._clear_messages_history() self._current_conversation_start = int(time.time()) + def set_conversation_record_folder_path(self, new_path): + self._conversation_record_folder_path = new_path + if new_path != '': + logger.info(f'Chat will be recorded to {new_path}') + else: + logger.info('Chat will not be recorded') + def communicate( self, messages: list, diff --git a/settings.yml b/settings.yml index 6e3e8df..3509091 100644 --- a/settings.yml +++ b/settings.yml @@ -1,7 +1,8 @@ application: refresh_conversation_after_ai_says_goodbye: 30 # seconds - conversation_records_folder: conversation_records # remove the characters after ":" to deactivate conversation recording - + records_folder_name: &name_anchor conversation_records + conversation_records_folder: *name_anchor # remove the characters after ":" to deactivate conversation recording + model: id: gpt-3.5-turbo @@ -36,6 +37,8 @@ guest: special_skill: # you can ___. - tell funny jokes - provide facts about science and nature in an understandable way + - talk like Barney Stinson + - talk like Winnie the Pooh - talk like a pirate - create incredible drinks (you actually named the party's signature drink "Rum Royale Bliss") - start playing a quiz on basic knowledge with the guest diff --git a/static/js/script.js b/static/js/script.js index 6c210b4..c34f2fe 100644 --- a/static/js/script.js +++ b/static/js/script.js @@ -1,4 +1,4 @@ -document.addEventListener("DOMContentLoaded", function() { +document.addEventListener("DOMContentLoaded", function () { // ---------------- USER INTERACTION ---------------- var conversation = document.getElementById("conversation"); @@ -6,25 +6,28 @@ document.addEventListener("DOMContentLoaded", function() { var buttonContainer = document.getElementById("button-container"); var sendButton = document.getElementById("send-button"); var closeButton = document.getElementById("close-button"); + var setRecordButton = document.getElementById("set-recording"); + var recordStatus = document.getElementById("recording-status"); + var chatInfo = document.getElementById("chat-info"); var newSessionHint = document.getElementById("new-session-hint"); + const featureFlags = { + allowRecords: true + }; - function setFocusOnInput() { - userInput.focus(); - } - setFocusOnInput(); + chatStartUIChange(); function addMessage(message, sender) { var messageElement = document.createElement("div"); messageElement.classList.add("message"); if (sender === "user") { - messageElement.classList.add("user-input"); - messageElement.textContent = "You: " + message; + messageElement.classList.add("user-input"); + messageElement.textContent = "You: " + message; } else if (sender === "assistant") { - messageElement.classList.add("ai-message"); - messageElement.textContent = "AI: " + message; // BACKUP: render newlines: .replace(/(?:\r\n|\r|\n)/g, '
'); + messageElement.classList.add("ai-message"); + messageElement.textContent = "AI: " + message; // BACKUP: render newlines: .replace(/(?:\r\n|\r|\n)/g, '
'); } else { console.error('Unknown message sender:', sender); @@ -46,34 +49,35 @@ document.addEventListener("DOMContentLoaded", function() { fetch('/process-input', { method: 'POST', headers: { - 'Content-Type': 'application/json', + 'Content-Type': 'application/json', }, body: JSON.stringify({ message }), }) - // and handle the response from the assistant - .then((response) => response.json()) - .then((data) => { - const reply = data.reply; - if (reply === '' || reply === null) { - console.log('Received null response from backend.'); - } - else { - addMessage(reply, 'assistant'); - } - }) - .catch((error) => { - console.error('Error:', error); - }); + // and handle the response from the assistant + .then((response) => response.json()) + .then((data) => { + const reply = data.reply; + if (reply === '' || reply === null) { + console.log('Received null response from backend.'); + } + else { + addMessage(reply, 'assistant'); + } + }) + .catch((error) => { + console.error('Error:', error); + }); } } - sendButton.addEventListener("click", function() { + sendButton.addEventListener("click", function () { handleUserInput(); + setFocusOnInput(); }); - document.addEventListener("keydown", function(event) { + document.addEventListener("keydown", function (event) { if (event.shiftKey && event.key === "Enter") { event.preventDefault(); // Prevent form submission var currentCursorPosition = userInput.selectionStart; // Get current cursor position @@ -85,6 +89,7 @@ document.addEventListener("DOMContentLoaded", function() { else if (event.key === "Enter") { event.preventDefault(); handleUserInput(); + setFocusOnInput(); } }); @@ -92,6 +97,7 @@ document.addEventListener("DOMContentLoaded", function() { function chatEndedUIChange() { lockInputField(); HideButtonContainer(); + hideChatInfo(); } function chatStartUIChange() { @@ -99,6 +105,20 @@ document.addEventListener("DOMContentLoaded", function() { unlockInputField(); setFocusOnInput(); showButtonContainer(); + showChatInfo(); + } + + function setFocusOnInput() { + userInput.focus(); + } + + + function hideChatInfo() { + chatInfo.style.display = 'none'; + } + + function showChatInfo() { + chatInfo.style.display = 'block'; } function lockInputField() { @@ -118,12 +138,30 @@ document.addEventListener("DOMContentLoaded", function() { } function showNewSessionHint() { - newSessionHint.style.display = 'block'; + newSessionHint.style.display = 'block'; } function hideNewSessionHint() { newSessionHint.style.display = 'none'; - } + } + + function setRecord(flag) { + const url = `/set-records?flag=${flag}`; + fetch(url, { + method: 'GET' + }) + .then((response) => { + if (response.status === 200) { + return response.json(); + } else { + console.error('Request failed with status:', response.status); + } + }) + .then((data) => { + recordStatus.textContent = data.message; + setRecordButton.textContent = featureFlags.allowRecords ? 'Disable recording' : 'Enable recording' + }); + } function saveRecords() { fetch('/save-records', { @@ -131,7 +169,7 @@ document.addEventListener("DOMContentLoaded", function() { }); } - function close_session(){ + function close_session() { conversation.innerHTML = ""; fetch('/close-session', { method: 'GET' @@ -142,14 +180,22 @@ document.addEventListener("DOMContentLoaded", function() { function refreshNewSession() { chatStartUIChange(); + featureFlags.allowRecords = true; //enable chat recording by default + setRecord(featureFlags.allowRecords); + } - closeButton.addEventListener("click", function() { + closeButton.addEventListener("click", function () { saveRecords(); close_session(); }); + setRecordButton.addEventListener("click", function () { + console.log("flag is now " + featureFlags.allowRecords); + featureFlags.allowRecords = !featureFlags.allowRecords; + setRecord(featureFlags.allowRecords); + }) // ---------------- IDLE REFRESH ---------------- @@ -158,10 +204,10 @@ document.addEventListener("DOMContentLoaded", function() { function startIdleTimer() { clearTimeout(idleTimer); - idleTimer = setTimeout (function () { + idleTimer = setTimeout(function () { saveRecords(); close_session(); - }, idleTimeoutDuration); + }, idleTimeoutDuration); } const inputElements = document.querySelectorAll('input, textarea'); @@ -176,11 +222,11 @@ document.addEventListener("DOMContentLoaded", function() { // ---------------- INSTRUCTIONS FROM BACKEND ---------------- var socket = io(); - socket.on('connect', function() { - socket.emit('connection', {state: 'success'}); + socket.on('connect', function () { + socket.emit('connection', { state: 'success' }); }); - socket.on('instruction', function(instruction) { + socket.on('instruction', function (instruction) { type = instruction.type if (type == 'refresh_session_timer') { addMessage(instruction.goodbye_msg, 'assistant'); @@ -195,4 +241,4 @@ document.addEventListener("DOMContentLoaded", function() { }); - }); +}); diff --git a/templates/index.html b/templates/index.html index f726b7a..c5213a2 100644 --- a/templates/index.html +++ b/templates/index.html @@ -9,6 +9,10 @@

PartyGPT

+
+

Chat will be recorded.

+ +
From 48a4d9c57e42fb60788ae32cbb054e611e2e5838 Mon Sep 17 00:00:00 2001 From: Yixin Date: Sun, 1 Oct 2023 00:41:41 +0200 Subject: [PATCH 5/6] feature: user can talk (en/de/zh/es) and ai can speak --- static/css/styles.css | 93 ++++-- static/images/mic-icon.png | Bin 0 -> 14041 bytes static/js/audio.mjs | 126 +++++++++ static/js/iso6393-to-bcp47.mjs | 419 ++++++++++++++++++++++++++++ static/js/{script.js => script.mjs} | 210 +++++++------- templates/index.html | 53 +++- 6 files changed, 767 insertions(+), 134 deletions(-) create mode 100644 static/images/mic-icon.png create mode 100644 static/js/audio.mjs create mode 100644 static/js/iso6393-to-bcp47.mjs rename static/js/{script.js => script.mjs} (52%) diff --git a/static/css/styles.css b/static/css/styles.css index fe44ff3..128a5bb 100644 --- a/static/css/styles.css +++ b/static/css/styles.css @@ -37,11 +37,62 @@ h1 { margin-bottom: 10px; } +.mode-container { + display: flex; + width: 100%; + justify-content: center; + align-items: center; + text-align: center; +} + +.button-container { + display: flex; + justify-content: space-between; +} + +.button-container button { + background-color: #4caf50; + color: #fff; + padding: 10px 20px; + font-size: 16px; + border: none; + border-radius: 5px; + cursor: pointer; +} + +#send-button:hover { + background-color: #45a049; +} + +#close-button { + background-color: #f44336; +} + +#close-button:hover { + background-color: #d32f2f; +} + +#mode-toggle-button { + background-color: #2fd2f3; +} + + +#chat-info { + display: flex; + justify-content: space-between; + margin-bottom: 20px; +} + +#recording-info { + margin-right: 10px; +} + #user-input { flex: 1; padding: 10px; font-size: 16px; border: none; + width: 100%; border-radius: 5px; } @@ -72,30 +123,28 @@ h1 { margin-bottom: 10px; box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1); white-space: pre-wrap; -} /* TODO: class for common properties of user-input and ai-message */ - -#send-button, -#close-button { - background-color: #4caf50; - color: #fff; - padding: 10px 20px; - font-size: 16px; - border: none; - border-radius: 5px; - cursor: pointer; } -#send-button:hover { - background-color: #45a049; -} +/* TODO: class for common properties of user-input and ai-message */ -#close-button { - position: absolute; - bottom: 20px; - right: 20px; - background-color: #f44336; -} -#close-button:hover { - background-color: #d32f2f; + +.icon-container { + display: inline-block; + text-decoration: none; + padding: 10px; + border-radius: 50%; + /* Makes it a circle */ + background-color: #007bff; + color: #fff; + /* Text color */ + transition: background-color 0.3s ease; + /* Smooth background color transition */ + margin-left: 30px; } + +#audio-input-icon { + width: 24px; + height: 24px; + vertical-align: middle; +} \ No newline at end of file diff --git a/static/images/mic-icon.png b/static/images/mic-icon.png new file mode 100644 index 0000000000000000000000000000000000000000..8a07e2e3b483825647888899226213ecbb3f2a04 GIT binary patch literal 14041 zcmc(Gc|4ST)b}+r7{u5`c8M&J#yS+$U?gN|w-pT{>tvE`q&xerkTs^HO*L6ZjN2{S zSVjwCGIxa`hNO_~{muQ{@B2K@fA2r<^Y;1l88g>)o!>d zkgzQd>kL6~@FyJN=K~+h*WUdGA3WhlY+d=mzi57+B=B7z1a~$Zf{A8v|HG1>3^jls zWeHYi2rj{R0`WrFC5T8Q>iP!-gnM5IxuhE$=3B%u-vvR+kS+GG>$T!}W@MiG$-=Gg z3r~;zDRUFL!4HS2;na9O@$i?c`8_V*mxOgXeEx7pkpGL?&A*`=^Mk1L6n^0(gQSn; z!gU`%Tf$Sym(*2Epc^oxsE_CooU-lv&CO5;pJR`Lm@~}v;%UmMeS`N;I-Z<=J>cRy zQpE@~63cEb9EBF;y@r>*oTa^?{iP8d*nB4rc}?cbsjVN?+bvTT+p-P!H&zp2Y||;XCn>K(yRy z0d?CPlZKN5ixUb9`}hu7hHA7DILBC(oI@6vB;0L4laO4fa4eKHp@*@va<_5fX^~UF zk_BBMEXE5joI+ij5Sf#euhI>I4q1O+CQ0wlHUiXMb<=}4 z%$tYNx3+iFEZ_-kp|e5!CK9Vzzr;ml<)ol&{r!|-SQ)G?qhHlA%y?0cv-O^>tlD@2 zCh8wwXn)UyX<>#*P4hd{aVfs2G9iiVz-#HlgRg^y99YzIC1!3tI%^erXUc%+DW+hp zb`psD?ZnpmIDy3tw$9KUPk8uMBTJw8u$aDwILE?K2B|V3uD?{|+8R|dyJ}Dt%GTIV z5spXi{l&?jHBq`&Mb*nhspbd*zH$oZ`P>BO6ickGe;%e~b7EZxqS=ycaT325KA3DW zS(6ZVb>1P=mX*#pjCC$YL(TzsBSjOrlBJrWAMy3n^+!MqCs&AE*Yj<20`ib9u{shh za;=uLBdf_7mUZp`r7wN~TDSJiK4F3I;BZ!KkYY2&Qc$%5dJMIC==phqwC$hvG`sQ5 zOt8RjTVvRp4KO*X4j%4n0ttg=Dkz z=EnqA51Zb>YS{onophsnBa>IE*g83{OGpd1y^oQdfgYUcfd^S(Ue^iI)oC-d$;lD% zY9S!55Lyn><-^~dySZiPXCE8%x}#+(V4{k->H0xT()&1Bd|8s@3Iu`*kU~IV z$K>Es4eOb}8VJXDEnoWL&%=$OF|2R)xrQAhZ@K z%4!2qGYVe|uHp$9hZ>|;jfvYHgsg~y(O!}WoKitqZWH5dSuj+vZ;k-67x8j1W7wFL z@65e5N#s@}<8V_Oed>f{yw<|ts*RU1F-{FQ?c-MxpyR$4pjHFDbbkI zjqCDWaVZNgQtB#Yid08|19jgjJec|UP3t3O=+;iwYfkJ1LRRd4iZ{s-Hjvgobm{Tl zApQ+|pruo6A=1~c#~)o?RO0K0M?CFqQ!3x^Al_g7l$h{^@b+4YzE%!z2`o#eU{sEj zS`kiQ1e1)Ls5Z!CA3Wd2&c$Z7jyvHMHEs{)W|yBUq!x+)CB%~UdJKMEy2L6QkBKB? zNfwOWqU|KbCG`*OSvgVsK5{Q6sS8ICunb$g@F;XKj>qH;bK*(D_LuRPMrv!}#RZbq zxK*oEM}d^>-F^r+#5HG>*CuOk$#1C;GIc^;LG2RWh_^wqU{}fgqnU8ChV_HMT|QX( zLTXnR)VCA1Z_{)XHf&CBrQw}6=wWq-I!2^y?*LLS>49k0ieyby-}^n1#(z9bL*+=I zi!4LmPLH#76t2mPd}40zQo#CLA@Er`N5CVMHU@YyhT7uhddfzfzHXjjVpH^Wa@2r6 z)Cx!6NXBhCrt}Xz5TUMDGgKbY=ng6qMYD?KCOe+%00~`y0g`90Egf8qb4G4+G~@1+ zrS&r&*h+ezRWrpPl)LpQKjX*ZpH@;{{S7D9^!%8!)u-qof>Zh#x2W+7%uNPxXh$*{ z%BiT&nr_=RqlGe5wB}4w%0h$c9>}~^wRM7JNp6M{NJefu=vPX=TMn7^3QR*4UbmVCFe+4A1Ycf-Iii4xZIlC8VoOO*jEPXe5Z`{sKmyMvXtll&ET$V%db^Ee<(L=rs>%t6`uiZTp z-LeeRbFYF;r>u@tLpZ@HEYgp$N$X~)_>?$2SXG!*&ezwIdH`3}?7F1CQB1XvS2*(F z*#LteW5{@3u7#{T~DRrmY4iQo~S;CuAIonBP7{63SnOrU9>qsb2W?r$x`HFeJQ=B%HkkQsM_<^rJ?jwCK}fNWQ!g! znp~JOW|&&P*wY~|Dxj~6A-(%}m;pN%^7B*F0`r%;6#{{1Pq+Tk^7r3=Q<(=7zYYCO z4{VQ*kAGU!HznQP-hO2}d&tk=8DHsehu`ZE(Zyc2sIwzrU}lX=CAyMf(!$&hr;L(E zbs~g~DEDbf@e9U9eZ9nw{#5~bjbC%z>T2eb8^fnjTO49*(R)7w6vE@kcWjq>dwYkS zv|M^mnaaQg6wV5$o$OwTSRPfUacCUC64UOEPGj(!G+9r7qavlNE9({QKXVJOF#HS- z4F(!b4Eh<=CG|@@3<|dtA3F16I2qSkILf4*Vwj3Qmt2)xR65yu+zT%UU*P*>=ae+d zQ6x@aFgM@j3X*Puv@O2XdKyPp+Liw1!mwxDW(8aM^?vU+X)}HJA7Lz0##wPnp;2KD z|8+sj3ii9{w+E|613l2-hTRnjgt0VPbcUZ%(QtXf_D4Tm9KRHlTwxVFZxm=$lz(?L z3fz)f_?r8KV4zIN9;drIxEoX9$2eclx7aO<#y4XyxT>CozjGxga-Wn;-MD!ImbKTV z0-^Q{q{ez90X455SRws~J)-6fp8_f*)K(?KElU(=-74;9T2Lj}o?!bD%+)V!WpBf) zv+vT8e0(jufs!SA5dW?H4;~!o?>&^f8<1uA5+`L2uF!5u`h3WJ`AM z?mhMNT9jV|ZveerL_EA%--@n8meTY-4gFHk}DTO1m+=%(-Uu>vw^xAkss8R}w z@#w%~)RuRpsY(nmZn0M*=}y21@U!o#B|s4Q=M!jEpJ94{{GYcl)xSNh|1`%It<*$c zPMvu1wm7Q=_#r>$;P|bj^hn!Y74HfI4e|mo`Wssx{RdQlab+ESBkAven*fHvvoB(P zG~(CXXm`t>RC?VNLdNm`eWipGI6zC0($0yw8dzu_a4d>T#^CH8E}6H3gM&s4*$;ar z17!#QUGbx0ojow?+zRv)yPnYl<^S?%!1-vffB#|?Tj*O(Se=j`Sl53r*@rwDsGHpX zTvyXqBD=~3CfeP}Yfp=C_}~ z{x1-cB5sdv8-lf;lz__T?xA^3Sq6$xy%g28wJut%3T}O+gUBGRtXLyw?kq)_mVSj!R|%AOpGsN{`LWyI+~D>Kx|rH3X17}q?tM>HHn1qb$&nnY7K^3PH>b{ORX<*^xCSCtgcX4n zd)F}F10Uea9CqysxFZPa!=p`Z<6g7#3UI|tlH(J%#w8=PSp;XiF@X_({R;OJA2HW+ zTRRwW=AeWMWV>yUwZtBv;UFK+hf&f%psN={Icc(?_A=PPhx zdD4qbfBXRw;VV=0LOpp!#Vy|b`!K0VlAv-Odbzf^xG2&$2gf8{~}Z+QI1L&r#ypH@ODw{~W~Jmjj2^&5bcQ>aS7XvT38DK08v+ z#Q5Hv-CU=+f-x{yZl(ARcvxV%&h(WYjM}_+@7^}AQF_0sY5<6eId$&&`_4xkAtb%f zLNDwBKKcRF4{D*nPFe=ipnikE@2KprjzZH;BZ??c-ANM@L z{M5_gvT5EN$qjqKOm}>AH~}Rl%XLph^?mM5%e?;tjABV{1mLcj4EE&=;TD4%eEtbV zCI%SG9o*0Zr26WUm^UGF20x|(eEgL)U`7d1&x{y5#ZdWhpZ>tmN=@SK6V*uDHN&QV zVjVYn;VKy;s+jgBaGxz=X==i$z_xqYVR*7fw z4~)GA0gh}4_H(W4Z@oGB11bFAZ_~B^8Xs*@$l-r4H2MK|jyw2IXzT;=>i~u;{l>zd zf9!~pP`GGt>CJXJ2)LEpAZ$q)-M|uVotn%$N$%%vHSzro-W{>h3Kb_WeRy0Ep48RF z{nbdQ82lt$<>a$78_QMjJeF9d^v`jqVGWdi70|LB))?RX!BF{l5Q zBJU3AaTx`2(bPa$>n@Y43$IsNo-q8PWk<9zxk;du!U=BiH2VTV>#p;xZR`$dorX~E zH)!GBYN?eP2q(BS^~IyC#qL~<1s11FbKCjZtF<4~H4!TV$4kv4k*tuEviLfbv_QEX zq`LWiZ%1m_vnvL>e()iY)7x{t7E}$4Yo48MqYrMsOoT-ILM4mIe?TLPA3Pe$Kup>VjY}(1&-4n|Wq+_GXJmEXc8Ty{&SL`_K^j zrSzbIfx*brnzw!ivHXGKPX{z|xF?8)T#x@4Q%fs#$bHggxM1{zu|*kSG%a6n&W!ltfo|Q*&^JH@kas?*Egfw60pfn`O_m?kEeBT0F@@aAJLT9x8kCM-Y52`=x-VaIqy3wFMP{ z9%k%`J3GX(PvGkmLWDbH`-9{+U~dq?JkQV+X4A6iB#0;1Y4iwqtl%7@QZE!*lt7O? zuqpGz`nckMnX}H7pGMBilLF|06N)ig=Uiay1L(1Y&n%0^PS49)Ax{hEhrZP8R$Vff zJiMkMN&LAoq{cdi?)@ch`^Xh)>Y}T|^AYEN&e#QsY-AQb9j>a;P`y!j1?6TP8p25w zj<=)Fna2FSURw?ybgx2~{aOofxZl7yfhBwP^VD&E3>Whk2jkNZ#W8!FVQg`96z}4l za=mR2bdl6@trgjtK4(Y^xY4rFMjK0LiI{u{5#kH42o=l8T!2o2za{q3r6 z-l{`8CeIO%5Hg0w`T}#=qK!cO@c_-zllE<{tUqxL`VtvKHD zm`rfkb=+h_yRR8wNVtp6r4pJod$P9973)ZmejSuo@RF3MM5M9qanmx}NC$HB9n0SV zp)XybkNLmL?IMRBG>;@>asTZ?y$*)9*SYkS2o@=7d%P!Hb+VRc$TKzfK73G}7{?=% z(XW3aPh)-)XjD9RsTdczPgZyMqJIKVwR2FZrsK!Foo3nmnghvs!#R1wtgqD&L47j7 zv4|VtUjOGqwvoaPrQBx}qc!V)MS(P2O?}cdd+A)x(yn52DN;W<`nTq!Z4{9JstOPi zojvTFJrZ)O;Dd#EW4=APzgqrNCUP^Mh9j^JO_3HKji*$R++Pxss+$IG{mHI-sCTk{ z{Y%h%6^I4dRf45de>P_Qm!rAv;gjd)MG;ZD8` zz-j(8H1Yq3p%qjr%L%*CjxyDzklY4Kq4R$nxP%@U zE%JOnAYL}Pf6B$fZ0p3c*F!KV5(l^D`VNn;J5NY*as#R+59BERn7sSq>_1{n;g9l` zn+0W{_NSPFqc~K>-_u%b=7D-gf#ccP0PER+AZZW+cmFiHyX+IOGD)kc3{P6_uIbe; zdr2M^8uARMfKaSZE%GLTuJd!^=QOb3W!oX|*KXRVlz#n`cf;d499Za|RlYc|vO(Vgx0$C%bL0{_gYq{2S^HTm#qczS z3?Jm>d{WySc7AZE8s?CUQXcX)P5b_CIA!8az%v2}15wVc+CMGPAj7+;#)%bPJV8KZ zD@Hz)|I)2BoP?+hPgDI_Yb2)yF>{yy;xppITPOOp)pM7@p}eEiJI-ald^h|t5Cr9c zW;V1BxA{(yh6G;1y=<0fsG3-7v_lJW|I9H)5Ur+Vtd?@enA`m0iX}(My+L}N5wp&o zyE$6J=VstQ=XaGr(4o)+H`${5O!UE~%(g zVl=GRlPqk&?&etJdt!d!gM+8@eGx7%hkY7uJl}-yA#mXTN(aJIkDCd%(L7X6Cs&gJ0O!?+p7wvJQ_LCpA+`+cWnHs#W1o8PW%IsmXxs!vp zQDUufq9K=+*V*k}C71AKHi>?JP|Hx`poc=XBIbD;MNaO@`aNZl)p_@>f)wdfgP;uP zf!vV4yS=N1XPN$p{LEJWRCsp#uKmYfJUEsQ5+oAU0@BIoXWTFb{o>oy3I1p_Cb4U8 z%h`F5r-&Kv-gy8q65?}Ii|#{4DItawID;fRmIsS{}ooxE_1#Pq9SXY43| zV>62sup{ygFgS~&9S@pmNl)4ZCT04pbi*z3YYn)JhwlmTBqT}G4Y?=ys~+Y6A}G0E zWl6F=&l|mCvbapHq>^{V>B;>ekl!D4=G1Q2f)??Jq=_zA_%)mNnrV7f}%0McQ)=`VLlaAFM?XV~)2W@Wma z9!5QlG0ubx={~(-hnag%q}S6X!?Xg7S&M8@?@7+~F-0@IV}?4qoo+#>s)8@3U6=ak z;B<(K&W-l)Uyo-)S1=G$MEUg!f5(|#C*ANU(~rNXdals#@@9u?Aihc48&i$~iBGPf zV>O$UEU#AyG#j*k+sX^SejQwOzw>3l#6}C+%zdl*)NRZ8nmDnQ&%_9m==$Nsq)e}q zLpY|Y_CNr*;m2;)GvLX0FX1U*H`|#5NC(hUyd559TKCeWfATC)Heu(-_10jp_Fv4z z_*4*W+7c@euX>pl@M4LQqgr2z!;VjCRW(M3Hl8}B^blG;?NC*RFB%mC1??m!7||kE za`fd2Gfd}Fb#c3CNnfvjI^Cmk^xZ+1O&|P@?rD6n1&lfweKe!cUE~Y_)ty6KPh*SD zAE2bi7m==jsgqwebwNs@!hyh7z8!YGv7?Tr2}*`nA|X)({_g&HNpB6`s;q%FrKNLn z+TbMHZV}Q+N9pq|9H20PNROLKO{7-s2-4keJW182o~gRib5wAsv^l5LI{gN_I556> zm_-M(qWvfnZ46dpJjV3-Zj6L-B~@YMg*@8SEcGI1>N7p;O5P!7SW+lvZBB?kq`Xi; zLZL!p@B%lQ1b6P_bMX1Q)u9ugV?PY-)w5r|#V^8m*7d zx0%i&M|Uy-+@Qh*U#A(eW0B6*@m@Sjw9v<36=sAiiyaY}z(5mx;Yz6}hVbN(ni#lG zdibM|MZK7`X4F>Zz52*oPB4COO2c=8l%uk~@a!JNYVET54;k@cVf_PZ7EY$d;p-l{ zdxwt*l*2{8O~k)vsJ!ulOddzqT>I6-WPZBgq3mj7&)QeGu()D8S!|x#{9*aMWI#Dj zxBqk34C(|b+7L|AWEX$2MVZc=g)R89*58;&o@^n=s!RHaFl@#2-KEAsC)u*wJF8v*jTrMwYg*iI)FH2g&>2u<&E}(4! z;&~a>Vi#yq(C2b#Q6y`E^vommMStS@z@(_2}RK{RM4 z$^GIvB&Sd@)`R#fkYHPWOc91ZJn0S?u>O>Dh}Q&lMy$leYpsmjTYI8gFB?6R2-|&Q1SuFX+*yDG@DB@!g!-#)GXcwoahu+`X;TK6Kf4d|7eQH ziQ-E=@3T2u>1@#@FqsfQ|3|NYO*_xr9A}G)(dYb9j7tt`W9+83gqjDKdMPOJHhRYH ziRn#P%K}F)9i;-gpIHDmCA%v1+=Fn(NZ9Gsc5g)jR-bYj4k~MF-){OmEt#|Rl&$PU zp9@K$kP?8-mSy(8X9lODxV_UX6SSF>i7CbgsmSYbuH)&M&OhmOl_u%y$5^s$q0T== zJ94Fz2rCxWi2z=p$MX#Okl8;J|8{&ck1fibc_eEuY;79uH^tZ?72zHaH(Ph$BcP7- zQe@8bmr~{{!Bv5fW4ApRV*=UVQ(yP8+#97LV6S)=dV*8l`si<6Wbdk_T-selMrcsU5IAhGswBx+W68H$&kTZk~ z6w}%Xz9p`pW{9s-2@wiWd%p9ze_+M4DA6;xQ2X^GL{G(H4O9HD<0%Q-{fx61LL*g4 zQQ-*a;ARN`yxFSoTCfR4)TE24l!3}QfG*;Zw{!YkDiSDPQVaDlFlN*1_29q$5zFE1eYyU;m_+3OC>Wd?kAfaXzd6 ztx6_r1=fm;cptA<1qGIY=KHy=0V zJ64isyb+VRQL|PPdWYJ&8|+BB2VY*5gru|EK`qEJg?Jd|PD%HqVOC+QiwYBC9>jIcOyg0Z6>&SnUJAqveV>nYWi1lbn5hU^@kWt zKyyE{Ra)T)rV4&S1^=r|a;)Q2pD@jh*lGxXCLm$o$rAS(VCx(V$)L9G!`vh!>cU)m zEfB`miR}1;@x1t79|4wRw`M0B{wQ*>HTz+4%29@@2s5-WMSRKv-0w) zHXMg+<{GT>aoad4g=h$so{5Wr^?=z`fojNpQ!%b^WvI4rTaF23~%XjKV85`vlM{L!`BrV?(wCuQ@`rc9V6VmgN2x2?) zVwCf1sz*~lop0~f=WU+%kpja_<94Bor>mdsmm=-jPk9}`<{HPi@mV103zJ!y0Qz8s zFkhH-$$b$;K0I~(23vXPycT450gps%BaZ__60)5Z2VWqrAS>H(E%ZPamN#weVW<6D z?f9ePn?2z6sH-xA@H~m%!jj0&M{-aGGDx4|UD1v!?RgQtqvOn@_H(8}&4-cOzkx#U z>QlbN>j8J5l$~&sUcE{XwJj?IaKxr<8e1pk7m8@1jk&oDlqOs_s!2<%3|}6-%%XrG z-J*j@BLjpUp9Aa@Z`zKGx4}vgD=y%Oi>rp-y$4wXiwcG73k2gEr2C9-F%@D0$hEJzdeP-Tz=l|I(+7(h^CdjI%e~1e;G~_J7C~ zv`jA?fr+6J;T2PuM!~&&Um5;w}+HlkdS#gil@_cuabo zP>DUy6D`?rJWlweOXP=dAt{LMWtIs;8bZ#*7F-9j7J8r^YZt2#fGa{I7DHJ9oE8iw zjnGej1G9l?z^J^Pwr#=ji1E!&Y-MrK^@H|PrXdUL9y1$m7)dVP@v_4Q zjd$!y4Tw)AZAYxMS|?T`&|^Z^ggfF}`!m1Seom#C{q#wFk}gO}d-!0Cuikslq5|s( zYkNHAWJu{Tead&53QeEZTitnnpV|HSm`hkK=)j{T-Xosniow`#x7IZy&Nd6JR)Ykt zSi=rqcj&oA2W7p6*C^9IFZ}|mZ26*+ETH}BFM0;Kf%CMr`OTDmN-qxpKzsVd#n)<= z4VMW3Tr?i`?T=eM1IJRnZRP`WSq3PJUizF@O6z3}a7vFHNEw!>(d03H_w#pPWLbBq zc`VIBbi~BK^^+nWinyW;{6^6g@;Ocq3{H6suS@H9785xU*kLYGFwHUUcr&vPJ(k`n zvWkS|9zmS!d@VV!ss~W1YXIn?kEXbjlpx^G)fKI90Nl>scKZ=}UN|zA5)dZ9D6%28k{-j6T zAJLA>1cPL$h~4{#nZe0Gu(xk?homNNfXc{$Af26pL5|oU<7T7*3=oVsDoNYeb>1XG zdpm7xd@}&J(m4RD)#$O|)w6OGNFrR+45S^n3N$oCoIGMIe9hvy zxk&~tZ&EVc>n>_b93;}v zWa|1JFd)v+K}Q*eV4hzAl#Dh6Oc=~oun7QRDPWm%Qn%OzAW|RS_>%PMGQfM5R3x;L zj|0AT;Oopn`1S*cc^Cf_Vu7&Y31HYw`;sD<{;^3-S!KhG*95NGL z$DgtLX`{&Mu8?f1r_#R|{(bvF{A!b25u(E$+x4qVx9P=AnG4{kIIVnqE>&99*=LvgSRtA0ibFI4qUpkmnQGkIfA_%ZkDOE2=L2H z!2e5ta~J}1EQFPqiS&q`k$v4WM1Q3TYeKBpiVUhx?Da1CC!T0fJv`WIV}gjI;(W3x z+4ld({OJG7(CYtcQ80;ys*&gb=sdy}F7d{&0bccdB{?i!yt@lQlcLp=w8vxo!ClHy(2N$^O!S&k2>&3k z3TNKOq#6XbO2Z~*&w`l$Y(HfutO~Ynuyk*Hg!v*b!KE5! zu_3k^_0Ov(U+s6)fw7?jq9y4~4z4v2>E?~ENkOSbzLkaiQwxP5UqmODR<-)#0JnM- z?atl0oExzzfmuz(UWUpHL4z&&2ZjkLm^5u5{s5{TB3AQbv{HiFk5 za#HL`+xnEIz-N2hE8xoB7xBRmNP1LM9balM*0Du%c1l6Ak5|CBF1r>pN>{f|%!&q{ z-P_~7^?-}psr@Jp8u2&3g0$kz=Mpd+^1<5JAhV*`nvj2T;T8T3sRB{3vh{K_ar-2J z-?pqD!aYV=BIB~<<)sxYm&)PFPwz?q@W{EP3O1C3l!E+ke_GPWNu0(av&_Ltk0ZcL zLUmu{DFW1;05XOiy3vu;6rAn{V~Q<9MJ+}P8cOlC0qp6Is$xiFM_$Ww_d->3cJ`f> mz0H^WKLSkqZveQ#ZDdJbn(%$C8174Kkgb&i_Sq4y8~+Q9c9^sP literal 0 HcmV?d00001 diff --git a/static/js/audio.mjs b/static/js/audio.mjs new file mode 100644 index 0000000..a072375 --- /dev/null +++ b/static/js/audio.mjs @@ -0,0 +1,126 @@ +import { handleUserInput } from './script.mjs'; +import { franc } from 'https://esm.sh/franc@6?bundle'; +import { iso6393To1 } from './iso6393-to-bcp47.mjs'; + + +var recognition = new (window.SpeechRecognition || window.webkitSpeechRecognition)(); +var isSpeakMode = false; +var isNowSpeaking = false; + + +export function textToSpeech(message, isByClicking) { + //Only in speak mode the new message will be spoken automatically + //every time when a new reply is received, this function will be called from script.mjs. + //But script.mjs doesn't hold the information whether it is speak mode now, so this condition will be checked here firstly + + if (isNowSpeaking && isByClicking) { + console.log('User stopped the speaking'); + speechSynthesis.cancel(); + return; + } + + if ((isSpeakMode || isByClicking)) { + console.log('message to speak:', message); + var utterance = new SpeechSynthesisUtterance(); + + utterance.onstart = function (event) { + console.log('Speech started'); + isNowSpeaking = true; + }; + + utterance.onend = function (event) { + console.log('Speech ended'); + isNowSpeaking = false; + }; + + utterance.onerror = function (event) { + //In tested chrome browser the speaking sometimes abrupts without calling onerror or onend + //but with Edge it is ok. + console.error('Speech synthesis error:', event.error); + isNowSpeaking = false; + }; + + var detectedLanguage = franc(message); + console.log("Detected language code by franc:", detectedLanguage); + utterance.lang = mapISO6393toISO6391(detectedLanguage) || 'en-US'; + //for code "zh", chrome uses mandarin, and edge uses cantonese :O maybe use BCP47 language code will be exacter + console.log("Language code in ISO639-1:", utterance.lang); + utterance.text = message; + utterance.rate = 1.3; + speechSynthesis.speak(utterance); + } +} + +export function setUpSpeechRecognition() { + recognition.continuous = true; // Enable continuous recognition + + recognition.onerror = (event) => { + console.error('Speech recognition error:', event.error); + }; + + recognition.onstart = () => { + console.log('Speech recognition started.'); + }; + + recognition.addEventListener('result', (event) => { + const transcript = event.results[0][0].transcript; + console.log('Recognized text:', transcript); + recognition.stop(); + let message = { value: transcript }; + handleUserInput(message); + }); +}; + + +function toggleMode(btn) { + console.log("Mode toggled. IsSpeakMode:", isSpeakMode); + + var speakModeDiv = document.getElementById('speak-mode-container'); + var writeModeDiv = document.getElementById('write-mode-container'); + + if (isSpeakMode) { + btn.textContent = 'Write Mode'; + speakModeDiv.style.display = 'block'; + writeModeDiv.style.display = 'none'; + + var LanguageDropdown = document.getElementById("language-dropdown"); + LanguageDropdown.addEventListener('change', () => { + //console.log('change language'); + setSpeechRecognitionLanguage(LanguageDropdown.value); + }); + + var audioInputIcon = document.getElementById("audio-input-icon"); + audioInputIcon.addEventListener('click', () => { + recognition.start(); + }); + + setUpSpeechRecognition(); + setSpeechRecognitionLanguage(LanguageDropdown.value); + } else { + btn.textContent = 'Speak Mode'; + speakModeDiv.style.display = 'none'; + writeModeDiv.style.display = 'block'; + var userInput = document.getElementById("user-input"); + userInput.focus(); + } +} + +function setSpeechRecognitionLanguage(lang) { + console.log('set language to: ', lang); + recognition.lang = lang; +}; + + +function mapISO6393toISO6391(iso6393Code) { + return iso6393To1[iso6393Code]; +} + +document.addEventListener("DOMContentLoaded", function () { + + var modeToggleButton = document.getElementById("mode-toggle-button"); + + modeToggleButton.addEventListener("click", function () { + isSpeakMode = !isSpeakMode; + toggleMode(modeToggleButton); + }); +}); \ No newline at end of file diff --git a/static/js/iso6393-to-bcp47.mjs b/static/js/iso6393-to-bcp47.mjs new file mode 100644 index 0000000..5edfd9e --- /dev/null +++ b/static/js/iso6393-to-bcp47.mjs @@ -0,0 +1,419 @@ +/** + * Map of ISO 639-3 codes to ISO 639-1 codes. + * + * @type {Record} + */ +export const iso6393To1 = { + "aae": "sq", + "aao": "ar", + "aar": "aa", + "aat": "sq", + "abh": "ar", + "abk": "ab", + "abv": "ar", + "acm": "ar", + "acq": "ar", + "acw": "ar", + "acx": "ar", + "acy": "ar", + "adf": "ar", + "aeb": "ar", + "aec": "ar", + "afb": "ar", + "afr": "af", + "ajp": "ar", + "aka": "ak", + "aln": "sq", + "als": "sq", + "amh": "am", + "apc": "ar", + "apd": "ar", + "ara": "ar", + "arb": "ar", + "arg": "an", + "arq": "ar", + "ars": "ar", + "ary": "ar", + "arz": "ar", + "asm": "as", + "auz": "ar", + "ava": "av", + "ave": "ae", + "avl": "ar", + "ayc": "ar", + "ayh": "ar", + "ayl": "ar", + "aym": "ay", + "ayn": "ar", + "ayp": "ar", + "ayr": "ay", + "azb": "az", + "aze": "az", + "azj": "az", + "bak": "ba", + "bam": "bm", + "bbz": "ar", + "bel": "be", + "ben": "bn", + "bhr": "mg", + "bis": "bi", + "bjn": "ms", + "bmm": "mg", + "bod": "bo", + "bos": "sh", + "bre": "br", + "btj": "ms", + "bul": "bg", + "bve": "ms", + "bvu": "ms", + "bzc": "mg", + "cat": "ca", + "cdo": "zh", + "ces": "cs", + "cha": "ch", + "che": "ce", + "chu": "cu", + "chv": "cv", + "cjy": "zh", + "ckb": "ku", + "cmn": "zh", + "coa": "ms", + "cor": "kw", + "cos": "co", + "cpx": "zh", + "cre": "cr", + "crj": "cr", + "crk": "cr", + "crl": "cr", + "crm": "cr", + "csw": "cr", + "cwd": "cr", + "cym": "cy", + "czh": "zh", + "czo": "zh", + "dan": "da", + "deu": "de", + "div": "dv", + "dty": "ne", + "dup": "ms", + "dzo": "dz", + "ekk": "et", + "ell": "el", + "eng": "en", + "epo": "eo", + "esi": "ik", + "esk": "ik", + "est": "et", + "eus": "eu", + "ewe": "ee", + "fao": "fo", + "fas": "fa", + "fat": "ak", + "ffm": "ff", + "fij": "fj", + "fin": "fi", + "fra": "fr", + "fry": "fy", + "fub": "ff", + "fuc": "ff", + "fue": "ff", + "fuf": "ff", + "fuh": "ff", + "fui": "ff", + "ful": "ff", + "fuq": "ff", + "fuv": "ff", + "gan": "zh", + "gax": "om", + "gaz": "om", + "gla": "gd", + "gle": "ga", + "glg": "gl", + "glv": "gv", + "gnw": "gn", + "grn": "gn", + "gug": "gn", + "gui": "gn", + "guj": "gu", + "gun": "gn", + "hae": "om", + "hak": "zh", + "hat": "ht", + "hau": "ha", + "hbs": "sh", + "heb": "he", + "her": "hz", + "hin": "hi", + "hji": "ms", + "hmo": "ho", + "hrv": "sh", + "hsn": "zh", + "hun": "hu", + "hye": "hy", + "ibo": "ig", + "ido": "io", + "iii": "ii", + "ike": "iu", + "ikt": "iu", + "iku": "iu", + "ile": "ie", + "ina": "ia", + "ind": "ms", + "ipk": "ik", + "isl": "is", + "ita": "it", + "jak": "ms", + "jav": "jv", + "jax": "ms", + "jpn": "ja", + "kal": "kl", + "kan": "kn", + "kas": "ks", + "kat": "ka", + "kau": "kr", + "kaz": "kk", + "kby": "kr", + "khk": "mn", + "khm": "km", + "kik": "ki", + "kin": "rw", + "kir": "ky", + "kmr": "ku", + "knc": "kr", + "kng": "kg", + "koi": "kv", + "kom": "kv", + "kon": "kg", + "kor": "ko", + "kpv": "kv", + "krt": "kr", + "kua": "kj", + "kur": "ku", + "kvb": "ms", + "kvr": "ms", + "kwy": "kg", + "kxd": "ms", + "lao": "lo", + "lat": "la", + "lav": "lv", + "lce": "ms", + "lcf": "ms", + "ldi": "kg", + "lim": "li", + "lin": "ln", + "lit": "lt", + "liw": "ms", + "ltg": "lv", + "ltz": "lb", + "lub": "lu", + "lug": "lg", + "lvs": "lv", + "lzh": "zh", + "mah": "mh", + "mal": "ml", + "mar": "mr", + "max": "ms", + "meo": "ms", + "mfa": "ms", + "mfb": "ms", + "min": "ms", + "mkd": "mk", + "mlg": "mg", + "mlt": "mt", + "mnp": "zh", + "mon": "mn", + "mqg": "ms", + "mri": "mi", + "msa": "ms", + "msh": "mg", + "msi": "ms", + "mui": "ms", + "mvf": "mn", + "mya": "my", + "nan": "zh", + "nau": "na", + "nav": "nv", + "nbl": "nr", + "nde": "nd", + "ndo": "ng", + "nep": "ne", + "nhd": "gn", + "nld": "nl", + "nno": "no", + "nob": "no", + "nor": "no", + "npi": "ne", + "nya": "ny", + "oci": "oc", + "ojb": "oj", + "ojc": "oj", + "ojg": "oj", + "oji": "oj", + "ojs": "oj", + "ojw": "oj", + "orc": "om", + "ori": "or", + "orm": "om", + "orn": "ms", + "ors": "ms", + "ory": "or", + "oss": "os", + "otw": "oj", + "pan": "pa", + "pbt": "ps", + "pbu": "ps", + "pel": "ms", + "pes": "fa", + "pga": "ar", + "pli": "pi", + "plt": "mg", + "pol": "pl", + "por": "pt", + "prs": "fa", + "pse": "ms", + "pst": "ps", + "pus": "ps", + "qub": "qu", + "qud": "qu", + "que": "qu", + "quf": "qu", + "qug": "qu", + "quh": "qu", + "quk": "qu", + "qul": "qu", + "qup": "qu", + "qur": "qu", + "qus": "qu", + "quw": "qu", + "qux": "qu", + "quy": "qu", + "quz": "qu", + "qva": "qu", + "qvc": "qu", + "qve": "qu", + "qvh": "qu", + "qvi": "qu", + "qvj": "qu", + "qvl": "qu", + "qvm": "qu", + "qvn": "qu", + "qvo": "qu", + "qvp": "qu", + "qvs": "qu", + "qvw": "qu", + "qvz": "qu", + "qwa": "qu", + "qwc": "qu", + "qwh": "qu", + "qws": "qu", + "qxa": "qu", + "qxc": "qu", + "qxh": "qu", + "qxl": "qu", + "qxn": "qu", + "qxo": "qu", + "qxp": "qu", + "qxr": "qu", + "qxt": "qu", + "qxu": "qu", + "qxw": "qu", + "roh": "rm", + "ron": "ro", + "run": "rn", + "rus": "ru", + "sag": "sg", + "san": "sa", + "sdc": "sc", + "sdh": "ku", + "sdn": "sc", + "shu": "ar", + "sin": "si", + "skg": "mg", + "slk": "sk", + "slv": "sl", + "sme": "se", + "smo": "sm", + "sna": "sn", + "snd": "sd", + "som": "so", + "sot": "st", + "spa": "es", + "spv": "or", + "sqi": "sq", + "src": "sc", + "srd": "sc", + "sro": "sc", + "srp": "sh", + "ssh": "ar", + "ssw": "ss", + "sun": "su", + "swa": "sw", + "swc": "sw", + "swe": "sv", + "swh": "sw", + "tah": "ty", + "tam": "ta", + "tat": "tt", + "tdx": "mg", + "tel": "te", + "tgk": "tg", + "tgl": "tl", + "tha": "th", + "tir": "ti", + "tkg": "mg", + "tmw": "ms", + "ton": "to", + "tsn": "tn", + "tso": "ts", + "tuk": "tk", + "tur": "tr", + "twi": "ak", + "txy": "mg", + "uig": "ug", + "ukr": "uk", + "urd": "ur", + "urk": "ms", + "uzb": "uz", + "uzn": "uz", + "uzs": "uz", + "ven": "ve", + "vie": "vi", + "vkk": "ms", + "vkt": "ms", + "vol": "vo", + "vro": "et", + "wln": "wa", + "wol": "wo", + "wuu": "zh", + "xho": "xh", + "xmm": "ms", + "xmv": "mg", + "xmw": "mg", + "ydd": "yi", + "yid": "yi", + "yih": "yi", + "yor": "yo", + "yue": "zh", + "zch": "za", + "zeh": "za", + "zgb": "za", + "zgm": "za", + "zgn": "za", + "zha": "za", + "zhd": "za", + "zhn": "za", + "zho": "zh", + "zlj": "za", + "zlm": "ms", + "zln": "za", + "zlq": "za", + "zmi": "ms", + "zqe": "za", + "zsm": "ms", + "zul": "zu", + "zyb": "za", + "zyg": "za", + "zyj": "za", + "zyn": "za", + "zzj": "za" +} \ No newline at end of file diff --git a/static/js/script.js b/static/js/script.mjs similarity index 52% rename from static/js/script.js rename to static/js/script.mjs index c34f2fe..df0df77 100644 --- a/static/js/script.js +++ b/static/js/script.mjs @@ -1,3 +1,66 @@ +import { textToSpeech } from './audio.mjs'; + +var speechSynthesis = 'speechSynthesis' in window; + +export function handleUserInput(userInput) { + var message = userInput.value; + if (message.trim() !== "") { + addMessage(message, "user"); + userInput.value = ""; + // TODO: sanitize input; avoid code injection + // Send the message to your backend using AJAX or WebSocket + fetch('/process-input', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ message }), + }) + // and handle the response from the assistant + .then((response) => response.json()) + .then((data) => { + const reply = data.reply; + if (reply === '' || reply === null) { + console.log('Received null response from backend.'); + } + else { + addMessage(reply, 'assistant'); + } + }) + .catch((error) => { + console.error('Error:', error); + }); + } +} + +function addMessage(message, sender) { + var messageElement = document.createElement("div"); + messageElement.classList.add("message"); + if (sender === "user") { + messageElement.classList.add("user-input"); + messageElement.textContent = "You: " + message; + } else if (sender === "assistant") { + messageElement.classList.add("ai-message"); + messageElement.textContent = "AI: " + message; // BACKUP: render newlines: .replace(/(?:\r\n|\r|\n)/g, '
'); + if(speechSynthesis){ + textToSpeech(message); + messageElement.addEventListener("click", function () { + textToSpeech(message, true); + }); + messageElement.addEventListener("contextmenu", function () { //on right click + stopSpeak(); + }); + } + } + else { + console.error('Unknown message sender:', sender); + return; + } + + conversation.appendChild(messageElement); //Why conversation can be accessed from here magically? 0_0 somewhere saved this as global? + conversation.scrollTop = conversation.scrollHeight; +} + document.addEventListener("DOMContentLoaded", function () { // ---------------- USER INTERACTION ---------------- @@ -8,71 +71,33 @@ document.addEventListener("DOMContentLoaded", function () { var closeButton = document.getElementById("close-button"); var setRecordButton = document.getElementById("set-recording"); var recordStatus = document.getElementById("recording-status"); - var chatInfo = document.getElementById("chat-info"); - var newSessionHint = document.getElementById("new-session-hint"); + var recordInfo = document.getElementById("recording-info"); + var newSessionHint = document.getElementById("new-session-hint"); + + checkSpeakMode(); + chatStartUIChange(); const featureFlags = { allowRecords: true }; + function checkSpeakMode() { + var SpeechRecognition = 'SpeechRecognition' in window || 'webkitSpeechRecognition' in window; - chatStartUIChange(); - - - function addMessage(message, sender) { - var messageElement = document.createElement("div"); - messageElement.classList.add("message"); - if (sender === "user") { - messageElement.classList.add("user-input"); - messageElement.textContent = "You: " + message; - } else if (sender === "assistant") { - messageElement.classList.add("ai-message"); - messageElement.textContent = "AI: " + message; // BACKUP: render newlines: .replace(/(?:\r\n|\r|\n)/g, '
'); + if (!SpeechRecognition) { + var modeToggleBtn = document.getElementById("mode-toggle-button"); + console.error('Speech recognition is not supported in this browser.'); + changeDivDisplay(modeToggleBtn, 'none'); } - else { - console.error('Unknown message sender:', sender); - return; + if (!speechSynthesis) { + console.error('Speech synthesis not supported in this browser.'); + var speakInfo = document.getElementById("speak-info"); + changeDivDisplay(speakInfo, 'none'); } - conversation.appendChild(messageElement); - conversation.scrollTop = conversation.scrollHeight; - setFocusOnInput(); - } - - - function handleUserInput() { - var message = userInput.value; - if (message.trim() !== "") { - addMessage(message, "user"); - userInput.value = ""; - // TODO: sanitize input; avoid code injection - // Send the message to your backend using AJAX or WebSocket - fetch('/process-input', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ message }), - }) - // and handle the response from the assistant - .then((response) => response.json()) - .then((data) => { - const reply = data.reply; - if (reply === '' || reply === null) { - console.log('Received null response from backend.'); - } - else { - addMessage(reply, 'assistant'); - } - }) - .catch((error) => { - console.error('Error:', error); - }); - } - } - + }; sendButton.addEventListener("click", function () { - handleUserInput(); + handleUserInput(userInput); setFocusOnInput(); }); @@ -88,61 +113,35 @@ document.addEventListener("DOMContentLoaded", function () { } else if (event.key === "Enter") { event.preventDefault(); - handleUserInput(); + handleUserInput(userInput); setFocusOnInput(); } }); - function chatEndedUIChange() { - lockInputField(); - HideButtonContainer(); - hideChatInfo(); + changeDivDisableStatus(userInput, true); + changeDivDisplay(buttonContainer, 'none'); + changeDivDisplay(recordInfo, 'none'); } function chatStartUIChange() { - hideNewSessionHint(); - unlockInputField(); + changeDivDisplay(newSessionHint, 'none'); + changeDivDisableStatus(userInput, false); setFocusOnInput(); - showButtonContainer(); - showChatInfo(); + changeDivDisplay(buttonContainer, 'flex'); + changeDivDisplay(recordInfo, 'block'); } function setFocusOnInput() { userInput.focus(); } - - function hideChatInfo() { - chatInfo.style.display = 'none'; - } - - function showChatInfo() { - chatInfo.style.display = 'block'; - } - - function lockInputField() { - userInput.disabled = true; - } - - function unlockInputField() { - userInput.disabled = false; - } - - function showButtonContainer() { - buttonContainer.style.display = 'block'; - } - - function HideButtonContainer() { - buttonContainer.style.display = 'none'; - } - - function showNewSessionHint() { - newSessionHint.style.display = 'block'; + function changeDivDisplay(div, display) { + div.style.display = display; } - function hideNewSessionHint() { - newSessionHint.style.display = 'none'; + function changeDivDisableStatus(div, disabled) { + div.disabled = disabled; } function setRecord(flag) { @@ -179,24 +178,28 @@ document.addEventListener("DOMContentLoaded", function () { } function refreshNewSession() { - chatStartUIChange(); featureFlags.allowRecords = true; //enable chat recording by default + chatStartUIChange(); setRecord(featureFlags.allowRecords); } - closeButton.addEventListener("click", function () { saveRecords(); close_session(); }); setRecordButton.addEventListener("click", function () { - console.log("flag is now " + featureFlags.allowRecords); featureFlags.allowRecords = !featureFlags.allowRecords; + console.log("chat recording flag is now " + featureFlags.allowRecords); setRecord(featureFlags.allowRecords); }) + window.addEventListener('beforeunload', function (event) { //before browser is closed + if(featureFlags.allowRecords){ + saveRecords(); + } + }); // ---------------- IDLE REFRESH ---------------- const idleTimeoutDuration = 3 * 60 * 1000; @@ -210,16 +213,13 @@ document.addEventListener("DOMContentLoaded", function () { }, idleTimeoutDuration); } - const inputElements = document.querySelectorAll('input, textarea'); - inputElements.forEach(function (input) { - input.addEventListener('input', startIdleTimer); - }); - + // Listen for clicks and input on the document + document.addEventListener('click', startIdleTimer); + document.addEventListener('input', startIdleTimer); + // startIdleTimer(); // TODO: remove? - - // ---------------- INSTRUCTIONS FROM BACKEND ---------------- var socket = io(); socket.on('connect', function () { @@ -227,13 +227,13 @@ document.addEventListener("DOMContentLoaded", function () { }); socket.on('instruction', function (instruction) { - type = instruction.type + let type = instruction.type; if (type == 'refresh_session_timer') { addMessage(instruction.goodbye_msg, 'assistant'); chatEndedUIChange(); saveRecords(); setTimeout(close_session, instruction.timer * 1000); // TODO: clear timer if second call here - showNewSessionHint(); + changeDivDisplay(newSessionHint, 'block'); } else { console.error('Unknown instruction:', type); diff --git a/templates/index.html b/templates/index.html index c5213a2..63e818f 100644 --- a/templates/index.html +++ b/templates/index.html @@ -1,33 +1,72 @@ + PartyGPT - + +

PartyGPT

-

Chat will be recorded.

- +
+

Click on the response of our special guest, it will speak to you! +
Click again to stop him talking 🤐 +

+
+
+

Chat will be recorded.

+ +
+
- + +
+ +
+
+
+
- +
+
- + + + - + + \ No newline at end of file From fb6b23c92b654c78aef56f56562857512de1dd13 Mon Sep 17 00:00:00 2001 From: Yixin Date: Tue, 3 Oct 2023 13:28:15 +0200 Subject: [PATCH 6/6] add doc links for used lib --- static/js/audio.mjs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/static/js/audio.mjs b/static/js/audio.mjs index a072375..1527b65 100644 --- a/static/js/audio.mjs +++ b/static/js/audio.mjs @@ -1,8 +1,8 @@ import { handleUserInput } from './script.mjs'; -import { franc } from 'https://esm.sh/franc@6?bundle'; +import { franc } from 'https://esm.sh/franc@6?bundle'; //https://github.com/wooorm/franc import { iso6393To1 } from './iso6393-to-bcp47.mjs'; - +//https://developer.mozilla.org/en-US/docs/Web/API/Web_Speech_API/Using_the_Web_Speech_API#speech_recognition var recognition = new (window.SpeechRecognition || window.webkitSpeechRecognition)(); var isSpeakMode = false; var isNowSpeaking = false; @@ -21,6 +21,7 @@ export function textToSpeech(message, isByClicking) { if ((isSpeakMode || isByClicking)) { console.log('message to speak:', message); + //https://developer.mozilla.org/en-US/docs/Web/API/SpeechSynthesisUtterance var utterance = new SpeechSynthesisUtterance(); utterance.onstart = function (event) {