/* FluentGerman.ai — Voice module (API-only TTS, browser + API STT) */ class VoiceManager { constructor() { this.mode = 'browser'; this.recognition = null; this.isRecording = false; this.isDisabled = false; this.lastInputWasVoice = false; this.mediaRecorder = null; this.audioChunks = []; this.onResult = null; this.onStateChange = null; this.browserSTTSupported = false; this.apiAvailable = false; this.onProcessing = null; // New callback for "Transcribing..." state this.currentAudio = null; // clip currently playing, if any this._resolvePlayback = null; // settles whoever awaits playAudio() this._resetPlayerUI = null; // puts that clip's player back to rest } async init() { this._initBrowserSTT(); try { const response = await api('/voice/config'); if (response?.ok) { const config = await response.json(); this.mode = config.voice_mode; this.apiAvailable = config.voice_api_available || false; console.log('[Voice] Server mode:', this.mode, '| API available:', this.apiAvailable); } } catch (e) { console.warn('[Voice] Could not fetch config, using browser mode'); this.mode = 'browser'; } // Determine best STT method if (this.mode === 'browser' && !this.browserSTTSupported) { if (this.apiAvailable) { this.mode = 'api'; } else { this.isDisabled = true; } } else if (this.mode === 'api' && !this.apiAvailable) { if (this.browserSTTSupported) { this.mode = 'browser'; } else { this.isDisabled = true; } } console.log('[Voice] Final mode:', this.isDisabled ? 'DISABLED' : this.mode); } _initBrowserSTT() { const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition; if (!SpeechRecognition) { this.browserSTTSupported = false; return; } this.browserSTTSupported = true; this.recognition = new SpeechRecognition(); this.recognition.continuous = false; this.recognition.interimResults = false; this.recognition.lang = 'de-DE'; this.recognition.onresult = (event) => { const text = event.results[0][0].transcript; console.log('[Voice] STT result:', text); this.lastInputWasVoice = true; if (this.onResult) this.onResult(text); }; this.recognition.onend = () => { this.isRecording = false; if (this.onStateChange) this.onStateChange(false); }; this.recognition.onerror = (event) => { console.error('[Voice] STT error:', event.error); this.isRecording = false; if (this.onStateChange) this.onStateChange(false); if (event.error === 'not-allowed') { showToast('Microphone access denied. Allow it in browser settings.', 'error'); } else if (event.error === 'no-speech') { showToast('No speech detected. Try again.', 'error'); } }; } async startRecording() { if (this.isDisabled) { showToast('Voice requires Chrome or Edge (HTTPS).', 'error'); return; } // Optimistic UI updates moved inside specific start blocks to prevent "fake" recording state // if hardware access fails or takes time. if (this.mode === 'api') { if (typeof MediaRecorder === 'undefined') { showToast('This browser cannot record audio.', 'error'); return; } try { const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); this.audioChunks = []; // Safari/iOS has no webm — let it fall through to mp4 const mimeType = VoiceManager.pickMimeType(); this.mediaRecorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined); this.mediaRecorder.ondataavailable = (e) => { if (e.data.size > 0) this.audioChunks.push(e.data); }; this.mediaRecorder.onstop = async () => { stream.getTracks().forEach(t => t.stop()); // Use what the recorder produced, not what we asked for. // A few old WebViews report an empty mimeType even though // they recorded something else — webm is the best guess left. const type = this.mediaRecorder.mimeType || mimeType || 'audio/webm'; const blob = new Blob(this.audioChunks, { type }); await this._transcribeAPI(blob); }; // Only update UI once recording has truly started this.mediaRecorder.onstart = () => { this.isRecording = true; this.lastInputWasVoice = true; if (this.onStateChange) this.onStateChange(true); }; this.mediaRecorder.start(); } catch (e) { console.error('[Voice] Mic access error:', e); showToast( e.name === 'NotAllowedError' ? 'Microphone access denied. Allow it in browser settings.' : `Could not start recording: ${e.message || e.name}`, 'error' ); this.isRecording = false; if (this.onStateChange) this.onStateChange(false); } } else { if (this.recognition) { try { this.recognition.start(); // Recognition 'onstart' would be better, but this is okay for browser mode this.isRecording = true; this.lastInputWasVoice = true; if (this.onStateChange) this.onStateChange(true); } catch (e) { this.isRecording = false; if (this.onStateChange) this.onStateChange(false); showToast('Voice recognition failed. Try again.', 'error'); } } } } stopRecording() { if (this.mode === 'api') { if (this.mediaRecorder?.state === 'recording') { // Show processing state immediately (don't wait for onstop callback) if (this.onProcessing) this.onProcessing(true); this.mediaRecorder.stop(); } else { this.isRecording = false; if (this.onStateChange) this.onStateChange(false); } } else { try { this.recognition?.stop(); } catch (e) { /* already stopped */ } this.isRecording = false; if (this.onStateChange) this.onStateChange(false); } } toggleRecording() { if (this.isDisabled) { showToast('Voice requires Chrome or Edge (HTTPS).', 'error'); return; } this.isRecording ? this.stopRecording() : this.startRecording(); } async _transcribeAPI(blob) { if (this.onProcessing) this.onProcessing(true); try { const formData = new FormData(); // The OpenAI SDK infers the audio format from this filename, so it // has to match what the recorder actually produced. formData.append('audio', blob, `recording.${VoiceManager.extensionFor(blob.type)}`); const response = await api('/voice/transcribe', { method: 'POST', body: formData, }); if (response?.ok) { const data = await response.json(); this.lastInputWasVoice = true; if (this.onResult) this.onResult(data.text); } else { const err = await response.json().catch(() => ({})); showToast(`Transcription failed: ${err.detail || 'Unknown error'}`, 'error'); } } catch (e) { showToast('Transcription network error', 'error'); } finally { this.isRecording = false; // Stop processing state if (this.onProcessing) this.onProcessing(false); if (this.onStateChange) this.onStateChange(false); } } /** * Fetch TTS audio blob for text (API only). * Returns audio URL or null. */ async fetchAudio(text, signal) { if (!this.apiAvailable) return null; const clean = VoiceManager.stripMarkdown(text); try { const response = await api('/voice/synthesize', { method: 'POST', body: JSON.stringify({ text: clean }), signal, }); if (response?.ok) { const audioBlob = await response.blob(); return URL.createObjectURL(audioBlob); } else { const err = await response.json().catch(() => ({})); console.warn('[Voice] TTS error:', err); } } catch (e) { console.warn('[Voice] TTS network error:', e); } return null; } /** * Play audio with an inline mini-player (progress bar, seek, replay). * @param {string} audioUrl – blob URL from fetchAudio() * @param {HTMLElement} [containerEl] – element to append the player into * @returns {Promise} resolves when first playback ends */ async playAudio(audioUrl, containerEl) { if (!audioUrl) return; this.stopPlayback(); // only one clip at a time const audio = new Audio(audioUrl); this.currentAudio = audio; // Settles on end, on error, or when stopPlayback() is called const finished = new Promise(resolve => { this._resolvePlayback = resolve; }); // Visual feedback — avatar pulse const avatarContainer = document.querySelector('.avatar-container'); if (avatarContainer) avatarContainer.classList.add('speaking'); // ── Build player DOM ────────────────────────────────────────── const player = document.createElement('div'); player.className = 'audio-player'; const playBtn = document.createElement('button'); playBtn.className = 'audio-player-btn playing'; playBtn.innerHTML = VoiceManager._pauseIcon(); playBtn.title = 'Pause'; const track = document.createElement('div'); track.className = 'audio-player-track'; const fill = document.createElement('div'); fill.className = 'audio-player-fill'; track.appendChild(fill); const timeLabel = document.createElement('span'); timeLabel.className = 'audio-player-time'; timeLabel.textContent = '0:00 / 0:00'; player.appendChild(playBtn); player.appendChild(track); player.appendChild(timeLabel); if (containerEl) { containerEl.appendChild(player); } // Lets stopPlayback() return this player to a resting state const resetPlayerUI = () => { playBtn.classList.remove('playing'); playBtn.innerHTML = VoiceManager._playIcon(); playBtn.title = 'Replay'; fill.style.width = '0%'; }; this._resetPlayerUI = resetPlayerUI; // ── Helpers ─────────────────────────────────────────────────── function fmt(s) { if (!isFinite(s)) return '0:00'; const m = Math.floor(s / 60); const sec = Math.floor(s % 60); return `${m}:${sec.toString().padStart(2, '0')}`; } function updateProgress() { if (!audio.duration) return; const pct = (audio.currentTime / audio.duration) * 100; fill.style.width = pct + '%'; timeLabel.textContent = `${fmt(audio.currentTime)} / ${fmt(audio.duration)}`; } // ── Events ──────────────────────────────────────────────────── audio.addEventListener('timeupdate', updateProgress); audio.addEventListener('loadedmetadata', () => { timeLabel.textContent = `0:00 / ${fmt(audio.duration)}`; }); // Seek on track click track.addEventListener('click', (e) => { const rect = track.getBoundingClientRect(); const pct = (e.clientX - rect.left) / rect.width; audio.currentTime = pct * audio.duration; updateProgress(); }); // Play/pause toggle playBtn.addEventListener('click', () => { if (audio.paused) { // Register the replay, otherwise stopPlayback() can't reach it this.stopPlayback(); this.currentAudio = audio; this._resetPlayerUI = resetPlayerUI; audio.play(); playBtn.classList.add('playing'); playBtn.innerHTML = VoiceManager._pauseIcon(); playBtn.title = 'Pause'; if (avatarContainer) avatarContainer.classList.add('speaking'); } else { audio.pause(); if (this.currentAudio === audio) { this.currentAudio = null; this._resetPlayerUI = null; } playBtn.classList.remove('playing'); playBtn.innerHTML = VoiceManager._playIcon(); playBtn.title = 'Play'; if (avatarContainer) avatarContainer.classList.remove('speaking'); } }); // ── Playback ────────────────────────────────────────────────── try { // Wait for audio to be fully buffered before playing — but give up // if stopPlayback() cuts in while it is still loading await Promise.race([ new Promise((resolve, reject) => { audio.addEventListener('canplaythrough', resolve, { once: true }); audio.addEventListener('error', reject, { once: true }); audio.load(); // Explicitly trigger loading }), finished, ]); if (this.currentAudio !== audio) return; // stopped while loading audio.currentTime = 0; // Ensure we start from the very beginning await audio.play(); audio.onended = () => { if (avatarContainer) avatarContainer.classList.remove('speaking'); playBtn.classList.remove('playing'); playBtn.innerHTML = VoiceManager._playIcon(); playBtn.title = 'Replay'; fill.style.width = '100%'; // Reset to beginning for replay audio.currentTime = 0; this._settlePlayback(); }; audio.onerror = () => { if (avatarContainer) avatarContainer.classList.remove('speaking'); this._settlePlayback(); }; return finished; } catch (e) { console.error('Playback failed', e); if (avatarContainer) avatarContainer.classList.remove('speaking'); playBtn.classList.remove('playing'); playBtn.innerHTML = VoiceManager._playIcon(); this._settlePlayback(); } } /** * Stop whatever is playing right now. Without this the Audio object is * unreachable once playAudio() returns, so removing the player from the * DOM leaves a detached clip still talking. */ stopPlayback() { const audio = this.currentAudio; if (audio) { audio.pause(); try { audio.currentTime = 0; } catch (e) { /* not seekable yet */ } } // Leave onended attached so a later replay still resets its own player const resetUI = this._resetPlayerUI; this._resetPlayerUI = null; if (resetUI) resetUI(); const avatarContainer = document.querySelector('.avatar-container'); if (avatarContainer) avatarContainer.classList.remove('speaking'); this._settlePlayback(); } /** Release anyone awaiting playAudio() and forget the current clip. */ _settlePlayback() { this.currentAudio = null; this._resetPlayerUI = null; const resolve = this._resolvePlayback; this._resolvePlayback = null; if (resolve) resolve(); } // ── SVG icons (inline, no external deps) ────────────────────────── static _playIcon() { return ``; } static _pauseIcon() { return ``; } /** * Legacy method for backward compatibility if needed, * or for simple direct speech. */ async speak(text) { const url = await this.fetchAudio(text); if (url) await this.playAudio(url); } /** * Pick a recording format this browser supports. * Chrome/Edge/Firefox give webm; Safari and iOS only do mp4. * Returns '' to let the browser choose its own default. */ static pickMimeType() { const candidates = [ 'audio/webm;codecs=opus', 'audio/webm', 'audio/mp4', 'audio/ogg;codecs=opus', ]; if (typeof MediaRecorder === 'undefined' || !MediaRecorder.isTypeSupported) return ''; return candidates.find(type => MediaRecorder.isTypeSupported(type)) || ''; } /** File extension matching a recorded blob's MIME type. */ static extensionFor(mimeType = '') { if (mimeType.includes('mp4')) return 'mp4'; if (mimeType.includes('mpeg')) return 'mp3'; if (mimeType.includes('ogg')) return 'ogg'; return 'webm'; } /** * Strip markdown formatting from text so TTS reads naturally. */ static stripMarkdown(text) { return text .replace(/```[\s\S]*?```/g, '') // code blocks .replace(/`([^`]+)`/g, '$1') // inline code .replace(/#{1,6}\s+/g, '') // headings .replace(/\*\*([^*]+)\*\*/g, '$1') // bold .replace(/\*([^*]+)\*/g, '$1') // italic .replace(/__([^_]+)__/g, '$1') // bold alt .replace(/_([^_]+)_/g, '$1') // italic alt .replace(/~~([^~]+)~~/g, '$1') // strikethrough .replace(/^\s*[-*+]\s+/gm, '') // unordered lists .replace(/^\s*\d+\.\s+/gm, '') // ordered lists .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1')// links .replace(/!\[([^\]]*)\]\([^)]+\)/g, '') // images .replace(/>\s+/g, '') // blockquotes .replace(/\n{2,}/g, '. ') // paragraph breaks → pause .replace(/\n/g, ' ') // newlines → space .trim(); } }