This commit is contained in:
Gal 2025-07-22 20:47:22 +02:00
parent c8651f3af0
commit 2ccb7f1955
Signed by: gal
GPG Key ID: F035BC65003BC00B
10 changed files with 902 additions and 159 deletions

View File

@ -73,7 +73,7 @@ export default {
{ type: 'wg_viewing', name: 'WG Room Viewing', emoji: '🏠' }, { type: 'wg_viewing', name: 'WG Room Viewing', emoji: '🏠' },
{ type: 'burgeramt', name: 'At the Bürgeramt', emoji: '🏛️' }, { type: 'burgeramt', name: 'At the Bürgeramt', emoji: '🏛️' },
{ type: 'biergarten', name: 'At a Biergarten', emoji: '🍺' }, { type: 'biergarten', name: 'At a Biergarten', emoji: '🍺' },
{ type: 'ubahn', name: 'U-Bahn Help', emoji: '🚇' } { type: 'ber_airport', name: 'BER Airport Train Help', emoji: '✈️' }
] ]
} }
}, },
@ -83,7 +83,7 @@ export default {
'wg_viewing': '🏠', 'wg_viewing': '🏠',
'burgeramt': '🏛️', 'burgeramt': '🏛️',
'biergarten': '🍺', 'biergarten': '🍺',
'ubahn': '🚇' 'ber_airport': '✈️'
} }
return emojiMap[type] || '📍' return emojiMap[type] || '📍'
}, },

View File

@ -1,5 +1,41 @@
<template> <template>
<div class="speech-interface"> <div class="speech-interface">
<!-- Tutorial Popup -->
<div v-if="showTutorial" class="tutorial-overlay" @click="closeTutorial">
<div class="tutorial-popup" @click.stop>
<div class="tutorial-header">
<h3>Willkommen bei Street Lingo! 🎉</h3>
<button @click="closeTutorial" class="tutorial-close"></button>
</div>
<div class="tutorial-content">
<div class="tutorial-step">
<div class="tutorial-icon">🎙</div>
<div class="tutorial-text">
<strong>Sprechen üben</strong>
<p>Klicken Sie auf "Gespräch beginnen", um mit KI-Charakteren in realen deutschen Szenarien zu sprechen</p>
</div>
</div>
<div class="tutorial-step">
<div class="tutorial-icon">💡</div>
<div class="tutorial-text">
<strong>Hilfe erhalten</strong>
<p>Aktivieren Sie die "Hilfe"-Schaltfläche und pausieren Sie während des Gesprächs, um hilfreiche Phrasen zu erhalten</p>
</div>
</div>
<div class="tutorial-step">
<div class="tutorial-icon">📊</div>
<div class="tutorial-text">
<strong>Gesprächsanalyse</strong>
<p>Klicken Sie auf "Gespräch beenden", um personalisiertes Feedback zu Ihrem Deutsch zu erhalten</p>
</div>
</div>
</div>
<div class="tutorial-footer">
<button @click="closeTutorial" class="tutorial-got-it">Verstanden!</button>
</div>
</div>
</div>
<div class="conversation-area"> <div class="conversation-area">
<div class="messages" ref="messagesContainer"> <div class="messages" ref="messagesContainer">
<div <div
@ -41,11 +77,6 @@
<span class="transcription-status">{{ getTranscriptionStatus() }}</span> <span class="transcription-status">{{ getTranscriptionStatus() }}</span>
</div> </div>
<!-- Recording Stopped Notification -->
<div v-if="showRecordingStoppedNotification" class="recording-stopped-notification">
<p>🛑 Aufnahme gestoppt. Klicken Sie auf "Sprechen", um fortzufahren.</p>
</div>
<!-- Inline Suggestion Panel --> <!-- Inline Suggestion Panel -->
<div v-if="showSuggestionPopup" class="suggestion-panel"> <div v-if="showSuggestionPopup" class="suggestion-panel">
<div class="suggestion-panel-header"> <div class="suggestion-panel-header">
@ -73,7 +104,18 @@
</div> </div>
<div class="controls"> <div class="controls">
<!-- Start Conversation Button -->
<button <button
v-if="showStartButton"
@click="startConversation"
class="start-conversation-btn"
:disabled="isConnecting"
>
🎙 Gespräch beginnen
</button>
<button
v-else
@click="toggleRecording" @click="toggleRecording"
:class="['record-btn', { recording: isRecording }]" :class="['record-btn', { recording: isRecording }]"
:disabled="isConnecting || isFinished" :disabled="isConnecting || isFinished"
@ -81,16 +123,6 @@
{{ isRecording ? '🛑 Stopp' : '🎤 Sprechen' }} {{ isRecording ? '🛑 Stopp' : '🎤 Sprechen' }}
</button> </button>
<button
v-if="isRecording"
@click="forceStopRecording"
class="force-stop-btn"
title="Sofort stoppen"
:disabled="isFinished"
>
</button>
<div class="suggestion-toggle-container"> <div class="suggestion-toggle-container">
<label class="suggestion-toggle-label" for="suggestion-toggle"> <label class="suggestion-toggle-label" for="suggestion-toggle">
💡 Hilfe 💡 Hilfe
@ -131,9 +163,9 @@
@click="finishConversation" @click="finishConversation"
class="finish-btn" class="finish-btn"
title="Gespräch beenden und Bewertung erhalten" title="Gespräch beenden und Bewertung erhalten"
:disabled="messages.length === 0 || isFinished" :disabled="messages.length === 0 || isFinished || isLoadingFeedback"
> >
Gespräch beenden {{ isLoadingFeedback ? 'Feedback wird analysiert...' : '✓ Gespräch beenden' }}
</button> </button>
</div> </div>
@ -255,12 +287,21 @@ export default {
isAICurrentlySpeaking: false, isAICurrentlySpeaking: false,
lastAIResponseTime: null, lastAIResponseTime: null,
isFinished: false, isFinished: false,
conversationFeedback: null conversationFeedback: null,
isLoadingFeedback: false,
hasRequestedInitialGreeting: false,
showStartButton: true,
showTutorial: false
} }
}, },
mounted() { mounted() {
this.connectWebSocket() this.connectWebSocket()
this.loadScenarioData() this.loadScenarioData()
// Check if user is new and show tutorial
this.checkAndShowTutorial()
// Don't automatically request greeting on mount - wait for user to click start button
}, },
beforeUnmount() { beforeUnmount() {
this.disconnect() this.disconnect()
@ -272,11 +313,8 @@ export default {
if (oldScenario && newScenario !== oldScenario) { if (oldScenario && newScenario !== oldScenario) {
this.resetConversationOnScenarioChange() this.resetConversationOnScenarioChange()
} else if (newScenario && !oldScenario) { } else if (newScenario && !oldScenario) {
// Initial scenario load - request greeting if connected // Initial scenario load - don't auto request greeting, wait for user to click start button
console.log('Scenario initially set to:', newScenario) console.log('Scenario initially set to:', newScenario)
if (this.connectionStatus === 'connected') {
this.requestInitialGreeting()
}
} }
} }
}, },
@ -300,8 +338,7 @@ export default {
this.isConnecting = false this.isConnecting = false
console.log('German WebSocket connected') console.log('German WebSocket connected')
// Request initial greeting from character // Don't automatically request greeting - wait for user to click start button
this.requestInitialGreeting()
} }
this.websocket.onmessage = (event) => { this.websocket.onmessage = (event) => {
@ -617,16 +654,18 @@ export default {
this.conversationComplete = false this.conversationComplete = false
this.isFinished = false this.isFinished = false
this.conversationFeedback = null this.conversationFeedback = null
this.hasRequestedInitialGreeting = false
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) { if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
const resetMessage = { const resetMessage = {
type: 'conversation_reset' type: 'conversation_reset'
} }
this.websocket.send(JSON.stringify(resetMessage)) this.websocket.send(JSON.stringify(resetMessage))
// Request initial greeting for new scenario
this.requestInitialGreeting()
} }
// For scenario changes, show start button again so user must start new conversation
this.showStartButton = true
// Don't automatically request greeting - wait for user to click start button
}, },
async requestTranslation(message) { async requestTranslation(message) {
@ -666,7 +705,7 @@ export default {
}, },
requestInitialGreeting() { requestInitialGreeting() {
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) { if (this.websocket && this.websocket.readyState === WebSocket.OPEN && !this.hasRequestedInitialGreeting) {
// Add a small delay to ensure scenario is properly set // Add a small delay to ensure scenario is properly set
setTimeout(() => { setTimeout(() => {
if (this.scenario) { if (this.scenario) {
@ -675,16 +714,47 @@ export default {
scenario_context: this.scenario scenario_context: this.scenario
} }
this.websocket.send(JSON.stringify(greetingMessage)) this.websocket.send(JSON.stringify(greetingMessage))
this.hasRequestedInitialGreeting = true
console.log('Sent initial greeting for scenario:', this.scenario) console.log('Sent initial greeting for scenario:', this.scenario)
} else { } else {
console.log('No scenario set, retrying in 200ms') console.log('No scenario set, retrying in 300ms')
// Retry if scenario not set yet // Retry if scenario not set yet with longer delay
setTimeout(() => this.requestInitialGreeting(), 200) setTimeout(() => this.requestInitialGreeting(), 300)
} }
}, 100) }, 200) // Increased delay to ensure scenario is set
} }
}, },
// Helper method to check if we should request greeting
checkAndRequestGreeting() {
if (this.websocket && this.websocket.readyState === WebSocket.OPEN &&
this.scenario && !this.hasRequestedInitialGreeting) {
this.requestInitialGreeting()
}
},
startConversation() {
// Hide the start button and show normal controls
this.showStartButton = false
// Request the initial greeting now that user has interacted
this.checkAndRequestGreeting()
},
checkAndShowTutorial() {
// Check if user has seen tutorial before
const hasSeenTutorial = localStorage.getItem('streetLingo_tutorialSeen')
if (!hasSeenTutorial) {
this.showTutorial = true
}
},
closeTutorial() {
this.showTutorial = false
// Mark tutorial as seen
localStorage.setItem('streetLingo_tutorialSeen', 'true')
},
setupAutoRecording(message) { setupAutoRecording(message) {
// Wait for the audio element to be created in the DOM // Wait for the audio element to be created in the DOM
this.$nextTick(() => { this.$nextTick(() => {
@ -939,6 +1009,7 @@ export default {
// Mark conversation as finished // Mark conversation as finished
this.isFinished = true this.isFinished = true
this.isLoadingFeedback = true
// Close suggestion popup if open // Close suggestion popup if open
if (this.showSuggestionPopup) { if (this.showSuggestionPopup) {
@ -966,6 +1037,7 @@ export default {
if (response.ok) { if (response.ok) {
const feedback = await response.json() const feedback = await response.json()
this.conversationFeedback = feedback this.conversationFeedback = feedback
this.isLoadingFeedback = false
// Scroll to show feedback // Scroll to show feedback
this.$nextTick(() => { this.$nextTick(() => {
@ -979,6 +1051,7 @@ export default {
suggestions: [], suggestions: [],
examples: [] examples: []
} }
this.isLoadingFeedback = false
} }
} catch (error) { } catch (error) {
console.error('Error getting conversation feedback:', error) console.error('Error getting conversation feedback:', error)
@ -988,6 +1061,7 @@ export default {
suggestions: [], suggestions: [],
examples: [] examples: []
} }
this.isLoadingFeedback = false
} }
}, },
@ -1805,4 +1879,203 @@ export default {
transform: none; transform: none;
box-shadow: none; box-shadow: none;
} }
/* Start Conversation Button Styles */
.start-conversation-btn {
background: var(--primary);
color: white;
border: none;
padding: 1rem 2rem;
border-radius: var(--radius-lg);
cursor: pointer;
font-family: 'DM Sans', sans-serif;
font-size: 0.9rem;
font-weight: 500;
transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1);
box-shadow: var(--shadow);
display: flex;
align-items: center;
gap: 0.5rem;
min-width: 200px;
justify-content: center;
}
.start-conversation-btn:hover:not(:disabled) {
background: var(--primary-dark);
transform: translateY(-2px);
box-shadow: var(--shadow-lg);
}
.start-conversation-btn:disabled {
background: var(--text-muted);
cursor: not-allowed;
transform: none;
box-shadow: var(--shadow-sm);
}
/* Tutorial Popup Styles */
.tutorial-overlay {
position: fixed;
top: 0;
left: 0;
right: 0;
bottom: 0;
background: rgba(0, 0, 0, 0.6);
display: flex;
align-items: center;
justify-content: center;
z-index: 9999;
backdrop-filter: blur(4px);
}
.tutorial-popup {
background: var(--surface);
border-radius: var(--radius-lg);
box-shadow: var(--shadow-lg);
max-width: 500px;
width: 90%;
max-height: 80vh;
overflow-y: auto;
animation: tutorialSlideIn 0.3s ease-out;
}
@keyframes tutorialSlideIn {
from {
opacity: 0;
transform: translateY(-20px) scale(0.95);
}
to {
opacity: 1;
transform: translateY(0) scale(1);
}
}
.tutorial-header {
display: flex;
justify-content: space-between;
align-items: center;
padding: 1.5rem 1.5rem 1rem 1.5rem;
border-bottom: 1px solid var(--border);
}
.tutorial-header h3 {
margin: 0;
color: var(--text);
font-size: 1.3rem;
font-weight: 700;
}
.tutorial-close {
background: none;
border: none;
font-size: 1.2rem;
cursor: pointer;
color: var(--text-muted);
padding: 0.25rem;
border-radius: var(--radius);
transition: all 0.2s ease;
}
.tutorial-close:hover {
background: var(--surface-alt);
color: var(--text);
}
.tutorial-content {
padding: 1.5rem;
}
.tutorial-step {
display: flex;
align-items: flex-start;
gap: 1rem;
margin-bottom: 1.5rem;
}
.tutorial-step:last-child {
margin-bottom: 0;
}
.tutorial-icon {
font-size: 2rem;
flex-shrink: 0;
}
.tutorial-text {
flex: 1;
}
.tutorial-text strong {
display: block;
color: var(--text);
font-size: 1rem;
font-weight: 600;
margin-bottom: 0.5rem;
}
.tutorial-text p {
margin: 0;
color: var(--text-light);
font-size: 0.9rem;
line-height: 1.4;
}
.tutorial-footer {
padding: 1rem 1.5rem 1.5rem 1.5rem;
border-top: 1px solid var(--border);
display: flex;
justify-content: center;
}
.tutorial-got-it {
background: var(--primary);
color: white;
border: none;
padding: 0.75rem 2rem;
border-radius: var(--radius-lg);
cursor: pointer;
font-family: 'DM Sans', sans-serif;
font-size: 0.9rem;
font-weight: 600;
transition: all 0.2s ease;
box-shadow: var(--shadow-sm);
}
.tutorial-got-it:hover {
background: var(--primary-dark);
transform: translateY(-1px);
box-shadow: var(--shadow);
}
@media (max-width: 768px) {
.tutorial-popup {
width: 95%;
max-height: 85vh;
}
.tutorial-header {
padding: 1rem 1rem 0.75rem 1rem;
}
.tutorial-header h3 {
font-size: 1.1rem;
}
.tutorial-content {
padding: 1rem;
}
.tutorial-step {
gap: 0.75rem;
margin-bottom: 1rem;
}
.tutorial-icon {
font-size: 1.5rem;
}
.tutorial-footer {
padding: 0.75rem 1rem 1rem 1rem;
}
}
</style> </style>

View File

@ -143,11 +143,11 @@ export default {
goal: 'Order a beer and traditional German food' goal: 'Order a beer and traditional German food'
}, },
{ {
type: 'ubahn', type: 'ber_airport',
name: 'U-Bahn Help', name: 'BER Airport Train Help',
description: 'Get help with public transport in Berlin', description: 'Get train directions from BER airport to Potsdamer Platz',
challenge: 'Transport terminology and directions', challenge: 'Airport-to-city transport and dealing with delays',
goal: 'Get directions and buy appropriate ticket' goal: 'Get clear directions to Potsdamer Platz despite train delays'
} }
] ]
} }
@ -158,7 +158,8 @@ export default {
'wg_viewing': '🏠', 'wg_viewing': '🏠',
'burgeramt': '🏛️', 'burgeramt': '🏛️',
'biergarten': '🍺', 'biergarten': '🍺',
'ubahn': '🚇' 'ber_airport': '✈️',
'arzt': '👨‍⚕️'
} }
return emojiMap[type] || '📍' return emojiMap[type] || '📍'
}, },

View File

@ -114,7 +114,8 @@ export default {
'wg_viewing': '🏠', 'wg_viewing': '🏠',
'burgeramt': '🏛️', 'burgeramt': '🏛️',
'biergarten': '🍺', 'biergarten': '🍺',
'ubahn': '🚇' 'ber_airport': '✈️',
'arzt': '👨‍⚕️'
} }
return emojiMap[type] || '📍' return emojiMap[type] || '📍'
}, },
@ -124,7 +125,8 @@ export default {
'wg_viewing': '👩‍🎓', 'wg_viewing': '👩‍🎓',
'burgeramt': '👩‍💼', 'burgeramt': '👩‍💼',
'biergarten': '👨‍🍳', 'biergarten': '👨‍🍳',
'ubahn': '👨‍🚀' 'ber_airport': '👩‍💼',
'arzt': '👨‍⚕️'
} }
return avatarMap[type] || '👤' return avatarMap[type] || '👤'
} }

View File

@ -3,7 +3,7 @@
<head> <head>
<meta charset="UTF-8"> <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0"> <meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Learn Indonesian - Realistic Scenarios</title> <title>Street Lingo - Learn Indonesian through everyday scenarios</title>
</head> </head>
<body> <body>
<div id="app"></div> <div id="app"></div>

View File

@ -5,7 +5,7 @@
<div class="logo-section"> <div class="logo-section">
<div class="logo"> <div class="logo">
<span class="flag">🇮🇩</span> <span class="flag">🇮🇩</span>
<h1>Learn Indonesian</h1> <h1>Street Lingo</h1>
</div> </div>
<p class="tagline">Learn Indonesian through everyday scenarios</p> <p class="tagline">Learn Indonesian through everyday scenarios</p>
</div> </div>

View File

@ -1,5 +1,41 @@
<template> <template>
<div class="speech-interface"> <div class="speech-interface">
<!-- Tutorial Popup -->
<div v-if="showTutorial" class="tutorial-overlay" @click="closeTutorial">
<div class="tutorial-popup" @click.stop>
<div class="tutorial-header">
<h3>Welcome to Street Lingo! 🎉</h3>
<button @click="closeTutorial" class="tutorial-close"></button>
</div>
<div class="tutorial-content">
<div class="tutorial-step">
<div class="tutorial-icon">🎙</div>
<div class="tutorial-text">
<strong>Practice Speaking</strong>
<p>Click "Start Conversation" to begin chatting with AI characters in real Indonesian scenarios</p>
</div>
</div>
<div class="tutorial-step">
<div class="tutorial-icon">💡</div>
<div class="tutorial-text">
<strong>Get Help</strong>
<p>Enable the "Help" toggle and pause during conversation to get helpful phrase suggestions</p>
</div>
</div>
<div class="tutorial-step">
<div class="tutorial-icon">📊</div>
<div class="tutorial-text">
<strong>Conversation Analysis</strong>
<p>Click "Finish Conversation" to get personalized feedback on your Indonesian</p>
</div>
</div>
</div>
<div class="tutorial-footer">
<button @click="closeTutorial" class="tutorial-got-it">Got it!</button>
</div>
</div>
</div>
<div class="conversation-area"> <div class="conversation-area">
<div class="messages" ref="messagesContainer"> <div class="messages" ref="messagesContainer">
<div <div
@ -73,7 +109,18 @@
</div> </div>
<div class="controls"> <div class="controls">
<!-- Start Conversation Button -->
<button <button
v-if="showStartButton"
@click="startConversation"
class="start-conversation-btn"
:disabled="isConnecting"
>
🎙 Start Conversation
</button>
<button
v-else
@click="toggleRecording" @click="toggleRecording"
:class="['record-btn', { recording: isRecording }]" :class="['record-btn', { recording: isRecording }]"
:disabled="isConnecting || isFinished" :disabled="isConnecting || isFinished"
@ -81,15 +128,6 @@
{{ isRecording ? '🛑 Stop' : '🎤 Speak' }} {{ isRecording ? '🛑 Stop' : '🎤 Speak' }}
</button> </button>
<button
v-if="isRecording"
@click="forceStopRecording"
class="force-stop-btn"
title="Force stop"
:disabled="isFinished"
>
</button>
<div class="suggestion-toggle-container"> <div class="suggestion-toggle-container">
<label class="suggestion-toggle-label" for="suggestion-toggle"> <label class="suggestion-toggle-label" for="suggestion-toggle">
@ -158,6 +196,14 @@
</div> </div>
</div> </div>
<!-- Feedback Loading Notification -->
<div class="feedback-loading" v-if="isLoadingFeedback">
<div class="loading-content">
<div class="loading-spinner"></div>
<p>Analyzing your conversation...</p>
</div>
</div>
<!-- Conversation Feedback --> <!-- Conversation Feedback -->
<div class="feedback-section" v-if="conversationFeedback"> <div class="feedback-section" v-if="conversationFeedback">
<h3 class="feedback-title"> <h3 class="feedback-title">
@ -254,12 +300,21 @@ export default {
showRecordingStoppedNotification: false, showRecordingStoppedNotification: false,
lastAIResponseTime: null, lastAIResponseTime: null,
isFinished: false, isFinished: false,
conversationFeedback: null conversationFeedback: null,
isLoadingFeedback: false,
hasRequestedInitialGreeting: false,
showStartButton: true,
showTutorial: false
} }
}, },
mounted() { mounted() {
this.connectWebSocket() this.connectWebSocket()
this.loadScenarioData() this.loadScenarioData()
// Check if user is new and show tutorial
this.checkAndShowTutorial()
// Don't automatically request greeting on mount - wait for user to click start button
}, },
beforeUnmount() { beforeUnmount() {
this.disconnect() this.disconnect()
@ -272,11 +327,8 @@ export default {
// Reset conversation when scenario changes // Reset conversation when scenario changes
this.resetConversationOnScenarioChange() this.resetConversationOnScenarioChange()
} else if (newScenario && !oldScenario) { } else if (newScenario && !oldScenario) {
// Initial scenario load - request greeting if connected // Initial scenario load - don't auto request greeting, wait for user to click start button
console.log('Scenario initially set to:', newScenario) console.log('Scenario initially set to:', newScenario)
if (this.connectionStatus === 'connected') {
this.requestInitialGreeting()
}
} }
} }
}, },
@ -301,8 +353,7 @@ export default {
this.isConnecting = false this.isConnecting = false
console.log('WebSocket connected') console.log('WebSocket connected')
// Request initial greeting from character // Don't automatically request greeting - wait for user to click start button
this.requestInitialGreeting()
} }
this.websocket.onmessage = (event) => { this.websocket.onmessage = (event) => {
@ -459,7 +510,7 @@ export default {
this.isRecording = false this.isRecording = false
this.isAutoListening = false this.isAutoListening = false
this.currentTranscription = 'Processing...' this.currentTranscription = '' // Don't show "Processing..." delay
this.isTranscriptionFinal = false this.isTranscriptionFinal = false
}, },
@ -558,6 +609,8 @@ export default {
this.isTranscriptionFinal = false this.isTranscriptionFinal = false
this.isFinished = false this.isFinished = false
this.conversationFeedback = null this.conversationFeedback = null
this.isLoadingFeedback = false
this.hasRequestedInitialGreeting = false
// Reset goals // Reset goals
this.resetGoals() this.resetGoals()
@ -614,6 +667,8 @@ export default {
this.conversationComplete = false this.conversationComplete = false
this.isFinished = false this.isFinished = false
this.conversationFeedback = null this.conversationFeedback = null
this.isLoadingFeedback = false
this.hasRequestedInitialGreeting = false
}, },
resetConversationOnScenarioChange() { resetConversationOnScenarioChange() {
@ -627,6 +682,8 @@ export default {
this.conversationComplete = false this.conversationComplete = false
this.isFinished = false this.isFinished = false
this.conversationFeedback = null this.conversationFeedback = null
this.isLoadingFeedback = false
this.hasRequestedInitialGreeting = false
// Send reset message to backend if connected // Send reset message to backend if connected
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) { if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
@ -634,10 +691,11 @@ export default {
type: 'conversation_reset' type: 'conversation_reset'
} }
this.websocket.send(JSON.stringify(resetMessage)) this.websocket.send(JSON.stringify(resetMessage))
// Request initial greeting for new scenario
this.requestInitialGreeting()
} }
// For scenario changes, show start button again so user must start new conversation
this.showStartButton = true
// Don't automatically request greeting - wait for user to click start button
}, },
async requestTranslation(message) { async requestTranslation(message) {
@ -678,7 +736,7 @@ export default {
}, },
requestInitialGreeting() { requestInitialGreeting() {
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) { if (this.websocket && this.websocket.readyState === WebSocket.OPEN && !this.hasRequestedInitialGreeting) {
// Add a small delay to ensure scenario is properly set // Add a small delay to ensure scenario is properly set
setTimeout(() => { setTimeout(() => {
if (this.scenario) { if (this.scenario) {
@ -687,16 +745,48 @@ export default {
scenario_context: this.scenario scenario_context: this.scenario
} }
this.websocket.send(JSON.stringify(greetingMessage)) this.websocket.send(JSON.stringify(greetingMessage))
this.hasRequestedInitialGreeting = true
console.log('Sent initial greeting for scenario:', this.scenario) console.log('Sent initial greeting for scenario:', this.scenario)
} else { } else {
console.log('No scenario set, retrying in 200ms') console.log('No scenario set, retrying in 300ms')
// Retry if scenario not set yet // Retry if scenario not set yet with longer delay
setTimeout(() => this.requestInitialGreeting(), 200) setTimeout(() => this.requestInitialGreeting(), 300)
} }
}, 100) }, 200) // Increased delay to ensure scenario is set
} }
}, },
// Helper method to check if we should request greeting
checkAndRequestGreeting() {
if (this.websocket && this.websocket.readyState === WebSocket.OPEN &&
this.scenario && !this.hasRequestedInitialGreeting) {
this.requestInitialGreeting()
}
},
startConversation() {
// Hide the start button and show normal controls
this.showStartButton = false
// Request the initial greeting now that user has interacted
this.checkAndRequestGreeting()
},
checkAndShowTutorial() {
// Check if user has seen tutorial before
const hasSeenTutorial = localStorage.getItem('streetLingo_tutorialSeen')
if (!hasSeenTutorial) {
this.showTutorial = true
}
},
closeTutorial() {
this.showTutorial = false
// Mark tutorial as seen
localStorage.setItem('streetLingo_tutorialSeen', 'true')
},
setupAutoRecording(message) { setupAutoRecording(message) {
// Wait for the audio element to be created in the DOM // Wait for the audio element to be created in the DOM
this.$nextTick(() => { this.$nextTick(() => {
@ -815,10 +905,6 @@ export default {
} }
}, },
forceStopRecording() {
console.log('Force stopping recording')
this.stopRecording()
},
startPauseDetection() { startPauseDetection() {
if (!this.suggestionsEnabled) { if (!this.suggestionsEnabled) {
@ -940,6 +1026,9 @@ export default {
} }
try { try {
// Set loading state
this.isLoadingFeedback = true
// Send conversation data to backend for feedback // Send conversation data to backend for feedback
const response = await fetch(`${this.wsBaseUrl.replace('ws', 'http')}/api/conversation-feedback`, { const response = await fetch(`${this.wsBaseUrl.replace('ws', 'http')}/api/conversation-feedback`, {
method: 'POST', method: 'POST',
@ -982,6 +1071,9 @@ export default {
suggestions: [], suggestions: [],
examples: [] examples: []
} }
} finally {
// Clear loading state
this.isLoadingFeedback = false
} }
}, },
@ -1379,27 +1471,6 @@ export default {
box-shadow: none; box-shadow: none;
} }
.force-stop-btn {
background: #dc2626;
color: white;
border: none;
padding: 0.75rem;
border-radius: var(--radius-lg);
cursor: pointer;
font-size: 1rem;
transition: all 0.2s ease;
margin-left: 0.5rem;
width: 40px;
height: 40px;
display: flex;
align-items: center;
justify-content: center;
}
.force-stop-btn:hover {
background: #b91c1c;
transform: translateY(-1px);
}
.status { .status {
padding: 0.75rem; padding: 0.75rem;
@ -1799,4 +1870,254 @@ export default {
border-radius: var(--radius); border-radius: var(--radius);
font-weight: 500; font-weight: 500;
} }
/* Feedback Loading Styles */
.feedback-loading {
background: var(--surface);
border: 1px solid var(--border);
border-radius: var(--radius-lg);
padding: 1.5rem;
margin-bottom: 1.5rem;
box-shadow: var(--shadow-sm);
position: relative;
overflow: hidden;
}
.feedback-loading::before {
content: '';
position: absolute;
top: 0;
left: 0;
right: 0;
height: 3px;
background: var(--primary);
}
.loading-content {
display: flex;
align-items: center;
justify-content: center;
gap: 1rem;
text-align: center;
}
.loading-content p {
margin: 0;
color: var(--text-light);
font-size: 1rem;
font-weight: 500;
}
.loading-spinner {
width: 24px;
height: 24px;
border: 3px solid var(--surface-alt);
border-top: 3px solid var(--primary);
border-radius: 50%;
animation: spin 1s linear infinite;
}
@keyframes spin {
0% { transform: rotate(0deg); }
100% { transform: rotate(360deg); }
}
/* Start Conversation Button Styles */
.start-conversation-btn {
background: var(--primary);
color: white;
border: none;
padding: 1rem 2rem;
border-radius: var(--radius-lg);
cursor: pointer;
font-family: 'DM Sans', sans-serif;
font-size: 0.9rem;
font-weight: 500;
transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1);
box-shadow: var(--shadow);
display: flex;
align-items: center;
gap: 0.5rem;
min-width: 200px;
justify-content: center;
}
.start-conversation-btn:hover:not(:disabled) {
background: var(--primary-dark);
transform: translateY(-2px);
box-shadow: var(--shadow-lg);
}
.start-conversation-btn:disabled {
background: var(--text-muted);
cursor: not-allowed;
transform: none;
box-shadow: var(--shadow-sm);
}
/* Tutorial Popup Styles */
.tutorial-overlay {
position: fixed;
top: 0;
left: 0;
right: 0;
bottom: 0;
background: rgba(0, 0, 0, 0.6);
display: flex;
align-items: center;
justify-content: center;
z-index: 9999;
backdrop-filter: blur(4px);
}
.tutorial-popup {
background: var(--surface);
border-radius: var(--radius-lg);
box-shadow: var(--shadow-lg);
max-width: 500px;
width: 90%;
max-height: 80vh;
overflow-y: auto;
animation: tutorialSlideIn 0.3s ease-out;
}
@keyframes tutorialSlideIn {
from {
opacity: 0;
transform: translateY(-20px) scale(0.95);
}
to {
opacity: 1;
transform: translateY(0) scale(1);
}
}
.tutorial-header {
display: flex;
justify-content: space-between;
align-items: center;
padding: 1.5rem 1.5rem 1rem 1.5rem;
border-bottom: 1px solid var(--border);
}
.tutorial-header h3 {
margin: 0;
color: var(--text);
font-size: 1.3rem;
font-weight: 700;
}
.tutorial-close {
background: none;
border: none;
font-size: 1.2rem;
cursor: pointer;
color: var(--text-muted);
padding: 0.25rem;
border-radius: var(--radius);
transition: all 0.2s ease;
}
.tutorial-close:hover {
background: var(--surface-alt);
color: var(--text);
}
.tutorial-content {
padding: 1.5rem;
}
.tutorial-step {
display: flex;
align-items: flex-start;
gap: 1rem;
margin-bottom: 1.5rem;
}
.tutorial-step:last-child {
margin-bottom: 0;
}
.tutorial-icon {
font-size: 2rem;
flex-shrink: 0;
}
.tutorial-text {
flex: 1;
}
.tutorial-text strong {
display: block;
color: var(--text);
font-size: 1rem;
font-weight: 600;
margin-bottom: 0.5rem;
}
.tutorial-text p {
margin: 0;
color: var(--text-light);
font-size: 0.9rem;
line-height: 1.4;
}
.tutorial-footer {
padding: 1rem 1.5rem 1.5rem 1.5rem;
border-top: 1px solid var(--border);
display: flex;
justify-content: center;
}
.tutorial-got-it {
background: var(--primary);
color: white;
border: none;
padding: 0.75rem 2rem;
border-radius: var(--radius-lg);
cursor: pointer;
font-family: 'DM Sans', sans-serif;
font-size: 0.9rem;
font-weight: 600;
transition: all 0.2s ease;
box-shadow: var(--shadow-sm);
}
.tutorial-got-it:hover {
background: var(--primary-dark);
transform: translateY(-1px);
box-shadow: var(--shadow);
}
@media (max-width: 768px) {
.tutorial-popup {
width: 95%;
max-height: 85vh;
}
.tutorial-header {
padding: 1rem 1rem 0.75rem 1rem;
}
.tutorial-header h3 {
font-size: 1.1rem;
}
.tutorial-content {
padding: 1rem;
}
.tutorial-step {
gap: 0.75rem;
margin-bottom: 1rem;
}
.tutorial-icon {
font-size: 1.5rem;
}
.tutorial-footer {
padding: 0.75rem 1rem 1rem 1rem;
}
}
</style> </style>

View File

@ -227,52 +227,114 @@ BIERGARTEN_PERSONALITIES = {
) )
} }
UBAHN_PERSONALITIES = { BER_AIRPORT_PERSONALITIES = {
"bvg_info": GermanPersonality( "airport_info": GermanPersonality(
character_type=CharacterType.SERVICE_WORKER, character_type=CharacterType.SERVICE_WORKER,
name="BVG Mitarbeiter", name="Flughafen-Mitarbeiter",
gender=Gender.MALE, gender=Gender.FEMALE,
tone=PersonalityTone.HELPFUL, tone=PersonalityTone.HELPFUL,
age_range="young", age_range="young",
background="Helpful BVG information staff at U-Bahn station", background="Helpful airport staff at BER who knows all train connections",
typical_phrases=[ typical_phrases=[
"Kann ich Ihnen helfen?", "Guten Tag! Kann ich Ihnen helfen?",
"Wohin möchten Sie denn?", "Ach, nach Potsdamer Platz?",
"Nehmen Sie die U6 Richtung...", "Ja, die Züge haben heute Verspätung",
"Steigen Sie an... um", "Schauen Sie mal auf die Anzeigetafel",
"Das sind drei Stationen", "Gleis 3 oder 4",
"Brauchen Sie eine Fahrkarte?", "Der Airport Express fährt alle 20 Minuten",
"Zone AB reicht", "Das dauert etwa 40 Minuten",
"Gute Fahrt!" "Nehmen Sie den FEX oder die RE7",
"Steigen Sie am Hauptbahnhof um"
], ],
response_style="Professional and helpful with public transport", response_style="Professional and understanding about travel delays",
location_context="U-Bahn station information desk", location_context="BER Airport terminal information desk",
scenario_title="U-Bahn Help", scenario_title="BER Airport Train Help",
scenario_description="You're at a Berlin U-Bahn station asking for directions and transport information. Practice asking about public transport in German.", scenario_description="You've just arrived at BER airport and need to get to your hotel at Potsdamer Platz, but Google Maps shows train delays. Ask for help with train connections and platforms.",
scenario_challenge="Understanding German public transport terminology, directions, and ticket system.", scenario_challenge="Understanding German train terminology, dealing with delays, and navigating airport-to-city connections.",
scenario_goal="Get directions and buy appropriate ticket", scenario_goal="Get clear directions to Potsdamer Platz despite train delays",
goal_items=[ goal_items=[
GoalItem( GoalItem(
id="ask_directions", id="explain_destination",
description="Ask for directions (Nach dem Weg fragen)" description="Explain you need to get to Potsdamer Platz (Ziel erklären)"
), ),
GoalItem( GoalItem(
id="buy_ticket", id="ask_about_delays",
description="Buy appropriate ticket (Passende Fahrkarte kaufen)" description="Ask about train delays (Nach Verspätungen fragen)"
),
GoalItem(
id="get_platform_info",
description="Find out which platform and train to take (Gleis und Zug erfragen)"
) )
], ],
helpful_phrases=[ helpful_phrases=[
HelpfulPhrase(native="Wie komme ich nach...?", english="How do I get to...?"), HelpfulPhrase(native="Ich muss nach Potsdamer Platz", english="I need to get to Potsdamer Platz"),
HelpfulPhrase(native="Welche Linie muss ich nehmen?", english="Which line do I need to take?"), HelpfulPhrase(native="Welcher Zug fährt dorthin?", english="Which train goes there?"),
HelpfulPhrase(native="Wo muss ich umsteigen?", english="Where do I need to change?"), HelpfulPhrase(native="Haben die Züge Verspätung?", english="Are the trains delayed?"),
HelpfulPhrase(native="Wie viele Stationen?", english="How many stations?"), HelpfulPhrase(native="Von welchem Gleis fährt der Zug?", english="From which platform does the train leave?"),
HelpfulPhrase(native="Welche Fahrkarte brauche ich?", english="Which ticket do I need?"), HelpfulPhrase(native="Wie lange dauert die Fahrt?", english="How long does the journey take?"),
HelpfulPhrase(native="Einzelfahrkarte", english="Single ticket"), HelpfulPhrase(native="Muss ich umsteigen?", english="Do I need to change trains?"),
HelpfulPhrase(native="Tageskarte", english="Day ticket"), HelpfulPhrase(native="Airport Express", english="Airport Express"),
HelpfulPhrase(native="Richtung", english="Direction") HelpfulPhrase(native="Anzeigetafel", english="Display board"),
HelpfulPhrase(native="Hauptbahnhof", english="Main station")
], ],
is_helpful=True, is_helpful=True,
is_talkative=False, is_talkative=True,
uses_slang=False
)
}
ARZT_PERSONALITIES = {
"dr_muller": GermanPersonality(
character_type=CharacterType.OFFICIAL,
name="Dr. Müller",
gender=Gender.MALE,
tone=PersonalityTone.FORMAL,
age_range="middle-aged",
background="General practitioner in Berlin who is thorough and caring",
typical_phrases=[
"Guten Tag, setzen Sie sich bitte.",
"Was führt Sie zu mir?",
"Seit wann haben Sie diese Beschwerden?",
"Wo genau tut es weh?",
"Auf einer Skala von 1 bis 10?",
"Ich schaue mir das mal an",
"Ich gebe Ihnen eine Überweisung",
"Nehmen Sie diese Medikamente",
"Gute Besserung!"
],
response_style="Professional, caring, asks detailed questions about symptoms",
location_context="Doctor's office in Berlin",
scenario_title="At the Doctor",
scenario_description="You've been having leg pain for the past week and need to see a doctor. Practice describing symptoms and asking for a specialist referral in German.",
scenario_challenge="Understanding medical terminology, describing pain and symptoms accurately, and navigating the German healthcare system.",
scenario_goal="Describe leg pain and get specialist referral",
goal_items=[
GoalItem(
id="describe_symptoms",
description="Describe your leg pain symptoms (Beinschmerzen beschreiben)"
),
GoalItem(
id="explain_duration",
description="Explain how long you've had the pain (Dauer der Schmerzen erklären)"
),
GoalItem(
id="request_referral",
description="Ask for a specialist referral (Überweisung zum Facharzt bitten)"
)
],
helpful_phrases=[
HelpfulPhrase(native="Mir tut das Bein weh", english="My leg hurts"),
HelpfulPhrase(native="Seit einer Woche", english="For a week"),
HelpfulPhrase(native="Die Schmerzen sind hier", english="The pain is here"),
HelpfulPhrase(native="Es tut sehr weh", english="It hurts a lot"),
HelpfulPhrase(native="Können Sie mir eine Überweisung geben?", english="Can you give me a referral?"),
HelpfulPhrase(native="Ich brauche einen Facharzt", english="I need a specialist"),
HelpfulPhrase(native="Orthopäde", english="Orthopedist"),
HelpfulPhrase(native="Schmerztabletten", english="Painkillers"),
HelpfulPhrase(native="Krankenschein", english="Sick note")
],
is_helpful=True,
is_talkative=True,
uses_slang=False uses_slang=False
) )
} }
@ -283,5 +345,6 @@ SCENARIO_PERSONALITIES = {
"wg_viewing": WG_PERSONALITIES, "wg_viewing": WG_PERSONALITIES,
"burgeramt": BURGERAMT_PERSONALITIES, "burgeramt": BURGERAMT_PERSONALITIES,
"biergarten": BIERGARTEN_PERSONALITIES, "biergarten": BIERGARTEN_PERSONALITIES,
"ubahn": UBAHN_PERSONALITIES "ber_airport": BER_AIRPORT_PERSONALITIES,
"arzt": ARZT_PERSONALITIES
} }

View File

@ -41,10 +41,16 @@ class GermanTextToSpeechService(TextToSpeechService):
"pitch": None, "pitch": None,
"ssml_gender": texttospeech.SsmlVoiceGender.MALE, "ssml_gender": texttospeech.SsmlVoiceGender.MALE,
}, },
"BVG Mitarbeiter": { "Flughafen-Mitarbeiter": {
"name": "de-DE-Chirp3-HD-Fenrir", # Professional male voice "name": "de-DE-Chirp3-HD-Leda", # Professional female voice
"speaking_rate": 0.95, "speaking_rate": 0.95,
"pitch": None, "pitch": None,
"ssml_gender": texttospeech.SsmlVoiceGender.FEMALE,
},
"Dr. Müller": {
"name": "de-DE-Chirp3-HD-Fenrir", # Professional male voice for doctor
"speaking_rate": 0.9,
"pitch": None,
"ssml_gender": texttospeech.SsmlVoiceGender.MALE, "ssml_gender": texttospeech.SsmlVoiceGender.MALE,
} }
} }
@ -129,8 +135,10 @@ class GermanConversationFlowService(BaseConversationFlowService):
detected_scenario = "burgeramt" detected_scenario = "burgeramt"
elif "biergarten" in context_lower or "beer" in context_lower or "restaurant" in context_lower: elif "biergarten" in context_lower or "beer" in context_lower or "restaurant" in context_lower:
detected_scenario = "biergarten" detected_scenario = "biergarten"
elif "ubahn" in context_lower or "u-bahn" in context_lower or "transport" in context_lower: elif "ber" in context_lower or "airport" in context_lower or "flughafen" in context_lower or "potsdamer" in context_lower:
detected_scenario = "ubahn" detected_scenario = "ber_airport"
elif "arzt" in context_lower or "doctor" in context_lower or "medical" in context_lower:
detected_scenario = "arzt"
else: else:
detected_scenario = "spati" # Default to späti detected_scenario = "spati" # Default to späti

View File

@ -373,10 +373,11 @@ Focus on common German language learning areas:
target_language = "Indonesian" target_language = "Indonesian"
language_specific_feedback = """ language_specific_feedback = """
Focus on common Indonesian language learning areas: Focus on common Indonesian language learning areas:
- Formal vs informal language (using proper pronouns) - Using everyday, natural Indonesian words and expressions
- Sounding more natural and conversational (not textbook formal)
- Common Indonesian idioms and colloquial expressions
- Sentence structure and word order - Sentence structure and word order
- Common Indonesian expressions - Building confidence in casual conversation
- Politeness levels and cultural context
""" """
feedback_prompt = f"""You are an encouraging {target_language} language teacher. A student has just finished a conversation practice session in a {request.scenario} scenario. feedback_prompt = f"""You are an encouraging {target_language} language teacher. A student has just finished a conversation practice session in a {request.scenario} scenario.
@ -386,35 +387,77 @@ Here's their conversation:
{language_specific_feedback} {language_specific_feedback}
MANDATORY ANALYSIS: Before providing feedback, carefully examine each thing the student said for language issues. Look for:
1. Unnatural phrasing or word choices
2. Grammar mistakes or awkward constructions
3. Word order problems
4. Missing words that would make meaning clearer
5. Overly formal or informal expressions for the context
If you find ANY of these issues in their actual speech, you MUST provide specific suggestions and examples. Do not give empty suggestions/examples arrays unless their language was genuinely perfect.
Provide helpful, encouraging feedback as a JSON object with: Provide helpful, encouraging feedback as a JSON object with:
- "encouragement": A positive, motivating message about their effort (2-3 sentences) - "encouragement": A positive, motivating message about their effort (2-3 sentences)
- "suggestions": Array of 2-3 objects with: - "suggestions": Array of 0-3 objects with:
- "category": Area of improvement (e.g., "Pronunciation", "Grammar", "Vocabulary") - "category": Area of improvement (e.g., "Pronunciation", "Grammar", "Vocabulary")
- "tip": Specific, actionable advice - "tip": Specific, actionable advice based ONLY on what they actually said in the conversation
- "examples": Array of 1-2 objects with: - "examples": Array of 0-2 objects with:
- "original": Something they actually said (from the conversation) - "original": Something they actually said (from the conversation)
- "improved": A better way to say it - "improved": A better way to say it
- "reason": Brief explanation of why it's better - "reason": Brief explanation of why it's better
CRITICAL REQUIREMENT: You MUST analyze the student's actual words and phrases for improvement opportunities. Common Indonesian learner issues to look for:
1. **Word Order**: "ayam indomi" should be "indomie ayam" (flavor comes after product)
2. **Phrasing**: "beli ayam indomi satu sama mau minum" is awkward - should be "mau beli indomie ayam sama minum"
3. **Missing Words**: "mau stroberi ultra milk" missing "yang" (mau yang stroberi)
4. **Unclear Intent**: "saya beli minum" should be "saya mau beli minum" (clearer intention)
Do NOT give empty suggestions/examples unless the conversation was genuinely flawless. If there are language issues (which there usually are), provide specific, helpful corrections.
Make it encouraging and supportive, focusing on growth rather than criticism. If they did well, focus on areas to sound more natural or confident. Make it encouraging and supportive, focusing on growth rather than criticism. If they did well, focus on areas to sound more natural or confident.
Example format: For Indonesian specifically:
- Focus on everyday conversational language rather than formal politeness
- Emphasize natural, casual expressions that locals actually use
- Include specific word examples in your tips
- Avoid focusing on formal grammar rules - prioritize natural communication
Example format for good conversation (no meaningful improvements needed):
{{
"encouragement": "Fantastic job in your conversation practice! You really engaged well and made your choices clear. Keep up the great work, and your confidence will only grow!",
"suggestions": [],
"examples": []
}}
Example format for conversation with meaningful improvements:
{{ {{
"encouragement": "You did a great job engaging in this conversation! Your effort to communicate is really paying off.", "encouragement": "You did a great job engaging in this conversation! Your effort to communicate is really paying off.",
"suggestions": [ "suggestions": [
{{ {{
"category": "Vocabulary", "category": "Word Order",
"tip": "Try using more common everyday words to sound more natural" "tip": "In Indonesian, the product name comes first, then the flavor - so 'indomie ayam' instead of 'ayam indomi'"
}},
{{
"category": "Phrasing",
"tip": "When expressing what you want to buy, use 'mau beli' to be clearer about your intention"
}} }}
], ],
"examples": [ "examples": [
{{ {{
"original": "I want to purchase this item", "original": "beli ayam indomi satu sama mau minum",
"improved": "I'd like to buy this", "improved": "mau beli indomie ayam satu sama minum",
"reason": "Sounds more natural and conversational" "reason": "Better word order and clearer intention - 'mau beli' shows you want to buy"
}},
{{
"original": "mau stroberi ultra milk",
"improved": "mau ultra milk yang stroberi",
"reason": "Adding 'yang' makes it clearer which flavor you want"
}} }}
] ]
}}""" }}
IMPORTANT: Analyze the student's actual phrases from the conversation above and provide specific corrections for any unnatural or incorrect expressions. Don't give empty arrays unless their Indonesian was perfect."""
response = client.chat.completions.create( response = client.chat.completions.create(
model=config.OPENAI_MODEL, model=config.OPENAI_MODEL,
@ -500,6 +543,7 @@ async def websocket_speech_endpoint(websocket: WebSocket, language: str):
transcript_repeat_count = 0 transcript_repeat_count = 0
last_transcript = "" last_transcript = ""
high_confidence_count = 0 high_confidence_count = 0
session_processed = False # Flag to prevent duplicate processing
import uuid import uuid
session_id = str(uuid.uuid4()) session_id = str(uuid.uuid4())
@ -518,6 +562,7 @@ async def websocket_speech_endpoint(websocket: WebSocket, language: str):
chunk_count = 0 chunk_count = 0
latest_transcript = "" latest_transcript = ""
recording_start_time = time.time() recording_start_time = time.time()
session_processed = False # Reset processing flag for new session
logger.info("Started recording session") logger.info("Started recording session")
elif message["type"] == "conversation_reset": elif message["type"] == "conversation_reset":
@ -527,17 +572,39 @@ async def websocket_speech_endpoint(websocket: WebSocket, language: str):
elif message["type"] == "audio_chunk": elif message["type"] == "audio_chunk":
if is_recording: if is_recording:
# Check for recording timeout # Check for recording timeout
if recording_start_time and time.time() - recording_start_time > max_recording_duration: if recording_start_time and time.time() - recording_start_time > max_recording_duration and not session_processed:
logger.warning("Recording timeout reached, auto-stopping") logger.warning("Recording timeout reached, auto-stopping")
is_recording = False
session_processed = True # Mark as processed to prevent duplicates
# Send timeout notification to frontend # Send timeout notification to frontend
timeout_notification = { timeout_notification = {
"type": "recording_timeout", "type": "recording_timeout",
"message": "Recording stopped due to timeout" "message": "Recording stopped due to timeout"
} }
await websocket.send_text(json.dumps(timeout_notification)) await websocket.send_text(json.dumps(timeout_notification))
# Force audio_end processing
message = {"type": "audio_end", "scenario_context": message.get("scenario_context", "")} # Process final transcript if available
# Don't return, let it fall through to audio_end processing if latest_transcript.strip():
transcription_result = {
"type": "transcription",
"transcript": latest_transcript,
"is_final": True,
"confidence": 0.8
}
await websocket.send_text(json.dumps(transcription_result))
# Process AI response
logger.info("Getting AI response after timeout...")
ai_response = await session_conversation_service.process_conversation_flow_fast(
latest_transcript,
message.get("scenario_context", "")
)
logger.info(f"AI response: {ai_response.get('text', 'No text')}")
await websocket.send_text(json.dumps(ai_response))
audio_buffer.clear()
continue # Skip further processing for this message
else: else:
audio_data = base64.b64decode(message["audio"]) audio_data = base64.b64decode(message["audio"])
logger.info(f"Received audio chunk: {len(audio_data)} bytes") logger.info(f"Received audio chunk: {len(audio_data)} bytes")
@ -548,8 +615,8 @@ async def websocket_speech_endpoint(websocket: WebSocket, language: str):
# Process chunk for real-time transcription # Process chunk for real-time transcription
chunk_count += 1 chunk_count += 1
try: try:
# Only process every 8th chunk to reduce log spam and API calls # Only process every 6th chunk for faster response (reduced from 8th)
if chunk_count % 8 == 0 and len(audio_buffer) >= 19200: # ~0.4 seconds of audio at 48kHz if chunk_count % 6 == 0 and len(audio_buffer) >= 14400: # ~0.3 seconds of audio at 48kHz
recognition_audio = speech.RecognitionAudio(content=bytes(audio_buffer)) recognition_audio = speech.RecognitionAudio(content=bytes(audio_buffer))
response = session_conversation_service.stt_service.client.recognize( response = session_conversation_service.stt_service.client.recognize(
config=session_conversation_service.stt_service.recognition_config, config=session_conversation_service.stt_service.recognition_config,
@ -561,7 +628,7 @@ async def websocket_speech_endpoint(websocket: WebSocket, language: str):
confidence = response.results[0].alternatives[0].confidence confidence = response.results[0].alternatives[0].confidence
# Store transcript if confidence is reasonable (lowered for speed) # Store transcript if confidence is reasonable (lowered for speed)
if confidence > 0.6: if confidence > 0.4: # Lowered threshold for faster processing
latest_transcript = transcript # Store latest transcript latest_transcript = transcript # Store latest transcript
# Check for repeated high-confidence transcripts # Check for repeated high-confidence transcripts
@ -570,13 +637,13 @@ async def websocket_speech_endpoint(websocket: WebSocket, language: str):
high_confidence_count += 1 high_confidence_count += 1
logger.info(f"Repeated high confidence transcript #{high_confidence_count}: '{transcript}' (confidence: {confidence})") logger.info(f"Repeated high confidence transcript #{high_confidence_count}: '{transcript}' (confidence: {confidence})")
# If we've seen the same high-confidence transcript 4+ times, auto-stop # If we've seen the same high-confidence transcript 3+ times, auto-stop (reduced from 4)
if high_confidence_count >= 4: if high_confidence_count >= 3 and not session_processed:
logger.info("Auto-stopping recording due to repeated high-confidence transcript") logger.info("Auto-stopping recording due to repeated high-confidence transcript")
is_recording = False is_recording = False
session_processed = True # Mark as processed to prevent duplicates
# Send final processing message # Send final processing message
final_message = {"type": "audio_end", "scenario_context": message.get("scenario_context", "")}
# Process immediately without waiting for more chunks
await websocket.send_text(json.dumps({ await websocket.send_text(json.dumps({
"type": "transcription", "type": "transcription",
"transcript": transcript, "transcript": transcript,
@ -640,15 +707,22 @@ async def websocket_speech_endpoint(websocket: WebSocket, language: str):
elif message["type"] == "audio_end": elif message["type"] == "audio_end":
is_recording = False is_recording = False
# Check if this session was already processed (e.g., by auto-stop logic)
if session_processed:
logger.info("Audio session already processed, skipping duplicate processing")
continue
final_transcript = "" final_transcript = ""
# Use latest interim transcript if available for faster response # Use latest interim transcript if available for faster response
logger.info(f"Checking latest_transcript: '{latest_transcript}'") logger.info(f"Checking latest_transcript: '{latest_transcript}'")
if latest_transcript.strip(): if latest_transcript.strip() and len(latest_transcript.strip()) > 3: # More aggressive check
final_transcript = latest_transcript final_transcript = latest_transcript
logger.info(f"Using latest interim transcript: '{final_transcript}'") logger.info(f"Using latest interim transcript: '{final_transcript}'")
session_processed = True # Mark as processed to prevent duplicates
# Send final transcription immediately # Send final transcription immediately - no "Processing..." delay
transcription_result = { transcription_result = {
"type": "transcription", "type": "transcription",
"transcript": final_transcript, "transcript": final_transcript,
@ -657,8 +731,8 @@ async def websocket_speech_endpoint(websocket: WebSocket, language: str):
} }
await websocket.send_text(json.dumps(transcription_result)) await websocket.send_text(json.dumps(transcription_result))
# Process AI response with faster flow # Process AI response with faster flow - start immediately
logger.info("Getting AI response...") logger.info("Getting AI response immediately...")
ai_response = await session_conversation_service.process_conversation_flow_fast( ai_response = await session_conversation_service.process_conversation_flow_fast(
final_transcript, final_transcript,
message.get("scenario_context", "") message.get("scenario_context", "")
@ -673,6 +747,7 @@ async def websocket_speech_endpoint(websocket: WebSocket, language: str):
elif len(audio_buffer) > 0: elif len(audio_buffer) > 0:
# Fallback to full transcription if no interim results # Fallback to full transcription if no interim results
logger.info(f"Processing final audio buffer: {len(audio_buffer)} bytes") logger.info(f"Processing final audio buffer: {len(audio_buffer)} bytes")
session_processed = True # Mark as processed to prevent duplicates
try: try:
recognition_audio = speech.RecognitionAudio(content=bytes(audio_buffer)) recognition_audio = speech.RecognitionAudio(content=bytes(audio_buffer))
response = session_conversation_service.stt_service.client.recognize( response = session_conversation_service.stt_service.client.recognize(