Implement Voice Agent Component for End-to-End Voice Pipeline Orchestration

Priority

CRITICAL - Production Blocker

Context

The Voice Agent Component is the orchestrator that connects VAD → STT → LLM → TTS into a seamless conversational pipeline. The iOS SDK has a fully functional VoiceAgent that manages the entire voice interaction lifecycle. The Kotlin SDK is missing this critical component.

Current State

• Status: Component does not exist
• Impact: No end-to-end voice conversation capability
• iOS Has: Complete VoiceAgent with state management, interruption handling, and pipeline coordination

iOS Reference Implementation

• Component: sdk/runanywhere-swift/Sources/RunAnywhere/Components/VoiceAgent/VoiceAgent.swift
• Models: sdk/runanywhere-swift/Sources/RunAnywhere/Components/VoiceAgent/Models/
• Pipeline: Orchestrates VAD, STT, LLM, TTS components

iOS Architecture Pattern

public final class VoiceAgent: Component {
    private let vadComponent: VADComponent
    private let sttComponent: STTComponent
    private let llmComponent: LLMComponent?
    private let ttsComponent: TTSComponent
    
    public func startConversation() async throws
    public func stopConversation() async
    public func handleInterruption() async
    
    public var conversationState: ConversationState { get }
    public var transcriptionStream: AsyncStream<String> { get }
    public var responseStream: AsyncStream<Data> { get }
}

Implementation Plan

1. Define Common Interfaces (commonMain)

// commonMain/kotlin/com/runanywhere/sdk/components/voiceagent/VoiceAgent.kt
class VoiceAgent(
    private val configuration: VoiceAgentConfiguration,
    private val serviceContainer: ServiceContainer
) : BaseComponent<Unit>(configuration, serviceContainer) {
    
    private var vadComponent: VADComponent? = null
    private var sttComponent: STTComponent? = null
    private var llmComponent: LLMComponent? = null
    private var ttsComponent: TTSComponent? = null
    
    private val _conversationState = MutableStateFlow(ConversationState.IDLE)
    val conversationState: StateFlow<ConversationState> = _conversationState.asStateFlow()
    
    private val _transcriptionStream = MutableSharedFlow<String>()
    val transcriptionStream: SharedFlow<String> = _transcriptionStream.asSharedFlow()
    
    private val _responseStream = MutableSharedFlow<ByteArray>()
    val responseStream: SharedFlow<ByteArray> = _responseStream.asSharedFlow()
    
    suspend fun startConversation() {
        _conversationState.value = ConversationState.LISTENING
        // Start VAD → STT pipeline
        startVoiceDetection()
    }
    
    suspend fun stopConversation() {
        _conversationState.value = ConversationState.IDLE
        // Stop all components
    }
    
    suspend fun handleInterruption() {
        // Interrupt TTS playback, restart listening
        _conversationState.value = ConversationState.LISTENING
    }
    
    private suspend fun startVoiceDetection() {
        vadComponent?.startDetection()?.collect { vadEvent ->
            when (vadEvent) {
                is VADEvent.SpeechStarted -> handleSpeechStart()
                is VADEvent.SpeechEnded -> handleSpeechEnd(vadEvent.audioData)
            }
        }
    }
    
    private suspend fun handleSpeechEnd(audioData: ByteArray) {
        _conversationState.value = ConversationState.TRANSCRIBING
        
        // Transcribe with STT
        val transcription = sttComponent?.transcribe(audioData)
        transcription?.let {
            _transcriptionStream.emit(it.text)
            
            // Generate response with LLM
            _conversationState.value = ConversationState.GENERATING
            val llmResponse = llmComponent?.generate(it.text)
            
            llmResponse?.let { response ->
                // Synthesize with TTS
                _conversationState.value = ConversationState.SPEAKING
                val audioResponse = ttsComponent?.synthesize(response, TTSOptions())
                audioResponse?.let { audio ->
                    _responseStream.emit(audio)
                    _conversationState.value = ConversationState.LISTENING
                }
            }
        }
    }
}

// commonMain/kotlin/com/runanywhere/sdk/components/voiceagent/models/VoiceAgentConfiguration.kt
data class VoiceAgentConfiguration(
    val vadConfig: VADConfiguration,
    val sttConfig: STTConfiguration,
    val llmConfig: LLMConfiguration? = null,
    val ttsConfig: TTSConfiguration,
    val enableInterruption: Boolean = true,
    val silenceTimeoutMs: Long = 2000
) : ComponentConfiguration

enum class ConversationState {
    IDLE,
    LISTENING,
    TRANSCRIBING,
    GENERATING,
    SPEAKING
}

2. Pipeline Orchestration Logic

// commonMain/kotlin/com/runanywhere/sdk/components/voiceagent/VoiceAgentPipeline.kt
internal class VoiceAgentPipeline(
    private val vadComponent: VADComponent,
    private val sttComponent: STTComponent,
    private val llmComponent: LLMComponent?,
    private val ttsComponent: TTSComponent
) {
    
    suspend fun executePipeline(audioInput: Flow<ByteArray>): Flow<ByteArray> = flow {
        // VAD stage
        val speechSegments = vadComponent.processAudioStream(audioInput)
        
        speechSegments.collect { segment ->
            // STT stage
            val transcription = sttComponent.transcribe(segment)
            
            // LLM stage (optional)
            val response = llmComponent?.generate(transcription.text) ?: transcription.text
            
            // TTS stage
            val audioResponse = ttsComponent.synthesize(response, TTSOptions())
            emit(audioResponse)
        }
    }
}

3. Interruption Handling

// commonMain/kotlin/com/runanywhere/sdk/components/voiceagent/InterruptionHandler.kt
internal class InterruptionHandler(
    private val configuration: VoiceAgentConfiguration
) {
    
    private var currentPlaybackJob: Job? = null
    
    suspend fun handleInterruption() {
        // Cancel current TTS playback
        currentPlaybackJob?.cancel()
        
        // Emit interruption event
        EventBus.publish(VoiceAgentEvent.InterruptionDetected)
    }
    
    fun setPlaybackJob(job: Job) {
        currentPlaybackJob = job
    }
}

Files to Create/Modify

New Files (commonMain)

• [ ] components/voiceagent/VoiceAgent.kt
• [ ] components/voiceagent/VoiceAgentPipeline.kt
• [ ] components/voiceagent/InterruptionHandler.kt
• [ ] components/voiceagent/models/VoiceAgentConfiguration.kt
• [ ] components/voiceagent/models/ConversationState.kt
• [ ] events/VoiceAgentEvent.kt

Files to Modify

• [ ] foundation/ServiceContainer.kt (add VoiceAgent registration)
• [ ] events/ComponentEvent.kt (add voice agent events)

Testing Requirements

• [ ] Unit tests for VoiceAgent lifecycle
• [ ] Integration tests for full pipeline (VAD → STT → LLM → TTS)
• [ ] Interruption handling tests
• [ ] State transition tests
• [ ] Concurrent conversation tests
• [ ] Error recovery tests (component failures)
• [ ] Stream backpressure tests

Success Criteria

• [ ] VoiceAgent orchestrates all 4 components successfully
• [ ] Conversation state transitions correctly
• [ ] Interruption handling works (user can interrupt AI speech)
• [ ] Audio streams flow without latency issues
• [ ] Proper error handling for component failures
• [ ] Thread-safe state management
• [ ] Graceful degradation when LLM is not configured

Estimated Effort

2-3 weeks (120-180 hours)

• Component architecture: 3-5 days
• Pipeline orchestration: 5-7 days
• Interruption handling: 3-4 days
• State management: 2-3 days
• Testing and integration: 5-7 days

Dependencies

• BLOCKS: #149 (TTS Component)
• Depends on: VADComponent (already exists)
• Depends on: STTComponent (already exists)
• Depends on: LLMComponent (already exists)
• Depends on: #142 (thread safety fixes)
• Depends on: #147 (privacy-first logging)

Related Issues

• Part of voice pipeline completion roadmap
• Required for conversational AI features
• Aligns with iOS VoiceAgent architecture

🤖 Generated with Claude Code