Implement VLM Component for Vision Language Model Support

Priority

HIGH

Context

The VLM (Vision Language Model) Component enables multimodal AI features by processing images alongside text. The iOS SDK has a complete VLM implementation. The Kotlin SDK has only a stub VLMComponent with no actual functionality.

Current State

• File: src/commonMain/kotlin/com/runanywhere/sdk/components/vlm/VLMComponent.kt (stub only)
• Status: Empty implementation, no service layer
• Impact: Cannot support vision-based AI features

iOS Reference Implementation

• Component: sdk/runanywhere-swift/Sources/RunAnywhere/Components/VLM/VLMComponent.swift
• Services: sdk/runanywhere-swift/Sources/RunAnywhere/Components/VLM/Services/VLMService.swift

iOS Architecture Pattern

public final class VLMComponent: ModelBasedComponent {
    private var service: VLMService?
    
    public func analyze(image: Data, prompt: String) async throws -> VLMResponse
    public func analyzeStream(images: AsyncStream<Data>, prompts: AsyncStream<String>) -> AsyncStream<VLMResponse>
}

Implementation Plan

1. Complete Component Implementation (commonMain)

// commonMain/kotlin/com/runanywhere/sdk/components/vlm/VLMComponent.kt
class VLMComponent(
    configuration: VLMConfiguration,
    serviceContainer: ServiceContainer? = null
) : BaseComponent<VLMService>(configuration, serviceContainer) {
    
    suspend fun analyze(image: ByteArray, prompt: String): VLMResponse {
        ensureReady()
        return service?.analyze(image, prompt) 
            ?: throw SDKError.ComponentNotReady("VLM service not initialized")
    }
    
    fun analyzeStream(
        images: Flow<ByteArray>,
        prompts: Flow<String>
    ): Flow<VLMResponse> = flow {
        ensureReady()
        images.zip(prompts) { image, prompt ->
            service?.analyze(image, prompt)
        }.collect { response ->
            response?.let { emit(it) }
        }
    }
    
    override suspend fun createService(): VLMService {
        val provider = ModuleRegistry.vlmProvider(configuration.modelId)
        return provider?.createVLMService(configuration)
            ?: throw SDKError.ComponentNotAvailable("No VLM provider for ${configuration.modelId}")
    }
}

// commonMain/kotlin/com/runanywhere/sdk/components/vlm/services/VLMService.kt
interface VLMService {
    suspend fun analyze(image: ByteArray, prompt: String): VLMResponse
    val supportedFormats: List<ImageFormat>
}

// commonMain/kotlin/com/runanywhere/sdk/components/vlm/models/VLMConfiguration.kt
data class VLMConfiguration(
    val modelId: String,
    val maxImageSize: Int = 1024,
    val imageFormat: ImageFormat = ImageFormat.JPEG
) : ComponentConfiguration

data class VLMResponse(
    val description: String,
    val confidence: Float,
    val objects: List<DetectedObject> = emptyList(),
    val processingTimeMs: Long
)

data class DetectedObject(
    val label: String,
    val confidence: Float,
    val boundingBox: BoundingBox? = null
)

data class BoundingBox(
    val x: Float,
    val y: Float,
    val width: Float,
    val height: Float
)

enum class ImageFormat {
    JPEG, PNG, BMP, WEBP
}

2. Platform Implementations

Android (androidMain)

// androidMain/kotlin/com/runanywhere/sdk/components/vlm/services/AndroidVLMService.kt
class AndroidVLMService(
    private val context: Context,
    private val configuration: VLMConfiguration
) : VLMService {
    
    // Use TensorFlow Lite or ML Kit for on-device vision
    private var interpreter: Interpreter? = null
    
    override suspend fun analyze(image: ByteArray, prompt: String): VLMResponse {
        // Load image
        val bitmap = BitmapFactory.decodeByteArray(image, 0, image.size)
        
        // Preprocess image
        val preprocessed = preprocessImage(bitmap)
        
        // Run inference
        val startTime = System.currentTimeMillis()
        val results = runInference(preprocessed, prompt)
        val duration = System.currentTimeMillis() - startTime
        
        return VLMResponse(
            description = results.description,
            confidence = results.confidence,
            objects = results.detectedObjects,
            processingTimeMs = duration
        )
    }
    
    override val supportedFormats = listOf(
        ImageFormat.JPEG,
        ImageFormat.PNG,
        ImageFormat.BMP,
        ImageFormat.WEBP
    )
}

JVM (jvmMain)

// jvmMain/kotlin/com/runanywhere/sdk/components/vlm/services/JvmVLMService.kt
class JvmVLMService(
    private val configuration: VLMConfiguration
) : VLMService {
    
    // Use DJL (Deep Java Library) or ONNX Runtime
    override suspend fun analyze(image: ByteArray, prompt: String): VLMResponse {
        // Implementation using DJL or ONNX
        return VLMResponse(
            description = "JVM VLM implementation",
            confidence = 0.0f,
            objects = emptyList(),
            processingTimeMs = 0L
        )
    }
    
    override val supportedFormats = listOf(
        ImageFormat.JPEG,
        ImageFormat.PNG,
        ImageFormat.BMP
    )
}

3. Provider Registration

// commonMain/kotlin/com/runanywhere/sdk/core/ModuleRegistry.kt
interface VLMServiceProvider {
    suspend fun createVLMService(configuration: VLMConfiguration): VLMService
    fun canHandle(modelId: String?): Boolean
    val name: String
}

object ModuleRegistry {
    private val vlmProviders = CopyOnWriteArrayList<VLMServiceProvider>()
    
    fun registerVLM(provider: VLMServiceProvider) {
        vlmProviders.add(provider)
    }
    
    fun vlmProvider(modelId: String? = null): VLMServiceProvider? {
        return vlmProviders.firstOrNull { it.canHandle(modelId) }
    }
}

Files to Create/Modify

Files to Modify

• [ ] components/vlm/VLMComponent.kt (replace stub with full implementation)

New Files (commonMain)

• [ ] components/vlm/services/VLMService.kt
• [ ] components/vlm/models/VLMConfiguration.kt
• [ ] components/vlm/models/VLMResponse.kt
• [ ] components/vlm/models/ImageFormat.kt
• [ ] components/vlm/providers/VLMServiceProvider.kt

New Files (androidMain)

• [ ] components/vlm/services/AndroidVLMService.kt

New Files (jvmMain)

• [ ] components/vlm/services/JvmVLMService.kt

Files to Modify

• [ ] core/ModuleRegistry.kt (add VLM provider registration)

Dependencies Required

Android

• TensorFlow Lite: org.tensorflow:tensorflow-lite:2.14.0
• OR ML Kit: com.google.mlkit:image-labeling:17.0.7

JVM

• Deep Java Library: ai.djl:api:0.24.0
• OR ONNX Runtime: com.microsoft.onnxruntime:onnxruntime:1.16.0

Testing Requirements

• [ ] Unit tests for VLMComponent lifecycle
• [ ] Integration tests with mock images
• [ ] Android platform tests with TensorFlow Lite
• [ ] JVM platform tests
• [ ] Image format support tests
• [ ] Stream processing tests
• [ ] Error handling tests (invalid images)

Success Criteria

• [ ] VLMComponent can analyze images with text prompts
• [ ] Android implementation uses TensorFlow Lite or ML Kit
• [ ] JVM implementation has working inference
• [ ] All image formats supported
• [ ] Stream processing works correctly
• [ ] Proper error handling
• [ ] Thread-safe provider registration

Estimated Effort

2-3 weeks (120-180 hours)

• Component implementation: 3-5 days
• Android implementation: 5-7 days
• JVM implementation: 5-7 days
• Testing: 3-5 days

Dependencies

• Depends on: #142 (thread safety fixes)
• Depends on: #147 (privacy-first logging)

Related Issues

• Part of multimodal AI roadmap
• Enables vision-based features
• Aligns with iOS VLM architecture

🤖 Generated with Claude Code