From 063c87fbceafcb23bde90e5e66b338d9f6e91de5 Mon Sep 17 00:00:00 2001 From: yhsung Date: Sat, 14 Jun 2025 17:42:52 +0800 Subject: [PATCH] Add image attachment support --- .gitignore | 1 + lib/providers/chat_providers.dart | 42 ++++++ lib/services/azure_openai_service.dart | 10 ++ lib/services/llm_service.dart | 6 + lib/services/ollama_service.dart | 9 ++ lib/services/openai_service.dart | 61 +++++++++ lib/ui/screens/chat_screen.dart | 45 +++++- macos/Flutter/GeneratedPluginRegistrant.swift | 2 + pubspec.lock | 128 ++++++++++++++++++ pubspec.yaml | 1 + test/app_test.dart | 10 ++ test/widget_test.dart | 30 ---- .../flutter/generated_plugin_registrant.cc | 3 + windows/flutter/generated_plugins.cmake | 1 + 14 files changed, 315 insertions(+), 34 deletions(-) create mode 100644 test/app_test.dart delete mode 100644 test/widget_test.dart diff --git a/.gitignore b/.gitignore index 79c113f..daaf835 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,4 @@ app.*.map.json /android/app/debug /android/app/profile /android/app/release +\nflutter/ diff --git a/lib/providers/chat_providers.dart b/lib/providers/chat_providers.dart index 44048f9..acef389 100644 --- a/lib/providers/chat_providers.dart +++ b/lib/providers/chat_providers.dart @@ -138,4 +138,46 @@ class ActiveChatSessionNotifier extends StateNotifier { _ref.read(isLoadingProvider.notifier).state = false; } } + + Future sendMessageWithImage( + String content, + String base64Image, + ) async { + if (state == null) return; + + final userMessage = Message( + role: MessageRole.user, + content: content, + ); + + final updatedSession = state!.addMessage(userMessage); + state = updatedSession; + _ref.read(chatSessionsProvider.notifier).updateSession(updatedSession); + + _ref.read(isLoadingProvider.notifier).state = true; + + try { + final llmService = _ref.read(selectedLlmServiceProvider); + + final assistantMessage = await llmService.sendMessageWithImage( + updatedSession.messages, + base64Image, + ); + + final finalSession = updatedSession.addMessage(assistantMessage); + state = finalSession; + _ref.read(chatSessionsProvider.notifier).updateSession(finalSession); + } catch (e) { + final errorMessage = Message( + role: MessageRole.assistant, + content: 'Error: ${e.toString()}', + ); + + final finalSession = updatedSession.addMessage(errorMessage); + state = finalSession; + _ref.read(chatSessionsProvider.notifier).updateSession(finalSession); + } finally { + _ref.read(isLoadingProvider.notifier).state = false; + } + } } \ No newline at end of file diff --git a/lib/services/azure_openai_service.dart b/lib/services/azure_openai_service.dart index 41c9b25..b11ccf5 100644 --- a/lib/services/azure_openai_service.dart +++ b/lib/services/azure_openai_service.dart @@ -74,6 +74,16 @@ class AzureOpenAiService implements LlmService { throw Exception('Error communicating with Azure OpenAI: $e'); } } + + @override + Future sendMessageWithImage( + List messages, + String base64Image, + ) async { + // Azure OpenAI does not currently support image inputs via this interface. + // Fallback to sending the text-only message. + return sendMessage(messages); + } @override Future> getAvailableModels() async { diff --git a/lib/services/llm_service.dart b/lib/services/llm_service.dart index ac159c3..25141ae 100644 --- a/lib/services/llm_service.dart +++ b/lib/services/llm_service.dart @@ -9,6 +9,12 @@ abstract class LlmService { /// Send a message to the LLM service and get a response Future sendMessage(List messages); + + /// Send a message with an attached image encoded as base64 + Future sendMessageWithImage( + List messages, + String base64Image, + ); /// Get the available models for this service Future> getAvailableModels(); diff --git a/lib/services/ollama_service.dart b/lib/services/ollama_service.dart index 98bb6dd..c80d9f6 100644 --- a/lib/services/ollama_service.dart +++ b/lib/services/ollama_service.dart @@ -51,6 +51,15 @@ class OllamaService implements LlmService { } } + @override + Future sendMessageWithImage( + List messages, + String base64Image, + ) async { + // Ollama currently has no direct image input support; ignore the image. + return sendMessage(messages); + } + String _formatMessagesForOllama(List messages) { // Simple formatting for Ollama // This is a basic implementation and might need to be adjusted based on the specific model diff --git a/lib/services/openai_service.dart b/lib/services/openai_service.dart index 00465ec..19f0b97 100644 --- a/lib/services/openai_service.dart +++ b/lib/services/openai_service.dart @@ -63,6 +63,67 @@ class OpenAiService implements LlmService { throw Exception('Error communicating with OpenAI: $e'); } } + + @override + Future sendMessageWithImage( + List messages, + String base64Image, + ) async { + final apiKey = await _secureStorage.read(key: _apiKeyKey); + if (apiKey == null || apiKey.isEmpty) { + throw Exception('OpenAI API key not configured'); + } + + final model = await getCurrentModel(); + + final formattedMessages = messages.map((m) { + final role = m.role.toString().split('.').last; + if (identical(m, messages.last) && m.role == MessageRole.user) { + return { + 'role': role, + 'content': [ + {'type': 'text', 'text': m.content}, + { + 'type': 'image_url', + 'image_url': 'data:image/png;base64,$base64Image' + }, + ], + }; + } + return { + 'role': role, + 'content': m.content, + }; + }).toList(); + + try { + final response = await _dio.post( + '$_baseUrl/chat/completions', + options: Options( + headers: { + 'Authorization': 'Bearer $apiKey', + 'Content-Type': 'application/json', + }, + ), + data: { + 'model': model, + 'messages': formattedMessages, + }, + ); + + if (response.statusCode == 200) { + final content = response.data['choices'][0]['message']['content']; + return Message( + role: MessageRole.assistant, + content: content, + ); + } else { + throw Exception('Failed to get response from OpenAI: ${response.statusCode}'); + } + } catch (e) { + throw Exception('Error communicating with OpenAI: $e'); + } + } @override Future> getAvailableModels() async { diff --git a/lib/ui/screens/chat_screen.dart b/lib/ui/screens/chat_screen.dart index abae906..ec0626c 100644 --- a/lib/ui/screens/chat_screen.dart +++ b/lib/ui/screens/chat_screen.dart @@ -1,5 +1,9 @@ +import 'dart:convert'; +import 'dart:io'; + import 'package:flutter/material.dart'; import 'package:flutter_riverpod/flutter_riverpod.dart'; +import 'package:image_picker/image_picker.dart'; import '../../models/message.dart'; import '../../providers/chat_providers.dart'; import '../widgets/message_bubble.dart'; @@ -89,6 +93,7 @@ class MessageInput extends ConsumerStatefulWidget { class _MessageInputState extends ConsumerState { final TextEditingController _controller = TextEditingController(); bool _canSend = false; + File? _selectedImage; @override void initState() { @@ -106,16 +111,40 @@ class _MessageInputState extends ConsumerState { void _updateCanSend() { final text = _controller.text.trim(); setState(() { - _canSend = text.isNotEmpty; + _canSend = text.isNotEmpty || _selectedImage != null; }); } - void _sendMessage() { + Future _pickImage() async { + final picker = ImagePicker(); + final picked = await picker.pickImage(source: ImageSource.gallery); + if (picked != null) { + setState(() { + _selectedImage = File(picked.path); + _updateCanSend(); + }); + } + } + + Future _sendMessage() async { final text = _controller.text.trim(); - if (text.isEmpty) return; + if (text.isEmpty && _selectedImage == null) return; + + if (_selectedImage != null) { + final bytes = await _selectedImage!.readAsBytes(); + final base64Image = base64Encode(bytes); + await ref + .read(activeChatSessionProvider.notifier) + .sendMessageWithImage(text, base64Image); + } else { + await ref.read(activeChatSessionProvider.notifier).sendMessage(text); + } - ref.read(activeChatSessionProvider.notifier).sendMessage(text); _controller.clear(); + setState(() { + _selectedImage = null; + _updateCanSend(); + }); } @override @@ -152,6 +181,14 @@ class _MessageInputState extends ConsumerState { ), ), const SizedBox(width: 8.0), + IconButton( + icon: Icon( + _selectedImage == null ? Icons.attach_file : Icons.check, + ), + onPressed: isLoading ? null : _pickImage, + color: Theme.of(context).colorScheme.secondary, + ), + const SizedBox(width: 8.0), FutureBuilder( future: isConfigured, builder: (context, snapshot) { diff --git a/macos/Flutter/GeneratedPluginRegistrant.swift b/macos/Flutter/GeneratedPluginRegistrant.swift index 15a1671..5d35054 100644 --- a/macos/Flutter/GeneratedPluginRegistrant.swift +++ b/macos/Flutter/GeneratedPluginRegistrant.swift @@ -5,10 +5,12 @@ import FlutterMacOS import Foundation +import file_selector_macos import flutter_secure_storage_macos import path_provider_foundation func RegisterGeneratedPlugins(registry: FlutterPluginRegistry) { + FileSelectorPlugin.register(with: registry.registrar(forPlugin: "FileSelectorPlugin")) FlutterSecureStoragePlugin.register(with: registry.registrar(forPlugin: "FlutterSecureStoragePlugin")) PathProviderPlugin.register(with: registry.registrar(forPlugin: "PathProviderPlugin")) } diff --git a/pubspec.lock b/pubspec.lock index 46fd8ec..49cee2a 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -49,6 +49,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.19.1" + cross_file: + dependency: transitive + description: + name: cross_file + sha256: "7caf6a750a0c04effbb52a676dce9a4a592e10ad35c34d6d2d0e4811160d5670" + url: "https://pub.dev" + source: hosted + version: "0.3.4+2" crypto: dependency: transitive description: @@ -97,6 +105,38 @@ packages: url: "https://pub.dev" source: hosted version: "2.1.4" + file_selector_linux: + dependency: transitive + description: + name: file_selector_linux + sha256: "54cbbd957e1156d29548c7d9b9ec0c0ebb6de0a90452198683a7d23aed617a33" + url: "https://pub.dev" + source: hosted + version: "0.9.3+2" + file_selector_macos: + dependency: transitive + description: + name: file_selector_macos + sha256: "8c9250b2bd2d8d4268e39c82543bacbaca0fda7d29e0728c3c4bbb7c820fd711" + url: "https://pub.dev" + source: hosted + version: "0.9.4+3" + file_selector_platform_interface: + dependency: transitive + description: + name: file_selector_platform_interface + sha256: a3994c26f10378a039faa11de174d7b78eb8f79e4dd0af2a451410c1a5c3f66b + url: "https://pub.dev" + source: hosted + version: "2.6.2" + file_selector_windows: + dependency: transitive + description: + name: file_selector_windows + sha256: "320fcfb6f33caa90f0b58380489fc5ac05d99ee94b61aa96ec2bff0ba81d3c2b" + url: "https://pub.dev" + source: hosted + version: "0.9.3+4" fixnum: dependency: transitive description: @@ -126,6 +166,14 @@ packages: url: "https://pub.dev" source: hosted version: "0.7.7+1" + flutter_plugin_android_lifecycle: + dependency: transitive + description: + name: flutter_plugin_android_lifecycle + sha256: f948e346c12f8d5480d2825e03de228d0eb8c3a737e4cdaa122267b89c022b5e + url: "https://pub.dev" + source: hosted + version: "2.0.28" flutter_riverpod: dependency: "direct main" description: @@ -192,6 +240,14 @@ packages: description: flutter source: sdk version: "0.0.0" + http: + dependency: transitive + description: + name: http + sha256: "2c11f3f94c687ee9bad77c171151672986360b2b001d109814ee7140b2cf261b" + url: "https://pub.dev" + source: hosted + version: "1.4.0" http_parser: dependency: transitive description: @@ -200,6 +256,70 @@ packages: url: "https://pub.dev" source: hosted version: "4.1.2" + image_picker: + dependency: "direct main" + description: + name: image_picker + sha256: "021834d9c0c3de46bf0fe40341fa07168407f694d9b2bb18d532dc1261867f7a" + url: "https://pub.dev" + source: hosted + version: "1.1.2" + image_picker_android: + dependency: transitive + description: + name: image_picker_android + sha256: "317a5d961cec5b34e777b9252393f2afbd23084aa6e60fcf601dcf6341b9ebeb" + url: "https://pub.dev" + source: hosted + version: "0.8.12+23" + image_picker_for_web: + dependency: transitive + description: + name: image_picker_for_web + sha256: "717eb042ab08c40767684327be06a5d8dbb341fe791d514e4b92c7bbe1b7bb83" + url: "https://pub.dev" + source: hosted + version: "3.0.6" + image_picker_ios: + dependency: transitive + description: + name: image_picker_ios + sha256: "05da758e67bc7839e886b3959848aa6b44ff123ab4b28f67891008afe8ef9100" + url: "https://pub.dev" + source: hosted + version: "0.8.12+2" + image_picker_linux: + dependency: transitive + description: + name: image_picker_linux + sha256: "34a65f6740df08bbbeb0a1abd8e6d32107941fd4868f67a507b25601651022c9" + url: "https://pub.dev" + source: hosted + version: "0.2.1+2" + image_picker_macos: + dependency: transitive + description: + name: image_picker_macos + sha256: "1b90ebbd9dcf98fb6c1d01427e49a55bd96b5d67b8c67cf955d60a5de74207c1" + url: "https://pub.dev" + source: hosted + version: "0.2.1+2" + image_picker_platform_interface: + dependency: transitive + description: + name: image_picker_platform_interface + sha256: "886d57f0be73c4b140004e78b9f28a8914a09e50c2d816bdd0520051a71236a0" + url: "https://pub.dev" + source: hosted + version: "2.10.1" + image_picker_windows: + dependency: transitive + description: + name: image_picker_windows + sha256: "6ad07afc4eb1bc25f3a01084d28520496c4a3bb0cb13685435838167c9dcedeb" + url: "https://pub.dev" + source: hosted + version: "0.2.1+1" intl: dependency: "direct main" description: @@ -280,6 +400,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.16.0" + mime: + dependency: transitive + description: + name: mime + sha256: "41a20518f0cb1256669420fdba0cd90d21561e560ac240f26ef8322e45bb7ed6" + url: "https://pub.dev" + source: hosted + version: "2.0.0" path: dependency: transitive description: diff --git a/pubspec.yaml b/pubspec.yaml index ee814aa..e05c5c4 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -51,6 +51,7 @@ dependencies: uuid: ^4.4.0 intl: ^0.19.0 path_provider: ^2.1.2 + image_picker: ^1.0.4 dev_dependencies: flutter_test: diff --git a/test/app_test.dart b/test/app_test.dart new file mode 100644 index 0000000..59ef2f5 --- /dev/null +++ b/test/app_test.dart @@ -0,0 +1,10 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:flutter_riverpod/flutter_riverpod.dart'; +import 'package:llm_chat_app/main.dart'; + +void main() { + testWidgets('App builds', (tester) async { + await tester.pumpWidget(const ProviderScope(child: MyApp())); + expect(find.byType(MainScreen), findsOneWidget); + }); +} diff --git a/test/widget_test.dart b/test/widget_test.dart deleted file mode 100644 index 694043f..0000000 --- a/test/widget_test.dart +++ /dev/null @@ -1,30 +0,0 @@ -// This is a basic Flutter widget test. -// -// To perform an interaction with a widget in your test, use the WidgetTester -// utility in the flutter_test package. For example, you can send tap and scroll -// gestures. You can also use WidgetTester to find child widgets in the widget -// tree, read text, and verify that the values of widget properties are correct. - -import 'package:flutter/material.dart'; -import 'package:flutter_test/flutter_test.dart'; - -import 'package:llm_chat_app/main.dart'; - -void main() { - testWidgets('Counter increments smoke test', (WidgetTester tester) async { - // Build our app and trigger a frame. - await tester.pumpWidget(const MyApp()); - - // Verify that our counter starts at 0. - expect(find.text('0'), findsOneWidget); - expect(find.text('1'), findsNothing); - - // Tap the '+' icon and trigger a frame. - await tester.tap(find.byIcon(Icons.add)); - await tester.pump(); - - // Verify that our counter has incremented. - expect(find.text('0'), findsNothing); - expect(find.text('1'), findsOneWidget); - }); -} diff --git a/windows/flutter/generated_plugin_registrant.cc b/windows/flutter/generated_plugin_registrant.cc index 0c50753..b53f20e 100644 --- a/windows/flutter/generated_plugin_registrant.cc +++ b/windows/flutter/generated_plugin_registrant.cc @@ -6,9 +6,12 @@ #include "generated_plugin_registrant.h" +#include #include void RegisterPlugins(flutter::PluginRegistry* registry) { + FileSelectorWindowsRegisterWithRegistrar( + registry->GetRegistrarForPlugin("FileSelectorWindows")); FlutterSecureStorageWindowsPluginRegisterWithRegistrar( registry->GetRegistrarForPlugin("FlutterSecureStorageWindowsPlugin")); } diff --git a/windows/flutter/generated_plugins.cmake b/windows/flutter/generated_plugins.cmake index 4fc759c..2b9f993 100644 --- a/windows/flutter/generated_plugins.cmake +++ b/windows/flutter/generated_plugins.cmake @@ -3,6 +3,7 @@ # list(APPEND FLUTTER_PLUGIN_LIST + file_selector_windows flutter_secure_storage_windows )