privatelm-cross-platform-llm-client

# PrivateLM Cross-Platform LLM Client > Skill by [ara.so](https://ara.so) — Devtools Skills collection. ## What It Does PrivateLM is a production-ready Flutter framework for building AI chat applications that seamlessly switch between: - **Local on-device inference** — GPU-accelerated GGUF model execution via `llama.cpp` (Android/iOS) - **Cloud API providers** — OpenAI, Anthropic Claude, Google Gemini, Kimi (Moonshot AI) - **Multimodal capabilities** — Text and vision support for both local (Qwen2-VL) and cloud models - **Offline-first architecture** — All data persisted locally via Hive; no cloud dependency for local mode **Key Features:** - Auto-detects device RAM/GPU and configures optimal inference parameters - Background service integration with Firebase Cloud Messaging - Cross-platform (Android full support, iOS via Metal, Web cloud-only) - Persistent chat sessions and task management ## Installation ### Add to Flutter Project ```yaml # pubspec.yaml dependencies: flutter: sdk: flutter get: ^4.6.5 hive: ^2.2.3 hive_flutter: ^1.1.0 dio: ^5.3.2 http: ^1.1.0 flutter_background_service: ^5.0.0 flutter_local_notifications: ^15.1.0 firebase_core: ^2.15.0 firebase_messaging: ^14.6.5 device_info_plus: ^9.0.3 path_provider: ^2.1.0 image_picker: ^1.0.2 permission_handler: ^11.0.0 dev_dependencies: flutter_test: sdk: flutter hive_generator: ^2.0.0 build_runner: ^2.4.6 ``` ### Platform Configuration **Android (android/app/build.gradle.kts):** ```kotlin android { namespace = "com.yourcompany.yourapp" compileSdk = 34 ndkVersion = "25.1.8937393" defaultConfig { minSdk = 28 // Required for llama_flutter_android targetSdk = 34 ndk { abiFilters.add("arm64-v8a") // 64-bit ARM only } } buildTypes { release { isMinifyEnabled = true isShrinkResources = true proguardFiles( getDefaultProguardFile("proguard-android-optimize.txt"), "proguard-rules.pro" ) } } } ``` **iOS (ios/Podfile):** ```ruby platform :ios, '12.0' target 'Runner' do use_frameworks! use_modular_headers! # Ensure Metal framework for GPU acceleration pod 'MetalKit' end ``` ### Initialize the App ```dart // lib/main.dart import 'package:flutter/material.dart'; import 'package:get/get.dart'; import 'package:hive_flutter/hive_flutter.dart'; import 'package:firebase_core/firebase_core.dart'; import 'services/hive_service.dart'; import 'services/device_info_service.dart'; import 'controllers/settings_controller.dart'; import 'controllers/model_controller.dart'; import 'controllers/chat_controller.dart'; void main() async { WidgetsFlutterBinding.ensureInitialized(); // Initialize Firebase await Firebase.initializeApp(); // Initialize Hive await Hive.initFlutter(); await HiveService.init(); // Initialize GetX controllers Get.put(DeviceInfoService()); Get.put(SettingsController()); Get.put(ModelController()); Get.put(ChatController()); runApp(MyApp()); } class MyApp extends StatelessWidget { @override Widget build(BuildContext context) { return GetMaterialApp( title: 'PrivateLM Chat', theme: ThemeData.dark(), home: HomeView(), ); } } ``` ## Architecture Overview ### Service Layer ```dart // lib/services/inference_service.dart import 'package:get/get.dart'; import 'dart:io' show Platform; class InferenceService extends GetxService { static bool get supportsLocalInference { if (Platform.isAndroid) return true; if (Platform.isIOS) return true; return false; // Web not yet supported } Future<void> loadModel(String modelPath, { required int contextSize, required int numThreads, required int gpuLayers, }) async { if (!supportsLocalInference) { throw UnsupportedError('Local inference not available on this platform'); } // Platform-specific implementation if (Platform.isAndroid || Platform.isIOS) { await _loadModelNative(modelPath, contextSize, numThreads, gpuLayers); } } Stream<String> generateChat({ required List<Map<String, String>> messages, required String template, int maxTokens = 512, double temperature = 0.7, }) async* { // Streaming token generation // Implementation calls native llama.cpp bridge } } ``` ### Cloud Service Integration ```dart // lib/services/cloud_service.dart import 'package:dio/dio.dart'; import 'package:get/get.dart'; class CloudService extends GetxService { final Dio _dio = Dio(); // OpenAI-compatible endpoint Stream<String> generateOpenAI({ required String apiKey, required List<Map<String, dynamic>> messages, String model = 'gpt-4', int maxTokens = 1024, double temperature = 0.7, }) async* { final response = await _dio.post( 'https://api.openai.com/v1/chat/completions', options: Options( headers: { 'Authorization': 'Bearer $apiKey', 'Content-Type': 'application/json', }, ), data: { 'model': model, 'messages': messages, 'max_tokens': maxTokens, 'temperature': temperature, 'stream': true, }, ); // Parse SSE stream await for (final chunk in response.data.stream) { yield _parseOpenAIChunk(chunk); } } // Anthropic Claude endpoint Stream<String> generateClaude({ required String apiKey, required List<Map<String, dynamic>> messages, String model = 'claude-3-opus-20240229', int maxTokens = 1024, }) async* { // Extract system message String? systemMessage; final userMessages = messages.where((m) { if (m['role'] == 'system') { systemMessage = m['content']; return false; } return true; }).toList(); final response = await _dio.post( 'https://api.anthropic.com/v1/messages', options: Options( headers: { 'x-api-key': apiKey, 'anthropic-version': '2023-06-01', 'Content-Type': 'application/json', }, ), data: { 'model': model, 'max_tokens': maxTokens, 'messages': userMessages, if (systemMessage != null) 'system': systemMessage, 'stream': true, }, ); await for (final chunk in response.data.stream) { yield _parseClaudeChunk(chunk); } } // Google Gemini endpoint Stream<String> generateGemini({ required String apiKey, required List<Map<String, dynamic>> messages, String model = 'gemini-1.5-pro', }) async* { final contents = messages.map((m) => { 'role': m['role'] == 'assistant' ? 'model' : 'user', 'parts': [{'text': m['content']}], }).toList(); final response = await _dio.post( 'https://generativelanguage.googleapis.com/v1beta/models/$model:streamGenerateContent?key=$apiKey', options: Options( headers: {'Content-Type': 'application/json'}, ), data: {'contents': contents}, ); await for (final chunk in response.data.stream) { yield _parseGeminiChunk(chunk); } } } ``` ## Device Detection & Auto-Configuration ```dart // lib/services/device_info_service.dart import 'package:device_info_plus/device_info_plus.dart'; import 'package:get/get.dart'; import 'dart:io'; enum DeviceTier { ultra, high, mid, low } class DeviceInfoService extends GetxService { final DeviceInfoPlugin _deviceInfo = DeviceInfoPlugin(); DeviceTier? _tier; int? _totalRamMB; DeviceTier get tier => _tier ?? DeviceTier.mid; int get totalRamMB => _totalRamMB ?? 4096; Future<void> detectDevice() async { if (Platform.isAndroid) { final androidInfo = await _deviceInfo.androidInfo; _totalRamMB = _estimateAndroidRAM(androidInfo); } else if (Platform.isIOS) { final iosInfo = await _deviceInfo.iosInfo; _totalRamMB = _estimateIOSRAM(iosInfo); } _tier = _calculateTier(_totalRamMB!); } DeviceTier _calculateTier(int ramMB) { if (ramMB >= 12000) return DeviceTier.ultra; // 12GB+ if (ramMB >= 8000) return DeviceTier.high; // 8-12GB if (ramMB >= 6000) return DeviceTier.mid; // 6-8GB return DeviceTier.low; // <6GB } Map<String, int> getOptimalSettings() { switch (tier) { case DeviceTier.ultra: return { 'contextSize': 8192, 'threads': 8, 'gpuLayers': 33, 'batchSize': 512, }; case DeviceTier.high: return { 'contextSize': 4096, 'threads': 6, 'gpuLayers': 25, 'batchSize': 256, }; case DeviceTier.mid: return { 'contextSize': 2048, 'threads': 4, 'gpuLayers': 15, 'batchSize': 128, }; case DeviceTier.low: return { 'contextSize': 1024, 'threads': 2, 'gpuLayers': 8, 'batchSize': 64, }; } } } ``` ## Local Model Management ```dart // lib/controllers/model_controller.dart import 'package:get/get.dart'; import 'package:dio/dio.dart'; import 'package:path_provider/path_provider.dart'; import 'dart:io'; class ModelController extends GetxController { final RxList<LocalModel> models = <LocalModel>[].obs; final RxMap<String, double> downloadProgress = <String, double>{}.obs; Future<void> downloadModel({ required String url, required String modelName, required String fileName, }) async { final appDir = await getApplicationDocumentsDirectory(); final modelDir = Directory('${appDir.path}/models'); if (!await modelDir.exists()) { await modelDir.create(recursive: true); } final filePath = '${modelDir.path}/$fileName'; try { await Dio().download( url, filePath, onReceiveProgress: (received, total) { if (total != -1) { final progress = received / total; downloadProgress[modelName] = progress; } }, ); // Save model metadata final model = LocalModel( name: modelName, path: filePath, sizeBytes: File(filePath).lengthSync(), downloadDate: DateTime.now(), ); models.add(model); await _saveModelsToHive(); Get.snackbar('Success', '$modelName downloaded successfully'); } catch (e) { Get.snackbar('Error', 'Failed to download $modelName: $e'); downloadProgress.remove(modelName); } } Future<void> deleteModel(String modelName) async { final model = models.firstWhere((m) => m.name == modelName); final file = File(model.path); if (await file.exists()) { await file.delete(); } models.removeWhere((m) => m.name == modelName); await _saveModelsToHive(); } } class LocalModel { final String name; final String path; final int sizeBytes; final DateTime downloadDate; LocalModel({ required this.name, required this.path, required this.sizeBytes, required this.downloadDate, }); } ``` ## Chat Implementation ```dart // lib/controllers/chat_controller.dart import 'package:get/get.dart'; import '../services/inference_service.dart'; import '../services/cloud_service.dart'; import '../services/hive_service.dart'; class ChatController extends GetxController { final InferenceService _inference = Get.find(); final CloudService _cloud = Get.find(); final HiveService _hive = Get.find(); final RxList<ChatMessage> messages = <ChatMessage>[].obs; final RxBool isGenerating = false.obs;