diff --git a/.github/workflows/debug_build.yml b/.github/workflows/debug_build.yml index 0752c0e..7e856b9 100644 --- a/.github/workflows/debug_build.yml +++ b/.github/workflows/debug_build.yml @@ -40,7 +40,7 @@ jobs: SIGNING_STORE_PASSWORD: ${{ secrets.SIGNING_STORE_PASSWORD }} - name: Upload APK - uses: actions/upload-artifact@v3.1.3 + uses: actions/upload-artifact@v4 with: name: app-debug-build path: app/build/outputs/apk/debug/app-debug.apk diff --git a/.gradle/8.2/checksums/checksums.lock b/.gradle/8.2/checksums/checksums.lock index 4a246f4..f199602 100644 Binary files a/.gradle/8.2/checksums/checksums.lock and b/.gradle/8.2/checksums/checksums.lock differ diff --git a/.gradle/8.2/checksums/md5-checksums.bin b/.gradle/8.2/checksums/md5-checksums.bin index 4c5c8cb..7edbcd5 100644 Binary files a/.gradle/8.2/checksums/md5-checksums.bin and b/.gradle/8.2/checksums/md5-checksums.bin differ diff --git a/.gradle/8.2/checksums/sha1-checksums.bin b/.gradle/8.2/checksums/sha1-checksums.bin index e1f4a78..7b63b06 100644 Binary files a/.gradle/8.2/checksums/sha1-checksums.bin and b/.gradle/8.2/checksums/sha1-checksums.bin differ diff --git a/.gradle/8.2/executionHistory/executionHistory.bin b/.gradle/8.2/executionHistory/executionHistory.bin index 7af2412..809d20c 100644 Binary files a/.gradle/8.2/executionHistory/executionHistory.bin and b/.gradle/8.2/executionHistory/executionHistory.bin differ diff --git a/.gradle/8.2/executionHistory/executionHistory.lock b/.gradle/8.2/executionHistory/executionHistory.lock index bd9fed9..42d2bc9 100644 Binary files a/.gradle/8.2/executionHistory/executionHistory.lock and b/.gradle/8.2/executionHistory/executionHistory.lock differ diff --git a/.gradle/8.2/fileHashes/fileHashes.bin b/.gradle/8.2/fileHashes/fileHashes.bin index e843cdf..6cb14de 100644 Binary files a/.gradle/8.2/fileHashes/fileHashes.bin and b/.gradle/8.2/fileHashes/fileHashes.bin differ diff --git a/.gradle/8.2/fileHashes/fileHashes.lock b/.gradle/8.2/fileHashes/fileHashes.lock index 10336c1..441629e 100644 Binary files a/.gradle/8.2/fileHashes/fileHashes.lock and b/.gradle/8.2/fileHashes/fileHashes.lock differ diff --git a/.gradle/8.2/fileHashes/resourceHashesCache.bin b/.gradle/8.2/fileHashes/resourceHashesCache.bin index ad3c434..ce7e436 100644 Binary files a/.gradle/8.2/fileHashes/resourceHashesCache.bin and b/.gradle/8.2/fileHashes/resourceHashesCache.bin differ diff --git a/.gradle/buildOutputCleanup/buildOutputCleanup.lock b/.gradle/buildOutputCleanup/buildOutputCleanup.lock index df039ce..24f1a6d 100644 Binary files a/.gradle/buildOutputCleanup/buildOutputCleanup.lock and b/.gradle/buildOutputCleanup/buildOutputCleanup.lock differ diff --git a/.gradle/buildOutputCleanup/outputFiles.bin b/.gradle/buildOutputCleanup/outputFiles.bin index 8bf9b1e..781d978 100644 Binary files a/.gradle/buildOutputCleanup/outputFiles.bin and b/.gradle/buildOutputCleanup/outputFiles.bin differ diff --git a/.gradle/config.properties b/.gradle/config.properties index 8bdd50e..0534dac 100644 --- a/.gradle/config.properties +++ b/.gradle/config.properties @@ -1,2 +1,2 @@ -#Thu Aug 29 09:52:32 CEST 2024 -java.home=C\:\\Users\\benek\\Desktop\\Android\\Android Studio\\jbr +#Sun Nov 24 11:49:21 CET 2024 +java.home=/Applications/Android Studio.app/Contents/jbr/Contents/Home diff --git a/.gradle/file-system.probe b/.gradle/file-system.probe index 9f7fae1..96dd2ea 100644 Binary files a/.gradle/file-system.probe and b/.gradle/file-system.probe differ diff --git a/README.md b/README.md index 196d749..6654bf6 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,9 @@ # Natural Language Interface -This is the code repository for the Android app developed in the studyproject Sensory augmentation and grasping movements. +This is the code repository for the Android app developed by graduate students at [Osnabrück University](https://www.uni-osnabrueck.de/) during the study projects supervised by [Prof. Dr. Peter König](https://www.ikw.uni-osnabrueck.de/en/research_groups/neurobiopsychology/pk.html): + +- Sensory Augmentation and Grasping Movements +- Spatial Navigation supported by AI [![Testing APK](https://github.com/StudyProject-NLI/NLInterface/actions/workflows/debug_build.yml/badge.svg)](https://github.com/StudyProject-NLI/NLInterface/actions/workflows/debug_build.yml) @@ -9,20 +12,24 @@ This is the code repository for the Android app developed in the studyproject Se ```mermaid gitGraph LR: commit id: "Current state" - commit id: "Workgroup Release" tag: "v0.0.1" - commit id: "Run example tensorflow model" + commit id: "Workgroup release" tag: "v0.0.1" + commit id: "Run example TensorFlow model" commit id: "Feedback round with experimental group" - commit id: "Integrate feedback" tag: "v0.0.2" + commit id: "Integrate feedback" tag: "v0.0.2" commit id: "Christmas break" commit id: "Object detection via any image input" commit id: "Polishing" commit id: "Optivist Presentation" tag: "v0.0.3" commit id: "Prepare for codebase transfer" + commit id: "Location functionalities" + commit id: "Improved GUI" tag: "v0.0.4" + commit id: "Integrated LLM interface" tag: "v0.0.5" ``` - # Building the app locally + Requirements: + - At least Java 17 JDK installed - Create a `local.properties` file at the root of this repository with the following content @@ -32,7 +39,7 @@ MAPS_API_KEY=your_google_maps_api_key You can request your keys via the [Google Maps Platform](https://developers.google.com/maps/documentation/embed/get-api-key). -Then build the application with +Then build the application with ``` ./gradlew build -x lint -x lintVitalRelease @@ -40,7 +47,7 @@ Then build the application with on MacOS or Linux based operating systems. -The resulting installable application can be found under +The resulting installable application can be found under `app/build/outputs/apk/debug/app-debug.apk` @@ -49,11 +56,16 @@ which you can copy and install on your device. # Documentation ## Running locally + - Install Python - `pip install -r docs/requirements.txt` -Run the documentation server via +Run the documentation server via + ```bash $ mkdocs serve ``` +## Integrations + +- [Context-Aware LLM Navigation Interface for Accessibility Apps](https://github.com/RillJ/llm-app-interface) diff --git a/app/build.gradle b/app/build.gradle index 85b7588..8ea3d0e 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -97,4 +97,7 @@ dependencies { //viewpager implementation 'androidx.viewpager2:viewpager2:1.1.0' + // Context-aware LLM API interface + implementation 'com.squareup.okhttp3:okhttp:4.9.3' + } diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index e269454..b5281b0 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -19,6 +19,7 @@ android:allowBackup="true" android:dataExtractionRules="@xml/data_extraction_rules" android:fullBackupContent="@xml/backup_rules" + android:networkSecurityConfig="@xml/network_security_config" android:icon="@mipmap/ic_logo" android:label="@string/app_name" android:roundIcon="@mipmap/ic_logo_round" diff --git a/app/src/main/java/com/nlinterface/activities/GroceryListActivity.kt b/app/src/main/java/com/nlinterface/activities/GroceryListActivity.kt index b92b926..0dc031a 100644 --- a/app/src/main/java/com/nlinterface/activities/GroceryListActivity.kt +++ b/app/src/main/java/com/nlinterface/activities/GroceryListActivity.kt @@ -114,6 +114,21 @@ class GroceryListActivity : AppCompatActivity(), GroceryListCallback { viewPagerSetUp() configureTTS() configureSTT() + + // Check if we were launched from an LLM voice command + if (intent.getBooleanExtra("FROM_VOICE_COMMAND", false)) { + // Process items to add + val itemsToAdd = intent.getStringArrayListExtra("ITEMS_TO_ADD") + itemsToAdd?.forEach { itemName -> + addGroceryItem(itemName) + } + + // Process items to remove + val itemsToRemove = intent.getStringArrayListExtra("ITEMS_TO_REMOVE") + itemsToRemove?.forEach { itemName -> + deleteGroceryItem(itemName) + } + } } /** diff --git a/app/src/main/java/com/nlinterface/activities/PlaceDetailsActivity.kt b/app/src/main/java/com/nlinterface/activities/PlaceDetailsActivity.kt index bdf6f7d..8cffd8d 100644 --- a/app/src/main/java/com/nlinterface/activities/PlaceDetailsActivity.kt +++ b/app/src/main/java/com/nlinterface/activities/PlaceDetailsActivity.kt @@ -105,11 +105,29 @@ class PlaceDetailsActivity : AppCompatActivity(), PlaceDetailsItemCallback { placeDetailsItemList = viewModel.placeDetailsItemList placeDetailsAdapter = PlaceDetailsAdapter(placeDetailsItemList, this) - configureAutocompleteFragment() viewPagerSetUp() configureTTS() configureSTT() + + val removalIds = intent.getStringArrayListExtra("PLACE_IDS_TO_REMOVE") + if (!removalIds.isNullOrEmpty()) { + val removedNames = mutableListOf() + removalIds.forEach { removalId -> + val itemToRemove = placeDetailsItemList.find { it.placeID == removalId } + if (itemToRemove != null) { + // Delete the item from the ViewModel list and update storage + viewModel.deletePlaceDetailsItem(itemToRemove) + // Also remove it from the local list and notify the adapter + val index = placeDetailsItemList.indexOf(itemToRemove) + if (index != -1) { + placeDetailsItemList.removeAt(index) + placeDetailsAdapter.notifyItemRemoved(index) + } + removedNames.add(itemToRemove.storeName) + } + } + } } diff --git a/app/src/main/java/com/nlinterface/activities/VoiceOnlyActivity.kt b/app/src/main/java/com/nlinterface/activities/VoiceOnlyActivity.kt index 719c920..5068a27 100644 --- a/app/src/main/java/com/nlinterface/activities/VoiceOnlyActivity.kt +++ b/app/src/main/java/com/nlinterface/activities/VoiceOnlyActivity.kt @@ -1,5 +1,9 @@ package com.nlinterface.activities +import com.google.gson.Gson +import com.google.gson.reflect.TypeToken +import com.nlinterface.dataclasses.GroceryItem +import java.io.File import android.Manifest import android.content.Intent import android.content.pm.PackageManager @@ -13,6 +17,8 @@ import androidx.lifecycle.ViewModelProvider import androidx.lifecycle.lifecycleScope import com.nlinterface.R import com.nlinterface.databinding.ActivityMainBinding +import com.nlinterface.dataclasses.PlaceDetailsItem +import com.nlinterface.utility.LLMAppConnector import com.nlinterface.utility.OnSwipeTouchListener import com.nlinterface.viewmodels.VoiceOnlyViewModel import kotlinx.coroutines.delay @@ -26,7 +32,7 @@ import kotlinx.coroutines.launch * TODO: Improve and Test */ class VoiceOnlyActivity: AppCompatActivity() { - + private var isProcessingCommand = false // State tracking private lateinit var binding: ActivityMainBinding private lateinit var viewModel: VoiceOnlyViewModel private lateinit var viewFlipper: ViewFlipper @@ -96,14 +102,13 @@ class VoiceOnlyActivity: AppCompatActivity() { * is finished. */ private fun configureSTT() { - viewModel.initSTT() val sttIsListeningObserver = Observer { isListening -> - if (isListening) { + if (isListening && !isProcessingCommand) { showListeningStage() } - else { + else if (!isListening && !isProcessingCommand) { viewModel.handleSTTSpeechBegin() } } @@ -111,8 +116,9 @@ class VoiceOnlyActivity: AppCompatActivity() { // observe LiveData change to be notified when the STT system is active(ly listening) viewModel.isListening.observe(this, sttIsListeningObserver) - val processObserver = Observer {isProcessing -> - if(isProcessing){ + val processObserver = Observer { isProcessing -> + if(isProcessing) { + isProcessingCommand = true showProcessingStage() } } @@ -183,21 +189,254 @@ class VoiceOnlyActivity: AppCompatActivity() { } } - /** - * Handles a received input and starts listening again on completion. - */ + private fun isValidCommand(label: String): Boolean { + val supportedCommands = listOf( + "grocery-list", + "barcode-scanner", + "navigation", + "object-and-hand-recognition" + ) + return supportedCommands.contains(label) + } + private fun processVoiceInput(command: String) { lifecycleScope.launch { showSpeakingStage() - viewModel.sayAndAwait(command) - showListeningStage() - startListening() + + try { + val llmConnector = LLMAppConnector.getInstance + + // Authenticate with the API + val token = llmConnector.authenticate() + + // Add prefix to the command + val prefixedCommand = "$command" + + // Send command to the LLM API + val apiResponse = llmConnector.sendCommandToLLM(prefixedCommand, token) + + // Parse the response and get label and additional data requirement + val (label, needsAdditionalData) = llmConnector.parseResponse(apiResponse) + + if (label != null) { + // If the label is found in the supported commands, execute it + if (isValidCommand(label)) { + executeCommand(label, needsAdditionalData) + } else { + // No valid command found, just say the response + viewModel.sayAndAwait(label) + } + } else { + viewModel.sayAndAwait("I heard: $command, but couldn't process it.") + } + } catch (e: Exception) { + viewModel.sayAndAwait("Sorry, I encountered an error: ${e.message}") + e.printStackTrace() + } finally { + // Reset processing state after command completion + isProcessingCommand = false + //startListening() + } } } - private fun startListening() { - viewModel.handleSTTSpeechBegin() + /** + * Gets the current grocery list formatted as a string. + * Format: "[quantity]Item; [quantity]Item2; ..." + * If the list is empty, returns "No items on the grocery list" + */ + private fun getCurrentGroceryList(): String { + val groceryListFile = File(applicationContext.filesDir, "GroceryList.json") + if (!groceryListFile.exists() || groceryListFile.length() == 0L) { + return "No items on the grocery list" + } + return try { + val json = groceryListFile.readText() + val type = object : TypeToken>() {}.type + val groceryList: ArrayList = Gson().fromJson(json, type) + if (groceryList.isEmpty()) { + "No items on the grocery list" + } else { + groceryList.joinToString("; ") { "[1]\\${it.itemName}" } + } + } catch (e: Exception) { + e.printStackTrace() + "No items on the grocery list" + } } + /** + * Executes the appropriate action based on the label from the LLM API. + */ + private suspend fun executeCommand(label: String, needsAdditionalData: Boolean) { + when (label) { + "grocery-list" -> { + if (needsAdditionalData) { + // Get the current grocery list + val groceryList = getCurrentGroceryList() + + // Send the grocery list to the LLM with prefix + val token = LLMAppConnector.getInstance.authenticate() + val prefixedData = "$groceryList" + val apiResponse = LLMAppConnector.getInstance.sendCommandToLLM(prefixedData, token) + + // Parse the response + val (response, _) = LLMAppConnector.getInstance.parseResponse(apiResponse) + + if (response != null) { + // Process grocery list updates + processGroceryListUpdates(response) + } else { + viewModel.sayAndAwait("Sorry, I couldn't process your request.") + } + } else { + viewModel.sayAndAwait(getString(R.string.navigate_to_grocery_list)) + val intent = Intent(this@VoiceOnlyActivity, GroceryListActivity::class.java) + startActivity(intent) + } + } + "barcode-scanner" -> { + viewModel.sayAndAwait(getString(R.string.barcode_scanner)) + val intent = Intent(this@VoiceOnlyActivity, BarcodeSettingsActivity::class.java) + startActivity(intent) + } + "navigation" -> { + if (needsAdditionalData) { + // Retrieve current place details list as additional data (JSON) + val placeList = getCurrentPlaceList() + val token = LLMAppConnector.getInstance.authenticate() + val prefixedData = "$placeList" + val apiResponse = LLMAppConnector.getInstance.sendCommandToLLM(prefixedData, token) + val (response, _) = LLMAppConnector.getInstance.parseResponse(apiResponse) + if (response != null) { + processNavigationUpdates(response) + } else { + viewModel.sayAndAwait("Sorry, I couldn't process your navigation request.") + } + } else { + viewModel.sayAndAwait(getString(R.string.place_details)) + val intent = Intent(this@VoiceOnlyActivity, PlaceDetailsActivity::class.java) + startActivity(intent) + } + } + "object-and-hand-recognition" -> { + viewModel.sayAndAwait(getString(R.string.classification)) + val intent = Intent(this@VoiceOnlyActivity, ClassificationActivity::class.java) + startActivity(intent) + } + else -> { + viewModel.sayAndAwait("I don't know how to handle: $label; additional-data-required=$needsAdditionalData") + } + } + } + + /** + * Process grocery list updates from LLM response + * Expected format: "[+/-quantity]Item; [quantity]Item2; [-quantity]Item3" + */ + private suspend fun processGroceryListUpdates(response: String) { + // Find the first occurrence of '[' to ignore irrelevant prefixed text + val startIndex = response.indexOf('[') + if (startIndex == -1) { + viewModel.sayAndAwait(response) // No valid grocery list format found, so print what the LLM said + return + } + + val itemsToProcess = response.substring(startIndex).split(";") + val itemsToAdd = mutableListOf() + val itemsToRemove = mutableListOf() + + // Parse the response to identify items to add or remove + for (item in itemsToProcess) { + val trimmedItem = item.trim() + if (trimmedItem.isNotEmpty()) { + // Parse the format [+/-quantity]Item + val regex = """^\[([+\-]?\d+)](.+)$""".toRegex() + val matchResult = regex.find(trimmedItem) + + if (matchResult != null) { + val (quantityStr, itemName) = matchResult.destructured + + if (quantityStr.startsWith("+") || !quantityStr.startsWith("-")) { + // Add item + itemsToAdd.add(itemName.trim()) + } else if (quantityStr.startsWith("-")) { + // Remove item + itemsToRemove.add(itemName.trim()) + } + } + } + } -} + // Launch GroceryListActivity and pass the items to add/remove + val intent = Intent(this, GroceryListActivity::class.java) + intent.putExtra("ITEMS_TO_ADD", ArrayList(itemsToAdd)) + intent.putExtra("ITEMS_TO_REMOVE", ArrayList(itemsToRemove)) + intent.putExtra("FROM_VOICE_COMMAND", true) + startActivity(intent) + + // Give feedback to user + val addMessage = if (itemsToAdd.isNotEmpty()) + "Adding ${itemsToAdd.joinToString(", ")} to your grocery list. " else "" + val removeMessage = if (itemsToRemove.isNotEmpty()) + "Removing ${itemsToRemove.joinToString(", ")} from your grocery list." else "" + + viewModel.sayAndAwait("$addMessage$removeMessage") + } + + // Helper to get the current place list from the saved PlaceDetailsItem JSON file. + private fun getCurrentPlaceList(): String { + val placeListFile = File(applicationContext.filesDir, "PlaceDetailsItemList.json") + if (!placeListFile.exists() || placeListFile.length() == 0L) { + return "{}" + } + return try { + placeListFile.readText() + } catch (e: Exception) { + "{}" + } + } + + private suspend fun processNavigationUpdates(response: String) { + // Process only the content after the tag if it exists. + val processedResponse = if (response.contains("")) { + response.substringAfter("") + } else { + response + } + + // Regex to capture removals of the format: [-1]PlaceID + val removalPattern = "\\[-1\\](\\S+)".toRegex() + val removals = removalPattern.findAll(processedResponse).map { it.groupValues[1] }.toList() + + if (removals.isNotEmpty()) { + // Start PlaceDetailsActivity with the removal IDs passed in the intent. + val intent = Intent(this@VoiceOnlyActivity, PlaceDetailsActivity::class.java) + intent.putStringArrayListExtra("PLACE_IDS_TO_REMOVE", ArrayList(removals)) + startActivity(intent) + + // Retrieve the currently stored PlaceDetailsItem list from JSON. + val placeListJson = getCurrentPlaceList() + val placeNames = try { + // Using the PlaceDetailsItem list from PlaceDetailsActivity. + val type = object : com.google.gson.reflect.TypeToken>() {}.type + val places: List = com.google.gson.Gson().fromJson(placeListJson, type) + // Map each removal ID to its corresponding store name (or fallback to the removal ID). + removals.map { removalId -> + places.find { it.placeID == removalId }?.storeName ?: removalId + } + } catch (e: Exception) { + removals // Fallback if JSON parsing fails. + } + + // Announce removal by speaking only the supermarket names. + viewModel.sayAndAwait("Removing supermarkets with names: ${placeNames.joinToString(", ")}.") + } else { + viewModel.sayAndAwait(response) + } + } + + private fun startListening() { + viewModel.handleSTTSpeechBegin() + } +} \ No newline at end of file diff --git a/app/src/main/java/com/nlinterface/fragments/MainScreen1.kt b/app/src/main/java/com/nlinterface/fragments/MainScreen1.kt index b7b597c..250de9b 100644 --- a/app/src/main/java/com/nlinterface/fragments/MainScreen1.kt +++ b/app/src/main/java/com/nlinterface/fragments/MainScreen1.kt @@ -82,7 +82,7 @@ class MainScreen1 : Fragment(), SwipeAction { } /** - * Navigates to the Voice Only Activity. + * Navigates to the Voice Only Activity (for LLM processing). */ override fun onDoubleTap() { val intent = Intent(activity, VoiceOnlyActivity::class.java) @@ -90,7 +90,7 @@ class MainScreen1 : Fragment(), SwipeAction { } /** - * Activates the LLM to start listening. + * Activates listening to facilitate executing hard-coded commands. */ override fun onLongPress() { if (viewModel.isListening.value == false) { diff --git a/app/src/main/java/com/nlinterface/fragments/MainScreen2.kt b/app/src/main/java/com/nlinterface/fragments/MainScreen2.kt index 6cf07d0..b2a43f1 100644 --- a/app/src/main/java/com/nlinterface/fragments/MainScreen2.kt +++ b/app/src/main/java/com/nlinterface/fragments/MainScreen2.kt @@ -82,7 +82,7 @@ class MainScreen2 : Fragment(), SwipeAction { } /** - * Navigates to the Voice Only Activity. + * Navigates to the Voice Only Activity (for LLM processing). */ override fun onDoubleTap() { val intent = Intent(activity, VoiceOnlyActivity::class.java) @@ -90,7 +90,7 @@ class MainScreen2 : Fragment(), SwipeAction { } /** - * Activates the LLM to start listening. + * Activates listening to facilitate executing hard-coded commands. */ override fun onLongPress() { if (viewModel.isListening.value == false) { diff --git a/app/src/main/java/com/nlinterface/utility/LLMAppConnector.kt b/app/src/main/java/com/nlinterface/utility/LLMAppConnector.kt new file mode 100644 index 0000000..bf6cddf --- /dev/null +++ b/app/src/main/java/com/nlinterface/utility/LLMAppConnector.kt @@ -0,0 +1,171 @@ +package com.nlinterface.utility + +import android.util.Log +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import okhttp3.* +import okhttp3.MediaType.Companion.toMediaTypeOrNull +import org.json.JSONObject +import java.io.IOException +import java.util.concurrent.TimeUnit + +/** + * LLMAppConnector handles communication with the LLM API + * for processing voice commands and getting appropriate action labels. + */ +class LLMAppConnector { + private val TAG = "LLMAppConnector" + private val API_URL = "http://192.168.50.21:8001" + private val USERNAME = "johndoe" + private val PASSWORD = "secret" + + // Create a client with custom timeout settings + private val client = OkHttpClient.Builder() + .connectTimeout(30, TimeUnit.SECONDS) + .readTimeout(30, TimeUnit.SECONDS) + .writeTimeout(30, TimeUnit.SECONDS) + .build() + + /** + * Authenticates with the LLM API and returns an access token. + * + * @return Authentication token string + * @throws Exception if authentication fails + */ + suspend fun authenticate(): String { + return withContext(Dispatchers.IO) { + try { + val formBody = FormBody.Builder() + .add("username", USERNAME) + .add("password", PASSWORD) + .build() + + val request = Request.Builder() + .url("$API_URL/token") + .post(formBody) + .build() + + val response = client.newCall(request).execute() + val responseBody = response.body?.string() ?: "" + + if (!response.isSuccessful) { + throw IOException("Authentication failed: ${response.code}") + } + + // Parse JSON response to get token + val jsonObject = JSONObject(responseBody) + jsonObject.getString("access_token") + } catch (e: Exception) { + Log.e(TAG, "Authentication failed: ${e.message}", e) + throw Exception("Authentication failed: ${e.message}") + } + } + } + + /** + * Sends a command to the LLM API and returns the raw response. + * + * @param command The user's voice command + * @param token The authentication token + * @return Raw JSON response from the API + * @throws Exception if the API request fails + */ + suspend fun sendCommandToLLM(command: String, token: String): String { + return withContext(Dispatchers.IO) { + try { + val jsonObject = JSONObject() + val inputObject = JSONObject() + inputObject.put("messages", command) + jsonObject.put("input", inputObject) + + val mediaType = "application/json; charset=utf-8".toMediaTypeOrNull() + val requestBody = RequestBody.create(mediaType, jsonObject.toString()) + + val request = Request.Builder() + .url("$API_URL/llm-app-interface/invoke") + .addHeader("Authorization", "Bearer $token") + .post(requestBody) + .build() + + val response = client.newCall(request).execute() + val responseBody = response.body?.string() ?: "" + + if (!response.isSuccessful) { + throw IOException("API request failed: ${response.code}") + } + + responseBody + } catch (e: Exception) { + Log.e(TAG, "API request failed: ${e.message}", e) + throw Exception("API request failed: ${e.message}") + } + } + } + + /** + * Parses the LLM response to extract labels and additional data requirements. + * + * @param responseJson Raw JSON response from the API + * @return Pair of the label and whether additional data is required + */ + fun parseResponse(responseJson: String): Pair { + try { + Log.d(TAG, "Parsing response: $responseJson") + + // Parse the API response - first get the output object + val jsonObject = JSONObject(responseJson) + val outputObject = jsonObject.getJSONObject("output") + val messages = outputObject.getJSONArray("messages") + + // Get the last message (which should be the AI response) + if (messages.length() > 0) { + val lastIndex = messages.length() - 1 + val message = messages.getJSONObject(lastIndex) + + // Check if it's an AI message + val type = message.optString("type") + if (type == "ai") { + val content = message.getString("content") + Log.d(TAG, "AI content: $content") + + // Extract label and additional-data-required flag + val labelPattern = """label=([^;]+)""".toRegex() + val additionalDataPattern = """additional-data-required=(True|False)""".toRegex() + + val labelMatch = labelPattern.find(content) + val additionalDataMatch = additionalDataPattern.find(content) + + if (labelMatch != null) { + // Extract only the value part (without "label=") + val label = labelMatch.groupValues[1].trim() + val needsAdditionalData = additionalDataMatch?.groupValues?.get(1) == "True" + return Pair(label, needsAdditionalData) + } + + // If no label pattern was found, return the content as is + return Pair(content, false) + } + } + + return Pair(null, false) + } catch (e: Exception) { + Log.e(TAG, "Failed to parse response: ${e.message}", e) + Log.e(TAG, "Response JSON: $responseJson") + e.printStackTrace() + return Pair(null, false) + } + } + + companion object { + private var instance: LLMAppConnector? = null + + @get:Synchronized + val getInstance: LLMAppConnector + get() { + if (instance == null) { + instance = LLMAppConnector() + } + return instance!! + } + } +} \ No newline at end of file diff --git a/app/src/main/java/com/nlinterface/viewmodels/BarcodeProductinfoViewModel.kt b/app/src/main/java/com/nlinterface/viewmodels/BarcodeProductinfoViewModel.kt index 8cc7ae9..e610bbc 100644 --- a/app/src/main/java/com/nlinterface/viewmodels/BarcodeProductinfoViewModel.kt +++ b/app/src/main/java/com/nlinterface/viewmodels/BarcodeProductinfoViewModel.kt @@ -180,7 +180,6 @@ class ScanningProcess{ * */ fun activateScanning(viewModel: MainViewModel, vibrator: Vibrator, context: Context) { - val selector = CameraSelector.Builder() .requireLensFacing(CameraSelector.LENS_FACING_BACK) .build() @@ -192,6 +191,9 @@ class ScanningProcess{ Scanner(viewModel, vibrator) ) cameraProvider = ProcessCameraProvider.getInstance(context).get() + + // Unbind previous use cases to avoid exceeding supported surfaces + cameraProvider.unbindAll() try { cameraProvider.bindToLifecycle( ProcessLifecycleOwner.get(), @@ -199,7 +201,6 @@ class ScanningProcess{ imageAnalysis ) Log.println(Log.INFO, "Camera", "Camera binding successful") - } catch (e: Exception) { e.printStackTrace().toString() } diff --git a/app/src/main/res/xml/network_security_config.xml b/app/src/main/res/xml/network_security_config.xml new file mode 100644 index 0000000..f18e1f0 --- /dev/null +++ b/app/src/main/res/xml/network_security_config.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file