Integration of Context-Aware LLM Navigation Interface for Accessibility Apps

.github/workflows/debug_build.yml

@@ -40,7 +40,7 @@ jobs:
           SIGNING_STORE_PASSWORD: ${{ secrets.SIGNING_STORE_PASSWORD }}
 
       - name: Upload APK
-        uses: actions/upload-artifact@v3.1.3
+        uses: actions/upload-artifact@v4
         with:
           name: app-debug-build
           path: app/build/outputs/apk/debug/app-debug.apk

.gradle/config.properties

@@ -1,2 +1,2 @@
-#Thu Aug 29 09:52:32 CEST 2024
-java.home=C\:\\Users\\benek\\Desktop\\Android\\Android Studio\\jbr
+#Sun Nov 24 11:49:21 CET 2024
+java.home=/Applications/Android Studio.app/Contents/jbr/Contents/Home

README.md

@@ -1,6 +1,9 @@
 # Natural Language Interface
 
-This is the code repository for the Android app developed in the studyproject Sensory augmentation and grasping movements.
+This is the code repository for the Android app developed by graduate students at [Osnabrück University](https://www.uni-osnabrueck.de/) during the study projects supervised by [Prof. Dr. Peter König](https://www.ikw.uni-osnabrueck.de/en/research_groups/neurobiopsychology/pk.html):
+
+- Sensory Augmentation and Grasping Movements
+- Spatial Navigation supported by AI
 
 [![Testing APK](https://github.com/StudyProject-NLI/NLInterface/actions/workflows/debug_build.yml/badge.svg)](https://github.com/StudyProject-NLI/NLInterface/actions/workflows/debug_build.yml)
 
@@ -9,20 +12,24 @@ This is the code repository for the Android app developed in the studyproject Se
 ```mermaid
 	gitGraph LR:
 		commit id: "Current state"
-		commit id: "Workgroup Release" tag: "v0.0.1"
-		commit id: "Run example tensorflow model"
+		commit id: "Workgroup release" tag: "v0.0.1"
+		commit id: "Run example TensorFlow model"
 		commit id: "Feedback round with experimental group"
-		commit id: "Integrate feedback"  tag: "v0.0.2"
+		commit id: "Integrate feedback" tag: "v0.0.2"
 		commit id: "Christmas break"
 		commit id: "Object detection via any image input"
 		commit id: "Polishing"
 		commit id: "Optivist Presentation" tag: "v0.0.3"
 		commit id: "Prepare for codebase transfer"
+		commit id: "Location functionalities"
+		commit id: "Improved GUI" tag: "v0.0.4"
+		commit id: "Integrated LLM interface" tag: "v0.0.5"
 ```
 
-
 # Building the app locally
+
 Requirements:
+
 - At least Java 17 JDK installed
 - Create a `local.properties` file at the root of this repository with the following content
 
@@ -32,15 +39,15 @@ MAPS_API_KEY=your_google_maps_api_key
 
 You can request your keys via the [Google Maps Platform](https://developers.google.com/maps/documentation/embed/get-api-key).
 
-Then build the application with 
+Then build the application with
 
 ```
 ./gradlew build -x lint -x lintVitalRelease
 ```
 
 on MacOS or Linux based operating systems.
 
-The resulting installable application can be found under 
+The resulting installable application can be found under
 
 `app/build/outputs/apk/debug/app-debug.apk`
 
@@ -49,11 +56,16 @@ which you can copy and install on your device.
 # Documentation
 
 ## Running locally
+
 - Install Python
 - `pip install -r docs/requirements.txt`
 
-Run the documentation server via 
+Run the documentation server via
+
 ```bash
 $ mkdocs serve
 ```
 
+## Integrations
+
+- [Context-Aware LLM Navigation Interface for Accessibility Apps](https://github.com/RillJ/llm-app-interface)

app/build.gradle

@@ -97,4 +97,7 @@ dependencies {
     //viewpager
     implementation 'androidx.viewpager2:viewpager2:1.1.0'
 
+    // Context-aware LLM API interface
+    implementation 'com.squareup.okhttp3:okhttp:4.9.3'
+
 }

app/src/main/AndroidManifest.xml

@@ -19,6 +19,7 @@
         android:allowBackup="true"
         android:dataExtractionRules="@xml/data_extraction_rules"
         android:fullBackupContent="@xml/backup_rules"
+        android:networkSecurityConfig="@xml/network_security_config"
         android:icon="@mipmap/ic_logo"
         android:label="@string/app_name"
         android:roundIcon="@mipmap/ic_logo_round"

app/src/main/java/com/nlinterface/activities/GroceryListActivity.kt

@@ -114,6 +114,21 @@ class GroceryListActivity : AppCompatActivity(), GroceryListCallback {
         viewPagerSetUp()
         configureTTS()
         configureSTT()
+
+        // Check if we were launched from an LLM voice command
+        if (intent.getBooleanExtra("FROM_VOICE_COMMAND", false)) {
+            // Process items to add
+            val itemsToAdd = intent.getStringArrayListExtra("ITEMS_TO_ADD")
+            itemsToAdd?.forEach { itemName ->
+                addGroceryItem(itemName)
+            }
+
+            // Process items to remove
+            val itemsToRemove = intent.getStringArrayListExtra("ITEMS_TO_REMOVE")
+            itemsToRemove?.forEach { itemName ->
+                deleteGroceryItem(itemName)
+            }
+        }
     }
 
     /**

app/src/main/java/com/nlinterface/activities/PlaceDetailsActivity.kt

@@ -105,11 +105,29 @@ class PlaceDetailsActivity : AppCompatActivity(), PlaceDetailsItemCallback {
         placeDetailsItemList = viewModel.placeDetailsItemList
         placeDetailsAdapter = PlaceDetailsAdapter(placeDetailsItemList, this)
 
-
         configureAutocompleteFragment()
         viewPagerSetUp()
         configureTTS()
         configureSTT()
+
+        val removalIds = intent.getStringArrayListExtra("PLACE_IDS_TO_REMOVE")
+        if (!removalIds.isNullOrEmpty()) {
+            val removedNames = mutableListOf<String>()
+            removalIds.forEach { removalId ->
+                val itemToRemove = placeDetailsItemList.find { it.placeID == removalId }
+                if (itemToRemove != null) {
+                    // Delete the item from the ViewModel list and update storage
+                    viewModel.deletePlaceDetailsItem(itemToRemove)
+                    // Also remove it from the local list and notify the adapter
+                    val index = placeDetailsItemList.indexOf(itemToRemove)
+                    if (index != -1) {
+                        placeDetailsItemList.removeAt(index)
+                        placeDetailsAdapter.notifyItemRemoved(index)
+                    }
+                    removedNames.add(itemToRemove.storeName)
+                }
+            }
+        }
     }