Skip to content

Commit c402fb8

Browse files
Android-PowerUserAndroid-PowerUser
authored
Add files via upload
1 parent 236ccb1 commit c402fb8

2 files changed

Lines changed: 105 additions & 65 deletions

File tree

‎app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningScreen.kt‎

Lines changed: 23 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -67,14 +67,9 @@ import coil.request.SuccessResult
6767
import com.google.ai.sample.GenerativeViewModelFactory
6868
import coil.size.Precision
6969
import com.google.ai.sample.R
70-
import com.google.ai.sample.ScreenOperatorAccessibilityService
7170
import com.google.ai.sample.util.UriSaver
72-
import kotlinx.coroutines.delay
7371
import kotlinx.coroutines.launch
74-
import java.io.File
75-
import android.content.Context
7672
import android.util.Log
77-
import android.widget.Toast
7873

7974
@Composable
8075
internal fun PhotoReasoningRoute(
@@ -91,71 +86,34 @@ internal fun PhotoReasoningRoute(
9186
uiState = photoReasoningUiState,
9287
onReasonClicked = { inputText, selectedItems ->
9388
coroutineScope.launch {
94-
Log.d("PhotoReasoningScreen", "Go button clicked, taking screenshot")
89+
Log.d("PhotoReasoningScreen", "Go button clicked, processing images")
9590

96-
// Show a toast to indicate we're taking a screenshot
97-
Toast.makeText(context, "Taking screenshot...", Toast.LENGTH_SHORT).show()
98-
99-
// Take a screenshot first
100-
ScreenOperatorAccessibilityService.takeScreenshot {
101-
// This will be called after the screenshot is taken
102-
coroutineScope.launch {
103-
Log.d("PhotoReasoningScreen", "Screenshot callback triggered")
104-
105-
// Give some time for the screenshot to be saved and processed
106-
delay(1000)
107-
108-
// Get the latest screenshot URI directly
109-
val screenshotUri = ScreenOperatorAccessibilityService.getLatestScreenshotUri()
110-
val updatedItems = selectedItems.toMutableList()
111-
112-
// Add the screenshot to the list if it exists
113-
if (screenshotUri != null) {
114-
try {
115-
updatedItems.add(screenshotUri)
116-
Log.d("PhotoReasoningScreen", "Added screenshot URI: $screenshotUri")
117-
118-
// Show a toast to indicate the screenshot was added
119-
Toast.makeText(context, "Screenshot added", Toast.LENGTH_SHORT).show()
120-
} catch (e: Exception) {
121-
Log.e("PhotoReasoningScreen", "Error adding screenshot URI: ${e.message}")
122-
Toast.makeText(context, "Error adding screenshot: ${e.message}", Toast.LENGTH_SHORT).show()
123-
}
91+
// Process all selected images
92+
val bitmaps = selectedItems.mapNotNull {
93+
Log.d("PhotoReasoningScreen", "Processing image: $it")
94+
val imageRequest = imageRequestBuilder
95+
.data(it)
96+
.precision(Precision.EXACT)
97+
.build()
98+
try {
99+
val result = imageLoader.execute(imageRequest)
100+
if (result is SuccessResult) {
101+
Log.d("PhotoReasoningScreen", "Successfully processed image")
102+
return@mapNotNull (result.drawable as BitmapDrawable).bitmap
124103
} else {
125-
Log.e("PhotoReasoningScreen", "Screenshot URI not found")
126-
Toast.makeText(context, "Screenshot not found", Toast.LENGTH_SHORT).show()
127-
}
128-
129-
// Process all images including the screenshot
130-
val bitmaps = updatedItems.mapNotNull {
131-
Log.d("PhotoReasoningScreen", "Processing image: $it")
132-
val imageRequest = imageRequestBuilder
133-
.data(it)
134-
// Scale the image down to 768px for faster uploads deaktiviert um genaue Auflösungen feedback zu bekommen
135-
// .size(size = 768)
136-
.precision(Precision.EXACT)
137-
.build()
138-
try {
139-
val result = imageLoader.execute(imageRequest)
140-
if (result is SuccessResult) {
141-
Log.d("PhotoReasoningScreen", "Successfully processed image")
142-
return@mapNotNull (result.drawable as BitmapDrawable).bitmap
143-
} else {
144-
Log.e("PhotoReasoningScreen", "Failed to process image: result is not SuccessResult")
145-
return@mapNotNull null
146-
}
147-
} catch (e: Exception) {
148-
Log.e("PhotoReasoningScreen", "Error processing image: ${e.message}")
149-
return@mapNotNull null
150-
}
104+
Log.e("PhotoReasoningScreen", "Failed to process image: result is not SuccessResult")
105+
return@mapNotNull null
151106
}
152-
153-
Log.d("PhotoReasoningScreen", "Processed ${bitmaps.size} images")
154-
155-
// Send to AI
156-
viewModel.reason(inputText, bitmaps)
107+
} catch (e: Exception) {
108+
Log.e("PhotoReasoningScreen", "Error processing image: ${e.message}")
109+
return@mapNotNull null
157110
}
158111
}
112+
113+
Log.d("PhotoReasoningScreen", "Processed ${bitmaps.size} images")
114+
115+
// Send to AI
116+
viewModel.reason(inputText, bitmaps)
159117
}
160118
}
161119
)

‎app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,16 @@
1717
package com.google.ai.sample.feature.multimodal
1818

1919
import android.graphics.Bitmap
20+
import android.graphics.drawable.BitmapDrawable
21+
import android.net.Uri
2022
import android.util.Log
23+
import android.widget.Toast
2124
import androidx.lifecycle.ViewModel
2225
import androidx.lifecycle.viewModelScope
26+
import coil.ImageLoader
27+
import coil.request.ImageRequest
28+
import coil.request.SuccessResult
29+
import coil.size.Precision
2330
import com.google.ai.client.generativeai.GenerativeModel
2431
import com.google.ai.client.generativeai.type.content
2532
import com.google.ai.sample.ScreenOperatorAccessibilityService
@@ -39,13 +46,30 @@ class PhotoReasoningViewModel(
3946
MutableStateFlow(PhotoReasoningUiState.Initial)
4047
val uiState: StateFlow<PhotoReasoningUiState> =
4148
_uiState.asStateFlow()
49+
50+
// Keep track of the latest screenshot URI
51+
private var latestScreenshotUri: Uri? = null
52+
53+
// Keep track of the current selected images
54+
private var currentSelectedImages: List<Bitmap> = emptyList()
55+
56+
// Keep track of the current user input
57+
private var currentUserInput: String = ""
58+
59+
// ImageLoader and ImageRequestBuilder for processing images
60+
private var imageLoader: ImageLoader? = null
61+
private var imageRequestBuilder: ImageRequest.Builder? = null
4262

4363
fun reason(
4464
userInput: String,
4565
selectedImages: List<Bitmap>
4666
) {
4767
_uiState.value = PhotoReasoningUiState.Loading
4868
val prompt = "Look at the image(s), and then answer the following question: $userInput"
69+
70+
// Store the current user input and selected images
71+
currentUserInput = userInput
72+
currentSelectedImages = selectedImages
4973

5074
viewModelScope.launch(Dispatchers.IO) {
5175
try {
@@ -100,4 +124,62 @@ class PhotoReasoningViewModel(
100124
}
101125
}
102126
}
127+
128+
/**
129+
* Add a screenshot to the conversation
130+
*/
131+
fun addScreenshotToConversation(screenshotUri: Uri, context: android.content.Context) {
132+
viewModelScope.launch(Dispatchers.Main) {
133+
try {
134+
Log.d(TAG, "Adding screenshot to conversation: $screenshotUri")
135+
136+
// Store the latest screenshot URI
137+
latestScreenshotUri = screenshotUri
138+
139+
// Initialize ImageLoader and ImageRequestBuilder if needed
140+
if (imageLoader == null) {
141+
imageLoader = ImageLoader.Builder(context).build()
142+
}
143+
if (imageRequestBuilder == null) {
144+
imageRequestBuilder = ImageRequest.Builder(context)
145+
}
146+
147+
// Process the screenshot
148+
val imageRequest = imageRequestBuilder!!
149+
.data(screenshotUri)
150+
.precision(Precision.EXACT)
151+
.build()
152+
153+
try {
154+
val result = imageLoader!!.execute(imageRequest)
155+
if (result is SuccessResult) {
156+
Log.d(TAG, "Successfully processed screenshot")
157+
val bitmap = (result.drawable as BitmapDrawable).bitmap
158+
159+
// Add the screenshot to the current images
160+
val updatedImages = currentSelectedImages.toMutableList()
161+
updatedImages.add(bitmap)
162+
163+
// Update the current selected images
164+
currentSelectedImages = updatedImages
165+
166+
// Re-send the query with the updated images
167+
reason(currentUserInput, updatedImages)
168+
169+
// Show a toast to indicate the screenshot was added
170+
Toast.makeText(context, "Screenshot added to conversation", Toast.LENGTH_SHORT).show()
171+
} else {
172+
Log.e(TAG, "Failed to process screenshot: result is not SuccessResult")
173+
Toast.makeText(context, "Failed to process screenshot", Toast.LENGTH_SHORT).show()
174+
}
175+
} catch (e: Exception) {
176+
Log.e(TAG, "Error processing screenshot: ${e.message}", e)
177+
Toast.makeText(context, "Error processing screenshot: ${e.message}", Toast.LENGTH_SHORT).show()
178+
}
179+
} catch (e: Exception) {
180+
Log.e(TAG, "Error adding screenshot to conversation: ${e.message}", e)
181+
Toast.makeText(context, "Error adding screenshot: ${e.message}", Toast.LENGTH_SHORT).show()
182+
}
183+
}
184+
}
103185
}

0 commit comments

Comments
 (0)