diff --git a/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt b/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt index fad120c..0daa352 100644 --- a/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt +++ b/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt @@ -424,12 +424,21 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { val delayMillis = pendingScreenshotDelayMillis pendingScreenshotDelayMillis = 0L + fun buildScreenInfoPayload(rawScreenInfo: String?): String? { + val termuxOutput = TermuxOutputPreferences.consumeOutput(applicationContext)?.trim().orEmpty() + if (termuxOutput.isBlank()) { + return rawScreenInfo + } + Log.i(TAG, "executeTakeScreenshotCommand: Overriding Screen elements payload with Termux output. chars=${termuxOutput.length}") + return "Termux output:\n$termuxOutput" + } + val captureAndRequestScreenshot = { val currentModel = GenerativeAiViewModelFactory.getCurrentModel() if (!currentModel.supportsScreenshot) { Log.d(TAG, "Command.TakeScreenshot: Model has no screenshot support, capturing screen info only.") showToast("Capturing screen info...", false) - val screenInfo = captureScreenInformation() + val screenInfo = buildScreenInfoPayload(captureScreenInformation()) val mainActivity = MainActivity.getInstance() mainActivity?.getPhotoReasoningViewModel()?.addScreenshotToConversation( Uri.EMPTY, @@ -440,7 +449,7 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { Log.d(TAG, "Command.TakeScreenshot: Capturing screen info and sending request broadcast to MainActivity.") showToast("Preparing screenshot...", false) - val screenInfo = captureScreenInformation() + val screenInfo = buildScreenInfoPayload(captureScreenInformation()) val intent = Intent(MainActivity.ACTION_REQUEST_MEDIAPROJECTION_SCREENSHOT).apply { putExtra(MainActivity.EXTRA_SCREEN_INFO, screenInfo) @@ -632,31 +641,53 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { } val resultBundle = intent.getBundleExtra("com.termux.app.extra.TERMUX_SERVICE.EXTRA_PLUGIN_RESULT_BUNDLE") ?: intent.getBundleExtra("result") - if (resultBundle == null) { - Log.w(TAG, "Termux result bundle missing; available extras=${intent.extras?.keySet()?.joinToString()}") - unregisterSelf() - return - } - val stdout = resultBundle.getString("com.termux.app.extra.TERMUX_SERVICE.EXTRA_PLUGIN_RESULT_BUNDLE_STDOUT") - ?: resultBundle.getString("stdout") - ?: "" - val stderr = resultBundle.getString("com.termux.app.extra.TERMUX_SERVICE.EXTRA_PLUGIN_RESULT_BUNDLE_STDERR") - ?: resultBundle.getString("stderr") - ?: "" + val extras = intent.extras + val stdout = sequenceOf( + resultBundle?.getString("com.termux.app.extra.TERMUX_SERVICE.EXTRA_PLUGIN_RESULT_BUNDLE_STDOUT"), + resultBundle?.getString("stdout"), + extras?.getString("com.termux.app.extra.TERMUX_SERVICE.EXTRA_PLUGIN_RESULT_BUNDLE_STDOUT"), + extras?.getString("stdout") + ).firstOrNull { !it.isNullOrBlank() }.orEmpty() + val stderr = sequenceOf( + resultBundle?.getString("com.termux.app.extra.TERMUX_SERVICE.EXTRA_PLUGIN_RESULT_BUNDLE_STDERR"), + resultBundle?.getString("stderr"), + extras?.getString("com.termux.app.extra.TERMUX_SERVICE.EXTRA_PLUGIN_RESULT_BUNDLE_STDERR"), + extras?.getString("stderr") + ).firstOrNull { !it.isNullOrBlank() }.orEmpty() val exitCode = when { - resultBundle.containsKey("com.termux.app.extra.TERMUX_SERVICE.EXTRA_PLUGIN_RESULT_BUNDLE_EXIT_CODE") -> { + resultBundle?.containsKey("com.termux.app.extra.TERMUX_SERVICE.EXTRA_PLUGIN_RESULT_BUNDLE_EXIT_CODE") == true -> { resultBundle.getInt("com.termux.app.extra.TERMUX_SERVICE.EXTRA_PLUGIN_RESULT_BUNDLE_EXIT_CODE", Int.MIN_VALUE) } - resultBundle.containsKey("exitCode") -> resultBundle.getInt("exitCode", Int.MIN_VALUE) + resultBundle?.containsKey("exitCode") == true -> resultBundle.getInt("exitCode", Int.MIN_VALUE) + extras?.containsKey("com.termux.app.extra.TERMUX_SERVICE.EXTRA_PLUGIN_RESULT_BUNDLE_EXIT_CODE") == true -> { + extras.getInt("com.termux.app.extra.TERMUX_SERVICE.EXTRA_PLUGIN_RESULT_BUNDLE_EXIT_CODE", Int.MIN_VALUE) + } + extras?.containsKey("exitCode") == true -> extras.getInt("exitCode", Int.MIN_VALUE) else -> Int.MIN_VALUE } - Log.i(TAG, "Termux result received: exitCode=$exitCode stdoutLen=${stdout.length} stderrLen=${stderr.length} keys=${resultBundle.keySet().joinToString()}") + val resultKeys = resultBundle?.keySet()?.joinToString().orEmpty() + val extraKeys = extras?.keySet()?.joinToString().orEmpty() + Log.i(TAG, "Termux result received: exitCode=$exitCode stdoutLen=${stdout.length} stderrLen=${stderr.length} bundleKeys=$resultKeys extraKeys=$extraKeys") val hasKnownResult = stdout.isNotBlank() || stderr.isNotBlank() || exitCode != Int.MIN_VALUE if (!hasKnownResult) { - Log.w(TAG, "Ignoring Termux callback without stdout/stderr/exitCode to avoid polluting pending output.") + val rawExtrasDump = extras?.keySet()?.joinToString("\n") { key -> "$key=${extras.get(key)}" }.orEmpty().trim() + if (rawExtrasDump.isBlank()) { + Log.w(TAG, "Ignoring Termux callback without stdout/stderr/exitCode and no readable extras.") + unregisterSelf() + return + } + Log.w(TAG, "Termux callback missing standard stdout/stderr/exitCode fields; falling back to raw extras dump for AI handoff.") + TermuxOutputPreferences.appendOutput(appContext, "Termux callback raw extras:\n$rawExtrasDump") + mainHandler.post { + MainActivity.getInstance()?.updateStatusMessage("Termux raw result captured", false) + } + serviceInstance?.handler?.post { + Log.d(TAG, "Termux raw callback captured, scheduling next command processing.") + serviceInstance?.scheduleNextCommandProcessing() + } unregisterSelf() return } diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt index 3393a9b..c3b4b05 100644 --- a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt @@ -2663,10 +2663,11 @@ private fun processCommands(text: String) { } val termuxOutputInfo = TermuxOutputPreferences.consumeOutput(appContext)?.let { "Termux output:\n$it" } if (!termuxOutputInfo.isNullOrBlank()) { - Log.i(TAG, "buildEnrichedScreenInfo: Injecting Termux output into next screen-info bubble. chars=${termuxOutputInfo.length}") + Log.i(TAG, "buildEnrichedScreenInfo: Replacing screen-elements bubble with Termux output. chars=${termuxOutputInfo.length}") + return termuxOutputInfo } val missingInfo = listOfNotNull(appNotFoundInfo, termuxNotFoundInfo).joinToString("\n").ifBlank { null } - val extraInfo = listOfNotNull(missingInfo, retrievedInfo, termuxOutputInfo).joinToString("\n\n").ifBlank { null } + val extraInfo = listOfNotNull(missingInfo, retrievedInfo).joinToString("\n\n").ifBlank { null } return when { !extraInfo.isNullOrBlank() && !screenInfo.isNullOrBlank() -> "$extraInfo\n\n$screenInfo" diff --git a/app/src/main/kotlin/com/google/ai/sample/util/TermuxOutputPreferences.kt b/app/src/main/kotlin/com/google/ai/sample/util/TermuxOutputPreferences.kt index ec2d0ab..8ee182a 100644 --- a/app/src/main/kotlin/com/google/ai/sample/util/TermuxOutputPreferences.kt +++ b/app/src/main/kotlin/com/google/ai/sample/util/TermuxOutputPreferences.kt @@ -11,14 +11,20 @@ object TermuxOutputPreferences { val prefs = context.getSharedPreferences(PREF_NAME, Context.MODE_PRIVATE) val existing = prefs.getString(KEY_PENDING_OUTPUT, "").orEmpty() val merged = if (existing.isBlank()) output else "$existing\n\n$output" - prefs.edit().putString(KEY_PENDING_OUTPUT, merged).apply() + val committed = prefs.edit().putString(KEY_PENDING_OUTPUT, merged).commit() + if (!committed) { + throw IllegalStateException("Failed to persist pending Termux output") + } } fun consumeOutput(context: Context): String? { val prefs = context.getSharedPreferences(PREF_NAME, Context.MODE_PRIVATE) val value = prefs.getString(KEY_PENDING_OUTPUT, "").orEmpty().trim() if (value.isBlank()) return null - prefs.edit().remove(KEY_PENDING_OUTPUT).apply() + val committed = prefs.edit().remove(KEY_PENDING_OUTPUT).commit() + if (!committed) { + throw IllegalStateException("Failed to clear consumed Termux output") + } return value } }