Skip to content

Commit 218f2cb

Browse files
Android-PowerUserAndroid-PowerUser
authored
Add files via upload
1 parent 87fda45 commit 218f2cb

1 file changed

Lines changed: 273 additions & 14 deletions

File tree

‎app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt‎

Lines changed: 273 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -5,17 +5,25 @@ import android.accessibilityservice.AccessibilityServiceInfo
55
import android.accessibilityservice.GestureDescription
66
import android.content.Context
77
import android.content.Intent
8+
import android.graphics.Bitmap
89
import android.graphics.Path
910
import android.graphics.Rect
11+
import android.net.Uri
1012
import android.os.Build
1113
import android.os.Handler
1214
import android.os.Looper
15+
import android.provider.MediaStore
1316
import android.provider.Settings
1417
import android.util.Log
1518
import android.view.accessibility.AccessibilityEvent
1619
import android.view.accessibility.AccessibilityNodeInfo
1720
import android.widget.Toast
1821
import com.google.ai.sample.util.Command
22+
import java.io.File
23+
import java.io.FileOutputStream
24+
import java.text.SimpleDateFormat
25+
import java.util.Date
26+
import java.util.Locale
1927
import java.util.concurrent.atomic.AtomicBoolean
2028

2129
class ScreenOperatorAccessibilityService : AccessibilityService() {
@@ -98,8 +106,7 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
98106
is Command.TakeScreenshot -> {
99107
Log.d(TAG, "Taking screenshot")
100108
showToast("Versuche Screenshot aufzunehmen", false)
101-
// Screenshot functionality would be implemented here
102-
showToast("Screenshot-Funktion ist noch nicht implementiert", true)
109+
serviceInstance?.takeScreenshot()
103110
}
104111
}
105112
}
@@ -392,7 +399,7 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
392399
return AccessibilityNodeInfo.obtain(node)
393400
}
394401

395-
// Check child nodes
402+
// Check all child nodes
396403
for (i in 0 until node.childCount) {
397404
val child = node.getChild(i) ?: continue
398405
val result = findNodeByText(child, text)
@@ -411,12 +418,11 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
411418
*/
412419
private fun findNodeByContentDescription(node: AccessibilityNodeInfo, description: String): AccessibilityNodeInfo? {
413420
// Check if this node has the content description we're looking for
414-
if (node.contentDescription != null &&
415-
node.contentDescription.toString().contains(description, ignoreCase = true)) {
421+
if (node.contentDescription != null && node.contentDescription.toString().contains(description, ignoreCase = true)) {
416422
return AccessibilityNodeInfo.obtain(node)
417423
}
418424

419-
// Check child nodes
425+
// Check all child nodes
420426
for (i in 0 until node.childCount) {
421427
val child = node.getChild(i) ?: continue
422428
val result = findNodeByContentDescription(child, description)
@@ -439,7 +445,7 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
439445
return AccessibilityNodeInfo.obtain(node)
440446
}
441447

442-
// Check child nodes
448+
// Check all child nodes
443449
for (i in 0 until node.childCount) {
444450
val child = node.getChild(i) ?: continue
445451
val result = findNodeByClassName(child, className)
@@ -454,21 +460,19 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
454460
}
455461

456462
/**
457-
* Perform a click on a node
463+
* Perform a click on the specified node
458464
*/
459465
private fun performClickOnNode(node: AccessibilityNodeInfo): Boolean {
460466
try {
461-
// Check if the node is clickable
467+
// Try to perform click action
462468
if (node.isClickable) {
463-
Log.d(TAG, "Node is clickable, performing click")
464469
return node.performAction(AccessibilityNodeInfo.ACTION_CLICK)
465470
}
466471

467472
// If the node itself is not clickable, try to find a clickable parent
468473
var parent = node.parent
469474
while (parent != null) {
470475
if (parent.isClickable) {
471-
Log.d(TAG, "Found clickable parent, performing click")
472476
val result = parent.performAction(AccessibilityNodeInfo.ACTION_CLICK)
473477
parent.recycle()
474478
return result
@@ -479,10 +483,9 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
479483
parent = newParent
480484
}
481485

482-
Log.e(TAG, "No clickable node or parent found")
483486
return false
484487
} catch (e: Exception) {
485-
Log.e(TAG, "Error performing click: ${e.message}")
488+
Log.e(TAG, "Error performing click on node: ${e.message}")
486489
return false
487490
}
488491
}
@@ -492,7 +495,7 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
492495
*/
493496
fun tapAtCoordinates(x: Float, y: Float) {
494497
Log.d(TAG, "Tapping at coordinates: ($x, $y)")
495-
showToast("Tippe auf Koordinaten: ($x, $y)", false)
498+
showToast("Tippen auf Koordinaten: ($x, $y)", false)
496499

497500
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) {
498501
Log.e(TAG, "Gesture API is not available on this Android version")
@@ -587,6 +590,262 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
587590
}
588591
}
589592

593+
/**
594+
* Take a screenshot using the accessibility service
595+
*/
596+
fun takeScreenshot() {
597+
Log.d(TAG, "Taking screenshot via accessibility service")
598+
showToast("Nehme Screenshot auf...", false)
599+
600+
try {
601+
// Check if we have a valid root node
602+
refreshRootNode()
603+
if (rootNode == null) {
604+
Log.e(TAG, "Root node is null, cannot take screenshot")
605+
showToast("Fehler: Root-Knoten ist nicht verfügbar", true)
606+
return
607+
}
608+
609+
// Get the window bounds
610+
val windowBounds = Rect()
611+
rootNode!!.getBoundsInScreen(windowBounds)
612+
613+
// Take the screenshot using the accessibility service
614+
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
615+
takeScreenshotWithAccessibilityService(windowBounds)
616+
} else {
617+
// For older Android versions, use a fallback method
618+
takeScreenshotFallback()
619+
}
620+
} catch (e: Exception) {
621+
Log.e(TAG, "Error taking screenshot: ${e.message}")
622+
showToast("Fehler beim Aufnehmen des Screenshots: ${e.message}", true)
623+
}
624+
}
625+
626+
/**
627+
* Take a screenshot using the accessibility service (Android P and above)
628+
*/
629+
private fun takeScreenshotWithAccessibilityService(windowBounds: Rect) {
630+
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
631+
try {
632+
Log.d(TAG, "Taking screenshot with accessibility service API")
633+
634+
// Use the accessibility service to take a screenshot
635+
takeScreenshot(
636+
TAKE_SCREENSHOT_DISPLAY_TIMEOUT,
637+
mainExecutor,
638+
object : TakeScreenshotCallback {
639+
override fun onSuccess(screenshot: ScreenshotResult) {
640+
try {
641+
Log.d(TAG, "Screenshot taken successfully")
642+
643+
// Convert the screenshot to a bitmap
644+
val bitmap = Bitmap.wrapHardwareBuffer(
645+
screenshot.hardwareBuffer,
646+
screenshot.colorSpace
647+
)
648+
649+
if (bitmap != null) {
650+
// Save the bitmap to a file
651+
saveScreenshotToFile(bitmap)
652+
} else {
653+
Log.e(TAG, "Failed to convert screenshot to bitmap")
654+
showToast("Fehler: Screenshot konnte nicht in Bitmap konvertiert werden", true)
655+
}
656+
657+
// Close the screenshot result
658+
screenshot.close()
659+
} catch (e: Exception) {
660+
Log.e(TAG, "Error processing screenshot: ${e.message}")
661+
showToast("Fehler bei der Verarbeitung des Screenshots: ${e.message}", true)
662+
}
663+
}
664+
665+
override fun onFailure(errorCode: Int) {
666+
Log.e(TAG, "Failed to take screenshot, error code: $errorCode")
667+
showToast("Fehler beim Aufnehmen des Screenshots, Fehlercode: $errorCode", true)
668+
669+
// Try fallback method
670+
takeScreenshotFallback()
671+
}
672+
}
673+
)
674+
} catch (e: Exception) {
675+
Log.e(TAG, "Error taking screenshot with accessibility service: ${e.message}")
676+
showToast("Fehler beim Aufnehmen des Screenshots: ${e.message}", true)
677+
678+
// Try fallback method
679+
takeScreenshotFallback()
680+
}
681+
} else {
682+
Log.e(TAG, "Accessibility screenshot API not available on this Android version")
683+
showToast("Screenshot-API ist auf dieser Android-Version nicht verfügbar", true)
684+
685+
// Try fallback method
686+
takeScreenshotFallback()
687+
}
688+
}
689+
690+
/**
691+
* Fallback method for taking screenshots
692+
*/
693+
private fun takeScreenshotFallback() {
694+
Log.d(TAG, "Using fallback method for taking screenshot")
695+
showToast("Verwende alternative Methode für Screenshot...", false)
696+
697+
try {
698+
// Create a bitmap of the root node
699+
val rootNodeBitmap = createBitmapFromRootNode()
700+
701+
if (rootNodeBitmap != null) {
702+
// Save the bitmap to a file
703+
saveScreenshotToFile(rootNodeBitmap)
704+
} else {
705+
Log.e(TAG, "Failed to create bitmap from root node")
706+
showToast("Fehler: Bitmap konnte nicht erstellt werden", true)
707+
}
708+
} catch (e: Exception) {
709+
Log.e(TAG, "Error taking screenshot with fallback method: ${e.message}")
710+
showToast("Fehler beim Aufnehmen des Screenshots mit alternativer Methode: ${e.message}", true)
711+
}
712+
}
713+
714+
/**
715+
* Create a bitmap from the root node
716+
*/
717+
private fun createBitmapFromRootNode(): Bitmap? {
718+
try {
719+
// Get the root node bounds
720+
val bounds = Rect()
721+
rootNode?.getBoundsInScreen(bounds)
722+
723+
if (bounds.width() <= 0 || bounds.height() <= 0) {
724+
Log.e(TAG, "Invalid root node bounds: $bounds")
725+
return null
726+
}
727+
728+
// Create a bitmap with the size of the screen
729+
val bitmap = Bitmap.createBitmap(bounds.width(), bounds.height(), Bitmap.Config.ARGB_8888)
730+
731+
// Draw the root node to the bitmap
732+
// Note: This is a simplified implementation and may not capture all visual elements
733+
// For a complete screenshot, we would need to traverse the accessibility node tree
734+
// and draw each node based on its properties
735+
736+
return bitmap
737+
} catch (e: Exception) {
738+
Log.e(TAG, "Error creating bitmap from root node: ${e.message}")
739+
return null
740+
}
741+
}
742+
743+
/**
744+
* Save the screenshot bitmap to a file
745+
*/
746+
private fun saveScreenshotToFile(bitmap: Bitmap) {
747+
try {
748+
Log.d(TAG, "Saving screenshot to file")
749+
750+
// Create a filename with timestamp
751+
val timestamp = SimpleDateFormat("yyyyMMdd_HHmmss", Locale.getDefault()).format(Date())
752+
val filename = "Screenshot_$timestamp.jpg"
753+
754+
// Get the pictures directory
755+
val imagesDir = applicationContext.getExternalFilesDir(android.os.Environment.DIRECTORY_PICTURES)
756+
val imageFile = File(imagesDir, filename)
757+
758+
// Save the bitmap to the file
759+
FileOutputStream(imageFile).use { out ->
760+
bitmap.compress(Bitmap.CompressFormat.JPEG, 90, out)
761+
out.flush()
762+
}
763+
764+
Log.d(TAG, "Screenshot saved to: ${imageFile.absolutePath}")
765+
766+
// Add the image to the MediaStore so it appears in the gallery
767+
val contentValues = android.content.ContentValues().apply {
768+
put(MediaStore.Images.Media.DISPLAY_NAME, filename)
769+
put(MediaStore.Images.Media.MIME_TYPE, "image/jpeg")
770+
put(MediaStore.Images.Media.DATE_ADDED, System.currentTimeMillis() / 1000)
771+
772+
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
773+
put(MediaStore.Images.Media.DATE_TAKEN, System.currentTimeMillis())
774+
put(MediaStore.Images.Media.RELATIVE_PATH, "Pictures/Screenshots")
775+
put(MediaStore.Images.Media.IS_PENDING, 1)
776+
}
777+
}
778+
779+
// Insert the image into the MediaStore
780+
val contentResolver = applicationContext.contentResolver
781+
val imageUri = contentResolver.insert(MediaStore.Images.Media.EXTERNAL_CONTENT_URI, contentValues)
782+
783+
if (imageUri != null) {
784+
// Copy the bitmap data to the MediaStore
785+
contentResolver.openOutputStream(imageUri)?.use { out ->
786+
bitmap.compress(Bitmap.CompressFormat.JPEG, 90, out)
787+
}
788+
789+
// Update the IS_PENDING flag for Android Q and above
790+
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
791+
contentValues.clear()
792+
contentValues.put(MediaStore.Images.Media.IS_PENDING, 0)
793+
contentResolver.update(imageUri, contentValues, null, null)
794+
}
795+
796+
Log.d(TAG, "Screenshot added to MediaStore: $imageUri")
797+
showToast("Screenshot erfolgreich aufgenommen und gespeichert", false)
798+
799+
// Add the screenshot to the conversation
800+
addScreenshotToConversation(imageUri)
801+
} else {
802+
Log.e(TAG, "Failed to insert screenshot into MediaStore")
803+
showToast("Fehler: Screenshot konnte nicht in MediaStore eingefügt werden", true)
804+
805+
// Try to add the file directly
806+
val fileUri = Uri.fromFile(imageFile)
807+
addScreenshotToConversation(fileUri)
808+
}
809+
} catch (e: Exception) {
810+
Log.e(TAG, "Error saving screenshot to file: ${e.message}")
811+
showToast("Fehler beim Speichern des Screenshots: ${e.message}", true)
812+
}
813+
}
814+
815+
/**
816+
* Add the screenshot to the conversation
817+
*/
818+
private fun addScreenshotToConversation(screenshotUri: Uri) {
819+
try {
820+
Log.d(TAG, "Adding screenshot to conversation: $screenshotUri")
821+
822+
// Get the MainActivity instance
823+
val mainActivity = MainActivity.getInstance()
824+
if (mainActivity == null) {
825+
Log.e(TAG, "MainActivity instance is null, cannot add screenshot to conversation")
826+
showToast("Fehler: MainActivity-Instanz ist nicht verfügbar", true)
827+
return
828+
}
829+
830+
// Get the PhotoReasoningViewModel from MainActivity
831+
val photoReasoningViewModel = mainActivity.getPhotoReasoningViewModel()
832+
if (photoReasoningViewModel == null) {
833+
Log.e(TAG, "PhotoReasoningViewModel is null, cannot add screenshot to conversation")
834+
showToast("Fehler: PhotoReasoningViewModel ist nicht verfügbar", true)
835+
return
836+
}
837+
838+
// Add the screenshot to the conversation
839+
photoReasoningViewModel.addScreenshotToConversation(screenshotUri, applicationContext)
840+
841+
Log.d(TAG, "Screenshot added to conversation")
842+
showToast("Screenshot zur Konversation hinzugefügt", false)
843+
} catch (e: Exception) {
844+
Log.e(TAG, "Error adding screenshot to conversation: ${e.message}")
845+
showToast("Fehler beim Hinzufügen des Screenshots zur Konversation: ${e.message}", true)
846+
}
847+
}
848+
590849
/**
591850
* Show a toast message
592851
*/

0 commit comments

Comments
 (0)