Skip to content

Commit 33b7a02

Browse files
google-labs-jules[bot]google-labs-jules[bot]
committed
Support percentage-based coordinates for input actions
This commit updates the application to support percentage-based coordinates (e.g., "50%", "25.5%") in addition to pixel-based coordinates for various input actions like `tapAtCoordinate` and coordinate-based scroll commands. Changes include: - Modified `Command.kt` to store coordinate values as Strings in relevant command data classes (`TapCoordinates`, `ScrollDownFromCoordinates`, etc.). - Updated `CommandParser.kt` to correctly parse these string coordinates, including those with a '%' suffix. Regex patterns and parsing logic were adjusted accordingly. - Introduced a `convertCoordinate(String, Int): Float` helper method in `ScreenOperatorAccessibilityService.kt` to convert coordinate strings (either pixel or percentage) into absolute pixel values based on screen dimensions. - Updated the `executeCommand` method in `ScreenOperatorAccessibilityService.kt` to use `convertCoordinate` before dispatching actions. - Added comprehensive unit tests for the new parsing logic in `CommandParserTest.kt` and for the `convertCoordinate` method in `ScreenOperatorAccessibilityServiceTest.kt`, covering various valid inputs, percentages, pixel values, and error conditions. This enhancement provides you with greater flexibility when specifying coordinates for screen interactions.
1 parent 9115e8c commit 33b7a02

4 files changed

Lines changed: 364 additions & 50 deletions

File tree

‎app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt‎

Lines changed: 49 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import com.google.ai.sample.GenerativeViewModelFactory
3232
import java.util.Date
3333
import java.util.Locale
3434
import java.util.concurrent.atomic.AtomicBoolean
35+
import java.lang.NumberFormatException
3536

3637
class ScreenOperatorAccessibilityService : AccessibilityService() {
3738
companion object {
@@ -97,6 +98,10 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
9798
showToast("Accessibility Service is not available. Please enable the service in settings.", true)
9899
return
99100
}
101+
102+
val displayMetrics = serviceInstance!!.resources.displayMetrics
103+
val screenWidth = displayMetrics.widthPixels
104+
val screenHeight = displayMetrics.heightPixels
100105

101106
// Execute the command
102107
when (command) {
@@ -106,9 +111,11 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
106111
serviceInstance?.findAndClickButtonByText(command.buttonText)
107112
}
108113
is Command.TapCoordinates -> {
109-
Log.d(TAG, "Tapping at coordinates: (${command.x}, ${command.y})")
110-
showToast("Trying to tap coordinates: (${command.x}, ${command.y})", false)
111-
serviceInstance?.tapAtCoordinates(command.x, command.y)
114+
val xPx = serviceInstance!!.convertCoordinate(command.x, screenWidth)
115+
val yPx = serviceInstance!!.convertCoordinate(command.y, screenHeight)
116+
Log.d(TAG, "Tapping at coordinates: (${command.x} -> $xPx, ${command.y} -> $yPx)")
117+
showToast("Trying to tap coordinates: ($xPx, $yPx)", false)
118+
serviceInstance?.tapAtCoordinates(xPx, yPx)
112119
}
113120
is Command.TakeScreenshot -> {
114121
Log.d(TAG, "Taking screenshot with 850ms delay")
@@ -154,24 +161,32 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
154161
serviceInstance?.scrollRight()
155162
}
156163
is Command.ScrollDownFromCoordinates -> {
157-
Log.d(TAG, "Scrolling down from coordinates (${command.x}, ${command.y}) with distance ${command.distance} and duration ${command.duration}ms")
158-
showToast("Trying to scroll down from position (${command.x}, ${command.y})", false)
159-
serviceInstance?.scrollDown(command.x, command.y, command.distance, command.duration)
164+
val xPx = serviceInstance!!.convertCoordinate(command.x, screenWidth)
165+
val yPx = serviceInstance!!.convertCoordinate(command.y, screenHeight)
166+
Log.d(TAG, "Scrolling down from coordinates (${command.x} -> $xPx, ${command.y} -> $yPx) with distance ${command.distance} and duration ${command.duration}ms")
167+
showToast("Trying to scroll down from position ($xPx, $yPx)", false)
168+
serviceInstance?.scrollDown(xPx, yPx, command.distance, command.duration)
160169
}
161170
is Command.ScrollUpFromCoordinates -> {
162-
Log.d(TAG, "Scrolling up from coordinates (${command.x}, ${command.y}) with distance ${command.distance} and duration ${command.duration}ms")
163-
showToast("Trying to scroll up from position (${command.x}, ${command.y})", false)
164-
serviceInstance?.scrollUp(command.x, command.y, command.distance, command.duration)
171+
val xPx = serviceInstance!!.convertCoordinate(command.x, screenWidth)
172+
val yPx = serviceInstance!!.convertCoordinate(command.y, screenHeight)
173+
Log.d(TAG, "Scrolling up from coordinates (${command.x} -> $xPx, ${command.y} -> $yPx) with distance ${command.distance} and duration ${command.duration}ms")
174+
showToast("Trying to scroll up from position ($xPx, $yPx)", false)
175+
serviceInstance?.scrollUp(xPx, yPx, command.distance, command.duration)
165176
}
166177
is Command.ScrollLeftFromCoordinates -> {
167-
Log.d(TAG, "Scrolling left from coordinates (${command.x}, ${command.y}) with distance ${command.distance} and duration ${command.duration}ms")
168-
showToast("Trying to scroll left from position (${command.x}, ${command.y})", false)
169-
serviceInstance?.scrollLeft(command.x, command.y, command.distance, command.duration)
178+
val xPx = serviceInstance!!.convertCoordinate(command.x, screenWidth)
179+
val yPx = serviceInstance!!.convertCoordinate(command.y, screenHeight)
180+
Log.d(TAG, "Scrolling left from coordinates (${command.x} -> $xPx, ${command.y} -> $yPx) with distance ${command.distance} and duration ${command.duration}ms")
181+
showToast("Trying to scroll left from position ($xPx, $yPx)", false)
182+
serviceInstance?.scrollLeft(xPx, yPx, command.distance, command.duration)
170183
}
171184
is Command.ScrollRightFromCoordinates -> {
172-
Log.d(TAG, "Scrolling right from coordinates (${command.x}, ${command.y}) with distance ${command.distance} and duration ${command.duration}ms")
173-
showToast("Trying to scroll right from position (${command.x}, ${command.y})", false)
174-
serviceInstance?.scrollRight(command.x, command.y, command.distance, command.duration)
185+
val xPx = serviceInstance!!.convertCoordinate(command.x, screenWidth)
186+
val yPx = serviceInstance!!.convertCoordinate(command.y, screenHeight)
187+
Log.d(TAG, "Scrolling right from coordinates (${command.x} -> $xPx, ${command.y} -> $yPx) with distance ${command.distance} and duration ${command.duration}ms")
188+
showToast("Trying to scroll right from position ($xPx, $yPx)", false)
189+
serviceInstance?.scrollRight(xPx, yPx, command.distance, command.duration)
175190
}
176191
is Command.OpenApp -> {
177192
Log.d(TAG, "Opening app: ${command.packageName}")
@@ -256,6 +271,25 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
256271
// Show a toast to indicate the service is connected
257272
showToast("Accessibility Service is enabled and connected", false)
258273
}
274+
275+
private fun convertCoordinate(coordinateString: String, screenSize: Int): Float {
276+
return try {
277+
if (coordinateString.endsWith("%")) {
278+
val numericValue = coordinateString.removeSuffix("%").toFloat()
279+
(numericValue / 100.0f) * screenSize
280+
} else {
281+
coordinateString.toFloat()
282+
}
283+
} catch (e: NumberFormatException) {
284+
Log.e(TAG, "Error converting coordinate string: '$coordinateString'", e)
285+
showToast("Error parsing coordinate: '$coordinateString'. Using 0f.", true)
286+
0f // Default to 0f or handle error as appropriate
287+
} catch (e: Exception) {
288+
Log.e(TAG, "Unexpected error converting coordinate string: '$coordinateString'", e)
289+
showToast("Unexpected error parsing coordinate: '$coordinateString'. Using 0f.", true)
290+
0f // Default to 0f or handle error as appropriate
291+
}
292+
}
259293

260294
override fun onInterrupt() {
261295
Log.d(TAG, "Accessibility service interrupted")

‎app/src/main/kotlin/com/google/ai/sample/util/CommandParser.kt‎

Lines changed: 36 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -77,13 +77,13 @@ object CommandParser {
7777
// Tap coordinates patterns - expanded to catch more variations
7878
private val TAP_COORDINATES_PATTERNS = listOf(
7979
// Standard patterns
80-
Regex("(?i)\\b(?:tap|click|press|tippe|klicke|tippe auf|klicke auf) (?:at|on|auf) (?:coordinates?|koordinaten|position|stelle|punkt)[:\\s]\\s*\\(?\\s*(\\d+(?:\\.\\d+)?)\\s*,\\s*(\\d+(?:\\.\\d+)?)\\s*\\)?"),
81-
Regex("(?i)\\b(?:tap|click|press|tippe|klicke|tippe auf|klicke auf) (?:at|on|auf) \\(?\\s*(\\d+(?:\\.\\d+)?)\\s*,\\s*(\\d+(?:\\.\\d+)?)\\s*\\)?"),
80+
Regex("(?i)\\b(?:tap|click|press|tippe|klicke|tippe auf|klicke auf) (?:at|on|auf) (?:coordinates?|koordinaten|position|stelle|punkt)[:\\s]\\s*\\(?\\s*([\\d\\.%]+)\\s*,\\s*([\\d\\.%]+)\\s*\\)?"),
81+
Regex("(?i)\\b(?:tap|click|press|tippe|klicke|tippe auf|klicke auf) (?:at|on|auf) \\(?\\s*([\\d\\.%]+)\\s*,\\s*([\\d\\.%]+)\\s*\\)?"),
8282

8383
// Function-like patterns
84-
Regex("(?i)\\btapAtCoordinates\\(\\s*(\\d+(?:\\.\\d+)?)\\s*,\\s*(\\d+(?:\\.\\d+)?)\\s*\\)"),
85-
Regex("(?i)\\bclickAtPosition\\(\\s*(\\d+(?:\\.\\d+)?)\\s*,\\s*(\\d+(?:\\.\\d+)?)\\s*\\)"),
86-
Regex("(?i)\\btapAt\\(\\s*(\\d+(?:\\.\\d+)?)\\s*,\\s*(\\d+(?:\\.\\d+)?)\\s*\\)")
84+
Regex("(?i)\\btapAtCoordinates\\(\\s*([\\d\\.%]+)\\s*,\\s*([\\d\\.%]+)\\s*\\)"),
85+
Regex("(?i)\\bclickAtPosition\\(\\s*([\\d\\.%]+)\\s*,\\s*([\\d\\.%]+)\\s*\\)"),
86+
Regex("(?i)\\btapAt\\(\\s*([\\d\\.%]+)\\s*,\\s*([\\d\\.%]+)\\s*\\)")
8787
)
8888

8989
// Screenshot patterns - expanded for consistency
@@ -475,13 +475,14 @@ object CommandParser {
475475
for (match in matches) {
476476
try {
477477
if (match.groupValues.size > 2) {
478-
val x = match.groupValues[1].trim().toFloat()
479-
val y = match.groupValues[2].trim().toFloat()
478+
val xString = match.groupValues[1].trim()
479+
val yString = match.groupValues[2].trim()
480480

481481
// Check if this command is already in the list (avoid duplicates)
482-
if (!commands.any { it is Command.TapCoordinates && it.x == x && it.y == y }) {
483-
Log.d(TAG, "Found tap coordinates command with pattern ${pattern.pattern}: ($x, $y)")
484-
commands.add(Command.TapCoordinates(x, y))
482+
// Note: Comparison now happens with strings directly.
483+
if (!commands.any { it is Command.TapCoordinates && it.x == xString && it.y == yString }) {
484+
Log.d(TAG, "Found tap coordinates command with pattern ${pattern.pattern}: ($xString, $yString)")
485+
commands.add(Command.TapCoordinates(xString, yString))
485486
}
486487
}
487488
} catch (e: Exception) {
@@ -568,19 +569,19 @@ object CommandParser {
568569
*/
569570
private fun findScrollDownCommands(text: String, commands: MutableList<Command>) {
570571
// First check for coordinate-based scroll down commands
571-
val coordPattern = Regex("(?i)\\bscrollDown\\s*\\(\\s*(\\d+)\\s*,\\s*(\\d+)\\s*,\\s*(\\d+)\\s*,\\s*(\\d+)\\s*\\)")
572+
val coordPattern = Regex("(?i)\\bscrollDown\\s*\\(\\s*([\\d\\.%]+)\\s*,\\s*([\\d\\.%]+)\\s*,\\s*(\\d+(?:\\.\\d+)?)\\s*,\\s*(\\d+)\\s*\\)")
572573
val matches = coordPattern.findAll(text)
573574

574575
for (match in matches) {
575576
if (match.groupValues.size >= 5) {
576577
try {
577-
val x = match.groupValues[1].toFloat()
578-
val y = match.groupValues[2].toFloat()
578+
val xString = match.groupValues[1].trim()
579+
val yString = match.groupValues[2].trim()
579580
val distance = match.groupValues[3].toFloat()
580581
val duration = match.groupValues[4].toLong()
581582

582-
Log.d(TAG, "Found coordinate-based scroll down command: scrollDown($x, $y, $distance, $duration)")
583-
commands.add(Command.ScrollDownFromCoordinates(x, y, distance, duration))
583+
Log.d(TAG, "Found coordinate-based scroll down command: scrollDown($xString, $yString, $distance, $duration)")
584+
commands.add(Command.ScrollDownFromCoordinates(xString, yString, distance, duration))
584585
} catch (e: Exception) {
585586
Log.e(TAG, "Error parsing coordinate-based scroll down command: ${e.message}")
586587
}
@@ -609,19 +610,19 @@ object CommandParser {
609610
*/
610611
private fun findScrollUpCommands(text: String, commands: MutableList<Command>) {
611612
// First check for coordinate-based scroll up commands
612-
val coordPattern = Regex("(?i)\\bscrollUp\\s*\\(\\s*(\\d+)\\s*,\\s*(\\d+)\\s*,\\s*(\\d+)\\s*,\\s*(\\d+)\\s*\\)")
613+
val coordPattern = Regex("(?i)\\bscrollUp\\s*\\(\\s*([\\d\\.%]+)\\s*,\\s*([\\d\\.%]+)\\s*,\\s*(\\d+(?:\\.\\d+)?)\\s*,\\s*(\\d+)\\s*\\)")
613614
val matches = coordPattern.findAll(text)
614615

615616
for (match in matches) {
616617
if (match.groupValues.size >= 5) {
617618
try {
618-
val x = match.groupValues[1].toFloat()
619-
val y = match.groupValues[2].toFloat()
619+
val xString = match.groupValues[1].trim()
620+
val yString = match.groupValues[2].trim()
620621
val distance = match.groupValues[3].toFloat()
621622
val duration = match.groupValues[4].toLong()
622623

623-
Log.d(TAG, "Found coordinate-based scroll up command: scrollUp($x, $y, $distance, $duration)")
624-
commands.add(Command.ScrollUpFromCoordinates(x, y, distance, duration))
624+
Log.d(TAG, "Found coordinate-based scroll up command: scrollUp($xString, $yString, $distance, $duration)")
625+
commands.add(Command.ScrollUpFromCoordinates(xString, yString, distance, duration))
625626
} catch (e: Exception) {
626627
Log.e(TAG, "Error parsing coordinate-based scroll up command: ${e.message}")
627628
}
@@ -650,19 +651,19 @@ object CommandParser {
650651
*/
651652
private fun findScrollLeftCommands(text: String, commands: MutableList<Command>) {
652653
// First check for coordinate-based scroll left commands
653-
val coordPattern = Regex("(?i)\\bscrollLeft\\s*\\(\\s*(\\d+)\\s*,\\s*(\\d+)\\s*,\\s*(\\d+)\\s*,\\s*(\\d+)\\s*\\)")
654+
val coordPattern = Regex("(?i)\\bscrollLeft\\s*\\(\\s*([\\d\\.%]+)\\s*,\\s*([\\d\\.%]+)\\s*,\\s*(\\d+(?:\\.\\d+)?)\\s*,\\s*(\\d+)\\s*\\)")
654655
val matches = coordPattern.findAll(text)
655656

656657
for (match in matches) {
657658
if (match.groupValues.size >= 5) {
658659
try {
659-
val x = match.groupValues[1].toFloat()
660-
val y = match.groupValues[2].toFloat()
660+
val xString = match.groupValues[1].trim()
661+
val yString = match.groupValues[2].trim()
661662
val distance = match.groupValues[3].toFloat()
662663
val duration = match.groupValues[4].toLong()
663664

664-
Log.d(TAG, "Found coordinate-based scroll left command: scrollLeft($x, $y, $distance, $duration)")
665-
commands.add(Command.ScrollLeftFromCoordinates(x, y, distance, duration))
665+
Log.d(TAG, "Found coordinate-based scroll left command: scrollLeft($xString, $yString, $distance, $duration)")
666+
commands.add(Command.ScrollLeftFromCoordinates(xString, yString, distance, duration))
666667
} catch (e: Exception) {
667668
Log.e(TAG, "Error parsing coordinate-based scroll left command: ${e.message}")
668669
}
@@ -691,19 +692,19 @@ object CommandParser {
691692
*/
692693
private fun findScrollRightCommands(text: String, commands: MutableList<Command>) {
693694
// First check for coordinate-based scroll right commands
694-
val coordPattern = Regex("(?i)\\bscrollRight\\s*\\(\\s*(\\d+)\\s*,\\s*(\\d+)\\s*,\\s*(\\d+)\\s*,\\s*(\\d+)\\s*\\)")
695+
val coordPattern = Regex("(?i)\\bscrollRight\\s*\\(\\s*([\\d\\.%]+)\\s*,\\s*([\\d\\.%]+)\\s*,\\s*(\\d+(?:\\.\\d+)?)\\s*,\\s*(\\d+)\\s*\\)")
695696
val matches = coordPattern.findAll(text)
696697

697698
for (match in matches) {
698699
if (match.groupValues.size >= 5) {
699700
try {
700-
val x = match.groupValues[1].toFloat()
701-
val y = match.groupValues[2].toFloat()
701+
val xString = match.groupValues[1].trim()
702+
val yString = match.groupValues[2].trim()
702703
val distance = match.groupValues[3].toFloat()
703704
val duration = match.groupValues[4].toLong()
704705

705-
Log.d(TAG, "Found coordinate-based scroll right command: scrollRight($x, $y, $distance, $duration)")
706-
commands.add(Command.ScrollRightFromCoordinates(x, y, distance, duration))
706+
Log.d(TAG, "Found coordinate-based scroll right command: scrollRight($xString, $yString, $distance, $duration)")
707+
commands.add(Command.ScrollRightFromCoordinates(xString, yString, distance, duration))
707708
} catch (e: Exception) {
708709
Log.e(TAG, "Error parsing coordinate-based scroll right command: ${e.message}")
709710
}
@@ -796,7 +797,7 @@ sealed class Command {
796797
/**
797798
* Command to tap at the specified coordinates
798799
*/
799-
data class TapCoordinates(val x: Float, val y: Float) : Command()
800+
data class TapCoordinates(val x: String, val y: String) : Command()
800801

801802
/**
802803
* Command to take a screenshot
@@ -846,22 +847,22 @@ sealed class Command {
846847
/**
847848
* Command to scroll down from specific coordinates with custom distance and duration
848849
*/
849-
data class ScrollDownFromCoordinates(val x: Float, val y: Float, val distance: Float, val duration: Long) : Command()
850+
data class ScrollDownFromCoordinates(val x: String, val y: String, val distance: Float, val duration: Long) : Command()
850851

851852
/**
852853
* Command to scroll up from specific coordinates with custom distance and duration
853854
*/
854-
data class ScrollUpFromCoordinates(val x: Float, val y: Float, val distance: Float, val duration: Long) : Command()
855+
data class ScrollUpFromCoordinates(val x: String, val y: String, val distance: Float, val duration: Long) : Command()
855856

856857
/**
857858
* Command to scroll left from specific coordinates with custom distance and duration
858859
*/
859-
data class ScrollLeftFromCoordinates(val x: Float, val y: Float, val distance: Float, val duration: Long) : Command()
860+
data class ScrollLeftFromCoordinates(val x: String, val y: String, val distance: Float, val duration: Long) : Command()
860861

861862
/**
862863
* Command to scroll right from specific coordinates with custom distance and duration
863864
*/
864-
data class ScrollRightFromCoordinates(val x: Float, val y: Float, val distance: Float, val duration: Long) : Command()
865+
data class ScrollRightFromCoordinates(val x: String, val y: String, val distance: Float, val duration: Long) : Command()
865866

866867
/**
867868
* Command to open an app by package name

0 commit comments

Comments
 (0)