feat: Phase 2 act primitives
Add the Artemis Go CLI and its packages: - internal/adb: timeout-bounded adb wrapper, device list, wireless-debugging port scan and auto-connect (never hardcodes the moving port). - internal/ui: uiautomator dump parsing, find-by-text, tap the centre of a text match (FR-4), type, swipe, scroll, back, home, launch, wake. - cmd/artemis: CLI for the primitives plus 'demo', the Phase 2 DoD run. Failures are loud: a missing or disabled element errors instead of tapping a guessed coordinate. The screen is woken before acting, and every adb call has a deadline. Also record the section 13 Q4 decision (adb lives in the user profile on .13, wireless ADB needs no udev rules; service packaging still open until Phase 6), add unit tests, and document the phase evidence.
This commit is contained in:
Paperclip Builder
324
internal/ui/actions.go
Normal file
324
internal/ui/actions.go
Normal file
@@ -0,0 +1,324 @@
|
||||
package ui
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.lab.audasmedia.com.au/sam/artemis_control/internal/adb"
|
||||
)
|
||||
|
||||
// Android key events we use.
|
||||
const (
|
||||
KeyHome = 3
|
||||
KeyBack = 4
|
||||
KeyEnter = 66
|
||||
KeyWakeup = 224
|
||||
)
|
||||
|
||||
// Well-known packages.
|
||||
const (
|
||||
PackageSpotify = "com.spotify.music"
|
||||
PackageMaps = "com.google.android.apps.maps"
|
||||
PackageSettings = "com.android.settings"
|
||||
)
|
||||
|
||||
// Tap sends a tap at absolute screen coordinates.
|
||||
func Tap(r *adb.Runner, x, y int) error {
|
||||
if x < 0 || y < 0 {
|
||||
return fmt.Errorf("refusing to tap negative coordinates (%d,%d)", x, y)
|
||||
}
|
||||
_, err := r.Shell(fmt.Sprintf("input tap %d %d", x, y))
|
||||
return err
|
||||
}
|
||||
|
||||
// KeyEvent sends a keyevent by numeric code.
|
||||
func KeyEvent(r *adb.Runner, code int) error {
|
||||
_, err := r.Shell(fmt.Sprintf("input keyevent %d", code))
|
||||
return err
|
||||
}
|
||||
|
||||
// Back presses the system back button.
|
||||
func Back(r *adb.Runner) error { return KeyEvent(r, KeyBack) }
|
||||
|
||||
// Home presses the system home button.
|
||||
func Home(r *adb.Runner) error { return KeyEvent(r, KeyHome) }
|
||||
|
||||
// Type types text into the focused field.
|
||||
func Type(r *adb.Runner, text string) error {
|
||||
if strings.TrimSpace(text) == "" {
|
||||
return fmt.Errorf("refusing to type empty text")
|
||||
}
|
||||
_, err := r.Shell("input text " + escapeInputText(text))
|
||||
return err
|
||||
}
|
||||
|
||||
// escapeInputText makes a string safe for `input text`.
|
||||
//
|
||||
// `input text` decodes %s as a space, so spaces are encoded first. The result
|
||||
// is then single-quoted for the device shell, with embedded quotes escaped.
|
||||
func escapeInputText(text string) string {
|
||||
encoded := strings.ReplaceAll(text, " ", "%s")
|
||||
return "'" + strings.ReplaceAll(encoded, "'", `'\''`) + "'"
|
||||
}
|
||||
|
||||
// Swipe drags from one point to another over duration.
|
||||
func Swipe(r *adb.Runner, x1, y1, x2, y2, durationMs int) error {
|
||||
if durationMs <= 0 {
|
||||
durationMs = 300
|
||||
}
|
||||
_, err := r.Shell(fmt.Sprintf("input swipe %d %d %d %d %d", x1, y1, x2, y2, durationMs))
|
||||
return err
|
||||
}
|
||||
|
||||
// tapCandidate ranks matches so the tap lands on something a human could press.
|
||||
type tapCandidate struct {
|
||||
node *Node
|
||||
clickable bool
|
||||
enabled bool
|
||||
area int
|
||||
}
|
||||
|
||||
// pickBestMatch chooses the most tappable node from a set of text matches.
|
||||
// Clickable and enabled nodes win; among equals the smallest node wins, because
|
||||
// the smallest enclosing element is the control itself rather than its container.
|
||||
func pickBestMatch(matches []*Node) (*Node, error) {
|
||||
candidates := make([]tapCandidate, 0, len(matches))
|
||||
for _, n := range matches {
|
||||
area := 1 << 30
|
||||
if r, err := n.BoundsRect(); err == nil {
|
||||
area = (r.X2 - r.X1) * (r.Y2 - r.Y1)
|
||||
}
|
||||
candidates = append(candidates, tapCandidate{node: n, clickable: n.Clickable, enabled: n.Enabled, area: area})
|
||||
}
|
||||
sort.SliceStable(candidates, func(i, j int) bool {
|
||||
a, b := candidates[i], candidates[j]
|
||||
if a.clickable != b.clickable {
|
||||
return a.clickable
|
||||
}
|
||||
if a.enabled != b.enabled {
|
||||
return a.enabled
|
||||
}
|
||||
return a.area < b.area
|
||||
})
|
||||
best := candidates[0]
|
||||
if !best.enabled {
|
||||
return nil, fmt.Errorf("best match %s is disabled", best.node.Label())
|
||||
}
|
||||
return best.node, nil
|
||||
}
|
||||
|
||||
// TapText finds an element by its text (or content description) and taps its
|
||||
// centre. This is FR-4: text match first, coordinates derived from bounds.
|
||||
//
|
||||
// It fails loudly when no element matches and never falls back to tapping a
|
||||
// guessed coordinate.
|
||||
func TapText(r *adb.Runner, query string, exact bool) (*Node, error) {
|
||||
if query == "" {
|
||||
return nil, fmt.Errorf("tap requires a non-empty text query")
|
||||
}
|
||||
h, err := Dump(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
matches := h.FindText(query, exact)
|
||||
if len(matches) == 0 {
|
||||
return nil, fmt.Errorf("element with text %q not found on screen; refusing to tap blindly", query)
|
||||
}
|
||||
node, err := pickBestMatch(matches)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
x, y, err := node.Center()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := Tap(r, x, y); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return node, nil
|
||||
}
|
||||
|
||||
// WaitForText polls the screen until an element matching query appears, then
|
||||
// returns the best tappable match. Use it after an action that loads a new
|
||||
// screen; apps do not draw instantly.
|
||||
func WaitForText(r *adb.Runner, query string, exact bool, timeout time.Duration) (*Node, error) {
|
||||
if timeout <= 0 {
|
||||
timeout = 10 * time.Second
|
||||
}
|
||||
deadline := time.Now().Add(timeout)
|
||||
var seen []*Node
|
||||
for time.Now().Before(deadline) {
|
||||
h, err := Dump(r)
|
||||
if err == nil {
|
||||
matches := h.FindText(query, exact)
|
||||
if len(matches) > 0 {
|
||||
seen = matches
|
||||
if n, perr := pickBestMatch(matches); perr == nil {
|
||||
return n, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
time.Sleep(400 * time.Millisecond)
|
||||
}
|
||||
if len(seen) > 0 {
|
||||
return nil, fmt.Errorf("text %q is on screen but no enabled match could be tapped", query)
|
||||
}
|
||||
return nil, fmt.Errorf("text %q did not appear within %s", query, timeout)
|
||||
}
|
||||
|
||||
// ScreenSize reads the physical screen size.
|
||||
func ScreenSize(r *adb.Runner) (width, height int, err error) {
|
||||
out, err := r.Shell("wm size")
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
// "Physical size: 1080x2400" (an Override size line may follow).
|
||||
re := regexp.MustCompile(`(\d+)x(\d+)`)
|
||||
m := re.FindStringSubmatch(out)
|
||||
if m == nil {
|
||||
return 0, 0, fmt.Errorf("could not parse screen size from %q", strings.TrimSpace(out))
|
||||
}
|
||||
w, _ := strconv.Atoi(m[1])
|
||||
h, _ := strconv.Atoi(m[2])
|
||||
return w, h, nil
|
||||
}
|
||||
|
||||
// Scroll scrolls the current screen in the given direction ("down", "up").
|
||||
//
|
||||
// It prefers a scrollable node from the dump, so the swipe stays inside a list
|
||||
// rather than on a toolbar. If no scrollable node is present it falls back to
|
||||
// the middle of the screen.
|
||||
func Scroll(r *adb.Runner, direction string) error {
|
||||
direction = strings.ToLower(strings.TrimSpace(direction))
|
||||
if direction != "down" && direction != "up" {
|
||||
return fmt.Errorf("scroll direction must be \"up\" or \"down\", got %q", direction)
|
||||
}
|
||||
|
||||
width, height, err := ScreenSize(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rect := Rect{X1: 0, Y1: 0, X2: width, Y2: height}
|
||||
h, dumpErr := Dump(r)
|
||||
if dumpErr == nil {
|
||||
// Pick the tallest scrollable node that is taller than it is wide.
|
||||
// Short or horizontal elements (carousels, toolbars) are ignored: a
|
||||
// vertical swipe inside them becomes a mis-gesture, which is exactly
|
||||
// what "never act blindly" forbids.
|
||||
var best *Node
|
||||
bestHeight := 0
|
||||
h.Walk(func(n *Node) {
|
||||
if !n.Scrollable || !n.Enabled {
|
||||
return
|
||||
}
|
||||
b, berr := n.BoundsRect()
|
||||
if berr != nil {
|
||||
return
|
||||
}
|
||||
nHeight := b.Y2 - b.Y1
|
||||
if nHeight <= b.X2-b.X1 {
|
||||
return
|
||||
}
|
||||
if nHeight > bestHeight {
|
||||
bestHeight, best = nHeight, n
|
||||
}
|
||||
})
|
||||
// Only trust it if it covers a meaningful slice of the screen.
|
||||
if best != nil && bestHeight >= height/3 {
|
||||
if b, berr := best.BoundsRect(); berr == nil {
|
||||
rect = b
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const margin = 0.25
|
||||
midX := (rect.X1 + rect.X2) / 2
|
||||
top := rect.Y1 + int(float64(rect.Y2-rect.Y1)*margin)
|
||||
bottom := rect.Y2 - int(float64(rect.Y2-rect.Y1)*margin)
|
||||
if direction == "down" {
|
||||
return Swipe(r, midX, bottom, midX, top, 400)
|
||||
}
|
||||
return Swipe(r, midX, top, midX, bottom, 400)
|
||||
}
|
||||
|
||||
// Wake makes sure the screen is on before any action (FR-7).
|
||||
func Wake(r *adb.Runner) error {
|
||||
if _, err := r.Shell("input keyevent " + strconv.Itoa(KeyWakeup)); err != nil {
|
||||
return fmt.Errorf("wake keyevent: %w", err)
|
||||
}
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
out, err := r.Shell("dumpsys power | grep mWakefulness=")
|
||||
if err == nil && strings.Contains(out, "mWakefulness=Awake") {
|
||||
// Dismiss a non-secure keyguard; harmless when the phone is unlocked.
|
||||
_, _ = r.Shell("wm dismiss-keyguard")
|
||||
return nil
|
||||
}
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
}
|
||||
return fmt.Errorf("screen did not reach Awake within 10s")
|
||||
}
|
||||
|
||||
var packageRe = regexp.MustCompile(`\bu0 ([a-zA-Z0-9_.]+)/`)
|
||||
|
||||
// ForegroundPackage returns the package of the current foreground window.
|
||||
func ForegroundPackage(r *adb.Runner) (string, error) {
|
||||
out, err := r.Shell("dumpsys window | grep mCurrentFocus")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
m := packageRe.FindStringSubmatch(out)
|
||||
if m == nil {
|
||||
return "", fmt.Errorf("could not determine foreground package from %q", strings.TrimSpace(out))
|
||||
}
|
||||
return m[1], nil
|
||||
}
|
||||
|
||||
// Launch starts an app by package name and waits until it is in the foreground.
|
||||
func Launch(r *adb.Runner, pkg string) error {
|
||||
if pkg == "" {
|
||||
return fmt.Errorf("launch requires a package name")
|
||||
}
|
||||
out, err := r.Shell("monkey -p " + pkg + " -c android.intent.category.LAUNCHER 1")
|
||||
if err != nil {
|
||||
return fmt.Errorf("launch %s: %w", pkg, err)
|
||||
}
|
||||
if strings.Contains(out, "monkey aborted") || strings.Contains(out, "No activities found") {
|
||||
return fmt.Errorf("launch %s: app not installed or has no launcher activity: %s", pkg, strings.TrimSpace(out))
|
||||
}
|
||||
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
fg, err := ForegroundPackage(r)
|
||||
if err == nil && fg == pkg {
|
||||
return nil
|
||||
}
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
}
|
||||
return fmt.Errorf("launched %s but it never became the foreground app within 10s", pkg)
|
||||
}
|
||||
|
||||
// AppInstalled reports whether a package is installed.
|
||||
func AppInstalled(r *adb.Runner, pkg string) (bool, error) {
|
||||
out, err := r.Shell("pm list packages " + pkg)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return strings.Contains(out, "package:"+pkg), nil
|
||||
}
|
||||
|
||||
// TextNodes returns every node with visible text, for dumps and debugging.
|
||||
func (h *Hierarchy) TextNodes() []*Node {
|
||||
var out []*Node
|
||||
h.Walk(func(n *Node) {
|
||||
if strings.TrimSpace(n.Text) != "" || strings.TrimSpace(n.ContentDesc) != "" {
|
||||
out = append(out, n)
|
||||
}
|
||||
})
|
||||
return out
|
||||
}
|
||||
Reference in New Issue
Block a user