package ui import ( "fmt" "regexp" "sort" "strconv" "strings" "time" "gitea.lab.audasmedia.com.au/sam/artemis_control/internal/adb" ) // PollInterval is how long the wait helpers sleep between screen dumps. // // A dump runs a whole uiautomator JVM on the phone, which is expensive on a // loaded device: polling every 400ms starves the very app Artemis is driving. // A slower poll costs a moment of latency and keeps the phone responsive. const PollInterval = 1500 * time.Millisecond // Android key events we use. const ( KeyHome = 3 KeyBack = 4 KeyEnter = 66 KeyWakeup = 224 ) // Well-known packages. const ( PackageSpotify = "com.spotify.music" PackageMaps = "com.google.android.apps.maps" PackageSettings = "com.android.settings" ) // Tap sends a tap at absolute screen coordinates. func Tap(r *adb.Runner, x, y int) error { if x < 0 || y < 0 { return fmt.Errorf("refusing to tap negative coordinates (%d,%d)", x, y) } _, err := r.Shell(fmt.Sprintf("input tap %d %d", x, y)) return err } // KeyEvent sends a keyevent by numeric code. func KeyEvent(r *adb.Runner, code int) error { _, err := r.Shell(fmt.Sprintf("input keyevent %d", code)) return err } // Back presses the system back button. func Back(r *adb.Runner) error { return KeyEvent(r, KeyBack) } // Home presses the system home button. func Home(r *adb.Runner) error { return KeyEvent(r, KeyHome) } // Type types text into the focused field. func Type(r *adb.Runner, text string) error { if strings.TrimSpace(text) == "" { return fmt.Errorf("refusing to type empty text") } _, err := r.Shell("input text " + escapeInputText(text)) return err } // escapeInputText makes a string safe for `input text`. // // `input text` decodes %s as a space, so spaces are encoded first. The result // is then single-quoted for the device shell, with embedded quotes escaped. func escapeInputText(text string) string { encoded := strings.ReplaceAll(text, " ", "%s") return "'" + strings.ReplaceAll(encoded, "'", `'\''`) + "'" } // Swipe drags from one point to another over duration. func Swipe(r *adb.Runner, x1, y1, x2, y2, durationMs int) error { if durationMs <= 0 { durationMs = 300 } _, err := r.Shell(fmt.Sprintf("input swipe %d %d %d %d %d", x1, y1, x2, y2, durationMs)) return err } // tapCandidate ranks matches so the tap lands on something a human could press. type tapCandidate struct { node *Node clickable bool enabled bool area int } // pickBestMatch chooses the most tappable node from a set of text matches. // Clickable and enabled nodes win; among equals the smallest node wins, because // the smallest enclosing element is the control itself rather than its container. func pickBestMatch(matches []*Node) (*Node, error) { candidates := make([]tapCandidate, 0, len(matches)) for _, n := range matches { area := 1 << 30 if r, err := n.BoundsRect(); err == nil { area = (r.X2 - r.X1) * (r.Y2 - r.Y1) } candidates = append(candidates, tapCandidate{node: n, clickable: n.Clickable, enabled: n.Enabled, area: area}) } sort.SliceStable(candidates, func(i, j int) bool { a, b := candidates[i], candidates[j] if a.clickable != b.clickable { return a.clickable } if a.enabled != b.enabled { return a.enabled } return a.area < b.area }) best := candidates[0] if !best.enabled { return nil, fmt.Errorf("best match %s is disabled", best.node.Label()) } return best.node, nil } // TapText finds an element by its text (or content description) and taps its // centre. This is FR-4: text match first, coordinates derived from bounds. // // It fails loudly when no element matches and never falls back to tapping a // guessed coordinate. func TapText(r *adb.Runner, query string, exact bool) (*Node, error) { if query == "" { return nil, fmt.Errorf("tap requires a non-empty text query") } h, err := Dump(r) if err != nil { return nil, err } matches := h.FindText(query, exact) if len(matches) == 0 { return nil, fmt.Errorf("element with text %q not found on screen; refusing to tap blindly", query) } return TapNode(r, matches) } // TapNode taps the centre of the most tappable node among matches. // // Callers that already hold a hierarchy use this instead of TapText, so the // coordinates come from the dump they just read rather than an older one. func TapNode(r *adb.Runner, matches []*Node) (*Node, error) { if len(matches) == 0 { return nil, fmt.Errorf("no element was matched, refusing to tap blindly") } node, err := pickBestMatch(matches) if err != nil { return nil, err } x, y, err := node.Center() if err != nil { return nil, err } if err := Tap(r, x, y); err != nil { return nil, err } return node, nil } // WaitForText polls the screen until an element matching query appears, then // returns the best tappable match. Use it after an action that loads a new // screen; apps do not draw instantly. func WaitForText(r *adb.Runner, query string, exact bool, timeout time.Duration) (*Node, error) { if timeout <= 0 { timeout = 10 * time.Second } deadline := time.Now().Add(timeout) var seen []*Node for time.Now().Before(deadline) { h, err := Dump(r) if err == nil { matches := h.FindText(query, exact) if len(matches) > 0 { seen = matches if n, perr := pickBestMatch(matches); perr == nil { return n, nil } } } time.Sleep(PollInterval) } if len(seen) > 0 { return nil, fmt.Errorf("text %q is on screen but no enabled match could be tapped", query) } return nil, fmt.Errorf("text %q did not appear within %s", query, timeout) } // WaitForNoText polls the screen until no element matches query. Recipes use it // to wait for one list to be replaced by another, for example a search result // list replacing the "Recents" list. func WaitForNoText(r *adb.Runner, query string, exact bool, timeout time.Duration) error { if timeout <= 0 { timeout = 10 * time.Second } deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { h, err := Dump(r) if err == nil && len(h.FindText(query, exact)) == 0 { return nil } time.Sleep(PollInterval) } return fmt.Errorf("text %q was still on screen after %s", query, timeout) } // ScreenSize reads the physical screen size. func ScreenSize(r *adb.Runner) (width, height int, err error) { out, err := r.Shell("wm size") if err != nil { return 0, 0, err } // "Physical size: 1080x2400" (an Override size line may follow). re := regexp.MustCompile(`(\d+)x(\d+)`) m := re.FindStringSubmatch(out) if m == nil { return 0, 0, fmt.Errorf("could not parse screen size from %q", strings.TrimSpace(out)) } w, _ := strconv.Atoi(m[1]) h, _ := strconv.Atoi(m[2]) return w, h, nil } // Scroll scrolls the current screen in the given direction ("down", "up"). // // It prefers a scrollable node from the dump, so the swipe stays inside a list // rather than on a toolbar. If no scrollable node is present it falls back to // the middle of the screen. func Scroll(r *adb.Runner, direction string) error { direction = strings.ToLower(strings.TrimSpace(direction)) if direction != "down" && direction != "up" { return fmt.Errorf("scroll direction must be \"up\" or \"down\", got %q", direction) } width, height, err := ScreenSize(r) if err != nil { return err } rect := Rect{X1: 0, Y1: 0, X2: width, Y2: height} h, dumpErr := Dump(r) if dumpErr == nil { // Pick the tallest scrollable node that is taller than it is wide. // Short or horizontal elements (carousels, toolbars) are ignored: a // vertical swipe inside them becomes a mis-gesture, which is exactly // what "never act blindly" forbids. var best *Node bestHeight := 0 h.Walk(func(n *Node) { if !n.Scrollable || !n.Enabled { return } b, berr := n.BoundsRect() if berr != nil { return } nHeight := b.Y2 - b.Y1 if nHeight <= b.X2-b.X1 { return } if nHeight > bestHeight { bestHeight, best = nHeight, n } }) // Only trust it if it covers a meaningful slice of the screen. if best != nil && bestHeight >= height/3 { if b, berr := best.BoundsRect(); berr == nil { rect = b } } } const margin = 0.25 midX := (rect.X1 + rect.X2) / 2 top := rect.Y1 + int(float64(rect.Y2-rect.Y1)*margin) bottom := rect.Y2 - int(float64(rect.Y2-rect.Y1)*margin) if direction == "down" { return Swipe(r, midX, bottom, midX, top, 400) } return Swipe(r, midX, top, midX, bottom, 400) } // Wake makes sure the screen is on before any action (FR-7). // // It dismisses the keyguard only when that keyguard is not secure. A secure // keyguard (PIN, pattern or password) is left exactly as it is: Artemis never // tries to get past it (AGENTS.md section 13 Q2). func Wake(r *adb.Runner) error { if _, err := r.Shell("input keyevent " + strconv.Itoa(KeyWakeup)); err != nil { return fmt.Errorf("wake keyevent: %w", err) } deadline := time.Now().Add(10 * time.Second) for time.Now().Before(deadline) { out, err := r.Shell("dumpsys power | grep mWakefulness=") if err == nil && strings.Contains(out, "mWakefulness=Awake") { // Only a non-secure keyguard may be dismissed. If the state cannot // be read we leave the keyguard alone rather than guess. if showing, secure, kerr := KeyguardState(r); kerr == nil && (!showing || !secure) { _, _ = r.Shell("wm dismiss-keyguard") } return nil } time.Sleep(300 * time.Millisecond) } return fmt.Errorf("screen did not reach Awake within 10s") } var packageRe = regexp.MustCompile(`\bu0 ([a-zA-Z0-9_.]+)/`) // ForegroundPackage returns the package of the current foreground window. func ForegroundPackage(r *adb.Runner) (string, error) { out, err := r.Shell("dumpsys window | grep mCurrentFocus") if err != nil { return "", err } m := packageRe.FindStringSubmatch(out) if m == nil { return "", fmt.Errorf("could not determine foreground package from %q", strings.TrimSpace(out)) } return m[1], nil } // Launch starts an app by package name and waits until it is in the foreground. func Launch(r *adb.Runner, pkg string) error { if pkg == "" { return fmt.Errorf("launch requires a package name") } out, err := r.Shell("monkey -p " + pkg + " -c android.intent.category.LAUNCHER 1") if err != nil { return fmt.Errorf("launch %s: %w", pkg, err) } if strings.Contains(out, "monkey aborted") || strings.Contains(out, "No activities found") { return fmt.Errorf("launch %s: app not installed or has no launcher activity: %s", pkg, strings.TrimSpace(out)) } deadline := time.Now().Add(10 * time.Second) for time.Now().Before(deadline) { fg, err := ForegroundPackage(r) if err == nil && fg == pkg { return nil } time.Sleep(300 * time.Millisecond) } return fmt.Errorf("launched %s but it never became the foreground app within 10s", pkg) } // AppInstalled reports whether a package is installed. func AppInstalled(r *adb.Runner, pkg string) (bool, error) { out, err := r.Shell("pm list packages " + pkg) if err != nil { return false, err } return strings.Contains(out, "package:"+pkg), nil } // TextNodes returns every node with visible text, for dumps and debugging. func (h *Hierarchy) TextNodes() []*Node { var out []*Node h.Walk(func(n *Node) { if strings.TrimSpace(n.Text) != "" || strings.TrimSpace(n.ContentDesc) != "" { out = append(out, n) } }) return out }