feat: Phase 2 act primitives

Add the Artemis Go CLI and its packages: - internal/adb: timeout-bounded adb wrapper, device list, wireless-debugging port scan and auto-connect (never hardcodes the moving port). - internal/ui: uiautomator dump parsing, find-by-text, tap the centre of a text match (FR-4), type, swipe, scroll, back, home, launch, wake. - cmd/artemis: CLI for the primitives plus 'demo', the Phase 2 DoD run. Failures are loud: a missing or disabled element errors instead of tapping a guessed coordinate. The screen is woken before acting, and every adb call has a deadline. Also record the section 13 Q4 decision (adb lives in the user profile on .13, wireless ADB needs no udev rules; service packaging still open until Phase 6), add unit tests, and document the phase evidence.
2026-10-16 11:36:03 +11:00
parent ecb8dd2c11
commit 1115531684
10 changed files with 1437 additions and 1 deletions
--- a/internal/adb/adb.go
+++ b/internal/adb/adb.go
@@ -0,0 +1,130 @@
+// Package adb is a thin, timeout-bounded wrapper around the adb command line
+// tool. Artemis never talks to the ADB wire protocol directly: it shells out to
+// the adb binary installed in the user profile on .13.
+package adb
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+)
+
+// DefaultTimeout bounds every single adb invocation. A recipe must never run
+// forever (FR-11), so no command is allowed to run without a deadline.
+const DefaultTimeout = 20 * time.Second
+
+// Runner executes adb commands against one device.
+type Runner struct {
+	Adb     string
+	Serial  string
+	Timeout time.Duration
+}
+
+// New returns a Runner. The adb binary is taken from ARTEMIS_ADB, else PATH.
+func New(serial string) *Runner {
+	adbPath := os.Getenv("ARTEMIS_ADB")
+	if adbPath == "" {
+		adbPath = "adb"
+	}
+	return &Runner{Adb: adbPath, Serial: serial, Timeout: DefaultTimeout}
+}
+
+func (r *Runner) adbArgs(args ...string) []string {
+	full := make([]string, 0, len(args)+2)
+	if r.Serial != "" {
+		full = append(full, "-s", r.Serial)
+	}
+	return append(full, args...)
+}
+
+func (r *Runner) run(timeout time.Duration, args ...string) ([]byte, error) {
+	if timeout <= 0 {
+		timeout = r.Timeout
+	}
+	if timeout <= 0 {
+		timeout = DefaultTimeout
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	cmd := exec.CommandContext(ctx, r.Adb, r.adbArgs(args...)...)
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+
+	err := cmd.Run()
+	if ctx.Err() == context.DeadlineExceeded {
+		return stdout.Bytes(), fmt.Errorf("adb %s: timed out after %s", strings.Join(args, " "), timeout)
+	}
+	if err != nil {
+		msg := strings.TrimSpace(stderr.String())
+		if msg == "" {
+			msg = strings.TrimSpace(stdout.String())
+		}
+		return stdout.Bytes(), fmt.Errorf("adb %s: %w: %s", strings.Join(args, " "), err, msg)
+	}
+	return stdout.Bytes(), nil
+}
+
+// Shell runs a command in the device shell and returns combined stdout.
+//
+// The whole command is passed as one argv element; adb joins argv with spaces
+// and the device's sh parses the result, exactly as a human at a terminal.
+func (r *Runner) Shell(command string) (string, error) {
+	out, err := r.run(r.Timeout, "shell", command)
+	return string(out), err
+}
+
+// ExecOut runs adb exec-out, which does not add carriage returns. Use it to
+// read files and binary data off the device.
+func (r *Runner) ExecOut(args ...string) ([]byte, error) {
+	return r.run(r.Timeout, append([]string{"exec-out"}, args...)...)
+}
+
+// Device is one row of `adb devices -l`.
+type Device struct {
+	Serial      string
+	State       string
+	Product     string
+	Model       string
+	Device      string
+	TransportID string
+}
+
+// Devices lists the devices the adb server knows about.
+func (r *Runner) Devices() ([]Device, error) {
+	out, err := r.run(10*time.Second, "devices", "-l")
+	if err != nil {
+		return nil, err
+	}
+	var devices []Device
+	for _, line := range strings.Split(string(out), "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" || strings.HasPrefix(line, "List of devices") {
+			continue
+		}
+		fields := strings.Fields(line)
+		if len(fields) < 2 {
+			continue
+		}
+		d := Device{Serial: fields[0], State: fields[1]}
+		for _, f := range fields[2:] {
+			switch {
+			case strings.HasPrefix(f, "product:"):
+				d.Product = strings.TrimPrefix(f, "product:")
+			case strings.HasPrefix(f, "model:"):
+				d.Model = strings.TrimPrefix(f, "model:")
+			case strings.HasPrefix(f, "device:"):
+				d.Device = strings.TrimPrefix(f, "device:")
+			case strings.HasPrefix(f, "transport_id:"):
+				d.TransportID = strings.TrimPrefix(f, "transport_id:")
+			}
+		}
+		devices = append(devices, d)
+	}
+	return devices, nil
+}
--- a/internal/adb/discover.go
+++ b/internal/adb/discover.go
@@ -0,0 +1,157 @@
+package adb
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"net"
+)
+
+// The wireless-debugging port changes on every phone reboot or toggle of
+// wireless debugging, so it is never hardcoded. Android opens two TCP ports:
+// a pairing port (useless here) and the connect port. `adb connect` succeeds on
+// the connect port and is refused on the pairing port.
+
+// DefaultScanRange is the range Android picks wireless-debugging ports from.
+const (
+	DefaultScanFrom = 30000
+	DefaultScanTo   = 65535
+)
+
+// Connect asks the adb server to connect to host:port.
+func (r *Runner) Connect(address string) (string, error) {
+	out, err := r.run(15*time.Second, "connect", address)
+	msg := strings.TrimSpace(string(out))
+	if err != nil {
+		return msg, err
+	}
+	// adb connect can exit 0 while reporting failure in its output.
+	lower := strings.ToLower(msg)
+	if strings.Contains(lower, "failed") || strings.Contains(lower, "cannot") {
+		return msg, fmt.Errorf("adb connect %s: %s", address, msg)
+	}
+	return msg, nil
+}
+
+// DeviceOnline reports whether serial is present and in state "device".
+func (r *Runner) DeviceOnline(serial string) bool {
+	devices, err := r.Devices()
+	if err != nil {
+		return false
+	}
+	for _, d := range devices {
+		if d.Serial == serial && d.State == "device" {
+			return true
+		}
+	}
+	return false
+}
+
+// ScanPorts does a concurrent TCP connect scan of host:from-to and returns the
+// open ports in ascending order.
+func ScanPorts(host string, from, to int, timeout time.Duration) []int {
+	if timeout <= 0 {
+		timeout = 500 * time.Millisecond
+	}
+	const workers = 512
+
+	type result struct {
+		port int
+		open bool
+	}
+
+	jobs := make(chan int)
+	results := make(chan result)
+	var wg sync.WaitGroup
+
+	for i := 0; i < workers; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for port := range jobs {
+				addr := net.JoinHostPort(host, fmt.Sprintf("%d", port))
+				conn, err := net.DialTimeout("tcp", addr, timeout)
+				if err != nil {
+					results <- result{port: port, open: false}
+					continue
+				}
+				conn.Close()
+				results <- result{port: port, open: true}
+			}
+		}()
+	}
+
+	go func() {
+		for p := from; p <= to; p++ {
+			jobs <- p
+		}
+		close(jobs)
+	}()
+
+	go func() {
+		wg.Wait()
+		close(results)
+	}()
+
+	var open []int
+	for res := range results {
+		if res.open {
+			open = append(open, res.port)
+		}
+	}
+	sort.Ints(open)
+	return open
+}
+
+// ConnectAuto brings the phone online. It reuses an already-connected device if
+// one is present, otherwise it scans host for open ports and tries
+// `adb connect` on each one until a port comes up in state "device".
+//
+// It returns the serial it connected to.
+func (r *Runner) ConnectAuto(host string) (string, error) {
+	// Already connected? Find a device whose serial is host:port.
+	for _, d := range mustDevices(r) {
+		if d.State == "device" && strings.HasPrefix(d.Serial, host+":") {
+			r.Serial = d.Serial
+			return d.Serial, nil
+		}
+	}
+
+	ports := ScanPorts(host, DefaultScanFrom, DefaultScanTo, 0)
+	if len(ports) == 0 {
+		return "", fmt.Errorf("no open TCP ports on %s in %d-%d; is wireless debugging on?",
+			host, DefaultScanFrom, DefaultScanTo)
+	}
+
+	var errs []string
+	for _, port := range ports {
+		serial := fmt.Sprintf("%s:%d", host, port)
+		if _, err := r.Connect(serial); err != nil {
+			errs = append(errs, fmt.Sprintf("%d: %v", port, err))
+			continue
+		}
+		// Give the adb server a moment to move the device to "device".
+		deadline := time.Now().Add(5 * time.Second)
+		for time.Now().Before(deadline) {
+			if r.DeviceOnline(serial) {
+				r.Serial = serial
+				return serial, nil
+			}
+			time.Sleep(250 * time.Millisecond)
+		}
+		errs = append(errs, fmt.Sprintf("%d: connect accepted but device never came online", port))
+	}
+	return "", fmt.Errorf("open ports on %s were %v but none accepted adb connect: %s",
+		host, ports, strings.Join(errs, "; "))
+}
+
+func mustDevices(r *Runner) []Device {
+	devices, err := r.Devices()
+	if err != nil {
+		return nil
+	}
+	return devices
+}
--- a/internal/ui/actions.go
+++ b/internal/ui/actions.go
@@ -0,0 +1,324 @@
+package ui
+
+import (
+	"fmt"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"gitea.lab.audasmedia.com.au/sam/artemis_control/internal/adb"
+)
+
+// Android key events we use.
+const (
+	KeyHome   = 3
+	KeyBack   = 4
+	KeyEnter  = 66
+	KeyWakeup = 224
+)
+
+// Well-known packages.
+const (
+	PackageSpotify  = "com.spotify.music"
+	PackageMaps     = "com.google.android.apps.maps"
+	PackageSettings = "com.android.settings"
+)
+
+// Tap sends a tap at absolute screen coordinates.
+func Tap(r *adb.Runner, x, y int) error {
+	if x < 0 || y < 0 {
+		return fmt.Errorf("refusing to tap negative coordinates (%d,%d)", x, y)
+	}
+	_, err := r.Shell(fmt.Sprintf("input tap %d %d", x, y))
+	return err
+}
+
+// KeyEvent sends a keyevent by numeric code.
+func KeyEvent(r *adb.Runner, code int) error {
+	_, err := r.Shell(fmt.Sprintf("input keyevent %d", code))
+	return err
+}
+
+// Back presses the system back button.
+func Back(r *adb.Runner) error { return KeyEvent(r, KeyBack) }
+
+// Home presses the system home button.
+func Home(r *adb.Runner) error { return KeyEvent(r, KeyHome) }
+
+// Type types text into the focused field.
+func Type(r *adb.Runner, text string) error {
+	if strings.TrimSpace(text) == "" {
+		return fmt.Errorf("refusing to type empty text")
+	}
+	_, err := r.Shell("input text " + escapeInputText(text))
+	return err
+}
+
+// escapeInputText makes a string safe for `input text`.
+//
+// `input text` decodes %s as a space, so spaces are encoded first. The result
+// is then single-quoted for the device shell, with embedded quotes escaped.
+func escapeInputText(text string) string {
+	encoded := strings.ReplaceAll(text, " ", "%s")
+	return "'" + strings.ReplaceAll(encoded, "'", `'\''`) + "'"
+}
+
+// Swipe drags from one point to another over duration.
+func Swipe(r *adb.Runner, x1, y1, x2, y2, durationMs int) error {
+	if durationMs <= 0 {
+		durationMs = 300
+	}
+	_, err := r.Shell(fmt.Sprintf("input swipe %d %d %d %d %d", x1, y1, x2, y2, durationMs))
+	return err
+}
+
+// tapCandidate ranks matches so the tap lands on something a human could press.
+type tapCandidate struct {
+	node      *Node
+	clickable bool
+	enabled   bool
+	area      int
+}
+
+// pickBestMatch chooses the most tappable node from a set of text matches.
+// Clickable and enabled nodes win; among equals the smallest node wins, because
+// the smallest enclosing element is the control itself rather than its container.
+func pickBestMatch(matches []*Node) (*Node, error) {
+	candidates := make([]tapCandidate, 0, len(matches))
+	for _, n := range matches {
+		area := 1 << 30
+		if r, err := n.BoundsRect(); err == nil {
+			area = (r.X2 - r.X1) * (r.Y2 - r.Y1)
+		}
+		candidates = append(candidates, tapCandidate{node: n, clickable: n.Clickable, enabled: n.Enabled, area: area})
+	}
+	sort.SliceStable(candidates, func(i, j int) bool {
+		a, b := candidates[i], candidates[j]
+		if a.clickable != b.clickable {
+			return a.clickable
+		}
+		if a.enabled != b.enabled {
+			return a.enabled
+		}
+		return a.area < b.area
+	})
+	best := candidates[0]
+	if !best.enabled {
+		return nil, fmt.Errorf("best match %s is disabled", best.node.Label())
+	}
+	return best.node, nil
+}
+
+// TapText finds an element by its text (or content description) and taps its
+// centre. This is FR-4: text match first, coordinates derived from bounds.
+//
+// It fails loudly when no element matches and never falls back to tapping a
+// guessed coordinate.
+func TapText(r *adb.Runner, query string, exact bool) (*Node, error) {
+	if query == "" {
+		return nil, fmt.Errorf("tap requires a non-empty text query")
+	}
+	h, err := Dump(r)
+	if err != nil {
+		return nil, err
+	}
+	matches := h.FindText(query, exact)
+	if len(matches) == 0 {
+		return nil, fmt.Errorf("element with text %q not found on screen; refusing to tap blindly", query)
+	}
+	node, err := pickBestMatch(matches)
+	if err != nil {
+		return nil, err
+	}
+	x, y, err := node.Center()
+	if err != nil {
+		return nil, err
+	}
+	if err := Tap(r, x, y); err != nil {
+		return nil, err
+	}
+	return node, nil
+}
+
+// WaitForText polls the screen until an element matching query appears, then
+// returns the best tappable match. Use it after an action that loads a new
+// screen; apps do not draw instantly.
+func WaitForText(r *adb.Runner, query string, exact bool, timeout time.Duration) (*Node, error) {
+	if timeout <= 0 {
+		timeout = 10 * time.Second
+	}
+	deadline := time.Now().Add(timeout)
+	var seen []*Node
+	for time.Now().Before(deadline) {
+		h, err := Dump(r)
+		if err == nil {
+			matches := h.FindText(query, exact)
+			if len(matches) > 0 {
+				seen = matches
+				if n, perr := pickBestMatch(matches); perr == nil {
+					return n, nil
+				}
+			}
+		}
+		time.Sleep(400 * time.Millisecond)
+	}
+	if len(seen) > 0 {
+		return nil, fmt.Errorf("text %q is on screen but no enabled match could be tapped", query)
+	}
+	return nil, fmt.Errorf("text %q did not appear within %s", query, timeout)
+}
+
+// ScreenSize reads the physical screen size.
+func ScreenSize(r *adb.Runner) (width, height int, err error) {
+	out, err := r.Shell("wm size")
+	if err != nil {
+		return 0, 0, err
+	}
+	// "Physical size: 1080x2400" (an Override size line may follow).
+	re := regexp.MustCompile(`(\d+)x(\d+)`)
+	m := re.FindStringSubmatch(out)
+	if m == nil {
+		return 0, 0, fmt.Errorf("could not parse screen size from %q", strings.TrimSpace(out))
+	}
+	w, _ := strconv.Atoi(m[1])
+	h, _ := strconv.Atoi(m[2])
+	return w, h, nil
+}
+
+// Scroll scrolls the current screen in the given direction ("down", "up").
+//
+// It prefers a scrollable node from the dump, so the swipe stays inside a list
+// rather than on a toolbar. If no scrollable node is present it falls back to
+// the middle of the screen.
+func Scroll(r *adb.Runner, direction string) error {
+	direction = strings.ToLower(strings.TrimSpace(direction))
+	if direction != "down" && direction != "up" {
+		return fmt.Errorf("scroll direction must be \"up\" or \"down\", got %q", direction)
+	}
+
+	width, height, err := ScreenSize(r)
+	if err != nil {
+		return err
+	}
+
+	rect := Rect{X1: 0, Y1: 0, X2: width, Y2: height}
+	h, dumpErr := Dump(r)
+	if dumpErr == nil {
+		// Pick the tallest scrollable node that is taller than it is wide.
+		// Short or horizontal elements (carousels, toolbars) are ignored: a
+		// vertical swipe inside them becomes a mis-gesture, which is exactly
+		// what "never act blindly" forbids.
+		var best *Node
+		bestHeight := 0
+		h.Walk(func(n *Node) {
+			if !n.Scrollable || !n.Enabled {
+				return
+			}
+			b, berr := n.BoundsRect()
+			if berr != nil {
+				return
+			}
+			nHeight := b.Y2 - b.Y1
+			if nHeight <= b.X2-b.X1 {
+				return
+			}
+			if nHeight > bestHeight {
+				bestHeight, best = nHeight, n
+			}
+		})
+		// Only trust it if it covers a meaningful slice of the screen.
+		if best != nil && bestHeight >= height/3 {
+			if b, berr := best.BoundsRect(); berr == nil {
+				rect = b
+			}
+		}
+	}
+
+	const margin = 0.25
+	midX := (rect.X1 + rect.X2) / 2
+	top := rect.Y1 + int(float64(rect.Y2-rect.Y1)*margin)
+	bottom := rect.Y2 - int(float64(rect.Y2-rect.Y1)*margin)
+	if direction == "down" {
+		return Swipe(r, midX, bottom, midX, top, 400)
+	}
+	return Swipe(r, midX, top, midX, bottom, 400)
+}
+
+// Wake makes sure the screen is on before any action (FR-7).
+func Wake(r *adb.Runner) error {
+	if _, err := r.Shell("input keyevent " + strconv.Itoa(KeyWakeup)); err != nil {
+		return fmt.Errorf("wake keyevent: %w", err)
+	}
+	deadline := time.Now().Add(10 * time.Second)
+	for time.Now().Before(deadline) {
+		out, err := r.Shell("dumpsys power | grep mWakefulness=")
+		if err == nil && strings.Contains(out, "mWakefulness=Awake") {
+			// Dismiss a non-secure keyguard; harmless when the phone is unlocked.
+			_, _ = r.Shell("wm dismiss-keyguard")
+			return nil
+		}
+		time.Sleep(300 * time.Millisecond)
+	}
+	return fmt.Errorf("screen did not reach Awake within 10s")
+}
+
+var packageRe = regexp.MustCompile(`\bu0 ([a-zA-Z0-9_.]+)/`)
+
+// ForegroundPackage returns the package of the current foreground window.
+func ForegroundPackage(r *adb.Runner) (string, error) {
+	out, err := r.Shell("dumpsys window | grep mCurrentFocus")
+	if err != nil {
+		return "", err
+	}
+	m := packageRe.FindStringSubmatch(out)
+	if m == nil {
+		return "", fmt.Errorf("could not determine foreground package from %q", strings.TrimSpace(out))
+	}
+	return m[1], nil
+}
+
+// Launch starts an app by package name and waits until it is in the foreground.
+func Launch(r *adb.Runner, pkg string) error {
+	if pkg == "" {
+		return fmt.Errorf("launch requires a package name")
+	}
+	out, err := r.Shell("monkey -p " + pkg + " -c android.intent.category.LAUNCHER 1")
+	if err != nil {
+		return fmt.Errorf("launch %s: %w", pkg, err)
+	}
+	if strings.Contains(out, "monkey aborted") || strings.Contains(out, "No activities found") {
+		return fmt.Errorf("launch %s: app not installed or has no launcher activity: %s", pkg, strings.TrimSpace(out))
+	}
+
+	deadline := time.Now().Add(10 * time.Second)
+	for time.Now().Before(deadline) {
+		fg, err := ForegroundPackage(r)
+		if err == nil && fg == pkg {
+			return nil
+		}
+		time.Sleep(300 * time.Millisecond)
+	}
+	return fmt.Errorf("launched %s but it never became the foreground app within 10s", pkg)
+}
+
+// AppInstalled reports whether a package is installed.
+func AppInstalled(r *adb.Runner, pkg string) (bool, error) {
+	out, err := r.Shell("pm list packages " + pkg)
+	if err != nil {
+		return false, err
+	}
+	return strings.Contains(out, "package:"+pkg), nil
+}
+
+// TextNodes returns every node with visible text, for dumps and debugging.
+func (h *Hierarchy) TextNodes() []*Node {
+	var out []*Node
+	h.Walk(func(n *Node) {
+		if strings.TrimSpace(n.Text) != "" || strings.TrimSpace(n.ContentDesc) != "" {
+			out = append(out, n)
+		}
+	})
+	return out
+}
--- a/internal/ui/dump.go
+++ b/internal/ui/dump.go
@@ -0,0 +1,170 @@
+// Package ui reads the phone screen and drives it.
+//
+// Reading uses `uiautomator dump`, which gives real element text and bounds.
+// Acting uses `input`, which takes coordinates because that is all Android
+// offers. Every tap is therefore derived from a text match in the dump — never
+// from a hardcoded coordinate (FR-4).
+package ui
+
+import (
+	"encoding/xml"
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"gitea.lab.audasmedia.com.au/sam/artemis_control/internal/adb"
+)
+
+const dumpPath = "/sdcard/artemis_window_dump.xml"
+
+// Node is one element in the uiautomator hierarchy.
+type Node struct {
+	Index       int    `xml:"index,attr"`
+	Text        string `xml:"text,attr"`
+	ResourceID  string `xml:"resource-id,attr"`
+	Class       string `xml:"class,attr"`
+	Package     string `xml:"package,attr"`
+	ContentDesc string `xml:"content-desc,attr"`
+	Clickable   bool   `xml:"clickable,attr"`
+	Enabled     bool   `xml:"enabled,attr"`
+	Scrollable  bool   `xml:"scrollable,attr"`
+	Focused     bool   `xml:"focused,attr"`
+	Selected    bool   `xml:"selected,attr"`
+	Bounds      string `xml:"bounds,attr"`
+	Children    []Node `xml:"node"`
+}
+
+// Hierarchy is a parsed uiautomator dump.
+type Hierarchy struct {
+	XMLName  xml.Name `xml:"hierarchy"`
+	Rotation int      `xml:"rotation,attr"`
+	Nodes    []Node   `xml:"node"`
+}
+
+// Dump fetches and parses the current screen.
+func Dump(r *adb.Runner) (*Hierarchy, error) {
+	if _, err := r.Shell("uiautomator dump " + dumpPath); err != nil {
+		return nil, fmt.Errorf("uiautomator dump: %w", err)
+	}
+	raw, err := r.ExecOut("cat", dumpPath)
+	if err != nil {
+		return nil, fmt.Errorf("read ui dump: %w", err)
+	}
+	if len(raw) == 0 {
+		return nil, fmt.Errorf("ui dump was empty")
+	}
+	return parseHierarchy(raw)
+}
+
+// parseHierarchy turns raw uiautomator XML into a Hierarchy.
+func parseHierarchy(raw []byte) (*Hierarchy, error) {
+	var h Hierarchy
+	if err := xml.Unmarshal(raw, &h); err != nil {
+		return nil, fmt.Errorf("parse ui dump: %w", err)
+	}
+	if len(h.Nodes) == 0 {
+		return nil, fmt.Errorf("ui dump contained no nodes")
+	}
+	return &h, nil
+}
+
+// Walk visits a node and all of its children depth first.
+func (n *Node) Walk(fn func(*Node)) {
+	fn(n)
+	for i := range n.Children {
+		n.Children[i].Walk(fn)
+	}
+}
+
+// Walk visits every node in the hierarchy.
+func (h *Hierarchy) Walk(fn func(*Node)) {
+	for i := range h.Nodes {
+		h.Nodes[i].Walk(fn)
+	}
+}
+
+// Label is the human-readable identity of a node, for logs and errors.
+func (n *Node) Label() string {
+	label := n.Text
+	if label == "" {
+		label = n.ContentDesc
+	}
+	return fmt.Sprintf("%s[%s] text=%q bounds=%s", n.Class, shortID(n.ResourceID), label, n.Bounds)
+}
+
+func shortID(id string) string {
+	if id == "" {
+		return "-"
+	}
+	if i := strings.LastIndex(id, "/"); i >= 0 {
+		return id[i+1:]
+	}
+	return id
+}
+
+// FindText returns nodes whose text or content description matches query.
+// When exact is false the match is case-insensitive and substring-based.
+func (h *Hierarchy) FindText(query string, exact bool) []*Node {
+	if query == "" {
+		return nil
+	}
+	needle := strings.ToLower(query)
+	var out []*Node
+	h.Walk(func(n *Node) {
+		for _, candidate := range []string{n.Text, n.ContentDesc} {
+			if candidate == "" {
+				continue
+			}
+			if exact {
+				if strings.EqualFold(candidate, query) {
+					out = append(out, n)
+					return
+				}
+				continue
+			}
+			if strings.Contains(strings.ToLower(candidate), needle) {
+				out = append(out, n)
+				return
+			}
+		}
+	})
+	return out
+}
+
+var boundsRe = regexp.MustCompile(`^\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]$`)
+
+// Rect is a screen rectangle in pixels.
+type Rect struct {
+	X1, Y1, X2, Y2 int
+}
+
+// Bounds parses the node's bounds attribute.
+func (n *Node) BoundsRect() (Rect, error) {
+	m := boundsRe.FindStringSubmatch(strings.TrimSpace(n.Bounds))
+	if m == nil {
+		return Rect{}, fmt.Errorf("node has no parsable bounds (%q)", n.Bounds)
+	}
+	vals := make([]int, 4)
+	for i := 0; i < 4; i++ {
+		v, err := strconv.Atoi(m[i+1])
+		if err != nil {
+			return Rect{}, fmt.Errorf("bad bounds %q: %w", n.Bounds, err)
+		}
+		vals[i] = v
+	}
+	r := Rect{X1: vals[0], Y1: vals[1], X2: vals[2], Y2: vals[3]}
+	if r.X2 <= r.X1 || r.Y2 <= r.Y1 {
+		return Rect{}, fmt.Errorf("node has empty bounds %q", n.Bounds)
+	}
+	return r, nil
+}
+
+// Center returns the centre point of the node, which is what a tap uses.
+func (n *Node) Center() (int, int, error) {
+	r, err := n.BoundsRect()
+	if err != nil {
+		return 0, 0, err
+	}
+	return (r.X1 + r.X2) / 2, (r.Y1 + r.Y2) / 2, nil
+}
--- a/internal/ui/dump_test.go
+++ b/internal/ui/dump_test.go
@@ -0,0 +1,118 @@
+package ui
+
+import (
+	"strings"
+	"testing"
+)
+
+const sampleDump = `<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
+<hierarchy rotation="0">
+  <node index="0" text="" class="android.widget.FrameLayout" package="com.tailscale.ipn" clickable="false" enabled="true" scrollable="false" bounds="[0,0][1080,2400]">
+    <node index="0" text="samuelrolfe@gmail.com" class="android.widget.TextView" package="com.tailscale.ipn" clickable="false" enabled="true" bounds="[221,155][798,217]"/>
+    <node index="1" text="Connected" class="android.widget.TextView" package="com.tailscale.ipn" clickable="false" enabled="true" bounds="[221,217][453,277]"/>
+    <node index="2" text="" content-desc="Navigate up" class="android.widget.ImageButton" package="com.tailscale.ipn" clickable="true" enabled="true" bounds="[42,121][189,268]"/>
+  </node>
+  <node index="1" text="Network and internet" class="android.widget.TextView" package="com.android.settings" clickable="true" enabled="true" bounds="[221,663][748,727]">
+    <node index="0" text="Network and internet" class="android.widget.TextView" package="com.android.settings" clickable="false" enabled="true" bounds="[221,663][748,727]"/>
+  </node>
+  <node index="2" text="Disabled thing" class="android.widget.TextView" package="com.android.settings" clickable="true" enabled="false" bounds="[0,0][10,10]"/>
+</hierarchy>`
+
+func parseSample(t *testing.T) *Hierarchy {
+	t.Helper()
+	h, err := parseHierarchy([]byte(sampleDump))
+	if err != nil {
+		t.Fatalf("parseHierarchy: %v", err)
+	}
+	return h
+}
+
+func TestBoundsAndCenter(t *testing.T) {
+	n := &Node{Bounds: "[221,155][798,217]"}
+	r, err := n.BoundsRect()
+	if err != nil {
+		t.Fatalf("BoundsRect: %v", err)
+	}
+	if r.X1 != 221 || r.Y1 != 155 || r.X2 != 798 || r.Y2 != 217 {
+		t.Fatalf("unexpected rect %+v", r)
+	}
+	cx, cy, err := n.Center()
+	if err != nil {
+		t.Fatalf("Center: %v", err)
+	}
+	if cx != (221+798)/2 || cy != (155+217)/2 {
+		t.Fatalf("unexpected centre (%d,%d)", cx, cy)
+	}
+}
+
+func TestBoundsRejectsGarbage(t *testing.T) {
+	for _, bad := range []string{"", "nope", "[1,2][3,4]junk", "[10,10][10,20]"} {
+		if _, err := (&Node{Bounds: bad}).BoundsRect(); err == nil {
+			t.Fatalf("expected error for bounds %q", bad)
+		}
+	}
+}
+
+func TestFindTextExactAndSubstring(t *testing.T) {
+	h := parseSample(t)
+
+	if got := len(h.FindText("Connected", true)); got != 1 {
+		t.Fatalf("exact match: want 1, got %d", got)
+	}
+	// Substring is case-insensitive and matches more than one node.
+	if got := len(h.FindText("network", false)); got != 2 {
+		t.Fatalf("substring match: want 2 (parent + child), got %d", got)
+	}
+	// Content descriptions count as text.
+	matches := h.FindText("Navigate up", true)
+	if len(matches) != 1 || !matches[0].Clickable {
+		t.Fatalf("content-desc match failed: %+v", matches)
+	}
+	if got := h.FindText("does not exist", false); got != nil {
+		t.Fatalf("expected no matches, got %d", len(got))
+	}
+}
+
+func TestPickBestMatchPrefersClickable(t *testing.T) {
+	h := parseSample(t)
+	matches := h.FindText("network", false)
+	if len(matches) != 2 {
+		t.Fatalf("want 2 matches, got %d", len(matches))
+	}
+	best, err := pickBestMatch(matches)
+	if err != nil {
+		t.Fatalf("pickBestMatch: %v", err)
+	}
+	if !best.Clickable {
+		t.Fatalf("expected a clickable match, got %s", best.Label())
+	}
+}
+
+func TestPickBestMatchRejectsDisabledOnly(t *testing.T) {
+	h := parseSample(t)
+	matches := h.FindText("Disabled thing", true)
+	if len(matches) != 1 {
+		t.Fatalf("want 1 match, got %d", len(matches))
+	}
+	if _, err := pickBestMatch(matches); err == nil {
+		t.Fatal("expected an error when the only match is disabled")
+	}
+}
+
+func TestEscapeInputText(t *testing.T) {
+	cases := map[string]string{
+		"hello world":    "'hello%sworld'",
+		"Rolling Stones": "'Rolling%sStones'",
+		"it's":           `'it'\''s'`,
+	}
+	for in, want := range cases {
+		if got := escapeInputText(in); got != want {
+			t.Errorf("escapeInputText(%q) = %q, want %q", in, got, want)
+		}
+	}
+	// A single quote must not be able to break out of the quoting.
+	got := escapeInputText("a'; rm -rf /; 'b")
+	if strings.Count(got, "'")%2 != 0 {
+		t.Fatalf("unbalanced quotes in %q", got)
+	}
+}