// Package ui reads the phone screen and drives it. // // Reading uses `uiautomator dump`, which gives real element text and bounds. // Acting uses `input`, which takes coordinates because that is all Android // offers. Every tap is therefore derived from a text match in the dump — never // from a hardcoded coordinate (FR-4). package ui import ( "encoding/xml" "fmt" "regexp" "strconv" "strings" "gitea.lab.audasmedia.com.au/sam/artemis_control/internal/adb" ) const dumpPath = "/sdcard/artemis_window_dump.xml" // Node is one element in the uiautomator hierarchy. type Node struct { Index int `xml:"index,attr"` Text string `xml:"text,attr"` ResourceID string `xml:"resource-id,attr"` Class string `xml:"class,attr"` Package string `xml:"package,attr"` ContentDesc string `xml:"content-desc,attr"` Clickable bool `xml:"clickable,attr"` Enabled bool `xml:"enabled,attr"` Scrollable bool `xml:"scrollable,attr"` Focused bool `xml:"focused,attr"` Selected bool `xml:"selected,attr"` Bounds string `xml:"bounds,attr"` Children []Node `xml:"node"` } // Hierarchy is a parsed uiautomator dump. type Hierarchy struct { XMLName xml.Name `xml:"hierarchy"` Rotation int `xml:"rotation,attr"` Nodes []Node `xml:"node"` } // Dump fetches and parses the current screen. func Dump(r *adb.Runner) (*Hierarchy, error) { if _, err := r.Shell("uiautomator dump " + dumpPath); err != nil { return nil, fmt.Errorf("uiautomator dump: %w", err) } raw, err := r.ExecOut("cat", dumpPath) if err != nil { return nil, fmt.Errorf("read ui dump: %w", err) } if len(raw) == 0 { return nil, fmt.Errorf("ui dump was empty") } return parseHierarchy(raw) } // parseHierarchy turns raw uiautomator XML into a Hierarchy. func parseHierarchy(raw []byte) (*Hierarchy, error) { var h Hierarchy if err := xml.Unmarshal(raw, &h); err != nil { return nil, fmt.Errorf("parse ui dump: %w", err) } if len(h.Nodes) == 0 { return nil, fmt.Errorf("ui dump contained no nodes") } return &h, nil } // Walk visits a node and all of its children depth first. func (n *Node) Walk(fn func(*Node)) { fn(n) for i := range n.Children { n.Children[i].Walk(fn) } } // Walk visits every node in the hierarchy. func (h *Hierarchy) Walk(fn func(*Node)) { for i := range h.Nodes { h.Nodes[i].Walk(fn) } } // Label is the human-readable identity of a node, for logs and errors. func (n *Node) Label() string { label := n.Text if label == "" { label = n.ContentDesc } return fmt.Sprintf("%s[%s] text=%q bounds=%s", n.Class, shortID(n.ResourceID), label, n.Bounds) } func shortID(id string) string { if id == "" { return "-" } if i := strings.LastIndex(id, "/"); i >= 0 { return id[i+1:] } return id } // IDSelector marks a selector that matches a resource id rather than text. A // step that names "id:search_omnibox_text_box" finds the element with that // resource id, whatever its visible label happens to be. // // This exists because some labels are not stable: the Google Maps search box // carries a rotating hint as its content description ("Search here", "Try // petrol stations, ATMs", ...), so a recipe that matched that text broke when // the phone picked a different hint. The resource id does not rotate. const IDSelector = "id:" // FindText returns nodes whose text or content description matches query. // When exact is false the match is case-insensitive and substring-based. func (h *Hierarchy) FindText(query string, exact bool) []*Node { return h.find(query, exact, false) } // FindID returns nodes whose resource id matches query. When exact is false the // match is case-insensitive and substring-based, so the short name // "search_omnibox_text_box" matches the full // "com.google.android.apps.maps:id/search_omnibox_text_box". func (h *Hierarchy) FindID(query string, exact bool) []*Node { return h.find(query, exact, true) } // Find returns nodes matching a recipe selector. A selector that starts with // "id:" is matched against the resource id; any other selector is matched // against text and content description (FR-4). func (h *Hierarchy) Find(query string, exact bool) []*Node { if id, ok := strings.CutPrefix(query, IDSelector); ok { return h.FindID(id, exact) } return h.FindText(query, exact) } func (h *Hierarchy) find(query string, exact, byID bool) []*Node { if query == "" { return nil } needle := strings.ToLower(query) var out []*Node h.Walk(func(n *Node) { var candidates []string if byID { if n.ResourceID != "" { candidates = []string{shortID(n.ResourceID), n.ResourceID} } } else { candidates = []string{n.Text, n.ContentDesc} } for _, candidate := range candidates { if candidate == "" { continue } if exact { if strings.EqualFold(candidate, query) { out = append(out, n) return } continue } if strings.Contains(strings.ToLower(candidate), needle) { out = append(out, n) return } } }) return out } var boundsRe = regexp.MustCompile(`^\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]$`) // Rect is a screen rectangle in pixels. type Rect struct { X1, Y1, X2, Y2 int } // Bounds parses the node's bounds attribute. func (n *Node) BoundsRect() (Rect, error) { m := boundsRe.FindStringSubmatch(strings.TrimSpace(n.Bounds)) if m == nil { return Rect{}, fmt.Errorf("node has no parsable bounds (%q)", n.Bounds) } vals := make([]int, 4) for i := 0; i < 4; i++ { v, err := strconv.Atoi(m[i+1]) if err != nil { return Rect{}, fmt.Errorf("bad bounds %q: %w", n.Bounds, err) } vals[i] = v } r := Rect{X1: vals[0], Y1: vals[1], X2: vals[2], Y2: vals[3]} if r.X2 <= r.X1 || r.Y2 <= r.Y1 { return Rect{}, fmt.Errorf("node has empty bounds %q", n.Bounds) } return r, nil } // Center returns the centre point of the node, which is what a tap uses. func (n *Node) Center() (int, int, error) { r, err := n.BoundsRect() if err != nil { return 0, 0, err } return (r.X1 + r.X2) / 2, (r.Y1 + r.Y2) / 2, nil }