feat: Phase 2 act primitives

Add the Artemis Go CLI and its packages:
- internal/adb: timeout-bounded adb wrapper, device list, wireless-debugging
  port scan and auto-connect (never hardcodes the moving port).
- internal/ui: uiautomator dump parsing, find-by-text, tap the centre of a
  text match (FR-4), type, swipe, scroll, back, home, launch, wake.
- cmd/artemis: CLI for the primitives plus 'demo', the Phase 2 DoD run.

Failures are loud: a missing or disabled element errors instead of tapping a
guessed coordinate. The screen is woken before acting, and every adb call has
a deadline.

Also record the section 13 Q4 decision (adb lives in the user profile on .13,
wireless ADB needs no udev rules; service packaging still open until Phase 6),
add unit tests, and document the phase evidence.
This commit is contained in:
Paperclip Builder
2026-10-16 11:36:03 +11:00
parent ecb8dd2c11
commit 1115531684
10 changed files with 1437 additions and 1 deletions

324
internal/ui/actions.go Normal file
View File

@@ -0,0 +1,324 @@
package ui
import (
"fmt"
"regexp"
"sort"
"strconv"
"strings"
"time"
"gitea.lab.audasmedia.com.au/sam/artemis_control/internal/adb"
)
// Android key events we use.
const (
KeyHome = 3
KeyBack = 4
KeyEnter = 66
KeyWakeup = 224
)
// Well-known packages.
const (
PackageSpotify = "com.spotify.music"
PackageMaps = "com.google.android.apps.maps"
PackageSettings = "com.android.settings"
)
// Tap sends a tap at absolute screen coordinates.
func Tap(r *adb.Runner, x, y int) error {
if x < 0 || y < 0 {
return fmt.Errorf("refusing to tap negative coordinates (%d,%d)", x, y)
}
_, err := r.Shell(fmt.Sprintf("input tap %d %d", x, y))
return err
}
// KeyEvent sends a keyevent by numeric code.
func KeyEvent(r *adb.Runner, code int) error {
_, err := r.Shell(fmt.Sprintf("input keyevent %d", code))
return err
}
// Back presses the system back button.
func Back(r *adb.Runner) error { return KeyEvent(r, KeyBack) }
// Home presses the system home button.
func Home(r *adb.Runner) error { return KeyEvent(r, KeyHome) }
// Type types text into the focused field.
func Type(r *adb.Runner, text string) error {
if strings.TrimSpace(text) == "" {
return fmt.Errorf("refusing to type empty text")
}
_, err := r.Shell("input text " + escapeInputText(text))
return err
}
// escapeInputText makes a string safe for `input text`.
//
// `input text` decodes %s as a space, so spaces are encoded first. The result
// is then single-quoted for the device shell, with embedded quotes escaped.
func escapeInputText(text string) string {
encoded := strings.ReplaceAll(text, " ", "%s")
return "'" + strings.ReplaceAll(encoded, "'", `'\''`) + "'"
}
// Swipe drags from one point to another over duration.
func Swipe(r *adb.Runner, x1, y1, x2, y2, durationMs int) error {
if durationMs <= 0 {
durationMs = 300
}
_, err := r.Shell(fmt.Sprintf("input swipe %d %d %d %d %d", x1, y1, x2, y2, durationMs))
return err
}
// tapCandidate ranks matches so the tap lands on something a human could press.
type tapCandidate struct {
node *Node
clickable bool
enabled bool
area int
}
// pickBestMatch chooses the most tappable node from a set of text matches.
// Clickable and enabled nodes win; among equals the smallest node wins, because
// the smallest enclosing element is the control itself rather than its container.
func pickBestMatch(matches []*Node) (*Node, error) {
candidates := make([]tapCandidate, 0, len(matches))
for _, n := range matches {
area := 1 << 30
if r, err := n.BoundsRect(); err == nil {
area = (r.X2 - r.X1) * (r.Y2 - r.Y1)
}
candidates = append(candidates, tapCandidate{node: n, clickable: n.Clickable, enabled: n.Enabled, area: area})
}
sort.SliceStable(candidates, func(i, j int) bool {
a, b := candidates[i], candidates[j]
if a.clickable != b.clickable {
return a.clickable
}
if a.enabled != b.enabled {
return a.enabled
}
return a.area < b.area
})
best := candidates[0]
if !best.enabled {
return nil, fmt.Errorf("best match %s is disabled", best.node.Label())
}
return best.node, nil
}
// TapText finds an element by its text (or content description) and taps its
// centre. This is FR-4: text match first, coordinates derived from bounds.
//
// It fails loudly when no element matches and never falls back to tapping a
// guessed coordinate.
func TapText(r *adb.Runner, query string, exact bool) (*Node, error) {
if query == "" {
return nil, fmt.Errorf("tap requires a non-empty text query")
}
h, err := Dump(r)
if err != nil {
return nil, err
}
matches := h.FindText(query, exact)
if len(matches) == 0 {
return nil, fmt.Errorf("element with text %q not found on screen; refusing to tap blindly", query)
}
node, err := pickBestMatch(matches)
if err != nil {
return nil, err
}
x, y, err := node.Center()
if err != nil {
return nil, err
}
if err := Tap(r, x, y); err != nil {
return nil, err
}
return node, nil
}
// WaitForText polls the screen until an element matching query appears, then
// returns the best tappable match. Use it after an action that loads a new
// screen; apps do not draw instantly.
func WaitForText(r *adb.Runner, query string, exact bool, timeout time.Duration) (*Node, error) {
if timeout <= 0 {
timeout = 10 * time.Second
}
deadline := time.Now().Add(timeout)
var seen []*Node
for time.Now().Before(deadline) {
h, err := Dump(r)
if err == nil {
matches := h.FindText(query, exact)
if len(matches) > 0 {
seen = matches
if n, perr := pickBestMatch(matches); perr == nil {
return n, nil
}
}
}
time.Sleep(400 * time.Millisecond)
}
if len(seen) > 0 {
return nil, fmt.Errorf("text %q is on screen but no enabled match could be tapped", query)
}
return nil, fmt.Errorf("text %q did not appear within %s", query, timeout)
}
// ScreenSize reads the physical screen size.
func ScreenSize(r *adb.Runner) (width, height int, err error) {
out, err := r.Shell("wm size")
if err != nil {
return 0, 0, err
}
// "Physical size: 1080x2400" (an Override size line may follow).
re := regexp.MustCompile(`(\d+)x(\d+)`)
m := re.FindStringSubmatch(out)
if m == nil {
return 0, 0, fmt.Errorf("could not parse screen size from %q", strings.TrimSpace(out))
}
w, _ := strconv.Atoi(m[1])
h, _ := strconv.Atoi(m[2])
return w, h, nil
}
// Scroll scrolls the current screen in the given direction ("down", "up").
//
// It prefers a scrollable node from the dump, so the swipe stays inside a list
// rather than on a toolbar. If no scrollable node is present it falls back to
// the middle of the screen.
func Scroll(r *adb.Runner, direction string) error {
direction = strings.ToLower(strings.TrimSpace(direction))
if direction != "down" && direction != "up" {
return fmt.Errorf("scroll direction must be \"up\" or \"down\", got %q", direction)
}
width, height, err := ScreenSize(r)
if err != nil {
return err
}
rect := Rect{X1: 0, Y1: 0, X2: width, Y2: height}
h, dumpErr := Dump(r)
if dumpErr == nil {
// Pick the tallest scrollable node that is taller than it is wide.
// Short or horizontal elements (carousels, toolbars) are ignored: a
// vertical swipe inside them becomes a mis-gesture, which is exactly
// what "never act blindly" forbids.
var best *Node
bestHeight := 0
h.Walk(func(n *Node) {
if !n.Scrollable || !n.Enabled {
return
}
b, berr := n.BoundsRect()
if berr != nil {
return
}
nHeight := b.Y2 - b.Y1
if nHeight <= b.X2-b.X1 {
return
}
if nHeight > bestHeight {
bestHeight, best = nHeight, n
}
})
// Only trust it if it covers a meaningful slice of the screen.
if best != nil && bestHeight >= height/3 {
if b, berr := best.BoundsRect(); berr == nil {
rect = b
}
}
}
const margin = 0.25
midX := (rect.X1 + rect.X2) / 2
top := rect.Y1 + int(float64(rect.Y2-rect.Y1)*margin)
bottom := rect.Y2 - int(float64(rect.Y2-rect.Y1)*margin)
if direction == "down" {
return Swipe(r, midX, bottom, midX, top, 400)
}
return Swipe(r, midX, top, midX, bottom, 400)
}
// Wake makes sure the screen is on before any action (FR-7).
func Wake(r *adb.Runner) error {
if _, err := r.Shell("input keyevent " + strconv.Itoa(KeyWakeup)); err != nil {
return fmt.Errorf("wake keyevent: %w", err)
}
deadline := time.Now().Add(10 * time.Second)
for time.Now().Before(deadline) {
out, err := r.Shell("dumpsys power | grep mWakefulness=")
if err == nil && strings.Contains(out, "mWakefulness=Awake") {
// Dismiss a non-secure keyguard; harmless when the phone is unlocked.
_, _ = r.Shell("wm dismiss-keyguard")
return nil
}
time.Sleep(300 * time.Millisecond)
}
return fmt.Errorf("screen did not reach Awake within 10s")
}
var packageRe = regexp.MustCompile(`\bu0 ([a-zA-Z0-9_.]+)/`)
// ForegroundPackage returns the package of the current foreground window.
func ForegroundPackage(r *adb.Runner) (string, error) {
out, err := r.Shell("dumpsys window | grep mCurrentFocus")
if err != nil {
return "", err
}
m := packageRe.FindStringSubmatch(out)
if m == nil {
return "", fmt.Errorf("could not determine foreground package from %q", strings.TrimSpace(out))
}
return m[1], nil
}
// Launch starts an app by package name and waits until it is in the foreground.
func Launch(r *adb.Runner, pkg string) error {
if pkg == "" {
return fmt.Errorf("launch requires a package name")
}
out, err := r.Shell("monkey -p " + pkg + " -c android.intent.category.LAUNCHER 1")
if err != nil {
return fmt.Errorf("launch %s: %w", pkg, err)
}
if strings.Contains(out, "monkey aborted") || strings.Contains(out, "No activities found") {
return fmt.Errorf("launch %s: app not installed or has no launcher activity: %s", pkg, strings.TrimSpace(out))
}
deadline := time.Now().Add(10 * time.Second)
for time.Now().Before(deadline) {
fg, err := ForegroundPackage(r)
if err == nil && fg == pkg {
return nil
}
time.Sleep(300 * time.Millisecond)
}
return fmt.Errorf("launched %s but it never became the foreground app within 10s", pkg)
}
// AppInstalled reports whether a package is installed.
func AppInstalled(r *adb.Runner, pkg string) (bool, error) {
out, err := r.Shell("pm list packages " + pkg)
if err != nil {
return false, err
}
return strings.Contains(out, "package:"+pkg), nil
}
// TextNodes returns every node with visible text, for dumps and debugging.
func (h *Hierarchy) TextNodes() []*Node {
var out []*Node
h.Walk(func(n *Node) {
if strings.TrimSpace(n.Text) != "" || strings.TrimSpace(n.ContentDesc) != "" {
out = append(out, n)
}
})
return out
}

170
internal/ui/dump.go Normal file
View File

@@ -0,0 +1,170 @@
// Package ui reads the phone screen and drives it.
//
// Reading uses `uiautomator dump`, which gives real element text and bounds.
// Acting uses `input`, which takes coordinates because that is all Android
// offers. Every tap is therefore derived from a text match in the dump — never
// from a hardcoded coordinate (FR-4).
package ui
import (
"encoding/xml"
"fmt"
"regexp"
"strconv"
"strings"
"gitea.lab.audasmedia.com.au/sam/artemis_control/internal/adb"
)
const dumpPath = "/sdcard/artemis_window_dump.xml"
// Node is one element in the uiautomator hierarchy.
type Node struct {
Index int `xml:"index,attr"`
Text string `xml:"text,attr"`
ResourceID string `xml:"resource-id,attr"`
Class string `xml:"class,attr"`
Package string `xml:"package,attr"`
ContentDesc string `xml:"content-desc,attr"`
Clickable bool `xml:"clickable,attr"`
Enabled bool `xml:"enabled,attr"`
Scrollable bool `xml:"scrollable,attr"`
Focused bool `xml:"focused,attr"`
Selected bool `xml:"selected,attr"`
Bounds string `xml:"bounds,attr"`
Children []Node `xml:"node"`
}
// Hierarchy is a parsed uiautomator dump.
type Hierarchy struct {
XMLName xml.Name `xml:"hierarchy"`
Rotation int `xml:"rotation,attr"`
Nodes []Node `xml:"node"`
}
// Dump fetches and parses the current screen.
func Dump(r *adb.Runner) (*Hierarchy, error) {
if _, err := r.Shell("uiautomator dump " + dumpPath); err != nil {
return nil, fmt.Errorf("uiautomator dump: %w", err)
}
raw, err := r.ExecOut("cat", dumpPath)
if err != nil {
return nil, fmt.Errorf("read ui dump: %w", err)
}
if len(raw) == 0 {
return nil, fmt.Errorf("ui dump was empty")
}
return parseHierarchy(raw)
}
// parseHierarchy turns raw uiautomator XML into a Hierarchy.
func parseHierarchy(raw []byte) (*Hierarchy, error) {
var h Hierarchy
if err := xml.Unmarshal(raw, &h); err != nil {
return nil, fmt.Errorf("parse ui dump: %w", err)
}
if len(h.Nodes) == 0 {
return nil, fmt.Errorf("ui dump contained no nodes")
}
return &h, nil
}
// Walk visits a node and all of its children depth first.
func (n *Node) Walk(fn func(*Node)) {
fn(n)
for i := range n.Children {
n.Children[i].Walk(fn)
}
}
// Walk visits every node in the hierarchy.
func (h *Hierarchy) Walk(fn func(*Node)) {
for i := range h.Nodes {
h.Nodes[i].Walk(fn)
}
}
// Label is the human-readable identity of a node, for logs and errors.
func (n *Node) Label() string {
label := n.Text
if label == "" {
label = n.ContentDesc
}
return fmt.Sprintf("%s[%s] text=%q bounds=%s", n.Class, shortID(n.ResourceID), label, n.Bounds)
}
func shortID(id string) string {
if id == "" {
return "-"
}
if i := strings.LastIndex(id, "/"); i >= 0 {
return id[i+1:]
}
return id
}
// FindText returns nodes whose text or content description matches query.
// When exact is false the match is case-insensitive and substring-based.
func (h *Hierarchy) FindText(query string, exact bool) []*Node {
if query == "" {
return nil
}
needle := strings.ToLower(query)
var out []*Node
h.Walk(func(n *Node) {
for _, candidate := range []string{n.Text, n.ContentDesc} {
if candidate == "" {
continue
}
if exact {
if strings.EqualFold(candidate, query) {
out = append(out, n)
return
}
continue
}
if strings.Contains(strings.ToLower(candidate), needle) {
out = append(out, n)
return
}
}
})
return out
}
var boundsRe = regexp.MustCompile(`^\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]$`)
// Rect is a screen rectangle in pixels.
type Rect struct {
X1, Y1, X2, Y2 int
}
// Bounds parses the node's bounds attribute.
func (n *Node) BoundsRect() (Rect, error) {
m := boundsRe.FindStringSubmatch(strings.TrimSpace(n.Bounds))
if m == nil {
return Rect{}, fmt.Errorf("node has no parsable bounds (%q)", n.Bounds)
}
vals := make([]int, 4)
for i := 0; i < 4; i++ {
v, err := strconv.Atoi(m[i+1])
if err != nil {
return Rect{}, fmt.Errorf("bad bounds %q: %w", n.Bounds, err)
}
vals[i] = v
}
r := Rect{X1: vals[0], Y1: vals[1], X2: vals[2], Y2: vals[3]}
if r.X2 <= r.X1 || r.Y2 <= r.Y1 {
return Rect{}, fmt.Errorf("node has empty bounds %q", n.Bounds)
}
return r, nil
}
// Center returns the centre point of the node, which is what a tap uses.
func (n *Node) Center() (int, int, error) {
r, err := n.BoundsRect()
if err != nil {
return 0, 0, err
}
return (r.X1 + r.X2) / 2, (r.Y1 + r.Y2) / 2, nil
}

118
internal/ui/dump_test.go Normal file
View File

@@ -0,0 +1,118 @@
package ui
import (
"strings"
"testing"
)
const sampleDump = `<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
<hierarchy rotation="0">
<node index="0" text="" class="android.widget.FrameLayout" package="com.tailscale.ipn" clickable="false" enabled="true" scrollable="false" bounds="[0,0][1080,2400]">
<node index="0" text="samuelrolfe@gmail.com" class="android.widget.TextView" package="com.tailscale.ipn" clickable="false" enabled="true" bounds="[221,155][798,217]"/>
<node index="1" text="Connected" class="android.widget.TextView" package="com.tailscale.ipn" clickable="false" enabled="true" bounds="[221,217][453,277]"/>
<node index="2" text="" content-desc="Navigate up" class="android.widget.ImageButton" package="com.tailscale.ipn" clickable="true" enabled="true" bounds="[42,121][189,268]"/>
</node>
<node index="1" text="Network and internet" class="android.widget.TextView" package="com.android.settings" clickable="true" enabled="true" bounds="[221,663][748,727]">
<node index="0" text="Network and internet" class="android.widget.TextView" package="com.android.settings" clickable="false" enabled="true" bounds="[221,663][748,727]"/>
</node>
<node index="2" text="Disabled thing" class="android.widget.TextView" package="com.android.settings" clickable="true" enabled="false" bounds="[0,0][10,10]"/>
</hierarchy>`
func parseSample(t *testing.T) *Hierarchy {
t.Helper()
h, err := parseHierarchy([]byte(sampleDump))
if err != nil {
t.Fatalf("parseHierarchy: %v", err)
}
return h
}
func TestBoundsAndCenter(t *testing.T) {
n := &Node{Bounds: "[221,155][798,217]"}
r, err := n.BoundsRect()
if err != nil {
t.Fatalf("BoundsRect: %v", err)
}
if r.X1 != 221 || r.Y1 != 155 || r.X2 != 798 || r.Y2 != 217 {
t.Fatalf("unexpected rect %+v", r)
}
cx, cy, err := n.Center()
if err != nil {
t.Fatalf("Center: %v", err)
}
if cx != (221+798)/2 || cy != (155+217)/2 {
t.Fatalf("unexpected centre (%d,%d)", cx, cy)
}
}
func TestBoundsRejectsGarbage(t *testing.T) {
for _, bad := range []string{"", "nope", "[1,2][3,4]junk", "[10,10][10,20]"} {
if _, err := (&Node{Bounds: bad}).BoundsRect(); err == nil {
t.Fatalf("expected error for bounds %q", bad)
}
}
}
func TestFindTextExactAndSubstring(t *testing.T) {
h := parseSample(t)
if got := len(h.FindText("Connected", true)); got != 1 {
t.Fatalf("exact match: want 1, got %d", got)
}
// Substring is case-insensitive and matches more than one node.
if got := len(h.FindText("network", false)); got != 2 {
t.Fatalf("substring match: want 2 (parent + child), got %d", got)
}
// Content descriptions count as text.
matches := h.FindText("Navigate up", true)
if len(matches) != 1 || !matches[0].Clickable {
t.Fatalf("content-desc match failed: %+v", matches)
}
if got := h.FindText("does not exist", false); got != nil {
t.Fatalf("expected no matches, got %d", len(got))
}
}
func TestPickBestMatchPrefersClickable(t *testing.T) {
h := parseSample(t)
matches := h.FindText("network", false)
if len(matches) != 2 {
t.Fatalf("want 2 matches, got %d", len(matches))
}
best, err := pickBestMatch(matches)
if err != nil {
t.Fatalf("pickBestMatch: %v", err)
}
if !best.Clickable {
t.Fatalf("expected a clickable match, got %s", best.Label())
}
}
func TestPickBestMatchRejectsDisabledOnly(t *testing.T) {
h := parseSample(t)
matches := h.FindText("Disabled thing", true)
if len(matches) != 1 {
t.Fatalf("want 1 match, got %d", len(matches))
}
if _, err := pickBestMatch(matches); err == nil {
t.Fatal("expected an error when the only match is disabled")
}
}
func TestEscapeInputText(t *testing.T) {
cases := map[string]string{
"hello world": "'hello%sworld'",
"Rolling Stones": "'Rolling%sStones'",
"it's": `'it'\''s'`,
}
for in, want := range cases {
if got := escapeInputText(in); got != want {
t.Errorf("escapeInputText(%q) = %q, want %q", in, got, want)
}
}
// A single quote must not be able to break out of the quoting.
got := escapeInputText("a'; rm -rf /; 'b")
if strings.Count(got, "'")%2 != 0 {
t.Fatalf("unbalanced quotes in %q", got)
}
}