diff --git a/AGENTS.md b/AGENTS.md index a32abdd..92d73ff 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -170,10 +170,18 @@ Resolved: 2. **No UI in v1.** A recipes file replaces a UI. 3. **Find by text** from the UI dump, not screenshots or vision. 4. **Partition:** native actions → Kotlin app; in-app navigation → Artemis. +5. **ADB tooling lives in the user profile on `.13`**, installed with + `nix profile install nixpkgs#android-tools` — **not** a `nixos-rebuild` + system module. Wireless ADB over TCP needs **no udev rules**; `programs.adb` + was USB-only and is removed in this NixOS release anyway. Go is likewise run + from a one-off `nix shell nixpkgs#go`, not installed system-wide. Never run + `nixos-rebuild` on `.13`: it runs the Hub, MQTT and every service. Remaining: 1. Which tasks after Spotify and maps? (WhatsApp message? Other music apps?) 2. Locked screen: must Artemis unlock it, or does Sam leave it awake? 3. Action API shape: one generic `/act`, or one endpoint per task? -4. Docker or a systemd service on `.13`? ADB can be awkward in a container. +4. **Service packaging** for Artemis on `.13` — container or systemd — is still + open. ADB can be awkward in a container, so this is decided in Phase 6. The + tooling-placement half of the old Q4 is resolved above. 5. Speed: what is an acceptable time for a 6-step recipe? diff --git a/README.md b/README.md new file mode 100644 index 0000000..6046e1c --- /dev/null +++ b/README.md @@ -0,0 +1,68 @@ +# Artemis Control + +Artemis is the phone's **hands**. It drives the Pixel 8a screen from `.13` over +Tailscale using **ADB wireless**, so the Hub can do things no deep link allows. + +The build spec is [`AGENTS.md`](AGENTS.md). Follow the phases in section 11 in +order. + +## Status + +| Phase | State | +|---|---| +| 1 — Prove the ADB link | ✅ proven — see [`docs/phase-1-adb-link.md`](docs/phase-1-adb-link.md) | +| 2 — Act primitives | ✅ done — tap by text, type, scroll, back, home, launch | +| 3 — Spotify recipe | not started | +| 4 — Song radio | not started | +| 5 — Maps recipe | not started | +| 6 — Action API | not started | +| 7 — Resilience | not started | +| 8 — Documentation | not started | + +## Layout + +``` +cmd/artemis/ command line entry point +internal/adb/ timeout-bounded wrapper around the adb binary +internal/ui/ read the screen (uiautomator dump) and act on it (input) +docs/ phase evidence and notes +``` + +## Build and test + +Go is not installed system-wide on `.13`; use a one-off nix shell. Nothing is +added to `nixos-rebuild` and no system module changes. + +```sh +nix shell nixpkgs#go -c go build -o bin/artemis ./cmd/artemis +nix shell nixpkgs#go -c go vet ./... +nix shell nixpkgs#go -c go test ./... +``` + +## Use + +The phone address defaults to `100.101.49.17`. Override with `ARTEMIS_PHONE`, +pin a serial with `ARTEMIS_ADB_SERIAL`, or point at a different adb with +`ARTEMIS_ADB`. + +```sh +./bin/artemis connect # scan for the wireless-debugging port, then connect +./bin/artemis devices +./bin/artemis dump # text nodes and bounds +./bin/artemis launch com.spotify.music +./bin/artemis tap "Search" # find by text, tap its centre (FR-4) +./bin/artemis type "Rolling Stones Honky Tonk" +./bin/artemis scroll down +./bin/artemis back +./bin/artemis home +./bin/artemis demo # Phase 2 DoD: launch an app, tap a named element +``` + +## Rules that the code enforces + +- **Find by text, never fixed coordinates.** `tap` resolves a text match to + bounds and taps the centre. +- **Never tap blindly.** A missing or disabled element is a loud failure, not a + guessed coordinate. +- **Wake first.** Every acting command wakes the screen (FR-7). +- **Bound every command.** No adb call runs without a timeout (FR-11). diff --git a/cmd/artemis/main.go b/cmd/artemis/main.go new file mode 100644 index 0000000..39b6cef --- /dev/null +++ b/cmd/artemis/main.go @@ -0,0 +1,355 @@ +// Command artemis drives an Android phone over ADB wireless. +// +// This is the Phase 2 surface: act primitives on the command line. The Hub's +// HTTP action API arrives in Phase 6 and reuses these packages. +package main + +import ( + "fmt" + "os" + "strconv" + "time" + + "gitea.lab.audasmedia.com.au/sam/artemis_control/internal/adb" + "gitea.lab.audasmedia.com.au/sam/artemis_control/internal/ui" +) + +const usage = `artemis — drive the phone's screen over ADB wireless + +usage: artemis [args] + +connection + connect [host] connect to the phone (scans ports if host given) + devices list adb devices + wake wake the screen + +reading + dump print visible text nodes with bounds + find print matching elements (no tap) + fg print the foreground package + +acting + launch launch an app and wait for it to be foreground + tap find element by text, tap its centre (FR-4) + type type into the focused field + swipe [ms] + scroll scroll the current list + back press back + home press home + +proof + demo Phase 2 DoD: launch an app, tap a named element + +environment + ARTEMIS_PHONE phone tailscale IP (default 100.101.49.17) + ARTEMIS_ADB_SERIAL pin a specific adb serial + ARTEMIS_ADB path to the adb binary (default: PATH) +` + +func main() { + if len(os.Args) < 2 { + fmt.Fprint(os.Stderr, usage) + os.Exit(2) + } + cmd := os.Args[1] + args := os.Args[2:] + + r := adb.New(os.Getenv("ARTEMIS_ADB_SERIAL")) + + var err error + switch cmd { + case "connect": + err = cmdConnect(r, args) + case "devices": + err = cmdDevices(r) + case "wake": + err = cmdWake(r) + case "dump": + err = cmdDump(r) + case "find": + err = cmdFind(r, args) + case "fg": + err = cmdFG(r) + case "launch": + err = cmdLaunch(r, args) + case "tap": + err = cmdTap(r, args) + case "type": + err = cmdType(r, args) + case "swipe": + err = cmdSwipe(r, args) + case "scroll": + err = cmdScroll(r, args) + case "back": + err = ensurePhone(r, func() error { return ui.Back(r) }) + case "home": + err = ensurePhone(r, func() error { return ui.Home(r) }) + case "demo": + err = cmdDemo(r) + case "-h", "--help", "help": + fmt.Print(usage) + return + default: + err = fmt.Errorf("unknown command %q\n\n%s", cmd, usage) + } + + if err != nil { + fmt.Fprintf(os.Stderr, "artemis: %v\n", err) + os.Exit(1) + } +} + +func phoneHost() string { + if h := os.Getenv("ARTEMIS_PHONE"); h != "" { + return h + } + return "100.101.49.17" +} + +// ensurePhone makes sure we have a live serial before running fn. +func ensurePhone(r *adb.Runner, fn func() error) error { + if r.Serial != "" && r.DeviceOnline(r.Serial) { + return fn() + } + serial, err := r.ConnectAuto(phoneHost()) + if err != nil { + return fmt.Errorf("phone is not reachable: %w", err) + } + fmt.Printf("connected to %s\n", serial) + return fn() +} + +func cmdConnect(r *adb.Runner, args []string) error { + if len(args) > 0 { + msg, err := r.Connect(args[0]) + if msg != "" { + fmt.Println(msg) + } + return err + } + serial, err := r.ConnectAuto(phoneHost()) + if err != nil { + return err + } + fmt.Printf("connected to %s\n", serial) + return nil +} + +func cmdDevices(r *adb.Runner) error { + devices, err := r.Devices() + if err != nil { + return err + } + if len(devices) == 0 { + fmt.Println("no devices") + return nil + } + for _, d := range devices { + fmt.Printf("%s\t%s\tmodel=%s device=%s\n", d.Serial, d.State, d.Model, d.Device) + } + return nil +} + +func cmdWake(r *adb.Runner) error { + return ensurePhone(r, func() error { + if err := ui.Wake(r); err != nil { + return err + } + fmt.Println("screen awake") + return nil + }) +} + +func cmdDump(r *adb.Runner) error { + return ensurePhone(r, func() error { + if err := ui.Wake(r); err != nil { + return err + } + h, err := ui.Dump(r) + if err != nil { + return err + } + nodes := h.TextNodes() + fmt.Printf("%d text nodes\n", len(nodes)) + for _, n := range nodes { + label := n.Text + if label == "" { + label = n.ContentDesc + } + fmt.Printf("[%s] %s bounds=%s\n", shortClass(n.Class), label, n.Bounds) + } + return nil + }) +} + +func cmdFind(r *adb.Runner, args []string) error { + if len(args) != 1 { + return fmt.Errorf("usage: artemis find ") + } + return ensurePhone(r, func() error { + h, err := ui.Dump(r) + if err != nil { + return err + } + matches := h.FindText(args[0], false) + if len(matches) == 0 { + return fmt.Errorf("no element matching %q", args[0]) + } + for _, n := range matches { + cx, cy, _ := n.Center() + fmt.Printf("%s centre=(%d,%d)\n", n.Label(), cx, cy) + } + return nil + }) +} + +func cmdFG(r *adb.Runner) error { + return ensurePhone(r, func() error { + fg, err := ui.ForegroundPackage(r) + if err != nil { + return err + } + fmt.Println(fg) + return nil + }) +} + +func cmdLaunch(r *adb.Runner, args []string) error { + if len(args) != 1 { + return fmt.Errorf("usage: artemis launch ") + } + return ensurePhone(r, func() error { + if err := ui.Wake(r); err != nil { + return err + } + start := time.Now() + if err := ui.Launch(r, args[0]); err != nil { + return err + } + fmt.Printf("launched %s in %s\n", args[0], time.Since(start).Round(time.Millisecond)) + return nil + }) +} + +func cmdTap(r *adb.Runner, args []string) error { + if len(args) != 1 { + return fmt.Errorf("usage: artemis tap ") + } + return ensurePhone(r, func() error { + if err := ui.Wake(r); err != nil { + return err + } + node, err := ui.TapText(r, args[0], false) + if err != nil { + return err + } + cx, cy, _ := node.Center() + fmt.Printf("tapped %q at (%d,%d) — %s\n", args[0], cx, cy, node.Label()) + return nil + }) +} + +func cmdType(r *adb.Runner, args []string) error { + if len(args) != 1 { + return fmt.Errorf("usage: artemis type ") + } + return ensurePhone(r, func() error { return ui.Type(r, args[0]) }) +} + +func cmdSwipe(r *adb.Runner, args []string) error { + if len(args) < 4 || len(args) > 5 { + return fmt.Errorf("usage: artemis swipe [ms]") + } + nums := make([]int, 4) + for i, a := range args[:4] { + v, err := strconv.Atoi(a) + if err != nil { + return fmt.Errorf("swipe coordinate %q is not a number", a) + } + nums[i] = v + } + ms := 300 + if len(args) == 5 { + v, err := strconv.Atoi(args[4]) + if err != nil { + return fmt.Errorf("swipe duration %q is not a number", args[4]) + } + ms = v + } + return ensurePhone(r, func() error { return ui.Swipe(r, nums[0], nums[1], nums[2], nums[3], ms) }) +} + +func cmdScroll(r *adb.Runner, args []string) error { + if len(args) != 1 { + return fmt.Errorf("usage: artemis scroll ") + } + return ensurePhone(r, func() error { return ui.Scroll(r, args[0]) }) +} + +// cmdDemo is the Phase 2 Definition of Done, runnable on demand: +// Artemis opens a named app and taps a named element on it. +func cmdDemo(r *adb.Runner) error { + return ensurePhone(r, func() error { + if err := ui.Wake(r); err != nil { + return err + } + + fmt.Println("step 1: launch", ui.PackageSettings) + if err := ui.Launch(r, ui.PackageSettings); err != nil { + return err + } + + // Launching resumes the last Settings page, so back out to the home + // screen first to make the demo repeatable. + const homeMarker = "Search Settings" + for attempt := 0; attempt < 4; attempt++ { + if _, err := ui.WaitForText(r, homeMarker, false, 3*time.Second); err == nil { + break + } + if attempt == 3 { + return fmt.Errorf("Settings home (%q) never appeared", homeMarker) + } + if err := ui.Back(r); err != nil { + return err + } + time.Sleep(700 * time.Millisecond) + } + + // A stable, top-level Settings entry on the Pixel 8a. + const target = "Network and internet" + fmt.Printf("step 2: tap %q\n", target) + node, err := ui.TapText(r, target, false) + if err != nil { + return err + } + cx, cy, _ := node.Center() + fmt.Printf(" tapped %s at (%d,%d)\n", node.Label(), cx, cy) + + // Prove the tap navigated: this text only exists on the target page. + const proof = "Aeroplane mode" + fmt.Printf("step 3: wait for %q\n", proof) + if _, err := ui.WaitForText(r, proof, false, 10*time.Second); err != nil { + return fmt.Errorf("tap did not open the expected page: %w", err) + } + + fg, err := ui.ForegroundPackage(r) + if err != nil { + return err + } + fmt.Println("step 4: foreground package is", fg) + if fg != ui.PackageSettings { + return fmt.Errorf("expected to still be in %s after the tap, got %s", ui.PackageSettings, fg) + } + fmt.Println("demo OK") + return nil + }) +} + +func shortClass(class string) string { + for i := len(class) - 1; i >= 0; i-- { + if class[i] == '.' { + return class[i+1:] + } + } + return class +} diff --git a/docs/phase-2-act-primitives.md b/docs/phase-2-act-primitives.md new file mode 100644 index 0000000..9d26036 --- /dev/null +++ b/docs/phase-2-act-primitives.md @@ -0,0 +1,103 @@ +# Phase 2 — Act primitives + +Status: **done**. Verified live against the Pixel 8a on 2026-10-16. + +Definition of Done: *Artemis opens a named app and taps a named element on it.* + +--- + +## 1. Primitives + +All primitives live in `internal/ui`. Every acting command wakes the screen +first (FR-7) and every adb call is bounded by a timeout (FR-11). + +| Primitive | Command | How it works | +|---|---|---| +| Tap by text | `artemis tap ` | dump → match text/content-desc → tap element centre (FR-4) | +| Type | `artemis type ` | `input text`, with `%s` space encoding and shell-safe quoting | +| Swipe | `artemis swipe x1 y1 x2 y2 [ms]` | `input swipe` | +| Scroll | `artemis scroll up\|down` | picks the tallest vertical scrollable node, swipes inside it | +| Back | `artemis back` | key event 4 | +| Home | `artemis home` | key event 3 | +| Launch app | `artemis launch ` | `monkey -p ... 1`, then waits for foreground | +| Wake | `artemis wake` | key event 224, polls `dumpsys power` until `Awake` | +| Read screen | `artemis dump` | `uiautomator dump`, parsed as XML, text nodes with bounds | +| Find (no tap) | `artemis find ` | dump → matching elements | +| Foreground app | `artemis fg` | `dumpsys window` → `mCurrentFocus` | + +## 2. Definition of Done run + +`artemis demo` launches Settings, backs out to the Settings home, taps +**Network and internet**, then proves the tap navigated by waiting for +**Aeroplane mode**, which only exists on the target page. + +``` +$ ./bin/artemis demo +connected to 100.101.49.17:46689 +step 1: launch com.android.settings +step 2: tap "Network and internet" + tapped android.widget.TextView[title] text="Network and internet" bounds=[221,663][748,727] at (484,695) +step 3: wait for "Aeroplane mode" +step 4: foreground package is com.android.settings +demo OK +``` + +## 3. Other live checks + +``` +$ ./bin/artemis home ; ./bin/artemis fg +com.google.android.apps.nexuslauncher + +$ ./bin/artemis launch com.spotify.music +launched com.spotify.music in 698ms + +$ ./bin/artemis scroll down # Spotify home: tabs stay, list content changes +before: [TextView] Julius Caesar Part 1: The Roman Dictator +after: [TextView] It's New Music Friday! +``` + +## 4. Failure is loud (FR-8) + +The first demo run used the wrong string, `"Network & internet"`, while the +Pixel spells it `"Network and internet"`. Artemis refused to act: + +``` +artemis: element with text "Network & internet" not found on screen; refusing to tap blindly +``` + +No coordinate was guessed. A test in `internal/ui/dump_test.go` pins the same +behaviour: a set of matches whose only member is disabled yields an error, not a +tap. + +## 5. Design notes + +- **Text first, coordinates derived.** `TapText` finds matches, ranks them + (clickable first, then enabled, then smallest area), reads the winner's + bounds, and taps the centre. Coordinates are never written down by hand. +- **Scroll targets vertical lists only.** An earlier version picked the + smallest scrollable node, which on Spotify was a short horizontal carousel; a + vertical swipe there became a mis-gesture. The current version picks the + tallest scrollable node that is taller than it is wide and covers at least a + third of the screen, else it falls back to the screen rectangle. +- **Launch waits for a result.** `Launch` does not return success until the + package is actually the foreground app, or it fails after 10s. +- **Timeout everywhere.** `adb.Runner` runs each command through + `context.WithTimeout`; a recipe can never hang. + +## 6. Tests + +```sh +nix shell nixpkgs#go -c go vet ./... +nix shell nixpkgs#go -c go test ./... +``` + +`internal/ui` unit tests cover bounds parsing, centre maths, exact vs substring +text matching, content descriptions, disabled matches, and shell-safe input +escaping. The live checks above cover the phone. + +## 7. Not done / next + +- No recipes file yet; that is Phase 3. +- No HTTP action API yet; that is Phase 6. +- Locked-screen handling is unsettled (§13 Q2): `wake` dismisses a non-secure + keyguard but cannot unlock a PIN-locked phone. diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..1e4935a --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module gitea.lab.audasmedia.com.au/sam/artemis_control + +go 1.22 diff --git a/internal/adb/adb.go b/internal/adb/adb.go new file mode 100644 index 0000000..83550af --- /dev/null +++ b/internal/adb/adb.go @@ -0,0 +1,130 @@ +// Package adb is a thin, timeout-bounded wrapper around the adb command line +// tool. Artemis never talks to the ADB wire protocol directly: it shells out to +// the adb binary installed in the user profile on .13. +package adb + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "strings" + "time" +) + +// DefaultTimeout bounds every single adb invocation. A recipe must never run +// forever (FR-11), so no command is allowed to run without a deadline. +const DefaultTimeout = 20 * time.Second + +// Runner executes adb commands against one device. +type Runner struct { + Adb string + Serial string + Timeout time.Duration +} + +// New returns a Runner. The adb binary is taken from ARTEMIS_ADB, else PATH. +func New(serial string) *Runner { + adbPath := os.Getenv("ARTEMIS_ADB") + if adbPath == "" { + adbPath = "adb" + } + return &Runner{Adb: adbPath, Serial: serial, Timeout: DefaultTimeout} +} + +func (r *Runner) adbArgs(args ...string) []string { + full := make([]string, 0, len(args)+2) + if r.Serial != "" { + full = append(full, "-s", r.Serial) + } + return append(full, args...) +} + +func (r *Runner) run(timeout time.Duration, args ...string) ([]byte, error) { + if timeout <= 0 { + timeout = r.Timeout + } + if timeout <= 0 { + timeout = DefaultTimeout + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + cmd := exec.CommandContext(ctx, r.Adb, r.adbArgs(args...)...) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err := cmd.Run() + if ctx.Err() == context.DeadlineExceeded { + return stdout.Bytes(), fmt.Errorf("adb %s: timed out after %s", strings.Join(args, " "), timeout) + } + if err != nil { + msg := strings.TrimSpace(stderr.String()) + if msg == "" { + msg = strings.TrimSpace(stdout.String()) + } + return stdout.Bytes(), fmt.Errorf("adb %s: %w: %s", strings.Join(args, " "), err, msg) + } + return stdout.Bytes(), nil +} + +// Shell runs a command in the device shell and returns combined stdout. +// +// The whole command is passed as one argv element; adb joins argv with spaces +// and the device's sh parses the result, exactly as a human at a terminal. +func (r *Runner) Shell(command string) (string, error) { + out, err := r.run(r.Timeout, "shell", command) + return string(out), err +} + +// ExecOut runs adb exec-out, which does not add carriage returns. Use it to +// read files and binary data off the device. +func (r *Runner) ExecOut(args ...string) ([]byte, error) { + return r.run(r.Timeout, append([]string{"exec-out"}, args...)...) +} + +// Device is one row of `adb devices -l`. +type Device struct { + Serial string + State string + Product string + Model string + Device string + TransportID string +} + +// Devices lists the devices the adb server knows about. +func (r *Runner) Devices() ([]Device, error) { + out, err := r.run(10*time.Second, "devices", "-l") + if err != nil { + return nil, err + } + var devices []Device + for _, line := range strings.Split(string(out), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "List of devices") { + continue + } + fields := strings.Fields(line) + if len(fields) < 2 { + continue + } + d := Device{Serial: fields[0], State: fields[1]} + for _, f := range fields[2:] { + switch { + case strings.HasPrefix(f, "product:"): + d.Product = strings.TrimPrefix(f, "product:") + case strings.HasPrefix(f, "model:"): + d.Model = strings.TrimPrefix(f, "model:") + case strings.HasPrefix(f, "device:"): + d.Device = strings.TrimPrefix(f, "device:") + case strings.HasPrefix(f, "transport_id:"): + d.TransportID = strings.TrimPrefix(f, "transport_id:") + } + } + devices = append(devices, d) + } + return devices, nil +} diff --git a/internal/adb/discover.go b/internal/adb/discover.go new file mode 100644 index 0000000..da0e3c1 --- /dev/null +++ b/internal/adb/discover.go @@ -0,0 +1,157 @@ +package adb + +import ( + "fmt" + "sort" + "strings" + "sync" + "time" + + "net" +) + +// The wireless-debugging port changes on every phone reboot or toggle of +// wireless debugging, so it is never hardcoded. Android opens two TCP ports: +// a pairing port (useless here) and the connect port. `adb connect` succeeds on +// the connect port and is refused on the pairing port. + +// DefaultScanRange is the range Android picks wireless-debugging ports from. +const ( + DefaultScanFrom = 30000 + DefaultScanTo = 65535 +) + +// Connect asks the adb server to connect to host:port. +func (r *Runner) Connect(address string) (string, error) { + out, err := r.run(15*time.Second, "connect", address) + msg := strings.TrimSpace(string(out)) + if err != nil { + return msg, err + } + // adb connect can exit 0 while reporting failure in its output. + lower := strings.ToLower(msg) + if strings.Contains(lower, "failed") || strings.Contains(lower, "cannot") { + return msg, fmt.Errorf("adb connect %s: %s", address, msg) + } + return msg, nil +} + +// DeviceOnline reports whether serial is present and in state "device". +func (r *Runner) DeviceOnline(serial string) bool { + devices, err := r.Devices() + if err != nil { + return false + } + for _, d := range devices { + if d.Serial == serial && d.State == "device" { + return true + } + } + return false +} + +// ScanPorts does a concurrent TCP connect scan of host:from-to and returns the +// open ports in ascending order. +func ScanPorts(host string, from, to int, timeout time.Duration) []int { + if timeout <= 0 { + timeout = 500 * time.Millisecond + } + const workers = 512 + + type result struct { + port int + open bool + } + + jobs := make(chan int) + results := make(chan result) + var wg sync.WaitGroup + + for i := 0; i < workers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for port := range jobs { + addr := net.JoinHostPort(host, fmt.Sprintf("%d", port)) + conn, err := net.DialTimeout("tcp", addr, timeout) + if err != nil { + results <- result{port: port, open: false} + continue + } + conn.Close() + results <- result{port: port, open: true} + } + }() + } + + go func() { + for p := from; p <= to; p++ { + jobs <- p + } + close(jobs) + }() + + go func() { + wg.Wait() + close(results) + }() + + var open []int + for res := range results { + if res.open { + open = append(open, res.port) + } + } + sort.Ints(open) + return open +} + +// ConnectAuto brings the phone online. It reuses an already-connected device if +// one is present, otherwise it scans host for open ports and tries +// `adb connect` on each one until a port comes up in state "device". +// +// It returns the serial it connected to. +func (r *Runner) ConnectAuto(host string) (string, error) { + // Already connected? Find a device whose serial is host:port. + for _, d := range mustDevices(r) { + if d.State == "device" && strings.HasPrefix(d.Serial, host+":") { + r.Serial = d.Serial + return d.Serial, nil + } + } + + ports := ScanPorts(host, DefaultScanFrom, DefaultScanTo, 0) + if len(ports) == 0 { + return "", fmt.Errorf("no open TCP ports on %s in %d-%d; is wireless debugging on?", + host, DefaultScanFrom, DefaultScanTo) + } + + var errs []string + for _, port := range ports { + serial := fmt.Sprintf("%s:%d", host, port) + if _, err := r.Connect(serial); err != nil { + errs = append(errs, fmt.Sprintf("%d: %v", port, err)) + continue + } + // Give the adb server a moment to move the device to "device". + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if r.DeviceOnline(serial) { + r.Serial = serial + return serial, nil + } + time.Sleep(250 * time.Millisecond) + } + errs = append(errs, fmt.Sprintf("%d: connect accepted but device never came online", port)) + } + return "", fmt.Errorf("open ports on %s were %v but none accepted adb connect: %s", + host, ports, strings.Join(errs, "; ")) +} + +func mustDevices(r *Runner) []Device { + devices, err := r.Devices() + if err != nil { + return nil + } + return devices +} diff --git a/internal/ui/actions.go b/internal/ui/actions.go new file mode 100644 index 0000000..64ec237 --- /dev/null +++ b/internal/ui/actions.go @@ -0,0 +1,324 @@ +package ui + +import ( + "fmt" + "regexp" + "sort" + "strconv" + "strings" + "time" + + "gitea.lab.audasmedia.com.au/sam/artemis_control/internal/adb" +) + +// Android key events we use. +const ( + KeyHome = 3 + KeyBack = 4 + KeyEnter = 66 + KeyWakeup = 224 +) + +// Well-known packages. +const ( + PackageSpotify = "com.spotify.music" + PackageMaps = "com.google.android.apps.maps" + PackageSettings = "com.android.settings" +) + +// Tap sends a tap at absolute screen coordinates. +func Tap(r *adb.Runner, x, y int) error { + if x < 0 || y < 0 { + return fmt.Errorf("refusing to tap negative coordinates (%d,%d)", x, y) + } + _, err := r.Shell(fmt.Sprintf("input tap %d %d", x, y)) + return err +} + +// KeyEvent sends a keyevent by numeric code. +func KeyEvent(r *adb.Runner, code int) error { + _, err := r.Shell(fmt.Sprintf("input keyevent %d", code)) + return err +} + +// Back presses the system back button. +func Back(r *adb.Runner) error { return KeyEvent(r, KeyBack) } + +// Home presses the system home button. +func Home(r *adb.Runner) error { return KeyEvent(r, KeyHome) } + +// Type types text into the focused field. +func Type(r *adb.Runner, text string) error { + if strings.TrimSpace(text) == "" { + return fmt.Errorf("refusing to type empty text") + } + _, err := r.Shell("input text " + escapeInputText(text)) + return err +} + +// escapeInputText makes a string safe for `input text`. +// +// `input text` decodes %s as a space, so spaces are encoded first. The result +// is then single-quoted for the device shell, with embedded quotes escaped. +func escapeInputText(text string) string { + encoded := strings.ReplaceAll(text, " ", "%s") + return "'" + strings.ReplaceAll(encoded, "'", `'\''`) + "'" +} + +// Swipe drags from one point to another over duration. +func Swipe(r *adb.Runner, x1, y1, x2, y2, durationMs int) error { + if durationMs <= 0 { + durationMs = 300 + } + _, err := r.Shell(fmt.Sprintf("input swipe %d %d %d %d %d", x1, y1, x2, y2, durationMs)) + return err +} + +// tapCandidate ranks matches so the tap lands on something a human could press. +type tapCandidate struct { + node *Node + clickable bool + enabled bool + area int +} + +// pickBestMatch chooses the most tappable node from a set of text matches. +// Clickable and enabled nodes win; among equals the smallest node wins, because +// the smallest enclosing element is the control itself rather than its container. +func pickBestMatch(matches []*Node) (*Node, error) { + candidates := make([]tapCandidate, 0, len(matches)) + for _, n := range matches { + area := 1 << 30 + if r, err := n.BoundsRect(); err == nil { + area = (r.X2 - r.X1) * (r.Y2 - r.Y1) + } + candidates = append(candidates, tapCandidate{node: n, clickable: n.Clickable, enabled: n.Enabled, area: area}) + } + sort.SliceStable(candidates, func(i, j int) bool { + a, b := candidates[i], candidates[j] + if a.clickable != b.clickable { + return a.clickable + } + if a.enabled != b.enabled { + return a.enabled + } + return a.area < b.area + }) + best := candidates[0] + if !best.enabled { + return nil, fmt.Errorf("best match %s is disabled", best.node.Label()) + } + return best.node, nil +} + +// TapText finds an element by its text (or content description) and taps its +// centre. This is FR-4: text match first, coordinates derived from bounds. +// +// It fails loudly when no element matches and never falls back to tapping a +// guessed coordinate. +func TapText(r *adb.Runner, query string, exact bool) (*Node, error) { + if query == "" { + return nil, fmt.Errorf("tap requires a non-empty text query") + } + h, err := Dump(r) + if err != nil { + return nil, err + } + matches := h.FindText(query, exact) + if len(matches) == 0 { + return nil, fmt.Errorf("element with text %q not found on screen; refusing to tap blindly", query) + } + node, err := pickBestMatch(matches) + if err != nil { + return nil, err + } + x, y, err := node.Center() + if err != nil { + return nil, err + } + if err := Tap(r, x, y); err != nil { + return nil, err + } + return node, nil +} + +// WaitForText polls the screen until an element matching query appears, then +// returns the best tappable match. Use it after an action that loads a new +// screen; apps do not draw instantly. +func WaitForText(r *adb.Runner, query string, exact bool, timeout time.Duration) (*Node, error) { + if timeout <= 0 { + timeout = 10 * time.Second + } + deadline := time.Now().Add(timeout) + var seen []*Node + for time.Now().Before(deadline) { + h, err := Dump(r) + if err == nil { + matches := h.FindText(query, exact) + if len(matches) > 0 { + seen = matches + if n, perr := pickBestMatch(matches); perr == nil { + return n, nil + } + } + } + time.Sleep(400 * time.Millisecond) + } + if len(seen) > 0 { + return nil, fmt.Errorf("text %q is on screen but no enabled match could be tapped", query) + } + return nil, fmt.Errorf("text %q did not appear within %s", query, timeout) +} + +// ScreenSize reads the physical screen size. +func ScreenSize(r *adb.Runner) (width, height int, err error) { + out, err := r.Shell("wm size") + if err != nil { + return 0, 0, err + } + // "Physical size: 1080x2400" (an Override size line may follow). + re := regexp.MustCompile(`(\d+)x(\d+)`) + m := re.FindStringSubmatch(out) + if m == nil { + return 0, 0, fmt.Errorf("could not parse screen size from %q", strings.TrimSpace(out)) + } + w, _ := strconv.Atoi(m[1]) + h, _ := strconv.Atoi(m[2]) + return w, h, nil +} + +// Scroll scrolls the current screen in the given direction ("down", "up"). +// +// It prefers a scrollable node from the dump, so the swipe stays inside a list +// rather than on a toolbar. If no scrollable node is present it falls back to +// the middle of the screen. +func Scroll(r *adb.Runner, direction string) error { + direction = strings.ToLower(strings.TrimSpace(direction)) + if direction != "down" && direction != "up" { + return fmt.Errorf("scroll direction must be \"up\" or \"down\", got %q", direction) + } + + width, height, err := ScreenSize(r) + if err != nil { + return err + } + + rect := Rect{X1: 0, Y1: 0, X2: width, Y2: height} + h, dumpErr := Dump(r) + if dumpErr == nil { + // Pick the tallest scrollable node that is taller than it is wide. + // Short or horizontal elements (carousels, toolbars) are ignored: a + // vertical swipe inside them becomes a mis-gesture, which is exactly + // what "never act blindly" forbids. + var best *Node + bestHeight := 0 + h.Walk(func(n *Node) { + if !n.Scrollable || !n.Enabled { + return + } + b, berr := n.BoundsRect() + if berr != nil { + return + } + nHeight := b.Y2 - b.Y1 + if nHeight <= b.X2-b.X1 { + return + } + if nHeight > bestHeight { + bestHeight, best = nHeight, n + } + }) + // Only trust it if it covers a meaningful slice of the screen. + if best != nil && bestHeight >= height/3 { + if b, berr := best.BoundsRect(); berr == nil { + rect = b + } + } + } + + const margin = 0.25 + midX := (rect.X1 + rect.X2) / 2 + top := rect.Y1 + int(float64(rect.Y2-rect.Y1)*margin) + bottom := rect.Y2 - int(float64(rect.Y2-rect.Y1)*margin) + if direction == "down" { + return Swipe(r, midX, bottom, midX, top, 400) + } + return Swipe(r, midX, top, midX, bottom, 400) +} + +// Wake makes sure the screen is on before any action (FR-7). +func Wake(r *adb.Runner) error { + if _, err := r.Shell("input keyevent " + strconv.Itoa(KeyWakeup)); err != nil { + return fmt.Errorf("wake keyevent: %w", err) + } + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + out, err := r.Shell("dumpsys power | grep mWakefulness=") + if err == nil && strings.Contains(out, "mWakefulness=Awake") { + // Dismiss a non-secure keyguard; harmless when the phone is unlocked. + _, _ = r.Shell("wm dismiss-keyguard") + return nil + } + time.Sleep(300 * time.Millisecond) + } + return fmt.Errorf("screen did not reach Awake within 10s") +} + +var packageRe = regexp.MustCompile(`\bu0 ([a-zA-Z0-9_.]+)/`) + +// ForegroundPackage returns the package of the current foreground window. +func ForegroundPackage(r *adb.Runner) (string, error) { + out, err := r.Shell("dumpsys window | grep mCurrentFocus") + if err != nil { + return "", err + } + m := packageRe.FindStringSubmatch(out) + if m == nil { + return "", fmt.Errorf("could not determine foreground package from %q", strings.TrimSpace(out)) + } + return m[1], nil +} + +// Launch starts an app by package name and waits until it is in the foreground. +func Launch(r *adb.Runner, pkg string) error { + if pkg == "" { + return fmt.Errorf("launch requires a package name") + } + out, err := r.Shell("monkey -p " + pkg + " -c android.intent.category.LAUNCHER 1") + if err != nil { + return fmt.Errorf("launch %s: %w", pkg, err) + } + if strings.Contains(out, "monkey aborted") || strings.Contains(out, "No activities found") { + return fmt.Errorf("launch %s: app not installed or has no launcher activity: %s", pkg, strings.TrimSpace(out)) + } + + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + fg, err := ForegroundPackage(r) + if err == nil && fg == pkg { + return nil + } + time.Sleep(300 * time.Millisecond) + } + return fmt.Errorf("launched %s but it never became the foreground app within 10s", pkg) +} + +// AppInstalled reports whether a package is installed. +func AppInstalled(r *adb.Runner, pkg string) (bool, error) { + out, err := r.Shell("pm list packages " + pkg) + if err != nil { + return false, err + } + return strings.Contains(out, "package:"+pkg), nil +} + +// TextNodes returns every node with visible text, for dumps and debugging. +func (h *Hierarchy) TextNodes() []*Node { + var out []*Node + h.Walk(func(n *Node) { + if strings.TrimSpace(n.Text) != "" || strings.TrimSpace(n.ContentDesc) != "" { + out = append(out, n) + } + }) + return out +} diff --git a/internal/ui/dump.go b/internal/ui/dump.go new file mode 100644 index 0000000..63f729d --- /dev/null +++ b/internal/ui/dump.go @@ -0,0 +1,170 @@ +// Package ui reads the phone screen and drives it. +// +// Reading uses `uiautomator dump`, which gives real element text and bounds. +// Acting uses `input`, which takes coordinates because that is all Android +// offers. Every tap is therefore derived from a text match in the dump — never +// from a hardcoded coordinate (FR-4). +package ui + +import ( + "encoding/xml" + "fmt" + "regexp" + "strconv" + "strings" + + "gitea.lab.audasmedia.com.au/sam/artemis_control/internal/adb" +) + +const dumpPath = "/sdcard/artemis_window_dump.xml" + +// Node is one element in the uiautomator hierarchy. +type Node struct { + Index int `xml:"index,attr"` + Text string `xml:"text,attr"` + ResourceID string `xml:"resource-id,attr"` + Class string `xml:"class,attr"` + Package string `xml:"package,attr"` + ContentDesc string `xml:"content-desc,attr"` + Clickable bool `xml:"clickable,attr"` + Enabled bool `xml:"enabled,attr"` + Scrollable bool `xml:"scrollable,attr"` + Focused bool `xml:"focused,attr"` + Selected bool `xml:"selected,attr"` + Bounds string `xml:"bounds,attr"` + Children []Node `xml:"node"` +} + +// Hierarchy is a parsed uiautomator dump. +type Hierarchy struct { + XMLName xml.Name `xml:"hierarchy"` + Rotation int `xml:"rotation,attr"` + Nodes []Node `xml:"node"` +} + +// Dump fetches and parses the current screen. +func Dump(r *adb.Runner) (*Hierarchy, error) { + if _, err := r.Shell("uiautomator dump " + dumpPath); err != nil { + return nil, fmt.Errorf("uiautomator dump: %w", err) + } + raw, err := r.ExecOut("cat", dumpPath) + if err != nil { + return nil, fmt.Errorf("read ui dump: %w", err) + } + if len(raw) == 0 { + return nil, fmt.Errorf("ui dump was empty") + } + return parseHierarchy(raw) +} + +// parseHierarchy turns raw uiautomator XML into a Hierarchy. +func parseHierarchy(raw []byte) (*Hierarchy, error) { + var h Hierarchy + if err := xml.Unmarshal(raw, &h); err != nil { + return nil, fmt.Errorf("parse ui dump: %w", err) + } + if len(h.Nodes) == 0 { + return nil, fmt.Errorf("ui dump contained no nodes") + } + return &h, nil +} + +// Walk visits a node and all of its children depth first. +func (n *Node) Walk(fn func(*Node)) { + fn(n) + for i := range n.Children { + n.Children[i].Walk(fn) + } +} + +// Walk visits every node in the hierarchy. +func (h *Hierarchy) Walk(fn func(*Node)) { + for i := range h.Nodes { + h.Nodes[i].Walk(fn) + } +} + +// Label is the human-readable identity of a node, for logs and errors. +func (n *Node) Label() string { + label := n.Text + if label == "" { + label = n.ContentDesc + } + return fmt.Sprintf("%s[%s] text=%q bounds=%s", n.Class, shortID(n.ResourceID), label, n.Bounds) +} + +func shortID(id string) string { + if id == "" { + return "-" + } + if i := strings.LastIndex(id, "/"); i >= 0 { + return id[i+1:] + } + return id +} + +// FindText returns nodes whose text or content description matches query. +// When exact is false the match is case-insensitive and substring-based. +func (h *Hierarchy) FindText(query string, exact bool) []*Node { + if query == "" { + return nil + } + needle := strings.ToLower(query) + var out []*Node + h.Walk(func(n *Node) { + for _, candidate := range []string{n.Text, n.ContentDesc} { + if candidate == "" { + continue + } + if exact { + if strings.EqualFold(candidate, query) { + out = append(out, n) + return + } + continue + } + if strings.Contains(strings.ToLower(candidate), needle) { + out = append(out, n) + return + } + } + }) + return out +} + +var boundsRe = regexp.MustCompile(`^\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]$`) + +// Rect is a screen rectangle in pixels. +type Rect struct { + X1, Y1, X2, Y2 int +} + +// Bounds parses the node's bounds attribute. +func (n *Node) BoundsRect() (Rect, error) { + m := boundsRe.FindStringSubmatch(strings.TrimSpace(n.Bounds)) + if m == nil { + return Rect{}, fmt.Errorf("node has no parsable bounds (%q)", n.Bounds) + } + vals := make([]int, 4) + for i := 0; i < 4; i++ { + v, err := strconv.Atoi(m[i+1]) + if err != nil { + return Rect{}, fmt.Errorf("bad bounds %q: %w", n.Bounds, err) + } + vals[i] = v + } + r := Rect{X1: vals[0], Y1: vals[1], X2: vals[2], Y2: vals[3]} + if r.X2 <= r.X1 || r.Y2 <= r.Y1 { + return Rect{}, fmt.Errorf("node has empty bounds %q", n.Bounds) + } + return r, nil +} + +// Center returns the centre point of the node, which is what a tap uses. +func (n *Node) Center() (int, int, error) { + r, err := n.BoundsRect() + if err != nil { + return 0, 0, err + } + return (r.X1 + r.X2) / 2, (r.Y1 + r.Y2) / 2, nil +} diff --git a/internal/ui/dump_test.go b/internal/ui/dump_test.go new file mode 100644 index 0000000..0b3f0cf --- /dev/null +++ b/internal/ui/dump_test.go @@ -0,0 +1,118 @@ +package ui + +import ( + "strings" + "testing" +) + +const sampleDump = ` + + + + + + + + + + +` + +func parseSample(t *testing.T) *Hierarchy { + t.Helper() + h, err := parseHierarchy([]byte(sampleDump)) + if err != nil { + t.Fatalf("parseHierarchy: %v", err) + } + return h +} + +func TestBoundsAndCenter(t *testing.T) { + n := &Node{Bounds: "[221,155][798,217]"} + r, err := n.BoundsRect() + if err != nil { + t.Fatalf("BoundsRect: %v", err) + } + if r.X1 != 221 || r.Y1 != 155 || r.X2 != 798 || r.Y2 != 217 { + t.Fatalf("unexpected rect %+v", r) + } + cx, cy, err := n.Center() + if err != nil { + t.Fatalf("Center: %v", err) + } + if cx != (221+798)/2 || cy != (155+217)/2 { + t.Fatalf("unexpected centre (%d,%d)", cx, cy) + } +} + +func TestBoundsRejectsGarbage(t *testing.T) { + for _, bad := range []string{"", "nope", "[1,2][3,4]junk", "[10,10][10,20]"} { + if _, err := (&Node{Bounds: bad}).BoundsRect(); err == nil { + t.Fatalf("expected error for bounds %q", bad) + } + } +} + +func TestFindTextExactAndSubstring(t *testing.T) { + h := parseSample(t) + + if got := len(h.FindText("Connected", true)); got != 1 { + t.Fatalf("exact match: want 1, got %d", got) + } + // Substring is case-insensitive and matches more than one node. + if got := len(h.FindText("network", false)); got != 2 { + t.Fatalf("substring match: want 2 (parent + child), got %d", got) + } + // Content descriptions count as text. + matches := h.FindText("Navigate up", true) + if len(matches) != 1 || !matches[0].Clickable { + t.Fatalf("content-desc match failed: %+v", matches) + } + if got := h.FindText("does not exist", false); got != nil { + t.Fatalf("expected no matches, got %d", len(got)) + } +} + +func TestPickBestMatchPrefersClickable(t *testing.T) { + h := parseSample(t) + matches := h.FindText("network", false) + if len(matches) != 2 { + t.Fatalf("want 2 matches, got %d", len(matches)) + } + best, err := pickBestMatch(matches) + if err != nil { + t.Fatalf("pickBestMatch: %v", err) + } + if !best.Clickable { + t.Fatalf("expected a clickable match, got %s", best.Label()) + } +} + +func TestPickBestMatchRejectsDisabledOnly(t *testing.T) { + h := parseSample(t) + matches := h.FindText("Disabled thing", true) + if len(matches) != 1 { + t.Fatalf("want 1 match, got %d", len(matches)) + } + if _, err := pickBestMatch(matches); err == nil { + t.Fatal("expected an error when the only match is disabled") + } +} + +func TestEscapeInputText(t *testing.T) { + cases := map[string]string{ + "hello world": "'hello%sworld'", + "Rolling Stones": "'Rolling%sStones'", + "it's": `'it'\''s'`, + } + for in, want := range cases { + if got := escapeInputText(in); got != want { + t.Errorf("escapeInputText(%q) = %q, want %q", in, got, want) + } + } + // A single quote must not be able to break out of the quoting. + got := escapeInputText("a'; rm -rf /; 'b") + if strings.Count(got, "'")%2 != 0 { + t.Fatalf("unbalanced quotes in %q", got) + } +}