Files
artemis_control/internal/recipe/engine.go
Sam Rolfe ab69f37aa4 feat: bounded retry for a dropped tap, and poll the screen more gently
Running spotify_play against the real phone showed a tap can be delivered
to the app and then never acted on (the dispatcher lists the Spotify window
as the touch target, and the screen does not move). The run log also showed
step durations were printed as 0s while the recorded ones were correct.

- wait steps accept "retry N": on timeout, re-run the step above and wait
  again, up to N extra times. Only a wait may use it, it needs a step above
  it, and it can only press something already matched by text. spotify_play
  uses it on the search box and on the playback check.
- The success log line printed the duration before the deferred assignment
  ran. It now records the duration first.
- uiautomator dump starts a JVM on the phone, and the wait helpers polled
  every 400ms. ui.PollInterval is now 1500ms, which cuts that load on a
  phone that is already short of memory.

docs/phase-3-spotify-recipe.md logs attempts 7-10: the recipe now reaches
tap_row and picks "Honky Tonk Women" at (540,878), but the app stops acting
on input on this loaded phone, so the end-to-end run is still outstanding.
2026-10-16 14:12:01 +11:00

426 lines
12 KiB
Go

package recipe
import (
"fmt"
"io"
"strings"
"time"
"gitea.lab.audasmedia.com.au/sam/artemis_control/internal/adb"
"gitea.lab.audasmedia.com.au/sam/artemis_control/internal/ui"
)
// StepResult is the record of one executed step (FR-10).
type StepResult struct {
Index int `json:"step"`
Verb string `json:"verb"`
Args []string `json:"args,omitempty"`
Detail string `json:"detail,omitempty"`
OK bool `json:"ok"`
Error string `json:"error,omitempty"`
DurationMS int64 `json:"duration_ms"`
Duration time.Duration `json:"-"`
}
// RunResult is the record of one whole recipe run (FR-10).
type RunResult struct {
Task string `json:"task"`
Params map[string]string `json:"params,omitempty"`
OK bool `json:"ok"`
Error string `json:"error,omitempty"`
Steps []StepResult `json:"steps"`
DurationMS int64 `json:"duration_ms"`
Duration time.Duration `json:"-"`
}
// Options carry the caller's parameters and where to write the step log.
type Options struct {
Params map[string]string
Log io.Writer
}
// Run executes a recipe against the phone.
//
// It refuses to start on a locked phone or a missing app, bounds every step
// (FR-11), and reports the reason for the first failing step. A run never
// returns an error silently: the returned RunResult always says what happened.
func Run(r *adb.Runner, rec *Recipe, opts Options) *RunResult {
started := time.Now()
params := opts.Params
if params == nil {
params = map[string]string{}
}
res := &RunResult{Task: rec.Task, Params: params}
defer func() {
res.Duration = time.Since(started)
res.DurationMS = res.Duration.Milliseconds()
}()
logf := func(format string, args ...any) {
if opts.Log != nil {
fmt.Fprintf(opts.Log, format+"\n", args...)
}
}
fail := func(err error) *RunResult {
res.OK = false
res.Error = err.Error()
logf("FAILED %s after %s: %s", rec.Task, time.Since(started).Round(time.Millisecond), res.Error)
return res
}
logf("task %s (%s)", rec.Task, rec.Description)
if missing := rec.MissingParams(params); len(missing) > 0 {
return fail(fmt.Errorf("missing required parameter(s): %s", strings.Join(missing, ", ")))
}
if rec.App != "" {
installed, err := ui.AppInstalled(r, rec.App)
if err != nil {
return fail(fmt.Errorf("could not check whether %s is installed: %w", rec.App, err))
}
if !installed {
return fail(fmt.Errorf("app is not installed: %s", rec.App))
}
}
if err := ui.Wake(r); err != nil {
return fail(err)
}
// A secure keyguard is a hard stop: Artemis has no credential and must
// never type one (AGENTS.md section 13 Q2).
if err := ui.RequireUnlocked(r); err != nil {
return fail(err)
}
w, h, _ := ui.ScreenSize(r) // best effort: bounds checking degrades gracefully
deadline := started.Add(rec.TotalTimeout)
for i := 0; i < len(rec.Steps); i++ {
step := rec.Steps[i]
var sr StepResult
for attempt := 0; ; attempt++ {
if remaining := time.Until(deadline); remaining <= 0 {
return fail(fmt.Errorf("recipe timed out after %s at step %d (%s)", rec.TotalTimeout, i+1, step.Verb))
}
sr = executeStep(r, rec, step, i, params, w, h, deadline, logf)
res.Steps = append(res.Steps, sr)
if sr.OK {
break
}
// A wait step with "retry N" presses the step above again: on a
// busy phone a tap can be delivered but never acted on, or dropped.
if attempt >= step.Retry {
return fail(fmt.Errorf("step %d (%s %s) failed: %s", i+1, step.Verb, strings.Join(step.Args, " "), sr.Error))
}
if i == 0 {
return fail(fmt.Errorf("step %d: retry needs a step above it to repeat", i+1))
}
logf(" step %d %s: retrying, pressing %s again", i+1, step.Verb, rec.Steps[i-1].Verb)
again := executeStep(r, rec, rec.Steps[i-1], i-1, params, w, h, deadline, logf)
res.Steps = append(res.Steps, again)
if !again.OK {
// The wait is the real failure; the repeat just could not run.
return fail(fmt.Errorf("step %d (%s %s) failed: %s; repeating %s failed too: %s",
i+1, step.Verb, strings.Join(step.Args, " "), sr.Error, rec.Steps[i-1].Verb, again.Error))
}
}
}
res.OK = true
logf("OK %s in %s", rec.Task, time.Since(started).Round(time.Millisecond))
return res
}
func executeStep(r *adb.Runner, rec *Recipe, step Step, index int, params map[string]string,
w, h int, deadline time.Time, logf func(string, ...any)) (sr StepResult) {
started := time.Now()
sr = StepResult{Index: index + 1, Verb: step.Verb, Args: step.Args}
defer func() {
sr.Duration = time.Since(started)
sr.DurationMS = sr.Duration.Milliseconds()
}()
timeout := step.Timeout
if timeout <= 0 {
timeout = rec.StepTimeout
}
if remaining := time.Until(deadline); timeout > remaining {
timeout = remaining
}
if timeout <= 0 {
sr.Error = "no time left in the recipe budget"
return sr
}
args := make([]string, len(step.Args))
for i, a := range step.Args {
v, err := Substitute(a, params)
if err != nil {
sr.Error = err.Error()
return sr
}
args[i] = v
}
stepErr := func() error {
switch step.Verb {
case "launch":
pkg := rec.App
if len(args) == 1 {
pkg = args[0]
}
if pkg == "" {
return fmt.Errorf("no app named: add an \"app:\" setting or pass a package")
}
if err := ui.Launch(r, pkg); err != nil {
return err
}
sr.Detail = "launched " + pkg
return nil
case "foreground":
// Confirm the recipe's app is really in front. Without this, a text
// like "Search" can match whatever app happened to be open.
pkg := rec.App
if len(args) == 1 {
pkg = args[0]
}
if pkg == "" {
return fmt.Errorf("foreground needs an app: add an \"app:\" setting or pass a package")
}
fgDeadline := time.Now().Add(timeout)
for {
fg, ferr := ui.ForegroundPackage(r)
if ferr == nil && fg == pkg {
sr.Detail = "foreground is " + pkg
return nil
}
if time.Now().After(fgDeadline) {
got := "unknown"
if ferr == nil {
got = fg
}
return fmt.Errorf("%s never came to the foreground within %s (foreground=%s)", pkg, timeout, got)
}
time.Sleep(ui.PollInterval)
}
case "wait":
node, err := ui.WaitForText(r, args[0], step.Exact, timeout)
if err != nil {
return describeFailure(r, rec, err)
}
sr.Detail = "found " + node.Label()
return nil
case "wait_gone":
if err := ui.WaitForNoText(r, args[0], step.Exact, timeout); err != nil {
return describeFailure(r, rec, err)
}
sr.Detail = fmt.Sprintf("%q is gone", args[0])
return nil
case "tap":
return tapByText(r, rec, args[0], step.Exact, timeout, &sr)
case "tap_if":
// Tap only if the text is already on screen. This makes a recipe
// tolerant of a screen that remembers state, without ever tapping
// something that was not matched by text.
h, err := ui.Dump(r)
if err != nil {
return err
}
matches := h.FindText(args[0], step.Exact)
if len(matches) == 0 {
sr.Detail = fmt.Sprintf("%q not on screen, skipped", args[0])
return nil
}
node, err := ui.TapNode(r, matches)
if err != nil {
return err
}
cx, cy, _ := node.Center()
sr.Detail = fmt.Sprintf("tapped %q at (%d,%d)", args[0], cx, cy)
return nil
case "tap_row":
return tapRow(r, rec, args[0], timeout, w, h, &sr)
case "type":
if err := ui.Type(r, args[0]); err != nil {
return err
}
sr.Detail = fmt.Sprintf("typed %d characters", len(args[0]))
return nil
case "key":
return sendKey(r, args[0], &sr)
case "back":
if err := ui.Back(r); err != nil {
return err
}
sr.Detail = "pressed back"
return nil
case "home":
if err := ui.Home(r); err != nil {
return err
}
sr.Detail = "pressed home"
return nil
case "sleep":
time.Sleep(timeout)
sr.Detail = "slept " + timeout.String()
return nil
}
return fmt.Errorf("unknown step verb %q", step.Verb)
}()
if stepErr != nil {
sr.OK = false
sr.Error = stepErr.Error()
logf(" step %d %s %v: %s", sr.Index, step.Verb, args, sr.Error)
return sr
}
sr.OK = true
sr.Duration = time.Since(started)
sr.DurationMS = sr.Duration.Milliseconds()
logf(" step %d %s %v: ok (%s)", sr.Index, step.Verb, args, sr.Duration.Round(time.Millisecond))
return sr
}
// tapByText finds an element by text and taps its centre (FR-4).
//
// The element is looked up twice: once to wait for it, and again immediately
// before the tap, so the coordinates come from the newest dump. On a busy
// phone the first dump can be seconds old by the time the tap is sent.
func tapByText(r *adb.Runner, rec *Recipe, query string, exact bool, timeout time.Duration, sr *StepResult) error {
if _, err := ui.WaitForText(r, query, exact, timeout); err != nil {
return describeFailure(r, rec, err)
}
h, err := ui.Dump(r)
if err != nil {
return err
}
matches := h.FindText(query, exact)
if len(matches) == 0 {
return describeFailure(r, rec, fmt.Errorf("element with text %q disappeared before the tap", query))
}
node, err := ui.TapNode(r, matches)
if err != nil {
return err
}
cx, cy, _ := node.Center()
sr.Detail = fmt.Sprintf("tapped %q at (%d,%d) %s", query, cx, cy, node.Label())
return nil
}
// tapRow taps the list row that best matches the query. It retries the dump
// until a matching row is on screen, then taps a point derived from that row's
// own bounds.
func tapRow(r *adb.Runner, rec *Recipe, query string, timeout time.Duration, w, h int, sr *StepResult) error {
deadline := time.Now().Add(timeout)
var lastErr error
for time.Now().Before(deadline) {
hier, err := ui.Dump(r)
if err == nil {
row, rerr := hier.BestRow(query, rec.RowID, w, h)
if rerr == nil {
if err := ui.Tap(r, row.Centre.X, row.Centre.Y); err != nil {
return err
}
sr.Detail = fmt.Sprintf("tapped row %q at (%d,%d) from %s", row.Title, row.Centre.X, row.Centre.Y,
strings.Join(row.Texts, " / "))
return nil
}
lastErr = rerr
} else {
lastErr = err
}
time.Sleep(ui.PollInterval)
}
if lastErr == nil {
lastErr = fmt.Errorf("no result row matches %q", query)
}
return describeFailure(r, rec, lastErr)
}
func sendKey(r *adb.Runner, name string, sr *StepResult) error {
codes := map[string]int{
"enter": ui.KeyEnter,
"back": ui.KeyBack,
"home": ui.KeyHome,
"power": 26,
"wakeup": ui.KeyWakeup,
"escape": 111,
}
code, ok := codes[strings.ToLower(name)]
if !ok {
return fmt.Errorf("unknown key %q", name)
}
if err := ui.KeyEvent(r, code); err != nil {
return err
}
sr.Detail = "pressed " + strings.ToLower(name)
return nil
}
// describeFailure turns a step failure into a clean, named failure (FR-8).
//
// A guard match wins: "Spotify is signed out" is far more useful than "text
// Search did not appear". Otherwise the message names the screen that was on
// the phone when the step gave up.
func describeFailure(r *adb.Runner, rec *Recipe, err error) error {
if err == nil {
return nil
}
h, dumpErr := ui.Dump(r)
if dumpErr == nil {
for _, g := range rec.Guards {
if len(h.FindText(g.Text, false)) > 0 {
return fmt.Errorf("%s (phone is showing %q): %w", g.Reason, g.Text, err)
}
}
}
screen := describeScreen(r, h, dumpErr)
if screen == "" {
return err
}
return fmt.Errorf("%w; %s", err, screen)
}
// describeScreen names the screen Artemis gave up on: the foreground package
// plus the first visible lines of text.
func describeScreen(r *adb.Runner, h *ui.Hierarchy, dumpErr error) string {
var parts []string
if fg, err := ui.ForegroundPackage(r); err == nil {
parts = append(parts, "foreground="+fg)
}
if dumpErr == nil && h != nil {
var labels []string
for _, n := range h.TextNodes() {
label := strings.TrimSpace(n.Text)
if label == "" {
label = strings.TrimSpace(n.ContentDesc)
}
if label == "" || len(label) > 60 {
continue
}
labels = append(labels, fmt.Sprintf("%q", label))
if len(labels) == 6 {
break
}
}
if len(labels) > 0 {
parts = append(parts, "screen shows: "+strings.Join(labels, ", "))
}
}
return strings.Join(parts, "; ")
}