Running spotify_play against the real phone showed a tap can be delivered to the app and then never acted on (the dispatcher lists the Spotify window as the touch target, and the screen does not move). The run log also showed step durations were printed as 0s while the recorded ones were correct. - wait steps accept "retry N": on timeout, re-run the step above and wait again, up to N extra times. Only a wait may use it, it needs a step above it, and it can only press something already matched by text. spotify_play uses it on the search box and on the playback check. - The success log line printed the duration before the deferred assignment ran. It now records the duration first. - uiautomator dump starts a JVM on the phone, and the wait helpers polled every 400ms. ui.PollInterval is now 1500ms, which cuts that load on a phone that is already short of memory. docs/phase-3-spotify-recipe.md logs attempts 7-10: the recipe now reaches tap_row and picks "Honky Tonk Women" at (540,878), but the app stops acting on input on this loaded phone, so the end-to-end run is still outstanding.
426 lines
12 KiB
Go
426 lines
12 KiB
Go
package recipe
|
|
|
|
import (
|
|
"fmt"
|
|
"io"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.lab.audasmedia.com.au/sam/artemis_control/internal/adb"
|
|
"gitea.lab.audasmedia.com.au/sam/artemis_control/internal/ui"
|
|
)
|
|
|
|
// StepResult is the record of one executed step (FR-10).
|
|
type StepResult struct {
|
|
Index int `json:"step"`
|
|
Verb string `json:"verb"`
|
|
Args []string `json:"args,omitempty"`
|
|
Detail string `json:"detail,omitempty"`
|
|
OK bool `json:"ok"`
|
|
Error string `json:"error,omitempty"`
|
|
DurationMS int64 `json:"duration_ms"`
|
|
Duration time.Duration `json:"-"`
|
|
}
|
|
|
|
// RunResult is the record of one whole recipe run (FR-10).
|
|
type RunResult struct {
|
|
Task string `json:"task"`
|
|
Params map[string]string `json:"params,omitempty"`
|
|
OK bool `json:"ok"`
|
|
Error string `json:"error,omitempty"`
|
|
Steps []StepResult `json:"steps"`
|
|
DurationMS int64 `json:"duration_ms"`
|
|
Duration time.Duration `json:"-"`
|
|
}
|
|
|
|
// Options carry the caller's parameters and where to write the step log.
|
|
type Options struct {
|
|
Params map[string]string
|
|
Log io.Writer
|
|
}
|
|
|
|
// Run executes a recipe against the phone.
|
|
//
|
|
// It refuses to start on a locked phone or a missing app, bounds every step
|
|
// (FR-11), and reports the reason for the first failing step. A run never
|
|
// returns an error silently: the returned RunResult always says what happened.
|
|
func Run(r *adb.Runner, rec *Recipe, opts Options) *RunResult {
|
|
started := time.Now()
|
|
params := opts.Params
|
|
if params == nil {
|
|
params = map[string]string{}
|
|
}
|
|
res := &RunResult{Task: rec.Task, Params: params}
|
|
defer func() {
|
|
res.Duration = time.Since(started)
|
|
res.DurationMS = res.Duration.Milliseconds()
|
|
}()
|
|
|
|
logf := func(format string, args ...any) {
|
|
if opts.Log != nil {
|
|
fmt.Fprintf(opts.Log, format+"\n", args...)
|
|
}
|
|
}
|
|
|
|
fail := func(err error) *RunResult {
|
|
res.OK = false
|
|
res.Error = err.Error()
|
|
logf("FAILED %s after %s: %s", rec.Task, time.Since(started).Round(time.Millisecond), res.Error)
|
|
return res
|
|
}
|
|
|
|
logf("task %s (%s)", rec.Task, rec.Description)
|
|
|
|
if missing := rec.MissingParams(params); len(missing) > 0 {
|
|
return fail(fmt.Errorf("missing required parameter(s): %s", strings.Join(missing, ", ")))
|
|
}
|
|
if rec.App != "" {
|
|
installed, err := ui.AppInstalled(r, rec.App)
|
|
if err != nil {
|
|
return fail(fmt.Errorf("could not check whether %s is installed: %w", rec.App, err))
|
|
}
|
|
if !installed {
|
|
return fail(fmt.Errorf("app is not installed: %s", rec.App))
|
|
}
|
|
}
|
|
|
|
if err := ui.Wake(r); err != nil {
|
|
return fail(err)
|
|
}
|
|
// A secure keyguard is a hard stop: Artemis has no credential and must
|
|
// never type one (AGENTS.md section 13 Q2).
|
|
if err := ui.RequireUnlocked(r); err != nil {
|
|
return fail(err)
|
|
}
|
|
|
|
w, h, _ := ui.ScreenSize(r) // best effort: bounds checking degrades gracefully
|
|
|
|
deadline := started.Add(rec.TotalTimeout)
|
|
for i := 0; i < len(rec.Steps); i++ {
|
|
step := rec.Steps[i]
|
|
var sr StepResult
|
|
for attempt := 0; ; attempt++ {
|
|
if remaining := time.Until(deadline); remaining <= 0 {
|
|
return fail(fmt.Errorf("recipe timed out after %s at step %d (%s)", rec.TotalTimeout, i+1, step.Verb))
|
|
}
|
|
sr = executeStep(r, rec, step, i, params, w, h, deadline, logf)
|
|
res.Steps = append(res.Steps, sr)
|
|
if sr.OK {
|
|
break
|
|
}
|
|
// A wait step with "retry N" presses the step above again: on a
|
|
// busy phone a tap can be delivered but never acted on, or dropped.
|
|
if attempt >= step.Retry {
|
|
return fail(fmt.Errorf("step %d (%s %s) failed: %s", i+1, step.Verb, strings.Join(step.Args, " "), sr.Error))
|
|
}
|
|
if i == 0 {
|
|
return fail(fmt.Errorf("step %d: retry needs a step above it to repeat", i+1))
|
|
}
|
|
logf(" step %d %s: retrying, pressing %s again", i+1, step.Verb, rec.Steps[i-1].Verb)
|
|
again := executeStep(r, rec, rec.Steps[i-1], i-1, params, w, h, deadline, logf)
|
|
res.Steps = append(res.Steps, again)
|
|
if !again.OK {
|
|
// The wait is the real failure; the repeat just could not run.
|
|
return fail(fmt.Errorf("step %d (%s %s) failed: %s; repeating %s failed too: %s",
|
|
i+1, step.Verb, strings.Join(step.Args, " "), sr.Error, rec.Steps[i-1].Verb, again.Error))
|
|
}
|
|
}
|
|
}
|
|
|
|
res.OK = true
|
|
logf("OK %s in %s", rec.Task, time.Since(started).Round(time.Millisecond))
|
|
return res
|
|
}
|
|
|
|
func executeStep(r *adb.Runner, rec *Recipe, step Step, index int, params map[string]string,
|
|
w, h int, deadline time.Time, logf func(string, ...any)) (sr StepResult) {
|
|
|
|
started := time.Now()
|
|
sr = StepResult{Index: index + 1, Verb: step.Verb, Args: step.Args}
|
|
defer func() {
|
|
sr.Duration = time.Since(started)
|
|
sr.DurationMS = sr.Duration.Milliseconds()
|
|
}()
|
|
|
|
timeout := step.Timeout
|
|
if timeout <= 0 {
|
|
timeout = rec.StepTimeout
|
|
}
|
|
if remaining := time.Until(deadline); timeout > remaining {
|
|
timeout = remaining
|
|
}
|
|
if timeout <= 0 {
|
|
sr.Error = "no time left in the recipe budget"
|
|
return sr
|
|
}
|
|
|
|
args := make([]string, len(step.Args))
|
|
for i, a := range step.Args {
|
|
v, err := Substitute(a, params)
|
|
if err != nil {
|
|
sr.Error = err.Error()
|
|
return sr
|
|
}
|
|
args[i] = v
|
|
}
|
|
|
|
stepErr := func() error {
|
|
switch step.Verb {
|
|
case "launch":
|
|
pkg := rec.App
|
|
if len(args) == 1 {
|
|
pkg = args[0]
|
|
}
|
|
if pkg == "" {
|
|
return fmt.Errorf("no app named: add an \"app:\" setting or pass a package")
|
|
}
|
|
if err := ui.Launch(r, pkg); err != nil {
|
|
return err
|
|
}
|
|
sr.Detail = "launched " + pkg
|
|
return nil
|
|
|
|
case "foreground":
|
|
// Confirm the recipe's app is really in front. Without this, a text
|
|
// like "Search" can match whatever app happened to be open.
|
|
pkg := rec.App
|
|
if len(args) == 1 {
|
|
pkg = args[0]
|
|
}
|
|
if pkg == "" {
|
|
return fmt.Errorf("foreground needs an app: add an \"app:\" setting or pass a package")
|
|
}
|
|
fgDeadline := time.Now().Add(timeout)
|
|
for {
|
|
fg, ferr := ui.ForegroundPackage(r)
|
|
if ferr == nil && fg == pkg {
|
|
sr.Detail = "foreground is " + pkg
|
|
return nil
|
|
}
|
|
if time.Now().After(fgDeadline) {
|
|
got := "unknown"
|
|
if ferr == nil {
|
|
got = fg
|
|
}
|
|
return fmt.Errorf("%s never came to the foreground within %s (foreground=%s)", pkg, timeout, got)
|
|
}
|
|
time.Sleep(ui.PollInterval)
|
|
}
|
|
|
|
case "wait":
|
|
node, err := ui.WaitForText(r, args[0], step.Exact, timeout)
|
|
if err != nil {
|
|
return describeFailure(r, rec, err)
|
|
}
|
|
sr.Detail = "found " + node.Label()
|
|
return nil
|
|
|
|
case "wait_gone":
|
|
if err := ui.WaitForNoText(r, args[0], step.Exact, timeout); err != nil {
|
|
return describeFailure(r, rec, err)
|
|
}
|
|
sr.Detail = fmt.Sprintf("%q is gone", args[0])
|
|
return nil
|
|
|
|
case "tap":
|
|
return tapByText(r, rec, args[0], step.Exact, timeout, &sr)
|
|
|
|
case "tap_if":
|
|
// Tap only if the text is already on screen. This makes a recipe
|
|
// tolerant of a screen that remembers state, without ever tapping
|
|
// something that was not matched by text.
|
|
h, err := ui.Dump(r)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
matches := h.FindText(args[0], step.Exact)
|
|
if len(matches) == 0 {
|
|
sr.Detail = fmt.Sprintf("%q not on screen, skipped", args[0])
|
|
return nil
|
|
}
|
|
node, err := ui.TapNode(r, matches)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
cx, cy, _ := node.Center()
|
|
sr.Detail = fmt.Sprintf("tapped %q at (%d,%d)", args[0], cx, cy)
|
|
return nil
|
|
|
|
case "tap_row":
|
|
return tapRow(r, rec, args[0], timeout, w, h, &sr)
|
|
|
|
case "type":
|
|
if err := ui.Type(r, args[0]); err != nil {
|
|
return err
|
|
}
|
|
sr.Detail = fmt.Sprintf("typed %d characters", len(args[0]))
|
|
return nil
|
|
|
|
case "key":
|
|
return sendKey(r, args[0], &sr)
|
|
|
|
case "back":
|
|
if err := ui.Back(r); err != nil {
|
|
return err
|
|
}
|
|
sr.Detail = "pressed back"
|
|
return nil
|
|
|
|
case "home":
|
|
if err := ui.Home(r); err != nil {
|
|
return err
|
|
}
|
|
sr.Detail = "pressed home"
|
|
return nil
|
|
|
|
case "sleep":
|
|
time.Sleep(timeout)
|
|
sr.Detail = "slept " + timeout.String()
|
|
return nil
|
|
}
|
|
return fmt.Errorf("unknown step verb %q", step.Verb)
|
|
}()
|
|
|
|
if stepErr != nil {
|
|
sr.OK = false
|
|
sr.Error = stepErr.Error()
|
|
logf(" step %d %s %v: %s", sr.Index, step.Verb, args, sr.Error)
|
|
return sr
|
|
}
|
|
|
|
sr.OK = true
|
|
sr.Duration = time.Since(started)
|
|
sr.DurationMS = sr.Duration.Milliseconds()
|
|
logf(" step %d %s %v: ok (%s)", sr.Index, step.Verb, args, sr.Duration.Round(time.Millisecond))
|
|
return sr
|
|
}
|
|
|
|
// tapByText finds an element by text and taps its centre (FR-4).
|
|
//
|
|
// The element is looked up twice: once to wait for it, and again immediately
|
|
// before the tap, so the coordinates come from the newest dump. On a busy
|
|
// phone the first dump can be seconds old by the time the tap is sent.
|
|
func tapByText(r *adb.Runner, rec *Recipe, query string, exact bool, timeout time.Duration, sr *StepResult) error {
|
|
if _, err := ui.WaitForText(r, query, exact, timeout); err != nil {
|
|
return describeFailure(r, rec, err)
|
|
}
|
|
h, err := ui.Dump(r)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
matches := h.FindText(query, exact)
|
|
if len(matches) == 0 {
|
|
return describeFailure(r, rec, fmt.Errorf("element with text %q disappeared before the tap", query))
|
|
}
|
|
node, err := ui.TapNode(r, matches)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
cx, cy, _ := node.Center()
|
|
sr.Detail = fmt.Sprintf("tapped %q at (%d,%d) %s", query, cx, cy, node.Label())
|
|
return nil
|
|
}
|
|
|
|
// tapRow taps the list row that best matches the query. It retries the dump
|
|
// until a matching row is on screen, then taps a point derived from that row's
|
|
// own bounds.
|
|
func tapRow(r *adb.Runner, rec *Recipe, query string, timeout time.Duration, w, h int, sr *StepResult) error {
|
|
deadline := time.Now().Add(timeout)
|
|
var lastErr error
|
|
for time.Now().Before(deadline) {
|
|
hier, err := ui.Dump(r)
|
|
if err == nil {
|
|
row, rerr := hier.BestRow(query, rec.RowID, w, h)
|
|
if rerr == nil {
|
|
if err := ui.Tap(r, row.Centre.X, row.Centre.Y); err != nil {
|
|
return err
|
|
}
|
|
sr.Detail = fmt.Sprintf("tapped row %q at (%d,%d) from %s", row.Title, row.Centre.X, row.Centre.Y,
|
|
strings.Join(row.Texts, " / "))
|
|
return nil
|
|
}
|
|
lastErr = rerr
|
|
} else {
|
|
lastErr = err
|
|
}
|
|
time.Sleep(ui.PollInterval)
|
|
}
|
|
if lastErr == nil {
|
|
lastErr = fmt.Errorf("no result row matches %q", query)
|
|
}
|
|
return describeFailure(r, rec, lastErr)
|
|
}
|
|
|
|
func sendKey(r *adb.Runner, name string, sr *StepResult) error {
|
|
codes := map[string]int{
|
|
"enter": ui.KeyEnter,
|
|
"back": ui.KeyBack,
|
|
"home": ui.KeyHome,
|
|
"power": 26,
|
|
"wakeup": ui.KeyWakeup,
|
|
"escape": 111,
|
|
}
|
|
code, ok := codes[strings.ToLower(name)]
|
|
if !ok {
|
|
return fmt.Errorf("unknown key %q", name)
|
|
}
|
|
if err := ui.KeyEvent(r, code); err != nil {
|
|
return err
|
|
}
|
|
sr.Detail = "pressed " + strings.ToLower(name)
|
|
return nil
|
|
}
|
|
|
|
// describeFailure turns a step failure into a clean, named failure (FR-8).
|
|
//
|
|
// A guard match wins: "Spotify is signed out" is far more useful than "text
|
|
// Search did not appear". Otherwise the message names the screen that was on
|
|
// the phone when the step gave up.
|
|
func describeFailure(r *adb.Runner, rec *Recipe, err error) error {
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
h, dumpErr := ui.Dump(r)
|
|
if dumpErr == nil {
|
|
for _, g := range rec.Guards {
|
|
if len(h.FindText(g.Text, false)) > 0 {
|
|
return fmt.Errorf("%s (phone is showing %q): %w", g.Reason, g.Text, err)
|
|
}
|
|
}
|
|
}
|
|
screen := describeScreen(r, h, dumpErr)
|
|
if screen == "" {
|
|
return err
|
|
}
|
|
return fmt.Errorf("%w; %s", err, screen)
|
|
}
|
|
|
|
// describeScreen names the screen Artemis gave up on: the foreground package
|
|
// plus the first visible lines of text.
|
|
func describeScreen(r *adb.Runner, h *ui.Hierarchy, dumpErr error) string {
|
|
var parts []string
|
|
if fg, err := ui.ForegroundPackage(r); err == nil {
|
|
parts = append(parts, "foreground="+fg)
|
|
}
|
|
if dumpErr == nil && h != nil {
|
|
var labels []string
|
|
for _, n := range h.TextNodes() {
|
|
label := strings.TrimSpace(n.Text)
|
|
if label == "" {
|
|
label = strings.TrimSpace(n.ContentDesc)
|
|
}
|
|
if label == "" || len(label) > 60 {
|
|
continue
|
|
}
|
|
labels = append(labels, fmt.Sprintf("%q", label))
|
|
if len(labels) == 6 {
|
|
break
|
|
}
|
|
}
|
|
if len(labels) > 0 {
|
|
parts = append(parts, "screen shows: "+strings.Join(labels, ", "))
|
|
}
|
|
}
|
|
return strings.Join(parts, "; ")
|
|
}
|