Files
artemis_control/internal/recipe/engine.go
Sam Rolfe c178f92851 feat: confirm playback from the media session, and stop demoting the top row
Phase 3 DoD reached end to end: `artemis run spotify_play query="Rolling
Stones Honky Tonk"` played the right song in 13 steps / 35.3s, with playback
paused first so the pass proves the row tap started it.

Two things stopped it before that, both found on the real phone:

- The playback check asserted on an on-screen "Pause" label that Spotify
  never shows: its transport button keeps the label "Play" while music is
  playing. The recipe now ends with a `playing` step that reads the app's
  media session (state PLAYING) and confirms the track title contains the
  query words. A failure reports the state and the track it found.
- tap_row pressed the wrong row. Once a track is loaded its row carries a
  `Paused` state indicator, and counting that content description as row text
  dropped the top hit below an identical duplicate further down, so the tap
  went to y=1443 instead of y=878. Rows are now scored on their title and
  subtitle only (state lives in content descriptions), and scores within
  rowScoreTolerance are treated as equal so the higher row wins, because
  Spotify already ranks by relevance.

internal/ui/rows_test.go reproduces that results screen, including the state
indicator and the duplicate row.

docs/phase-3-spotify-recipe.md records the successful run, the independent
media-session and audio-output reads, and the failure that preceded it.
2026-10-16 14:32:24 +11:00

458 lines
13 KiB
Go

package recipe
import (
"fmt"
"io"
"strings"
"time"
"gitea.lab.audasmedia.com.au/sam/artemis_control/internal/adb"
"gitea.lab.audasmedia.com.au/sam/artemis_control/internal/ui"
)
// StepResult is the record of one executed step (FR-10).
type StepResult struct {
Index int `json:"step"`
Verb string `json:"verb"`
Args []string `json:"args,omitempty"`
Detail string `json:"detail,omitempty"`
OK bool `json:"ok"`
Error string `json:"error,omitempty"`
DurationMS int64 `json:"duration_ms"`
Duration time.Duration `json:"-"`
}
// RunResult is the record of one whole recipe run (FR-10).
type RunResult struct {
Task string `json:"task"`
Params map[string]string `json:"params,omitempty"`
OK bool `json:"ok"`
Error string `json:"error,omitempty"`
Steps []StepResult `json:"steps"`
DurationMS int64 `json:"duration_ms"`
Duration time.Duration `json:"-"`
}
// Options carry the caller's parameters and where to write the step log.
type Options struct {
Params map[string]string
Log io.Writer
}
// Run executes a recipe against the phone.
//
// It refuses to start on a locked phone or a missing app, bounds every step
// (FR-11), and reports the reason for the first failing step. A run never
// returns an error silently: the returned RunResult always says what happened.
func Run(r *adb.Runner, rec *Recipe, opts Options) *RunResult {
started := time.Now()
params := opts.Params
if params == nil {
params = map[string]string{}
}
res := &RunResult{Task: rec.Task, Params: params}
defer func() {
res.Duration = time.Since(started)
res.DurationMS = res.Duration.Milliseconds()
}()
logf := func(format string, args ...any) {
if opts.Log != nil {
fmt.Fprintf(opts.Log, format+"\n", args...)
}
}
fail := func(err error) *RunResult {
res.OK = false
res.Error = err.Error()
logf("FAILED %s after %s: %s", rec.Task, time.Since(started).Round(time.Millisecond), res.Error)
return res
}
logf("task %s (%s)", rec.Task, rec.Description)
if missing := rec.MissingParams(params); len(missing) > 0 {
return fail(fmt.Errorf("missing required parameter(s): %s", strings.Join(missing, ", ")))
}
if rec.App != "" {
installed, err := ui.AppInstalled(r, rec.App)
if err != nil {
return fail(fmt.Errorf("could not check whether %s is installed: %w", rec.App, err))
}
if !installed {
return fail(fmt.Errorf("app is not installed: %s", rec.App))
}
}
if err := ui.Wake(r); err != nil {
return fail(err)
}
// A secure keyguard is a hard stop: Artemis has no credential and must
// never type one (AGENTS.md section 13 Q2).
if err := ui.RequireUnlocked(r); err != nil {
return fail(err)
}
w, h, _ := ui.ScreenSize(r) // best effort: bounds checking degrades gracefully
deadline := started.Add(rec.TotalTimeout)
for i := 0; i < len(rec.Steps); i++ {
step := rec.Steps[i]
var sr StepResult
for attempt := 0; ; attempt++ {
if remaining := time.Until(deadline); remaining <= 0 {
return fail(fmt.Errorf("recipe timed out after %s at step %d (%s)", rec.TotalTimeout, i+1, step.Verb))
}
sr = executeStep(r, rec, step, i, params, w, h, deadline, logf)
res.Steps = append(res.Steps, sr)
if sr.OK {
break
}
// A wait step with "retry N" presses the step above again: on a
// busy phone a tap can be delivered but never acted on, or dropped.
if attempt >= step.Retry {
return fail(fmt.Errorf("step %d (%s %s) failed: %s", i+1, step.Verb, strings.Join(step.Args, " "), sr.Error))
}
if i == 0 {
return fail(fmt.Errorf("step %d: retry needs a step above it to repeat", i+1))
}
logf(" step %d %s: retrying, pressing %s again", i+1, step.Verb, rec.Steps[i-1].Verb)
again := executeStep(r, rec, rec.Steps[i-1], i-1, params, w, h, deadline, logf)
res.Steps = append(res.Steps, again)
if !again.OK {
// The wait is the real failure; the repeat just could not run.
return fail(fmt.Errorf("step %d (%s %s) failed: %s; repeating %s failed too: %s",
i+1, step.Verb, strings.Join(step.Args, " "), sr.Error, rec.Steps[i-1].Verb, again.Error))
}
}
}
res.OK = true
logf("OK %s in %s", rec.Task, time.Since(started).Round(time.Millisecond))
return res
}
func executeStep(r *adb.Runner, rec *Recipe, step Step, index int, params map[string]string,
w, h int, deadline time.Time, logf func(string, ...any)) (sr StepResult) {
started := time.Now()
sr = StepResult{Index: index + 1, Verb: step.Verb, Args: step.Args}
defer func() {
sr.Duration = time.Since(started)
sr.DurationMS = sr.Duration.Milliseconds()
}()
timeout := step.Timeout
if timeout <= 0 {
timeout = rec.StepTimeout
}
if remaining := time.Until(deadline); timeout > remaining {
timeout = remaining
}
if timeout <= 0 {
sr.Error = "no time left in the recipe budget"
return sr
}
args := make([]string, len(step.Args))
for i, a := range step.Args {
v, err := Substitute(a, params)
if err != nil {
sr.Error = err.Error()
return sr
}
args[i] = v
}
stepErr := func() error {
switch step.Verb {
case "launch":
pkg := rec.App
if len(args) == 1 {
pkg = args[0]
}
if pkg == "" {
return fmt.Errorf("no app named: add an \"app:\" setting or pass a package")
}
if err := ui.Launch(r, pkg); err != nil {
return err
}
sr.Detail = "launched " + pkg
return nil
case "foreground":
// Confirm the recipe's app is really in front. Without this, a text
// like "Search" can match whatever app happened to be open.
pkg := rec.App
if len(args) == 1 {
pkg = args[0]
}
if pkg == "" {
return fmt.Errorf("foreground needs an app: add an \"app:\" setting or pass a package")
}
fgDeadline := time.Now().Add(timeout)
for {
fg, ferr := ui.ForegroundPackage(r)
if ferr == nil && fg == pkg {
sr.Detail = "foreground is " + pkg
return nil
}
if time.Now().After(fgDeadline) {
got := "unknown"
if ferr == nil {
got = fg
}
return fmt.Errorf("%s never came to the foreground within %s (foreground=%s)", pkg, timeout, got)
}
time.Sleep(ui.PollInterval)
}
case "wait":
node, err := ui.WaitForText(r, args[0], step.Exact, timeout)
if err != nil {
return describeFailure(r, rec, err)
}
sr.Detail = "found " + node.Label()
return nil
case "playing":
// The screen is not a trustworthy place to read playback: Spotify's
// transport button keeps the label "Play" while music is playing.
// Ask the media session instead, and confirm the track title.
pkg := rec.App
if len(args) == 2 {
pkg = args[1]
}
if pkg == "" {
return fmt.Errorf("playing needs an app: add an \"app:\" setting")
}
checkDeadline := time.Now().Add(timeout)
var last *ui.Playback
for {
state, err := ui.PlaybackState(r, pkg)
if err != nil {
return err
}
last = state
if state.Playing() && state.Title != "" {
matched, total := ui.MatchesText(state.Title, args[0])
if total > 0 && matched*2 >= total {
sr.Detail = fmt.Sprintf("%s is playing %q", pkg, state.Title)
return nil
}
}
if time.Now().After(checkDeadline) {
return fmt.Errorf("%s is not playing %q within %s (%s)", pkg, args[0], timeout, last)
}
time.Sleep(ui.PollInterval)
}
case "wait_gone":
if err := ui.WaitForNoText(r, args[0], step.Exact, timeout); err != nil {
return describeFailure(r, rec, err)
}
sr.Detail = fmt.Sprintf("%q is gone", args[0])
return nil
case "tap":
return tapByText(r, rec, args[0], step.Exact, timeout, &sr)
case "tap_if":
// Tap only if the text is already on screen. This makes a recipe
// tolerant of a screen that remembers state, without ever tapping
// something that was not matched by text.
h, err := ui.Dump(r)
if err != nil {
return err
}
matches := h.FindText(args[0], step.Exact)
if len(matches) == 0 {
sr.Detail = fmt.Sprintf("%q not on screen, skipped", args[0])
return nil
}
node, err := ui.TapNode(r, matches)
if err != nil {
return err
}
cx, cy, _ := node.Center()
sr.Detail = fmt.Sprintf("tapped %q at (%d,%d)", args[0], cx, cy)
return nil
case "tap_row":
return tapRow(r, rec, args[0], timeout, w, h, &sr)
case "type":
if err := ui.Type(r, args[0]); err != nil {
return err
}
sr.Detail = fmt.Sprintf("typed %d characters", len(args[0]))
return nil
case "key":
return sendKey(r, args[0], &sr)
case "back":
if err := ui.Back(r); err != nil {
return err
}
sr.Detail = "pressed back"
return nil
case "home":
if err := ui.Home(r); err != nil {
return err
}
sr.Detail = "pressed home"
return nil
case "sleep":
time.Sleep(timeout)
sr.Detail = "slept " + timeout.String()
return nil
}
return fmt.Errorf("unknown step verb %q", step.Verb)
}()
if stepErr != nil {
sr.OK = false
sr.Error = stepErr.Error()
logf(" step %d %s %v: %s", sr.Index, step.Verb, args, sr.Error)
return sr
}
sr.OK = true
sr.Duration = time.Since(started)
sr.DurationMS = sr.Duration.Milliseconds()
logf(" step %d %s %v: ok (%s)", sr.Index, step.Verb, args, sr.Duration.Round(time.Millisecond))
return sr
}
// tapByText finds an element by text and taps its centre (FR-4).
//
// The element is looked up twice: once to wait for it, and again immediately
// before the tap, so the coordinates come from the newest dump. On a busy
// phone the first dump can be seconds old by the time the tap is sent.
func tapByText(r *adb.Runner, rec *Recipe, query string, exact bool, timeout time.Duration, sr *StepResult) error {
if _, err := ui.WaitForText(r, query, exact, timeout); err != nil {
return describeFailure(r, rec, err)
}
h, err := ui.Dump(r)
if err != nil {
return err
}
matches := h.FindText(query, exact)
if len(matches) == 0 {
return describeFailure(r, rec, fmt.Errorf("element with text %q disappeared before the tap", query))
}
node, err := ui.TapNode(r, matches)
if err != nil {
return err
}
cx, cy, _ := node.Center()
sr.Detail = fmt.Sprintf("tapped %q at (%d,%d) %s", query, cx, cy, node.Label())
return nil
}
// tapRow taps the list row that best matches the query. It retries the dump
// until a matching row is on screen, then taps a point derived from that row's
// own bounds.
func tapRow(r *adb.Runner, rec *Recipe, query string, timeout time.Duration, w, h int, sr *StepResult) error {
deadline := time.Now().Add(timeout)
var lastErr error
for time.Now().Before(deadline) {
hier, err := ui.Dump(r)
if err == nil {
row, rerr := hier.BestRow(query, rec.RowID, w, h)
if rerr == nil {
if err := ui.Tap(r, row.Centre.X, row.Centre.Y); err != nil {
return err
}
sr.Detail = fmt.Sprintf("tapped row %q at (%d,%d) from %s", row.Title, row.Centre.X, row.Centre.Y,
strings.Join(row.Texts, " / "))
return nil
}
lastErr = rerr
} else {
lastErr = err
}
time.Sleep(ui.PollInterval)
}
if lastErr == nil {
lastErr = fmt.Errorf("no result row matches %q", query)
}
return describeFailure(r, rec, lastErr)
}
func sendKey(r *adb.Runner, name string, sr *StepResult) error {
codes := map[string]int{
"enter": ui.KeyEnter,
"back": ui.KeyBack,
"home": ui.KeyHome,
"power": 26,
"wakeup": ui.KeyWakeup,
"escape": 111,
}
code, ok := codes[strings.ToLower(name)]
if !ok {
return fmt.Errorf("unknown key %q", name)
}
if err := ui.KeyEvent(r, code); err != nil {
return err
}
sr.Detail = "pressed " + strings.ToLower(name)
return nil
}
// describeFailure turns a step failure into a clean, named failure (FR-8).
//
// A guard match wins: "Spotify is signed out" is far more useful than "text
// Search did not appear". Otherwise the message names the screen that was on
// the phone when the step gave up.
func describeFailure(r *adb.Runner, rec *Recipe, err error) error {
if err == nil {
return nil
}
h, dumpErr := ui.Dump(r)
if dumpErr == nil {
for _, g := range rec.Guards {
if len(h.FindText(g.Text, false)) > 0 {
return fmt.Errorf("%s (phone is showing %q): %w", g.Reason, g.Text, err)
}
}
}
screen := describeScreen(r, h, dumpErr)
if screen == "" {
return err
}
return fmt.Errorf("%w; %s", err, screen)
}
// describeScreen names the screen Artemis gave up on: the foreground package
// plus the first visible lines of text.
func describeScreen(r *adb.Runner, h *ui.Hierarchy, dumpErr error) string {
var parts []string
if fg, err := ui.ForegroundPackage(r); err == nil {
parts = append(parts, "foreground="+fg)
}
if dumpErr == nil && h != nil {
var labels []string
for _, n := range h.TextNodes() {
label := strings.TrimSpace(n.Text)
if label == "" {
label = strings.TrimSpace(n.ContentDesc)
}
if label == "" || len(label) > 60 {
continue
}
labels = append(labels, fmt.Sprintf("%q", label))
if len(labels) == 6 {
break
}
}
if len(labels) > 0 {
parts = append(parts, "screen shows: "+strings.Join(labels, ", "))
}
}
return strings.Join(parts, "; ")
}