Phase 3 DoD reached end to end: `artemis run spotify_play query="Rolling Stones Honky Tonk"` played the right song in 13 steps / 35.3s, with playback paused first so the pass proves the row tap started it. Two things stopped it before that, both found on the real phone: - The playback check asserted on an on-screen "Pause" label that Spotify never shows: its transport button keeps the label "Play" while music is playing. The recipe now ends with a `playing` step that reads the app's media session (state PLAYING) and confirms the track title contains the query words. A failure reports the state and the track it found. - tap_row pressed the wrong row. Once a track is loaded its row carries a `Paused` state indicator, and counting that content description as row text dropped the top hit below an identical duplicate further down, so the tap went to y=1443 instead of y=878. Rows are now scored on their title and subtitle only (state lives in content descriptions), and scores within rowScoreTolerance are treated as equal so the higher row wins, because Spotify already ranks by relevance. internal/ui/rows_test.go reproduces that results screen, including the state indicator and the duplicate row. docs/phase-3-spotify-recipe.md records the successful run, the independent media-session and audio-output reads, and the failure that preceded it.
458 lines
13 KiB
Go
458 lines
13 KiB
Go
package recipe
|
|
|
|
import (
|
|
"fmt"
|
|
"io"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.lab.audasmedia.com.au/sam/artemis_control/internal/adb"
|
|
"gitea.lab.audasmedia.com.au/sam/artemis_control/internal/ui"
|
|
)
|
|
|
|
// StepResult is the record of one executed step (FR-10).
|
|
type StepResult struct {
|
|
Index int `json:"step"`
|
|
Verb string `json:"verb"`
|
|
Args []string `json:"args,omitempty"`
|
|
Detail string `json:"detail,omitempty"`
|
|
OK bool `json:"ok"`
|
|
Error string `json:"error,omitempty"`
|
|
DurationMS int64 `json:"duration_ms"`
|
|
Duration time.Duration `json:"-"`
|
|
}
|
|
|
|
// RunResult is the record of one whole recipe run (FR-10).
|
|
type RunResult struct {
|
|
Task string `json:"task"`
|
|
Params map[string]string `json:"params,omitempty"`
|
|
OK bool `json:"ok"`
|
|
Error string `json:"error,omitempty"`
|
|
Steps []StepResult `json:"steps"`
|
|
DurationMS int64 `json:"duration_ms"`
|
|
Duration time.Duration `json:"-"`
|
|
}
|
|
|
|
// Options carry the caller's parameters and where to write the step log.
|
|
type Options struct {
|
|
Params map[string]string
|
|
Log io.Writer
|
|
}
|
|
|
|
// Run executes a recipe against the phone.
|
|
//
|
|
// It refuses to start on a locked phone or a missing app, bounds every step
|
|
// (FR-11), and reports the reason for the first failing step. A run never
|
|
// returns an error silently: the returned RunResult always says what happened.
|
|
func Run(r *adb.Runner, rec *Recipe, opts Options) *RunResult {
|
|
started := time.Now()
|
|
params := opts.Params
|
|
if params == nil {
|
|
params = map[string]string{}
|
|
}
|
|
res := &RunResult{Task: rec.Task, Params: params}
|
|
defer func() {
|
|
res.Duration = time.Since(started)
|
|
res.DurationMS = res.Duration.Milliseconds()
|
|
}()
|
|
|
|
logf := func(format string, args ...any) {
|
|
if opts.Log != nil {
|
|
fmt.Fprintf(opts.Log, format+"\n", args...)
|
|
}
|
|
}
|
|
|
|
fail := func(err error) *RunResult {
|
|
res.OK = false
|
|
res.Error = err.Error()
|
|
logf("FAILED %s after %s: %s", rec.Task, time.Since(started).Round(time.Millisecond), res.Error)
|
|
return res
|
|
}
|
|
|
|
logf("task %s (%s)", rec.Task, rec.Description)
|
|
|
|
if missing := rec.MissingParams(params); len(missing) > 0 {
|
|
return fail(fmt.Errorf("missing required parameter(s): %s", strings.Join(missing, ", ")))
|
|
}
|
|
if rec.App != "" {
|
|
installed, err := ui.AppInstalled(r, rec.App)
|
|
if err != nil {
|
|
return fail(fmt.Errorf("could not check whether %s is installed: %w", rec.App, err))
|
|
}
|
|
if !installed {
|
|
return fail(fmt.Errorf("app is not installed: %s", rec.App))
|
|
}
|
|
}
|
|
|
|
if err := ui.Wake(r); err != nil {
|
|
return fail(err)
|
|
}
|
|
// A secure keyguard is a hard stop: Artemis has no credential and must
|
|
// never type one (AGENTS.md section 13 Q2).
|
|
if err := ui.RequireUnlocked(r); err != nil {
|
|
return fail(err)
|
|
}
|
|
|
|
w, h, _ := ui.ScreenSize(r) // best effort: bounds checking degrades gracefully
|
|
|
|
deadline := started.Add(rec.TotalTimeout)
|
|
for i := 0; i < len(rec.Steps); i++ {
|
|
step := rec.Steps[i]
|
|
var sr StepResult
|
|
for attempt := 0; ; attempt++ {
|
|
if remaining := time.Until(deadline); remaining <= 0 {
|
|
return fail(fmt.Errorf("recipe timed out after %s at step %d (%s)", rec.TotalTimeout, i+1, step.Verb))
|
|
}
|
|
sr = executeStep(r, rec, step, i, params, w, h, deadline, logf)
|
|
res.Steps = append(res.Steps, sr)
|
|
if sr.OK {
|
|
break
|
|
}
|
|
// A wait step with "retry N" presses the step above again: on a
|
|
// busy phone a tap can be delivered but never acted on, or dropped.
|
|
if attempt >= step.Retry {
|
|
return fail(fmt.Errorf("step %d (%s %s) failed: %s", i+1, step.Verb, strings.Join(step.Args, " "), sr.Error))
|
|
}
|
|
if i == 0 {
|
|
return fail(fmt.Errorf("step %d: retry needs a step above it to repeat", i+1))
|
|
}
|
|
logf(" step %d %s: retrying, pressing %s again", i+1, step.Verb, rec.Steps[i-1].Verb)
|
|
again := executeStep(r, rec, rec.Steps[i-1], i-1, params, w, h, deadline, logf)
|
|
res.Steps = append(res.Steps, again)
|
|
if !again.OK {
|
|
// The wait is the real failure; the repeat just could not run.
|
|
return fail(fmt.Errorf("step %d (%s %s) failed: %s; repeating %s failed too: %s",
|
|
i+1, step.Verb, strings.Join(step.Args, " "), sr.Error, rec.Steps[i-1].Verb, again.Error))
|
|
}
|
|
}
|
|
}
|
|
|
|
res.OK = true
|
|
logf("OK %s in %s", rec.Task, time.Since(started).Round(time.Millisecond))
|
|
return res
|
|
}
|
|
|
|
func executeStep(r *adb.Runner, rec *Recipe, step Step, index int, params map[string]string,
|
|
w, h int, deadline time.Time, logf func(string, ...any)) (sr StepResult) {
|
|
|
|
started := time.Now()
|
|
sr = StepResult{Index: index + 1, Verb: step.Verb, Args: step.Args}
|
|
defer func() {
|
|
sr.Duration = time.Since(started)
|
|
sr.DurationMS = sr.Duration.Milliseconds()
|
|
}()
|
|
|
|
timeout := step.Timeout
|
|
if timeout <= 0 {
|
|
timeout = rec.StepTimeout
|
|
}
|
|
if remaining := time.Until(deadline); timeout > remaining {
|
|
timeout = remaining
|
|
}
|
|
if timeout <= 0 {
|
|
sr.Error = "no time left in the recipe budget"
|
|
return sr
|
|
}
|
|
|
|
args := make([]string, len(step.Args))
|
|
for i, a := range step.Args {
|
|
v, err := Substitute(a, params)
|
|
if err != nil {
|
|
sr.Error = err.Error()
|
|
return sr
|
|
}
|
|
args[i] = v
|
|
}
|
|
|
|
stepErr := func() error {
|
|
switch step.Verb {
|
|
case "launch":
|
|
pkg := rec.App
|
|
if len(args) == 1 {
|
|
pkg = args[0]
|
|
}
|
|
if pkg == "" {
|
|
return fmt.Errorf("no app named: add an \"app:\" setting or pass a package")
|
|
}
|
|
if err := ui.Launch(r, pkg); err != nil {
|
|
return err
|
|
}
|
|
sr.Detail = "launched " + pkg
|
|
return nil
|
|
|
|
case "foreground":
|
|
// Confirm the recipe's app is really in front. Without this, a text
|
|
// like "Search" can match whatever app happened to be open.
|
|
pkg := rec.App
|
|
if len(args) == 1 {
|
|
pkg = args[0]
|
|
}
|
|
if pkg == "" {
|
|
return fmt.Errorf("foreground needs an app: add an \"app:\" setting or pass a package")
|
|
}
|
|
fgDeadline := time.Now().Add(timeout)
|
|
for {
|
|
fg, ferr := ui.ForegroundPackage(r)
|
|
if ferr == nil && fg == pkg {
|
|
sr.Detail = "foreground is " + pkg
|
|
return nil
|
|
}
|
|
if time.Now().After(fgDeadline) {
|
|
got := "unknown"
|
|
if ferr == nil {
|
|
got = fg
|
|
}
|
|
return fmt.Errorf("%s never came to the foreground within %s (foreground=%s)", pkg, timeout, got)
|
|
}
|
|
time.Sleep(ui.PollInterval)
|
|
}
|
|
|
|
case "wait":
|
|
node, err := ui.WaitForText(r, args[0], step.Exact, timeout)
|
|
if err != nil {
|
|
return describeFailure(r, rec, err)
|
|
}
|
|
sr.Detail = "found " + node.Label()
|
|
return nil
|
|
|
|
case "playing":
|
|
// The screen is not a trustworthy place to read playback: Spotify's
|
|
// transport button keeps the label "Play" while music is playing.
|
|
// Ask the media session instead, and confirm the track title.
|
|
pkg := rec.App
|
|
if len(args) == 2 {
|
|
pkg = args[1]
|
|
}
|
|
if pkg == "" {
|
|
return fmt.Errorf("playing needs an app: add an \"app:\" setting")
|
|
}
|
|
checkDeadline := time.Now().Add(timeout)
|
|
var last *ui.Playback
|
|
for {
|
|
state, err := ui.PlaybackState(r, pkg)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
last = state
|
|
if state.Playing() && state.Title != "" {
|
|
matched, total := ui.MatchesText(state.Title, args[0])
|
|
if total > 0 && matched*2 >= total {
|
|
sr.Detail = fmt.Sprintf("%s is playing %q", pkg, state.Title)
|
|
return nil
|
|
}
|
|
}
|
|
if time.Now().After(checkDeadline) {
|
|
return fmt.Errorf("%s is not playing %q within %s (%s)", pkg, args[0], timeout, last)
|
|
}
|
|
time.Sleep(ui.PollInterval)
|
|
}
|
|
|
|
case "wait_gone":
|
|
if err := ui.WaitForNoText(r, args[0], step.Exact, timeout); err != nil {
|
|
return describeFailure(r, rec, err)
|
|
}
|
|
sr.Detail = fmt.Sprintf("%q is gone", args[0])
|
|
return nil
|
|
|
|
case "tap":
|
|
return tapByText(r, rec, args[0], step.Exact, timeout, &sr)
|
|
|
|
case "tap_if":
|
|
// Tap only if the text is already on screen. This makes a recipe
|
|
// tolerant of a screen that remembers state, without ever tapping
|
|
// something that was not matched by text.
|
|
h, err := ui.Dump(r)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
matches := h.FindText(args[0], step.Exact)
|
|
if len(matches) == 0 {
|
|
sr.Detail = fmt.Sprintf("%q not on screen, skipped", args[0])
|
|
return nil
|
|
}
|
|
node, err := ui.TapNode(r, matches)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
cx, cy, _ := node.Center()
|
|
sr.Detail = fmt.Sprintf("tapped %q at (%d,%d)", args[0], cx, cy)
|
|
return nil
|
|
|
|
case "tap_row":
|
|
return tapRow(r, rec, args[0], timeout, w, h, &sr)
|
|
|
|
case "type":
|
|
if err := ui.Type(r, args[0]); err != nil {
|
|
return err
|
|
}
|
|
sr.Detail = fmt.Sprintf("typed %d characters", len(args[0]))
|
|
return nil
|
|
|
|
case "key":
|
|
return sendKey(r, args[0], &sr)
|
|
|
|
case "back":
|
|
if err := ui.Back(r); err != nil {
|
|
return err
|
|
}
|
|
sr.Detail = "pressed back"
|
|
return nil
|
|
|
|
case "home":
|
|
if err := ui.Home(r); err != nil {
|
|
return err
|
|
}
|
|
sr.Detail = "pressed home"
|
|
return nil
|
|
|
|
case "sleep":
|
|
time.Sleep(timeout)
|
|
sr.Detail = "slept " + timeout.String()
|
|
return nil
|
|
}
|
|
return fmt.Errorf("unknown step verb %q", step.Verb)
|
|
}()
|
|
|
|
if stepErr != nil {
|
|
sr.OK = false
|
|
sr.Error = stepErr.Error()
|
|
logf(" step %d %s %v: %s", sr.Index, step.Verb, args, sr.Error)
|
|
return sr
|
|
}
|
|
|
|
sr.OK = true
|
|
sr.Duration = time.Since(started)
|
|
sr.DurationMS = sr.Duration.Milliseconds()
|
|
logf(" step %d %s %v: ok (%s)", sr.Index, step.Verb, args, sr.Duration.Round(time.Millisecond))
|
|
return sr
|
|
}
|
|
|
|
// tapByText finds an element by text and taps its centre (FR-4).
|
|
//
|
|
// The element is looked up twice: once to wait for it, and again immediately
|
|
// before the tap, so the coordinates come from the newest dump. On a busy
|
|
// phone the first dump can be seconds old by the time the tap is sent.
|
|
func tapByText(r *adb.Runner, rec *Recipe, query string, exact bool, timeout time.Duration, sr *StepResult) error {
|
|
if _, err := ui.WaitForText(r, query, exact, timeout); err != nil {
|
|
return describeFailure(r, rec, err)
|
|
}
|
|
h, err := ui.Dump(r)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
matches := h.FindText(query, exact)
|
|
if len(matches) == 0 {
|
|
return describeFailure(r, rec, fmt.Errorf("element with text %q disappeared before the tap", query))
|
|
}
|
|
node, err := ui.TapNode(r, matches)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
cx, cy, _ := node.Center()
|
|
sr.Detail = fmt.Sprintf("tapped %q at (%d,%d) %s", query, cx, cy, node.Label())
|
|
return nil
|
|
}
|
|
|
|
// tapRow taps the list row that best matches the query. It retries the dump
|
|
// until a matching row is on screen, then taps a point derived from that row's
|
|
// own bounds.
|
|
func tapRow(r *adb.Runner, rec *Recipe, query string, timeout time.Duration, w, h int, sr *StepResult) error {
|
|
deadline := time.Now().Add(timeout)
|
|
var lastErr error
|
|
for time.Now().Before(deadline) {
|
|
hier, err := ui.Dump(r)
|
|
if err == nil {
|
|
row, rerr := hier.BestRow(query, rec.RowID, w, h)
|
|
if rerr == nil {
|
|
if err := ui.Tap(r, row.Centre.X, row.Centre.Y); err != nil {
|
|
return err
|
|
}
|
|
sr.Detail = fmt.Sprintf("tapped row %q at (%d,%d) from %s", row.Title, row.Centre.X, row.Centre.Y,
|
|
strings.Join(row.Texts, " / "))
|
|
return nil
|
|
}
|
|
lastErr = rerr
|
|
} else {
|
|
lastErr = err
|
|
}
|
|
time.Sleep(ui.PollInterval)
|
|
}
|
|
if lastErr == nil {
|
|
lastErr = fmt.Errorf("no result row matches %q", query)
|
|
}
|
|
return describeFailure(r, rec, lastErr)
|
|
}
|
|
|
|
func sendKey(r *adb.Runner, name string, sr *StepResult) error {
|
|
codes := map[string]int{
|
|
"enter": ui.KeyEnter,
|
|
"back": ui.KeyBack,
|
|
"home": ui.KeyHome,
|
|
"power": 26,
|
|
"wakeup": ui.KeyWakeup,
|
|
"escape": 111,
|
|
}
|
|
code, ok := codes[strings.ToLower(name)]
|
|
if !ok {
|
|
return fmt.Errorf("unknown key %q", name)
|
|
}
|
|
if err := ui.KeyEvent(r, code); err != nil {
|
|
return err
|
|
}
|
|
sr.Detail = "pressed " + strings.ToLower(name)
|
|
return nil
|
|
}
|
|
|
|
// describeFailure turns a step failure into a clean, named failure (FR-8).
|
|
//
|
|
// A guard match wins: "Spotify is signed out" is far more useful than "text
|
|
// Search did not appear". Otherwise the message names the screen that was on
|
|
// the phone when the step gave up.
|
|
func describeFailure(r *adb.Runner, rec *Recipe, err error) error {
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
h, dumpErr := ui.Dump(r)
|
|
if dumpErr == nil {
|
|
for _, g := range rec.Guards {
|
|
if len(h.FindText(g.Text, false)) > 0 {
|
|
return fmt.Errorf("%s (phone is showing %q): %w", g.Reason, g.Text, err)
|
|
}
|
|
}
|
|
}
|
|
screen := describeScreen(r, h, dumpErr)
|
|
if screen == "" {
|
|
return err
|
|
}
|
|
return fmt.Errorf("%w; %s", err, screen)
|
|
}
|
|
|
|
// describeScreen names the screen Artemis gave up on: the foreground package
|
|
// plus the first visible lines of text.
|
|
func describeScreen(r *adb.Runner, h *ui.Hierarchy, dumpErr error) string {
|
|
var parts []string
|
|
if fg, err := ui.ForegroundPackage(r); err == nil {
|
|
parts = append(parts, "foreground="+fg)
|
|
}
|
|
if dumpErr == nil && h != nil {
|
|
var labels []string
|
|
for _, n := range h.TextNodes() {
|
|
label := strings.TrimSpace(n.Text)
|
|
if label == "" {
|
|
label = strings.TrimSpace(n.ContentDesc)
|
|
}
|
|
if label == "" || len(label) > 60 {
|
|
continue
|
|
}
|
|
labels = append(labels, fmt.Sprintf("%q", label))
|
|
if len(labels) == 6 {
|
|
break
|
|
}
|
|
}
|
|
if len(labels) > 0 {
|
|
parts = append(parts, "screen shows: "+strings.Join(labels, ", "))
|
|
}
|
|
}
|
|
return strings.Join(parts, "; ")
|
|
}
|