sam/artemis_control
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Phase 4 — spotify_radio, and a bound on logs/runs.jsonl

Browse Source 
spotify_radio opens a track's own context menu and starts its song radio.
The labels came from a fresh uiautomator dump on the phone, not from a guess:
the entry is "Go to song radio" and it sits below the fold, so the recipe
scrolls before it matches.

Recipe format:
- menu_row "{query}" finds the row like tap_row, then presses the action
  button inside that row's own subtree (ui.Row.Action), so the More options
  button of any other row is never a candidate. It records the row as {row}.
- a step can name its own failure: wait ... else "reason". The named reason
  comes first and the screen's labels still follow (FR-8).
- scroll up|down moves the current list.
- playing with no title waits for the media session to be PLAYING. A radio
  changes track every few minutes, so a title match would fail on a working
  radio; spotify_play still passes the title it wants.

ui:
- pickBestMatch ignores a match with no usable bounds. An element scrolled out
  of its list reports [0,0][0,0], and tapping its "centre" would be a blind tap
  in the corner. A wait now only succeeds once the element is on screen.

Run log (FR-10): appendRunLog rotates runs.jsonl when a write would pass
ARTEMIS_LOG_MAX_BYTES (default 1 MiB), keeping ARTEMIS_LOG_KEEP (default 5)
files. History is bounded, never deleted wholesale, and the per-entry cap of
6 screen labels of 60 characters is unchanged.

Live evidence in docs/phase-4-song-radio.md: ok:true in 56.6s, the menu that
opened belonged to the pressed row, the station name "Honky Tonk Women Radio"
on the phone, and the radio playing past its seed track (Layla) per the media
session and dumpsys audio. Both new failure modes were demonstrated failing
with distinct reasons. The mini-bar's transport label stays "Play" while
playing on this build, which is why playback is confirmed from the media
session.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Sam Rolfe
2026-10-16 15:25:50 +11:00
parent 40a8a51c9e
commit 36b4dd2391
11 changed files with 1116 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

127
internal/recipe/engine.go
View File

@@ -46,9 +46,12 @@ type Options struct {
// returns an error silently: the returned RunResult always says what happened.
func Run(r *adb.Runner, rec *Recipe, opts Options) *RunResult {
started := time.Now()
params := opts.Params
if params == nil {
params = map[string]string{}
// The caller's map is copied: a run adds its own values to it (the row a
// tap_row or menu_row step chose, under "row"), and the caller's map must
// not change because of a run.
params := make(map[string]string, len(opts.Params)+1)
for name, value := range opts.Params {
params[name] = value
}
res := &RunResult{Task: rec.Task, Params: params}
defer func() {
@@ -111,7 +114,7 @@ func Run(r *adb.Runner, rec *Recipe, opts Options) *RunResult {
// A wait step with "retry N" presses the step above again: on a
// busy phone a tap can be delivered but never acted on, or dropped.
if attempt >= step.Retry {
return fail(fmt.Errorf("step %d (%s %s) failed: %s", i+1, step.Verb, strings.Join(step.Args, " "), sr.Error))
return fail(fmt.Errorf("step %d (%s %s) failed: %s", i+1, step.Verb, strings.Join(step.Args, " "), stepFailure(sr, step)))
}
if i == 0 {
return fail(fmt.Errorf("step %d: retry needs a step above it to repeat", i+1))
@@ -218,7 +221,15 @@ func executeStep(r *adb.Runner, rec *Recipe, step Step, index int, params map[st
case "playing":
// The screen is not a trustworthy place to read playback: Spotify's
// transport button keeps the label "Play" while music is playing.
// Ask the media session instead, and confirm the track title.
// Ask the media session instead. An extra text argument also
// confirms the playing title, which is how spotify_play proves it
// started the right song. With no text, only the state is checked:
// a radio plays a different track every few minutes, so a title
// match would fail on a radio that is working perfectly.
title := ""
if len(args) >= 1 {
title = args[0]
}
pkg := rec.App
if len(args) == 2 {
pkg = args[1]
@@ -235,14 +246,22 @@ func executeStep(r *adb.Runner, rec *Recipe, step Step, index int, params map[st
}
last = state
if state.Playing() && state.Title != "" {
matched, total := ui.MatchesText(state.Title, args[0])
if title == "" {
sr.Detail = fmt.Sprintf("%s is playing %q", pkg, state.Title)
return nil
}
matched, total := ui.MatchesText(state.Title, title)
if total > 0 && matched*2 >= total {
sr.Detail = fmt.Sprintf("%s is playing %q", pkg, state.Title)
return nil
}
}
if time.Now().After(checkDeadline) {
return fmt.Errorf("%s is not playing %q within %s (%s)", pkg, args[0], timeout, last)
want := fmt.Sprintf("playing %q", title)
if title == "" {
want = "playing"
}
return fmt.Errorf("%s is not %s within %s (%s)", pkg, want, timeout, last)
}
time.Sleep(ui.PollInterval)
}
@@ -279,7 +298,20 @@ func executeStep(r *adb.Runner, rec *Recipe, step Step, index int, params map[st
return nil
case "tap_row":
return tapRow(r, rec, args[0], timeout, w, h, &sr)
title, err := tapRow(r, rec, args[0], timeout, w, h, &sr)
if err != nil {
return err
}
params["row"] = title
return nil
case "menu_row":
title, err := tapRowMenu(r, rec, args[0], timeout, w, h, &sr)
if err != nil {
return err
}
params["row"] = title
return nil
case "type":
if err := ui.Type(r, args[0]); err != nil {
@@ -288,6 +320,17 @@ func executeStep(r *adb.Runner, rec *Recipe, step Step, index int, params map[st
sr.Detail = fmt.Sprintf("typed %d characters", len(args[0]))
return nil
case "scroll":
// Scroll the current list. ui.Scroll prefers a scrollable node from
// the dump, so a swipe stays inside the list rather than landing on
// a toolbar. A recipe uses it when a menu's entries sit below the
// fold and are not on screen yet.
if err := ui.Scroll(r, args[0]); err != nil {
return err
}
sr.Detail = "scrolled " + strings.ToLower(args[0])
return nil
case "key":
return sendKey(r, args[0], &sr)
@@ -355,8 +398,9 @@ func tapByText(r *adb.Runner, rec *Recipe, query string, exact bool, timeout tim
// tapRow taps the list row that best matches the query. It retries the dump
// until a matching row is on screen, then taps a point derived from that row's
// own bounds.
func tapRow(r *adb.Runner, rec *Recipe, query string, timeout time.Duration, w, h int, sr *StepResult) error {
// own bounds. It returns the chosen row's title, which later steps can use as
// the "row" value.
func tapRow(r *adb.Runner, rec *Recipe, query string, timeout time.Duration, w, h int, sr *StepResult) (string, error) {
deadline := time.Now().Add(timeout)
var lastErr error
for time.Now().Before(deadline) {
@@ -365,11 +409,11 @@ func tapRow(r *adb.Runner, rec *Recipe, query string, timeout time.Duration, w,
row, rerr := hier.BestRow(query, rec.RowID, w, h)
if rerr == nil {
if err := ui.Tap(r, row.Centre.X, row.Centre.Y); err != nil {
return err
return "", err
}
sr.Detail = fmt.Sprintf("tapped row %q at (%d,%d) from %s", row.Title, row.Centre.X, row.Centre.Y,
strings.Join(row.Texts, " / "))
return nil
return row.Title, nil
}
lastErr = rerr
} else {
@@ -380,7 +424,64 @@ func tapRow(r *adb.Runner, rec *Recipe, query string, timeout time.Duration, w,
if lastErr == nil {
lastErr = fmt.Errorf("no result row matches %q", query)
}
return describeFailure(r, rec, lastErr)
return "", describeFailure(r, rec, lastErr)
}
// tapRowMenu opens the More options menu of the row that matches query.
//
// The button is looked up *inside the chosen row's own subtree*, so the menu
// always belongs to the row the query matched; the More options button of any
// other row is never a candidate. The button carries no text of its own on the
// real screen (its label sits on a child), so the search covers each candidate
// button's subtree. It returns the chosen row's title, which later steps can use
// as the "row" value to check that the menu that opened is that row's.
func tapRowMenu(r *adb.Runner, rec *Recipe, query string, timeout time.Duration, w, h int, sr *StepResult) (string, error) {
deadline := time.Now().Add(timeout)
var lastErr error
for time.Now().Before(deadline) {
hier, err := ui.Dump(r)
if err != nil {
lastErr = err
time.Sleep(ui.PollInterval)
continue
}
row, rerr := hier.BestRow(query, rec.RowID, w, h)
if rerr != nil {
lastErr = rerr
time.Sleep(ui.PollInterval)
continue
}
button, berr := row.Action(rec.RowMenu)
if berr != nil {
return "", describeFailure(r, rec, berr)
}
cx, cy, cerr := button.Center()
if cerr != nil {
return "", cerr
}
if err := ui.Tap(r, cx, cy); err != nil {
return "", err
}
sr.Detail = fmt.Sprintf("opened the %q menu of row %q at (%d,%d)", rec.RowMenu, row.Title, cx, cy)
return row.Title, nil
}
if lastErr == nil {
lastErr = fmt.Errorf("no result row matches %q", query)
}
return "", describeFailure(r, rec, lastErr)
}
// stepFailure is the message for a failed step. A step's own "else" reason
// comes first, because it names the real problem, and the technical detail
// follows so the screen Artemis saw is still in the record (FR-8).
func stepFailure(sr StepResult, step Step) string {
if step.Reason == "" {
return sr.Error
}
if sr.Error == "" {
return step.Reason
}
return step.Reason + ": " + sr.Error
}
func sendKey(r *adb.Runner, name string, sr *StepResult) error {

55
internal/recipe/recipe.go
View File

@@ -18,6 +18,7 @@
// app: com.spotify.music the package the recipe drives
// param: query a parameter the caller must supply (repeatable)
// row_id: com.spotify:id/row_root container id for list rows (optional)
// row_menu: More options label of a row's own action button (optional)
// step_timeout: 20s default per-step timeout (FR-11)
// total_timeout: 120s whole-recipe timeout (FR-11)
//
@@ -34,18 +35,35 @@
// - wait exact "Pause" 60s retry 3
// if it never appears, re-run the step above and
// wait again (a busy phone can drop a tap)
// - playing "{query}" 30s retry 3
// wait until the app's media session is PLAYING and its
// title matches the text; with no text, wait for PLAYING
// alone (a radio plays a different track every few minutes)
// - wait_gone "Recents" 30s wait until no element matches the text
// - tap "Search" tap the element whose text matches (FR-4)
// - tap exact "Pause" tap an element whose text matches exactly
// - tap_if "Clear search query" tap it when it is on screen, else skip
// - tap_row "{query}" 30s tap the list row that best matches the text
// - menu_row "{query}" 30s open the More options menu of that row
// - scroll down scroll the current list ("down" or "up")
// - type "{query}" type into the focused field
// - key enter send a named key event
// - back / home press a system key
// - sleep 1s wait a fixed time
//
// A step may name its own failure reason with "else":
//
// - wait "Go to song radio" 20s else "this track offers no Song radio"
//
// {name} placeholders are replaced with the value of parameter "name". The
// trailing "30s" on wait/tap/tap_row steps is that step's timeout.
//
// tap_row and menu_row also record the row they chose as "row", so a later step
// can refer to it with {row} — for example to check that the context menu that
// opened belongs to the row that was pressed:
//
// - menu_row "{query}" 40s
// - wait "{row}" 20s else "the context menu opened on the wrong row"
package recipe
import (
@@ -65,6 +83,7 @@ type Recipe struct {
App string
Params []string
RowID string
RowMenu string
StepTimeout time.Duration
TotalTimeout time.Duration
Guards []Guard
@@ -87,13 +106,20 @@ type Step struct {
// Retry, on a wait step, re-runs the preceding step and waits again, up to
// this many extra times. A busy phone can drop a tap.
Retry int
Line int
// Reason, when set with "else", is the step's named failure (FR-8). A wait
// for a label a screen may simply not offer reads better as "this track
// offers no Song radio" than as "text did not appear".
Reason string
Line int
}
// Defaults used when a recipe does not set its own timeouts.
// Defaults used when a recipe does not set its own timeouts or labels.
const (
DefaultStepTimeout = 20 * time.Second
DefaultTotalTimeout = 120 * time.Second
// DefaultRowMenu is the label of a row's own action button. Spotify's list
// row action carries the content description "More options for song ...".
DefaultRowMenu = "More options"
)
// validKeys are the directive names a recipe file may use.
@@ -103,6 +129,7 @@ var validKeys = map[string]bool{
"app": true,
"param": true,
"row_id": true,
"row_menu": true,
"step_timeout": true,
"total_timeout": true,
}
@@ -116,11 +143,13 @@ var verbs = map[string]struct {
"launch": {0, 1, false, false},
"foreground": {0, 1, false, false},
"wait": {1, 2, false, true},
"playing": {1, 2, false, false},
"playing": {0, 2, false, false},
"wait_gone": {1, 2, false, true},
"tap": {1, 2, false, true},
"tap_if": {1, 1, false, true},
"tap_row": {1, 2, false, true},
"menu_row": {1, 2, false, false},
"scroll": {1, 1, false, false},
"type": {1, 1, false, false},
"key": {1, 1, false, false},
"back": {0, 0, false, false},
@@ -202,6 +231,7 @@ func Parse(src string) (*Recipe, error) {
rec := &Recipe{
StepTimeout: DefaultStepTimeout,
TotalTimeout: DefaultTotalTimeout,
RowMenu: DefaultRowMenu,
}
for i, raw := range strings.Split(src, "\n") {
@@ -262,6 +292,11 @@ func (rec *Recipe) parseDirective(line string, lineNo int) error {
rec.Params = append(rec.Params, value)
case "row_id":
rec.RowID = value
case "row_menu":
if value == "" {
return fmt.Errorf("line %d: row_menu needs a label", lineNo)
}
rec.RowMenu = value
case "step_timeout":
d, err := parseDuration(value)
if err != nil {
@@ -340,6 +375,20 @@ func parseStep(body string, lineNo int) (*Step, error) {
break
}
// "else \"reason\"" names this step's own failure, so a screen that simply
// lacks a label can fail with the reason a human needs (FR-8).
for i := 0; i < len(args); i++ {
if !strings.EqualFold(args[i], "else") {
continue
}
if i+1 >= len(args) {
return nil, fmt.Errorf("line %d: else needs a reason, as in else \"that track has no radio\"", lineNo)
}
step.Reason = args[i+1]
args = append(args[:i], args[i+2:]...)
break
}
// A trailing duration is the step timeout, except for sleep where the
// duration is the step itself.
if !spec.durArg && len(args) > spec.min {

158
internal/recipe/recipe_test.go
View File

@@ -3,6 +3,7 @@ package recipe
import (
"os"
"path/filepath"
"strconv"
"strings"
"testing"
"time"
@@ -279,3 +280,160 @@ func TestStepDurationsAreRecorded(t *testing.T) {
t.Fatalf("duration not recorded: %v (%d ms)", sr.Duration, sr.DurationMS)
}
}
// TestElseNamesAStepFailure covers the "else" modifier: a screen that simply
// does not offer a label must fail with a reason a human can act on (FR-8).
func TestElseNamesAStepFailure(t *testing.T) {
rec, err := Parse("steps:\n - wait \"Go to song radio\" 20s else \"this track offers no Song radio\"\n")
if err != nil {
t.Fatalf("Parse: %v", err)
}
step := rec.Steps[0]
if step.Args[0] != "Go to song radio" || step.Timeout != 20*time.Second {
t.Fatalf("step = %+v", step)
}
if step.Reason != "this track offers no Song radio" {
t.Fatalf("reason = %q", step.Reason)
}
// The reason must not leak into the arguments the engine substitutes.
if len(step.Args) != 1 {
t.Fatalf("args = %v, want the query only", step.Args)
}
}
func TestElseNeedsAReason(t *testing.T) {
if _, err := Parse("steps:\n - wait \"Search\" 20s else\n"); err == nil {
t.Fatal("expected an error when else has no reason")
}
}
func TestStepFailurePrefersTheStepsOwnReason(t *testing.T) {
step := Step{Verb: "wait", Reason: "this track offers no Song radio"}
sr := StepResult{Error: "text \"Go to song radio\" did not appear within 20s; screen shows: \"Share\""}
got := stepFailure(sr, step)
if !strings.HasPrefix(got, "this track offers no Song radio:") {
t.Fatalf("stepFailure = %q, want the step's own reason first", got)
}
if !strings.Contains(got, "did not appear") {
t.Fatalf("stepFailure = %q, want the technical detail kept", got)
}
if plain := stepFailure(sr, Step{Verb: "wait"}); plain != sr.Error {
t.Fatalf("a step without a reason must report the error unchanged, got %q", plain)
}
}
// TestRowMenuDefaults keeps menu_row usable without extra settings, and proves
// a recipe can override the label.
func TestRowMenuDefaults(t *testing.T) {
rec, err := Parse("steps:\n - home\n")
if err != nil {
t.Fatalf("Parse: %v", err)
}
if rec.RowMenu != DefaultRowMenu {
t.Fatalf("row_menu default = %q, want %q", rec.RowMenu, DefaultRowMenu)
}
rec, err = Parse("row_menu: More options\nsteps:\n - menu_row \"{query}\" 30s\n")
if err != nil {
t.Fatalf("Parse: %v", err)
}
if rec.RowMenu != "More options" {
t.Fatalf("row_menu = %q", rec.RowMenu)
}
if rec.Steps[0].Verb != "menu_row" || rec.Steps[0].Args[0] != "{query}" {
t.Fatalf("step = %+v", rec.Steps[0])
}
}
// TestRowValueIsAvailableToLaterSteps covers {row}: menu_row/tap_row record the
// row they chose, so a recipe can check that a menu belongs to the row it
// pressed. The value is not a declared parameter, so it must not be required
// up front.
func TestRowValueIsAvailableToLaterSteps(t *testing.T) {
rec, err := Parse("param: query\nsteps:\n - menu_row \"{query}\" 40s\n - wait \"{row}\" 20s else \"the context menu opened on the wrong row\"\n")
if err != nil {
t.Fatalf("Parse: %v", err)
}
if missing := rec.MissingParams(map[string]string{"query": "x"}); len(missing) != 0 {
t.Fatalf("row must not be a required parameter, got missing %v", missing)
}
params := map[string]string{"query": "Rolling Stones Honky Tonk"}
if _, err := Substitute(rec.Steps[1].Args[0], params); err == nil {
t.Fatal("expected {row} to be unresolved before a row step has run")
}
params["row"] = "Honky Tonk Women"
got, err := Substitute(rec.Steps[1].Args[0], params)
if err != nil {
t.Fatalf("Substitute: %v", err)
}
if got != "Honky Tonk Women" {
t.Fatalf("substituted %q", got)
}
}
// TestSpotifyRadioRecipeIsValid keeps the shipped Phase 4 recipe honest: a typo
// in recipes/spotify_radio.recipe fails the test suite, not the phone.
func TestSpotifyRadioRecipeIsValid(t *testing.T) {
rec, err := Load(filepath.Join("..", "..", "recipes", "spotify_radio.recipe"))
if err != nil {
t.Fatalf("Load: %v", err)
}
if rec.Task != "spotify_radio" || rec.App != "com.spotify.music" {
t.Fatalf("header = %q / %q", rec.Task, rec.App)
}
if rec.RowID != "com.spotify.music:id/row_root" {
t.Fatalf("row_id = %q", rec.RowID)
}
if rec.RowMenu != DefaultRowMenu {
t.Fatalf("row_menu = %q", rec.RowMenu)
}
if len(rec.MissingParams(map[string]string{"query": "x"})) != 0 {
t.Fatal("spotify_radio must accept a query parameter")
}
// The app must be pinned in front before any text is trusted, and the
// screen must be searched before a row is picked.
var sawForeground, sawMenuRow int
reasons := map[string]bool{}
var last Step
for _, s := range rec.Steps {
switch s.Verb {
case "foreground":
sawForeground++
case "menu_row":
sawMenuRow++
}
if s.Reason != "" {
reasons[s.Reason] = true
}
// No coordinates, nowhere: every step argument is text, a duration or a
// placeholder, never a bare screen coordinate.
for _, a := range s.Args {
if _, err := strconv.Atoi(a); err == nil {
t.Fatalf("step %s has a bare number %q as an argument: this recipe must name text, not coordinates", s.Verb, a)
}
}
last = s
}
if sawForeground == 0 {
t.Fatal("spotify_radio must pin the foreground app before matching text")
}
if sawMenuRow != 1 {
t.Fatalf("spotify_radio must open exactly one row menu, got %d", sawMenuRow)
}
for _, want := range []string{
"the context menu opened on the wrong row",
"this track offers no Song radio",
} {
if !reasons[want] {
t.Fatalf("spotify_radio must fail with the distinct reason %q; it has %v", want, reasons)
}
}
if last.Verb != "playing" || len(last.Args) != 0 {
t.Fatalf("the last step must confirm playback by state (a radio changes track), got %+v", last)
}
if last.Retry < 1 {
t.Fatal("the playback check must be allowed to press the step above again")
}
}