feat: confirm playback from the media session, and stop demoting the top row

Phase 3 DoD reached end to end: `artemis run spotify_play query="Rolling
Stones Honky Tonk"` played the right song in 13 steps / 35.3s, with playback
paused first so the pass proves the row tap started it.

Two things stopped it before that, both found on the real phone:

- The playback check asserted on an on-screen "Pause" label that Spotify
  never shows: its transport button keeps the label "Play" while music is
  playing. The recipe now ends with a `playing` step that reads the app's
  media session (state PLAYING) and confirms the track title contains the
  query words. A failure reports the state and the track it found.
- tap_row pressed the wrong row. Once a track is loaded its row carries a
  `Paused` state indicator, and counting that content description as row text
  dropped the top hit below an identical duplicate further down, so the tap
  went to y=1443 instead of y=878. Rows are now scored on their title and
  subtitle only (state lives in content descriptions), and scores within
  rowScoreTolerance are treated as equal so the higher row wins, because
  Spotify already ranks by relevance.

internal/ui/rows_test.go reproduces that results screen, including the state
indicator and the duplicate row.

docs/phase-3-spotify-recipe.md records the successful run, the independent
media-session and audio-output reads, and the failure that preceded it.
This commit is contained in:
2026-10-16 14:32:24 +11:00
parent ab69f37aa4
commit c178f92851
10 changed files with 403 additions and 37 deletions

View File

@@ -215,6 +215,38 @@ func executeStep(r *adb.Runner, rec *Recipe, step Step, index int, params map[st
sr.Detail = "found " + node.Label()
return nil
case "playing":
// The screen is not a trustworthy place to read playback: Spotify's
// transport button keeps the label "Play" while music is playing.
// Ask the media session instead, and confirm the track title.
pkg := rec.App
if len(args) == 2 {
pkg = args[1]
}
if pkg == "" {
return fmt.Errorf("playing needs an app: add an \"app:\" setting")
}
checkDeadline := time.Now().Add(timeout)
var last *ui.Playback
for {
state, err := ui.PlaybackState(r, pkg)
if err != nil {
return err
}
last = state
if state.Playing() && state.Title != "" {
matched, total := ui.MatchesText(state.Title, args[0])
if total > 0 && matched*2 >= total {
sr.Detail = fmt.Sprintf("%s is playing %q", pkg, state.Title)
return nil
}
}
if time.Now().After(checkDeadline) {
return fmt.Errorf("%s is not playing %q within %s (%s)", pkg, args[0], timeout, last)
}
time.Sleep(ui.PollInterval)
}
case "wait_gone":
if err := ui.WaitForNoText(r, args[0], step.Exact, timeout); err != nil {
return describeFailure(r, rec, err)

View File

@@ -116,6 +116,7 @@ var verbs = map[string]struct {
"launch": {0, 1, false, false},
"foreground": {0, 1, false, false},
"wait": {1, 2, false, true},
"playing": {1, 2, false, false},
"wait_gone": {1, 2, false, true},
"tap": {1, 2, false, true},
"tap_if": {1, 1, false, true},
@@ -331,8 +332,8 @@ func parseStep(body string, lineNo int) (*Step, error) {
if err != nil || n < 1 {
return nil, fmt.Errorf("line %d: retry needs a positive number, got %q", lineNo, args[i+1])
}
if verb != "wait" {
return nil, fmt.Errorf("line %d: retry is only allowed on a wait step", lineNo)
if verb != "wait" && verb != "playing" {
return nil, fmt.Errorf("line %d: retry is only allowed on a wait or playing step", lineNo)
}
step.Retry = n
args = append(args[:i], args[i+2:]...)

View File

@@ -158,7 +158,7 @@ func TestSpotifyRecipeIsValid(t *testing.T) {
}
}
// The recipe must confirm it is in Spotify before it trusts a text match,
// and it must wait for playback rather than assume it.
// and it must confirm playback rather than assume it.
var sawForeground bool
for _, s := range rec.Steps {
if s.Verb == "foreground" {
@@ -169,8 +169,8 @@ func TestSpotifyRecipeIsValid(t *testing.T) {
t.Fatal("spotify_play must pin the foreground app before matching text")
}
last := rec.Steps[len(rec.Steps)-1]
if last.Verb != "wait" || last.Args[0] != "Pause" {
t.Fatalf("the last step must confirm playback, got %+v", last)
if last.Verb != "playing" || last.Args[0] != "{query}" {
t.Fatalf("the last step must confirm the right track is playing, got %+v", last)
}
if last.Retry < 1 {
t.Fatal("the playback check must be allowed to press the row again")
@@ -209,16 +209,16 @@ func TestLoadAndList(t *testing.T) {
}
func TestRetryOnWait(t *testing.T) {
rec, err := Parse("steps:\n - tap_row \"{query}\" 40s\n - wait exact \"Pause\" 45s retry 3\n")
rec, err := Parse("steps:\n - tap_row \"{query}\" 40s\n - wait exact \"Search\" 45s retry 3\n")
if err != nil {
t.Fatalf("Parse: %v", err)
}
wait := rec.Steps[1]
if wait.Retry != 3 || wait.Timeout != 45*time.Second || !wait.Exact || wait.Args[0] != "Pause" {
if wait.Retry != 3 || wait.Timeout != 45*time.Second || !wait.Exact || wait.Args[0] != "Search" {
t.Fatalf("retry wait step = %+v", wait)
}
// retry without a duration still works.
rec, err = Parse("steps:\n - tap_row \"x\"\n - wait \"Pause\" retry 2\n")
rec, err = Parse("steps:\n - tap_row \"x\"\n - wait \"Search\" retry 2\n")
if err != nil {
t.Fatalf("Parse: %v", err)
}
@@ -230,6 +230,7 @@ func TestRetryOnWait(t *testing.T) {
func TestRetryRejectsBadUse(t *testing.T) {
for name, src := range map[string]string{
"on a tap": "steps:\n - tap \"x\" retry 3\n",
"on tap_row": "steps:\n - tap_row \"x\" retry 3\n",
"no number": "steps:\n - tap_row \"x\"\n - wait \"y\" retry\n",
"zero": "steps:\n - tap_row \"x\"\n - wait \"y\" retry 0\n",
"not a number": "steps:\n - tap_row \"x\"\n - wait \"y\" retry soon\n",
@@ -240,6 +241,17 @@ func TestRetryRejectsBadUse(t *testing.T) {
}
}
func TestPlayingStep(t *testing.T) {
rec, err := Parse("app: com.spotify.music\nsteps:\n - tap_row \"{query}\"\n - playing \"{query}\" 30s retry 3\n")
if err != nil {
t.Fatalf("Parse: %v", err)
}
p := rec.Steps[1]
if p.Verb != "playing" || p.Args[0] != "{query}" || p.Timeout != 30*time.Second || p.Retry != 3 {
t.Fatalf("playing step = %+v", p)
}
}
func TestForegroundStep(t *testing.T) {
rec, err := Parse("app: com.spotify.music\nsteps:\n - launch\n - foreground 30s\n")
if err != nil {