From c178f928511febc84ad8238848a77b09e900f062 Mon Sep 17 00:00:00 2001 From: Sam Rolfe Date: Fri, 16 Oct 2026 14:32:24 +1100 Subject: [PATCH] feat: confirm playback from the media session, and stop demoting the top row Phase 3 DoD reached end to end: `artemis run spotify_play query="Rolling Stones Honky Tonk"` played the right song in 13 steps / 35.3s, with playback paused first so the pass proves the row tap started it. Two things stopped it before that, both found on the real phone: - The playback check asserted on an on-screen "Pause" label that Spotify never shows: its transport button keeps the label "Play" while music is playing. The recipe now ends with a `playing` step that reads the app's media session (state PLAYING) and confirms the track title contains the query words. A failure reports the state and the track it found. - tap_row pressed the wrong row. Once a track is loaded its row carries a `Paused` state indicator, and counting that content description as row text dropped the top hit below an identical duplicate further down, so the tap went to y=1443 instead of y=878. Rows are now scored on their title and subtitle only (state lives in content descriptions), and scores within rowScoreTolerance are treated as equal so the higher row wins, because Spotify already ranks by relevance. internal/ui/rows_test.go reproduces that results screen, including the state indicator and the duplicate row. docs/phase-3-spotify-recipe.md records the successful run, the independent media-session and audio-output reads, and the failure that preceded it. --- README.md | 5 +- docs/phase-3-spotify-recipe.md | 90 ++++++++++++++++++++++--- internal/recipe/engine.go | 32 +++++++++ internal/recipe/recipe.go | 5 +- internal/recipe/recipe_test.go | 24 +++++-- internal/ui/media.go | 116 +++++++++++++++++++++++++++++++++ internal/ui/media_test.go | 77 ++++++++++++++++++++++ internal/ui/rows.go | 26 +++++--- internal/ui/rows_test.go | 49 +++++++++++++- recipes/spotify_play.recipe | 16 +++-- 10 files changed, 403 insertions(+), 37 deletions(-) create mode 100644 internal/ui/media.go create mode 100644 internal/ui/media_test.go diff --git a/README.md b/README.md index 38176d3..c09117b 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ order. |---|---| | 1 — Prove the ADB link | ✅ proven — see [`docs/phase-1-adb-link.md`](docs/phase-1-adb-link.md) | | 2 — Act primitives | ✅ done — tap by text, type, scroll, back, home, launch | -| 3 — Spotify recipe | 🟡 built — recipes file, guards, `spotify_play`; the live end-to-end run is still to do, see [`docs/phase-3-spotify-recipe.md`](docs/phase-3-spotify-recipe.md) | +| 3 — Spotify recipe | ✅ done — `spotify_play` plays the right song end to end (13 steps, 35s), see [`docs/phase-3-spotify-recipe.md`](docs/phase-3-spotify-recipe.md) | | 4 — Song radio | not started | | 5 — Maps recipe | not started | | 6 — Action API | not started | @@ -86,6 +86,9 @@ file format, the verbs, and why list rows need their own step. - **Never touch a locked phone.** A secure keyguard stops Artemis with `phone is locked; unlock it and retry`. No PIN is stored, read or typed (AGENTS.md section 13 Q2). +- **Confirm playback from the media session, not the screen.** Spotify's + transport button keeps the label "Play" while music plays; the `playing` step + asks Android what is actually playing and checks the track title. - **Wake first.** Every acting command wakes the screen before it acts (FR-7), and it checks the keyguard only after the screen is on. - **Bound every command.** No adb call runs without a timeout, and a recipe has diff --git a/docs/phase-3-spotify-recipe.md b/docs/phase-3-spotify-recipe.md index 4ce8f16..e8ebd8d 100644 --- a/docs/phase-3-spotify-recipe.md +++ b/docs/phase-3-spotify-recipe.md @@ -192,10 +192,46 @@ keyguard; reading the state first would say "unlocked" about a locked phone. | Step durations reach the run log | unit tested (`internal/recipe`) | | All acting commands refuse on a locked phone | **live, verified** (above) | | `artemis run spotify_play` refuses on a locked phone | **live, verified** (above) | -| `spotify_play` plays the right song | **not yet run end to end** — see below | +| `spotify_play` plays the right song | **live, verified end to end** — 13 steps, 35.3s | | `spotify_play` picks the right row on the real results list | **live, verified** — `tapped row "Honky Tonk Women" at (540,878)` | | Spotify search reaches the result list | live (Home → Search → type → results) | +## Confirming playback + +The screen is not a trustworthy place to read playback state. Spotify's +transport button keeps the accessibility label **"Play"** while music is +actually playing, so the first version of this recipe asserted on a label that +never appears and failed on a phone that was playing the track perfectly. + +The last step asks Android instead: + +``` +- playing "{query}" 30s retry 3 +``` + +`playing` waits until the app's media session reports `PLAYING` **and** its +metadata contains the query's words. It fails with the state and the track it +found, for example: + +``` +com.spotify.music is not playing "Rolling Stones Honky Tonk" within 30s + (state=PAUSED, playing "Honky Tonk Women, The Rolling Stones, ...") +``` + +## Why the top row wins + +Spotify ranks results by relevance, so the highest acceptable row is the one a +human would press. Two rules keep it that way: + +- A row's own text is its title and subtitle. Text inside a clickable child is + an action button, and **content descriptions are state**, not text: the row of + the track that is already loaded carries `content-desc="Paused"`. Counting + that made the top hit score *worse* than an identical duplicate further down, + and `tap_row` pressed the duplicate at `y=1443` while the right row sat at + `y=878`. `internal/ui/rows_test.go` reproduces that screen exactly. +- Scores within `rowScoreTolerance` (0.15) are treated as equal, and then the + higher row wins. A clearly better text match still wins outright. + ## Retrying a dropped tap A step can ask for the step above it to be pressed again: @@ -256,6 +292,44 @@ nothing random — which is the point of the phase. no dumps and no input, the screen had moved to Home — queued input had been processed — so a fresh run was started immediately. It failed at `tap "Search"`: the tap reported ok, the screen never left Home. +11. **Wrong row chosen, quietly.** After Sam restarted Spotify the whole recipe + ran fast and ended at `playing` with `state=PAUSED`. The run log showed why: + `tapped row "Honky Tonk Women" at (540,1443)` — the fifth result, not the top + hit at `(540,878)`. Once the track is loaded its top row carries a `Paused` + state indicator, and counting that description as row text dropped the top + hit's F1 below an identical duplicate further down. Fixed, and the top row + now loses no points for carrying state. +12. **Success.** With the ranking fixed, and with playback confirmed through the + media session rather than the on-screen label (which stays `Play` even while + music plays): + + ```sh + $ ./bin/artemis run spotify_play query="Rolling Stones Honky Tonk" + step 12 tap_row [Rolling Stones Honky Tonk]: ok (2.922s) + step 13 playing [Rolling Stones Honky Tonk]: ok (1.714s) + OK spotify_play in 35.311s + ``` + + ``` + tapped row "Honky Tonk Women" at (540,878) + from Honky Tonk Women / Song • The Rolling Stones + com.spotify.music is playing + "Honky Tonk Women, The Rolling Stones, Honky Tonk Women / You Can't Always Get What You Want" + ``` + + Independent reads, not Artemis's own claim: + + ``` + $ adb shell dumpsys media_session | grep -A9 'package=com.spotify.music' + state=PlaybackState {state=PLAYING(3), position=546, ...} + $ adb shell dumpsys audio | grep -i 'state:started' + AudioPlaybackConfiguration piid:22575 ... type:AAudio u/pid:10340/11987 + state:started ... sampleRate=44100 + ``` + + uid 10340 is Spotify, so audio really was being produced. A run was started + with the track deliberately **paused** first, so the pass proves the row tap + started playback rather than finding it already playing. ### Diagnosis @@ -271,17 +345,17 @@ Artemis made this worse than it needed to be: `uiautomator dump` starts a JVM on the phone, and the wait helpers were polling every 400ms. `ui.PollInterval` is now 1500ms, which cuts that load by roughly two thirds. -### What is left - -The live end-to-end run of `spotify_play` (Phase 3 DoD) has not completed. It -needs the phone's Spotify restarted — swipe it away from Recents, or force-stop -it — and the phone left awake and unlocked. Then: +### Reproducing it ```sh +cd ~/paseo/projects/artemis_control ./bin/artemis run spotify_play query="Rolling Stones Honky Tonk" -./bin/artemis fg -./bin/artemis dump | grep -i pause tail -1 logs/runs.jsonl +adb shell dumpsys media_session | grep -A9 'package=com.spotify.music' ``` +The phone must be awake and unlocked. On a quiet phone this takes about 35 +seconds; on the loaded phone above it ranged from 90 seconds to over four +minutes, which is what `total_timeout: 420s` is there to bound. + Phase 4 (song radio) and Phase 5 (maps) have not been started, on instruction. diff --git a/internal/recipe/engine.go b/internal/recipe/engine.go index 4a18314..9a16603 100644 --- a/internal/recipe/engine.go +++ b/internal/recipe/engine.go @@ -215,6 +215,38 @@ func executeStep(r *adb.Runner, rec *Recipe, step Step, index int, params map[st sr.Detail = "found " + node.Label() return nil + case "playing": + // The screen is not a trustworthy place to read playback: Spotify's + // transport button keeps the label "Play" while music is playing. + // Ask the media session instead, and confirm the track title. + pkg := rec.App + if len(args) == 2 { + pkg = args[1] + } + if pkg == "" { + return fmt.Errorf("playing needs an app: add an \"app:\" setting") + } + checkDeadline := time.Now().Add(timeout) + var last *ui.Playback + for { + state, err := ui.PlaybackState(r, pkg) + if err != nil { + return err + } + last = state + if state.Playing() && state.Title != "" { + matched, total := ui.MatchesText(state.Title, args[0]) + if total > 0 && matched*2 >= total { + sr.Detail = fmt.Sprintf("%s is playing %q", pkg, state.Title) + return nil + } + } + if time.Now().After(checkDeadline) { + return fmt.Errorf("%s is not playing %q within %s (%s)", pkg, args[0], timeout, last) + } + time.Sleep(ui.PollInterval) + } + case "wait_gone": if err := ui.WaitForNoText(r, args[0], step.Exact, timeout); err != nil { return describeFailure(r, rec, err) diff --git a/internal/recipe/recipe.go b/internal/recipe/recipe.go index 1897bb9..edf1066 100644 --- a/internal/recipe/recipe.go +++ b/internal/recipe/recipe.go @@ -116,6 +116,7 @@ var verbs = map[string]struct { "launch": {0, 1, false, false}, "foreground": {0, 1, false, false}, "wait": {1, 2, false, true}, + "playing": {1, 2, false, false}, "wait_gone": {1, 2, false, true}, "tap": {1, 2, false, true}, "tap_if": {1, 1, false, true}, @@ -331,8 +332,8 @@ func parseStep(body string, lineNo int) (*Step, error) { if err != nil || n < 1 { return nil, fmt.Errorf("line %d: retry needs a positive number, got %q", lineNo, args[i+1]) } - if verb != "wait" { - return nil, fmt.Errorf("line %d: retry is only allowed on a wait step", lineNo) + if verb != "wait" && verb != "playing" { + return nil, fmt.Errorf("line %d: retry is only allowed on a wait or playing step", lineNo) } step.Retry = n args = append(args[:i], args[i+2:]...) diff --git a/internal/recipe/recipe_test.go b/internal/recipe/recipe_test.go index 5aa023e..0dc3a0f 100644 --- a/internal/recipe/recipe_test.go +++ b/internal/recipe/recipe_test.go @@ -158,7 +158,7 @@ func TestSpotifyRecipeIsValid(t *testing.T) { } } // The recipe must confirm it is in Spotify before it trusts a text match, - // and it must wait for playback rather than assume it. + // and it must confirm playback rather than assume it. var sawForeground bool for _, s := range rec.Steps { if s.Verb == "foreground" { @@ -169,8 +169,8 @@ func TestSpotifyRecipeIsValid(t *testing.T) { t.Fatal("spotify_play must pin the foreground app before matching text") } last := rec.Steps[len(rec.Steps)-1] - if last.Verb != "wait" || last.Args[0] != "Pause" { - t.Fatalf("the last step must confirm playback, got %+v", last) + if last.Verb != "playing" || last.Args[0] != "{query}" { + t.Fatalf("the last step must confirm the right track is playing, got %+v", last) } if last.Retry < 1 { t.Fatal("the playback check must be allowed to press the row again") @@ -209,16 +209,16 @@ func TestLoadAndList(t *testing.T) { } func TestRetryOnWait(t *testing.T) { - rec, err := Parse("steps:\n - tap_row \"{query}\" 40s\n - wait exact \"Pause\" 45s retry 3\n") + rec, err := Parse("steps:\n - tap_row \"{query}\" 40s\n - wait exact \"Search\" 45s retry 3\n") if err != nil { t.Fatalf("Parse: %v", err) } wait := rec.Steps[1] - if wait.Retry != 3 || wait.Timeout != 45*time.Second || !wait.Exact || wait.Args[0] != "Pause" { + if wait.Retry != 3 || wait.Timeout != 45*time.Second || !wait.Exact || wait.Args[0] != "Search" { t.Fatalf("retry wait step = %+v", wait) } // retry without a duration still works. - rec, err = Parse("steps:\n - tap_row \"x\"\n - wait \"Pause\" retry 2\n") + rec, err = Parse("steps:\n - tap_row \"x\"\n - wait \"Search\" retry 2\n") if err != nil { t.Fatalf("Parse: %v", err) } @@ -230,6 +230,7 @@ func TestRetryOnWait(t *testing.T) { func TestRetryRejectsBadUse(t *testing.T) { for name, src := range map[string]string{ "on a tap": "steps:\n - tap \"x\" retry 3\n", + "on tap_row": "steps:\n - tap_row \"x\" retry 3\n", "no number": "steps:\n - tap_row \"x\"\n - wait \"y\" retry\n", "zero": "steps:\n - tap_row \"x\"\n - wait \"y\" retry 0\n", "not a number": "steps:\n - tap_row \"x\"\n - wait \"y\" retry soon\n", @@ -240,6 +241,17 @@ func TestRetryRejectsBadUse(t *testing.T) { } } +func TestPlayingStep(t *testing.T) { + rec, err := Parse("app: com.spotify.music\nsteps:\n - tap_row \"{query}\"\n - playing \"{query}\" 30s retry 3\n") + if err != nil { + t.Fatalf("Parse: %v", err) + } + p := rec.Steps[1] + if p.Verb != "playing" || p.Args[0] != "{query}" || p.Timeout != 30*time.Second || p.Retry != 3 { + t.Fatalf("playing step = %+v", p) + } +} + func TestForegroundStep(t *testing.T) { rec, err := Parse("app: com.spotify.music\nsteps:\n - launch\n - foreground 30s\n") if err != nil { diff --git a/internal/ui/media.go b/internal/ui/media.go new file mode 100644 index 0000000..a71cef2 --- /dev/null +++ b/internal/ui/media.go @@ -0,0 +1,116 @@ +package ui + +import ( + "fmt" + "regexp" + "strings" + + "gitea.lab.audasmedia.com.au/sam/artemis_control/internal/adb" +) + +// Playback is what an app's media session says it is doing. +// +// The screen is not a reliable place to read this. Spotify's play/pause button +// keeps the accessibility label "Play" while music is playing, so a recipe that +// asserted on the label failed while the phone was busy playing the track. The +// media session is the authoritative answer, and it carries the track title. +type Playback struct { + Package string + State string // PLAYING, PAUSED, STOPPED, NONE, ... + Title string // the metadata description, usually "title, artist, album" +} + +// Playing reports whether anything is being played. +func (p *Playback) Playing() bool { + return p != nil && strings.EqualFold(p.State, "PLAYING") +} + +// String is the one-line form used in log lines and error messages. +func (p *Playback) String() string { + if p == nil { + return "no media session" + } + if p.Title == "" { + return fmt.Sprintf("state=%s", p.State) + } + return fmt.Sprintf("state=%s, playing %q", p.State, p.Title) +} + +var ( + playbackStateRe = regexp.MustCompile(`state=PlaybackState \{state=([A-Z]+)`) + metadataRe = regexp.MustCompile(`metadata:.*description=(.*)$`) +) + +// PlaybackState reads the media session of one package. +// +// It returns an error only when the dump cannot be read; an app that is simply +// not playing comes back with a state of PAUSED or NONE. +func PlaybackState(r *adb.Runner, pkg string) (*Playback, error) { + if pkg == "" { + return nil, fmt.Errorf("playback check needs a package name") + } + out, err := r.Shell("dumpsys media_session") + if err != nil { + return nil, fmt.Errorf("read media sessions: %w", err) + } + return parsePlayback(out, pkg), nil +} + +// parsePlayback finds the block for pkg and reads its state and metadata. A +// session block is delimited by its "package=" line, so state and metadata are +// only read from the block that belongs to the package. +func parsePlayback(dump, pkg string) *Playback { + var found *Playback + var cur *Playback + + for _, raw := range strings.Split(dump, "\n") { + line := strings.TrimSpace(raw) + if name, ok := strings.CutPrefix(line, "package="); ok { + if cur != nil && cur.Package == pkg && found == nil { + found = cur + } + cur = &Playback{Package: strings.TrimSpace(name)} + continue + } + if cur == nil { + continue + } + if m := playbackStateRe.FindStringSubmatch(line); m != nil { + cur.State = m[1] + continue + } + if m := metadataRe.FindStringSubmatch(line); m != nil && cur.Title == "" { + cur.Title = strings.TrimSpace(m[1]) + } + } + if cur != nil && cur.Package == pkg && found == nil { + found = cur + } + if found == nil { + return &Playback{Package: pkg, State: "NONE"} + } + if found.State == "" { + found.State = "NONE" + } + return found +} + +// MatchesText reports how many of the query's words appear in text. It is used +// to confirm that the session is playing the song the recipe asked for. +func MatchesText(text, query string) (matched, total int) { + want := tokenize(query) + if len(want) == 0 { + return 0, 0 + } + have := map[string]bool{} + for _, w := range tokenize(text) { + have[w] = true + } + seen := map[string]bool{} + for _, w := range want { + if have[w] { + seen[w] = true + } + } + return len(seen), len(want) +} diff --git a/internal/ui/media_test.go b/internal/ui/media_test.go new file mode 100644 index 0000000..6a493d9 --- /dev/null +++ b/internal/ui/media_test.go @@ -0,0 +1,77 @@ +package ui + +import "testing" + +// A real (trimmed) `dumpsys media_session` capture, taken while Spotify was +// playing the track the recipe asked for. Note the transport button on screen +// still read "Play". +const mediaSessionDump = `MEDIA SESSION SERVICE (dumpsys media_session) + + 12 sessions listeners: + 1: com.spotify.music/SpotifySession/1 (userId=0) + package=com.spotify.music + launchIntent=null + active=true + flags=3 + rating type=2 + controllers: 12 + state=PlaybackState {state=PLAYING(3), position=606, buffered position=0, speed=1.0, updated=1207066569, actions=3025843, custom actions=[Action:mName='Start radio, mIcon=2131233633, mExtras=Bundle[mParcelledData.dataSize=144]], active item id=18, error=null} + audioAttrs=AudioAttributes: usage=USAGE_MEDIA content=CONTENT_TYPE_UNKNOWN flags=0x800(FLAG_MUTE_HAPTIC) + volumeType=LOCAL, controlType=ABSOLUTE, max=0, current=0 + metadata: size=18, description=Honky Tonk Women, The Rolling Stones, Honky Tonk Women / You Can't Always Get What You Want + queueTitle=Play Queue, size=50 + 2: org.kde.kdeconnect_tp.media_session org.kde.kdeconnect_tp/org.kde.kdeconnect_tp.media_session/630 (userId=0) + package=org.kde.kdeconnect_tp + active=false + state=PlaybackState {state=PAUSED(2), position=881, buffered position=0, speed=0.0, updated=1206521908, actions=260, custom actions=[], active item id=-1, error=null} + metadata: size=0, description=, , +` + +func TestPlaybackStateReadsTheRightSession(t *testing.T) { + got := parsePlayback(mediaSessionDump, "com.spotify.music") + if got == nil || !got.Playing() { + t.Fatalf("spotify playback = %+v", got) + } + if got.Title != "Honky Tonk Women, The Rolling Stones, Honky Tonk Women / You Can't Always Get What You Want" { + t.Fatalf("title = %q", got.Title) + } + + // The block is delimited by its package line: Spotify's state must not leak + // into another package and vice versa. + other := parsePlayback(mediaSessionDump, "org.kde.kdeconnect_tp") + if other.Playing() { + t.Fatalf("kdeconnect playback = %+v, want paused", other) + } + if other.Title != ", ," && other.Title != "" { + t.Fatalf("kdeconnect title = %q", other.Title) + } + + missing := parsePlayback(mediaSessionDump, "com.example.absent") + if missing.Playing() || missing.State != "NONE" { + t.Fatalf("missing package = %+v, want state NONE", missing) + } +} + +func TestMatchesText(t *testing.T) { + const meta = "Honky Tonk Women, The Rolling Stones, Honky Tonk Women / You Can't Always Get What You Want" + if matched, total := MatchesText(meta, "Rolling Stones Honky Tonk"); matched != 4 || total != 4 { + t.Fatalf("matched %d/%d, want 4/4", matched, total) + } + if matched, _ := MatchesText(meta, "Chopin Nocturne"); matched != 0 { + t.Fatalf("matched %d for an unrelated query, want 0", matched) + } + if _, total := MatchesText(meta, ""); total != 0 { + t.Fatalf("empty query should have no words, got %d", total) + } +} + +func TestPlaybackString(t *testing.T) { + p := &Playback{Package: "com.spotify.music", State: "PLAYING", Title: "Honky Tonk Women"} + if s := p.String(); s != `state=PLAYING, playing "Honky Tonk Women"` { + t.Fatalf("String() = %q", s) + } + var nilSession *Playback + if s := nilSession.String(); s != "no media session" { + t.Fatalf("nil String() = %q", s) + } +} diff --git a/internal/ui/rows.go b/internal/ui/rows.go index 915e7a3..2e02a35 100644 --- a/internal/ui/rows.go +++ b/internal/ui/rows.go @@ -133,15 +133,22 @@ func (h *Hierarchy) rowCandidates(rowID string, screenW, screenH int) []*Node { return out } -// betterRow ranks rows: the best text match wins, and equal matches go to the -// row nearest the top, because search results are ranked by relevance. +// betterRow ranks rows. Spotify already ranks results by relevance, so a higher +// row wins whenever the text match is comparable: the top hit can carry extra +// state text ("Paused" on the row of the track that is already loaded) that +// must not push it below a duplicate further down the list. Scores that are +// further apart are not comparable, and the better text match wins. func betterRow(a, b *Row) bool { - if math.Abs(a.Score-b.Score) > 1e-9 { + if math.Abs(a.Score-b.Score) > rowScoreTolerance { return a.Score > b.Score } return a.Top < b.Top } +// rowScoreTolerance is how close two row scores must be before the higher row +// is preferred. +const rowScoreTolerance = 0.15 + func rowMatchesID(n *Node, rowID string) bool { if rowID == "" { return true @@ -170,10 +177,12 @@ func textDescendants(n *Node) []string { // rowTexts returns the text that belongs to the row itself: its title and // subtitle. // -// Anything inside a clickable child is an action button ("More options", "Add -// to library", "Add suggestion"), not the row's own text, so that subtree is -// skipped. This is what keeps a one-line search suggestion from looking like a -// two-line result. +// Two things are left out on purpose. Anything inside a clickable child is an +// action button ("More options", "Add to library", "Add suggestion"), not the +// row's own text, so that subtree is skipped. And content descriptions are not +// counted, because that is where apps put *state* ("Paused", "Playing") rather +// than the row's title; counting them made the row of the track that was +// already loaded score worse than an identical duplicate further down. func rowTexts(n *Node) []string { var out []string seen := map[string]bool{} @@ -192,9 +201,6 @@ func rowTexts(n *Node) []string { return } add(c.Text) - if !c.Clickable { - add(c.ContentDesc) - } for i := range c.Children { visit(&c.Children[i], false) } diff --git a/internal/ui/rows_test.go b/internal/ui/rows_test.go index 745b9f5..d5ace4f 100644 --- a/internal/ui/rows_test.go +++ b/internal/ui/rows_test.go @@ -18,9 +18,12 @@ const spotifyRowDump = ` - - - + + + + + + @@ -124,6 +127,46 @@ func TestRowScorePrefersPrecision(t *testing.T) { } } +// TestBestRowPrefersTheTopHitWhenItsRowCarriesStateText is the regression that +// cost the most: the top hit's row carries a "Paused" state indicator once it is +// the loaded track, and counting that description as row text pushed the score +// below an identical duplicate further down the list. tap_row then pressed the +// duplicate at y=1443 and playback never started. +func TestBestRowPrefersTheTopHitWhenItsRowCarriesStateText(t *testing.T) { + h := spotifyRows(t) + row, err := h.BestRow("Rolling Stones Honky Tonk", spotifyRowID, 1080, 2400) + if err != nil { + t.Fatalf("BestRow: %v", err) + } + if row.Title != "Honky Tonk Women" { + t.Fatalf("chose %q", row.Title) + } + if row.Top != 794 { + t.Fatalf("chose the row at y=%d, want the top hit at y=794", row.Top) + } + if row.Centre.Y != (794+962)/2 { + t.Fatalf("tap point y=%d is not the top row's centre", row.Centre.Y) + } +} + +func TestRowTextsIgnoreStateDescriptions(t *testing.T) { + h := spotifyRows(t) + var top *Node + h.Walk(func(n *Node) { + if n.ResourceID == spotifyRowID && n.Bounds == "[0,794][1080,962]" { + top = n + } + }) + if top == nil { + t.Fatal("top row not found in the sample dump") + } + for _, line := range rowTexts(top) { + if line == "Paused" { + t.Fatal("the row's state description must not count as row text") + } + } +} + func TestRowTextsSkipActionButtons(t *testing.T) { h := spotifyRows(t) diff --git a/recipes/spotify_play.recipe b/recipes/spotify_play.recipe index b58a9a2..d701e1c 100644 --- a/recipes/spotify_play.recipe +++ b/recipes/spotify_play.recipe @@ -65,11 +65,13 @@ steps: # go to the highest row, which is how Spotify ranks them. - tap_row "{query}" 40s - # 7. Playback started when the transport button says "Pause". Exact match, so - # Spotify's "Paused" labels do not count. + # 7. Confirm playback by asking Android, not by reading the screen. Spotify's + # transport button keeps the label "Play" while music is playing, so a + # label check fails on a phone that is happily playing the track. "playing" + # waits until the app's media session says PLAYING and its track title + # matches the query. # - # This is the phone's slowest moment: the app is rendering a new list and - # can stop answering input for a while, so a tap can be dropped. "retry 3" - # presses the row again and waits, up to three more times. It never taps - # anything it did not match by text. - - wait exact "Pause" 45s retry 3 + # This is the phone's slowest moment, and a tap can be dropped, so "retry 3" + # presses the row again and checks again, up to three more times. It never + # taps anything it did not match by text. + - playing "{query}" 30s retry 3