feat: Phase 4 — spotify_radio, and a bound on logs/runs.jsonl

spotify_radio opens a track's own context menu and starts its song radio.
The labels came from a fresh uiautomator dump on the phone, not from a guess:
the entry is "Go to song radio" and it sits below the fold, so the recipe
scrolls before it matches.

Recipe format:
- menu_row "{query}" finds the row like tap_row, then presses the action
  button inside that row's own subtree (ui.Row.Action), so the More options
  button of any other row is never a candidate. It records the row as {row}.
- a step can name its own failure: wait ... else "reason". The named reason
  comes first and the screen's labels still follow (FR-8).
- scroll up|down moves the current list.
- playing with no title waits for the media session to be PLAYING. A radio
  changes track every few minutes, so a title match would fail on a working
  radio; spotify_play still passes the title it wants.

ui:
- pickBestMatch ignores a match with no usable bounds. An element scrolled out
  of its list reports [0,0][0,0], and tapping its "centre" would be a blind tap
  in the corner. A wait now only succeeds once the element is on screen.

Run log (FR-10): appendRunLog rotates runs.jsonl when a write would pass
ARTEMIS_LOG_MAX_BYTES (default 1 MiB), keeping ARTEMIS_LOG_KEEP (default 5)
files. History is bounded, never deleted wholesale, and the per-entry cap of
6 screen labels of 60 characters is unchanged.

Live evidence in docs/phase-4-song-radio.md: ok:true in 56.6s, the menu that
opened belonged to the pressed row, the station name "Honky Tonk Women Radio"
on the phone, and the radio playing past its seed track (Layla) per the media
session and dumpsys audio. Both new failure modes were demonstrated failing
with distinct reasons. The mini-bar's transport label stays "Play" while
playing on this build, which is why playback is confirmed from the media
session.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
2026-10-16 15:25:50 +11:00
parent 40a8a51c9e
commit 36b4dd2391
11 changed files with 1116 additions and 34 deletions

View File

@@ -92,15 +92,25 @@ type tapCandidate struct {
// pickBestMatch chooses the most tappable node from a set of text matches.
// Clickable and enabled nodes win; among equals the smallest node wins, because
// the smallest enclosing element is the control itself rather than its container.
//
// A node without usable bounds is never a tap target. An element that is not on
// screen (scrolled out of a list, or not laid out yet) reports bounds of
// [0,0][0,0], and tapping its "centre" would be a blind tap at the top-left of
// the screen. Such a match is skipped, and a set with no usable match fails
// (FR-8) rather than tapping anything.
func pickBestMatch(matches []*Node) (*Node, error) {
candidates := make([]tapCandidate, 0, len(matches))
for _, n := range matches {
area := 1 << 30
if r, err := n.BoundsRect(); err == nil {
area = (r.X2 - r.X1) * (r.Y2 - r.Y1)
r, err := n.BoundsRect()
if err != nil {
continue
}
area := (r.X2 - r.X1) * (r.Y2 - r.Y1)
candidates = append(candidates, tapCandidate{node: n, clickable: n.Clickable, enabled: n.Enabled, area: area})
}
if len(candidates) == 0 {
return nil, fmt.Errorf("the matched element has no usable bounds: it is not on screen; refusing to tap blindly")
}
sort.SliceStable(candidates, func(i, j int) bool {
a, b := candidates[i], candidates[j]
if a.clickable != b.clickable {
@@ -168,13 +178,13 @@ func WaitForText(r *adb.Runner, query string, exact bool, timeout time.Duration)
timeout = 10 * time.Second
}
deadline := time.Now().Add(timeout)
var seen []*Node
sawMatch := false
for time.Now().Before(deadline) {
h, err := Dump(r)
if err == nil {
matches := h.FindText(query, exact)
if len(matches) > 0 {
seen = matches
sawMatch = true
if n, perr := pickBestMatch(matches); perr == nil {
return n, nil
}
@@ -182,8 +192,8 @@ func WaitForText(r *adb.Runner, query string, exact bool, timeout time.Duration)
}
time.Sleep(PollInterval)
}
if len(seen) > 0 {
return nil, fmt.Errorf("text %q is on screen but no enabled match could be tapped", query)
if sawMatch {
return nil, fmt.Errorf("text %q is present but never became a usable tap target within %s (it is off screen or disabled)", query, timeout)
}
return nil, fmt.Errorf("text %q did not appear within %s", query, timeout)
}

View File

@@ -89,6 +89,50 @@ func (h *Hierarchy) BestRow(query, rowID string, screenW, screenH int) (*Row, er
return best, nil
}
// Action returns the row's own action button: the smallest clickable element
// inside the row whose text or content description contains label.
//
// The search is confined to the chosen row's subtree, which is what stops a
// recipe from pressing the More options button of a different row. The button
// on the real Spotify screen carries no label of its own (the description sits
// on a non-clickable child), so each candidate button's subtree is searched
// too, and the smallest match wins: a larger ancestor is the row itself.
func (row *Row) Action(label string) (*Node, error) {
needle := strings.ToLower(strings.TrimSpace(label))
if needle == "" {
return nil, fmt.Errorf("a row action needs a non-empty label")
}
var best *Node
bestArea := 1 << 30
row.Node.Walk(func(n *Node) {
if n == row.Node || !n.Clickable || !n.Enabled {
return
}
matched := false
for _, text := range textDescendants(n) {
if strings.Contains(strings.ToLower(text), needle) {
matched = true
break
}
}
if !matched {
return
}
area := 1 << 30
if r, err := n.BoundsRect(); err == nil {
area = (r.X2 - r.X1) * (r.Y2 - r.Y1)
}
if best == nil || area < bestArea {
best, bestArea = n, area
}
})
if best == nil {
return nil, fmt.Errorf("row %q has no %q button", row.Title, label)
}
return best, nil
}
// minRowScore is the F1 floor for an acceptable row match.
const minRowScore = 0.5

View File

@@ -215,3 +215,54 @@ func TestTokenize(t *testing.T) {
}
}
}
// TestRowActionFindsTheButtonsInsideTheChosenRow is the safety property behind
// menu_row: the More options button is taken from the row's own subtree, so a
// recipe can never press the button of a different row.
func TestRowActionFindsTheButtonsInsideTheChosenRow(t *testing.T) {
h := spotifyRows(t)
row, err := h.BestRow("Rolling Stones Honky Tonk", spotifyRowID, 1080, 2400)
if err != nil {
t.Fatalf("BestRow: %v", err)
}
if row.Title != "Honky Tonk Women" {
t.Fatalf("chose %q", row.Title)
}
button, err := row.Action("More options")
if err != nil {
t.Fatalf("Action: %v", err)
}
// The top hit's menu button, not the duplicate row's at y=1380.
if button.Bounds != "[828,815][954,941]" {
t.Fatalf("chose button %s, want the top row's", button.Bounds)
}
if !strings.Contains(button.ContentDesc, "More options for song Honky Tonk Women") {
t.Fatalf("chose %q", button.ContentDesc)
}
// A label the row does not carry fails instead of falling back to any
// clickable node.
if _, err := row.Action("Share"); err == nil {
t.Fatal("expected an error for a label the row does not have")
}
}
// TestRowActionRefusesARowWithoutAButton covers a result row that has no action
// bar at all: the step must fail, not tap the row itself.
func TestRowActionRefusesARowWithoutAButton(t *testing.T) {
h := spotifyRows(t)
var bare *Node
h.Walk(func(n *Node) {
if n.ResourceID == spotifyRowID && n.Bounds == "[0,962][1080,1130]" {
bare = n
}
})
if bare == nil {
t.Fatal("the sample row without an action bar was not found")
}
row := &Row{Node: bare, Title: "Honky Tonk Women - Mono Version"}
if _, err := row.Action("More options"); err == nil {
t.Fatal("a row with no action bar must not yield a button")
}
}