Files
artemis_control/internal/ui/rows.go
Sam Rolfe 2604fea483 feat: Phase 3 recipes file, spotify_play, and locked-screen refusal
Phase 3 of AGENTS.md section 11.

Recipes file (FR-5)
- recipes/ holds one hand-editable file per task; recipes/spotify_play.recipe
  is the first one.
- internal/recipe parses directives, guards, parameters and steps, and runs
  them with a per-step and a whole-run timeout (FR-11).
- Steps name text ("tap the element whose text is Search"), never coordinates.
- guards name screens Artemis must never tap on, so a signed-out or
  first-run Spotify fails with its own reason (FR-8).
- Every run appends task, params, steps, result and duration to
  logs/runs.jsonl (FR-10).

spotify_play
- launch, search, play, then wait for the transport button to say "Pause".
- internal/ui/rows.go adds tap_row: a result row is a clickable container with
  a title and a subtitle, so a one-line search suggestion is not a result, and
  the tap lands on the row's centre rather than on its "More options" button.
  This fixes a real wrong tap: a plain text tap hit a 126x12 pixel
  "More options for song Honky Tonk Women - Live" button.

Locked screen (AGENTS.md section 13 Q2, decided)
- Artemis fails loudly on a secure keyguard and never types a PIN. No secret is
  stored. A non-secure keyguard is still dismissed.
- internal/ui/keyguard.go reads the KeyguardServiceDelegate block plus
  cmd lock_settings is-user-secure.
- Every acting command wakes the screen before it checks the keyguard: a dozing
  phone reports showing=false, because the lock screen is not drawn yet.
- AGENTS.md records the decision and that storing a PIN for auto-unlock is
  rejected for now.

Verified live: the phone locked itself mid-session, and keyguard, tap and
run spotify_play all refused with "phone is locked; unlock it and retry".
The live end-to-end playback run still needs an unlocked phone.
2026-10-16 13:24:22 +11:00

253 lines
6.5 KiB
Go

package ui
import (
"fmt"
"math"
"strings"
"unicode"
)
// Rows are the only safe way to press a list result.
//
// A search result row is a clickable container that holds a title and a
// subtitle ("Honky Tonk Women" / "Song • The Rolling Stones"). The container is
// the text-matched element, and its centre is the tap point: that lands on the
// title, never on the row's "More options" or "Add to library" buttons, which
// sit in a small action bar at the row's right edge.
//
// Selecting the row instead of the smallest clickable child matters. On the
// real Spotify screen a plain text match resolved to a 126x12 pixel "More
// options for song ..." button, because that was the smallest clickable node.
// Row is one pressable list row.
type Row struct {
Node *Node
Texts []string
Score float64
Title string
Top int
Centre Point
}
// Point is a screen coordinate.
type Point struct {
X, Y int
}
// BestRow chooses the row that best matches query.
//
// rowID, when set, restricts candidates to elements with that resource id
// (either the full "package:id/name" form or a suffix of it).
func (h *Hierarchy) BestRow(query, rowID string, screenW, screenH int) (*Row, error) {
query = strings.TrimSpace(query)
if query == "" {
return nil, fmt.Errorf("row match needs a non-empty query")
}
qTokens := tokenize(query)
if len(qTokens) == 0 {
return nil, fmt.Errorf("query %q has no matchable words", query)
}
candidates := h.rowCandidates(rowID, screenW, screenH)
if len(candidates) == 0 {
return nil, fmt.Errorf("no result row is fully drawn on screen; refusing to tap blindly")
}
var best *Row
for _, node := range candidates {
rect, err := node.BoundsRect()
if err != nil {
continue
}
texts := rowTexts(node)
score, matched := rowScore(qTokens, texts)
// More than half of the query words must be on the row, and the row
// must not be buried under unrelated words.
if matched*2 < len(qTokens) || score < minRowScore {
continue
}
cx, cy, err := node.Center()
if err != nil {
continue
}
candidate := &Row{
Node: node,
Texts: texts,
Score: score,
Title: texts[0],
Top: rect.Y1,
Centre: Point{X: cx, Y: cy},
}
if best == nil || betterRow(candidate, best) {
best = candidate
}
}
if best == nil {
return nil, fmt.Errorf("no result row matches %q; refusing to tap blindly", query)
}
return best, nil
}
// minRowScore is the F1 floor for an acceptable row match.
const minRowScore = 0.5
// rowCandidates returns the fully drawn rows on screen: clickable containers
// that carry at least a title and a subtitle.
//
// Two shapes are rejected on purpose. A one-line search *suggestion* is not a
// result. And a row clipped by the bottom of the list is much shorter than a
// row that is fully scrolled into view, so its centre is not a point a human
// could press; rows shorter than half the tallest candidate are dropped.
func (h *Hierarchy) rowCandidates(rowID string, screenW, screenH int) []*Node {
var found []*Node
tallest := 0
h.Walk(func(n *Node) {
if !n.Clickable || !n.Enabled || !rowMatchesID(n, rowID) {
return
}
rect, err := n.BoundsRect()
if err != nil || !onScreen(rect, screenW, screenH) {
return
}
if len(rowTexts(n)) < 2 {
return
}
if h := rect.Y2 - rect.Y1; h > tallest {
tallest = h
}
found = append(found, n)
})
var out []*Node
for _, n := range found {
rect, err := n.BoundsRect()
if err != nil {
continue
}
if (rect.Y2-rect.Y1)*2 < tallest {
continue
}
out = append(out, n)
}
return out
}
// betterRow ranks rows: the best text match wins, and equal matches go to the
// row nearest the top, because search results are ranked by relevance.
func betterRow(a, b *Row) bool {
if math.Abs(a.Score-b.Score) > 1e-9 {
return a.Score > b.Score
}
return a.Top < b.Top
}
func rowMatchesID(n *Node, rowID string) bool {
if rowID == "" {
return true
}
return n.ResourceID == rowID || strings.HasSuffix(n.ResourceID, rowID)
}
// textDescendants returns the visible text lines inside a node, in reading
// order, without duplicates. It is used for dumps and diagnostics.
func textDescendants(n *Node) []string {
var out []string
seen := map[string]bool{}
n.Walk(func(c *Node) {
for _, t := range []string{c.Text, c.ContentDesc} {
t = strings.TrimSpace(t)
if t == "" || seen[t] {
continue
}
seen[t] = true
out = append(out, t)
}
})
return out
}
// rowTexts returns the text that belongs to the row itself: its title and
// subtitle.
//
// Anything inside a clickable child is an action button ("More options", "Add
// to library", "Add suggestion"), not the row's own text, so that subtree is
// skipped. This is what keeps a one-line search suggestion from looking like a
// two-line result.
func rowTexts(n *Node) []string {
var out []string
seen := map[string]bool{}
add := func(s string) {
s = strings.TrimSpace(s)
if s == "" || seen[s] {
return
}
seen[s] = true
out = append(out, s)
}
var visit func(c *Node, root bool)
visit = func(c *Node, root bool) {
if !root && c.Clickable {
return
}
add(c.Text)
if !c.Clickable {
add(c.ContentDesc)
}
for i := range c.Children {
visit(&c.Children[i], false)
}
}
visit(n, true)
return out
}
// onScreen reports whether the whole rectangle is inside the display. A row
// clipped by the screen edge would be tapped at a point the user cannot see,
// so it is not a candidate.
func onScreen(r Rect, w, h int) bool {
if w <= 0 || h <= 0 {
return true
}
return r.X1 >= 0 && r.Y1 >= 0 && r.X2 <= w && r.Y2 <= h
}
// rowScore returns the F1 score of the query words against the row's words,
// plus how many distinct query words were found.
func rowScore(query []string, texts []string) (score float64, matched int) {
row := map[string]bool{}
for _, t := range texts {
for _, w := range tokenize(t) {
row[w] = true
}
}
hit := map[string]bool{}
for _, w := range query {
if row[w] {
hit[w] = true
}
}
if len(hit) == 0 || len(row) == 0 {
return 0, 0
}
precision := float64(len(hit)) / float64(len(row))
recall := float64(len(hit)) / float64(len(query))
if precision+recall == 0 {
return 0, len(hit)
}
return 2 * precision * recall / (precision + recall), len(hit)
}
// tokenize splits text into lowercase words.
func tokenize(s string) []string {
fields := strings.FieldsFunc(s, func(r rune) bool {
return !unicode.IsLetter(r) && !unicode.IsDigit(r)
})
out := make([]string, 0, len(fields))
for _, f := range fields {
out = append(out, strings.ToLower(f))
}
return out
}