Phase 3 of AGENTS.md section 11.
Recipes file (FR-5)
- recipes/ holds one hand-editable file per task; recipes/spotify_play.recipe
is the first one.
- internal/recipe parses directives, guards, parameters and steps, and runs
them with a per-step and a whole-run timeout (FR-11).
- Steps name text ("tap the element whose text is Search"), never coordinates.
- guards name screens Artemis must never tap on, so a signed-out or
first-run Spotify fails with its own reason (FR-8).
- Every run appends task, params, steps, result and duration to
logs/runs.jsonl (FR-10).
spotify_play
- launch, search, play, then wait for the transport button to say "Pause".
- internal/ui/rows.go adds tap_row: a result row is a clickable container with
a title and a subtitle, so a one-line search suggestion is not a result, and
the tap lands on the row's centre rather than on its "More options" button.
This fixes a real wrong tap: a plain text tap hit a 126x12 pixel
"More options for song Honky Tonk Women - Live" button.
Locked screen (AGENTS.md section 13 Q2, decided)
- Artemis fails loudly on a secure keyguard and never types a PIN. No secret is
stored. A non-secure keyguard is still dismissed.
- internal/ui/keyguard.go reads the KeyguardServiceDelegate block plus
cmd lock_settings is-user-secure.
- Every acting command wakes the screen before it checks the keyguard: a dozing
phone reports showing=false, because the lock screen is not drawn yet.
- AGENTS.md records the decision and that storing a PIN for auto-unlock is
rejected for now.
Verified live: the phone locked itself mid-session, and keyguard, tap and
run spotify_play all refused with "phone is locked; unlock it and retry".
The live end-to-end playback run still needs an unlocked phone.
253 lines
6.5 KiB
Go
253 lines
6.5 KiB
Go
package ui
|
|
|
|
import (
|
|
"fmt"
|
|
"math"
|
|
"strings"
|
|
"unicode"
|
|
)
|
|
|
|
// Rows are the only safe way to press a list result.
|
|
//
|
|
// A search result row is a clickable container that holds a title and a
|
|
// subtitle ("Honky Tonk Women" / "Song • The Rolling Stones"). The container is
|
|
// the text-matched element, and its centre is the tap point: that lands on the
|
|
// title, never on the row's "More options" or "Add to library" buttons, which
|
|
// sit in a small action bar at the row's right edge.
|
|
//
|
|
// Selecting the row instead of the smallest clickable child matters. On the
|
|
// real Spotify screen a plain text match resolved to a 126x12 pixel "More
|
|
// options for song ..." button, because that was the smallest clickable node.
|
|
|
|
// Row is one pressable list row.
|
|
type Row struct {
|
|
Node *Node
|
|
Texts []string
|
|
Score float64
|
|
Title string
|
|
Top int
|
|
Centre Point
|
|
}
|
|
|
|
// Point is a screen coordinate.
|
|
type Point struct {
|
|
X, Y int
|
|
}
|
|
|
|
// BestRow chooses the row that best matches query.
|
|
//
|
|
// rowID, when set, restricts candidates to elements with that resource id
|
|
// (either the full "package:id/name" form or a suffix of it).
|
|
func (h *Hierarchy) BestRow(query, rowID string, screenW, screenH int) (*Row, error) {
|
|
query = strings.TrimSpace(query)
|
|
if query == "" {
|
|
return nil, fmt.Errorf("row match needs a non-empty query")
|
|
}
|
|
qTokens := tokenize(query)
|
|
if len(qTokens) == 0 {
|
|
return nil, fmt.Errorf("query %q has no matchable words", query)
|
|
}
|
|
|
|
candidates := h.rowCandidates(rowID, screenW, screenH)
|
|
if len(candidates) == 0 {
|
|
return nil, fmt.Errorf("no result row is fully drawn on screen; refusing to tap blindly")
|
|
}
|
|
|
|
var best *Row
|
|
for _, node := range candidates {
|
|
rect, err := node.BoundsRect()
|
|
if err != nil {
|
|
continue
|
|
}
|
|
texts := rowTexts(node)
|
|
score, matched := rowScore(qTokens, texts)
|
|
// More than half of the query words must be on the row, and the row
|
|
// must not be buried under unrelated words.
|
|
if matched*2 < len(qTokens) || score < minRowScore {
|
|
continue
|
|
}
|
|
cx, cy, err := node.Center()
|
|
if err != nil {
|
|
continue
|
|
}
|
|
candidate := &Row{
|
|
Node: node,
|
|
Texts: texts,
|
|
Score: score,
|
|
Title: texts[0],
|
|
Top: rect.Y1,
|
|
Centre: Point{X: cx, Y: cy},
|
|
}
|
|
if best == nil || betterRow(candidate, best) {
|
|
best = candidate
|
|
}
|
|
}
|
|
|
|
if best == nil {
|
|
return nil, fmt.Errorf("no result row matches %q; refusing to tap blindly", query)
|
|
}
|
|
return best, nil
|
|
}
|
|
|
|
// minRowScore is the F1 floor for an acceptable row match.
|
|
const minRowScore = 0.5
|
|
|
|
// rowCandidates returns the fully drawn rows on screen: clickable containers
|
|
// that carry at least a title and a subtitle.
|
|
//
|
|
// Two shapes are rejected on purpose. A one-line search *suggestion* is not a
|
|
// result. And a row clipped by the bottom of the list is much shorter than a
|
|
// row that is fully scrolled into view, so its centre is not a point a human
|
|
// could press; rows shorter than half the tallest candidate are dropped.
|
|
func (h *Hierarchy) rowCandidates(rowID string, screenW, screenH int) []*Node {
|
|
var found []*Node
|
|
tallest := 0
|
|
h.Walk(func(n *Node) {
|
|
if !n.Clickable || !n.Enabled || !rowMatchesID(n, rowID) {
|
|
return
|
|
}
|
|
rect, err := n.BoundsRect()
|
|
if err != nil || !onScreen(rect, screenW, screenH) {
|
|
return
|
|
}
|
|
if len(rowTexts(n)) < 2 {
|
|
return
|
|
}
|
|
if h := rect.Y2 - rect.Y1; h > tallest {
|
|
tallest = h
|
|
}
|
|
found = append(found, n)
|
|
})
|
|
|
|
var out []*Node
|
|
for _, n := range found {
|
|
rect, err := n.BoundsRect()
|
|
if err != nil {
|
|
continue
|
|
}
|
|
if (rect.Y2-rect.Y1)*2 < tallest {
|
|
continue
|
|
}
|
|
out = append(out, n)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// betterRow ranks rows: the best text match wins, and equal matches go to the
|
|
// row nearest the top, because search results are ranked by relevance.
|
|
func betterRow(a, b *Row) bool {
|
|
if math.Abs(a.Score-b.Score) > 1e-9 {
|
|
return a.Score > b.Score
|
|
}
|
|
return a.Top < b.Top
|
|
}
|
|
|
|
func rowMatchesID(n *Node, rowID string) bool {
|
|
if rowID == "" {
|
|
return true
|
|
}
|
|
return n.ResourceID == rowID || strings.HasSuffix(n.ResourceID, rowID)
|
|
}
|
|
|
|
// textDescendants returns the visible text lines inside a node, in reading
|
|
// order, without duplicates. It is used for dumps and diagnostics.
|
|
func textDescendants(n *Node) []string {
|
|
var out []string
|
|
seen := map[string]bool{}
|
|
n.Walk(func(c *Node) {
|
|
for _, t := range []string{c.Text, c.ContentDesc} {
|
|
t = strings.TrimSpace(t)
|
|
if t == "" || seen[t] {
|
|
continue
|
|
}
|
|
seen[t] = true
|
|
out = append(out, t)
|
|
}
|
|
})
|
|
return out
|
|
}
|
|
|
|
// rowTexts returns the text that belongs to the row itself: its title and
|
|
// subtitle.
|
|
//
|
|
// Anything inside a clickable child is an action button ("More options", "Add
|
|
// to library", "Add suggestion"), not the row's own text, so that subtree is
|
|
// skipped. This is what keeps a one-line search suggestion from looking like a
|
|
// two-line result.
|
|
func rowTexts(n *Node) []string {
|
|
var out []string
|
|
seen := map[string]bool{}
|
|
add := func(s string) {
|
|
s = strings.TrimSpace(s)
|
|
if s == "" || seen[s] {
|
|
return
|
|
}
|
|
seen[s] = true
|
|
out = append(out, s)
|
|
}
|
|
|
|
var visit func(c *Node, root bool)
|
|
visit = func(c *Node, root bool) {
|
|
if !root && c.Clickable {
|
|
return
|
|
}
|
|
add(c.Text)
|
|
if !c.Clickable {
|
|
add(c.ContentDesc)
|
|
}
|
|
for i := range c.Children {
|
|
visit(&c.Children[i], false)
|
|
}
|
|
}
|
|
visit(n, true)
|
|
return out
|
|
}
|
|
|
|
// onScreen reports whether the whole rectangle is inside the display. A row
|
|
// clipped by the screen edge would be tapped at a point the user cannot see,
|
|
// so it is not a candidate.
|
|
func onScreen(r Rect, w, h int) bool {
|
|
if w <= 0 || h <= 0 {
|
|
return true
|
|
}
|
|
return r.X1 >= 0 && r.Y1 >= 0 && r.X2 <= w && r.Y2 <= h
|
|
}
|
|
|
|
// rowScore returns the F1 score of the query words against the row's words,
|
|
// plus how many distinct query words were found.
|
|
func rowScore(query []string, texts []string) (score float64, matched int) {
|
|
row := map[string]bool{}
|
|
for _, t := range texts {
|
|
for _, w := range tokenize(t) {
|
|
row[w] = true
|
|
}
|
|
}
|
|
hit := map[string]bool{}
|
|
for _, w := range query {
|
|
if row[w] {
|
|
hit[w] = true
|
|
}
|
|
}
|
|
if len(hit) == 0 || len(row) == 0 {
|
|
return 0, 0
|
|
}
|
|
precision := float64(len(hit)) / float64(len(row))
|
|
recall := float64(len(hit)) / float64(len(query))
|
|
if precision+recall == 0 {
|
|
return 0, len(hit)
|
|
}
|
|
return 2 * precision * recall / (precision + recall), len(hit)
|
|
}
|
|
|
|
// tokenize splits text into lowercase words.
|
|
func tokenize(s string) []string {
|
|
fields := strings.FieldsFunc(s, func(r rune) bool {
|
|
return !unicode.IsLetter(r) && !unicode.IsDigit(r)
|
|
})
|
|
out := make([]string, 0, len(fields))
|
|
for _, f := range fields {
|
|
out = append(out, strings.ToLower(f))
|
|
}
|
|
return out
|
|
}
|