feat: Phase 3 recipes file, spotify_play, and locked-screen refusal
Phase 3 of AGENTS.md section 11.
Recipes file (FR-5)
- recipes/ holds one hand-editable file per task; recipes/spotify_play.recipe
is the first one.
- internal/recipe parses directives, guards, parameters and steps, and runs
them with a per-step and a whole-run timeout (FR-11).
- Steps name text ("tap the element whose text is Search"), never coordinates.
- guards name screens Artemis must never tap on, so a signed-out or
first-run Spotify fails with its own reason (FR-8).
- Every run appends task, params, steps, result and duration to
logs/runs.jsonl (FR-10).
spotify_play
- launch, search, play, then wait for the transport button to say "Pause".
- internal/ui/rows.go adds tap_row: a result row is a clickable container with
a title and a subtitle, so a one-line search suggestion is not a result, and
the tap lands on the row's centre rather than on its "More options" button.
This fixes a real wrong tap: a plain text tap hit a 126x12 pixel
"More options for song Honky Tonk Women - Live" button.
Locked screen (AGENTS.md section 13 Q2, decided)
- Artemis fails loudly on a secure keyguard and never types a PIN. No secret is
stored. A non-secure keyguard is still dismissed.
- internal/ui/keyguard.go reads the KeyguardServiceDelegate block plus
cmd lock_settings is-user-secure.
- Every acting command wakes the screen before it checks the keyguard: a dozing
phone reports showing=false, because the lock screen is not drawn yet.
- AGENTS.md records the decision and that storing a PIN for auto-unlock is
rejected for now.
Verified live: the phone locked itself mid-session, and keyguard, tap and
run spotify_play all refused with "phone is locked; unlock it and retry".
The live end-to-end playback run still needs an unlocked phone.
This commit is contained in:
252
internal/ui/rows.go
Normal file
252
internal/ui/rows.go
Normal file
@@ -0,0 +1,252 @@
|
||||
package ui
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// Rows are the only safe way to press a list result.
|
||||
//
|
||||
// A search result row is a clickable container that holds a title and a
|
||||
// subtitle ("Honky Tonk Women" / "Song • The Rolling Stones"). The container is
|
||||
// the text-matched element, and its centre is the tap point: that lands on the
|
||||
// title, never on the row's "More options" or "Add to library" buttons, which
|
||||
// sit in a small action bar at the row's right edge.
|
||||
//
|
||||
// Selecting the row instead of the smallest clickable child matters. On the
|
||||
// real Spotify screen a plain text match resolved to a 126x12 pixel "More
|
||||
// options for song ..." button, because that was the smallest clickable node.
|
||||
|
||||
// Row is one pressable list row.
|
||||
type Row struct {
|
||||
Node *Node
|
||||
Texts []string
|
||||
Score float64
|
||||
Title string
|
||||
Top int
|
||||
Centre Point
|
||||
}
|
||||
|
||||
// Point is a screen coordinate.
|
||||
type Point struct {
|
||||
X, Y int
|
||||
}
|
||||
|
||||
// BestRow chooses the row that best matches query.
|
||||
//
|
||||
// rowID, when set, restricts candidates to elements with that resource id
|
||||
// (either the full "package:id/name" form or a suffix of it).
|
||||
func (h *Hierarchy) BestRow(query, rowID string, screenW, screenH int) (*Row, error) {
|
||||
query = strings.TrimSpace(query)
|
||||
if query == "" {
|
||||
return nil, fmt.Errorf("row match needs a non-empty query")
|
||||
}
|
||||
qTokens := tokenize(query)
|
||||
if len(qTokens) == 0 {
|
||||
return nil, fmt.Errorf("query %q has no matchable words", query)
|
||||
}
|
||||
|
||||
candidates := h.rowCandidates(rowID, screenW, screenH)
|
||||
if len(candidates) == 0 {
|
||||
return nil, fmt.Errorf("no result row is fully drawn on screen; refusing to tap blindly")
|
||||
}
|
||||
|
||||
var best *Row
|
||||
for _, node := range candidates {
|
||||
rect, err := node.BoundsRect()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
texts := rowTexts(node)
|
||||
score, matched := rowScore(qTokens, texts)
|
||||
// More than half of the query words must be on the row, and the row
|
||||
// must not be buried under unrelated words.
|
||||
if matched*2 < len(qTokens) || score < minRowScore {
|
||||
continue
|
||||
}
|
||||
cx, cy, err := node.Center()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
candidate := &Row{
|
||||
Node: node,
|
||||
Texts: texts,
|
||||
Score: score,
|
||||
Title: texts[0],
|
||||
Top: rect.Y1,
|
||||
Centre: Point{X: cx, Y: cy},
|
||||
}
|
||||
if best == nil || betterRow(candidate, best) {
|
||||
best = candidate
|
||||
}
|
||||
}
|
||||
|
||||
if best == nil {
|
||||
return nil, fmt.Errorf("no result row matches %q; refusing to tap blindly", query)
|
||||
}
|
||||
return best, nil
|
||||
}
|
||||
|
||||
// minRowScore is the F1 floor for an acceptable row match.
|
||||
const minRowScore = 0.5
|
||||
|
||||
// rowCandidates returns the fully drawn rows on screen: clickable containers
|
||||
// that carry at least a title and a subtitle.
|
||||
//
|
||||
// Two shapes are rejected on purpose. A one-line search *suggestion* is not a
|
||||
// result. And a row clipped by the bottom of the list is much shorter than a
|
||||
// row that is fully scrolled into view, so its centre is not a point a human
|
||||
// could press; rows shorter than half the tallest candidate are dropped.
|
||||
func (h *Hierarchy) rowCandidates(rowID string, screenW, screenH int) []*Node {
|
||||
var found []*Node
|
||||
tallest := 0
|
||||
h.Walk(func(n *Node) {
|
||||
if !n.Clickable || !n.Enabled || !rowMatchesID(n, rowID) {
|
||||
return
|
||||
}
|
||||
rect, err := n.BoundsRect()
|
||||
if err != nil || !onScreen(rect, screenW, screenH) {
|
||||
return
|
||||
}
|
||||
if len(rowTexts(n)) < 2 {
|
||||
return
|
||||
}
|
||||
if h := rect.Y2 - rect.Y1; h > tallest {
|
||||
tallest = h
|
||||
}
|
||||
found = append(found, n)
|
||||
})
|
||||
|
||||
var out []*Node
|
||||
for _, n := range found {
|
||||
rect, err := n.BoundsRect()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if (rect.Y2-rect.Y1)*2 < tallest {
|
||||
continue
|
||||
}
|
||||
out = append(out, n)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// betterRow ranks rows: the best text match wins, and equal matches go to the
|
||||
// row nearest the top, because search results are ranked by relevance.
|
||||
func betterRow(a, b *Row) bool {
|
||||
if math.Abs(a.Score-b.Score) > 1e-9 {
|
||||
return a.Score > b.Score
|
||||
}
|
||||
return a.Top < b.Top
|
||||
}
|
||||
|
||||
func rowMatchesID(n *Node, rowID string) bool {
|
||||
if rowID == "" {
|
||||
return true
|
||||
}
|
||||
return n.ResourceID == rowID || strings.HasSuffix(n.ResourceID, rowID)
|
||||
}
|
||||
|
||||
// textDescendants returns the visible text lines inside a node, in reading
|
||||
// order, without duplicates. It is used for dumps and diagnostics.
|
||||
func textDescendants(n *Node) []string {
|
||||
var out []string
|
||||
seen := map[string]bool{}
|
||||
n.Walk(func(c *Node) {
|
||||
for _, t := range []string{c.Text, c.ContentDesc} {
|
||||
t = strings.TrimSpace(t)
|
||||
if t == "" || seen[t] {
|
||||
continue
|
||||
}
|
||||
seen[t] = true
|
||||
out = append(out, t)
|
||||
}
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// rowTexts returns the text that belongs to the row itself: its title and
|
||||
// subtitle.
|
||||
//
|
||||
// Anything inside a clickable child is an action button ("More options", "Add
|
||||
// to library", "Add suggestion"), not the row's own text, so that subtree is
|
||||
// skipped. This is what keeps a one-line search suggestion from looking like a
|
||||
// two-line result.
|
||||
func rowTexts(n *Node) []string {
|
||||
var out []string
|
||||
seen := map[string]bool{}
|
||||
add := func(s string) {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" || seen[s] {
|
||||
return
|
||||
}
|
||||
seen[s] = true
|
||||
out = append(out, s)
|
||||
}
|
||||
|
||||
var visit func(c *Node, root bool)
|
||||
visit = func(c *Node, root bool) {
|
||||
if !root && c.Clickable {
|
||||
return
|
||||
}
|
||||
add(c.Text)
|
||||
if !c.Clickable {
|
||||
add(c.ContentDesc)
|
||||
}
|
||||
for i := range c.Children {
|
||||
visit(&c.Children[i], false)
|
||||
}
|
||||
}
|
||||
visit(n, true)
|
||||
return out
|
||||
}
|
||||
|
||||
// onScreen reports whether the whole rectangle is inside the display. A row
|
||||
// clipped by the screen edge would be tapped at a point the user cannot see,
|
||||
// so it is not a candidate.
|
||||
func onScreen(r Rect, w, h int) bool {
|
||||
if w <= 0 || h <= 0 {
|
||||
return true
|
||||
}
|
||||
return r.X1 >= 0 && r.Y1 >= 0 && r.X2 <= w && r.Y2 <= h
|
||||
}
|
||||
|
||||
// rowScore returns the F1 score of the query words against the row's words,
|
||||
// plus how many distinct query words were found.
|
||||
func rowScore(query []string, texts []string) (score float64, matched int) {
|
||||
row := map[string]bool{}
|
||||
for _, t := range texts {
|
||||
for _, w := range tokenize(t) {
|
||||
row[w] = true
|
||||
}
|
||||
}
|
||||
hit := map[string]bool{}
|
||||
for _, w := range query {
|
||||
if row[w] {
|
||||
hit[w] = true
|
||||
}
|
||||
}
|
||||
if len(hit) == 0 || len(row) == 0 {
|
||||
return 0, 0
|
||||
}
|
||||
precision := float64(len(hit)) / float64(len(row))
|
||||
recall := float64(len(hit)) / float64(len(query))
|
||||
if precision+recall == 0 {
|
||||
return 0, len(hit)
|
||||
}
|
||||
return 2 * precision * recall / (precision + recall), len(hit)
|
||||
}
|
||||
|
||||
// tokenize splits text into lowercase words.
|
||||
func tokenize(s string) []string {
|
||||
fields := strings.FieldsFunc(s, func(r rune) bool {
|
||||
return !unicode.IsLetter(r) && !unicode.IsDigit(r)
|
||||
})
|
||||
out := make([]string, 0, len(fields))
|
||||
for _, f := range fields {
|
||||
out = append(out, strings.ToLower(f))
|
||||
}
|
||||
return out
|
||||
}
|
||||
Reference in New Issue
Block a user