feat: Phase 3 recipes file, spotify_play, and locked-screen refusal
Phase 3 of AGENTS.md section 11.
Recipes file (FR-5)
- recipes/ holds one hand-editable file per task; recipes/spotify_play.recipe
is the first one.
- internal/recipe parses directives, guards, parameters and steps, and runs
them with a per-step and a whole-run timeout (FR-11).
- Steps name text ("tap the element whose text is Search"), never coordinates.
- guards name screens Artemis must never tap on, so a signed-out or
first-run Spotify fails with its own reason (FR-8).
- Every run appends task, params, steps, result and duration to
logs/runs.jsonl (FR-10).
spotify_play
- launch, search, play, then wait for the transport button to say "Pause".
- internal/ui/rows.go adds tap_row: a result row is a clickable container with
a title and a subtitle, so a one-line search suggestion is not a result, and
the tap lands on the row's centre rather than on its "More options" button.
This fixes a real wrong tap: a plain text tap hit a 126x12 pixel
"More options for song Honky Tonk Women - Live" button.
Locked screen (AGENTS.md section 13 Q2, decided)
- Artemis fails loudly on a secure keyguard and never types a PIN. No secret is
stored. A non-secure keyguard is still dismissed.
- internal/ui/keyguard.go reads the KeyguardServiceDelegate block plus
cmd lock_settings is-user-secure.
- Every acting command wakes the screen before it checks the keyguard: a dozing
phone reports showing=false, because the lock screen is not drawn yet.
- AGENTS.md records the decision and that storing a PIN for auto-unlock is
rejected for now.
Verified live: the phone locked itself mid-session, and keyguard, tap and
run spotify_play all refused with "phone is locked; unlock it and retry".
The live end-to-end playback run still needs an unlocked phone.
This commit is contained in:
376
internal/recipe/engine.go
Normal file
376
internal/recipe/engine.go
Normal file
@@ -0,0 +1,376 @@
|
||||
package recipe
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.lab.audasmedia.com.au/sam/artemis_control/internal/adb"
|
||||
"gitea.lab.audasmedia.com.au/sam/artemis_control/internal/ui"
|
||||
)
|
||||
|
||||
// StepResult is the record of one executed step (FR-10).
|
||||
type StepResult struct {
|
||||
Index int `json:"step"`
|
||||
Verb string `json:"verb"`
|
||||
Args []string `json:"args,omitempty"`
|
||||
Detail string `json:"detail,omitempty"`
|
||||
OK bool `json:"ok"`
|
||||
Error string `json:"error,omitempty"`
|
||||
DurationMS int64 `json:"duration_ms"`
|
||||
Duration time.Duration `json:"-"`
|
||||
}
|
||||
|
||||
// RunResult is the record of one whole recipe run (FR-10).
|
||||
type RunResult struct {
|
||||
Task string `json:"task"`
|
||||
Params map[string]string `json:"params,omitempty"`
|
||||
OK bool `json:"ok"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Steps []StepResult `json:"steps"`
|
||||
DurationMS int64 `json:"duration_ms"`
|
||||
Duration time.Duration `json:"-"`
|
||||
}
|
||||
|
||||
// Options carry the caller's parameters and where to write the step log.
|
||||
type Options struct {
|
||||
Params map[string]string
|
||||
Log io.Writer
|
||||
}
|
||||
|
||||
// Run executes a recipe against the phone.
|
||||
//
|
||||
// It refuses to start on a locked phone or a missing app, bounds every step
|
||||
// (FR-11), and reports the reason for the first failing step. A run never
|
||||
// returns an error silently: the returned RunResult always says what happened.
|
||||
func Run(r *adb.Runner, rec *Recipe, opts Options) *RunResult {
|
||||
started := time.Now()
|
||||
params := opts.Params
|
||||
if params == nil {
|
||||
params = map[string]string{}
|
||||
}
|
||||
res := &RunResult{Task: rec.Task, Params: params}
|
||||
defer func() {
|
||||
res.Duration = time.Since(started)
|
||||
res.DurationMS = res.Duration.Milliseconds()
|
||||
}()
|
||||
|
||||
logf := func(format string, args ...any) {
|
||||
if opts.Log != nil {
|
||||
fmt.Fprintf(opts.Log, format+"\n", args...)
|
||||
}
|
||||
}
|
||||
|
||||
fail := func(err error) *RunResult {
|
||||
res.OK = false
|
||||
res.Error = err.Error()
|
||||
logf("FAILED %s after %s: %s", rec.Task, time.Since(started).Round(time.Millisecond), res.Error)
|
||||
return res
|
||||
}
|
||||
|
||||
logf("task %s (%s)", rec.Task, rec.Description)
|
||||
|
||||
if missing := rec.MissingParams(params); len(missing) > 0 {
|
||||
return fail(fmt.Errorf("missing required parameter(s): %s", strings.Join(missing, ", ")))
|
||||
}
|
||||
if rec.App != "" {
|
||||
installed, err := ui.AppInstalled(r, rec.App)
|
||||
if err != nil {
|
||||
return fail(fmt.Errorf("could not check whether %s is installed: %w", rec.App, err))
|
||||
}
|
||||
if !installed {
|
||||
return fail(fmt.Errorf("app is not installed: %s", rec.App))
|
||||
}
|
||||
}
|
||||
|
||||
if err := ui.Wake(r); err != nil {
|
||||
return fail(err)
|
||||
}
|
||||
// A secure keyguard is a hard stop: Artemis has no credential and must
|
||||
// never type one (AGENTS.md section 13 Q2).
|
||||
if err := ui.RequireUnlocked(r); err != nil {
|
||||
return fail(err)
|
||||
}
|
||||
|
||||
w, h, _ := ui.ScreenSize(r) // best effort: bounds checking degrades gracefully
|
||||
|
||||
deadline := started.Add(rec.TotalTimeout)
|
||||
for i, step := range rec.Steps {
|
||||
if remaining := time.Until(deadline); remaining <= 0 {
|
||||
return fail(fmt.Errorf("recipe timed out after %s at step %d (%s)", rec.TotalTimeout, i+1, step.Verb))
|
||||
}
|
||||
sr := executeStep(r, rec, step, i, params, w, h, deadline, logf)
|
||||
res.Steps = append(res.Steps, sr)
|
||||
if !sr.OK {
|
||||
return fail(fmt.Errorf("step %d (%s %s) failed: %s", i+1, step.Verb, strings.Join(step.Args, " "), sr.Error))
|
||||
}
|
||||
}
|
||||
|
||||
res.OK = true
|
||||
logf("OK %s in %s", rec.Task, time.Since(started).Round(time.Millisecond))
|
||||
return res
|
||||
}
|
||||
|
||||
func executeStep(r *adb.Runner, rec *Recipe, step Step, index int, params map[string]string,
|
||||
w, h int, deadline time.Time, logf func(string, ...any)) StepResult {
|
||||
|
||||
started := time.Now()
|
||||
sr := StepResult{Index: index + 1, Verb: step.Verb, Args: step.Args}
|
||||
defer func() {
|
||||
sr.Duration = time.Since(started)
|
||||
sr.DurationMS = sr.Duration.Milliseconds()
|
||||
}()
|
||||
|
||||
timeout := step.Timeout
|
||||
if timeout <= 0 {
|
||||
timeout = rec.StepTimeout
|
||||
}
|
||||
if remaining := time.Until(deadline); timeout > remaining {
|
||||
timeout = remaining
|
||||
}
|
||||
if timeout <= 0 {
|
||||
sr.Error = "no time left in the recipe budget"
|
||||
return sr
|
||||
}
|
||||
|
||||
args := make([]string, len(step.Args))
|
||||
for i, a := range step.Args {
|
||||
v, err := Substitute(a, params)
|
||||
if err != nil {
|
||||
sr.Error = err.Error()
|
||||
return sr
|
||||
}
|
||||
args[i] = v
|
||||
}
|
||||
|
||||
stepErr := func() error {
|
||||
switch step.Verb {
|
||||
case "launch":
|
||||
pkg := rec.App
|
||||
if len(args) == 1 {
|
||||
pkg = args[0]
|
||||
}
|
||||
if pkg == "" {
|
||||
return fmt.Errorf("no app named: add an \"app:\" setting or pass a package")
|
||||
}
|
||||
if err := ui.Launch(r, pkg); err != nil {
|
||||
return err
|
||||
}
|
||||
sr.Detail = "launched " + pkg
|
||||
return nil
|
||||
|
||||
case "wait":
|
||||
node, err := ui.WaitForText(r, args[0], step.Exact, timeout)
|
||||
if err != nil {
|
||||
return describeFailure(r, rec, err)
|
||||
}
|
||||
sr.Detail = "found " + node.Label()
|
||||
return nil
|
||||
|
||||
case "wait_gone":
|
||||
if err := ui.WaitForNoText(r, args[0], step.Exact, timeout); err != nil {
|
||||
return describeFailure(r, rec, err)
|
||||
}
|
||||
sr.Detail = fmt.Sprintf("%q is gone", args[0])
|
||||
return nil
|
||||
|
||||
case "tap":
|
||||
return tapByText(r, rec, args[0], step.Exact, timeout, &sr)
|
||||
|
||||
case "tap_if":
|
||||
// Tap only if the text is already on screen. This makes a recipe
|
||||
// tolerant of a screen that remembers state, without ever tapping
|
||||
// something that was not matched by text.
|
||||
h, err := ui.Dump(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
matches := h.FindText(args[0], step.Exact)
|
||||
if len(matches) == 0 {
|
||||
sr.Detail = fmt.Sprintf("%q not on screen, skipped", args[0])
|
||||
return nil
|
||||
}
|
||||
node, err := ui.TapNode(r, matches)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cx, cy, _ := node.Center()
|
||||
sr.Detail = fmt.Sprintf("tapped %q at (%d,%d)", args[0], cx, cy)
|
||||
return nil
|
||||
|
||||
case "tap_row":
|
||||
return tapRow(r, rec, args[0], timeout, w, h, &sr)
|
||||
|
||||
case "type":
|
||||
if err := ui.Type(r, args[0]); err != nil {
|
||||
return err
|
||||
}
|
||||
sr.Detail = fmt.Sprintf("typed %d characters", len(args[0]))
|
||||
return nil
|
||||
|
||||
case "key":
|
||||
return sendKey(r, args[0], &sr)
|
||||
|
||||
case "back":
|
||||
if err := ui.Back(r); err != nil {
|
||||
return err
|
||||
}
|
||||
sr.Detail = "pressed back"
|
||||
return nil
|
||||
|
||||
case "home":
|
||||
if err := ui.Home(r); err != nil {
|
||||
return err
|
||||
}
|
||||
sr.Detail = "pressed home"
|
||||
return nil
|
||||
|
||||
case "sleep":
|
||||
time.Sleep(timeout)
|
||||
sr.Detail = "slept " + timeout.String()
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("unknown step verb %q", step.Verb)
|
||||
}()
|
||||
|
||||
if stepErr != nil {
|
||||
sr.OK = false
|
||||
sr.Error = stepErr.Error()
|
||||
logf(" step %d %s %v: %s", sr.Index, step.Verb, args, sr.Error)
|
||||
return sr
|
||||
}
|
||||
|
||||
sr.OK = true
|
||||
logf(" step %d %s %v: ok (%s)", sr.Index, step.Verb, args, sr.Duration.Round(time.Millisecond))
|
||||
return sr
|
||||
}
|
||||
|
||||
// tapByText finds an element by text and taps its centre (FR-4).
|
||||
//
|
||||
// The element is looked up twice: once to wait for it, and again immediately
|
||||
// before the tap, so the coordinates come from the newest dump. On a busy
|
||||
// phone the first dump can be seconds old by the time the tap is sent.
|
||||
func tapByText(r *adb.Runner, rec *Recipe, query string, exact bool, timeout time.Duration, sr *StepResult) error {
|
||||
if _, err := ui.WaitForText(r, query, exact, timeout); err != nil {
|
||||
return describeFailure(r, rec, err)
|
||||
}
|
||||
h, err := ui.Dump(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
matches := h.FindText(query, exact)
|
||||
if len(matches) == 0 {
|
||||
return describeFailure(r, rec, fmt.Errorf("element with text %q disappeared before the tap", query))
|
||||
}
|
||||
node, err := ui.TapNode(r, matches)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cx, cy, _ := node.Center()
|
||||
sr.Detail = fmt.Sprintf("tapped %q at (%d,%d) %s", query, cx, cy, node.Label())
|
||||
return nil
|
||||
}
|
||||
|
||||
// tapRow taps the list row that best matches the query. It retries the dump
|
||||
// until a matching row is on screen, then taps a point derived from that row's
|
||||
// own bounds.
|
||||
func tapRow(r *adb.Runner, rec *Recipe, query string, timeout time.Duration, w, h int, sr *StepResult) error {
|
||||
deadline := time.Now().Add(timeout)
|
||||
var lastErr error
|
||||
for time.Now().Before(deadline) {
|
||||
hier, err := ui.Dump(r)
|
||||
if err == nil {
|
||||
row, rerr := hier.BestRow(query, rec.RowID, w, h)
|
||||
if rerr == nil {
|
||||
if err := ui.Tap(r, row.Centre.X, row.Centre.Y); err != nil {
|
||||
return err
|
||||
}
|
||||
sr.Detail = fmt.Sprintf("tapped row %q at (%d,%d) from %s", row.Title, row.Centre.X, row.Centre.Y,
|
||||
strings.Join(row.Texts, " / "))
|
||||
return nil
|
||||
}
|
||||
lastErr = rerr
|
||||
} else {
|
||||
lastErr = err
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
if lastErr == nil {
|
||||
lastErr = fmt.Errorf("no result row matches %q", query)
|
||||
}
|
||||
return describeFailure(r, rec, lastErr)
|
||||
}
|
||||
|
||||
func sendKey(r *adb.Runner, name string, sr *StepResult) error {
|
||||
codes := map[string]int{
|
||||
"enter": ui.KeyEnter,
|
||||
"back": ui.KeyBack,
|
||||
"home": ui.KeyHome,
|
||||
"power": 26,
|
||||
"wakeup": ui.KeyWakeup,
|
||||
"escape": 111,
|
||||
}
|
||||
code, ok := codes[strings.ToLower(name)]
|
||||
if !ok {
|
||||
return fmt.Errorf("unknown key %q", name)
|
||||
}
|
||||
if err := ui.KeyEvent(r, code); err != nil {
|
||||
return err
|
||||
}
|
||||
sr.Detail = "pressed " + strings.ToLower(name)
|
||||
return nil
|
||||
}
|
||||
|
||||
// describeFailure turns a step failure into a clean, named failure (FR-8).
|
||||
//
|
||||
// A guard match wins: "Spotify is signed out" is far more useful than "text
|
||||
// Search did not appear". Otherwise the message names the screen that was on
|
||||
// the phone when the step gave up.
|
||||
func describeFailure(r *adb.Runner, rec *Recipe, err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
h, dumpErr := ui.Dump(r)
|
||||
if dumpErr == nil {
|
||||
for _, g := range rec.Guards {
|
||||
if len(h.FindText(g.Text, false)) > 0 {
|
||||
return fmt.Errorf("%s (phone is showing %q): %w", g.Reason, g.Text, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
screen := describeScreen(r, h, dumpErr)
|
||||
if screen == "" {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("%w; %s", err, screen)
|
||||
}
|
||||
|
||||
// describeScreen names the screen Artemis gave up on: the foreground package
|
||||
// plus the first visible lines of text.
|
||||
func describeScreen(r *adb.Runner, h *ui.Hierarchy, dumpErr error) string {
|
||||
var parts []string
|
||||
if fg, err := ui.ForegroundPackage(r); err == nil {
|
||||
parts = append(parts, "foreground="+fg)
|
||||
}
|
||||
if dumpErr == nil && h != nil {
|
||||
var labels []string
|
||||
for _, n := range h.TextNodes() {
|
||||
label := strings.TrimSpace(n.Text)
|
||||
if label == "" {
|
||||
label = strings.TrimSpace(n.ContentDesc)
|
||||
}
|
||||
if label == "" || len(label) > 60 {
|
||||
continue
|
||||
}
|
||||
labels = append(labels, fmt.Sprintf("%q", label))
|
||||
if len(labels) == 6 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(labels) > 0 {
|
||||
parts = append(parts, "screen shows: "+strings.Join(labels, ", "))
|
||||
}
|
||||
}
|
||||
return strings.Join(parts, "; ")
|
||||
}
|
||||
Reference in New Issue
Block a user