Files
artemis_control/internal/recipe/recipe_test.go
Sam Rolfe a7c07c645e feat: Phase 5 — maps_navigate, and read the screen before the input
The Maps recipe: open the search box, type the place, and confirm the
result list names it. Every label comes from a live uiautomator dump
(search_omnibox_text_box, typed_suggest_container, the account sheet),
never from a guess. Text only, one step_timeout/total_timeout, guards for
the account sheet.

The first live run exposed an ordering bug in expect_change. A dump is not
instant, so a baseline taken after the input already showed the app's
response and the check could never see a wedge; it failed against a
healthy Maps. The engine now reads the screen before the step above runs
and hands that signature to the check. Each retry re-reads it.

Live: maps_navigate ok in 27.6s, result list names "Melbourne Museum".
The wedge reason and its bounded retry are demonstrated with a probe
recipe whose input changes nothing; a real wedge could not be forced, and
docs/phase-5-maps-recipe.md says so plainly.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-10-16 16:18:06 +11:00

511 lines
17 KiB
Go

package recipe
import (
"os"
"path/filepath"
"strconv"
"strings"
"testing"
"time"
"gitea.lab.audasmedia.com.au/sam/artemis_control/internal/adb"
)
const sample = `# a comment
task: spotify_play
description: Search Spotify for a song and play it.
app: com.spotify.music
param: query
row_id: com.spotify.music:id/row_root
step_timeout: 20s
total_timeout: 150s
guard "Log in" => Spotify is signed out
guard "Update Spotify" => Spotify wants an update
steps:
- launch
- wait "Search" 30s
- tap "Search"
- tap_if "Clear search query"
- wait_gone "Recents" 30s
- type "{query}"
- tap_row "{query}" 40s
- wait exact "Pause" 40s
- key enter
- sleep 1s
`
func TestParseSampleRecipe(t *testing.T) {
rec, err := Parse(sample)
if err != nil {
t.Fatalf("Parse: %v", err)
}
if rec.Task != "spotify_play" || rec.App != "com.spotify.music" {
t.Fatalf("unexpected header: %+v", rec)
}
if len(rec.Params) != 1 || rec.Params[0] != "query" {
t.Fatalf("params = %v", rec.Params)
}
if rec.StepTimeout != 20*time.Second || rec.TotalTimeout != 150*time.Second {
t.Fatalf("timeouts = %s / %s", rec.StepTimeout, rec.TotalTimeout)
}
if len(rec.Guards) != 2 || rec.Guards[0].Text != "Log in" || rec.Guards[0].Reason != "Spotify is signed out" {
t.Fatalf("guards = %+v", rec.Guards)
}
if len(rec.Steps) != 10 {
t.Fatalf("steps = %d, want 10", len(rec.Steps))
}
wait := rec.Steps[1]
if wait.Verb != "wait" || wait.Args[0] != "Search" || wait.Timeout != 30*time.Second {
t.Fatalf("wait step = %+v", wait)
}
exact := rec.Steps[7]
if exact.Verb != "wait" || !exact.Exact || exact.Args[0] != "Pause" || exact.Timeout != 40*time.Second {
t.Fatalf("exact wait step = %+v", exact)
}
sleep := rec.Steps[9]
if sleep.Verb != "sleep" || sleep.Timeout != time.Second {
t.Fatalf("sleep step = %+v", sleep)
}
}
func TestParseRejectsBadInput(t *testing.T) {
cases := map[string]string{
"unknown verb": "steps:\n - teleport \"Home\"\n",
"missing settings": "steps:\n - wait \"x\"\n - frobnicate: 3\n",
"bad duration": "step_timeout: soon\nsteps:\n - wait \"x\"\n",
"no steps": "task: nothing\n",
"guard needs text": "guard only a reason => nope\nsteps:\n - home\n",
"guard needs why": "guard \"Log in\" =>\nsteps:\n - home\n",
"unknown key name": "steps:\n - key volup\n",
"unclosed quote": "steps:\n - wait \"Search\n",
"too many args": "steps:\n - type \"a\" \"b\"\n",
"unknown setting": "colour: blue\nsteps:\n - home\n",
}
for name, src := range cases {
if _, err := Parse(src); err == nil {
t.Errorf("%s: expected a parse error", name)
}
}
}
func TestSubstitute(t *testing.T) {
got, err := Substitute("play {query} now", map[string]string{"query": "Honky Tonk Women"})
if err != nil {
t.Fatalf("Substitute: %v", err)
}
if got != "play Honky Tonk Women now" {
t.Fatalf("Substitute = %q", got)
}
for _, bad := range []map[string]string{{}, {"query": " "}} {
if _, err := Substitute("{query}", bad); err == nil {
t.Fatalf("expected an error for params %v", bad)
}
}
if _, err := Substitute("{other}", map[string]string{"query": "x"}); err == nil {
t.Fatal("expected an error for an undeclared placeholder")
}
}
func TestMissingParams(t *testing.T) {
rec, err := Parse(sample)
if err != nil {
t.Fatalf("Parse: %v", err)
}
if got := rec.MissingParams(map[string]string{"query": "x"}); len(got) != 0 {
t.Fatalf("MissingParams = %v", got)
}
if got := rec.MissingParams(nil); len(got) != 1 || got[0] != "query" {
t.Fatalf("MissingParams = %v", got)
}
}
func TestParseRecipeWithoutTimeoutsUsesDefaults(t *testing.T) {
rec, err := Parse("steps:\n - home\n")
if err != nil {
t.Fatalf("Parse: %v", err)
}
if rec.StepTimeout != DefaultStepTimeout || rec.TotalTimeout != DefaultTotalTimeout {
t.Fatalf("defaults = %s / %s", rec.StepTimeout, rec.TotalTimeout)
}
}
// TestSpotifyRecipeIsValid keeps the shipped recipe honest: a typo in
// recipes/spotify_play.recipe fails the test suite, not the phone.
func TestSpotifyRecipeIsValid(t *testing.T) {
dir := filepath.Join("..", "..", "recipes")
rec, err := Load(filepath.Join(dir, "spotify_play.recipe"))
if err != nil {
t.Fatalf("Load: %v", err)
}
if rec.Task != "spotify_play" {
t.Fatalf("task = %q", rec.Task)
}
if rec.App != "com.spotify.music" {
t.Fatalf("app = %q", rec.App)
}
if len(rec.MissingParams(map[string]string{"query": "x"})) != 0 {
t.Fatal("spotify_play must accept a query parameter")
}
if len(rec.Guards) < 3 {
t.Fatalf("spotify_play needs guards for its bad screens, got %d", len(rec.Guards))
}
// Every step must be runnable: the parsed steps are what the engine gets.
for i, s := range rec.Steps {
if _, ok := verbs[s.Verb]; !ok {
t.Fatalf("step %d uses unknown verb %q", i+1, s.Verb)
}
}
// The recipe must confirm it is in Spotify before it trusts a text match,
// and it must confirm playback rather than assume it.
var sawForeground bool
for _, s := range rec.Steps {
if s.Verb == "foreground" {
sawForeground = true
}
}
if !sawForeground {
t.Fatal("spotify_play must pin the foreground app before matching text")
}
last := rec.Steps[len(rec.Steps)-1]
if last.Verb != "playing" || last.Args[0] != "{query}" {
t.Fatalf("the last step must confirm the right track is playing, got %+v", last)
}
if last.Retry < 1 {
t.Fatal("the playback check must be allowed to press the row again")
}
}
// TestMapsRecipeIsValid keeps the shipped Phase 5 recipe honest: a typo in
// recipes/maps_navigate.recipe fails the test suite, not the phone.
func TestMapsRecipeIsValid(t *testing.T) {
rec, err := Load(filepath.Join("..", "..", "recipes", "maps_navigate.recipe"))
if err != nil {
t.Fatalf("Load: %v", err)
}
if rec.Task != "maps_navigate" || rec.App != "com.google.android.apps.maps" {
t.Fatalf("header = %q / %q", rec.Task, rec.App)
}
if len(rec.MissingParams(map[string]string{"query": "x"})) != 0 {
t.Fatal("maps_navigate must accept a query parameter")
}
if len(rec.Guards) == 0 {
t.Fatal("maps_navigate needs guards for screens it must not tap")
}
var sawForeground, sawChange int
for i, s := range rec.Steps {
switch s.Verb {
case "foreground":
sawForeground++
case "expect_change":
sawChange++
// The engine reads the screen before the step above, so the check
// needs a step above it.
if i == 0 {
t.Fatal("expect_change must have a step above it")
}
}
// No coordinates, nowhere: every argument is text, a duration or a
// placeholder, never a bare screen coordinate.
for _, a := range s.Args {
if _, err := strconv.Atoi(a); err == nil {
t.Fatalf("step %s has a bare number %q as an argument: this recipe must name text, not coordinates", s.Verb, a)
}
}
}
if sawForeground == 0 {
t.Fatal("maps_navigate must pin the foreground app before matching text")
}
if sawChange == 0 {
t.Fatal("maps_navigate must check that its inputs changed the screen (the wedged-app check)")
}
last := rec.Steps[len(rec.Steps)-1]
if last.Verb != "wait" || last.Args[0] != "{query}" {
t.Fatalf("the last step must confirm the result list names the place, got %+v", last)
}
}
func TestLoadAndList(t *testing.T) {
dir := t.TempDir()
write := func(name, body string) {
t.Helper()
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil {
t.Fatal(err)
}
}
write("b.recipe", "task: beta\nsteps:\n - home\n")
write("a.recipe", "task: alpha\nsteps:\n - home\n")
list, err := List(dir)
if err != nil {
t.Fatalf("List: %v", err)
}
if len(list) != 2 || list[0].Task != "alpha" || list[1].Task != "beta" {
t.Fatalf("List = %+v", list)
}
rec, err := Find(dir, "beta")
if err != nil {
t.Fatalf("Find: %v", err)
}
if rec.Task != "beta" {
t.Fatalf("Find = %+v", rec)
}
if _, err := Find(dir, "gamma"); err == nil || !strings.Contains(err.Error(), "alpha") {
t.Fatalf("Find unknown task: %v", err)
}
}
func TestRetryOnWait(t *testing.T) {
rec, err := Parse("steps:\n - tap_row \"{query}\" 40s\n - wait exact \"Search\" 45s retry 3\n")
if err != nil {
t.Fatalf("Parse: %v", err)
}
wait := rec.Steps[1]
if wait.Retry != 3 || wait.Timeout != 45*time.Second || !wait.Exact || wait.Args[0] != "Search" {
t.Fatalf("retry wait step = %+v", wait)
}
// retry without a duration still works.
rec, err = Parse("steps:\n - tap_row \"x\"\n - wait \"Search\" retry 2\n")
if err != nil {
t.Fatalf("Parse: %v", err)
}
if rec.Steps[1].Retry != 2 || rec.Steps[1].Timeout != 0 {
t.Fatalf("retry step = %+v", rec.Steps[1])
}
}
func TestRetryRejectsBadUse(t *testing.T) {
for name, src := range map[string]string{
"on a tap": "steps:\n - tap \"x\" retry 3\n",
"on tap_row": "steps:\n - tap_row \"x\" retry 3\n",
"no number": "steps:\n - tap_row \"x\"\n - wait \"y\" retry\n",
"zero": "steps:\n - tap_row \"x\"\n - wait \"y\" retry 0\n",
"not a number": "steps:\n - tap_row \"x\"\n - wait \"y\" retry soon\n",
"expect_change without a duration": "steps:\n - key enter\n - expect_change\n",
} {
if _, err := Parse(src); err == nil {
t.Errorf("%s: expected a parse error", name)
}
}
}
// TestExpectChangeStep covers the wedged-app check: a delivered input that the
// app never acts on must fail with its own reason, and may re-press the step
// above a bounded number of times (FR-11).
func TestExpectChangeStep(t *testing.T) {
rec, err := Parse("steps:\n - key enter\n - expect_change 12s retry 2 else \"the app stopped accepting input\"\n")
if err != nil {
t.Fatalf("Parse: %v", err)
}
step := rec.Steps[1]
if step.Verb != "expect_change" || step.Timeout != 12*time.Second {
t.Fatalf("expect_change step = %+v", step)
}
if step.Retry != 2 || step.Reason != "the app stopped accepting input" {
t.Fatalf("retry/reason = %d / %q", step.Retry, step.Reason)
}
if len(step.Args) != 1 || step.Args[0] != "12s" {
t.Fatalf("args = %v, want the duration as the only argument", step.Args)
}
}
func TestPlayingStep(t *testing.T) {
rec, err := Parse("app: com.spotify.music\nsteps:\n - tap_row \"{query}\"\n - playing \"{query}\" 30s retry 3\n")
if err != nil {
t.Fatalf("Parse: %v", err)
}
p := rec.Steps[1]
if p.Verb != "playing" || p.Args[0] != "{query}" || p.Timeout != 30*time.Second || p.Retry != 3 {
t.Fatalf("playing step = %+v", p)
}
}
func TestForegroundStep(t *testing.T) {
rec, err := Parse("app: com.spotify.music\nsteps:\n - launch\n - foreground 30s\n")
if err != nil {
t.Fatalf("Parse: %v", err)
}
fg := rec.Steps[1]
if fg.Verb != "foreground" || len(fg.Args) != 0 || fg.Timeout != 30*time.Second {
t.Fatalf("foreground step = %+v", fg)
}
}
// TestStepDurationsAreRecorded guards the run log (FR-10): a step's duration
// must reach the caller, not be swallowed by the return value.
func TestStepDurationsAreRecorded(t *testing.T) {
rec := &Recipe{StepTimeout: time.Second}
step := Step{Verb: "sleep", Timeout: 50 * time.Millisecond}
sr := executeStep(adb.New(""), rec, step, 2, nil, 0, 0, time.Now().Add(time.Minute), "", func(string, ...any) {})
if !sr.OK {
t.Fatalf("sleep step failed: %s", sr.Error)
}
if sr.Index != 3 {
t.Fatalf("index = %d, want 3", sr.Index)
}
if sr.Duration < 40*time.Millisecond || sr.DurationMS < 40 {
t.Fatalf("duration not recorded: %v (%d ms)", sr.Duration, sr.DurationMS)
}
}
// TestElseNamesAStepFailure covers the "else" modifier: a screen that simply
// does not offer a label must fail with a reason a human can act on (FR-8).
func TestElseNamesAStepFailure(t *testing.T) {
rec, err := Parse("steps:\n - wait \"Go to song radio\" 20s else \"this track offers no Song radio\"\n")
if err != nil {
t.Fatalf("Parse: %v", err)
}
step := rec.Steps[0]
if step.Args[0] != "Go to song radio" || step.Timeout != 20*time.Second {
t.Fatalf("step = %+v", step)
}
if step.Reason != "this track offers no Song radio" {
t.Fatalf("reason = %q", step.Reason)
}
// The reason must not leak into the arguments the engine substitutes.
if len(step.Args) != 1 {
t.Fatalf("args = %v, want the query only", step.Args)
}
}
func TestElseNeedsAReason(t *testing.T) {
if _, err := Parse("steps:\n - wait \"Search\" 20s else\n"); err == nil {
t.Fatal("expected an error when else has no reason")
}
}
func TestStepFailurePrefersTheStepsOwnReason(t *testing.T) {
step := Step{Verb: "wait", Reason: "this track offers no Song radio"}
sr := StepResult{Error: "text \"Go to song radio\" did not appear within 20s; screen shows: \"Share\""}
got := stepFailure(sr, step)
if !strings.HasPrefix(got, "this track offers no Song radio:") {
t.Fatalf("stepFailure = %q, want the step's own reason first", got)
}
if !strings.Contains(got, "did not appear") {
t.Fatalf("stepFailure = %q, want the technical detail kept", got)
}
if plain := stepFailure(sr, Step{Verb: "wait"}); plain != sr.Error {
t.Fatalf("a step without a reason must report the error unchanged, got %q", plain)
}
}
// TestRowMenuDefaults keeps menu_row usable without extra settings, and proves
// a recipe can override the label.
func TestRowMenuDefaults(t *testing.T) {
rec, err := Parse("steps:\n - home\n")
if err != nil {
t.Fatalf("Parse: %v", err)
}
if rec.RowMenu != DefaultRowMenu {
t.Fatalf("row_menu default = %q, want %q", rec.RowMenu, DefaultRowMenu)
}
rec, err = Parse("row_menu: More options\nsteps:\n - menu_row \"{query}\" 30s\n")
if err != nil {
t.Fatalf("Parse: %v", err)
}
if rec.RowMenu != "More options" {
t.Fatalf("row_menu = %q", rec.RowMenu)
}
if rec.Steps[0].Verb != "menu_row" || rec.Steps[0].Args[0] != "{query}" {
t.Fatalf("step = %+v", rec.Steps[0])
}
}
// TestRowValueIsAvailableToLaterSteps covers {row}: menu_row/tap_row record the
// row they chose, so a recipe can check that a menu belongs to the row it
// pressed. The value is not a declared parameter, so it must not be required
// up front.
func TestRowValueIsAvailableToLaterSteps(t *testing.T) {
rec, err := Parse("param: query\nsteps:\n - menu_row \"{query}\" 40s\n - wait \"{row}\" 20s else \"the context menu opened on the wrong row\"\n")
if err != nil {
t.Fatalf("Parse: %v", err)
}
if missing := rec.MissingParams(map[string]string{"query": "x"}); len(missing) != 0 {
t.Fatalf("row must not be a required parameter, got missing %v", missing)
}
params := map[string]string{"query": "Rolling Stones Honky Tonk"}
if _, err := Substitute(rec.Steps[1].Args[0], params); err == nil {
t.Fatal("expected {row} to be unresolved before a row step has run")
}
params["row"] = "Honky Tonk Women"
got, err := Substitute(rec.Steps[1].Args[0], params)
if err != nil {
t.Fatalf("Substitute: %v", err)
}
if got != "Honky Tonk Women" {
t.Fatalf("substituted %q", got)
}
}
// TestSpotifyRadioRecipeIsValid keeps the shipped Phase 4 recipe honest: a typo
// in recipes/spotify_radio.recipe fails the test suite, not the phone.
func TestSpotifyRadioRecipeIsValid(t *testing.T) {
rec, err := Load(filepath.Join("..", "..", "recipes", "spotify_radio.recipe"))
if err != nil {
t.Fatalf("Load: %v", err)
}
if rec.Task != "spotify_radio" || rec.App != "com.spotify.music" {
t.Fatalf("header = %q / %q", rec.Task, rec.App)
}
if rec.RowID != "com.spotify.music:id/row_root" {
t.Fatalf("row_id = %q", rec.RowID)
}
if rec.RowMenu != DefaultRowMenu {
t.Fatalf("row_menu = %q", rec.RowMenu)
}
if len(rec.MissingParams(map[string]string{"query": "x"})) != 0 {
t.Fatal("spotify_radio must accept a query parameter")
}
// The app must be pinned in front before any text is trusted, and the
// screen must be searched before a row is picked.
var sawForeground, sawMenuRow int
reasons := map[string]bool{}
var last Step
for _, s := range rec.Steps {
switch s.Verb {
case "foreground":
sawForeground++
case "menu_row":
sawMenuRow++
}
if s.Reason != "" {
reasons[s.Reason] = true
}
// No coordinates, nowhere: every step argument is text, a duration or a
// placeholder, never a bare screen coordinate.
for _, a := range s.Args {
if _, err := strconv.Atoi(a); err == nil {
t.Fatalf("step %s has a bare number %q as an argument: this recipe must name text, not coordinates", s.Verb, a)
}
}
last = s
}
if sawForeground == 0 {
t.Fatal("spotify_radio must pin the foreground app before matching text")
}
if sawMenuRow != 1 {
t.Fatalf("spotify_radio must open exactly one row menu, got %d", sawMenuRow)
}
for _, want := range []string{
"the context menu opened on the wrong row",
"this track offers no Song radio",
} {
if !reasons[want] {
t.Fatalf("spotify_radio must fail with the distinct reason %q; it has %v", want, reasons)
}
}
if last.Verb != "playing" || len(last.Args) != 0 {
t.Fatalf("the last step must confirm playback by state (a radio changes track), got %+v", last)
}
if last.Retry < 1 {
t.Fatal("the playback check must be allowed to press the step above again")
}
}