feat: Phase 5 — maps_navigate, and read the screen before the input

The Maps recipe: open the search box, type the place, and confirm the
result list names it. Every label comes from a live uiautomator dump
(search_omnibox_text_box, typed_suggest_container, the account sheet),
never from a guess. Text only, one step_timeout/total_timeout, guards for
the account sheet.

The first live run exposed an ordering bug in expect_change. A dump is not
instant, so a baseline taken after the input already showed the app's
response and the check could never see a wedge; it failed against a
healthy Maps. The engine now reads the screen before the step above runs
and hands that signature to the check. Each retry re-reads it.

Live: maps_navigate ok in 27.6s, result list names "Melbourne Museum".
The wedge reason and its bounded retry are demonstrated with a probe
recipe whose input changes nothing; a real wedge could not be forced, and
docs/phase-5-maps-recipe.md says so plainly.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
2026-10-16 16:18:06 +11:00
parent 43cd4282d0
commit a7c07c645e
7 changed files with 420 additions and 34 deletions

View File

@@ -99,14 +99,30 @@ func Run(r *adb.Runner, rec *Recipe, opts Options) *RunResult {
w, h, _ := ui.ScreenSize(r) // best effort: bounds checking degrades gracefully
deadline := started.Add(rec.TotalTimeout)
// beforeChange is the screen signature captured immediately before the step
// above, for a following expect_change step. The capture happens before that
// step's input runs, because an app that never acts on the input leaves the
// screen exactly as it was — and a dump taken afterwards would already show
// the app's response (AGENTS.md section 11 Phase 5).
var beforeChange string
for i := 0; i < len(rec.Steps); i++ {
step := rec.Steps[i]
if step.Verb != "expect_change" && i+1 < len(rec.Steps) && rec.Steps[i+1].Verb == "expect_change" {
// A dump runs a JVM on the phone, so this happens only when a recipe
// actually asks for the wedge check.
if sig, err := ui.ScreenSignature(r); err == nil {
beforeChange = sig
} else {
beforeChange = ""
logf(" step %d %s: could not read the screen before the wedge check: %v", i+1, step.Verb, err)
}
}
var sr StepResult
for attempt := 0; ; attempt++ {
if remaining := time.Until(deadline); remaining <= 0 {
return fail(fmt.Errorf("recipe timed out after %s at step %d (%s)", rec.TotalTimeout, i+1, step.Verb))
}
sr = executeStep(r, rec, step, i, params, w, h, deadline, logf)
sr = executeStep(r, rec, step, i, params, w, h, deadline, beforeChange, logf)
res.Steps = append(res.Steps, sr)
if sr.OK {
break
@@ -120,7 +136,16 @@ func Run(r *adb.Runner, rec *Recipe, opts Options) *RunResult {
return fail(fmt.Errorf("step %d: retry needs a step above it to repeat", i+1))
}
logf(" step %d %s: retrying, pressing %s again", i+1, step.Verb, rec.Steps[i-1].Verb)
again := executeStep(r, rec, rec.Steps[i-1], i-1, params, w, h, deadline, logf)
if step.Verb == "expect_change" {
// Judge the re-press on its own: capture the screen again before
// the step above runs a second time.
if sig, err := ui.ScreenSignature(r); err == nil {
beforeChange = sig
} else {
beforeChange = ""
}
}
again := executeStep(r, rec, rec.Steps[i-1], i-1, params, w, h, deadline, "", logf)
res.Steps = append(res.Steps, again)
if !again.OK {
// The wait is the real failure; the repeat just could not run.
@@ -136,7 +161,7 @@ func Run(r *adb.Runner, rec *Recipe, opts Options) *RunResult {
}
func executeStep(r *adb.Runner, rec *Recipe, step Step, index int, params map[string]string,
w, h int, deadline time.Time, logf func(string, ...any)) (sr StepResult) {
w, h int, deadline time.Time, beforeChange string, logf func(string, ...any)) (sr StepResult) {
started := time.Now()
sr = StepResult{Index: index + 1, Verb: step.Verb, Args: step.Args}
@@ -274,12 +299,15 @@ func executeStep(r *adb.Runner, rec *Recipe, step Step, index int, params map[st
return nil
case "expect_change":
// The wedged-app check. An app that has stopped consuming input
// still draws its old screen, so a delivered tap or key changes
// nothing. Failing here names the real problem instead of blaming
// the label the next step would have waited for (Phase 5).
_, _, err := ui.ExpectChange(r, timeout)
if err != nil {
// The wedged-app check. beforeChange was captured before the step
// above ran. An app that has stopped consuming input still draws
// its old screen, so the signature does not move; failing here
// names the real problem instead of blaming the label the next
// step would have waited for (Phase 5).
if beforeChange == "" {
return fmt.Errorf("the screen was not read before the step above, so a change cannot be judged")
}
if _, err := ui.WaitForChange(r, beforeChange, timeout); err != nil {
return describeFailure(r, rec, fmt.Errorf("the app stopped consuming input: %w", err))
}
sr.Detail = "screen changed"

View File

@@ -40,8 +40,9 @@
// title matches the text; with no text, wait for PLAYING
// alone (a radio plays a different track every few minutes)
// - wait_gone "Recents" 30s wait until no element matches the text
// - expect_change 10s fail unless the screen changes within the time
// (the wedged-app check: a delivered input that the app never acts on)
// - expect_change 10s the screen is read before the step above, so this
// fails with "the app stopped consuming input" when
// that step's input left the screen unchanged (a wedged app)
// - tap "Search" tap the element whose text matches (FR-4)
// - tap exact "Pause" tap an element whose text matches exactly
// - tap_if "Clear search query" tap it when it is on screen, else skip

View File

@@ -178,6 +178,56 @@ func TestSpotifyRecipeIsValid(t *testing.T) {
}
}
// TestMapsRecipeIsValid keeps the shipped Phase 5 recipe honest: a typo in
// recipes/maps_navigate.recipe fails the test suite, not the phone.
func TestMapsRecipeIsValid(t *testing.T) {
rec, err := Load(filepath.Join("..", "..", "recipes", "maps_navigate.recipe"))
if err != nil {
t.Fatalf("Load: %v", err)
}
if rec.Task != "maps_navigate" || rec.App != "com.google.android.apps.maps" {
t.Fatalf("header = %q / %q", rec.Task, rec.App)
}
if len(rec.MissingParams(map[string]string{"query": "x"})) != 0 {
t.Fatal("maps_navigate must accept a query parameter")
}
if len(rec.Guards) == 0 {
t.Fatal("maps_navigate needs guards for screens it must not tap")
}
var sawForeground, sawChange int
for i, s := range rec.Steps {
switch s.Verb {
case "foreground":
sawForeground++
case "expect_change":
sawChange++
// The engine reads the screen before the step above, so the check
// needs a step above it.
if i == 0 {
t.Fatal("expect_change must have a step above it")
}
}
// No coordinates, nowhere: every argument is text, a duration or a
// placeholder, never a bare screen coordinate.
for _, a := range s.Args {
if _, err := strconv.Atoi(a); err == nil {
t.Fatalf("step %s has a bare number %q as an argument: this recipe must name text, not coordinates", s.Verb, a)
}
}
}
if sawForeground == 0 {
t.Fatal("maps_navigate must pin the foreground app before matching text")
}
if sawChange == 0 {
t.Fatal("maps_navigate must check that its inputs changed the screen (the wedged-app check)")
}
last := rec.Steps[len(rec.Steps)-1]
if last.Verb != "wait" || last.Args[0] != "{query}" {
t.Fatalf("the last step must confirm the result list names the place, got %+v", last)
}
}
func TestLoadAndList(t *testing.T) {
dir := t.TempDir()
write := func(name, body string) {
@@ -290,7 +340,7 @@ func TestForegroundStep(t *testing.T) {
func TestStepDurationsAreRecorded(t *testing.T) {
rec := &Recipe{StepTimeout: time.Second}
step := Step{Verb: "sleep", Timeout: 50 * time.Millisecond}
sr := executeStep(adb.New(""), rec, step, 2, nil, 0, 0, time.Now().Add(time.Minute), func(string, ...any) {})
sr := executeStep(adb.New(""), rec, step, 2, nil, 0, 0, time.Now().Add(time.Minute), "", func(string, ...any) {})
if !sr.OK {
t.Fatalf("sleep step failed: %s", sr.Error)
}