feat: Phase 5 — maps_navigate, and read the screen before the input
The Maps recipe: open the search box, type the place, and confirm the result list names it. Every label comes from a live uiautomator dump (search_omnibox_text_box, typed_suggest_container, the account sheet), never from a guess. Text only, one step_timeout/total_timeout, guards for the account sheet. The first live run exposed an ordering bug in expect_change. A dump is not instant, so a baseline taken after the input already showed the app's response and the check could never see a wedge; it failed against a healthy Maps. The engine now reads the screen before the step above runs and hands that signature to the check. Each retry re-reads it. Live: maps_navigate ok in 27.6s, result list names "Melbourne Museum". The wedge reason and its bounded retry are demonstrated with a probe recipe whose input changes nothing; a real wedge could not be forced, and docs/phase-5-maps-recipe.md says so plainly. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
@@ -99,14 +99,30 @@ func Run(r *adb.Runner, rec *Recipe, opts Options) *RunResult {
|
||||
w, h, _ := ui.ScreenSize(r) // best effort: bounds checking degrades gracefully
|
||||
|
||||
deadline := started.Add(rec.TotalTimeout)
|
||||
// beforeChange is the screen signature captured immediately before the step
|
||||
// above, for a following expect_change step. The capture happens before that
|
||||
// step's input runs, because an app that never acts on the input leaves the
|
||||
// screen exactly as it was — and a dump taken afterwards would already show
|
||||
// the app's response (AGENTS.md section 11 Phase 5).
|
||||
var beforeChange string
|
||||
for i := 0; i < len(rec.Steps); i++ {
|
||||
step := rec.Steps[i]
|
||||
if step.Verb != "expect_change" && i+1 < len(rec.Steps) && rec.Steps[i+1].Verb == "expect_change" {
|
||||
// A dump runs a JVM on the phone, so this happens only when a recipe
|
||||
// actually asks for the wedge check.
|
||||
if sig, err := ui.ScreenSignature(r); err == nil {
|
||||
beforeChange = sig
|
||||
} else {
|
||||
beforeChange = ""
|
||||
logf(" step %d %s: could not read the screen before the wedge check: %v", i+1, step.Verb, err)
|
||||
}
|
||||
}
|
||||
var sr StepResult
|
||||
for attempt := 0; ; attempt++ {
|
||||
if remaining := time.Until(deadline); remaining <= 0 {
|
||||
return fail(fmt.Errorf("recipe timed out after %s at step %d (%s)", rec.TotalTimeout, i+1, step.Verb))
|
||||
}
|
||||
sr = executeStep(r, rec, step, i, params, w, h, deadline, logf)
|
||||
sr = executeStep(r, rec, step, i, params, w, h, deadline, beforeChange, logf)
|
||||
res.Steps = append(res.Steps, sr)
|
||||
if sr.OK {
|
||||
break
|
||||
@@ -120,7 +136,16 @@ func Run(r *adb.Runner, rec *Recipe, opts Options) *RunResult {
|
||||
return fail(fmt.Errorf("step %d: retry needs a step above it to repeat", i+1))
|
||||
}
|
||||
logf(" step %d %s: retrying, pressing %s again", i+1, step.Verb, rec.Steps[i-1].Verb)
|
||||
again := executeStep(r, rec, rec.Steps[i-1], i-1, params, w, h, deadline, logf)
|
||||
if step.Verb == "expect_change" {
|
||||
// Judge the re-press on its own: capture the screen again before
|
||||
// the step above runs a second time.
|
||||
if sig, err := ui.ScreenSignature(r); err == nil {
|
||||
beforeChange = sig
|
||||
} else {
|
||||
beforeChange = ""
|
||||
}
|
||||
}
|
||||
again := executeStep(r, rec, rec.Steps[i-1], i-1, params, w, h, deadline, "", logf)
|
||||
res.Steps = append(res.Steps, again)
|
||||
if !again.OK {
|
||||
// The wait is the real failure; the repeat just could not run.
|
||||
@@ -136,7 +161,7 @@ func Run(r *adb.Runner, rec *Recipe, opts Options) *RunResult {
|
||||
}
|
||||
|
||||
func executeStep(r *adb.Runner, rec *Recipe, step Step, index int, params map[string]string,
|
||||
w, h int, deadline time.Time, logf func(string, ...any)) (sr StepResult) {
|
||||
w, h int, deadline time.Time, beforeChange string, logf func(string, ...any)) (sr StepResult) {
|
||||
|
||||
started := time.Now()
|
||||
sr = StepResult{Index: index + 1, Verb: step.Verb, Args: step.Args}
|
||||
@@ -274,12 +299,15 @@ func executeStep(r *adb.Runner, rec *Recipe, step Step, index int, params map[st
|
||||
return nil
|
||||
|
||||
case "expect_change":
|
||||
// The wedged-app check. An app that has stopped consuming input
|
||||
// still draws its old screen, so a delivered tap or key changes
|
||||
// nothing. Failing here names the real problem instead of blaming
|
||||
// the label the next step would have waited for (Phase 5).
|
||||
_, _, err := ui.ExpectChange(r, timeout)
|
||||
if err != nil {
|
||||
// The wedged-app check. beforeChange was captured before the step
|
||||
// above ran. An app that has stopped consuming input still draws
|
||||
// its old screen, so the signature does not move; failing here
|
||||
// names the real problem instead of blaming the label the next
|
||||
// step would have waited for (Phase 5).
|
||||
if beforeChange == "" {
|
||||
return fmt.Errorf("the screen was not read before the step above, so a change cannot be judged")
|
||||
}
|
||||
if _, err := ui.WaitForChange(r, beforeChange, timeout); err != nil {
|
||||
return describeFailure(r, rec, fmt.Errorf("the app stopped consuming input: %w", err))
|
||||
}
|
||||
sr.Detail = "screen changed"
|
||||
|
||||
@@ -40,8 +40,9 @@
|
||||
// title matches the text; with no text, wait for PLAYING
|
||||
// alone (a radio plays a different track every few minutes)
|
||||
// - wait_gone "Recents" 30s wait until no element matches the text
|
||||
// - expect_change 10s fail unless the screen changes within the time
|
||||
// (the wedged-app check: a delivered input that the app never acts on)
|
||||
// - expect_change 10s the screen is read before the step above, so this
|
||||
// fails with "the app stopped consuming input" when
|
||||
// that step's input left the screen unchanged (a wedged app)
|
||||
// - tap "Search" tap the element whose text matches (FR-4)
|
||||
// - tap exact "Pause" tap an element whose text matches exactly
|
||||
// - tap_if "Clear search query" tap it when it is on screen, else skip
|
||||
|
||||
@@ -178,6 +178,56 @@ func TestSpotifyRecipeIsValid(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestMapsRecipeIsValid keeps the shipped Phase 5 recipe honest: a typo in
|
||||
// recipes/maps_navigate.recipe fails the test suite, not the phone.
|
||||
func TestMapsRecipeIsValid(t *testing.T) {
|
||||
rec, err := Load(filepath.Join("..", "..", "recipes", "maps_navigate.recipe"))
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
if rec.Task != "maps_navigate" || rec.App != "com.google.android.apps.maps" {
|
||||
t.Fatalf("header = %q / %q", rec.Task, rec.App)
|
||||
}
|
||||
if len(rec.MissingParams(map[string]string{"query": "x"})) != 0 {
|
||||
t.Fatal("maps_navigate must accept a query parameter")
|
||||
}
|
||||
if len(rec.Guards) == 0 {
|
||||
t.Fatal("maps_navigate needs guards for screens it must not tap")
|
||||
}
|
||||
|
||||
var sawForeground, sawChange int
|
||||
for i, s := range rec.Steps {
|
||||
switch s.Verb {
|
||||
case "foreground":
|
||||
sawForeground++
|
||||
case "expect_change":
|
||||
sawChange++
|
||||
// The engine reads the screen before the step above, so the check
|
||||
// needs a step above it.
|
||||
if i == 0 {
|
||||
t.Fatal("expect_change must have a step above it")
|
||||
}
|
||||
}
|
||||
// No coordinates, nowhere: every argument is text, a duration or a
|
||||
// placeholder, never a bare screen coordinate.
|
||||
for _, a := range s.Args {
|
||||
if _, err := strconv.Atoi(a); err == nil {
|
||||
t.Fatalf("step %s has a bare number %q as an argument: this recipe must name text, not coordinates", s.Verb, a)
|
||||
}
|
||||
}
|
||||
}
|
||||
if sawForeground == 0 {
|
||||
t.Fatal("maps_navigate must pin the foreground app before matching text")
|
||||
}
|
||||
if sawChange == 0 {
|
||||
t.Fatal("maps_navigate must check that its inputs changed the screen (the wedged-app check)")
|
||||
}
|
||||
last := rec.Steps[len(rec.Steps)-1]
|
||||
if last.Verb != "wait" || last.Args[0] != "{query}" {
|
||||
t.Fatalf("the last step must confirm the result list names the place, got %+v", last)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadAndList(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
write := func(name, body string) {
|
||||
@@ -290,7 +340,7 @@ func TestForegroundStep(t *testing.T) {
|
||||
func TestStepDurationsAreRecorded(t *testing.T) {
|
||||
rec := &Recipe{StepTimeout: time.Second}
|
||||
step := Step{Verb: "sleep", Timeout: 50 * time.Millisecond}
|
||||
sr := executeStep(adb.New(""), rec, step, 2, nil, 0, 0, time.Now().Add(time.Minute), func(string, ...any) {})
|
||||
sr := executeStep(adb.New(""), rec, step, 2, nil, 0, 0, time.Now().Add(time.Minute), "", func(string, ...any) {})
|
||||
if !sr.OK {
|
||||
t.Fatalf("sleep step failed: %s", sr.Error)
|
||||
}
|
||||
|
||||
@@ -50,34 +50,33 @@ func (h *Hierarchy) Signature(foreground string) string {
|
||||
return foreground + "\n" + strings.Join(parts, "\n")
|
||||
}
|
||||
|
||||
// ExpectChange waits until the screen no longer looks like it did when the
|
||||
// function was called, and returns the signatures before and after.
|
||||
// WaitForChange waits until the screen no longer matches before, and returns
|
||||
// the new signature.
|
||||
//
|
||||
// This is the wedged-app check. An app that has stopped consuming input still
|
||||
// draws the screen it had, so a delivered tap changes nothing. A timeout here
|
||||
// means the last input was accepted by Android but never acted on by the app,
|
||||
// which the recipe reports as "the app stopped consuming input" instead of
|
||||
// blaming the label it was waiting for (AGENTS.md section 11 Phase 5).
|
||||
func ExpectChange(r *adb.Runner, timeout time.Duration) (before, after string, err error) {
|
||||
// The caller captures before with ScreenSignature *before* the input it wants to
|
||||
// judge. That ordering is the whole check: an app that has stopped consuming
|
||||
// input still draws the screen it had, so the signature after the input is
|
||||
// identical to the one before it. A dump is not instant, so a baseline taken
|
||||
// after the input would already include the app's response and could never see
|
||||
// the wedge.
|
||||
func WaitForChange(r *adb.Runner, before string, timeout time.Duration) (string, error) {
|
||||
if strings.TrimSpace(before) == "" {
|
||||
return "", fmt.Errorf("no screen signature was captured before the input")
|
||||
}
|
||||
if timeout <= 0 {
|
||||
timeout = 10 * time.Second
|
||||
}
|
||||
before, err = ScreenSignature(r)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
deadline := time.Now().Add(timeout)
|
||||
for {
|
||||
sig, derr := ScreenSignature(r)
|
||||
if derr != nil {
|
||||
return before, "", derr
|
||||
sig, err := ScreenSignature(r)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if sig != before {
|
||||
return before, sig, nil
|
||||
return sig, nil
|
||||
}
|
||||
if !time.Now().Before(deadline) {
|
||||
return before, before, fmt.Errorf("the screen did not change within %s", timeout)
|
||||
if time.Now().After(deadline) {
|
||||
return before, fmt.Errorf("the screen did not change within %s", timeout)
|
||||
}
|
||||
time.Sleep(PollInterval)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user