From 3ef11da8086bb70e80ea0d4fec8f119adef3a35a Mon Sep 17 00:00:00 2001 From: Sam Rolfe Date: Sat, 17 Oct 2026 09:12:01 +1100 Subject: [PATCH] fix: address the Maps search box by id, and re-prove Phase 5 The Maps omnibox hint rotates: it read "Search here" in the Phase 5 run and "Try petrol stations, ATMs" in the AUD-128 re-verification, so the recipe's text match on it failed. The resource id does not rotate. - internal/ui: add Find/FindID and the "id:" selector; Find dispatches an id: selector to the resource id and everything else to text as before. - internal/recipe: wait/tap/wait_gone/tap_if and the tap helper use Find, so a recipe may name a resource id. Guards stay text-only. - recipes/maps_navigate.recipe: wait/tap id:search_omnibox_text_box. - tests for the id selector; docs and README updated. Co-Authored-By: Paperclip --- README.md | 6 ++- cmd/artemis/main.go | 4 +- docs/phase-5-maps-recipe.md | 77 +++++++++++++++++++++++++++++++++++- internal/recipe/engine.go | 9 +++-- internal/recipe/recipe.go | 4 ++ internal/ui/actions.go | 23 ++++++----- internal/ui/dump.go | 42 +++++++++++++++++++- internal/ui/dump_test.go | 53 +++++++++++++++++++++++++ recipes/maps_navigate.recipe | 14 ++++--- 9 files changed, 206 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index d188505..d98f516 100644 --- a/README.md +++ b/README.md @@ -84,10 +84,12 @@ labels, the `expect_change` check, and the one ordering rule it depends on. ## Rules that the code enforces -- **Find by text, never fixed coordinates.** `tap` resolves a text match to +- **Find by text, never fixed coordinates.** `tap` resolves a match to bounds and taps the centre; `tap_row` scores result rows against the query and taps the winning row's centre; `menu_row` opens the action menu of that - same row, from the button inside it. + same row, from the button inside it. Where a visible label rotates (the Maps + search box), a step names the stable resource id with the `id:` selector — + still a named match, still no coordinates. - **Never tap blindly.** A missing or disabled element is a loud failure, not a guessed coordinate. A named bad screen (`guard`) fails with its own reason, and a step may name its own failure with `else "..."`. diff --git a/cmd/artemis/main.go b/cmd/artemis/main.go index 894e97a..088999a 100644 --- a/cmd/artemis/main.go +++ b/cmd/artemis/main.go @@ -245,14 +245,14 @@ func cmdDump(r *adb.Runner) error { func cmdFind(r *adb.Runner, args []string) error { if len(args) != 1 { - return fmt.Errorf("usage: artemis find ") + return fmt.Errorf("usage: artemis find ") } return ensurePhone(r, func() error { h, err := ui.Dump(r) if err != nil { return err } - matches := h.FindText(args[0], false) + matches := h.Find(args[0], false) if len(matches) == 0 { return fmt.Errorf("no element matching %q", args[0]) } diff --git a/docs/phase-5-maps-recipe.md b/docs/phase-5-maps-recipe.md index 83d657c..541c8e3 100644 --- a/docs/phase-5-maps-recipe.md +++ b/docs/phase-5-maps-recipe.md @@ -11,6 +11,7 @@ Date: 2026-10-16. Phone: `Pixel_8a (akita)` over Tailscale at | File | What it holds | |---|---| | `recipes/maps_navigate.recipe` | The recipe: open the search box, type the place, confirm the result list names it. | +| `internal/ui/dump.go` | `Find` / `FindID`: the `id:` selector, for a label that rotates. | | `internal/ui/change.go` | `ScreenSignature` and `WaitForChange`: the screen fingerprint and the bounded change check. | | `internal/recipe/recipe.go` | The new `expect_change ` verb. | | `internal/recipe/engine.go` | Runs the check, and reads the screen **before** the step above it. | @@ -31,8 +32,8 @@ guard "Turn on Incognito mode" => Maps is showing its account sheet; press b steps: - launch - foreground 30s - - wait "Search here" 30s retry 2 - - tap "Search here" + - wait id:search_omnibox_text_box 30s retry 2 + - tap id:search_omnibox_text_box - expect_change 15s retry 2 - type "{query}" - expect_change 15s retry 2 @@ -45,6 +46,41 @@ Run it with: ./bin/artemis run maps_navigate query="Melbourne Museum" ``` +## The search box is addressed by id, not by its label + +The first version of the recipe waited for the text **`Search here`**. On +2026-10-17 a re-verification run failed at that step: the box was on screen, +but its label had rotated to **`Try petrol stations, ATMs`**: + +``` +step 3 wait [Search here]: element matching "Search here" did not appear within 30s; + foreground=com.google.android.apps.maps; + screen shows: "Layers", "Enter compass mode", "Directions", "Ivanhoe", ... +``` + +The Maps omnibox carries its hint as a **rotating content description**, so a +text match on it is not stable. A fresh dump of the live node shows the stable +half and the rotating half side by side: + +``` + +``` + +The recipe now names the resource id with the **`id:` selector**: +`wait id:search_omnibox_text_box` and `tap id:search_omnibox_text_box`. This is +still a named match with the tap coordinate derived from the matched element's +bounds — not a fixed coordinate, so FR-4 holds. `internal/ui/dump.go` dispatches +`Find`: a selector that starts with `id:` is matched against the resource id +(full or short name), and every other selector against text and content +description as before. `guards` remain text-only, because a guard names visible +screen text. + +The rule to keep: **use a text selector where a stable label exists, and an +`id:` selector where the label rotates.** The final `wait "{query}"` stays a +text match on purpose — the result row must name the place the caller asked for. + ## The labels came from the phone, not from a guess A fresh `uiautomator dump` of the Maps home and the search screen on the real @@ -223,10 +259,47 @@ and the typing that draws the result list. | `expect_change` parsing, `retry`, `else` | unit tested (`internal/recipe`) | | All shipped recipes parse | unit tested (`internal/recipe`) | | `maps_navigate` opens a real search and names the place | **live, verified** — 8 steps, 27.6s, result list above | +| `maps_navigate` re-verified after the label rotated | **live, verified** — 8 steps, 27.2s, id selector; see “Re-verification 2026-10-17” below | | The wedge reason fires on an input that changes nothing | **live, verified** — probe above | | A wedged app was reproduced | **no** — could not be forced; see the limit above | | The wedge check is bounded by `retry` | **live, verified** — 3 attempts, then stop | +## Re-verification 2026-10-17 + +AUD-128 re-ran the recipe on the current tree and the current phone session. +The text selector failed (the label had rotated, above), so the recipe moved to +the `id:` selector and the run went green: + +```sh +$ ./bin/artemis run maps_navigate query="Melbourne Museum" +task maps_navigate (Search Google Maps for a place and show the matching result list.) + step 1 launch []: ok (676ms) + step 2 foreground []: ok (166ms) + step 3 wait [id:search_omnibox_text_box]: ok (2.795s) + step 4 tap [id:search_omnibox_text_box]: ok (5.768s) + step 5 expect_change [15s]: ok (3.21s) + step 6 type [Melbourne Museum]: ok (420ms) + step 7 expect_change [15s]: ok (3.479s) + step 8 wait [Melbourne Museum]: ok (2.992s) +OK maps_navigate in 27.177s +``` + +The run log records `"ok":true`, and the wedge probe fired on a live no-op input +(`key escape` on the map home), three attempts in total, then stopped: + +``` +step 3 key [escape]: ok (151ms) +step 4 expect_change [6s]: the app stopped consuming input: the screen did not change within 6s; + foreground=com.google.android.apps.maps; screen shows: "Layers", "Enter compass mode", ... +step 4 expect_change: retrying, pressing key again +step 3 key [escape]: ok (207ms) +step 4 expect_change [6s]: the app stopped consuming input: ... +step 4 expect_change: retrying, pressing key again +step 3 key [escape]: ok (180ms) +step 4 expect_change [6s]: the app stopped consuming input: ... +FAILED maps_wedge_probe after 44.346s: ... +``` + ## Reproducing it ```sh diff --git a/internal/recipe/engine.go b/internal/recipe/engine.go index b20c942..0d8d94d 100644 --- a/internal/recipe/engine.go +++ b/internal/recipe/engine.go @@ -324,7 +324,7 @@ func executeStep(r *adb.Runner, rec *Recipe, step Step, index int, params map[st if err != nil { return err } - matches := h.FindText(args[0], step.Exact) + matches := h.Find(args[0], step.Exact) if len(matches) == 0 { sr.Detail = fmt.Sprintf("%q not on screen, skipped", args[0]) return nil @@ -410,7 +410,8 @@ func executeStep(r *adb.Runner, rec *Recipe, step Step, index int, params map[st return sr } -// tapByText finds an element by text and taps its centre (FR-4). +// tapByText finds an element by text and taps its centre (FR-4). The selector +// may name a resource id with the "id:" prefix. // // The element is looked up twice: once to wait for it, and again immediately // before the tap, so the coordinates come from the newest dump. On a busy @@ -423,9 +424,9 @@ func tapByText(r *adb.Runner, rec *Recipe, query string, exact bool, timeout tim if err != nil { return err } - matches := h.FindText(query, exact) + matches := h.Find(query, exact) if len(matches) == 0 { - return describeFailure(r, rec, fmt.Errorf("element with text %q disappeared before the tap", query)) + return describeFailure(r, rec, fmt.Errorf("element matching %q disappeared before the tap", query)) } node, err := ui.TapNode(r, matches) if err != nil { diff --git a/internal/recipe/recipe.go b/internal/recipe/recipe.go index 41c399b..b205cd8 100644 --- a/internal/recipe/recipe.go +++ b/internal/recipe/recipe.go @@ -32,6 +32,10 @@ // - launch start the recipe's app // - foreground 30s wait until the recipe's app is in front // - wait "Search" 30s wait for an element whose text matches +// - wait id:search_box 30s wait for an element by resource id. The short +// name after the "/" is enough. Use this when a +// label rotates (the Maps search box does); it is +// still a named match, never a coordinate (FR-4). // - wait exact "Pause" 60s retry 3 // if it never appears, re-run the step above and // wait again (a busy phone can drop a tap) diff --git a/internal/ui/actions.go b/internal/ui/actions.go index 812e6c9..5972e4c 100644 --- a/internal/ui/actions.go +++ b/internal/ui/actions.go @@ -129,7 +129,8 @@ func pickBestMatch(matches []*Node) (*Node, error) { } // TapText finds an element by its text (or content description) and taps its -// centre. This is FR-4: text match first, coordinates derived from bounds. +// centre. This is FR-4: a named match first, coordinates derived from bounds. +// A selector may also name a resource id with the "id:" prefix. // // It fails loudly when no element matches and never falls back to tapping a // guessed coordinate. @@ -141,9 +142,9 @@ func TapText(r *adb.Runner, query string, exact bool) (*Node, error) { if err != nil { return nil, err } - matches := h.FindText(query, exact) + matches := h.Find(query, exact) if len(matches) == 0 { - return nil, fmt.Errorf("element with text %q not found on screen; refusing to tap blindly", query) + return nil, fmt.Errorf("element matching %q not found on screen; refusing to tap blindly", query) } return TapNode(r, matches) } @@ -172,7 +173,8 @@ func TapNode(r *adb.Runner, matches []*Node) (*Node, error) { // WaitForText polls the screen until an element matching query appears, then // returns the best tappable match. Use it after an action that loads a new -// screen; apps do not draw instantly. +// screen; apps do not draw instantly. A selector may name a resource id with +// the "id:" prefix. func WaitForText(r *adb.Runner, query string, exact bool, timeout time.Duration) (*Node, error) { if timeout <= 0 { timeout = 10 * time.Second @@ -182,7 +184,7 @@ func WaitForText(r *adb.Runner, query string, exact bool, timeout time.Duration) for time.Now().Before(deadline) { h, err := Dump(r) if err == nil { - matches := h.FindText(query, exact) + matches := h.Find(query, exact) if len(matches) > 0 { sawMatch = true if n, perr := pickBestMatch(matches); perr == nil { @@ -193,14 +195,15 @@ func WaitForText(r *adb.Runner, query string, exact bool, timeout time.Duration) time.Sleep(PollInterval) } if sawMatch { - return nil, fmt.Errorf("text %q is present but never became a usable tap target within %s (it is off screen or disabled)", query, timeout) + return nil, fmt.Errorf("element matching %q is present but never became a usable tap target within %s (it is off screen or disabled)", query, timeout) } - return nil, fmt.Errorf("text %q did not appear within %s", query, timeout) + return nil, fmt.Errorf("element matching %q did not appear within %s", query, timeout) } // WaitForNoText polls the screen until no element matches query. Recipes use it // to wait for one list to be replaced by another, for example a search result -// list replacing the "Recents" list. +// list replacing the "Recents" list. A selector may name a resource id with the +// "id:" prefix. func WaitForNoText(r *adb.Runner, query string, exact bool, timeout time.Duration) error { if timeout <= 0 { timeout = 10 * time.Second @@ -208,12 +211,12 @@ func WaitForNoText(r *adb.Runner, query string, exact bool, timeout time.Duratio deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { h, err := Dump(r) - if err == nil && len(h.FindText(query, exact)) == 0 { + if err == nil && len(h.Find(query, exact)) == 0 { return nil } time.Sleep(PollInterval) } - return fmt.Errorf("text %q was still on screen after %s", query, timeout) + return fmt.Errorf("element matching %q was still on screen after %s", query, timeout) } // ScreenSize reads the physical screen size. diff --git a/internal/ui/dump.go b/internal/ui/dump.go index 63f729d..33d972f 100644 --- a/internal/ui/dump.go +++ b/internal/ui/dump.go @@ -103,16 +103,56 @@ func shortID(id string) string { return id } +// IDSelector marks a selector that matches a resource id rather than text. A +// step that names "id:search_omnibox_text_box" finds the element with that +// resource id, whatever its visible label happens to be. +// +// This exists because some labels are not stable: the Google Maps search box +// carries a rotating hint as its content description ("Search here", "Try +// petrol stations, ATMs", ...), so a recipe that matched that text broke when +// the phone picked a different hint. The resource id does not rotate. +const IDSelector = "id:" + // FindText returns nodes whose text or content description matches query. // When exact is false the match is case-insensitive and substring-based. func (h *Hierarchy) FindText(query string, exact bool) []*Node { + return h.find(query, exact, false) +} + +// FindID returns nodes whose resource id matches query. When exact is false the +// match is case-insensitive and substring-based, so the short name +// "search_omnibox_text_box" matches the full +// "com.google.android.apps.maps:id/search_omnibox_text_box". +func (h *Hierarchy) FindID(query string, exact bool) []*Node { + return h.find(query, exact, true) +} + +// Find returns nodes matching a recipe selector. A selector that starts with +// "id:" is matched against the resource id; any other selector is matched +// against text and content description (FR-4). +func (h *Hierarchy) Find(query string, exact bool) []*Node { + if id, ok := strings.CutPrefix(query, IDSelector); ok { + return h.FindID(id, exact) + } + return h.FindText(query, exact) +} + +func (h *Hierarchy) find(query string, exact, byID bool) []*Node { if query == "" { return nil } needle := strings.ToLower(query) var out []*Node h.Walk(func(n *Node) { - for _, candidate := range []string{n.Text, n.ContentDesc} { + var candidates []string + if byID { + if n.ResourceID != "" { + candidates = []string{shortID(n.ResourceID), n.ResourceID} + } + } else { + candidates = []string{n.Text, n.ContentDesc} + } + for _, candidate := range candidates { if candidate == "" { continue } diff --git a/internal/ui/dump_test.go b/internal/ui/dump_test.go index 0b3f0cf..0b69335 100644 --- a/internal/ui/dump_test.go +++ b/internal/ui/dump_test.go @@ -99,6 +99,59 @@ func TestPickBestMatchRejectsDisabledOnly(t *testing.T) { } } +// The Maps search box keeps a stable resource id but a rotating label, so it is +// addressed with the "id:" selector. This covers the full and short id, the +// prefix dispatch, and that a plain selector is still matched by text. +const sampleSelectorDump = ` + + + +` + +func parseSelectorSample(t *testing.T) *Hierarchy { + t.Helper() + h, err := parseHierarchy([]byte(sampleSelectorDump)) + if err != nil { + t.Fatalf("parseHierarchy: %v", err) + } + return h +} + +func TestFindIDSelector(t *testing.T) { + h := parseSelectorSample(t) + + // Full resource id, and the short name after the "/". + for _, sel := range []string{ + "id:search_omnibox_text_box", + "id:com.google.android.apps.maps:id/search_omnibox_text_box", + } { + matches := h.Find(sel, false) + if len(matches) != 1 { + t.Fatalf("Find(%q): want 1 match, got %d", sel, len(matches)) + } + if matches[0].Class != "android.widget.EditText" { + t.Fatalf("Find(%q) matched %q", sel, matches[0].Class) + } + } + + // Exact matching of the short id. + if got := len(h.Find("id:search_omnibox_text_box", true)); got != 1 { + t.Fatalf("exact id: want 1, got %d", got) + } + // A plain selector is not diverted to the id match. + if got := len(h.Find("search_omnibox_text_box", false)); got != 0 { + t.Fatalf("plain text must not match a resource id, got %d", got) + } + // Text still works through Find. + if got := len(h.Find("Search", true)); got != 1 { + t.Fatalf("text via Find: want 1, got %d", got) + } + // An unknown id is a clean miss, not a fallthrough to text. + if got := h.Find("id:no_such_id", false); len(got) != 0 { + t.Fatalf("unknown id: want 0, got %d", len(got)) + } +} + func TestEscapeInputText(t *testing.T) { cases := map[string]string{ "hello world": "'hello%sworld'", diff --git a/recipes/maps_navigate.recipe b/recipes/maps_navigate.recipe index d4100ec..1f0009e 100644 --- a/recipes/maps_navigate.recipe +++ b/recipes/maps_navigate.recipe @@ -30,11 +30,15 @@ steps: - launch - foreground 30s - # 2. Open the search box. On the map, the box is the EditText labelled - # "Search here" (resource id search_omnibox_text_box); tapping it opens the - # search screen with the box focused and a suggestions list below it. - - wait "Search here" 30s retry 2 - - tap "Search here" + # 2. Open the search box. The box is the EditText with resource id + # "search_omnibox_text_box". It is addressed by id, not by its label, + # because its label is a rotating hint: it reads "Search here" one day and + # "Try petrol stations, ATMs" the next, so a text match here broke in a + # re-verification run. The resource id is stable, and it is still a named + # match rather than a coordinate (FR-4). Tapping it opens the search screen + # with the box focused and a suggestions list below it. + - wait id:search_omnibox_text_box 30s retry 2 + - tap id:search_omnibox_text_box # 3. The search screen is a new screen, and the wedge check sits here. If the # tap was delivered but Maps never acted on it, the screen does not change: