package ui import ( "fmt" "math" "strings" "unicode" ) // Rows are the only safe way to press a list result. // // A search result row is a clickable container that holds a title and a // subtitle ("Honky Tonk Women" / "Song • The Rolling Stones"). The container is // the text-matched element, and its centre is the tap point: that lands on the // title, never on the row's "More options" or "Add to library" buttons, which // sit in a small action bar at the row's right edge. // // Selecting the row instead of the smallest clickable child matters. On the // real Spotify screen a plain text match resolved to a 126x12 pixel "More // options for song ..." button, because that was the smallest clickable node. // Row is one pressable list row. type Row struct { Node *Node Texts []string Score float64 Title string Top int Centre Point } // Point is a screen coordinate. type Point struct { X, Y int } // BestRow chooses the row that best matches query. // // rowID, when set, restricts candidates to elements with that resource id // (either the full "package:id/name" form or a suffix of it). func (h *Hierarchy) BestRow(query, rowID string, screenW, screenH int) (*Row, error) { query = strings.TrimSpace(query) if query == "" { return nil, fmt.Errorf("row match needs a non-empty query") } qTokens := tokenize(query) if len(qTokens) == 0 { return nil, fmt.Errorf("query %q has no matchable words", query) } candidates := h.rowCandidates(rowID, screenW, screenH) if len(candidates) == 0 { return nil, fmt.Errorf("no result row is fully drawn on screen; refusing to tap blindly") } var best *Row for _, node := range candidates { rect, err := node.BoundsRect() if err != nil { continue } texts := rowTexts(node) score, matched := rowScore(qTokens, texts) // More than half of the query words must be on the row, and the row // must not be buried under unrelated words. if matched*2 < len(qTokens) || score < minRowScore { continue } cx, cy, err := node.Center() if err != nil { continue } candidate := &Row{ Node: node, Texts: texts, Score: score, Title: texts[0], Top: rect.Y1, Centre: Point{X: cx, Y: cy}, } if best == nil || betterRow(candidate, best) { best = candidate } } if best == nil { return nil, fmt.Errorf("no result row matches %q; refusing to tap blindly", query) } return best, nil } // Action returns the row's own action button: the smallest clickable element // inside the row whose text or content description contains label. // // The search is confined to the chosen row's subtree, which is what stops a // recipe from pressing the More options button of a different row. The button // on the real Spotify screen carries no label of its own (the description sits // on a non-clickable child), so each candidate button's subtree is searched // too, and the smallest match wins: a larger ancestor is the row itself. func (row *Row) Action(label string) (*Node, error) { needle := strings.ToLower(strings.TrimSpace(label)) if needle == "" { return nil, fmt.Errorf("a row action needs a non-empty label") } var best *Node bestArea := 1 << 30 row.Node.Walk(func(n *Node) { if n == row.Node || !n.Clickable || !n.Enabled { return } matched := false for _, text := range textDescendants(n) { if strings.Contains(strings.ToLower(text), needle) { matched = true break } } if !matched { return } area := 1 << 30 if r, err := n.BoundsRect(); err == nil { area = (r.X2 - r.X1) * (r.Y2 - r.Y1) } if best == nil || area < bestArea { best, bestArea = n, area } }) if best == nil { return nil, fmt.Errorf("row %q has no %q button", row.Title, label) } return best, nil } // minRowScore is the F1 floor for an acceptable row match. const minRowScore = 0.5 // rowCandidates returns the fully drawn rows on screen: clickable containers // that carry at least a title and a subtitle. // // Two shapes are rejected on purpose. A one-line search *suggestion* is not a // result. And a row clipped by the bottom of the list is much shorter than a // row that is fully scrolled into view, so its centre is not a point a human // could press; rows shorter than half the tallest candidate are dropped. func (h *Hierarchy) rowCandidates(rowID string, screenW, screenH int) []*Node { var found []*Node tallest := 0 h.Walk(func(n *Node) { if !n.Clickable || !n.Enabled || !rowMatchesID(n, rowID) { return } rect, err := n.BoundsRect() if err != nil || !onScreen(rect, screenW, screenH) { return } if len(rowTexts(n)) < 2 { return } if h := rect.Y2 - rect.Y1; h > tallest { tallest = h } found = append(found, n) }) var out []*Node for _, n := range found { rect, err := n.BoundsRect() if err != nil { continue } if (rect.Y2-rect.Y1)*2 < tallest { continue } out = append(out, n) } return out } // betterRow ranks rows. Spotify already ranks results by relevance, so a higher // row wins whenever the text match is comparable: the top hit can carry extra // state text ("Paused" on the row of the track that is already loaded) that // must not push it below a duplicate further down the list. Scores that are // further apart are not comparable, and the better text match wins. func betterRow(a, b *Row) bool { if math.Abs(a.Score-b.Score) > rowScoreTolerance { return a.Score > b.Score } return a.Top < b.Top } // rowScoreTolerance is how close two row scores must be before the higher row // is preferred. const rowScoreTolerance = 0.15 func rowMatchesID(n *Node, rowID string) bool { if rowID == "" { return true } return n.ResourceID == rowID || strings.HasSuffix(n.ResourceID, rowID) } // textDescendants returns the visible text lines inside a node, in reading // order, without duplicates. It is used for dumps and diagnostics. func textDescendants(n *Node) []string { var out []string seen := map[string]bool{} n.Walk(func(c *Node) { for _, t := range []string{c.Text, c.ContentDesc} { t = strings.TrimSpace(t) if t == "" || seen[t] { continue } seen[t] = true out = append(out, t) } }) return out } // rowTexts returns the text that belongs to the row itself: its title and // subtitle. // // Two things are left out on purpose. Anything inside a clickable child is an // action button ("More options", "Add to library", "Add suggestion"), not the // row's own text, so that subtree is skipped. And content descriptions are not // counted, because that is where apps put *state* ("Paused", "Playing") rather // than the row's title; counting them made the row of the track that was // already loaded score worse than an identical duplicate further down. func rowTexts(n *Node) []string { var out []string seen := map[string]bool{} add := func(s string) { s = strings.TrimSpace(s) if s == "" || seen[s] { return } seen[s] = true out = append(out, s) } var visit func(c *Node, root bool) visit = func(c *Node, root bool) { if !root && c.Clickable { return } add(c.Text) for i := range c.Children { visit(&c.Children[i], false) } } visit(n, true) return out } // onScreen reports whether the whole rectangle is inside the display. A row // clipped by the screen edge would be tapped at a point the user cannot see, // so it is not a candidate. func onScreen(r Rect, w, h int) bool { if w <= 0 || h <= 0 { return true } return r.X1 >= 0 && r.Y1 >= 0 && r.X2 <= w && r.Y2 <= h } // rowScore returns the F1 score of the query words against the row's words, // plus how many distinct query words were found. func rowScore(query []string, texts []string) (score float64, matched int) { row := map[string]bool{} for _, t := range texts { for _, w := range tokenize(t) { row[w] = true } } hit := map[string]bool{} for _, w := range query { if row[w] { hit[w] = true } } if len(hit) == 0 || len(row) == 0 { return 0, 0 } precision := float64(len(hit)) / float64(len(row)) recall := float64(len(hit)) / float64(len(query)) if precision+recall == 0 { return 0, len(hit) } return 2 * precision * recall / (precision + recall), len(hit) } // tokenize splits text into lowercase words. func tokenize(s string) []string { fields := strings.FieldsFunc(s, func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) }) out := make([]string, 0, len(fields)) for _, f := range fields { out = append(out, strings.ToLower(f)) } return out }