diff --git a/src/algo/algo.go b/src/algo/algo.go index c3a4ca7d..f4c128b6 100644 --- a/src/algo/algo.go +++ b/src/algo/algo.go @@ -347,15 +347,49 @@ func isAscii(runes []rune) bool { return true } -// runeFuzzyIndex is asciiFuzzyIndex for rune-mode input. Only valid when the -// input cannot fold to ASCII, see the caller. +// runePrefilterable reports whether scanning the rune array can decide this +// pattern against this item without missing a match. Phase 2 lowercases an +// uppercase text rune and then normalizes it, and the scan sees neither +// transform, so every pattern rune must be unreachable by them. +func runePrefilterable(input *util.Chars, pattern []rune, caseSensitive bool) bool { + if input.MayFoldToAscii() { + // A non-ASCII rune of this item could fold onto an ASCII pattern char + for _, r := range pattern { + if r < utf8.RuneSelf { + return false + } + } + } + if caseSensitive { + // No case transform is applied, and normalization only ever produces + // ASCII, so nothing can reach a non-ASCII pattern rune + return true + } + for _, r := range pattern { + // Another rune must not lowercase onto this one. Being uncased is not + // enough by itself: U+00DF has no simple uppercase yet U+1E9E + // lowercases to it. Excluding the foldable set covers that. + if r >= utf8.RuneSelf && + (unicode.ToUpper(r) != r || unicode.ToLower(r) != r || util.MayFoldToAscii(r)) { + return false + } + } + return true +} + +// runeFuzzyIndex is asciiFuzzyIndex for rune-mode input. Only valid when +// runePrefilterable says so. func runeFuzzyIndex(input *util.Chars, pattern []rune, caseSensitive bool) (int, int) { runes := input.Runes() firstIdx, idx, lastIdx := 0, 0, 0 - var b byte + var last rune for pidx := range pattern { - b = byte(pattern[pidx]) - idx = indexAsciiRune(runes, caseSensitive, b, idx) + last = pattern[pidx] + if last < utf8.RuneSelf { + idx = indexAsciiRune(runes, caseSensitive, byte(last), idx) + } else { + idx = indexRune(runes, last, idx) + } if idx < 0 { return -1, -1 } @@ -370,7 +404,13 @@ func runeFuzzyIndex(input *util.Chars, pattern []rune, caseSensitive bool) (int, // Find the last appearance of the last character of the pattern to limit // the search scope if lastIdx+1 < len(runes) { - if end := lastIndexAsciiRune(runes, caseSensitive, b, lastIdx+1); end >= 0 { + var end int + if last < utf8.RuneSelf { + end = lastIndexAsciiRune(runes, caseSensitive, byte(last), lastIdx+1) + } else { + end = lastIndexRune(runes, last, lastIdx+1) + } + if end >= 0 { return firstIdx, end + 1 } } @@ -378,24 +418,26 @@ func runeFuzzyIndex(input *util.Chars, pattern []rune, caseSensitive bool) (int, } func asciiFuzzyIndex(input *util.Chars, pattern []rune, caseSensitive bool) (int, int) { - if !isAscii(pattern) { - // An ASCII string cannot contain a non-ASCII character - if input.IsBytes() { - return -1, -1 - } - // Rune input with a non-ASCII pattern is not filtered yet - return 0, input.Length() - } - if !input.IsBytes() { - // Case folding or normalization can turn a non-ASCII rune into the - // ASCII character we are looking for, which the scan cannot see - if disableRunePrefilter || input.MayFoldToAscii() { + if disableRunePrefilter { return 0, input.Length() } + // runePrefilterable does not inline, so keep the common case out of + // it: an ASCII pattern against an item that cannot fold to ASCII is + // always scannable, and both of these checks do inline. + if input.MayFoldToAscii() || !isAscii(pattern) { + if !runePrefilterable(input, pattern, caseSensitive) { + return 0, input.Length() + } + } return runeFuzzyIndex(input, pattern, caseSensitive) } + // Not possible + if !isAscii(pattern) { + return -1, -1 + } + firstIdx, idx, lastIdx := 0, 0, 0 var b byte for pidx := range pattern { diff --git a/src/algo/runeindex_others.go b/src/algo/runeindex_others.go index 4a0b73d9..c6961447 100644 --- a/src/algo/runeindex_others.go +++ b/src/algo/runeindex_others.go @@ -13,3 +13,11 @@ func indexAsciiRune(runes []rune, caseSensitive bool, b byte, from int) int { func lastIndexAsciiRune(runes []rune, caseSensitive bool, b byte, from int) int { return lastIndexAsciiRuneRef(runes, caseSensitive, b, from) } + +func indexRune(runes []rune, r rune, from int) int { + return indexRuneRef(runes, r, from) +} + +func lastIndexRune(runes []rune, r rune, from int) int { + return lastIndexRuneRef(runes, r, from) +} diff --git a/src/algo/runeindex_ref.go b/src/algo/runeindex_ref.go index 53bcd038..8bc07399 100644 --- a/src/algo/runeindex_ref.go +++ b/src/algo/runeindex_ref.go @@ -35,3 +35,21 @@ func lastIndexAsciiRuneRef(runes []rune, caseSensitive bool, b byte, from int) i } return -1 } + +func indexRuneRef(runes []rune, r rune, from int) int { + for i := from; i < len(runes); i++ { + if runes[i] == r { + return i + } + } + return -1 +} + +func lastIndexRuneRef(runes []rune, r rune, from int) int { + for i := len(runes) - 1; i >= from; i-- { + if runes[i] == r { + return i + } + } + return -1 +} diff --git a/src/algo/runeindex_x86.go b/src/algo/runeindex_x86.go index 56d31db3..aea5c335 100644 --- a/src/algo/runeindex_x86.go +++ b/src/algo/runeindex_x86.go @@ -42,6 +42,69 @@ func indexAsciiRune(runes []rune, caseSensitive bool, b byte, from int) int { return -1 } +// runeNeedle picks which of the rune's four bytes to scan for, and returns its +// lane index and value. A zero byte is a useless needle because every ASCII +// rune contributes three of them, so U+AE00 scanned by its low byte would hit +// on almost every character of an ASCII-heavy line. Prefer a byte that cannot +// occur in an ASCII rune at all, then any non-zero byte. +func runeNeedle(r rune) (int, byte) { + var b [4]byte + b[0], b[1], b[2], b[3] = byte(r), byte(r>>8), byte(r>>16), byte(r>>24) + for i, v := range b { + if v >= 0x80 { + return i, v + } + } + for i, v := range b { + if v != 0 { + return i, v + } + } + return 0, 0 +} + +func runeAt(view []byte, start int) rune { + return rune(view[start]) | rune(view[start+1])<<8 | rune(view[start+2])<<16 | rune(view[start+3])<<24 +} + +// indexRune returns the index of the first rune equal to r at or after rune +// index from. Case is not folded, so the caller must have established that no +// other rune can transform into r. +func indexRune(runes []rune, r rune, from int) int { + view := runeBytes(runes) + lane, needle := runeNeedle(r) + for off := from*4 + lane; off < len(view); { + idx := bytes.IndexByte(view[off:], needle) + if idx < 0 { + return -1 + } + pos := off + idx + if start := pos - lane; start&3 == 0 && runeAt(view, start) == r { + return start >> 2 + } + off = pos + 1 + } + return -1 +} + +// lastIndexRune is indexRune scanning backwards from the end. +func lastIndexRune(runes []rune, r rune, from int) int { + view := runeBytes(runes) + lane, needle := runeNeedle(r) + for end := len(view); end > from*4+lane; { + idx := bytes.LastIndexByte(view[from*4+lane:end], needle) + if idx < 0 { + return -1 + } + pos := from*4 + lane + idx + if start := pos - lane; start&3 == 0 && runeAt(view, start) == r { + return start >> 2 + } + end = pos + } + return -1 +} + // lastIndexAsciiRune is indexAsciiRune scanning backwards from the end. func lastIndexAsciiRune(runes []rune, caseSensitive bool, b byte, from int) int { view := runeBytes(runes)[from*4:] diff --git a/src/algo/runeprefilter_test.go b/src/algo/runeprefilter_test.go index 59c2d283..fa566364 100644 --- a/src/algo/runeprefilter_test.go +++ b/src/algo/runeprefilter_test.go @@ -255,6 +255,131 @@ func TestNormalizeRuneUnchangedByGuard(t *testing.T) { } } +// Step G lets non-ASCII pattern runes use the scan, but only when no other +// rune can transform into them. Being uncased is not sufficient: U+00DF has no +// simple uppercase yet U+1E9E lowercases onto it. This pins the guard against +// the full preimage relation over all of Unicode. +func TestRunePrefilterableGuardIsSound(t *testing.T) { + preimage := map[rune][]rune{} + for r := rune(0); r <= unicode.MaxRune; r++ { + if r >= 0xD800 && r <= 0xDFFF { + continue + } + if charClassOfNonAscii(r) == charUpper { + if l := unicode.To(unicode.LowerCase, r); l != r { + preimage[l] = append(preimage[l], r) + } + } + } + + clean := util.ToChars([]byte("漢字")) // rune mode, fold bit clear + admitted, violations := 0, 0 + for r := rune(utf8.RuneSelf); r <= unicode.MaxRune; r++ { + if r >= 0xD800 && r <= 0xDFFF { + continue + } + if !runePrefilterable(&clean, []rune{r}, false) { + continue + } + admitted++ + if extra := preimage[r]; len(extra) > 0 { + violations++ + if violations <= 5 { + t.Errorf("guard admits U+%04X but %U lowercases onto it", r, extra) + } + } + } + t.Logf("guard admits %d non-ASCII pattern runes, unsound for %d", admitted, violations) + + // The scripts this step exists for must be fully admitted. + for _, s := range []struct { + name string + lo, hi rune + }{{"CJK", 0x4E00, 0x9FFF}, {"Hangul", 0xAC00, 0xD7A3}, {"kana", 0x3040, 0x30FF}, + {"Thai", 0x0E00, 0x0E7F}, {"emoji", 0x1F300, 0x1FAFF}} { + for r := s.lo; r <= s.hi; r++ { + if !runePrefilterable(&clean, []rune{r}, false) { + t.Errorf("%s U+%04X should be admitted", s.name, r) + break + } + } + } +} + +// The Step G path must actually engage and reject, otherwise the equivalence +// test above proves nothing about non-ASCII patterns. +func TestNonAsciiPatternPrefilterEngages(t *testing.T) { + rng := rand.New(rand.NewSource(6)) + parts := []string{"漢字", "한글", "src", "conf", "/", "мир", "🎉"} + engaged, rejected, narrowed, bypassed := 0, 0, 0, 0 + for range 5000 { + var sb strings.Builder + for range 1 + rng.Intn(6) { + sb.WriteString(parts[rng.Intn(len(parts))]) + } + chars := util.ToChars([]byte(sb.String())) + if chars.IsBytes() { + continue + } + pattern := []rune([]string{"漢", "漢字", "한글", "мир", "🎉", "é", "ß"}[rng.Intn(7)]) + if !runePrefilterable(&chars, pattern, false) { + bypassed++ + continue + } + engaged++ + lo, hi := asciiFuzzyIndex(&chars, pattern, false) + switch { + case lo < 0: + rejected++ + case hi-lo < chars.Length(): + narrowed++ + } + } + t.Logf("non-ASCII patterns: engaged %d, bypassed %d, rejected %d, narrowed %d", + engaged, bypassed, rejected, narrowed) + if engaged == 0 || rejected == 0 { + t.Fatalf("non-ASCII pattern path not exercised (engaged=%d rejected=%d)", engaged, rejected) + } + if bypassed == 0 { + t.Fatal("cased and foldable patterns should still bypass") + } +} + +// indexRune picks which byte lane to scan, and is checked against the shipped +// reference scanners rather than a copy of them. Zero-low-byte runes (U+AE00) and +// runes whose lanes collide with common ASCII bytes are the cases that break a +// naive low-byte scan, so the alphabet includes both. +func TestIndexRuneMatchesReference(t *testing.T) { + + alphabet := []rune{ + 'a', 'e', 'N', '/', 0x00, + 0xAE00, 0xAC00, 0xD55C, // Hangul, low byte zero for U+AE00 + 0x4E00, 0x6587, 0x9FFF, // CJK, U+4E00 has zero low byte + 0x3040, 0x0E00, // kana, Thai with zero low byte + 0x1F389, 0x1F300, // emoji, 3 significant bytes + 0x0100, 0x0165, 0x00E9, // low byte zero / ASCII-colliding lanes + } + rng := rand.New(rand.NewSource(7)) + for trial := range 30000 { + n := rng.Intn(20) + runes := make([]rune, n) + for i := range runes { + runes[i] = alphabet[rng.Intn(len(alphabet))] + } + r := alphabet[rng.Intn(len(alphabet))] + from := 0 + if n > 0 { + from = rng.Intn(n) + } + if got, exp := indexRune(runes, r, from), indexRuneRef(runes, r, from); got != exp { + t.Fatalf("trial %d: indexRune(%U, U+%04X, %d) = %d, expected %d", trial, runes, r, from, got, exp) + } + if got, exp := lastIndexRune(runes, r, from), lastIndexRuneRef(runes, r, from); got != exp { + t.Fatalf("trial %d: lastIndexRune(%U, U+%04X, %d) = %d, expected %d", trial, runes, r, from, got, exp) + } + } +} + // preparePattern mirrors what pattern.go guarantees the algo functions: // lowercased when case-insensitive, normalized when normalize is on. func preparePattern(pat string, caseSensitive, normalize bool) []rune {