mirror of
https://github.com/genuinetools/reg.git
synced 2024-07-04 12:34:17 -04:00
702 lines
14 KiB
Go
702 lines
14 KiB
Go
// Copyright 2015 The Go Authors. All rights reserved.
|
||
// Use of this source code is governed by a BSD-style
|
||
// license that can be found in the LICENSE file.
|
||
|
||
package width
|
||
|
||
import (
|
||
"bytes"
|
||
"strings"
|
||
"testing"
|
||
|
||
"golang.org/x/text/internal/testtext"
|
||
"golang.org/x/text/transform"
|
||
)
|
||
|
||
func foldRune(r rune) (folded rune, ok bool) {
|
||
alt, ok := mapRunes[r]
|
||
if ok && alt.e&tagNeedsFold != 0 {
|
||
return alt.r, true
|
||
}
|
||
return r, false
|
||
}
|
||
|
||
func widenRune(r rune) (wide rune, ok bool) {
|
||
alt, ok := mapRunes[r]
|
||
if k := alt.e.kind(); k == EastAsianHalfwidth || k == EastAsianNarrow {
|
||
return alt.r, true
|
||
}
|
||
return r, false
|
||
}
|
||
|
||
func narrowRune(r rune) (narrow rune, ok bool) {
|
||
alt, ok := mapRunes[r]
|
||
if k := alt.e.kind(); k == EastAsianFullwidth || k == EastAsianWide || k == EastAsianAmbiguous {
|
||
return alt.r, true
|
||
}
|
||
return r, false
|
||
}
|
||
|
||
func TestFoldSingleRunes(t *testing.T) {
|
||
for r := rune(0); r < 0x1FFFF; r++ {
|
||
if loSurrogate <= r && r <= hiSurrogate {
|
||
continue
|
||
}
|
||
x, _ := foldRune(r)
|
||
want := string(x)
|
||
got := Fold.String(string(r))
|
||
if got != want {
|
||
t.Errorf("Fold().String(%U) = %+q; want %+q", r, got, want)
|
||
}
|
||
}
|
||
}
|
||
|
||
type transformTest struct {
|
||
desc string
|
||
src string
|
||
nBuf int
|
||
nDst int
|
||
atEOF bool
|
||
dst string
|
||
nSrc int
|
||
err error
|
||
nSpan int
|
||
errSpan error
|
||
}
|
||
|
||
func (tc *transformTest) doTest(t *testing.T, tr Transformer) {
|
||
testtext.Run(t, tc.desc, func(t *testing.T) {
|
||
b := make([]byte, tc.nBuf)
|
||
nDst, nSrc, err := tr.Transform(b, []byte(tc.src), tc.atEOF)
|
||
if got := string(b[:nDst]); got != tc.dst[:nDst] {
|
||
t.Errorf("dst was %+q; want %+q", got, tc.dst)
|
||
}
|
||
if nDst != tc.nDst {
|
||
t.Errorf("nDst was %d; want %d", nDst, tc.nDst)
|
||
}
|
||
if nSrc != tc.nSrc {
|
||
t.Errorf("nSrc was %d; want %d", nSrc, tc.nSrc)
|
||
}
|
||
if err != tc.err {
|
||
t.Errorf("error was %v; want %v", err, tc.err)
|
||
}
|
||
if got := tr.String(tc.src); got != tc.dst {
|
||
t.Errorf("String(%q) = %q; want %q", tc.src, got, tc.dst)
|
||
}
|
||
n, err := tr.Span([]byte(tc.src), tc.atEOF)
|
||
if n != tc.nSpan || err != tc.errSpan {
|
||
t.Errorf("Span: got %d, %v; want %d, %v", n, err, tc.nSpan, tc.errSpan)
|
||
}
|
||
})
|
||
}
|
||
|
||
func TestFold(t *testing.T) {
|
||
for _, tc := range []transformTest{{
|
||
desc: "empty",
|
||
src: "",
|
||
nBuf: 10,
|
||
dst: "",
|
||
nDst: 0,
|
||
nSrc: 0,
|
||
atEOF: false,
|
||
err: nil,
|
||
nSpan: 0,
|
||
errSpan: nil,
|
||
}, {
|
||
desc: "short source 1",
|
||
src: "a\xc2",
|
||
nBuf: 10,
|
||
dst: "a\xc2",
|
||
nDst: 1,
|
||
nSrc: 1,
|
||
atEOF: false,
|
||
err: transform.ErrShortSrc,
|
||
nSpan: 1,
|
||
errSpan: transform.ErrShortSrc,
|
||
}, {
|
||
desc: "short source 2",
|
||
src: "a\xe0\x80",
|
||
nBuf: 10,
|
||
dst: "a\xe0\x80",
|
||
nDst: 1,
|
||
nSrc: 1,
|
||
atEOF: false,
|
||
err: transform.ErrShortSrc,
|
||
nSpan: 1,
|
||
errSpan: transform.ErrShortSrc,
|
||
}, {
|
||
desc: "incomplete but terminated source 1",
|
||
src: "a\xc2",
|
||
nBuf: 10,
|
||
dst: "a\xc2",
|
||
nDst: 2,
|
||
nSrc: 2,
|
||
atEOF: true,
|
||
err: nil,
|
||
nSpan: 2,
|
||
errSpan: nil,
|
||
}, {
|
||
desc: "incomplete but terminated source 2",
|
||
src: "a\xe0\x80",
|
||
nBuf: 10,
|
||
dst: "a\xe0\x80",
|
||
nDst: 3,
|
||
nSrc: 3,
|
||
atEOF: true,
|
||
err: nil,
|
||
nSpan: 3,
|
||
errSpan: nil,
|
||
}, {
|
||
desc: "exact fit dst",
|
||
src: "a\uff01",
|
||
nBuf: 2,
|
||
dst: "a!",
|
||
nDst: 2,
|
||
nSrc: 4,
|
||
atEOF: false,
|
||
err: nil,
|
||
nSpan: 1,
|
||
errSpan: transform.ErrEndOfSpan,
|
||
}, {
|
||
desc: "exact fit dst and src ascii",
|
||
src: "ab",
|
||
nBuf: 2,
|
||
dst: "ab",
|
||
nDst: 2,
|
||
nSrc: 2,
|
||
atEOF: true,
|
||
err: nil,
|
||
nSpan: 2,
|
||
errSpan: nil,
|
||
}, {
|
||
desc: "empty dst",
|
||
src: "\u0300",
|
||
nBuf: 0,
|
||
dst: "\u0300",
|
||
nDst: 0,
|
||
nSrc: 0,
|
||
atEOF: true,
|
||
err: transform.ErrShortDst,
|
||
nSpan: 2,
|
||
errSpan: nil,
|
||
}, {
|
||
desc: "empty dst ascii",
|
||
src: "a",
|
||
nBuf: 0,
|
||
dst: "a",
|
||
nDst: 0,
|
||
nSrc: 0,
|
||
atEOF: true,
|
||
err: transform.ErrShortDst,
|
||
nSpan: 1,
|
||
errSpan: nil,
|
||
}, {
|
||
desc: "short dst 1",
|
||
src: "a\uffe0", // ￠
|
||
nBuf: 2,
|
||
dst: "a\u00a2", // ¢
|
||
nDst: 1,
|
||
nSrc: 1,
|
||
atEOF: false,
|
||
err: transform.ErrShortDst,
|
||
nSpan: 1,
|
||
errSpan: transform.ErrEndOfSpan,
|
||
}, {
|
||
desc: "short dst 2",
|
||
src: "不夠",
|
||
nBuf: 3,
|
||
dst: "不夠",
|
||
nDst: 3,
|
||
nSrc: 3,
|
||
atEOF: true,
|
||
err: transform.ErrShortDst,
|
||
nSpan: 6,
|
||
errSpan: nil,
|
||
}, {
|
||
desc: "short dst fast path",
|
||
src: "fast",
|
||
nDst: 3,
|
||
dst: "fast",
|
||
nBuf: 3,
|
||
nSrc: 3,
|
||
atEOF: true,
|
||
err: transform.ErrShortDst,
|
||
nSpan: 4,
|
||
errSpan: nil,
|
||
}, {
|
||
desc: "short dst larger buffer",
|
||
src: "\uff21" + strings.Repeat("0", 127) + "B",
|
||
nBuf: 128,
|
||
dst: "A" + strings.Repeat("0", 127) + "B",
|
||
nDst: 128,
|
||
nSrc: 130,
|
||
atEOF: true,
|
||
err: transform.ErrShortDst,
|
||
nSpan: 0,
|
||
errSpan: transform.ErrEndOfSpan,
|
||
}, {
|
||
desc: "fast path alternation",
|
||
src: "fast路徑fast路徑",
|
||
nBuf: 20,
|
||
dst: "fast路徑fast路徑",
|
||
nDst: 20,
|
||
nSrc: 20,
|
||
atEOF: true,
|
||
err: nil,
|
||
nSpan: 20,
|
||
errSpan: nil,
|
||
}} {
|
||
tc.doTest(t, Fold)
|
||
}
|
||
}
|
||
|
||
func TestWidenSingleRunes(t *testing.T) {
|
||
for r := rune(0); r < 0x1FFFF; r++ {
|
||
if loSurrogate <= r && r <= hiSurrogate {
|
||
continue
|
||
}
|
||
alt, _ := widenRune(r)
|
||
want := string(alt)
|
||
got := Widen.String(string(r))
|
||
if got != want {
|
||
t.Errorf("Widen().String(%U) = %+q; want %+q", r, got, want)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestWiden(t *testing.T) {
|
||
for _, tc := range []transformTest{{
|
||
desc: "empty",
|
||
src: "",
|
||
nBuf: 10,
|
||
dst: "",
|
||
nDst: 0,
|
||
nSrc: 0,
|
||
atEOF: false,
|
||
err: nil,
|
||
nSpan: 0,
|
||
errSpan: nil,
|
||
}, {
|
||
desc: "short source 1",
|
||
src: "a\xc2",
|
||
nBuf: 10,
|
||
dst: "ａ\xc2",
|
||
nDst: 3,
|
||
nSrc: 1,
|
||
atEOF: false,
|
||
err: transform.ErrShortSrc,
|
||
nSpan: 0,
|
||
errSpan: transform.ErrEndOfSpan,
|
||
}, {
|
||
desc: "short source 2",
|
||
src: "a\xe0\x80",
|
||
nBuf: 10,
|
||
dst: "ａ\xe0\x80",
|
||
nDst: 3,
|
||
nSrc: 1,
|
||
atEOF: false,
|
||
err: transform.ErrShortSrc,
|
||
nSpan: 0,
|
||
errSpan: transform.ErrEndOfSpan,
|
||
}, {
|
||
desc: "incomplete but terminated source 1",
|
||
src: "a\xc2",
|
||
nBuf: 10,
|
||
dst: "ａ\xc2",
|
||
nDst: 4,
|
||
nSrc: 2,
|
||
atEOF: true,
|
||
err: nil,
|
||
nSpan: 0,
|
||
errSpan: transform.ErrEndOfSpan,
|
||
}, {
|
||
desc: "incomplete but terminated source 2",
|
||
src: "a\xe0\x80",
|
||
nBuf: 10,
|
||
dst: "ａ\xe0\x80",
|
||
nDst: 5,
|
||
nSrc: 3,
|
||
atEOF: true,
|
||
err: nil,
|
||
nSpan: 0,
|
||
errSpan: transform.ErrEndOfSpan,
|
||
}, {
|
||
desc: "short source 1 some span",
|
||
src: "ａ\xc2",
|
||
nBuf: 10,
|
||
dst: "ａ\xc2",
|
||
nDst: 3,
|
||
nSrc: 3,
|
||
atEOF: false,
|
||
err: transform.ErrShortSrc,
|
||
nSpan: 3,
|
||
errSpan: transform.ErrShortSrc,
|
||
}, {
|
||
desc: "short source 2 some span",
|
||
src: "ａ\xe0\x80",
|
||
nBuf: 10,
|
||
dst: "ａ\xe0\x80",
|
||
nDst: 3,
|
||
nSrc: 3,
|
||
atEOF: false,
|
||
err: transform.ErrShortSrc,
|
||
nSpan: 3,
|
||
errSpan: transform.ErrShortSrc,
|
||
}, {
|
||
desc: "incomplete but terminated source 1 some span",
|
||
src: "ａ\xc2",
|
||
nBuf: 10,
|
||
dst: "ａ\xc2",
|
||
nDst: 4,
|
||
nSrc: 4,
|
||
atEOF: true,
|
||
err: nil,
|
||
nSpan: 4,
|
||
errSpan: nil,
|
||
}, {
|
||
desc: "incomplete but terminated source 2 some span",
|
||
src: "ａ\xe0\x80",
|
||
nBuf: 10,
|
||
dst: "ａ\xe0\x80",
|
||
nDst: 5,
|
||
nSrc: 5,
|
||
atEOF: true,
|
||
err: nil,
|
||
nSpan: 5,
|
||
errSpan: nil,
|
||
}, {
|
||
desc: "exact fit dst",
|
||
src: "a!",
|
||
nBuf: 6,
|
||
dst: "ａ\uff01",
|
||
nDst: 6,
|
||
nSrc: 2,
|
||
atEOF: false,
|
||
err: nil,
|
||
nSpan: 0,
|
||
errSpan: transform.ErrEndOfSpan,
|
||
}, {
|
||
desc: "empty dst",
|
||
src: "\u0300",
|
||
nBuf: 0,
|
||
dst: "\u0300",
|
||
nDst: 0,
|
||
nSrc: 0,
|
||
atEOF: true,
|
||
err: transform.ErrShortDst,
|
||
nSpan: 2,
|
||
errSpan: nil,
|
||
}, {
|
||
desc: "empty dst ascii",
|
||
src: "a",
|
||
nBuf: 0,
|
||
dst: "ａ",
|
||
nDst: 0,
|
||
nSrc: 0,
|
||
atEOF: true,
|
||
err: transform.ErrShortDst,
|
||
nSpan: 0,
|
||
errSpan: transform.ErrEndOfSpan,
|
||
}, {
|
||
desc: "short dst 1",
|
||
src: "a\uffe0",
|
||
nBuf: 4,
|
||
dst: "ａ\uffe0",
|
||
nDst: 3,
|
||
nSrc: 1,
|
||
atEOF: false,
|
||
err: transform.ErrShortDst,
|
||
nSpan: 0,
|
||
errSpan: transform.ErrEndOfSpan,
|
||
}, {
|
||
desc: "short dst 2",
|
||
src: "不夠",
|
||
nBuf: 3,
|
||
dst: "不夠",
|
||
nDst: 3,
|
||
nSrc: 3,
|
||
atEOF: true,
|
||
err: transform.ErrShortDst,
|
||
nSpan: 6,
|
||
errSpan: nil,
|
||
}, {
|
||
desc: "short dst ascii",
|
||
src: "ascii",
|
||
nBuf: 3,
|
||
dst: "ａｓｃｉｉ", // U+ff41, ...
|
||
nDst: 3,
|
||
nSrc: 1,
|
||
atEOF: true,
|
||
err: transform.ErrShortDst,
|
||
nSpan: 0,
|
||
errSpan: transform.ErrEndOfSpan,
|
||
}, {
|
||
desc: "ambiguous",
|
||
src: "\uffe9",
|
||
nBuf: 4,
|
||
dst: "\u2190",
|
||
nDst: 3,
|
||
nSrc: 3,
|
||
atEOF: false,
|
||
err: nil,
|
||
nSpan: 0,
|
||
errSpan: transform.ErrEndOfSpan,
|
||
}} {
|
||
tc.doTest(t, Widen)
|
||
}
|
||
}
|
||
|
||
func TestNarrowSingleRunes(t *testing.T) {
|
||
for r := rune(0); r < 0x1FFFF; r++ {
|
||
if loSurrogate <= r && r <= hiSurrogate {
|
||
continue
|
||
}
|
||
alt, _ := narrowRune(r)
|
||
want := string(alt)
|
||
got := Narrow.String(string(r))
|
||
if got != want {
|
||
t.Errorf("Narrow().String(%U) = %+q; want %+q", r, got, want)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestNarrow(t *testing.T) {
|
||
for _, tc := range []transformTest{{
|
||
desc: "empty",
|
||
src: "",
|
||
nBuf: 10,
|
||
dst: "",
|
||
nDst: 0,
|
||
nSrc: 0,
|
||
atEOF: false,
|
||
err: nil,
|
||
nSpan: 0,
|
||
errSpan: nil,
|
||
}, {
|
||
desc: "short source 1",
|
||
src: "a\xc2",
|
||
nBuf: 10,
|
||
dst: "a\xc2",
|
||
nDst: 1,
|
||
nSrc: 1,
|
||
atEOF: false,
|
||
err: transform.ErrShortSrc,
|
||
nSpan: 1,
|
||
errSpan: transform.ErrShortSrc,
|
||
}, {
|
||
desc: "short source 2",
|
||
src: "ａ\xe0\x80",
|
||
nBuf: 10,
|
||
dst: "a\xe0\x80",
|
||
nDst: 1,
|
||
nSrc: 3,
|
||
atEOF: false,
|
||
err: transform.ErrShortSrc,
|
||
nSpan: 0,
|
||
errSpan: transform.ErrEndOfSpan,
|
||
}, {
|
||
desc: "incomplete but terminated source 1",
|
||
src: "ａ\xc2",
|
||
nBuf: 10,
|
||
dst: "a\xc2",
|
||
nDst: 2,
|
||
nSrc: 4,
|
||
atEOF: true,
|
||
err: nil,
|
||
nSpan: 0,
|
||
errSpan: transform.ErrEndOfSpan,
|
||
}, {
|
||
desc: "incomplete but terminated source 2",
|
||
src: "ａ\xe0\x80",
|
||
nBuf: 10,
|
||
dst: "a\xe0\x80",
|
||
nDst: 3,
|
||
nSrc: 5,
|
||
atEOF: true,
|
||
err: nil,
|
||
nSpan: 0,
|
||
errSpan: transform.ErrEndOfSpan,
|
||
}, {
|
||
desc: "exact fit dst",
|
||
src: "ａ\uff01",
|
||
nBuf: 2,
|
||
dst: "a!",
|
||
nDst: 2,
|
||
nSrc: 6,
|
||
atEOF: false,
|
||
err: nil,
|
||
nSpan: 0,
|
||
errSpan: transform.ErrEndOfSpan,
|
||
}, {
|
||
desc: "exact fit dst some span",
|
||
src: "a\uff01",
|
||
nBuf: 2,
|
||
dst: "a!",
|
||
nDst: 2,
|
||
nSrc: 4,
|
||
atEOF: false,
|
||
err: nil,
|
||
nSpan: 1,
|
||
errSpan: transform.ErrEndOfSpan,
|
||
}, {
|
||
desc: "empty dst",
|
||
src: "\u0300",
|
||
nBuf: 0,
|
||
dst: "\u0300",
|
||
nDst: 0,
|
||
nSrc: 0,
|
||
atEOF: true,
|
||
err: transform.ErrShortDst,
|
||
nSpan: 2,
|
||
errSpan: nil,
|
||
}, {
|
||
desc: "empty dst ascii",
|
||
src: "a",
|
||
nBuf: 0,
|
||
dst: "a",
|
||
nDst: 0,
|
||
nSrc: 0,
|
||
atEOF: true,
|
||
err: transform.ErrShortDst,
|
||
nSpan: 1,
|
||
errSpan: nil,
|
||
}, {
|
||
desc: "short dst 1",
|
||
src: "ａ\uffe0", // ￠
|
||
nBuf: 2,
|
||
dst: "a\u00a2", // ¢
|
||
nDst: 1,
|
||
nSrc: 3,
|
||
atEOF: false,
|
||
err: transform.ErrShortDst,
|
||
nSpan: 0,
|
||
errSpan: transform.ErrEndOfSpan,
|
||
}, {
|
||
desc: "short dst 2",
|
||
src: "不夠",
|
||
nBuf: 3,
|
||
dst: "不夠",
|
||
nDst: 3,
|
||
nSrc: 3,
|
||
atEOF: true,
|
||
err: transform.ErrShortDst,
|
||
nSpan: 6,
|
||
errSpan: nil,
|
||
}, {
|
||
// Create a narrow variant of ambiguous runes, if they exist.
|
||
desc: "ambiguous",
|
||
src: "\u2190",
|
||
nBuf: 4,
|
||
dst: "\uffe9",
|
||
nDst: 3,
|
||
nSrc: 3,
|
||
atEOF: false,
|
||
err: nil,
|
||
nSpan: 0,
|
||
errSpan: transform.ErrEndOfSpan,
|
||
}, {
|
||
desc: "short dst fast path",
|
||
src: "fast",
|
||
nBuf: 3,
|
||
dst: "fast",
|
||
nDst: 3,
|
||
nSrc: 3,
|
||
atEOF: true,
|
||
err: transform.ErrShortDst,
|
||
nSpan: 4,
|
||
errSpan: nil,
|
||
}, {
|
||
desc: "short dst larger buffer",
|
||
src: "\uff21" + strings.Repeat("0", 127) + "B",
|
||
nBuf: 128,
|
||
dst: "A" + strings.Repeat("0", 127) + "B",
|
||
nDst: 128,
|
||
nSrc: 130,
|
||
atEOF: true,
|
||
err: transform.ErrShortDst,
|
||
nSpan: 0,
|
||
errSpan: transform.ErrEndOfSpan,
|
||
}, {
|
||
desc: "fast path alternation",
|
||
src: "fast路徑fast路徑",
|
||
nBuf: 20,
|
||
dst: "fast路徑fast路徑",
|
||
nDst: 20,
|
||
nSrc: 20,
|
||
atEOF: true,
|
||
err: nil,
|
||
nSpan: 20,
|
||
errSpan: nil,
|
||
}} {
|
||
tc.doTest(t, Narrow)
|
||
}
|
||
}
|
||
|
||
func bench(b *testing.B, t Transformer, s string) {
|
||
dst := make([]byte, 1024)
|
||
src := []byte(s)
|
||
b.SetBytes(int64(len(src)))
|
||
b.ResetTimer()
|
||
for i := 0; i < b.N; i++ {
|
||
t.Transform(dst, src, true)
|
||
}
|
||
}
|
||
|
||
func changingRunes(f func(r rune) (rune, bool)) string {
|
||
buf := &bytes.Buffer{}
|
||
for r := rune(0); r <= 0xFFFF; r++ {
|
||
if _, ok := foldRune(r); ok {
|
||
buf.WriteRune(r)
|
||
}
|
||
}
|
||
return buf.String()
|
||
}
|
||
|
||
func BenchmarkFoldASCII(b *testing.B) {
|
||
bench(b, Fold, testtext.ASCII)
|
||
}
|
||
|
||
func BenchmarkFoldCJK(b *testing.B) {
|
||
bench(b, Fold, testtext.CJK)
|
||
}
|
||
|
||
func BenchmarkFoldNonCanonical(b *testing.B) {
|
||
bench(b, Fold, changingRunes(foldRune))
|
||
}
|
||
|
||
func BenchmarkFoldOther(b *testing.B) {
|
||
bench(b, Fold, testtext.TwoByteUTF8+testtext.ThreeByteUTF8)
|
||
}
|
||
|
||
func BenchmarkWideASCII(b *testing.B) {
|
||
bench(b, Widen, testtext.ASCII)
|
||
}
|
||
|
||
func BenchmarkWideCJK(b *testing.B) {
|
||
bench(b, Widen, testtext.CJK)
|
||
}
|
||
|
||
func BenchmarkWideNonCanonical(b *testing.B) {
|
||
bench(b, Widen, changingRunes(widenRune))
|
||
}
|
||
|
||
func BenchmarkWideOther(b *testing.B) {
|
||
bench(b, Widen, testtext.TwoByteUTF8+testtext.ThreeByteUTF8)
|
||
}
|
||
|
||
func BenchmarkNarrowASCII(b *testing.B) {
|
||
bench(b, Narrow, testtext.ASCII)
|
||
}
|
||
|
||
func BenchmarkNarrowCJK(b *testing.B) {
|
||
bench(b, Narrow, testtext.CJK)
|
||
}
|
||
|
||
func BenchmarkNarrowNonCanonical(b *testing.B) {
|
||
bench(b, Narrow, changingRunes(narrowRune))
|
||
}
|
||
|
||
func BenchmarkNarrowOther(b *testing.B) {
|
||
bench(b, Narrow, testtext.TwoByteUTF8+testtext.ThreeByteUTF8)
|
||
}
|