yves
/
rclone
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

lib/encoder: add LeftPeriod encoding

s3-about
Nick Craig-Wood 2019-09-30 16:41:48 +01:00
parent 8d8fad724b
commit 5cef5f8b49
4 changed files with 27404 additions and 21501 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
fstest/fstests/fstests.go
View File

@ -554,6 +554,7 @@ func Run(t *testing.T, opt *Opt) {
{"leading LF", "\nleading LF"},
{"leading HT", "\tleading HT"},
{"leading VT", "\vleading VT"},
{"leading dot", ".leading dot"},
{"trailing space", "trailing space "},
{"trailing CR", "trailing CR\r"},
{"trailing LF", "trailing LF\n"},

12
lib/encoder/encoder.go
View File

@ -53,6 +53,7 @@ const (
EncodeDel // DEL(0x7F)
EncodeCtl // CTRL(0x01-0x1F)
EncodeLeftSpace // Leading SPACE
EncodeLeftPeriod // Leading .
EncodeLeftTilde // Leading ~
EncodeLeftCrLfHtVt // Leading CR LF HT VT
EncodeRightSpace // Trailing SPACE
@ -127,6 +128,13 @@ func (mask MultiEncoder) Encode(in string) string {
prefix, in = string(QuoteRune)+"␠", in[l:] // SYMBOL FOR SPACE
}
}
if mask.Has(EncodeLeftPeriod) && prefix == "" { // Leading PERIOD
if in[0] == '.' {
prefix, in = "．", in[1:] // FULLWIDTH FULL STOP
} else if r, l := utf8.DecodeRuneInString(in); r == '．' { // FULLWIDTH FULL STOP
prefix, in = string(QuoteRune)+"．", in[l:] // FULLWIDTH FULL STOP
}
}
if mask.Has(EncodeLeftTilde) && prefix == "" { // Leading ~
if in[0] == '~' {
prefix, in = string('~'+fullOffset), in[1:] // FULLWIDTH TILDE
@ -533,6 +541,8 @@ func (mask MultiEncoder) Decode(in string) string {
prefix := ""
if r, l1 := utf8.DecodeRuneInString(in); mask.Has(EncodeLeftSpace) && r == '␠' { // SYMBOL FOR SPACE
prefix, in = " ", in[l1:]
} else if mask.Has(EncodeLeftPeriod) && r == '．' { // FULLWIDTH FULL STOP
prefix, in = ".", in[l1:]
} else if mask.Has(EncodeLeftTilde) && r == '～' { // FULLWIDTH TILDE
prefix, in = "~", in[l1:]
} else if mask.Has(EncodeLeftCrLfHtVt) && (r == '␀'+'\t' || r == '␀'+'\n' || r == '␀'+'\v' || r == '␀'+'\r') {
@ -540,6 +550,8 @@ func (mask MultiEncoder) Decode(in string) string {
} else if r == QuoteRune {
if r, l2 := utf8.DecodeRuneInString(in[l1:]); mask.Has(EncodeLeftSpace) && r == '␠' { // SYMBOL FOR SPACE
prefix, in = "␠", in[l1+l2:]
} else if mask.Has(EncodeLeftPeriod) && r == '．' { // FULLWIDTH FULL STOP
prefix, in = "．", in[l1+l2:]
} else if mask.Has(EncodeLeftTilde) && r == '～' { // FULLWIDTH TILDE
prefix, in = "～", in[l1+l2:]
} else if mask.Has(EncodeLeftCrLfHtVt) && (r == '␀'+'\t' || r == '␀'+'\n' || r == '␀'+'\v' || r == '␀'+'\r') {

48880
lib/encoder/encoder_cases_test.go
View File

File diff suppressed because it is too large Load Diff

12
lib/encoder/internal/gen/main.go
View File

@ -57,6 +57,7 @@ var maskBits = []struct {
{encoder.EncodeDel, "EncodeDel"},
{encoder.EncodeCtl, "EncodeCtl"},
{encoder.EncodeLeftSpace, "EncodeLeftSpace"},
{encoder.EncodeLeftPeriod, "EncodeLeftPeriod"},
{encoder.EncodeLeftTilde, "EncodeLeftTilde"},
{encoder.EncodeLeftCrLfHtVt, "EncodeLeftCrLfHtVt"},
{encoder.EncodeRightSpace, "EncodeRightSpace"},
@ -76,6 +77,7 @@ type edge struct {
var allEdges = []edge{
{encoder.EncodeLeftSpace, "EncodeLeftSpace", edgeLeft, []rune{' '}, []rune{'␠'}},
{encoder.EncodeLeftPeriod, "EncodeLeftPeriod", edgeLeft, []rune{'.'}, []rune{'．'}},
{encoder.EncodeLeftTilde, "EncodeLeftTilde", edgeLeft, []rune{'~'}, []rune{'～'}},
{encoder.EncodeLeftCrLfHtVt, "EncodeLeftCrLfHtVt", edgeLeft,
[]rune{'\t', '\n', '\v', '\r'},
@ -311,6 +313,10 @@ var testCasesSingleEdge = []testCase{
mask: EncodeLeftSpace,
in: " ",
out: "␠ ",
}, { // %d
mask: EncodeLeftPeriod,
in: "..",
out: "．.",
}, { // %d
mask: EncodeLeftTilde,
in: "~~",
@ -339,6 +345,10 @@ var testCasesSingleEdge = []testCase{
mask: EncodeLeftSpace | EncodeRightSpace,
in: " ",
out: "␠ ␠",
}, { // %d
mask: EncodeLeftPeriod | EncodeRightPeriod,
in: "...",
out: "．.．",
}, { // %d
mask: EncodeRightPeriod | EncodeRightSpace,
in: "a. ",
@ -351,7 +361,7 @@ var testCasesSingleEdge = []testCase{
}
var testCasesDoubleEdge = []testCase{
`, i(), i(), i(), i(), i(), i(), i(), i(), i(), i()))("Error writing test case:")
`, i(), i(), i(), i(), i(), i(), i(), i(), i(), i(), i(), i()))("Error writing test case:")
_i = 0
for _, e1 := range allEdges {
for _, e2 := range allEdges {