yves
/
rclone
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

lib/encoder: move definitions here and remove uint casts

s3-about
Nick Craig-Wood 2020-01-14 21:22:02 +00:00
parent 3c620d521d
commit c555dc71c2
5 changed files with 172 additions and 200 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

246
fs/encodings/encodings.go
View File

@ -9,24 +9,14 @@ import (
"github.com/rclone/rclone/lib/encoder"
)
// Base only encodes the zero byte and slash
const Base = encoder.MultiEncoder(
encoder.EncodeZero |
encoder.EncodeSlash |
encoder.EncodeDot)
// Display is the internal encoding for logging and output
const Display = encoder.Standard
// LocalUnix is the encoding used by the local backend for non windows platforms
const LocalUnix = Base
const LocalUnix = encoder.Base
// LocalMacOS is the encoding used by the local backend for macOS
//
// macOS can't store invalid UTF-8, it converts them into %XX encoding
const LocalMacOS = encoder.MultiEncoder(
uint(Base) |
encoder.EncodeInvalidUtf8)
const LocalMacOS = (encoder.Base |
encoder.EncodeInvalidUtf8)
// LocalWindows is the encoding used by the local backend for windows platforms
//
@ -48,31 +38,28 @@ const LocalMacOS = encoder.MultiEncoder(
// Also encode invalid UTF-8 bytes as Go can't convert them to UTF-16.
//
// https://docs.microsoft.com/de-de/windows/desktop/FileIO/naming-a-file#naming-conventions
const LocalWindows = encoder.MultiEncoder(
uint(Base) |
encoder.EncodeWin |
encoder.EncodeBackSlash |
encoder.EncodeCtl |
encoder.EncodeRightSpace |
encoder.EncodeRightPeriod |
encoder.EncodeInvalidUtf8)
const LocalWindows = (encoder.Base |
encoder.EncodeWin |
encoder.EncodeBackSlash |
encoder.EncodeCtl |
encoder.EncodeRightSpace |
encoder.EncodeRightPeriod |
encoder.EncodeInvalidUtf8)
// AmazonCloudDrive is the encoding used by the amazonclouddrive backend
//
// Encode invalid UTF-8 bytes as json doesn't handle them properly.
const AmazonCloudDrive = encoder.MultiEncoder(
uint(Base) |
encoder.EncodeInvalidUtf8)
const AmazonCloudDrive = (encoder.Base |
encoder.EncodeInvalidUtf8)
// B2 is the encoding used by the b2 backend
//
// See: https://www.backblaze.com/b2/docs/files.html
// Encode invalid UTF-8 bytes as json doesn't handle them properly.
// FIXME: allow /, but not leading, trailing or double
const B2 = encoder.MultiEncoder(
uint(Display) |
encoder.EncodeBackSlash |
encoder.EncodeInvalidUtf8)
const B2 = (encoder.Display |
encoder.EncodeBackSlash |
encoder.EncodeInvalidUtf8)
// Box is the encoding used by the box backend
//
@ -83,18 +70,16 @@ const B2 = encoder.MultiEncoder(
//
// Testing revealed names with leading spaces work fine.
// Also encode invalid UTF-8 bytes as json doesn't handle them properly.
const Box = encoder.MultiEncoder(
uint(Display) |
encoder.EncodeBackSlash |
encoder.EncodeRightSpace |
encoder.EncodeInvalidUtf8)
const Box = (encoder.Display |
encoder.EncodeBackSlash |
encoder.EncodeRightSpace |
encoder.EncodeInvalidUtf8)
// Drive is the encoding used by the drive backend
//
// Encode invalid UTF-8 bytes as json doesn't handle them properly.
// Don't encode / as it's a valid name character in drive.
const Drive = encoder.MultiEncoder(
encoder.EncodeInvalidUtf8)
const Drive = (encoder.EncodeInvalidUtf8)
// Dropbox is the encoding used by the dropbox backend
//
@ -102,52 +87,46 @@ const Drive = encoder.MultiEncoder(
// as invalid characters.
// Testing revealed names with trailing spaces and the DEL character don't work.
// Also encode invalid UTF-8 bytes as json doesn't handle them properly.
const Dropbox = encoder.MultiEncoder(
uint(Base) |
encoder.EncodeBackSlash |
encoder.EncodeDel |
encoder.EncodeRightSpace |
encoder.EncodeInvalidUtf8)
const Dropbox = (encoder.Base |
encoder.EncodeBackSlash |
encoder.EncodeDel |
encoder.EncodeRightSpace |
encoder.EncodeInvalidUtf8)
// GoogleCloudStorage is the encoding used by the googlecloudstorage backend
const GoogleCloudStorage = encoder.MultiEncoder(
uint(Base) |
encoder.EncodeCrLf |
encoder.EncodeInvalidUtf8)
const GoogleCloudStorage = (encoder.Base |
encoder.EncodeCrLf |
encoder.EncodeInvalidUtf8)
// JottaCloud is the encoding used by the jottacloud backend
//
// Encode invalid UTF-8 bytes as xml doesn't handle them properly.
//
// Also: '*', '/', ':', '<', '>', '?', '\"', '\x00', '|'
const JottaCloud = encoder.MultiEncoder(
uint(Display) |
encoder.EncodeWin | // :?"*<>|
encoder.EncodeInvalidUtf8)
const JottaCloud = (encoder.Display |
encoder.EncodeWin | // :?"*<>|
encoder.EncodeInvalidUtf8)
// Koofr is the encoding used by the koofr backend
//
// Encode invalid UTF-8 bytes as json doesn't handle them properly.
const Koofr = encoder.MultiEncoder(
uint(Display) |
encoder.EncodeBackSlash |
encoder.EncodeInvalidUtf8)
const Koofr = (encoder.Display |
encoder.EncodeBackSlash |
encoder.EncodeInvalidUtf8)
// Mailru is the encoding used by the mailru backend
//
// Encode invalid UTF-8 bytes as json doesn't handle them properly.
const Mailru = encoder.MultiEncoder(
uint(Display) |
encoder.EncodeWin | // :?"*<>|
encoder.EncodeBackSlash |
encoder.EncodeInvalidUtf8)
const Mailru = (encoder.Display |
encoder.EncodeWin | // :?"*<>|
encoder.EncodeBackSlash |
encoder.EncodeInvalidUtf8)
// Mega is the encoding used by the mega backend
//
// Encode invalid UTF-8 bytes as json doesn't handle them properly.
const Mega = encoder.MultiEncoder(
uint(Base) |
encoder.EncodeInvalidUtf8)
const Mega = (encoder.Base |
encoder.EncodeInvalidUtf8)
// OneDrive is the encoding used by the onedrive backend
//
@ -184,16 +163,15 @@ const Mega = encoder.MultiEncoder(
// the same rules as the Windows naming conventions.
//
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/concepts/addressing-driveitems?view=odsp-graph-online#path-encoding
const OneDrive = encoder.MultiEncoder(
uint(Display) |
encoder.EncodeBackSlash |
encoder.EncodeHashPercent |
encoder.EncodeLeftSpace |
encoder.EncodeLeftTilde |
encoder.EncodeRightPeriod |
encoder.EncodeRightSpace |
encoder.EncodeWin |
encoder.EncodeInvalidUtf8)
const OneDrive = (encoder.Display |
encoder.EncodeBackSlash |
encoder.EncodeHashPercent |
encoder.EncodeLeftSpace |
encoder.EncodeLeftTilde |
encoder.EncodeRightPeriod |
encoder.EncodeRightSpace |
encoder.EncodeWin |
encoder.EncodeInvalidUtf8)
// OpenDrive is the encoding used by the opendrive backend
//
@ -218,44 +196,40 @@ const OneDrive = encoder.MultiEncoder(
// Also encode invalid UTF-8 bytes as json doesn't handle them properly.
//
// https://www.opendrive.com/wp-content/uploads/guides/OpenDrive_API_guide.pdf
const OpenDrive = encoder.MultiEncoder(
uint(Base) |
encoder.EncodeWin |
encoder.EncodeLeftCrLfHtVt |
encoder.EncodeRightCrLfHtVt |
encoder.EncodeBackSlash |
encoder.EncodeLeftSpace |
encoder.EncodeRightSpace |
encoder.EncodeInvalidUtf8)
const OpenDrive = (encoder.Base |
encoder.EncodeWin |
encoder.EncodeLeftCrLfHtVt |
encoder.EncodeRightCrLfHtVt |
encoder.EncodeBackSlash |
encoder.EncodeLeftSpace |
encoder.EncodeRightSpace |
encoder.EncodeInvalidUtf8)
// PremiumizeMe is the encoding used by the premiumizeme backend
//
// Encode invalid UTF-8 bytes as json doesn't handle them properly.
const PremiumizeMe = encoder.MultiEncoder(
uint(Display) |
encoder.EncodeBackSlash |
encoder.EncodeDoubleQuote |
encoder.EncodeInvalidUtf8)
const PremiumizeMe = (encoder.Display |
encoder.EncodeBackSlash |
encoder.EncodeDoubleQuote |
encoder.EncodeInvalidUtf8)
// Pcloud is the encoding used by the pcloud backend
//
// Encode invalid UTF-8 bytes as json doesn't handle them properly.
//
// TODO: Investigate Unicode simplification (＼ gets converted to \ server-side)
const Pcloud = encoder.MultiEncoder(
uint(Display) |
encoder.EncodeBackSlash |
encoder.EncodeInvalidUtf8)
const Pcloud = (encoder.Display |
encoder.EncodeBackSlash |
encoder.EncodeInvalidUtf8)
// Putio is the encoding used by the putio backend
//
// Note that \ is renamed to -
//
// Encode invalid UTF-8 bytes as json doesn't handle them properly.
const Putio = encoder.MultiEncoder(
uint(Display) |
encoder.EncodeBackSlash |
encoder.EncodeInvalidUtf8)
const Putio = (encoder.Display |
encoder.EncodeBackSlash |
encoder.EncodeInvalidUtf8)
// Fichier is the encoding used by the fichier backend
//
@ -270,17 +244,16 @@ const Putio = encoder.MultiEncoder(
// '`': '｀', // FULLWIDTH GRAVE ACCENT
//
// Leading space and trailing space
const Fichier = encoder.MultiEncoder(
uint(Display) |
encoder.EncodeBackSlash |
encoder.EncodeSingleQuote |
encoder.EncodeBackQuote |
encoder.EncodeDoubleQuote |
encoder.EncodeLtGt |
encoder.EncodeDollar |
encoder.EncodeLeftSpace |
encoder.EncodeRightSpace |
encoder.EncodeInvalidUtf8)
const Fichier = (encoder.Display |
encoder.EncodeBackSlash |
encoder.EncodeSingleQuote |
encoder.EncodeBackQuote |
encoder.EncodeDoubleQuote |
encoder.EncodeLtGt |
encoder.EncodeDollar |
encoder.EncodeLeftSpace |
encoder.EncodeRightSpace |
encoder.EncodeInvalidUtf8)
// FTP is the encoding used by the ftp backend
//
@ -289,9 +262,8 @@ const Fichier = encoder.MultiEncoder(
//
// proftpd can't handle '*' in file names
// pureftpd can't handle '[', ']' or '*'
const FTP = encoder.MultiEncoder(
uint(Display) |
encoder.EncodeRightSpace)
const FTP = (encoder.Display |
encoder.EncodeRightSpace)
// S3 is the encoding used by the s3 backend
//
@ -305,58 +277,52 @@ const FTP = encoder.MultiEncoder(
// - doubled / encoding
// - trailing / encoding
// so that AWS keys are always valid file names
const S3 = encoder.MultiEncoder(
encoder.EncodeInvalidUtf8 |
encoder.EncodeSlash |
encoder.EncodeDot)
const S3 = (encoder.EncodeInvalidUtf8 |
encoder.EncodeSlash |
encoder.EncodeDot)
// Swift is the encoding used by the swift backend
const Swift = encoder.MultiEncoder(
encoder.EncodeInvalidUtf8 |
encoder.EncodeSlash)
const Swift = (encoder.EncodeInvalidUtf8 |
encoder.EncodeSlash)
// AzureBlob is the encoding used by the azureblob backend
const AzureBlob = encoder.MultiEncoder(
encoder.EncodeInvalidUtf8 |
encoder.EncodeSlash |
encoder.EncodeCtl |
encoder.EncodeDel |
encoder.EncodeBackSlash |
encoder.EncodeRightPeriod)
const AzureBlob = (encoder.EncodeInvalidUtf8 |
encoder.EncodeSlash |
encoder.EncodeCtl |
encoder.EncodeDel |
encoder.EncodeBackSlash |
encoder.EncodeRightPeriod)
// QingStor is the encoding used by the qingstor backend
const QingStor = encoder.MultiEncoder(
encoder.EncodeInvalidUtf8 |
encoder.EncodeCtl |
encoder.EncodeSlash)
const QingStor = (encoder.EncodeInvalidUtf8 |
encoder.EncodeCtl |
encoder.EncodeSlash)
// Sharefile is the encoding used by the sharefile backend
const Sharefile = encoder.MultiEncoder(
uint(Base) |
encoder.EncodeWin | // :?"*<>|
encoder.EncodeBackSlash | // \
encoder.EncodeCtl |
encoder.EncodeRightSpace |
encoder.EncodeRightPeriod |
encoder.EncodeLeftSpace |
encoder.EncodeLeftPeriod |
encoder.EncodeInvalidUtf8)
const Sharefile = (encoder.Base |
encoder.EncodeWin | // :?"*<>|
encoder.EncodeBackSlash | // \
encoder.EncodeCtl |
encoder.EncodeRightSpace |
encoder.EncodeRightPeriod |
encoder.EncodeLeftSpace |
encoder.EncodeLeftPeriod |
encoder.EncodeInvalidUtf8)
// Yandex is the encoding used by the yandex backend
//
// Of the control characters \t \n \r are allowed
// it doesn't seem worth making an exception for this
const Yandex = encoder.MultiEncoder(
uint(Display) |
encoder.EncodeInvalidUtf8)
const Yandex = (encoder.Display |
encoder.EncodeInvalidUtf8)
// ByName returns the encoder for a give backend name or nil
func ByName(name string) encoder.Encoder {
switch strings.ToLower(name) {
case "base":
return Base
return encoder.Base
case "display":
return Display
return encoder.Display
case "amazonclouddrive":
return AmazonCloudDrive
case "azureblob":

56
lib/encoder/encoder.go
View File

@ -37,32 +37,32 @@ const (
// Possible flags for the MultiEncoder
const (
EncodeZero uint = 0 // NUL(0x00)
EncodeSlash uint = 1 << iota // /
EncodeLtGt // <>
EncodeDoubleQuote // "
EncodeSingleQuote // '
EncodeBackQuote // `
EncodeDollar // $
EncodeColon // :
EncodeQuestion // ?
EncodeAsterisk // *
EncodePipe // |
EncodeHash // #
EncodePercent // %
EncodeBackSlash // \
EncodeCrLf // CR(0x0D), LF(0x0A)
EncodeDel // DEL(0x7F)
EncodeCtl // CTRL(0x01-0x1F)
EncodeLeftSpace // Leading SPACE
EncodeLeftPeriod // Leading .
EncodeLeftTilde // Leading ~
EncodeLeftCrLfHtVt // Leading CR LF HT VT
EncodeRightSpace // Trailing SPACE
EncodeRightPeriod // Trailing .
EncodeRightCrLfHtVt // Trailing CR LF HT VT
EncodeInvalidUtf8 // Invalid UTF-8 bytes
EncodeDot // . and .. names
EncodeZero MultiEncoder = 0 // NUL(0x00)
EncodeSlash MultiEncoder = 1 << iota // /
EncodeLtGt // <>
EncodeDoubleQuote // "
EncodeSingleQuote // '
EncodeBackQuote // `
EncodeDollar // $
EncodeColon // :
EncodeQuestion // ?
EncodeAsterisk // *
EncodePipe // |
EncodeHash // #
EncodePercent // %
EncodeBackSlash // \
EncodeCrLf // CR(0x0D), LF(0x0A)
EncodeDel // DEL(0x7F)
EncodeCtl // CTRL(0x01-0x1F)
EncodeLeftSpace // Leading SPACE
EncodeLeftPeriod // Leading .
EncodeLeftTilde // Leading ~
EncodeLeftCrLfHtVt // Leading CR LF HT VT
EncodeRightSpace // Trailing SPACE
EncodeRightPeriod // Trailing .
EncodeRightCrLfHtVt // Trailing CR LF HT VT
EncodeInvalidUtf8 // Invalid UTF-8 bytes
EncodeDot // . and .. names
// Synthetic
EncodeWin = EncodeColon | EncodeQuestion | EncodeDoubleQuote | EncodeAsterisk | EncodeLtGt | EncodePipe // :?"*<>|
@ -70,8 +70,8 @@ const (
)
// Has returns true if flag is contained in mask
func (mask MultiEncoder) Has(flag uint) bool {
return uint(mask)&flag != 0
func (mask MultiEncoder) Has(flag MultiEncoder) bool {
return mask&flag != 0
}
// Encoder can transform names to and from the original and translated version.

41
lib/encoder/encoder_test.go
View File

@ -19,7 +19,7 @@ var (
func TestEncodeString(t *testing.T) {
for _, test := range []struct {
mask uint
mask MultiEncoder
want string
}{
{0, "None"},
@ -31,7 +31,7 @@ func TestEncodeString(t *testing.T) {
{EncodeSlash | EncodeDollar | EncodeColon, "Slash,Dollar,Colon"},
{EncodeSlash | (1 << 31), "Slash,0x80000000"},
} {
got := MultiEncoder(test.mask).String()
got := test.mask.String()
assert.Equal(t, test.want, got)
}
@ -40,7 +40,7 @@ func TestEncodeString(t *testing.T) {
func TestEncodeSet(t *testing.T) {
for _, test := range []struct {
in string
want uint
want MultiEncoder
wantErr bool
}{
{"", 0, true},
@ -58,20 +58,20 @@ func TestEncodeSet(t *testing.T) {
var got MultiEncoder
err := got.Set(test.in)
assert.Equal(t, test.wantErr, err != nil, err)
assert.Equal(t, MultiEncoder(test.want), got, test.in)
assert.Equal(t, test.want, got, test.in)
}
}
type testCase struct {
mask uint
mask MultiEncoder
in string
out string
}
func TestEncodeSingleMask(t *testing.T) {
for i, tc := range testCasesSingle {
e := MultiEncoder(tc.mask)
e := tc.mask
t.Run(strconv.FormatInt(int64(i), 10), func(t *testing.T) {
got := e.Encode(tc.in)
if got != tc.out {
@ -87,7 +87,7 @@ func TestEncodeSingleMask(t *testing.T) {
func TestEncodeSingleMaskEdge(t *testing.T) {
for i, tc := range testCasesSingleEdge {
e := MultiEncoder(tc.mask)
e := tc.mask
t.Run(strconv.FormatInt(int64(i), 10), func(t *testing.T) {
got := e.Encode(tc.in)
if got != tc.out {
@ -103,7 +103,7 @@ func TestEncodeSingleMaskEdge(t *testing.T) {
func TestEncodeDoubleMaskEdge(t *testing.T) {
for i, tc := range testCasesDoubleEdge {
e := MultiEncoder(tc.mask)
e := tc.mask
t.Run(strconv.FormatInt(int64(i), 10), func(t *testing.T) {
got := e.Encode(tc.in)
if got != tc.out {
@ -161,7 +161,7 @@ func TestEncodeInvalidUnicode(t *testing.T) {
out: "a\xBF＼\xFEb",
},
} {
e := MultiEncoder(tc.mask)
e := tc.mask
t.Run(strconv.FormatInt(int64(i), 10), func(t *testing.T) {
got := e.Encode(tc.in)
if got != tc.out {
@ -203,7 +203,7 @@ func TestEncodeDot(t *testing.T) {
out: ". .",
},
} {
e := MultiEncoder(tc.mask)
e := tc.mask
t.Run(strconv.FormatInt(int64(i), 10), func(t *testing.T) {
got := e.Encode(tc.in)
if got != tc.out {
@ -245,7 +245,7 @@ func TestDecodeHalf(t *testing.T) {
out: "a‛B\\‛Eg",
},
} {
e := MultiEncoder(tc.mask)
e := tc.mask
t.Run(strconv.FormatInt(int64(i), 10), func(t *testing.T) {
got := e.Decode(tc.in)
if got != tc.out {
@ -255,16 +255,15 @@ func TestDecodeHalf(t *testing.T) {
}
}
const oneDrive = MultiEncoder(
uint(Standard) |
EncodeWin |
EncodeBackSlash |
EncodeHashPercent |
EncodeDel |
EncodeCtl |
EncodeLeftTilde |
EncodeRightSpace |
EncodeRightPeriod)
const oneDrive = (Standard |
EncodeWin |
EncodeBackSlash |
EncodeHashPercent |
EncodeDel |
EncodeCtl |
EncodeLeftTilde |
EncodeRightSpace |
EncodeRightPeriod)
var benchTests = []struct {
in string

10
lib/encoder/internal/gen/main.go
View File

@ -20,7 +20,7 @@ const (
)
type mapping struct {
mask uint
mask encoder.MultiEncoder
src, dst []rune
}
type stringPair struct {
@ -36,7 +36,7 @@ package encoder
`
var maskBits = []struct {
mask uint
mask encoder.MultiEncoder
name string
}{
{encoder.EncodeZero, "EncodeZero"},
@ -68,7 +68,7 @@ var maskBits = []struct {
}
type edge struct {
mask uint
mask encoder.MultiEncoder
name string
edge int
orig []rune
@ -429,7 +429,7 @@ func fatalW(_ int, err error) func(...interface{}) {
return func(s ...interface{}) {}
}
func invalidMask(mask uint) bool {
func invalidMask(mask encoder.MultiEncoder) bool {
return mask&(encoder.EncodeCtl|encoder.EncodeCrLf) != 0 && mask&(encoder.EncodeLeftCrLfHtVt|encoder.EncodeRightCrLfHtVt) != 0
}
@ -445,7 +445,7 @@ func runeRange(l, h rune) []rune {
return out
}
func getMapping(mask uint) mapping {
func getMapping(mask encoder.MultiEncoder) mapping {
for _, m := range allMappings {
if m.mask == mask {
return m

19
lib/encoder/standard.go
View File

@ -7,9 +7,16 @@ package encoder
// List of replaced characters:
// (0x00) -> '␀' // SYMBOL FOR NULL
// / (slash) -> '／' // FULLWIDTH SOLIDUS
const Standard = MultiEncoder(
EncodeZero |
EncodeSlash |
EncodeCtl |
EncodeDel |
EncodeDot)
const Standard = (EncodeZero |
EncodeSlash |
EncodeCtl |
EncodeDel |
EncodeDot)
// Base only encodes the zero byte and slash
const Base = (EncodeZero |
EncodeSlash |
EncodeDot)
// Display is the internal encoding for logging and output
const Display = Standard