yves
/
rclone
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

backends: move encoding definitions from fs/encodings

s3-about
Nick Craig-Wood 2020-01-14 21:51:49 +00:00
parent c555dc71c2
commit bafe7d5a73
28 changed files with 243 additions and 534 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
backend/amazonclouddrive/amazonclouddrive.go
View File

@ -28,7 +28,6 @@ import (
"github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/encodings"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
@ -141,7 +140,9 @@ underlying S3 storage.`,
Name: config.ConfigEncoding,
Help: config.ConfigEncodingHelp,
Advanced: true,
Default: encodings.AmazonCloudDrive,
// Encode invalid UTF-8 bytes as json doesn't handle them properly.
Default: (encoder.Base |
encoder.EncodeInvalidUtf8),
}},
})
}

8
backend/azureblob/azureblob.go
View File

@ -29,7 +29,6 @@ import (
"github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/encodings"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
@ -131,7 +130,12 @@ tiering blob to "Hot" or "Cool".`,
Name: config.ConfigEncoding,
Help: config.ConfigEncodingHelp,
Advanced: true,
Default: encodings.AzureBlob,
Default: (encoder.EncodeInvalidUtf8 |
encoder.EncodeSlash |
encoder.EncodeCtl |
encoder.EncodeDel |
encoder.EncodeBackSlash |
encoder.EncodeRightPeriod),
}},
})
}

8
backend/b2/b2.go
View File

@ -26,7 +26,6 @@ import (
"github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/encodings"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
@ -150,7 +149,12 @@ The minimum value is 1 second. The maximum value is one week.`,
Name: config.ConfigEncoding,
Help: config.ConfigEncodingHelp,
Advanced: true,
Default: encodings.B2,
// See: https://www.backblaze.com/b2/docs/files.html
// Encode invalid UTF-8 bytes as json doesn't handle them properly.
// FIXME: allow /, but not leading, trailing or double
Default: (encoder.Display |
encoder.EncodeBackSlash |
encoder.EncodeInvalidUtf8),
}},
})
}

13
backend/box/box.go
View File

@ -37,7 +37,6 @@ import (
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/fs/encodings"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
@ -149,7 +148,17 @@ func init() {
Name: config.ConfigEncoding,
Help: config.ConfigEncodingHelp,
Advanced: true,
Default: encodings.Box,
// From https://developer.box.com/docs/error-codes#section-400-bad-request :
// > Box only supports file or folder names that are 255 characters or less.
// > File names containing non-printable ascii, "/" or "\", names with leading
// > or trailing spaces, and the special names “.” and “..” are also unsupported.
//
// Testing revealed names with leading spaces work fine.
// Also encode invalid UTF-8 bytes as json doesn't handle them properly.
Default: (encoder.Display |
encoder.EncodeBackSlash |
encoder.EncodeRightSpace |
encoder.EncodeInvalidUtf8),
}},
})
}

5
backend/drive/drive.go
View File

@ -33,7 +33,6 @@ import (
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/fs/encodings"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
@ -459,7 +458,9 @@ See: https://github.com/rclone/rclone/issues/3857
Name: config.ConfigEncoding,
Help: config.ConfigEncodingHelp,
Advanced: true,
Default: encodings.Drive,
// Encode invalid UTF-8 bytes as json doesn't handle them properly.
// Don't encode / as it's a valid name character in drive.
Default: encoder.EncodeInvalidUtf8,
}},
})

11
backend/dropbox/dropbox.go
View File

@ -45,7 +45,6 @@ import (
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/fs/encodings"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/lib/encoder"
@ -150,7 +149,15 @@ memory. It can be set smaller if you are tight on memory.`, maxChunkSize),
Name: config.ConfigEncoding,
Help: config.ConfigEncodingHelp,
Advanced: true,
Default: encodings.Dropbox,
// https://www.dropbox.com/help/syncing-uploads/files-not-syncing lists / and \
// as invalid characters.
// Testing revealed names with trailing spaces and the DEL character don't work.
// Also encode invalid UTF-8 bytes as json doesn't handle them properly.
Default: (encoder.Base |
encoder.EncodeBackSlash |
encoder.EncodeDel |
encoder.EncodeRightSpace |
encoder.EncodeInvalidUtf8),
}},
})
}

23
backend/fichier/fichier.go
View File

@ -14,7 +14,6 @@ import (
"github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/encodings"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/lib/dircache"
@ -50,7 +49,27 @@ func init() {
Name: config.ConfigEncoding,
Help: config.ConfigEncodingHelp,
Advanced: true,
Default: encodings.Fichier,
// Characters that need escaping
//
// '\\': '＼', // FULLWIDTH REVERSE SOLIDUS
// '<': '＜', // FULLWIDTH LESS-THAN SIGN
// '>': '＞', // FULLWIDTH GREATER-THAN SIGN
// '"': '＂', // FULLWIDTH QUOTATION MARK - not on the list but seems to be reserved
// '\'': '＇', // FULLWIDTH APOSTROPHE
// '$': '＄', // FULLWIDTH DOLLAR SIGN
// '`': '｀', // FULLWIDTH GRAVE ACCENT
//
// Leading space and trailing space
Default: (encoder.Display |
encoder.EncodeBackSlash |
encoder.EncodeSingleQuote |
encoder.EncodeBackQuote |
encoder.EncodeDoubleQuote |
encoder.EncodeLtGt |
encoder.EncodeDollar |
encoder.EncodeLeftSpace |
encoder.EncodeRightSpace |
encoder.EncodeInvalidUtf8),
}},
})
}

9
backend/ftp/ftp.go
View File

@ -18,7 +18,6 @@ import (
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/fs/encodings"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/lib/encoder"
"github.com/rclone/rclone/lib/pacer"
@ -73,7 +72,13 @@ func init() {
Name: config.ConfigEncoding,
Help: config.ConfigEncodingHelp,
Advanced: true,
Default: encodings.FTP,
// The FTP protocal can't handle trailing spaces (for instance
// pureftpd turns them into _)
//
// proftpd can't handle '*' in file names
// pureftpd can't handle '[', ']' or '*'
Default: (encoder.Display |
encoder.EncodeRightSpace),
}},
})
}

5
backend/googlecloudstorage/googlecloudstorage.go
View File

@ -32,7 +32,6 @@ import (
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/fs/encodings"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
@ -251,7 +250,9 @@ Docs: https://cloud.google.com/storage/docs/bucket-policy-only
Name: config.ConfigEncoding,
Help: config.ConfigEncodingHelp,
Advanced: true,
Default: encodings.GoogleCloudStorage,
Default: (encoder.Base |
encoder.EncodeCrLf |
encoder.EncodeInvalidUtf8),
}},
})
}

8
backend/jottacloud/jottacloud.go
View File

@ -26,7 +26,6 @@ import (
"github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/encodings"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
@ -160,7 +159,12 @@ func init() {
Name: config.ConfigEncoding,
Help: config.ConfigEncodingHelp,
Advanced: true,
Default: encodings.JottaCloud,
// Encode invalid UTF-8 bytes as xml doesn't handle them properly.
//
// Also: '*', '/', ':', '<', '>', '?', '\"', '\x00', '|'
Default: (encoder.Display |
encoder.EncodeWin | // :?"*<>|
encoder.EncodeInvalidUtf8),
}},
})
}

6
backend/koofr/koofr.go
View File

@ -16,7 +16,6 @@ import (
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/fs/encodings"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/lib/encoder"
@ -62,7 +61,10 @@ func init() {
Name: config.ConfigEncoding,
Help: config.ConfigEncodingHelp,
Advanced: true,
Default: encodings.Koofr,
// Encode invalid UTF-8 bytes as json doesn't handle them properly.
Default: (encoder.Display |
encoder.EncodeBackSlash |
encoder.EncodeInvalidUtf8),
}},
})
}

10
backend/local/encode_darwin.go
View File

@ -2,8 +2,10 @@
package local
import (
"github.com/rclone/rclone/fs/encodings"
)
import "github.com/rclone/rclone/lib/encoder"
const defaultEnc = encodings.LocalMacOS
// This is the encoding used by the local backend for macOS
//
// macOS can't store invalid UTF-8, it converts them into %XX encoding
const defaultEnc = (encoder.Base |
encoder.EncodeInvalidUtf8)

7
backend/local/encode_other.go
View File

@ -2,8 +2,7 @@
package local
import (
"github.com/rclone/rclone/fs/encodings"
)
import "github.com/rclone/rclone/lib/encoder"
const defaultEnc = encodings.LocalUnix
// This is the encoding used by the local backend for non windows platforms
const defaultEnc = encoder.Base

32
backend/local/encode_windows.go
View File

@ -2,8 +2,32 @@
package local
import (
"github.com/rclone/rclone/fs/encodings"
)
import "github.com/rclone/rclone/lib/encoder"
const defaultEnc = encodings.LocalWindows
// This is the encoding used by the local backend for windows platforms
//
// List of replaced characters:
// < (less than) -> '＜' // FULLWIDTH LESS-THAN SIGN
// > (greater than) -> '＞' // FULLWIDTH GREATER-THAN SIGN
// : (colon) -> '：' // FULLWIDTH COLON
// " (double quote) -> '＂' // FULLWIDTH QUOTATION MARK
// \ (backslash) -> '＼' // FULLWIDTH REVERSE SOLIDUS
// | (vertical line) -> '｜' // FULLWIDTH VERTICAL LINE
// ? (question mark) -> '？' // FULLWIDTH QUESTION MARK
// * (asterisk) -> '＊' // FULLWIDTH ASTERISK
//
// Additionally names can't end with a period (.) or space ( ).
// List of replaced characters:
// . (period) -> '．' // FULLWIDTH FULL STOP
// (space) -> '␠' // SYMBOL FOR SPACE
//
// Also encode invalid UTF-8 bytes as Go can't convert them to UTF-16.
//
// https://docs.microsoft.com/de-de/windows/desktop/FileIO/naming-a-file#naming-conventions
const defaultEnc = (encoder.Base |
encoder.EncodeWin |
encoder.EncodeBackSlash |
encoder.EncodeCtl |
encoder.EncodeRightSpace |
encoder.EncodeRightPeriod |
encoder.EncodeInvalidUtf8)

7
backend/mailru/mailru.go
View File

@ -28,7 +28,6 @@ import (
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/fs/encodings"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
@ -197,7 +196,11 @@ Supported quirks: atomicmkdir binlist gzip insecure retry400`,
Name: config.ConfigEncoding,
Help: config.ConfigEncodingHelp,
Advanced: true,
Default: encodings.Mailru,
// Encode invalid UTF-8 bytes as json doesn't handle them properly.
Default: (encoder.Display |
encoder.EncodeWin | // :?"*<>|
encoder.EncodeBackSlash |
encoder.EncodeInvalidUtf8),
}},
})
}

5
backend/mega/mega.go
View File

@ -30,7 +30,6 @@ import (
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/fs/encodings"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/lib/encoder"
@ -87,7 +86,9 @@ permanently delete objects instead.`,
Name: config.ConfigEncoding,
Help: config.ConfigEncodingHelp,
Advanced: true,
Default: encodings.Mega,
// Encode invalid UTF-8 bytes as json doesn't handle them properly.
Default: (encoder.Base |
encoder.EncodeInvalidUtf8),
}},
})
}

44
backend/onedrive/onedrive.go
View File

@ -23,7 +23,6 @@ import (
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/fs/encodings"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/lib/atexit"
@ -255,7 +254,48 @@ listing, set this option.`,
Name: config.ConfigEncoding,
Help: config.ConfigEncodingHelp,
Advanced: true,
Default: encodings.OneDrive,
// List of replaced characters:
// < (less than) -> '＜' // FULLWIDTH LESS-THAN SIGN
// > (greater than) -> '＞' // FULLWIDTH GREATER-THAN SIGN
// : (colon) -> '：' // FULLWIDTH COLON
// " (double quote) -> '＂' // FULLWIDTH QUOTATION MARK
// \ (backslash) -> '＼' // FULLWIDTH REVERSE SOLIDUS
// | (vertical line) -> '｜' // FULLWIDTH VERTICAL LINE
// ? (question mark) -> '？' // FULLWIDTH QUESTION MARK
// * (asterisk) -> '＊' // FULLWIDTH ASTERISK
// # (number sign) -> '＃' // FULLWIDTH NUMBER SIGN
// % (percent sign) -> '％' // FULLWIDTH PERCENT SIGN
//
// Folder names cannot begin with a tilde ('~')
// List of replaced characters:
// ~ (tilde) -> '～' // FULLWIDTH TILDE
//
// Additionally names can't begin with a space ( ) or end with a period (.) or space ( ).
// List of replaced characters:
// . (period) -> '．' // FULLWIDTH FULL STOP
// (space) -> '␠' // SYMBOL FOR SPACE
//
// Also encode invalid UTF-8 bytes as json doesn't handle them.
//
// The OneDrive API documentation lists the set of reserved characters, but
// testing showed this list is incomplete. This are the differences:
// - " (double quote) is rejected, but missing in the documentation
// - space at the end of file and folder names is rejected, but missing in the documentation
// - period at the end of file names is rejected, but missing in the documentation
//
// Adding these restrictions to the OneDrive API documentation yields exactly
// the same rules as the Windows naming conventions.
//
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/concepts/addressing-driveitems?view=odsp-graph-online#path-encoding
Default: (encoder.Display |
encoder.EncodeBackSlash |
encoder.EncodeHashPercent |
encoder.EncodeLeftSpace |
encoder.EncodeLeftTilde |
encoder.EncodeRightPeriod |
encoder.EncodeRightSpace |
encoder.EncodeWin |
encoder.EncodeInvalidUtf8),
}},
})
}

31
backend/opendrive/opendrive.go
View File

@ -17,7 +17,6 @@ import (
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/fs/encodings"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
@ -54,7 +53,35 @@ func init() {
Name: config.ConfigEncoding,
Help: config.ConfigEncodingHelp,
Advanced: true,
Default: encodings.OpenDrive,
// List of replaced characters:
// < (less than) -> '＜' // FULLWIDTH LESS-THAN SIGN
// > (greater than) -> '＞' // FULLWIDTH GREATER-THAN SIGN
// : (colon) -> '：' // FULLWIDTH COLON
// " (double quote) -> '＂' // FULLWIDTH QUOTATION MARK
// \ (backslash) -> '＼' // FULLWIDTH REVERSE SOLIDUS
// | (vertical line) -> '｜' // FULLWIDTH VERTICAL LINE
// ? (question mark) -> '？' // FULLWIDTH QUESTION MARK
// * (asterisk) -> '＊' // FULLWIDTH ASTERISK
//
// Additionally names can't begin or end with a ASCII whitespace.
// List of replaced characters:
// (space) -> '␠' // SYMBOL FOR SPACE
// (horizontal tab) -> '␉' // SYMBOL FOR HORIZONTAL TABULATION
// (line feed) -> '␊' // SYMBOL FOR LINE FEED
// (vertical tab) -> '␋' // SYMBOL FOR VERTICAL TABULATION
// (carriage return) -> '␍' // SYMBOL FOR CARRIAGE RETURN
//
// Also encode invalid UTF-8 bytes as json doesn't handle them properly.
//
// https://www.opendrive.com/wp-content/uploads/guides/OpenDrive_API_guide.pdf
Default: (encoder.Base |
encoder.EncodeWin |
encoder.EncodeLeftCrLfHtVt |
encoder.EncodeRightCrLfHtVt |
encoder.EncodeBackSlash |
encoder.EncodeLeftSpace |
encoder.EncodeRightSpace |
encoder.EncodeInvalidUtf8),
}},
})
}

8
backend/pcloud/pcloud.go
View File

@ -26,7 +26,6 @@ import (
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/fs/encodings"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/lib/dircache"
@ -84,7 +83,12 @@ func init() {
Name: config.ConfigEncoding,
Help: config.ConfigEncodingHelp,
Advanced: true,
Default: encodings.Pcloud,
// Encode invalid UTF-8 bytes as json doesn't handle them properly.
//
// TODO: Investigate Unicode simplification (＼ gets converted to \ server-side)
Default: (encoder.Display |
encoder.EncodeBackSlash |
encoder.EncodeInvalidUtf8),
}},
})
}

7
backend/premiumizeme/premiumizeme.go
View File

@ -35,7 +35,6 @@ import (
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/fs/encodings"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
@ -97,7 +96,11 @@ This is not normally used - use oauth instead.
Name: config.ConfigEncoding,
Help: config.ConfigEncodingHelp,
Advanced: true,
Default: encodings.PremiumizeMe,
// Encode invalid UTF-8 bytes as json doesn't handle them properly.
Default: (encoder.Display |
encoder.EncodeBackSlash |
encoder.EncodeDoubleQuote |
encoder.EncodeInvalidUtf8),
}},
})
}

8
backend/putio/putio.go
View File

@ -9,7 +9,6 @@ import (
"github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/fs/encodings"
"github.com/rclone/rclone/lib/dircache"
"github.com/rclone/rclone/lib/encoder"
"github.com/rclone/rclone/lib/oauthutil"
@ -70,7 +69,12 @@ func init() {
Name: config.ConfigEncoding,
Help: config.ConfigEncodingHelp,
Advanced: true,
Default: encodings.Putio,
// Note that \ is renamed to -
//
// Encode invalid UTF-8 bytes as json doesn't handle them properly.
Default: (encoder.Display |
encoder.EncodeBackSlash |
encoder.EncodeInvalidUtf8),
}},
})
}

5
backend/qingstor/qingstor.go
View File

@ -21,7 +21,6 @@ import (
"github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/encodings"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/walk"
@ -117,7 +116,9 @@ this may help to speed up the transfers.`,
Name: config.ConfigEncoding,
Help: config.ConfigEncodingHelp,
Advanced: true,
Default: encodings.QingStor,
Default: (encoder.EncodeInvalidUtf8 |
encoder.EncodeCtl |
encoder.EncodeSlash),
}},
})
}

15
backend/s3/s3.go
View File

@ -49,7 +49,6 @@ import (
"github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/encodings"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
@ -815,7 +814,19 @@ In Ceph, this can be increased with the "rgw list buckets max chunk" option.
Name: config.ConfigEncoding,
Help: config.ConfigEncodingHelp,
Advanced: true,
Default: encodings.S3,
// Any UTF-8 character is valid in a key, however it can't handle
// invalid UTF-8 and / have a special meaning.
//
// The SDK can't seem to handle uploading files called '.'
//
// FIXME would be nice to add
// - initial / encoding
// - doubled / encoding
// - trailing / encoding
// so that AWS keys are always valid file names
Default: (encoder.EncodeInvalidUtf8 |
encoder.EncodeSlash |
encoder.EncodeDot),
}},
})
}

11
backend/sharefile/sharefile.go
View File

@ -91,7 +91,6 @@ import (
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/fs/encodings"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/lib/dircache"
@ -208,7 +207,15 @@ be set manually to something like: https://XXX.sharefile.com
Name: config.ConfigEncoding,
Help: config.ConfigEncodingHelp,
Advanced: true,
Default: encodings.Sharefile,
Default: (encoder.Base |
encoder.EncodeWin | // :?"*<>|
encoder.EncodeBackSlash | // \
encoder.EncodeCtl |
encoder.EncodeRightSpace |
encoder.EncodeRightPeriod |
encoder.EncodeLeftSpace |
encoder.EncodeLeftPeriod |
encoder.EncodeInvalidUtf8),
}},
})
}

4
backend/swift/swift.go
View File

@ -19,7 +19,6 @@ import (
"github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/encodings"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
@ -66,7 +65,8 @@ copy operations.`,
Name: config.ConfigEncoding,
Help: config.ConfigEncodingHelp,
Advanced: true,
Default: encodings.Swift,
Default: (encoder.EncodeInvalidUtf8 |
encoder.EncodeSlash),
}}
// Register with Fs

6
backend/yandex/yandex.go
View File

@ -20,7 +20,6 @@ import (
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/fs/encodings"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/lib/encoder"
@ -83,7 +82,10 @@ func init() {
Name: config.ConfigEncoding,
Help: config.ConfigEncodingHelp,
Advanced: true,
Default: encodings.Yandex,
// Of the control characters \t \n \r are allowed
// it doesn't seem worth making an exception for this
Default: (encoder.Display |
encoder.EncodeInvalidUtf8),
}},
})
}

419
fs/encodings/encodings.go
View File

@ -1,419 +0,0 @@
// +build !noencode
package encodings
import (
"runtime"
"strings"
"github.com/rclone/rclone/lib/encoder"
)
// LocalUnix is the encoding used by the local backend for non windows platforms
const LocalUnix = encoder.Base
// LocalMacOS is the encoding used by the local backend for macOS
//
// macOS can't store invalid UTF-8, it converts them into %XX encoding
const LocalMacOS = (encoder.Base |
encoder.EncodeInvalidUtf8)
// LocalWindows is the encoding used by the local backend for windows platforms
//
// List of replaced characters:
// < (less than) -> '＜' // FULLWIDTH LESS-THAN SIGN
// > (greater than) -> '＞' // FULLWIDTH GREATER-THAN SIGN
// : (colon) -> '：' // FULLWIDTH COLON
// " (double quote) -> '＂' // FULLWIDTH QUOTATION MARK
// \ (backslash) -> '＼' // FULLWIDTH REVERSE SOLIDUS
// | (vertical line) -> '｜' // FULLWIDTH VERTICAL LINE
// ? (question mark) -> '？' // FULLWIDTH QUESTION MARK
// * (asterisk) -> '＊' // FULLWIDTH ASTERISK
//
// Additionally names can't end with a period (.) or space ( ).
// List of replaced characters:
// . (period) -> '．' // FULLWIDTH FULL STOP
// (space) -> '␠' // SYMBOL FOR SPACE
//
// Also encode invalid UTF-8 bytes as Go can't convert them to UTF-16.
//
// https://docs.microsoft.com/de-de/windows/desktop/FileIO/naming-a-file#naming-conventions
const LocalWindows = (encoder.Base |
encoder.EncodeWin |
encoder.EncodeBackSlash |
encoder.EncodeCtl |
encoder.EncodeRightSpace |
encoder.EncodeRightPeriod |
encoder.EncodeInvalidUtf8)
// AmazonCloudDrive is the encoding used by the amazonclouddrive backend
//
// Encode invalid UTF-8 bytes as json doesn't handle them properly.
const AmazonCloudDrive = (encoder.Base |
encoder.EncodeInvalidUtf8)
// B2 is the encoding used by the b2 backend
//
// See: https://www.backblaze.com/b2/docs/files.html
// Encode invalid UTF-8 bytes as json doesn't handle them properly.
// FIXME: allow /, but not leading, trailing or double
const B2 = (encoder.Display |
encoder.EncodeBackSlash |
encoder.EncodeInvalidUtf8)
// Box is the encoding used by the box backend
//
// From https://developer.box.com/docs/error-codes#section-400-bad-request :
// > Box only supports file or folder names that are 255 characters or less.
// > File names containing non-printable ascii, "/" or "\", names with leading
// > or trailing spaces, and the special names “.” and “..” are also unsupported.
//
// Testing revealed names with leading spaces work fine.
// Also encode invalid UTF-8 bytes as json doesn't handle them properly.
const Box = (encoder.Display |
encoder.EncodeBackSlash |
encoder.EncodeRightSpace |
encoder.EncodeInvalidUtf8)
// Drive is the encoding used by the drive backend
//
// Encode invalid UTF-8 bytes as json doesn't handle them properly.
// Don't encode / as it's a valid name character in drive.
const Drive = (encoder.EncodeInvalidUtf8)
// Dropbox is the encoding used by the dropbox backend
//
// https://www.dropbox.com/help/syncing-uploads/files-not-syncing lists / and \
// as invalid characters.
// Testing revealed names with trailing spaces and the DEL character don't work.
// Also encode invalid UTF-8 bytes as json doesn't handle them properly.
const Dropbox = (encoder.Base |
encoder.EncodeBackSlash |
encoder.EncodeDel |
encoder.EncodeRightSpace |
encoder.EncodeInvalidUtf8)
// GoogleCloudStorage is the encoding used by the googlecloudstorage backend
const GoogleCloudStorage = (encoder.Base |
encoder.EncodeCrLf |
encoder.EncodeInvalidUtf8)
// JottaCloud is the encoding used by the jottacloud backend
//
// Encode invalid UTF-8 bytes as xml doesn't handle them properly.
//
// Also: '*', '/', ':', '<', '>', '?', '\"', '\x00', '|'
const JottaCloud = (encoder.Display |
encoder.EncodeWin | // :?"*<>|
encoder.EncodeInvalidUtf8)
// Koofr is the encoding used by the koofr backend
//
// Encode invalid UTF-8 bytes as json doesn't handle them properly.
const Koofr = (encoder.Display |
encoder.EncodeBackSlash |
encoder.EncodeInvalidUtf8)
// Mailru is the encoding used by the mailru backend
//
// Encode invalid UTF-8 bytes as json doesn't handle them properly.
const Mailru = (encoder.Display |
encoder.EncodeWin | // :?"*<>|
encoder.EncodeBackSlash |
encoder.EncodeInvalidUtf8)
// Mega is the encoding used by the mega backend
//
// Encode invalid UTF-8 bytes as json doesn't handle them properly.
const Mega = (encoder.Base |
encoder.EncodeInvalidUtf8)
// OneDrive is the encoding used by the onedrive backend
//
// List of replaced characters:
// < (less than) -> '＜' // FULLWIDTH LESS-THAN SIGN
// > (greater than) -> '＞' // FULLWIDTH GREATER-THAN SIGN
// : (colon) -> '：' // FULLWIDTH COLON
// " (double quote) -> '＂' // FULLWIDTH QUOTATION MARK
// \ (backslash) -> '＼' // FULLWIDTH REVERSE SOLIDUS
// | (vertical line) -> '｜' // FULLWIDTH VERTICAL LINE
// ? (question mark) -> '？' // FULLWIDTH QUESTION MARK
// * (asterisk) -> '＊' // FULLWIDTH ASTERISK
// # (number sign) -> '＃' // FULLWIDTH NUMBER SIGN
// % (percent sign) -> '％' // FULLWIDTH PERCENT SIGN
//
// Folder names cannot begin with a tilde ('~')
// List of replaced characters:
// ~ (tilde) -> '～' // FULLWIDTH TILDE
//
// Additionally names can't begin with a space ( ) or end with a period (.) or space ( ).
// List of replaced characters:
// . (period) -> '．' // FULLWIDTH FULL STOP
// (space) -> '␠' // SYMBOL FOR SPACE
//
// Also encode invalid UTF-8 bytes as json doesn't handle them.
//
// The OneDrive API documentation lists the set of reserved characters, but
// testing showed this list is incomplete. This are the differences:
// - " (double quote) is rejected, but missing in the documentation
// - space at the end of file and folder names is rejected, but missing in the documentation
// - period at the end of file names is rejected, but missing in the documentation
//
// Adding these restrictions to the OneDrive API documentation yields exactly
// the same rules as the Windows naming conventions.
//
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/concepts/addressing-driveitems?view=odsp-graph-online#path-encoding
const OneDrive = (encoder.Display |
encoder.EncodeBackSlash |
encoder.EncodeHashPercent |
encoder.EncodeLeftSpace |
encoder.EncodeLeftTilde |
encoder.EncodeRightPeriod |
encoder.EncodeRightSpace |
encoder.EncodeWin |
encoder.EncodeInvalidUtf8)
// OpenDrive is the encoding used by the opendrive backend
//
// List of replaced characters:
// < (less than) -> '＜' // FULLWIDTH LESS-THAN SIGN
// > (greater than) -> '＞' // FULLWIDTH GREATER-THAN SIGN
// : (colon) -> '：' // FULLWIDTH COLON
// " (double quote) -> '＂' // FULLWIDTH QUOTATION MARK
// \ (backslash) -> '＼' // FULLWIDTH REVERSE SOLIDUS
// | (vertical line) -> '｜' // FULLWIDTH VERTICAL LINE
// ? (question mark) -> '？' // FULLWIDTH QUESTION MARK
// * (asterisk) -> '＊' // FULLWIDTH ASTERISK
//
// Additionally names can't begin or end with a ASCII whitespace.
// List of replaced characters:
// (space) -> '␠' // SYMBOL FOR SPACE
// (horizontal tab) -> '␉' // SYMBOL FOR HORIZONTAL TABULATION
// (line feed) -> '␊' // SYMBOL FOR LINE FEED
// (vertical tab) -> '␋' // SYMBOL FOR VERTICAL TABULATION
// (carriage return) -> '␍' // SYMBOL FOR CARRIAGE RETURN
//
// Also encode invalid UTF-8 bytes as json doesn't handle them properly.
//
// https://www.opendrive.com/wp-content/uploads/guides/OpenDrive_API_guide.pdf
const OpenDrive = (encoder.Base |
encoder.EncodeWin |
encoder.EncodeLeftCrLfHtVt |
encoder.EncodeRightCrLfHtVt |
encoder.EncodeBackSlash |
encoder.EncodeLeftSpace |
encoder.EncodeRightSpace |
encoder.EncodeInvalidUtf8)
// PremiumizeMe is the encoding used by the premiumizeme backend
//
// Encode invalid UTF-8 bytes as json doesn't handle them properly.
const PremiumizeMe = (encoder.Display |
encoder.EncodeBackSlash |
encoder.EncodeDoubleQuote |
encoder.EncodeInvalidUtf8)
// Pcloud is the encoding used by the pcloud backend
//
// Encode invalid UTF-8 bytes as json doesn't handle them properly.
//
// TODO: Investigate Unicode simplification (＼ gets converted to \ server-side)
const Pcloud = (encoder.Display |
encoder.EncodeBackSlash |
encoder.EncodeInvalidUtf8)
// Putio is the encoding used by the putio backend
//
// Note that \ is renamed to -
//
// Encode invalid UTF-8 bytes as json doesn't handle them properly.
const Putio = (encoder.Display |
encoder.EncodeBackSlash |
encoder.EncodeInvalidUtf8)
// Fichier is the encoding used by the fichier backend
//
// Characters that need escaping
//
// '\\': '＼', // FULLWIDTH REVERSE SOLIDUS
// '<': '＜', // FULLWIDTH LESS-THAN SIGN
// '>': '＞', // FULLWIDTH GREATER-THAN SIGN
// '"': '＂', // FULLWIDTH QUOTATION MARK - not on the list but seems to be reserved
// '\'': '＇', // FULLWIDTH APOSTROPHE
// '$': '＄', // FULLWIDTH DOLLAR SIGN
// '`': '｀', // FULLWIDTH GRAVE ACCENT
//
// Leading space and trailing space
const Fichier = (encoder.Display |
encoder.EncodeBackSlash |
encoder.EncodeSingleQuote |
encoder.EncodeBackQuote |
encoder.EncodeDoubleQuote |
encoder.EncodeLtGt |
encoder.EncodeDollar |
encoder.EncodeLeftSpace |
encoder.EncodeRightSpace |
encoder.EncodeInvalidUtf8)
// FTP is the encoding used by the ftp backend
//
// The FTP protocal can't handle trailing spaces (for instance
// pureftpd turns them into _)
//
// proftpd can't handle '*' in file names
// pureftpd can't handle '[', ']' or '*'
const FTP = (encoder.Display |
encoder.EncodeRightSpace)
// S3 is the encoding used by the s3 backend
//
// Any UTF-8 character is valid in a key, however it can't handle
// invalid UTF-8 and / have a special meaning.
//
// The SDK can't seem to handle uploading files called '.'
//
// FIXME would be nice to add
// - initial / encoding
// - doubled / encoding
// - trailing / encoding
// so that AWS keys are always valid file names
const S3 = (encoder.EncodeInvalidUtf8 |
encoder.EncodeSlash |
encoder.EncodeDot)
// Swift is the encoding used by the swift backend
const Swift = (encoder.EncodeInvalidUtf8 |
encoder.EncodeSlash)
// AzureBlob is the encoding used by the azureblob backend
const AzureBlob = (encoder.EncodeInvalidUtf8 |
encoder.EncodeSlash |
encoder.EncodeCtl |
encoder.EncodeDel |
encoder.EncodeBackSlash |
encoder.EncodeRightPeriod)
// QingStor is the encoding used by the qingstor backend
const QingStor = (encoder.EncodeInvalidUtf8 |
encoder.EncodeCtl |
encoder.EncodeSlash)
// Sharefile is the encoding used by the sharefile backend
const Sharefile = (encoder.Base |
encoder.EncodeWin | // :?"*<>|
encoder.EncodeBackSlash | // \
encoder.EncodeCtl |
encoder.EncodeRightSpace |
encoder.EncodeRightPeriod |
encoder.EncodeLeftSpace |
encoder.EncodeLeftPeriod |
encoder.EncodeInvalidUtf8)
// Yandex is the encoding used by the yandex backend
//
// Of the control characters \t \n \r are allowed
// it doesn't seem worth making an exception for this
const Yandex = (encoder.Display |
encoder.EncodeInvalidUtf8)
// ByName returns the encoder for a give backend name or nil
func ByName(name string) encoder.Encoder {
switch strings.ToLower(name) {
case "base":
return encoder.Base
case "display":
return encoder.Display
case "amazonclouddrive":
return AmazonCloudDrive
case "azureblob":
return AzureBlob
case "b2":
return B2
case "box":
return Box
//case "cache":
case "drive":
return Drive
case "dropbox":
return Dropbox
//case "ftp":
case "ficher":
return Fichier
case "googlecloudstorage":
return GoogleCloudStorage
//case "http":
//case "hubic":
case "jottacloud":
return JottaCloud
case "koofr":
return Koofr
case "local":
return Local()
case "local-windows", "windows":
return LocalWindows
case "local-unix", "unix":
return LocalUnix
case "local-macos", "macos":
return LocalMacOS
case "mailru":
return Mailru
case "mega":
return Mega
case "onedrive":
return OneDrive
case "opendrive":
return OpenDrive
case "premiumizeme":
return PremiumizeMe
case "pcloud":
return Pcloud
case "qingstor":
return QingStor
case "s3":
return S3
case "sharefile":
return Sharefile
//case "sftp":
case "swift":
return Swift
//case "webdav":
case "yandex":
return Yandex
default:
return nil
}
}
// Local returns the local encoding for the current platform
func Local() encoder.MultiEncoder {
if runtime.GOOS == "windows" {
return LocalWindows
}
return LocalUnix
}
// Names returns the list of known encodings as accepted by ByName
func Names() []string {
return []string{
"base",
"display",
"amazonclouddrive",
"b2",
"box",
"drive",
"dropbox",
"googlecloudstorage",
"jottacloud",
"koofr",
"local-unix",
"local-windows",
"local-macos",
"local",
"mega",
"onedrive",
"opendrive",
"premiumizeme",
"pcloud",
"sharefile",
}
}

57
fs/encodings/encodings_noencode.go
View File

@ -1,57 +0,0 @@
// +build noencode
package encodings
import (
"github.com/rclone/rclone/lib/encoder"
)
// Fake encodings used for testing
const (
Base = encoder.MultiEncoder(
encoder.EncodeZero |
encoder.EncodeSlash)
Display = Base
LocalUnix = Base
LocalWindows = Base
AmazonCloudDrive = Base
AzureBlob = Base
B2 = Base
Box = Base
Drive = Base
Dropbox = Base
FTP = Base
Fichier = Base
GoogleCloudStorage = Base
JottaCloud = Base
Koofr = Base
Mailru = Base
Mega = Base
OneDrive = Base
OpenDrive = Base
Pcloud = Base
PremiumizeMe = Base
Putio = Base
QingStor = Base
S3 = Base
Sharefile = Base
Swift = Base
Yandex = Base
)
// ByName returns the encoder for a give backend name or nil
func ByName(name string) encoder.Encoder {
return Base
}
// Local returns the local encoding for the current platform
func Local() encoder.MultiEncoder {
return Base
}
// Names returns the list of known encodings as accepted by ByName
func Names() []string {
return []string{
"base",
}
}