From 43eadf278c6b5fc4e4e3b929348d0927f01c1f33 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sat, 20 Aug 2016 18:46:10 +0100 Subject: [PATCH] Remove flattening and replace with {off, standard} name encryption --- crypt/cipher.go | 151 +++++++++++++++++++++--------------- crypt/cipher_test.go | 173 +++++++++++++++++++++++------------------- crypt/crypt.go | 116 +++++++++------------------- docs/content/crypt.md | 124 +++++++++++++++++------------- 4 files changed, 289 insertions(+), 275 deletions(-) diff --git a/crypt/cipher.go b/crypt/cipher.go index 184dc54ce..4ac3de52a 100644 --- a/crypt/cipher.go +++ b/crypt/cipher.go @@ -6,6 +6,7 @@ import ( gocipher "crypto/cipher" "crypto/rand" "encoding/base32" + "fmt" "io" "strings" "sync" @@ -30,6 +31,7 @@ const ( blockHeaderSize = secretbox.Overhead blockDataSize = 64 * 1024 blockSize = blockHeaderSize + blockDataSize + encryptedSuffix = ".bin" // when file name encryption is off we add this suffix to make sure the cloud provider doesn't process the file ) // Errors returned by cipher @@ -43,10 +45,8 @@ var ( ErrorEncryptedBadMagic = errors.New("not an encrypted file - bad magic string") ErrorEncryptedBadBlock = errors.New("failed to authenticate decrypted block - bad password?") ErrorBadBase32Encoding = errors.New("bad base32 filename encoding") - ErrorBadSpreadNotSingleChar = errors.New("bad unspread - not single character") - ErrorBadSpreadResultTooShort = errors.New("bad unspread - result too short") - ErrorBadSpreadDidntMatch = errors.New("bad unspread - directory prefix didn't match") ErrorFileClosed = errors.New("file already closed") + ErrorNotAnEncryptedFile = errors.New("not an encrypted file - no \"" + encryptedSuffix + "\" suffix") defaultSalt = []byte{0xA8, 0x0D, 0xF4, 0x3A, 0x8F, 0xBD, 0x03, 0x08, 0xA7, 0xCA, 0xB8, 0x3E, 0x58, 0x1F, 0x86, 0xB1} ) @@ -57,10 +57,14 @@ var ( // Cipher is used to swap out the encryption implementations type Cipher interface { - // EncryptName encrypts a file path - EncryptName(string) string - // DecryptName decrypts a file path, returns error if decrypt was invalid - DecryptName(string) (string, error) + // EncryptFileName encrypts a file path + EncryptFileName(string) string + // DecryptFileName decrypts a file path, returns error if decrypt was invalid + DecryptFileName(string) (string, error) + // EncryptDirName encrypts a directory path + EncryptDirName(string) string + // DecryptDirName decrypts a directory path, returns error if decrypt was invalid + DecryptDirName(string) (string, error) // EncryptData EncryptData(io.Reader) (io.Reader, error) // DecryptData @@ -71,20 +75,56 @@ type Cipher interface { DecryptedSize(int64) (int64, error) } +// NameEncryptionMode is the type of file name encryption in use +type NameEncryptionMode int + +// NameEncryptionMode levels +const ( + NameEncryptionOff NameEncryptionMode = iota + NameEncryptionStandard +) + +// NewNameEncryptionMode turns a string into a NameEncryptionMode +func NewNameEncryptionMode(s string) (mode NameEncryptionMode, err error) { + s = strings.ToLower(s) + switch s { + case "off": + mode = NameEncryptionOff + case "standard": + mode = NameEncryptionStandard + default: + err = errors.Errorf("Unknown file name encryption mode %q", s) + } + return mode, err +} + +// String turns mode into a human readable string +func (mode NameEncryptionMode) String() (out string) { + switch mode { + case NameEncryptionOff: + out = "off" + case NameEncryptionStandard: + out = "standard" + default: + out = fmt.Sprintf("Unknown mode #%d", mode) + } + return out +} + type cipher struct { dataKey [32]byte // Key for secretbox nameKey [32]byte // 16,24 or 32 bytes nameTweak [nameCipherBlockSize]byte // used to tweak the name crypto block gocipher.Block - flatten int // set flattening level - 0 is off + mode NameEncryptionMode buffers sync.Pool // encrypt/decrypt buffers cryptoRand io.Reader // read crypto random numbers from here } // newCipher initialises the cipher. If salt is "" then it uses a built in salt val -func newCipher(flatten int, password, salt string) (*cipher, error) { +func newCipher(mode NameEncryptionMode, password, salt string) (*cipher, error) { c := &cipher{ - flatten: flatten, + mode: mode, cryptoRand: rand.Reader, } c.buffers.New = func() interface{} { @@ -231,50 +271,8 @@ func (c *cipher) decryptSegment(ciphertext string) (string, error) { return string(plaintext), err } -// spread a name over the given number of directory levels -// -// if in isn't long enough dirs will be reduces -func spreadName(dirs int, in string) string { - if dirs > len(in) { - dirs = len(in) - } - prefix := "" - for i := 0; i < dirs; i++ { - prefix += string(in[i]) + "/" - } - return prefix + in -} - -// reverse spreadName, returning an error if not in spread format -// -// This decodes any level of spreading -func unspreadName(in string) (string, error) { - in = strings.ToLower(in) - segments := strings.Split(in, "/") - if len(segments) == 0 { - return in, nil - } - out := segments[len(segments)-1] - segments = segments[:len(segments)-1] - for i, s := range segments { - if len(s) != 1 { - return "", ErrorBadSpreadNotSingleChar - } - if i >= len(out) { - return "", ErrorBadSpreadResultTooShort - } - if s[0] != out[i] { - return "", ErrorBadSpreadDidntMatch - } - } - return out, nil -} - -// EncryptName encrypts a file path -func (c *cipher) EncryptName(in string) string { - if c.flatten > 0 { - return spreadName(c.flatten, c.encryptSegment(in)) - } +// encryptFileName encrypts a file path +func (c *cipher) encryptFileName(in string) string { segments := strings.Split(in, "/") for i := range segments { segments[i] = c.encryptSegment(segments[i]) @@ -282,15 +280,24 @@ func (c *cipher) EncryptName(in string) string { return strings.Join(segments, "/") } -// DecryptName decrypts a file path -func (c *cipher) DecryptName(in string) (string, error) { - if c.flatten > 0 { - unspread, err := unspreadName(in) - if err != nil { - return "", err - } - return c.decryptSegment(unspread) +// EncryptFileName encrypts a file path +func (c *cipher) EncryptFileName(in string) string { + if c.mode == NameEncryptionOff { + return in + encryptedSuffix } + return c.encryptFileName(in) +} + +// EncryptDirName encrypts a directory path +func (c *cipher) EncryptDirName(in string) string { + if c.mode == NameEncryptionOff { + return in + } + return c.encryptFileName(in) +} + +// decryptFileName decrypts a file path +func (c *cipher) decryptFileName(in string) (string, error) { segments := strings.Split(in, "/") for i := range segments { var err error @@ -302,6 +309,26 @@ func (c *cipher) DecryptName(in string) (string, error) { return strings.Join(segments, "/"), nil } +// DecryptFileName decrypts a file path +func (c *cipher) DecryptFileName(in string) (string, error) { + if c.mode == NameEncryptionOff { + remainingLength := len(in) - len(encryptedSuffix) + if remainingLength > 0 && strings.HasSuffix(in, encryptedSuffix) { + return in[:remainingLength], nil + } + return "", ErrorNotAnEncryptedFile + } + return c.decryptFileName(in) +} + +// DecryptDirName decrypts a directory path +func (c *cipher) DecryptDirName(in string) (string, error) { + if c.mode == NameEncryptionOff { + return in, nil + } + return c.decryptFileName(in) +} + // nonce is an NACL secretbox nonce type nonce [fileNonceSize]byte diff --git a/crypt/cipher_test.go b/crypt/cipher_test.go index 06897bf51..6ec09cc10 100644 --- a/crypt/cipher_test.go +++ b/crypt/cipher_test.go @@ -15,6 +15,32 @@ import ( "github.com/stretchr/testify/require" ) +func TestNewNameEncryptionMode(t *testing.T) { + for _, test := range []struct { + in string + expected NameEncryptionMode + expectedErr string + }{ + {"off", NameEncryptionOff, ""}, + {"standard", NameEncryptionStandard, ""}, + {"potato", NameEncryptionMode(0), "Unknown file name encryption mode \"potato\""}, + } { + actual, actualErr := NewNameEncryptionMode(test.in) + assert.Equal(t, actual, test.expected) + if test.expectedErr == "" { + assert.NoError(t, actualErr) + } else { + assert.Error(t, actualErr, test.expectedErr) + } + } +} + +func TestNewNameEncryptionModeString(t *testing.T) { + assert.Equal(t, NameEncryptionOff.String(), "off") + assert.Equal(t, NameEncryptionStandard.String(), "standard") + assert.Equal(t, NameEncryptionMode(2).String(), "Unknown mode #2") +} + func TestValidString(t *testing.T) { for _, test := range []struct { in string @@ -129,7 +155,7 @@ func TestDecodeFileName(t *testing.T) { } func TestEncryptSegment(t *testing.T) { - c, _ := newCipher(0, "", "") + c, _ := newCipher(NameEncryptionStandard, "", "") for _, test := range []struct { in string expected string @@ -166,7 +192,7 @@ func TestEncryptSegment(t *testing.T) { func TestDecryptSegment(t *testing.T) { // We've tested the forwards above, now concentrate on the errors - c, _ := newCipher(0, "", "") + c, _ := newCipher(NameEncryptionStandard, "", "") for _, test := range []struct { in string expectedErr error @@ -184,87 +210,78 @@ func TestDecryptSegment(t *testing.T) { } } -func TestSpreadName(t *testing.T) { - for _, test := range []struct { - n int - in string - expected string - }{ - {3, "", ""}, - {0, "abcdefg", "abcdefg"}, - {1, "abcdefg", "a/abcdefg"}, - {2, "abcdefg", "a/b/abcdefg"}, - {3, "abcdefg", "a/b/c/abcdefg"}, - {4, "abcdefg", "a/b/c/d/abcdefg"}, - {4, "abcd", "a/b/c/d/abcd"}, - {4, "abc", "a/b/c/abc"}, - {4, "ab", "a/b/ab"}, - {4, "a", "a/a"}, - } { - actual := spreadName(test.n, test.in) - assert.Equal(t, test.expected, actual, fmt.Sprintf("Testing %d,%q", test.n, test.in)) - recovered, err := unspreadName(test.expected) - assert.NoError(t, err, fmt.Sprintf("Testing reverse %q", test.expected)) - assert.Equal(t, test.in, recovered, fmt.Sprintf("Testing reverse %q", test.expected)) - } +func TestEncryptFileName(t *testing.T) { + // First standard mode + c, _ := newCipher(NameEncryptionStandard, "", "") + assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s", c.EncryptFileName("1")) + assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", c.EncryptFileName("1/12")) + assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", c.EncryptFileName("1/12/123")) + // Now off mode + c, _ = newCipher(NameEncryptionOff, "", "") + assert.Equal(t, "1/12/123.bin", c.EncryptFileName("1/12/123")) } -func TestUnspreadName(t *testing.T) { - // We've tested the forwards above, now concentrate on the errors +func TestDecryptFileName(t *testing.T) { for _, test := range []struct { - in string - expectedErr error - }{ - {"aa/bc", ErrorBadSpreadNotSingleChar}, - {"/", ErrorBadSpreadNotSingleChar}, - {"a/", ErrorBadSpreadResultTooShort}, - {"a/b/c/ab", ErrorBadSpreadResultTooShort}, - {"a/b/x/abc", ErrorBadSpreadDidntMatch}, - {"a/b/c/ABC", nil}, - } { - actual, actualErr := unspreadName(test.in) - assert.Equal(t, test.expectedErr, actualErr, fmt.Sprintf("in=%q got actual=%q, err = %v %T", test.in, actual, actualErr, actualErr)) - } -} - -func TestEncryptName(t *testing.T) { - // First no flatten - c, _ := newCipher(0, "", "") - assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s", c.EncryptName("1")) - assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", c.EncryptName("1/12")) - assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", c.EncryptName("1/12/123")) - // Now with flatten - c, _ = newCipher(3, "", "") - assert.Equal(t, "k/g/t/kgtickdcigo7600huebjl3ubu4", c.EncryptName("1/12/123")) -} - -func TestDecryptName(t *testing.T) { - for _, test := range []struct { - flatten int + mode NameEncryptionMode in string expected string expectedErr error }{ - {0, "p0e52nreeaj0a5ea7s64m4j72s", "1", nil}, - {0, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", "1/12", nil}, - {0, "p0e52nreeAJ0A5EA7S64M4J72S/L42G6771HNv3an9cgc8cr2n1ng", "1/12", nil}, - {0, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", "1/12/123", nil}, - {0, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1/qgm4avr35m5loi1th53ato71v0", "", ErrorNotAMultipleOfBlocksize}, - {3, "k/g/t/kgtickdcigo7600huebjl3ubu4", "1/12/123", nil}, - {1, "k/g/t/kgtickdcigo7600huebjl3ubu4", "1/12/123", nil}, - {1, "k/g/t/i/kgtickdcigo7600huebjl3ubu4", "1/12/123", nil}, - {1, "k/x/t/i/kgtickdcigo7600huebjl3ubu4", "", ErrorBadSpreadDidntMatch}, + {NameEncryptionStandard, "p0e52nreeaj0a5ea7s64m4j72s", "1", nil}, + {NameEncryptionStandard, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", "1/12", nil}, + {NameEncryptionStandard, "p0e52nreeAJ0A5EA7S64M4J72S/L42G6771HNv3an9cgc8cr2n1ng", "1/12", nil}, + {NameEncryptionStandard, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", "1/12/123", nil}, + {NameEncryptionStandard, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1/qgm4avr35m5loi1th53ato71v0", "", ErrorNotAMultipleOfBlocksize}, + {NameEncryptionOff, "1/12/123.bin", "1/12/123", nil}, + {NameEncryptionOff, "1/12/123.bix", "", ErrorNotAnEncryptedFile}, + {NameEncryptionOff, ".bin", "", ErrorNotAnEncryptedFile}, } { - c, _ := newCipher(test.flatten, "", "") - actual, actualErr := c.DecryptName(test.in) - what := fmt.Sprintf("Testing %q (flatten=%d)", test.in, test.flatten) + c, _ := newCipher(test.mode, "", "") + actual, actualErr := c.DecryptFileName(test.in) + what := fmt.Sprintf("Testing %q (mode=%v)", test.in, test.mode) + assert.Equal(t, test.expected, actual, what) + assert.Equal(t, test.expectedErr, actualErr, what) + } +} + +func TestEncryptDirName(t *testing.T) { + // First standard mode + c, _ := newCipher(NameEncryptionStandard, "", "") + assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s", c.EncryptDirName("1")) + assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", c.EncryptDirName("1/12")) + assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", c.EncryptDirName("1/12/123")) + // Now off mode + c, _ = newCipher(NameEncryptionOff, "", "") + assert.Equal(t, "1/12/123", c.EncryptDirName("1/12/123")) +} + +func TestDecryptDirName(t *testing.T) { + for _, test := range []struct { + mode NameEncryptionMode + in string + expected string + expectedErr error + }{ + {NameEncryptionStandard, "p0e52nreeaj0a5ea7s64m4j72s", "1", nil}, + {NameEncryptionStandard, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", "1/12", nil}, + {NameEncryptionStandard, "p0e52nreeAJ0A5EA7S64M4J72S/L42G6771HNv3an9cgc8cr2n1ng", "1/12", nil}, + {NameEncryptionStandard, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", "1/12/123", nil}, + {NameEncryptionStandard, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1/qgm4avr35m5loi1th53ato71v0", "", ErrorNotAMultipleOfBlocksize}, + {NameEncryptionOff, "1/12/123.bin", "1/12/123.bin", nil}, + {NameEncryptionOff, "1/12/123", "1/12/123", nil}, + {NameEncryptionOff, ".bin", ".bin", nil}, + } { + c, _ := newCipher(test.mode, "", "") + actual, actualErr := c.DecryptDirName(test.in) + what := fmt.Sprintf("Testing %q (mode=%v)", test.in, test.mode) assert.Equal(t, test.expected, actual, what) assert.Equal(t, test.expectedErr, actualErr, what) } } func TestEncryptedSize(t *testing.T) { - c, _ := newCipher(0, "", "") + c, _ := newCipher(NameEncryptionStandard, "", "") for _, test := range []struct { in int64 expected int64 @@ -288,7 +305,7 @@ func TestEncryptedSize(t *testing.T) { func TestDecryptedSize(t *testing.T) { // Test the errors since we tested the reverse above - c, _ := newCipher(0, "", "") + c, _ := newCipher(NameEncryptionStandard, "", "") for _, test := range []struct { in int64 expectedErr error @@ -521,7 +538,7 @@ func (z *zeroes) Read(p []byte) (n int, err error) { // Test encrypt decrypt with different buffer sizes func testEncryptDecrypt(t *testing.T, bufSize int, copySize int64) { - c, err := newCipher(0, "", "") + c, err := newCipher(NameEncryptionStandard, "", "") assert.NoError(t, err) c.cryptoRand = &zeroes{} // zero out the nonce buf := make([]byte, bufSize) @@ -591,7 +608,7 @@ func TestEncryptData(t *testing.T) { {[]byte{1}, file1}, {[]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}, file16}, } { - c, err := newCipher(0, "", "") + c, err := newCipher(NameEncryptionStandard, "", "") assert.NoError(t, err) c.cryptoRand = newRandomSource(1E8) // nodge the crypto rand generator @@ -614,7 +631,7 @@ func TestEncryptData(t *testing.T) { } func TestNewEncrypter(t *testing.T) { - c, err := newCipher(0, "", "") + c, err := newCipher(NameEncryptionStandard, "", "") assert.NoError(t, err) c.cryptoRand = newRandomSource(1E8) // nodge the crypto rand generator @@ -658,7 +675,7 @@ func (c *closeDetector) Close() error { } func TestNewDecrypter(t *testing.T) { - c, err := newCipher(0, "", "") + c, err := newCipher(NameEncryptionStandard, "", "") assert.NoError(t, err) c.cryptoRand = newRandomSource(1E8) // nodge the crypto rand generator @@ -700,7 +717,7 @@ func TestNewDecrypter(t *testing.T) { } func TestDecrypterRead(t *testing.T) { - c, err := newCipher(0, "", "") + c, err := newCipher(NameEncryptionStandard, "", "") assert.NoError(t, err) // Test truncating the header @@ -744,7 +761,7 @@ func TestDecrypterRead(t *testing.T) { } func TestDecrypterClose(t *testing.T) { - c, err := newCipher(0, "", "") + c, err := newCipher(NameEncryptionStandard, "", "") assert.NoError(t, err) cd := newCloseDetector(bytes.NewBuffer(file16)) @@ -780,7 +797,7 @@ func TestDecrypterClose(t *testing.T) { } func TestPutGetBlock(t *testing.T) { - c, err := newCipher(0, "", "") + c, err := newCipher(NameEncryptionStandard, "", "") assert.NoError(t, err) block := c.getBlock() @@ -791,7 +808,7 @@ func TestPutGetBlock(t *testing.T) { } func TestKey(t *testing.T) { - c, err := newCipher(0, "", "") + c, err := newCipher(NameEncryptionStandard, "", "") assert.NoError(t, err) // Check zero keys OK diff --git a/crypt/crypt.go b/crypt/crypt.go index a25cfc416..68dad6ebf 100644 --- a/crypt/crypt.go +++ b/crypt/crypt.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "path" - "strings" "sync" "github.com/ncw/rclone/fs" @@ -22,27 +21,15 @@ func init() { Name: "remote", Help: "Remote to encrypt/decrypt.", }, { - Name: "flatten", - Help: "Flatten the directory structure - more secure, less useful - see docs for tradeoffs.", + Name: "filename_encryption", + Help: "How to encrypt the filenames.", Examples: []fs.OptionExample{ { - Value: "0", - Help: "Don't flatten files (default) - good for unlimited files, but doesn't hide directory structure.", + Value: "off", + Help: "Don't encrypt the file names. Adds a \".bin\" extension only.", }, { - Value: "1", - Help: "Spread files over 1 directory good for <10,000 files.", - }, { - Value: "2", - Help: "Spread files over 32 directories good for <320,000 files.", - }, { - Value: "3", - Help: "Spread files over 1024 directories good for <10,000,000 files.", - }, { - Value: "4", - Help: "Spread files over 32,768 directories good for <320,000,000 files.", - }, { - Value: "5", - Help: "Spread files over 1,048,576 levels good for <10,000,000,000 files.", + Value: "standard", + Help: "Encrypt the filenames see the docs for the details.", }, }, }, { @@ -60,12 +47,15 @@ func init() { // NewFs contstructs an Fs from the path, container:path func NewFs(name, rpath string) (fs.Fs, error) { - flatten := fs.ConfigFile.MustInt(name, "flatten", 0) + mode, err := NewNameEncryptionMode(fs.ConfigFile.MustValue(name, "filename_encryption", "standard")) + if err != nil { + return nil, err + } password := fs.ConfigFile.MustValue(name, "password", "") if password == "" { return nil, errors.New("password not set in config file") } - password, err := fs.Reveal(password) + password, err = fs.Reveal(password) if err != nil { return nil, errors.Wrap(err, "failed to decrypt password") } @@ -76,20 +66,26 @@ func NewFs(name, rpath string) (fs.Fs, error) { return nil, errors.Wrap(err, "failed to decrypt password2") } } - cipher, err := newCipher(flatten, password, salt) + cipher, err := newCipher(mode, password, salt) if err != nil { return nil, errors.Wrap(err, "failed to make cipher") } remote := fs.ConfigFile.MustValue(name, "remote") - remotePath := path.Join(remote, cipher.EncryptName(rpath)) + // Look for a file first + remotePath := path.Join(remote, cipher.EncryptFileName(rpath)) wrappedFs, err := fs.NewFs(remotePath) + // if that didn't produce a file, look for a directory + if err != fs.ErrorIsFile { + remotePath = path.Join(remote, cipher.EncryptDirName(rpath)) + wrappedFs, err = fs.NewFs(remotePath) + } if err != fs.ErrorIsFile && err != nil { return nil, errors.Wrapf(err, "failed to make remote %q to wrap", remotePath) } f := &Fs{ - Fs: wrappedFs, - cipher: cipher, - flatten: flatten, + Fs: wrappedFs, + cipher: cipher, + mode: mode, } return f, err } @@ -97,8 +93,8 @@ func NewFs(name, rpath string) (fs.Fs, error) { // Fs represents a wrapped fs.Fs type Fs struct { fs.Fs - cipher Cipher - flatten int + cipher Cipher + mode NameEncryptionMode } // String returns a description of the FS @@ -108,12 +104,12 @@ func (f *Fs) String() string { // List the Fs into a channel func (f *Fs) List(opts fs.ListOpts, dir string) { - f.Fs.List(f.newListOpts(opts, dir), f.cipher.EncryptName(dir)) + f.Fs.List(f.newListOpts(opts, dir), f.cipher.EncryptDirName(dir)) } // NewObject finds the Object at remote. func (f *Fs) NewObject(remote string) (fs.Object, error) { - o, err := f.Fs.NewObject(f.cipher.EncryptName(remote)) + o, err := f.Fs.NewObject(f.cipher.EncryptFileName(remote)) if err != nil { return nil, err } @@ -174,7 +170,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) { if !ok { return nil, fs.ErrorCantCopy } - oResult, err := do.Copy(o.Object, f.cipher.EncryptName(remote)) + oResult, err := do.Copy(o.Object, f.cipher.EncryptFileName(remote)) if err != nil { return nil, err } @@ -199,7 +195,7 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) { if !ok { return nil, fs.ErrorCantCopy } - oResult, err := do.Move(o.Object, f.cipher.EncryptName(remote)) + oResult, err := do.Move(o.Object, f.cipher.EncryptFileName(remote)) if err != nil { return nil, err } @@ -242,7 +238,7 @@ func (o *Object) String() string { // Remote returns the remote path func (o *Object) Remote() string { remote := o.Object.Remote() - decryptedName, err := o.f.cipher.DecryptName(remote) + decryptedName, err := o.f.cipher.DecryptFileName(remote) if err != nil { fs.Debug(remote, "Undecryptable file name: %v", err) return remote @@ -287,7 +283,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error { func (f *Fs) newDir(dir *fs.Dir) *fs.Dir { new := *dir remote := dir.Name - decryptedRemote, err := f.cipher.DecryptName(remote) + decryptedRemote, err := f.cipher.DecryptDirName(remote) if err != nil { fs.Debug(remote, "Undecryptable dir name: %v", err) } else { @@ -318,7 +314,7 @@ func (o *ObjectInfo) Fs() fs.Info { // Remote returns the remote path func (o *ObjectInfo) Remote() string { - return o.f.cipher.EncryptName(o.ObjectInfo.Remote()) + return o.f.cipher.EncryptFileName(o.ObjectInfo.Remote()) } // Size returns the size of the file @@ -358,55 +354,19 @@ func (f *Fs) newListOpts(lo fs.ListOpts, dir string) *ListOpts { // // Each returned item must have less than level `/`s in. func (lo *ListOpts) Level() int { - // If flattened recurse fully - if lo.f.flatten > 0 { - return fs.MaxLevel - } return lo.ListOpts.Level() } -// addSyntheticDirs makes up directory objects for the path passed in -func (lo *ListOpts) addSyntheticDirs(path string) { - lo.mu.Lock() - defer lo.mu.Unlock() - for { - i := strings.LastIndexByte(path, '/') - if i < 0 { - break - } - path = path[:i] - if path == "" { - break - } - if _, found := lo.dirs[path]; found { - break - } - slashes := strings.Count(path, "/") - if slashes < lo.ListOpts.Level() { - lo.ListOpts.AddDir(&fs.Dir{Name: path}) - } - lo.dirs[path] = struct{}{} - } -} - // Add an object to the output. // If the function returns true, the operation has been aborted. // Multiple goroutines can safely add objects concurrently. func (lo *ListOpts) Add(obj fs.Object) (abort bool) { remote := obj.Remote() - decryptedRemote, err := lo.f.cipher.DecryptName(remote) + _, err := lo.f.cipher.DecryptFileName(remote) if err != nil { fs.Debug(remote, "Skipping undecryptable file name: %v", err) return lo.ListOpts.IsFinished() } - // If flattened add synthetic directories - if lo.f.flatten > 0 { - lo.addSyntheticDirs(decryptedRemote) - slashes := strings.Count(decryptedRemote, "/") - if slashes >= lo.ListOpts.Level() { - return lo.ListOpts.IsFinished() - } - } return lo.ListOpts.Add(lo.f.newObject(obj)) } @@ -414,12 +374,8 @@ func (lo *ListOpts) Add(obj fs.Object) (abort bool) { // If the function returns true, the operation has been aborted. // Multiple goroutines can safely add objects concurrently. func (lo *ListOpts) AddDir(dir *fs.Dir) (abort bool) { - // If flattened we don't add any directories from the underlying remote - if lo.f.flatten > 0 { - return lo.ListOpts.IsFinished() - } remote := dir.Name - _, err := lo.f.cipher.DecryptName(remote) + _, err := lo.f.cipher.DecryptDirName(remote) if err != nil { fs.Debug(remote, "Skipping undecryptable dir name: %v", err) return lo.ListOpts.IsFinished() @@ -430,11 +386,7 @@ func (lo *ListOpts) AddDir(dir *fs.Dir) (abort bool) { // IncludeDirectory returns whether this directory should be // included in the listing (and recursed into or not). func (lo *ListOpts) IncludeDirectory(remote string) bool { - // If flattened we look in all directories - if lo.f.flatten > 0 { - return true - } - decryptedRemote, err := lo.f.cipher.DecryptName(remote) + decryptedRemote, err := lo.f.cipher.DecryptDirName(remote) if err != nil { fs.Debug(remote, "Not including undecryptable directory name: %v", err) return false diff --git a/docs/content/crypt.md b/docs/content/crypt.md index 5ac4ffb26..ba69590fe 100644 --- a/docs/content/crypt.md +++ b/docs/content/crypt.md @@ -19,8 +19,8 @@ First check your chosen remote is working - we'll call it will be encrypted and anything outside won't. This means that if you are using a bucket based remote (eg S3, B2, swift) then you should probably put the bucket in the remote `s3:bucket`. If you just use -`s3:` then rclone will make encrypted bucket names too which may or -may not be what you want. +`s3:` then rclone will make encrypted bucket names too (if using file +name encryption) which may or may not be what you want. Now configure `crypt` using `rclone config`. We will call this one `secret` to differentiate it from the `remote`. @@ -30,7 +30,7 @@ No remotes found - make a new one n) New remote s) Set configuration password q) Quit config -n/s/q> n +n/s/q> n name> secret Type of storage to configure. Choose a number from below, or type in your own value @@ -61,32 +61,44 @@ Choose a number from below, or type in your own value Storage> 5 Remote to encrypt/decrypt. remote> remote:path -Flatten the directory structure - more secure, less useful - see docs for tradeoffs. +How to encrypt the filenames. Choose a number from below, or type in your own value - 1 / Don't flatten files (default) - good for unlimited files, but doesn't hide directory structure. - \ "0" - 2 / Spread files over 1 directory good for <10,000 files. - \ "1" - 3 / Spread files over 32 directories good for <320,000 files. - \ "2" - 4 / Spread files over 1024 directories good for <10,000,000 files. - \ "3" - 5 / Spread files over 32,768 directories good for <320,000,000 files. - \ "4" - 6 / Spread files over 1,048,576 levels good for <10,000,000,000 files. - \ "5" -flatten> 1 + 1 / Don't encrypt the file names. Adds a ".bin" extension only. + \ "off" + 2 / Encrypt the filenames see the docs for the details. + \ "standard" +filename_encryption> 2 Password or pass phrase for encryption. +y) Yes type in my own password +g) Generate random password +y/g> y Enter the password: password: Confirm the password: password: +Password or pass phrase for salt. Optional but recommended. +Should be different to the previous password. +y) Yes type in my own password +g) Generate random password +n) No leave this optional password blank +y/g/n> g +Password strength in bits. +64 is just about memorable +128 is secure +1024 is the maximum +Bits> 128 +Your password is: JAsJvRcgR-_veXNfy_sGmQ +Use this password? +y) Yes +n) No +y/n> y Remote config -------------------- [secret] remote = remote:path -flatten = 0 -password = 0_gtCJ422bzwAWP0UN2lggrjhA-sSg +filename_encryption = standard +password = CfDxopZIXFG0Oo-ac7dPLWWOHkNJbw +password2 = HYUpfuzHJL8qnX9fOaIYijq0xnVLwyVzp3y4SF3TwYqAU6HLysk -------------------- y) Yes this is OK e) Edit this remote @@ -99,9 +111,9 @@ obscured so it isn't immediately obvious what it is. It is in no way secure unless you use config file encryption. A long passphrase is recommended, or you can use a random one. Note -that if you reconfigure rclone with the same password/passphrase +that if you reconfigure rclone with the same passwords/passphrases elsewhere it will be compatible - all the secrets used are derived -from that one password/passphrase. +from those two passwords/passphrases. Note that rclone does not encrypt * file length - this can be calcuated within 16 bytes @@ -109,7 +121,8 @@ Note that rclone does not encrypt ## Example ## -To test I made a little directory of files +To test I made a little directory of files using "standard" file name +encryption. ``` plaintext/ @@ -154,39 +167,43 @@ $ rclone -q ls secret:subdir 10 subsubdir/file4.txt ``` -If you use the flattened flag then the listing will look and that last command will not work. +If don't use file name encryption then the remote will look like this +- note the `.bin` extensions added to prevent the cloud provider +attempting to interpret the data. ``` $ rclone -q ls remote:path - 56 t/tsdtcpdu6g9dpamn6poqc248tll9dj5ok78a363etmq8ushr821g - 57 g/gsrp2g0u85pgsi6kso74bjsrsafe11odpfln8qqpj6n9p20of0a0 - 55 h/hagjclgavj2mbiqm6u6cnjjqcg - 58 4/4jsbao3dhi0jfoubt2oo493pbqmsshn92q01ddu7dg6428rlluhg - 54 v/v05749mltvv1tf4onltun46gls + 54 file0.txt.bin + 57 subdir/file3.txt.bin + 56 subdir/file2.txt.bin + 58 subdir/subsubdir/file4.txt.bin + 55 file1.txt.bin ``` -### Flattened vs non-Flattened ### +### File name encryption modes ### -Pros and cons of each +Here are some of the features of the file name encryption modes -Flattened - * hides directory structures - * identical file names won't have identical encrypted names - * can't use a sub path - * doesn't work: `rclone copy crypt:sub/dir /tmp/recovered` - * use: `rclone copy --include "/sub/dir/**" crypt: /tmp/recovered` - * will always have to recurse through the entire directory structure - * can't copy a single file directly - * doesn't work: `rclone copy crypt:path/to/file /tmp/recovered` - * use: `rclone copy --include "/path/to/file" crypt: /tmp/recovered` +Off + * doesn't hide file names or directory structure + * allows for longer file names (~246 characters) + * can use sub paths and copy single files -Normal +Standard + * file names encrypted + * file names can't be as long (~156 characters) * can use sub paths and copy single files * directory structure visibile * identical files names will have identical uploaded names * can use shortcuts to shorten the directory recursion -You can swap between flattened levels without re-uploading your files. +Cloud storage systems have various limits on file name length and +total path length which you are more likely to hit using "Standard" +file name encryption. If you keep your file names to below 156 +characters in length then you should be OK on all providers. + +There may be an even more secure file name encryption mode in the +future which will address the long file name problem. ## File formats ## @@ -245,25 +262,23 @@ files. ### Name encryption ### -File names are encrypted by crypt. These are either encrypted segment -by segment - the path is broken up into `/` separated strings and -these are encrypted individually, or if working in flattened mode the -whole path is encrypted `/`s and all. +File names are encrypted segment by segment - the path is broken up +into `/` separated strings and these are encrypted individually. -First file names are padded using using PKCS#7 to a multiple of 16 -bytes before encryption. +File segments are padded using using PKCS#7 to a multiple of 16 bytes +before encryption. They are then encrypted with EME using AES with 256 bit key. EME (ECB-Mix-ECB) is a wide-block encryption mode presented in the 2003 paper "A Parallelizable Enciphering Mode" by Halevi and Rogaway. This makes for determinstic encryption which is what we want - the -same filename must encrypt to the same thing. +same filename must encrypt to the same thing otherwise we can't find +it on the cloud storage system. This means that * filenames with the same name will encrypt the same - * (though we can use directory flattening to avoid this if required) * filenames which start the same won't have a common prefix This uses a 32 byte key (256 bits) and a 16 byte (128 bits) IV both of @@ -281,8 +296,11 @@ used on case insensitive remotes (eg Windows, Amazon Drive). ### Key derivation ### -Rclone uses `scrypt` with parameters `N=16384, r=8, p=1` with a fixed -salt to derive the 32+32+16 = 80 bytes of key material required. +Rclone uses `scrypt` with parameters `N=16384, r=8, p=1` with a an +optional user supplied salt (password2) to derive the 32+32+16 = 80 +bytes of key material required. If the user doesn't supply a salt +then rclone uses an internal one. `scrypt` makes it impractical to mount a dictionary attack on rclone -encrypted data. +encrypted data. For full protection agains this you should always use +a salt.