From 00e6462de37bca7c95758c7b3afebb33969f505f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Ericson?= Date: Thu, 24 Sep 2015 12:59:43 -0300 Subject: [PATCH] Now when sending binary data messages one should use bytearray, this way there is no ambiguity, on python2, about byte/str. --- ably/types/message.py | 19 ++++---- ably/types/mixins.py | 9 ++-- ably/types/typedbuffer.py | 2 +- ably/util/crypto.py | 6 ++- test/ably/encoders_test.py | 89 ++++++++++++++++++++++++++++---------- 5 files changed, 88 insertions(+), 37 deletions(-) diff --git a/ably/types/message.py b/ably/types/message.py index b92fa60c..5a77c2a4 100644 --- a/ably/types/message.py +++ b/ably/types/message.py @@ -116,15 +116,15 @@ def as_dict(self, binary=False): if isinstance(data, dict) or isinstance(data, list): encoding.append('json') data = json.dumps(data) - - elif isinstance(self.data, six.binary_type) and not binary: - data = base64.b64encode(data).decode('ascii') - encoding.append('base64') elif isinstance(data, six.text_type) and not binary: - encoding.append('utf-8') - elif isinstance(data, (six.text_type, six.binary_type)): - # only if is binary + # text_type is always a unicode string pass + elif (not binary and isinstance(data, bytearray) or + # bytearray is always bytes + isinstance(data, six.binary_type) and six.binary_type != str): + # in py3k we will understand as bytes + data = base64.b64encode(data).decode('ascii') + encoding.append('base64') elif isinstance(data, CipherData): encoding.append(data.encoding_str) data_type = data.type @@ -133,8 +133,11 @@ def as_dict(self, binary=False): encoding.append('base64') else: data = data.buffer + elif binary and isinstance(data, bytearray): + data = six.binary_type(data) - if not (isinstance(data, (six.binary_type, six.text_type, list, dict)) or + if not (isinstance(data, (six.binary_type, six.text_type, list, dict, + bytearray)) or data is None): raise AblyException("Invalid data payload", 400, 40011) diff --git a/ably/types/mixins.py b/ably/types/mixins.py index e35bae9b..8e20f5cf 100644 --- a/ably/types/mixins.py +++ b/ably/types/mixins.py @@ -21,6 +21,8 @@ def decode(data, encoding='', cipher=None): while encoding_list: encoding = encoding_list.pop() + if not encoding: + continue if encoding == 'json': if isinstance(data, six.binary_type): data = data.decode() @@ -28,9 +30,9 @@ def decode(data, encoding='', cipher=None): continue data = json.loads(data) elif encoding == 'base64' and isinstance(data, six.binary_type): - data = base64.b64decode(data) + data = bytearray(base64.b64decode(data)) elif encoding == 'base64': - data = base64.b64decode(data.encode('utf-8')) + data = bytearray(base64.b64decode(data.encode('utf-8'))) elif encoding.startswith('%s+' % CipherData.ENCODING_ID): if not cipher: log.error('Message cannot be decrypted as the channel is ' @@ -38,7 +40,8 @@ def decode(data, encoding='', cipher=None): encoding_list.append(encoding) break data = cipher.decrypt(data) - elif encoding == 'utf-8' and isinstance(data, six.binary_type): + elif encoding == 'utf-8' and isinstance(data, (six.binary_type, + bytearray)): data = data.decode('utf-8') elif encoding == 'utf-8': pass diff --git a/ably/types/typedbuffer.py b/ably/types/typedbuffer.py index 3308c120..270aeaf0 100644 --- a/ably/types/typedbuffer.py +++ b/ably/types/typedbuffer.py @@ -65,7 +65,7 @@ def from_obj(obj): if isinstance(obj, TypedBuffer): return obj - elif isinstance(obj, six.binary_type): + elif isinstance(obj, (six.binary_type, bytearray)): type = DataType.BUFFER buffer = obj elif isinstance(obj, six.string_types): diff --git a/ably/util/crypto.py b/ably/util/crypto.py index 29f20cb6..e48e44f2 100644 --- a/ably/util/crypto.py +++ b/ably/util/crypto.py @@ -90,17 +90,21 @@ def __random(self, length): return rndfile.read(length) def encrypt(self, plaintext): + if isinstance(plaintext, bytearray): + plaintext = six.binary_type(plaintext) padded_plaintext = self.__pad(plaintext) encrypted = self.__iv + self.__encryptor.encrypt(padded_plaintext) self.__iv = encrypted[-self.__block_size:] return encrypted def decrypt(self, ciphertext): + if isinstance(ciphertext, bytearray): + ciphertext = six.binary_type(ciphertext) iv = ciphertext[:self.__block_size] ciphertext = ciphertext[self.__block_size:] decryptor = AES.new(self.__secret_key, AES.MODE_CBC, iv) decrypted = decryptor.decrypt(ciphertext) - return self.__unpad(decrypted) + return bytearray(self.__unpad(decrypted)) @property def secret_key(self): diff --git a/test/ably/encoders_test.py b/test/ably/encoders_test.py index 0c2205d2..817984c3 100644 --- a/test/ably/encoders_test.py +++ b/test/ably/encoders_test.py @@ -37,21 +37,44 @@ def test_text_utf8(self): channel.publish('event', six.u('foó')) _, kwargs = post_mock.call_args self.assertEqual(json.loads(kwargs['body'])['data'], six.u('foó')) - self.assertEqual(json.loads(kwargs['body']).get('encoding', '').strip('/'), - 'utf-8') + self.assertFalse(json.loads(kwargs['body']).get('encoding', '')) + + def test_str(self): + # This test only makes sense for py2 + channel = self.ably.channels["persisted:publish"] + + with mock.patch('ably.rest.rest.Http.post') as post_mock: + channel.publish('event', 'foo') + _, kwargs = post_mock.call_args + self.assertEqual(json.loads(kwargs['body'])['data'], 'foo') + self.assertFalse(json.loads(kwargs['body']).get('encoding', '')) def test_with_binary_type(self): channel = self.ably.channels["persisted:publish"] with mock.patch('ably.rest.rest.Http.post') as post_mock: - channel.publish('event', six.b('foo')) + channel.publish('event', bytearray(b'foo')) _, kwargs = post_mock.call_args raw_data = json.loads(kwargs['body'])['data'] self.assertEqual(base64.b64decode(raw_data.encode('ascii')), - six.b('foo')) + bytearray(b'foo')) self.assertEqual(json.loads(kwargs['body'])['encoding'].strip('/'), 'base64') + def test_with_bytes_type(self): + # this test is only relevant for python3 + if six.PY3: + channel = self.ably.channels["persisted:publish"] + + with mock.patch('ably.rest.rest.Http.post') as post_mock: + channel.publish('event', b'foo') + _, kwargs = post_mock.call_args + raw_data = json.loads(kwargs['body'])['data'] + self.assertEqual(base64.b64decode(raw_data.encode('ascii')), + bytearray(b'foo')) + self.assertEqual(json.loads(kwargs['body'])['encoding'].strip('/'), + 'base64') + def test_with_json_dict_data(self): channel = self.ably.channels["persisted:publish"] data = {six.u('foó'): six.u('bár')} @@ -83,13 +106,22 @@ def test_text_utf8_decode(self): self.assertIsInstance(message.data, six.text_type) self.assertFalse(message.encoding) + def test_text_str_decode(self): + channel = self.ably.channels["persisted:stringnonutf8decode"] + + channel.publish('event', 'foo') + message = channel.history().items[0] + self.assertEqual(message.data, six.u('foo')) + self.assertIsInstance(message.data, six.text_type) + self.assertFalse(message.encoding) + def test_with_binary_type_decode(self): channel = self.ably.channels["persisted:binarydecode"] - channel.publish('event', six.b('foob')) + channel.publish('event', bytearray(b'foob')) message = channel.history().items[0] - self.assertEqual(message.data, six.b('foob')) - self.assertIsInstance(message.data, six.binary_type) + self.assertEqual(message.data, bytearray(b'foob')) + self.assertIsInstance(message.data, bytearray) self.assertFalse(message.encoding) def test_with_json_dict_data_decode(self): @@ -146,6 +178,16 @@ def test_text_utf8(self): data = self.decrypt(json.loads(kwargs['body'])['data']).decode('utf-8') self.assertEquals(data, six.u('fóo')) + def test_str(self): + # This test only makes sense for py2 + channel = self.ably.channels["persisted:publish"] + + with mock.patch('ably.rest.rest.Http.post') as post_mock: + channel.publish('event', 'foo') + _, kwargs = post_mock.call_args + self.assertEqual(json.loads(kwargs['body'])['data'], 'foo') + self.assertFalse(json.loads(kwargs['body']).get('encoding', '')) + def test_with_binary_type(self): channel = self.ably.channels.get("persisted:publish_enc", options=ChannelOptions( @@ -153,14 +195,14 @@ def test_with_binary_type(self): cipher_params=self.cipher_params)) with mock.patch('ably.rest.rest.Http.post') as post_mock: - channel.publish('event', six.b('foo')) + channel.publish('event', bytearray(b'foo')) _, kwargs = post_mock.call_args self.assertEquals(json.loads(kwargs['body'])['encoding'].strip('/'), 'cipher+aes-128-cbc/base64') data = self.decrypt(json.loads(kwargs['body'])['data']) - self.assertEqual(data, six.b('foo')) - self.assertIsInstance(data, six.binary_type) + self.assertEqual(data, bytearray(b'foo')) + self.assertIsInstance(data, bytearray) def test_with_json_dict_data(self): channel = self.ably.channels.get("persisted:publish_enc", @@ -207,10 +249,10 @@ def test_with_binary_type_decode(self): encrypted=True, cipher_params=self.cipher_params)) - channel.publish('event', six.b('foob')) + channel.publish('event', bytearray(b'foob')) message = channel.history().items[0] - self.assertEqual(message.data, six.b('foob')) - self.assertIsInstance(message.data, six.binary_type) + self.assertEqual(message.data, bytearray(b'foob')) + self.assertIsInstance(message.data, bytearray) self.assertFalse(message.encoding) def test_with_json_dict_data_decode(self): @@ -263,9 +305,9 @@ def test_with_binary_type(self): with mock.patch('ably.rest.rest.Http.post', wraps=channel.ably.http.post) as post_mock: - channel.publish('event', six.b('foo')) + channel.publish('event', bytearray(b'foo')) _, kwargs = post_mock.call_args - self.assertEqual(self.decode(kwargs['body'])['data'], six.b('foo')) + self.assertEqual(self.decode(kwargs['body'])['data'], bytearray(b'foo')) self.assertEqual(self.decode(kwargs['body']).get('encoding', '').strip('/'), '') def test_with_json_dict_data(self): @@ -304,10 +346,9 @@ def test_text_utf8_decode(self): def test_with_binary_type_decode(self): channel = self.ably.channels["persisted:binarydecode-bin"] - channel.publish('event', six.b('foob')) + channel.publish('event', bytearray(b'foob')) message = channel.history().items[0] - self.assertEqual(message.data, six.b('foob')) - self.assertIsInstance(message.data, six.binary_type) + self.assertEqual(message.data, bytearray(b'foob')) self.assertFalse(message.encoding) def test_with_json_dict_data_decode(self): @@ -367,14 +408,14 @@ def test_with_binary_type(self): with mock.patch('ably.rest.rest.Http.post', wraps=channel.ably.http.post) as post_mock: - channel.publish('event', six.b('foo')) + channel.publish('event', bytearray(b'foo')) _, kwargs = post_mock.call_args self.assertEquals(self.decode(kwargs['body'])['encoding'].strip('/'), 'cipher+aes-128-cbc') data = self.decrypt(self.decode(kwargs['body'])['data']) - self.assertEqual(data, six.b('foo')) - self.assertIsInstance(data, six.binary_type) + self.assertEqual(data, bytearray(b'foo')) + self.assertIsInstance(data, bytearray) def test_with_json_dict_data(self): channel = self.ably.channels.get("persisted:publish_enc", @@ -423,10 +464,10 @@ def test_with_binary_type_decode(self): encrypted=True, cipher_params=self.cipher_params)) - channel.publish('event', six.b('foob')) + channel.publish('event', bytearray(b'foob')) message = channel.history().items[0] - self.assertEqual(message.data, six.b('foob')) - self.assertIsInstance(message.data, six.binary_type) + self.assertEqual(message.data, bytearray(b'foob')) + self.assertIsInstance(message.data, bytearray) self.assertFalse(message.encoding) def test_with_json_dict_data_decode(self):