Skip to content

Commit 75aa299

Browse files
committed
Use bytearrays for building up bytes for I/O.
Bytes are immutable in python, which means that repeatedly appending to them is _quadratic_ in time. That is, we would like and expect that deleting 10k objects 10 times would take about as long as deleting 100k, but because we repeatedly concatenate to a bytes object in the loop, they are not: timeit.timeit(lambda: server.delete_multi(["foo"]*10000),number=10) # 0.5087270829999966 timeit.timeit(lambda: server.delete_multi(["foo"]*100000),number=1) # 10.650619775999985 Switch to using `bytearray` in all palces which handle variable numbers of bytes. After this change: timeit.timeit(lambda: server.delete_multi(["foo"]*10000),number=10) # 0.1197161969999998 timeit.timeit(lambda: server.delete_multi(["foo"]*100000),number=1) # 0.1269589350000011
1 parent 41ed613 commit 75aa299

File tree

1 file changed

+18
-22
lines changed

1 file changed

+18
-22
lines changed

bmemcached/protocol.py

Lines changed: 18 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -195,7 +195,7 @@ def _read_socket(self, size):
195195
:param size: Size in bytes to be read.
196196
:return: Data from socket
197197
"""
198-
value = b''
198+
value = bytearray()
199199
while len(value) < size:
200200
data = self.connection.recv(size - len(value))
201201
if not data:
@@ -206,7 +206,7 @@ def _read_socket(self, size):
206206
if len(value) < size:
207207
raise socket.error()
208208

209-
return value
209+
return bytes(value)
210210

211211
def _get_response(self):
212212
"""
@@ -501,7 +501,7 @@ def get_multi(self, keys):
501501
if n == 0:
502502
return {}
503503

504-
msg = b''
504+
msg = bytearray()
505505
for i, key in enumerate(keys):
506506
keybytes = str_to_bytes(key)
507507
command = self.COMMANDS['getk' if i == n - 1 else 'getkq']
@@ -690,7 +690,7 @@ def set_multi(self, mappings, time=100, compress_level=-1):
690690
:rtype: list
691691
"""
692692
mappings = list(mappings.items())
693-
msg = []
693+
msg = bytearray()
694694

695695
for opaque, (key, value) in enumerate(mappings):
696696
if isinstance(key, tuple):
@@ -707,23 +707,19 @@ def set_multi(self, mappings, time=100, compress_level=-1):
707707

708708
keybytes = str_to_bytes(key)
709709
flags, value = self.serialize(value, compress_level=compress_level)
710-
m = struct.pack(self.HEADER_STRUCT +
711-
self.COMMANDS[command]['struct'] % (len(keybytes), len(value)),
712-
self.MAGIC['request'],
713-
self.COMMANDS[command]['command'],
714-
len(keybytes),
715-
8, 0, 0, len(keybytes) + len(value) + 8, opaque, cas or 0,
716-
flags, time, keybytes, value)
717-
msg.append(m)
718-
719-
m = struct.pack(self.HEADER_STRUCT +
720-
self.COMMANDS['noop']['struct'],
721-
self.MAGIC['request'],
722-
self.COMMANDS['noop']['command'],
723-
0, 0, 0, 0, 0, 0, 0)
724-
msg.append(m)
725-
726-
msg = b''.join(msg)
710+
msg += struct.pack(self.HEADER_STRUCT +
711+
self.COMMANDS[command]['struct'] % (len(keybytes), len(value)),
712+
self.MAGIC['request'],
713+
self.COMMANDS[command]['command'],
714+
len(keybytes),
715+
8, 0, 0, len(keybytes) + len(value) + 8, opaque, cas or 0,
716+
flags, time, keybytes, value)
717+
718+
msg += struct.pack(self.HEADER_STRUCT +
719+
self.COMMANDS['noop']['struct'],
720+
self.MAGIC['request'],
721+
self.COMMANDS['noop']['command'],
722+
0, 0, 0, 0, 0, 0, 0)
727723

728724
self._send(msg)
729725

@@ -854,7 +850,7 @@ def delete_multi(self, keys):
854850
:rtype: bool
855851
"""
856852
logger.debug('Deleting keys %r', keys)
857-
msg = b''
853+
msg = bytearray()
858854
for key in keys:
859855
keybytes = str_to_bytes(key)
860856
msg += struct.pack(

0 commit comments

Comments
 (0)