Skip to content

Refactor fallback' read header #441

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Jan 2, 2021
Merged

Refactor fallback' read header #441

Changes from 4 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
6d4050a
Removed unused parameter.
GuyTuval Oct 22, 2020
424e81a
Added MessagePack namedtuple and MESSAGE_PACK_DICT to store the diffe…
GuyTuval Oct 22, 2020
db85cfa
Removed duplicated code in 'elif' statements using the MESSAGE_PACK_D…
GuyTuval Oct 22, 2020
5eb8c5f
Combined 'elif' statements for bool check.
GuyTuval Oct 22, 2020
912c082
Renamed MESSAGE_PACK_DICT to _MSGPACK_HEADERS and changed its type to…
GuyTuval Jan 1, 2021
b02d115
Reverted combination of bool check.
GuyTuval Jan 1, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
265 changes: 96 additions & 169 deletions msgpack/fallback.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
"""Fallback pure Python implementation of msgpack"""

from collections import namedtuple
from datetime import datetime as _DateTime
import sys
import struct
Expand Down Expand Up @@ -149,6 +149,42 @@ def _unpack_from(f, b, o=0):
_unpack_from = struct.unpack_from


MessagePack = namedtuple("MessagePack", ["size", "format", "type"])
MessagePack.__new__.__defaults__ = ("", -1)


MESSAGE_PACK_DICT = {
0xC4: MessagePack(1, type=TYPE_BIN),
0xC5: MessagePack(2, ">H", TYPE_BIN),
0xC6: MessagePack(4, ">I", TYPE_BIN),
0xC7: MessagePack(2, "Bb", TYPE_EXT),
0xC8: MessagePack(3, ">Hb", TYPE_EXT),
0xC9: MessagePack(5, ">Ib", TYPE_EXT),
0xCA: MessagePack(4, ">f"),
0xCB: MessagePack(8, ">d"),
0xCC: MessagePack(1),
0xCD: MessagePack(2, ">H"),
0xCE: MessagePack(4, ">I"),
0xCF: MessagePack(8, ">Q"),
0xD0: MessagePack(1, "b"),
0xD1: MessagePack(2, ">h"),
0xD2: MessagePack(4, ">i"),
0xD3: MessagePack(8, ">q"),
0xD4: MessagePack(1, "b1s", TYPE_EXT),
0xD5: MessagePack(2, "b2s", TYPE_EXT),
0xD6: MessagePack(4, "b4s", TYPE_EXT),
0xD7: MessagePack(8, "b8s", TYPE_EXT),
0xD8: MessagePack(16, "b16s", TYPE_EXT),
0xD9: MessagePack(1, type=TYPE_RAW),
0xDA: MessagePack(2, ">H", TYPE_RAW),
0xDB: MessagePack(4, ">I", TYPE_RAW),
0xDC: MessagePack(2, ">H", TYPE_ARRAY),
0xDD: MessagePack(4, ">I", TYPE_ARRAY),
0xDE: MessagePack(2, ">H", TYPE_MAP),
0xDF: MessagePack(4, ">I", TYPE_MAP),
}


class Unpacker(object):
"""Streaming unpacker.

Expand Down Expand Up @@ -409,7 +445,7 @@ def _reserve(self, n, raise_outofdata=True):
self._buff_i = 0 # rollback
raise OutOfData

def _read_header(self, execute=EX_CONSTRUCT):
def _read_header(self):
typ = TYPE_IMMEDIATE
n = 0
obj = None
Expand Down Expand Up @@ -438,191 +474,82 @@ def _read_header(self, execute=EX_CONSTRUCT):
raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
elif b == 0xC0:
obj = None
elif b == 0xC2:
obj = False
elif b == 0xC3:
obj = True
elif b == 0xC4:
typ = TYPE_BIN
self._reserve(1)
n = self._buffer[self._buff_i]
self._buff_i += 1
if n > self._max_bin_len:
raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
obj = self._read(n)
elif b == 0xC5:
typ = TYPE_BIN
self._reserve(2)
n = _unpack_from(">H", self._buffer, self._buff_i)[0]
self._buff_i += 2
if n > self._max_bin_len:
raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
obj = self._read(n)
elif b == 0xC6:
typ = TYPE_BIN
self._reserve(4)
n = _unpack_from(">I", self._buffer, self._buff_i)[0]
self._buff_i += 4
elif 0xC2 <= b <= 0xC3:
obj = b == 0xC3
elif 0xC4 <= b <= 0xC6:
message_pack = MESSAGE_PACK_DICT[b]
typ = message_pack.type
self._reserve(message_pack.size)
if len(message_pack.format) > 0:
n = _unpack_from(message_pack.format, self._buffer, self._buff_i)[0]
else:
n = self._buffer[self._buff_i]
self._buff_i += message_pack.size
if n > self._max_bin_len:
raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
obj = self._read(n)
elif b == 0xC7: # ext 8
typ = TYPE_EXT
self._reserve(2)
L, n = _unpack_from("Bb", self._buffer, self._buff_i)
self._buff_i += 2
elif 0xC7 <= b <= 0xC9:
message_pack = MESSAGE_PACK_DICT[b]
typ = message_pack.type
self._reserve(message_pack.size)
L, n = _unpack_from(message_pack.format, self._buffer, self._buff_i)
self._buff_i += message_pack.size
if L > self._max_ext_len:
raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
obj = self._read(L)
elif b == 0xC8: # ext 16
typ = TYPE_EXT
self._reserve(3)
L, n = _unpack_from(">Hb", self._buffer, self._buff_i)
self._buff_i += 3
if L > self._max_ext_len:
raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
obj = self._read(L)
elif b == 0xC9: # ext 32
typ = TYPE_EXT
self._reserve(5)
L, n = _unpack_from(">Ib", self._buffer, self._buff_i)
self._buff_i += 5
if L > self._max_ext_len:
raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
obj = self._read(L)
elif b == 0xCA:
self._reserve(4)
obj = _unpack_from(">f", self._buffer, self._buff_i)[0]
self._buff_i += 4
elif b == 0xCB:
self._reserve(8)
obj = _unpack_from(">d", self._buffer, self._buff_i)[0]
self._buff_i += 8
elif b == 0xCC:
self._reserve(1)
obj = self._buffer[self._buff_i]
self._buff_i += 1
elif b == 0xCD:
self._reserve(2)
obj = _unpack_from(">H", self._buffer, self._buff_i)[0]
self._buff_i += 2
elif b == 0xCE:
self._reserve(4)
obj = _unpack_from(">I", self._buffer, self._buff_i)[0]
self._buff_i += 4
elif b == 0xCF:
self._reserve(8)
obj = _unpack_from(">Q", self._buffer, self._buff_i)[0]
self._buff_i += 8
elif b == 0xD0:
self._reserve(1)
obj = _unpack_from("b", self._buffer, self._buff_i)[0]
self._buff_i += 1
elif b == 0xD1:
self._reserve(2)
obj = _unpack_from(">h", self._buffer, self._buff_i)[0]
self._buff_i += 2
elif b == 0xD2:
self._reserve(4)
obj = _unpack_from(">i", self._buffer, self._buff_i)[0]
self._buff_i += 4
elif b == 0xD3:
self._reserve(8)
obj = _unpack_from(">q", self._buffer, self._buff_i)[0]
self._buff_i += 8
elif b == 0xD4: # fixext 1
typ = TYPE_EXT
if self._max_ext_len < 1:
raise ValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len))
self._reserve(2)
n, obj = _unpack_from("b1s", self._buffer, self._buff_i)
self._buff_i += 2
elif b == 0xD5: # fixext 2
typ = TYPE_EXT
if self._max_ext_len < 2:
raise ValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len))
self._reserve(3)
n, obj = _unpack_from("b2s", self._buffer, self._buff_i)
self._buff_i += 3
elif b == 0xD6: # fixext 4
typ = TYPE_EXT
if self._max_ext_len < 4:
raise ValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len))
self._reserve(5)
n, obj = _unpack_from("b4s", self._buffer, self._buff_i)
self._buff_i += 5
elif b == 0xD7: # fixext 8
typ = TYPE_EXT
if self._max_ext_len < 8:
raise ValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len))
self._reserve(9)
n, obj = _unpack_from("b8s", self._buffer, self._buff_i)
self._buff_i += 9
elif b == 0xD8: # fixext 16
typ = TYPE_EXT
if self._max_ext_len < 16:
raise ValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len))
self._reserve(17)
n, obj = _unpack_from("b16s", self._buffer, self._buff_i)
self._buff_i += 17
elif b == 0xD9:
typ = TYPE_RAW
self._reserve(1)
n = self._buffer[self._buff_i]
self._buff_i += 1
if n > self._max_str_len:
raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
obj = self._read(n)
elif b == 0xDA:
typ = TYPE_RAW
self._reserve(2)
(n,) = _unpack_from(">H", self._buffer, self._buff_i)
self._buff_i += 2
if n > self._max_str_len:
raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
obj = self._read(n)
elif b == 0xDB:
typ = TYPE_RAW
self._reserve(4)
(n,) = _unpack_from(">I", self._buffer, self._buff_i)
self._buff_i += 4
elif 0xCA <= b <= 0xD3:
message_pack = MESSAGE_PACK_DICT[b]
self._reserve(message_pack.size)
if len(message_pack.format) > 0:
obj = _unpack_from(message_pack.format, self._buffer, self._buff_i)[0]
else:
obj = self._buffer[self._buff_i]
self._buff_i += message_pack.size
elif 0xD4 <= b <= 0xD8:
message_pack = MESSAGE_PACK_DICT[b]
typ = message_pack.type
if self._max_ext_len < message_pack.size:
raise ValueError(
"%s exceeds max_ext_len(%s)"
% (message_pack.size, self._max_ext_len)
)
self._reserve(message_pack.size + 1)
n, obj = _unpack_from(message_pack.format, self._buffer, self._buff_i)
self._buff_i += message_pack.size + 1
elif 0xD9 <= b <= 0xDB:
message_pack = MESSAGE_PACK_DICT[b]
typ = message_pack.type
self._reserve(message_pack.size)
if len(message_pack.format) > 0:
(n,) = _unpack_from(message_pack.format, self._buffer, self._buff_i)
else:
n = self._buffer[self._buff_i]
self._buff_i += message_pack.size
if n > self._max_str_len:
raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
obj = self._read(n)
elif b == 0xDC:
typ = TYPE_ARRAY
self._reserve(2)
(n,) = _unpack_from(">H", self._buffer, self._buff_i)
self._buff_i += 2
if n > self._max_array_len:
raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
elif b == 0xDD:
typ = TYPE_ARRAY
self._reserve(4)
(n,) = _unpack_from(">I", self._buffer, self._buff_i)
self._buff_i += 4
elif 0xDC <= b <= 0xDD:
message_pack = MESSAGE_PACK_DICT[b]
typ = message_pack.type
self._reserve(message_pack.size)
(n,) = _unpack_from(message_pack.format, self._buffer, self._buff_i)
self._buff_i += message_pack.size
if n > self._max_array_len:
raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
elif b == 0xDE:
self._reserve(2)
(n,) = _unpack_from(">H", self._buffer, self._buff_i)
self._buff_i += 2
elif 0xDE <= b <= 0xDF:
message_pack = MESSAGE_PACK_DICT[b]
self._reserve(message_pack.size)
(n,) = _unpack_from(message_pack.format, self._buffer, self._buff_i)
self._buff_i += message_pack.size
if n > self._max_map_len:
raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
typ = TYPE_MAP
elif b == 0xDF:
self._reserve(4)
(n,) = _unpack_from(">I", self._buffer, self._buff_i)
self._buff_i += 4
if n > self._max_map_len:
raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
typ = TYPE_MAP
typ = message_pack.type
else:
raise FormatError("Unknown header: 0x%x" % b)
return typ, n, obj

def _unpack(self, execute=EX_CONSTRUCT):
typ, n, obj = self._read_header(execute)
typ, n, obj = self._read_header()

if execute == EX_READ_ARRAY_HEADER:
if typ != TYPE_ARRAY:
Expand Down