diff --git a/runtime/pb/wire.lua b/runtime/pb/wire.lua index 9482ba76201a18749d88c6479777acb04ffec593..6262045a766e3063d3e08a1b790d7b20326b4fc3 100644 --- a/runtime/pb/wire.lua +++ b/runtime/pb/wire.lua @@ -97,7 +97,22 @@ return encode_varint(field_id * 8 + wire_type) end M.encode_tag = encode_tag +-- The 1-byte varint fast path is duplicated at every hot decode call +-- site (decode_tag, decode_len, decode_int32 / int64 / uint32 / uint64 / +-- sint32 / sint64 / bool, skip_field VARINT branch) instead of being +-- factored into a helper. Reason: LuaJIT 2.1 inlines a small called +-- function into the caller's trace, so when the parent guard exits to +-- a side trace for the multi-byte case, the side trace has to return +-- from the inlined frame — and LuaJIT can't stitch that return back +-- to the parent, dropping to interpreter dispatch. Inlining the fast +-- path literally keeps the side trace inside the caller's own frame, +-- where stitching works. local function decode_tag(buf, pos) + local b = buf:byte(pos) + if b == nil then error("truncated varint at offset " .. pos, 0) end + if b < 0x80 then + return bit.rshift(b, 3), bit.band(b, 7), pos + 1 + end local v, npos = decode_varint(buf, pos) v = tonumber(v) return bit.rshift(v, 3), bit.band(v, 7), npos @@ -242,8 +257,16 @@ M.encode_len = encode_len -- decode_len(buf, pos) -> string, new_pos local function decode_len(buf, pos) - local len, npos = decode_varint(buf, pos) - len = tonumber(len) + local b = buf:byte(pos) + if b == nil then error("truncated varint at offset " .. pos, 0) end + local len, npos + if b < 0x80 then + len = b; npos = pos + 1 + else + local v + v, npos = decode_varint(buf, pos) + len = tonumber(v) + end if npos + len - 1 > #buf then error("truncated LEN payload", 0) end return buf:sub(npos, npos + len - 1), npos + len end @@ -277,33 +300,58 @@ M.encode_bytes = encode_len -- (encode_fixed32, encode_fixed64, encode_float, encode_double already on M) -- Decoders -------------------------------------------------------------------- +-- The 1-byte fast path is inlined at every varint-based scalar decoder +-- (see comment above decode_tag). Each decoder reads the first byte, +-- handles the common 0..127 case in straight-line code, and falls +-- through to decode_varint only for multi-byte values. local function decode_int32(buf, pos) + local b = buf:byte(pos) + if b == nil then error("truncated varint at offset " .. pos, 0) end + if b < 0x80 then return b, pos + 1 end -- 0..127 fits int32 directly local u, np = decode_varint(buf, pos) return tonumber(INT64(u)), np -- truncated to int32 range via int64 sign-extension end local function decode_int64(buf, pos) + local b = buf:byte(pos) + if b == nil then error("truncated varint at offset " .. pos, 0) end + if b < 0x80 then return INT64(b), pos + 1 end local u, np = decode_varint(buf, pos) return INT64(u), np end local function decode_uint32(buf, pos) + local b = buf:byte(pos) + if b == nil then error("truncated varint at offset " .. pos, 0) end + if b < 0x80 then return b, pos + 1 end local u, np = decode_varint(buf, pos) local n = tonumber(UINT64(u)) if n < 0 then n = n + 0x100000000 end return n, np end local function decode_uint64(buf, pos) + local b = buf:byte(pos) + if b == nil then error("truncated varint at offset " .. pos, 0) end + if b < 0x80 then return UINT64(b), pos + 1 end local u, np = decode_varint(buf, pos) return UINT64(u), np end local function decode_sint32(buf, pos) + local b = buf:byte(pos) + if b == nil then error("truncated varint at offset " .. pos, 0) end + if b < 0x80 then return zigzag_decode32(b), pos + 1 end local u, np = decode_varint(buf, pos) return zigzag_decode32(tonumber(u)), np end local function decode_sint64(buf, pos) + local b = buf:byte(pos) + if b == nil then error("truncated varint at offset " .. pos, 0) end + if b < 0x80 then return zigzag_decode64(b), pos + 1 end local u, np = decode_varint(buf, pos) return zigzag_decode64(u), np end local function decode_bool(buf, pos) + local b = buf:byte(pos) + if b == nil then error("truncated varint at offset " .. pos, 0) end + if b < 0x80 then return b ~= 0, pos + 1 end local u, np = decode_varint(buf, pos) return u ~= UINT64_ZERO, np end @@ -363,11 +411,17 @@ -- skip_field(buf, pos, wire_type) -> new_pos -- --------------------------------------------------------------------------- local function skip_field(buf, pos, wire_type) if wire_type == M.WIRE_VARINT then + local b = buf:byte(pos) + if b == nil then error("truncated varint at offset " .. pos, 0) end + if b < 0x80 then return pos + 1 end local _, npos = decode_varint(buf, pos) return npos elseif wire_type == M.WIRE_I64 then return pos + 8 elseif wire_type == M.WIRE_LEN then + local b = buf:byte(pos) + if b == nil then error("truncated varint at offset " .. pos, 0) end + if b < 0x80 then return pos + 1 + b end local len, npos = decode_varint(buf, pos) return npos + tonumber(len) elseif wire_type == M.WIRE_I32 then