diff --git a/runtime/pb/wire.lua b/runtime/pb/wire.lua index dda47bfc60616d091d075c4249b2f9da43f663ad..20dbd4241ae896e11520c3ff99850855311b8a4a 100644 --- a/runtime/pb/wire.lua +++ b/runtime/pb/wire.lua @@ -419,8 +419,62 @@ M.decode_sint64 = decode_sint64 M.decode_bool = decode_bool M.decode_sfixed32 = decode_sfixed32 M.decode_sfixed64 = decode_sfixed64 -M.decode_string = decode_len -M.decode_bytes = decode_len +-- RFC 3629 UTF-8 validator. Rejects: out-of-range continuation bytes, +-- truncated multi-byte sequences, overlong encodings, UTF-16 surrogate +-- code points (U+D800..U+DFFF), and code points above U+10FFFF. +local function is_valid_utf8(s) + local i, n = 1, #s + while i <= n do + local b = s:byte(i) + if b < 0x80 then + i = i + 1 + elseif b < 0xC2 then + return false -- stray continuation or overlong 2-byte + elseif b < 0xE0 then + if i + 1 > n then return false end + local b2 = s:byte(i + 1) + if b2 < 0x80 or b2 > 0xBF then return false end + i = i + 2 + elseif b < 0xF0 then + if i + 2 > n then return false end + local b2 = s:byte(i + 1) + local b3 = s:byte(i + 2) + if b == 0xE0 and b2 < 0xA0 then return false end -- overlong + if b == 0xED and b2 > 0x9F then return false end -- surrogate + if b2 < 0x80 or b2 > 0xBF or b3 < 0x80 or b3 > 0xBF then + return false + end + i = i + 3 + elseif b < 0xF5 then + if i + 3 > n then return false end + local b2 = s:byte(i + 1) + local b3 = s:byte(i + 2) + local b4 = s:byte(i + 3) + if b == 0xF0 and b2 < 0x90 then return false end -- overlong + if b == 0xF4 and b2 > 0x8F then return false end -- > U+10FFFF + if b2 < 0x80 or b2 > 0xBF + or b3 < 0x80 or b3 > 0xBF + or b4 < 0x80 or b4 > 0xBF then + return false + end + i = i + 4 + else + return false -- 5-byte+ sequences or 0xF5..0xFF + end + end + return true +end +M.is_valid_utf8 = is_valid_utf8 + +local function decode_string(buf, pos) + local s, np = decode_len(buf, pos) + if not is_valid_utf8(s) then + error("invalid UTF-8 in string field at offset " .. pos, 0) + end + return s, np +end +M.decode_string = decode_string +M.decode_bytes = decode_len -- (decode_fixed32, decode_fixed64, decode_float, decode_double already on M -- and have the right semantics for their proto types: fixed32 -> uint32 Lua -- number 0..2^32-1, fixed64 -> uint64 cdata.) diff --git a/test/conformance/known_failures.txt b/test/conformance/known_failures.txt index a1294cb1459836241ad4b229b9e8fe2e1fcf2cbe..229c94eee22d78cfebe868a2e41bf058a72291e8 100644 --- a/test/conformance/known_failures.txt +++ b/test/conformance/known_failures.txt @@ -20,11 +20,6 @@ # # Re-generate this file after fixes via `just conformance-refresh-failures`. Recommended.Proto3.JsonInput.FieldNameWithDoubleUnderscores.ProtobufOutput Recommended.Proto3.JsonInput.NullValueInOtherOneofOldFormat.Validator -Recommended.Proto3.ProtobufInput.RejectInvalidUtf8.String.MapKey -Recommended.Proto3.ProtobufInput.RejectInvalidUtf8.String.MapValue -Recommended.Proto3.ProtobufInput.RejectInvalidUtf8.String.Oneof -Recommended.Proto3.ProtobufInput.RejectInvalidUtf8.String.Repeated -Recommended.Proto3.ProtobufInput.RejectInvalidUtf8.String.Singular Required.Proto3.JsonInput.AllFieldAcceptNull.JsonOutput Required.Proto3.JsonInput.AllFieldAcceptNull.ProtobufOutput Required.Proto3.JsonInput.AnyNested.JsonOutput diff --git a/test/conformance_test.lua b/test/conformance_test.lua index 34f862d2301769af0917658266a1b222768ec062..4859cdd8676fd7b01ef5b1a5b75b9719b81e7283 100644 --- a/test/conformance_test.lua +++ b/test/conformance_test.lua @@ -580,6 +580,60 @@ local input = '\x88\x80\x80\x80\x00\x01' t.assert_not_equals(pb_roundtrip(input).parse_error, nil) end +-- ========================================================================= +-- Fix 8: UTF-8 validation on proto3 string fields. Singular, repeated, +-- oneof, map key, and map value all share decode_string in wire.lua, so +-- pinning a few distinct shapes is enough. +-- ========================================================================= + +core_g.test_invalid_utf8_singular_string_rejected = function() + -- field 14 = optional_string. tag(14, LEN) = 0x72. One-byte payload + -- 0xff is an isolated start of a 5-byte UTF-8 sequence (illegal). + t.assert_not_equals(pb_roundtrip('\x72\x01\xff').parse_error, nil) +end + +core_g.test_invalid_utf8_repeated_string_rejected = function() + -- field 44 = repeated_string. Encode three entries, middle invalid. + -- tag(44, LEN) = (44<<3)|2 = 354 → varint 0xe2 0x02. + local tag = '\xe2\x02' + local input = tag .. '\x02ok' .. tag .. '\x01\xff' .. tag .. '\x02ok' + t.assert_not_equals(pb_roundtrip(input).parse_error, nil) +end + +core_g.test_invalid_utf8_oneof_string_rejected = function() + -- oneof_string id 113, tag (113<<3)|2 = 906 → varint 0x8a 0x07. + t.assert_not_equals(pb_roundtrip('\x8a\x07\x01\xff').parse_error, nil) +end + +core_g.test_invalid_utf8_lone_surrogate_rejected = function() + -- 0xED 0xA0 0x80 is U+D800 (a UTF-16 surrogate), invalid as a code + -- point. Pin the surrogate-range branch of is_valid_utf8. + t.assert_not_equals(pb_roundtrip('\x72\x03\xed\xa0\x80').parse_error, nil) +end + +core_g.test_invalid_utf8_above_max_codepoint_rejected = function() + -- 0xF4 0x90 0x80 0x80 is U+110000, one past Unicode's max. Pin the + -- > U+10FFFF branch. + local input = '\x72\x04\xf4\x90\x80\x80' + t.assert_not_equals(pb_roundtrip(input).parse_error, nil) +end + +core_g.test_valid_utf8_singular_string_round_trips = function() + -- Positive control: a multi-byte UTF-8 string survives unchanged. + local s = '\xe2\x9c\x85' -- ✅ U+2705 + local input = '\x72' .. string.char(#s) .. s + local resp = pb_roundtrip(input) + t.assert_not(resp.parse_error, resp.parse_error) + t.assert_equals(resp.protobuf_payload, input) +end + +core_g.test_bytes_field_accepts_arbitrary_bytes = function() + -- bytes (not string) must NOT validate UTF-8. field 15 = optional_bytes. + -- tag(15, LEN) = 0x7a. + local resp = pb_roundtrip('\x7a\x03\xff\xfe\xfd') + t.assert_not(resp.parse_error, resp.parse_error) +end + core_g.test_oneof_merge_still_clears_sibling_branches = function() -- The post-fix merge code must still clear oneof siblings: setting -- oneof_uint32 first, then merging two oneof_nested_message entries,