diff --git a/runtime/pb/json.lua b/runtime/pb/json.lua index 7f9618eb884624bedd27c48edd7b1569ba72e942..bff5f2aa73c63941146a7d89b0e0bce0abfb7642 100644 --- a/runtime/pb/json.lua +++ b/runtime/pb/json.lua @@ -10,13 +10,19 @@ -- * bytes -> base64 string -- * enum -> string name when known, else number -- * message -> JSON object (camelCase keys) -- * map -> JSON object (keys stringified per spec) --- * Timestamp -> ISO 8601 "YYYY-MM-DDTHH:MM:SS[.nnnnnnnnn]Z" --- * Duration -> "s" (decimal seconds with up to 9 fractional digits) +-- * Timestamp -> RFC 3339 "YYYY-MM-DDTHH:MM:SS[.fff]Z" +-- * Duration -> "[.]s" (decimal seconds with 0/3/6/9 frac digits) -- * Empty -> {} -- * Wrapper messages -> unwrapped scalar +-- * Any -> {"@type": "", ...} for user types; nested under "value" for WKTs. -- --- Field names: emitted camelCase per spec; decoder accepts both camelCase --- and the original snake_case so users aren't punished for either convention. +-- Field names: emitted lowerCamelCase per spec; decoder accepts both +-- lowerCamelCase and the original snake_case. +-- +-- JSON output uses a hand-rolled encoder so the double formatter can pick +-- the shortest round-tripping representation (Tarantool's json.encode is +-- locked to a global precision that wouldn't satisfy the conformance +-- suite). JSON input still goes through Tarantool's json.decode. local ffi = require('ffi') local json = require('json') local digest = require('digest') @@ -26,68 +32,550 @@ local pbwkt = require('pb.wkt') local M = {} -local TYPE_INFO = wire.TYPE_INFO -local INT64_FAMILY = {int64=true, uint64=true, sint64=true, fixed64=true, sfixed64=true} +local INT64_FAMILY = {int64=true, uint64=true, sint64=true, + fixed64=true, sfixed64=true} + +local INT64_T = ffi.typeof('int64_t') +local UINT64_T = ffi.typeof('uint64_t') + +local PB_NULL = pbwkt.NULL -- --------------------------------------------------------------------------- --- Helpers +-- WKT classification (used by Any encoding and elsewhere). +-- WKTs that have a non-object JSON representation must be nested under a +-- "value" key when wrapped in google.protobuf.Any. -- --------------------------------------------------------------------------- +local WKT_NAMES = { + ['google.protobuf.Timestamp'] = true, + ['google.protobuf.Duration'] = true, + ['google.protobuf.FieldMask'] = true, + ['google.protobuf.Any'] = true, + ['google.protobuf.Struct'] = true, + ['google.protobuf.ListValue'] = true, + ['google.protobuf.Value'] = true, + ['google.protobuf.Empty'] = true, +} +local function is_wkt_name(name) + return WKT_NAMES[name] or (name:match('^google%.protobuf%.%w+Value$') ~= nil) +end --- snake_case -> lowerCamelCase per the proto3 JSON spec. Multi-underscore --- runs collapse to one capitalized letter; trailing underscores drop. A --- leading underscore causes the next letter to be capitalized so the JSON --- name has no leading underscore (e.g. `_field_name3` → `FieldName3`). +-- --------------------------------------------------------------------------- +-- Field name conversion (snake_case ↔ lowerCamelCase per spec). +-- Multi-underscore runs collapse to a single capitalized letter; trailing +-- underscores drop. A leading underscore capitalizes the next letter so the +-- JSON name has no leading underscore. +-- --------------------------------------------------------------------------- local function to_camel(name) name = name:gsub('_+$', '') return (name:gsub('_+(%w)', function(c) return c:upper() end)) end --- Lossless stringification of an integer/cdata for JSON output. -local function int_to_string(v) +-- --------------------------------------------------------------------------- +-- Number/integer parsing +-- --------------------------------------------------------------------------- + +-- Strict integer-literal validation. Returns true if s matches +-- /^-?[0-9]+$/. JSON spec disallows leading + and whitespace. +local function is_int_string(s) + return type(s) == 'string' and s:match('^%-?%d+$') ~= nil +end + +-- Strict JSON-number-literal validation. Accepts: +-- -?[0-9]+(\.[0-9]+)?([eE][+-]?[0-9]+)? +-- -?\.[0-9]+([eE][+-]?[0-9]+)? +-- Plus the three sentinel strings ("NaN", "Infinity", "-Infinity"). +local function is_number_string(s) + if type(s) ~= 'string' or s == '' then return false end + if s == 'NaN' or s == 'Infinity' or s == '-Infinity' then return true end + local i, n = 1, #s + if s:sub(1, 1) == '-' then i = 2 end + if i > n then return false end + local saw_int = false + while i <= n and s:sub(i, i):match('%d') do saw_int = true; i = i + 1 end + local saw_frac = false + if i <= n and s:sub(i, i) == '.' then + i = i + 1 + while i <= n and s:sub(i, i):match('%d') do saw_frac = true; i = i + 1 end + end + if not (saw_int or saw_frac) then return false end + if i <= n then + local c = s:sub(i, i) + if c ~= 'e' and c ~= 'E' then return false end + i = i + 1 + if i <= n and (s:sub(i, i) == '+' or s:sub(i, i) == '-') then i = i + 1 end + local saw_exp = false + while i <= n and s:sub(i, i):match('%d') do saw_exp = true; i = i + 1 end + if not saw_exp then return false end + end + return i > n +end + +-- Compare two strings of digits as unsigned integers (longer = larger; +-- equal length = lexicographic). +local function digits_cmp(a, b) + -- Strip leading zeros for an apples-to-apples comparison. + a = a:gsub('^0+', ''); if a == '' then a = '0' end + b = b:gsub('^0+', ''); if b == '' then b = '0' end + if #a ~= #b then return #a < #b and -1 or 1 end + if a == b then return 0 end + return a < b and -1 or 1 +end + +-- Range checks for integer strings (no need to materialize 64-bit values). +local function int64_string_in_range(s) + local neg = s:sub(1, 1) == '-' + local digits = neg and s:sub(2) or s + local max = neg and '9223372036854775808' or '9223372036854775807' + return digits_cmp(digits, max) <= 0 +end +local function uint64_string_in_range(s) + if s:sub(1, 1) == '-' then + -- "-0" is acceptable (it's still zero); any other negative is not. + return s:match('^%-0+$') ~= nil + end + return digits_cmp(s, '18446744073709551615') <= 0 +end + +local function int_string_from_number(v) + -- Re-render a Lua number that is integer-valued. Used to fold a JSON + -- number literal into the string-based int validators. + if v ~= v or v == math.huge or v == -math.huge then return nil end + if v % 1 ~= 0 then return nil end + -- string.format('%.0f', ...) rounds; we want truncate-to-integer, but + -- since we already checked v % 1 == 0 the rounding is harmless. + if v >= 0 and v < 2^53 then return string.format('%.0f', v) end + if v < 0 and v > -2^53 then return string.format('%.0f', v) end + -- For magnitudes past 2^53, the Lua number can't represent v exactly. + -- The JSON number was parsed lossily; reject it rather than guess. + return nil +end + +-- Resolve a JSON-string integer field. Accepts strict-int literal +-- ("-?[0-9]+") OR a full JSON number string with integer value. The +-- second form covers conformance "Int32FieldQuotedExponentialValue" +-- ("1e5" → 100000). +local function int_string_from_string(s, typename) + if is_int_string(s) then return s end + if not is_number_string(s) then + error(typename .. ': invalid string "' .. s .. '"', 0) + end + local n = tonumber(s) + if n == nil or n ~= n or n == math.huge or n == -math.huge then + error(typename .. ': out of range "' .. s .. '"', 0) + end + if n % 1 ~= 0 then + error(typename .. ': non-integer value "' .. s .. '"', 0) + end + local canonical = int_string_from_number(n) + if canonical == nil then + error(typename .. ': out of representable range "' .. s .. '"', 0) + end + return canonical +end + +-- Decode a JSON value into a Lua number that fits in [INT32_MIN, INT32_MAX]. +local function decode_int32(v) + local s if type(v) == 'cdata' then - return tostring(v):gsub('U?LL$', '') + s = tostring(v):gsub('U?LL$', '') + elseif type(v) == 'number' then + s = int_string_from_number(v) + if s == nil then error('int32: not an integer-valued JSON number', 0) end + elseif type(v) == 'string' then + s = int_string_from_string(v, 'int32') + else + error('int32: expected JSON number/string, got ' .. type(v), 0) end - return tostring(v) + if not int64_string_in_range(s) then error('int32 out of range: ' .. s, 0) end + local n = tonumber(s) + if n < -2147483648 or n > 2147483647 then + error('int32 out of range: ' .. s, 0) + end + return n end --- Convert a JSON string/number/cdata back to int64 or uint64 cdata. --- Tarantool's json.decode parses out-of-double-range integer literals as --- int64_t / uint64_t cdata, so that branch is hit even when the proto3 --- JSON spec says 64-bit integers should be quoted. -local function string_to_int64(v, is_unsigned) +local function decode_uint32(v) + local s if type(v) == 'cdata' then - if is_unsigned then return ffi.cast('uint64_t', v) end - return ffi.cast('int64_t', v) + s = tostring(v):gsub('U?LL$', '') + elseif type(v) == 'number' then + s = int_string_from_number(v) + if s == nil then error('uint32: not an integer-valued JSON number', 0) end + elseif type(v) == 'string' then + s = int_string_from_string(v, 'uint32') + else + error('uint32: expected JSON number/string, got ' .. type(v), 0) + end + if s:sub(1, 1) == '-' and s:match('^%-0+$') == nil then + error('uint32 cannot be negative: ' .. s, 0) + end + if not uint64_string_in_range(s) then error('uint32 out of range: ' .. s, 0) end + local n = tonumber(s) + if n < 0 or n > 4294967295 then + error('uint32 out of range: ' .. s, 0) + end + return n +end + +local function decode_int64_value(v, is_unsigned) + local s + if type(v) == 'cdata' then + s = tostring(v):gsub('U?LL$', '') + elseif type(v) == 'number' then + s = int_string_from_number(v) + if s == nil then error('int64: not an integer-valued JSON number', 0) end + elseif type(v) == 'string' then + s = int_string_from_string(v, is_unsigned and 'uint64' or 'int64') + else + error('int64: expected JSON number/string, got ' .. type(v), 0) + end + if is_unsigned then + if s:sub(1, 1) == '-' and s:match('^%-0+$') == nil then + error('uint64 cannot be negative: ' .. s, 0) + end + if not uint64_string_in_range(s) then error('uint64 out of range: ' .. s, 0) end + local c = tonumber64(s) + if c == nil then error('uint64 parse failed: ' .. s, 0) end + return ffi.cast(UINT64_T, c) + end + if not int64_string_in_range(s) then error('int64 out of range: ' .. s, 0) end + local c = tonumber64(s) + if c == nil then error('int64 parse failed: ' .. s, 0) end + return ffi.cast(INT64_T, c) +end + +-- Float/double decode. NaN/Infinity sentinels and JSON numbers / numeric +-- strings both accepted; out-of-range strings produce a parse error. +local FLOAT_MAX = 3.4028234663852886e+38 +local FLOAT_MIN = -3.4028234663852886e+38 + +local function decode_float_value(v, is_float) + local n + local typename = is_float and 'float' or 'double' + if type(v) == 'number' then + -- JSON numeric literals that overflow double parse to inf — that's + -- a Too-Large/Too-Small input and must be rejected. + if v == math.huge or v == -math.huge then + error(typename .. ': value out of range (Infinity from JSON number)', 0) + end + n = v + elseif type(v) == 'cdata' then + n = tonumber(v) + elseif type(v) == 'string' then + if v == 'NaN' then return 0/0 end + if v == 'Infinity' then return math.huge end + if v == '-Infinity' then return -math.huge end + if not is_number_string(v) then + error(typename .. ': invalid string "' .. v .. '"', 0) + end + n = tonumber(v) + if n == nil then + error(typename .. ': unparseable "' .. v .. '"', 0) + end + if n == math.huge or n == -math.huge then + error(typename .. ': out of range "' .. v .. '"', 0) + end + else + error(typename .. ': expected JSON number/string, got ' .. type(v), 0) + end + if is_float and n == n and n ~= math.huge and n ~= -math.huge then + if n > FLOAT_MAX or n < FLOAT_MIN then + error('float out of range: ' .. tostring(n), 0) + end + end + return n +end + +-- --------------------------------------------------------------------------- +-- Timestamp / Duration helpers +-- --------------------------------------------------------------------------- + +local TS_MIN_SECONDS = -62135596800 -- 0001-01-01T00:00:00Z +local TS_MAX_SECONDS = 253402300799 -- 9999-12-31T23:59:59Z +local DUR_MAX_SECONDS = 315576000000 -- 10000 years, per spec + +-- Format a fractional-seconds string with 0/3/6/9 digits per spec. +local function fractional_seconds(nanos) + if nanos == 0 then return '' end + local frac = string.format('%09d', nanos) + if nanos % 1000000 == 0 then frac = frac:sub(1, 3) + elseif nanos % 1000 == 0 then frac = frac:sub(1, 6) + end + return '.' .. frac +end + +-- Convert UTC epoch seconds to (year, month, day, hour, minute, second). +-- Uses Howard Hinnant's date algorithm, which works for any integer epoch +-- without depending on the platform's gmtime — POSIX %Y formats year 1 as +-- "1" on glibc, which would fall over the conformance round-trip. +local function epoch_to_ymdhms(secs) + secs = math.floor(secs) + local days = math.floor(secs / 86400) + local tod = secs - days * 86400 + if tod < 0 then tod = tod + 86400; days = days - 1 end + days = days + 719468 + local era = math.floor(days / 146097) + local doe = days - era * 146097 + local yoe = math.floor((doe - math.floor(doe / 1460) + + math.floor(doe / 36524) + - math.floor(doe / 146096)) / 365) + local y = yoe + era * 400 + local doy = doe - (365 * yoe + math.floor(yoe / 4) + - math.floor(yoe / 100)) + local mp = math.floor((5 * doy + 2) / 153) + local d = doy - math.floor((153 * mp + 2) / 5) + 1 + local m = mp < 10 and mp + 3 or mp - 9 + if m <= 2 then y = y + 1 end + local h = math.floor(tod / 3600) + local mi = math.floor((tod - h * 3600) / 60) + local s = tod - h * 3600 - mi * 60 + return y, m, d, h, mi, s +end + +-- Parse an RFC 3339 timestamp. Strict: uppercase 'T' separator, either 'Z' +-- or '±HH:MM' offset, fraction (if present) up to 9 digits. +local function parse_timestamp(s) + if type(s) ~= 'string' then error('Timestamp: expected string', 0) end + local base = '%d%d%d%d%-%d%d%-%d%dT%d%d:%d%d:%d%d' + local body, frac, tz + body, frac = s:match('^(' .. base .. ')%.(%d+)Z$') + if body then tz = 'Z' end + if not body then + body, frac, tz = s:match('^(' .. base .. ')%.(%d+)([+%-]%d%d:%d%d)$') + end + if not body then + body = s:match('^(' .. base .. ')Z$') + if body then tz = 'Z'; frac = '' end + end + if not body then + body, tz = s:match('^(' .. base .. ')([+%-]%d%d:%d%d)$') + frac = frac or '' + end + if not body then error('Timestamp: invalid format "' .. s .. '"', 0) end + if #frac > 9 then + error('Timestamp: fraction has more than 9 digits', 0) + end + local y, mo, d, h, mi, sec = body:match( + '^(%d%d%d%d)%-(%d%d)%-(%d%d)T(%d%d):(%d%d):(%d%d)$') + y = tonumber(y); mo = tonumber(mo); d = tonumber(d) + h = tonumber(h); mi = tonumber(mi); sec = tonumber(sec) + if mo < 1 or mo > 12 or d < 1 or d > 31 then + error('Timestamp: invalid date "' .. s .. '"', 0) + end + if h > 23 or mi > 59 or sec > 59 then + error('Timestamp: invalid time "' .. s .. '"', 0) + end + local nanos = 0 + if frac ~= '' then nanos = tonumber((frac .. '000000000'):sub(1, 9)) end + local tz_minutes = 0 + if tz ~= 'Z' then + local sign, oh, om = tz:match('^([+%-])(%d%d):(%d%d)$') + tz_minutes = (sign == '+' and 1 or -1) * + (tonumber(oh) * 60 + tonumber(om)) + if tonumber(oh) > 23 or tonumber(om) > 59 then + error('Timestamp: invalid tz offset "' .. tz .. '"', 0) + end + end + local ok, dt = pcall(datetime.new, { + year = y, month = mo, day = d, + hour = h, min = mi, sec = sec, + nsec = nanos, tzoffset = tz_minutes, + }) + if not ok then error('Timestamp: ' .. tostring(dt), 0) end + local epoch_n = tonumber(dt.epoch) + if epoch_n < TS_MIN_SECONDS or epoch_n > TS_MAX_SECONDS then + error('Timestamp: out of range "' .. s .. '"', 0) + end + return dt +end + +-- Format a datetime cdata (or {seconds=,nanos=} table) as UTC RFC 3339. +local function format_timestamp(v) + local seconds_n, nanos + if type(v) == 'cdata' and datetime.is_datetime(v) then + seconds_n = tonumber(v.epoch) + nanos = v.nsec + elseif type(v) == 'table' then + local s = v.seconds or 0 + seconds_n = (type(s) == 'cdata') and tonumber(s) or s + nanos = v.nanos or 0 + elseif type(v) == 'number' then + seconds_n = math.floor(v) + nanos = math.floor((v - seconds_n) * 1e9 + 0.5) + else + error('Timestamp: expected datetime/table/number, got ' .. type(v), 0) + end + if nanos < 0 or nanos >= 1000000000 then + error('Timestamp: nanos out of range (' .. tostring(nanos) .. ')', 0) + end + if seconds_n < TS_MIN_SECONDS or seconds_n > TS_MAX_SECONDS then + error('Timestamp: seconds out of range (' .. tostring(seconds_n) .. ')', 0) + end + local y, mo, d, h, mi, s = epoch_to_ymdhms(seconds_n) + return string.format('%04d-%02d-%02dT%02d:%02d:%02d', y, mo, d, h, mi, s) + .. fractional_seconds(nanos) .. 'Z' +end + +local function parse_duration(s) + if type(s) ~= 'string' then error('Duration: expected string', 0) end + if s:sub(-1) ~= 's' then + error('Duration: missing "s" suffix in "' .. s .. '"', 0) + end + local body = s:sub(1, -2) + local neg = body:sub(1, 1) == '-' + if neg then body = body:sub(2) end + local sec_str, frac = body:match('^(%d+)%.(%d+)$') + if sec_str == nil then + sec_str = body:match('^(%d+)$') + frac = '' + end + if sec_str == nil then + error('Duration: invalid format "' .. s .. '"', 0) + end + if #frac > 9 then + error('Duration: fraction has more than 9 digits', 0) + end + local seconds = tonumber(sec_str) + local nanos = 0 + if frac ~= '' then nanos = tonumber((frac .. '000000000'):sub(1, 9)) end + if seconds > DUR_MAX_SECONDS then + error('Duration: out of range "' .. s .. '"', 0) end - if type(v) == 'number' then v = string.format('%.0f', v) end - if type(v) ~= 'string' then error('expected JSON string or number for int64', 0) end - local cdata = tonumber64(v) - if cdata == nil then error('invalid int64 string: ' .. v, 0) end - if is_unsigned then return ffi.cast('uint64_t', cdata) end - return ffi.cast('int64_t', cdata) + if neg then seconds = -seconds; nanos = -nanos end + return {seconds = ffi.cast(INT64_T, seconds), nanos = nanos} +end + +local function format_duration(v) + local seconds, nanos + if type(v) == 'table' then + seconds = v.seconds or 0 + nanos = v.nanos or 0 + elseif type(v) == 'number' then + seconds = math.floor(v) + nanos = math.floor((v - seconds) * 1e9 + 0.5) + else + error('Duration: expected table/number, got ' .. type(v), 0) + end + local seconds_n = (type(seconds) == 'cdata') and tonumber(seconds) or seconds + if nanos <= -1000000000 or nanos >= 1000000000 then + error('Duration: nanos out of range (' .. tostring(nanos) .. ')', 0) + end + if (seconds_n > 0 and nanos < 0) or (seconds_n < 0 and nanos > 0) then + error('Duration: seconds and nanos must have the same sign', 0) + end + if seconds_n < -DUR_MAX_SECONDS or seconds_n > DUR_MAX_SECONDS then + error('Duration: seconds out of range (' .. tostring(seconds_n) .. ')', 0) + end + local negative = seconds_n < 0 or nanos < 0 + local abs_s = math.abs(seconds_n) + local abs_n = math.abs(nanos) + local out = string.format('%d', abs_s) + if abs_n ~= 0 then out = out .. fractional_seconds(abs_n) end + return (negative and '-' or '') .. out .. 's' end -- --------------------------------------------------------------------------- --- Encode (proto-Lua table -> Lua table suitable for json.encode) +-- Hand-rolled JSON encoder (gives us shortest round-trip doubles) -- --------------------------------------------------------------------------- -local to_json_value -- forward -local encode_message -- forward +local ESCAPES = {} +for i = 0, 0x1f do ESCAPES[string.char(i)] = string.format('\\u%04x', i) end +ESCAPES['\b'] = '\\b'; ESCAPES['\f'] = '\\f' +ESCAPES['\n'] = '\\n'; ESCAPES['\r'] = '\\r'; ESCAPES['\t'] = '\\t' +ESCAPES['"'] = '\\"'; ESCAPES['\\'] = '\\\\' --- Encode a single scalar (not repeated/map). Returns a JSON-friendly value. +local function encode_json_string(s) + return '"' .. (s:gsub('[%z\1-\31"\\]', ESCAPES)) .. '"' +end + +local function encode_json_number(n) + if n ~= n then return '"NaN"' end + if n == math.huge then return '"Infinity"' end + if n == -math.huge then return '"-Infinity"' end + -- Integer-valued doubles within Lua's safe integer range: emit as int. + if n == math.floor(n) and math.abs(n) < 1e16 then + if n == 0 then return '0' end + return string.format('%d', n) + end + -- Shortest round-trip: try increasing precision until tonumber round-trips. + for p = 15, 17 do + local s = string.format('%.' .. p .. 'g', n) + if tonumber(s) == n then return s end + end + return string.format('%.17g', n) +end + +local encode_json -- forward + +local function is_array_table(t, mt) + if mt and mt.__serialize == 'seq' then return true end + if mt and mt.__serialize == 'map' then return false end + if mt and mt.__pb_kind == 'list' then return true end + if mt and mt.__pb_kind == 'struct' then return false end + if next(t) == nil then return false end -- empty defaults to object + return t[1] ~= nil +end + +local function encode_json_array(t) + local parts = {} + for i = 1, #t do parts[i] = encode_json(t[i]) end + return '[' .. table.concat(parts, ',') .. ']' +end + +local function encode_json_object(t) + local parts, n = {}, 0 + for k, v in pairs(t) do + n = n + 1 + parts[n] = encode_json_string(tostring(k)) .. ':' .. encode_json(v) + end + return '{' .. table.concat(parts, ',') .. '}' +end + +encode_json = function(v) + if v == nil then return 'null' end + local ty = type(v) + if ty == 'cdata' then + if v == box.NULL then return 'null' end + -- 64-bit ints from non-JSON sources are stringified by the proto + -- scalar encoder before we get here; any cdata that slips through + -- is rendered as an unquoted decimal (best-effort fallback). + return (tostring(v):gsub('U?LL$', '')) + end + if ty == 'boolean' then return v and 'true' or 'false' end + if ty == 'number' then return encode_json_number(v) end + if ty == 'string' then return encode_json_string(v) end + if ty == 'table' then + local mt = getmetatable(v) + if is_array_table(v, mt) then return encode_json_array(v) end + return encode_json_object(v) + end + error('JSON encode: unsupported type ' .. ty, 0) +end + +-- --------------------------------------------------------------------------- +-- Encode (proto-Lua table -> Lua structure suitable for our JSON emitter) +-- --------------------------------------------------------------------------- + +local to_json_value +local encode_message + +-- Marker so the JSON encoder emits a quoted JSON value verbatim. The +-- proto3 spec mandates 64-bit ints as JSON strings; this keeps the +-- formatter simple while letting the scalar encoder produce ready-made +-- string output. local function encode_scalar(proto_type, v) - if INT64_FAMILY[proto_type] then return int_to_string(v) end - if proto_type == 'uint32' then - -- Lua double can hold 0..2^32 - 1; emit as number. - return v + if INT64_FAMILY[proto_type] then + -- Always emit as JSON string per spec. + if type(v) == 'cdata' then + return (tostring(v):gsub('U?LL$', '')) + end + if type(v) == 'number' then return string.format('%.0f', v) end + return tostring(v) end + if proto_type == 'uint32' then return v end if proto_type == 'bytes' then return digest.base64_encode(v) end if proto_type == 'float' or proto_type == 'double' then - -- JSON has no NaN/Infinity literals; spec mandates string sentinels. - if v ~= v then return 'NaN' end - if v == math.huge then return 'Infinity' end - if v == -math.huge then return '-Infinity' end - return v + return v -- handled by encode_json_number end return v -- string, bool, int32, sint32, fixed32, sfixed32 end @@ -98,49 +586,46 @@ local name = enum_desc.by_value[v] return name or v -- unknown numeric value: emit as number end --- Convert a single proto value to a JSON-encodable value, based on field kind. local function encode_field_value(field, v) local kind = field.kind - if kind == 'scalar' then - return encode_scalar(field.proto_type, v) - elseif kind == 'enum' then - return encode_enum(field.enum, v) - elseif kind == 'message' then - return encode_message(field.message, v) - end + if kind == 'scalar' then return encode_scalar(field.proto_type, v) end + if kind == 'enum' then return encode_enum(field.enum, v) end + if kind == 'message' then return encode_message(field.message, v) end error('encode_field_value: unknown kind ' .. tostring(kind), 0) end --- Map key encoding: per spec, all keys are strings in JSON output. local function encode_map_key(key_field, k) local pt = key_field.proto_type if pt == 'bool' then return k and 'true' or 'false' end - if INT64_FAMILY[pt] then return int_to_string(k) end + if INT64_FAMILY[pt] then + if type(k) == 'cdata' then return (tostring(k):gsub('U?LL$', '')) end + return tostring(k) + end if pt == 'string' then return k end - return tostring(k) -- int32/uint32/sint32/etc. + return tostring(k) end --- Struct/Value/ListValue JSON mappings. --- --- Per the proto3 JSON spec: --- * Struct ↔ JSON object (the `fields` map is hoisted away) --- * ListValue ↔ JSON array (the `values` array is hoisted away) --- * Value ↔ any JSON value (null/number/string/bool/object/array) --- --- Lua representations mirror what runtime/pb/wkt.lua produces: --- * box.NULL → JSON null --- * boolean/number/string → JSON true|false/number/string --- * table tagged with pb.wkt.list → JSON array; otherwise → JSON object - -local PB_NULL = pbwkt.NULL - -local value_to_json, value_to_json_struct, value_to_json_list -- forwards +local value_to_json, value_to_json_struct, value_to_json_list value_to_json = function(v) if v == nil or v == PB_NULL then return box.NULL end local ty = type(v) - if ty == 'boolean' or ty == 'number' or ty == 'string' then return v end - if ty == 'cdata' then return tonumber(v) end + if ty == 'number' then + -- google.protobuf.Value's number_value is a double, but JSON has no + -- NaN/Infinity literals so the spec forbids them here. JSON output + -- must fail rather than emit a "NaN"/"Infinity" string (which would + -- be silently treated as the string_value branch by readers). + if v ~= v then error('Value JSON: NaN is not a valid number_value', 0) end + if v == math.huge or v == -math.huge then + error('Value JSON: Infinity is not a valid number_value', 0) + end + return v + end + if ty == 'boolean' or ty == 'string' then return v end + if ty == 'cdata' then + if v == box.NULL then return box.NULL end + return tonumber(v) + end if ty == 'table' then local mt = getmetatable(v) if mt and mt.__pb_kind == 'list' then return value_to_json_list(v) end @@ -169,7 +654,6 @@ for i = 1, #t do out[i] = value_to_json(t[i]) end return setmetatable(out, {__serialize='seq'}) end --- FieldMask JSON: paths joined by `,`. snake_case → lowerCamelCase per spec. local function fieldmask_to_json(v) if v == nil or #v == 0 then return '' end local parts = {} @@ -181,15 +665,13 @@ local function fieldmask_from_json(s) if type(s) ~= 'string' or s == '' then return {} end local out = {} for part in (s .. ','):gmatch('([^,]+),') do - -- lowerCamelCase -> snake_case out[#out + 1] = (part:gsub('(%u)', function(c) return '_' .. c:lower() end)) end return out end --- Any JSON: a flat object {"@type": "", ...fields...}. Decoded by --- consulting pb.wkt registry. Pack/unpack the payload through the registered --- descriptor; unregistered types fall back to the opaque {type_url,value} form. +-- Encode Any to a Lua structure ready for the JSON emitter. Returns a +-- table with the `@type`+`value` shape per spec. local function any_to_json(v) if v == nil then return setmetatable({}, {__serialize='map'}) end if type(v) ~= 'table' then @@ -197,106 +679,78 @@ error('Any JSON: expected table, got ' .. type(v), 0) end local type_url = v.type_url or '' local bytes = v.value or '' - local desc = pbwkt.lookup(type_url) - if desc == nil or bytes == '' then - -- Opaque fallback: emit the protobuf representation as-is so a round - -- trip is still possible without a registered descriptor. - local obj = {['@type'] = type_url} + if type_url == '' and bytes == '' then + return setmetatable({}, {__serialize='map'}) + end + local desc = pbwkt.lookup(type_url) + if desc == nil then + -- Unknown type: opaque pass-through with base64-encoded value (this + -- keeps round-trips through user-registered types stable without + -- forcing every embedded payload into a Wkt shape). + local obj = {} + if type_url ~= '' then obj['@type'] = type_url end if bytes ~= '' then obj.value = digest.base64_encode(bytes) end return obj end local inner = desc.decode and desc.decode(bytes) or require('pb.codec').decode(desc, bytes) local payload = encode_message(desc, inner) - -- For Value/Struct/ListValue/wrappers the JSON form is not an object; the - -- spec says to nest under "value" then. - if type(payload) ~= 'table' or getmetatable(payload) and - getmetatable(payload).__serialize == 'seq' then + if is_wkt_name(desc.name) then + -- Empty's JSON form is {} and the reference implementation rejects + -- the explicit {"value": {}} shape inside Any, so emit only @type. + if desc.name == 'google.protobuf.Empty' then + return {['@type'] = type_url} + end + return {['@type'] = type_url, value = payload} + end + -- User-type Any: flatten the message fields next to "@type". + if type(payload) ~= 'table' then return {['@type'] = type_url, value = payload} end payload['@type'] = type_url return payload end --- WKT special-case encoders. Return either a JSON-encodable Lua value or --- nil to indicate "no override; fall back to generic message walk". +-- WKT special-case encoders. Return the value to emit in place of the +-- generic message walk, or nil to fall back. local function encode_wkt(desc, v) local name = desc.name if name == 'google.protobuf.Empty' then - return setmetatable({}, {__serialize='map'}) -- emit as {} + return setmetatable({}, {__serialize='map'}) end - if name == 'google.protobuf.Timestamp' then - local dt = v - if type(v) == 'table' then - dt = datetime.new({timestamp = tonumber(v.seconds or 0), - nsec = v.nanos or 0}) - elseif type(v) == 'cdata' and datetime.is_datetime(v) then - dt = v - elseif type(v) == 'number' then - dt = datetime.new({timestamp = math.floor(v), - nsec = math.floor((v - math.floor(v)) * 1e9 + 0.5)}) - else - error('Timestamp JSON: expected datetime/table/number', 0) - end - return tostring(dt):gsub('Z$', 'Z') -- Tarantool already emits ISO 8601 - end - if name == 'google.protobuf.Duration' then - local seconds, nanos - if type(v) == 'table' then - seconds = tonumber(v.seconds or 0) - nanos = v.nanos or 0 - elseif type(v) == 'number' then - seconds = math.floor(v) - nanos = math.floor((v - seconds) * 1e9 + 0.5) - else - error('Duration JSON: expected table or number', 0) - end - local total_secs = seconds - if nanos == 0 then return string.format('%ds', total_secs) end - local fraction = string.format('.%09d', nanos):gsub('0+$', '') - if fraction == '.' then fraction = '' end - return string.format('%d%ss', total_secs, fraction) - end - -- Wrappers: encode is just the unwrapped value. + if name == 'google.protobuf.Timestamp' then return format_timestamp(v) end + if name == 'google.protobuf.Duration' then return format_duration(v) end + if name == 'google.protobuf.FieldMask' then return fieldmask_to_json(v) end + if name == 'google.protobuf.Any' then return any_to_json(v) end + if name == 'google.protobuf.Struct' then return value_to_json_struct(v) end + if name == 'google.protobuf.ListValue' then return value_to_json_list(v) end + if name == 'google.protobuf.Value' then return value_to_json(v) end local wrap = name:match('^google%.protobuf%.(%w+)Value$') if wrap then local wrapper_proto = { - Int32 = 'int32', UInt32 = 'uint32', Int64 = 'int64', UInt64 = 'uint64', - Float = 'float', Double = 'double', Bool = 'bool', - String = 'string', Bytes = 'bytes', + Int32='int32', UInt32='uint32', Int64='int64', UInt64='uint64', + Float='float', Double='double', Bool='bool', + String='string', Bytes='bytes', } local pt = wrapper_proto[wrap] if pt then return encode_scalar(pt, v) end end - if name == 'google.protobuf.Struct' then - return value_to_json_struct(v) - end - if name == 'google.protobuf.ListValue' then - return value_to_json_list(v) - end - if name == 'google.protobuf.Value' then - return value_to_json(v) - end - if name == 'google.protobuf.FieldMask' then - return fieldmask_to_json(v) - end - if name == 'google.protobuf.Any' then - return any_to_json(v) - end return nil end encode_message = function(desc, t) - if t == nil then return nil end + if rawequal(t, nil) then return nil end local override = encode_wkt(desc, t) - if override ~= nil then return override end + -- box.NULL ~= nil is false under __eq; use rawequal so a Value WKT can + -- legitimately return null as its override. + if not rawequal(override, nil) then return override end - -- A proto3 message with no set fields must serialize as a JSON object - -- `{}`, not the empty-table default `[]`. Mark the table as a map. - local out = setmetatable({}, {__serialize = 'map'}) + local out = setmetatable({}, {__serialize='map'}) for _, f in ipairs(desc.fields) do local v = t[f.name] - if v ~= nil then + -- box.NULL == nil under Tarantool's __eq metamethod; use rawequal + -- so a deliberately-stored null sentinel survives the field walk. + if not rawequal(v, nil) then local key = to_camel(f.name) if f.kind == 'map' then if next(v) ~= nil then @@ -308,12 +762,11 @@ out[key] = obj end elseif f.repeated then if #v > 0 then - local arr = {} + local arr = setmetatable({}, {__serialize='seq'}) for i = 1, #v do arr[i] = encode_field_value(f, v[i]) end out[key] = arr end else - -- proto3 default elision (unless presence is meaningful). local emit = true if not (f.optional or f.oneof) then if f.kind == 'scalar' then @@ -323,8 +776,7 @@ if v == '' then emit = false end elseif pt == 'bool' then if v == false then emit = false end else - -- numeric default - if v == 0 or (type(v) == 'cdata' and v == ffi.cast('int64_t', 0)) then + if v == 0 or (type(v) == 'cdata' and v == ffi.cast(INT64_T, 0)) then emit = false end end @@ -342,7 +794,7 @@ to_json_value = encode_message function M.encode(desc, t) - return json.encode(encode_message(desc, t)) + return encode_json(encode_message(desc, t)) end -- --------------------------------------------------------------------------- @@ -352,44 +804,74 @@ local decode_message -- forward local function decode_scalar(proto_type, v) - if INT64_FAMILY[proto_type] then - return string_to_int64(v, proto_type:sub(1, 1) == 'u' or proto_type == 'fixed64') + if proto_type == 'int32' or proto_type == 'sint32' or + proto_type == 'fixed32' or proto_type == 'sfixed32' then + return decode_int32(v) end - if proto_type == 'bytes' then return digest.base64_decode(v) end - if proto_type == 'float' or proto_type == 'double' then - if v == 'NaN' then return 0/0 end - if v == 'Infinity' then return math.huge end - if v == '-Infinity' then return -math.huge end - if type(v) == 'string' then return tonumber(v) end + if proto_type == 'uint32' then return decode_uint32(v) end + if proto_type == 'int64' or proto_type == 'sint64' or + proto_type == 'sfixed64' then + return decode_int64_value(v, false) + end + if proto_type == 'uint64' or proto_type == 'fixed64' then + return decode_int64_value(v, true) + end + if proto_type == 'float' then return decode_float_value(v, true) end + if proto_type == 'double' then return decode_float_value(v, false) end + if proto_type == 'bool' then + if type(v) ~= 'boolean' then + error('bool: expected JSON true/false, got ' .. type(v), 0) + end + return v + end + if proto_type == 'string' then + if type(v) ~= 'string' then + error('string: expected JSON string, got ' .. type(v), 0) + end + if not wire.is_valid_utf8(v) then + error('string: invalid UTF-8', 0) + end return v end - if proto_type == 'bool' then - if type(v) == 'string' then return v == 'true' end - return v and true or false + if proto_type == 'bytes' then + if type(v) ~= 'string' then + error('bytes: expected base64 JSON string, got ' .. type(v), 0) + end + return digest.base64_decode(v) end - -- 32-bit ints + string: accept JSON string or number defensively. - if type(v) == 'string' and proto_type ~= 'string' then return tonumber(v) end - return v + error('decode_scalar: unsupported proto type ' .. tostring(proto_type), 0) end local function decode_enum(enum_desc, v) if type(v) == 'string' then local n = enum_desc.by_name[v] if n ~= nil then return n end + -- Numeric string (e.g. "999"): JSON spec allows it; treat as integer. + if is_int_string(v) then + local n2 = tonumber(v) + if n2 >= -2147483648 and n2 <= 2147483647 then return n2 end + end + -- Unknown name: signal "ignore" via nil. + return nil end - -- proto3 JSON: unrecognized integer enum values pass through; unrecognized - -- string names yield nil so the caller can drop the element (repeated/map) - -- or fall back to the field default (singular). - return tonumber(v) + if type(v) == 'number' then + if v ~= v or v ~= math.floor(v) then + error('enum: non-integer JSON number', 0) + end + if v < -2147483648 or v > 2147483647 then + error('enum: integer out of int32 range', 0) + end + return v + end + if type(v) == 'cdata' then return tonumber(v) end + error('enum: expected JSON string or integer, got ' .. type(v), 0) end local function decode_field_value(field, v) if v == box.NULL then - -- proto3 JSON: a null value means "use the field's default" — i.e. - -- the field is treated as absent. The lone exception is - -- google.protobuf.Value, where null is itself a valid value - -- (NullValue.NULL_VALUE); pass through so json_to_value returns - -- PB_NULL. + -- proto3 JSON: null on a non-message field means "use the default" + -- (treat as absent). The lone exception is google.protobuf.Value + -- whose null is the NullValue.NULL_VALUE member. if field.kind == 'message' and field.message and field.message.name == 'google.protobuf.Value' then return PB_NULL @@ -397,51 +879,105 @@ end return nil end local kind = field.kind - if kind == 'scalar' then return decode_scalar(field.proto_type, v) - elseif kind == 'enum' then return decode_enum(field.enum, v) - elseif kind == 'message' then return decode_message(field.message, v) end + if kind == 'scalar' then return decode_scalar(field.proto_type, v) end + if kind == 'enum' then return decode_enum(field.enum, v) end + if kind == 'message' then return decode_message(field.message, v) end error('decode_field_value: unknown kind ' .. tostring(kind), 0) end local function decode_map_key(key_field, k) local pt = key_field.proto_type if pt == 'string' then return k end - if pt == 'bool' then return k == 'true' end - if INT64_FAMILY[pt] then return string_to_int64(k, pt:sub(1, 1) == 'u' or pt == 'fixed64') end - return tonumber(k) + if pt == 'bool' then + if k == 'true' then return true end + if k == 'false' then return false end + error('map: invalid key "' .. tostring(k) .. '"', 0) + end + if INT64_FAMILY[pt] then + return decode_int64_value(k, pt:sub(1, 1) == 'u' or pt == 'fixed64') + end + -- 32-bit integer key. JSON map keys are always strings; require strict + -- digits-only input. + if pt == 'uint32' or pt == 'fixed32' then return decode_uint32(k) end + return decode_int32(k) end local json_to_value, json_to_struct, json_to_list, json_to_any -- forwards json_to_any = function(v) - if v == nil then return {type_url = '', value = ''} end + if v == nil then return {type_url='', value=''} end if type(v) ~= 'table' then error('Any JSON: expected object, got ' .. type(v), 0) end - local type_url = v['@type'] or '' + local type_url = v['@type'] + if type_url == nil then + -- Bare `{}` is an empty Any. Anything else without @type is an error. + if next(v) ~= nil then + error('Any JSON: missing @type', 0) + end + return {type_url='', value=''} + end + if type(type_url) ~= 'string' then + error('Any JSON: @type must be a string', 0) + end + if type_url == '' then + -- Empty @type with any other key is malformed + -- (AnyWktRepresentationWithEmptyTypeAndValue). + for k, _ in pairs(v) do + if k ~= '@type' then + error('Any JSON: empty @type with sibling fields', 0) + end + end + return {type_url='', value=''} + end + -- A valid Any type URL has the shape "/". + -- "not_a_url" or anything else without a slash is rejected + -- (AnyWktRepresentationWithBadType). + if type_url:find('/', 1, true) == nil then + error('Any JSON: invalid @type URL "' .. type_url .. '"', 0) + end local desc = pbwkt.lookup(type_url) if desc == nil then - -- Opaque fallback (no registered descriptor): expect a base64 `value` - -- field, mirroring our encode-side fallback. + -- Opaque fallback: only accept base64-encoded `value`. A non-string + -- value implies a Wkt representation that we cannot dispatch. local raw = v.value - return {type_url = type_url, - value = raw and digest.base64_decode(raw) or ''} + if raw == nil or raw == box.NULL then + return {type_url = type_url, value = ''} + end + if type(raw) ~= 'string' then + error('Any JSON: unknown type "' .. type_url .. '"', 0) + end + return {type_url = type_url, value = digest.base64_decode(raw)} end - -- Reconstruct the inner message from the flat JSON, skipping @type. - local payload = {} - for k, mv in pairs(v) do - if k ~= '@type' then payload[k] = mv end - end - -- For Value/Struct/ListValue/wrappers the spec nests under `value`. - if payload.value ~= nil and next(payload, next(payload)) == nil then - payload = payload.value + local payload + if is_wkt_name(desc.name) then + -- Spec requires WKTs to be nested under "value". Empty is the lone + -- exception: its JSON form is `{}` so the value key is optional + -- (AnyEmpty test sends `{"@type": ".../Empty"}` and expects success). + local raw = v.value + if raw == nil then + if desc.name == 'google.protobuf.Empty' then + payload = {} + else + error('Any JSON: WKT @type "' .. type_url + .. '" requires a "value" key', 0) + end + elseif raw == box.NULL and desc.name ~= 'google.protobuf.Value' then + error('Any JSON: null value for WKT "' .. type_url .. '"', 0) + else + payload = raw + end + else + payload = {} + for k, mv in pairs(v) do + if k ~= '@type' then payload[k] = mv end + end end local inner = decode_message(desc, payload) local bytes = desc.encode and desc.encode(inner) - or require('pb.codec').encode(desc, inner) + or require('pb.codec').encode(desc, inner) return {type_url = type_url, value = bytes} end - json_to_value = function(v) if v == nil or v == box.NULL then return PB_NULL end @@ -449,11 +985,9 @@ local ty = type(v) if ty == 'boolean' or ty == 'number' or ty == 'string' then return v end if ty == 'cdata' then return tonumber(v) end if ty == 'table' then - if v[1] ~= nil or next(v) == nil and getmetatable(v) - and getmetatable(v).__serialize == 'seq' then - return json_to_list(v) - end - -- Heuristic: integer-keyed → list, else → struct. + local mt = getmetatable(v) + if mt and mt.__serialize == 'seq' then return json_to_list(v) end + if mt and mt.__serialize == 'map' then return json_to_struct(v) end if v[1] ~= nil then return json_to_list(v) end return json_to_struct(v) end @@ -476,48 +1010,53 @@ end local function decode_wkt(desc, v) local name = desc.name - if name == 'google.protobuf.Empty' then return {} end - if name == 'google.protobuf.Timestamp' then - return datetime.parse(v) + if name == 'google.protobuf.Empty' then + if v ~= nil and type(v) ~= 'table' then + error('Empty JSON: expected object, got ' .. type(v), 0) + end + return {} + end + if name == 'google.protobuf.Timestamp' then return parse_timestamp(v) end + if name == 'google.protobuf.Duration' then return parse_duration(v) end + if name == 'google.protobuf.FieldMask' then return fieldmask_from_json(v) end + if name == 'google.protobuf.Any' then return json_to_any(v) end + if name == 'google.protobuf.Value' then return json_to_value(v) end + if name == 'google.protobuf.Struct' then + if type(v) ~= 'table' then + error('Struct JSON: expected object, got ' .. type(v), 0) + end + return json_to_struct(v) end - if name == 'google.protobuf.Duration' then - local body = v:gsub('s$', '') - local sign = 1 - if body:sub(1, 1) == '-' then sign = -1; body = body:sub(2) end - local sec_str, frac = body:match('^(%d+)%.?(%d*)$') - if sec_str == nil then error('invalid Duration JSON: ' .. v, 0) end - local nanos = 0 - if frac and frac ~= '' then - nanos = tonumber((frac .. '000000000'):sub(1, 9)) + if name == 'google.protobuf.ListValue' then + if type(v) ~= 'table' then + error('ListValue JSON: expected array, got ' .. type(v), 0) end - return {seconds = sign * tonumber(sec_str), nanos = sign * nanos} + return json_to_list(v) end local wrap = name:match('^google%.protobuf%.(%w+)Value$') if wrap then local wrapper_proto = { - Int32 = 'int32', UInt32 = 'uint32', Int64 = 'int64', UInt64 = 'uint64', - Float = 'float', Double = 'double', Bool = 'bool', - String = 'string', Bytes = 'bytes', + Int32='int32', UInt32='uint32', Int64='int64', UInt64='uint64', + Float='float', Double='double', Bool='bool', + String='string', Bytes='bytes', } local pt = wrapper_proto[wrap] if pt then return decode_scalar(pt, v) end end - if name == 'google.protobuf.Value' then - return json_to_value(v) - end - if name == 'google.protobuf.Struct' then - return json_to_struct(v) - end - if name == 'google.protobuf.ListValue' then - return json_to_list(v) - end - if name == 'google.protobuf.FieldMask' then - return fieldmask_from_json(v) - end - if name == 'google.protobuf.Any' then - return json_to_any(v) + return nil +end + +local function is_json_array(t) + local mt = getmetatable(t) + if mt and mt.__serialize == 'seq' then return true end + if mt and mt.__serialize == 'map' then return false end + if next(t) == nil then + -- Empty `{}` from Tarantool's json.decode has no metatable; treat + -- empty as either depending on context. For repeated-field decode + -- we allow `[]` (empty array) or null (the caller short-circuits). + return true end - return nil + return t[1] ~= nil end decode_message = function(desc, v) @@ -525,7 +1064,8 @@ if v == nil then return nil end local override = decode_wkt(desc, v) if override ~= nil then return override end if type(v) ~= 'table' then - error('expected JSON object for ' .. desc.name .. ', got ' .. type(v), 0) + error('expected JSON object for ' .. desc.name .. + ', got ' .. type(v), 0) end -- Build a name -> field map covering both camelCase and snake_case. @@ -540,16 +1080,30 @@ desc._json_field_by_name = field_by_json_name end local out = {} + local oneof_seen -- lazily allocated for k, jv in pairs(v) do local f = field_by_json_name[k] if f ~= nil then - if jv == box.NULL and (f.kind ~= 'message' - or f.message == nil - or f.message.name ~= 'google.protobuf.Value') then - -- proto3 JSON: null on any non-Value field means "use the - -- default" — i.e. leave the field unset. + local is_value_field = (f.kind == 'message' and f.message + and f.message.name == 'google.protobuf.Value') + local is_null_default = (jv == box.NULL) and not is_value_field + -- Reject duplicate oneof branches. A `null` JSON value for a + -- oneof branch means "field absent" and does NOT count as + -- setting the oneof (matches OneofFieldNullFirst/Second tests). + if f.oneof and not is_null_default then + oneof_seen = oneof_seen or {} + if oneof_seen[f.oneof] then + error('oneof "' .. f.oneof .. '" set multiple times', 0) + end + oneof_seen[f.oneof] = true + end + if is_null_default then + -- proto3 JSON: null on non-Value fields = "use default" elseif f.kind == 'map' then if jv ~= box.NULL then + if type(jv) ~= 'table' then + error('field "' .. k .. '": expected JSON object for map', 0) + end local m = {} for mk, mv in pairs(jv) do local dv = decode_field_value(f.value, mv) @@ -561,6 +1115,9 @@ out[f.name] = m end elseif f.repeated then if jv ~= box.NULL then + if type(jv) ~= 'table' or not is_json_array(jv) then + error('field "' .. k .. '": expected JSON array', 0) + end local arr = {} local n = 0 for i = 1, #jv do @@ -582,7 +1139,16 @@ return out end function M.decode(desc, s) - return decode_message(desc, json.decode(s)) + local v = json.decode(s) + if v == nil or v == box.NULL then + -- Top-level JSON null is rejected for regular messages but is a + -- legal Value (it maps to NullValue.NULL_VALUE). + if desc.name == 'google.protobuf.Value' then + return PB_NULL + end + error('top-level JSON null is not a valid message', 0) + end + return decode_message(desc, v) end M.to_json_value = to_json_value diff --git a/runtime/pb/wkt.lua b/runtime/pb/wkt.lua index b652bfbe8cb395db2aa096fa4fdd94ffc32c1a49..c13bda34c6cdb022b2fb1907acb71bd49908fa2d 100644 --- a/runtime/pb/wkt.lua +++ b/runtime/pb/wkt.lua @@ -83,7 +83,14 @@ else pos = wire.skip_field(buf, pos, wt) end end - return datetime.new({timestamp = tonumber(seconds), nsec = nanos}) + -- datetime.new validates nanos/seconds ranges. Out-of-spec Timestamps + -- (e.g. negative nanos, year > 9999) are kept as raw {seconds, nanos} + -- so JSON serialization can reject them with serialize_error rather + -- than crashing here with parse_error. + local ok, dt = pcall(datetime.new, + {timestamp = tonumber(seconds), nsec = nanos}) + if not ok then return {seconds = seconds, nanos = nanos} end + return dt end M.Timestamp_encode = timestamp_encode diff --git a/test/conformance/known_failures.txt b/test/conformance/known_failures.txt index 950f0aaf1b30ef7d9fb095e8fb534a46699bcff1..c8730389374b64603ca13df4d4bd446ec0fdd8ec 100644 --- a/test/conformance/known_failures.txt +++ b/test/conformance/known_failures.txt @@ -1,99 +1,32 @@ # conformance_test_runner --failure_list # -# Tests we know fail today. Captured from `just conformance` on -# 2026-05-15 against protobuf v34.1's conformance corpus. -# Summary at capture time: 798 successes, 1864 skipped, 144 failures. +# Recommended-only failures (Required tests all pass as of 2026-05-16). +# Re-captured after the proto3 JSON strict-validation pass; see commits +# spanning runtime/pb/json.lua + runtime/pb/wire.lua. # -# JSON output is currently disabled in cmd/conformance/core.lua (returns -# `skipped`) so the harness's jsoncpp comparator doesn't crash on our -# half-finished JSON. Failures below are split into a few clusters: -# * Required.Proto3.JsonInput.* — our JSON *decoder* path (we still -# consume JSON input even though we don't emit it). -# * Required.Proto3.ProtobufInput.RejectInvalidUtf8.* — we don't yet -# enforce strict UTF-8 on proto3 string fields. -# * Recommended.Proto3.ProtobufInput.ValidDataRepeated.ENUM.* — packed -# vs unpacked enum input/output corners. -# * Required.MapFieldsHaveNoPresence.* — Map presence semantics. -# * Required.{TimestampProtoInputTooLarge,…}.JsonOutput — synthetic -# because of the global JSON-output skip. -# -# Re-generate this file after fixes via `just conformance-refresh-failures`. -Required.Proto3.JsonInput.AnyNested.JsonOutput -Required.Proto3.JsonInput.AnyWithStruct.JsonOutput -Required.Proto3.JsonInput.AnyWithValueForJsonObject.JsonOutput -Required.Proto3.TimestampProtoNegativeNanos.JsonOutput -Required.Proto3.DurationProtoInputTooLarge.JsonOutput -Required.Proto3.DurationProtoInputTooSmall.JsonOutput -Required.Proto3.DurationProtoNanosTooLarge.JsonOutput -Required.Proto3.DurationProtoNanosTooSmall.JsonOutput -Required.Proto3.DurationProtoNanosWrongSign.JsonOutput -Required.Proto3.DurationProtoNanosWrongSignNegativeSecs.JsonOutput -Required.Proto3.JsonInput.AnyWithNoType.JsonOutput -Required.Proto3.JsonInput.AnyWktRepresentationWithBadType -Required.Proto3.JsonInput.AnyWktRepresentationWithEmptyTypeAndValue -Required.Proto3.JsonInput.DoubleFieldEmptyString -Required.Proto3.JsonInput.DoubleFieldStringValueNonNumeric -Required.Proto3.JsonInput.DoubleFieldStringValuePartiallyNumeric -Required.Proto3.JsonInput.DoubleFieldTooLarge -Required.Proto3.JsonInput.DoubleFieldTooSmall -Required.Proto3.JsonInput.DurationJsonInputTooLarge -Required.Proto3.JsonInput.DurationJsonInputTooSmall -Required.Proto3.JsonInput.DurationMinValue.JsonOutput -Required.Proto3.JsonInput.DurationMissingS -Required.Proto3.JsonInput.DurationNegativeNanos.JsonOutput -Required.Proto3.JsonInput.DurationRepeatedValue.JsonOutput -Required.Proto3.JsonInput.FloatFieldEmptyString -Required.Proto3.JsonInput.FloatFieldStringValueNonNumeric -Required.Proto3.JsonInput.FloatFieldStringValuePartiallyNumeric -Required.Proto3.JsonInput.FloatFieldStringValuePartiallyNumericComma -Required.Proto3.JsonInput.FloatFieldStringValuePartiallyNumericSpace -Required.Proto3.JsonInput.FloatFieldStringValuePartiallyNumericUnicode -Required.Proto3.JsonInput.FloatFieldTooLarge -Required.Proto3.JsonInput.FloatFieldTooSmall -Required.Proto3.JsonInput.Int32FieldEmptyString -Required.Proto3.JsonInput.Int32FieldLeadingSpace -Required.Proto3.JsonInput.Int32FieldNotInteger -Required.Proto3.JsonInput.Int32FieldNotNumber -Required.Proto3.JsonInput.Int32FieldStringValueNonNumeric -Required.Proto3.JsonInput.Int32FieldStringValuePartiallyNumeric -Required.Proto3.JsonInput.Int32FieldStringValuePartiallyNumericComma -Required.Proto3.JsonInput.Int32FieldStringValuePartiallyNumericSpace -Required.Proto3.JsonInput.Int32FieldStringValuePartiallyNumericUnicode -Required.Proto3.JsonInput.Int32FieldTooLarge -Required.Proto3.JsonInput.Int32FieldTooSmall -Required.Proto3.JsonInput.Int32FieldTrailingSpace -Required.Proto3.JsonInput.Int64FieldTooLarge -Required.Proto3.JsonInput.OneofFieldDuplicate -Required.Proto3.JsonInput.RejectTopLevelNull -Required.Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotBool -Required.Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotMessage -Required.Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotString -Required.Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotBool -Required.Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotInt -Required.Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotMessage -Required.Proto3.JsonInput.SingleValueForRepeatedFieldMessage -Required.Proto3.JsonInput.StringFieldNotAString -Required.Proto3.JsonInput.TimestampJsonInputLowercaseT -Required.Proto3.JsonInput.TimestampJsonInputLowercaseZ -Required.Proto3.JsonInput.TimestampJsonInputMissingT -Required.Proto3.JsonInput.TimestampJsonInputMissingZ -Required.Proto3.JsonInput.TimestampJsonInputTooLarge -Required.Proto3.JsonInput.TimestampJsonInputTooSmall -Required.Proto3.JsonInput.TimestampWithMissingColonInOffset -Required.Proto3.JsonInput.TimestampWithNegativeOffset.JsonOutput -Required.Proto3.JsonInput.TimestampWithPositiveOffset.JsonOutput -Required.Proto3.JsonInput.Uint32FieldEmptyString -Required.Proto3.JsonInput.Uint32FieldNotInteger -Required.Proto3.JsonInput.Uint32FieldNotNumber -Required.Proto3.JsonInput.Uint32FieldTooLarge -Required.Proto3.JsonInput.ValueAcceptNull.JsonOutput -Required.Proto3.ProtobufInput.DoubleFieldNormalizeSignalingNan.JsonOutput -Required.Proto3.ProtobufInput.FloatFieldNormalizeSignalingNan.JsonOutput -Required.Proto3.ProtobufInput.RepeatedScalarSelectsLast.DOUBLE.JsonOutput -Required.Proto3.ProtobufInput.ValidDataRepeated.DOUBLE.PackedInput.JsonOutput -Required.Proto3.ProtobufInput.ValidDataRepeated.DOUBLE.UnpackedInput.JsonOutput -Required.Proto3.ProtobufInput.ValidDataScalar.DOUBLE[2].JsonOutput -Required.Proto3.ProtobufInput.ValidDataScalar.DOUBLE[3].JsonOutput -Required.Proto3.TimestampProtoInputTooLarge.JsonOutput -Required.Proto3.TimestampProtoInputTooSmall.JsonOutput -Required.Proto3.TimestampProtoNanoTooLarge.JsonOutput +# Remaining categories: +# * FieldMask — round-trip tolerates names that can't survive +# lowerCamel → snake → lowerCamel. +# * FieldNameDuplicate — duplicate keys aren't rejected (Tarantool's +# json.decode silently keeps the last). +# * MapFieldValueIsNull / RepeatedField*ElementIsNull — null elements +# inside a map/repeated should be parse_error; we currently drop them. +# * NullValueInOtherOneof[New|Old]Format.Validator — Validator harness +# disagrees with our round-trip even though semantics match. +# * RejectUnknownEnumStringValueIn[Optional|Repeated|Map]Field — proto3 +# mode should reject unknown enum names (we silently drop them). +Recommended.Proto3.FieldMaskNumbersDontRoundTrip.JsonOutput +Recommended.Proto3.FieldMaskPathsDontRoundTrip.JsonOutput +Recommended.Proto3.FieldMaskTooManyUnderscore.JsonOutput +Recommended.Proto3.JsonInput.FieldMaskInvalidCharacter +Recommended.Proto3.JsonInput.FieldNameDuplicate +Recommended.Proto3.JsonInput.FieldNameDuplicateDifferentCasing1 +Recommended.Proto3.JsonInput.FieldNameDuplicateDifferentCasing2 +Recommended.Proto3.JsonInput.MapFieldValueIsNull +Recommended.Proto3.JsonInput.NullValueInOtherOneofNewFormat.Validator +Recommended.Proto3.JsonInput.NullValueInOtherOneofOldFormat.Validator +Recommended.Proto3.JsonInput.RejectUnknownEnumStringValueInMapValue +Recommended.Proto3.JsonInput.RejectUnknownEnumStringValueInOptionalField +Recommended.Proto3.JsonInput.RejectUnknownEnumStringValueInRepeatedField +Recommended.Proto3.JsonInput.RepeatedFieldMessageElementIsNull +Recommended.Proto3.JsonInput.RepeatedFieldPrimitiveElementIsNull diff --git a/test/conformance_test.lua b/test/conformance_test.lua index 58f8a8c7d078715ba664ecd1d2bd74e076911819..1968906723c5c69eb9bf52d8fb243fc420008be2 100644 --- a/test/conformance_test.lua +++ b/test/conformance_test.lua @@ -753,6 +753,360 @@ 'oneof sibling must be cleared after the message branch is set') t.assert_equals(decoded.oneof_nested_message.a, 2) end +-- ========================================================================= +-- Fix 12: strict JSON scalar validation. proto3 spec rejects every shape +-- of malformed numeric/string scalar input. We pin one example per +-- distinct rejection path (per type, per shape) instead of enumerating +-- the full conformance matrix — the validator code path is shared. +-- ========================================================================= + +local function pb_to_json(pb_bytes) + return decode_resp(core.handle_request(encode_req({ + protobuf_payload = pb_bytes, + requested_output_format = JSON, + message_type = PROTO3_NAME, + }))) +end + +core_g.test_json_int32_rejects_empty_string = function() + t.assert_not_equals(json_to_pb('{"optionalInt32":""}').parse_error, nil) +end + +core_g.test_json_int32_rejects_leading_space = function() + t.assert_not_equals(json_to_pb('{"optionalInt32":" 1"}').parse_error, nil) +end + +core_g.test_json_int32_rejects_partial_numeric = function() + t.assert_not_equals(json_to_pb('{"optionalInt32":"1abc"}').parse_error, nil) +end + +core_g.test_json_int32_rejects_non_integer_number = function() + t.assert_not_equals(json_to_pb('{"optionalInt32":1.5}').parse_error, nil) +end + +core_g.test_json_int32_rejects_non_numeric_type = function() + t.assert_not_equals(json_to_pb('{"optionalInt32":true}').parse_error, nil) +end + +core_g.test_json_int32_rejects_out_of_range_high = function() + t.assert_not_equals(json_to_pb('{"optionalInt32":2147483648}').parse_error, nil) +end + +core_g.test_json_int32_rejects_out_of_range_low = function() + t.assert_not_equals(json_to_pb('{"optionalInt32":-2147483649}').parse_error, nil) +end + +core_g.test_json_int32_accepts_quoted_exponential = function() + -- "1e5" must decode to 100000 — the JSON spec lets numbers in string + -- form use exponential notation as long as the value is integer. + local resp = json_to_pb('{"optionalInt32":"1e5"}') + t.assert_not(resp.parse_error, resp.parse_error) + local decoded = proto3.TestAllTypesProto3_decode(resp.protobuf_payload) + t.assert_equals(decoded.optional_int32, 100000) +end + +core_g.test_json_uint32_rejects_negative = function() + t.assert_not_equals(json_to_pb('{"optionalUint32":-1}').parse_error, nil) +end + +core_g.test_json_double_rejects_overflow_number = function() + -- 1.79769e+309 overflows IEEE 754 → inf, which must be parse_error + -- (NOT silently coerced to "Infinity"). + t.assert_not_equals(json_to_pb('{"optionalDouble":1.79769e309}').parse_error, nil) +end + +core_g.test_json_double_rejects_partial_numeric_string = function() + t.assert_not_equals(json_to_pb('{"optionalDouble":"1.0abc"}').parse_error, nil) +end + +core_g.test_json_bool_rejects_string = function() + t.assert_not_equals(json_to_pb('{"optionalBool":"true"}').parse_error, nil) +end + +core_g.test_json_string_rejects_number = function() + t.assert_not_equals(json_to_pb('{"optionalString":123}').parse_error, nil) +end + +core_g.test_json_repeated_rejects_object = function() + -- A repeated field must be a JSON array; an object is malformed. + t.assert_not_equals(json_to_pb( + '{"repeatedNestedMessage":{"a":1}}').parse_error, nil) +end + +core_g.test_json_oneof_rejects_duplicate_branches = function() + -- Setting two branches of the same oneof in one JSON object is a + -- parse_error per the proto3 JSON spec. + t.assert_not_equals(json_to_pb( + '{"oneofUint32":1,"oneofString":"x"}').parse_error, nil) +end + +core_g.test_json_oneof_null_branch_does_not_count = function() + -- A null-valued oneof branch counts as "absent", so a second non-null + -- branch in the same object is the only set branch (not a duplicate). + local resp = json_to_pb( + '{"oneofUint32":null,"oneofString":"x"}') + t.assert_not(resp.parse_error, resp.parse_error) + local decoded = proto3.TestAllTypesProto3_decode(resp.protobuf_payload) + t.assert_equals(decoded.oneof_string, 'x') + t.assert_equals(decoded.oneof_uint32, nil) +end + +core_g.test_json_top_level_null_rejected_for_messages = function() + -- Decoding `null` for a regular message is a parse_error. The lone + -- exception is google.protobuf.Value (covered by json_test.lua). + local req = encode_req({ + json_payload = 'null', + requested_output_format = PROTOBUF, + message_type = PROTO3_NAME, + }) + t.assert_not_equals(decode_resp(core.handle_request(req)).parse_error, nil) +end + +-- ========================================================================= +-- Fix 13: strict Timestamp parser + canonical output. Strict format +-- (uppercase T/Z, colon in offset, max 9 frac digits, range check). +-- Output uses a portable epoch_to_ymdhms so years 0001-9999 always pad +-- to 4 digits — glibc's POSIX %Y emits "1" for year 1, which the +-- conformance harness can't parse back. +-- ========================================================================= + +local function json_round_trip(json_str) + return decode_resp(core.handle_request(encode_req({ + json_payload = json_str, + requested_output_format = JSON, + message_type = PROTO3_NAME, + }))) +end + +core_g.test_json_timestamp_rejects_lowercase_t = function() + t.assert_not_equals(json_to_pb( + '{"optionalTimestamp":"1970-01-01t00:00:00Z"}').parse_error, nil) +end + +core_g.test_json_timestamp_rejects_lowercase_z = function() + t.assert_not_equals(json_to_pb( + '{"optionalTimestamp":"1970-01-01T00:00:00z"}').parse_error, nil) +end + +core_g.test_json_timestamp_rejects_missing_z_and_offset = function() + t.assert_not_equals(json_to_pb( + '{"optionalTimestamp":"1970-01-01T00:00:00"}').parse_error, nil) +end + +core_g.test_json_timestamp_rejects_offset_without_colon = function() + t.assert_not_equals(json_to_pb( + '{"optionalTimestamp":"1970-01-01T00:00:00+0100"}').parse_error, nil) +end + +core_g.test_json_timestamp_min_value_round_trips = function() + -- Year 0001 must pad to 4 digits ("0001-...") on output. POSIX %Y on + -- glibc emits just "1" for that year, which would break round-trip. + local resp = json_round_trip( + '{"optionalTimestamp":"0001-01-01T00:00:00Z"}') + t.assert_not(resp.parse_error, resp.parse_error) + t.assert_str_contains(resp.json_payload, '0001-01-01T00:00:00Z') +end + +core_g.test_json_timestamp_normalizes_offset_to_utc = function() + -- "+01:00" offset must be applied to compute the UTC epoch, then the + -- output must use "Z" form. 12:00:00+01:00 == 11:00:00 UTC. + local resp = json_round_trip( + '{"optionalTimestamp":"2020-01-01T12:00:00+01:00"}') + t.assert_not(resp.parse_error, resp.parse_error) + t.assert_str_contains(resp.json_payload, '2020-01-01T11:00:00Z') +end + +core_g.test_proto_timestamp_negative_nanos_serialize_error = function() + -- Binary input with nanos = -1 is a valid wire shape but per spec + -- the JSON serializer must reject it (Timestamp.nanos >= 0). + -- optional_timestamp (id 302), tag (302<<3)|2 = 0xf2 0x12. + -- Inner Timestamp: tag(2, VARINT) nanos=-1 (10-byte int32 form). + local nanos_field = '\x10' .. '\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01' + local input = '\xf2\x12' .. string.char(#nanos_field) .. nanos_field + local resp = pb_to_json(input) + t.assert_not_equals(resp.serialize_error, nil) +end + +-- ========================================================================= +-- Fix 14: strict Duration parser + canonical output. Suffix "s" is +-- mandatory; range is ±10000 years; nanos sign must match seconds. +-- Fractional output uses 0/3/6/9 digits. +-- ========================================================================= + +core_g.test_json_duration_rejects_missing_s_suffix = function() + t.assert_not_equals(json_to_pb( + '{"optionalDuration":"1.5"}').parse_error, nil) +end + +core_g.test_json_duration_rejects_out_of_range = function() + -- Spec: seconds within ±315576000000. + t.assert_not_equals(json_to_pb( + '{"optionalDuration":"315576000001s"}').parse_error, nil) +end + +core_g.test_proto_duration_negative_nanos_canonical_output = function() + -- Encoded Duration with seconds=0, nanos=-500000000 must serialize + -- as "-0.500s" with both sign and 3-digit fraction. + -- optional_duration (id 301), tag (301<<3)|2 = 0xea 0x12. + -- Inner: tag(2,VARINT)=0x10 + nanos as 10-byte int32 -500000000. + local nanos_field = '\x10\x80\xb6\xca\x91\xfe\xff\xff\xff\xff\x01' + local input = '\xea\x12' .. string.char(#nanos_field) .. nanos_field + local resp = pb_to_json(input) + t.assert_not(resp.serialize_error, resp.serialize_error) + t.assert_str_contains(resp.json_payload, '-0.500s') +end + +-- ========================================================================= +-- Fix 15: Any JSON output rules. +-- - Empty Any (no @type, no value) → emit `{}`. +-- - URL with no `/` is malformed (AnyWktRepresentationWithBadType). +-- - WKT-typed Any nests payload under "value"; user-typed Any flattens. +-- - Empty WKT inside Any drops the "value" key (reference parser +-- rejects {"value":{}} for Empty). +-- ========================================================================= + +core_g.test_json_any_empty_round_trips_as_empty_object = function() + local resp = json_round_trip('{"optionalAny":{}}') + t.assert_not(resp.parse_error, resp.parse_error) + t.assert_str_contains(resp.json_payload, '"optionalAny":{}') +end + +core_g.test_json_any_with_only_empty_wkt_type_emits_no_value = function() + -- Round-trip pins the reference-implementation expectation that + -- {"@type":".../Empty"} stays in that shape (not promoted to + -- {"@type":".../Empty","value":{}}). + local resp = json_round_trip( + '{"optionalAny":{"@type":"type.googleapis.com/google.protobuf.Empty"}}') + t.assert_not(resp.parse_error, resp.parse_error) + t.assert_str_contains(resp.json_payload, + '"@type":"type.googleapis.com/google.protobuf.Empty"') + t.assert_not(resp.json_payload:find('"value"', 1, true), + 'Empty WKT in Any must not emit a "value" key') +end + +core_g.test_json_any_rejects_malformed_url = function() + -- @type without a `/` is malformed (AnyWktRepresentationWithBadType). + t.assert_not_equals(json_to_pb( + '{"optionalAny":{"@type":"not_a_url","value":""}}').parse_error, nil) +end + +core_g.test_json_any_rejects_empty_type_with_sibling_fields = function() + t.assert_not_equals(json_to_pb( + '{"optionalAny":{"@type":"","value":""}}').parse_error, nil) +end + +core_g.test_json_any_wkt_struct_nests_under_value = function() + -- A WKT payload must be wrapped: {"@type":"...Struct","value":{...}} + -- — flattening the struct fields next to "@type" is wrong because + -- "@type" would clash with a user "@type" key inside the struct. + local resp = json_round_trip( + '{"optionalAny":{"@type":"type.googleapis.com/google.protobuf.Struct","value":{"k":"v"}}}') + t.assert_not(resp.parse_error, resp.parse_error) + t.assert_str_contains(resp.json_payload, '"value":{"k":"v"}') +end + +-- ========================================================================= +-- Fix 16: google.protobuf.Value's number_value cannot represent NaN or +-- ±Infinity (JSON has no such literals). The serializer must reject. +-- ========================================================================= + +core_g.test_json_value_nan_serialize_error = function() + -- optional_value (id 306), tag (306<<3)|2 = 0x92 0x13. + -- Inner Value: tag(2,I64)=0x11 + NaN bytes. + local value_payload = '\x11\x00\x00\x00\x00\x00\x00\xf8\x7f' + local input = '\x92\x13' .. string.char(#value_payload) .. value_payload + local resp = pb_to_json(input) + t.assert_not_equals(resp.serialize_error, nil) +end + +core_g.test_json_value_infinity_serialize_error = function() + -- Value.number_value = +Infinity (bit pattern 0x7FF0000000000000) + local value_payload = '\x11\x00\x00\x00\x00\x00\x00\xf0\x7f' + local input = '\x92\x13' .. string.char(#value_payload) .. value_payload + local resp = pb_to_json(input) + t.assert_not_equals(resp.serialize_error, nil) +end + +-- ========================================================================= +-- Fix 17: ValueAcceptNull round-trips JSON `null` for a Value-typed +-- field. The decode side preserves it as PB_NULL (box.NULL); the encode +-- side must walk past `box.NULL == nil` (rawequal, not `==`). +-- ========================================================================= + +core_g.test_json_value_accept_null_round_trips = function() + local resp = json_round_trip('{"optionalValue":null}') + t.assert_not(resp.parse_error, resp.parse_error) + t.assert_str_contains(resp.json_payload, '"optionalValue":null') +end + +-- ========================================================================= +-- Fix 18: LuaJIT NaN-boxing collision. Certain IEEE NaN bit patterns +-- alias internal Lua type tags (nil, function, …) when read through +-- ffi.cast — tonumber returns a non-number for those values. The wire +-- decoder must detect NaN/Inf from the integer bit pattern before +-- touching the float field. +-- ========================================================================= + +core_g.test_wire_double_nan_box_collision_normalizes_to_nan = function() + -- 0x7FFBCBA987654321 — the specific bit pattern observed crashing + -- DoubleFieldNormalizeSignalingNan.JsonOutput in the conformance + -- suite. Must produce "NaN" without serialize_error. + local input = '\x61\x21\x43\x65\x87\xa9\xcb\xfb\xff' + local resp = pb_to_json(input) + t.assert_not(resp.serialize_error, resp.serialize_error) + t.assert_str_contains(resp.json_payload, '"optionalDouble":"NaN"') +end + +core_g.test_wire_double_all_ones_normalizes_to_nan = function() + -- 0xFFFFFFFFFFFFFFFF triggered the NaN-boxing collision pre-fix + -- (tonumber returned nil → field silently dropped). + local input = '\x61\xff\xff\xff\xff\xff\xff\xff\xff' + local resp = pb_to_json(input) + t.assert_not(resp.serialize_error, resp.serialize_error) + t.assert_str_contains(resp.json_payload, '"optionalDouble":"NaN"') +end + +core_g.test_wire_double_positive_infinity_decodes = function() + -- 0x7FF0000000000000 → +Infinity. Bit-pattern detection must + -- distinguish Inf from NaN. + local input = '\x61\x00\x00\x00\x00\x00\x00\xf0\x7f' + local resp = pb_to_json(input) + t.assert_str_contains(resp.json_payload, '"optionalDouble":"Infinity"') +end + +core_g.test_wire_float_sNaN_normalizes_to_nan = function() + -- 0x7FBFFFFF — classic single-precision signaling NaN. + local input = '\x5d\xff\xff\xbf\x7f' + local resp = pb_to_json(input) + t.assert_not(resp.serialize_error, resp.serialize_error) + t.assert_str_contains(resp.json_payload, '"optionalFloat":"NaN"') +end + +-- ========================================================================= +-- Fix 19: hand-rolled JSON encoder for shortest round-trip doubles. +-- Tarantool's json.encode uses a fixed global precision (14 by default) +-- so doubles like 0.1 don't round-trip; we emit them via a custom +-- precision-escalating formatter. Pin a few specific values. +-- ========================================================================= + +core_g.test_json_double_shortest_round_trip = function() + -- 0.1 doesn't have an exact double representation; %.14g would drop + -- the precision-bearing trailing digit. We need %.17g (or shortest) + -- so the value survives JSON → proto → JSON. + local resp = json_round_trip('{"optionalDouble":0.1}') + t.assert_not(resp.parse_error, resp.parse_error) + t.assert_str_contains(resp.json_payload, '0.1') +end + +core_g.test_json_double_integer_valued_emitted_without_decimal = function() + -- Integer-valued doubles in safe-int range emit without a decimal + -- point. (json.encode would write "1" too, but the hand-rolled path + -- has its own integer fast path — pin it.) + local resp = json_round_trip('{"optionalDouble":1.0}') + t.assert_not(resp.parse_error, resp.parse_error) + t.assert_str_contains(resp.json_payload, '"optionalDouble":1') +end + -- --------------------------------------------------------------------------- -- 2. Subprocess: stdin/stdout framing -- ---------------------------------------------------------------------------