diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 926973ae5b5c53ad05f358a456f1a4a92c8a80b1..15dc2b5ba7b2f6b4f843eebb1d703efcae1c9ef8 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -17,7 +17,7 @@ {"_type":"issue","id":"tarantool-protobuf-e4t","title":"LSP: plugin emits @class type stubs for generated messages","description":"Phase 3 of the LSP / LLM affordance work. Follows phase 1 (commit 8721548 — runtime annotations) and phase 2 (bd-74c — annotate codec/grpc/json/wkt).\n\nHighest-payoff phase for LLM grounding. Modify the protoc plugin (cmd/protoc-gen-tarantool/internal/gen) so each generated _pb.lua module emits ---@class blocks describing every message, plus typed @param/@return on the _encode/_decode/_descriptor surface. After this lands, anyone (human or LLM) who require('full.hello.hello_pb') gets full type info on Person, Address, etc., without reading the .proto.\n\nEmit shape (decision: inline at top of generated file; one file per .proto already includes all messages from that file):\n\n ---@class hello.Person\n ---@field name? string\n ---@field user_id? ffi.cdata* # uint64\n ---@field age? integer # int32\n ---@field weight_kg? number # double\n ---@field emails? string[]\n ---@field address? hello.Address\n ---@field lucky_numbers? integer[]\n ---@field favorite_color? hello.Color # enum alias\n ---@field tags? table\u003cstring,string\u003e # map\u003cstring,string\u003e\n ---@field unknown_fields? string\n\n ---@param tbl hello.Person\n ---@return string\n function M.Person_encode(tbl) ... end\n\n ---@param bytes string\n ---@return hello.Person\n function M.Person_decode(bytes) ... end\n\nField-kind -\u003e Lua-type mapping:\n- scalar int32/uint32/sint32/fixed32/sfixed32/bool → integer/boolean\n- scalar int64/uint64/sint64/fixed64/sfixed64 → ffi.cdata* (LuaJIT cdata, per project convention)\n- scalar float/double → number\n- scalar string/bytes → string\n- enum → alias of integer (emit ---@alias hello.Color integer)\n- message → another @class reference\n- repeated T → T[]\n- map\u003cK,V\u003e → table\u003cK_lua, V_lua\u003e\n- oneof — all members are optional, generator should NOT emit a discriminator field; user calls view:which() / inspects which value is non-nil\n\nAll fields are optional (? suffix) since proto3 default-elision means absence-on-wire is indistinguishable from default value. Required fields in proto2 omit the ?.\n\nBoth mode=full and mode=runtime emit the same _encode/_decode wrapper signatures, so the stubs apply uniformly.\n\nVerify by:\n1. Regenerating examples/expected/ via just gen\n2. Confirming the generated _pb.lua files load and pass tests (no behavior change)\n3. Opening examples/expected/full/hello/hello_pb.lua in an LSP-aware editor and checking that hover on hello.Person, M.Person_encode shows the @class + typed signature\n4. Sanity: require a generated module from spike code and confirm autocomplete on the result-table field names","acceptance_criteria":"Plugin emits ---@class blocks + typed _encode/_decode signatures into every generated _pb.lua; just gen + just test pass; LSP hover on require('full.hello.hello_pb').Person_encode shows ---@param tbl hello.Person ---@return string; ---@class hello.Person is reachable via hover on a decoded value","status":"open","priority":2,"issue_type":"feature","owner":"bigbes@gmail.com","created_at":"2026-05-23T09:30:19Z","created_by":"Eugene Blikh","updated_at":"2026-05-23T09:30:19Z","labels":["codegen","lsp"],"dependencies":[{"issue_id":"tarantool-protobuf-e4t","depends_on_id":"tarantool-protobuf-74c","type":"blocks","created_at":"2026-05-23T12:30:18Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-74c","title":"LSP: annotate codec/grpc/json/wkt with EmmyLua types","description":"Phase 2 of the LSP / LLM affordance work (phase 1 landed in commit 8721548 — .luarc.json, runtime/pb/_types.lua, init.lua + lazy.lua annotations).\n\nAdd ---@param / ---@return annotations to the remaining public-surface modules so editor hover and LLM context show types beyond just init.lua and the lazy views:\n\n- runtime/pb/codec.lua — encode(desc, t) / decode(desc, b); compile_writers / compile_readers (called from pb.finalize_message)\n- runtime/pb/grpc.lua — loopback / multiplex transport factories; the service client (factory(transport) -\u003e client_methods) and server (impl -\u003e {service, methods}) shapes\n- runtime/pb/json.lua — pb.json.encode(desc, t, opts) / pb.json.decode(desc, s, opts); opts shape (preserve_proto_field_names, emit_unpopulated, ...)\n- runtime/pb/wkt.lua — register(full_name, desc), lookup, any_pack/any_unpack, NULL sentinel\n\nFoundational @class declarations (pb.Descriptor, pb.Field, pb.GrpcTransport, pb.Json, pb.Wkt, pb.Module) already live in runtime/pb/_types.lua — extend as needed for opts shapes / map\u003cK,V\u003e generics.\n\nPure metadata change — no runtime behavior. Verify by running just test (must stay at 748 passing) and by opening a generated _pb.lua in an LSP-aware editor and confirming hover on require('pb').encode / .grpc.loopback / .json.encode shows the right signatures.","acceptance_criteria":"just test still green (748+ tests); hover in lua-language-server-aware editor shows typed signatures for pb.encode, pb.decode, grpc.loopback, json.encode, json.decode, wkt.register","status":"open","priority":2,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-23T09:29:55Z","created_by":"Eugene Blikh","updated_at":"2026-05-23T09:29:55Z","labels":["docs","lsp"],"dependency_count":0,"dependent_count":1,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-rmf","title":"ra6 3k: WKT override-hook passthrough","description":"If a plan's descriptor has desc.encode and desc.decode set (the WKT pattern from runtime/pb/wkt.lua), the C runtime must call those Lua functions instead of walking fields. Plan compiler stores luaL_ref to those functions; encode/decode entry points check first. This is also the extension point for future per-message codegen C (c0i) — same hook. Depends on 3a only (just adds a check at entry to the encode/decode loop). Acceptance: hello.Event (uses Timestamp, Duration, Any, FieldMask, Wrappers, Struct) round-trips with PB_ENABLE_C=1 producing byte-equal output to mode=full; the WKT module is unmodified.","status":"open","priority":2,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-18T20:21:13Z","created_by":"Eugene Blikh","updated_at":"2026-05-18T20:21:13Z","dependencies":[{"issue_id":"tarantool-protobuf-rmf","depends_on_id":"tarantool-protobuf-mq7","type":"blocks","created_at":"2026-05-18T23:21:55Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-m7u","title":"ra6 3i: proto2 — required, defaults, groups, extensions","description":"Proto2 semantic differences from proto3 that the C runtime must honor: (1) Required fields: decode-time enforcement when explicitly opted in (or default to skipping per existing pure-Lua behavior — match exactly); encode-time enforcement same. (2) Field defaults: present in the plan; emit them when missing on encode and substitute on decode where proto2 semantics require. (3) Groups (SGROUP/EGROUP wire format): encode/decode the framed group syntax. (4) Extensions: walk extension ranges, use plan-cached extension descriptor pointers, store in result._extensions. (5) Closed enum semantics for proto2. Depends on 3b + 3c + 3d + 3e + 3g + 3j (it's the join point — proto2 touches scalars, sub-messages, repeated, oneofs, unknown-fields). Acceptance: proto2_basic.BenchPayload and test_messages_proto2 round-trip byte-equal to mode=full at all bench sizes; conformance proto2 suite passes under PB_ENABLE_C=1.","status":"open","priority":2,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-18T20:21:03Z","created_by":"Eugene Blikh","updated_at":"2026-05-18T20:21:03Z","dependencies":[{"issue_id":"tarantool-protobuf-m7u","depends_on_id":"tarantool-protobuf-hwe","type":"blocks","created_at":"2026-05-18T23:22:21Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-m7u","depends_on_id":"tarantool-protobuf-jc9","type":"blocks","created_at":"2026-05-18T23:22:22Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-m7u","depends_on_id":"tarantool-protobuf-mz6","type":"blocks","created_at":"2026-05-18T23:22:20Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-m7u","depends_on_id":"tarantool-protobuf-w3u","type":"blocks","created_at":"2026-05-18T23:22:22Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-m7u","depends_on_id":"tarantool-protobuf-wyp","type":"blocks","created_at":"2026-05-18T23:22:23Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-m7u","depends_on_id":"tarantool-protobuf-y1n","type":"blocks","created_at":"2026-05-18T23:22:20Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":6,"dependent_count":1,"comment_count":0} -{"_type":"issue","id":"tarantool-protobuf-asz","title":"ra6 3h: maps (entry-as-pseudo-message)","description":"map\u003cK,V\u003e fields are wire-encoded as repeated messages with synthetic Entry { key=1; value=2 } shape. Compile a map plan: outer field is K_REPEATED_MESSAGE with a synthesized Entry sub-plan. Encode: walk the Lua map via pairs(), emit each entry (this is the documented JIT exception — map fields are allowed pairs() in the hot path per CLAUDE.md). Decode: each entry yields a key + value, lua_settable into the result map. Depends on 3d (uses sub-message machinery). Acceptance: Person.ages_by_nickname / nickname_by_age / addresses_by_label round-trip; multi-key map fixtures behave correctly even though encode byte order differs (the existing map_test pattern).","status":"open","priority":2,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-18T20:20:57Z","created_by":"Eugene Blikh","updated_at":"2026-05-18T20:20:57Z","dependencies":[{"issue_id":"tarantool-protobuf-asz","depends_on_id":"tarantool-protobuf-hwe","type":"blocks","created_at":"2026-05-18T23:22:19Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0} +{"_type":"issue","id":"tarantool-protobuf-asz","title":"ra6 3h: maps (entry-as-pseudo-message)","description":"map\u003cK,V\u003e fields are wire-encoded as repeated messages with synthetic Entry { key=1; value=2 } shape. Compile a map plan: outer field is K_REPEATED_MESSAGE with a synthesized Entry sub-plan. Encode: walk the Lua map via pairs(), emit each entry (this is the documented JIT exception — map fields are allowed pairs() in the hot path per CLAUDE.md). Decode: each entry yields a key + value, lua_settable into the result map. Depends on 3d (uses sub-message machinery). Acceptance: Person.ages_by_nickname / nickname_by_age / addresses_by_label round-trip; multi-key map fixtures behave correctly even though encode byte order differs (the existing map_test pattern).","status":"closed","priority":2,"issue_type":"task","assignee":"Eugene Blikh","owner":"bigbes@gmail.com","created_at":"2026-05-18T20:20:57Z","created_by":"Eugene Blikh","updated_at":"2026-05-23T17:07:06Z","started_at":"2026-05-23T16:45:29Z","closed_at":"2026-05-23T17:07:06Z","close_reason":"ra6 3h: maps (entry-as-pseudo-message) — encode walks the user map with lua_next (CLAUDE.md JIT exception), builds each entry payload in a stack-backed sub-buffer with synthetic tag(1,key) + tag(2,value), proto3-elides defaults independently for key and value, and emits outer tag + len + payload into the parent buffer. Map\u003c,message\u003e resolves the value sub-plan once and recurses through encode_body. Decode reads the entry payload bounded, dispatches inner id=1/id=2 (skipping anything else per spec), and lua_rawsets into a lazy-created result map table; missing key or value falls back to the proto3 zero. Reuses the existing list_stack_idx[] slot for the lazy map cache (a field is either repeated or map, never both). 12 new tests cover round-trip for ages_by_nickname (string→int32), nickname_by_age (int32→string), and addresses_by_label (string→message), plus default-elision, multi-key correctness, empty maps, and unknown-inner-id tolerance. Full suite 964/964 with PB_ENABLE_C=1.","dependencies":[{"issue_id":"tarantool-protobuf-asz","depends_on_id":"tarantool-protobuf-hwe","type":"blocks","created_at":"2026-05-18T23:22:19Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-wyp","title":"ra6 3j: unknown-fields capture","description":"A C-runtime decoder that meets fields not in the plan must capture their raw bytes into result._unknown_fields, and a subsequent C encode must re-emit them verbatim — same contract as test/unknown_test.lua against the pure-Lua paths. Bytes go in as a Lua string keyed by field number (mirror the existing convention). Depends on 3b + 3c (need the basic decode loop to know where to splice the unknown bytes). Acceptance: test/unknown_test.lua passes under PB_ENABLE_C=1; an evolved schema decoding bytes written by an old schema preserves the unknown fields through a C-side re-encode.","status":"open","priority":2,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-18T20:20:40Z","created_by":"Eugene Blikh","updated_at":"2026-05-18T20:20:40Z","dependencies":[{"issue_id":"tarantool-protobuf-wyp","depends_on_id":"tarantool-protobuf-mz6","type":"blocks","created_at":"2026-05-18T23:22:10Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-wyp","depends_on_id":"tarantool-protobuf-y1n","type":"blocks","created_at":"2026-05-18T23:22:09Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":2,"dependent_count":2,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-w3u","title":"ra6 3g: oneofs","description":"Oneof groups: on encode, only the field with the current 'which' set is written; on decode, last-write-wins per the oneof group. Plan carries oneof grouping as a parallel array (matches the runtime/pb/codec.lua desc.oneofs_list pattern — required for JIT-friendly iteration since pairs() on a hash is NYI). Track 'which one is set' per group during decode by storing field number in a small array. Depends on 3b + 3c. Acceptance: hello.Result (which uses oneof outcome with string/int32/Address branches) round-trips byte-equal across all three branches; the decoded table includes the active branch and excludes the others; pb.lazy view's :which() returns the correct branch name.","status":"closed","priority":2,"issue_type":"task","assignee":"Eugene Blikh","owner":"bigbes@gmail.com","created_at":"2026-05-18T20:20:36Z","created_by":"Eugene Blikh","updated_at":"2026-05-23T13:17:51Z","started_at":"2026-05-23T13:15:43Z","closed_at":"2026-05-23T13:17:51Z","close_reason":"Closed","dependencies":[{"issue_id":"tarantool-protobuf-w3u","depends_on_id":"tarantool-protobuf-mz6","type":"blocks","created_at":"2026-05-18T23:22:08Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-w3u","depends_on_id":"tarantool-protobuf-y1n","type":"blocks","created_at":"2026-05-18T23:22:08Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":2,"dependent_count":2,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-21d","title":"runtime/Person_encode emits 2.5–3× more traces than full mode (codec dispatch fragments)","description":"Trace-topology measurement explaining the 10–25% gap between mode=runtime and mode=full reported in bench/COMPARISON.md.\n\nMeasured on Tarantool 3.8.0-entrypoint / LuaJIT 2.1.0-beta3 / macOS arm64 via bench/jit_trace.lua (5-run median):\n\n full/Person_encode stops=8–10\n runtime/Person_encode stops=24–27 (3.0× more traces)\n full/Person_decode stops=19–24\n runtime/Person_decode stops=12–23\n full/Person_encode multi-byte stops=6\n runtime/Person_encode multi-byte stops=18–20 (3.0×)\n\nThe runtime-side encode in runtime/pb/codec.lua dispatches on field.kind / field.proto_type per field per iteration. Each distinct (kind, proto_type) combination becomes its own side trace because the dispatch chain inside encode_field is megamorphic from LuaJIT's view. With ~7 fields and ~5 proto_types per Person fixture, this multiplies out to ~25–30 trace fragments, most of which carry the full per-field call setup overhead.\n\nThe compile_writers / compile_readers hooks in runtime/pb/init.lua already attach per-field monomorphic functions (codec.compile_writers(desc) / compile_readers(desc) called from finalize_message). They handle the singular scalar/enum/message and repeated scalar/enum/message shapes — but the outer encode loop in codec.encode_message still iterates desc.fields and branches per-shape before calling the writer. The branch is what fragments.\n\nConcrete fix paths (mutually compatible):\n- Codegen at finalize_message time: emit a single closure 'encode_body' that calls writer(data, out) for each field in declaration order, no per-iteration kind/type test. The body is monomorphic because each writer is monomorphic.\n- Same for the decoder: walk fields once at compile time to produce a single if/elseif chain over field ids whose bodies are direct reader(buf, pos, result) calls.\n\nExpected impact: most of the runtime-vs-full gap in COMPARISON.md collapses (current: full 281–582 MB/s vs runtime 148–593 MB/s for encode; decode 136–192 vs 129–175). Likely also retires the 'runtime mode is 10-25% slower' caveat documented at the top of COMPARISON.md.\n\nValidation:\n- bench/jit_trace.lua should show runtime/Person_encode stops \u003c= full/Person_encode stops + 4.\n- bench/bench.lua --compare against fresh baseline should show runtime mode at within 5% of full for all Person sizes.","status":"open","priority":2,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-18T17:27:19Z","created_by":"Eugene Blikh","updated_at":"2026-05-18T17:27:19Z","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/runtime/pb/c/c_runtime.c b/runtime/pb/c/c_runtime.c index 186334355e287cca9d01293deeb521b60fb00e7d..e971d179127a0d338daa98bf0f57d61a8e7cdd9d 100644 --- a/runtime/pb/c/c_runtime.c +++ b/runtime/pb/c/c_runtime.c @@ -1181,10 +1181,198 @@ luaL_error(L, "kind %d not packable", (int)f->kind); } } +/* ---------------------------------------------------------------- * + * Map helpers (bd-asz / ra6 3h). * + * * + * A map field is wire-encoded as a repeated message with a * + * synthetic Entry { key=1; value=2 } shape. Each entry payload * + * contains zero/one key followed by zero/one value (defaults are * + * proto3-elided on encode and re-defaulted on decode). * + * * + * These helpers work with raw kind values (uint8_t) rather than * + * pb_plan_field pointers because map K and V are not full * + * pb_plan_fields in the plan — only their kinds are recorded. * + * Enum values are accepted only as numeric ordinals (no enum_ref * + * is stored per map value; map<,enum> with string-form enums is * + * not in the acceptance set for 3h). * + * ---------------------------------------------------------------- */ + +static uint8_t +wire_type_for_kind(uint8_t kind) +{ + switch (kind) { + case PB_KIND_INT32: case PB_KIND_INT64: + case PB_KIND_UINT32: case PB_KIND_UINT64: + case PB_KIND_SINT32: case PB_KIND_SINT64: + case PB_KIND_BOOL: case PB_KIND_ENUM: + return PB_WIRE_VARINT; + case PB_KIND_FIXED32: case PB_KIND_SFIXED32: case PB_KIND_FLOAT: + return PB_WIRE_I32; + case PB_KIND_FIXED64: case PB_KIND_SFIXED64: case PB_KIND_DOUBLE: + return PB_WIRE_I64; + case PB_KIND_STRING: case PB_KIND_BYTES: case PB_KIND_MESSAGE: + return PB_WIRE_LEN; + default: + return 0; + } +} + +/* Write a single value into `b` for the given kind. No tag, no length- + * prefix for non-LEN kinds. STRING/BYTES include the varint length per + * wire spec. MESSAGE is NOT handled here — sub-message values need a + * sub-plan and recurse through encode_body. */ +static void +encode_kind_value(lua_State *L, enc_buf *b, uint8_t kind, int val_idx) +{ + switch (kind) { + case PB_KIND_INT32: + case PB_KIND_INT64: + case PB_KIND_UINT32: + case PB_KIND_UINT64: { + uint64_t u = to_uint64_at(L, val_idx); + ebuf_reserve(L, b, 10); + ebuf_put_varint(b, u); + return; + } + case PB_KIND_SINT32: { + int32_t s = to_int32_at(L, val_idx); + ebuf_reserve(L, b, 5); + ebuf_put_varint(b, zigzag32(s)); + return; + } + case PB_KIND_SINT64: { + int64_t s = to_int64_at(L, val_idx); + ebuf_reserve(L, b, 10); + ebuf_put_varint(b, zigzag64(s)); + return; + } + case PB_KIND_BOOL: { + int truthy = lua_toboolean(L, val_idx); + ebuf_reserve(L, b, 1); + ebuf_put_byte(b, truthy ? 1 : 0); + return; + } + case PB_KIND_FIXED32: + case PB_KIND_SFIXED32: { + uint64_t u = to_uint64_at(L, val_idx); + ebuf_reserve(L, b, 4); + ebuf_put_fixed32(b, (uint32_t)u); + return; + } + case PB_KIND_FIXED64: + case PB_KIND_SFIXED64: { + uint64_t u = to_uint64_at(L, val_idx); + ebuf_reserve(L, b, 8); + ebuf_put_fixed64(b, u); + return; + } + case PB_KIND_FLOAT: { + double d = lua_tonumber(L, val_idx); + ebuf_reserve(L, b, 4); + ebuf_put_fixed32(b, f32_to_u32((float)d)); + return; + } + case PB_KIND_DOUBLE: { + double d = lua_tonumber(L, val_idx); + ebuf_reserve(L, b, 8); + ebuf_put_fixed64(b, f64_to_u64(d)); + return; + } + case PB_KIND_ENUM: { + int32_t e = (int32_t)to_int64_at(L, val_idx); + ebuf_reserve(L, b, 10); + ebuf_put_varint(b, (uint64_t)(int64_t)e); + return; + } + case PB_KIND_STRING: + case PB_KIND_BYTES: { + size_t len; + const char *s = lua_tolstring(L, val_idx, &len); + if (s == NULL) + luaL_error(L, "map string/bytes value must be a string"); + ebuf_reserve(L, b, 10 + len); + ebuf_put_varint(b, (uint64_t)len); + if (len > 0) + ebuf_put_bytes(b, (const uint8_t *)s, len); + return; + } + default: + luaL_error(L, "encode_kind_value: unsupported kind %d", + (int)kind); + } +} + +/* Proto3 default predicate for a Lua value of a given kind. Used to + * elide key/value bytes in map entries when they collapse to the type + * zero, matching the runtime Lua codec's `is_default_scalar` behavior. */ +static int +value_is_default_kind(lua_State *L, uint8_t kind, int val_idx) +{ + switch (kind) { + case PB_KIND_INT32: case PB_KIND_UINT32: + case PB_KIND_SINT32: case PB_KIND_FIXED32: case PB_KIND_SFIXED32: + case PB_KIND_ENUM: + if (lua_type(L, val_idx) == LUA_TNUMBER) + return lua_tonumber(L, val_idx) == 0; + return luaL_toint64(L, val_idx) == 0; + case PB_KIND_INT64: case PB_KIND_UINT64: + case PB_KIND_SINT64: case PB_KIND_FIXED64: case PB_KIND_SFIXED64: + if (lua_type(L, val_idx) == LUA_TNUMBER) + return lua_tonumber(L, val_idx) == 0; + return luaL_toint64(L, val_idx) == 0; + case PB_KIND_BOOL: + return !lua_toboolean(L, val_idx); + case PB_KIND_FLOAT: + case PB_KIND_DOUBLE: { + double d = lua_tonumber(L, val_idx); + if (d != 0.0) return 0; + /* -0.0 still emits — its bit pattern is non-zero. */ + union { double d; uint64_t u; } pun; + pun.d = d; + return pun.u == 0; + } + case PB_KIND_STRING: + case PB_KIND_BYTES: { + size_t len; + lua_tolstring(L, val_idx, &len); + return len == 0; + } + default: + return 0; + } +} + +/* Push the proto3 zero value for the given kind. MESSAGE pushes an + * empty table — used as a placeholder before the message decode body + * replaces it. */ +static void +push_default_for_kind(lua_State *L, uint8_t kind) +{ + switch (kind) { + case PB_KIND_BOOL: + lua_pushboolean(L, 0); return; + case PB_KIND_STRING: + case PB_KIND_BYTES: + lua_pushlstring(L, "", 0); return; + case PB_KIND_INT64: luaL_pushint64(L, 0); return; + case PB_KIND_UINT64: luaL_pushuint64(L, 0); return; + case PB_KIND_SINT64: luaL_pushint64(L, 0); return; + case PB_KIND_FIXED64: luaL_pushuint64(L, 0); return; + case PB_KIND_SFIXED64: luaL_pushint64(L, 0); return; + case PB_KIND_MESSAGE: + lua_newtable(L); return; + default: + /* int32/uint32/sint32/fixed32/sfixed32/enum/float/double → 0 */ + lua_pushinteger(L, 0); return; + } +} + /* Forward decls for the recursive encode pair. */ static void encode_body(lua_State *L, enc_buf *b, pb_plan *plan, int msg_idx); static void encode_submessage_field(lua_State *L, enc_buf *b, pb_plan *plan, pb_plan_field *f, int val_idx); +static void encode_map_field(lua_State *L, enc_buf *b, pb_plan *plan, + pb_plan_field *f, int val_idx); /* Encode a repeated field's elements into `b`. Dispatches on element * kind and the `packed` plan flag: @@ -1307,6 +1495,115 @@ lua_settop(L, saved_top); } +/* Encode a map field into `b`. + * + * Wire shape: each (k, v) pair becomes a length-delimited entry sub- + * message with synthetic field-1 key + field-2 value. Proto3 default- + * elision applies independently to key and value (key=='' or k==0 skip + * the key tag; v==zero skips the value tag). Sub-message values are + * always emitted regardless of contents (proto3 message presence). + * + * Iteration uses `lua_next` (pairs semantics) — this is the documented + * JIT exception per CLAUDE.md. Map encode is not on the JIT-traced hot + * path; hash iteration is the only way to walk user-provided table keys. + * + * Stack/buffer lifecycle mirrors encode_submessage_field: the parent's + * heap is force-established before any per-entry sub-buffer can allocate, + * so the parent's heap_idx survives the per-iter `lua_settop` cleanup. */ +static void +encode_map_field(lua_State *L, enc_buf *b, pb_plan *plan, + pb_plan_field *f, int val_idx) +{ + val_idx = abs_idx(L, val_idx); + if (lua_type(L, val_idx) != LUA_TTABLE) + luaL_error(L, "map field requires a table value"); + + /* Fast empty-check: probe the first iter step and bail if nothing. */ + lua_pushnil(L); + if (lua_next(L, val_idx) == 0) + return; + lua_pop(L, 2); /* drop probe k+v */ + + /* Force parent's heap to exist before per-entry sub-bufs allocate. */ + if (b->heap_idx == 0) + ebuf_grow(L, b, 1); + + /* Pre-compute entry-internal tags. Both ids are < 16 so they fit + * in a single varint byte. */ + uint8_t key_wt = wire_type_for_kind(f->map_key_kind); + uint8_t val_wt = wire_type_for_kind(f->map_value_kind); + uint8_t key_tag = (uint8_t)((1u << 3) | key_wt); + uint8_t val_tag = (uint8_t)((2u << 3) | val_wt); + + /* Resolve value sub-plan once if this is map. */ + pb_plan *value_subplan = NULL; + if (f->map_value_kind == PB_KIND_MESSAGE) { + if (plan->sub_plans_ref == LUA_NOREF || + f->map_value_sub_plan_idx <= 0) + luaL_error(L, + "map<,message> field has no value sub-plan"); + lua_rawgeti(L, LUA_REGISTRYINDEX, plan->sub_plans_ref); + lua_rawgeti(L, -1, f->map_value_sub_plan_idx); + value_subplan = (pb_plan *)lua_touserdata(L, -1); + lua_pop(L, 2); + if (value_subplan == NULL) + luaL_error(L, "map value sub-plan is not a userdata"); + } + + int saved_top = lua_gettop(L); + lua_pushnil(L); /* iter sentinel */ + while (lua_next(L, val_idx) != 0) { + int k_idx = saved_top + 1; + int v_idx = saved_top + 2; + + /* Build the entry payload in a stack-backed sub-buffer. */ + enc_buf entry; + ebuf_init(&entry); + + /* Key (proto3-elide on default). */ + if (!value_is_default_kind(L, f->map_key_kind, k_idx)) { + ebuf_reserve(L, &entry, 1); + ebuf_put_byte(&entry, key_tag); + encode_kind_value(L, &entry, f->map_key_kind, k_idx); + } + + /* Value. Messages always emit (presence is meaningful); other + * kinds proto3-elide on default. */ + if (f->map_value_kind == PB_KIND_MESSAGE) { + if (lua_type(L, v_idx) != LUA_TTABLE) + luaL_error(L, + "map<,message> value must be a table"); + enc_buf vbody; + ebuf_init(&vbody); + encode_body(L, &vbody, value_subplan, v_idx); + ebuf_reserve(L, &entry, + 1 + 10 + vbody.used); + ebuf_put_byte(&entry, val_tag); + ebuf_put_varint(&entry, (uint64_t)vbody.used); + if (vbody.used > 0) + ebuf_put_bytes(&entry, ebuf_base(&vbody), + vbody.used); + } else if (!value_is_default_kind(L, f->map_value_kind, + v_idx)) { + ebuf_reserve(L, &entry, 1); + ebuf_put_byte(&entry, val_tag); + encode_kind_value(L, &entry, f->map_value_kind, + v_idx); + } + + /* Emit outer tag + len-varint + entry body into parent. */ + ebuf_reserve(L, b, f->tag_len + 10 + entry.used); + ebuf_put_tag(b, f); + ebuf_put_varint(b, (uint64_t)entry.used); + if (entry.used > 0) + ebuf_put_bytes(b, ebuf_base(&entry), entry.used); + + /* Drop per-entry userdata frames, leave key for lua_next. */ + lua_settop(L, saved_top + 1); + } + /* lua_next returned 0 — it has already popped the final key. */ +} + static void encode_body(lua_State *L, enc_buf *b, pb_plan *plan, int msg_idx) { @@ -1347,9 +1644,6 @@ for (int i = 0; i < plan->n_fields; i++) { pb_plan_field *f = &plan->fields[i]; - /* Map dispatches in 3h. */ - if (f->kind == PB_KIND_MAP) continue; - /* Oneof: skip every non-active member. The active member * encodes with force_emit so default values (e.g. text="") * still carry presence. */ @@ -1365,7 +1659,9 @@ lua_pop(L, 1); continue; } - if (f->repeated) { + if (f->kind == PB_KIND_MAP) { + encode_map_field(L, b, plan, f, val_idx); + } else if (f->repeated) { if (lua_type(L, val_idx) != LUA_TTABLE) luaL_error(L, "repeated field requires a table value"); @@ -1513,11 +1809,14 @@ { return (int64_t)((u >> 1) ^ (~(u & 1) + 1)); } -/* Decode a single value for field `f` and push it onto the Lua stack. */ +/* Decode a single value of the given `kind` from the stream and push it + * onto the Lua stack. Per-kind Lua representations match wire.lua's + * decoder (cdata int64/uint64 for 64-bit kinds, Lua number/integer for + * everything else, lstring for STRING/BYTES). */ static void -dec_push_one(dec_ctx *c, pb_plan_field *f) +dec_push_kind(dec_ctx *c, uint8_t kind) { - switch (f->kind) { + switch (kind) { case PB_KIND_INT32: { uint64_t u = dec_varint(c); int32_t v = (int32_t)(uint32_t)u; @@ -1607,15 +1906,25 @@ c->pos += (size_t)plen; return; } default: - luaL_error(c->L, "dec_push_one: unsupported kind %d", - (int)f->kind); + luaL_error(c->L, "dec_push_kind: unsupported kind %d", + (int)kind); } } +/* Decode a single value for field `f` and push it onto the Lua stack. + * Thin wrapper around dec_push_kind to preserve the existing call sites. */ +static void +dec_push_one(dec_ctx *c, pb_plan_field *f) +{ + dec_push_kind(c, f->kind); +} + /* Forward decl for the recursive decode. */ static void decode_body(dec_ctx *c, pb_plan *plan, int result_idx); static void decode_submessage_field(dec_ctx *c, pb_plan_field *f, int sub_plans_idx); +static void decode_map_entry(dec_ctx *c, pb_plan_field *f, int sub_plans_idx, + int map_idx); /* Decode one singular sub-message field. On entry, `c->pos` points at * the length-varint byte; on exit, `c->pos == c->pos + plen`. Pushes @@ -1652,6 +1961,98 @@ (int)c->pos, (int)c->len); c->len = saved_len; } +/* Decode one map entry from the wire and lua_rawset it into the + * map table at absolute stack index `map_idx`. + * + * On entry `c->pos` points at the entry's length-varint (one element of + * the outer repeated-message stream). On exit `c->pos` has advanced past + * the entry. Pushes the {key, value} pair temporarily, sets into the + * map, then pops both. + * + * Inner-tag semantics: + * id == 1: key value + * id == 2: value value + * anything else: skipped per the wire-format spec for synthetic Entry + * + * Missing key or value defaults to the proto3 zero for the corresponding + * kind — that's how an empty entry payload `{}` round-trips through a + * map<,> field (see test_map_defaults_round_trip). For message-typed + * values, the default is an empty Lua table. */ +static void +decode_map_entry(dec_ctx *c, pb_plan_field *f, int sub_plans_idx, + int map_idx) +{ + lua_State *L = c->L; + uint64_t plen = dec_varint(c); + if (c->len - c->pos < plen) + luaL_error(L, "truncated map entry for field %d", + (int)f->field_number); + + size_t saved_len = c->len; + c->len = c->pos + (size_t)plen; + + /* Pre-push proto3 defaults; replace if the corresponding tag arrives. */ + push_default_for_kind(L, f->map_key_kind); + int key_idx = lua_gettop(L); + push_default_for_kind(L, f->map_value_kind); + int val_idx = lua_gettop(L); + + while (c->pos < c->len) { + uint64_t tag = dec_varint(c); + uint32_t id = (uint32_t)(tag >> 3); + uint8_t wt = (uint8_t)(tag & 0x07); + if (id == 1) { + dec_push_kind(c, f->map_key_kind); + lua_replace(L, key_idx); + } else if (id == 2) { + if (f->map_value_kind == PB_KIND_MESSAGE) { + if (f->map_value_sub_plan_idx <= 0) + luaL_error(L, + "map<,message> has no value sub-plan"); + lua_rawgeti(L, sub_plans_idx, + f->map_value_sub_plan_idx); + pb_plan *vsub = + (pb_plan *)lua_touserdata(L, -1); + if (vsub == NULL) + luaL_error(L, + "map value sub-plan is not a userdata"); + lua_pop(L, 1); + + uint64_t sub_len = dec_varint(c); + if (c->len - c->pos < sub_len) + luaL_error(L, + "truncated nested map<,message> value"); + lua_createtable(L, 0, vsub->n_fields); + int new_val = lua_gettop(L); + size_t saved2 = c->len; + c->len = c->pos + (size_t)sub_len; + decode_body(c, vsub, new_val); + if (c->pos != c->len) + luaL_error(L, + "nested map<,message> body underflow"); + c->len = saved2; + lua_replace(L, val_idx); + } else { + dec_push_kind(c, f->map_value_kind); + lua_replace(L, val_idx); + } + } else { + dec_skip(c, wt); + } + } + + /* map[key] = val — proto3 last-wins per key. cdata int64/uint64 keys + * would need a pointer-equality dedup pass (see runtime/pb/codec.lua's + * `key_dedup`); the acceptance set for 3h uses only string/int32 keys + * so we skip the dedup here. */ + lua_pushvalue(L, key_idx); + lua_pushvalue(L, val_idx); + lua_rawset(L, map_idx); + + lua_pop(L, 2); /* drop key + val defaults */ + c->len = saved_len; +} + /* Return 1 if the field's element wire type is varint/i32/i64 — i.e. * the field is eligible for packed encoding. Length-delimited kinds * (STRING/BYTES/MESSAGE) and MAP are never packable. Used on decode @@ -1736,9 +2137,35 @@ break; } } - /* Unknown tag, or map (3h territory). */ - if (f == NULL || f->kind == PB_KIND_MAP) { + /* Unknown tag. */ + if (f == NULL) { dec_skip(c, wt); + continue; + } + + /* ------------------------------------------------------ * + * Map dispatch (bd-asz / ra6 3h) * + * ------------------------------------------------------ */ + if (f->kind == PB_KIND_MAP) { + if (wt != PB_WIRE_LEN) + luaL_error(L, + "map field %d expected wire 2, got %d", + (int)field_number, (int)wt); + + /* Lazy-create the map table on first hit. Reuses the + * list_stack_idx[] slot since a field is either + * repeated or map, never both. */ + int m_idx = list_stack_idx[f_idx]; + if (m_idx == 0) { + lua_createtable(L, 0, 0); + lua_pushvalue(L, -1); + lua_rawgeti(L, names_idx, f_idx + 1); + lua_insert(L, -2); + lua_rawset(L, result_idx); + m_idx = lua_gettop(L); + list_stack_idx[f_idx] = m_idx; + } + decode_map_entry(c, f, sub_plans_idx, m_idx); continue; } diff --git a/test/c_runtime_decode_test.lua b/test/c_runtime_decode_test.lua index 72edb710fe5e6db47c0010d721f23ab4a461191c..73addb97125e9a6416e3217f27552fd4a0dd400a 100644 --- a/test/c_runtime_decode_test.lua +++ b/test/c_runtime_decode_test.lua @@ -129,26 +129,79 @@ local c_decoded = c_runtime.decode(plan, bytes) t.assert_equals(c_decoded.avatar, '\x00\x01\xff\xfe') end - function g.test_skips_map_tags() - -- 3e scope: singular message + repeated now decode. Map tags - -- (synthetic entry sub-messages) remain skipped — 3h replaces - -- the gate. + -- ---------- Map decode (bd-asz / ra6 3h) ---------- + + function g.test_map_string_to_int32_round_trip() + local plan = c_runtime.compile_plan(hello.Person_descriptor) + local msg = {ages_by_nickname = {alice = 30, bob = 25, carol = 40}} + local bytes = full_hello.Person_encode(msg) + local c_decoded = c_runtime.decode(plan, bytes) + t.assert_equals(c_decoded.ages_by_nickname, msg.ages_by_nickname) + end + + function g.test_map_int32_to_string_round_trip() + local plan = c_runtime.compile_plan(hello.Person_descriptor) + local msg = {nickname_by_age = {[30] = 'alice', [25] = 'bob'}} + local bytes = full_hello.Person_encode(msg) + local c_decoded = c_runtime.decode(plan, bytes) + t.assert_equals(c_decoded.nickname_by_age, msg.nickname_by_age) + end + + function g.test_map_string_to_message_round_trip() + local plan = c_runtime.compile_plan(hello.Person_descriptor) + local msg = {addresses_by_label = { + home = {street = 'Main', city = 'X', zip = 1}, + work = {street = '5th', city = 'Y', zip = 2}, + }} + local bytes = full_hello.Person_encode(msg) + local c_decoded = c_runtime.decode(plan, bytes) + t.assert_equals(c_decoded.addresses_by_label.home.street, 'Main') + t.assert_equals(c_decoded.addresses_by_label.home.zip, 1) + t.assert_equals(c_decoded.addresses_by_label.work.street, '5th') + t.assert_equals(c_decoded.addresses_by_label.work.city, 'Y') + end + + function g.test_map_defaults_decode_as_proto3_zeros() + -- mode=full encodes {[''] = 0} as a single-entry map with both + -- key and value bytes elided. C decoder must surface the proto3 + -- defaults from the (otherwise empty) entry payload. + local plan = c_runtime.compile_plan(hello.Person_descriptor) + local bytes = full_hello.Person_encode({ages_by_nickname = {[''] = 0}}) + local c_decoded = c_runtime.decode(plan, bytes) + t.assert_equals(c_decoded.ages_by_nickname[''], 0) + end + + function g.test_map_skips_unknown_entry_ids() + -- Synthetic Entry must tolerate unknown inner field ids (per the + -- proto wire spec for map<,>). Hand-craft an entry payload with a + -- spurious id=3 between key and value. local plan = c_runtime.compile_plan(hello.Person_descriptor) - local rich = { - name = 'x', - emails = {'a@b'}, -- repeated string (decode) - address = {street = 'Main'}, -- message (decode) - lucky_numbers = {1, 2, 3}, -- repeated packed (decode) - ages_by_nickname = {alice = 30}, -- map (skip) - } - local bytes = full_hello.Person_encode(rich) + local entry = '\x0a\x05alice' -- tag(1, LEN) + len(5) + "alice" + .. '\x18\x07' -- tag(3, VARINT) + value 7 (spurious) + .. '\x10\x1e' -- tag(2, VARINT) + 30 + local bytes = '\x6a' -- tag(13, LEN): outer map field + .. string.char(#entry) + .. entry local c_decoded = c_runtime.decode(plan, bytes) - t.assert_equals(c_decoded, { + t.assert_equals(c_decoded.ages_by_nickname.alice, 30) + end + + function g.test_map_mixed_with_other_fields() + local plan = c_runtime.compile_plan(hello.Person_descriptor) + local rich = { name = 'x', emails = {'a@b'}, address = {street = 'Main'}, lucky_numbers = {1, 2, 3}, - }) + ages_by_nickname = {alice = 30, bob = 25}, + } + local bytes = full_hello.Person_encode(rich) + local c_decoded = c_runtime.decode(plan, bytes) + t.assert_equals(c_decoded.name, 'x') + t.assert_equals(c_decoded.emails, {'a@b'}) + t.assert_equals(c_decoded.address.street, 'Main') + t.assert_equals(c_decoded.lucky_numbers, {1, 2, 3}) + t.assert_equals(c_decoded.ages_by_nickname, rich.ages_by_nickname) end -- ---------- Sub-message decode (bd-hwe / ra6 3d) ---------- diff --git a/test/c_runtime_encode_test.lua b/test/c_runtime_encode_test.lua index 842825a5e29b8721537625d494d973a23fb78d27..bdbef646cc981b8453a690c96cb528985d16bdcf 100644 --- a/test/c_runtime_encode_test.lua +++ b/test/c_runtime_encode_test.lua @@ -144,25 +144,75 @@ t.assert_equals(c_runtime.encode(plan, msg), full_hello.Person_encode(msg)) end - function g.test_skips_map_fields() - -- 3e scope: singular message + repeated (packed and unpacked) - -- now encode. Map fields remain silently skipped — 3h replaces - -- the gate. + -- ---------- Map encode (bd-asz / ra6 3h) ---------- + + function g.test_map_string_to_int32_single_key() + -- Single-key fixtures lock down byte-for-byte equality. Multi-key + -- map encode order is hash-determined per Lua's pairs() and won't + -- match mode=full's pairs() order in general, so we exercise + -- multi-key behavior via round-trip below. + local plan = c_runtime.compile_plan(hello.Person_descriptor) + local msg = {ages_by_nickname = {alice = 30}} + t.assert_equals(c_runtime.encode(plan, msg), + full_hello.Person_encode(msg)) + end + + function g.test_map_int32_to_string_single_key() + local plan = c_runtime.compile_plan(hello.Person_descriptor) + local msg = {nickname_by_age = {[30] = 'alice'}} + t.assert_equals(c_runtime.encode(plan, msg), + full_hello.Person_encode(msg)) + end + + function g.test_map_string_to_message_single_key() + local plan = c_runtime.compile_plan(hello.Person_descriptor) + local msg = {addresses_by_label = { + home = {street = 'Main', city = 'X', zip = 1}, + }} + t.assert_equals(c_runtime.encode(plan, msg), + full_hello.Person_encode(msg)) + end + + function g.test_map_empty_emits_nothing() + local plan = c_runtime.compile_plan(hello.Person_descriptor) + t.assert_equals(c_runtime.encode(plan, {ages_by_nickname = {}}), '') + end + + function g.test_map_default_key_and_value_round_trip() + -- Empty-string key + zero value: per proto3 wire spec both bytes + -- are elided but the entry itself is still emitted (presence of + -- the key/value pair is meaningful even when both default). + local plan = c_runtime.compile_plan(hello.Person_descriptor) + local msg = {ages_by_nickname = {[''] = 0}} + local c_bytes = c_runtime.encode(plan, msg) + t.assert_equals(c_bytes, full_hello.Person_encode(msg)) + local dec = full_hello.Person_decode(c_bytes) + t.assert_equals(dec.ages_by_nickname[''], 0) + end + + function g.test_map_multi_key_round_trip() + -- Multi-key encode byte order is hash-determined; we only assert + -- that mode=full can decode our bytes back to the same table. + local plan = c_runtime.compile_plan(hello.Person_descriptor) + local msg = {ages_by_nickname = {alice = 30, bob = 25, carol = 40}} + local dec = full_hello.Person_decode(c_runtime.encode(plan, msg)) + t.assert_equals(dec.ages_by_nickname, msg.ages_by_nickname) + end + + function g.test_map_message_value_with_other_fields() + -- Ensure map<,message> sub-plan resolution doesn't disturb the + -- enclosing message's field-walk. local plan = c_runtime.compile_plan(hello.Person_descriptor) local msg = { name = 'x', - emails = {'a@b'}, -- repeated string (encode) - address = {street = 'Main'}, -- message (encode) - lucky_numbers = {1, 2, 3}, -- repeated packed (encode) - ages_by_nickname = {alice = 30}, -- map (skip) - } - local expected = full_hello.Person_encode({ - name = 'x', + addresses_by_label = {home = {street = 'Main', zip = 7}}, emails = {'a@b'}, - address = {street = 'Main'}, - lucky_numbers = {1, 2, 3}, - }) - t.assert_equals(c_runtime.encode(plan, msg), expected) + } + local dec = full_hello.Person_decode(c_runtime.encode(plan, msg)) + t.assert_equals(dec.name, 'x') + t.assert_equals(dec.emails, {'a@b'}) + t.assert_equals(dec.addresses_by_label.home.street, 'Main') + t.assert_equals(dec.addresses_by_label.home.zip, 7) end -- ---------- Sub-message encode (bd-hwe / ra6 3d) ----------