diff --git a/.beads/interactions.jsonl b/.beads/interactions.jsonl index 65d63ac3e2a7274ea6acdba86e188ba48b431da6..2ea8797bf22ccaf3d69e697793901d2eb8773d54 100644 --- a/.beads/interactions.jsonl +++ b/.beads/interactions.jsonl @@ -19,3 +19,4 @@ {"id":"int-ba4ac242","kind":"field_change","created_at":"2026-05-19T04:23:56.245983Z","actor":"Eugene Blikh","issue_id":"tarantool-protobuf-47e","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Compatibility contract formalized in docs/specs/c_accel_compat.md. Spec covers: PB_ENABLE_C=1 single activation switch, silent fallback, full public surface preservation (pb.encode/decode/decode_lazy/parse/from_pb/json/text/grpc), generated module wrappers, 64-bit cdata invariant, WKT shapes, unknown fields, extensions, error compatibility, install paths (pure-Lua and C-enabled), ABI versioning, conformance via existing-suite re-run, explicit out-of-scope list. Lazy view stays Lua-only per spike rationale. Downstream tasks (mq7, ra6, 43t, c0i) reference this spec for compat boundaries."}} {"id":"int-951d03f7","kind":"field_change","created_at":"2026-05-19T04:24:12.141983Z","actor":"Eugene Blikh","issue_id":"tarantool-protobuf-z7x","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"C-side strategy formalized in docs/specs/c_accel_strategy.md. Spec covers: pb_plan / pb_plan_field struct layout, field-name luaL_ref caching (spike-validated), 4 KB stack-backed pb_buf with malloc promotion, lua_createtable(0, n_fields) result pre-sizing, cached per-field stack indices for repeated/packed (spike Phase B: 2x speedup at 100 KB), 64-bit cdata via luaT_pushint64/checkint64, sub-buffer (chosen) vs backpatching (deferred) for sub-messages, two-pass packed encode, map iteration via lua_next, oneof grouping, unknown field passthrough, deferred micro-opts (manual varint inlining, SIMD scan), C99/-O2 build environment, deferred PB_C_TRACE/PB_C_CHECK_PARITY env knobs. ra6/mq7 references this spec for low-level decisions."}} {"id":"int-7ce7c344","kind":"field_change","created_at":"2026-05-19T04:24:13.236096Z","actor":"Eugene Blikh","issue_id":"tarantool-protobuf-wky","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Build and packaging scaffolding landed: docs/specs/c_accel_build_packaging.md spec, runtime/pb/init.lua PB_ENABLE_C=1 opt-in pcall hook (exposed as pb.c_runtime field), Justfile build-c/clean-c recipes (stub erroring cleanly until bd-ra6 lands runtime/pb/c/), Justfile lua_cpath constant + LUA_CPATH on test/test-one, .gitignore for compiled .so/.dylib, sourcehut .builds/{pure-lua,c-enabled}.yml manifests (ubuntu/noble, parallel jobs via separate files since sourcehut has no matrix), .sourcehut/conformance.yml for manual conformance triggers. Rockspec changes deferred to first C-source landing (described in spec). All 745 tests pass with PB_ENABLE_C unset and PB_ENABLE_C=1 (silent fallback verified)."}} {"id":"int-b6e91276","kind":"field_change","created_at":"2026-05-19T04:46:39.175032Z","actor":"Eugene Blikh","issue_id":"tarantool-protobuf-mq7","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Descriptor -> C plan compiler landed in runtime/pb/c/c_runtime.c. Compiles a finalized Lua descriptor into an opaque pb_plan userdata with: per-field records (field_number, wire_type, kind, repeated/packed/optional, pre-encoded tag bytes, sub_plan_idx, oneof_idx, enum_ref), oneof grouping (parallel array with member indices + field oneof_idx back-pointers), WKT override pointers (has_override + override_encode_ref/decode_ref skipping field-walk), extension range hooks (proto2 scaffolding), field-name luaL_ref cache, sub-plan table for cycle-safe recursion. Self-references (Person.friends -> Person) handled by stashing c_plan on desc BEFORE recursing, so resolve_sub_plan returns the same userdata. Idempotent: re-compiling returns cached plan. Build via 'just build-c'. Smoke test at test/c_runtime_plan_test.lua: 26 assertions across both codegen modes (full + runtime) verify module surface, scalar/enum/message/map/repeated/packed shapes, sub-plan resolution, self-reference cycle break, oneof grouping (Result.outcome), idempotency, WKT override detection (Timestamp). Full suite: 771/771 with PB_ENABLE_C=1, 745+26 skipped without (silent fallback verified). Unblocks 3b/3c/3l/3k (y1n, mz6, awv, rmf)."}} +{"id":"int-59f687bd","kind":"field_change","created_at":"2026-05-23T09:41:30.983997Z","actor":"Eugene Blikh","issue_id":"tarantool-protobuf-y1n","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"C-side singular-scalar encode landed in runtime/pb/c/c_runtime.c. New entry pb.c_runtime.encode(plan, msg) -> string. Buffer strategy: 4KB stack scratch promoted to lua_newuserdata on overflow — mid-encode luaL_error doesn't leak (userdata GC'd at unwind). Covered kinds: int32/64, uint32/64, sint32/64, bool, fixed32/64, sfixed32/64, float, double, enum (number/string lookup via field->enum_ref's by_name), string, bytes. Repeated/map/message silently skipped (3d/3e scope). Proto3 zero-suppression mirrors mode=full exactly: empty strings, zero ints/fixed, +0.0 double skipped; -0.0 emitted via type-pun byte equality (matches Lua's 1/v == -math.huge guard); proto3-optional fields bypass suppression. WKT override plans rejected (3k/bd-rmf scope). New test test/c_runtime_encode_test.lua: 16 cases per mode (full + runtime) = 32 tests covering acceptance (Person {name='x', age=42, balance=-7, user_id=0xDEADBEEFCAFEBABEULL, weight_kg=3.14} byte-equal to mode=full), per-kind sweep, edge cases (proto3-optional empty string, -0.0 double, enum-string lookup + unknown-name error, cdata uint64 fixed64, heap-buffer grow at 8KB string, repeated/message-skip silent). Full suite: 806/806 with PB_ENABLE_C=1, 748/748 + 58 skipped without (silent fallback intact). Unblocks 3d/3e/3g/3i/3j."}} diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 0839441d170858d4833c29fb6433219b66eaf307..2a8a622787626d0ca663e34947f72502e720609d 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -3,7 +3,7 @@ {"_type":"issue","id":"tarantool-protobuf-jc9","title":"ra6 3e: repeated/packed scalar encode/decode","description":"Repeated scalar fields: encode-time, walk the Lua array via cached stack idx; decode-time, lazy-create the array on first hit and append via lua_rawseti with a cached stack idx (NOT lazy lua_getfield per element — that's 2x slower at 100KB per the spike). Packed repeated: length-prefixed varint blob; tight loop in C. Covers int32/sint32/uint32/int64/uint64/fixed32/fixed64/double/bool/enum. Depends on 3b + 3c (need the scalar primitives). Acceptance: Person.lucky_numbers (packed int32) round-trips byte-equal; a fixture with both packed and unpacked repeated scalars round-trips against mode=full at 10/100/1000-element counts.","status":"open","priority":1,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-18T20:20:33Z","created_by":"Eugene Blikh","updated_at":"2026-05-18T20:20:33Z","dependencies":[{"issue_id":"tarantool-protobuf-jc9","depends_on_id":"tarantool-protobuf-mz6","type":"blocks","created_at":"2026-05-18T23:22:07Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-jc9","depends_on_id":"tarantool-protobuf-y1n","type":"blocks","created_at":"2026-05-18T23:22:07Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":2,"dependent_count":2,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-hwe","title":"ra6 3d: sub-message encode/decode (recursion)","description":"Nested-message support. Encode: open a sub-buffer, recurse with the submsg plan, prefix parent buffer with tag + length. Decode: bound the byte range, recurse, lua_setfield the resulting table. Sub-buffer stack-backed (4KB) and malloc-promoted, same pattern as the outer buffer. Recursion is by C function call into the same encode/decode routine with a different plan, so depth is bounded by C stack. Depends on 3b + 3c. Acceptance: Person.address (1-level) round-trips; deeper nesting via a 5-level test fixture round-trips byte-equal to mode=full.","status":"open","priority":1,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-18T20:20:24Z","created_by":"Eugene Blikh","updated_at":"2026-05-18T20:20:24Z","dependencies":[{"issue_id":"tarantool-protobuf-hwe","depends_on_id":"tarantool-protobuf-mz6","type":"blocks","created_at":"2026-05-18T23:22:06Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-hwe","depends_on_id":"tarantool-protobuf-y1n","type":"blocks","created_at":"2026-05-18T23:22:05Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":2,"dependent_count":4,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-awv","title":"ra6 3l: 64-bit cdata fidelity","description":"int64/uint64/fixed64/sfixed64 must round-trip as LuaJIT cdata in both encode and decode — never narrowed to a Lua double. Same convention as msgpackffi, net.box, box.tuple, the built-in protobuf. Use luaT_pushuint64/luaT_pushint64 from Tarantool's module.h for the C side. Independent transverse concern — can land in parallel with 3b/3c since it's a per-kind fix rather than a new pipeline stage. Acceptance: a value \u003e 2^53 round-trips byte-equal and remains cdata after decode; luatest test/protobuf_test.lua int64-cdata cases pass with PB_ENABLE_C=1.","status":"open","priority":1,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-18T20:20:10Z","created_by":"Eugene Blikh","updated_at":"2026-05-18T20:20:10Z","dependencies":[{"issue_id":"tarantool-protobuf-awv","depends_on_id":"tarantool-protobuf-mq7","type":"blocks","created_at":"2026-05-18T23:21:55Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-mz6","title":"ra6 3c: decode scalars","description":"C-side decode for scalar wire types. Pre-sized result table via lua_createtable(0, n_fields) using descriptor stats; per-field plan-driven dispatch on wire type and kind; cached luaL_ref-based lua_setfield. Wire→table output must equal mode=full Lua output for the same input bytes. Depends on 3a. Acceptance: Person decode round-trip works for the bytes produced by 3b; the decoded table is shape-identical to pure-Lua decode (verified by running test/interop tests with PB_ENABLE_C=1).","status":"open","priority":1,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-18T20:20:06Z","created_by":"Eugene Blikh","updated_at":"2026-05-18T20:20:06Z","dependencies":[{"issue_id":"tarantool-protobuf-mz6","depends_on_id":"tarantool-protobuf-mq7","type":"blocks","created_at":"2026-05-18T23:21:54Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":1,"dependent_count":6,"comment_count":0} -{"_type":"issue","id":"tarantool-protobuf-y1n","title":"ra6 3b: encode scalars (varint, fixed32/64, double, bool, enum, sint)","description":"C-side encode for scalar wire types into the message's output buffer. Reads field values via cached luaL_ref + lua_rawgeti, dispatches on plan field kind, writes varint/fixed32/fixed64/double/bool/enum/sint into the 4KB stack-backed buffer with malloc-promotion on overflow. Wire-byte output must be byte-equal to mode=full Lua output. Depends on 3a (needs the plan). Acceptance: Person encode round-trip works for {name='x', age=42, balance=-7, user_id=0xDEADBEEFCAFEBABEULL, weight_kg=3.14} with byte-equal output to mode=full pure Lua.","status":"open","priority":1,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-18T20:20:03Z","created_by":"Eugene Blikh","updated_at":"2026-05-18T20:20:03Z","dependencies":[{"issue_id":"tarantool-protobuf-y1n","depends_on_id":"tarantool-protobuf-mq7","type":"blocks","created_at":"2026-05-18T23:21:53Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":1,"dependent_count":6,"comment_count":0} +{"_type":"issue","id":"tarantool-protobuf-y1n","title":"ra6 3b: encode scalars (varint, fixed32/64, double, bool, enum, sint)","description":"C-side encode for scalar wire types into the message's output buffer. Reads field values via cached luaL_ref + lua_rawgeti, dispatches on plan field kind, writes varint/fixed32/fixed64/double/bool/enum/sint into the 4KB stack-backed buffer with malloc-promotion on overflow. Wire-byte output must be byte-equal to mode=full Lua output. Depends on 3a (needs the plan). Acceptance: Person encode round-trip works for {name='x', age=42, balance=-7, user_id=0xDEADBEEFCAFEBABEULL, weight_kg=3.14} with byte-equal output to mode=full pure Lua.","status":"closed","priority":1,"issue_type":"task","assignee":"Eugene Blikh","owner":"bigbes@gmail.com","created_at":"2026-05-18T20:20:03Z","created_by":"Eugene Blikh","updated_at":"2026-05-23T09:41:31Z","started_at":"2026-05-23T09:32:14Z","closed_at":"2026-05-23T09:41:31Z","close_reason":"C-side singular-scalar encode landed in runtime/pb/c/c_runtime.c. New entry pb.c_runtime.encode(plan, msg) -\u003e string. Buffer strategy: 4KB stack scratch promoted to lua_newuserdata on overflow — mid-encode luaL_error doesn't leak (userdata GC'd at unwind). Covered kinds: int32/64, uint32/64, sint32/64, bool, fixed32/64, sfixed32/64, float, double, enum (number/string lookup via field-\u003eenum_ref's by_name), string, bytes. Repeated/map/message silently skipped (3d/3e scope). Proto3 zero-suppression mirrors mode=full exactly: empty strings, zero ints/fixed, +0.0 double skipped; -0.0 emitted via type-pun byte equality (matches Lua's 1/v == -math.huge guard); proto3-optional fields bypass suppression. WKT override plans rejected (3k/bd-rmf scope). New test test/c_runtime_encode_test.lua: 16 cases per mode (full + runtime) = 32 tests covering acceptance (Person {name='x', age=42, balance=-7, user_id=0xDEADBEEFCAFEBABEULL, weight_kg=3.14} byte-equal to mode=full), per-kind sweep, edge cases (proto3-optional empty string, -0.0 double, enum-string lookup + unknown-name error, cdata uint64 fixed64, heap-buffer grow at 8KB string, repeated/message-skip silent). Full suite: 806/806 with PB_ENABLE_C=1, 748/748 + 58 skipped without (silent fallback intact). Unblocks 3d/3e/3g/3i/3j.","dependencies":[{"issue_id":"tarantool-protobuf-y1n","depends_on_id":"tarantool-protobuf-mq7","type":"blocks","created_at":"2026-05-18T23:21:53Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":1,"dependent_count":6,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-mq7","title":"ra6 3a: descriptor → C plan compiler","description":"Compile finalized Lua descriptors into a C-side plan userdata, stashed on desc.c_plan. The plan carries per-field records (field number, wire type, kind tag, presence offset, default index), pre-encoded tag bytes, luaL_ref slots for cached field names, sub-descriptor pointers, oneof grouping metadata (parallel array, matches desc.oneofs_list), extension range hooks, and WKT override pointers (desc.encode/desc.decode passthrough). Rebuilt at pb.finalize_message time when PB_ENABLE_C=1. Not serialized, not shared across processes. Foundation for all other ra6 sub-issues — no other 3* issue can start until this lands. Acceptance: pb.c_runtime.compile_plan(desc) returns a userdata; plan-\u003en_fields and plan-\u003efields[i].tag are readable from a C-level smoke test; plans for hello.Person and Address build without error.","status":"closed","priority":1,"issue_type":"task","assignee":"Eugene Blikh","owner":"bigbes@gmail.com","created_at":"2026-05-18T20:19:58Z","created_by":"Eugene Blikh","updated_at":"2026-05-19T04:46:39Z","started_at":"2026-05-19T04:26:53Z","closed_at":"2026-05-19T04:46:39Z","close_reason":"Descriptor -\u003e C plan compiler landed in runtime/pb/c/c_runtime.c. Compiles a finalized Lua descriptor into an opaque pb_plan userdata with: per-field records (field_number, wire_type, kind, repeated/packed/optional, pre-encoded tag bytes, sub_plan_idx, oneof_idx, enum_ref), oneof grouping (parallel array with member indices + field oneof_idx back-pointers), WKT override pointers (has_override + override_encode_ref/decode_ref skipping field-walk), extension range hooks (proto2 scaffolding), field-name luaL_ref cache, sub-plan table for cycle-safe recursion. Self-references (Person.friends -\u003e Person) handled by stashing c_plan on desc BEFORE recursing, so resolve_sub_plan returns the same userdata. Idempotent: re-compiling returns cached plan. Build via 'just build-c'. Smoke test at test/c_runtime_plan_test.lua: 26 assertions across both codegen modes (full + runtime) verify module surface, scalar/enum/message/map/repeated/packed shapes, sub-plan resolution, self-reference cycle break, oneof grouping (Result.outcome), idempotency, WKT override detection (Timestamp). Full suite: 771/771 with PB_ENABLE_C=1, 745+26 skipped without (silent fallback verified). Unblocks 3b/3c/3l/3k (y1n, mz6, awv, rmf).","dependencies":[{"issue_id":"tarantool-protobuf-mq7","depends_on_id":"tarantool-protobuf-47e","type":"blocks","created_at":"2026-05-18T23:21:52Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-mq7","depends_on_id":"tarantool-protobuf-pf6","type":"blocks","created_at":"2026-05-18T23:21:52Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-mq7","depends_on_id":"tarantool-protobuf-z7x","type":"blocks","created_at":"2026-05-18T23:21:53Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":3,"dependent_count":5,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-1eq","title":"C accel spike: generic one-call C codec variant (strategy 3)","description":"Sub-issue of tarantool-protobuf-04c. Implement strategy 3: a generic C codec that gets a descriptor + Lua table once per top-level encode/decode call and owns the inner loop. Internally dispatches per-field on descriptor kind (the work pf6 / ra6 ends up shipping in production). Measure with bench/c_accel/spike_bench.lua. Acceptance: numbers in bench/c_accel/README.md showing the gap to strategy 4 (hand-written, no dispatch).","notes":"Phase A measured strategy 4 ceiling at 3-10x over pure Lua. The 4 vs 3 gap is the key signal for pf6 architecture: \u003c20-40% gap means generic C runtime (ra6) alone is sufficient; \u003e40% gap means we need codegen-emitted C (c0i). For the spike, dispatch can be simple (switch on protoreflect.Kind); no need to design the compiled-plan format here.","status":"closed","priority":1,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-18T19:28:08Z","created_by":"Eugene Blikh","updated_at":"2026-05-18T19:44:07Z","closed_at":"2026-05-18T19:44:07Z","close_reason":"Measured as part of the 04c spike; numbers and architecture conclusion captured in 04c close, bench/c_accel/README.md, and bd memory c-accel-spike-04c-final-2026-05-18.","dependencies":[{"issue_id":"tarantool-protobuf-1eq","depends_on_id":"tarantool-protobuf-04c","type":"blocks","created_at":"2026-05-18T22:28:15Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-80t","title":"C accel spike: per-primitive FFI variant (strategy 2)","description":"Sub-issue of tarantool-protobuf-04c. Implement strategy 2 of the C-acceleration benchmark: replace wire.lua's encode_varint / decode_varint / encode_string / decode_string / fixed32/64 / double with ffi.C.\u003cfn\u003e calls into a small C shim. The Lua dispatch loop stays; only the primitives cross. Measure with bench/c_accel/spike_bench.lua across the same 5 sizes. Acceptance: numbers added to bench/c_accel/README.md results table.","notes":"Phase A landed strategies 1 + 4. Strategy 4 (hand-written Person C codec) sets the upper bound at 3-10x over pure Lua. Strategy 2 must clear ~3x at every size to be competitive with 4; otherwise FFI per-primitive boundary cost is the wrong place to spend complexity. See bench/c_accel/README.md for full Phase A results.","status":"closed","priority":1,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-18T19:28:02Z","created_by":"Eugene Blikh","updated_at":"2026-05-18T19:44:07Z","closed_at":"2026-05-18T19:44:07Z","close_reason":"Measured as part of the 04c spike; numbers and architecture conclusion captured in 04c close, bench/c_accel/README.md, and bd memory c-accel-spike-04c-final-2026-05-18.","dependencies":[{"issue_id":"tarantool-protobuf-80t","depends_on_id":"tarantool-protobuf-04c","type":"blocks","created_at":"2026-05-18T22:28:15Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0} @@ -13,6 +13,8 @@ {"_type":"issue","id":"tarantool-protobuf-ra6","title":"C accel: generic C runtime codec with compiled descriptor plans","description":"Implement a pb.c runtime module that compiles finalized Lua descriptors into C-side plan userdata: field ids, wire types, cached Lua field-name refs, oneof metadata, extension ranges, map entry metadata, and WKT hooks. pb.encode/pb.decode can dispatch to the C plan when available and fall back to Lua otherwise. This targets runtime mode, dynamic schemas, and a shared substrate for generated C. Avoid reading descriptor Lua tables in the hot loop; compile once.","status":"open","priority":1,"issue_type":"feature","owner":"bigbes@gmail.com","created_at":"2026-05-17T16:32:31Z","created_by":"Eugene Blikh","updated_at":"2026-05-17T16:32:31Z","dependencies":[{"issue_id":"tarantool-protobuf-ra6","depends_on_id":"tarantool-protobuf-47e","type":"blocks","created_at":"2026-05-17T19:33:28Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-ra6","depends_on_id":"tarantool-protobuf-asz","type":"blocks","created_at":"2026-05-18T23:22:36Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-ra6","depends_on_id":"tarantool-protobuf-awv","type":"blocks","created_at":"2026-05-18T23:22:39Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-ra6","depends_on_id":"tarantool-protobuf-exy","type":"blocks","created_at":"2026-05-18T23:22:35Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-ra6","depends_on_id":"tarantool-protobuf-hwe","type":"blocks","created_at":"2026-05-18T23:22:34Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-ra6","depends_on_id":"tarantool-protobuf-jc9","type":"blocks","created_at":"2026-05-18T23:22:34Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-ra6","depends_on_id":"tarantool-protobuf-m7u","type":"blocks","created_at":"2026-05-18T23:22:37Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-ra6","depends_on_id":"tarantool-protobuf-mq7","type":"blocks","created_at":"2026-05-18T23:22:31Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-ra6","depends_on_id":"tarantool-protobuf-mz6","type":"blocks","created_at":"2026-05-18T23:22:33Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-ra6","depends_on_id":"tarantool-protobuf-pf6","type":"blocks","created_at":"2026-05-17T19:33:12Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-ra6","depends_on_id":"tarantool-protobuf-rmf","type":"blocks","created_at":"2026-05-18T23:22:38Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-ra6","depends_on_id":"tarantool-protobuf-w3u","type":"blocks","created_at":"2026-05-18T23:22:36Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-ra6","depends_on_id":"tarantool-protobuf-wyp","type":"blocks","created_at":"2026-05-18T23:22:37Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-ra6","depends_on_id":"tarantool-protobuf-y1n","type":"blocks","created_at":"2026-05-18T23:22:32Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-ra6","depends_on_id":"tarantool-protobuf-z7x","type":"blocks","created_at":"2026-05-17T19:33:28Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":15,"dependent_count":2,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-4kj","title":"Decoder: generated tag/length fast path for full-mode decode","description":"Inspection against starwing/lua-protobuf shows the 1KB+ Person decode gap is mostly repeated Lua-level tag/field dispatch, not a fatal JIT abort. On the 930B Person fixture, generated full decode is ~6.9-7.5 us/op, tag+length scan is ~2.1 us/op, skip_field scan is ~3.6 us/op, and an order-specialized decoder is ~4.3 us/op. Implement a full-mode generated fast path that decodes expected 1-byte tags and LEN prefixes inline at the call site, falling back to the generic decode_tag/skip path for unknown/out-of-order/multi-byte cases. Keep correctness for arbitrary field order, repeated occurrences, unknown fields, and proto2 semantics.","status":"closed","priority":1,"issue_type":"task","assignee":"Eugene Blikh","owner":"bigbes@gmail.com","created_at":"2026-05-17T16:02:41Z","created_by":"Eugene Blikh","updated_at":"2026-05-18T18:56:20Z","started_at":"2026-05-18T18:44:48Z","closed_at":"2026-05-18T18:56:20Z","close_reason":"Inlined wire.decode_tag's 1-byte fast path at every M.X_decode while-loop site, plus header localization of string.byte/bit.band/bit.rshift. Median-of-3 Person full decode: +7-15% across all sizes. Full encode flat to -2.6% (likely header upvalue layout). 745/745 tests, 37/37 JIT, 0 bridges. See bench/PERF_LOG.md entry. Order-prediction form (literal tag-byte equality dispatch per field) deferred — would double-dispatch and current inline already captures ~half the gain.","dependencies":[{"issue_id":"tarantool-protobuf-4kj","depends_on_id":"tarantool-protobuf-0an","type":"blocks","created_at":"2026-05-17T19:02:53Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-gcy","title":"Decoder: inline nested-message decode at the call site","description":"Today Person_decode calls Address_decode(slice) across a function boundary; the side trace into Address_decode may not stitch back (memory: luajit_side_trace_inlined_return). For field-typed messages in mode=full, emit the decode body inline at the call site. Expected: 20-40% on payloads with nested messages. Codegen growth is bounded by recursion depth — pick a depth limit and call out beyond it.","status":"closed","priority":1,"issue_type":"task","assignee":"Eugene Blikh","owner":"bigbes@gmail.com","created_at":"2026-05-17T15:47:09Z","created_by":"Eugene Blikh","updated_at":"2026-05-18T18:44:22Z","started_at":"2026-05-18T18:32:15Z","closed_at":"2026-05-18T18:44:22Z","close_reason":"Attempted but reverted. Inlining Address_decode body into Person_decode (singular non-recursive candidate) showed 745/745 tests passing and 37/37 JIT, but bench median-of-3 regressed Person 1KB/10KB/100KB decode by 5-8% and encode by 3-6%. Profile's '100% interpreter bail' claim turned out to be a vl trace-attribution artifact; LuaJIT was already inlining Address_decode into Person_decode's trace effectively. Larger root traces (22 stops) compiled less efficiently. See bench/PERF_LOG.md entry for full data and analysis. Real decode bottlenecks remain decode_string utf8 (6bb), decode_tag fast path (4kj), repeated append (cch).","labels":["codegen","decoder","perf"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-h8v","title":"Encoder: codegen-time inline FFI writes (mode=full)","description":"Replace per-field wire.encode_* calls + ../table.concat chain with directly-emitted FFI byte writes at every field site in mode=full. Today every encode_len(s) is 'encode_varint(#s) .. s' — two allocations and a concat per string. Sidesteps the per-byte b:alloc(1) cliff that sank the earlier ibuf attempt (memory: tarantool_ibuf_perf). Expected: 1.5-2x encode throughput; alloc/op drops from ~130 KB at 100 KB Person to near zero. Biggest single perf move. Bench reference: bench/starwing_bench.lua + bench/COMPARISON.md show starwing C encoder at 2.3-3x ours across all sizes.","status":"closed","priority":1,"issue_type":"task","assignee":"Eugene Blikh","owner":"bigbes@gmail.com","created_at":"2026-05-17T15:47:04Z","created_by":"Eugene Blikh","updated_at":"2026-05-18T18:31:54Z","started_at":"2026-05-18T18:25:39Z","closed_at":"2026-05-18T18:31:54Z","close_reason":"Inlined 1-byte varint length prefix at every LEN emit site; +25-48% encode at 1KB+ Person. Full FFI-buffer rewrite deferred — this slice captured the dominant profile target without that level of disruption. See bench/PERF_LOG.md entry for details.","labels":["codegen","encoder","perf"],"dependency_count":0,"dependent_count":2,"comment_count":0} +{"_type":"issue","id":"tarantool-protobuf-e4t","title":"LSP: plugin emits @class type stubs for generated messages","description":"Phase 3 of the LSP / LLM affordance work. Follows phase 1 (commit 8721548 — runtime annotations) and phase 2 (bd-74c — annotate codec/grpc/json/wkt).\n\nHighest-payoff phase for LLM grounding. Modify the protoc plugin (cmd/protoc-gen-tarantool/internal/gen) so each generated _pb.lua module emits ---@class blocks describing every message, plus typed @param/@return on the _encode/_decode/_descriptor surface. After this lands, anyone (human or LLM) who require('full.hello.hello_pb') gets full type info on Person, Address, etc., without reading the .proto.\n\nEmit shape (decision: inline at top of generated file; one file per .proto already includes all messages from that file):\n\n ---@class hello.Person\n ---@field name? string\n ---@field user_id? ffi.cdata* # uint64\n ---@field age? integer # int32\n ---@field weight_kg? number # double\n ---@field emails? string[]\n ---@field address? hello.Address\n ---@field lucky_numbers? integer[]\n ---@field favorite_color? hello.Color # enum alias\n ---@field tags? table\u003cstring,string\u003e # map\u003cstring,string\u003e\n ---@field unknown_fields? string\n\n ---@param tbl hello.Person\n ---@return string\n function M.Person_encode(tbl) ... end\n\n ---@param bytes string\n ---@return hello.Person\n function M.Person_decode(bytes) ... end\n\nField-kind -\u003e Lua-type mapping:\n- scalar int32/uint32/sint32/fixed32/sfixed32/bool → integer/boolean\n- scalar int64/uint64/sint64/fixed64/sfixed64 → ffi.cdata* (LuaJIT cdata, per project convention)\n- scalar float/double → number\n- scalar string/bytes → string\n- enum → alias of integer (emit ---@alias hello.Color integer)\n- message → another @class reference\n- repeated T → T[]\n- map\u003cK,V\u003e → table\u003cK_lua, V_lua\u003e\n- oneof — all members are optional, generator should NOT emit a discriminator field; user calls view:which() / inspects which value is non-nil\n\nAll fields are optional (? suffix) since proto3 default-elision means absence-on-wire is indistinguishable from default value. Required fields in proto2 omit the ?.\n\nBoth mode=full and mode=runtime emit the same _encode/_decode wrapper signatures, so the stubs apply uniformly.\n\nVerify by:\n1. Regenerating examples/expected/ via just gen\n2. Confirming the generated _pb.lua files load and pass tests (no behavior change)\n3. Opening examples/expected/full/hello/hello_pb.lua in an LSP-aware editor and checking that hover on hello.Person, M.Person_encode shows the @class + typed signature\n4. Sanity: require a generated module from spike code and confirm autocomplete on the result-table field names","acceptance_criteria":"Plugin emits ---@class blocks + typed _encode/_decode signatures into every generated _pb.lua; just gen + just test pass; LSP hover on require('full.hello.hello_pb').Person_encode shows ---@param tbl hello.Person ---@return string; ---@class hello.Person is reachable via hover on a decoded value","status":"open","priority":2,"issue_type":"feature","owner":"bigbes@gmail.com","created_at":"2026-05-23T09:30:19Z","created_by":"Eugene Blikh","updated_at":"2026-05-23T09:30:19Z","labels":["codegen","lsp"],"dependencies":[{"issue_id":"tarantool-protobuf-e4t","depends_on_id":"tarantool-protobuf-74c","type":"blocks","created_at":"2026-05-23T12:30:18Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"tarantool-protobuf-74c","title":"LSP: annotate codec/grpc/json/wkt with EmmyLua types","description":"Phase 2 of the LSP / LLM affordance work (phase 1 landed in commit 8721548 — .luarc.json, runtime/pb/_types.lua, init.lua + lazy.lua annotations).\n\nAdd ---@param / ---@return annotations to the remaining public-surface modules so editor hover and LLM context show types beyond just init.lua and the lazy views:\n\n- runtime/pb/codec.lua — encode(desc, t) / decode(desc, b); compile_writers / compile_readers (called from pb.finalize_message)\n- runtime/pb/grpc.lua — loopback / multiplex transport factories; the service client (factory(transport) -\u003e client_methods) and server (impl -\u003e {service, methods}) shapes\n- runtime/pb/json.lua — pb.json.encode(desc, t, opts) / pb.json.decode(desc, s, opts); opts shape (preserve_proto_field_names, emit_unpopulated, ...)\n- runtime/pb/wkt.lua — register(full_name, desc), lookup, any_pack/any_unpack, NULL sentinel\n\nFoundational @class declarations (pb.Descriptor, pb.Field, pb.GrpcTransport, pb.Json, pb.Wkt, pb.Module) already live in runtime/pb/_types.lua — extend as needed for opts shapes / map\u003cK,V\u003e generics.\n\nPure metadata change — no runtime behavior. Verify by running just test (must stay at 748 passing) and by opening a generated _pb.lua in an LSP-aware editor and confirming hover on require('pb').encode / .grpc.loopback / .json.encode shows the right signatures.","acceptance_criteria":"just test still green (748+ tests); hover in lua-language-server-aware editor shows typed signatures for pb.encode, pb.decode, grpc.loopback, json.encode, json.decode, wkt.register","status":"open","priority":2,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-23T09:29:55Z","created_by":"Eugene Blikh","updated_at":"2026-05-23T09:29:55Z","labels":["docs","lsp"],"dependency_count":0,"dependent_count":1,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-rmf","title":"ra6 3k: WKT override-hook passthrough","description":"If a plan's descriptor has desc.encode and desc.decode set (the WKT pattern from runtime/pb/wkt.lua), the C runtime must call those Lua functions instead of walking fields. Plan compiler stores luaL_ref to those functions; encode/decode entry points check first. This is also the extension point for future per-message codegen C (c0i) — same hook. Depends on 3a only (just adds a check at entry to the encode/decode loop). Acceptance: hello.Event (uses Timestamp, Duration, Any, FieldMask, Wrappers, Struct) round-trips with PB_ENABLE_C=1 producing byte-equal output to mode=full; the WKT module is unmodified.","status":"open","priority":2,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-18T20:21:13Z","created_by":"Eugene Blikh","updated_at":"2026-05-18T20:21:13Z","dependencies":[{"issue_id":"tarantool-protobuf-rmf","depends_on_id":"tarantool-protobuf-mq7","type":"blocks","created_at":"2026-05-18T23:21:55Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-m7u","title":"ra6 3i: proto2 — required, defaults, groups, extensions","description":"Proto2 semantic differences from proto3 that the C runtime must honor: (1) Required fields: decode-time enforcement when explicitly opted in (or default to skipping per existing pure-Lua behavior — match exactly); encode-time enforcement same. (2) Field defaults: present in the plan; emit them when missing on encode and substitute on decode where proto2 semantics require. (3) Groups (SGROUP/EGROUP wire format): encode/decode the framed group syntax. (4) Extensions: walk extension ranges, use plan-cached extension descriptor pointers, store in result._extensions. (5) Closed enum semantics for proto2. Depends on 3b + 3c + 3d + 3e + 3g + 3j (it's the join point — proto2 touches scalars, sub-messages, repeated, oneofs, unknown-fields). Acceptance: proto2_basic.BenchPayload and test_messages_proto2 round-trip byte-equal to mode=full at all bench sizes; conformance proto2 suite passes under PB_ENABLE_C=1.","status":"open","priority":2,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-18T20:21:03Z","created_by":"Eugene Blikh","updated_at":"2026-05-18T20:21:03Z","dependencies":[{"issue_id":"tarantool-protobuf-m7u","depends_on_id":"tarantool-protobuf-hwe","type":"blocks","created_at":"2026-05-18T23:22:21Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-m7u","depends_on_id":"tarantool-protobuf-jc9","type":"blocks","created_at":"2026-05-18T23:22:22Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-m7u","depends_on_id":"tarantool-protobuf-mz6","type":"blocks","created_at":"2026-05-18T23:22:20Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-m7u","depends_on_id":"tarantool-protobuf-w3u","type":"blocks","created_at":"2026-05-18T23:22:22Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-m7u","depends_on_id":"tarantool-protobuf-wyp","type":"blocks","created_at":"2026-05-18T23:22:23Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-m7u","depends_on_id":"tarantool-protobuf-y1n","type":"blocks","created_at":"2026-05-18T23:22:20Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":6,"dependent_count":1,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-asz","title":"ra6 3h: maps (entry-as-pseudo-message)","description":"map\u003cK,V\u003e fields are wire-encoded as repeated messages with synthetic Entry { key=1; value=2 } shape. Compile a map plan: outer field is K_REPEATED_MESSAGE with a synthesized Entry sub-plan. Encode: walk the Lua map via pairs(), emit each entry (this is the documented JIT exception — map fields are allowed pairs() in the hot path per CLAUDE.md). Decode: each entry yields a key + value, lua_settable into the result map. Depends on 3d (uses sub-message machinery). Acceptance: Person.ages_by_nickname / nickname_by_age / addresses_by_label round-trip; multi-key map fixtures behave correctly even though encode byte order differs (the existing map_test pattern).","status":"open","priority":2,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-18T20:20:57Z","created_by":"Eugene Blikh","updated_at":"2026-05-18T20:20:57Z","dependencies":[{"issue_id":"tarantool-protobuf-asz","depends_on_id":"tarantool-protobuf-hwe","type":"blocks","created_at":"2026-05-18T23:22:19Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0} diff --git a/runtime/pb/c/c_runtime.c b/runtime/pb/c/c_runtime.c index b540ce1cf0b83e660d43b870a7bedb02b94a8baf..82ecf6d733701573df4d73527e49dc054e2419a1 100644 --- a/runtime/pb/c/c_runtime.c +++ b/runtime/pb/c/c_runtime.c @@ -775,6 +775,375 @@ return 1; } /* ---------------------------------------------------------------- * + * Encode (bd-y1n / ra6 3b): singular scalar fields. * + * * + * Repeated/packed, map, and message-typed fields are skipped — * + * those land in 3d (sub-message) and 3e (repeated/packed). * + * * + * Buffer strategy: a 4KB stack-backed scratch buffer that promotes * + * to a Lua userdata (GC'd automatically) on overflow. Using * + * `lua_newuserdata` for heap growth means a luaL_error mid-encode * + * doesn't leak — the userdata is still on the stack at the unwind * + * point and gets collected normally. * + * ---------------------------------------------------------------- */ + +#define ENC_STACK_BUF 4096 + +typedef struct enc_buf { + uint8_t stack[ENC_STACK_BUF]; + uint8_t *heap; /* pointer into Lua userdata when grown; NULL while on stack */ + int heap_idx; /* stack slot of the userdata; 0 if not yet on heap */ + size_t cap; + size_t used; +} enc_buf; + +static inline uint8_t * +ebuf_base(enc_buf *b) +{ + return b->heap != NULL ? b->heap : b->stack; +} + +static void +ebuf_init(enc_buf *b) +{ + b->heap = NULL; + b->heap_idx = 0; + b->cap = ENC_STACK_BUF; + b->used = 0; +} + +static void +ebuf_grow(lua_State *L, enc_buf *b, size_t needed) +{ + size_t new_cap = b->cap * 2; + while (new_cap - b->used < needed) + new_cap *= 2; + + uint8_t *new_buf = (uint8_t *)lua_newuserdata(L, new_cap); + memcpy(new_buf, ebuf_base(b), b->used); + if (b->heap_idx == 0) { + b->heap_idx = lua_gettop(L); + } else { + lua_replace(L, b->heap_idx); + } + b->heap = new_buf; + b->cap = new_cap; +} + +static inline void +ebuf_reserve(lua_State *L, enc_buf *b, size_t needed) +{ + if (b->cap - b->used < needed) + ebuf_grow(L, b, needed); +} + +static inline void +ebuf_put_byte(enc_buf *b, uint8_t v) +{ + ebuf_base(b)[b->used++] = v; +} + +static inline void +ebuf_put_bytes(enc_buf *b, const uint8_t *src, size_t n) +{ + memcpy(ebuf_base(b) + b->used, src, n); + b->used += n; +} + +static inline void +ebuf_put_varint(enc_buf *b, uint64_t v) +{ + uint8_t *p = ebuf_base(b) + b->used; + while (v >= 0x80) { + *p++ = (uint8_t)(v | 0x80); + v >>= 7; + } + *p++ = (uint8_t)v; + b->used = (size_t)(p - ebuf_base(b)); +} + +static inline void +ebuf_put_fixed32(enc_buf *b, uint32_t v) +{ + uint8_t *p = ebuf_base(b) + b->used; + p[0] = (uint8_t)v; + p[1] = (uint8_t)(v >> 8); + p[2] = (uint8_t)(v >> 16); + p[3] = (uint8_t)(v >> 24); + b->used += 4; +} + +static inline void +ebuf_put_fixed64(enc_buf *b, uint64_t v) +{ + uint8_t *p = ebuf_base(b) + b->used; + p[0] = (uint8_t)v; + p[1] = (uint8_t)(v >> 8); + p[2] = (uint8_t)(v >> 16); + p[3] = (uint8_t)(v >> 24); + p[4] = (uint8_t)(v >> 32); + p[5] = (uint8_t)(v >> 40); + p[6] = (uint8_t)(v >> 48); + p[7] = (uint8_t)(v >> 56); + b->used += 8; +} + +static inline void +ebuf_put_tag(enc_buf *b, const pb_plan_field *f) +{ + memcpy(ebuf_base(b) + b->used, f->tag_bytes, f->tag_len); + b->used += f->tag_len; +} + +/* Read a Lua value as uint64. Mirrors wire.lua's to_uint64: negative + * Lua numbers are sign-extended through int64 (proto3 wire spec for + * int32 fields). int64/uint64 cdata flow through luaL_touint64. */ +static uint64_t +to_uint64_at(lua_State *L, int idx) +{ + if (lua_type(L, idx) == LUA_TNUMBER) { + double d = lua_tonumber(L, idx); + if (d < 0) + return (uint64_t)(int64_t)d; + return (uint64_t)d; + } + return luaL_touint64(L, idx); +} + +static int32_t +to_int32_at(lua_State *L, int idx) +{ + if (lua_type(L, idx) == LUA_TNUMBER) + return (int32_t)lua_tointeger(L, idx); + return (int32_t)luaL_toint64(L, idx); +} + +static int64_t +to_int64_at(lua_State *L, int idx) +{ + if (lua_type(L, idx) == LUA_TNUMBER) + return (int64_t)lua_tonumber(L, idx); + return luaL_toint64(L, idx); +} + +static inline uint32_t +zigzag32(int32_t n) +{ + return ((uint32_t)n << 1) ^ (uint32_t)(n >> 31); +} + +static inline uint64_t +zigzag64(int64_t n) +{ + return ((uint64_t)n << 1) ^ (uint64_t)(n >> 63); +} + +static inline uint32_t +f32_to_u32(float f) +{ + union { float f; uint32_t u; } pun; + pun.f = f; + return pun.u; +} + +static inline uint64_t +f64_to_u64(double d) +{ + union { double d; uint64_t u; } pun; + pun.d = d; + return pun.u; +} + +/* Resolve an enum field's Lua value to its int32 ordinal. Accepts + * numbers (returned directly), cdata int64 (downcast), or strings + * (looked up in field->enum_ref's by_name table). Mirrors the + * `if type(v) == 'string' then nv = M.Status[v]` branch in mode=full. */ +static int32_t +enum_value_at(lua_State *L, pb_plan_field *f, int idx) +{ + int t = lua_type(L, idx); + if (t == LUA_TSTRING) { + const char *s = lua_tostring(L, idx); + if (f->enum_ref == LUA_NOREF) + luaL_error(L, "enum field '%s' has no enum descriptor", s); + lua_rawgeti(L, LUA_REGISTRYINDEX, f->enum_ref); + lua_getfield(L, -1, "by_name"); + if (!lua_istable(L, -1)) { + luaL_error(L, "enum descriptor missing by_name"); + } + lua_pushvalue(L, idx); + lua_rawget(L, -2); + if (lua_isnil(L, -1)) { + luaL_error(L, "unknown enum value '%s'", s); + } + int32_t v = (int32_t)lua_tointeger(L, -1); + lua_pop(L, 3); /* value + by_name + desc */ + return v; + } + return (int32_t)to_int64_at(L, idx); +} + +/* Encode one singular scalar/enum/string field. Returns 1 if bytes + * were written, 0 if the value collapsed to its proto3 default and + * was suppressed. Zero-suppression skipped for proto3-optional. */ +static int +encode_one_field(lua_State *L, enc_buf *b, pb_plan_field *f, int val_idx) +{ + switch (f->kind) { + case PB_KIND_INT32: + case PB_KIND_INT64: + case PB_KIND_UINT32: + case PB_KIND_UINT64: { + uint64_t u = to_uint64_at(L, val_idx); + if (!f->optional && u == 0) return 0; + ebuf_reserve(L, b, f->tag_len + 10); + ebuf_put_tag(b, f); + ebuf_put_varint(b, u); + return 1; + } + case PB_KIND_SINT32: { + int32_t s = to_int32_at(L, val_idx); + if (!f->optional && s == 0) return 0; + ebuf_reserve(L, b, f->tag_len + 5); + ebuf_put_tag(b, f); + ebuf_put_varint(b, zigzag32(s)); + return 1; + } + case PB_KIND_SINT64: { + int64_t s = to_int64_at(L, val_idx); + if (!f->optional && s == 0) return 0; + ebuf_reserve(L, b, f->tag_len + 10); + ebuf_put_tag(b, f); + ebuf_put_varint(b, zigzag64(s)); + return 1; + } + case PB_KIND_BOOL: { + int truthy = lua_toboolean(L, val_idx); + if (!f->optional && !truthy) return 0; + ebuf_reserve(L, b, f->tag_len + 1); + ebuf_put_tag(b, f); + ebuf_put_byte(b, truthy ? 1 : 0); + return 1; + } + case PB_KIND_FIXED32: + case PB_KIND_SFIXED32: { + uint64_t u = to_uint64_at(L, val_idx); + uint32_t u32 = (uint32_t)u; + if (!f->optional && u32 == 0) return 0; + ebuf_reserve(L, b, f->tag_len + 4); + ebuf_put_tag(b, f); + ebuf_put_fixed32(b, u32); + return 1; + } + case PB_KIND_FIXED64: + case PB_KIND_SFIXED64: { + uint64_t u = to_uint64_at(L, val_idx); + if (!f->optional && u == 0) return 0; + ebuf_reserve(L, b, f->tag_len + 8); + ebuf_put_tag(b, f); + ebuf_put_fixed64(b, u); + return 1; + } + case PB_KIND_FLOAT: { + double d = lua_tonumber(L, val_idx); + uint32_t u = f32_to_u32((float)d); + /* +0.0 -> u==0 (skip); -0.0 -> u==0x80000000 (emit). Matches the + * Lua-side `(v ~= 0 or 1/v == -math.huge)` guard. */ + if (!f->optional && u == 0) return 0; + ebuf_reserve(L, b, f->tag_len + 4); + ebuf_put_tag(b, f); + ebuf_put_fixed32(b, u); + return 1; + } + case PB_KIND_DOUBLE: { + double d = lua_tonumber(L, val_idx); + uint64_t u = f64_to_u64(d); + if (!f->optional && u == 0) return 0; + ebuf_reserve(L, b, f->tag_len + 8); + ebuf_put_tag(b, f); + ebuf_put_fixed64(b, u); + return 1; + } + case PB_KIND_ENUM: { + int32_t e = enum_value_at(L, f, val_idx); + if (!f->optional && e == 0) return 0; + ebuf_reserve(L, b, f->tag_len + 10); + ebuf_put_tag(b, f); + /* enum is wire-equivalent to int32: sign-extend to uint64 then varint. */ + ebuf_put_varint(b, (uint64_t)(int64_t)e); + return 1; + } + case PB_KIND_STRING: + case PB_KIND_BYTES: { + if (lua_type(L, val_idx) != LUA_TSTRING) + luaL_error(L, "string/bytes field requires a string value"); + size_t len; + const char *s = lua_tolstring(L, val_idx, &len); + if (!f->optional && len == 0) return 0; + ebuf_reserve(L, b, f->tag_len + 10 + len); + ebuf_put_tag(b, f); + ebuf_put_varint(b, (uint64_t)len); + if (len > 0) + ebuf_put_bytes(b, (const uint8_t *)s, len); + return 1; + } + default: + /* Repeated/map/message are 3d/3e scope — silently skip. */ + return 0; + } +} + +static int +encode_lua(lua_State *L) +{ + pb_plan *plan = (pb_plan *)luaL_checkudata(L, 1, PB_PLAN_MT); + luaL_checktype(L, 2, LUA_TTABLE); + int msg_idx = 2; + + if (plan->has_override) { + return luaL_error(L, + "C encode on plan with override is unsupported (bd-rmf)"); + } + + /* Pin the field-names table on the stack for the loop's rawgeti. */ + if (plan->field_names_ref == LUA_NOREF || plan->n_fields == 0) { + lua_pushliteral(L, ""); + return 1; + } + lua_rawgeti(L, LUA_REGISTRYINDEX, plan->field_names_ref); + int names_idx = lua_gettop(L); + + enc_buf b; + ebuf_init(&b); + + for (int i = 0; i < plan->n_fields; i++) { + pb_plan_field *f = &plan->fields[i]; + + /* 3b scope: singular scalar/enum/string/bytes. Repeated, map, + * and message-typed fields are silently skipped — they land + * in 3d/3e and will overwrite this short-circuit. */ + if (f->repeated) continue; + if (f->kind == PB_KIND_MAP || f->kind == PB_KIND_MESSAGE) + continue; + + lua_rawgeti(L, names_idx, i + 1); /* push field name */ + lua_rawget(L, msg_idx); /* push msg[name] */ + int val_idx = lua_gettop(L); + + if (lua_isnil(L, val_idx)) { + lua_pop(L, 1); + continue; + } + + encode_one_field(L, &b, f, val_idx); + lua_pop(L, 1); + } + + lua_pushlstring(L, (const char *)ebuf_base(&b), b.used); + return 1; +} + +/* ---------------------------------------------------------------- * * Module entry. * * ---------------------------------------------------------------- */ @@ -787,6 +1156,7 @@ {"plan_n_oneofs", plan_n_oneofs}, {"plan_oneof_info", plan_oneof_info}, {"plan_has_override", plan_has_override}, {"plan_sub_plan", plan_sub_plan}, + {"encode", encode_lua}, {NULL, NULL}, }; diff --git a/test/c_runtime_encode_test.lua b/test/c_runtime_encode_test.lua new file mode 100644 index 0000000000000000000000000000000000000000..669817033581357d9762e0b99c84fb738f89eab1 --- /dev/null +++ b/test/c_runtime_encode_test.lua @@ -0,0 +1,178 @@ +-- Test for bd-y1n / ra6 3b: C-side scalar encode. +-- +-- Only runs when PB_ENABLE_C=1 is set in the environment AND the C +-- runtime module is loadable. Otherwise the group is skipped, which +-- keeps `just test` green on hosts without the C module built. +-- +-- Acceptance per bd-y1n: +-- Person encode for {name='x', age=42, balance=-7, user_id=..., +-- weight_kg=3.14} byte-equal to mode=full pure-Lua output. + +local t = require('luatest') +local ffi = require('ffi') + +local pb = require('pb') +local c_runtime = pb.c_runtime + +local function skip_if_no_c() + if c_runtime == nil then + t.skip('PB_ENABLE_C not set or pb.c_runtime not available') + end +end + +-- The mode=full reference is the byte-equality target; the test runs +-- against both codegen modes' descriptors (the C plan is mode-agnostic). +local full_hello + +for _, mode in ipairs({'full', 'runtime'}) do + local g = t.group('c_runtime_encode.' .. mode) + local hello + + g.before_all(function() + skip_if_no_c() + hello = require(mode .. '.hello.hello_pb') + full_hello = require('full.hello.hello_pb') + end) + + g.before_each(skip_if_no_c) + + -- ---------- Acceptance per bd-y1n ---------- + + function g.test_acceptance_person_scalar_subset() + local msg = { + name = 'x', + age = 42, + balance = -7, + user_id = 0xDEADBEEFCAFEBABEULL, + weight_kg = 3.14, + } + local plan = c_runtime.compile_plan(hello.Person_descriptor) + local c_bytes = c_runtime.encode(plan, msg) + local lua_bytes = full_hello.Person_encode(msg) + t.assert_equals(c_bytes, lua_bytes, + 'C encode matches mode=full pure-Lua encode byte-for-byte') + end + + -- ---------- Per-kind coverage ---------- + + function g.test_empty_message_produces_empty_string() + local plan = c_runtime.compile_plan(hello.Address_descriptor) + t.assert_equals(c_runtime.encode(plan, {}), '') + end + + function g.test_address_strings_and_int32() + local plan = c_runtime.compile_plan(hello.Address_descriptor) + local msg = {street = 'Main', city = 'Springfield', zip = 12345} + t.assert_equals(c_runtime.encode(plan, msg), + full_hello.Address_encode(msg)) + end + + function g.test_proto3_optional_emits_empty_string() + -- Address.apartment is proto3-optional; presence beats default. + local plan = c_runtime.compile_plan(hello.Address_descriptor) + local msg = {apartment = ''} + t.assert_equals(c_runtime.encode(plan, msg), + full_hello.Address_encode(msg)) + end + + function g.test_double_negative_zero_emits() + local plan = c_runtime.compile_plan(hello.Person_descriptor) + -- +0.0 -> skip; -0.0 -> emit (sign bit preserved by wire spec). + local m_pos = {weight_kg = 0.0} + local m_neg = {weight_kg = -0.0} + t.assert_equals(c_runtime.encode(plan, m_pos), + full_hello.Person_encode(m_pos)) + t.assert_equals(c_runtime.encode(plan, m_neg), + full_hello.Person_encode(m_neg)) + end + + function g.test_enum_as_number() + local plan = c_runtime.compile_plan(hello.Person_descriptor) + local msg = {status = 2} -- ERROR + t.assert_equals(c_runtime.encode(plan, msg), + full_hello.Person_encode(msg)) + end + + function g.test_enum_as_string_lookup() + local plan = c_runtime.compile_plan(hello.Person_descriptor) + local msg = {status = 'OK'} + t.assert_equals(c_runtime.encode(plan, msg), + full_hello.Person_encode(msg)) + end + + function g.test_enum_zero_value_suppressed() + local plan = c_runtime.compile_plan(hello.Person_descriptor) + local msg = {status = 'UNKNOWN'} -- = 0; default-suppressed + t.assert_equals(c_runtime.encode(plan, msg), '') + end + + function g.test_enum_unknown_string_errors() + local plan = c_runtime.compile_plan(hello.Person_descriptor) + t.assert_error_msg_contains("unknown enum value 'NOPE'", function() + c_runtime.encode(plan, {status = 'NOPE'}) + end) + end + + function g.test_fixed64_cdata_round_trip() + local plan = c_runtime.compile_plan(hello.Person_descriptor) + local msg = {user_id = ffi.new('uint64_t', 0x123456789ABCDEF0)} + t.assert_equals(c_runtime.encode(plan, msg), + full_hello.Person_encode(msg)) + end + + function g.test_sint32_negative_and_zero() + local plan = c_runtime.compile_plan(hello.Person_descriptor) + t.assert_equals(c_runtime.encode(plan, {balance = 0}), '') + t.assert_equals( + c_runtime.encode(plan, {balance = -1}), + full_hello.Person_encode({balance = -1})) + t.assert_equals( + c_runtime.encode(plan, {balance = 0x7fffffff}), + full_hello.Person_encode({balance = 0x7fffffff})) + end + + function g.test_int32_zero_suppressed() + local plan = c_runtime.compile_plan(hello.Person_descriptor) + t.assert_equals(c_runtime.encode(plan, {age = 0}), '') + end + + function g.test_bytes_field() + -- Person.avatar is bytes @8 + local plan = c_runtime.compile_plan(hello.Person_descriptor) + local msg = {avatar = '\x00\x01\xff\xfe'} + t.assert_equals(c_runtime.encode(plan, msg), + full_hello.Person_encode(msg)) + end + + function g.test_skips_repeated_and_message_fields() + -- 3b scope: repeated/map/message silently skipped (3d/3e will + -- replace this). Singular fields still encode. + local plan = c_runtime.compile_plan(hello.Person_descriptor) + local msg = { + name = 'x', + emails = {'a@b'}, -- repeated string + address = {street = 'Main'}, -- message + lucky_numbers = {1, 2, 3}, -- repeated packed + ages_by_nickname = {alice = 30}, -- map + } + local expected = full_hello.Person_encode({name = 'x'}) + t.assert_equals(c_runtime.encode(plan, msg), expected) + end + + function g.test_long_string_grows_buffer() + -- Stack buffer is 4KB; force the heap-promotion path with a + -- string that pushes past it. + local plan = c_runtime.compile_plan(hello.Person_descriptor) + local msg = {name = string.rep('a', 8192)} + t.assert_equals(c_runtime.encode(plan, msg), + full_hello.Person_encode(msg)) + end + + function g.test_wkt_override_rejected() + -- has_override plans skip field-walk; 3b does not handle them. + local plan = c_runtime.compile_plan(pb.wkt.Timestamp_descriptor) + t.assert_error_msg_contains('override', function() + c_runtime.encode(plan, {seconds = 1}) + end) + end +end