diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index b7a7e7a0d53fc3c65cdfbaf9f9c14f17c71fd243..f87e9ca80666b19c05cacb7316c58eb46f267f44 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1,4 +1,4 @@ -{"_type":"issue","id":"tarantool-protobuf-exy","title":"ra6 3f: repeated strings + repeated messages (cached stack-idx)","description":"Repeated string fields and repeated message fields. Encode: iterate the Lua array via cached stack idx, write each element (string field or recursive submsg encode). Decode: first hit lazy-creates the result array and stashes its stack index in a per-field slot (sized by plan-\u003en_fields, cap 16 for the spike fixed-size); subsequent hits lua_rawseti directly without re-lookup; one lua_setfield at decode_message exit attaches the array to the result table. This is the SPIKE-VALIDATED PATTERN — naive lazy-getfield is 2x slower at 100KB. Depends on 3d (needs sub-message machinery for repeated messages). Acceptance: Person.emails (repeated string) and Person.friends (repeated Person, self-reference) both round-trip byte-equal to mode=full at 1KB, 10KB, 100KB sizes.","status":"open","priority":1,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-18T20:20:52Z","created_by":"Eugene Blikh","updated_at":"2026-05-18T20:20:52Z","dependencies":[{"issue_id":"tarantool-protobuf-exy","depends_on_id":"tarantool-protobuf-hwe","type":"blocks","created_at":"2026-05-18T23:22:18Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0} +{"_type":"issue","id":"tarantool-protobuf-exy","title":"ra6 3f: repeated strings + repeated messages (cached stack-idx)","description":"Repeated string fields and repeated message fields. Encode: iterate the Lua array via cached stack idx, write each element (string field or recursive submsg encode). Decode: first hit lazy-creates the result array and stashes its stack index in a per-field slot (sized by plan-\u003en_fields, cap 16 for the spike fixed-size); subsequent hits lua_rawseti directly without re-lookup; one lua_setfield at decode_message exit attaches the array to the result table. This is the SPIKE-VALIDATED PATTERN — naive lazy-getfield is 2x slower at 100KB. Depends on 3d (needs sub-message machinery for repeated messages). Acceptance: Person.emails (repeated string) and Person.friends (repeated Person, self-reference) both round-trip byte-equal to mode=full at 1KB, 10KB, 100KB sizes.","status":"closed","priority":1,"issue_type":"task","assignee":"Eugene Blikh","owner":"bigbes@gmail.com","created_at":"2026-05-18T20:20:52Z","created_by":"Eugene Blikh","updated_at":"2026-05-23T11:07:38Z","started_at":"2026-05-23T11:05:14Z","closed_at":"2026-05-23T11:07:38Z","close_reason":"Acceptance tests landed: Person.emails (repeated string) and Person.friends (repeated Person, self-reference) round-trip byte-equal to mode=full at 1KB/10KB/100KB. Repeated string + message dispatch was already implemented as part of 3e (encode_repeated_field handles MESSAGE + string/bytes branches alongside scalars, decode_body uses cached list_stack_idx[] per-field); this ticket adds the formal size-scaled acceptance coverage.","dependencies":[{"issue_id":"tarantool-protobuf-exy","depends_on_id":"tarantool-protobuf-hwe","type":"blocks","created_at":"2026-05-18T23:22:18Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-jc9","title":"ra6 3e: repeated/packed scalar encode/decode","description":"Repeated scalar fields: encode-time, walk the Lua array via cached stack idx; decode-time, lazy-create the array on first hit and append via lua_rawseti with a cached stack idx (NOT lazy lua_getfield per element — that's 2x slower at 100KB per the spike). Packed repeated: length-prefixed varint blob; tight loop in C. Covers int32/sint32/uint32/int64/uint64/fixed32/fixed64/double/bool/enum. Depends on 3b + 3c (need the scalar primitives). Acceptance: Person.lucky_numbers (packed int32) round-trips byte-equal; a fixture with both packed and unpacked repeated scalars round-trips against mode=full at 10/100/1000-element counts.","status":"closed","priority":1,"issue_type":"task","assignee":"Eugene Blikh","owner":"bigbes@gmail.com","created_at":"2026-05-18T20:20:33Z","created_by":"Eugene Blikh","updated_at":"2026-05-23T10:39:55Z","started_at":"2026-05-23T10:29:02Z","closed_at":"2026-05-23T10:39:55Z","close_reason":"Closed","dependencies":[{"issue_id":"tarantool-protobuf-jc9","depends_on_id":"tarantool-protobuf-mz6","type":"blocks","created_at":"2026-05-18T23:22:07Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-jc9","depends_on_id":"tarantool-protobuf-y1n","type":"blocks","created_at":"2026-05-18T23:22:07Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":2,"dependent_count":2,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-hwe","title":"ra6 3d: sub-message encode/decode (recursion)","description":"Nested-message support. Encode: open a sub-buffer, recurse with the submsg plan, prefix parent buffer with tag + length. Decode: bound the byte range, recurse, lua_setfield the resulting table. Sub-buffer stack-backed (4KB) and malloc-promoted, same pattern as the outer buffer. Recursion is by C function call into the same encode/decode routine with a different plan, so depth is bounded by C stack. Depends on 3b + 3c. Acceptance: Person.address (1-level) round-trips; deeper nesting via a 5-level test fixture round-trips byte-equal to mode=full.","status":"closed","priority":1,"issue_type":"task","assignee":"Eugene Blikh","owner":"bigbes@gmail.com","created_at":"2026-05-18T20:20:24Z","created_by":"Eugene Blikh","updated_at":"2026-05-23T10:24:19Z","started_at":"2026-05-23T10:11:41Z","closed_at":"2026-05-23T10:24:19Z","close_reason":"ra6 3d: sub-message encode/decode (recursion) — refactored encode_lua/decode_lua into reusable encode_body/decode_body, added singular sub-message encode (force-promoted parent heap_idx before recursion to make lua_settop safe) and decode (temporarily shrunk c-\u003elen for bounded inner read). 22 new tests cover round-trip, empty sub-message presence, proto3-optional fields, 5-level depth (new test/proto/c_nested.proto), parent-buffer regrow path, truncated-input errors. Full suite 854/854 with PB_ENABLE_C=1.","dependencies":[{"issue_id":"tarantool-protobuf-hwe","depends_on_id":"tarantool-protobuf-mz6","type":"blocks","created_at":"2026-05-18T23:22:06Z","created_by":"Eugene Blikh","metadata":"{}"},{"issue_id":"tarantool-protobuf-hwe","depends_on_id":"tarantool-protobuf-y1n","type":"blocks","created_at":"2026-05-18T23:22:05Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":2,"dependent_count":4,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-awv","title":"ra6 3l: 64-bit cdata fidelity","description":"int64/uint64/fixed64/sfixed64 must round-trip as LuaJIT cdata in both encode and decode — never narrowed to a Lua double. Same convention as msgpackffi, net.box, box.tuple, the built-in protobuf. Use luaT_pushuint64/luaT_pushint64 from Tarantool's module.h for the C side. Independent transverse concern — can land in parallel with 3b/3c since it's a per-kind fix rather than a new pipeline stage. Acceptance: a value \u003e 2^53 round-trips byte-equal and remains cdata after decode; luatest test/protobuf_test.lua int64-cdata cases pass with PB_ENABLE_C=1.","status":"open","priority":1,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-18T20:20:10Z","created_by":"Eugene Blikh","updated_at":"2026-05-18T20:20:10Z","dependencies":[{"issue_id":"tarantool-protobuf-awv","depends_on_id":"tarantool-protobuf-mq7","type":"blocks","created_at":"2026-05-18T23:21:55Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0} diff --git a/test/c_runtime_decode_test.lua b/test/c_runtime_decode_test.lua index a78fc2267110eb7ed8a51fafacb345ff28512943..72edb710fe5e6db47c0010d721f23ab4a461191c 100644 --- a/test/c_runtime_decode_test.lua +++ b/test/c_runtime_decode_test.lua @@ -359,6 +359,49 @@ ('messages n=%d'):format(n)) end end + -- ---------- Acceptance per bd-exy / ra6 3f ---------- + -- + -- Decode-side mirror of c_runtime_encode_test's acceptance: the + -- cached-stack-idx repeated dispatch (decode_body's per-field + -- list_stack_idx[]) must round-trip Person.emails and Person. + -- friends byte-equal to mode=full at 1KB, 10KB, 100KB. This + -- exercises the lazy-create + cached-idx + lua_rawseti loop at + -- counts where the spike measured naive lazy-getfield at 2x + -- slower. + + function g.test_acceptance_repeated_strings_round_trip_at_sizes() + local plan = c_runtime.compile_plan(hello.Person_descriptor) + for _, c in ipairs({{n=50}, {n=500}, {n=5000}}) do + local emails = {} + for i = 1, c.n do + emails[i] = string.rep('e', 16) + .. string.format('%02d', i % 100) + end + local bytes = full_hello.Person_encode({emails = emails}) + t.assert_equals(c_runtime.decode(plan, bytes), + full_hello.Person_decode(bytes), + ('emails round-trip n=%d'):format(c.n)) + end + end + + function g.test_acceptance_repeated_messages_round_trip_at_sizes() + local plan = c_runtime.compile_plan(hello.Person_descriptor) + local pad = 'xxxxxxx' + for _, c in ipairs({{n=56}, {n=560}, {n=5600}}) do + local friends = {} + for i = 1, c.n do + friends[i] = { + name = pad .. string.format('f%04d', i), + age = i, + } + end + local bytes = full_hello.Person_encode({friends = friends}) + t.assert_equals(c_runtime.decode(plan, bytes), + full_hello.Person_decode(bytes), + ('friends round-trip n=%d'):format(c.n)) + end + end + function g.test_fixture_mixed_round_trip() local cr = require(mode .. '.c_repeated.c_repeated_pb') local full_cr = require('full.c_repeated.c_repeated_pb') diff --git a/test/c_runtime_encode_test.lua b/test/c_runtime_encode_test.lua index d9cfe7fb8eba67dec13aa2539b0e5ec861264d81..842825a5e29b8721537625d494d973a23fb78d27 100644 --- a/test/c_runtime_encode_test.lua +++ b/test/c_runtime_encode_test.lua @@ -401,6 +401,87 @@ ('messages n=%d'):format(n)) end end + -- ---------- Acceptance per bd-exy / ra6 3f ---------- + -- + -- Person.emails (repeated string) and Person.friends (repeated + -- Person, self-reference) round-trip byte-equal to mode=full at + -- 1KB, 10KB, 100KB wire sizes. The repeated dispatch landed + -- with 3e (encode_repeated_field handles MESSAGE + string/bytes + -- branches alongside the scalars); this acceptance pins it under + -- the cached-stack-idx pattern at scale. + -- + -- Sizing chosen to hit the named targets without per-test math + -- in the loop: + -- emails @ 50 elements ≈ 1KB, ≈ 10KB, ≈ 100KB + -- friends @ 60 elements ≈ 1KB, ≈ 10KB, ≈ 100KB + -- + -- Bytes are checked at runtime against the named target band so + -- a future schema or encoding shift surfaces as a test failure + -- rather than silently moving off-target. + + local function size_band(actual, target) + -- Accept anything within 30% of the named size — generous + -- enough to survive small wire-format shifts, tight enough + -- to flag a regression to the wrong order of magnitude. + return actual >= target * 0.7 and actual <= target * 1.3 + end + + function g.test_acceptance_repeated_strings_at_size_targets() + local plan = c_runtime.compile_plan(hello.Person_descriptor) + local cases = { + {label = '1KB', n = 50, elem_len = 18, target = 1024}, + {label = '10KB', n = 500, elem_len = 18, target = 10240}, + {label = '100KB', n = 5000, elem_len = 18, target = 102400}, + } + for _, c in ipairs(cases) do + local emails = {} + for i = 1, c.n do + emails[i] = string.rep('e', c.elem_len - 2) + .. string.format('%02d', i % 100) + end + local msg = {emails = emails} + local c_bytes = c_runtime.encode(plan, msg) + local lua_bytes = full_hello.Person_encode(msg) + t.assert_equals(c_bytes, lua_bytes, + ('emails %s byte-equal'):format(c.label)) + t.assert(size_band(#c_bytes, c.target), + ('emails %s actual=%dB target=%dB'):format( + c.label, #c_bytes, c.target)) + end + end + + function g.test_acceptance_repeated_messages_self_ref_at_size_targets() + -- Friend payload shape: {name = '<12-char>', age = N}. Each + -- emitted friend is around 18 bytes: parent tag (1) + body + -- length varint (1) + inner name tag (1) + name len (1) + + -- 12 chars + age tag (1) + age varint (1-2). 56 → ~1KB, + -- 560 → ~10KB, 5600 → ~100KB. + local plan = c_runtime.compile_plan(hello.Person_descriptor) + local cases = { + {label = '1KB', n = 56, target = 1024}, + {label = '10KB', n = 560, target = 10240}, + {label = '100KB', n = 5600, target = 102400}, + } + local pad = 'xxxxxxx' -- 7 chars; combined with 'fNNNN' → 12 + for _, c in ipairs(cases) do + local friends = {} + for i = 1, c.n do + friends[i] = { + name = pad .. string.format('f%04d', i), + age = i, + } + end + local msg = {friends = friends} + local c_bytes = c_runtime.encode(plan, msg) + local lua_bytes = full_hello.Person_encode(msg) + t.assert_equals(c_bytes, lua_bytes, + ('friends %s byte-equal'):format(c.label)) + t.assert(size_band(#c_bytes, c.target), + ('friends %s actual=%dB target=%dB'):format( + c.label, #c_bytes, c.target)) + end + end + function g.test_fixture_mixed_packed_and_unpacked() -- All branches simultaneously: packed + unpacked + string/bytes -- + message, on the same message instance.