diff --git a/.beads/interactions.jsonl b/.beads/interactions.jsonl index 78956653bb32e8e92f9b20dbb39a7aa0c900822d..a8ba8d8ad566a48b1414da7ac6a58b86cccf9b0f 100644 --- a/.beads/interactions.jsonl +++ b/.beads/interactions.jsonl @@ -49,3 +49,5 @@ {"id":"int-643d6ba3","kind":"field_change","created_at":"2026-05-24T16:53:43.706654Z","actor":"Eugene Blikh","issue_id":"tarantool-protobuf-86g","extra":{"field":"status","new_value":"closed","old_value":"open"}} {"id":"int-7db2afd6","kind":"field_change","created_at":"2026-05-24T17:05:10.538707Z","actor":"Eugene Blikh","issue_id":"tarantool-protobuf-ch2","extra":{"field":"status","new_value":"closed","old_value":"in_progress"}} {"id":"int-b981997e","kind":"field_change","created_at":"2026-05-24T17:20:53.622551Z","actor":"Eugene Blikh","issue_id":"tarantool-protobuf-aah","extra":{"field":"status","new_value":"closed","old_value":"in_progress"}} {"id":"int-2a1c9c90","kind":"field_change","created_at":"2026-05-24T18:20:11.709034Z","actor":"Eugene Blikh","issue_id":"tarantool-protobuf-6bb","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Full-mode codegen emits _decode_unsafe alongside _decode that skips utf8_len at every string site (singular/repeated/map-kv/extensions/>=128-byte fallback) and recurses into sub-messages' _decode_unsafe. Skips pb.c_runtime dispatch (C runtime validates today; tracked in kyt). Runtime mode deferred to 58u (compile parallel _reader_unsafe closures). 6 tests in test/decode_unsafe_test.lua; full suite + conformance pass; ~20% perf win on string-heavy 1KB Person microbench. Generated code ~35% larger as expected. Docs in docs/api-modes.md."}} +{"id":"int-27d2602e","kind":"field_change","created_at":"2026-05-24T19:00:34.358869Z","actor":"Eugene Blikh","issue_id":"tarantool-protobuf-b12","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Already fixed in commit 2656c97 (2026-05-17, same day issue filed). register_extension appends to extensions_list (parallel array); codec.encode_message, text.emit_message, json.encode_message all iterate via 'for i=1,#elist'. Inline-mode codegen also walks the list via 'for _i = N+1, #_elist' for runtime-registered extensions past the statically-known set. No pairs() over extensions remains anywhere on the hot path. The issue was never closed in bd."}} +{"id":"int-a28aba03","kind":"field_change","created_at":"2026-05-24T19:11:19.58095Z","actor":"Eugene Blikh","issue_id":"tarantool-protobuf-58u","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Runtime mode now exposes pb.decode_unsafe and M._decode_unsafe. codec.lua adds scalar_unsafe table, parameterizes build_reader/build_repeated_reader/decode_one to accept (scalar_tbl, decode_msg_fn, decode_group_fn), and adds compile_readers_unsafe + decode_message_unsafe + decode_group_unsafe + decode_extension_unsafe as literal clones with the three substitutions documented in codec.lua. Tests parameterized over both modes (14 cases), perf microbench shows ~8% gain in runtime mode (~20% in full mode). Conformance + JIT trace gates still pass. kyt remains open for unifying with C accel."}} diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index fef400fa38a45d5aaf9313cc40bbaf697f31df37..222a8a50d01c6709ceffb1e3f5389125808db69a 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -28,7 +28,7 @@ {"_type":"issue","id":"tarantool-protobuf-qwt","title":"Codegen: emit per-extension writers for proto2 extensions (skip pb.codec.encode_field dispatch)","description":"proto2 extensions are the slowest line in the bench: proto2_basic.BenchPayload 'min' fixture encodes at 11.0 MB/s — vs 281 MB/s for the 100B Person. Cause: generated _extensions walk in mode=full falls through to pb.codec.encode_field, the slow runtime dispatch that mode=full otherwise avoids.\n\nFix: at codegen time, when 'extend Foo { ... ext_count = 100; ... }' is seen, emit a dedicated writer per extension. The Foo_encode body's extensions block becomes:\n\n local _ev = _exts['proto2_basic.ext_count']\n if _ev ~= nil then\n -- direct inline write for the int32 ext, just like a regular int32 field\n n = n + 1; out[n] = '\u003cprecomputed tag bytes\u003e'\n n = n + 1; out[n] = wire.encode_int32(_ev)\n end\n -- repeat per extension\n\nDecoder side: extensions_by_id[id] dispatch in the else branch currently calls pb.codec.decode_extension; can similarly be replaced by inline-emitted per-extension decoder blocks alongside the regular field branches.\n\nConcretely closes the proto2 min throughput gap (3.4x vs Go apiv2) which is bench's worst data point. Has zero impact on payloads without extensions.","notes":"Generated code grows by one if-block per registered extension. For a message with 50 extensions this could be significant — but proto2 extensions are typically used sparingly. Worth gating on extension count if size becomes a problem.\n\nExisting extensions_by_id / extensions_list / extensions_by_full_name registries can stay (they're used by text/json/runtime dispatch); the fast path just bypasses them in mode=full.","status":"closed","priority":2,"issue_type":"task","assignee":"Eugene Blikh","owner":"bigbes@gmail.com","created_at":"2026-05-18T17:14:38Z","created_by":"Eugene Blikh","updated_at":"2026-05-24T10:55:07Z","started_at":"2026-05-24T10:32:44Z","closed_at":"2026-05-24T10:55:07Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-ch2","title":"Encoder: JIT-friendly map\u003cK,V\u003e field iteration (avoid pairs() ISNEXT NYI)","description":"map\u003cK,V\u003e field encoding walks the user-supplied table with pairs() — runtime/pb/codec.lua:162 and every codegen-emitted map block in mode=full. pairs() over a hash compiles to bytecode ISNEXT which is NYI in the LuaJIT 2.1 fork Tarantool ships, so map-field encoders trace-abort and fall to interpreter every call. bench/jit_trace.lua already pins this as the only intentional NYI in the hot path.\n\nUnlike oneofs/extensions (which are flattened to *_list arrays at finalize-time in init.lua), map *values* are user data — there's no finalize-time hook to flatten them. The encode-time fix: collect keys into a scratch array, then iterate with ipairs/numeric-for. Pattern:\n\n local _mkeys, _mn = {}, 0\n for k in pairs(value) do _mn = _mn + 1; _mkeys[_mn] = k end\n for _i = 1, _mn do\n local k = _mkeys[_i]\n local v = value[k]\n -- emit entry as today\n end\n\nCosts one extra alloc (scratch keys table) per map field per call but lets the inner emit-loop stay on a JIT trace. For maps with \u003e3 entries the trace-stable inner loop should net out positive. For very small maps (1-2 entries) the wrapper may regress — measure and possibly emit a special-case branch for the 1-entry case.\n\nPairs naturally with x9f (deterministic map option) — when on, the scratch keys array can be sorted, giving deterministic output for free vs the current pairs()-order-undefined behavior. Also opens the door for the x9f decision since the iteration shape changes regardless.","notes":"Generated code lives in protoc-gen-tarantool. Both runtime mode (codec.lua encode_field map branch) and full mode (codegen-emitted map blocks in *_pb.lua) need updating. Bench fixture: hello.Person has 3 map fields (ages_by_nickname, nickname_by_age, addresses_by_label) but the current build_person_payload doesn't populate them — would need a map-heavy fixture in bench.lua to measure the win.\n2026-05-24 — investigated, intervention regresses across all map sizes.\n\nHand-implemented the materialize-keys-then-ipairs pattern in both\ncodegen (inline.go:emitInlineEncodeMap) and runtime (codec.lua kind=='map'\nbranch). Parity-clean — 752/752 tests pass.\n\nBench (work.lab.local, median of 3 trials, full mode, Person.ages_by_nickname\nmap\u003cstring,int32\u003e):\n\n map size | BEFORE | AFTER | Δ\n ---------|-------------|-------------|-------\n 1 | 1,282,180 | 1,128,545 | -12%\n 3 | 634,880 | 549,761 | -13%\n 10 | 215,745 | 196,800 | -9%\n 50 | 48,162 | 45,614 | -5%\n 200 | 11,613 | 11,282 | -3%\n\nDiagnosis: the issue's prediction that 'inner emit-loop stays on a JIT\ntrace' was theoretically correct but empirically irrelevant. LuaJIT's\nside-trace machinery was already JIT-ing the inner body via a side trace\nfrom the pairs()-induced ISNEXT abort point — the body was already\non-trace before the change. The change adds:\n - scratch _mkeys table allocation per encode\n - O(N) key-collection walk (still pairs(), still off-trace)\n - extra v[_k] hash lookup per entry\n\nThese costs scale with map size. Regression shrinks as N grows (the\nwrapper amortizes) but never crosses zero even at 200 entries.\n\nThe jit_trace.lua [PIN] still passed under the change because pairs()\nis still called for key collection — the NYI was simply moved, not\neliminated. The encoder cannot fully shed the NYI without a\ndifferent data shape (e.g., keys stored as a parallel array at\ntable-construction time, which is a user-visible API change).\n\nConclusion: not viable. The \"for \u003e3 entries net positive\" prediction is\nempirically false on Tarantool's LuaJIT 2.1 fork. Closing.\n\nRelated future direction: if x9f (deterministic map encoding) ships\nwith sorted-keys semantics, the user-side cost of producing a sorted\nkeys array could be amortized at construction time rather than at encode\ntime, opening a different (non-pairs) iteration shape. That's a feature\ndecision, not a perf one.","status":"closed","priority":2,"issue_type":"task","assignee":"Eugene Blikh","owner":"bigbes@gmail.com","created_at":"2026-05-18T17:13:53Z","created_by":"Eugene Blikh","updated_at":"2026-05-24T17:05:10Z","started_at":"2026-05-24T16:59:19Z","closed_at":"2026-05-24T17:05:10Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-cch","title":"Decoder: use local counter instead of #list at repeated-field append","description":"jit.p profile (Person 1KB): line 1021 'list[#list + 1] = val' is 6.2% of total decode time. Each #list invocation re-traverses to find the array length. Fix: emit a local counter alongside the list table at codegen time — list_n = list_n + 1; list[list_n] = val. Already done on encode side ('out, n' pattern). Tiny 1-line codegen change. Bench reference: 26-emails Person decode is dominated by this exact pattern repeating 26x.","notes":"At decode start, when initializing a repeated-field list, also init the counter: 'local emails, emails_n = result.emails, #(result.emails or {}) ' — for the first-encounter case, counter starts at 0. Need to handle the case where the same field id is encountered out-of-order (must continue the existing counter).","status":"closed","priority":2,"issue_type":"task","assignee":"Eugene Blikh","owner":"bigbes@gmail.com","created_at":"2026-05-18T16:52:38Z","created_by":"Eugene Blikh","updated_at":"2026-05-18T19:01:17Z","started_at":"2026-05-18T18:56:39Z","closed_at":"2026-05-18T19:01:17Z","close_reason":"Local counter per repeated non-map field; median-of-3 decode wins +5.0% (1KB), +5.0% (10KB), +7.1% (100KB). Profile target was 6.2%; landed wins of 5-7%. Encode flat, runtime flat. 745/745 tests, 37/37 JIT. See bench/PERF_LOG.md.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-a6n","title":"Decoder: ffi-cast buffer ptr once per decode; replace buf:byte/buf:sub at fast paths","description":"Decode hot path uses buf:byte(pos) and buf:sub(start, end-1) — both are method calls dispatched via the string metatable per invocation. Wire.lua already has U8CP = ffi.typeof('const uint8_t*'). The fix: at the top of each generated Type_decode, cast once (local ptr = ffi.cast(U8CP, buf)) and use ptr[pos-1] for byte reads. For string returns, keep buf:sub (or use ffi.string per bgu issue) since the result must be a Lua string. Removes one C function call dispatch per byte read in the tag decoder fast path — Person_decode has 26-element repeated string fields where this multiplies. Compatible with bgu (the ffi.string change for decode_string) — both rely on the same cast.","notes":"Caveat: ffi.cast holds the string pinned. Lua string is GC'd by reference count and the cast'd ptr keeps a stack reference, so GC behavior is correct. Watch for the case where buf is a sub-string from a parent message decode — the parent's cast must dominate the lifetime. Easy fix: each Type_decode re-casts the buf it owns.","status":"closed","priority":2,"issue_type":"task","assignee":"Eugene Blikh","owner":"bigbes@gmail.com","created_at":"2026-05-18T16:51:24Z","created_by":"Eugene Blikh","updated_at":"2026-05-23T20:48:33Z","started_at":"2026-05-23T20:34:10Z","closed_at":"2026-05-23T20:48:33Z","close_reason":"Inlined 1-byte LEN fast path for string/bytes scalar + repeated fields in generated full-mode _decode; skips wire.decode_string/_bytes function call frame and inlines utf8_len. Dropped the U8CP cast-at-top idea — 24B/decode cdata allocation that LuaJIT couldn't sink, regressed small messages. Person_decode full: 10B -6.9%, 100B -7.2%, 1KB -10.7%, 10KB -12.2%, 100KB -14.0%. Zero alloc impact. Tests: 1043 pass, 37 JIT trace checks pass.","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"tarantool-protobuf-6bb","title":"Decoder: opt-in skip_utf8_validation flag for trusted sources","description":"decode_string runs utf8_len (ICU) on every decoded string. Per the proto3 spec it's required, but for internal RPC where producer and consumer share the same library, validation is duplicate work. Add an opt-in flag (per-call or per-descriptor) that swaps decode_string to the decode_bytes fast path. Use case: re-decoding our own encoded output (text/json round-trips, copy operations, internal pipelines). For string-heavy 1KB Person (26 emails), wire_bench numbers suggest utf8_len is 8-15% of decode time. Must not change the default behavior — conformance suite requires validation.","notes":"Plumbing options: pb.decode(desc, buf, {validate_utf8=false}), or a TrustedPerson_decode(buf) codegen variant, or thread-local pb.set_validate_utf8(false). Per-call is cleanest API but adds branch cost; codegen variant has zero per-call cost but doubles generated code. Per-descriptor (desc.skip_validation) is a middle ground. Decide during implementation.","status":"in_progress","priority":2,"issue_type":"task","assignee":"Eugene Blikh","owner":"bigbes@gmail.com","created_at":"2026-05-18T16:51:23Z","created_by":"Eugene Blikh","updated_at":"2026-05-24T18:03:10Z","started_at":"2026-05-24T18:03:10Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"tarantool-protobuf-6bb","title":"Decoder: opt-in skip_utf8_validation flag for trusted sources","description":"decode_string runs utf8_len (ICU) on every decoded string. Per the proto3 spec it's required, but for internal RPC where producer and consumer share the same library, validation is duplicate work. Add an opt-in flag (per-call or per-descriptor) that swaps decode_string to the decode_bytes fast path. Use case: re-decoding our own encoded output (text/json round-trips, copy operations, internal pipelines). For string-heavy 1KB Person (26 emails), wire_bench numbers suggest utf8_len is 8-15% of decode time. Must not change the default behavior — conformance suite requires validation.","notes":"Plumbing options: pb.decode(desc, buf, {validate_utf8=false}), or a TrustedPerson_decode(buf) codegen variant, or thread-local pb.set_validate_utf8(false). Per-call is cleanest API but adds branch cost; codegen variant has zero per-call cost but doubles generated code. Per-descriptor (desc.skip_validation) is a middle ground. Decide during implementation.","status":"closed","priority":2,"issue_type":"task","assignee":"Eugene Blikh","owner":"bigbes@gmail.com","created_at":"2026-05-18T16:51:23Z","created_by":"Eugene Blikh","updated_at":"2026-05-24T18:20:12Z","started_at":"2026-05-24T18:03:10Z","closed_at":"2026-05-24T18:20:12Z","close_reason":"Full-mode codegen emits \u003cMsg\u003e_decode_unsafe alongside \u003cMsg\u003e_decode that skips utf8_len at every string site (singular/repeated/map-kv/extensions/\u003e=128-byte fallback) and recurses into sub-messages' _decode_unsafe. Skips pb.c_runtime dispatch (C runtime validates today; tracked in kyt). Runtime mode deferred to 58u (compile parallel _reader_unsafe closures). 6 tests in test/decode_unsafe_test.lua; full suite + conformance pass; ~20% perf win on string-heavy 1KB Person microbench. Generated code ~35% larger as expected. Docs in docs/api-modes.md.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-kot","title":"Codegen: localize wire.* upvalues at top of each generated message function","description":"examples/expected/full/hello/hello_pb.lua has 106 wire.* accesses — each is a hash-table lookup on the upvalue. LuaJIT hoists this when traces stay hot, but every nested-message boundary breaks the trace (see gcy notes), forcing re-lookup in the side trace / interpreter. Fix: codegen-emit at the top of each Type_encode and Type_decode function the locals it actually uses — only for the wire.* entries referenced in that function body — so the in-body calls are direct LJ_FUNCC dispatches with no table lookup. One-line codegen change; broad impact whenever traces stitch poorly. Pairs naturally with h8v and 4kj (both move more work into the same generated function bodies, magnifying the per-call lookup cost). Expected: 3-8% across the board; larger when traces break.","notes":"Codegen-side change in the Go plugin (protoc-gen-tarantool). Pre-compute the set of wire.* symbols used in each function (e.g., {encode_varint, decode_string, decode_tag}) and emit 'local encode_varint = wire.encode_varint' style preambles. Keep the existing 'local wire = pb.wire' so non-emitted entries still work. Verify with bench/jit_trace.lua that we don't increase trace size past LuaJIT inline budget on large messages (test_messages_proto3 is 4212 lines).","status":"closed","priority":2,"issue_type":"task","assignee":"Eugene Blikh","owner":"bigbes@gmail.com","created_at":"2026-05-18T16:51:23Z","created_by":"Eugene Blikh","updated_at":"2026-05-24T05:40:24Z","started_at":"2026-05-24T04:41:18Z","closed_at":"2026-05-24T05:40:24Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-lkz","title":"Encoder: single-pass two-phase (size + emit) into single buffer","description":"Eliminate the per-call out table and the per-field intermediate string allocations by emitting once into a pre-sized buffer. Two viable shapes (per drm notes):\n\n1. Two-pass: walk fields once to compute exact size, allocate one buffer, walk again to emit. Mirrors vtproto's Size()+MarshalTo() pattern — vtproto's 1-alloc encode is what gives it 13-20x throughput over us at 1KB-10KB Person.\n2. Single-pass with backpatched length varints: write tag + 1-byte placeholder for the length, recurse, fill in (memmove if final length \u003e= 128). Skips the size pass (~3 us at 1KB).\n\nImplementation can live in codegen (mode=full) — emit a _encode_to_buf per message that takes (data, ibuf, offset) and returns new_offset, then a public Person_encode that wraps it with the buffer allocation. Constraint: per-field closure count must stay \u003c= current pb.encode writers count, else the per-field dispatch cost re-emerges (drm: two prior ibuf prototypes both regressed by ~2x because of dispatch).\n\nRelates to h8v (codegen FFI direct writes) which is the per-field building block this work composes.","notes":"drm notes have the deeper analysis including why M6 ibuf path naive byte-write fails (20x interpreter dispatch overhead). The codegen approach sidesteps that by inlining all writes at codegen time so there's no runtime closure dispatch.\n2026-05-24 — verdict: not viable as a pure-Lua path.\n\nHand-written spike (ffi.new uint8_t[?] + two-pass size+emit, all inlined,\nno closures, exact-size buffer) was byte-exact for parity but regressed\nencode throughput across all 5 size buckets:\n\n 10B 0.77x 100B 0.71x 1KB 0.50x 10KB 0.31x 100KB 0.31x\n\nProfile (jit.p) of the current encoder shows 84% of time in the inlined\nmessage bodies (Person_encode + Address_encode); wire.encode_int32 is 2%\nof total. The \"out table + table.concat\" pattern is hitting LuaJIT's\nspecialized table-grow + interned-short-string + C-level concat path —\nffi.copy per string genuinely loses to `out[n] = string` for repeated\nLEN fields. The 136 B/op encode floor is not a perf wall.\n\nWhat WAS in this bucket and shipped: 2ri (replace string.char(_len) with\nCHARS[_len] lookup at the inlined length-prefix sites). That captured the\nsingle hottest line the profile flagged (39% of Person_encode share) for\n+17%-34% encode throughput on 1KB+ payloads.\n\nRemaining encode-perf headroom is in c0i (C-runtime backend) — escape\npure Lua to beat the pure-Lua ceiling. Closing lkz as superseded.","status":"closed","priority":2,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-18T16:49:16Z","created_by":"Eugene Blikh","updated_at":"2026-05-24T16:53:34Z","closed_at":"2026-05-24T16:53:34Z","dependencies":[{"issue_id":"tarantool-protobuf-lkz","depends_on_id":"tarantool-protobuf-h8v","type":"blocks","created_at":"2026-05-18T19:49:21Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-2sn","title":"Decoder: emit table.new(N, 0) for repeated-field lists","description":"Decode result table rehashes as nested fields populate. u39 covers table.new(0, N) for the message table itself, but per-field list tables (result.emails = {}, result.lucky_numbers = {}, ...) are also bare {} and re-hash as they grow. Worth a follow-up: emit table.new(N, 0) for repeated lists where the encoded count is recoverable from a single quick scan (count tag occurrences for non-packed, or read the LEN prefix and divide by per-element size for packed). Quick scan cost vs allocation savings is the tradeoff to measure — on the 1KB Person fixture, 26 emails + 5 packed lucky_numbers means 2 re-hash cycles per repeated list. Pairs with u39 to fully close the rehash-on-grow alloc pattern.","notes":"See bench/COMPARISON.md and bd show drm. For packed scalars where per-element size is fixed (fixed32/fixed64/float/double), count = payload_len / elem_size — O(1). For varint-packed and non-packed repeated, count requires a scan; might still be net-positive on payloads with \u003e8 elements but needs measurement.","status":"closed","priority":2,"issue_type":"task","assignee":"Eugene Blikh","owner":"bigbes@gmail.com","created_at":"2026-05-18T16:49:15Z","created_by":"Eugene Blikh","updated_at":"2026-05-24T10:55:08Z","started_at":"2026-05-24T10:32:45Z","closed_at":"2026-05-24T10:55:08Z","close_reason":"Closed","dependencies":[{"issue_id":"tarantool-protobuf-2sn","depends_on_id":"tarantool-protobuf-u39","type":"blocks","created_at":"2026-05-18T19:49:20Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} @@ -41,13 +41,15 @@ {"_type":"issue","id":"tarantool-protobuf-801","title":"Release: tagged releases + rockspec + Go plugin binaries","description":"Once the CI pipeline is green, ship tagged releases. Rockspec for the Lua runtime (publishable via tt rocks). Pre-built protoc-gen-tarantool binaries for darwin/arm64, darwin/amd64, linux/amd64, linux/arm64. Sourcecraft has 'sc release' tooling (see sc-release skill) for the release workflow itself.","status":"open","priority":2,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-17T15:47:16Z","created_by":"Eugene Blikh","updated_at":"2026-05-17T15:47:16Z","labels":["release"],"dependencies":[{"issue_id":"tarantool-protobuf-801","depends_on_id":"tarantool-protobuf-1eu","type":"blocks","created_at":"2026-05-17T18:47:20Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-1eu","title":"Release: Sourcecraft.dev project + CI pipeline","description":"Set up the canonical sourcecraft.dev project for the repo and a CI pipeline. Matrix: Tarantool 2.11 (CE+EE) and 3.x (CE+EE), Linux + macOS. Targets to run: just gen, just test, just bench-compare (alloc regression gate), just conformance (gated on the cached Docker image — see related CI wire-up bead). The repo lives under ~/data/home which by convention publishes to sourcecraft.dev (not github).","status":"open","priority":2,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-17T15:47:15Z","created_by":"Eugene Blikh","updated_at":"2026-05-17T15:47:15Z","labels":["ci","release"],"dependencies":[{"issue_id":"tarantool-protobuf-1eu","depends_on_id":"tarantool-protobuf-7lf","type":"blocks","created_at":"2026-05-17T18:47:20Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-7lf","title":"CI: wire up conformance suite with cached Docker image","description":"The Docker image build (docker/conformance.Dockerfile) is the long pole at ~10-15 min on a clean cache. A registry push from a scheduled job would let CI runs reuse a warm image. Today the conformance suite runs locally via 'just conformance' but isn't gated on pushes. Goal: every push to master runs the binary+JSON and text-format suites; PRs run the same. Pre-requisite for M8 sourcecraft setup (this defines what the CI pipeline runs).","status":"open","priority":2,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-17T15:47:15Z","created_by":"Eugene Blikh","updated_at":"2026-05-17T15:47:15Z","labels":["ci","conformance"],"dependency_count":0,"dependent_count":1,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-0an","title":"Profile the 1 KB decode cliff in hello.Person","description":"bench/bench.lua shows hello.Person decode dropping 260 MB/s @ 100 B to 128 MB/s @ 1 KB — per-MB throughput halves at the payload-shape transition (scalars-only -\u003e emails[] + nested address + packed lucky_numbers). starwing's C decoder scales smoothly across the same transition. Smoking gun: a trace abort or side-trace stitching failure at the array/nested branch. Capture jit.dump=tbim over the transition, identify the abort, fix it. Likely small change once located. Prereq for several decoder fixes — pin down what we're actually hitting first.","notes":"2026-05-17 inspection vs starwing/lua-protobuf 0.5.3 on Tarantool 3.8.0: reproduced starwing hello.Person encode at 75/688/888/1391/1431 MB/s and decode at 11.5/101.6/378/698.7/702.8 MB/s for 10B/100B/1KB/10KB/100KB. Current full mode is encode 30/257/287/541/643 MB/s and decode 30.7/247.5/128.8/186.4/184.3 MB/s. So we beat starwing on tiny decode, but lose 2.9-3.8x once Person switches to repeated strings + nested Address + packed ints. jit_trace passes including multi-byte varint paths; no fatal abort or bridge smoking gun. Focused probes on 930B Person: generated full decode ~6.9-7.5 us/op; scan tags+skip_field ~3.6 us/op; scan tags/lengths only ~2.1 us/op; hand order-specialized decoder with same string extraction/Address decode/lucky parse ~4.3 us/op. Patching decode_string to skip UTF-8 improves only ~16% (7.1 -\u003e 6.1 us/op), so UTF-8 is material but not the main cliff. Likely cause: per-field Lua dispatch/function-call/table growth overhead repeated across ~30 LEN/VARINT fields; C starwing parses from pointer/buffer and lands near our tag-only scan cost. Existing follow-ups bgu (ffi.string string decode), gcy (inline nested decode), and 0u1 (typed decoder byte fast paths) are relevant, but the largest remaining gap likely needs generated ordered/specialized decode loops or ptr+offset readers that avoid decode_tag/decode_len helper dispatch and substring slicing.","status":"closed","priority":2,"issue_type":"bug","assignee":"Eugene Blikh","owner":"bigbes@gmail.com","created_at":"2026-05-17T15:47:12Z","created_by":"Eugene Blikh","updated_at":"2026-05-17T16:02:54Z","started_at":"2026-05-17T15:56:33Z","closed_at":"2026-05-17T16:02:54Z","close_reason":"Profiled against starwing and local focused probes. No fatal JIT abort found; cliff attributed mainly to repeated Lua tag/field dispatch plus LEN/string slicing/validation. Follow-up fast-path work captured in tarantool-protobuf-4kj and existing decoder tasks.","labels":["decoder","investigation","perf"],"dependency_count":0,"dependent_count":3,"comment_count":0} -{"_type":"issue","id":"tarantool-protobuf-b12","title":"Encoder: parallel-array iteration for proto2 extensions","description":"Mirror the desc.oneofs_list finalize-time pattern: build desc.extensions_list once in pb.finalize_message, iterate with 'for i=1,#' in encode instead of walking the descriptor's hash table on every call. The 4-7x proto2 gap (vs 2-4x proto3 in bench/starwing_bench.lua) is partly explained by hash-walked extensions on the hot path. Expected: brings proto2 BenchPayload encode closer to proto3 Person encode at the same shape.","status":"open","priority":2,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-17T15:47:12Z","created_by":"Eugene Blikh","updated_at":"2026-05-17T15:47:12Z","labels":["encoder","perf","proto2"],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"tarantool-protobuf-b12","title":"Encoder: parallel-array iteration for proto2 extensions","description":"Mirror the desc.oneofs_list finalize-time pattern: build desc.extensions_list once in pb.finalize_message, iterate with 'for i=1,#' in encode instead of walking the descriptor's hash table on every call. The 4-7x proto2 gap (vs 2-4x proto3 in bench/starwing_bench.lua) is partly explained by hash-walked extensions on the hot path. Expected: brings proto2 BenchPayload encode closer to proto3 Person encode at the same shape.","status":"closed","priority":2,"issue_type":"task","assignee":"Eugene Blikh","owner":"bigbes@gmail.com","created_at":"2026-05-17T15:47:12Z","created_by":"Eugene Blikh","updated_at":"2026-05-24T19:00:34Z","started_at":"2026-05-24T18:59:15Z","closed_at":"2026-05-24T19:00:34Z","close_reason":"Already fixed in commit 2656c97 (2026-05-17, same day issue filed). register_extension appends to extensions_list (parallel array); codec.encode_message, text.emit_message, json.encode_message all iterate via 'for i=1,#elist'. Inline-mode codegen also walks the list via 'for _i = N+1, #_elist' for runtime-registered extensions past the statically-known set. No pairs() over extensions remains anywhere on the hot path. The issue was never closed in bd.","labels":["encoder","perf","proto2"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-bgu","title":"Decoder: ffi.string(ptr+off, len) for string/bytes decode","description":"Replace buf:sub(npos, npos+len-1) in decode_string/decode_bytes with ffi.string against a cached ffi.cast('const uint8_t*', buf). Removes method-call dispatch and lets the JIT fuse the read with surrounding code. Hits hardest on repeated-string fields (26 emails strings in 1KB Person -\u003e 26 sub calls today). Watch for LuaJIT string-intern collision behavior to stay identical to sub().","status":"closed","priority":2,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-17T15:47:10Z","created_by":"Eugene Blikh","updated_at":"2026-05-24T04:20:11Z","closed_at":"2026-05-24T04:20:11Z","close_reason":"Spike (bench/c_accel/bgu_probe.lua, removed) measured ffi.string vs buf:sub for the 1-byte LEN string fast path on Tarantool 3.8.0/LuaJIT 2.1.0-beta3 / Apple M-series. Results (ns/call):\n\n size buf:sub per-call-cast amortized/26 amortized/4\n 10B 89.7 280.5 (3.1x) 224.9 (2.5x) 246.2 (2.7x)\n 32B 107.6 299.4 (2.8x) 243.8 (2.3x) 260.9 (2.4x)\n 80B 154.3 362.4 (2.3x) 300.1 (1.9x) 314.9 (2.0x)\n\nV1 (per-call ffi.cast(U8CP, buf) inside decode_string): 2.3-3.1x slower.\nV2 (cast hoisted to caller, threaded as ptr arg, amortized over 26 emails — the Person 1KB peak shape): still 1.9-2.5x slower. Cast-once-amortize-many fails because 'ptr + np - 1' mints a fresh cdata wrapper per call that the JIT cannot sink across the decode_string frame — same root cause as a6n's failed top-of-_decode cast. The amortization helps a little (V2 \u003c V1) but does not close the gap.\n\nConclusion: bgu's premise (ffi.string lets the JIT fuse the string read with surrounding code) does not hold on the current LuaJIT fork — the cdata allocation cost dominates any reduction in dispatch. Closing as superseded by a6n-decode-inline-len-fast-path-2026-05-23. Revival criteria: a measurable cdata-sink win on a future LuaJIT (likely needs upstream allocation-sink improvements for pointer-arith cdata), OR an API redesign where the decoder works on a cdata buffer end-to-end (not a Lua string).","labels":["decoder","perf","wire"],"dependencies":[{"issue_id":"tarantool-protobuf-bgu","depends_on_id":"tarantool-protobuf-0an","type":"blocks","created_at":"2026-05-17T18:47:21Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-u39","title":"Decoder: emit table.new(0, N) for result tables","description":"Codegen knows the field count of every message at emission time. Emit table.new(0, N) instead of {} so the result table is sized correctly from the start. Cheap, broad win — especially closes the 1KB cliff where hello.Person rehashes as nested address + repeated fields populate. LuaJIT-only (table.new is in require('table.new')).","status":"closed","priority":2,"issue_type":"task","assignee":"Eugene Blikh","owner":"bigbes@gmail.com","created_at":"2026-05-17T15:47:08Z","created_by":"Eugene Blikh","updated_at":"2026-05-18T19:06:00Z","started_at":"2026-05-18T19:01:45Z","closed_at":"2026-05-18T19:06:00Z","close_reason":"Attempted and reverted. table_new(0, N) for decode result tables passed tests + JIT but regressed small-payload decode (10B -15.4%, 100B -17.7%) because the call cost exceeds rehash savings at small scale, and the small case doesn't reach the first rehash. Large-payload decode flat. Confirms drm bug: alloc is not the throughput bottleneck. See bench/PERF_LOG.md.","labels":["codegen","decoder","perf"],"dependency_count":0,"dependent_count":1,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-aah","title":"Encoder: codegen-emitted packed-scalar tight loops","description":"Replace per-element wire.encode_int32 calls with one inlined varint-emitting loop per packed field. Applies to packed int32, int64, sint32, sint64, bool, enum. The 1KB Person has 5 packed lucky_numbers + 26 emails — current per-element function-call boundary costs add up. Expected: 30%+ on packed-heavy payloads.","notes":"Shipped. work.lab.local 3-trial medians: packed_int32 +90/+113/+106%, packed_sint32 +66/+236/+156%, packed_uint32 +73/+105/+102%, packed_bool +44/+51/+39% (at 10/100/1000 elements). packed_int64 +22% (cdata path; fast path skipped, gain from table_new pre-sizing). Headline Person 1KB +3.6%, proto2 mid +10.5%.","status":"closed","priority":2,"issue_type":"task","assignee":"Eugene Blikh","owner":"bigbes@gmail.com","created_at":"2026-05-17T15:47:07Z","created_by":"Eugene Blikh","updated_at":"2026-05-24T17:20:54Z","started_at":"2026-05-24T17:08:14Z","closed_at":"2026-05-24T17:20:54Z","labels":["codegen","encoder","perf"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-4ql","title":"Encoder: optional caller-owned ffi.cdata ibuf API","description":"Separate API surface, not the default. ibuf-based encoder that writes into a caller-owned ffi.cdata buffer instead of returning a fresh Lua string. Targets hot RPC paths where the caller already owns a reusable buffer (e.g. net.box send path). Independent of the codegen rewrite — different API contract. Earlier attempt parked in stash@{0}; revisit after the codegen-time emission lands so we can compare apples-to-apples.","status":"closed","priority":2,"issue_type":"feature","owner":"bigbes@gmail.com","created_at":"2026-05-17T15:47:06Z","created_by":"Eugene Blikh","updated_at":"2026-05-24T04:30:27Z","closed_at":"2026-05-24T04:30:27Z","close_reason":"Spike (bench/c_accel/ibuf_probe.lua, removed) implemented a hand-coded Person_encode_ibuf mirroring what protoc-gen-tarantool would emit: stable pre-allocated cdata scratch buffer, direct p[i] byte writes, single-pass with backpatched length for nested Address + packed lucky_numbers, ffi.copy(p+i, lua_str, n) for strings. Same-byte correctness verified across 10B/100B/1KB/10KB.\n\nBench (ns/op, Tarantool 3.8.0 / Apple M-series), Person_encode vs ibuf scratch-only:\n\n size Person_encode ibuf scratch ibuf+ffi.string speedup\n 10B 462.8 239.1 273.7 1.69x (win)\n 100B 470.1 243.1 286.3 1.64x (win)\n 1KB 3693.8 4872.6 5141.6 0.72x (loss)\n 10KB 17146 42807 43023 0.40x (loss)\n 100KB 159081 422510 432701 0.37x (loss)\n\nCrossover ~26 emails. Root cause: each email pays an ffi.copy(p+i, lua_str, n) boundary (~50 ns/call). At 26 emails = ~1.3 us pure boundary; at 2800 emails (100KB) = ~140 us pure boundary. Meanwhile Person_encode appends Lua-string refs to an out table (no FFI boundary) and pays ONE bulk table.concat memcpy at the end regardless of count. Per-field boundary work beats per-message bulk work only when field count is very small.\n\nFor 4ql's stated use case (net.box send path), typical Tarantool RPC payloads are \u003e=1 KB — exactly the regression zone (1.4-2.7x slower). Win window (\u003c100B) is too narrow to justify a separate API surface, especially since pb.encode is already 462-470ns at that size — saving 200ns on a sub-microsecond operation is not a meaningful net.box gain.\n\nThree abandoned attempts now (per-byte b:alloc cliff, v2 two-pass bulk reserve, this single-pass backpatch). All hit the same per-field FFI boundary tax. Closing as 'not viable on current LuaJIT'. REVIVAL CRITERIA: a future LuaJIT that can sink FFI calls into traces (so per-field ffi.copy stops paying the boundary cost), OR a use case where caller passes pre-cdata-cast strings (no per-email lua-string -\u003e cdata copy), OR a hand-written C encoder behind the c_runtime FFI surface (different path entirely — see c0i).","labels":["api","encoder","perf"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-86g","title":"Encoder: two-pass with exact size precomputation","description":"Walk fields once to sum byte sizes, allocate the final string at exact size, walk again to write. Eliminates buffer-grow realloc; one lua_pushlstring. Same pattern as vtprotobuf's Size() + MarshalToVT. Stacks with the codegen-time inline writes (depends-on). Expected: additional 30-50% on large-message encode beyond the inline-FFI baseline.","notes":"2026-05-24 — verdict: not viable as a pure-Lua path. Same evidence as lkz close.\n\nHand-spike of the two-pass exact-size+emit shape (ffi.new uint8_t[?],\ninlined per-message sizer + writer, recursive) regressed encode 0.31x-0.77x\nacross all 5 size buckets vs the current `out` table + `table.concat` shape.\nProfile (jit.p) showed 84% of time in the inlined message bodies, not in\nbuffering — the perceived buffering inefficiency simply isn't there.\n\nThe single profile-hottest line (39% of Person_encode share) was replaced\nin 2ri (CHARS[_len] lookup) for +17%-34% encode throughput on 1KB+ payloads.\n\nBeating the pure-Lua ceiling requires escaping it — c0i (C-runtime backend)\nis the remaining path. Closing 86g as superseded.","status":"closed","priority":2,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-17T15:47:05Z","created_by":"Eugene Blikh","updated_at":"2026-05-24T16:53:44Z","closed_at":"2026-05-24T16:53:44Z","labels":["codegen","encoder","perf"],"dependencies":[{"issue_id":"tarantool-protobuf-86g","depends_on_id":"tarantool-protobuf-h8v","type":"blocks","created_at":"2026-05-17T18:47:19Z","created_by":"Eugene Blikh","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-drm","title":"Inspect: ~120 B/op fixed encode/decode overhead in Lua hot path","description":"Cross-runtime bench (bench/COMPARISON.md) shows a fixed per-call\nallocation floor on both encode and decode in `mode=full`:\n\n- encode: 136 B/op even at 10 B payload (output is 10 bytes; ~126 B overhead)\n- decode: 112 B/op even at 10 B payload (top-level table = 2 hash-tables)\n\nFor comparison, Go vtproto runs encode at 16 B/op and decode at 8 B/op\non the same 10 B fixture.\n\nInspect:\n 1. What's the source of the 100+ B encode overhead? Suspect candidates:\n - per-call buffer.ibuf workspace allocation in wire.lua\n - Lua string concat building the result\n - GCstr header on the output string itself (~24-32 B)\n 2. Decode floor: 112 B/op. Likely 2x table headers + small alloc for\n the top-level message table.\n 3. For large payloads (100 KB encode = 131605 B/op vs output 96674 B)\n the overhead is ~28 KB. Is that one big realloc tail, or many small?\n\nOutcome: a writeup pointing to specific lines and a recommendation\non whether the M6 ibuf path would actually help.","notes":"Findings from bench/alloc_probe.lua:\n\nENCODE 10B floor = 136 B/op is the `out` Lua table:\n- `local out, n = {}, 0` alone: 64 B (Lua GCtab base)\n- After 5 array entries: 136 B (matches the encode floor exactly)\n- table.concat: 0 B in the bench because output bytes intern\n- Varints: 0 B per call because 1-byte string.char outputs intern\n globally (small string dedup)\n- 2-byte varints DO allocate: ~33 B per fresh value (encode_varint_slow\n path returns a fresh string from string.char + bit.bor)\n\nDECODE 10B floor = 112 B/op is the top-level result table\n{name=..., age=...} — same shape across iters means it'd allocate the\nsame in a real workload.\n\nLARGE PAYLOAD (1KB Person):\n- same input (output interned): 1368 B/op\n- varying age (output unique): 2368 B/op\n- Delta = ~1000 B is the result string for 930 bytes of output (the\n GCstr header + 930 content; the extra ~70 B is presumably padding /\n alignment / the Address's nested concat).\n- So real cost per call has TWO components: (a) the small fixed tables\n for the encoder workspace, (b) the output bytes themselves.\n\nKEY INSIGHT: bench numbers UNDERSTATE real allocation. The bench\niterates the SAME input → output string interns → bench reads only\nthe table cost. Real workloads where every message is unique pay\noutput-size + table-cost.\n\nOPTIMIZATION RANKING:\n1. Highest ROI: encoder workspace table. ~136 B per top-level encode\n + ~136 B per nested message encode (Address adds its own). For a\n 1KB Person we have ~5 nested encoders → ~700 B of tables. Replacing\n the array-of-string-pieces with a single growable `buffer.ibuf` cuts\n this to ~0. This is the M6 ibuf path.\n2. Medium: encode_varint_slow returns a fresh string on each call for\n non-fast-path values. Tag bytes are precomputed as literals; only\n payload varints hit this. Inlining the slow path into wire.lua's\n hot caller (or returning into a passed-in buffer) drops this.\n3. Lowest: result string. Unavoidable for the encoder's API contract\n (returns a string). Only the lazy path avoids it.\n\nDECISION POINT: M6 ibuf path (already prototyped, deferred per\nmemory/decode_perf_deferred.md) is the right intervention.\nConservatively halves encode B/op on small messages, larger savings\non nested-heavy payloads.\nCORRECTION to earlier note.\n\nRecommendation to \"use M6 ibuf path to cut the 136 B/op floor\" was\nwrong. Per memory/tarantool_ibuf_perf.md (verified against current\nrepo state — feature NOT in HEAD):\n\n- Naive per-byte b:alloc(1): ~20× slower realistic, ~36× synthetic\n- Two-pass bulk-reserve: byte-equal correct, ~2× slower at every\n payload size. Sitting in git stash@{0}.\n\nBoth prototypes lose on wall time because:\n (a) Bench harness runs with jit.off, closure dispatch interpreted\n (b) Realistic encoder cost is dominated by per-field closures\n (sizer + writer + emit_tag + pwrite_*), NOT byte writes\n (c) Two-pass adds an extra walk on top\n\nAlso: the 136 B/op encode floor is NOT the throughput bottleneck.\nCross-runtime gap (5× apiv2, 13-20× vtproto) is JIT/dispatch\noverhead per field, not allocator pressure. Cutting 136 B doesn't\nclose that gap.\n\nVIABLE PATHS (neither tried):\n1. Single-pass with backpatched length varints. Walk once. For nested\n msgs: write tag + 1-byte placeholder, recurse, fill in (or memmove\n if final length ≥ 128). Eliminates the size pass (~3 µs at 1 KB).\n2. Codegen-time `_encode_ibuf` per message in protoc-gen-tarantool\n (mode=full). Straight-line ibuf writes, no descriptor walk at\n runtime. Mirrors what pb.encode already does for the table path.\n Probably the only approach that actually closes the throughput gap.\n\nEither has to keep per-field closure count ≤ pb.encode's writers,\notherwise we re-introduce the dispatch cost both stashed prototypes\nfell on.\n2026-05-24 re-verification (post-21d/qwt):\n\nAllocation floor unchanged. Fresh `tarantool bench/alloc_probe.lua`:\n Person_encode 10B (same input) 136.0 B/op\n Person_encode 100B (same input) 136.0 B/op\n Person_decode 10B (same input) 112.0 B/op\n\nDrill-down isolates the source line-by-line — `local out, n = {}, 0` (64 B base) + 5 array entries (8 B each at LuaJIT tab growth = +72 B) = 136 B exactly. Lines: examples/expected/full/hello/hello_pb.lua:1044 (the `out` table) and 1049/1056/1061+ (the `n = n + 1; out[n] = ...` writes that populate it).\n\n21d closed dispatch fragmentation (small-encode +42% throughput per memory/21d-encode-dispatch-codegen-2026-05-24) and the alloc floor did not move — consistent with the CORRECTION note: the 5x cross-runtime throughput gap is not allocator pressure, and naive byte-buffer rewrites already lost in two stashed prototypes.\n\nInvestigation complete. Actionable interventions live in:\n- lkz (single-pass with backpatched length varints into one buffer)\n- 86g (two-pass with exact-size precompute; vtproto's Size+MarshalTo)\n\nBoth depend on closed h8v (codegen FFI direct writes). Closing drm.","status":"closed","priority":2,"issue_type":"bug","owner":"bigbes@gmail.com","created_at":"2026-05-17T15:13:38Z","created_by":"Eugene Blikh","updated_at":"2026-05-24T15:24:55Z","closed_at":"2026-05-24T15:24:55Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"tarantool-protobuf-58u","title":"Runtime-mode decode_unsafe support (compile parallel _reader_unsafe closures)","description":"6bb landed _decode_unsafe in full mode only. Runtime mode (pb.decode(desc, buf) and the runtime-mode codegen wrapper) currently has no unsafe path because compile_readers builds f._reader closures that capture handler.decode by value — a runtime swap of scalar.string.decode wouldn't reach them. Implementation sketch: add M.compile_readers_unsafe(desc) that builds f._reader_unsafe by passing an alternate scalar table where scalar.string = scalar.bytes. Call it from pb.finalize_message alongside compile_readers. Add M.decode_unsafe(desc, buf) as a near-clone of decode_message that uses f._reader_unsafe and routes the map-fallback scalar dispatch through the unsafe scalar table. Then emit M.\u003cName\u003e_decode_unsafe = function(b) return pb.decode_unsafe(M.\u003cName\u003e_descriptor, b) end in gen.go's runtime-mode emitMessageWrappers for API symmetry with full mode.","status":"in_progress","priority":3,"issue_type":"task","assignee":"Eugene Blikh","owner":"bigbes@gmail.com","created_at":"2026-05-24T18:20:01Z","created_by":"Eugene Blikh","updated_at":"2026-05-24T18:59:16Z","started_at":"2026-05-24T18:59:16Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"tarantool-protobuf-kyt","title":"C runtime: skip_utf8_validation plan flag (unblock _decode_unsafe + C accel)","description":"6bb's full-mode _decode_unsafe skips the pb.c_runtime dispatch because runtime/pb/c/c_runtime.c calls is_valid_utf8 unconditionally on every string field. Result: when PB_ENABLE_C=1 the safe _decode wins on C but _decode_unsafe runs the inline Lua path and may be slower than C. To unify: add a skip_utf8_validation flag to the decode plan (or expose pb.c_runtime.decode_unsafe(plan, buf)), gate the is_valid_utf8 call on it in c_runtime.c, and wire _decode_unsafe to take the C path when c_runtime is available.","status":"open","priority":3,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-24T18:20:01Z","created_by":"Eugene Blikh","updated_at":"2026-05-24T18:20:01Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-h8x","title":"decode_group: 'inner loop in root trace' abort makes trace topology bimodal","description":"decode_group (runtime/pb/codec.lua:1089) has an 'inner loop in root trace' abort condition — the per-tag while-loop is hot enough to be a trace root itself but is reached from another root trace that tries to extend through it.\n\nObserved via bench/jit_trace.lua probe in isolation on full/WithGroup_decode (group):\n Mode A (typical, ~9/10 runs): starts=38, stops=5, aborts=33 (recompile loop)\n Mode B (rare, ~1/10 runs): starts=102, stops=100, aborts=2 (side-trace cascade)\n\nPre-existing on master (probed before/after the compile_encode_body fix landed for tarantool-protobuf-21d). Neither mode breaks the gate (both have stops\u003e0, no FATAL aborts), but the bimodal behavior is unstable and confused 21d's measurements.\n\nFix direction: same idea as 21d — emit a generated per-descriptor decode body (or at minimum, refactor decode_group so the inner while-loop is a separate function the JIT can compile as its own root trace). Mirrors compile_encode_body from the 21d fix.","status":"closed","priority":3,"issue_type":"bug","owner":"bigbes@gmail.com","created_at":"2026-05-24T09:53:50Z","created_by":"Eugene Blikh","updated_at":"2026-05-24T14:18:34Z","closed_at":"2026-05-24T14:18:34Z","close_reason":"wont_fix","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-3qu","title":"bench/*.lua: apply mcode arena hardening + re-snapshot baseline.json if numbers shift","description":"Sibling to 3o2 (closed). The trace gate had an intermittent 'fails silently with no JIT' mode on macOS arm64 caused by the default mcode arena being too small for our codegen footprint. The fix in 3o2 added jit.opt.start('sizemcode=64','maxmcode=4096') to bench/jit_trace.lua and 20/20 runs are now stable.\n\nThe other bench scripts have the same risk and none have the fix:\n bench/bench.lua\n bench/lazy_bench.lua\n bench/profile.lua\n bench/shapes_bench.lua\n bench/starwing_bench.lua\n bench/wire_bench.lua\n bench/alloc_probe.lua\n\nThese scripts have larger codegen footprints than the trace gate (they require more modules, run for longer, and accumulate more traces), so they're MORE likely to hit the same intermittent JIT-fails-silently mode than the gate was. When that happens the script reports throughput that includes interpreter-only iterations — underreporting the real numbers without any diagnostic.\n\nConcrete steps:\n1) Add the jit.opt.start line to each script (same comment block as 3o2, or factor into a tiny bench/_setup.lua included from each).\n2) Re-run bench/bench.lua --baseline to refresh bench/baseline.json.\n3) Re-run bench/bench.lua --print and update the MB/s tables in bench/COMPARISON.md if any number moved \u003e5%.\n4) bench/COMPARISON.md notes 'Numbers will drift run-to-run by 5–10%' — verify that variance band shrinks after the fix.\n\nalloc_probe.lua doesn't need it (allocator counters don't depend on JIT), but adding the line costs nothing and keeps the bench/ scripts uniform.","status":"open","priority":3,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-18T17:29:35Z","created_by":"Eugene Blikh","updated_at":"2026-05-18T17:29:35Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"tarantool-protobuf-auj","title":"Quantify and address Person_decode multi-byte varint side-trace bridge (intermittent ~1/5 runs)","description":"Trace-topology finding from bench/jit_trace.lua. The 'full/Person_decode multi-byte varint' check intermittently reports one bridge — a side trace whose linktype=interpreter, costing interp dispatch per multi-byte tag/length-prefix on the hot decode path.\n\nCaptured output (1–2 runs out of 10):\n\n [ OK ] full/Person_decode multi-byte varint (stops=10, bridges=1)\n info: bridge tr9 side-of tr5 hello_pb.lua:997 pc=55\n\ntr5 is the Person_decode while loop (entry at hello_pb.lua:997). pc=55 falls inside the wire.decode_tag inlined fast path: the guard 'if b \u003c 0x80' fails on a 2+ byte tag, exits to side trace tr9, which contains the multi-byte continuation loop but can't self-link back to the parent — drops to the interpreter to walk the rest of the dispatcher and re-enter on next iteration.\n\nExisting context:\n- wire.lua duplicates the 1-byte varint fast path at every hot decode call site precisely because LuaJIT side traces can't stitch returns from an inlined helper frame. That works for the 1-byte case. The 2+ byte case still calls decode_varint() (the multi-byte fallback) which has its own internal while loop.\n- gcy (inline nested decode) and kot (localize wire.* upvalues) are the structurally related items already filed; they don't cover this specific bridge though.\n\nWhy P3 (not P2):\n- Intermittent (~1/5 runs in the gate). The trace topology is mostly stable.\n- The multi-byte tag path is \u003c 5% of typical RPC payloads (field IDs 1..15 = 1-byte tag, lengths \u003c 128 = 1-byte length). Larger impact would require \u003e127-byte fields or field IDs \u003e= 16.\n- The 'multi-byte varint' fixture in bench/jit_trace.lua (200-byte name + lucky_numbers including 200000, 500000) was added specifically to expose this — and it does, intermittently. The intermittency is the JIT settling on different trace shapes across runs.\n\nConcrete approaches to investigate:\n1) Inline a 2-byte varint fast path inside decode_tag (and decode_string LEN prefix, etc.) — 'if b \u003c 0x80 then ... elseif b2 \u003c 0x80 then ...' — keeping the 3+ byte case in the fallback. Covers field IDs up to 4095 and length prefixes up to 16383, which is almost all real payloads.\n2) Profile-driven: run bench/jit_trace.lua 100x with a fixed seed, collect bridges by location, and decide whether the intermittency rate justifies (1) at all.\n\nAcceptance: 50 consecutive runs of bench/jit_trace.lua report bridges=0 for full/Person_decode multi-byte varint, OR a measured throughput improvement on the bench at the multi-byte-varint fixture.","status":"open","priority":3,"issue_type":"task","owner":"bigbes@gmail.com","created_at":"2026-05-18T17:27:36Z","created_by":"Eugene Blikh","updated_at":"2026-05-18T17:27:36Z","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/cmd/protoc-gen-tarantool/internal/gen/gen.go b/cmd/protoc-gen-tarantool/internal/gen/gen.go index 8194a195ad3fa978fb78cf826a692b3cd3f29e24..f089b21e659da34a65a8265d457ad0264d4875af 100644 --- a/cmd/protoc-gen-tarantool/internal/gen/gen.go +++ b/cmd/protoc-gen-tarantool/internal/gen/gen.go @@ -820,6 +820,12 @@ emitEmmyWrapperAnnotations(w, name, full, wrapperEncode) w.line("function M.%s_encode(t) return pb.encode(M.%s_descriptor, t) end", name, name) emitEmmyWrapperAnnotations(w, name, full, wrapperDecode) w.line("function M.%s_decode(b) return pb.decode(M.%s_descriptor, b) end", name, name) + // 58u: API-symmetric unsafe-decode wrapper. Routes through + // pb.decode_unsafe which uses the parallel `_reader_unsafe` + // closures compiled in pb.finalize_message. Full mode emits + // inline (a literal sister decoder); runtime mode shares one + // dispatcher. + w.line("function M.%s_decode_unsafe(b) return pb.decode_unsafe(M.%s_descriptor, b) end", name, name) emitEmmyWrapperAnnotations(w, name, full, wrapperDecodeLazy) w.line("function M.%s_decode_lazy(b) return pb.decode_lazy(M.%s_descriptor, b) end", name, name) emitEmmyWrapperAnnotations(w, name, full, wrapperText) diff --git a/docs/api-modes.md b/docs/api-modes.md index 9cd65d15d847be5b13381457fdc7e6f774d8d7ef..f991e081683bd28647b1acca42e8bc7a0eb555de 100644 --- a/docs/api-modes.md +++ b/docs/api-modes.md @@ -92,10 +92,17 @@ name** rather than a per-call flag so the JIT trace shape and inlining budget for the safe path are unchanged. The cost is generated code size: each message with string content emits a second decoder body. +Runtime mode (`mode=runtime` codegen and the reflective `pb.decode_unsafe` +entry) also supports the unsafe path: `pb.finalize_message` eagerly +compiles a parallel `f._reader_unsafe` set against a swapped scalar +table (`string -> bytes` handler), and `pb.decode_unsafe(desc, buf)` +dispatches through those readers with sub-message / extension / group +recursion all on the unsafe path. Runtime-mode wins are smaller — +~5-10% on the same payload — because the descriptor-dispatch and +closure indirection swamp the utf8_len savings, but it's still net +positive and the API is symmetric with full mode. + Caveats: -- **Runtime mode does not currently emit the unsafe variant.** The - reflective path (`pb.decode`) always validates. Migrate the hot decode - to full mode if you need the unsafe path. - **C-acceleration is bypassed in the unsafe variant** — the C runtime validates unconditionally today, so `_decode_unsafe` skips the `pb.c_runtime` dispatch and runs the inline Lua decoder. On payloads diff --git a/examples/expected/runtime/c_int64/c_int64_pb.lua b/examples/expected/runtime/c_int64/c_int64_pb.lua index ae36756f1a96540371d8c7f873d9837cfe1f4aa9..6c2c32240f76be8db90edf3f31770c295d844093 100644 --- a/examples/expected/runtime/c_int64/c_int64_pb.lua +++ b/examples/expected/runtime/c_int64/c_int64_pb.lua @@ -55,6 +55,7 @@ function M.Wide_encode(t) return pb.encode(M.Wide_descriptor, t) end ---@param b string ---@return c_int64.Wide function M.Wide_decode(b) return pb.decode(M.Wide_descriptor, b) end +function M.Wide_decode_unsafe(b) return pb.decode_unsafe(M.Wide_descriptor, b) end ---@param b string ---@return pb.MessageView function M.Wide_decode_lazy(b) return pb.decode_lazy(M.Wide_descriptor, b) end diff --git a/examples/expected/runtime/c_nested/c_nested_pb.lua b/examples/expected/runtime/c_nested/c_nested_pb.lua index 39c507b7e6ec0c705987971a546b002e6c52f570..e6703fceaad5df20f0b37280ba953a28d5c3efab 100644 --- a/examples/expected/runtime/c_nested/c_nested_pb.lua +++ b/examples/expected/runtime/c_nested/c_nested_pb.lua @@ -111,6 +111,7 @@ function M.L1_encode(t) return pb.encode(M.L1_descriptor, t) end ---@param b string ---@return c_nested.L1 function M.L1_decode(b) return pb.decode(M.L1_descriptor, b) end +function M.L1_decode_unsafe(b) return pb.decode_unsafe(M.L1_descriptor, b) end ---@param b string ---@return pb.MessageView function M.L1_decode_lazy(b) return pb.decode_lazy(M.L1_descriptor, b) end @@ -128,6 +129,7 @@ function M.L2_encode(t) return pb.encode(M.L2_descriptor, t) end ---@param b string ---@return c_nested.L2 function M.L2_decode(b) return pb.decode(M.L2_descriptor, b) end +function M.L2_decode_unsafe(b) return pb.decode_unsafe(M.L2_descriptor, b) end ---@param b string ---@return pb.MessageView function M.L2_decode_lazy(b) return pb.decode_lazy(M.L2_descriptor, b) end @@ -145,6 +147,7 @@ function M.L3_encode(t) return pb.encode(M.L3_descriptor, t) end ---@param b string ---@return c_nested.L3 function M.L3_decode(b) return pb.decode(M.L3_descriptor, b) end +function M.L3_decode_unsafe(b) return pb.decode_unsafe(M.L3_descriptor, b) end ---@param b string ---@return pb.MessageView function M.L3_decode_lazy(b) return pb.decode_lazy(M.L3_descriptor, b) end @@ -162,6 +165,7 @@ function M.L4_encode(t) return pb.encode(M.L4_descriptor, t) end ---@param b string ---@return c_nested.L4 function M.L4_decode(b) return pb.decode(M.L4_descriptor, b) end +function M.L4_decode_unsafe(b) return pb.decode_unsafe(M.L4_descriptor, b) end ---@param b string ---@return pb.MessageView function M.L4_decode_lazy(b) return pb.decode_lazy(M.L4_descriptor, b) end @@ -179,6 +183,7 @@ function M.L5_encode(t) return pb.encode(M.L5_descriptor, t) end ---@param b string ---@return c_nested.L5 function M.L5_decode(b) return pb.decode(M.L5_descriptor, b) end +function M.L5_decode_unsafe(b) return pb.decode_unsafe(M.L5_descriptor, b) end ---@param b string ---@return pb.MessageView function M.L5_decode_lazy(b) return pb.decode_lazy(M.L5_descriptor, b) end diff --git a/examples/expected/runtime/c_repeated/c_repeated_pb.lua b/examples/expected/runtime/c_repeated/c_repeated_pb.lua index 8f829d63afc30df75f869d9a660d28d10c180185..291e961b951c68aeb45833f34edbfc8f4e1b6c44 100644 --- a/examples/expected/runtime/c_repeated/c_repeated_pb.lua +++ b/examples/expected/runtime/c_repeated/c_repeated_pb.lua @@ -101,6 +101,7 @@ function M.Inner_encode(t) return pb.encode(M.Inner_descriptor, t) end ---@param b string ---@return c_repeated.Inner function M.Inner_decode(b) return pb.decode(M.Inner_descriptor, b) end +function M.Inner_decode_unsafe(b) return pb.decode_unsafe(M.Inner_descriptor, b) end ---@param b string ---@return pb.MessageView function M.Inner_decode_lazy(b) return pb.decode_lazy(M.Inner_descriptor, b) end @@ -118,6 +119,7 @@ function M.Holder_encode(t) return pb.encode(M.Holder_descriptor, t) end ---@param b string ---@return c_repeated.Holder function M.Holder_decode(b) return pb.decode(M.Holder_descriptor, b) end +function M.Holder_decode_unsafe(b) return pb.decode_unsafe(M.Holder_descriptor, b) end ---@param b string ---@return pb.MessageView function M.Holder_decode_lazy(b) return pb.decode_lazy(M.Holder_descriptor, b) end diff --git a/examples/expected/runtime/conformance/conformance_pb.lua b/examples/expected/runtime/conformance/conformance_pb.lua index 938ccc20f5a5741fa8bbd868cae6639259a9166b..5087cbb0105e3237c93830d93a5379a1adf4e513 100644 --- a/examples/expected/runtime/conformance/conformance_pb.lua +++ b/examples/expected/runtime/conformance/conformance_pb.lua @@ -207,6 +207,7 @@ function M.TestStatus_encode(t) return pb.encode(M.TestStatus_descriptor, t) end ---@param b string ---@return conformance.TestStatus function M.TestStatus_decode(b) return pb.decode(M.TestStatus_descriptor, b) end +function M.TestStatus_decode_unsafe(b) return pb.decode_unsafe(M.TestStatus_descriptor, b) end ---@param b string ---@return pb.MessageView function M.TestStatus_decode_lazy(b) return pb.decode_lazy(M.TestStatus_descriptor, b) end @@ -224,6 +225,7 @@ function M.FailureSet_encode(t) return pb.encode(M.FailureSet_descriptor, t) end ---@param b string ---@return conformance.FailureSet function M.FailureSet_decode(b) return pb.decode(M.FailureSet_descriptor, b) end +function M.FailureSet_decode_unsafe(b) return pb.decode_unsafe(M.FailureSet_descriptor, b) end ---@param b string ---@return pb.MessageView function M.FailureSet_decode_lazy(b) return pb.decode_lazy(M.FailureSet_descriptor, b) end @@ -241,6 +243,7 @@ function M.ConformanceRequest_encode(t) return pb.encode(M.ConformanceRequest_descriptor, t) end ---@param b string ---@return conformance.ConformanceRequest function M.ConformanceRequest_decode(b) return pb.decode(M.ConformanceRequest_descriptor, b) end +function M.ConformanceRequest_decode_unsafe(b) return pb.decode_unsafe(M.ConformanceRequest_descriptor, b) end ---@param b string ---@return pb.MessageView function M.ConformanceRequest_decode_lazy(b) return pb.decode_lazy(M.ConformanceRequest_descriptor, b) end @@ -258,6 +261,7 @@ function M.ConformanceResponse_encode(t) return pb.encode(M.ConformanceResponse_descriptor, t) end ---@param b string ---@return conformance.ConformanceResponse function M.ConformanceResponse_decode(b) return pb.decode(M.ConformanceResponse_descriptor, b) end +function M.ConformanceResponse_decode_unsafe(b) return pb.decode_unsafe(M.ConformanceResponse_descriptor, b) end ---@param b string ---@return pb.MessageView function M.ConformanceResponse_decode_lazy(b) return pb.decode_lazy(M.ConformanceResponse_descriptor, b) end @@ -275,6 +279,7 @@ function M.JspbEncodingConfig_encode(t) return pb.encode(M.JspbEncodingConfig_descriptor, t) end ---@param b string ---@return conformance.JspbEncodingConfig function M.JspbEncodingConfig_decode(b) return pb.decode(M.JspbEncodingConfig_descriptor, b) end +function M.JspbEncodingConfig_decode_unsafe(b) return pb.decode_unsafe(M.JspbEncodingConfig_descriptor, b) end ---@param b string ---@return pb.MessageView function M.JspbEncodingConfig_decode_lazy(b) return pb.decode_lazy(M.JspbEncodingConfig_descriptor, b) end diff --git a/examples/expected/runtime/hello/hello_pb.lua b/examples/expected/runtime/hello/hello_pb.lua index 376ba660e0ad979fb982d34243803b4f8d4c7476..2efa22c0752da09e31f00fe4ca6d089e8fa7c326 100644 --- a/examples/expected/runtime/hello/hello_pb.lua +++ b/examples/expected/runtime/hello/hello_pb.lua @@ -218,6 +218,7 @@ function M.Result_encode(t) return pb.encode(M.Result_descriptor, t) end ---@param b string ---@return hello.Result function M.Result_decode(b) return pb.decode(M.Result_descriptor, b) end +function M.Result_decode_unsafe(b) return pb.decode_unsafe(M.Result_descriptor, b) end ---@param b string ---@return pb.MessageView function M.Result_decode_lazy(b) return pb.decode_lazy(M.Result_descriptor, b) end @@ -235,6 +236,7 @@ function M.HelloRequest_encode(t) return pb.encode(M.HelloRequest_descriptor, t) end ---@param b string ---@return hello.HelloRequest function M.HelloRequest_decode(b) return pb.decode(M.HelloRequest_descriptor, b) end +function M.HelloRequest_decode_unsafe(b) return pb.decode_unsafe(M.HelloRequest_descriptor, b) end ---@param b string ---@return pb.MessageView function M.HelloRequest_decode_lazy(b) return pb.decode_lazy(M.HelloRequest_descriptor, b) end @@ -252,6 +254,7 @@ function M.HelloReply_encode(t) return pb.encode(M.HelloReply_descriptor, t) end ---@param b string ---@return hello.HelloReply function M.HelloReply_decode(b) return pb.decode(M.HelloReply_descriptor, b) end +function M.HelloReply_decode_unsafe(b) return pb.decode_unsafe(M.HelloReply_descriptor, b) end ---@param b string ---@return pb.MessageView function M.HelloReply_decode_lazy(b) return pb.decode_lazy(M.HelloReply_descriptor, b) end @@ -269,6 +272,7 @@ function M.Event_encode(t) return pb.encode(M.Event_descriptor, t) end ---@param b string ---@return hello.Event function M.Event_decode(b) return pb.decode(M.Event_descriptor, b) end +function M.Event_decode_unsafe(b) return pb.decode_unsafe(M.Event_descriptor, b) end ---@param b string ---@return pb.MessageView function M.Event_decode_lazy(b) return pb.decode_lazy(M.Event_descriptor, b) end @@ -286,6 +290,7 @@ function M.Address_encode(t) return pb.encode(M.Address_descriptor, t) end ---@param b string ---@return hello.Address function M.Address_decode(b) return pb.decode(M.Address_descriptor, b) end +function M.Address_decode_unsafe(b) return pb.decode_unsafe(M.Address_descriptor, b) end ---@param b string ---@return pb.MessageView function M.Address_decode_lazy(b) return pb.decode_lazy(M.Address_descriptor, b) end @@ -308,6 +313,7 @@ function M.Person_encode(t) return pb.encode(M.Person_descriptor, t) end ---@param b string ---@return hello.Person function M.Person_decode(b) return pb.decode(M.Person_descriptor, b) end +function M.Person_decode_unsafe(b) return pb.decode_unsafe(M.Person_descriptor, b) end ---@param b string ---@return pb.MessageView function M.Person_decode_lazy(b) return pb.decode_lazy(M.Person_descriptor, b) end diff --git a/examples/expected/runtime/proto2_basic/proto2_basic_pb.lua b/examples/expected/runtime/proto2_basic/proto2_basic_pb.lua index 391fed3700a34dcb246a1c51c295a1a420565b22..747fca0f3f5317291876802b65833a1e0c567e8b 100644 --- a/examples/expected/runtime/proto2_basic/proto2_basic_pb.lua +++ b/examples/expected/runtime/proto2_basic/proto2_basic_pb.lua @@ -245,6 +245,7 @@ function M.Defaults_encode(t) return pb.encode(M.Defaults_descriptor, t) end ---@param b string ---@return proto2_basic.Defaults function M.Defaults_decode(b) return pb.decode(M.Defaults_descriptor, b) end +function M.Defaults_decode_unsafe(b) return pb.decode_unsafe(M.Defaults_descriptor, b) end ---@param b string ---@return pb.MessageView function M.Defaults_decode_lazy(b) return pb.decode_lazy(M.Defaults_descriptor, b) end @@ -307,6 +308,7 @@ function M.Cardinality_encode(t) return pb.encode(M.Cardinality_descriptor, t) end ---@param b string ---@return proto2_basic.Cardinality function M.Cardinality_decode(b) return pb.decode(M.Cardinality_descriptor, b) end +function M.Cardinality_decode_unsafe(b) return pb.decode_unsafe(M.Cardinality_descriptor, b) end ---@param b string ---@return pb.MessageView function M.Cardinality_decode_lazy(b) return pb.decode_lazy(M.Cardinality_descriptor, b) end @@ -329,6 +331,7 @@ function M.Nested_encode(t) return pb.encode(M.Nested_descriptor, t) end ---@param b string ---@return proto2_basic.Nested function M.Nested_decode(b) return pb.decode(M.Nested_descriptor, b) end +function M.Nested_decode_unsafe(b) return pb.decode_unsafe(M.Nested_descriptor, b) end ---@param b string ---@return pb.MessageView function M.Nested_decode_lazy(b) return pb.decode_lazy(M.Nested_descriptor, b) end @@ -351,6 +354,7 @@ function M.Nested_Inner_encode(t) return pb.encode(M.Nested_Inner_descriptor, t) end ---@param b string ---@return proto2_basic.Nested.Inner function M.Nested_Inner_decode(b) return pb.decode(M.Nested_Inner_descriptor, b) end +function M.Nested_Inner_decode_unsafe(b) return pb.decode_unsafe(M.Nested_Inner_descriptor, b) end ---@param b string ---@return pb.MessageView function M.Nested_Inner_decode_lazy(b) return pb.decode_lazy(M.Nested_Inner_descriptor, b) end @@ -368,6 +372,7 @@ function M.WithGroup_encode(t) return pb.encode(M.WithGroup_descriptor, t) end ---@param b string ---@return proto2_basic.WithGroup function M.WithGroup_decode(b) return pb.decode(M.WithGroup_descriptor, b) end +function M.WithGroup_decode_unsafe(b) return pb.decode_unsafe(M.WithGroup_descriptor, b) end ---@param b string ---@return pb.MessageView function M.WithGroup_decode_lazy(b) return pb.decode_lazy(M.WithGroup_descriptor, b) end @@ -390,6 +395,7 @@ function M.WithGroup_SingleGroup_encode(t) return pb.encode(M.WithGroup_SingleGroup_descriptor, t) end ---@param b string ---@return proto2_basic.WithGroup.SingleGroup function M.WithGroup_SingleGroup_decode(b) return pb.decode(M.WithGroup_SingleGroup_descriptor, b) end +function M.WithGroup_SingleGroup_decode_unsafe(b) return pb.decode_unsafe(M.WithGroup_SingleGroup_descriptor, b) end ---@param b string ---@return pb.MessageView function M.WithGroup_SingleGroup_decode_lazy(b) return pb.decode_lazy(M.WithGroup_SingleGroup_descriptor, b) end @@ -417,6 +423,7 @@ function M.WithGroup_RepGroup_encode(t) return pb.encode(M.WithGroup_RepGroup_descriptor, t) end ---@param b string ---@return proto2_basic.WithGroup.RepGroup function M.WithGroup_RepGroup_decode(b) return pb.decode(M.WithGroup_RepGroup_descriptor, b) end +function M.WithGroup_RepGroup_decode_unsafe(b) return pb.decode_unsafe(M.WithGroup_RepGroup_descriptor, b) end ---@param b string ---@return pb.MessageView function M.WithGroup_RepGroup_decode_lazy(b) return pb.decode_lazy(M.WithGroup_RepGroup_descriptor, b) end @@ -439,6 +446,7 @@ function M.BenchPayload_encode(t) return pb.encode(M.BenchPayload_descriptor, t) end ---@param b string ---@return proto2_basic.BenchPayload function M.BenchPayload_decode(b) return pb.decode(M.BenchPayload_descriptor, b) end +function M.BenchPayload_decode_unsafe(b) return pb.decode_unsafe(M.BenchPayload_descriptor, b) end ---@param b string ---@return pb.MessageView function M.BenchPayload_decode_lazy(b) return pb.decode_lazy(M.BenchPayload_descriptor, b) end @@ -476,6 +484,7 @@ function M.BenchPayload_Stats_encode(t) return pb.encode(M.BenchPayload_Stats_descriptor, t) end ---@param b string ---@return proto2_basic.BenchPayload.Stats function M.BenchPayload_Stats_decode(b) return pb.decode(M.BenchPayload_Stats_descriptor, b) end +function M.BenchPayload_Stats_decode_unsafe(b) return pb.decode_unsafe(M.BenchPayload_Stats_descriptor, b) end ---@param b string ---@return pb.MessageView function M.BenchPayload_Stats_decode_lazy(b) return pb.decode_lazy(M.BenchPayload_Stats_descriptor, b) end @@ -503,6 +512,7 @@ function M.BenchPayload_Inner_encode(t) return pb.encode(M.BenchPayload_Inner_descriptor, t) end ---@param b string ---@return proto2_basic.BenchPayload.Inner function M.BenchPayload_Inner_decode(b) return pb.decode(M.BenchPayload_Inner_descriptor, b) end +function M.BenchPayload_Inner_decode_unsafe(b) return pb.decode_unsafe(M.BenchPayload_Inner_descriptor, b) end ---@param b string ---@return pb.MessageView function M.BenchPayload_Inner_decode_lazy(b) return pb.decode_lazy(M.BenchPayload_Inner_descriptor, b) end diff --git a/examples/expected/runtime/protobuf_test_messages/proto2/test_messages_proto2_pb.lua b/examples/expected/runtime/protobuf_test_messages/proto2/test_messages_proto2_pb.lua index 02e6926fd5c46dc3ed7acd61ab47b25e15bc8f3d..838c54d509cc4b5338b3d60930f200c571d5275b 100644 --- a/examples/expected/runtime/protobuf_test_messages/proto2/test_messages_proto2_pb.lua +++ b/examples/expected/runtime/protobuf_test_messages/proto2/test_messages_proto2_pb.lua @@ -907,6 +907,7 @@ function M.TestAllTypesProto2_encode(t) return pb.encode(M.TestAllTypesProto2_descriptor, t) end ---@param b string ---@return protobuf_test_messages.proto2.TestAllTypesProto2 function M.TestAllTypesProto2_decode(b) return pb.decode(M.TestAllTypesProto2_descriptor, b) end +function M.TestAllTypesProto2_decode_unsafe(b) return pb.decode_unsafe(M.TestAllTypesProto2_descriptor, b) end ---@param b string ---@return pb.MessageView function M.TestAllTypesProto2_decode_lazy(b) return pb.decode_lazy(M.TestAllTypesProto2_descriptor, b) end @@ -1209,6 +1210,7 @@ function M.TestAllTypesProto2_NestedMessage_encode(t) return pb.encode(M.TestAllTypesProto2_NestedMessage_descriptor, t) end ---@param b string ---@return protobuf_test_messages.proto2.TestAllTypesProto2.NestedMessage function M.TestAllTypesProto2_NestedMessage_decode(b) return pb.decode(M.TestAllTypesProto2_NestedMessage_descriptor, b) end +function M.TestAllTypesProto2_NestedMessage_decode_unsafe(b) return pb.decode_unsafe(M.TestAllTypesProto2_NestedMessage_descriptor, b) end ---@param b string ---@return pb.MessageView function M.TestAllTypesProto2_NestedMessage_decode_lazy(b) return pb.decode_lazy(M.TestAllTypesProto2_NestedMessage_descriptor, b) end @@ -1236,6 +1238,7 @@ function M.TestAllTypesProto2_Data_encode(t) return pb.encode(M.TestAllTypesProto2_Data_descriptor, t) end ---@param b string ---@return protobuf_test_messages.proto2.TestAllTypesProto2.Data function M.TestAllTypesProto2_Data_decode(b) return pb.decode(M.TestAllTypesProto2_Data_descriptor, b) end +function M.TestAllTypesProto2_Data_decode_unsafe(b) return pb.decode_unsafe(M.TestAllTypesProto2_Data_descriptor, b) end ---@param b string ---@return pb.MessageView function M.TestAllTypesProto2_Data_decode_lazy(b) return pb.decode_lazy(M.TestAllTypesProto2_Data_descriptor, b) end @@ -1263,6 +1266,7 @@ function M.TestAllTypesProto2_MultiWordGroupField_encode(t) return pb.encode(M.TestAllTypesProto2_MultiWordGroupField_descriptor, t) end ---@param b string ---@return protobuf_test_messages.proto2.TestAllTypesProto2.MultiWordGroupField function M.TestAllTypesProto2_MultiWordGroupField_decode(b) return pb.decode(M.TestAllTypesProto2_MultiWordGroupField_descriptor, b) end +function M.TestAllTypesProto2_MultiWordGroupField_decode_unsafe(b) return pb.decode_unsafe(M.TestAllTypesProto2_MultiWordGroupField_descriptor, b) end ---@param b string ---@return pb.MessageView function M.TestAllTypesProto2_MultiWordGroupField_decode_lazy(b) return pb.decode_lazy(M.TestAllTypesProto2_MultiWordGroupField_descriptor, b) end @@ -1290,6 +1294,7 @@ function M.ForeignMessageProto2_encode(t) return pb.encode(M.ForeignMessageProto2_descriptor, t) end ---@param b string ---@return protobuf_test_messages.proto2.ForeignMessageProto2 function M.ForeignMessageProto2_decode(b) return pb.decode(M.ForeignMessageProto2_descriptor, b) end +function M.ForeignMessageProto2_decode_unsafe(b) return pb.decode_unsafe(M.ForeignMessageProto2_descriptor, b) end ---@param b string ---@return pb.MessageView function M.ForeignMessageProto2_decode_lazy(b) return pb.decode_lazy(M.ForeignMessageProto2_descriptor, b) end @@ -1312,6 +1317,7 @@ function M.GroupField_encode(t) return pb.encode(M.GroupField_descriptor, t) end ---@param b string ---@return protobuf_test_messages.proto2.GroupField function M.GroupField_decode(b) return pb.decode(M.GroupField_descriptor, b) end +function M.GroupField_decode_unsafe(b) return pb.decode_unsafe(M.GroupField_descriptor, b) end ---@param b string ---@return pb.MessageView function M.GroupField_decode_lazy(b) return pb.decode_lazy(M.GroupField_descriptor, b) end @@ -1339,6 +1345,7 @@ function M.UnknownToTestAllTypes_encode(t) return pb.encode(M.UnknownToTestAllTypes_descriptor, t) end ---@param b string ---@return protobuf_test_messages.proto2.UnknownToTestAllTypes function M.UnknownToTestAllTypes_decode(b) return pb.decode(M.UnknownToTestAllTypes_descriptor, b) end +function M.UnknownToTestAllTypes_decode_unsafe(b) return pb.decode_unsafe(M.UnknownToTestAllTypes_descriptor, b) end ---@param b string ---@return pb.MessageView function M.UnknownToTestAllTypes_decode_lazy(b) return pb.decode_lazy(M.UnknownToTestAllTypes_descriptor, b) end @@ -1381,6 +1388,7 @@ function M.UnknownToTestAllTypes_OptionalGroup_encode(t) return pb.encode(M.UnknownToTestAllTypes_OptionalGroup_descriptor, t) end ---@param b string ---@return protobuf_test_messages.proto2.UnknownToTestAllTypes.OptionalGroup function M.UnknownToTestAllTypes_OptionalGroup_decode(b) return pb.decode(M.UnknownToTestAllTypes_OptionalGroup_descriptor, b) end +function M.UnknownToTestAllTypes_OptionalGroup_decode_unsafe(b) return pb.decode_unsafe(M.UnknownToTestAllTypes_OptionalGroup_descriptor, b) end ---@param b string ---@return pb.MessageView function M.UnknownToTestAllTypes_OptionalGroup_decode_lazy(b) return pb.decode_lazy(M.UnknownToTestAllTypes_OptionalGroup_descriptor, b) end @@ -1403,6 +1411,7 @@ function M.NullHypothesisProto2_encode(t) return pb.encode(M.NullHypothesisProto2_descriptor, t) end ---@param b string ---@return protobuf_test_messages.proto2.NullHypothesisProto2 function M.NullHypothesisProto2_decode(b) return pb.decode(M.NullHypothesisProto2_descriptor, b) end +function M.NullHypothesisProto2_decode_unsafe(b) return pb.decode_unsafe(M.NullHypothesisProto2_descriptor, b) end ---@param b string ---@return pb.MessageView function M.NullHypothesisProto2_decode_lazy(b) return pb.decode_lazy(M.NullHypothesisProto2_descriptor, b) end @@ -1420,6 +1429,7 @@ function M.EnumOnlyProto2_encode(t) return pb.encode(M.EnumOnlyProto2_descriptor, t) end ---@param b string ---@return protobuf_test_messages.proto2.EnumOnlyProto2 function M.EnumOnlyProto2_decode(b) return pb.decode(M.EnumOnlyProto2_descriptor, b) end +function M.EnumOnlyProto2_decode_unsafe(b) return pb.decode_unsafe(M.EnumOnlyProto2_descriptor, b) end ---@param b string ---@return pb.MessageView function M.EnumOnlyProto2_decode_lazy(b) return pb.decode_lazy(M.EnumOnlyProto2_descriptor, b) end @@ -1437,6 +1447,7 @@ function M.OneStringProto2_encode(t) return pb.encode(M.OneStringProto2_descriptor, t) end ---@param b string ---@return protobuf_test_messages.proto2.OneStringProto2 function M.OneStringProto2_decode(b) return pb.decode(M.OneStringProto2_descriptor, b) end +function M.OneStringProto2_decode_unsafe(b) return pb.decode_unsafe(M.OneStringProto2_descriptor, b) end ---@param b string ---@return pb.MessageView function M.OneStringProto2_decode_lazy(b) return pb.decode_lazy(M.OneStringProto2_descriptor, b) end @@ -1459,6 +1470,7 @@ function M.ProtoWithKeywords_encode(t) return pb.encode(M.ProtoWithKeywords_descriptor, t) end ---@param b string ---@return protobuf_test_messages.proto2.ProtoWithKeywords function M.ProtoWithKeywords_decode(b) return pb.decode(M.ProtoWithKeywords_descriptor, b) end +function M.ProtoWithKeywords_decode_unsafe(b) return pb.decode_unsafe(M.ProtoWithKeywords_descriptor, b) end ---@param b string ---@return pb.MessageView function M.ProtoWithKeywords_decode_lazy(b) return pb.decode_lazy(M.ProtoWithKeywords_descriptor, b) end @@ -1486,6 +1498,7 @@ function M.TestAllRequiredTypesProto2_encode(t) return pb.encode(M.TestAllRequiredTypesProto2_descriptor, t) end ---@param b string ---@return protobuf_test_messages.proto2.TestAllRequiredTypesProto2 function M.TestAllRequiredTypesProto2_decode(b) return pb.decode(M.TestAllRequiredTypesProto2_descriptor, b) end +function M.TestAllRequiredTypesProto2_decode_unsafe(b) return pb.decode_unsafe(M.TestAllRequiredTypesProto2_descriptor, b) end ---@param b string ---@return pb.MessageView function M.TestAllRequiredTypesProto2_decode_lazy(b) return pb.decode_lazy(M.TestAllRequiredTypesProto2_descriptor, b) end @@ -1508,6 +1521,7 @@ function M.TestAllRequiredTypesProto2_NestedMessage_encode(t) return pb.encode(M.TestAllRequiredTypesProto2_NestedMessage_descriptor, t) end ---@param b string ---@return protobuf_test_messages.proto2.TestAllRequiredTypesProto2.NestedMessage function M.TestAllRequiredTypesProto2_NestedMessage_decode(b) return pb.decode(M.TestAllRequiredTypesProto2_NestedMessage_descriptor, b) end +function M.TestAllRequiredTypesProto2_NestedMessage_decode_unsafe(b) return pb.decode_unsafe(M.TestAllRequiredTypesProto2_NestedMessage_descriptor, b) end ---@param b string ---@return pb.MessageView function M.TestAllRequiredTypesProto2_NestedMessage_decode_lazy(b) return pb.decode_lazy(M.TestAllRequiredTypesProto2_NestedMessage_descriptor, b) end @@ -1530,6 +1544,7 @@ function M.TestAllRequiredTypesProto2_Data_encode(t) return pb.encode(M.TestAllRequiredTypesProto2_Data_descriptor, t) end ---@param b string ---@return protobuf_test_messages.proto2.TestAllRequiredTypesProto2.Data function M.TestAllRequiredTypesProto2_Data_decode(b) return pb.decode(M.TestAllRequiredTypesProto2_Data_descriptor, b) end +function M.TestAllRequiredTypesProto2_Data_decode_unsafe(b) return pb.decode_unsafe(M.TestAllRequiredTypesProto2_Data_descriptor, b) end ---@param b string ---@return pb.MessageView function M.TestAllRequiredTypesProto2_Data_decode_lazy(b) return pb.decode_lazy(M.TestAllRequiredTypesProto2_Data_descriptor, b) end @@ -1547,6 +1562,7 @@ function M.TestLargeOneof_encode(t) return pb.encode(M.TestLargeOneof_descriptor, t) end ---@param b string ---@return protobuf_test_messages.proto2.TestLargeOneof function M.TestLargeOneof_decode(b) return pb.decode(M.TestLargeOneof_descriptor, b) end +function M.TestLargeOneof_decode_unsafe(b) return pb.decode_unsafe(M.TestLargeOneof_descriptor, b) end ---@param b string ---@return pb.MessageView function M.TestLargeOneof_decode_lazy(b) return pb.decode_lazy(M.TestLargeOneof_descriptor, b) end @@ -1564,6 +1580,7 @@ function M.TestLargeOneof_A1_encode(t) return pb.encode(M.TestLargeOneof_A1_descriptor, t) end ---@param b string ---@return protobuf_test_messages.proto2.TestLargeOneof.A1 function M.TestLargeOneof_A1_decode(b) return pb.decode(M.TestLargeOneof_A1_descriptor, b) end +function M.TestLargeOneof_A1_decode_unsafe(b) return pb.decode_unsafe(M.TestLargeOneof_A1_descriptor, b) end ---@param b string ---@return pb.MessageView function M.TestLargeOneof_A1_decode_lazy(b) return pb.decode_lazy(M.TestLargeOneof_A1_descriptor, b) end @@ -1581,6 +1598,7 @@ function M.TestLargeOneof_A2_encode(t) return pb.encode(M.TestLargeOneof_A2_descriptor, t) end ---@param b string ---@return protobuf_test_messages.proto2.TestLargeOneof.A2 function M.TestLargeOneof_A2_decode(b) return pb.decode(M.TestLargeOneof_A2_descriptor, b) end +function M.TestLargeOneof_A2_decode_unsafe(b) return pb.decode_unsafe(M.TestLargeOneof_A2_descriptor, b) end ---@param b string ---@return pb.MessageView function M.TestLargeOneof_A2_decode_lazy(b) return pb.decode_lazy(M.TestLargeOneof_A2_descriptor, b) end @@ -1598,6 +1616,7 @@ function M.TestLargeOneof_A3_encode(t) return pb.encode(M.TestLargeOneof_A3_descriptor, t) end ---@param b string ---@return protobuf_test_messages.proto2.TestLargeOneof.A3 function M.TestLargeOneof_A3_decode(b) return pb.decode(M.TestLargeOneof_A3_descriptor, b) end +function M.TestLargeOneof_A3_decode_unsafe(b) return pb.decode_unsafe(M.TestLargeOneof_A3_descriptor, b) end ---@param b string ---@return pb.MessageView function M.TestLargeOneof_A3_decode_lazy(b) return pb.decode_lazy(M.TestLargeOneof_A3_descriptor, b) end @@ -1615,6 +1634,7 @@ function M.TestLargeOneof_A4_encode(t) return pb.encode(M.TestLargeOneof_A4_descriptor, t) end ---@param b string ---@return protobuf_test_messages.proto2.TestLargeOneof.A4 function M.TestLargeOneof_A4_decode(b) return pb.decode(M.TestLargeOneof_A4_descriptor, b) end +function M.TestLargeOneof_A4_decode_unsafe(b) return pb.decode_unsafe(M.TestLargeOneof_A4_descriptor, b) end ---@param b string ---@return pb.MessageView function M.TestLargeOneof_A4_decode_lazy(b) return pb.decode_lazy(M.TestLargeOneof_A4_descriptor, b) end @@ -1632,6 +1652,7 @@ function M.TestLargeOneof_A5_encode(t) return pb.encode(M.TestLargeOneof_A5_descriptor, t) end ---@param b string ---@return protobuf_test_messages.proto2.TestLargeOneof.A5 function M.TestLargeOneof_A5_decode(b) return pb.decode(M.TestLargeOneof_A5_descriptor, b) end +function M.TestLargeOneof_A5_decode_unsafe(b) return pb.decode_unsafe(M.TestLargeOneof_A5_descriptor, b) end ---@param b string ---@return pb.MessageView function M.TestLargeOneof_A5_decode_lazy(b) return pb.decode_lazy(M.TestLargeOneof_A5_descriptor, b) end diff --git a/examples/expected/runtime/protobuf_test_messages/proto3/test_messages_proto3_pb.lua b/examples/expected/runtime/protobuf_test_messages/proto3/test_messages_proto3_pb.lua index 19397fb4351de8247374d5cad9173c1d5db466af..ae0235c377e36a5ccba595aed232696104951f2b 100644 --- a/examples/expected/runtime/protobuf_test_messages/proto3/test_messages_proto3_pb.lua +++ b/examples/expected/runtime/protobuf_test_messages/proto3/test_messages_proto3_pb.lua @@ -601,6 +601,7 @@ function M.TestAllTypesProto3_encode(t) return pb.encode(M.TestAllTypesProto3_descriptor, t) end ---@param b string ---@return protobuf_test_messages.proto3.TestAllTypesProto3 function M.TestAllTypesProto3_decode(b) return pb.decode(M.TestAllTypesProto3_descriptor, b) end +function M.TestAllTypesProto3_decode_unsafe(b) return pb.decode_unsafe(M.TestAllTypesProto3_descriptor, b) end ---@param b string ---@return pb.MessageView function M.TestAllTypesProto3_decode_lazy(b) return pb.decode_lazy(M.TestAllTypesProto3_descriptor, b) end @@ -618,6 +619,7 @@ function M.TestAllTypesProto3_NestedMessage_encode(t) return pb.encode(M.TestAllTypesProto3_NestedMessage_descriptor, t) end ---@param b string ---@return protobuf_test_messages.proto3.TestAllTypesProto3.NestedMessage function M.TestAllTypesProto3_NestedMessage_decode(b) return pb.decode(M.TestAllTypesProto3_NestedMessage_descriptor, b) end +function M.TestAllTypesProto3_NestedMessage_decode_unsafe(b) return pb.decode_unsafe(M.TestAllTypesProto3_NestedMessage_descriptor, b) end ---@param b string ---@return pb.MessageView function M.TestAllTypesProto3_NestedMessage_decode_lazy(b) return pb.decode_lazy(M.TestAllTypesProto3_NestedMessage_descriptor, b) end @@ -635,6 +637,7 @@ function M.ForeignMessage_encode(t) return pb.encode(M.ForeignMessage_descriptor, t) end ---@param b string ---@return protobuf_test_messages.proto3.ForeignMessage function M.ForeignMessage_decode(b) return pb.decode(M.ForeignMessage_descriptor, b) end +function M.ForeignMessage_decode_unsafe(b) return pb.decode_unsafe(M.ForeignMessage_descriptor, b) end ---@param b string ---@return pb.MessageView function M.ForeignMessage_decode_lazy(b) return pb.decode_lazy(M.ForeignMessage_descriptor, b) end @@ -652,6 +655,7 @@ function M.NullHypothesisProto3_encode(t) return pb.encode(M.NullHypothesisProto3_descriptor, t) end ---@param b string ---@return protobuf_test_messages.proto3.NullHypothesisProto3 function M.NullHypothesisProto3_decode(b) return pb.decode(M.NullHypothesisProto3_descriptor, b) end +function M.NullHypothesisProto3_decode_unsafe(b) return pb.decode_unsafe(M.NullHypothesisProto3_descriptor, b) end ---@param b string ---@return pb.MessageView function M.NullHypothesisProto3_decode_lazy(b) return pb.decode_lazy(M.NullHypothesisProto3_descriptor, b) end @@ -669,6 +673,7 @@ function M.EnumOnlyProto3_encode(t) return pb.encode(M.EnumOnlyProto3_descriptor, t) end ---@param b string ---@return protobuf_test_messages.proto3.EnumOnlyProto3 function M.EnumOnlyProto3_decode(b) return pb.decode(M.EnumOnlyProto3_descriptor, b) end +function M.EnumOnlyProto3_decode_unsafe(b) return pb.decode_unsafe(M.EnumOnlyProto3_descriptor, b) end ---@param b string ---@return pb.MessageView function M.EnumOnlyProto3_decode_lazy(b) return pb.decode_lazy(M.EnumOnlyProto3_descriptor, b) end diff --git a/examples/expected/runtime/quickstart/quickstart_pb.lua b/examples/expected/runtime/quickstart/quickstart_pb.lua index c605756e07badcc434959495602e5decbe5889b1..ba02be7439622b84980d3295842ed6d5e921ab5a 100644 --- a/examples/expected/runtime/quickstart/quickstart_pb.lua +++ b/examples/expected/runtime/quickstart/quickstart_pb.lua @@ -62,6 +62,7 @@ function M.User_encode(t) return pb.encode(M.User_descriptor, t) end ---@param b string ---@return quickstart.User function M.User_decode(b) return pb.decode(M.User_descriptor, b) end +function M.User_decode_unsafe(b) return pb.decode_unsafe(M.User_descriptor, b) end ---@param b string ---@return pb.MessageView function M.User_decode_lazy(b) return pb.decode_lazy(M.User_descriptor, b) end diff --git a/runtime/pb/codec.lua b/runtime/pb/codec.lua index ce699affe9bb107cb293c0235bc4e5b53da3fb6f..15a65a9d3447eb07a35fc7e7e0a44a45511af801 100644 --- a/runtime/pb/codec.lua +++ b/runtime/pb/codec.lua @@ -34,6 +34,17 @@ -- Reuse the single source of truth defined in pb.wire. local scalar = wire.TYPE_INFO M.scalar = scalar +-- Parallel scalar table for the opt-in unsafe decode path: identical to +-- `scalar` except `string` decoder skips UTF-8 validation by routing +-- through the bytes handler (same wire format, no utf8_len). Compiled +-- readers built against this table inherit the swap as a captured +-- upvalue — the alternative (swap-on-call) cannot reach pre-compiled +-- f._reader closures because they capture handler.decode by value. +-- See M.decode_unsafe / pb.compile_readers_unsafe. (6bb, 58u) +local scalar_unsafe = {} +for k, v in pairs(scalar) do scalar_unsafe[k] = v end +scalar_unsafe.string = scalar.bytes + -- --------------------------------------------------------------------------- -- proto3 default-value detection (for elision on encode) -- --------------------------------------------------------------------------- @@ -71,6 +82,11 @@ -- the per-tag dispatch shares fast-path knowledge with decode_message. local encode_message local decode_msg local decode_group +-- 58u: unsafe-decode twins. The unsafe reader closures (one per field, +-- built by compile_readers_unsafe) capture these so sub-message recursion +-- and group decoding stay on the unsafe path all the way down. +local decode_msg_unsafe +local decode_group_unsafe -- merge_message(desc, prev, decoded): recursively merge `decoded` into -- `prev` per proto3 spec semantics: @@ -723,13 +739,23 @@ -- one entry per wire-encounter, which is awkward to express as a closure -- without churning the result table); same as on the encoder side. -- --------------------------------------------------------------------------- -local function build_repeated_reader(f) +-- build_repeated_reader / build_reader are parameterized by the scalar +-- handler table and the sub-message dispatcher so the same builders can +-- emit either the standard validating readers (scalar_tbl=scalar, +-- decode_msg_fn=decode_msg) or the 58u unsafe readers +-- (scalar_tbl=scalar_unsafe, decode_msg_fn=decode_msg_unsafe). All +-- references that would otherwise tie the closure to the safe path +-- (handler.decode capture, decode_msg, decode_group) flow through the +-- arguments. Compile-time capture: the closures bind these upvalues +-- by value, so a runtime swap on `scalar` afterwards has no effect — +-- that's why the parallel unsafe path exists. +local function build_repeated_reader(f, scalar_tbl, decode_msg_fn, decode_group_fn) local fname = f.name local kind = f.kind local siblings = f.oneof_siblings -- nil if not in a oneof if kind == 'scalar' then - local handler = scalar[f.proto_type] + local handler = scalar_tbl[f.proto_type] if not handler then return nil end local decode_value = handler.decode local packable = handler.packable and (handler.wire ~= wire.WIRE_LEN) @@ -787,7 +813,7 @@ return function(buf, pos, wt, result) local list = result[fname] if list == nil then list = {}; result[fname] = list end local payload, np = decode_len_fn(buf, pos) - list[#list + 1] = decode_msg(sub_desc, payload) + list[#list + 1] = decode_msg_fn(sub_desc, payload) return np end end @@ -798,7 +824,7 @@ local stop_id = f.id return function(buf, pos, wt, result) local list = result[fname] if list == nil then list = {}; result[fname] = list end - local decoded, np = decode_group(sub_desc, buf, pos, stop_id) + local decoded, np = decode_group_fn(sub_desc, buf, pos, stop_id) list[#list + 1] = decoded return np end @@ -807,7 +833,7 @@ return nil end -local function build_reader(f) +local function build_reader(f, scalar_tbl, decode_msg_fn, decode_group_fn) -- Maps stay on the in-loop dispatch path. if f.kind == 'map' then return nil end @@ -815,10 +841,10 @@ local fname = f.name local kind = f.kind local siblings = f.oneof_siblings -- nil if not in a oneof - if f.repeated then return build_repeated_reader(f) end + if f.repeated then return build_repeated_reader(f, scalar_tbl, decode_msg_fn, decode_group_fn) end if kind == 'scalar' then - local handler = scalar[f.proto_type] + local handler = scalar_tbl[f.proto_type] if not handler then return nil end local decode_value = handler.decode if siblings then @@ -868,21 +894,21 @@ if has_custom_decode then if siblings then return function(buf, pos, wt, result) local payload, np = decode_len_fn(buf, pos) - result[fname] = decode_msg(sub_desc, payload) + result[fname] = decode_msg_fn(sub_desc, payload) for i = 1, #siblings do result[siblings[i]] = nil end return np end end return function(buf, pos, wt, result) local payload, np = decode_len_fn(buf, pos) - result[fname] = decode_msg(sub_desc, payload) + result[fname] = decode_msg_fn(sub_desc, payload) return np end end if siblings then return function(buf, pos, wt, result) local payload, np = decode_len_fn(buf, pos) - local decoded = decode_msg(sub_desc, payload) + local decoded = decode_msg_fn(sub_desc, payload) local prev = result[fname] if prev == nil then result[fname] = decoded @@ -895,7 +921,7 @@ end end return function(buf, pos, wt, result) local payload, np = decode_len_fn(buf, pos) - local decoded = decode_msg(sub_desc, payload) + local decoded = decode_msg_fn(sub_desc, payload) local prev = result[fname] if prev == nil then result[fname] = decoded @@ -911,7 +937,7 @@ local sub_desc = f.message local stop_id = f.id if siblings then return function(buf, pos, wt, result) - local decoded, np = decode_group(sub_desc, buf, pos, stop_id) + local decoded, np = decode_group_fn(sub_desc, buf, pos, stop_id) local prev = result[fname] if prev == nil then result[fname] = decoded @@ -923,7 +949,7 @@ return np end end return function(buf, pos, wt, result) - local decoded, np = decode_group(sub_desc, buf, pos, stop_id) + local decoded, np = decode_group_fn(sub_desc, buf, pos, stop_id) local prev = result[fname] if prev == nil then result[fname] = decoded @@ -940,7 +966,20 @@ ---@param desc pb.Descriptor function M.compile_readers(desc) for _, f in ipairs(desc.fields) do - f._reader = build_reader(f) + f._reader = build_reader(f, scalar, decode_msg, decode_group) + end +end + +-- 58u: compile_readers_unsafe builds the parallel f._reader_unsafe set +-- against the swapped scalar table (string -> bytes) and the unsafe +-- sub-message / group dispatchers. Called by pb.finalize_message right +-- after compile_readers so every descriptor carries both reader shapes +-- and pb.decode_unsafe stays at full reader-fastpath speed. +---@param desc pb.Descriptor +function M.compile_readers_unsafe(desc) + for _, f in ipairs(desc.fields) do + f._reader_unsafe = build_reader(f, scalar_unsafe, + decode_msg_unsafe, decode_group_unsafe) end end @@ -1036,16 +1075,19 @@ return decode_message(desc, buf) end -- decode_one returns (value, new_pos) for a single value based on field kind. -local function decode_one(field, buf, pos) +-- scalar_tbl and decode_msg_fn are parameterized so the same helper serves +-- both the validating (decode_message) and unsafe (decode_message_unsafe) +-- map fallback paths. (58u) +local function decode_one(field, buf, pos, scalar_tbl, decode_msg_fn) local kind = field.kind if kind == 'scalar' then - return scalar[field.proto_type].decode(buf, pos) + return scalar_tbl[field.proto_type].decode(buf, pos) elseif kind == 'enum' then local u, np = wire.decode_varint(buf, pos) return wire.varint_to_int32(u), np elseif kind == 'message' then local payload, np = wire.decode_len(buf, pos) - return decode_msg(field.message, payload), np + return decode_msg_fn(field.message, payload), np end error("decode_one: unknown kind " .. tostring(kind), 0) end @@ -1264,9 +1306,9 @@ while ep <= elim do local eid, ewt eid, ewt, ep = wire.decode_tag(payload, ep) if eid == 1 then - key, ep = decode_one(f.key, payload, ep) + key, ep = decode_one(f.key, payload, ep, scalar, decode_msg) elseif eid == 2 then - val, ep = decode_one(f.value, payload, ep) + val, ep = decode_one(f.value, payload, ep, scalar, decode_msg) else ep = wire.skip_field(payload, ep, ewt, eid) end @@ -1364,5 +1406,280 @@ if unknown ~= nil then result._unknown_fields = table.concat(unknown) end return result end M.decode = decode_message + +-- --------------------------------------------------------------------------- +-- 58u: unsafe-decode twins (opt-out of per-string utf8_len validation) +-- +-- Each twin is a literal clone of its safe counterpart with three +-- substitutions: +-- * `f._reader` → `f._reader_unsafe` (compiled against scalar_unsafe) +-- * `decode_msg` / `decode_group` / `decode_extension` → +-- `decode_msg_unsafe` / `decode_group_unsafe` / `decode_extension_unsafe` +-- * direct `scalar[...]` accesses in the slow / extension paths → +-- `scalar_unsafe[...]` (so string fields take the bytes handler) +-- +-- A factory-pattern refactor would save lines but force the reader +-- access through a string-keyed indirection (`f[reader_key]`) — that +-- regresses the hot dispatch. Literal duplication keeps both paths +-- monomorphic and lets LuaJIT specialize each independently. +-- +-- WKT sub-messages keep going through `desc.decode` (no _unsafe twin in +-- pb.wkt); they have no string-validation hot path so the asymmetry is +-- intentional and matches the inline-mode (6bb) behavior. +-- --------------------------------------------------------------------------- + +local decode_extension_unsafe + +decode_group_unsafe = function(desc, buf, pos, stop_id) + local result = {} + local fbi = desc.field_by_id + local len = #buf + local WIRE_EG = wire.WIRE_EGROUP + local unknown + while pos <= len do + local tag_start = pos + local id, wt + id, wt, pos = wire.decode_tag(buf, pos) + if wt == WIRE_EG then + if id ~= stop_id then + error(("EGROUP id %d does not match SGROUP id %d"): + format(id, stop_id), 0) + end + if unknown ~= nil then + result._unknown_fields = table.concat(unknown) + end + return result, pos + end + local f = fbi[id] + if f == nil then + pos = wire.skip_field(buf, pos, wt, id) + if unknown == nil then unknown = {} end + unknown[#unknown + 1] = buf:sub(tag_start, pos - 1) + else + local reader = f._reader_unsafe + if reader ~= nil then + pos = reader(buf, pos, wt, result) + else + pos = wire.skip_field(buf, pos, wt, id) + end + end + end + error("group not terminated by EGROUP id " .. tostring(stop_id), 0) +end + +decode_extension_unsafe = function(ext, buf, pos, wt, result) + local exts = result._extensions + if exts == nil then exts = {}; result._extensions = exts end + local key = ext.full_name + local kind = ext.kind + + if ext.repeated then + local list = exts[key] + if list == nil then list = {}; exts[key] = list end + if kind == 'scalar' then + local h = scalar_unsafe[ext.proto_type] + if h.packable and wt == wire.WIRE_LEN and h.wire ~= wire.WIRE_LEN then + local payload, np = wire.decode_len(buf, pos) + local items = decode_packed(ext, payload) + local base = #list + for i = 1, #items do list[base + i] = items[i] end + return np + end + local v, np = h.decode(buf, pos) + list[#list + 1] = v + return np + elseif kind == 'enum' then + if wt == wire.WIRE_LEN then + local payload, np = wire.decode_len(buf, pos) + local p2, lim = 1, #payload + while p2 <= lim do + local u, np2 = wire.decode_varint(payload, p2) + p2 = np2 + list[#list + 1] = wire.varint_to_int32(u) + end + return np + end + local u, np = wire.decode_varint(buf, pos) + list[#list + 1] = wire.varint_to_int32(u) + return np + elseif kind == 'message' then + local payload, np = wire.decode_len(buf, pos) + list[#list + 1] = decode_msg_unsafe(ext.message, payload) + return np + elseif kind == 'group' then + local decoded, np = decode_group_unsafe(ext.message, buf, pos, ext.id) + list[#list + 1] = decoded + return np + end + error("decode_extension_unsafe: unknown repeated kind " .. tostring(kind), 0) + end + + if kind == 'scalar' then + local h = scalar_unsafe[ext.proto_type] + local v, np = h.decode(buf, pos) + exts[key] = v + return np + elseif kind == 'enum' then + local u, np = wire.decode_varint(buf, pos) + exts[key] = wire.varint_to_int32(u) + return np + elseif kind == 'message' then + local payload, np = wire.decode_len(buf, pos) + local decoded = decode_msg_unsafe(ext.message, payload) + local prev = exts[key] + if prev == nil then + exts[key] = decoded + else + M.merge_message(ext.message, prev, decoded) + end + return np + elseif kind == 'group' then + local decoded, np = decode_group_unsafe(ext.message, buf, pos, ext.id) + local prev = exts[key] + if prev == nil then + exts[key] = decoded + else + M.merge_message(ext.message, prev, decoded) + end + return np + end + error("decode_extension_unsafe: unknown kind " .. tostring(kind), 0) +end + +local decode_message_unsafe = function(desc, buf) + if type(buf) ~= 'string' then + error(("expected string for decode of %s, got %s"):format(desc.name, type(buf)), 0) + end + local result = {} + local pos, len = 1, #buf + local fbi = desc.field_by_id + local unknown + + while pos <= len do + local tag_start = pos + local id, wt, npos = wire.decode_tag(buf, pos) + pos = npos + local f = fbi[id] + if f == nil then + local ext = desc.extensions_by_id and desc.extensions_by_id[id] + if ext ~= nil then + pos = decode_extension_unsafe(ext, buf, pos, wt, result) + else + pos = wire.skip_field(buf, pos, wt, id) + if unknown == nil then unknown = {} end + unknown[#unknown + 1] = buf:sub(tag_start, pos - 1) + end + else + local reader = f._reader_unsafe + if reader ~= nil then + pos = reader(buf, pos, wt, result) + else + local kind = f.kind + if kind == 'map' then + local map_t = result[f.name] + if map_t == nil then map_t = {}; result[f.name] = map_t end + local payload, np = wire.decode_len(buf, pos) + pos = np + local key, val + local ep, elim = 1, #payload + while ep <= elim do + local eid, ewt + eid, ewt, ep = wire.decode_tag(payload, ep) + if eid == 1 then + key, ep = decode_one(f.key, payload, ep, scalar_unsafe, decode_msg_unsafe) + elseif eid == 2 then + val, ep = decode_one(f.value, payload, ep, scalar_unsafe, decode_msg_unsafe) + else + ep = wire.skip_field(payload, ep, ewt, eid) + end + end + if key == nil then key = default_value(f.key) end + if val == nil then val = default_value(f.value) end + if f.key_dedup then + for k in pairs(map_t) do + if k == key then key = k; break end + end + end + map_t[key] = val + elseif f.repeated then + local list = result[f.name] + if list == nil then list = {}; result[f.name] = list end + + if kind == 'scalar' then + local h = scalar_unsafe[f.proto_type] + if h.packable and wt == wire.WIRE_LEN and h.wire ~= wire.WIRE_LEN then + local payload, np = wire.decode_len(buf, pos) + pos = np + local items = decode_packed(f, payload) + local base = #list + for i = 1, #items do list[base + i] = items[i] end + else + local v, np = h.decode(buf, pos) + list[#list + 1] = v + pos = np + end + elseif kind == 'enum' then + if wt == wire.WIRE_LEN then + local payload, np = wire.decode_len(buf, pos) + pos = np + local p2, lim = 1, #payload + while p2 <= lim do + local u, np2 = wire.decode_varint(payload, p2) + p2 = np2 + list[#list + 1] = wire.varint_to_int32(u) + end + else + local u, np = wire.decode_varint(buf, pos) + list[#list + 1] = wire.varint_to_int32(u) + pos = np + end + elseif kind == 'message' then + local payload, np = wire.decode_len(buf, pos) + pos = np + list[#list + 1] = decode_msg_unsafe(f.message, payload) + end + else + if kind == 'scalar' then + local h = scalar_unsafe[f.proto_type] + local v, np = h.decode(buf, pos) + pos = np + result[f.name] = v + elseif kind == 'enum' then + local u, np = wire.decode_varint(buf, pos) + pos = np + result[f.name] = wire.varint_to_int32(u) + elseif kind == 'message' then + local payload, np = wire.decode_len(buf, pos) + pos = np + local decoded = decode_msg_unsafe(f.message, payload) + local prev = result[f.name] + if prev == nil or f.oneof or f.message.decode then + result[f.name] = decoded + else + for k, v in pairs(decoded) do prev[k] = v end + end + end + if f.oneof_siblings then + for _, s in ipairs(f.oneof_siblings) do result[s] = nil end + end + end + end + end + end + if unknown ~= nil then result._unknown_fields = table.concat(unknown) end + return result +end + +-- decode_msg_unsafe dispatches WKT custom decoders normally (they have +-- no _unsafe twin and don't run utf8_len) and routes everything else +-- through decode_message_unsafe. +decode_msg_unsafe = function(desc, buf) + if desc.decode then return desc.decode(buf) end + return decode_message_unsafe(desc, buf) +end + +M.decode_unsafe = decode_msg_unsafe +M.decode_group_unsafe = decode_group_unsafe +M.decode_extension_unsafe = function(...) return decode_extension_unsafe(...) end return M diff --git a/runtime/pb/init.lua b/runtime/pb/init.lua index d0dca2c5a803a70226e67dcd50b5d6994cfdd5ae..476f69fcd574ff9a441ec1470bc40b20cea4abf3 100644 --- a/runtime/pb/init.lua +++ b/runtime/pb/init.lua @@ -58,6 +58,15 @@ return { -- High-level codec encode = pb_encode, decode = pb_decode, + -- Opt-in non-validating decode for trusted producers (re-decoding + -- bytes from our own encoder, JSON/text round-trips, in-process + -- typed RPC). Skips utf8_len on every string field; sub-message + -- recursion stays on the unsafe path. C runtime dispatch is + -- bypassed because the C path validates unconditionally today + -- (tracked in kyt). Generated runtime-mode wrappers + -- `M._decode_unsafe` forward into this; full-mode codegen + -- inlines a literal sister `_decode_unsafe` body. (6bb, 58u) + decode_unsafe = codec.decode_unsafe, -- Lazy / zero-copy decode view. See runtime/pb/lazy.lua for the -- :get / :has / :which / :iter / :names surface on the returned @@ -226,6 +235,12 @@ -- typed value extraction, list bookkeeping, message-merge -- rules, and oneof sibling clearing. Maps fall through to -- the existing in-loop dispatch. codec.compile_readers(desc) + -- 58u: parallel reader set for pb.decode_unsafe. Built against + -- scalar_unsafe (string -> bytes handler) and the unsafe + -- sub-message dispatchers so the entire decode tree skips + -- utf8_len when the caller opted in. Per-field cost at module + -- load is O(fields); negligible at descriptor scale. + codec.compile_readers_unsafe(desc) -- Emit a generated per-descriptor `_encode_body(data, out, active)` -- with one monomorphic call site per field. Must follow -- compile_writers so it can capture each f._writer as a fixed diff --git a/test/decode_unsafe_test.lua b/test/decode_unsafe_test.lua index 0f7bd9c87cec9676fcf155d04ebf02265e6687e7..6d9fb17382de2aa818a3a1180a9e5a07839e72a8 100644 --- a/test/decode_unsafe_test.lua +++ b/test/decode_unsafe_test.lua @@ -1,99 +1,118 @@ --- decode_unsafe: codegen emits a sister _decode_unsafe alongside --- _decode that skips the per-string utf8_len validation. Intended --- for re-decoding bytes from a trusted producer (own encoder, JSON/text --- round-trip, in-process typed RPC). Emitted in full mode only; runtime --- mode does not currently expose the unsafe path (compiled f._reader --- closures capture handler.decode by value, so a swap-on-call would not --- reach them — a proper runtime-mode unsafe path would need parallel --- _reader_unsafe closures and is intentionally deferred). (6bb) +-- decode_unsafe: opt-in non-validating decode for trusted producers +-- (re-decoding bytes from own encoder, JSON/text round-trips, in-process +-- typed RPC). Skips per-string utf8_len on every singular/repeated/map +-- string field; sub-message recursion stays on the unsafe path. +-- +-- Full mode emits a literal sister _decode_unsafe body (6bb). +-- Runtime mode wraps pb.decode_unsafe, which dispatches through +-- f._reader_unsafe closures compiled in pb.finalize_message against +-- a swapped scalar table (scalar.string = scalar.bytes) — see +-- codec.compile_readers_unsafe (58u). +-- +-- This file runs the same suite against both modes to pin the API +-- contract (one function name, same semantics) regardless of which +-- code path executed. local t = require('luatest') -local hello = require('full.hello.hello_pb') -local g = t.group('decode_unsafe.full') - --- Hand-rolled wire bytes for hello.Address{street=}. Tag for field 1 --- (wire 2, LEN) is 0x0A; length-prefix is one varint byte for len<128. local function address_with_street(s) return string.char(0x0A, #s) .. s end --- Hand-rolled wire bytes for hello.Person{name=}. Tag for field 1 --- is identical (0x0A). -local function person_with_name(s) - return string.char(0x0A, #s) .. s -end +for _, mode in ipairs({'full', 'runtime'}) do + local g = t.group('decode_unsafe.' .. mode) + local hello = require(mode .. '.hello.hello_pb') -g.test_valid_string_matches_safe_decode = function() - local addr = {street = 'Pushkina 1', city = 'Moscow', zip = 123456} - local bytes = hello.Address_encode(addr) - t.assert_equals(hello.Address_decode_unsafe(bytes), - hello.Address_decode(bytes)) -end + g.test_valid_string_matches_safe_decode = function() + local addr = {street = 'Pushkina 1', city = 'Moscow', zip = 123456} + local bytes = hello.Address_encode(addr) + t.assert_equals(hello.Address_decode_unsafe(bytes), + hello.Address_decode(bytes)) + end -g.test_safe_decode_rejects_invalid_utf8 = function() - -- 0xC0 0x80 is the classic overlong NUL — rejected by RFC 3629 - -- (also banned in proto3 strings). - local bytes = address_with_street('\xC0\x80') - t.assert_error_msg_contains( - 'invalid UTF-8', - function() hello.Address_decode(bytes) end) -end + g.test_safe_decode_rejects_invalid_utf8 = function() + -- 0xC0 0x80 is the classic overlong NUL — rejected by RFC 3629 + -- (also banned in proto3 strings). + local bytes = address_with_street('\xC0\x80') + t.assert_error_msg_contains( + 'invalid UTF-8', + function() hello.Address_decode(bytes) end) + end + + g.test_unsafe_decode_accepts_invalid_utf8 = function() + local bytes = address_with_street('\xC0\x80') + local dec = hello.Address_decode_unsafe(bytes) + t.assert_equals(dec.street, '\xC0\x80') + end + + g.test_unsafe_decode_repeated_string = function() + -- Person.emails is a repeated string; two entries, second invalid. + local good = 'alice@example.com' + local bad = '\xFF\xFE' + local bytes = string.char(0x1A, #good) .. good + .. string.char(0x1A, #bad) .. bad + t.assert_error_msg_contains( + 'invalid UTF-8', + function() hello.Person_decode(bytes) end) + local dec = hello.Person_decode_unsafe(bytes) + t.assert_equals(dec.emails, {good, bad}) + end -g.test_unsafe_decode_accepts_invalid_utf8 = function() - local bytes = address_with_street('\xC0\x80') - local dec = hello.Address_decode_unsafe(bytes) - t.assert_equals(dec.street, '\xC0\x80') -end + g.test_unsafe_decode_recurses_into_sub_messages = function() + -- Person{address = Address{street = '\xC0\x80'}}. + -- Pins that the unsafe path threads through sub-messages — a + -- mistake here would call the safe Address decoder and error. + local inner = address_with_street('\xC0\x80') + local bytes = string.char(0x2A, #inner) .. inner -g.test_unsafe_decode_repeated_string = function() - -- Person.emails is a repeated string; two entries, second is invalid. - -- Tag 0x1A = field 3 (emails), wire 2. - local good = 'alice@example.com' - local bad = '\xFF\xFE' - local bytes = string.char(0x1A, #good) .. good - .. string.char(0x1A, #bad) .. bad - t.assert_error_msg_contains( - 'invalid UTF-8', - function() hello.Person_decode(bytes) end) - local dec = hello.Person_decode_unsafe(bytes) - t.assert_equals(dec.emails, {good, bad}) -end + t.assert_error_msg_contains( + 'invalid UTF-8', + function() hello.Person_decode(bytes) end) -g.test_unsafe_decode_recurses_into_sub_messages = function() - -- Person{address = Address{street = '\xC0\x80'}}. - -- Tag 0x2A = field 5 (address), wire 2; payload is the Address bytes. - local inner = address_with_street('\xC0\x80') - local bytes = string.char(0x2A, #inner) .. inner + local dec = hello.Person_decode_unsafe(bytes) + t.assert_equals(dec.address.street, '\xC0\x80') + end - -- Safe path: nested string rejected (proves nested validation runs by - -- default). - t.assert_error_msg_contains( - 'invalid UTF-8', - function() hello.Person_decode(bytes) end) + g.test_unsafe_decode_handles_long_string_fallback = function() + -- >=128 byte payload exits the 1-byte LEN inline fast path. Full + -- mode falls through to wire.decode_bytes (codegen swap); runtime + -- mode lands in the compiled _reader_unsafe scalar handler which + -- is scalar_unsafe.string (= bytes). Both paths must accept + -- invalid UTF-8 in the long-string regime. + local big = string.rep('x', 200) .. '\xFF' -- 201 bytes, trailing bad + local bytes = string.char(0x0A) .. string.char(0xC9, 0x01) .. big + -- 201 in varint = 0xC9 0x01. + t.assert_error_msg_contains( + 'invalid UTF-8', + function() hello.Address_decode(bytes) end) + local dec = hello.Address_decode_unsafe(bytes) + t.assert_equals(#dec.street, 201) + t.assert_equals(dec.street:byte(201), 0xFF) + end - -- Unsafe path: nested call must also be the unsafe variant. If - -- Person_decode_unsafe were to call Address_decode (the safe variant) - -- for sub-messages, this would still error. The recursive dispatch is - -- emitted by inline.go and pinned by this assertion. - local dec = hello.Person_decode_unsafe(bytes) - t.assert_equals(dec.address.street, '\xC0\x80') -end + g.test_unsafe_decode_map_string_value = function() + -- Person.ages_by_nickname is map — string KEYS go + -- through the map-fallback decode_one path in runtime mode (no + -- _reader for map fields). The unsafe twin passes scalar_unsafe + -- to decode_one so the key's utf8_len is bypassed. In full mode + -- the inline map decoder routes through wire.decode_bytes. + -- map: entry message = {1: string key, 2: int32 value}. + local bad_key = '\xFE' + local entry = string.char(0x0A, #bad_key) .. bad_key -- key tag + .. string.char(0x10, 7) -- value tag + varint 7 + -- Person.ages_by_nickname id is per the .proto — check it. + -- Use bench/dynamic introspection: look up field id from descriptor. + local f = nil + for _, fld in ipairs(hello.Person_descriptor.fields) do + if fld.name == 'ages_by_nickname' then f = fld; break end + end + t.assert_not_equals(f, nil) + local tag = require('pb.wire').encode_tag(f.id, 2) + local bytes = tag .. string.char(#entry) .. entry -g.test_unsafe_decode_handles_long_string_fallback = function() - -- >=128 byte payload exits the 1-byte LEN inline fast path and falls - -- through to wire.decode_bytes (instead of wire.decode_string) on - -- the unsafe path. Exercises the fallback branch in the emitted code. - local big = string.rep('x', 200) .. '\xFF' -- 201 bytes, trailing bad - local bytes = string.char(0x0A) .. string.char(0xC9, 0x01) .. big - -- 201 in varint = 0xC9 0x01. - t.assert_error_msg_contains( - 'invalid UTF-8', - function() hello.Address_decode(bytes) end) - local dec = hello.Address_decode_unsafe(bytes) - t.assert_equals(#dec.street, 201) - t.assert_equals(dec.street:byte(201), 0xFF) + t.assert_error_msg_contains( + 'invalid UTF-8', + function() hello.Person_decode(bytes) end) + local dec = hello.Person_decode_unsafe(bytes) + t.assert_equals(dec.ages_by_nickname[bad_key], 7) + end end - --- Person.name omitted from the above explicitly to keep tests focused; --- the singular-string scalar path is already covered by Address.street. -_ = person_with_name -- silence unused-local under future trimming