RFC: Tachyon binary format chunking - #2

% ./python.exe -m profiling.sampling run \
 --binary \
 --flush interval:180s \
 -- python zażółćgęsląjaźn.py
% ./python.exe -m profiling.sampling run \
 --binary \
 # I believe that bytes: could be supported but I think it requires a number of complex decisions
 --flush samples:5000 \
 -- python zażółćgęsląjaźn.py

% ./python.exe -m profiling.sampling run \
 --binary \
 --flush interval:180s \
 --output just-a-file-by-default.txt
 -- python zażółćgęsląjaźn.py
% ./python.exe -m profiling.sampling run \
 --jsonl \
 --flush interval:180s \
 --output file:just-a-file-by-default.txt
 -- python zażółćgęsląjaźn.py
% ./python.exe -m profiling.sampling run \
 --jsonl \
 --flush interval:180s \
 --output ws://213.25.252.50:9000 # nice but not in the stdlib!
 -- python zażółćgęsląjaźn.py
% ./python.exe -m profiling.sampling run \
 --binary \
 --flush interval:180s \
 # or tcp-connect: and tcp-listen:
 # what about auth?
 --output tcp:213.25.252.50:9000
 -- python zażółćgęsląjaźn.py

+------------------+ Offset 0
| Chunk 0 | Header, sample data, tables, footer
+------------------+ chunk0_size
| Chunk 1 | Header, sample data, tables, footer
+------------------+ chunk0_size + chunk1_size
| ... |
+------------------+

One CHUNK = a complete, independently-decodable mini-profile
(byte-for-byte the same shape as a whole v1 file)
+-----------------------------------------------------------------+ <- chunk_start
| HEADER (64 B) |
| @ 0 magic <- written LAST = commit marker |
| @ 4 version = 2 |
| @12 start_time |
| @28 sample_count (THIS chunk only) |
| @36 string_table_offset |
| @44 frame_table_offset |
| @56 chunk_size <- locator: where THIS chunk ends |
+-----------------------------------------------------------------+
| SAMPLE DATA delta-encoded; stack/timestamp baselines |
| reset at every chunk boundary |
+-----------------------------------------------------------------+
| STRING TABLE local to this chunk |
+-----------------------------------------------------------------+
| FRAME TABLE local to this chunk |
+-----------------------------------------------------------------+
| FOOTER (32 B) string_count | frame_count | chunk_end |
+-----------------------------------------------------------------+ <- chunk_start + chunk_size
(+ one fresh zstd frame per chunk when compression is on)

import collections
import tempfile
import types
import _remote_debugging
def sample(fn):
 fr = _remote_debugging.FrameInfo(
 (fn, _remote_debugging.LocationInfo((1, 1, 0, 0)), "f", None)
 )
 return [
 _remote_debugging.InterpreterInfo(
 (0, [_remote_debugging.ThreadInfo((1, 0, [fr]))])
 )
 ]
BATCHES = (("c0", 3), ("c1", 2), ("c2", 4)) # one chunk per batch
path = tempfile.mktemp(suffix=".bin")
w = _remote_debugging.BinaryWriter(path, 1000, 0, compression=0, chunked=True)
for name, count in BATCHES:
 for _ in range(count):
 w.write_sample(sample(name), 1000)
 w.flush()
w.finalize()
seen = collections.Counter()
def collect(stack_frames, timestamps_us=None):
 weight = len(timestamps_us) if timestamps_us else 1
 for interp in stack_frames:
 for thread in interp[1]:
 for frame in thread[2]:
 seen[frame[0]] += weight
with _remote_debugging.BinaryReader(path) as r:
 replayed = r.replay(types.SimpleNamespace(collect=collect))
 version = r.get_info()["version"]
expected = dict(BATCHES)
total = sum(expected.values())
assert version == 2, f"version {version} != 2 (not chunked)"
assert replayed == total, f"replayed {replayed} != {total}"
assert dict(seen) == expected, f"content {dict(seen)} != {expected}"
print(
 f"OK: {replayed} samples across {len(BATCHES)} chunks, content == {expected}"
)