m/market labResearch, with a record.
Source on GitHub ↗

agent_probe_records.py

1"""Column storage for benchmark samples; materialize dictionaries after playback.23Numeric payloads contain no Python object references for GC to traverse. Text columns retain only4immutable strings. Every sample is retained, so tail statistics and request5correlation do not silently become a sample of the final few seconds.6"""78from array import array9import math10from threading import Lock, local111213class ProbeRecords:14    def __init__(self, integers=(), floats=(), texts=(), flags=()):15        self.columns = {name: array('q') for name in integers}16        self.columns.update({name: array('d') for name in floats})17        self.columns.update({name: [] for name in texts})18        self.columns.update({name: array('b') for name in flags})19        self.flags = set(flags)20        self.lock = Lock()2122    def __len__(self):23        return len(next(iter(self.columns.values())))2425    def append(self, value):26        # Phase samples arrive from several parent threads. Keep column writes27        # together; GIL switching between columns must not corrupt correlation.28        with self.lock:29            index = len(self)30            for name, column in self.columns.items():31                column.append(self.encode(column, value.get(name)))32            return index3334    def update(self, index, values):35        with self.lock:36            for name, value in values.items():37                column = self.columns[name]38                column[index] = self.encode(column, value)3940    @staticmethod41    def encode(column, value):42        if isinstance(column, list):43            return value44        if value is None:45            return math.nan if column.typecode == 'd' else -146        return value4748    def __iter__(self):49        for index in range(len(self)):50            value = {}51            for name, column in self.columns.items():52                item = column[index]53                if isinstance(column, array):54                    missing = math.isnan(item) if column.typecode == 'd' else item == -155                    if missing:56                        continue57                if name in self.flags:58                    item = bool(item)59                value[name] = item60            # Request-less phases still participate in phase summaries.61            if 'request_id' in self.columns:62                value.setdefault('request_id', None)63            yield value6465    def storage_bytes(self):66        # Numeric payload plus string references; excludes text contents and67        # array/list spare capacity. This is not process RSS.68        return sum(69            len(c) * (c.itemsize if isinstance(c, array) else 8) for c in self.columns.values()70        )717273class ThreadProbeRecords:74    """Separate columns per producer; no shared hot-path recorder lock.7576    Registration is a single list append in this CPython benchmark. Export only77    after workers close. The returned token identifies the receiving thread's78    exact row for the subsequent request-ID decode update.79    """8081    def __init__(self, **schema):82        self.schema = schema83        self.local = local()84        self.stores = []8586    def append(self, value):87        store = getattr(self.local, 'store', None)88        if store is None:89            store = self.local.store = ProbeRecords(**self.schema)90            self.stores.append(store)91        return store, store.append(value)9293    def update(self, token, values):94        store, index = token95        store.update(index, values)9697    def __iter__(self):98        for store in self.stores:99            yield from store100101    def storage_bytes(self):102        return sum(store.storage_bytes() for store in self.stores)