m/market labResearch, with a record.
Source on GitHub ↗

checkpoint.cpp

1#include "itch/replay/checkpoint.hpp"2#include "itch/common/binary.hpp"3#include "itch/common/file_utils.hpp"4#include "itch/feed/itch/decoder.hpp"5#include "itch/output/csv.hpp"6#include "itch/output/staged_output.hpp"7#include <algorithm>8#include <array>9#include <filesystem>10#include <fstream>11#include <iterator>12#include <limits>13#include <stdexcept>1415namespace itch {16namespace {17[[noreturn]] void invalid(const std::string& reason) {18    throw std::runtime_error("checkpoint: " + reason);19}2021void text_field(std::string& out, const std::string& value) {22    append_big_endian(out, value.size());23    out += value;24}2526class Reader {27    const std::string& bytes_;28    size_t pos_ = 0;2930  public:31    explicit Reader(const std::string& bytes) : bytes_(bytes) {}3233    size_t remaining() const {34        return bytes_.size() - pos_;35    }3637    std::string take(size_t n) {38        if (n > remaining()) {39            invalid("truncated state");40        }41        auto value = bytes_.substr(pos_, n);42        pos_ += n;43        return value;44    }4546    uint64_t number(size_t width = 8) {47        const auto bytes = take(width);48        return read_big_endian(reinterpret_cast<const unsigned char*>(bytes.data()), width);49    }5051    bool flag() {52        const auto n = number(1);53        if (n > 1) {54            invalid("invalid flag");55        }56        return n != 0;57    }5859    std::string field(size_t maximum) {60        const auto n = number();61        if (n > maximum) {62            invalid("invalid string length");63        }64        return take(n);65    }66};67} // namespace6869std::string encode_checkpoint(const Checkpoint& c) {70    std::string out = "ITCHCP01";71    append_big_endian(out, c.csv_path.empty() ? 1 : 2, 4);72    append_big_endian(out, c.input.size);73    text_field(out, c.input.digest);74    text_field(out, c.target);75    append_big_endian(out, c.next_frame);76    append_big_endian(out, c.offset);77    append_big_endian(out, c.selected);78    append_big_endian(out, c.hash);79    append_big_endian(out, c.depth);80    append_big_endian(out, c.with_offset, 1);81    append_big_endian(out, c.allow_implicit_eof, 1);82    append_big_endian(out, c.last_end_messages, 1);83    append_big_endian(out, c.book.locate.has_value(), 1);84    append_big_endian(out, c.book.locate.value_or(0), 2);85    append_big_endian(out, c.book.orders.size());86    for (const auto& o : c.book.orders) {87        append_big_endian(out, o.id);88        append_big_endian(out, static_cast<unsigned char>(o.side), 1);89        append_big_endian(out, o.shares, 4);90        append_big_endian(out, o.price, 4);91    }92    for (const auto* levels : {&c.book.bids, &c.book.asks}) {93        append_big_endian(out, levels->size());94        for (const auto& l : *levels) {95            append_big_endian(out, l.price, 4);96            append_big_endian(out, l.shares);97        }98    }99    if (!c.csv_path.empty()) {100        text_field(out, c.csv_path);101        append_big_endian(out, c.csv_bytes);102    }103    return out + sha256(out);104}105106Checkpoint decode_checkpoint(const std::string& bytes) {107    if (bytes.size() < 76) {108        invalid("truncated file");109    }110    const auto body = bytes.substr(0, bytes.size() - 64);111    if (sha256(body) != bytes.substr(bytes.size() - 64)) {112        invalid("checksum mismatch");113    }114    Reader r(body);115    if (r.take(8) != "ITCHCP01") {116        invalid("invalid magic");117    }118    const auto version = r.number(4);119    if (version != 1 && version != 2) {120        invalid("unsupported version");121    }122    Checkpoint c;123    c.input.size = r.number();124    c.input.digest = r.field(64);125    c.target = r.field(8);126    if (c.input.digest.size() != 64 ||127        c.input.digest.find_first_not_of("0123456789abcdef") != std::string::npos) {128        invalid("invalid input SHA-256");129    }130    if (c.target.empty()) {131        invalid("empty symbol");132    }133    c.next_frame = r.number();134    c.offset = r.number();135    c.selected = r.number();136    c.hash = r.number();137    const auto depth = r.number();138    if (!depth || depth > 1000) {139        invalid("invalid depth");140    }141    c.depth = depth;142    c.with_offset = r.flag();143    c.allow_implicit_eof = r.flag();144    c.last_end_messages = r.flag();145    const bool has_locate = r.flag();146    const auto locate = r.number(2);147    if (has_locate) {148        c.book.locate = static_cast<uint16_t>(locate);149    } else if (locate) {150        invalid("locate without symbol mapping");151    }152    if (!c.next_frame || c.next_frame == std::numeric_limits<uint64_t>::max() ||153        c.offset > c.input.size || c.selected >= c.next_frame) {154        invalid("invalid replay counters");155    }156    const auto orders = r.number();157    if (orders > r.remaining() / 17) {158        invalid("invalid order count");159    }160    for (uint64_t i = 0; i < orders; ++i) {161        const auto id = r.number();162        const char side = static_cast<char>(r.number(1));163        const auto shares = r.number(4), price = r.number(4);164        c.book.orders.push_back(165            {id, side, static_cast<uint32_t>(shares), static_cast<uint32_t>(price)});166    }167    for (auto* levels : {&c.book.bids, &c.book.asks}) {168        const auto count = r.number();169        if (count > r.remaining() / 12) {170            invalid("invalid level count");171        }172        for (uint64_t i = 0; i < count; ++i) {173            const auto price = r.number(4), shares = r.number();174            levels->push_back({static_cast<uint32_t>(price), shares});175        }176    }177    if (version == 2) {178        c.csv_path = r.field(4096);179        c.csv_bytes = r.number();180        if (c.csv_path.empty() || c.csv_path.find('\0') != std::string::npos || !c.csv_bytes) {181            invalid("invalid CSV prefix binding");182        }183    }184    if (r.remaining()) {185        invalid("trailing state bytes");186    }187    Book book(c.target);188    book.restore(c.book, c.next_frame - 1);189    return c;190}191192Checkpoint load_checkpoint(const std::string& path) {193    std::ifstream in(path, std::ios::binary);194    if (!in) {195        invalid("cannot open " + path);196    }197    const std::string bytes{std::istreambuf_iterator<char>(in), std::istreambuf_iterator<char>()};198    if (in.bad()) {199        invalid("read failed");200    }201    return decode_checkpoint(bytes);202}203204void save_checkpoint(const Checkpoint& c, const std::string& path, const std::string& input) {205    const auto bytes = encode_checkpoint(c);206    // Round-trip validation before atomic publication.207    decode_checkpoint(bytes);208    StagedOutput output(path, input);209    output.stream().write(bytes.data(), static_cast<std::streamsize>(bytes.size()));210    output.commit([&bytes](const std::string& temporary) {211        if (encode_checkpoint(load_checkpoint(temporary)) != bytes) {212            invalid("temporary file round-trip mismatch");213        }214    });215}216217void validate_boundary(const Checkpoint& c, const std::string& input) {218    std::ifstream in(input, std::ios::binary);219    if (!in) {220        invalid("cannot open input");221    }222    uint64_t offset = 0, frames = 0;223    bool last = false;224    while (offset < c.offset) {225        unsigned char prefix[2];226        in.read(reinterpret_cast<char*>(prefix), 2);227        if (in.gcount() != 2) {228            invalid("offset is not a frame boundary");229        }230        const size_t length = (static_cast<size_t>(prefix[0]) << 8) | prefix[1];231        if (!length || length + 2 > c.offset - offset) {232            invalid("offset is not a frame boundary");233        }234        Bytes bytes(length);235        in.read(reinterpret_cast<char*>(bytes.data()), length);236        if (in.gcount() != static_cast<std::streamsize>(length)) {237            invalid("truncated input before offset");238        }239        decode(bytes, ++frames);240        last = bytes.size() == 12 && bytes[0] == 'S' && bytes[1] == 0 && bytes[2] == 0 &&241               bytes[11] == 'C';242        offset += 2 + length;243    }244    if (frames != c.next_frame - 1 || last != c.last_end_messages) {245        invalid("frame/offset or end-message state mismatch");246    }247}248249void copy_checkpoint_csv(const Checkpoint& c, std::ostream& output) {250    if (c.csv_path.empty()) {251        invalid("no saved CSV prefix; use suffix replay or a CSV-bound checkpoint");252    }253    std::ifstream in(c.csv_path, std::ios::binary);254    if (!in) {255        invalid("cannot open saved CSV prefix: " + c.csv_path);256    }257    const std::string header = std::string(csv_header(c.with_offset)) + "\n";258    if (c.csv_bytes < header.size()) {259        invalid("invalid CSV prefix length");260    }261    std::string actual(header.size(), '\0');262    in.read(actual.data(), static_cast<std::streamsize>(actual.size()));263    if (actual != header || in.gcount() != static_cast<std::streamsize>(header.size())) {264        invalid("CSV prefix header mismatch");265    }266    output << header;267    uint64_t remaining = c.csv_bytes - header.size(), rows = 0, hash = 14695981039346656037ULL;268    char last = '\n';269    std::array<char, 65536> buffer;270    while (remaining) {271        const auto count =272            static_cast<std::streamsize>(std::min<uint64_t>(remaining, buffer.size()));273        in.read(buffer.data(), count);274        if (in.gcount() != count) {275            invalid("truncated CSV prefix");276        }277        hash = fnv1a_bytes(hash, std::string_view(buffer.data(), static_cast<size_t>(count)));278        for (std::streamsize i = 0; i < count; ++i) {279            const auto ch = static_cast<unsigned char>(buffer[i]);280            if (ch == '\n') {281                ++rows;282            }283        }284        last = buffer[static_cast<size_t>(count) - 1];285        output.write(buffer.data(), count);286        remaining -= static_cast<uint64_t>(count);287    }288    if (last != '\n' || rows != c.selected || hash != c.hash) {289        invalid("CSV prefix row count/hash mismatch");290    }291    if (!output) {292        invalid("CSV prefix copy failed");293    }294    // Bytes written after the saved boundary were never checkpointed and are ignored.295}296} // namespace itch