checkpoint.cpp
1#include "itch/replay/checkpoint.hpp"2#include "itch/common/binary.hpp"3#include "itch/common/file_utils.hpp"4#include "itch/feed/itch/decoder.hpp"5#include "itch/output/csv.hpp"6#include "itch/output/staged_output.hpp"7#include <algorithm>8#include <array>9#include <filesystem>10#include <fstream>11#include <iterator>12#include <limits>13#include <stdexcept>1415namespace itch {16namespace {17[[noreturn]] void invalid(const std::string& reason) {18 throw std::runtime_error("checkpoint: " + reason);19}2021void text_field(std::string& out, const std::string& value) {22 append_big_endian(out, value.size());23 out += value;24}2526class Reader {27 const std::string& bytes_;28 size_t pos_ = 0;2930 public:31 explicit Reader(const std::string& bytes) : bytes_(bytes) {}3233 size_t remaining() const {34 return bytes_.size() - pos_;35 }3637 std::string take(size_t n) {38 if (n > remaining()) {39 invalid("truncated state");40 }41 auto value = bytes_.substr(pos_, n);42 pos_ += n;43 return value;44 }4546 uint64_t number(size_t width = 8) {47 const auto bytes = take(width);48 return read_big_endian(reinterpret_cast<const unsigned char*>(bytes.data()), width);49 }5051 bool flag() {52 const auto n = number(1);53 if (n > 1) {54 invalid("invalid flag");55 }56 return n != 0;57 }5859 std::string field(size_t maximum) {60 const auto n = number();61 if (n > maximum) {62 invalid("invalid string length");63 }64 return take(n);65 }66};67} // namespace6869std::string encode_checkpoint(const Checkpoint& c) {70 std::string out = "ITCHCP01";71 append_big_endian(out, c.csv_path.empty() ? 1 : 2, 4);72 append_big_endian(out, c.input.size);73 text_field(out, c.input.digest);74 text_field(out, c.target);75 append_big_endian(out, c.next_frame);76 append_big_endian(out, c.offset);77 append_big_endian(out, c.selected);78 append_big_endian(out, c.hash);79 append_big_endian(out, c.depth);80 append_big_endian(out, c.with_offset, 1);81 append_big_endian(out, c.allow_implicit_eof, 1);82 append_big_endian(out, c.last_end_messages, 1);83 append_big_endian(out, c.book.locate.has_value(), 1);84 append_big_endian(out, c.book.locate.value_or(0), 2);85 append_big_endian(out, c.book.orders.size());86 for (const auto& o : c.book.orders) {87 append_big_endian(out, o.id);88 append_big_endian(out, static_cast<unsigned char>(o.side), 1);89 append_big_endian(out, o.shares, 4);90 append_big_endian(out, o.price, 4);91 }92 for (const auto* levels : {&c.book.bids, &c.book.asks}) {93 append_big_endian(out, levels->size());94 for (const auto& l : *levels) {95 append_big_endian(out, l.price, 4);96 append_big_endian(out, l.shares);97 }98 }99 if (!c.csv_path.empty()) {100 text_field(out, c.csv_path);101 append_big_endian(out, c.csv_bytes);102 }103 return out + sha256(out);104}105106Checkpoint decode_checkpoint(const std::string& bytes) {107 if (bytes.size() < 76) {108 invalid("truncated file");109 }110 const auto body = bytes.substr(0, bytes.size() - 64);111 if (sha256(body) != bytes.substr(bytes.size() - 64)) {112 invalid("checksum mismatch");113 }114 Reader r(body);115 if (r.take(8) != "ITCHCP01") {116 invalid("invalid magic");117 }118 const auto version = r.number(4);119 if (version != 1 && version != 2) {120 invalid("unsupported version");121 }122 Checkpoint c;123 c.input.size = r.number();124 c.input.digest = r.field(64);125 c.target = r.field(8);126 if (c.input.digest.size() != 64 ||127 c.input.digest.find_first_not_of("0123456789abcdef") != std::string::npos) {128 invalid("invalid input SHA-256");129 }130 if (c.target.empty()) {131 invalid("empty symbol");132 }133 c.next_frame = r.number();134 c.offset = r.number();135 c.selected = r.number();136 c.hash = r.number();137 const auto depth = r.number();138 if (!depth || depth > 1000) {139 invalid("invalid depth");140 }141 c.depth = depth;142 c.with_offset = r.flag();143 c.allow_implicit_eof = r.flag();144 c.last_end_messages = r.flag();145 const bool has_locate = r.flag();146 const auto locate = r.number(2);147 if (has_locate) {148 c.book.locate = static_cast<uint16_t>(locate);149 } else if (locate) {150 invalid("locate without symbol mapping");151 }152 if (!c.next_frame || c.next_frame == std::numeric_limits<uint64_t>::max() ||153 c.offset > c.input.size || c.selected >= c.next_frame) {154 invalid("invalid replay counters");155 }156 const auto orders = r.number();157 if (orders > r.remaining() / 17) {158 invalid("invalid order count");159 }160 for (uint64_t i = 0; i < orders; ++i) {161 const auto id = r.number();162 const char side = static_cast<char>(r.number(1));163 const auto shares = r.number(4), price = r.number(4);164 c.book.orders.push_back(165 {id, side, static_cast<uint32_t>(shares), static_cast<uint32_t>(price)});166 }167 for (auto* levels : {&c.book.bids, &c.book.asks}) {168 const auto count = r.number();169 if (count > r.remaining() / 12) {170 invalid("invalid level count");171 }172 for (uint64_t i = 0; i < count; ++i) {173 const auto price = r.number(4), shares = r.number();174 levels->push_back({static_cast<uint32_t>(price), shares});175 }176 }177 if (version == 2) {178 c.csv_path = r.field(4096);179 c.csv_bytes = r.number();180 if (c.csv_path.empty() || c.csv_path.find('\0') != std::string::npos || !c.csv_bytes) {181 invalid("invalid CSV prefix binding");182 }183 }184 if (r.remaining()) {185 invalid("trailing state bytes");186 }187 Book book(c.target);188 book.restore(c.book, c.next_frame - 1);189 return c;190}191192Checkpoint load_checkpoint(const std::string& path) {193 std::ifstream in(path, std::ios::binary);194 if (!in) {195 invalid("cannot open " + path);196 }197 const std::string bytes{std::istreambuf_iterator<char>(in), std::istreambuf_iterator<char>()};198 if (in.bad()) {199 invalid("read failed");200 }201 return decode_checkpoint(bytes);202}203204void save_checkpoint(const Checkpoint& c, const std::string& path, const std::string& input) {205 const auto bytes = encode_checkpoint(c);206 // Round-trip validation before atomic publication.207 decode_checkpoint(bytes);208 StagedOutput output(path, input);209 output.stream().write(bytes.data(), static_cast<std::streamsize>(bytes.size()));210 output.commit([&bytes](const std::string& temporary) {211 if (encode_checkpoint(load_checkpoint(temporary)) != bytes) {212 invalid("temporary file round-trip mismatch");213 }214 });215}216217void validate_boundary(const Checkpoint& c, const std::string& input) {218 std::ifstream in(input, std::ios::binary);219 if (!in) {220 invalid("cannot open input");221 }222 uint64_t offset = 0, frames = 0;223 bool last = false;224 while (offset < c.offset) {225 unsigned char prefix[2];226 in.read(reinterpret_cast<char*>(prefix), 2);227 if (in.gcount() != 2) {228 invalid("offset is not a frame boundary");229 }230 const size_t length = (static_cast<size_t>(prefix[0]) << 8) | prefix[1];231 if (!length || length + 2 > c.offset - offset) {232 invalid("offset is not a frame boundary");233 }234 Bytes bytes(length);235 in.read(reinterpret_cast<char*>(bytes.data()), length);236 if (in.gcount() != static_cast<std::streamsize>(length)) {237 invalid("truncated input before offset");238 }239 decode(bytes, ++frames);240 last = bytes.size() == 12 && bytes[0] == 'S' && bytes[1] == 0 && bytes[2] == 0 &&241 bytes[11] == 'C';242 offset += 2 + length;243 }244 if (frames != c.next_frame - 1 || last != c.last_end_messages) {245 invalid("frame/offset or end-message state mismatch");246 }247}248249void copy_checkpoint_csv(const Checkpoint& c, std::ostream& output) {250 if (c.csv_path.empty()) {251 invalid("no saved CSV prefix; use suffix replay or a CSV-bound checkpoint");252 }253 std::ifstream in(c.csv_path, std::ios::binary);254 if (!in) {255 invalid("cannot open saved CSV prefix: " + c.csv_path);256 }257 const std::string header = std::string(csv_header(c.with_offset)) + "\n";258 if (c.csv_bytes < header.size()) {259 invalid("invalid CSV prefix length");260 }261 std::string actual(header.size(), '\0');262 in.read(actual.data(), static_cast<std::streamsize>(actual.size()));263 if (actual != header || in.gcount() != static_cast<std::streamsize>(header.size())) {264 invalid("CSV prefix header mismatch");265 }266 output << header;267 uint64_t remaining = c.csv_bytes - header.size(), rows = 0, hash = 14695981039346656037ULL;268 char last = '\n';269 std::array<char, 65536> buffer;270 while (remaining) {271 const auto count =272 static_cast<std::streamsize>(std::min<uint64_t>(remaining, buffer.size()));273 in.read(buffer.data(), count);274 if (in.gcount() != count) {275 invalid("truncated CSV prefix");276 }277 hash = fnv1a_bytes(hash, std::string_view(buffer.data(), static_cast<size_t>(count)));278 for (std::streamsize i = 0; i < count; ++i) {279 const auto ch = static_cast<unsigned char>(buffer[i]);280 if (ch == '\n') {281 ++rows;282 }283 }284 last = buffer[static_cast<size_t>(count) - 1];285 output.write(buffer.data(), count);286 remaining -= static_cast<uint64_t>(count);287 }288 if (last != '\n' || rows != c.selected || hash != c.hash) {289 invalid("CSV prefix row count/hash mismatch");290 }291 if (!output) {292 invalid("CSV prefix copy failed");293 }294 // Bytes written after the saved boundary were never checkpointed and are ignored.295}296} // namespace itch