live.cpp
1#include "itch/feed/moldudp/live.hpp"2#include "itch/book/book.hpp"3#include "itch/common/binary.hpp"4#include "itch/common/digest.hpp"5#include "itch/feed/itch/decoder.hpp"6#include "itch/feed/moldudp/moldudp.hpp"7#include "itch/output/csv.hpp"8#include <algorithm>9#include <arpa/inet.h>10#include <array>11#include <cerrno>12#include <chrono>13#include <cstring>14#include <iostream>15#include <map>16#include <optional>17#include <poll.h>18#include <stdexcept>19#include <sys/socket.h>20#include <unistd.h>21#include <vector>2223namespace itch {24namespace {25using Clock = std::chrono::steady_clock;26using Ms = std::chrono::milliseconds;2728/** Close the descriptor on all exit paths, including exceptions. */29struct Socket {30 int fd = socket(AF_INET, SOCK_DGRAM, 0);3132 Socket() {33 if (fd < 0) {34 throw std::runtime_error("cannot create UDP socket");35 }36 }3738 ~Socket() {39 close(fd);40 }41};4243/** Parse an explicit unicast endpoint; no DNS, wildcard bind or multicast membership. */44sockaddr_in address(const std::string& text) {45 const auto colon = text.find(':');46 if (colon == std::string::npos) {47 throw std::runtime_error("endpoint must be IPv4:port");48 }49 const auto port = text.substr(colon + 1);50 if (port.empty() || port.size() > 5 || !std::all_of(port.begin(), port.end(), [](char c) {51 return c >= '0' && c <= '9';52 })) {53 throw std::runtime_error("invalid endpoint port");54 }55 const auto n = std::stoul(port);56 if (!n || n > 65535) {57 throw std::runtime_error("endpoint port must be 1..65535");58 }59 sockaddr_in a{};60 a.sin_family = AF_INET;61 a.sin_port = htons(static_cast<uint16_t>(n));62 if (inet_pton(AF_INET, text.substr(0, colon).c_str(), &a.sin_addr) != 1) {63 throw std::runtime_error("endpoint must use numeric IPv4 address");64 }65 const auto host = ntohl(a.sin_addr.s_addr);66 if (host == 0 || host == 0xffffffff || (host >> 28) == 14) {67 throw std::runtime_error("Only explicit unicast endpoints are supported");68 }69 return a;70}7172bool same(const sockaddr_in& a, const sockaddr_in& b) {73 return a.sin_family == b.sin_family && a.sin_port == b.sin_port &&74 a.sin_addr.s_addr == b.sin_addr.s_addr;75}7677/** Sequence, duplicate and recovery state for one bounded finite session.78 * No message becomes readable until the full observed gap is filled. Transport79 * and CSV publication borrow this state but never mutate its buffers directly.80 */81class ReorderBuffer {82 const LiveOptions& options;83 std::map<uint64_t, Bytes> pending, recent;84 size_t pending_bytes = 0, recent_bytes = 0;85 uint64_t next = 1, high = 1;86 std::optional<uint64_t> end;87 std::optional<Clock::time_point> gap_start;88 Clock::time_point last_progress, last_request;8990 void insert(uint64_t sequence, Bytes bytes) {91 if (sequence < next) {92 const auto old = recent.find(sequence);93 if (old != recent.end() && old->second != bytes) {94 fail("conflicting processed duplicate");95 }96 ++duplicates;97 return;98 }99 const auto old = pending.find(sequence);100 if (old != pending.end()) {101 if (old->second != bytes) {102 fail("conflicting buffered duplicate");103 }104 ++duplicates;105 return;106 }107 if (pending.size() >= options.max_messages ||108 bytes.size() > options.max_bytes - pending_bytes) {109 fail("reorder buffer limit exceeded");110 }111 pending_bytes += bytes.size();112 pending.emplace(sequence, std::move(bytes));113 }114115 public:116 uint64_t requests = 0, duplicates = 0, gaps = 0;117 size_t peak_messages = 0, peak_bytes = 0;118119 explicit ReorderBuffer(const LiveOptions& limits)120 : options(limits), last_progress(Clock::now()),121 last_request(last_progress - Ms(limits.retry_ms)) {}122123 uint64_t next_sequence() const {124 return next;125 }126127 bool has_pending() const {128 return !pending.empty();129 }130131 bool complete() const {132 return end && next == *end;133 }134135 [[noreturn]] void fail(const std::string& reason) const {136 std::cerr << "state=FAILED next_sequence=" << next << " highest_boundary=" << high137 << " buffered_messages=" << pending.size() << '\n';138 throw std::runtime_error(reason + "; next_sequence=" + std::to_string(next));139 }140141 void check_deadlines(Clock::time_point now) const {142 if (gap_start && now - *gap_start >= Ms(options.recovery_ms)) {143 fail("recovery timeout");144 }145 if (now - last_progress >= Ms(options.idle_ms)) {146 fail("feed progress timeout (freshness unknown)");147 }148 }149150 /** Return the first missing range; a successful send must call requested(). */151 std::optional<std::pair<uint64_t, uint16_t>> recovery_request(Clock::time_point now) const {152 if (!gap_start || now - last_request < Ms(options.retry_ms)) {153 return std::nullopt;154 }155 uint64_t missing = next;156 for (const auto& entry : pending) {157 if (entry.first != missing) {158 break;159 }160 ++missing;161 }162 const auto following = pending.lower_bound(missing);163 const auto until = following == pending.end() ? high : following->first;164 const auto count = std::min<uint64_t>(until - missing, 65534);165 if (!count) {166 fail("invalid recovery range");167 }168 return std::make_pair(missing, static_cast<uint16_t>(count));169 }170171 void requested(Clock::time_point now) {172 ++requests;173 last_request = now;174 }175176 void admit(MoldDatagram datagram) {177 if (end && datagram.boundary > *end) {178 fail("data beyond end-of-session boundary");179 }180 if (datagram.ending) {181 if ((end && *end != datagram.sequence) || datagram.sequence < high) {182 fail("inconsistent end-of-session boundary");183 }184 end = datagram.sequence;185 }186 if (datagram.boundary > high) {187 high = datagram.boundary;188 last_progress = Clock::now();189 }190 for (size_t i = 0; i < datagram.messages.size(); ++i) {191 insert(datagram.sequence + i, std::move(datagram.messages[i]));192 }193 peak_messages = std::max(peak_messages, pending.size());194 peak_bytes = std::max(peak_bytes, pending_bytes);195 }196197 bool ready() {198 // Every pending key is in [next, high); equal counts prove continuity.199 if (high - next != pending.size()) {200 if (!gap_start) {201 gap_start = Clock::now();202 ++gaps;203 last_request = *gap_start - Ms(options.retry_ms);204 std::cerr << "state=RECOVERING next_sequence=" << next205 << " highest_boundary=" << high << '\n';206 }207 return false;208 }209 if (gap_start || next == 1) {210 std::cerr << "state=SYNCED next_sequence=" << next << '\n';211 }212 gap_start.reset();213 return true;214 }215216 const Bytes& front() const {217 if (pending.empty() || pending.begin()->first != next) {218 fail("internal noncontiguous queue");219 }220 return pending.begin()->second;221 }222223 /** Retire only a successfully applied/published frame and retain bounded duplicate evidence. */224 void acknowledge() {225 auto entry = pending.begin();226 pending_bytes -= entry->second.size();227 recent_bytes += entry->second.size();228 recent.emplace(next, std::move(entry->second));229 pending.erase(entry);230 while (recent.size() > options.max_messages || recent_bytes > options.max_bytes) {231 recent_bytes -= recent.begin()->second.size();232 recent.erase(recent.begin());233 }234 ++next;235 last_progress = Clock::now();236 }237};238239/** Poll and validate one datagram; unrelated endpoints are counted and ignored. */240std::optional<MoldDatagram> receive_datagram(int socket, const sockaddr_in& source,241 const sockaddr_in& recovery,242 const std::string& session,243 const ReorderBuffer& buffer, uint64_t& ignored) {244 pollfd wait{socket, POLLIN, 0};245 const int ready = poll(&wait, 1, 10);246 if (ready < 0) {247 if (errno == EINTR) {248 return std::nullopt;249 }250 buffer.fail("socket poll failed");251 }252 if (!ready) {253 return std::nullopt;254 }255 if (wait.revents & (POLLERR | POLLHUP | POLLNVAL)) {256 buffer.fail("socket poll error");257 }258 std::array<unsigned char, 65536> packet{};259 sockaddr_in from{};260 socklen_t from_len = sizeof(from);261 const auto received = recvfrom(socket, packet.data(), packet.size(), 0,262 reinterpret_cast<sockaddr*>(&from), &from_len);263 if (received < 0) {264 if (errno == EINTR) {265 return std::nullopt;266 }267 buffer.fail("socket receive failed");268 }269 if (!same(from, source) && !same(from, recovery)) {270 ++ignored;271 return std::nullopt;272 }273 return parse_moldudp64(packet.data(), static_cast<size_t>(received), session);274}275276/** Send a missing range before marking its retry deadline as advanced. */277void send_recovery(int socket, const sockaddr_in& endpoint, const std::string& session,278 ReorderBuffer& buffer, Clock::time_point now) {279 const auto range = buffer.recovery_request(now);280 if (!range) {281 return;282 }283 std::array<unsigned char, 20> request{};284 std::copy(session.begin(), session.end(), request.begin());285 write_big_endian(request.data() + 10, range->first, 8);286 write_big_endian(request.data() + 18, range->second, 2);287 if (sendto(socket, request.data(), request.size(), 0,288 reinterpret_cast<const sockaddr*>(&endpoint), sizeof(endpoint)) != 20) {289 buffer.fail("cannot send recovery request");290 }291 buffer.requested(now);292 std::cerr << "recovery_request sequence=" << range->first << " count=" << range->second << '\n';293}294295/** Apply one ordered frame and publish its selected book snapshot. */296void publish_frame(const ReorderBuffer& buffer, Book& book, std::ostream& output, size_t depth,297 bool check_book, Result& result) {298 const auto event = decode(buffer.front(), buffer.next_sequence());299 if (event) {300 const auto snapshot = book.apply(*event, depth, check_book);301 if (snapshot) {302 const auto line = csv_line(*snapshot, false);303 output << line << '\n';304 if (!output) {305 buffer.fail("output write failed");306 }307 result.hash = hash_csv_line(result.hash, line);308 ++result.selected;309 ++result.emitted;310 }311 }312}313314} // namespace315316Result live(const LiveOptions& opt, const std::string& target, std::ostream& output, size_t depth,317 bool check_book) {318 if (opt.session.size() != 10 ||319 !std::all_of(opt.session.begin(), opt.session.end(),320 [](unsigned char c) {321 return c >= 32 && c <= 126;322 }) ||323 !opt.idle_ms || !opt.recovery_ms || !opt.retry_ms || opt.retry_ms > opt.recovery_ms ||324 !opt.max_messages || !opt.max_bytes) {325 throw std::runtime_error("invalid live limits/session (session must be exactly 10 bytes)");326 }327 const auto listen = address(opt.listen), source = address(opt.source),328 recovery = address(opt.recovery);329 Socket socket;330 if (bind(socket.fd, reinterpret_cast<const sockaddr*>(&listen), sizeof(listen)) != 0) {331 throw std::runtime_error("cannot bind UDP socket: " + std::string(std::strerror(errno)));332 }333 Book book(target);334 Result result;335 result.hash = fnv1a_offset_basis;336 ReorderBuffer buffer(opt);337 uint64_t ignored = 0;338 const auto begin = Clock::now();339 std::cerr << "state=STARTING session=" << opt.session << " next_sequence=1\n";340 std::cerr << "listening=" << opt.listen341 << " transport=MoldUDP64-unicast validation=simulated-only\n";342 output << csv_header(false) << '\n';343 try {344 while (true) {345 const auto now = Clock::now();346 buffer.check_deadlines(now);347 send_recovery(socket.fd, recovery, opt.session, buffer, now);348 auto datagram =349 receive_datagram(socket.fd, source, recovery, opt.session, buffer, ignored);350 if (!datagram) {351 continue;352 }353 buffer.admit(std::move(*datagram));354 if (!buffer.ready()) {355 continue;356 }357 while (buffer.has_pending()) {358 publish_frame(buffer, book, output, depth, check_book, result);359 buffer.acknowledge();360 ++result.frames;361 }362 if (buffer.complete()) {363 break;364 }365 }366 } catch (const std::exception& error) {367 std::cerr << "state=UNTRUSTED next_sequence=" << buffer.next_sequence() << '\n';368 throw std::runtime_error(std::string(error.what()) +369 "; next_sequence=" + std::to_string(buffer.next_sequence()) +370 "; no complete session published");371 }372 result.elapsed_ns =373 std::chrono::duration_cast<std::chrono::nanoseconds>(Clock::now() - begin).count();374 std::cerr << "state=COMPLETE next_sequence=" << buffer.next_sequence()375 << " frames=" << result.frames << " selected=" << result.selected376 << " gaps=" << buffer.gaps << " requests=" << buffer.requests377 << " duplicates=" << buffer.duplicates << " ignored_sources=" << ignored378 << " peak_buffer_messages=" << buffer.peak_messages379 << " peak_buffer_bytes=" << buffer.peak_bytes380 << " validation=simulated-only nasdaq_connection=unverified\n";381 return result;382}383} // namespace itch