m/market labResearch, with a record.
Source on GitHub ↗

live.cpp

1#include "itch/feed/moldudp/live.hpp"2#include "itch/book/book.hpp"3#include "itch/common/binary.hpp"4#include "itch/common/digest.hpp"5#include "itch/feed/itch/decoder.hpp"6#include "itch/feed/moldudp/moldudp.hpp"7#include "itch/output/csv.hpp"8#include <algorithm>9#include <arpa/inet.h>10#include <array>11#include <cerrno>12#include <chrono>13#include <cstring>14#include <iostream>15#include <map>16#include <optional>17#include <poll.h>18#include <stdexcept>19#include <sys/socket.h>20#include <unistd.h>21#include <vector>2223namespace itch {24namespace {25using Clock = std::chrono::steady_clock;26using Ms = std::chrono::milliseconds;2728/** Close the descriptor on all exit paths, including exceptions. */29struct Socket {30    int fd = socket(AF_INET, SOCK_DGRAM, 0);3132    Socket() {33        if (fd < 0) {34            throw std::runtime_error("cannot create UDP socket");35        }36    }3738    ~Socket() {39        close(fd);40    }41};4243/** Parse an explicit unicast endpoint; no DNS, wildcard bind or multicast membership. */44sockaddr_in address(const std::string& text) {45    const auto colon = text.find(':');46    if (colon == std::string::npos) {47        throw std::runtime_error("endpoint must be IPv4:port");48    }49    const auto port = text.substr(colon + 1);50    if (port.empty() || port.size() > 5 || !std::all_of(port.begin(), port.end(), [](char c) {51            return c >= '0' && c <= '9';52        })) {53        throw std::runtime_error("invalid endpoint port");54    }55    const auto n = std::stoul(port);56    if (!n || n > 65535) {57        throw std::runtime_error("endpoint port must be 1..65535");58    }59    sockaddr_in a{};60    a.sin_family = AF_INET;61    a.sin_port = htons(static_cast<uint16_t>(n));62    if (inet_pton(AF_INET, text.substr(0, colon).c_str(), &a.sin_addr) != 1) {63        throw std::runtime_error("endpoint must use numeric IPv4 address");64    }65    const auto host = ntohl(a.sin_addr.s_addr);66    if (host == 0 || host == 0xffffffff || (host >> 28) == 14) {67        throw std::runtime_error("Only explicit unicast endpoints are supported");68    }69    return a;70}7172bool same(const sockaddr_in& a, const sockaddr_in& b) {73    return a.sin_family == b.sin_family && a.sin_port == b.sin_port &&74           a.sin_addr.s_addr == b.sin_addr.s_addr;75}7677/** Sequence, duplicate and recovery state for one bounded finite session.78 * No message becomes readable until the full observed gap is filled. Transport79 * and CSV publication borrow this state but never mutate its buffers directly.80 */81class ReorderBuffer {82    const LiveOptions& options;83    std::map<uint64_t, Bytes> pending, recent;84    size_t pending_bytes = 0, recent_bytes = 0;85    uint64_t next = 1, high = 1;86    std::optional<uint64_t> end;87    std::optional<Clock::time_point> gap_start;88    Clock::time_point last_progress, last_request;8990    void insert(uint64_t sequence, Bytes bytes) {91        if (sequence < next) {92            const auto old = recent.find(sequence);93            if (old != recent.end() && old->second != bytes) {94                fail("conflicting processed duplicate");95            }96            ++duplicates;97            return;98        }99        const auto old = pending.find(sequence);100        if (old != pending.end()) {101            if (old->second != bytes) {102                fail("conflicting buffered duplicate");103            }104            ++duplicates;105            return;106        }107        if (pending.size() >= options.max_messages ||108            bytes.size() > options.max_bytes - pending_bytes) {109            fail("reorder buffer limit exceeded");110        }111        pending_bytes += bytes.size();112        pending.emplace(sequence, std::move(bytes));113    }114115  public:116    uint64_t requests = 0, duplicates = 0, gaps = 0;117    size_t peak_messages = 0, peak_bytes = 0;118119    explicit ReorderBuffer(const LiveOptions& limits)120        : options(limits), last_progress(Clock::now()),121          last_request(last_progress - Ms(limits.retry_ms)) {}122123    uint64_t next_sequence() const {124        return next;125    }126127    bool has_pending() const {128        return !pending.empty();129    }130131    bool complete() const {132        return end && next == *end;133    }134135    [[noreturn]] void fail(const std::string& reason) const {136        std::cerr << "state=FAILED next_sequence=" << next << " highest_boundary=" << high137                  << " buffered_messages=" << pending.size() << '\n';138        throw std::runtime_error(reason + "; next_sequence=" + std::to_string(next));139    }140141    void check_deadlines(Clock::time_point now) const {142        if (gap_start && now - *gap_start >= Ms(options.recovery_ms)) {143            fail("recovery timeout");144        }145        if (now - last_progress >= Ms(options.idle_ms)) {146            fail("feed progress timeout (freshness unknown)");147        }148    }149150    /** Return the first missing range; a successful send must call requested(). */151    std::optional<std::pair<uint64_t, uint16_t>> recovery_request(Clock::time_point now) const {152        if (!gap_start || now - last_request < Ms(options.retry_ms)) {153            return std::nullopt;154        }155        uint64_t missing = next;156        for (const auto& entry : pending) {157            if (entry.first != missing) {158                break;159            }160            ++missing;161        }162        const auto following = pending.lower_bound(missing);163        const auto until = following == pending.end() ? high : following->first;164        const auto count = std::min<uint64_t>(until - missing, 65534);165        if (!count) {166            fail("invalid recovery range");167        }168        return std::make_pair(missing, static_cast<uint16_t>(count));169    }170171    void requested(Clock::time_point now) {172        ++requests;173        last_request = now;174    }175176    void admit(MoldDatagram datagram) {177        if (end && datagram.boundary > *end) {178            fail("data beyond end-of-session boundary");179        }180        if (datagram.ending) {181            if ((end && *end != datagram.sequence) || datagram.sequence < high) {182                fail("inconsistent end-of-session boundary");183            }184            end = datagram.sequence;185        }186        if (datagram.boundary > high) {187            high = datagram.boundary;188            last_progress = Clock::now();189        }190        for (size_t i = 0; i < datagram.messages.size(); ++i) {191            insert(datagram.sequence + i, std::move(datagram.messages[i]));192        }193        peak_messages = std::max(peak_messages, pending.size());194        peak_bytes = std::max(peak_bytes, pending_bytes);195    }196197    bool ready() {198        // Every pending key is in [next, high); equal counts prove continuity.199        if (high - next != pending.size()) {200            if (!gap_start) {201                gap_start = Clock::now();202                ++gaps;203                last_request = *gap_start - Ms(options.retry_ms);204                std::cerr << "state=RECOVERING next_sequence=" << next205                          << " highest_boundary=" << high << '\n';206            }207            return false;208        }209        if (gap_start || next == 1) {210            std::cerr << "state=SYNCED next_sequence=" << next << '\n';211        }212        gap_start.reset();213        return true;214    }215216    const Bytes& front() const {217        if (pending.empty() || pending.begin()->first != next) {218            fail("internal noncontiguous queue");219        }220        return pending.begin()->second;221    }222223    /** Retire only a successfully applied/published frame and retain bounded duplicate evidence. */224    void acknowledge() {225        auto entry = pending.begin();226        pending_bytes -= entry->second.size();227        recent_bytes += entry->second.size();228        recent.emplace(next, std::move(entry->second));229        pending.erase(entry);230        while (recent.size() > options.max_messages || recent_bytes > options.max_bytes) {231            recent_bytes -= recent.begin()->second.size();232            recent.erase(recent.begin());233        }234        ++next;235        last_progress = Clock::now();236    }237};238239/** Poll and validate one datagram; unrelated endpoints are counted and ignored. */240std::optional<MoldDatagram> receive_datagram(int socket, const sockaddr_in& source,241                                             const sockaddr_in& recovery,242                                             const std::string& session,243                                             const ReorderBuffer& buffer, uint64_t& ignored) {244    pollfd wait{socket, POLLIN, 0};245    const int ready = poll(&wait, 1, 10);246    if (ready < 0) {247        if (errno == EINTR) {248            return std::nullopt;249        }250        buffer.fail("socket poll failed");251    }252    if (!ready) {253        return std::nullopt;254    }255    if (wait.revents & (POLLERR | POLLHUP | POLLNVAL)) {256        buffer.fail("socket poll error");257    }258    std::array<unsigned char, 65536> packet{};259    sockaddr_in from{};260    socklen_t from_len = sizeof(from);261    const auto received = recvfrom(socket, packet.data(), packet.size(), 0,262                                   reinterpret_cast<sockaddr*>(&from), &from_len);263    if (received < 0) {264        if (errno == EINTR) {265            return std::nullopt;266        }267        buffer.fail("socket receive failed");268    }269    if (!same(from, source) && !same(from, recovery)) {270        ++ignored;271        return std::nullopt;272    }273    return parse_moldudp64(packet.data(), static_cast<size_t>(received), session);274}275276/** Send a missing range before marking its retry deadline as advanced. */277void send_recovery(int socket, const sockaddr_in& endpoint, const std::string& session,278                   ReorderBuffer& buffer, Clock::time_point now) {279    const auto range = buffer.recovery_request(now);280    if (!range) {281        return;282    }283    std::array<unsigned char, 20> request{};284    std::copy(session.begin(), session.end(), request.begin());285    write_big_endian(request.data() + 10, range->first, 8);286    write_big_endian(request.data() + 18, range->second, 2);287    if (sendto(socket, request.data(), request.size(), 0,288               reinterpret_cast<const sockaddr*>(&endpoint), sizeof(endpoint)) != 20) {289        buffer.fail("cannot send recovery request");290    }291    buffer.requested(now);292    std::cerr << "recovery_request sequence=" << range->first << " count=" << range->second << '\n';293}294295/** Apply one ordered frame and publish its selected book snapshot. */296void publish_frame(const ReorderBuffer& buffer, Book& book, std::ostream& output, size_t depth,297                   bool check_book, Result& result) {298    const auto event = decode(buffer.front(), buffer.next_sequence());299    if (event) {300        const auto snapshot = book.apply(*event, depth, check_book);301        if (snapshot) {302            const auto line = csv_line(*snapshot, false);303            output << line << '\n';304            if (!output) {305                buffer.fail("output write failed");306            }307            result.hash = hash_csv_line(result.hash, line);308            ++result.selected;309            ++result.emitted;310        }311    }312}313314} // namespace315316Result live(const LiveOptions& opt, const std::string& target, std::ostream& output, size_t depth,317            bool check_book) {318    if (opt.session.size() != 10 ||319        !std::all_of(opt.session.begin(), opt.session.end(),320                     [](unsigned char c) {321                         return c >= 32 && c <= 126;322                     }) ||323        !opt.idle_ms || !opt.recovery_ms || !opt.retry_ms || opt.retry_ms > opt.recovery_ms ||324        !opt.max_messages || !opt.max_bytes) {325        throw std::runtime_error("invalid live limits/session (session must be exactly 10 bytes)");326    }327    const auto listen = address(opt.listen), source = address(opt.source),328               recovery = address(opt.recovery);329    Socket socket;330    if (bind(socket.fd, reinterpret_cast<const sockaddr*>(&listen), sizeof(listen)) != 0) {331        throw std::runtime_error("cannot bind UDP socket: " + std::string(std::strerror(errno)));332    }333    Book book(target);334    Result result;335    result.hash = fnv1a_offset_basis;336    ReorderBuffer buffer(opt);337    uint64_t ignored = 0;338    const auto begin = Clock::now();339    std::cerr << "state=STARTING session=" << opt.session << " next_sequence=1\n";340    std::cerr << "listening=" << opt.listen341              << " transport=MoldUDP64-unicast validation=simulated-only\n";342    output << csv_header(false) << '\n';343    try {344        while (true) {345            const auto now = Clock::now();346            buffer.check_deadlines(now);347            send_recovery(socket.fd, recovery, opt.session, buffer, now);348            auto datagram =349                receive_datagram(socket.fd, source, recovery, opt.session, buffer, ignored);350            if (!datagram) {351                continue;352            }353            buffer.admit(std::move(*datagram));354            if (!buffer.ready()) {355                continue;356            }357            while (buffer.has_pending()) {358                publish_frame(buffer, book, output, depth, check_book, result);359                buffer.acknowledge();360                ++result.frames;361            }362            if (buffer.complete()) {363                break;364            }365        }366    } catch (const std::exception& error) {367        std::cerr << "state=UNTRUSTED next_sequence=" << buffer.next_sequence() << '\n';368        throw std::runtime_error(std::string(error.what()) +369                                 "; next_sequence=" + std::to_string(buffer.next_sequence()) +370                                 "; no complete session published");371    }372    result.elapsed_ns =373        std::chrono::duration_cast<std::chrono::nanoseconds>(Clock::now() - begin).count();374    std::cerr << "state=COMPLETE next_sequence=" << buffer.next_sequence()375              << " frames=" << result.frames << " selected=" << result.selected376              << " gaps=" << buffer.gaps << " requests=" << buffer.requests377              << " duplicates=" << buffer.duplicates << " ignored_sources=" << ignored378              << " peak_buffer_messages=" << buffer.peak_messages379              << " peak_buffer_bytes=" << buffer.peak_bytes380              << " validation=simulated-only nasdaq_connection=unverified\n";381    return result;382}383} // namespace itch