parser.h

// Copyright (c) 2020-now by the Zeek Project. See LICENSE for details.
 
#pragma once
 
#include <algorithm>
#include <string>
#include <string_view>
#include <type_traits>
#include <utility>
#include <vector>
 
#include <hilti/rt/exception.h>
#include <hilti/rt/fiber.h>
#include <hilti/rt/result.h>
#include <hilti/rt/type-info.h>
#include <hilti/rt/types/bytes.h>
#include <hilti/rt/types/null.h>
#include <hilti/rt/types/port.h>
#include <hilti/rt/types/reference.h>
#include <hilti/rt/types/string.h>
#include <hilti/rt/types/struct.h>
#include <hilti/rt/types/tuple.h>
#include <hilti/rt/util.h>
 
#include <spicy/rt/filter.h>
#include <spicy/rt/global-state.h>
#include <spicy/rt/mime.h>
#include <spicy/rt/parser-fwd.h>
#include <spicy/rt/sink.h>
#include <spicy/rt/typedefs.h>
 
namespace spicy::rt {
 
HILTI_RT_ENUM(Direction, Originator, Responder, Both);
 
} // namespace spicy::rt
 
namespace hilti::rt::detail::adl {
 
inline std::string to_string(const ::spicy::rt::Direction& x, adl::tag /*unused*/) {
    switch ( x.value() ) {
        case spicy::rt::Direction::Originator: return "originator";
        case spicy::rt::Direction::Responder: return "responder";
        case spicy::rt::Direction::Both: return "both";
        case spicy::rt::Direction::Undef: return "undefined";
    }
 
    cannot_be_reached();
};
 
} // namespace hilti::rt::detail::adl
 
namespace spicy::rt {
 
inline std::ostream& operator<<(std::ostream& out, const Direction& d) { return out << hilti::rt::to_string(d); }
 
struct ParserPort {
    hilti::rt::Port port;
    Direction direction;
 
    // Constructor used by code generator.
    ParserPort(hilti::rt::Tuple<hilti::rt::Port, Direction> args)
        : port(hilti::rt::tuple::get<0>(args)), direction(hilti::rt::tuple::get<1>(args)) {}
};
 
inline std::ostream& operator<<(std::ostream& out, const ParserPort& p) { return out << hilti::rt::to_string(p); }
 
} // namespace spicy::rt
 
namespace hilti::rt::detail::adl {
 
inline std::string to_string(const spicy::rt::ParserPort& x, adl::tag /*unused*/) {
    // TODO: Not sure why we need to explicit to_string() here.
    if ( x.direction == spicy::rt::Direction::Both )
        return std::string(static_cast<hilti::rt::String>(x.port));
    else
        return fmt("%s (%s direction)", x.port, x.direction);
}
 
} // namespace hilti::rt::detail::adl
 
namespace spicy::rt {
 
namespace detail {
 
// Helper traits to detect whether a parser implements sink hooks.
 
template<typename P>
struct has_on_gap {
    template<typename U>
    // If `->` gets wrapped to the next line cpplint misdetects this as a C-style cast.
    // NOLINTNEXTLINE(readability/casting)
    static auto test(int)
        -> decltype(std::declval<U>().HILTI_INTERNAL(on_0x25_gap)(std::declval<uint64_t>(), std::declval<uint64_t>()),
                    std::true_type());
    template<typename U>
    static std::false_type test(...);
    static constexpr bool value = std::is_same_v<decltype(test<P>(0)), std::true_type>;
};
 
template<typename P>
struct has_on_skipped {
    template<typename U>
    // NOLINTNEXTLINE(readability/casting)
    static auto test(int)
        -> decltype(std::declval<U>().HILTI_INTERNAL(on_0x25_skipped)(std::declval<uint64_t>()), std::true_type());
    template<typename U>
    static std::false_type test(...);
    static constexpr bool value = std::is_same_v<decltype(test<P>(0)), std::true_type>;
};
 
template<typename P>
struct has_on_overlap {
    template<typename U>
    // NOLINTNEXTLINE(readability/casting)
    static auto test(int)
        -> decltype(std::declval<U>().HILTI_INTERNAL(on_0x25_overlap)(std::declval<uint64_t>(),
                                                                      std::declval<const hilti::rt::Bytes&>(),
                                                                      std::declval<const hilti::rt::Bytes&>()),
                    std::true_type());
    template<typename U>
    static std::false_type test(...);
    static constexpr bool value = std::is_same_v<decltype(test<P>(0)), std::true_type>;
};
 
template<typename P>
struct has_on_undelivered {
    template<typename U>
    // NOLINTNEXTLINE(readability/casting)
    static auto test(int)
        -> decltype(std::declval<U>().HILTI_INTERNAL(on_0x25_undelivered)(std::declval<uint64_t>(),
                                                                          std::declval<const hilti::rt::Bytes&>()),
                    std::true_type());
    template<typename U>
    static std::false_type test(...);
    static constexpr bool value = std::is_same_v<decltype(test<P>(0)), std::true_type>;
};
 
} // namespace detail
 
struct Parser {
    Parser(::hilti::rt::struct_::tag::Inits,
           std::string_view name,
           bool is_public,
           Parse1Function parse1,
           hilti::rt::any parse2,
           Parse3Function parse3,
           ContextNewFunction context_new,
           const hilti::rt::TypeInfo* type,
           hilti::rt::String description,
           hilti::rt::Vector<MIMEType> mime_types,
           hilti::rt::Vector<ParserPort> ports)
        : name(name),
          is_public(is_public),
          parse1(parse1),
          parse2(std::move(parse2)),
          parse3(parse3),
          context_new(context_new),
          type_info(type),
          description(std::move(description)),
          mime_types(std::move(mime_types)),
          ports(std::move(ports)) {
        _initProfiling();
    }
 
    Parser(::hilti::rt::struct_::tag::Inits,
           std::string_view name,
           bool is_public,
           Parse1Function parse1,
           hilti::rt::any parse2,
           Parse3Function parse3,
           hilti::rt::Null /* null */,
           const hilti::rt::TypeInfo* type,
           hilti::rt::String description,
           hilti::rt::Vector<MIMEType> mime_types,
           hilti::rt::Vector<ParserPort> ports)
        : name(name),
          is_public(is_public),
          parse1(parse1),
          parse2(std::move(parse2)),
          parse3(parse3),
          type_info(type),
          description(std::move(description)),
          mime_types(std::move(mime_types)),
          ports(std::move(ports)) {
        _initProfiling();
    }
 
    Parser(::hilti::rt::struct_::tag::Inits,
           std::string_view name,
           bool is_public,
           hilti::rt::Null /* null */,
           hilti::rt::any parse2,
           hilti::rt::Null /* null */,
           hilti::rt::Null /* null */,
           const hilti::rt::TypeInfo* type,
           hilti::rt::String description,
           hilti::rt::Vector<MIMEType> mime_types,
           hilti::rt::Vector<ParserPort> ports)
        : Parser(::hilti::rt::struct_::tag::Inits(),
                 name,
                 is_public,
                 nullptr,
                 std::move(parse2),
                 nullptr,
                 nullptr,
                 type,
                 std::move(description),
                 std::move(mime_types),
                 std::move(ports)) {
        _initProfiling();
    }
 
    Parser(::hilti::rt::struct_::tag::Inits,
           std::string_view name,
           bool is_public,
           hilti::rt::Null /* null */,
           hilti::rt::any parse2,
           hilti::rt::Null /* null */,
           ContextNewFunction context_new,
           const hilti::rt::TypeInfo* type,
           hilti::rt::String description,
           hilti::rt::Vector<MIMEType> mime_types,
           hilti::rt::Vector<ParserPort> ports)
        : Parser(::hilti::rt::struct_::tag::Inits(),
                 name,
                 is_public,
                 nullptr,
                 std::move(parse2),
                 nullptr,
                 context_new,
                 type,
                 std::move(description),
                 std::move(mime_types),
                 std::move(ports)) {
        _initProfiling();
    }
 
    Parser(const Parser&) = default;
 
    Parser() = default;
    ~Parser() = default;
    Parser(Parser&&) noexcept = default;
    Parser& operator=(const Parser&) = default;
    Parser& operator=(Parser&&) noexcept = default;
 
    hilti::rt::Optional<UnitContext> createContext() const {
        if ( context_new )
            return (*context_new)();
        else
            return {};
    }
 
    hilti::rt::String name;
 
    bool is_public = false;
 
    uint64_t linker_scope = 0;
 
    Parse1Function parse1{};
 
    hilti::rt::any parse2;
 
    Parse3Function parse3{};
 
    ContextNewFunction context_new = nullptr;
 
    const hilti::rt::TypeInfo* type_info = nullptr;
 
    struct {
        std::string prepare_block = "spicy/prepare/block/";
        std::string prepare_input = "spicy/prepare/input/";
        std::string prepare_stream = "spicy/prepare/stream/";
 
        operator bool() const {
            // ensure initialization code has run
            return ! prepare_input.empty();
        }
    } profiler_tags;
 
    hilti::rt::String description;
 
    hilti::rt::Vector<MIMEType> mime_types;
 
    hilti::rt::Vector<ParserPort> ports;
 
    detail::ParseSinkFunction __parse_sink = nullptr;
 
    void (*__hook_gap)(const hilti::rt::StrongReferenceGeneric&, uint64_t, uint64_t) = nullptr;
 
    void (*__hook_overlap)(const hilti::rt::StrongReferenceGeneric&,
                           uint64_t,
                           const hilti::rt::Bytes&,
                           const hilti::rt::Bytes&) = nullptr;
 
    void (*__hook_skipped)(const hilti::rt::StrongReferenceGeneric&, uint64_t) = nullptr;
 
    void (*__hook_undelivered)(const hilti::rt::StrongReferenceGeneric&, uint64_t, const hilti::rt::Bytes&) = nullptr;
 
private:
    void _initProfiling();
};
 
inline auto parsers() {
    const auto& parsers = detail::globalState()->parsers;
 
    std::vector<const Parser*> public_parsers;
    std::ranges::copy_if(parsers, std::back_inserter(public_parsers), [](const auto& p) { return p->is_public; });
 
    return public_parsers;
}
 
inline const auto& parserNames() { return detail::globalState()->parsers_by_name; }
 
hilti::rt::Result<hilti::rt::Nothing> registerParserAlias(const hilti::rt::String& parser,
                                                          const hilti::rt::String& alias);
 
hilti::rt::Result<const spicy::rt::Parser*> lookupParser(const hilti::rt::String& name = {},
                                                         const hilti::rt::Optional<uint64_t>& linker_scope = {});
 
class ParseError : public hilti::rt::RecoverableFailure {
public:
    ParseError(std::string_view msg, std::string_view location = "") : RecoverableFailure(msg, location) {}
 
    ParseError(const hilti::rt::result::Error& e) : RecoverableFailure(e.description()) {}
 
    ~ParseError() override; /* required to create vtable, see hilti::rt::Exception */
};
 
class Backtrack : public ParseError {
public:
    Backtrack() : ParseError("backtracking outside of &try scope") {}
    ~Backtrack() override;
};
 
class MissingData : public ParseError {
public:
    MissingData(std::string_view location = "") : ParseError("missing data", location) {}
    ~MissingData() override; /* required to create vtable, see hilti::rt::Exception */
};
 
extern void accept_input();
 
extern void decline_input(const hilti::rt::String& reason);
 
namespace detail {
 
template<typename UnitRef>
inline void registerParser(::spicy::rt::Parser& p, // NOLINT(google-runtime-references)
                           uint64_t linker_scope,
                           UnitRef /* not used, just for template instantiation */,
                           const hilti::rt::TypeInfo* /* utype */) {
    // Note: This may may be called before spicy::rt::init(), and during
    // hilti::rt::init(). Cannot rely on any library functionality being
    // initialized yet.
 
    p.linker_scope = linker_scope;
    globalState()->parsers.emplace_back(&p);
 
    using unit_type = typename UnitRef::element_type;
 
    if constexpr ( sink::detail::supports_sinks<unit_type>::value &&
                   ! std::is_base_of_v<hilti::rt::trait::hasParameters, unit_type> )
        p.__parse_sink = []() {
            auto unit = spicy::rt::UnitRef<unit_type>(unit_type());
            return std::make_pair(hilti::rt::StrongReferenceGeneric(unit), spicy::rt::sink::detail::connectUnit(unit));
        };
 
    if constexpr ( detail::has_on_gap<unit_type>::value )
        p.__hook_gap = [](const hilti::rt::StrongReferenceGeneric& u, uint64_t seq, uint64_t len) -> void {
            (u.as<unit_type>()->HILTI_INTERNAL(on_0x25_gap))(seq, len);
        };
 
    if constexpr ( detail::has_on_skipped<unit_type>::value )
        p.__hook_skipped = [](const hilti::rt::StrongReferenceGeneric& u, uint64_t seq) -> void {
            (u.as<unit_type>()->HILTI_INTERNAL(on_0x25_skipped))(seq);
        };
 
    if constexpr ( detail::has_on_overlap<unit_type>::value )
        p.__hook_overlap = [](const hilti::rt::StrongReferenceGeneric& u,
                              uint64_t seq,
                              const hilti::rt::Bytes& old,
                              const hilti::rt::Bytes& new_) -> void {
            (u.as<unit_type>()->HILTI_INTERNAL(on_0x25_overlap))(seq, old, new_);
        };
 
    if constexpr ( detail::has_on_undelivered<unit_type>::value )
        p.__hook_undelivered =
            [](const hilti::rt::StrongReferenceGeneric& u, uint64_t seq, const hilti::rt::Bytes& bytes) -> void {
            (u.as<unit_type>()->HILTI_INTERNAL(on_0x25_undelivered))(seq, bytes);
        };
}
 
void printParserState(std::string_view unit_id,
                      const hilti::rt::ValueReference<hilti::rt::Stream>& data,
                      const hilti::rt::Optional<hilti::rt::stream::SafeConstIterator>& begin,
                      const hilti::rt::stream::View& cur,
                      int64_t lahead,
                      const hilti::rt::stream::SafeConstIterator& lahead_end,
                      std::string_view literal_mode,
                      bool trim,
                      const hilti::rt::Optional<hilti::rt::RecoverableFailure>& error);
 
extern bool waitForInputOrEod(hilti::rt::ValueReference<hilti::rt::Stream>& data, // NOLINT(google-runtime-references)
                              const hilti::rt::stream::View& cur,
                              uint64_t min,
                              hilti::rt::StrongReference<spicy::rt::filter::detail::Filters> filters);
 
extern void waitForEod(hilti::rt::ValueReference<hilti::rt::Stream>& data, // NOLINT(google-runtime-references)
                       const hilti::rt::stream::View& cur,
                       hilti::rt::StrongReference<spicy::rt::filter::detail::Filters> filters);
 
extern void waitForInput(hilti::rt::ValueReference<hilti::rt::Stream>& data, // NOLINT(google-runtime-references)
                         const hilti::rt::stream::View& cur,
                         uint64_t min,
                         std::string_view error_msg,
                         std::string_view location,
                         hilti::rt::StrongReference<spicy::rt::filter::detail::Filters> filters);
 
extern bool waitForInputNoThrow(hilti::rt::ValueReference<hilti::rt::Stream>& data, // NOLINT(google-runtime-references)
                                const hilti::rt::stream::View& cur,
                                uint64_t min,
                                hilti::rt::StrongReference<spicy::rt::filter::detail::Filters> filters);
 
extern bool waitForInputOrEod(hilti::rt::ValueReference<hilti::rt::Stream>& data, // NOLINT(google-runtime-references)
                              const hilti::rt::stream::View& cur,
                              const hilti::rt::StrongReference<spicy::rt::filter::detail::Filters>& filters);
 
extern void waitForInput(hilti::rt::ValueReference<hilti::rt::Stream>& data, // NOLINT(google-runtime-references)
                         const hilti::rt::stream::View& cur,
                         std::string_view error_msg,
                         std::string_view location,
                         const hilti::rt::StrongReference<spicy::rt::filter::detail::Filters>& filters);
 
extern bool atEod(hilti::rt::ValueReference<hilti::rt::Stream>& data,
                  const hilti::rt::stream::View& cur,
                  const hilti::rt::StrongReference<spicy::rt::filter::detail::Filters>& filters);
 
inline void backtrack() { throw Backtrack(); }
 
hilti::rt::Optional<hilti::rt::stream::SafeConstIterator> unitFind(
    const hilti::rt::stream::SafeConstIterator& begin,
    const hilti::rt::stream::SafeConstIterator& end,
    const hilti::rt::Optional<hilti::rt::stream::SafeConstIterator>& i,
    const hilti::rt::Bytes& needle,
    hilti::rt::stream::Direction d);
 
hilti::rt::Bytes extractBytes(hilti::rt::ValueReference<hilti::rt::Stream>& data,
                              const hilti::rt::stream::View& cur,
                              uint64_t size,
                              bool eod_ok,
                              std::string_view location,
                              const hilti::rt::StrongReference<spicy::rt::filter::detail::Filters>& filters);
 
void expectBytesLiteral(hilti::rt::ValueReference<hilti::rt::Stream>& data,
                        const hilti::rt::stream::View& cur,
                        const hilti::rt::Bytes& literal,
                        std::string_view location,
                        const hilti::rt::StrongReference<spicy::rt::filter::detail::Filters>& filters);
 
} // namespace detail
} // namespace spicy::rt