Spicy
parser.h
1 // Copyright (c) 2020-2021 by the Zeek Project. See LICENSE for details.
2 
3 #pragma once
4 
5 #include <algorithm>
6 #include <string>
7 #include <tuple>
8 #include <type_traits>
9 #include <utility>
10 #include <vector>
11 
12 #include <hilti/rt/exception.h>
13 #include <hilti/rt/fiber.h>
14 #include <hilti/rt/result.h>
15 #include <hilti/rt/type-info.h>
16 #include <hilti/rt/types/bytes.h>
17 #include <hilti/rt/types/null.h>
18 #include <hilti/rt/types/port.h>
19 #include <hilti/rt/types/reference.h>
20 #include <hilti/rt/types/struct.h>
21 #include <hilti/rt/util.h>
22 
23 #include <spicy/rt/filter.h>
24 #include <spicy/rt/global-state.h>
25 #include <spicy/rt/mime.h>
26 #include <spicy/rt/parser-fwd.h>
27 #include <spicy/rt/sink.h>
28 #include <spicy/rt/typedefs.h>
29 
30 namespace spicy::rt {
31 
33 HILTI_RT_ENUM(Direction, Originator, Responder, Both, Undef);
34 
35 } // namespace spicy::rt
36 
37 namespace hilti::rt::detail::adl {
38 
39 inline std::string to_string(const ::spicy::rt::Direction& x, adl::tag /*unused*/) {
40  switch ( x.value() ) {
41  case spicy::rt::Direction::Originator: return "originator";
42  case spicy::rt::Direction::Responder: return "responder";
43  case spicy::rt::Direction::Both: return "both";
44  case spicy::rt::Direction::Undef: return "undefined";
45  }
46 
48 };
49 
50 } // namespace hilti::rt::detail::adl
51 
52 namespace spicy::rt {
53 
54 inline std::ostream& operator<<(std::ostream& out, const Direction& d) { return out << hilti::rt::to_string(d); }
55 
57 struct ParserPort {
58  hilti::rt::Port port;
59  Direction direction;
60 
61  // Constructor used by code generator.
62  ParserPort(std::tuple<hilti::rt::Port, Direction> args) : port(std::get<0>(args)), direction(std::get<1>(args)) {}
63 };
64 
65 inline std::ostream& operator<<(std::ostream& out, const ParserPort& p) { return out << hilti::rt::to_string(p); }
66 
67 } // namespace spicy::rt
68 
69 namespace hilti::rt::detail::adl {
70 
71 inline std::string to_string(const spicy::rt::ParserPort& x, adl::tag /*unused*/) {
72  // TODO: Not sure why we need to explicit to_string() here.
73  if ( x.direction == spicy::rt::Direction::Both )
74  return x.port;
75  else
76  return fmt("%s (%s direction)", x.port, x.direction);
77 }
78 
79 } // namespace hilti::rt::detail::adl
80 
81 namespace spicy::rt {
82 
83 namespace detail {
84 
85 // Helper traits to detect whether a parser implements sink hooks.
86 
87 template<typename P>
88 struct has_on_gap {
89  template<typename U>
90  // If `->` gets wrapped to the next line cpplint misdetects this as a C-style cast.
91  // clang-format off
92  static auto test(int) -> decltype(
93  std::declval<U>().__on_0x25_gap(std::declval<uint64_t>(), std::declval<uint64_t>()), std::true_type());
94  // clang-format on
95  template<typename U>
96  static std::false_type test(...);
97  static constexpr bool value = std::is_same_v<decltype(test<P>(0)), std::true_type>;
98 };
99 
100 template<typename P>
102  template<typename U>
103  static auto test(int) -> decltype(std::declval<U>().__on_0x25_skipped(std::declval<uint64_t>()), std::true_type());
104  template<typename U>
105  static std::false_type test(...);
106  static constexpr bool value = std::is_same_v<decltype(test<P>(0)), std::true_type>;
107 };
108 
109 template<typename P>
111  template<typename U>
112  static auto test(int) -> decltype(std::declval<U>().__on_0x25_overlap(std::declval<uint64_t>(),
113  std::declval<const hilti::rt::Bytes&>(),
114  std::declval<const hilti::rt::Bytes&>()),
115  std::true_type());
116  template<typename U>
117  static std::false_type test(...);
118  static constexpr bool value = std::is_same_v<decltype(test<P>(0)), std::true_type>;
119 };
120 
121 template<typename P>
123  template<typename U>
124  static auto test(int) -> decltype(std::declval<U>().__on_0x25_undelivered(std::declval<uint64_t>(),
125  std::declval<const hilti::rt::Bytes&>()),
126  std::true_type());
127  template<typename U>
128  static std::false_type test(...);
129  static constexpr bool value = std::is_same_v<decltype(test<P>(0)), std::true_type>;
130 };
131 
132 } // namespace detail
133 
140 struct Parser {
141  Parser(std::string name, bool is_public, Parse1Function parse1, hilti::rt::any parse2, Parse3Function parse3,
142  ContextNewFunction context_new, const hilti::rt::TypeInfo* type, std::string description,
144  : name(std::move(name)),
145  is_public(is_public),
146  parse1(parse1),
147  parse2(std::move(parse2)),
148  parse3(parse3),
149  context_new(context_new),
150  type_info(type),
151  description(std::move(description)),
152  mime_types(std::move(mime_types)),
153  ports(std::move(ports)) {}
154 
155  Parser(std::string name, bool is_public, Parse1Function parse1, hilti::rt::any parse2, Parse3Function parse3,
156  hilti::rt::Null /* null */, const hilti::rt::TypeInfo* type, std::string description,
158  : name(std::move(name)),
159  is_public(is_public),
160  parse1(parse1),
161  parse2(std::move(parse2)),
162  parse3(parse3),
163  type_info(type),
164  description(std::move(description)),
165  mime_types(std::move(mime_types)),
166  ports(std::move(ports)) {}
167 
168  Parser(std::string name, bool is_public, hilti::rt::Null /* null */, hilti::rt::any parse2,
169  hilti::rt::Null /* null */, hilti::rt::Null /* null */, const hilti::rt::TypeInfo* type,
170  std::string description, hilti::rt::Vector<MIMEType> mime_types, hilti::rt::Vector<ParserPort> ports)
171  : Parser(std::move(name), is_public, nullptr, std::move(parse2), nullptr, nullptr, type, std::move(description),
172  std::move(mime_types), std::move(ports)) {}
173 
174  Parser(std::string name, bool is_public, hilti::rt::Null /* null */, hilti::rt::any parse2,
175  hilti::rt::Null /* null */, ContextNewFunction context_new, const hilti::rt::TypeInfo* type,
176  std::string description, hilti::rt::Vector<MIMEType> mime_types, hilti::rt::Vector<ParserPort> ports)
177  : Parser(std::move(name), is_public, nullptr, std::move(parse2), nullptr, context_new, type,
178  std::move(description), std::move(mime_types), std::move(ports)) {}
179 
180  Parser(const Parser&) = default;
181 
182  Parser() = default;
183  ~Parser() = default;
184  Parser(Parser&&) noexcept = default;
185  Parser& operator=(const Parser&) = default;
186  Parser& operator=(Parser&&) noexcept = default;
187 
192  std::optional<UnitContext> createContext() const {
193  if ( context_new )
194  return (*context_new)();
195  else
196  return {};
197  }
198 
200  std::string name;
201 
203  bool is_public;
204 
210  std::string linker_scope;
211 
217  Parse1Function parse1{};
218 
223  hilti::rt::any parse2;
224 
231  Parse3Function parse3{};
232 
237  ContextNewFunction context_new = nullptr;
238 
239  const hilti::rt::TypeInfo* type_info;
240 
244  std::string description;
245 
250 
255 
260  std::optional<detail::ParseSinkFunction> __parse_sink;
261 
263  std::optional<std::function<void(hilti::rt::StrongReferenceGeneric, uint64_t, uint64_t)>> __hook_gap;
264 
266  std::optional<std::function<void(hilti::rt::StrongReferenceGeneric, uint64_t, const hilti::rt::Bytes&,
267  const hilti::rt::Bytes&)>>
269 
271  std::optional<std::function<void(hilti::rt::StrongReferenceGeneric, uint64_t)>> __hook_skipped;
272 
274  std::optional<std::function<void(hilti::rt::StrongReferenceGeneric, uint64_t, const hilti::rt::Bytes&)>>
276 };
277 
279 inline auto parsers() {
280  const auto& parsers = detail::globalState()->parsers;
281 
282  std::vector<const Parser*> public_parsers;
283  std::copy_if(parsers.begin(), parsers.end(), std::back_inserter(public_parsers),
284  [](const auto& p) { return p->is_public; });
285 
286  return public_parsers;
287 }
288 
289 
293 class ParseError : public hilti::rt::RecoverableFailure {
294 public:
295  ParseError(const std::string& msg, const std::string& location = "") : RecoverableFailure(msg, location) {}
296 
297  ParseError(const hilti::rt::result::Error& e) : RecoverableFailure(e.description()) {}
298 
299  ~ParseError() override; /* required to create vtable, see hilti::rt::Exception */
300 };
301 
307 class Backtrack : public ParseError {
308 public:
309  Backtrack() : ParseError("backtracking outside of &try scope") {}
310  ~Backtrack() override;
311 };
312 
313 class MissingData : public ParseError {
314 public:
315  MissingData(const std::string& location = "") : ParseError("missing data", location) {}
316  ~MissingData() override; /* required to create vtable, see hilti::rt::Exception */
317 };
318 
323 extern void accept_input();
324 
331 extern void decline_input(const std::string& reason);
332 
333 namespace detail {
334 
348 template<typename UnitRef>
349 inline void registerParser(::spicy::rt::Parser& p, // NOLINT(google-runtime-references)
350  std::string linker_scope, UnitRef /* not used, just for template instantiation */) {
351  // Note: This may may be called before spicy::rt::init(), and during
352  // hilti::rt::init(). Cannot rely on any library functionality being
353  // initialized yet.
354 
355  p.linker_scope = std::move(linker_scope);
356  globalState()->parsers.emplace_back(&p);
357 
358  using unit_type = typename UnitRef::element_type;
359 
361  ! std::is_base_of<hilti::rt::trait::hasParameters, unit_type>::value )
362  p.__parse_sink = sink::detail::parseFunction<unit_type>();
363 
364  if constexpr ( detail::has_on_gap<unit_type>::value )
365  p.__hook_gap = sink::detail::hookFunction<unit_type, &unit_type::__on_0x25_gap, uint64_t, uint64_t>();
366 
368  p.__hook_skipped = sink::detail::hookFunction<unit_type, &unit_type::__on_0x25_skipped, uint64_t>();
369 
371  p.__hook_overlap = sink::detail::hookFunction<unit_type, &unit_type::__on_0x25_overlap, uint64_t,
372  const hilti::rt::Bytes&, const hilti::rt::Bytes&>();
373 
375  p.__hook_undelivered = sink::detail::hookFunction<unit_type, &unit_type::__on_0x25_undelivered, uint64_t,
376  const hilti::rt::Bytes&>();
377 }
378 
383 void printParserState(const std::string& unit_id, const hilti::rt::ValueReference<hilti::rt::Stream>& data,
384  const hilti::rt::stream::View& cur, int64_t lahead,
385  const hilti::rt::stream::SafeConstIterator& lahead_end, const std::string& literal_mode,
386  bool trim, const std::optional<hilti::rt::RecoverableFailure>& error);
387 
399 extern bool waitForInputOrEod(hilti::rt::ValueReference<hilti::rt::Stream>& data, // NOLINT(google-runtime-references)
400  const hilti::rt::stream::View& cur, uint64_t min,
402 
411 extern void waitForEod(hilti::rt::ValueReference<hilti::rt::Stream>& data, // NOLINT(google-runtime-references)
412  const hilti::rt::stream::View& cur,
414 
429 extern void waitForInput(hilti::rt::ValueReference<hilti::rt::Stream>& data, // NOLINT(google-runtime-references)
430  const hilti::rt::stream::View& cur, uint64_t min, const std::string& error_msg,
431  const std::string& location,
433 
444 extern bool waitForInputOrEod(hilti::rt::ValueReference<hilti::rt::Stream>& data, // NOLINT(google-runtime-references)
445  const hilti::rt::stream::View& cur,
447 
461 extern void waitForInput(hilti::rt::ValueReference<hilti::rt::Stream>& data, // NOLINT(google-runtime-references)
462  const hilti::rt::stream::View& cur, const std::string& error_msg, const std::string& location,
464 
474 
478 inline void backtrack() { throw Backtrack(); }
479 
491 std::optional<hilti::rt::stream::SafeConstIterator> unitFind(
493  const std::optional<hilti::rt::stream::SafeConstIterator>& i, const hilti::rt::Bytes& needle,
494  hilti::rt::stream::Direction d);
495 } // namespace detail
496 } // namespace spicy::rt
std::string to_string(T &&x)
Definition: extension-points.h:26
Definition: result.h:18
Definition: reference.h:663
Definition: parser.h:110
std::optional< std::function< void(hilti::rt::StrongReferenceGeneric, uint64_t, uint64_t)> > __hook_gap
Definition: parser.h:263
Definition: null.h:19
Definition: port.h:22
Definition: parser.h:307
hilti::rt::Vector< ParserPort > ports
Definition: parser.h:254
std::string linker_scope
Definition: parser.h:210
std::optional< std::function< void(hilti::rt::StrongReferenceGeneric, uint64_t, const hilti::rt::Bytes &, const hilti::rt::Bytes &)> > __hook_overlap
Definition: parser.h:268
Definition: parser.h:293
Definition: bytes.h:157
void cannot_be_reached() __attribute__((noreturn))
Definition: util.cc:42
Definition: stream.h:984
std::optional< detail::ParseSinkFunction > __parse_sink
Definition: parser.h:260
Definition: reference.h:345
Definition: parser.h:57
std::optional< UnitContext > createContext() const
Definition: parser.h:192
Definition: vector.h:256
Definition: deferred-expression.h:41
std::optional< std::function< void(hilti::rt::StrongReferenceGeneric, uint64_t, const hilti::rt::Bytes &)> > __hook_undelivered
Definition: parser.h:275
Definition: parser.h:88
Definition: parser.h:140
Definition: location.h:94
bool is_public
Definition: parser.h:203
Definition: parser.h:101
Definition: stream.h:393
Definition: type-info.h:1143
hilti::rt::any parse2
Definition: parser.h:223
std::optional< std::function< void(hilti::rt::StrongReferenceGeneric, uint64_t)> > __hook_skipped
Definition: parser.h:271
hilti::rt::Vector< MIMEType > mime_types
Definition: parser.h:249
std::string description
Definition: parser.h:244
Definition: parser.h:313
std::string name
Definition: parser.h:200