Spicy
parser.h
1 // Copyright (c) 2020-2021 by the Zeek Project. See LICENSE for details.
2 
3 #pragma once
4 
5 #include <algorithm>
6 #include <string>
7 #include <tuple>
8 #include <type_traits>
9 #include <utility>
10 #include <vector>
11 
12 #include <hilti/rt/exception.h>
13 #include <hilti/rt/fiber.h>
14 #include <hilti/rt/result.h>
15 #include <hilti/rt/type-info.h>
16 #include <hilti/rt/types/bytes.h>
17 #include <hilti/rt/types/null.h>
18 #include <hilti/rt/types/port.h>
19 #include <hilti/rt/types/reference.h>
20 #include <hilti/rt/types/struct.h>
21 
22 #include <spicy/rt/filter.h>
23 #include <spicy/rt/global-state.h>
24 #include <spicy/rt/mime.h>
25 #include <spicy/rt/parser-fwd.h>
26 #include <spicy/rt/sink.h>
27 #include <spicy/rt/typedefs.h>
28 
29 namespace spicy::rt {
30 
32 enum class Direction { Originator, Responder, Both, Undef };
33 
34 } // namespace spicy::rt
35 
36 namespace hilti::rt::detail::adl {
37 
38 inline std::string to_string(const ::spicy::rt::Direction& x, adl::tag /*unused*/) {
39  switch ( x ) {
40  case spicy::rt::Direction::Originator: return "originator";
41  case spicy::rt::Direction::Responder: return "responder";
42  case spicy::rt::Direction::Both: return "both";
43  case spicy::rt::Direction::Undef: return "undefined";
44  }
45 
47 };
48 
49 } // namespace hilti::rt::detail::adl
50 
51 namespace spicy::rt {
52 
53 inline std::ostream& operator<<(std::ostream& out, const Direction& d) { return out << hilti::rt::to_string(d); }
54 
56 struct ParserPort {
57  hilti::rt::Port port;
58  Direction direction;
59 
60  // Constructor used by code generator.
61  ParserPort(std::tuple<hilti::rt::Port, Direction> args) : port(std::get<0>(args)), direction(std::get<1>(args)) {}
62 };
63 
64 inline std::ostream& operator<<(std::ostream& out, const ParserPort& p) { return out << hilti::rt::to_string(p); }
65 
66 } // namespace spicy::rt
67 
68 namespace hilti::rt::detail::adl {
69 
70 inline std::string to_string(const spicy::rt::ParserPort& x, adl::tag /*unused*/) {
71  // TODO: Not sure why we need to explicit to_string() here.
72  if ( x.direction == spicy::rt::Direction::Both )
73  return x.port;
74  else
75  return fmt("%s (%s direction)", x.port, x.direction);
76 }
77 
78 } // namespace hilti::rt::detail::adl
79 
80 namespace spicy::rt {
81 
82 namespace detail {
83 
84 // Helper traits to detect whether a parser implements sink hooks.
85 
86 template<typename P>
87 struct has_on_gap {
88  template<typename U>
89  // If `->` gets wrapped to the next line cpplint misdetects this as a C-style cast.
90  // clang-format off
91  static auto test(int) -> decltype(
92  std::declval<U>().__on_0x25_gap(std::declval<uint64_t>(), std::declval<uint64_t>()), std::true_type());
93  // clang-format on
94  template<typename U>
95  static std::false_type test(...);
96  static constexpr bool value = std::is_same_v<decltype(test<P>(0)), std::true_type>;
97 };
98 
99 template<typename P>
101  template<typename U>
102  static auto test(int) -> decltype(std::declval<U>().__on_0x25_skipped(std::declval<uint64_t>()), std::true_type());
103  template<typename U>
104  static std::false_type test(...);
105  static constexpr bool value = std::is_same_v<decltype(test<P>(0)), std::true_type>;
106 };
107 
108 template<typename P>
110  template<typename U>
111  static auto test(int) -> decltype(std::declval<U>().__on_0x25_overlap(std::declval<uint64_t>(),
112  std::declval<const hilti::rt::Bytes&>(),
113  std::declval<const hilti::rt::Bytes&>()),
114  std::true_type());
115  template<typename U>
116  static std::false_type test(...);
117  static constexpr bool value = std::is_same_v<decltype(test<P>(0)), std::true_type>;
118 };
119 
120 template<typename P>
122  template<typename U>
123  static auto test(int) -> decltype(std::declval<U>().__on_0x25_undelivered(std::declval<uint64_t>(),
124  std::declval<const hilti::rt::Bytes&>()),
125  std::true_type());
126  template<typename U>
127  static std::false_type test(...);
128  static constexpr bool value = std::is_same_v<decltype(test<P>(0)), std::true_type>;
129 };
130 
131 } // namespace detail
132 
139 struct Parser {
140  Parser(std::string name, bool is_public, Parse1Function parse1, hilti::rt::any parse2, Parse3Function parse3,
141  ContextNewFunction context_new, const hilti::rt::TypeInfo* type, std::string description,
143  : name(std::move(name)),
144  is_public(is_public),
145  parse1(parse1),
146  parse2(std::move(parse2)),
147  parse3(parse3),
148  context_new(context_new),
149  type_info(type),
150  description(std::move(description)),
151  mime_types(std::move(mime_types)),
152  ports(std::move(ports)) {}
153 
154  Parser(std::string name, bool is_public, Parse1Function parse1, hilti::rt::any parse2, Parse3Function parse3,
155  hilti::rt::Null /* null */, const hilti::rt::TypeInfo* type, std::string description,
157  : name(std::move(name)),
158  is_public(is_public),
159  parse1(parse1),
160  parse2(std::move(parse2)),
161  parse3(parse3),
162  type_info(type),
163  description(std::move(description)),
164  mime_types(std::move(mime_types)),
165  ports(std::move(ports)) {}
166 
167  Parser(std::string name, bool is_public, hilti::rt::Null /* null */, hilti::rt::any parse2,
168  hilti::rt::Null /* null */, hilti::rt::Null /* null */, const hilti::rt::TypeInfo* type,
169  std::string description, hilti::rt::Vector<MIMEType> mime_types, hilti::rt::Vector<ParserPort> ports)
170  : Parser(std::move(name), is_public, nullptr, parse2, nullptr, nullptr, type, std::move(description),
171  std::move(mime_types), std::move(ports)) {}
172 
173  Parser(std::string name, bool is_public, hilti::rt::Null /* null */, hilti::rt::any parse2,
174  hilti::rt::Null /* null */, ContextNewFunction context_new, const hilti::rt::TypeInfo* type,
175  std::string description, hilti::rt::Vector<MIMEType> mime_types, hilti::rt::Vector<ParserPort> ports)
176  : Parser(std::move(name), is_public, nullptr, parse2, nullptr, context_new, type, std::move(description),
177  std::move(mime_types), std::move(ports)) {}
178 
179  Parser(const Parser&) = default;
180 
181  Parser() = default;
182  ~Parser() = default;
183  Parser(Parser&&) noexcept = default;
184  Parser& operator=(const Parser&) = default;
185  Parser& operator=(Parser&&) noexcept = default;
186 
191  std::optional<UnitContext> createContext() const {
192  if ( context_new )
193  return (*context_new)();
194  else
195  return {};
196  }
197 
199  std::string name;
200 
202  bool is_public;
203 
209  std::string linker_scope;
210 
216  Parse1Function parse1{};
217 
222  hilti::rt::any parse2;
223 
230  Parse3Function parse3{};
231 
236  ContextNewFunction context_new = nullptr;
237 
238  const hilti::rt::TypeInfo* type_info;
239 
243  std::string description;
244 
249 
254 
259  std::optional<detail::ParseSinkFunction> __parse_sink;
260 
262  std::optional<std::function<void(hilti::rt::StrongReferenceGeneric, uint64_t, uint64_t)>> __hook_gap;
263 
265  std::optional<std::function<void(hilti::rt::StrongReferenceGeneric, uint64_t, const hilti::rt::Bytes&,
266  const hilti::rt::Bytes&)>>
268 
270  std::optional<std::function<void(hilti::rt::StrongReferenceGeneric, uint64_t)>> __hook_skipped;
271 
273  std::optional<std::function<void(hilti::rt::StrongReferenceGeneric, uint64_t, const hilti::rt::Bytes&)>>
275 };
276 
278 inline auto parsers() {
279  const auto& parsers = detail::globalState()->parsers;
280 
281  std::vector<const Parser*> public_parsers;
282  std::copy_if(parsers.begin(), parsers.end(), std::back_inserter(public_parsers),
283  [](const auto& p) { return p->is_public; });
284 
285  return public_parsers;
286 }
287 
288 
292 class ParseError : public hilti::rt::RecoverableFailure {
293 public:
294  ParseError(const std::string& msg, const std::string& location = "") : RecoverableFailure(msg, location) {}
295 
296  ParseError(const hilti::rt::result::Error& e) : RecoverableFailure(e.description()) {}
297 
298  ~ParseError() override; /* required to create vtable, see hilti::rt::Exception */
299 };
300 
306 class Backtrack : public ParseError {
307 public:
308  Backtrack() : ParseError("backtracking outside of &try scope") {}
309  ~Backtrack() override;
310 };
311 
312 class MissingData : public ParseError {
313 public:
314  MissingData(const std::string& location = "") : ParseError("missing data", location) {}
315  ~MissingData() override; /* required to create vtable, see hilti::rt::Exception */
316 };
317 
318 namespace detail {
319 
333 template<typename UnitRef>
334 inline void registerParser(::spicy::rt::Parser& p, // NOLINT(google-runtime-references)
335  std::string linker_scope, UnitRef /* not used, just for template instantiation */) {
336  // Note: This may may be called before spicy::rt::init(), and during
337  // hilti::rt::init(). Cannot rely on any library functionality being
338  // initialized yet.
339 
340  p.linker_scope = std::move(linker_scope);
341  globalState()->parsers.emplace_back(&p);
342 
343  using unit_type = typename UnitRef::element_type;
344 
346  ! std::is_base_of<hilti::rt::trait::hasParameters, unit_type>::value )
347  p.__parse_sink = sink::detail::parseFunction<unit_type>();
348 
349  if constexpr ( detail::has_on_gap<unit_type>::value )
350  p.__hook_gap = sink::detail::hookFunction<unit_type, &unit_type::__on_0x25_gap, uint64_t, uint64_t>();
351 
353  p.__hook_skipped = sink::detail::hookFunction<unit_type, &unit_type::__on_0x25_skipped, uint64_t>();
354 
356  p.__hook_overlap = sink::detail::hookFunction<unit_type, &unit_type::__on_0x25_overlap, uint64_t,
357  const hilti::rt::Bytes&, const hilti::rt::Bytes&>();
358 
360  p.__hook_undelivered = sink::detail::hookFunction<unit_type, &unit_type::__on_0x25_undelivered, uint64_t,
361  const hilti::rt::Bytes&>();
362 }
363 
368 void printParserState(const std::string& unit_id, const hilti::rt::ValueReference<hilti::rt::Stream>& data,
369  const hilti::rt::stream::View& cur, int64_t lahead,
370  const hilti::rt::stream::SafeConstIterator& lahead_end, const std::string& literal_mode,
371  bool trim, const std::optional<hilti::rt::RecoverableFailure>& error);
372 
384 extern bool waitForInputOrEod(hilti::rt::ValueReference<hilti::rt::Stream>& data, // NOLINT(google-runtime-references)
385  const hilti::rt::stream::View& cur, uint64_t min,
387 
396 extern void waitForEod(hilti::rt::ValueReference<hilti::rt::Stream>& data, // NOLINT(google-runtime-references)
397  const hilti::rt::stream::View& cur,
399 
414 extern void waitForInput(hilti::rt::ValueReference<hilti::rt::Stream>& data, // NOLINT(google-runtime-references)
415  const hilti::rt::stream::View& cur, uint64_t min, const std::string& error_msg,
416  const std::string& location,
418 
429 extern bool waitForInputOrEod(hilti::rt::ValueReference<hilti::rt::Stream>& data, // NOLINT(google-runtime-references)
430  const hilti::rt::stream::View& cur,
432 
446 extern void waitForInput(hilti::rt::ValueReference<hilti::rt::Stream>& data, // NOLINT(google-runtime-references)
447  const hilti::rt::stream::View& cur, const std::string& error_msg, const std::string& location,
449 
459 
463 inline void backtrack() { throw Backtrack(); }
464 
476 std::optional<hilti::rt::stream::SafeConstIterator> unitFind(
478  const std::optional<hilti::rt::stream::SafeConstIterator>& i, const hilti::rt::Bytes& needle,
479  hilti::rt::stream::Direction d);
480 } // namespace detail
481 } // namespace spicy::rt
std::string to_string(T &&x)
Definition: extension-points.h:26
Definition: result.h:18
Definition: reference.h:652
Definition: parser.h:109
std::optional< std::function< void(hilti::rt::StrongReferenceGeneric, uint64_t, uint64_t)> > __hook_gap
Definition: parser.h:262
Definition: null.h:19
Definition: port.h:22
Definition: parser.h:306
hilti::rt::Vector< ParserPort > ports
Definition: parser.h:253
std::string linker_scope
Definition: parser.h:209
std::optional< std::function< void(hilti::rt::StrongReferenceGeneric, uint64_t, const hilti::rt::Bytes &, const hilti::rt::Bytes &)> > __hook_overlap
Definition: parser.h:267
Definition: parser.h:292
Definition: bytes.h:155
void cannot_be_reached() __attribute__((noreturn))
Definition: util.cc:52
Definition: stream.h:1001
std::optional< detail::ParseSinkFunction > __parse_sink
Definition: parser.h:259
Definition: reference.h:340
Definition: parser.h:56
std::optional< UnitContext > createContext() const
Definition: parser.h:191
Definition: vector.h:251
Definition: deferred-expression.h:41
std::optional< std::function< void(hilti::rt::StrongReferenceGeneric, uint64_t, const hilti::rt::Bytes &)> > __hook_undelivered
Definition: parser.h:274
Definition: parser.h:87
Definition: parser.h:139
Definition: location.h:93
bool is_public
Definition: parser.h:202
Definition: parser.h:100
Definition: stream.h:391
Definition: type-info.h:1146
hilti::rt::any parse2
Definition: parser.h:222
std::optional< std::function< void(hilti::rt::StrongReferenceGeneric, uint64_t)> > __hook_skipped
Definition: parser.h:270
hilti::rt::Vector< MIMEType > mime_types
Definition: parser.h:248
std::string description
Definition: parser.h:243
Definition: parser.h:312
std::string name
Definition: parser.h:199