|
Prism Ruby parser
|
The parser used to parse Ruby source. More...
#include "prism/compiler/nodiscard.h"#include "prism/compiler/nonnull.h"#include "prism/ast.h"#include "prism/comments.h"#include "prism/diagnostic.h"#include "prism/line_offset_list.h"#include "prism/magic_comments.h"#include "prism/options.h"

Go to the source code of this file.
Typedefs | |
| typedef struct pm_parser_t | pm_parser_t |
| The parser used to parse Ruby source. | |
| typedef void(* | pm_encoding_changed_callback_t) (pm_parser_t *parser) |
| When the encoding that is being used to parse the source is changed by prism, we provide the ability here to call out to a user-defined function. | |
| typedef void(* | pm_lex_callback_t) (pm_parser_t *parser, pm_token_t *token, void *data) |
| This is the callback that is called when a token is lexed. | |
Functions | |
| PRISM_NODISCARD PRISM_EXPORTED_FUNCTION pm_parser_t * | pm_parser_new (pm_arena_t *arena, const uint8_t *source, size_t size, const pm_options_t *options) PRISM_NONNULL(1) |
| Allocate and initialize a parser with the given start and end pointers. | |
| PRISM_EXPORTED_FUNCTION void | pm_parser_free (pm_parser_t *parser) PRISM_NONNULL(1) |
| Free both the memory held by the given parser and the parser itself. | |
| PRISM_EXPORTED_FUNCTION void | pm_parser_encoding_changed_callback_set (pm_parser_t *parser, pm_encoding_changed_callback_t callback) PRISM_NONNULL(1) |
| Register a callback that will be called whenever prism changes the encoding it is using to parse based on the magic comment. | |
| PRISM_EXPORTED_FUNCTION void | pm_parser_lex_callback_set (pm_parser_t *parser, pm_lex_callback_t callback, void *data) PRISM_NONNULL(1) |
| Register a callback that will be called whenever a token is lexed. | |
| PRISM_EXPORTED_FUNCTION void * | pm_parser_lex_callback_data (const pm_parser_t *parser) PRISM_NONNULL(1) |
| Returns the opaque data that is passed to the lex callback when it is called. | |
| PRISM_EXPORTED_FUNCTION const uint8_t * | pm_parser_start (const pm_parser_t *parser) PRISM_NONNULL(1) |
| Returns the raw pointer to the start of the source that is being parsed. | |
| PRISM_EXPORTED_FUNCTION const uint8_t * | pm_parser_end (const pm_parser_t *parser) PRISM_NONNULL(1) |
| Returns the raw pointer to the end of the source that is being parsed. | |
| PRISM_EXPORTED_FUNCTION int32_t | pm_parser_start_line (const pm_parser_t *parser) PRISM_NONNULL(1) |
| Returns the line that the parser was considered to have started on. | |
| PRISM_EXPORTED_FUNCTION const char * | pm_parser_encoding_name (const pm_parser_t *parser) PRISM_NONNULL(1) |
| Returns the name of the encoding that is being used to parse the source. | |
| PRISM_EXPORTED_FUNCTION size_t | pm_parser_encoding_char_width (const pm_parser_t *parser, const uint8_t *start, ptrdiff_t remaining) PRISM_NONNULL(1 |
| Returns the width of the character at the given pointer in the encoding that is being used to parse the source. | |
| PRISM_EXPORTED_FUNCTION size_t PRISM_EXPORTED_FUNCTION bool | pm_parser_encoding_us_ascii (const pm_parser_t *parser) PRISM_NONNULL(1) |
| Returns whether or not the parser is using the US-ASCII encoding. | |
| PRISM_EXPORTED_FUNCTION const pm_string_t * | pm_parser_filepath (const pm_parser_t *parser) PRISM_NONNULL(1) |
| Returns the filepath that is being used to parse the source. | |
| PRISM_EXPORTED_FUNCTION pm_constant_id_t | pm_parser_constant_find (const pm_parser_t *parser, const uint8_t *start, size_t length) PRISM_NONNULL(1 |
| Find a constant in the parser's constant pool. | |
| PRISM_EXPORTED_FUNCTION pm_constant_id_t PRISM_EXPORTED_FUNCTION pm_constant_id_t | pm_parser_constant_insert (pm_parser_t *parser, const uint8_t *start, size_t length) PRISM_NONNULL(1 |
| Add a constant to the parser's constant pool, for naming a node that was not parsed from the source. | |
| PRISM_EXPORTED_FUNCTION pm_constant_id_t PRISM_EXPORTED_FUNCTION pm_constant_id_t PRISM_EXPORTED_FUNCTION int8_t | pm_parser_frozen_string_literal (const pm_parser_t *parser) PRISM_NONNULL(1) |
| Returns the frozen string literal value of the parser, as determined by the frozen_string_literal magic comment or the option set on the parser. | |
| PRISM_EXPORTED_FUNCTION const pm_line_offset_list_t * | pm_parser_line_offsets (const pm_parser_t *parser) PRISM_NONNULL(1) |
| Returns the line offsets that are associated with the given parser. | |
| PRISM_EXPORTED_FUNCTION const pm_location_t * | pm_parser_data_loc (const pm_parser_t *parser) PRISM_NONNULL(1) |
| Returns the location of the DATA section that is associated with the given parser. | |
| PRISM_EXPORTED_FUNCTION uint64_t | pm_parser_checksum (const pm_parser_t *parser) PRISM_NONNULL(1) |
| Returns the checksum of the source code that was parsed. | |
| PRISM_EXPORTED_FUNCTION bool | pm_parser_continuable (const pm_parser_t *parser) PRISM_NONNULL(1) |
| Returns whether the given parser is continuable, meaning that it could become valid if more input were appended, as opposed to being definitively invalid. | |
| PRISM_EXPORTED_FUNCTION int | pm_parser_lex_state (const pm_parser_t *parser) PRISM_NONNULL(1) |
| Returns the lex state of the parser. | |
| PRISM_EXPORTED_FUNCTION size_t | pm_parser_comments_size (const pm_parser_t *parser) PRISM_NONNULL(1) |
| Returns the number of comments associated with the given parser. | |
| PRISM_EXPORTED_FUNCTION const pm_comment_t * | pm_parser_comments_first (const pm_parser_t *parser) PRISM_NONNULL(1) |
| Returns the first comment associated with the given parser. | |
| PRISM_EXPORTED_FUNCTION uint32_t | pm_node_comments_size (const pm_parser_t *parser, const pm_node_t *node) PRISM_NONNULL(1 |
| Returns the number of comments owned by the given node. | |
| PRISM_EXPORTED_FUNCTION uint32_t PRISM_EXPORTED_FUNCTION const pm_comment_t * | pm_node_comment (const pm_parser_t *parser, const pm_node_t *node, uint32_t index) PRISM_NONNULL(1 |
| Returns the comment at the given index among the comments owned by the given node, which are ordered by start offset. | |
| PRISM_EXPORTED_FUNCTION uint32_t PRISM_EXPORTED_FUNCTION const pm_comment_t PRISM_EXPORTED_FUNCTION uint32_t | pm_parser_comment_attachment (const pm_parser_t *parser, uint32_t index) PRISM_NONNULL(1) |
| Returns the index, among the comments associated with the given parser, of the comment at the given index among the comment attachments. | |
| PRISM_EXPORTED_FUNCTION uint32_t | pm_parser_comment_ranges_size (const pm_parser_t *parser) PRISM_NONNULL(1) |
| Returns the number of node ids that have a comment range, which is one more than the largest id of a node that owns a comment, or 0 if the attach_comments option was not set or there are no comments. | |
| PRISM_EXPORTED_FUNCTION pm_comment_range_t | pm_parser_comment_range (const pm_parser_t *parser, uint32_t node_id) PRISM_NONNULL(1) |
| Returns the range of comment attachments owned by the node with the given id. | |
| PRISM_EXPORTED_FUNCTION size_t | pm_parser_magic_comments_size (const pm_parser_t *parser) PRISM_NONNULL(1) |
| Returns the number of magic comments associated with the given parser. | |
| PRISM_EXPORTED_FUNCTION const pm_magic_comment_t * | pm_parser_magic_comments_first (const pm_parser_t *parser) PRISM_NONNULL(1) |
| Returns the first magic comment associated with the given parser. | |
| PRISM_EXPORTED_FUNCTION size_t | pm_parser_errors_size (const pm_parser_t *parser) PRISM_NONNULL(1) |
| Returns the number of errors associated with the given parser. | |
| PRISM_EXPORTED_FUNCTION size_t | pm_parser_warnings_size (const pm_parser_t *parser) PRISM_NONNULL(1) |
| Returns the number of warnings associated with the given parser. | |
| PRISM_EXPORTED_FUNCTION const pm_diagnostic_t * | pm_parser_errors_first (const pm_parser_t *parser) PRISM_NONNULL(1) |
| Returns the first error associated with the given parser. | |
| PRISM_EXPORTED_FUNCTION const pm_diagnostic_t * | pm_parser_warnings_first (const pm_parser_t *parser) PRISM_NONNULL(1) |
| Returns the first warning associated with the given parser. | |
| PRISM_EXPORTED_FUNCTION size_t | pm_parser_constants_size (const pm_parser_t *parser) PRISM_NONNULL(1) |
| Returns the number of constants in the constant pool associated with the given parser. | |
| PRISM_EXPORTED_FUNCTION const pm_constant_t * | pm_parser_constant_at (const pm_parser_t *parser, size_t index) PRISM_NONNULL(1) |
| Returns the constant at the given index in the constant pool associated with the given parser. | |
| PRISM_EXPORTED_FUNCTION const pm_constant_t * | pm_parser_constant (const pm_parser_t *parser, pm_constant_id_t constant_id) PRISM_NONNULL(1) |
| Returns a pointer to the constant at the given id in the constant pool associated with the given parser. | |
| PRISM_EXPORTED_FUNCTION pm_node_t * | pm_parse (pm_parser_t *parser) PRISM_NONNULL(1) |
| Initiate the parser with the given parser. | |
The parser used to parse Ruby source.
| typedef void(* pm_lex_callback_t) (pm_parser_t *parser, pm_token_t *token, void *data) |
This is the callback that is called when a token is lexed.
It is passed the opaque data pointer, the parser, and the token that was lexed.
| PRISM_NODISCARD PRISM_EXPORTED_FUNCTION pm_parser_t * pm_parser_new | ( | pm_arena_t * | arena, |
| const uint8_t * | source, | ||
| size_t | size, | ||
| const pm_options_t * | options | ||
| ) |
Allocate and initialize a parser with the given start and end pointers.
| arena | The arena to use for all AST-lifetime allocations. It is caller- owned and must outlive the parser. |
| source | The source to parse. |
| size | The size of the source. |
| options | The optional options to use when parsing. These options must live for the whole lifetime of this parser. |
pm_parser_free(). | PRISM_EXPORTED_FUNCTION void pm_parser_free | ( | pm_parser_t * | parser | ) |
Free both the memory held by the given parser and the parser itself.
| parser | The parser to free. |
| PRISM_EXPORTED_FUNCTION void pm_parser_encoding_changed_callback_set | ( | pm_parser_t * | parser, |
| pm_encoding_changed_callback_t | callback | ||
| ) |
Register a callback that will be called whenever prism changes the encoding it is using to parse based on the magic comment.
| parser | The parser to register the callback with. |
| callback | The callback to register. |
| PRISM_EXPORTED_FUNCTION void pm_parser_lex_callback_set | ( | pm_parser_t * | parser, |
| pm_lex_callback_t | callback, | ||
| void * | data | ||
| ) |
Register a callback that will be called whenever a token is lexed.
| parser | The parser to register the callback with. |
| data | The opaque data to pass to the callback when it is called. |
| callback | The callback to register. |
| PRISM_EXPORTED_FUNCTION void * pm_parser_lex_callback_data | ( | const pm_parser_t * | parser | ) |
Returns the opaque data that is passed to the lex callback when it is called.
| parser | The parser whose lex callback data we want to get. |
| PRISM_EXPORTED_FUNCTION const uint8_t * pm_parser_start | ( | const pm_parser_t * | parser | ) |
Returns the raw pointer to the start of the source that is being parsed.
| parser | the parser whose start pointer we want to get |
| PRISM_EXPORTED_FUNCTION const uint8_t * pm_parser_end | ( | const pm_parser_t * | parser | ) |
Returns the raw pointer to the end of the source that is being parsed.
| parser | the parser whose end pointer we want to get |
| PRISM_EXPORTED_FUNCTION int32_t pm_parser_start_line | ( | const pm_parser_t * | parser | ) |
Returns the line that the parser was considered to have started on.
| parser | the parser whose start line we want to get |
| PRISM_EXPORTED_FUNCTION const char * pm_parser_encoding_name | ( | const pm_parser_t * | parser | ) |
Returns the name of the encoding that is being used to parse the source.
| parser | the parser whose encoding name we want to get |
| PRISM_EXPORTED_FUNCTION size_t pm_parser_encoding_char_width | ( | const pm_parser_t * | parser, |
| const uint8_t * | start, | ||
| ptrdiff_t | remaining | ||
| ) |
Returns the width of the character at the given pointer in the encoding that is being used to parse the source.
| parser | the parser whose encoding we want to use |
| start | a pointer to the start of the character |
| remaining | the number of bytes remaining in the source |
| PRISM_EXPORTED_FUNCTION size_t PRISM_EXPORTED_FUNCTION bool pm_parser_encoding_us_ascii | ( | const pm_parser_t * | parser | ) |
Returns whether or not the parser is using the US-ASCII encoding.
| parser | the parser to check |
| PRISM_EXPORTED_FUNCTION const pm_string_t * pm_parser_filepath | ( | const pm_parser_t * | parser | ) |
Returns the filepath that is being used to parse the source.
| parser | the parser whose filepath we want to get |
| PRISM_EXPORTED_FUNCTION pm_constant_id_t pm_parser_constant_find | ( | const pm_parser_t * | parser, |
| const uint8_t * | start, | ||
| size_t | length | ||
| ) |
Find a constant in the parser's constant pool.
Returns the id of the constant, or 0 if the constant is not found.
| parser | the parser whose constant pool we want to search |
| start | a pointer to the start of the string to search for |
| length | the length of the string to search for |
| PRISM_EXPORTED_FUNCTION pm_constant_id_t PRISM_EXPORTED_FUNCTION pm_constant_id_t pm_parser_constant_insert | ( | pm_parser_t * | parser, |
| const uint8_t * | start, | ||
| size_t | length | ||
| ) |
Add a constant to the parser's constant pool, for naming a node that was not parsed from the source.
If the constant is already in the pool, its existing id is returned. Otherwise the parser copies the bytes, so they only need to remain valid for the duration of the call.
Adding a constant makes the pool larger, so a cache with one entry per constant built beforehand from pm_parser_constants_size and pm_parser_constant_at does not cover it. Build such caches after adding any constants.
| parser | the parser whose constant pool we want to add to |
| start | a pointer to the start of the string to add |
| length | the length of the string to add |
| PRISM_EXPORTED_FUNCTION pm_constant_id_t PRISM_EXPORTED_FUNCTION pm_constant_id_t PRISM_EXPORTED_FUNCTION int8_t pm_parser_frozen_string_literal | ( | const pm_parser_t * | parser | ) |
Returns the frozen string literal value of the parser, as determined by the frozen_string_literal magic comment or the option set on the parser.
| parser | the parser whose frozen string literal value we want to get |
| PRISM_EXPORTED_FUNCTION const pm_line_offset_list_t * pm_parser_line_offsets | ( | const pm_parser_t * | parser | ) |
Returns the line offsets that are associated with the given parser.
| parser | the parser whose line offsets we want to get |
| PRISM_EXPORTED_FUNCTION const pm_location_t * pm_parser_data_loc | ( | const pm_parser_t * | parser | ) |
Returns the location of the DATA section that is associated with the given parser.
| parser | the parser whose data location we want to get |
| PRISM_EXPORTED_FUNCTION uint64_t pm_parser_checksum | ( | const pm_parser_t * | parser | ) |
Returns the checksum of the source code that was parsed.
The data section after an END marker is not code, so the hash covers the source only up to and including the newline that ends the END line. The END marker is only found by parsing, so this must be called after pm_parse.
The hash is computed from the source on each call rather than during parsing, so that parses that never ask for it do not pay for it. This requires the source to still be alive, and callers that need the value more than once should keep it.
The hash is currently the 64-bit variant of XXH3 with a seed of 0. It is not cryptographic, and the algorithm may change between versions of prism, so hashes should only be compared against other hashes computed by the same version of prism.
| parser | the parser whose checksum we want to get |
| PRISM_EXPORTED_FUNCTION bool pm_parser_continuable | ( | const pm_parser_t * | parser | ) |
Returns whether the given parser is continuable, meaning that it could become valid if more input were appended, as opposed to being definitively invalid.
| parser | the parser whose continuable status we want to get |
| PRISM_EXPORTED_FUNCTION int pm_parser_lex_state | ( | const pm_parser_t * | parser | ) |
Returns the lex state of the parser.
Note that this is an internal detail, and we are purposefully not returning an instance of the internal enum that we use to track this. This is only exposed because we need it for some very niche use cases. Most consumers should avoid this function.
| parser | the parser whose lex state we want to get |
| PRISM_EXPORTED_FUNCTION size_t pm_parser_comments_size | ( | const pm_parser_t * | parser | ) |
Returns the number of comments associated with the given parser.
| parser | the parser whose comments we want to get the size of |
| PRISM_EXPORTED_FUNCTION const pm_comment_t * pm_parser_comments_first | ( | const pm_parser_t * | parser | ) |
Returns the first comment associated with the given parser.
Iterate over the rest with pm_comment_next, as in:
| parser | the parser whose comments we want to iterate over |
| PRISM_EXPORTED_FUNCTION uint32_t pm_node_comments_size | ( | const pm_parser_t * | parser, |
| const pm_node_t * | node | ||
| ) |
Returns the number of comments owned by the given node.
This is 0 if the node owns no comments or if the attach_comments option was not set.
| parser | the parser that produced the node |
| node | the node whose comments we want to count |
| PRISM_EXPORTED_FUNCTION uint32_t PRISM_EXPORTED_FUNCTION const pm_comment_t * pm_node_comment | ( | const pm_parser_t * | parser, |
| const pm_node_t * | node, | ||
| uint32_t | index | ||
| ) |
Returns the comment at the given index among the comments owned by the given node, which are ordered by start offset.
| parser | the parser that produced the node |
| node | the node whose comment we want to get |
| index | the index of the comment among those owned by the node |
| PRISM_EXPORTED_FUNCTION uint32_t PRISM_EXPORTED_FUNCTION const pm_comment_t PRISM_EXPORTED_FUNCTION uint32_t pm_parser_comment_attachment | ( | const pm_parser_t * | parser, |
| uint32_t | index | ||
| ) |
Returns the index, among the comments associated with the given parser, of the comment at the given index among the comment attachments.
The attachments are every comment, grouped by owning node in order of node id and ordered by start offset within each node, so there are as many attachments as there are comments when the attach_comments option is set.
| parser | the parser whose comment attachments we want to read |
| index | the index of the attachment |
| PRISM_EXPORTED_FUNCTION uint32_t pm_parser_comment_ranges_size | ( | const pm_parser_t * | parser | ) |
Returns the number of node ids that have a comment range, which is one more than the largest id of a node that owns a comment, or 0 if the attach_comments option was not set or there are no comments.
| parser | the parser whose comment ranges we want to count |
| PRISM_EXPORTED_FUNCTION pm_comment_range_t pm_parser_comment_range | ( | const pm_parser_t * | parser, |
| uint32_t | node_id | ||
| ) |
Returns the range of comment attachments owned by the node with the given id.
| parser | the parser that produced the node |
| node_id | the id of the node |
| PRISM_EXPORTED_FUNCTION size_t pm_parser_magic_comments_size | ( | const pm_parser_t * | parser | ) |
Returns the number of magic comments associated with the given parser.
| parser | the parser whose magic comments we want to get the size of |
| PRISM_EXPORTED_FUNCTION const pm_magic_comment_t * pm_parser_magic_comments_first | ( | const pm_parser_t * | parser | ) |
Returns the first magic comment associated with the given parser.
Iterate over the rest with pm_magic_comment_next.
| parser | the parser whose magic comments we want to iterate over |
| PRISM_EXPORTED_FUNCTION size_t pm_parser_errors_size | ( | const pm_parser_t * | parser | ) |
Returns the number of errors associated with the given parser.
| parser | the parser whose errors we want to get the size of |
| PRISM_EXPORTED_FUNCTION size_t pm_parser_warnings_size | ( | const pm_parser_t * | parser | ) |
Returns the number of warnings associated with the given parser.
| parser | the parser whose warnings we want to get the size of |
| PRISM_EXPORTED_FUNCTION const pm_diagnostic_t * pm_parser_errors_first | ( | const pm_parser_t * | parser | ) |
Returns the first error associated with the given parser.
Iterate over the rest with pm_diagnostic_next.
| parser | the parser whose errors we want to iterate over |
| PRISM_EXPORTED_FUNCTION const pm_diagnostic_t * pm_parser_warnings_first | ( | const pm_parser_t * | parser | ) |
Returns the first warning associated with the given parser.
Iterate over the rest with pm_diagnostic_next.
| parser | the parser whose warnings we want to iterate over |
| PRISM_EXPORTED_FUNCTION size_t pm_parser_constants_size | ( | const pm_parser_t * | parser | ) |
Returns the number of constants in the constant pool associated with the given parser.
| parser | the parser whose constant pool constants we want to get the size of |
| PRISM_EXPORTED_FUNCTION const pm_constant_t * pm_parser_constant_at | ( | const pm_parser_t * | parser, |
| size_t | index | ||
| ) |
Returns the constant at the given index in the constant pool associated with the given parser.
Together with pm_constant_id_index, this lets a caller build a cache with one entry per constant and then find the entry for any constant id, as in:
For any constant id, pm_parser_constant_at(parser, pm_constant_id_index(constant_id)) is pm_parser_constant(parser, constant_id).
| parser | the parser whose constant pool we want to look up from |
| index | the index of the constant, which must be less than pm_parser_constants_size(parser) |
| PRISM_EXPORTED_FUNCTION const pm_constant_t * pm_parser_constant | ( | const pm_parser_t * | parser, |
| pm_constant_id_t | constant_id | ||
| ) |
Returns a pointer to the constant at the given id in the constant pool associated with the given parser.
| parser | the parser whose constant pool we want to look up from |
| constant_id | the id of the constant to look up. Optional constant fields on nodes hold 0 when they are absent, which is not a valid id. |
| PRISM_EXPORTED_FUNCTION pm_node_t * pm_parse | ( | pm_parser_t * | parser | ) |
Initiate the parser with the given parser.
| parser | The parser to use. |