#include <lexer.h>

Public Types
enum	Token { ERROR, BUILD, COLON, DEFAULT, EQUALS, IDENT, INCLUDE, INDENT, NEWLINE, PIPE, PIPE2, POOL, RULE, SUBNINJA, TEOF }

Public Member Functions
string	DescribeLastError ()
	If the last token read was an ERROR token, provide more info or the empty string. More...

bool	Error (const string &message, string *err)
	Construct an error message with context. More...

	Lexer ()

	Lexer (const char *input)
	Helper ctor useful for tests. More...

bool	PeekToken (Token token)
	If the next token is token, read it and return true. More...

bool	ReadIdent (string *out)
	Read a simple identifier (a rule or variable name). More...

bool	ReadPath (EvalString path, string err)
	Read a path (complete with $escapes). More...

Token	ReadToken ()
	Read a Token from the Token enum. More...

bool	ReadVarValue (EvalString value, string err)
	Read the value side of a var = value line (complete with $escapes). More...

void	Start (StringPiece filename, StringPiece input)
	Start parsing some input. More...

void	UnreadToken ()
	Rewind to the last read Token. More...

Static Public Member Functions
static const char *	TokenErrorHint (Token expected)
	Return a human-readable token hint, used in error messages. More...

static const char *	TokenName (Token t)
	Return a human-readable form of a token, used in error messages. More...

Private Member Functions
void	EatWhitespace ()
	Skip past whitespace (called after each read token/ident/etc.). More...

bool	ReadEvalString (EvalString eval, bool path, string err)
	Read a $-escaped string. More...

Private Attributes
StringPiece	filename_

StringPiece	input_

const char *	last_token_

const char *	ofs_

Detailed Description

Definition at line 27 of file lexer.h.

Member Enumeration Documentation

enum Lexer::Token

Enumerator
ERROR
BUILD
COLON
DEFAULT
EQUALS
IDENT
INCLUDE
INDENT
NEWLINE
PIPE
PIPE2
POOL
RULE
SUBNINJA
TEOF

Definition at line 32 of file lexer.h.

Constructor & Destructor Documentation

Lexer::Lexer ( )

inline

Definition at line 28 of file lexer.h.

Lexer::Lexer ( const char * input )

explicit

Helper ctor useful for tests.

Definition at line 62 of file lexer.cc.

References Start().

Member Function Documentation

string Lexer::DescribeLastError ( )

If the last token read was an ERROR token, provide more info or the empty string.

Definition at line 103 of file lexer.cc.

References last_token_.

Referenced by ManifestParser::Parse(), ReadEvalString(), and TEST().

void Lexer::EatWhitespace ( )

private

Skip past whitespace (called after each read token/ident/etc.).

re2c [ ]+ { continue; } "$\n" { continue; } nul { break; } [^] { break; }

Definition at line 428 of file lexer.cc.

References ofs_.

Referenced by ReadEvalString(), ReadIdent(), and ReadToken().

bool Lexer::Error	(	const string &	message,
		string *	err
	)

Construct an error message with context.

Definition at line 23 of file lexer.cc.

References StringPiece::AsString(), filename_, input_, last_token_, and StringPiece::str_.

Referenced by ManifestParser::ExpectToken(), ManifestParser::Parse(), ManifestParser::ParseDefault(), ManifestParser::ParseEdge(), ManifestParser::ParseFileInclude(), ManifestParser::ParsePool(), ManifestParser::ParseRule(), and ReadEvalString().

bool Lexer::PeekToken ( Token token )

If the next token is token, read it and return true.

Definition at line 420 of file lexer.cc.

References ReadToken(), and UnreadToken().

Referenced by ManifestParser::ParseEdge(), ManifestParser::ParsePool(), and ManifestParser::ParseRule().

bool Lexer::ReadEvalString	(	EvalString *	eval,
		bool	path,
		string *	err
	)

private

Read a $-escaped string.

re2c [^$ :
|\000]+ { eval->AddText(StringPiece(start, p - start)); continue; } [ :|
] { if (path) { p = start; break; } else { if (*start == '
') break; eval->AddText(StringPiece(start, 1)); continue; } } "$$" { eval->AddText(StringPiece("$", 1)); continue; } "$ " { eval->AddText(StringPiece(" ", 1)); continue; } "$\n"[ ]* { continue; } "${"varname"}" { eval->AddSpecial(StringPiece(start + 2, p - start - 3)); continue; } "$"simple_varname { eval->AddSpecial(StringPiece(start + 1, p - start - 1)); continue; } "$:" { eval->AddText(StringPiece(":", 1)); continue; } "$". { last_token_ = start; return Error("bad $-escape (literal $ must be written as $$)", err); } nul { last_token_ = start; return Error("unexpected EOF", err); } [^] { last_token_ = start; return Error(DescribeLastError(), err); }

Definition at line 595 of file lexer.cc.

References EvalString::AddSpecial(), EvalString::AddText(), DescribeLastError(), EatWhitespace(), Error(), last_token_, and ofs_.

Referenced by ReadPath(), and ReadVarValue().

bool Lexer::ReadIdent ( string * out )

Read a simple identifier (a rule or variable name).

Returns false if a name can't be read.

re2c varname { out->assign(start, p - start); break; } [^] { return false; }

Definition at line 509 of file lexer.cc.

References EatWhitespace(), and ofs_.

Referenced by ManifestParser::ParseEdge(), ManifestParser::ParseLet(), ManifestParser::ParsePool(), ManifestParser::ParseRule(), and TEST().

bool Lexer::ReadPath	(	EvalString *	path,
		string *	err
	)

inline

Read a path (complete with $escapes).

Returns false only on error, returned path may be empty if a delimiter (space, newline) is hit.

Definition at line 79 of file lexer.h.

References ReadEvalString().

Referenced by ManifestParser::ParseDefault(), ManifestParser::ParseEdge(), and ManifestParser::ParseFileInclude().

Lexer::Token Lexer::ReadToken ( )

Read a Token from the Token enum.

re2c re2c:define:YYCTYPE = "unsigned char"; re2c:define:YYCURSOR = p; re2c:define:YYMARKER = q; re2c:yyfill:enable = 0;

nul = "\000"; simple_varname = [a-zA-Z0-9_-]+; varname = [a-zA-Z0-9_.-]+;

[ ]*"#"[^\000
]*"\n" { continue; } [ ]*[
] { token = NEWLINE; break; } [ ]+ { token = INDENT; break; } "build" { token = BUILD; break; } "pool" { token = POOL; break; } "rule" { token = RULE; break; } "default" { token = DEFAULT; break; } "=" { token = EQUALS; break; } ":" { token = COLON; break; } "||" { token = PIPE2; break; } "|" { token = PIPE; break; } "include" { token = INCLUDE; break; } "subninja" { token = SUBNINJA; break; } varname { token = IDENT; break; } nul { token = TEOF; break; } [^] { token = ERROR; break; }

Definition at line 119 of file lexer.cc.

References BUILD, COLON, DEFAULT, EatWhitespace(), EQUALS, ERROR, IDENT, INCLUDE, INDENT, last_token_, NEWLINE, ofs_, PIPE, PIPE2, POOL, RULE, SUBNINJA, and TEOF.