|
145
|
1
|
|
|
2 /* Compiler implementation of the D programming language
|
|
|
3 * Copyright (C) 1999-2019 by The D Language Foundation, All Rights Reserved
|
|
|
4 * written by Walter Bright
|
|
|
5 * http://www.digitalmars.com
|
|
|
6 * Distributed under the Boost Software License, Version 1.0.
|
|
|
7 * http://www.boost.org/LICENSE_1_0.txt
|
|
|
8 * https://github.com/D-Programming-Language/dmd/blob/master/src/lexer.h
|
|
|
9 */
|
|
|
10
|
|
|
11 #pragma once
|
|
|
12
|
|
|
13 #include "root/root.h"
|
|
|
14 #include "globals.h"
|
|
|
15 #include "tokens.h"
|
|
|
16
|
|
|
17 struct StringTable;
|
|
|
18 class Identifier;
|
|
|
19
|
|
|
20 class Lexer
|
|
|
21 {
|
|
|
22 public:
|
|
|
23 static OutBuffer stringbuffer;
|
|
|
24
|
|
|
25 Loc scanloc; // for error messages
|
|
|
26
|
|
|
27 const utf8_t *base; // pointer to start of buffer
|
|
|
28 const utf8_t *end; // past end of buffer
|
|
|
29 const utf8_t *p; // current character
|
|
|
30 const utf8_t *line; // start of current line
|
|
|
31 Token token;
|
|
|
32 bool doDocComment; // collect doc comment information
|
|
|
33 bool anyToken; // !=0 means seen at least one token
|
|
|
34 bool commentToken; // !=0 means comments are TOKcomment's
|
|
|
35 bool errors; // errors occurred during lexing or parsing
|
|
|
36
|
|
|
37 Lexer(const char *filename,
|
|
|
38 const utf8_t *base, size_t begoffset, size_t endoffset,
|
|
|
39 bool doDocComment, bool commentToken);
|
|
|
40
|
|
|
41 TOK nextToken();
|
|
|
42 TOK peekNext();
|
|
|
43 TOK peekNext2();
|
|
|
44 void scan(Token *t);
|
|
|
45 Token *peek(Token *t);
|
|
|
46 Token *peekPastParen(Token *t);
|
|
|
47 unsigned escapeSequence();
|
|
|
48 TOK wysiwygStringConstant(Token *t, int tc);
|
|
|
49 TOK hexStringConstant(Token *t);
|
|
|
50 TOK delimitedStringConstant(Token *t);
|
|
|
51 TOK tokenStringConstant(Token *t);
|
|
|
52 TOK escapeStringConstant(Token *t);
|
|
|
53 TOK charConstant(Token *t);
|
|
|
54 void stringPostfix(Token *t);
|
|
|
55 TOK number(Token *t);
|
|
|
56 TOK inreal(Token *t);
|
|
|
57
|
|
|
58 Loc loc()
|
|
|
59 {
|
|
|
60 scanloc.charnum = (unsigned)(1 + p-line);
|
|
|
61 return scanloc;
|
|
|
62 }
|
|
|
63
|
|
|
64 void error(const char *format, ...);
|
|
|
65 void error(Loc loc, const char *format, ...);
|
|
|
66 void deprecation(const char *format, ...);
|
|
|
67 void poundLine();
|
|
|
68 unsigned decodeUTF();
|
|
|
69 void getDocComment(Token *t, unsigned lineComment);
|
|
|
70
|
|
|
71 static const utf8_t *combineComments(const utf8_t *c1, const utf8_t *c2);
|
|
|
72
|
|
|
73 private:
|
|
|
74 void endOfLine();
|
|
|
75 };
|
