The data contained in this repository can be downloaded to your computer using one of several clients.
Please see the documentation of your version control software client for more information.
Please select the desired protocol below to get the URL.
This URL has Read-Only access.
main_repo / deps / v8 / src / scanner.h @ 40c0f755
History | View | Annotate | Download (7.63 KB)
1 |
// Copyright 2006-2008 the V8 project authors. All rights reserved.
|
---|---|
2 |
// Redistribution and use in source and binary forms, with or without
|
3 |
// modification, are permitted provided that the following conditions are
|
4 |
// met:
|
5 |
//
|
6 |
// * Redistributions of source code must retain the above copyright
|
7 |
// notice, this list of conditions and the following disclaimer.
|
8 |
// * Redistributions in binary form must reproduce the above
|
9 |
// copyright notice, this list of conditions and the following
|
10 |
// disclaimer in the documentation and/or other materials provided
|
11 |
// with the distribution.
|
12 |
// * Neither the name of Google Inc. nor the names of its
|
13 |
// contributors may be used to endorse or promote products derived
|
14 |
// from this software without specific prior written permission.
|
15 |
//
|
16 |
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
17 |
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
18 |
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
19 |
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
20 |
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
21 |
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
22 |
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
23 |
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
24 |
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
25 |
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
26 |
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
27 |
|
28 |
#ifndef V8_SCANNER_H_
|
29 |
#define V8_SCANNER_H_
|
30 |
|
31 |
#include "token.h" |
32 |
#include "char-predicates-inl.h" |
33 |
|
34 |
namespace v8 { namespace internal { |
35 |
|
36 |
|
37 |
class UTF8Buffer { |
38 |
public:
|
39 |
UTF8Buffer(); |
40 |
~UTF8Buffer(); |
41 |
|
42 |
void Initialize(char* src, int length); |
43 |
void AddChar(uc32 c);
|
44 |
void Reset() { pos_ = 0; } |
45 |
int pos() const { return pos_; } |
46 |
char* data() const { return data_; } |
47 |
|
48 |
private:
|
49 |
char* data_;
|
50 |
int size_;
|
51 |
int pos_;
|
52 |
}; |
53 |
|
54 |
|
55 |
class UTF16Buffer { |
56 |
public:
|
57 |
UTF16Buffer(); |
58 |
|
59 |
void Initialize(Handle<String> data, unibrow::CharacterStream* stream);
|
60 |
void PushBack(uc32 ch);
|
61 |
uc32 Advance(); // returns a value < 0 when the buffer end is reached
|
62 |
uint16_t CharAt(int index);
|
63 |
int pos() const { return pos_; } |
64 |
int size() const { return size_; } |
65 |
Handle<String> SubString(int start, int end); |
66 |
List<uc32>* pushback_buffer() { return &pushback_buffer_; }
|
67 |
void SeekForward(int pos); |
68 |
|
69 |
private:
|
70 |
Handle<String> data_; |
71 |
int pos_;
|
72 |
int size_;
|
73 |
List<uc32> pushback_buffer_; |
74 |
uc32 last_; |
75 |
unibrow::CharacterStream* stream_; |
76 |
}; |
77 |
|
78 |
|
79 |
class Scanner { |
80 |
public:
|
81 |
|
82 |
typedef unibrow::Utf8InputBuffer<1024> Utf8Decoder; |
83 |
|
84 |
// Construction
|
85 |
explicit Scanner(bool is_pre_parsing);
|
86 |
|
87 |
// Initialize the Scanner to scan source:
|
88 |
void Init(Handle<String> source,
|
89 |
unibrow::CharacterStream* stream, |
90 |
int position);
|
91 |
|
92 |
// Returns the next token.
|
93 |
Token::Value Next(); |
94 |
|
95 |
// One token look-ahead (past the token returned by Next()).
|
96 |
Token::Value peek() const { return next_.token; } |
97 |
|
98 |
// Returns true if there was a line terminator before the peek'ed token.
|
99 |
bool has_line_terminator_before_next() const { |
100 |
return has_line_terminator_before_next_;
|
101 |
} |
102 |
|
103 |
struct Location {
|
104 |
Location(int b, int e) : beg_pos(b), end_pos(e) { } |
105 |
Location() : beg_pos(0), end_pos(0) { } |
106 |
int beg_pos;
|
107 |
int end_pos;
|
108 |
}; |
109 |
|
110 |
// Returns the location information for the current token
|
111 |
// (the token returned by Next()).
|
112 |
Location location() const { return current_.location; } |
113 |
Location peek_location() const { return next_.location; } |
114 |
|
115 |
// Returns the literal string, if any, for the current token (the
|
116 |
// token returned by Next()). The string is 0-terminated and in
|
117 |
// UTF-8 format; they may contain 0-characters. Literal strings are
|
118 |
// collected for identifiers, strings, and numbers.
|
119 |
const char* literal_string() const { |
120 |
return &literals_.data()[current_.literal_pos];
|
121 |
} |
122 |
int literal_length() const { |
123 |
return current_.literal_end - current_.literal_pos;
|
124 |
} |
125 |
|
126 |
Vector<const char> next_literal() const { |
127 |
return Vector<const char>(next_literal_string(), next_literal_length()); |
128 |
} |
129 |
|
130 |
// Returns the literal string for the next token (the token that
|
131 |
// would be returned if Next() were called).
|
132 |
const char* next_literal_string() const { |
133 |
return &literals_.data()[next_.literal_pos];
|
134 |
} |
135 |
// Returns the length of the next token (that would be returned if
|
136 |
// Next() were called).
|
137 |
int next_literal_length() const { |
138 |
return next_.literal_end - next_.literal_pos;
|
139 |
} |
140 |
|
141 |
// Scans the input as a regular expression pattern, previous
|
142 |
// character(s) must be /(=). Returns true if a pattern is scanned.
|
143 |
bool ScanRegExpPattern(bool seen_equal); |
144 |
// Returns true if regexp flags are scanned (always since flags can
|
145 |
// be empty).
|
146 |
bool ScanRegExpFlags();
|
147 |
|
148 |
// Seek forward to the given position. This operation does not
|
149 |
// work in general, for instance when there are pushed back
|
150 |
// characters, but works for seeking forward until simple delimiter
|
151 |
// tokens, which is what it is used for.
|
152 |
void SeekForward(int pos); |
153 |
|
154 |
Handle<String> SubString(int start_pos, int end_pos); |
155 |
bool stack_overflow() { return stack_overflow_; } |
156 |
|
157 |
static StaticResource<Utf8Decoder>* utf8_decoder() { return &utf8_decoder_; } |
158 |
|
159 |
// Tells whether the buffer contains an identifier (no escapes).
|
160 |
// Used for checking if a property name is an identifier.
|
161 |
static bool IsIdentifier(unibrow::CharacterStream* buffer); |
162 |
|
163 |
static unibrow::Predicate<IdentifierStart, 128> kIsIdentifierStart; |
164 |
static unibrow::Predicate<IdentifierPart, 128> kIsIdentifierPart; |
165 |
static unibrow::Predicate<unibrow::LineTerminator, 128> kIsLineTerminator; |
166 |
static unibrow::Predicate<unibrow::WhiteSpace, 128> kIsWhiteSpace; |
167 |
|
168 |
private:
|
169 |
// Source.
|
170 |
UTF16Buffer source_; |
171 |
int position_;
|
172 |
|
173 |
// Buffer to hold literal values (identifiers, strings, numbers)
|
174 |
// using 0-terminated UTF-8 encoding.
|
175 |
UTF8Buffer literals_; |
176 |
|
177 |
bool stack_overflow_;
|
178 |
static StaticResource<Utf8Decoder> utf8_decoder_;
|
179 |
|
180 |
// One Unicode character look-ahead; c0_ < 0 at the end of the input.
|
181 |
uc32 c0_; |
182 |
|
183 |
// The current and look-ahead token.
|
184 |
struct TokenDesc {
|
185 |
Token::Value token; |
186 |
Location location; |
187 |
int literal_pos, literal_end;
|
188 |
}; |
189 |
|
190 |
TokenDesc current_; // desc for current token (as returned by Next())
|
191 |
TokenDesc next_; // desc for next token (one token look-ahead)
|
192 |
bool has_line_terminator_before_next_;
|
193 |
bool is_pre_parsing_;
|
194 |
|
195 |
static const int kCharacterLookaheadBufferSize = 1; |
196 |
|
197 |
// Literal buffer support
|
198 |
void StartLiteral();
|
199 |
void AddChar(uc32 ch);
|
200 |
void AddCharAdvance();
|
201 |
void TerminateLiteral();
|
202 |
|
203 |
// Low-level scanning support.
|
204 |
void Advance();
|
205 |
void PushBack(uc32 ch);
|
206 |
|
207 |
void SkipWhiteSpace(bool initial); |
208 |
Token::Value SkipSingleLineComment(); |
209 |
Token::Value SkipMultiLineComment(); |
210 |
|
211 |
inline Token::Value Select(Token::Value tok);
|
212 |
inline Token::Value Select(uc32 next, Token::Value then, Token::Value else_);
|
213 |
|
214 |
void Scan();
|
215 |
Token::Value ScanToken(); |
216 |
void ScanDecimalDigits();
|
217 |
Token::Value ScanNumber(bool seen_period);
|
218 |
Token::Value ScanIdentifier(); |
219 |
uc32 ScanHexEscape(uc32 c, int length);
|
220 |
uc32 ScanOctalEscape(uc32 c, int length);
|
221 |
void ScanEscape();
|
222 |
Token::Value ScanString(); |
223 |
|
224 |
// Scans a possible HTML comment -- begins with '<!'.
|
225 |
Token::Value ScanHtmlComment(); |
226 |
|
227 |
// Return the current source position.
|
228 |
int source_pos() {
|
229 |
return source_.pos() - kCharacterLookaheadBufferSize + position_;
|
230 |
} |
231 |
|
232 |
// Decodes a unicode escape-sequence which is part of an identifier.
|
233 |
// If the escape sequence cannot be decoded the result is kBadRune.
|
234 |
uc32 ScanIdentifierUnicodeEscape(); |
235 |
}; |
236 |
|
237 |
} } // namespace v8::internal
|
238 |
|
239 |
#endif // V8_SCANNER_H_ |