1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
|
#ifndef INC_TokenBuffer_hpp__
#define INC_TokenBuffer_hpp__
/**
* <b>SOFTWARE RIGHTS</b>
* <p>
* ANTLR 2.6.0 MageLang Insitute, 1999
* <p>
* We reserve no legal rights to the ANTLR--it is fully in the
* public domain. An individual or company may do whatever
* they wish with source code distributed with ANTLR or the
* code generated by ANTLR, including the incorporation of
* ANTLR, or its output, into commerical software.
* <p>
* We encourage users to develop software with ANTLR. However,
* we do ask that credit is given to us for developing
* ANTLR. By "credit", we mean that if you use ANTLR or
* incorporate any source code into one of your programs
* (commercial product, research project, or otherwise) that
* you acknowledge this fact somewhere in the documentation,
* research report, etc... If you like ANTLR and have
* developed a nice tool with the output, please mention that
* you developed it using ANTLR. In addition, we ask that the
* headers remain intact in our source code. As long as these
* guidelines are kept, we expect to continue enhancing this
* system and expect to make other tools available as they are
* completed.
* <p>
* The ANTLR gang:
* @version ANTLR 2.6.0 MageLang Insitute, 1999
* @author Terence Parr, <a href=http://www.MageLang.com>MageLang Institute</a>
* @author <br>John Lilley, <a href=http://www.Empathy.com>Empathy Software</a>
* @author <br><a href="mailto:pete@yamuna.demon.co.uk">Pete Wells</a>
*/
#include "antlr/config.hpp"
#include "antlr/TokenStream.hpp"
#include "antlr/CircularQueue.hpp"
ANTLR_BEGIN_NAMESPACE(antlr)
/**A Stream of Token objects fed to the parser from a TokenStream that can
* be rewound via mark()/rewind() methods.
* <p>
* A dynamic array is used to buffer up all the input tokens. Normally,
* "k" tokens are stored in the buffer. More tokens may be stored during
* guess mode (testing syntactic predicate), or when LT(i>k) is referenced.
* Consumption of tokens is deferred. In other words, reading the next
* token is not done by conume(), but deferred until needed by LA or LT.
* <p>
*
* @see antlr.Token
* @see antlr.TokenStream
* @see antlr.TokenQueue
*/
class TokenBuffer {
protected:
// Token source
TokenStream& input;
private:
// Number of active markers
int nMarkers;
// Additional offset used when markers are active
int markerOffset;
// Number of calls to consume() since last LA() or LT() call
int numToConsume;
// Circular queue
CircularQueue<RefToken> queue;
public:
/** Create a token buffer */
TokenBuffer(TokenStream& input_);
/** Mark another token for deferred consumption */
void consume();
private:
/** Ensure that the token buffer is sufficiently full */
void fill(int amount);
public:
/** Get a lookahead token value */
int LA(int i);
/** Get a lookahead token */
RefToken LT(int i);
/**Return an integer marker that can be used to rewind the buffer to
* its current state.
*/
int mark();
/**Rewind the token buffer to a marker.
* @param mark Marker returned previously from mark()
*/
void rewind(int mark);
private:
/** Sync up deferred consumption */
void syncConsume();
private:
TokenBuffer(const TokenBuffer& other);
const TokenBuffer& operator=(const TokenBuffer& other);
public:
// virtual ~TokenBuffer() {}
};
/** Sync up deferred consumption */
inline void TokenBuffer::syncConsume()
{
#ifdef OLD_CODE
while (numToConsume > 0) {
if (nMarkers > 0) {
// guess mode -- leave leading tokens and bump offset.
markerOffset++;
} else {
// normal mode -- remove first token
queue.removeFirst();
}
numToConsume--;
}
#endif
if (numToConsume > 0) {
if (nMarkers > 0) {
markerOffset += numToConsume;
} else {
queue.removeItems( numToConsume );
}
numToConsume = 0;
}
}
ANTLR_END_NAMESPACE
#endif //INC_TokenBuffer_hpp__
|
