diff options
Diffstat (limited to 'lib/antlr/src')
30 files changed, 3249 insertions, 0 deletions
diff --git a/lib/antlr/src/ANTLRUtil.cpp b/lib/antlr/src/ANTLRUtil.cpp new file mode 100644 index 00000000..30e7ba21 --- /dev/null +++ b/lib/antlr/src/ANTLRUtil.cpp @@ -0,0 +1,163 @@ +/* ANTLR Translator Generator + * Project led by Terence Parr at http://www.jGuru.com + * Software rights: http://www.antlr.org/license.html + * + * $Id$ + */ + +#include <antlr/config.hpp> +#include <antlr/IOException.hpp> + +#include <iostream> +#include <cctype> +#include <string> + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +namespace antlr { +#endif + +/** Eat whitespace from the input stream + * @param is the stream to read from + */ +ANTLR_USE_NAMESPACE(std)istream& eatwhite( ANTLR_USE_NAMESPACE(std)istream& is ) +{ + char c; + while( is.get(c) ) + { +#ifdef ANTLR_CCTYPE_NEEDS_STD + if( !ANTLR_USE_NAMESPACE(std)isspace(c) ) +#else + if( !isspace(c) ) +#endif + { + is.putback(c); + break; + } + } + return is; +} + +/** Read a string enclosed by '"' from a stream. Also handles escaping of \". + * Skips leading whitespace. + * @param in the istream to read from. + * @returns the string read from file exclusive the '"' + * @throws IOException if string is badly formatted + */ +ANTLR_USE_NAMESPACE(std)string read_string( ANTLR_USE_NAMESPACE(std)istream& in ) +{ + char ch; + ANTLR_USE_NAMESPACE(std)string ret(""); + // States for a simple state machine... + enum { START, READING, ESCAPE, FINISHED }; + int state = START; + + eatwhite(in); + + while( state != FINISHED && in.get(ch) ) + { + switch( state ) + { + case START: + // start state: check wether starting with " then switch to READING + if( ch != '"' ) + throw IOException("string must start with '\"'"); + state = READING; + continue; + case READING: + // reading state: look out for escape sequences and closing " + if( ch == '\\' ) // got escape sequence + { + state = ESCAPE; + continue; + } + if( ch == '"' ) // close quote -> stop + { + state = FINISHED; + continue; + } + ret += ch; // else append... + continue; + case ESCAPE: + switch(ch) + { + case '\\': + ret += ch; + state = READING; + continue; + case '"': + ret += ch; + state = READING; + continue; + case '0': + ret += '\0'; + state = READING; + continue; + default: // unrecognized escape is not mapped + ret += '\\'; + ret += ch; + state = READING; + continue; + } + } + } + if( state != FINISHED ) + throw IOException("badly formatted string: "+ret); + + return ret; +} + +/* Read a ([A-Z][0-9][a-z]_)* kindoff thing. Skips leading whitespace. + * @param in the istream to read from. + */ +ANTLR_USE_NAMESPACE(std)string read_identifier( ANTLR_USE_NAMESPACE(std)istream& in ) +{ + char ch; + ANTLR_USE_NAMESPACE(std)string ret(""); + + eatwhite(in); + + while( in.get(ch) ) + { +#ifdef ANTLR_CCTYPE_NEEDS_STD + if( ANTLR_USE_NAMESPACE(std)isupper(ch) || + ANTLR_USE_NAMESPACE(std)islower(ch) || + ANTLR_USE_NAMESPACE(std)isdigit(ch) || + ch == '_' ) +#else + if( isupper(ch) || islower(ch) || isdigit(ch) || ch == '_' ) +#endif + ret += ch; + else + { + in.putback(ch); + break; + } + } + return ret; +} + +/** Read a attribute="value" thing. Leading whitespace is skipped. + * Between attribute and '=' no whitespace is allowed. After the '=' it is + * permitted. + * @param in the istream to read from. + * @param attribute string the attribute name is put in + * @param value string the value of the attribute is put in + * @throws IOException if something is fishy. E.g. malformed quoting + * or missing '=' + */ +void read_AttributeNValue( ANTLR_USE_NAMESPACE(std)istream& in, + ANTLR_USE_NAMESPACE(std)string& attribute, + ANTLR_USE_NAMESPACE(std)string& value ) +{ + attribute = read_identifier(in); + + char ch; + if( in.get(ch) && ch == '=' ) + value = read_string(in); + else + throw IOException("invalid attribute=value thing "+attribute); +} + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +} +#endif diff --git a/lib/antlr/src/ASTFactory.cpp b/lib/antlr/src/ASTFactory.cpp new file mode 100644 index 00000000..98ce6b7a --- /dev/null +++ b/lib/antlr/src/ASTFactory.cpp @@ -0,0 +1,504 @@ +/* ANTLR Translator Generator + * Project led by Terence Parr at http://www.jGuru.com + * Software rights: http://www.antlr.org/license.html + * + * $Id$ + */ + +#include "antlr/CommonAST.hpp" +#include "antlr/ANTLRException.hpp" +#include "antlr/IOException.hpp" +#include "antlr/ASTFactory.hpp" +#include "antlr/ANTLRUtil.hpp" + +#include <iostream> +#include <istream> + +using namespace std; + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +namespace antlr { +#endif + +/** AST Support code shared by TreeParser and Parser. + * We use delegation to share code (and have only one + * bit of code to maintain) rather than subclassing + * or superclassing (forces AST support code to be + * loaded even when you don't want to do AST stuff). + * + * This class collects all factories of AST types used inside the code. + * New AST node types are registered with the registerFactory method. + * On creation of an ASTFactory object a default AST node factory may be + * specified. + * + * When registering types gaps between different types are filled with entries + * for the default factory. + */ + +/// Initialize factory +ASTFactory::ASTFactory() +: default_factory_descriptor(ANTLR_USE_NAMESPACE(std)make_pair(CommonAST::TYPE_NAME,&CommonAST::factory)) +{ + nodeFactories.resize( Token::MIN_USER_TYPE, &default_factory_descriptor ); +} + +/** Initialize factory with a non default node type. + * factory_node_name should be the name of the AST node type the factory + * generates. (should exist during the existance of this ASTFactory instance) + */ +ASTFactory::ASTFactory( const char* factory_node_name, factory_type fact ) +: default_factory_descriptor(ANTLR_USE_NAMESPACE(std)make_pair(factory_node_name, fact)) +{ + nodeFactories.resize( Token::MIN_USER_TYPE, &default_factory_descriptor ); +} + +/// Delete ASTFactory +ASTFactory::~ASTFactory() +{ + factory_descriptor_list::iterator i = nodeFactories.begin(); + + while( i != nodeFactories.end() ) + { + if( *i != &default_factory_descriptor ) + delete *i; + i++; + } +} + +/// Register a factory for a given AST type +void ASTFactory::registerFactory( int type, const char* ast_name, factory_type factory ) +{ + // check validity of arguments... + if( type < Token::MIN_USER_TYPE ) + throw ANTLRException("Internal parser error invalid type passed to RegisterFactory"); + if( factory == 0 ) + throw ANTLRException("Internal parser error 0 factory passed to RegisterFactory"); + + // resize up to and including 'type' and initalize any gaps to default + // factory. + if( nodeFactories.size() < (static_cast<unsigned int>(type)+1) ) + nodeFactories.resize( type+1, &default_factory_descriptor ); + + // And add new thing.. + nodeFactories[type] = new ANTLR_USE_NAMESPACE(std)pair<const char*, factory_type>( ast_name, factory ); +} + +void ASTFactory::setMaxNodeType( int type ) +{ + if( nodeFactories.size() < (static_cast<unsigned int>(type)+1) ) + nodeFactories.resize( type+1, &default_factory_descriptor ); +} + +/** Create a new empty AST node; if the user did not specify + * an AST node type, then create a default one: CommonAST. + */ +RefAST ASTFactory::create() +{ + RefAST node = nodeFactories[0]->second(); + node->setType(Token::INVALID_TYPE); + return node; +} + +RefAST ASTFactory::create(int type) +{ + RefAST t = nodeFactories[type]->second(); + t->initialize(type,""); + return t; +} + +RefAST ASTFactory::create(int type, const ANTLR_USE_NAMESPACE(std)string& txt) +{ + RefAST t = nodeFactories[type]->second(); + t->initialize(type,txt); + return t; +} + +#ifdef ANTLR_SUPPORT_XML +RefAST ASTFactory::create(const ANTLR_USE_NAMESPACE(std)string& type_name, ANTLR_USE_NAMESPACE(std)istream& infile ) +{ + factory_descriptor_list::iterator fact = nodeFactories.begin(); + + while( fact != nodeFactories.end() ) + { + if( type_name == (*fact)->first ) + { + RefAST t = (*fact)->second(); + t->initialize(infile); + return t; + } + fact++; + } + + string error = "ASTFactory::create: Unknown AST type '" + type_name + "'"; + throw ANTLRException(error); +} +#endif + +/** Create a new empty AST node; if the user did not specify + * an AST node type, then create a default one: CommonAST. + */ +RefAST ASTFactory::create(RefAST tr) +{ + if (!tr) + return nullAST; + +// cout << "create(tr)" << endl; + + RefAST t = nodeFactories[tr->getType()]->second(); + t->initialize(tr); + return t; +} + +RefAST ASTFactory::create(RefToken tok) +{ +// cout << "create( tok="<< tok->getType() << ", " << tok->getText() << ")" << nodeFactories.size() << endl; + RefAST t = nodeFactories[tok->getType()]->second(); + t->initialize(tok); + return t; +} + +/** Add a child to the current AST */ +void ASTFactory::addASTChild(ASTPair& currentAST, RefAST child) +{ + if (child) + { + if (!currentAST.root) + { + // Make new child the current root + currentAST.root = child; + } + else + { + if (!currentAST.child) + { + // Add new child to current root + currentAST.root->setFirstChild(child); + } + else + { + currentAST.child->setNextSibling(child); + } + } + // Make new child the current child + currentAST.child = child; + currentAST.advanceChildToEnd(); + } +} + +/** Deep copy a single node. This function the new clone() methods in the AST + * interface. Returns nullAST if t is null. + */ +RefAST ASTFactory::dup(RefAST t) +{ + if( t ) + return t->clone(); + else + return RefAST(nullASTptr); +} + +/** Duplicate tree including siblings of root. */ +RefAST ASTFactory::dupList(RefAST t) +{ + RefAST result = dupTree(t); // if t == null, then result==null + RefAST nt = result; + + while( t ) + { // for each sibling of the root + t = t->getNextSibling(); + nt->setNextSibling(dupTree(t)); // dup each subtree, building new tree + nt = nt->getNextSibling(); + } + return result; +} + +/** Duplicate a tree, assuming this is a root node of a tree + * duplicate that node and what's below; ignore siblings of root node. + */ +RefAST ASTFactory::dupTree(RefAST t) +{ + RefAST result = dup(t); // make copy of root + // copy all children of root. + if( t ) + result->setFirstChild( dupList(t->getFirstChild()) ); + return result; +} + +/** Make a tree from a list of nodes. The first element in the + * array is the root. If the root is null, then the tree is + * a simple list not a tree. Handles null children nodes correctly. + * For example, make(a, b, null, c) yields tree (a b c). make(null,a,b) + * yields tree (nil a b). + */ +RefAST ASTFactory::make(ANTLR_USE_NAMESPACE(std)vector<RefAST>& nodes) +{ + if ( nodes.size() == 0 ) + return RefAST(nullASTptr); + + RefAST root = nodes[0]; + RefAST tail = RefAST(nullASTptr); + + if( root ) + root->setFirstChild(RefAST(nullASTptr)); // don't leave any old pointers set + + // link in children; + for( unsigned int i = 1; i < nodes.size(); i++ ) + { + if ( nodes[i] == 0 ) // ignore null nodes + continue; + + if ( root == 0 ) // Set the root and set it up for a flat list + root = tail = nodes[i]; + else if ( tail == 0 ) + { + root->setFirstChild(nodes[i]); + tail = root->getFirstChild(); + } + else + { + tail->setNextSibling(nodes[i]); + tail = tail->getNextSibling(); + } + + if( tail ) // RK: I cannot fathom why this missing check didn't bite anyone else... + { + // Chase tail to last sibling + while (tail->getNextSibling()) + tail = tail->getNextSibling(); + } + } + + return root; +} + +/** Make a tree from a list of nodes, where the nodes are contained + * in an ASTArray object + */ +RefAST ASTFactory::make(ASTArray* nodes) +{ + RefAST ret = make(nodes->array); + delete nodes; + return ret; +} + +/// Make an AST the root of current AST +void ASTFactory::makeASTRoot( ASTPair& currentAST, RefAST root ) +{ + if (root) + { + // Add the current root as a child of new root + root->addChild(currentAST.root); + // The new current child is the last sibling of the old root + currentAST.child = currentAST.root; + currentAST.advanceChildToEnd(); + // Set the new root + currentAST.root = root; + } +} + +void ASTFactory::setASTNodeFactory( const char* factory_node_name, + factory_type factory ) +{ + default_factory_descriptor.first = factory_node_name; + default_factory_descriptor.second = factory; +} + +#ifdef ANTLR_SUPPORT_XML +bool ASTFactory::checkCloseTag( ANTLR_USE_NAMESPACE(std)istream& in ) +{ + char ch; + + if( in.get(ch) ) + { + if( ch == '<' ) + { + char ch2; + if( in.get(ch2) ) + { + if( ch2 == '/' ) + { + in.putback(ch2); + in.putback(ch); + return true; + } + in.putback(ch2); + in.putback(ch); + return false; + } + } + in.putback(ch); + return false; + } + return false; +} + +void ASTFactory::loadChildren( ANTLR_USE_NAMESPACE(std)istream& infile, + RefAST current ) +{ + char ch; + + for(;;) // for all children of this node.... + { + eatwhite(infile); + + infile.get(ch); // '<' + if( ch != '<' ) + { + string error = "Invalid XML file... no '<' found ("; + error += ch + ")"; + throw IOException(error); + } + + infile.get(ch); // / or text.... + + if( ch == '/' ) // check for close tag... + { + string temp; + + // read until '>' and see if it matches the open tag... if not trouble + temp = read_identifier( infile ); + + if( strcmp(temp.c_str(), current->typeName() ) != 0 ) + { + string error = "Invalid XML file... close tag does not match start tag: "; + error += current->typeName(); + error += " closed by " + temp; + throw IOException(error); + } + + infile.get(ch); // must be a '>' + + if( ch != '>' ) + { + string error = "Invalid XML file... no '>' found ("; + error += ch + ")"; + throw IOException(error); + } + // close tag => exit loop + break; + } + + // put our 'look ahead' back where it came from + infile.putback(ch); + infile.putback('<'); + + // and recurse into the tree... + RefAST child = LoadAST(infile); + + current->addChild( child ); + } +} + +void ASTFactory::loadSiblings(ANTLR_USE_NAMESPACE(std)istream& infile, + RefAST current ) +{ + for(;;) + { + eatwhite(infile); + + if( infile.eof() ) + break; + + if( checkCloseTag(infile) ) + break; + + RefAST sibling = LoadAST(infile); + current->setNextSibling(sibling); + } +} + +RefAST ASTFactory::LoadAST( ANTLR_USE_NAMESPACE(std)istream& infile ) +{ + RefAST current = nullAST; + char ch; + + eatwhite(infile); + + if( !infile.get(ch) ) + return nullAST; + + if( ch != '<' ) + { + string error = "Invalid XML file... no '<' found ("; + error += ch + ")"; + throw IOException(error); + } + + string ast_type = read_identifier(infile); + + // create the ast of type 'ast_type' + current = create( ast_type, infile ); + if( current == nullAST ) + { + string error = "Unsuported AST type: " + ast_type; + throw IOException(error); + } + + eatwhite(infile); + + infile.get(ch); + + // now if we have a '/' here it's a single node. If it's a '>' we get + // a tree with children + + if( ch == '/' ) + { + infile.get(ch); // get the closing '>' + if( ch != '>' ) + { + string error = "Invalid XML file... no '>' found after '/' ("; + error += ch + ")"; + throw IOException(error); + } + + // get the rest on this level + loadSiblings( infile, current ); + + return current; + } + + // and finaly see if we got the close tag... + if( ch != '>' ) + { + string error = "Invalid XML file... no '>' found ("; + error += ch + ")"; + throw IOException(error); + } + + // handle the ones below this level.. + loadChildren( infile, current ); + + // load the rest on this level... + loadSiblings( infile, current ); + + return current; +} +#endif // ANTLR_SUPPORT_XML + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +} +#endif + +/* Heterogeneous AST/XML-I/O ramblings... + * + * So there is some heterogeneous AST support.... + * basically in the code generators a new custom ast is generated without + * going throug the factory. It also expects the RefXAST to be defined. + * + * Is it maybe better to register all AST types with the ASTFactory class + * together with the respective factory methods. + * + * More and more I get the impression that hetero ast was a kindoff hack + * on top of ANTLR's normal AST system. + * + * The heteroast stuff will generate trouble for all astFactory.create( ... ) + * invocations. Most of this is handled via getASTCreateString methods in the + * codegenerator. At the moment getASTCreateString(GrammarAtom, String) has + * slightly to little info to do it's job (ok the hack that is in now + * works, but it's an ugly hack) + * + * An extra caveat is the 'nice' action.g thing. Which also judiciously calls + * getASTCreateString methods because it handles the #( ... ) syntax. + * And converts that to ASTFactory calls. + * + * + */ diff --git a/lib/antlr/src/ASTNULLType.cpp b/lib/antlr/src/ASTNULLType.cpp new file mode 100644 index 00000000..7dd62724 --- /dev/null +++ b/lib/antlr/src/ASTNULLType.cpp @@ -0,0 +1,157 @@ +/* ANTLR Translator Generator + * Project led by Terence Parr at http://www.jGuru.com + * Software rights: http://www.antlr.org/license.html + * + * $Id$ + */ + +#include "antlr/config.hpp" +#include "antlr/AST.hpp" +#include "antlr/ASTNULLType.hpp" + +#include <iostream> + +ANTLR_USING_NAMESPACE(std) + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +namespace antlr { +#endif + +RefAST ASTNULLType::clone( void ) const +{ + return RefAST(this); +} + +void ASTNULLType::addChild( RefAST ) +{ +} + +size_t ASTNULLType::getNumberOfChildren() const +{ + return 0; +} + +bool ASTNULLType::equals( RefAST ) const +{ + return false; +} + +bool ASTNULLType::equalsList( RefAST ) const +{ + return false; +} + +bool ASTNULLType::equalsListPartial( RefAST ) const +{ + return false; +} + +bool ASTNULLType::equalsTree( RefAST ) const +{ + return false; +} + +bool ASTNULLType::equalsTreePartial( RefAST ) const +{ + return false; +} + +vector<RefAST> ASTNULLType::findAll( RefAST ) +{ + return vector<RefAST>(); +} + +vector<RefAST> ASTNULLType::findAllPartial( RefAST ) +{ + return vector<RefAST>(); +} + +RefAST ASTNULLType::getFirstChild() const +{ + return this; +} + +RefAST ASTNULLType::getNextSibling() const +{ + return this; +} + +string ASTNULLType::getText() const +{ + return "<ASTNULL>"; +} + +int ASTNULLType::getType() const +{ + return Token::NULL_TREE_LOOKAHEAD; +} + +void ASTNULLType::initialize( int, const string& ) +{ +} + +void ASTNULLType::initialize( RefAST ) +{ +} + +void ASTNULLType::initialize( RefToken ) +{ +} + +#ifdef ANTLR_SUPPORT_XML +void ASTNULLType::initialize( istream& ) +{ +} +#endif + +void ASTNULLType::setFirstChild( RefAST ) +{ +} + +void ASTNULLType::setNextSibling( RefAST ) +{ +} + +void ASTNULLType::setText( const string& ) +{ +} + +void ASTNULLType::setType( int ) +{ +} + +string ASTNULLType::toString() const +{ + return getText(); +} + +string ASTNULLType::toStringList() const +{ + return getText(); +} + +string ASTNULLType::toStringTree() const +{ + return getText(); +} + +#ifdef ANTLR_SUPPORT_XML +bool ASTNULLType::attributesToStream( ostream& ) const +{ + return false; +} + +void ASTNULLType::toStream( ostream& out ) const +{ + out << "</ASTNULL>" << endl; +} +#endif + +const char* ASTNULLType::typeName( void ) const +{ + return "ASTNULLType"; +} + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +} +#endif diff --git a/lib/antlr/src/ASTRefCount.cpp b/lib/antlr/src/ASTRefCount.cpp new file mode 100644 index 00000000..0ca54df0 --- /dev/null +++ b/lib/antlr/src/ASTRefCount.cpp @@ -0,0 +1,41 @@ +/* ANTLR Translator Generator + * Project led by Terence Parr at http://www.jGuru.com + * Software rights: http://www.antlr.org/license.html + * + * $Id$ + */ +#include "antlr/ASTRefCount.hpp" +#include "antlr/AST.hpp" + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +namespace antlr { +#endif + +ASTRef::ASTRef(AST* p) +: ptr(p), count(1) +{ + if (p && !p->ref) + p->ref = this; +} + +ASTRef::~ASTRef() +{ + delete ptr; +} + +ASTRef* ASTRef::getRef(const AST* p) +{ + if (p) { + AST* pp = const_cast<AST*>(p); + if (pp->ref) + return pp->ref->increment(); + else + return new ASTRef(pp); + } else + return 0; +} + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +} +#endif + diff --git a/lib/antlr/src/BaseAST.cpp b/lib/antlr/src/BaseAST.cpp new file mode 100644 index 00000000..f10f1e16 --- /dev/null +++ b/lib/antlr/src/BaseAST.cpp @@ -0,0 +1,281 @@ +/* ANTLR Translator Generator + * Project led by Terence Parr at http://www.jGuru.com + * Software rights: http://www.antlr.org/license.html + * + * $Id$ + */ + +#include "antlr/config.hpp" + +#include <iostream> + +#include "antlr/AST.hpp" +#include "antlr/BaseAST.hpp" + +ANTLR_USING_NAMESPACE(std) +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +namespace antlr { +#endif + +size_t BaseAST::getNumberOfChildren() const +{ + RefBaseAST t = this->down; + size_t n = 0; + if( t ) + { + n = 1; + while( t->right ) + { + t = t->right; + n++; + } + return n; + } + return n; +} + +void BaseAST::doWorkForFindAll( + ANTLR_USE_NAMESPACE(std)vector<RefAST>& v, + RefAST target,bool partialMatch) +{ + // Start walking sibling lists, looking for matches. + for (RefAST sibling=this; + sibling; + sibling=sibling->getNextSibling()) + { + if ( (partialMatch && sibling->equalsTreePartial(target)) || + (!partialMatch && sibling->equalsTree(target)) ) { + v.push_back(sibling); + } + // regardless of match or not, check any children for matches + if ( sibling->getFirstChild() ) { + RefBaseAST(sibling->getFirstChild())->doWorkForFindAll(v, target, partialMatch); + } + } +} + +/** Is t an exact structural and equals() match of this tree. The + * 'this' reference is considered the start of a sibling list. + */ +bool BaseAST::equalsList(RefAST t) const +{ + // the empty tree is not a match of any non-null tree. + if (!t) + return false; + + // Otherwise, start walking sibling lists. First mismatch, return false. + RefAST sibling=this; + for (;sibling && t; + sibling=sibling->getNextSibling(), t=t->getNextSibling()) { + // as a quick optimization, check roots first. + if (!sibling->equals(t)) + return false; + // if roots match, do full list match test on children. + if (sibling->getFirstChild()) { + if (!sibling->getFirstChild()->equalsList(t->getFirstChild())) + return false; + } + // sibling has no kids, make sure t doesn't either + else if (t->getFirstChild()) + return false; + } + + if (!sibling && !t) + return true; + + // one sibling list has more than the other + return false; +} + +/** Is 'sub' a subtree of this list? + * The siblings of the root are NOT ignored. + */ +bool BaseAST::equalsListPartial(RefAST sub) const +{ + // the empty tree is always a subset of any tree. + if (!sub) + return true; + + // Otherwise, start walking sibling lists. First mismatch, return false. + RefAST sibling=this; + for (;sibling && sub; + sibling=sibling->getNextSibling(), sub=sub->getNextSibling()) { + // as a quick optimization, check roots first. + if (!sibling->equals(sub)) + return false; + // if roots match, do partial list match test on children. + if (sibling->getFirstChild()) + if (!sibling->getFirstChild()->equalsListPartial(sub->getFirstChild())) + return false; + } + + if (!sibling && sub) + // nothing left to match in this tree, but subtree has more + return false; + + // either both are null or sibling has more, but subtree doesn't + return true; +} + +/** Is tree rooted at 'this' equal to 't'? The siblings + * of 'this' are ignored. + */ +bool BaseAST::equalsTree(RefAST t) const +{ + // check roots first + if (!equals(t)) + return false; + // if roots match, do full list match test on children. + if (getFirstChild()) { + if (!getFirstChild()->equalsList(t->getFirstChild())) + return false; + } + // sibling has no kids, make sure t doesn't either + else if (t->getFirstChild()) + return false; + + return true; +} + +/** Is 'sub' a subtree of the tree rooted at 'this'? The siblings + * of 'this' are ignored. + */ +bool BaseAST::equalsTreePartial(RefAST sub) const +{ + // the empty tree is always a subset of any tree. + if (!sub) + return true; + + // check roots first + if (!equals(sub)) + return false; + // if roots match, do full list partial match test on children. + if (getFirstChild()) + if (!getFirstChild()->equalsListPartial(sub->getFirstChild())) + return false; + + return true; +} + +/** Walk the tree looking for all exact subtree matches. Return + * an ASTEnumerator that lets the caller walk the list + * of subtree roots found herein. + */ +ANTLR_USE_NAMESPACE(std)vector<RefAST> BaseAST::findAll(RefAST target) +{ + ANTLR_USE_NAMESPACE(std)vector<RefAST> roots; + + // the empty tree cannot result in an enumeration + if (target) { + doWorkForFindAll(roots,target,false); // find all matches recursively + } + + return roots; +} + +/** Walk the tree looking for all subtrees. Return + * an ASTEnumerator that lets the caller walk the list + * of subtree roots found herein. + */ +ANTLR_USE_NAMESPACE(std)vector<RefAST> BaseAST::findAllPartial(RefAST target) +{ + ANTLR_USE_NAMESPACE(std)vector<RefAST> roots; + + // the empty tree cannot result in an enumeration + if (target) + doWorkForFindAll(roots,target,true); // find all matches recursively + + return roots; +} + +ANTLR_USE_NAMESPACE(std)string BaseAST::toStringList() const +{ + ANTLR_USE_NAMESPACE(std)string ts=""; + + if (getFirstChild()) + { + ts+=" ( "; + ts+=toString(); + ts+=getFirstChild()->toStringList(); + ts+=" )"; + } + else + { + ts+=" "; + ts+=toString(); + } + + if (getNextSibling()) + ts+=getNextSibling()->toStringList(); + + return ts; +} + +ANTLR_USE_NAMESPACE(std)string BaseAST::toStringTree() const +{ + ANTLR_USE_NAMESPACE(std)string ts = ""; + + if (getFirstChild()) + { + ts+=" ( "; + ts+=toString(); + ts+=getFirstChild()->toStringList(); + ts+=" )"; + } + else + { + ts+=" "; + ts+=toString(); + } + return ts; +} + +#ifdef ANTLR_SUPPORT_XML +/* This whole XML output stuff needs a little bit more thought + * I'd like to store extra XML data in the node. e.g. for custom ast's + * with for instance symboltable references. This + * should be more pluggable.. + * @returns boolean value indicating wether a closetag should be produced. + */ +bool BaseAST::attributesToStream( ANTLR_USE_NAMESPACE(std)ostream& out ) const +{ + out << "text=\"" << this->getText() + << "\" type=\"" << this->getType() << "\""; + + return false; +} + +void BaseAST::toStream( ANTLR_USE_NAMESPACE(std)ostream& out ) const +{ + for( RefAST node = this; node != 0; node = node->getNextSibling() ) + { + out << "<" << this->typeName() << " "; + + // Write out attributes and if there is extra data... + bool need_close_tag = node->attributesToStream( out ); + + if( need_close_tag ) + { + // got children so write them... + if( node->getFirstChild() != 0 ) + node->getFirstChild()->toStream( out ); + + // and a closing tag.. + out << "</" << node->typeName() << ">" << endl; + } + } +} +#endif + +// this is nasty, but it makes the code generation easier +ANTLR_API RefAST nullAST; + +#if defined(_MSC_VER) && !defined(__ICL) // Microsoft Visual C++ +extern ANTLR_API AST* const nullASTptr = 0; +#else +ANTLR_API AST* const nullASTptr = 0; +#endif + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +} +#endif diff --git a/lib/antlr/src/BitSet.cpp b/lib/antlr/src/BitSet.cpp new file mode 100644 index 00000000..2a32404a --- /dev/null +++ b/lib/antlr/src/BitSet.cpp @@ -0,0 +1,62 @@ +/* ANTLR Translator Generator + * Project led by Terence Parr at http://www.jGuru.com + * Software rights: http://www.antlr.org/license.html + * + * $Id$ + */ +#include "antlr/BitSet.hpp" +#include <string> + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +namespace antlr { +#endif + +BitSet::BitSet(unsigned int nbits) +: storage(nbits) +{ + for (unsigned int i = 0; i < nbits ; i++ ) + storage[i] = false; +} + +BitSet::BitSet( const unsigned long* bits_, unsigned int nlongs ) +: storage(nlongs*32) +{ + for ( unsigned int i = 0 ; i < (nlongs * 32); i++) + storage[i] = (bits_[i>>5] & (1UL << (i&31))) ? true : false; +} + +BitSet::~BitSet() +{ +} + +void BitSet::add(unsigned int el) +{ + if( el >= storage.size() ) + storage.resize( el+1, false ); + + storage[el] = true; +} + +bool BitSet::member(unsigned int el) const +{ + if ( el >= storage.size()) + return false; + + return storage[el]; +} + +ANTLR_USE_NAMESPACE(std)vector<unsigned int> BitSet::toArray() const +{ + ANTLR_USE_NAMESPACE(std)vector<unsigned int> elems; + for (unsigned int i = 0; i < storage.size(); i++) + { + if (storage[i]) + elems.push_back(i); + } + + return elems; +} + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +} +#endif diff --git a/lib/antlr/src/CharBuffer.cpp b/lib/antlr/src/CharBuffer.cpp new file mode 100644 index 00000000..a0e11eae --- /dev/null +++ b/lib/antlr/src/CharBuffer.cpp @@ -0,0 +1,52 @@ +/* ANTLR Translator Generator + * Project led by Terence Parr at http://www.jGuru.com + * Software rights: http://www.antlr.org/license.html + * + * $Id$ + */ + +#include "antlr/CharBuffer.hpp" +#include <iostream> + +//#include <ios> + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +namespace antlr { +#endif + +/* RK: Per default istream does not throw exceptions. This can be + * enabled with: + * stream.exceptions(ios_base::badbit|ios_base::failbit|ios_base::eofbit); + * + * We could try catching the bad/fail stuff. But handling eof via this is + * not a good idea. EOF is best handled as a 'normal' character. + * + * So this does not work yet with gcc... Comment it until I get to a platform + * that does.. + */ + +/** Create a character buffer. Enable fail and bad exceptions, if supported + * by platform. */ +CharBuffer::CharBuffer(ANTLR_USE_NAMESPACE(std)istream& input_) +: input(input_) +{ +// input.exceptions(ANTLR_USE_NAMESPACE(std)ios_base::badbit| +// ANTLR_USE_NAMESPACE(std)ios_base::failbit); +} + +/** Get the next character from the stream. May throw CharStreamIOException + * when something bad happens (not EOF) (if supported by platform). + */ +int CharBuffer::getChar() +{ +// try { + return input.get(); +// } +// catch (ANTLR_USE_NAMESPACE(std)ios_base::failure& e) { +// throw CharStreamIOException(e); +// } +} + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +} +#endif diff --git a/lib/antlr/src/CharScanner.cpp b/lib/antlr/src/CharScanner.cpp new file mode 100644 index 00000000..d5b1f753 --- /dev/null +++ b/lib/antlr/src/CharScanner.cpp @@ -0,0 +1,108 @@ +/* ANTLR Translator Generator + * Project led by Terence Parr at http://www.jGuru.com + * Software rights: http://www.antlr.org/license.html + * + * $Id$ + */ + +#include <iostream> + +#include "antlr/CharScanner.hpp" +#include "antlr/CommonToken.hpp" + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +namespace antlr { +#endif +ANTLR_C_USING(exit) + +CharScanner::CharScanner(InputBuffer& cb, bool case_sensitive ) + : saveConsumedInput(true) //, caseSensitiveLiterals(true) + , caseSensitive(case_sensitive) + , literals(CharScannerLiteralsLess(this)) + , inputState(new LexerInputState(cb)) + , commitToPath(false) + , tabsize(8) + , traceDepth(0) +{ + setTokenObjectFactory(&CommonToken::factory); +} + +CharScanner::CharScanner(InputBuffer* cb, bool case_sensitive ) + : saveConsumedInput(true) //, caseSensitiveLiterals(true) + , caseSensitive(case_sensitive) + , literals(CharScannerLiteralsLess(this)) + , inputState(new LexerInputState(cb)) + , commitToPath(false) + , tabsize(8) + , traceDepth(0) +{ + setTokenObjectFactory(&CommonToken::factory); +} + +CharScanner::CharScanner( const LexerSharedInputState& state, bool case_sensitive ) + : saveConsumedInput(true) //, caseSensitiveLiterals(true) + , caseSensitive(case_sensitive) + , literals(CharScannerLiteralsLess(this)) + , inputState(state) + , commitToPath(false) + , tabsize(8) + , traceDepth(0) +{ + setTokenObjectFactory(&CommonToken::factory); +} + +/** Report exception errors caught in nextToken() */ +void CharScanner::reportError(const RecognitionException& ex) +{ + ANTLR_USE_NAMESPACE(std)cerr << ex.toString().c_str() << ANTLR_USE_NAMESPACE(std)endl; +} + +/** Parser error-reporting function can be overridden in subclass */ +void CharScanner::reportError(const ANTLR_USE_NAMESPACE(std)string& s) +{ + if (getFilename() == "") + ANTLR_USE_NAMESPACE(std)cerr << "error: " << s.c_str() << ANTLR_USE_NAMESPACE(std)endl; + else + ANTLR_USE_NAMESPACE(std)cerr << getFilename().c_str() << ": error: " << s.c_str() << ANTLR_USE_NAMESPACE(std)endl; +} + +/** Parser warning-reporting function can be overridden in subclass */ +void CharScanner::reportWarning(const ANTLR_USE_NAMESPACE(std)string& s) +{ + if (getFilename() == "") + ANTLR_USE_NAMESPACE(std)cerr << "warning: " << s.c_str() << ANTLR_USE_NAMESPACE(std)endl; + else + ANTLR_USE_NAMESPACE(std)cerr << getFilename().c_str() << ": warning: " << s.c_str() << ANTLR_USE_NAMESPACE(std)endl; +} + +void CharScanner::traceIndent() +{ + for( int i = 0; i < traceDepth; i++ ) + ANTLR_USE_NAMESPACE(std)cout << " "; +} + +void CharScanner::traceIn(const char* rname) +{ + traceDepth++; + traceIndent(); + ANTLR_USE_NAMESPACE(std)cout << "> lexer " << rname + << "; c==" << LA(1) << ANTLR_USE_NAMESPACE(std)endl; +} + +void CharScanner::traceOut(const char* rname) +{ + traceIndent(); + ANTLR_USE_NAMESPACE(std)cout << "< lexer " << rname + << "; c==" << LA(1) << ANTLR_USE_NAMESPACE(std)endl; + traceDepth--; +} + +#ifndef NO_STATIC_CONSTS +const int CharScanner::NO_CHAR; +const int CharScanner::EOF_CHAR; +#endif + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +} +#endif + diff --git a/lib/antlr/src/CommonAST.cpp b/lib/antlr/src/CommonAST.cpp new file mode 100644 index 00000000..e132c0aa --- /dev/null +++ b/lib/antlr/src/CommonAST.cpp @@ -0,0 +1,49 @@ +/* ANTLR Translator Generator + * Project led by Terence Parr at http://www.jGuru.com + * Software rights: http://www.antlr.org/license.html + * + * $Id$ + */ +#include "antlr/config.hpp" + +#include <cstdlib> +#include <iostream> + +#include "antlr/CommonAST.hpp" +#include "antlr/ANTLRUtil.hpp" + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +namespace antlr { +#endif + +const char* const CommonAST::TYPE_NAME = "CommonAST"; + +#ifdef ANTLR_SUPPORT_XML +void CommonAST::initialize( ANTLR_USE_NAMESPACE(std)istream& in ) +{ + ANTLR_USE_NAMESPACE(std)string t1, t2, text; + + // text + read_AttributeNValue( in, t1, text ); + + read_AttributeNValue( in, t1, t2 ); +#ifdef ANTLR_ATOI_IN_STD + int type = ANTLR_USE_NAMESPACE(std)atoi(t2.c_str()); +#else + int type = atoi(t2.c_str()); +#endif + + // initialize first part of AST. + this->initialize( type, text ); +} +#endif + +RefAST CommonAST::factory() +{ + return RefAST(new CommonAST); +} + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +} +#endif + diff --git a/lib/antlr/src/CommonASTWithHiddenTokens.cpp b/lib/antlr/src/CommonASTWithHiddenTokens.cpp new file mode 100644 index 00000000..b0e1a3dd --- /dev/null +++ b/lib/antlr/src/CommonASTWithHiddenTokens.cpp @@ -0,0 +1,64 @@ +/* ANTLR Translator Generator + * Project led by Terence Parr at http://www.jGuru.com + * Software rights: http://www.antlr.org/license.html + * + * $Id$ + */ +#include "antlr/config.hpp" +#include "antlr/AST.hpp" +#include "antlr/BaseAST.hpp" +#include "antlr/CommonAST.hpp" +#include "antlr/CommonASTWithHiddenTokens.hpp" +#include "antlr/CommonHiddenStreamToken.hpp" + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +namespace antlr { +#endif + +const char* const CommonASTWithHiddenTokens::TYPE_NAME = "CommonASTWithHiddenTokens"; +// RK: Do not put constructor and destructor into the header file here.. +// this triggers something very obscure in gcc 2.95.3 (and 3.0) +// missing vtables and stuff. +// Although this may be a problem with with binutils. +CommonASTWithHiddenTokens::CommonASTWithHiddenTokens() +: CommonAST() +{ +} + +CommonASTWithHiddenTokens::~CommonASTWithHiddenTokens() +{ +} + +void CommonASTWithHiddenTokens::initialize(int t,const ANTLR_USE_NAMESPACE(std)string& txt) +{ + CommonAST::initialize(t,txt); +} + +void CommonASTWithHiddenTokens::initialize(RefAST t) +{ + CommonAST::initialize(t); + hiddenBefore = RefCommonASTWithHiddenTokens(t)->getHiddenBefore(); + hiddenAfter = RefCommonASTWithHiddenTokens(t)->getHiddenAfter(); +} + +void CommonASTWithHiddenTokens::initialize(RefToken t) +{ + CommonAST::initialize(t); + hiddenBefore = static_cast<CommonHiddenStreamToken*>(t.get())->getHiddenBefore(); + hiddenAfter = static_cast<CommonHiddenStreamToken*>(t.get())->getHiddenAfter(); +} + +RefAST CommonASTWithHiddenTokens::factory() +{ + return RefAST(new CommonASTWithHiddenTokens); +} + +RefAST CommonASTWithHiddenTokens::clone( void ) const +{ + CommonASTWithHiddenTokens *ast = new CommonASTWithHiddenTokens( *this ); + return RefAST(ast); +} + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +} +#endif diff --git a/lib/antlr/src/CommonHiddenStreamToken.cpp b/lib/antlr/src/CommonHiddenStreamToken.cpp new file mode 100644 index 00000000..9396a43b --- /dev/null +++ b/lib/antlr/src/CommonHiddenStreamToken.cpp @@ -0,0 +1,56 @@ +/* ANTLR Translator Generator + * Project led by Terence Parr at http://www.jGuru.com + * Software rights: http://www.antlr.org/license.html + * + * $Id$ + */ +#include "antlr/CommonHiddenStreamToken.hpp" + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +namespace antlr { +#endif + +CommonHiddenStreamToken::CommonHiddenStreamToken() +: CommonToken() +{ +} + +CommonHiddenStreamToken::CommonHiddenStreamToken(int t, const ANTLR_USE_NAMESPACE(std)string& txt) +: CommonToken(t,txt) +{ +} + +CommonHiddenStreamToken::CommonHiddenStreamToken(const ANTLR_USE_NAMESPACE(std)string& s) +: CommonToken(s) +{ +} + +RefToken CommonHiddenStreamToken::getHiddenAfter() +{ + return hiddenAfter; +} + +RefToken CommonHiddenStreamToken::getHiddenBefore() +{ + return hiddenBefore; +} + +RefToken CommonHiddenStreamToken::factory() +{ + return RefToken(new CommonHiddenStreamToken); +} + +void CommonHiddenStreamToken::setHiddenAfter(RefToken t) +{ + hiddenAfter = t; +} + +void CommonHiddenStreamToken::setHiddenBefore(RefToken t) +{ + hiddenBefore = t; +} + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +} +#endif + diff --git a/lib/antlr/src/CommonToken.cpp b/lib/antlr/src/CommonToken.cpp new file mode 100644 index 00000000..a223b239 --- /dev/null +++ b/lib/antlr/src/CommonToken.cpp @@ -0,0 +1,45 @@ +/* ANTLR Translator Generator + * Project led by Terence Parr at http://www.jGuru.com + * Software rights: http://www.antlr.org/license.html + * + * $Id$ + */ + +#include "antlr/CommonToken.hpp" +#include "antlr/String.hpp" + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +namespace antlr { +#endif + +CommonToken::CommonToken() : Token(), line(1), col(1), text("") +{} + +CommonToken::CommonToken(int t, const ANTLR_USE_NAMESPACE(std)string& txt) +: Token(t) +, line(1) +, col(1) +, text(txt) +{} + +CommonToken::CommonToken(const ANTLR_USE_NAMESPACE(std)string& s) +: Token() +, line(1) +, col(1) +, text(s) +{} + +ANTLR_USE_NAMESPACE(std)string CommonToken::toString() const +{ + return "[\""+getText()+"\",<"+getType()+">,line="+getLine()+",column="+getColumn()+"]"; +} + +RefToken CommonToken::factory() +{ + return RefToken(new CommonToken); +} + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +} +#endif + diff --git a/lib/antlr/src/InputBuffer.cpp b/lib/antlr/src/InputBuffer.cpp new file mode 100644 index 00000000..c9eced7d --- /dev/null +++ b/lib/antlr/src/InputBuffer.cpp @@ -0,0 +1,81 @@ +/* ANTLR Translator Generator + * Project led by Terence Parr at http://www.jGuru.com + * Software rights: http://www.antlr.org/license.html + * + * $Id$ + */ + +#include "antlr/config.hpp" +#include "antlr/InputBuffer.hpp" + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +namespace antlr { +#endif + +/** Ensure that the character buffer is sufficiently full */ +void InputBuffer::fill(unsigned int amount) +{ + syncConsume(); + // Fill the buffer sufficiently to hold needed characters + while (queue.entries() < amount + markerOffset) + { + // Append the next character + queue.append(getChar()); + } +} + +/** get the current lookahead characters as a string + * @warning it may treat 0 and EOF values wrong + */ +ANTLR_USE_NAMESPACE(std)string InputBuffer::getLAChars( void ) const +{ + ANTLR_USE_NAMESPACE(std)string ret; + + for(unsigned int i = markerOffset; i < queue.entries(); i++) + ret += queue.elementAt(i); + + return ret; +} + +/** get the current marked characters as a string + * @warning it may treat 0 and EOF values wrong + */ +ANTLR_USE_NAMESPACE(std)string InputBuffer::getMarkedChars( void ) const +{ + ANTLR_USE_NAMESPACE(std)string ret; + + for(unsigned int i = 0; i < markerOffset; i++) + ret += queue.elementAt(i); + + return ret; +} + +/** Return an integer marker that can be used to rewind the buffer to + * its current state. + */ +unsigned int InputBuffer::mark() +{ + syncConsume(); + nMarkers++; + return markerOffset; +} + +/** Rewind the character buffer to a marker. + * @param mark Marker returned previously from mark() + */ +void InputBuffer::rewind(unsigned int mark) +{ + syncConsume(); + markerOffset = mark; + nMarkers--; +} + +unsigned int InputBuffer::entries() const +{ + //assert(queue.entries() >= markerOffset); + return queue.entries() - markerOffset; +} + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +} +#endif diff --git a/lib/antlr/src/LLkParser.cpp b/lib/antlr/src/LLkParser.cpp new file mode 100644 index 00000000..ab0d672a --- /dev/null +++ b/lib/antlr/src/LLkParser.cpp @@ -0,0 +1,85 @@ +/* ANTLR Translator Generator + * Project led by Terence Parr at http://www.jGuru.com + * Software rights: http://www.antlr.org/license.html + * + * $Id$ + */ + +#include "antlr/LLkParser.hpp" +#include <iostream> + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +namespace antlr { +#endif + +ANTLR_USING_NAMESPACE(std) + +/**An LL(k) parser. + * + * @see antlr.Token + * @see antlr.TokenBuffer + * @see antlr.LL1Parser + */ + +// LLkParser(int k_); + +LLkParser::LLkParser(const ParserSharedInputState& state, int k_) +: Parser(state), k(k_) +{ +} + +LLkParser::LLkParser(TokenBuffer& tokenBuf, int k_) +: Parser(tokenBuf), k(k_) +{ +} + +LLkParser::LLkParser(TokenStream& lexer, int k_) +: Parser(new TokenBuffer(lexer)), k(k_) +{ +} + +void LLkParser::trace(const char* ee, const char* rname) +{ + traceIndent(); + + cout << ee << rname << ((inputState->guessing>0)?"; [guessing]":"; "); + + for (int i = 1; i <= k; i++) + { + if (i != 1) { + cout << ", "; + } + cout << "LA(" << i << ")=="; + + string temp; + + try { + temp = LT(i)->getText().c_str(); + } + catch( ANTLRException& ae ) + { + temp = "[error: "; + temp += ae.toString(); + temp += ']'; + } + cout << temp; + } + + cout << endl; +} + +void LLkParser::traceIn(const char* rname) +{ + traceDepth++; + trace("> ",rname); +} + +void LLkParser::traceOut(const char* rname) +{ + trace("< ",rname); + traceDepth--; +} + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +} +#endif diff --git a/lib/antlr/src/Makefile.am b/lib/antlr/src/Makefile.am new file mode 100644 index 00000000..b1af9fca --- /dev/null +++ b/lib/antlr/src/Makefile.am @@ -0,0 +1,10 @@ + +# Make #include <antlr/xxx> work.. +INCLUDES = -I$(srcdir)/.. +KDE_CXXFLAGS = $(USE_EXCEPTIONS) + +noinst_LTLIBRARIES = libantlr.la + +libantlr_la_LDFLAGS = -no-undefined + +libantlr_la_SOURCES = ANTLRUtil.cpp ASTFactory.cpp ASTNULLType.cpp ASTRefCount.cpp BaseAST.cpp BitSet.cpp CharBuffer.cpp CharScanner.cpp CommonAST.cpp CommonASTWithHiddenTokens.cpp CommonHiddenStreamToken.cpp CommonToken.cpp InputBuffer.cpp LLkParser.cpp MismatchedCharException.cpp MismatchedTokenException.cpp NoViableAltException.cpp NoViableAltForCharException.cpp Parser.cpp RecognitionException.cpp String.cpp Token.cpp TokenBuffer.cpp TokenRefCount.cpp TokenStreamBasicFilter.cpp TokenStreamHiddenTokenFilter.cpp TokenStreamRewriteEngine.cpp TokenStreamSelector.cpp TreeParser.cpp diff --git a/lib/antlr/src/MismatchedCharException.cpp b/lib/antlr/src/MismatchedCharException.cpp new file mode 100644 index 00000000..e73dbf36 --- /dev/null +++ b/lib/antlr/src/MismatchedCharException.cpp @@ -0,0 +1,120 @@ +/* ANTLR Translator Generator + * Project led by Terence Parr at http://www.jGuru.com + * Software rights: http://www.antlr.org/license.html + * + * $Id$ + */ + +#include "antlr/CharScanner.hpp" +#include "antlr/MismatchedCharException.hpp" +#include "antlr/String.hpp" + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +namespace antlr { +#endif + +MismatchedCharException::MismatchedCharException() + : RecognitionException("Mismatched char") +{} + +// Expected range / not range +MismatchedCharException::MismatchedCharException( + int c, + int lower, + int upper_, + bool matchNot, + CharScanner* scanner_ +) : RecognitionException("Mismatched char", + scanner_->getFilename(), + scanner_->getLine(), scanner_->getColumn()) + , mismatchType(matchNot ? NOT_RANGE : RANGE) + , foundChar(c) + , expecting(lower) + , upper(upper_) + , scanner(scanner_) +{ +} + +// Expected token / not token +MismatchedCharException::MismatchedCharException( + int c, + int expecting_, + bool matchNot, + CharScanner* scanner_ +) : RecognitionException("Mismatched char", + scanner_->getFilename(), + scanner_->getLine(), scanner_->getColumn()) + , mismatchType(matchNot ? NOT_CHAR : CHAR) + , foundChar(c) + , expecting(expecting_) + , scanner(scanner_) +{ +} + +// Expected BitSet / not BitSet +MismatchedCharException::MismatchedCharException( + int c, + BitSet set_, + bool matchNot, + CharScanner* scanner_ +) : RecognitionException("Mismatched char", + scanner_->getFilename(), + scanner_->getLine(), scanner_->getColumn()) + , mismatchType(matchNot ? NOT_SET : SET) + , foundChar(c) + , set(set_) + , scanner(scanner_) +{ +} + +ANTLR_USE_NAMESPACE(std)string MismatchedCharException::getMessage() const +{ + ANTLR_USE_NAMESPACE(std)string s; + + switch (mismatchType) { + case CHAR : + s += "expecting '" + charName(expecting) + "', found '" + charName(foundChar) + "'"; + break; + case NOT_CHAR : + s += "expecting anything but '" + charName(expecting) + "'; got it anyway"; + break; + case RANGE : + s += "expecting token in range: '" + charName(expecting) + "'..'" + charName(upper) + "', found '" + charName(foundChar) + "'"; + break; + case NOT_RANGE : + s += "expecting token NOT in range: " + charName(expecting) + "'..'" + charName(upper) + "', found '" + charName(foundChar) + "'"; + break; + case SET : + case NOT_SET : + { + s += ANTLR_USE_NAMESPACE(std)string("expecting ") + (mismatchType == NOT_SET ? "NOT " : "") + "one of ("; + ANTLR_USE_NAMESPACE(std)vector<unsigned int> elems = set.toArray(); + for ( unsigned int i = 0; i < elems.size(); i++ ) + { + s += " '"; + s += charName(elems[i]); + s += "'"; + } + s += "), found '" + charName(foundChar) + "'"; + } + break; + default : + s += RecognitionException::getMessage(); + break; + } + + return s; +} + +#ifndef NO_STATIC_CONSTS +const int MismatchedCharException::CHAR; +const int MismatchedCharException::NOT_CHAR; +const int MismatchedCharException::RANGE; +const int MismatchedCharException::NOT_RANGE; +const int MismatchedCharException::SET; +const int MismatchedCharException::NOT_SET; +#endif + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +} +#endif diff --git a/lib/antlr/src/MismatchedTokenException.cpp b/lib/antlr/src/MismatchedTokenException.cpp new file mode 100644 index 00000000..7fc349f7 --- /dev/null +++ b/lib/antlr/src/MismatchedTokenException.cpp @@ -0,0 +1,196 @@ +/* ANTLR Translator Generator + * Project led by Terence Parr at http://www.jGuru.com + * Software rights: http://www.antlr.org/license.html + * + * $Id$ + */ + +#include "antlr/MismatchedTokenException.hpp" +#include "antlr/String.hpp" + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +namespace antlr { +#endif + +MismatchedTokenException::MismatchedTokenException() + : RecognitionException("Mismatched Token: expecting any AST node","<AST>",-1,-1) + , token(0) + , node(nullASTptr) + , tokenNames(0) + , numTokens(0) +{ +} + +// Expected range / not range +MismatchedTokenException::MismatchedTokenException( + const char* const* tokenNames_, + const int numTokens_, + RefAST node_, + int lower, + int upper_, + bool matchNot +) : RecognitionException("Mismatched Token","<AST>",-1,-1) + , token(0) + , node(node_) + , tokenText( (node_ ? node_->toString(): ANTLR_USE_NAMESPACE(std)string("<empty tree>")) ) + , mismatchType(matchNot ? NOT_RANGE : RANGE) + , expecting(lower) + , upper(upper_) + , tokenNames(tokenNames_) + , numTokens(numTokens_) +{ +} + +// Expected token / not token +MismatchedTokenException::MismatchedTokenException( + const char* const* tokenNames_, + const int numTokens_, + RefAST node_, + int expecting_, + bool matchNot +) : RecognitionException("Mismatched Token","<AST>",-1,-1) + , token(0) + , node(node_) + , tokenText( (node_ ? node_->toString(): ANTLR_USE_NAMESPACE(std)string("<empty tree>")) ) + , mismatchType(matchNot ? NOT_TOKEN : TOKEN) + , expecting(expecting_) + , tokenNames(tokenNames_) + , numTokens(numTokens_) +{ +} + +// Expected BitSet / not BitSet +MismatchedTokenException::MismatchedTokenException( + const char* const* tokenNames_, + const int numTokens_, + RefAST node_, + BitSet set_, + bool matchNot +) : RecognitionException("Mismatched Token","<AST>",-1,-1) + , token(0) + , node(node_) + , tokenText( (node_ ? node_->toString(): ANTLR_USE_NAMESPACE(std)string("<empty tree>")) ) + , mismatchType(matchNot ? NOT_SET : SET) + , set(set_) + , tokenNames(tokenNames_) + , numTokens(numTokens_) +{ +} + +// Expected range / not range +MismatchedTokenException::MismatchedTokenException( + const char* const* tokenNames_, + const int numTokens_, + RefToken token_, + int lower, + int upper_, + bool matchNot, + const ANTLR_USE_NAMESPACE(std)string& fileName_ +) : RecognitionException("Mismatched Token",fileName_,token_->getLine(),token_->getColumn()) + , token(token_) + , node(nullASTptr) + , tokenText(token_->getText()) + , mismatchType(matchNot ? NOT_RANGE : RANGE) + , expecting(lower) + , upper(upper_) + , tokenNames(tokenNames_) + , numTokens(numTokens_) +{ +} + +// Expected token / not token +MismatchedTokenException::MismatchedTokenException( + const char* const* tokenNames_, + const int numTokens_, + RefToken token_, + int expecting_, + bool matchNot, + const ANTLR_USE_NAMESPACE(std)string& fileName_ +) : RecognitionException("Mismatched Token",fileName_,token_->getLine(),token_->getColumn()) + , token(token_) + , node(nullASTptr) + , tokenText(token_->getText()) + , mismatchType(matchNot ? NOT_TOKEN : TOKEN) + , expecting(expecting_) + , tokenNames(tokenNames_) + , numTokens(numTokens_) +{ +} + +// Expected BitSet / not BitSet +MismatchedTokenException::MismatchedTokenException( + const char* const* tokenNames_, + const int numTokens_, + RefToken token_, + BitSet set_, + bool matchNot, + const ANTLR_USE_NAMESPACE(std)string& fileName_ +) : RecognitionException("Mismatched Token",fileName_,token_->getLine(),token_->getColumn()) + , token(token_) + , node(nullASTptr) + , tokenText(token_->getText()) + , mismatchType(matchNot ? NOT_SET : SET) + , set(set_) + , tokenNames(tokenNames_) + , numTokens(numTokens_) +{ +} + +ANTLR_USE_NAMESPACE(std)string MismatchedTokenException::getMessage() const +{ + ANTLR_USE_NAMESPACE(std)string s; + switch (mismatchType) { + case TOKEN: + s += "expecting " + tokenName(expecting) + ", found '" + tokenText + "'"; + break; + case NOT_TOKEN: + s += "expecting anything but " + tokenName(expecting) + "; got it anyway"; + break; + case RANGE: + s += "expecting token in range: " + tokenName(expecting) + ".." + tokenName(upper) + ", found '" + tokenText + "'"; + break; + case NOT_RANGE: + s += "expecting token NOT in range: " + tokenName(expecting) + ".." + tokenName(upper) + ", found '" + tokenText + "'"; + break; + case SET: + case NOT_SET: + { + s += ANTLR_USE_NAMESPACE(std)string("expecting ") + (mismatchType == NOT_SET ? "NOT " : "") + "one of ("; + ANTLR_USE_NAMESPACE(std)vector<unsigned int> elems = set.toArray(); + for ( unsigned int i = 0; i < elems.size(); i++ ) + { + s += " "; + s += tokenName(elems[i]); + } + s += "), found '" + tokenText + "'"; + } + break; + default: + s = RecognitionException::getMessage(); + break; + } + return s; +} + +ANTLR_USE_NAMESPACE(std)string MismatchedTokenException::tokenName(int tokenType) const +{ + if (tokenType == Token::INVALID_TYPE) + return "<Set of tokens>"; + else if (tokenType < 0 || tokenType >= numTokens) + return ANTLR_USE_NAMESPACE(std)string("<") + tokenType + ">"; + else + return tokenNames[tokenType]; +} + +#ifndef NO_STATIC_CONSTS +const int MismatchedTokenException::TOKEN; +const int MismatchedTokenException::NOT_TOKEN; +const int MismatchedTokenException::RANGE; +const int MismatchedTokenException::NOT_RANGE; +const int MismatchedTokenException::SET; +const int MismatchedTokenException::NOT_SET; +#endif + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +} +#endif diff --git a/lib/antlr/src/NoViableAltException.cpp b/lib/antlr/src/NoViableAltException.cpp new file mode 100644 index 00000000..94da2cc4 --- /dev/null +++ b/lib/antlr/src/NoViableAltException.cpp @@ -0,0 +1,52 @@ +/* ANTLR Translator Generator + * Project led by Terence Parr at http://www.jGuru.com + * Software rights: http://www.antlr.org/license.html + * + * $Id$ + */ + +#include "antlr/NoViableAltException.hpp" +#include "antlr/String.hpp" + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +namespace antlr { +#endif + +ANTLR_USING_NAMESPACE(std) + +NoViableAltException::NoViableAltException(RefAST t) + : RecognitionException("NoViableAlt","<AST>",-1,-1), + token(0), node(t) +{ +} + +NoViableAltException::NoViableAltException( + RefToken t, + const ANTLR_USE_NAMESPACE(std)string& fileName_ +) : RecognitionException("NoViableAlt",fileName_,t->getLine(),t->getColumn()), + token(t), node(nullASTptr) +{ +} + +ANTLR_USE_NAMESPACE(std)string NoViableAltException::getMessage() const +{ + if (token) + { + if( token->getType() == Token::EOF_TYPE ) + return string("unexpected end of file"); + else if( token->getType() == Token::NULL_TREE_LOOKAHEAD ) + return string("unexpected end of tree"); + else + return string("unexpected token: ")+token->getText(); + } + + // must a tree parser error if token==null + if (!node) + return "unexpected end of subtree"; + + return string("unexpected AST node: ")+node->toString(); +} + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +} +#endif diff --git a/lib/antlr/src/NoViableAltForCharException.cpp b/lib/antlr/src/NoViableAltForCharException.cpp new file mode 100644 index 00000000..10d9447a --- /dev/null +++ b/lib/antlr/src/NoViableAltForCharException.cpp @@ -0,0 +1,39 @@ +/* ANTLR Translator Generator + * Project led by Terence Parr at http://www.jGuru.com + * Software rights: http://www.antlr.org/license.html + * + * $Id$ + */ + +#include "antlr/NoViableAltForCharException.hpp" +#include "antlr/String.hpp" + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +namespace antlr { +#endif + +NoViableAltForCharException::NoViableAltForCharException(int c, CharScanner* scanner) + : RecognitionException("NoViableAlt", + scanner->getFilename(), + scanner->getLine(),scanner->getColumn()), + foundChar(c) +{ +} + +NoViableAltForCharException::NoViableAltForCharException( + int c, + const ANTLR_USE_NAMESPACE(std)string& fileName_, + int line_, int column_) + : RecognitionException("NoViableAlt",fileName_,line_,column_), + foundChar(c) +{ +} + +ANTLR_USE_NAMESPACE(std)string NoViableAltForCharException::getMessage() const +{ + return ANTLR_USE_NAMESPACE(std)string("unexpected char: ")+charName(foundChar); +} + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +} +#endif diff --git a/lib/antlr/src/Parser.cpp b/lib/antlr/src/Parser.cpp new file mode 100644 index 00000000..640da7b6 --- /dev/null +++ b/lib/antlr/src/Parser.cpp @@ -0,0 +1,113 @@ +/* ANTLR Translator Generator + * Project led by Terence Parr at http://www.jGuru.com + * Software rights: http://www.antlr.org/license.html + * + * $Id$ + */ + +#include "antlr/Parser.hpp" + +#include <iostream> + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +namespace antlr { +#endif + +/** A generic ANTLR parser (LL(k) for k>=1) containing a bunch of + * utility routines useful at any lookahead depth. We distinguish between + * the LL(1) and LL(k) parsers because of efficiency. This may not be + * necessary in the near future. + * + * Each parser object contains the state of the parse including a lookahead + * cache (the form of which is determined by the subclass), whether or + * not the parser is in guess mode, where tokens come from, etc... + * + * <p> + * During <b>guess</b> mode, the current lookahead token(s) and token type(s) + * cache must be saved because the token stream may not have been informed + * to save the token (via <tt>mark</tt>) before the <tt>try</tt> block. + * Guessing is started by: + * <ol> + * <li>saving the lookahead cache. + * <li>marking the current position in the TokenBuffer. + * <li>increasing the guessing level. + * </ol> + * + * After guessing, the parser state is restored by: + * <ol> + * <li>restoring the lookahead cache. + * <li>rewinding the TokenBuffer. + * <li>decreasing the guessing level. + * </ol> + * + * @see antlr.Token + * @see antlr.TokenBuffer + * @see antlr.TokenStream + * @see antlr.LL1Parser + * @see antlr.LLkParser + */ + +bool DEBUG_PARSER = false; + +/** Parser error-reporting function can be overridden in subclass */ +void Parser::reportError(const RecognitionException& ex) +{ + ANTLR_USE_NAMESPACE(std)cerr << ex.toString().c_str() << ANTLR_USE_NAMESPACE(std)endl; +} + +/** Parser error-reporting function can be overridden in subclass */ +void Parser::reportError(const ANTLR_USE_NAMESPACE(std)string& s) +{ + if ( getFilename()=="" ) + ANTLR_USE_NAMESPACE(std)cerr << "error: " << s.c_str() << ANTLR_USE_NAMESPACE(std)endl; + else + ANTLR_USE_NAMESPACE(std)cerr << getFilename().c_str() << ": error: " << s.c_str() << ANTLR_USE_NAMESPACE(std)endl; +} + +/** Parser warning-reporting function can be overridden in subclass */ +void Parser::reportWarning(const ANTLR_USE_NAMESPACE(std)string& s) +{ + if ( getFilename()=="" ) + ANTLR_USE_NAMESPACE(std)cerr << "warning: " << s.c_str() << ANTLR_USE_NAMESPACE(std)endl; + else + ANTLR_USE_NAMESPACE(std)cerr << getFilename().c_str() << ": warning: " << s.c_str() << ANTLR_USE_NAMESPACE(std)endl; +} + +/** Set or change the input token buffer */ +// void setTokenBuffer(TokenBuffer<Token>* t); + +void Parser::traceIndent() +{ + for( int i = 0; i < traceDepth; i++ ) + ANTLR_USE_NAMESPACE(std)cout << " "; +} + +void Parser::traceIn(const char* rname) +{ + traceDepth++; + + for( int i = 0; i < traceDepth; i++ ) + ANTLR_USE_NAMESPACE(std)cout << " "; + + ANTLR_USE_NAMESPACE(std)cout << "> " << rname + << "; LA(1)==" << LT(1)->getText().c_str() + << ((inputState->guessing>0)?" [guessing]":"") + << ANTLR_USE_NAMESPACE(std)endl; +} + +void Parser::traceOut(const char* rname) +{ + for( int i = 0; i < traceDepth; i++ ) + ANTLR_USE_NAMESPACE(std)cout << " "; + + ANTLR_USE_NAMESPACE(std)cout << "< " << rname + << "; LA(1)==" << LT(1)->getText().c_str() + << ((inputState->guessing>0)?" [guessing]":"") + << ANTLR_USE_NAMESPACE(std)endl; + + traceDepth--; +} + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +} +#endif diff --git a/lib/antlr/src/RecognitionException.cpp b/lib/antlr/src/RecognitionException.cpp new file mode 100644 index 00000000..9c185ccc --- /dev/null +++ b/lib/antlr/src/RecognitionException.cpp @@ -0,0 +1,71 @@ +/* ANTLR Translator Generator + * Project led by Terence Parr at http://www.jGuru.com + * Software rights: http://www.antlr.org/license.html + * + * $Id$ + */ + +#include "antlr/RecognitionException.hpp" +#include "antlr/String.hpp" + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +namespace antlr { +#endif + +RecognitionException::RecognitionException() +: ANTLRException("parsing error") +, line(-1) +, column(-1) +{ +} + +RecognitionException::RecognitionException(const ANTLR_USE_NAMESPACE(std)string& s) +: ANTLRException(s) +, line(-1) +, column(-1) +{ +} + +RecognitionException::RecognitionException(const ANTLR_USE_NAMESPACE(std)string& s, + const ANTLR_USE_NAMESPACE(std)string& fileName_, + int line_,int column_) +: ANTLRException(s) +, fileName(fileName_) +, line(line_) +, column(column_) +{ +} + +ANTLR_USE_NAMESPACE(std)string RecognitionException::getFileLineColumnString() const +{ + ANTLR_USE_NAMESPACE(std)string fileLineColumnString; + + if ( fileName.length() > 0 ) + fileLineColumnString = fileName + ":"; + + if ( line != -1 ) + { + if ( fileName.length() == 0 ) + fileLineColumnString = fileLineColumnString + "line "; + + fileLineColumnString = fileLineColumnString + line; + + if ( column != -1 ) + fileLineColumnString = fileLineColumnString + ":" + column; + + fileLineColumnString = fileLineColumnString + ":"; + } + + fileLineColumnString = fileLineColumnString + " "; + + return fileLineColumnString; +} + +ANTLR_USE_NAMESPACE(std)string RecognitionException::toString() const +{ + return getFileLineColumnString()+getMessage(); +} + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +} +#endif diff --git a/lib/antlr/src/String.cpp b/lib/antlr/src/String.cpp new file mode 100644 index 00000000..ae70f479 --- /dev/null +++ b/lib/antlr/src/String.cpp @@ -0,0 +1,90 @@ +/* ANTLR Translator Generator + * Project led by Terence Parr at http://www.jGuru.com + * Software rights: http://www.antlr.org/license.html + * + * $Id$ + */ + +#include "antlr/String.hpp" + +#include <cctype> + +#ifdef HAS_NOT_CSTDIO_H +#include <stdio.h> +#else +#include <cstdio> +#endif + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +namespace antlr { +#endif + +// wh: hack for Borland C++ 5.6 +#if __BORLANDC__ + using std::sprintf; +#endif + + +// RK: should be using snprintf actually... (or stringstream) +ANTLR_C_USING(sprintf) + +ANTLR_USE_NAMESPACE(std)string operator+( const ANTLR_USE_NAMESPACE(std)string& lhs, const int rhs ) +{ + char tmp[100]; + sprintf(tmp,"%d",rhs); + return lhs+tmp; +} + +ANTLR_USE_NAMESPACE(std)string operator+( const ANTLR_USE_NAMESPACE(std)string& lhs, size_t rhs ) +{ + char tmp[100]; + sprintf(tmp,"%u",rhs); + return lhs+tmp; +} + +/** Convert character to readable string + */ +ANTLR_USE_NAMESPACE(std)string charName(int ch) +{ + if (ch == EOF) + return "EOF"; + else + { + ANTLR_USE_NAMESPACE(std)string s; + + // when you think you've seen it all.. an isprint that crashes... + ch = ch & 0xFF; +#ifdef ANTLR_CCTYPE_NEEDS_STD + if( ANTLR_USE_NAMESPACE(std)isprint( ch ) ) +#else + if( isprint( ch ) ) +#endif + { + s.append("'"); + s += ch; + s.append("'"); +// s += "'"+ch+"'"; + } + else + { + s += "0x"; + + unsigned int t = ch >> 4; + if( t < 10 ) + s += t | 0x30; + else + s += t + 0x37; + t = ch & 0xF; + if( t < 10 ) + s += t | 0x30; + else + s += t + 0x37; + } + return s; + } +} + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +} +#endif + diff --git a/lib/antlr/src/Token.cpp b/lib/antlr/src/Token.cpp new file mode 100644 index 00000000..f8181634 --- /dev/null +++ b/lib/antlr/src/Token.cpp @@ -0,0 +1,80 @@ +/* ANTLR Translator Generator + * Project led by Terence Parr at http://www.jGuru.com + * Software rights: http://www.antlr.org/license.html + * + * $Id$ + */ + +#include "antlr/Token.hpp" +#include "antlr/String.hpp" + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +namespace antlr { +#endif + +int Token::getColumn() const +{ + return 0; +} + +int Token::getLine() const +{ + return 0; +} + +ANTLR_USE_NAMESPACE(std)string Token::getText() const +{ + return "<no text>"; +} + +int Token::getType() const +{ + return type; +} + +void Token::setColumn(int) +{ +} + +void Token::setLine(int) +{ +} + +void Token::setText(const ANTLR_USE_NAMESPACE(std)string&) +{ +} + +void Token::setType(int t) +{ + type = t; +} + +void Token::setFilename(const ANTLR_USE_NAMESPACE(std)string&) +{ +} + +ANTLR_USE_NAMESPACE(std)string emptyString(""); + +const ANTLR_USE_NAMESPACE(std)string& Token::getFilename() const +{ + return emptyString; +} + +ANTLR_USE_NAMESPACE(std)string Token::toString() const +{ + return "[\""+getText()+"\",<"+type+">]"; +} + +ANTLR_API RefToken nullToken; + +#ifndef NO_STATIC_CONSTS +const int Token::MIN_USER_TYPE; +const int Token::NULL_TREE_LOOKAHEAD; +const int Token::INVALID_TYPE; +const int Token::EOF_TYPE; +const int Token::SKIP; +#endif + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +} +#endif diff --git a/lib/antlr/src/TokenBuffer.cpp b/lib/antlr/src/TokenBuffer.cpp new file mode 100644 index 00000000..ed69d6e0 --- /dev/null +++ b/lib/antlr/src/TokenBuffer.cpp @@ -0,0 +1,96 @@ +/* ANTLR Translator Generator + * Project led by Terence Parr at http://www.jGuru.com + * Software rights: http://www.antlr.org/license.html + * + * $Id$ + */ + +#include "antlr/TokenBuffer.hpp" + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +namespace antlr { +#endif + +/**A Stream of Token objects fed to the parser from a TokenStream that can + * be rewound via mark()/rewind() methods. + * <p> + * A dynamic array is used to buffer up all the input tokens. Normally, + * "k" tokens are stored in the buffer. More tokens may be stored during + * guess mode (testing syntactic predicate), or when LT(i>k) is referenced. + * Consumption of tokens is deferred. In other words, reading the next + * token is not done by conume(), but deferred until needed by LA or LT. + * <p> + * + * @see antlr.Token + * @see antlr.TokenStream + * @see antlr.TokenQueue + */ + +/** Create a token buffer */ +TokenBuffer::TokenBuffer( TokenStream& inp ) +: input(inp) +, nMarkers(0) +, markerOffset(0) +, numToConsume(0) +{ +} + +TokenBuffer::~TokenBuffer( void ) +{ +} + +/** Ensure that the token buffer is sufficiently full */ +void TokenBuffer::fill(unsigned int amount) +{ + syncConsume(); + // Fill the buffer sufficiently to hold needed tokens + while (queue.entries() < (amount + markerOffset)) + { + // Append the next token + queue.append(input.nextToken()); + } +} + +/** Get a lookahead token value */ +int TokenBuffer::LA(unsigned int i) +{ + fill(i); + return queue.elementAt(markerOffset+i-1)->getType(); +} + +/** Get a lookahead token */ +RefToken TokenBuffer::LT(unsigned int i) +{ + fill(i); + return queue.elementAt(markerOffset+i-1); +} + +/** Return an integer marker that can be used to rewind the buffer to + * its current state. + */ +unsigned int TokenBuffer::mark() +{ + syncConsume(); + nMarkers++; + return markerOffset; +} + +/**Rewind the token buffer to a marker. + * @param mark Marker returned previously from mark() + */ +void TokenBuffer::rewind(unsigned int mark) +{ + syncConsume(); + markerOffset=mark; + nMarkers--; +} + +/// Get number of non-consumed tokens +unsigned int TokenBuffer::entries() const +{ + return queue.entries() - markerOffset; +} + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE + } +#endif diff --git a/lib/antlr/src/TokenRefCount.cpp b/lib/antlr/src/TokenRefCount.cpp new file mode 100644 index 00000000..0afb0f84 --- /dev/null +++ b/lib/antlr/src/TokenRefCount.cpp @@ -0,0 +1,41 @@ +/* ANTLR Translator Generator + * Project led by Terence Parr at http://www.jGuru.com + * Software rights: http://www.antlr.org/license.html + * + * $Id:$ + */ +#include "antlr/TokenRefCount.hpp" +#include "antlr/Token.hpp" + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +namespace antlr { +#endif + +TokenRef::TokenRef(Token* p) +: ptr(p), count(1) +{ + if (p && !p->ref) + p->ref = this; +} + +TokenRef::~TokenRef() +{ + delete ptr; +} + +TokenRef* TokenRef::getRef(const Token* p) +{ + if (p) { + Token* pp = const_cast<Token*>(p); + if (pp->ref) + return pp->ref->increment(); + else + return new TokenRef(pp); + } else + return 0; +} + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +} +#endif + diff --git a/lib/antlr/src/TokenStreamBasicFilter.cpp b/lib/antlr/src/TokenStreamBasicFilter.cpp new file mode 100644 index 00000000..982e8645 --- /dev/null +++ b/lib/antlr/src/TokenStreamBasicFilter.cpp @@ -0,0 +1,44 @@ +/* ANTLR Translator Generator + * Project led by Terence Parr at http://www.jGuru.com + * Software rights: http://www.antlr.org/license.html + * + * $Id$ + */ +#include "antlr/TokenStreamBasicFilter.hpp" + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +namespace antlr { +#endif + +/** This object is a TokenStream that passes through all + * tokens except for those that you tell it to discard. + * There is no buffering of the tokens. + */ +TokenStreamBasicFilter::TokenStreamBasicFilter(TokenStream& input_) +: input(&input_) +{ +} + +void TokenStreamBasicFilter::discard(int ttype) +{ + discardMask.add(ttype); +} + +void TokenStreamBasicFilter::discard(const BitSet& mask) +{ + discardMask = mask; +} + +RefToken TokenStreamBasicFilter::nextToken() +{ + RefToken tok = input->nextToken(); + while ( tok && discardMask.member(tok->getType()) ) { + tok = input->nextToken(); + } + return tok; +} + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +} +#endif + diff --git a/lib/antlr/src/TokenStreamHiddenTokenFilter.cpp b/lib/antlr/src/TokenStreamHiddenTokenFilter.cpp new file mode 100644 index 00000000..431df0c3 --- /dev/null +++ b/lib/antlr/src/TokenStreamHiddenTokenFilter.cpp @@ -0,0 +1,156 @@ +/* ANTLR Translator Generator + * Project led by Terence Parr at http://www.jGuru.com + * Software rights: http://www.antlr.org/license.html + * + * $Id$ + */ +#include "antlr/TokenStreamHiddenTokenFilter.hpp" +#include "antlr/CommonHiddenStreamToken.hpp" + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +namespace antlr { +#endif + +/**This object filters a token stream coming from a lexer + * or another TokenStream so that only certain token channels + * get transmitted to the parser. + * + * Any of the channels can be filtered off as "hidden" channels whose + * tokens can be accessed from the parser. + */ + +TokenStreamHiddenTokenFilter::TokenStreamHiddenTokenFilter(TokenStream& input) +: TokenStreamBasicFilter(input) +{ +} + +void TokenStreamHiddenTokenFilter::consume() +{ + nextMonitoredToken = input->nextToken(); +} + +void TokenStreamHiddenTokenFilter::consumeFirst() +{ + consume(); + + // Handle situation where hidden or discarded tokens + // appear first in input stream + RefToken p; + // while hidden or discarded scarf tokens + while ( hideMask.member(LA(1)->getType()) || discardMask.member(LA(1)->getType()) ) { + if ( hideMask.member(LA(1)->getType()) ) { + if ( !p ) { + p = LA(1); + } + else { + static_cast<CommonHiddenStreamToken*>(p.get())->setHiddenAfter(LA(1)); + static_cast<CommonHiddenStreamToken*>(LA(1).get())->setHiddenBefore(p); // double-link + p = LA(1); + } + lastHiddenToken = p; + if (!firstHidden) + firstHidden = p; // record hidden token if first + } + consume(); + } +} + +BitSet TokenStreamHiddenTokenFilter::getDiscardMask() const +{ + return discardMask; +} + +/** Return a ptr to the hidden token appearing immediately after + * token t in the input stream. + */ +RefToken TokenStreamHiddenTokenFilter::getHiddenAfter(RefToken t) +{ + return static_cast<CommonHiddenStreamToken*>(t.get())->getHiddenAfter(); +} + +/** Return a ptr to the hidden token appearing immediately before + * token t in the input stream. + */ +RefToken TokenStreamHiddenTokenFilter::getHiddenBefore(RefToken t) +{ + return static_cast<CommonHiddenStreamToken*>(t.get())->getHiddenBefore(); +} + +BitSet TokenStreamHiddenTokenFilter::getHideMask() const +{ + return hideMask; +} + +/** Return the first hidden token if one appears + * before any monitored token. + */ +RefToken TokenStreamHiddenTokenFilter::getInitialHiddenToken() +{ + return firstHidden; +} + +void TokenStreamHiddenTokenFilter::hide(int m) +{ + hideMask.add(m); +} + +void TokenStreamHiddenTokenFilter::hide(const BitSet& mask) +{ + hideMask = mask; +} + +RefToken TokenStreamHiddenTokenFilter::LA(int) +{ + return nextMonitoredToken; +} + +/** Return the next monitored token. +* Test the token following the monitored token. +* If following is another monitored token, save it +* for the next invocation of nextToken (like a single +* lookahead token) and return it then. +* If following is unmonitored, nondiscarded (hidden) +* channel token, add it to the monitored token. +* +* Note: EOF must be a monitored Token. +*/ +RefToken TokenStreamHiddenTokenFilter::nextToken() +{ + // handle an initial condition; don't want to get lookahead + // token of this splitter until first call to nextToken + if ( !LA(1) ) { + consumeFirst(); + } + + // we always consume hidden tokens after monitored, thus, + // upon entry LA(1) is a monitored token. + RefToken monitored = LA(1); + // point to hidden tokens found during last invocation + static_cast<CommonHiddenStreamToken*>(monitored.get())->setHiddenBefore(lastHiddenToken); + lastHiddenToken = nullToken; + + // Look for hidden tokens, hook them into list emanating + // from the monitored tokens. + consume(); + RefToken p = monitored; + // while hidden or discarded scarf tokens + while ( hideMask.member(LA(1)->getType()) || discardMask.member(LA(1)->getType()) ) { + if ( hideMask.member(LA(1)->getType()) ) { + // attach the hidden token to the monitored in a chain + // link forwards + static_cast<CommonHiddenStreamToken*>(p.get())->setHiddenAfter(LA(1)); + // link backwards + if (p != monitored) { //hidden cannot point to monitored tokens + static_cast<CommonHiddenStreamToken*>(LA(1).get())->setHiddenBefore(p); + } + p = lastHiddenToken = LA(1); + } + consume(); + } + return monitored; +} + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +} +#endif + diff --git a/lib/antlr/src/TokenStreamRewriteEngine.cpp b/lib/antlr/src/TokenStreamRewriteEngine.cpp new file mode 100644 index 00000000..2f171eb6 --- /dev/null +++ b/lib/antlr/src/TokenStreamRewriteEngine.cpp @@ -0,0 +1,214 @@ +#include <antlr/config.hpp> + +#include <string> +#include <list> +#include <vector> +#include <map> +#include <utility> +#include <iostream> +#include <iterator> +#include <sstream> +#include <cassert> + +#include <antlr/TokenStream.hpp> +#include <antlr/TokenWithIndex.hpp> +#include <antlr/BitSet.hpp> +#include <antlr/TokenStreamRewriteEngine.hpp> + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +namespace antlr { +#endif + +#ifndef NO_STATIC_CONSTS +const size_t TokenStreamRewriteEngine::MIN_TOKEN_INDEX = 0; +const int TokenStreamRewriteEngine::PROGRAM_INIT_SIZE = 100; +#endif + +const char* TokenStreamRewriteEngine::DEFAULT_PROGRAM_NAME = "default"; + +namespace { + + struct compareOperationIndex { + typedef TokenStreamRewriteEngine::RewriteOperation RewriteOperation; + bool operator() ( const RewriteOperation* a, const RewriteOperation* b ) const + { + return a->getIndex() < b->getIndex(); + } + }; + struct dumpTokenWithIndex { + dumpTokenWithIndex( ANTLR_USE_NAMESPACE(std)ostream& o ) : out(o) {} + void operator() ( const RefTokenWithIndex& t ) { + out << "[txt='" << t->getText() << "' tp=" << t->getType() << " idx=" << t->getIndex() << "]\n"; + } + ANTLR_USE_NAMESPACE(std)ostream& out; + }; +} + +TokenStreamRewriteEngine::TokenStreamRewriteEngine(TokenStream& upstream) +: stream(upstream) +, index(MIN_TOKEN_INDEX) +, tokens() +, programs() +, discardMask() +{ +} + +TokenStreamRewriteEngine::TokenStreamRewriteEngine(TokenStream& upstream, size_t initialSize ) +: stream(upstream) +, index(MIN_TOKEN_INDEX) +, tokens(initialSize) +, programs() +, discardMask() +{ +} + +RefToken TokenStreamRewriteEngine::nextToken( void ) +{ + RefTokenWithIndex t; + // suck tokens until end of stream or we find a non-discarded token + do { + t = RefTokenWithIndex(stream.nextToken()); + if ( t ) + { + t->setIndex(index); // what is t's index in list? + if ( t->getType() != Token::EOF_TYPE ) { + tokens.push_back(t); // track all tokens except EOF + } + index++; // move to next position + } + } while ( t && discardMask.member(t->getType()) ); + return RefToken(t); +} + +void TokenStreamRewriteEngine::rollback( const std::string& programName, + size_t instructionIndex ) +{ + program_map::iterator rewrite = programs.find(programName); + if( rewrite != programs.end() ) + { + operation_list& prog = rewrite->second; + operation_list::iterator + j = prog.begin(), + end = prog.end(); + + std::advance(j,instructionIndex); + if( j != end ) + prog.erase(j, end); + } +} + +void TokenStreamRewriteEngine::originalToStream( std::ostream& out, + size_t start, + size_t end ) const +{ + token_list::const_iterator s = tokens.begin(); + std::advance( s, start ); + token_list::const_iterator e = s; + std::advance( e, end-start ); + std::for_each( s, e, tokenToStream(out) ); +} + +void TokenStreamRewriteEngine::toStream( std::ostream& out, + const std::string& programName, + size_t firstToken, + size_t lastToken ) const +{ + if( tokens.size() == 0 ) + return; + + program_map::const_iterator rewriter = programs.find(programName); + + if ( rewriter == programs.end() ) + return; + + // get the prog and some iterators in it... + const operation_list& prog = rewriter->second; + operation_list::const_iterator + rewriteOpIndex = prog.begin(), + rewriteOpEnd = prog.end(); + + size_t tokenCursor = firstToken; + // make sure we don't run out of the tokens we have... + if( lastToken > (tokens.size() - 1) ) + lastToken = tokens.size() - 1; + + while ( tokenCursor <= lastToken ) + { +// std::cout << "tokenCursor = " << tokenCursor << " first prog index = " << (*rewriteOpIndex)->getIndex() << std::endl; + + if( rewriteOpIndex != rewriteOpEnd ) + { + size_t up_to_here = std::min(lastToken,(*rewriteOpIndex)->getIndex()); + while( tokenCursor < up_to_here ) + out << tokens[tokenCursor++]->getText(); + } + while ( rewriteOpIndex != rewriteOpEnd && + tokenCursor == (*rewriteOpIndex)->getIndex() && + tokenCursor <= lastToken ) + { + tokenCursor = (*rewriteOpIndex)->execute(out); + ++rewriteOpIndex; + } + if( tokenCursor <= lastToken ) + out << tokens[tokenCursor++]->getText(); + } + // std::cout << "Handling tail operations # left = " << std::distance(rewriteOpIndex,rewriteOpEnd) << std::endl; + // now see if there are operations (append) beyond last token index + std::for_each( rewriteOpIndex, rewriteOpEnd, executeOperation(out) ); + rewriteOpIndex = rewriteOpEnd; +} + +void TokenStreamRewriteEngine::toDebugStream( std::ostream& out, + size_t start, + size_t end ) const +{ + token_list::const_iterator s = tokens.begin(); + std::advance( s, start ); + token_list::const_iterator e = s; + std::advance( e, end-start ); + std::for_each( s, e, dumpTokenWithIndex(out) ); +} + +void TokenStreamRewriteEngine::addToSortedRewriteList( const std::string& programName, + RewriteOperation* op ) +{ + program_map::iterator rewrites = programs.find(programName); + // check if we got the program already.. + if ( rewrites == programs.end() ) + { + // no prog make a new one... + operation_list ops; + ops.push_back(op); + programs.insert(std::make_pair(programName,ops)); + return; + } + operation_list& prog = rewrites->second; + + if( prog.empty() ) + { + prog.push_back(op); + return; + } + + operation_list::iterator i, end = prog.end(); + i = end; + --i; + // if at or beyond last op's index, just append + if ( op->getIndex() >= (*i)->getIndex() ) { + prog.push_back(op); // append to list of operations + return; + } + i = prog.begin(); + + if( i != end ) + { + operation_list::iterator pos = std::upper_bound( i, end, op, compareOperationIndex() ); + prog.insert(pos,op); + } + else + prog.push_back(op); +} + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +} +#endif diff --git a/lib/antlr/src/TokenStreamSelector.cpp b/lib/antlr/src/TokenStreamSelector.cpp new file mode 100644 index 00000000..602e50dc --- /dev/null +++ b/lib/antlr/src/TokenStreamSelector.cpp @@ -0,0 +1,107 @@ +/* ANTLR Translator Generator + * Project led by Terence Parr at http://www.jGuru.com + * Software rights: http://www.antlr.org/license.html + * + * $Id$ + */ +#include "antlr/TokenStreamSelector.hpp" +#include "antlr/TokenStreamRetryException.hpp" + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +namespace antlr { +#endif + +/** A token stream MUX (multiplexor) knows about n token streams + * and can multiplex them onto the same channel for use by token + * stream consumer like a parser. This is a way to have multiple + * lexers break up the same input stream for a single parser. + * Or, you can have multiple instances of the same lexer handle + * multiple input streams; this works great for includes. + */ + +TokenStreamSelector::TokenStreamSelector() +: input(0) +{ +} + +TokenStreamSelector::~TokenStreamSelector() +{ +} + +void TokenStreamSelector::addInputStream(TokenStream* stream, const ANTLR_USE_NAMESPACE(std)string& key) +{ + inputStreamNames[key] = stream; +} + +TokenStream* TokenStreamSelector::getCurrentStream() const +{ + return input; +} + +TokenStream* TokenStreamSelector::getStream(const ANTLR_USE_NAMESPACE(std)string& sname) const +{ + inputStreamNames_coll::const_iterator i = inputStreamNames.find(sname); + if (i == inputStreamNames.end()) { + throw ANTLR_USE_NAMESPACE(std)string("TokenStream ")+sname+" not found"; + } + return (*i).second; +} + +RefToken TokenStreamSelector::nextToken() +{ + // keep looking for a token until you don't + // get a retry exception + for (;;) { + try { + return input->nextToken(); + } + catch (TokenStreamRetryException&) { + // just retry "forever" + } + } +} + +TokenStream* TokenStreamSelector::pop() +{ + TokenStream* stream = streamStack.top(); + streamStack.pop(); + select(stream); + return stream; +} + +void TokenStreamSelector::push(TokenStream* stream) +{ + streamStack.push(input); + select(stream); +} + +void TokenStreamSelector::push(const ANTLR_USE_NAMESPACE(std)string& sname) +{ + streamStack.push(input); + select(sname); +} + +void TokenStreamSelector::retry() +{ + throw TokenStreamRetryException(); +} + +/** Set the stream without pushing old stream */ +void TokenStreamSelector::select(TokenStream* stream) +{ + input = stream; +} + +void TokenStreamSelector::select(const ANTLR_USE_NAMESPACE(std)string& sname) +{ + inputStreamNames_coll::const_iterator i = inputStreamNames.find(sname); + if (i == inputStreamNames.end()) { + throw ANTLR_USE_NAMESPACE(std)string("TokenStream ")+sname+" not found"; + } + input = (*i).second; +} + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +} +#endif + diff --git a/lib/antlr/src/TreeParser.cpp b/lib/antlr/src/TreeParser.cpp new file mode 100644 index 00000000..6b3f2ca1 --- /dev/null +++ b/lib/antlr/src/TreeParser.cpp @@ -0,0 +1,72 @@ +/* ANTLR Translator Generator + * Project led by Terence Parr at http://www.jGuru.com + * Software rights: http://www.antlr.org/license.html + * + * $Id$ + */ + +#include "antlr/TreeParser.hpp" +#include "antlr/ASTNULLType.hpp" + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +namespace antlr { +#endif + +/** The AST Null object; the parsing cursor is set to this when + * it is found to be null. This way, we can test the + * token type of a node without having to have tests for null + * everywhere. + */ +RefAST TreeParser::ASTNULL(new ASTNULLType); + +/** Parser error-reporting function can be overridden in subclass */ +void TreeParser::reportError(const RecognitionException& ex) +{ + ANTLR_USE_NAMESPACE(std)cerr << ex.toString().c_str() << ANTLR_USE_NAMESPACE(std)endl; +} + +/** Parser error-reporting function can be overridden in subclass */ +void TreeParser::reportError(const ANTLR_USE_NAMESPACE(std)string& s) +{ + ANTLR_USE_NAMESPACE(std)cerr << "error: " << s.c_str() << ANTLR_USE_NAMESPACE(std)endl; +} + +/** Parser warning-reporting function can be overridden in subclass */ +void TreeParser::reportWarning(const ANTLR_USE_NAMESPACE(std)string& s) +{ + ANTLR_USE_NAMESPACE(std)cerr << "warning: " << s.c_str() << ANTLR_USE_NAMESPACE(std)endl; +} + +/** Procedure to write out an indent for traceIn and traceOut */ +void TreeParser::traceIndent() +{ + for( int i = 0; i < traceDepth; i++ ) + ANTLR_USE_NAMESPACE(std)cout << " "; +} + +void TreeParser::traceIn(const char* rname, RefAST t) +{ + traceDepth++; + traceIndent(); + + ANTLR_USE_NAMESPACE(std)cout << "> " << rname + << "(" << (t ? t->toString().c_str() : "null") << ")" + << ((inputState->guessing>0)?" [guessing]":"") + << ANTLR_USE_NAMESPACE(std)endl; +} + +void TreeParser::traceOut(const char* rname, RefAST t) +{ + traceIndent(); + + ANTLR_USE_NAMESPACE(std)cout << "< " << rname + << "(" << (t ? t->toString().c_str() : "null") << ")" + << ((inputState->guessing>0)?" [guessing]":"") + << ANTLR_USE_NAMESPACE(std)endl; + + traceDepth--; +} + +#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE +} +#endif |